projects / mesa.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
history | raw | HEAD
ac: set swizzled bit in cache policy as a hint not to merge loads/stores
[mesa.git] / src / gallium / drivers / radeonsi / si_shader.c
1 /*
2 * Copyright 2012 Advanced Micro Devices, Inc.
3 * All Rights Reserved.
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * on the rights to use, copy, modify, merge, publish, distribute, sub
9 * license, and/or sell copies of the Software, and to permit persons to whom
10 * the Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice (including the next
13 * paragraph) shall be included in all copies or substantial portions of the
14 * Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
20 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
21 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
22 * USE OR OTHER DEALINGS IN THE SOFTWARE.
23 */
24
25 #include <llvm/Config/llvm-config.h>
26
27 #include "util/u_memory.h"
28 #include "util/u_string.h"
29 #include "tgsi/tgsi_build.h"
30 #include "tgsi/tgsi_strings.h"
31 #include "tgsi/tgsi_util.h"
32 #include "tgsi/tgsi_dump.h"
33 #include "tgsi/tgsi_from_mesa.h"
34
35 #include "ac_binary.h"
36 #include "ac_exp_param.h"
37 #include "ac_shader_util.h"
38 #include "ac_rtld.h"
39 #include "ac_llvm_util.h"
40 #include "si_shader_internal.h"
41 #include "si_pipe.h"
42 #include "sid.h"
43
44 #include "compiler/nir/nir.h"
45 #include "compiler/nir/nir_serialize.h"
46
47 static const char scratch_rsrc_dword0_symbol[] =
48 "SCRATCH_RSRC_DWORD0";
49
50 static const char scratch_rsrc_dword1_symbol[] =
51 "SCRATCH_RSRC_DWORD1";
52
53 static void si_init_shader_ctx(struct si_shader_context *ctx,
54 struct si_screen *sscreen,
55 struct ac_llvm_compiler *compiler,
56 unsigned wave_size,
57 bool nir);
58
59 static void si_llvm_emit_barrier(const struct lp_build_tgsi_action *action,
60 struct lp_build_tgsi_context *bld_base,
61 struct lp_build_emit_data *emit_data);
62
63 static void si_dump_shader_key(const struct si_shader *shader, FILE *f);
64
65 static void si_build_vs_prolog_function(struct si_shader_context *ctx,
66 union si_shader_part_key *key);
67 static void si_build_tcs_epilog_function(struct si_shader_context *ctx,
68 union si_shader_part_key *key);
69 static void si_build_ps_prolog_function(struct si_shader_context *ctx,
70 union si_shader_part_key *key);
71 static void si_build_ps_epilog_function(struct si_shader_context *ctx,
72 union si_shader_part_key *key);
73 static void si_fix_resource_usage(struct si_screen *sscreen,
74 struct si_shader *shader);
75
76 /* Ideally pass the sample mask input to the PS epilog as v14, which
77 * is its usual location, so that the shader doesn't have to add v_mov.
78 */
79 #define PS_EPILOG_SAMPLEMASK_MIN_LOC 14
80
81 static bool llvm_type_is_64bit(struct si_shader_context *ctx,
82 LLVMTypeRef type)
83 {
84 if (type == ctx->ac.i64 || type == ctx->ac.f64)
85 return true;
86
87 return false;
88 }
89
90 /** Whether the shader runs as a combination of multiple API shaders */
91 static bool is_multi_part_shader(struct si_shader_context *ctx)
92 {
93 if (ctx->screen->info.chip_class <= GFX8)
94 return false;
95
96 return ctx->shader->key.as_ls ||
97 ctx->shader->key.as_es ||
98 ctx->type == PIPE_SHADER_TESS_CTRL ||
99 ctx->type == PIPE_SHADER_GEOMETRY;
100 }
101
102 /** Whether the shader runs on a merged HW stage (LSHS or ESGS) */
103 static bool is_merged_shader(struct si_shader_context *ctx)
104 {
105 return ctx->shader->key.as_ngg || is_multi_part_shader(ctx);
106 }
107
108 /**
109 * Returns a unique index for a per-patch semantic name and index. The index
110 * must be less than 32, so that a 32-bit bitmask of used inputs or outputs
111 * can be calculated.
112 */
113 unsigned si_shader_io_get_unique_index_patch(unsigned semantic_name, unsigned index)
114 {
115 switch (semantic_name) {
116 case TGSI_SEMANTIC_TESSOUTER:
117 return 0;
118 case TGSI_SEMANTIC_TESSINNER:
119 return 1;
120 case TGSI_SEMANTIC_PATCH:
121 assert(index < 30);
122 return 2 + index;
123
124 default:
125 assert(!"invalid semantic name");
126 return 0;
127 }
128 }
129
130 /**
131 * Returns a unique index for a semantic name and index. The index must be
132 * less than 64, so that a 64-bit bitmask of used inputs or outputs can be
133 * calculated.
134 */
135 unsigned si_shader_io_get_unique_index(unsigned semantic_name, unsigned index,
136 unsigned is_varying)
137 {
138 switch (semantic_name) {
139 case TGSI_SEMANTIC_POSITION:
140 return 0;
141 case TGSI_SEMANTIC_GENERIC:
142 /* Since some shader stages use the the highest used IO index
143 * to determine the size to allocate for inputs/outputs
144 * (in LDS, tess and GS rings). GENERIC should be placed right
145 * after POSITION to make that size as small as possible.
146 */
147 if (index < SI_MAX_IO_GENERIC)
148 return 1 + index;
149
150 assert(!"invalid generic index");
151 return 0;
152 case TGSI_SEMANTIC_FOG:
153 return SI_MAX_IO_GENERIC + 1;
154 case TGSI_SEMANTIC_COLOR:
155 assert(index < 2);
156 return SI_MAX_IO_GENERIC + 2 + index;
157 case TGSI_SEMANTIC_BCOLOR:
158 assert(index < 2);
159 /* If it's a varying, COLOR and BCOLOR alias. */
160 if (is_varying)
161 return SI_MAX_IO_GENERIC + 2 + index;
162 else
163 return SI_MAX_IO_GENERIC + 4 + index;
164 case TGSI_SEMANTIC_TEXCOORD:
165 assert(index < 8);
166 return SI_MAX_IO_GENERIC + 6 + index;
167
168 /* These are rarely used between LS and HS or ES and GS. */
169 case TGSI_SEMANTIC_CLIPDIST:
170 assert(index < 2);
171 return SI_MAX_IO_GENERIC + 6 + 8 + index;
172 case TGSI_SEMANTIC_CLIPVERTEX:
173 return SI_MAX_IO_GENERIC + 6 + 8 + 2;
174 case TGSI_SEMANTIC_PSIZE:
175 return SI_MAX_IO_GENERIC + 6 + 8 + 3;
176
177 /* These can't be written by LS, HS, and ES. */
178 case TGSI_SEMANTIC_LAYER:
179 return SI_MAX_IO_GENERIC + 6 + 8 + 4;
180 case TGSI_SEMANTIC_VIEWPORT_INDEX:
181 return SI_MAX_IO_GENERIC + 6 + 8 + 5;
182 case TGSI_SEMANTIC_PRIMID:
183 STATIC_ASSERT(SI_MAX_IO_GENERIC + 6 + 8 + 6 <= 63);
184 return SI_MAX_IO_GENERIC + 6 + 8 + 6;
185 default:
186 fprintf(stderr, "invalid semantic name = %u\n", semantic_name);
187 assert(!"invalid semantic name");
188 return 0;
189 }
190 }
191
192 /**
193 * Get the value of a shader input parameter and extract a bitfield.
194 */
195 static LLVMValueRef unpack_llvm_param(struct si_shader_context *ctx,
196 LLVMValueRef value, unsigned rshift,
197 unsigned bitwidth)
198 {
199 if (LLVMGetTypeKind(LLVMTypeOf(value)) == LLVMFloatTypeKind)
200 value = ac_to_integer(&ctx->ac, value);
201
202 if (rshift)
203 value = LLVMBuildLShr(ctx->ac.builder, value,
204 LLVMConstInt(ctx->i32, rshift, 0), "");
205
206 if (rshift + bitwidth < 32) {
207 unsigned mask = (1 << bitwidth) - 1;
208 value = LLVMBuildAnd(ctx->ac.builder, value,
209 LLVMConstInt(ctx->i32, mask, 0), "");
210 }
211
212 return value;
213 }
214
215 LLVMValueRef si_unpack_param(struct si_shader_context *ctx,
216 struct ac_arg param, unsigned rshift,
217 unsigned bitwidth)
218 {
219 LLVMValueRef value = ac_get_arg(&ctx->ac, param);
220
221 return unpack_llvm_param(ctx, value, rshift, bitwidth);
222 }
223
224 static LLVMValueRef get_rel_patch_id(struct si_shader_context *ctx)
225 {
226 switch (ctx->type) {
227 case PIPE_SHADER_TESS_CTRL:
228 return si_unpack_param(ctx, ctx->args.tcs_rel_ids, 0, 8);
229
230 case PIPE_SHADER_TESS_EVAL:
231 return ac_get_arg(&ctx->ac, ctx->tes_rel_patch_id);
232
233 default:
234 assert(0);
235 return NULL;
236 }
237 }
238
239 /* Tessellation shaders pass outputs to the next shader using LDS.
240 *
241 * LS outputs = TCS inputs
242 * TCS outputs = TES inputs
243 *
244 * The LDS layout is:
245 * - TCS inputs for patch 0
246 * - TCS inputs for patch 1
247 * - TCS inputs for patch 2 = get_tcs_in_current_patch_offset (if RelPatchID==2)
248 * - ...
249 * - TCS outputs for patch 0 = get_tcs_out_patch0_offset
250 * - Per-patch TCS outputs for patch 0 = get_tcs_out_patch0_patch_data_offset
251 * - TCS outputs for patch 1
252 * - Per-patch TCS outputs for patch 1
253 * - TCS outputs for patch 2 = get_tcs_out_current_patch_offset (if RelPatchID==2)
254 * - Per-patch TCS outputs for patch 2 = get_tcs_out_current_patch_data_offset (if RelPatchID==2)
255 * - ...
256 *
257 * All three shaders VS(LS), TCS, TES share the same LDS space.
258 */
259
260 static LLVMValueRef
261 get_tcs_in_patch_stride(struct si_shader_context *ctx)
262 {
263 return si_unpack_param(ctx, ctx->vs_state_bits, 8, 13);
264 }
265
266 static unsigned get_tcs_out_vertex_dw_stride_constant(struct si_shader_context *ctx)
267 {
268 assert(ctx->type == PIPE_SHADER_TESS_CTRL);
269
270 if (ctx->shader->key.mono.u.ff_tcs_inputs_to_copy)
271 return util_last_bit64(ctx->shader->key.mono.u.ff_tcs_inputs_to_copy) * 4;
272
273 return util_last_bit64(ctx->shader->selector->outputs_written) * 4;
274 }
275
276 static LLVMValueRef get_tcs_out_vertex_dw_stride(struct si_shader_context *ctx)
277 {
278 unsigned stride = get_tcs_out_vertex_dw_stride_constant(ctx);
279
280 return LLVMConstInt(ctx->i32, stride, 0);
281 }
282
283 static LLVMValueRef get_tcs_out_patch_stride(struct si_shader_context *ctx)
284 {
285 if (ctx->shader->key.mono.u.ff_tcs_inputs_to_copy)
286 return si_unpack_param(ctx, ctx->tcs_out_lds_layout, 0, 13);
287
288 const struct tgsi_shader_info *info = &ctx->shader->selector->info;
289 unsigned tcs_out_vertices = info->properties[TGSI_PROPERTY_TCS_VERTICES_OUT];
290 unsigned vertex_dw_stride = get_tcs_out_vertex_dw_stride_constant(ctx);
291 unsigned num_patch_outputs = util_last_bit64(ctx->shader->selector->patch_outputs_written);
292 unsigned patch_dw_stride = tcs_out_vertices * vertex_dw_stride +
293 num_patch_outputs * 4;
294 return LLVMConstInt(ctx->i32, patch_dw_stride, 0);
295 }
296
297 static LLVMValueRef
298 get_tcs_out_patch0_offset(struct si_shader_context *ctx)
299 {
300 return LLVMBuildMul(ctx->ac.builder,
301 si_unpack_param(ctx, ctx->tcs_out_lds_offsets, 0, 16),
302 LLVMConstInt(ctx->i32, 4, 0), "");
303 }
304
305 static LLVMValueRef
306 get_tcs_out_patch0_patch_data_offset(struct si_shader_context *ctx)
307 {
308 return LLVMBuildMul(ctx->ac.builder,
309 si_unpack_param(ctx, ctx->tcs_out_lds_offsets, 16, 16),
310 LLVMConstInt(ctx->i32, 4, 0), "");
311 }
312
313 static LLVMValueRef
314 get_tcs_in_current_patch_offset(struct si_shader_context *ctx)
315 {
316 LLVMValueRef patch_stride = get_tcs_in_patch_stride(ctx);
317 LLVMValueRef rel_patch_id = get_rel_patch_id(ctx);
318
319 return LLVMBuildMul(ctx->ac.builder, patch_stride, rel_patch_id, "");
320 }
321
322 static LLVMValueRef
323 get_tcs_out_current_patch_offset(struct si_shader_context *ctx)
324 {
325 LLVMValueRef patch0_offset = get_tcs_out_patch0_offset(ctx);
326 LLVMValueRef patch_stride = get_tcs_out_patch_stride(ctx);
327 LLVMValueRef rel_patch_id = get_rel_patch_id(ctx);
328
329 return ac_build_imad(&ctx->ac, patch_stride, rel_patch_id, patch0_offset);
330 }
331
332 static LLVMValueRef
333 get_tcs_out_current_patch_data_offset(struct si_shader_context *ctx)
334 {
335 LLVMValueRef patch0_patch_data_offset =
336 get_tcs_out_patch0_patch_data_offset(ctx);
337 LLVMValueRef patch_stride = get_tcs_out_patch_stride(ctx);
338 LLVMValueRef rel_patch_id = get_rel_patch_id(ctx);
339
340 return ac_build_imad(&ctx->ac, patch_stride, rel_patch_id, patch0_patch_data_offset);
341 }
342
343 static LLVMValueRef get_num_tcs_out_vertices(struct si_shader_context *ctx)
344 {
345 unsigned tcs_out_vertices =
346 ctx->shader->selector ?
347 ctx->shader->selector->info.properties[TGSI_PROPERTY_TCS_VERTICES_OUT] : 0;
348
349 /* If !tcs_out_vertices, it's either the fixed-func TCS or the TCS epilog. */
350 if (ctx->type == PIPE_SHADER_TESS_CTRL && tcs_out_vertices)
351 return LLVMConstInt(ctx->i32, tcs_out_vertices, 0);
352
353 return si_unpack_param(ctx, ctx->tcs_offchip_layout, 6, 6);
354 }
355
356 static LLVMValueRef get_tcs_in_vertex_dw_stride(struct si_shader_context *ctx)
357 {
358 unsigned stride;
359
360 switch (ctx->type) {
361 case PIPE_SHADER_VERTEX:
362 stride = ctx->shader->selector->lshs_vertex_stride / 4;
363 return LLVMConstInt(ctx->i32, stride, 0);
364
365 case PIPE_SHADER_TESS_CTRL:
366 if (ctx->screen->info.chip_class >= GFX9 &&
367 ctx->shader->is_monolithic) {
368 stride = ctx->shader->key.part.tcs.ls->lshs_vertex_stride / 4;
369 return LLVMConstInt(ctx->i32, stride, 0);
370 }
371 return si_unpack_param(ctx, ctx->vs_state_bits, 24, 8);
372
373 default:
374 assert(0);
375 return NULL;
376 }
377 }
378
379 static LLVMValueRef unpack_sint16(struct si_shader_context *ctx,
380 LLVMValueRef i32, unsigned index)
381 {
382 assert(index <= 1);
383
384 if (index == 1)
385 return LLVMBuildAShr(ctx->ac.builder, i32,
386 LLVMConstInt(ctx->i32, 16, 0), "");
387
388 return LLVMBuildSExt(ctx->ac.builder,
389 LLVMBuildTrunc(ctx->ac.builder, i32,
390 ctx->ac.i16, ""),
391 ctx->i32, "");
392 }
393
394 void si_llvm_load_input_vs(
395 struct si_shader_context *ctx,
396 unsigned input_index,
397 LLVMValueRef out[4])
398 {
399 const struct tgsi_shader_info *info = &ctx->shader->selector->info;
400 unsigned vs_blit_property = info->properties[TGSI_PROPERTY_VS_BLIT_SGPRS_AMD];
401
402 if (vs_blit_property) {
403 LLVMValueRef vertex_id = ctx->abi.vertex_id;
404 LLVMValueRef sel_x1 = LLVMBuildICmp(ctx->ac.builder,
405 LLVMIntULE, vertex_id,
406 ctx->i32_1, "");
407 /* Use LLVMIntNE, because we have 3 vertices and only
408 * the middle one should use y2.
409 */
410 LLVMValueRef sel_y1 = LLVMBuildICmp(ctx->ac.builder,
411 LLVMIntNE, vertex_id,
412 ctx->i32_1, "");
413
414 unsigned param_vs_blit_inputs = ctx->vs_blit_inputs.arg_index;
415 if (input_index == 0) {
416 /* Position: */
417 LLVMValueRef x1y1 = LLVMGetParam(ctx->main_fn,
418 param_vs_blit_inputs);
419 LLVMValueRef x2y2 = LLVMGetParam(ctx->main_fn,
420 param_vs_blit_inputs + 1);
421
422 LLVMValueRef x1 = unpack_sint16(ctx, x1y1, 0);
423 LLVMValueRef y1 = unpack_sint16(ctx, x1y1, 1);
424 LLVMValueRef x2 = unpack_sint16(ctx, x2y2, 0);
425 LLVMValueRef y2 = unpack_sint16(ctx, x2y2, 1);
426
427 LLVMValueRef x = LLVMBuildSelect(ctx->ac.builder, sel_x1,
428 x1, x2, "");
429 LLVMValueRef y = LLVMBuildSelect(ctx->ac.builder, sel_y1,
430 y1, y2, "");
431
432 out[0] = LLVMBuildSIToFP(ctx->ac.builder, x, ctx->f32, "");
433 out[1] = LLVMBuildSIToFP(ctx->ac.builder, y, ctx->f32, "");
434 out[2] = LLVMGetParam(ctx->main_fn,
435 param_vs_blit_inputs + 2);
436 out[3] = ctx->ac.f32_1;
437 return;
438 }
439
440 /* Color or texture coordinates: */
441 assert(input_index == 1);
442
443 if (vs_blit_property == SI_VS_BLIT_SGPRS_POS_COLOR) {
444 for (int i = 0; i < 4; i++) {
445 out[i] = LLVMGetParam(ctx->main_fn,
446 param_vs_blit_inputs + 3 + i);
447 }
448 } else {
449 assert(vs_blit_property == SI_VS_BLIT_SGPRS_POS_TEXCOORD);
450 LLVMValueRef x1 = LLVMGetParam(ctx->main_fn,
451 param_vs_blit_inputs + 3);
452 LLVMValueRef y1 = LLVMGetParam(ctx->main_fn,
453 param_vs_blit_inputs + 4);
454 LLVMValueRef x2 = LLVMGetParam(ctx->main_fn,
455 param_vs_blit_inputs + 5);
456 LLVMValueRef y2 = LLVMGetParam(ctx->main_fn,
457 param_vs_blit_inputs + 6);
458
459 out[0] = LLVMBuildSelect(ctx->ac.builder, sel_x1,
460 x1, x2, "");
461 out[1] = LLVMBuildSelect(ctx->ac.builder, sel_y1,
462 y1, y2, "");
463 out[2] = LLVMGetParam(ctx->main_fn,
464 param_vs_blit_inputs + 7);
465 out[3] = LLVMGetParam(ctx->main_fn,
466 param_vs_blit_inputs + 8);
467 }
468 return;
469 }
470
471 union si_vs_fix_fetch fix_fetch;
472 LLVMValueRef t_list_ptr;
473 LLVMValueRef t_offset;
474 LLVMValueRef t_list;
475 LLVMValueRef vertex_index;
476 LLVMValueRef tmp;
477
478 /* Load the T list */
479 t_list_ptr = ac_get_arg(&ctx->ac, ctx->vertex_buffers);
480
481 t_offset = LLVMConstInt(ctx->i32, input_index, 0);
482
483 t_list = ac_build_load_to_sgpr(&ctx->ac, t_list_ptr, t_offset);
484
485 vertex_index = LLVMGetParam(ctx->main_fn,
486 ctx->vertex_index0.arg_index +
487 input_index);
488
489 /* Use the open-coded implementation for all loads of doubles and
490 * of dword-sized data that needs fixups. We need to insert conversion
491 * code anyway, and the amd/common code does it for us.
492 *
493 * Note: On LLVM <= 8, we can only open-code formats with
494 * channel size >= 4 bytes.
495 */
496 bool opencode = ctx->shader->key.mono.vs_fetch_opencode & (1 << input_index);
497 fix_fetch.bits = ctx->shader->key.mono.vs_fix_fetch[input_index].bits;
498 if (opencode ||
499 (fix_fetch.u.log_size == 3 && fix_fetch.u.format == AC_FETCH_FORMAT_FLOAT) ||
500 (fix_fetch.u.log_size == 2)) {
501 tmp = ac_build_opencoded_load_format(
502 &ctx->ac, fix_fetch.u.log_size, fix_fetch.u.num_channels_m1 + 1,
503 fix_fetch.u.format, fix_fetch.u.reverse, !opencode,
504 t_list, vertex_index, ctx->ac.i32_0, ctx->ac.i32_0, 0, true);
505 for (unsigned i = 0; i < 4; ++i)
506 out[i] = LLVMBuildExtractElement(ctx->ac.builder, tmp, LLVMConstInt(ctx->i32, i, false), "");
507 return;
508 }
509
510 /* Do multiple loads for special formats. */
511 unsigned required_channels = util_last_bit(info->input_usage_mask[input_index]);
512 LLVMValueRef fetches[4];
513 unsigned num_fetches;
514 unsigned fetch_stride;
515 unsigned channels_per_fetch;
516
517 if (fix_fetch.u.log_size <= 1 && fix_fetch.u.num_channels_m1 == 2) {
518 num_fetches = MIN2(required_channels, 3);
519 fetch_stride = 1 << fix_fetch.u.log_size;
520 channels_per_fetch = 1;
521 } else {
522 num_fetches = 1;
523 fetch_stride = 0;
524 channels_per_fetch = required_channels;
525 }
526
527 for (unsigned i = 0; i < num_fetches; ++i) {
528 LLVMValueRef voffset = LLVMConstInt(ctx->i32, fetch_stride * i, 0);
529 fetches[i] = ac_build_buffer_load_format(&ctx->ac, t_list, vertex_index, voffset,
530 channels_per_fetch, 0, true);
531 }
532
533 if (num_fetches == 1 && channels_per_fetch > 1) {
534 LLVMValueRef fetch = fetches[0];
535 for (unsigned i = 0; i < channels_per_fetch; ++i) {
536 tmp = LLVMConstInt(ctx->i32, i, false);
537 fetches[i] = LLVMBuildExtractElement(
538 ctx->ac.builder, fetch, tmp, "");
539 }
540 num_fetches = channels_per_fetch;
541 channels_per_fetch = 1;
542 }
543
544 for (unsigned i = num_fetches; i < 4; ++i)
545 fetches[i] = LLVMGetUndef(ctx->f32);
546
547 if (fix_fetch.u.log_size <= 1 && fix_fetch.u.num_channels_m1 == 2 &&
548 required_channels == 4) {
549 if (fix_fetch.u.format == AC_FETCH_FORMAT_UINT || fix_fetch.u.format == AC_FETCH_FORMAT_SINT)
550 fetches[3] = ctx->ac.i32_1;
551 else
552 fetches[3] = ctx->ac.f32_1;
553 } else if (fix_fetch.u.log_size == 3 &&
554 (fix_fetch.u.format == AC_FETCH_FORMAT_SNORM ||
555 fix_fetch.u.format == AC_FETCH_FORMAT_SSCALED ||
556 fix_fetch.u.format == AC_FETCH_FORMAT_SINT) &&
557 required_channels == 4) {
558 /* For 2_10_10_10, the hardware returns an unsigned value;
559 * convert it to a signed one.
560 */
561 LLVMValueRef tmp = fetches[3];
562 LLVMValueRef c30 = LLVMConstInt(ctx->i32, 30, 0);
563
564 /* First, recover the sign-extended signed integer value. */
565 if (fix_fetch.u.format == AC_FETCH_FORMAT_SSCALED)
566 tmp = LLVMBuildFPToUI(ctx->ac.builder, tmp, ctx->i32, "");
567 else
568 tmp = ac_to_integer(&ctx->ac, tmp);
569
570 /* For the integer-like cases, do a natural sign extension.
571 *
572 * For the SNORM case, the values are 0.0, 0.333, 0.666, 1.0
573 * and happen to contain 0, 1, 2, 3 as the two LSBs of the
574 * exponent.
575 */
576 tmp = LLVMBuildShl(ctx->ac.builder, tmp,
577 fix_fetch.u.format == AC_FETCH_FORMAT_SNORM ?
578 LLVMConstInt(ctx->i32, 7, 0) : c30, "");
579 tmp = LLVMBuildAShr(ctx->ac.builder, tmp, c30, "");
580
581 /* Convert back to the right type. */
582 if (fix_fetch.u.format == AC_FETCH_FORMAT_SNORM) {
583 LLVMValueRef clamp;
584 LLVMValueRef neg_one = LLVMConstReal(ctx->f32, -1.0);
585 tmp = LLVMBuildSIToFP(ctx->ac.builder, tmp, ctx->f32, "");
586 clamp = LLVMBuildFCmp(ctx->ac.builder, LLVMRealULT, tmp, neg_one, "");
587 tmp = LLVMBuildSelect(ctx->ac.builder, clamp, neg_one, tmp, "");
588 } else if (fix_fetch.u.format == AC_FETCH_FORMAT_SSCALED) {
589 tmp = LLVMBuildSIToFP(ctx->ac.builder, tmp, ctx->f32, "");
590 }
591
592 fetches[3] = tmp;
593 }
594
595 for (unsigned i = 0; i < 4; ++i)
596 out[i] = ac_to_float(&ctx->ac, fetches[i]);
597 }
598
599 static void declare_input_vs(
600 struct si_shader_context *ctx,
601 unsigned input_index,
602 const struct tgsi_full_declaration *decl,
603 LLVMValueRef out[4])
604 {
605 si_llvm_load_input_vs(ctx, input_index, out);
606 }
607
608 LLVMValueRef si_get_primitive_id(struct si_shader_context *ctx,
609 unsigned swizzle)
610 {
611 if (swizzle > 0)
612 return ctx->i32_0;
613
614 switch (ctx->type) {
615 case PIPE_SHADER_VERTEX:
616 return ac_get_arg(&ctx->ac, ctx->vs_prim_id);
617 case PIPE_SHADER_TESS_CTRL:
618 return ac_get_arg(&ctx->ac, ctx->args.tcs_patch_id);
619 case PIPE_SHADER_TESS_EVAL:
620 return ac_get_arg(&ctx->ac, ctx->args.tes_patch_id);
621 case PIPE_SHADER_GEOMETRY:
622 return ac_get_arg(&ctx->ac, ctx->args.gs_prim_id);
623 default:
624 assert(0);
625 return ctx->i32_0;
626 }
627 }
628
629 /**
630 * Return the value of tgsi_ind_register for indexing.
631 * This is the indirect index with the constant offset added to it.
632 */
633 LLVMValueRef si_get_indirect_index(struct si_shader_context *ctx,
634 const struct tgsi_ind_register *ind,
635 unsigned addr_mul,
636 int rel_index)
637 {
638 LLVMValueRef result;
639
640 if (ind->File == TGSI_FILE_ADDRESS) {
641 result = ctx->addrs[ind->Index][ind->Swizzle];
642 result = LLVMBuildLoad(ctx->ac.builder, result, "");
643 } else {
644 struct tgsi_full_src_register src = {};
645
646 src.Register.File = ind->File;
647 src.Register.Index = ind->Index;
648
649 /* Set the second index to 0 for constants. */
650 if (ind->File == TGSI_FILE_CONSTANT)
651 src.Register.Dimension = 1;
652
653 result = ctx->bld_base.emit_fetch_funcs[ind->File](&ctx->bld_base, &src,
654 TGSI_TYPE_SIGNED,
655 ind->Swizzle);
656 result = ac_to_integer(&ctx->ac, result);
657 }
658
659 return ac_build_imad(&ctx->ac, result, LLVMConstInt(ctx->i32, addr_mul, 0),
660 LLVMConstInt(ctx->i32, rel_index, 0));
661 }
662
663 /**
664 * Like si_get_indirect_index, but restricts the return value to a (possibly
665 * undefined) value inside [0..num).
666 */
667 LLVMValueRef si_get_bounded_indirect_index(struct si_shader_context *ctx,
668 const struct tgsi_ind_register *ind,
669 int rel_index, unsigned num)
670 {
671 LLVMValueRef result = si_get_indirect_index(ctx, ind, 1, rel_index);
672
673 return si_llvm_bound_index(ctx, result, num);
674 }
675
676 static LLVMValueRef get_dw_address_from_generic_indices(struct si_shader_context *ctx,
677 LLVMValueRef vertex_dw_stride,
678 LLVMValueRef base_addr,
679 LLVMValueRef vertex_index,
680 LLVMValueRef param_index,
681 unsigned input_index,
682 ubyte *name,
683 ubyte *index,
684 bool is_patch)
685 {
686 if (vertex_dw_stride) {
687 base_addr = ac_build_imad(&ctx->ac, vertex_index,
688 vertex_dw_stride, base_addr);
689 }
690
691 if (param_index) {
692 base_addr = ac_build_imad(&ctx->ac, param_index,
693 LLVMConstInt(ctx->i32, 4, 0), base_addr);
694 }
695
696 int param = is_patch ?
697 si_shader_io_get_unique_index_patch(name[input_index],
698 index[input_index]) :
699 si_shader_io_get_unique_index(name[input_index],
700 index[input_index], false);
701
702 /* Add the base address of the element. */
703 return LLVMBuildAdd(ctx->ac.builder, base_addr,
704 LLVMConstInt(ctx->i32, param * 4, 0), "");
705 }
706
707 /**
708 * Calculate a dword address given an input or output register and a stride.
709 */
710 static LLVMValueRef get_dw_address(struct si_shader_context *ctx,
711 const struct tgsi_full_dst_register *dst,
712 const struct tgsi_full_src_register *src,
713 LLVMValueRef vertex_dw_stride,
714 LLVMValueRef base_addr)
715 {
716 struct tgsi_shader_info *info = &ctx->shader->selector->info;
717 ubyte *name, *index, *array_first;
718 int input_index;
719 struct tgsi_full_dst_register reg;
720 LLVMValueRef vertex_index = NULL;
721 LLVMValueRef ind_index = NULL;
722
723 /* Set the register description. The address computation is the same
724 * for sources and destinations. */
725 if (src) {
726 reg.Register.File = src->Register.File;
727 reg.Register.Index = src->Register.Index;
728 reg.Register.Indirect = src->Register.Indirect;
729 reg.Register.Dimension = src->Register.Dimension;
730 reg.Indirect = src->Indirect;
731 reg.Dimension = src->Dimension;
732 reg.DimIndirect = src->DimIndirect;
733 } else
734 reg = *dst;
735
736 /* If the register is 2-dimensional (e.g. an array of vertices
737 * in a primitive), calculate the base address of the vertex. */
738 if (reg.Register.Dimension) {
739 if (reg.Dimension.Indirect)
740 vertex_index = si_get_indirect_index(ctx, &reg.DimIndirect,
741 1, reg.Dimension.Index);
742 else
743 vertex_index = LLVMConstInt(ctx->i32, reg.Dimension.Index, 0);
744 }
745
746 /* Get information about the register. */
747 if (reg.Register.File == TGSI_FILE_INPUT) {
748 name = info->input_semantic_name;
749 index = info->input_semantic_index;
750 array_first = info->input_array_first;
751 } else if (reg.Register.File == TGSI_FILE_OUTPUT) {
752 name = info->output_semantic_name;
753 index = info->output_semantic_index;
754 array_first = info->output_array_first;
755 } else {
756 assert(0);
757 return NULL;
758 }
759
760 if (reg.Register.Indirect) {
761 /* Add the relative address of the element. */
762 if (reg.Indirect.ArrayID)
763 input_index = array_first[reg.Indirect.ArrayID];
764 else
765 input_index = reg.Register.Index;
766
767 ind_index = si_get_indirect_index(ctx, &reg.Indirect,
768 1, reg.Register.Index - input_index);
769 } else {
770 input_index = reg.Register.Index;
771 }
772
773 return get_dw_address_from_generic_indices(ctx, vertex_dw_stride,
774 base_addr, vertex_index,
775 ind_index, input_index,
776 name, index,
777 !reg.Register.Dimension);
778 }
779
780 /* The offchip buffer layout for TCS->TES is
781 *
782 * - attribute 0 of patch 0 vertex 0
783 * - attribute 0 of patch 0 vertex 1
784 * - attribute 0 of patch 0 vertex 2
785 * ...
786 * - attribute 0 of patch 1 vertex 0
787 * - attribute 0 of patch 1 vertex 1
788 * ...
789 * - attribute 1 of patch 0 vertex 0
790 * - attribute 1 of patch 0 vertex 1
791 * ...
792 * - per patch attribute 0 of patch 0
793 * - per patch attribute 0 of patch 1
794 * ...
795 *
796 * Note that every attribute has 4 components.
797 */
798 static LLVMValueRef get_tcs_tes_buffer_address(struct si_shader_context *ctx,
799 LLVMValueRef rel_patch_id,
800 LLVMValueRef vertex_index,
801 LLVMValueRef param_index)
802 {
803 LLVMValueRef base_addr, vertices_per_patch, num_patches, total_vertices;
804 LLVMValueRef param_stride, constant16;
805
806 vertices_per_patch = get_num_tcs_out_vertices(ctx);
807 num_patches = si_unpack_param(ctx, ctx->tcs_offchip_layout, 0, 6);
808 total_vertices = LLVMBuildMul(ctx->ac.builder, vertices_per_patch,
809 num_patches, "");
810
811 constant16 = LLVMConstInt(ctx->i32, 16, 0);
812 if (vertex_index) {
813 base_addr = ac_build_imad(&ctx->ac, rel_patch_id,
814 vertices_per_patch, vertex_index);
815 param_stride = total_vertices;
816 } else {
817 base_addr = rel_patch_id;
818 param_stride = num_patches;
819 }
820
821 base_addr = ac_build_imad(&ctx->ac, param_index, param_stride, base_addr);
822 base_addr = LLVMBuildMul(ctx->ac.builder, base_addr, constant16, "");
823
824 if (!vertex_index) {
825 LLVMValueRef patch_data_offset =
826 si_unpack_param(ctx, ctx->tcs_offchip_layout, 12, 20);
827
828 base_addr = LLVMBuildAdd(ctx->ac.builder, base_addr,
829 patch_data_offset, "");
830 }
831 return base_addr;
832 }
833
834 /* This is a generic helper that can be shared by the NIR and TGSI backends */
835 static LLVMValueRef get_tcs_tes_buffer_address_from_generic_indices(
836 struct si_shader_context *ctx,
837 LLVMValueRef vertex_index,
838 LLVMValueRef param_index,
839 unsigned param_base,
840 ubyte *name,
841 ubyte *index,
842 bool is_patch)
843 {
844 unsigned param_index_base;
845
846 param_index_base = is_patch ?
847 si_shader_io_get_unique_index_patch(name[param_base], index[param_base]) :
848 si_shader_io_get_unique_index(name[param_base], index[param_base], false);
849
850 if (param_index) {
851 param_index = LLVMBuildAdd(ctx->ac.builder, param_index,
852 LLVMConstInt(ctx->i32, param_index_base, 0),
853 "");
854 } else {
855 param_index = LLVMConstInt(ctx->i32, param_index_base, 0);
856 }
857
858 return get_tcs_tes_buffer_address(ctx, get_rel_patch_id(ctx),
859 vertex_index, param_index);
860 }
861
862 static LLVMValueRef get_tcs_tes_buffer_address_from_reg(
863 struct si_shader_context *ctx,
864 const struct tgsi_full_dst_register *dst,
865 const struct tgsi_full_src_register *src)
866 {
867 struct tgsi_shader_info *info = &ctx->shader->selector->info;
868 ubyte *name, *index, *array_first;
869 struct tgsi_full_src_register reg;
870 LLVMValueRef vertex_index = NULL;
871 LLVMValueRef param_index = NULL;
872 unsigned param_base;
873
874 reg = src ? *src : tgsi_full_src_register_from_dst(dst);
875
876 if (reg.Register.Dimension) {
877
878 if (reg.Dimension.Indirect)
879 vertex_index = si_get_indirect_index(ctx, &reg.DimIndirect,
880 1, reg.Dimension.Index);
881 else
882 vertex_index = LLVMConstInt(ctx->i32, reg.Dimension.Index, 0);
883 }
884
885 /* Get information about the register. */
886 if (reg.Register.File == TGSI_FILE_INPUT) {
887 name = info->input_semantic_name;
888 index = info->input_semantic_index;
889 array_first = info->input_array_first;
890 } else if (reg.Register.File == TGSI_FILE_OUTPUT) {
891 name = info->output_semantic_name;
892 index = info->output_semantic_index;
893 array_first = info->output_array_first;
894 } else {
895 assert(0);
896 return NULL;
897 }
898
899 if (reg.Register.Indirect) {
900 if (reg.Indirect.ArrayID)
901 param_base = array_first[reg.Indirect.ArrayID];
902 else
903 param_base = reg.Register.Index;
904
905 param_index = si_get_indirect_index(ctx, &reg.Indirect,
906 1, reg.Register.Index - param_base);
907
908 } else {
909 param_base = reg.Register.Index;
910 }
911
912 return get_tcs_tes_buffer_address_from_generic_indices(ctx, vertex_index,
913 param_index, param_base,
914 name, index, !reg.Register.Dimension);
915 }
916
917 static LLVMValueRef buffer_load(struct lp_build_tgsi_context *bld_base,
918 LLVMTypeRef type, unsigned swizzle,
919 LLVMValueRef buffer, LLVMValueRef offset,
920 LLVMValueRef base, bool can_speculate)
921 {
922 struct si_shader_context *ctx = si_shader_context(bld_base);
923 LLVMValueRef value, value2;
924 LLVMTypeRef vec_type = LLVMVectorType(type, 4);
925
926 if (swizzle == ~0) {
927 value = ac_build_buffer_load(&ctx->ac, buffer, 4, NULL, base, offset,
928 0, ac_glc, can_speculate, false);
929
930 return LLVMBuildBitCast(ctx->ac.builder, value, vec_type, "");
931 }
932
933 if (!llvm_type_is_64bit(ctx, type)) {
934 value = ac_build_buffer_load(&ctx->ac, buffer, 4, NULL, base, offset,
935 0, ac_glc, can_speculate, false);
936
937 value = LLVMBuildBitCast(ctx->ac.builder, value, vec_type, "");
938 return LLVMBuildExtractElement(ctx->ac.builder, value,
939 LLVMConstInt(ctx->i32, swizzle, 0), "");
940 }
941
942 value = ac_build_buffer_load(&ctx->ac, buffer, 1, NULL, base, offset,
943 swizzle * 4, ac_glc, can_speculate, false);
944
945 value2 = ac_build_buffer_load(&ctx->ac, buffer, 1, NULL, base, offset,
946 swizzle * 4 + 4, ac_glc, can_speculate, false);
947
948 return si_llvm_emit_fetch_64bit(bld_base, type, value, value2);
949 }
950
951 /**
952 * Load from LSHS LDS storage.
953 *
954 * \param type output value type
955 * \param swizzle offset (typically 0..3); it can be ~0, which loads a vec4
956 * \param dw_addr address in dwords
957 */
958 static LLVMValueRef lshs_lds_load(struct lp_build_tgsi_context *bld_base,
959 LLVMTypeRef type, unsigned swizzle,
960 LLVMValueRef dw_addr)
961 {
962 struct si_shader_context *ctx = si_shader_context(bld_base);
963 LLVMValueRef value;
964
965 if (swizzle == ~0) {
966 LLVMValueRef values[TGSI_NUM_CHANNELS];
967
968 for (unsigned chan = 0; chan < TGSI_NUM_CHANNELS; chan++)
969 values[chan] = lshs_lds_load(bld_base, type, chan, dw_addr);
970
971 return ac_build_gather_values(&ctx->ac, values,
972 TGSI_NUM_CHANNELS);
973 }
974
975 /* Split 64-bit loads. */
976 if (llvm_type_is_64bit(ctx, type)) {
977 LLVMValueRef lo, hi;
978
979 lo = lshs_lds_load(bld_base, ctx->i32, swizzle, dw_addr);
980 hi = lshs_lds_load(bld_base, ctx->i32, swizzle + 1, dw_addr);
981 return si_llvm_emit_fetch_64bit(bld_base, type, lo, hi);
982 }
983
984 dw_addr = LLVMBuildAdd(ctx->ac.builder, dw_addr,
985 LLVMConstInt(ctx->i32, swizzle, 0), "");
986
987 value = ac_lds_load(&ctx->ac, dw_addr);
988
989 return LLVMBuildBitCast(ctx->ac.builder, value, type, "");
990 }
991
992 /**
993 * Store to LSHS LDS storage.
994 *
995 * \param swizzle offset (typically 0..3)
996 * \param dw_addr address in dwords
997 * \param value value to store
998 */
999 static void lshs_lds_store(struct si_shader_context *ctx,
1000 unsigned dw_offset_imm, LLVMValueRef dw_addr,
1001 LLVMValueRef value)
1002 {
1003 dw_addr = LLVMBuildAdd(ctx->ac.builder, dw_addr,
1004 LLVMConstInt(ctx->i32, dw_offset_imm, 0), "");
1005
1006 ac_lds_store(&ctx->ac, dw_addr, value);
1007 }
1008
1009 enum si_tess_ring {
1010 TCS_FACTOR_RING,
1011 TESS_OFFCHIP_RING_TCS,
1012 TESS_OFFCHIP_RING_TES,
1013 };
1014
1015 static LLVMValueRef get_tess_ring_descriptor(struct si_shader_context *ctx,
1016 enum si_tess_ring ring)
1017 {
1018 LLVMBuilderRef builder = ctx->ac.builder;
1019 LLVMValueRef addr = ac_get_arg(&ctx->ac,
1020 ring == TESS_OFFCHIP_RING_TES ?
1021 ctx->tes_offchip_addr :
1022 ctx->tcs_out_lds_layout);
1023
1024 /* TCS only receives high 13 bits of the address. */
1025 if (ring == TESS_OFFCHIP_RING_TCS || ring == TCS_FACTOR_RING) {
1026 addr = LLVMBuildAnd(builder, addr,
1027 LLVMConstInt(ctx->i32, 0xfff80000, 0), "");
1028 }
1029
1030 if (ring == TCS_FACTOR_RING) {
1031 unsigned tf_offset = ctx->screen->tess_offchip_ring_size;
1032 addr = LLVMBuildAdd(builder, addr,
1033 LLVMConstInt(ctx->i32, tf_offset, 0), "");
1034 }
1035
1036 uint32_t rsrc3 = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) |
1037 S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
1038 S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) |
1039 S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W);
1040
1041 if (ctx->screen->info.chip_class >= GFX10)
1042 rsrc3 |= S_008F0C_FORMAT(V_008F0C_IMG_FORMAT_32_FLOAT) |
1043 S_008F0C_OOB_SELECT(3) |
1044 S_008F0C_RESOURCE_LEVEL(1);
1045 else
1046 rsrc3 |= S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
1047 S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
1048
1049 LLVMValueRef desc[4];
1050 desc[0] = addr;
1051 desc[1] = LLVMConstInt(ctx->i32,
1052 S_008F04_BASE_ADDRESS_HI(ctx->screen->info.address32_hi), 0);
1053 desc[2] = LLVMConstInt(ctx->i32, 0xffffffff, 0);
1054 desc[3] = LLVMConstInt(ctx->i32, rsrc3, false);
1055
1056 return ac_build_gather_values(&ctx->ac, desc, 4);
1057 }
1058
1059 static LLVMValueRef fetch_input_tcs(
1060 struct lp_build_tgsi_context *bld_base,
1061 const struct tgsi_full_src_register *reg,
1062 enum tgsi_opcode_type type, unsigned swizzle_in)
1063 {
1064 struct si_shader_context *ctx = si_shader_context(bld_base);
1065 LLVMValueRef dw_addr, stride;
1066 unsigned swizzle = swizzle_in & 0xffff;
1067 stride = get_tcs_in_vertex_dw_stride(ctx);
1068 dw_addr = get_tcs_in_current_patch_offset(ctx);
1069 dw_addr = get_dw_address(ctx, NULL, reg, stride, dw_addr);
1070
1071 return lshs_lds_load(bld_base, tgsi2llvmtype(bld_base, type), swizzle, dw_addr);
1072 }
1073
1074 static LLVMValueRef si_nir_load_tcs_varyings(struct ac_shader_abi *abi,
1075 LLVMTypeRef type,
1076 LLVMValueRef vertex_index,
1077 LLVMValueRef param_index,
1078 unsigned const_index,
1079 unsigned location,
1080 unsigned driver_location,
1081 unsigned component,
1082 unsigned num_components,
1083 bool is_patch,
1084 bool is_compact,
1085 bool load_input)
1086 {
1087 struct si_shader_context *ctx = si_shader_context_from_abi(abi);
1088 struct tgsi_shader_info *info = &ctx->shader->selector->info;
1089 struct lp_build_tgsi_context *bld_base = &ctx->bld_base;
1090 LLVMValueRef dw_addr, stride;
1091
1092 driver_location = driver_location / 4;
1093
1094 if (load_input) {
1095 stride = get_tcs_in_vertex_dw_stride(ctx);
1096 dw_addr = get_tcs_in_current_patch_offset(ctx);
1097 } else {
1098 if (is_patch) {
1099 stride = NULL;
1100 dw_addr = get_tcs_out_current_patch_data_offset(ctx);
1101 } else {
1102 stride = get_tcs_out_vertex_dw_stride(ctx);
1103 dw_addr = get_tcs_out_current_patch_offset(ctx);
1104 }
1105 }
1106
1107 if (!param_index) {
1108 param_index = LLVMConstInt(ctx->i32, const_index, 0);
1109 }
1110
1111 ubyte *names;
1112 ubyte *indices;
1113 if (load_input) {
1114 names = info->input_semantic_name;
1115 indices = info->input_semantic_index;
1116 } else {
1117 names = info->output_semantic_name;
1118 indices = info->output_semantic_index;
1119 }
1120
1121 dw_addr = get_dw_address_from_generic_indices(ctx, stride, dw_addr,
1122 vertex_index, param_index,
1123 driver_location,
1124 names, indices,
1125 is_patch);
1126
1127 LLVMValueRef value[4];
1128 for (unsigned i = 0; i < num_components; i++) {
1129 unsigned offset = i;
1130 if (llvm_type_is_64bit(ctx, type))
1131 offset *= 2;
1132
1133 offset += component;
1134 value[i + component] = lshs_lds_load(bld_base, type, offset, dw_addr);
1135 }
1136
1137 return ac_build_varying_gather_values(&ctx->ac, value, num_components, component);
1138 }
1139
1140 static LLVMValueRef fetch_output_tcs(
1141 struct lp_build_tgsi_context *bld_base,
1142 const struct tgsi_full_src_register *reg,
1143 enum tgsi_opcode_type type, unsigned swizzle_in)
1144 {
1145 struct si_shader_context *ctx = si_shader_context(bld_base);
1146 LLVMValueRef dw_addr, stride;
1147 unsigned swizzle = (swizzle_in & 0xffff);
1148
1149 if (reg->Register.Dimension) {
1150 stride = get_tcs_out_vertex_dw_stride(ctx);
1151 dw_addr = get_tcs_out_current_patch_offset(ctx);
1152 dw_addr = get_dw_address(ctx, NULL, reg, stride, dw_addr);
1153 } else {
1154 dw_addr = get_tcs_out_current_patch_data_offset(ctx);
1155 dw_addr = get_dw_address(ctx, NULL, reg, NULL, dw_addr);
1156 }
1157
1158 return lshs_lds_load(bld_base, tgsi2llvmtype(bld_base, type), swizzle, dw_addr);
1159 }
1160
1161 static LLVMValueRef fetch_input_tes(
1162 struct lp_build_tgsi_context *bld_base,
1163 const struct tgsi_full_src_register *reg,
1164 enum tgsi_opcode_type type, unsigned swizzle_in)
1165 {
1166 struct si_shader_context *ctx = si_shader_context(bld_base);
1167 LLVMValueRef base, addr;
1168 unsigned swizzle = (swizzle_in & 0xffff);
1169
1170 base = ac_get_arg(&ctx->ac, ctx->tcs_offchip_offset);
1171 addr = get_tcs_tes_buffer_address_from_reg(ctx, NULL, reg);
1172
1173 return buffer_load(bld_base, tgsi2llvmtype(bld_base, type), swizzle,
1174 ctx->tess_offchip_ring, base, addr, true);
1175 }
1176
1177 LLVMValueRef si_nir_load_input_tes(struct ac_shader_abi *abi,
1178 LLVMTypeRef type,
1179 LLVMValueRef vertex_index,
1180 LLVMValueRef param_index,
1181 unsigned const_index,
1182 unsigned location,
1183 unsigned driver_location,
1184 unsigned component,
1185 unsigned num_components,
1186 bool is_patch,
1187 bool is_compact,
1188 bool load_input)
1189 {
1190 struct si_shader_context *ctx = si_shader_context_from_abi(abi);
1191 struct tgsi_shader_info *info = &ctx->shader->selector->info;
1192 LLVMValueRef base, addr;
1193
1194 driver_location = driver_location / 4;
1195
1196 base = ac_get_arg(&ctx->ac, ctx->tcs_offchip_offset);
1197
1198 if (!param_index) {
1199 param_index = LLVMConstInt(ctx->i32, const_index, 0);
1200 }
1201
1202 addr = get_tcs_tes_buffer_address_from_generic_indices(ctx, vertex_index,
1203 param_index, driver_location,
1204 info->input_semantic_name,
1205 info->input_semantic_index,
1206 is_patch);
1207
1208 /* TODO: This will generate rather ordinary llvm code, although it
1209 * should be easy for the optimiser to fix up. In future we might want
1210 * to refactor buffer_load(), but for now this maximises code sharing
1211 * between the NIR and TGSI backends.
1212 */
1213 LLVMValueRef value[4];
1214 for (unsigned i = 0; i < num_components; i++) {
1215 unsigned offset = i;
1216 if (llvm_type_is_64bit(ctx, type)) {
1217 offset *= 2;
1218 if (offset == 4) {
1219 addr = get_tcs_tes_buffer_address_from_generic_indices(ctx,
1220 vertex_index,
1221 param_index,
1222 driver_location + 1,
1223 info->input_semantic_name,
1224 info->input_semantic_index,
1225 is_patch);
1226 }
1227
1228 offset = offset % 4;
1229 }
1230
1231 offset += component;
1232 value[i + component] = buffer_load(&ctx->bld_base, type, offset,
1233 ctx->tess_offchip_ring, base, addr, true);
1234 }
1235
1236 return ac_build_varying_gather_values(&ctx->ac, value, num_components, component);
1237 }
1238
1239 static void store_output_tcs(struct lp_build_tgsi_context *bld_base,
1240 const struct tgsi_full_instruction *inst,
1241 const struct tgsi_opcode_info *info,
1242 unsigned index,
1243 LLVMValueRef dst[4])
1244 {
1245 struct si_shader_context *ctx = si_shader_context(bld_base);
1246 const struct tgsi_full_dst_register *reg = &inst->Dst[index];
1247 const struct tgsi_shader_info *sh_info = &ctx->shader->selector->info;
1248 unsigned chan_index;
1249 LLVMValueRef dw_addr, stride;
1250 LLVMValueRef buffer, base, buf_addr;
1251 LLVMValueRef values[4];
1252 bool skip_lds_store;
1253 bool is_tess_factor = false, is_tess_inner = false;
1254
1255 /* Only handle per-patch and per-vertex outputs here.
1256 * Vectors will be lowered to scalars and this function will be called again.
1257 */
1258 if (reg->Register.File != TGSI_FILE_OUTPUT ||
1259 (dst[0] && LLVMGetTypeKind(LLVMTypeOf(dst[0])) == LLVMVectorTypeKind)) {
1260 si_llvm_emit_store(bld_base, inst, info, index, dst);
1261 return;
1262 }
1263
1264 if (reg->Register.Dimension) {
1265 stride = get_tcs_out_vertex_dw_stride(ctx);
1266 dw_addr = get_tcs_out_current_patch_offset(ctx);
1267 dw_addr = get_dw_address(ctx, reg, NULL, stride, dw_addr);
1268 skip_lds_store = !sh_info->reads_pervertex_outputs;
1269 } else {
1270 dw_addr = get_tcs_out_current_patch_data_offset(ctx);
1271 dw_addr = get_dw_address(ctx, reg, NULL, NULL, dw_addr);
1272 skip_lds_store = !sh_info->reads_perpatch_outputs;
1273
1274 if (!reg->Register.Indirect) {
1275 int name = sh_info->output_semantic_name[reg->Register.Index];
1276
1277 /* Always write tess factors into LDS for the TCS epilog. */
1278 if (name == TGSI_SEMANTIC_TESSINNER ||
1279 name == TGSI_SEMANTIC_TESSOUTER) {
1280 /* The epilog doesn't read LDS if invocation 0 defines tess factors. */
1281 skip_lds_store = !sh_info->reads_tessfactor_outputs &&
1282 ctx->shader->selector->tcs_info.tessfactors_are_def_in_all_invocs;
1283 is_tess_factor = true;
1284 is_tess_inner = name == TGSI_SEMANTIC_TESSINNER;
1285 }
1286 }
1287 }
1288
1289 buffer = get_tess_ring_descriptor(ctx, TESS_OFFCHIP_RING_TCS);
1290
1291 base = ac_get_arg(&ctx->ac, ctx->tcs_offchip_offset);
1292 buf_addr = get_tcs_tes_buffer_address_from_reg(ctx, reg, NULL);
1293
1294 uint32_t writemask = reg->Register.WriteMask;
1295 while (writemask) {
1296 chan_index = u_bit_scan(&writemask);
1297 LLVMValueRef value = dst[chan_index];
1298
1299 if (inst->Instruction.Saturate)
1300 value = ac_build_clamp(&ctx->ac, value);
1301
1302 /* Skip LDS stores if there is no LDS read of this output. */
1303 if (!skip_lds_store)
1304 lshs_lds_store(ctx, chan_index, dw_addr, value);
1305
1306 value = ac_to_integer(&ctx->ac, value);
1307 values[chan_index] = value;
1308
1309 if (reg->Register.WriteMask != 0xF && !is_tess_factor) {
1310 ac_build_buffer_store_dword(&ctx->ac, buffer, value, 1,
1311 buf_addr, base,
1312 4 * chan_index, ac_glc);
1313 }
1314
1315 /* Write tess factors into VGPRs for the epilog. */
1316 if (is_tess_factor &&
1317 ctx->shader->selector->tcs_info.tessfactors_are_def_in_all_invocs) {
1318 if (!is_tess_inner) {
1319 LLVMBuildStore(ctx->ac.builder, value, /* outer */
1320 ctx->invoc0_tess_factors[chan_index]);
1321 } else if (chan_index < 2) {
1322 LLVMBuildStore(ctx->ac.builder, value, /* inner */
1323 ctx->invoc0_tess_factors[4 + chan_index]);
1324 }
1325 }
1326 }
1327
1328 if (reg->Register.WriteMask == 0xF && !is_tess_factor) {
1329 LLVMValueRef value = ac_build_gather_values(&ctx->ac,
1330 values, 4);
1331 ac_build_buffer_store_dword(&ctx->ac, buffer, value, 4, buf_addr,
1332 base, 0, ac_glc);
1333 }
1334 }
1335
1336 static void si_nir_store_output_tcs(struct ac_shader_abi *abi,
1337 const struct nir_variable *var,
1338 LLVMValueRef vertex_index,
1339 LLVMValueRef param_index,
1340 unsigned const_index,
1341 LLVMValueRef src,
1342 unsigned writemask)
1343 {
1344 struct si_shader_context *ctx = si_shader_context_from_abi(abi);
1345 struct tgsi_shader_info *info = &ctx->shader->selector->info;
1346 const unsigned component = var->data.location_frac;
1347 const bool is_patch = var->data.patch;
1348 unsigned driver_location = var->data.driver_location;
1349 LLVMValueRef dw_addr, stride;
1350 LLVMValueRef buffer, base, addr;
1351 LLVMValueRef values[8];
1352 bool skip_lds_store;
1353 bool is_tess_factor = false, is_tess_inner = false;
1354
1355 driver_location = driver_location / 4;
1356
1357 bool is_const = !param_index;
1358 if (!param_index)
1359 param_index = LLVMConstInt(ctx->i32, const_index, 0);
1360
1361 if (!is_patch) {
1362 stride = get_tcs_out_vertex_dw_stride(ctx);
1363 dw_addr = get_tcs_out_current_patch_offset(ctx);
1364 dw_addr = get_dw_address_from_generic_indices(ctx, stride, dw_addr,
1365 vertex_index, param_index,
1366 driver_location,
1367 info->output_semantic_name,
1368 info->output_semantic_index,
1369 is_patch);
1370
1371 skip_lds_store = !info->reads_pervertex_outputs;
1372 } else {
1373 dw_addr = get_tcs_out_current_patch_data_offset(ctx);
1374 dw_addr = get_dw_address_from_generic_indices(ctx, NULL, dw_addr,
1375 vertex_index, param_index,
1376 driver_location,
1377 info->output_semantic_name,
1378 info->output_semantic_index,
1379 is_patch);
1380
1381 skip_lds_store = !info->reads_perpatch_outputs;
1382
1383 if (is_const && const_index == 0) {
1384 int name = info->output_semantic_name[driver_location];
1385
1386 /* Always write tess factors into LDS for the TCS epilog. */
1387 if (name == TGSI_SEMANTIC_TESSINNER ||
1388 name == TGSI_SEMANTIC_TESSOUTER) {
1389 /* The epilog doesn't read LDS if invocation 0 defines tess factors. */
1390 skip_lds_store = !info->reads_tessfactor_outputs &&
1391 ctx->shader->selector->tcs_info.tessfactors_are_def_in_all_invocs;
1392 is_tess_factor = true;
1393 is_tess_inner = name == TGSI_SEMANTIC_TESSINNER;
1394 }
1395 }
1396 }
1397
1398 buffer = get_tess_ring_descriptor(ctx, TESS_OFFCHIP_RING_TCS);
1399
1400 base = ac_get_arg(&ctx->ac, ctx->tcs_offchip_offset);
1401
1402 addr = get_tcs_tes_buffer_address_from_generic_indices(ctx, vertex_index,
1403 param_index, driver_location,
1404 info->output_semantic_name,
1405 info->output_semantic_index,
1406 is_patch);
1407
1408 for (unsigned chan = 0; chan < 8; chan++) {
1409 if (!(writemask & (1 << chan)))
1410 continue;
1411 LLVMValueRef value = ac_llvm_extract_elem(&ctx->ac, src, chan - component);
1412
1413 unsigned buffer_store_offset = chan % 4;
1414 if (chan == 4) {
1415 addr = get_tcs_tes_buffer_address_from_generic_indices(ctx,
1416 vertex_index,
1417 param_index,
1418 driver_location + 1,
1419 info->output_semantic_name,
1420 info->output_semantic_index,
1421 is_patch);
1422 }
1423
1424 /* Skip LDS stores if there is no LDS read of this output. */
1425 if (!skip_lds_store)
1426 lshs_lds_store(ctx, chan, dw_addr, value);
1427
1428 value = ac_to_integer(&ctx->ac, value);
1429 values[chan] = value;
1430
1431 if (writemask != 0xF && !is_tess_factor) {
1432 ac_build_buffer_store_dword(&ctx->ac, buffer, value, 1,
1433 addr, base,
1434 4 * buffer_store_offset,
1435 ac_glc);
1436 }
1437
1438 /* Write tess factors into VGPRs for the epilog. */
1439 if (is_tess_factor &&
1440 ctx->shader->selector->tcs_info.tessfactors_are_def_in_all_invocs) {
1441 if (!is_tess_inner) {
1442 LLVMBuildStore(ctx->ac.builder, value, /* outer */
1443 ctx->invoc0_tess_factors[chan]);
1444 } else if (chan < 2) {
1445 LLVMBuildStore(ctx->ac.builder, value, /* inner */
1446 ctx->invoc0_tess_factors[4 + chan]);
1447 }
1448 }
1449 }
1450
1451 if (writemask == 0xF && !is_tess_factor) {
1452 LLVMValueRef value = ac_build_gather_values(&ctx->ac,
1453 values, 4);
1454 ac_build_buffer_store_dword(&ctx->ac, buffer, value, 4, addr,
1455 base, 0, ac_glc);
1456 }
1457 }
1458
1459 LLVMValueRef si_llvm_load_input_gs(struct ac_shader_abi *abi,
1460 unsigned input_index,
1461 unsigned vtx_offset_param,
1462 LLVMTypeRef type,
1463 unsigned swizzle)
1464 {
1465 struct si_shader_context *ctx = si_shader_context_from_abi(abi);
1466 struct lp_build_tgsi_context *bld_base = &ctx->bld_base;
1467 struct si_shader *shader = ctx->shader;
1468 LLVMValueRef vtx_offset, soffset;
1469 struct tgsi_shader_info *info = &shader->selector->info;
1470 unsigned semantic_name = info->input_semantic_name[input_index];
1471 unsigned semantic_index = info->input_semantic_index[input_index];
1472 unsigned param;
1473 LLVMValueRef value;
1474
1475 param = si_shader_io_get_unique_index(semantic_name, semantic_index, false);
1476
1477 /* GFX9 has the ESGS ring in LDS. */
1478 if (ctx->screen->info.chip_class >= GFX9) {
1479 unsigned index = vtx_offset_param;
1480
1481 switch (index / 2) {
1482 case 0:
1483 vtx_offset = si_unpack_param(ctx, ctx->gs_vtx01_offset,
1484 index % 2 ? 16 : 0, 16);
1485 break;
1486 case 1:
1487 vtx_offset = si_unpack_param(ctx, ctx->gs_vtx23_offset,
1488 index % 2 ? 16 : 0, 16);
1489 break;
1490 case 2:
1491 vtx_offset = si_unpack_param(ctx, ctx->gs_vtx45_offset,
1492 index % 2 ? 16 : 0, 16);
1493 break;
1494 default:
1495 assert(0);
1496 return NULL;
1497 }
1498
1499 unsigned offset = param * 4 + swizzle;
1500 vtx_offset = LLVMBuildAdd(ctx->ac.builder, vtx_offset,
1501 LLVMConstInt(ctx->i32, offset, false), "");
1502
1503 LLVMValueRef ptr = ac_build_gep0(&ctx->ac, ctx->esgs_ring, vtx_offset);
1504 LLVMValueRef value = LLVMBuildLoad(ctx->ac.builder, ptr, "");
1505 if (llvm_type_is_64bit(ctx, type)) {
1506 ptr = LLVMBuildGEP(ctx->ac.builder, ptr,
1507 &ctx->ac.i32_1, 1, "");
1508 LLVMValueRef values[2] = {
1509 value,
1510 LLVMBuildLoad(ctx->ac.builder, ptr, "")
1511 };
1512 value = ac_build_gather_values(&ctx->ac, values, 2);
1513 }
1514 return LLVMBuildBitCast(ctx->ac.builder, value, type, "");
1515 }
1516
1517 /* GFX6: input load from the ESGS ring in memory. */
1518 if (swizzle == ~0) {
1519 LLVMValueRef values[TGSI_NUM_CHANNELS];
1520 unsigned chan;
1521 for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) {
1522 values[chan] = si_llvm_load_input_gs(abi, input_index, vtx_offset_param,
1523 type, chan);
1524 }
1525 return ac_build_gather_values(&ctx->ac, values,
1526 TGSI_NUM_CHANNELS);
1527 }
1528
1529 /* Get the vertex offset parameter on GFX6. */
1530 LLVMValueRef gs_vtx_offset = ac_get_arg(&ctx->ac,
1531 ctx->gs_vtx_offset[vtx_offset_param]);
1532
1533 vtx_offset = LLVMBuildMul(ctx->ac.builder, gs_vtx_offset,
1534 LLVMConstInt(ctx->i32, 4, 0), "");
1535
1536 soffset = LLVMConstInt(ctx->i32, (param * 4 + swizzle) * 256, 0);
1537
1538 value = ac_build_buffer_load(&ctx->ac, ctx->esgs_ring, 1, ctx->i32_0,
1539 vtx_offset, soffset, 0, ac_glc, true, false);
1540 if (llvm_type_is_64bit(ctx, type)) {
1541 LLVMValueRef value2;
1542 soffset = LLVMConstInt(ctx->i32, (param * 4 + swizzle + 1) * 256, 0);
1543
1544 value2 = ac_build_buffer_load(&ctx->ac, ctx->esgs_ring, 1,
1545 ctx->i32_0, vtx_offset, soffset,
1546 0, ac_glc, true, false);
1547 return si_llvm_emit_fetch_64bit(bld_base, type, value, value2);
1548 }
1549 return LLVMBuildBitCast(ctx->ac.builder, value, type, "");
1550 }
1551
1552 static LLVMValueRef si_nir_load_input_gs(struct ac_shader_abi *abi,
1553 unsigned location,
1554 unsigned driver_location,
1555 unsigned component,
1556 unsigned num_components,
1557 unsigned vertex_index,
1558 unsigned const_index,
1559 LLVMTypeRef type)
1560 {
1561 struct si_shader_context *ctx = si_shader_context_from_abi(abi);
1562
1563 LLVMValueRef value[4];
1564 for (unsigned i = 0; i < num_components; i++) {
1565 unsigned offset = i;
1566 if (llvm_type_is_64bit(ctx, type))
1567 offset *= 2;
1568
1569 offset += component;
1570 value[i + component] = si_llvm_load_input_gs(&ctx->abi, driver_location / 4 + const_index,
1571 vertex_index, type, offset);
1572 }
1573
1574 return ac_build_varying_gather_values(&ctx->ac, value, num_components, component);
1575 }
1576
1577 static LLVMValueRef fetch_input_gs(
1578 struct lp_build_tgsi_context *bld_base,
1579 const struct tgsi_full_src_register *reg,
1580 enum tgsi_opcode_type type,
1581 unsigned swizzle_in)
1582 {
1583 struct si_shader_context *ctx = si_shader_context(bld_base);
1584 struct tgsi_shader_info *info = &ctx->shader->selector->info;
1585 unsigned swizzle = swizzle_in & 0xffff;
1586
1587 unsigned semantic_name = info->input_semantic_name[reg->Register.Index];
1588 if (swizzle != ~0 && semantic_name == TGSI_SEMANTIC_PRIMID)
1589 return si_get_primitive_id(ctx, swizzle);
1590
1591 if (!reg->Register.Dimension)
1592 return NULL;
1593
1594 return si_llvm_load_input_gs(&ctx->abi, reg->Register.Index,
1595 reg->Dimension.Index,
1596 tgsi2llvmtype(bld_base, type),
1597 swizzle);
1598 }
1599
1600 static int lookup_interp_param_index(unsigned interpolate, unsigned location)
1601 {
1602 switch (interpolate) {
1603 case TGSI_INTERPOLATE_CONSTANT:
1604 return 0;
1605
1606 case TGSI_INTERPOLATE_LINEAR:
1607 if (location == TGSI_INTERPOLATE_LOC_SAMPLE)
1608 return SI_PARAM_LINEAR_SAMPLE;
1609 else if (location == TGSI_INTERPOLATE_LOC_CENTROID)
1610 return SI_PARAM_LINEAR_CENTROID;
1611 else
1612 return SI_PARAM_LINEAR_CENTER;
1613 break;
1614 case TGSI_INTERPOLATE_COLOR:
1615 case TGSI_INTERPOLATE_PERSPECTIVE:
1616 if (location == TGSI_INTERPOLATE_LOC_SAMPLE)
1617 return SI_PARAM_PERSP_SAMPLE;
1618 else if (location == TGSI_INTERPOLATE_LOC_CENTROID)
1619 return SI_PARAM_PERSP_CENTROID;
1620 else
1621 return SI_PARAM_PERSP_CENTER;
1622 break;
1623 default:
1624 fprintf(stderr, "Warning: Unhandled interpolation mode.\n");
1625 return -1;
1626 }
1627 }
1628
1629 static LLVMValueRef si_build_fs_interp(struct si_shader_context *ctx,
1630 unsigned attr_index, unsigned chan,
1631 LLVMValueRef prim_mask,
1632 LLVMValueRef i, LLVMValueRef j)
1633 {
1634 if (i || j) {
1635 return ac_build_fs_interp(&ctx->ac,
1636 LLVMConstInt(ctx->i32, chan, 0),
1637 LLVMConstInt(ctx->i32, attr_index, 0),
1638 prim_mask, i, j);
1639 }
1640 return ac_build_fs_interp_mov(&ctx->ac,
1641 LLVMConstInt(ctx->i32, 2, 0), /* P0 */
1642 LLVMConstInt(ctx->i32, chan, 0),
1643 LLVMConstInt(ctx->i32, attr_index, 0),
1644 prim_mask);
1645 }
1646
1647 /**
1648 * Interpolate a fragment shader input.
1649 *
1650 * @param ctx context
1651 * @param input_index index of the input in hardware
1652 * @param semantic_name TGSI_SEMANTIC_*
1653 * @param semantic_index semantic index
1654 * @param num_interp_inputs number of all interpolated inputs (= BCOLOR offset)
1655 * @param colors_read_mask color components read (4 bits for each color, 8 bits in total)
1656 * @param interp_param interpolation weights (i,j)
1657 * @param prim_mask SI_PARAM_PRIM_MASK
1658 * @param face SI_PARAM_FRONT_FACE
1659 * @param result the return value (4 components)
1660 */
1661 static void interp_fs_input(struct si_shader_context *ctx,
1662 unsigned input_index,
1663 unsigned semantic_name,
1664 unsigned semantic_index,
1665 unsigned num_interp_inputs,
1666 unsigned colors_read_mask,
1667 LLVMValueRef interp_param,
1668 LLVMValueRef prim_mask,
1669 LLVMValueRef face,
1670 LLVMValueRef result[4])
1671 {
1672 LLVMValueRef i = NULL, j = NULL;
1673 unsigned chan;
1674
1675 /* fs.constant returns the param from the middle vertex, so it's not
1676 * really useful for flat shading. It's meant to be used for custom
1677 * interpolation (but the intrinsic can't fetch from the other two
1678 * vertices).
1679 *
1680 * Luckily, it doesn't matter, because we rely on the FLAT_SHADE state
1681 * to do the right thing. The only reason we use fs.constant is that
1682 * fs.interp cannot be used on integers, because they can be equal
1683 * to NaN.
1684 *
1685 * When interp is false we will use fs.constant or for newer llvm,
1686 * amdgcn.interp.mov.
1687 */
1688 bool interp = interp_param != NULL;
1689
1690 if (interp) {
1691 interp_param = LLVMBuildBitCast(ctx->ac.builder, interp_param,
1692 LLVMVectorType(ctx->f32, 2), "");
1693
1694 i = LLVMBuildExtractElement(ctx->ac.builder, interp_param,
1695 ctx->i32_0, "");
1696 j = LLVMBuildExtractElement(ctx->ac.builder, interp_param,
1697 ctx->i32_1, "");
1698 }
1699
1700 if (semantic_name == TGSI_SEMANTIC_COLOR &&
1701 ctx->shader->key.part.ps.prolog.color_two_side) {
1702 LLVMValueRef is_face_positive;
1703
1704 /* If BCOLOR0 is used, BCOLOR1 is at offset "num_inputs + 1",
1705 * otherwise it's at offset "num_inputs".
1706 */
1707 unsigned back_attr_offset = num_interp_inputs;
1708 if (semantic_index == 1 && colors_read_mask & 0xf)
1709 back_attr_offset += 1;
1710
1711 is_face_positive = LLVMBuildICmp(ctx->ac.builder, LLVMIntNE,
1712 face, ctx->i32_0, "");
1713
1714 for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) {
1715 LLVMValueRef front, back;
1716
1717 front = si_build_fs_interp(ctx,
1718 input_index, chan,
1719 prim_mask, i, j);
1720 back = si_build_fs_interp(ctx,
1721 back_attr_offset, chan,
1722 prim_mask, i, j);
1723
1724 result[chan] = LLVMBuildSelect(ctx->ac.builder,
1725 is_face_positive,
1726 front,
1727 back,
1728 "");
1729 }
1730 } else if (semantic_name == TGSI_SEMANTIC_FOG) {
1731 result[0] = si_build_fs_interp(ctx, input_index,
1732 0, prim_mask, i, j);
1733 result[1] =
1734 result[2] = LLVMConstReal(ctx->f32, 0.0f);
1735 result[3] = LLVMConstReal(ctx->f32, 1.0f);
1736 } else {
1737 for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) {
1738 result[chan] = si_build_fs_interp(ctx,
1739 input_index, chan,
1740 prim_mask, i, j);
1741 }
1742 }
1743 }
1744
1745 void si_llvm_load_input_fs(
1746 struct si_shader_context *ctx,
1747 unsigned input_index,
1748 LLVMValueRef out[4])
1749 {
1750 struct si_shader *shader = ctx->shader;
1751 struct tgsi_shader_info *info = &shader->selector->info;
1752 LLVMValueRef main_fn = ctx->main_fn;
1753 LLVMValueRef interp_param = NULL;
1754 int interp_param_idx;
1755 enum tgsi_semantic semantic_name = info->input_semantic_name[input_index];
1756 unsigned semantic_index = info->input_semantic_index[input_index];
1757 enum tgsi_interpolate_mode interp_mode = info->input_interpolate[input_index];
1758 enum tgsi_interpolate_loc interp_loc = info->input_interpolate_loc[input_index];
1759
1760 /* Get colors from input VGPRs (set by the prolog). */
1761 if (semantic_name == TGSI_SEMANTIC_COLOR) {
1762 unsigned colors_read = shader->selector->info.colors_read;
1763 unsigned mask = colors_read >> (semantic_index * 4);
1764 unsigned offset = SI_PARAM_POS_FIXED_PT + 1 +
1765 (semantic_index ? util_bitcount(colors_read & 0xf) : 0);
1766 LLVMValueRef undef = LLVMGetUndef(ctx->f32);
1767
1768 out[0] = mask & 0x1 ? LLVMGetParam(main_fn, offset++) : undef;
1769 out[1] = mask & 0x2 ? LLVMGetParam(main_fn, offset++) : undef;
1770 out[2] = mask & 0x4 ? LLVMGetParam(main_fn, offset++) : undef;
1771 out[3] = mask & 0x8 ? LLVMGetParam(main_fn, offset++) : undef;
1772 return;
1773 }
1774
1775 interp_param_idx = lookup_interp_param_index(interp_mode, interp_loc);
1776 if (interp_param_idx == -1)
1777 return;
1778 else if (interp_param_idx) {
1779 interp_param = LLVMGetParam(ctx->main_fn, interp_param_idx);
1780 }
1781
1782 interp_fs_input(ctx, input_index, semantic_name,
1783 semantic_index, 0, /* this param is unused */
1784 shader->selector->info.colors_read, interp_param,
1785 ac_get_arg(&ctx->ac, ctx->args.prim_mask),
1786 LLVMGetParam(main_fn, SI_PARAM_FRONT_FACE),
1787 &out[0]);
1788 }
1789
1790 static void declare_input_fs(
1791 struct si_shader_context *ctx,
1792 unsigned input_index,
1793 const struct tgsi_full_declaration *decl,
1794 LLVMValueRef out[4])
1795 {
1796 si_llvm_load_input_fs(ctx, input_index, out);
1797 }
1798
1799 LLVMValueRef si_get_sample_id(struct si_shader_context *ctx)
1800 {
1801 return si_unpack_param(ctx, ctx->args.ancillary, 8, 4);
1802 }
1803
1804 static LLVMValueRef get_base_vertex(struct ac_shader_abi *abi)
1805 {
1806 struct si_shader_context *ctx = si_shader_context_from_abi(abi);
1807
1808 /* For non-indexed draws, the base vertex set by the driver
1809 * (for direct draws) or the CP (for indirect draws) is the
1810 * first vertex ID, but GLSL expects 0 to be returned.
1811 */
1812 LLVMValueRef vs_state = ac_get_arg(&ctx->ac,
1813 ctx->vs_state_bits);
1814 LLVMValueRef indexed;
1815
1816 indexed = LLVMBuildLShr(ctx->ac.builder, vs_state, ctx->i32_1, "");
1817 indexed = LLVMBuildTrunc(ctx->ac.builder, indexed, ctx->i1, "");
1818
1819 return LLVMBuildSelect(ctx->ac.builder, indexed,
1820 ac_get_arg(&ctx->ac, ctx->args.base_vertex),
1821 ctx->i32_0, "");
1822 }
1823
1824 static LLVMValueRef get_block_size(struct ac_shader_abi *abi)
1825 {
1826 struct si_shader_context *ctx = si_shader_context_from_abi(abi);
1827
1828 LLVMValueRef values[3];
1829 LLVMValueRef result;
1830 unsigned i;
1831 unsigned *properties = ctx->shader->selector->info.properties;
1832
1833 if (properties[TGSI_PROPERTY_CS_FIXED_BLOCK_WIDTH] != 0) {
1834 unsigned sizes[3] = {
1835 properties[TGSI_PROPERTY_CS_FIXED_BLOCK_WIDTH],
1836 properties[TGSI_PROPERTY_CS_FIXED_BLOCK_HEIGHT],
1837 properties[TGSI_PROPERTY_CS_FIXED_BLOCK_DEPTH]
1838 };
1839
1840 for (i = 0; i < 3; ++i)
1841 values[i] = LLVMConstInt(ctx->i32, sizes[i], 0);
1842
1843 result = ac_build_gather_values(&ctx->ac, values, 3);
1844 } else {
1845 result = ac_get_arg(&ctx->ac, ctx->block_size);
1846 }
1847
1848 return result;
1849 }
1850
1851 /**
1852 * Load a dword from a constant buffer.
1853 */
1854 static LLVMValueRef buffer_load_const(struct si_shader_context *ctx,
1855 LLVMValueRef resource,
1856 LLVMValueRef offset)
1857 {
1858 return ac_build_buffer_load(&ctx->ac, resource, 1, NULL, offset, NULL,
1859 0, 0, true, true);
1860 }
1861
1862 static LLVMValueRef load_sample_position(struct ac_shader_abi *abi, LLVMValueRef sample_id)
1863 {
1864 struct si_shader_context *ctx = si_shader_context_from_abi(abi);
1865 LLVMValueRef desc = ac_get_arg(&ctx->ac, ctx->rw_buffers);
1866 LLVMValueRef buf_index = LLVMConstInt(ctx->i32, SI_PS_CONST_SAMPLE_POSITIONS, 0);
1867 LLVMValueRef resource = ac_build_load_to_sgpr(&ctx->ac, desc, buf_index);
1868
1869 /* offset = sample_id * 8 (8 = 2 floats containing samplepos.xy) */
1870 LLVMValueRef offset0 = LLVMBuildMul(ctx->ac.builder, sample_id, LLVMConstInt(ctx->i32, 8, 0), "");
1871 LLVMValueRef offset1 = LLVMBuildAdd(ctx->ac.builder, offset0, LLVMConstInt(ctx->i32, 4, 0), "");
1872
1873 LLVMValueRef pos[4] = {
1874 buffer_load_const(ctx, resource, offset0),
1875 buffer_load_const(ctx, resource, offset1),
1876 LLVMConstReal(ctx->f32, 0),
1877 LLVMConstReal(ctx->f32, 0)
1878 };
1879
1880 return ac_build_gather_values(&ctx->ac, pos, 4);
1881 }
1882
1883 static LLVMValueRef load_sample_mask_in(struct ac_shader_abi *abi)
1884 {
1885 struct si_shader_context *ctx = si_shader_context_from_abi(abi);
1886 return ac_to_integer(&ctx->ac, ac_get_arg(&ctx->ac, ctx->args.sample_coverage));
1887 }
1888
1889 static LLVMValueRef si_load_tess_coord(struct ac_shader_abi *abi)
1890 {
1891 struct si_shader_context *ctx = si_shader_context_from_abi(abi);
1892 LLVMValueRef coord[4] = {
1893 ac_get_arg(&ctx->ac, ctx->tes_u),
1894 ac_get_arg(&ctx->ac, ctx->tes_v),
1895 ctx->ac.f32_0,
1896 ctx->ac.f32_0
1897 };
1898
1899 /* For triangles, the vector should be (u, v, 1-u-v). */
1900 if (ctx->shader->selector->info.properties[TGSI_PROPERTY_TES_PRIM_MODE] ==
1901 PIPE_PRIM_TRIANGLES) {
1902 coord[2] = LLVMBuildFSub(ctx->ac.builder, ctx->ac.f32_1,
1903 LLVMBuildFAdd(ctx->ac.builder,
1904 coord[0], coord[1], ""), "");
1905 }
1906 return ac_build_gather_values(&ctx->ac, coord, 4);
1907 }
1908
1909 static LLVMValueRef load_tess_level(struct si_shader_context *ctx,
1910 unsigned semantic_name)
1911 {
1912 LLVMValueRef base, addr;
1913
1914 int param = si_shader_io_get_unique_index_patch(semantic_name, 0);
1915
1916 base = ac_get_arg(&ctx->ac, ctx->tcs_offchip_offset);
1917 addr = get_tcs_tes_buffer_address(ctx, get_rel_patch_id(ctx), NULL,
1918 LLVMConstInt(ctx->i32, param, 0));
1919
1920 return buffer_load(&ctx->bld_base, ctx->f32,
1921 ~0, ctx->tess_offchip_ring, base, addr, true);
1922
1923 }
1924
1925 static LLVMValueRef load_tess_level_default(struct si_shader_context *ctx,
1926 unsigned semantic_name)
1927 {
1928 LLVMValueRef buf, slot, val[4];
1929 int i, offset;
1930
1931 slot = LLVMConstInt(ctx->i32, SI_HS_CONST_DEFAULT_TESS_LEVELS, 0);
1932 buf = ac_get_arg(&ctx->ac, ctx->rw_buffers);
1933 buf = ac_build_load_to_sgpr(&ctx->ac, buf, slot);
1934 offset = semantic_name == TGSI_SEMANTIC_TESS_DEFAULT_INNER_LEVEL ? 4 : 0;
1935
1936 for (i = 0; i < 4; i++)
1937 val[i] = buffer_load_const(ctx, buf,
1938 LLVMConstInt(ctx->i32, (offset + i) * 4, 0));
1939 return ac_build_gather_values(&ctx->ac, val, 4);
1940 }
1941
1942 static LLVMValueRef si_load_tess_level(struct ac_shader_abi *abi,
1943 unsigned varying_id,
1944 bool load_default_state)
1945 {
1946 struct si_shader_context *ctx = si_shader_context_from_abi(abi);
1947 unsigned semantic_name;
1948
1949 if (load_default_state) {
1950 switch (varying_id) {
1951 case VARYING_SLOT_TESS_LEVEL_INNER:
1952 semantic_name = TGSI_SEMANTIC_TESS_DEFAULT_INNER_LEVEL;
1953 break;
1954 case VARYING_SLOT_TESS_LEVEL_OUTER:
1955 semantic_name = TGSI_SEMANTIC_TESS_DEFAULT_OUTER_LEVEL;
1956 break;
1957 default:
1958 unreachable("unknown tess level");
1959 }
1960 return load_tess_level_default(ctx, semantic_name);
1961 }
1962
1963 switch (varying_id) {
1964 case VARYING_SLOT_TESS_LEVEL_INNER:
1965 semantic_name = TGSI_SEMANTIC_TESSINNER;
1966 break;
1967 case VARYING_SLOT_TESS_LEVEL_OUTER:
1968 semantic_name = TGSI_SEMANTIC_TESSOUTER;
1969 break;
1970 default:
1971 unreachable("unknown tess level");
1972 }
1973
1974 return load_tess_level(ctx, semantic_name);
1975
1976 }
1977
1978 static LLVMValueRef si_load_patch_vertices_in(struct ac_shader_abi *abi)
1979 {
1980 struct si_shader_context *ctx = si_shader_context_from_abi(abi);
1981 if (ctx->type == PIPE_SHADER_TESS_CTRL)
1982 return si_unpack_param(ctx, ctx->tcs_out_lds_layout, 13, 6);
1983 else if (ctx->type == PIPE_SHADER_TESS_EVAL)
1984 return get_num_tcs_out_vertices(ctx);
1985 else
1986 unreachable("invalid shader stage for TGSI_SEMANTIC_VERTICESIN");
1987 }
1988
1989 void si_load_system_value(struct si_shader_context *ctx,
1990 unsigned index,
1991 const struct tgsi_full_declaration *decl)
1992 {
1993 LLVMValueRef value = 0;
1994
1995 assert(index < RADEON_LLVM_MAX_SYSTEM_VALUES);
1996
1997 switch (decl->Semantic.Name) {
1998 case TGSI_SEMANTIC_INSTANCEID:
1999 value = ctx->abi.instance_id;
2000 break;
2001
2002 case TGSI_SEMANTIC_VERTEXID:
2003 value = LLVMBuildAdd(ctx->ac.builder,
2004 ctx->abi.vertex_id,
2005 ac_get_arg(&ctx->ac, ctx->args.base_vertex), "");
2006 break;
2007
2008 case TGSI_SEMANTIC_VERTEXID_NOBASE:
2009 /* Unused. Clarify the meaning in indexed vs. non-indexed
2010 * draws if this is ever used again. */
2011 assert(false);
2012 break;
2013
2014 case TGSI_SEMANTIC_BASEVERTEX:
2015 value = get_base_vertex(&ctx->abi);
2016 break;
2017
2018 case TGSI_SEMANTIC_BASEINSTANCE:
2019 value = ac_get_arg(&ctx->ac, ctx->args.start_instance);
2020 break;
2021
2022 case TGSI_SEMANTIC_DRAWID:
2023 value = ac_get_arg(&ctx->ac, ctx->args.draw_id);
2024 break;
2025
2026 case TGSI_SEMANTIC_INVOCATIONID:
2027 if (ctx->type == PIPE_SHADER_TESS_CTRL) {
2028 value = si_unpack_param(ctx, ctx->args.tcs_rel_ids, 8, 5);
2029 } else if (ctx->type == PIPE_SHADER_GEOMETRY) {
2030 if (ctx->screen->info.chip_class >= GFX10) {
2031 value = LLVMBuildAnd(ctx->ac.builder,
2032 ac_get_arg(&ctx->ac, ctx->args.gs_invocation_id),
2033 LLVMConstInt(ctx->i32, 127, 0), "");
2034 } else {
2035 value = ac_get_arg(&ctx->ac, ctx->args.gs_invocation_id);
2036 }
2037 } else {
2038 assert(!"INVOCATIONID not implemented");
2039 }
2040 break;
2041
2042 case TGSI_SEMANTIC_POSITION:
2043 {
2044 LLVMValueRef pos[4] = {
2045 LLVMGetParam(ctx->main_fn, SI_PARAM_POS_X_FLOAT),
2046 LLVMGetParam(ctx->main_fn, SI_PARAM_POS_Y_FLOAT),
2047 LLVMGetParam(ctx->main_fn, SI_PARAM_POS_Z_FLOAT),
2048 ac_build_fdiv(&ctx->ac, ctx->ac.f32_1,
2049 LLVMGetParam(ctx->main_fn, SI_PARAM_POS_W_FLOAT)),
2050 };
2051 value = ac_build_gather_values(&ctx->ac, pos, 4);
2052 break;
2053 }
2054
2055 case TGSI_SEMANTIC_FACE:
2056 value = ac_get_arg(&ctx->ac, ctx->args.front_face);
2057 break;
2058
2059 case TGSI_SEMANTIC_SAMPLEID:
2060 value = si_get_sample_id(ctx);
2061 break;
2062
2063 case TGSI_SEMANTIC_SAMPLEPOS: {
2064 LLVMValueRef pos[4] = {
2065 LLVMGetParam(ctx->main_fn, SI_PARAM_POS_X_FLOAT),
2066 LLVMGetParam(ctx->main_fn, SI_PARAM_POS_Y_FLOAT),
2067 LLVMConstReal(ctx->f32, 0),
2068 LLVMConstReal(ctx->f32, 0)
2069 };
2070 pos[0] = ac_build_fract(&ctx->ac, pos[0], 32);
2071 pos[1] = ac_build_fract(&ctx->ac, pos[1], 32);
2072 value = ac_build_gather_values(&ctx->ac, pos, 4);
2073 break;
2074 }
2075
2076 case TGSI_SEMANTIC_SAMPLEMASK:
2077 /* This can only occur with the OpenGL Core profile, which
2078 * doesn't support smoothing.
2079 */
2080 value = LLVMGetParam(ctx->main_fn, SI_PARAM_SAMPLE_COVERAGE);
2081 break;
2082
2083 case TGSI_SEMANTIC_TESSCOORD:
2084 value = si_load_tess_coord(&ctx->abi);
2085 break;
2086
2087 case TGSI_SEMANTIC_VERTICESIN:
2088 value = si_load_patch_vertices_in(&ctx->abi);
2089 break;
2090
2091 case TGSI_SEMANTIC_TESSINNER:
2092 case TGSI_SEMANTIC_TESSOUTER:
2093 value = load_tess_level(ctx, decl->Semantic.Name);
2094 break;
2095
2096 case TGSI_SEMANTIC_TESS_DEFAULT_OUTER_LEVEL:
2097 case TGSI_SEMANTIC_TESS_DEFAULT_INNER_LEVEL:
2098 value = load_tess_level_default(ctx, decl->Semantic.Name);
2099 break;
2100
2101 case TGSI_SEMANTIC_PRIMID:
2102 value = si_get_primitive_id(ctx, 0);
2103 break;
2104
2105 case TGSI_SEMANTIC_GRID_SIZE:
2106 value = ac_get_arg(&ctx->ac, ctx->args.num_work_groups);
2107 break;
2108
2109 case TGSI_SEMANTIC_BLOCK_SIZE:
2110 value = get_block_size(&ctx->abi);
2111 break;
2112
2113 case TGSI_SEMANTIC_BLOCK_ID:
2114 {
2115 LLVMValueRef values[3];
2116
2117 for (int i = 0; i < 3; i++) {
2118 values[i] = ctx->i32_0;
2119 if (ctx->args.workgroup_ids[i].used) {
2120 values[i] = ac_get_arg(&ctx->ac, ctx->args.workgroup_ids[i]);
2121 }
2122 }
2123 value = ac_build_gather_values(&ctx->ac, values, 3);
2124 break;
2125 }
2126
2127 case TGSI_SEMANTIC_THREAD_ID:
2128 value = ac_get_arg(&ctx->ac, ctx->args.local_invocation_ids);
2129 break;
2130
2131 case TGSI_SEMANTIC_HELPER_INVOCATION:
2132 value = ac_build_load_helper_invocation(&ctx->ac);
2133 break;
2134
2135 case TGSI_SEMANTIC_SUBGROUP_SIZE:
2136 value = LLVMConstInt(ctx->i32, ctx->ac.wave_size, 0);
2137 break;
2138
2139 case TGSI_SEMANTIC_SUBGROUP_INVOCATION:
2140 value = ac_get_thread_id(&ctx->ac);
2141 break;
2142
2143 case TGSI_SEMANTIC_SUBGROUP_EQ_MASK:
2144 {
2145 LLVMValueRef id = ac_get_thread_id(&ctx->ac);
2146 if (ctx->ac.wave_size == 64)
2147 id = LLVMBuildZExt(ctx->ac.builder, id, ctx->i64, "");
2148 value = LLVMBuildShl(ctx->ac.builder,
2149 LLVMConstInt(ctx->ac.iN_wavemask, 1, 0), id, "");
2150 if (ctx->ac.wave_size == 32)
2151 value = LLVMBuildZExt(ctx->ac.builder, value, ctx->i64, "");
2152 value = LLVMBuildBitCast(ctx->ac.builder, value, ctx->v2i32, "");
2153 break;
2154 }
2155
2156 case TGSI_SEMANTIC_SUBGROUP_GE_MASK:
2157 case TGSI_SEMANTIC_SUBGROUP_GT_MASK:
2158 case TGSI_SEMANTIC_SUBGROUP_LE_MASK:
2159 case TGSI_SEMANTIC_SUBGROUP_LT_MASK:
2160 {
2161 LLVMValueRef id = ac_get_thread_id(&ctx->ac);
2162 if (decl->Semantic.Name == TGSI_SEMANTIC_SUBGROUP_GT_MASK ||
2163 decl->Semantic.Name == TGSI_SEMANTIC_SUBGROUP_LE_MASK) {
2164 /* All bits set except LSB */
2165 value = LLVMConstInt(ctx->ac.iN_wavemask, -2, 0);
2166 } else {
2167 /* All bits set */
2168 value = LLVMConstInt(ctx->ac.iN_wavemask, -1, 0);
2169 }
2170 if (ctx->ac.wave_size == 64)
2171 id = LLVMBuildZExt(ctx->ac.builder, id, ctx->i64, "");
2172 value = LLVMBuildShl(ctx->ac.builder, value, id, "");
2173 if (decl->Semantic.Name == TGSI_SEMANTIC_SUBGROUP_LE_MASK ||
2174 decl->Semantic.Name == TGSI_SEMANTIC_SUBGROUP_LT_MASK)
2175 value = LLVMBuildNot(ctx->ac.builder, value, "");
2176 if (ctx->ac.wave_size == 32)
2177 value = LLVMBuildZExt(ctx->ac.builder, value, ctx->i64, "");
2178 value = LLVMBuildBitCast(ctx->ac.builder, value, ctx->v2i32, "");
2179 break;
2180 }
2181
2182 case TGSI_SEMANTIC_CS_USER_DATA_AMD:
2183 value = ac_get_arg(&ctx->ac, ctx->cs_user_data);
2184 break;
2185
2186 default:
2187 assert(!"unknown system value");
2188 return;
2189 }
2190
2191 ctx->system_values[index] = value;
2192 }
2193
2194 void si_declare_compute_memory(struct si_shader_context *ctx)
2195 {
2196 struct si_shader_selector *sel = ctx->shader->selector;
2197 unsigned lds_size = sel->info.properties[TGSI_PROPERTY_CS_LOCAL_SIZE];
2198
2199 LLVMTypeRef i8p = LLVMPointerType(ctx->i8, AC_ADDR_SPACE_LDS);
2200 LLVMValueRef var;
2201
2202 assert(!ctx->ac.lds);
2203
2204 var = LLVMAddGlobalInAddressSpace(ctx->ac.module,
2205 LLVMArrayType(ctx->i8, lds_size),
2206 "compute_lds",
2207 AC_ADDR_SPACE_LDS);
2208 LLVMSetAlignment(var, 64 * 1024);
2209
2210 ctx->ac.lds = LLVMBuildBitCast(ctx->ac.builder, var, i8p, "");
2211 }
2212
2213 void si_tgsi_declare_compute_memory(struct si_shader_context *ctx,
2214 const struct tgsi_full_declaration *decl)
2215 {
2216 assert(decl->Declaration.MemType == TGSI_MEMORY_TYPE_SHARED);
2217 assert(decl->Range.First == decl->Range.Last);
2218
2219 si_declare_compute_memory(ctx);
2220 }
2221
2222 static LLVMValueRef load_const_buffer_desc_fast_path(struct si_shader_context *ctx)
2223 {
2224 LLVMValueRef ptr =
2225 ac_get_arg(&ctx->ac, ctx->const_and_shader_buffers);
2226 struct si_shader_selector *sel = ctx->shader->selector;
2227
2228 /* Do the bounds checking with a descriptor, because
2229 * doing computation and manual bounds checking of 64-bit
2230 * addresses generates horrible VALU code with very high
2231 * VGPR usage and very low SIMD occupancy.
2232 */
2233 ptr = LLVMBuildPtrToInt(ctx->ac.builder, ptr, ctx->ac.intptr, "");
2234
2235 LLVMValueRef desc0, desc1;
2236 desc0 = ptr;
2237 desc1 = LLVMConstInt(ctx->i32,
2238 S_008F04_BASE_ADDRESS_HI(ctx->screen->info.address32_hi), 0);
2239
2240 uint32_t rsrc3 = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) |
2241 S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
2242 S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) |
2243 S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W);
2244
2245 if (ctx->screen->info.chip_class >= GFX10)
2246 rsrc3 |= S_008F0C_FORMAT(V_008F0C_IMG_FORMAT_32_FLOAT) |
2247 S_008F0C_OOB_SELECT(3) |
2248 S_008F0C_RESOURCE_LEVEL(1);
2249 else
2250 rsrc3 |= S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
2251 S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
2252
2253 LLVMValueRef desc_elems[] = {
2254 desc0,
2255 desc1,
2256 LLVMConstInt(ctx->i32, (sel->info.const_file_max[0] + 1) * 16, 0),
2257 LLVMConstInt(ctx->i32, rsrc3, false)
2258 };
2259
2260 return ac_build_gather_values(&ctx->ac, desc_elems, 4);
2261 }
2262
2263 static LLVMValueRef load_const_buffer_desc(struct si_shader_context *ctx, int i)
2264 {
2265 LLVMValueRef list_ptr = ac_get_arg(&ctx->ac,
2266 ctx->const_and_shader_buffers);
2267
2268 return ac_build_load_to_sgpr(&ctx->ac, list_ptr,
2269 LLVMConstInt(ctx->i32, si_get_constbuf_slot(i), 0));
2270 }
2271
2272 static LLVMValueRef load_ubo(struct ac_shader_abi *abi, LLVMValueRef index)
2273 {
2274 struct si_shader_context *ctx = si_shader_context_from_abi(abi);
2275 struct si_shader_selector *sel = ctx->shader->selector;
2276
2277 LLVMValueRef ptr = ac_get_arg(&ctx->ac, ctx->const_and_shader_buffers);
2278
2279 if (sel->info.const_buffers_declared == 1 &&
2280 sel->info.shader_buffers_declared == 0) {
2281 return load_const_buffer_desc_fast_path(ctx);
2282 }
2283
2284 index = si_llvm_bound_index(ctx, index, ctx->num_const_buffers);
2285 index = LLVMBuildAdd(ctx->ac.builder, index,
2286 LLVMConstInt(ctx->i32, SI_NUM_SHADER_BUFFERS, 0), "");
2287
2288 return ac_build_load_to_sgpr(&ctx->ac, ptr, index);
2289 }
2290
2291 static LLVMValueRef
2292 load_ssbo(struct ac_shader_abi *abi, LLVMValueRef index, bool write)
2293 {
2294 struct si_shader_context *ctx = si_shader_context_from_abi(abi);
2295 LLVMValueRef rsrc_ptr = ac_get_arg(&ctx->ac,
2296 ctx->const_and_shader_buffers);
2297
2298 index = si_llvm_bound_index(ctx, index, ctx->num_shader_buffers);
2299 index = LLVMBuildSub(ctx->ac.builder,
2300 LLVMConstInt(ctx->i32, SI_NUM_SHADER_BUFFERS - 1, 0),
2301 index, "");
2302
2303 return ac_build_load_to_sgpr(&ctx->ac, rsrc_ptr, index);
2304 }
2305
2306 static LLVMValueRef fetch_constant(
2307 struct lp_build_tgsi_context *bld_base,
2308 const struct tgsi_full_src_register *reg,
2309 enum tgsi_opcode_type type,
2310 unsigned swizzle_in)
2311 {
2312 struct si_shader_context *ctx = si_shader_context(bld_base);
2313 struct si_shader_selector *sel = ctx->shader->selector;
2314 const struct tgsi_ind_register *ireg = &reg->Indirect;
2315 unsigned buf, idx;
2316 unsigned swizzle = swizzle_in & 0xffff;
2317
2318 LLVMValueRef addr, bufp;
2319
2320 if (swizzle_in == LP_CHAN_ALL) {
2321 unsigned chan;
2322 LLVMValueRef values[4];
2323 for (chan = 0; chan < TGSI_NUM_CHANNELS; ++chan)
2324 values[chan] = fetch_constant(bld_base, reg, type, chan);
2325
2326 return ac_build_gather_values(&ctx->ac, values, 4);
2327 }
2328
2329 /* Split 64-bit loads. */
2330 if (tgsi_type_is_64bit(type)) {
2331 LLVMValueRef lo, hi;
2332
2333 lo = fetch_constant(bld_base, reg, TGSI_TYPE_UNSIGNED, swizzle);
2334 hi = fetch_constant(bld_base, reg, TGSI_TYPE_UNSIGNED, (swizzle_in >> 16));
2335 return si_llvm_emit_fetch_64bit(bld_base, tgsi2llvmtype(bld_base, type),
2336 lo, hi);
2337 }
2338
2339 idx = reg->Register.Index * 4 + swizzle;
2340 if (reg->Register.Indirect) {
2341 addr = si_get_indirect_index(ctx, ireg, 16, idx * 4);
2342 } else {
2343 addr = LLVMConstInt(ctx->i32, idx * 4, 0);
2344 }
2345
2346 /* Fast path when user data SGPRs point to constant buffer 0 directly. */
2347 if (sel->info.const_buffers_declared == 1 &&
2348 sel->info.shader_buffers_declared == 0) {
2349 LLVMValueRef desc = load_const_buffer_desc_fast_path(ctx);
2350 LLVMValueRef result = buffer_load_const(ctx, desc, addr);
2351 return bitcast(bld_base, type, result);
2352 }
2353
2354 assert(reg->Register.Dimension);
2355 buf = reg->Dimension.Index;
2356
2357 if (reg->Dimension.Indirect) {
2358 LLVMValueRef ptr = ac_get_arg(&ctx->ac, ctx->const_and_shader_buffers);
2359 LLVMValueRef index;
2360 index = si_get_bounded_indirect_index(ctx, &reg->DimIndirect,
2361 reg->Dimension.Index,
2362 ctx->num_const_buffers);
2363 index = LLVMBuildAdd(ctx->ac.builder, index,
2364 LLVMConstInt(ctx->i32, SI_NUM_SHADER_BUFFERS, 0), "");
2365 bufp = ac_build_load_to_sgpr(&ctx->ac, ptr, index);
2366 } else
2367 bufp = load_const_buffer_desc(ctx, buf);
2368
2369 return bitcast(bld_base, type, buffer_load_const(ctx, bufp, addr));
2370 }
2371
2372 /* Initialize arguments for the shader export intrinsic */
2373 static void si_llvm_init_export_args(struct si_shader_context *ctx,
2374 LLVMValueRef *values,
2375 unsigned target,
2376 struct ac_export_args *args)
2377 {
2378 LLVMValueRef f32undef = LLVMGetUndef(ctx->ac.f32);
2379 unsigned spi_shader_col_format = V_028714_SPI_SHADER_32_ABGR;
2380 unsigned chan;
2381 bool is_int8, is_int10;
2382
2383 /* Default is 0xf. Adjusted below depending on the format. */
2384 args->enabled_channels = 0xf; /* writemask */
2385
2386 /* Specify whether the EXEC mask represents the valid mask */
2387 args->valid_mask = 0;
2388
2389 /* Specify whether this is the last export */
2390 args->done = 0;
2391
2392 /* Specify the target we are exporting */
2393 args->target = target;
2394
2395 if (ctx->type == PIPE_SHADER_FRAGMENT) {
2396 const struct si_shader_key *key = &ctx->shader->key;
2397 unsigned col_formats = key->part.ps.epilog.spi_shader_col_format;
2398 int cbuf = target - V_008DFC_SQ_EXP_MRT;
2399
2400 assert(cbuf >= 0 && cbuf < 8);
2401 spi_shader_col_format = (col_formats >> (cbuf * 4)) & 0xf;
2402 is_int8 = (key->part.ps.epilog.color_is_int8 >> cbuf) & 0x1;
2403 is_int10 = (key->part.ps.epilog.color_is_int10 >> cbuf) & 0x1;
2404 }
2405
2406 args->compr = false;
2407 args->out[0] = f32undef;
2408 args->out[1] = f32undef;
2409 args->out[2] = f32undef;
2410 args->out[3] = f32undef;
2411
2412 LLVMValueRef (*packf)(struct ac_llvm_context *ctx, LLVMValueRef args[2]) = NULL;
2413 LLVMValueRef (*packi)(struct ac_llvm_context *ctx, LLVMValueRef args[2],
2414 unsigned bits, bool hi) = NULL;
2415
2416 switch (spi_shader_col_format) {
2417 case V_028714_SPI_SHADER_ZERO:
2418 args->enabled_channels = 0; /* writemask */
2419 args->target = V_008DFC_SQ_EXP_NULL;
2420 break;
2421
2422 case V_028714_SPI_SHADER_32_R:
2423 args->enabled_channels = 1; /* writemask */
2424 args->out[0] = values[0];
2425 break;
2426
2427 case V_028714_SPI_SHADER_32_GR:
2428 args->enabled_channels = 0x3; /* writemask */
2429 args->out[0] = values[0];
2430 args->out[1] = values[1];
2431 break;
2432
2433 case V_028714_SPI_SHADER_32_AR:
2434 if (ctx->screen->info.chip_class >= GFX10) {
2435 args->enabled_channels = 0x3; /* writemask */
2436 args->out[0] = values[0];
2437 args->out[1] = values[3];
2438 } else {
2439 args->enabled_channels = 0x9; /* writemask */
2440 args->out[0] = values[0];
2441 args->out[3] = values[3];
2442 }
2443 break;
2444
2445 case V_028714_SPI_SHADER_FP16_ABGR:
2446 packf = ac_build_cvt_pkrtz_f16;
2447 break;
2448
2449 case V_028714_SPI_SHADER_UNORM16_ABGR:
2450 packf = ac_build_cvt_pknorm_u16;
2451 break;
2452
2453 case V_028714_SPI_SHADER_SNORM16_ABGR:
2454 packf = ac_build_cvt_pknorm_i16;
2455 break;
2456
2457 case V_028714_SPI_SHADER_UINT16_ABGR:
2458 packi = ac_build_cvt_pk_u16;
2459 break;
2460
2461 case V_028714_SPI_SHADER_SINT16_ABGR:
2462 packi = ac_build_cvt_pk_i16;
2463 break;
2464
2465 case V_028714_SPI_SHADER_32_ABGR:
2466 memcpy(&args->out[0], values, sizeof(values[0]) * 4);
2467 break;
2468 }
2469
2470 /* Pack f16 or norm_i16/u16. */
2471 if (packf) {
2472 for (chan = 0; chan < 2; chan++) {
2473 LLVMValueRef pack_args[2] = {
2474 values[2 * chan],
2475 values[2 * chan + 1]
2476 };
2477 LLVMValueRef packed;
2478
2479 packed = packf(&ctx->ac, pack_args);
2480 args->out[chan] = ac_to_float(&ctx->ac, packed);
2481 }
2482 args->compr = 1; /* COMPR flag */
2483 }
2484 /* Pack i16/u16. */
2485 if (packi) {
2486 for (chan = 0; chan < 2; chan++) {
2487 LLVMValueRef pack_args[2] = {
2488 ac_to_integer(&ctx->ac, values[2 * chan]),
2489 ac_to_integer(&ctx->ac, values[2 * chan + 1])
2490 };
2491 LLVMValueRef packed;
2492
2493 packed = packi(&ctx->ac, pack_args,
2494 is_int8 ? 8 : is_int10 ? 10 : 16,
2495 chan == 1);
2496 args->out[chan] = ac_to_float(&ctx->ac, packed);
2497 }
2498 args->compr = 1; /* COMPR flag */
2499 }
2500 }
2501
2502 static void si_alpha_test(struct lp_build_tgsi_context *bld_base,
2503 LLVMValueRef alpha)
2504 {
2505 struct si_shader_context *ctx = si_shader_context(bld_base);
2506
2507 if (ctx->shader->key.part.ps.epilog.alpha_func != PIPE_FUNC_NEVER) {
2508 static LLVMRealPredicate cond_map[PIPE_FUNC_ALWAYS + 1] = {
2509 [PIPE_FUNC_LESS] = LLVMRealOLT,
2510 [PIPE_FUNC_EQUAL] = LLVMRealOEQ,
2511 [PIPE_FUNC_LEQUAL] = LLVMRealOLE,
2512 [PIPE_FUNC_GREATER] = LLVMRealOGT,
2513 [PIPE_FUNC_NOTEQUAL] = LLVMRealONE,
2514 [PIPE_FUNC_GEQUAL] = LLVMRealOGE,
2515 };
2516 LLVMRealPredicate cond = cond_map[ctx->shader->key.part.ps.epilog.alpha_func];
2517 assert(cond);
2518
2519 LLVMValueRef alpha_ref = LLVMGetParam(ctx->main_fn,
2520 SI_PARAM_ALPHA_REF);
2521 LLVMValueRef alpha_pass =
2522 LLVMBuildFCmp(ctx->ac.builder, cond, alpha, alpha_ref, "");
2523 ac_build_kill_if_false(&ctx->ac, alpha_pass);
2524 } else {
2525 ac_build_kill_if_false(&ctx->ac, ctx->i1false);
2526 }
2527 }
2528
2529 static LLVMValueRef si_scale_alpha_by_sample_mask(struct lp_build_tgsi_context *bld_base,
2530 LLVMValueRef alpha,
2531 unsigned samplemask_param)
2532 {
2533 struct si_shader_context *ctx = si_shader_context(bld_base);
2534 LLVMValueRef coverage;
2535
2536 /* alpha = alpha * popcount(coverage) / SI_NUM_SMOOTH_AA_SAMPLES */
2537 coverage = LLVMGetParam(ctx->main_fn,
2538 samplemask_param);
2539 coverage = ac_to_integer(&ctx->ac, coverage);
2540
2541 coverage = ac_build_intrinsic(&ctx->ac, "llvm.ctpop.i32",
2542 ctx->i32,
2543 &coverage, 1, AC_FUNC_ATTR_READNONE);
2544
2545 coverage = LLVMBuildUIToFP(ctx->ac.builder, coverage,
2546 ctx->f32, "");
2547
2548 coverage = LLVMBuildFMul(ctx->ac.builder, coverage,
2549 LLVMConstReal(ctx->f32,
2550 1.0 / SI_NUM_SMOOTH_AA_SAMPLES), "");
2551
2552 return LLVMBuildFMul(ctx->ac.builder, alpha, coverage, "");
2553 }
2554
2555 static void si_llvm_emit_clipvertex(struct si_shader_context *ctx,
2556 struct ac_export_args *pos, LLVMValueRef *out_elts)
2557 {
2558 unsigned reg_index;
2559 unsigned chan;
2560 unsigned const_chan;
2561 LLVMValueRef base_elt;
2562 LLVMValueRef ptr = ac_get_arg(&ctx->ac, ctx->rw_buffers);
2563 LLVMValueRef constbuf_index = LLVMConstInt(ctx->i32,
2564 SI_VS_CONST_CLIP_PLANES, 0);
2565 LLVMValueRef const_resource = ac_build_load_to_sgpr(&ctx->ac, ptr, constbuf_index);
2566
2567 for (reg_index = 0; reg_index < 2; reg_index ++) {
2568 struct ac_export_args *args = &pos[2 + reg_index];
2569
2570 args->out[0] =
2571 args->out[1] =
2572 args->out[2] =
2573 args->out[3] = LLVMConstReal(ctx->f32, 0.0f);
2574
2575 /* Compute dot products of position and user clip plane vectors */
2576 for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) {
2577 for (const_chan = 0; const_chan < TGSI_NUM_CHANNELS; const_chan++) {
2578 LLVMValueRef addr =
2579 LLVMConstInt(ctx->i32, ((reg_index * 4 + chan) * 4 +
2580 const_chan) * 4, 0);
2581 base_elt = buffer_load_const(ctx, const_resource,
2582 addr);
2583 args->out[chan] = ac_build_fmad(&ctx->ac, base_elt,
2584 out_elts[const_chan], args->out[chan]);
2585 }
2586 }
2587
2588 args->enabled_channels = 0xf;
2589 args->valid_mask = 0;
2590 args->done = 0;
2591 args->target = V_008DFC_SQ_EXP_POS + 2 + reg_index;
2592 args->compr = 0;
2593 }
2594 }
2595
2596 static void si_dump_streamout(struct pipe_stream_output_info *so)
2597 {
2598 unsigned i;
2599
2600 if (so->num_outputs)
2601 fprintf(stderr, "STREAMOUT\n");
2602
2603 for (i = 0; i < so->num_outputs; i++) {
2604 unsigned mask = ((1 << so->output[i].num_components) - 1) <<
2605 so->output[i].start_component;
2606 fprintf(stderr, " %i: BUF%i[%i..%i] <- OUT[%i].%s%s%s%s\n",
2607 i, so->output[i].output_buffer,
2608 so->output[i].dst_offset, so->output[i].dst_offset + so->output[i].num_components - 1,
2609 so->output[i].register_index,
2610 mask & 1 ? "x" : "",
2611 mask & 2 ? "y" : "",
2612 mask & 4 ? "z" : "",
2613 mask & 8 ? "w" : "");
2614 }
2615 }
2616
2617 void si_emit_streamout_output(struct si_shader_context *ctx,
2618 LLVMValueRef const *so_buffers,
2619 LLVMValueRef const *so_write_offsets,
2620 struct pipe_stream_output *stream_out,
2621 struct si_shader_output_values *shader_out)
2622 {
2623 unsigned buf_idx = stream_out->output_buffer;
2624 unsigned start = stream_out->start_component;
2625 unsigned num_comps = stream_out->num_components;
2626 LLVMValueRef out[4];
2627
2628 assert(num_comps && num_comps <= 4);
2629 if (!num_comps || num_comps > 4)
2630 return;
2631
2632 /* Load the output as int. */
2633 for (int j = 0; j < num_comps; j++) {
2634 assert(stream_out->stream == shader_out->vertex_stream[start + j]);
2635
2636 out[j] = ac_to_integer(&ctx->ac, shader_out->values[start + j]);
2637 }
2638
2639 /* Pack the output. */
2640 LLVMValueRef vdata = NULL;
2641
2642 switch (num_comps) {
2643 case 1: /* as i32 */
2644 vdata = out[0];
2645 break;
2646 case 2: /* as v2i32 */
2647 case 3: /* as v3i32 */
2648 if (ac_has_vec3_support(ctx->screen->info.chip_class, false)) {
2649 vdata = ac_build_gather_values(&ctx->ac, out, num_comps);
2650 break;
2651 }
2652 /* as v4i32 (aligned to 4) */
2653 out[3] = LLVMGetUndef(ctx->i32);
2654 /* fall through */
2655 case 4: /* as v4i32 */
2656 vdata = ac_build_gather_values(&ctx->ac, out, util_next_power_of_two(num_comps));
2657 break;
2658 }
2659
2660 ac_build_buffer_store_dword(&ctx->ac, so_buffers[buf_idx],
2661 vdata, num_comps,
2662 so_write_offsets[buf_idx],
2663 ctx->i32_0,
2664 stream_out->dst_offset * 4, ac_glc | ac_slc);
2665 }
2666
2667 /**
2668 * Write streamout data to buffers for vertex stream @p stream (different
2669 * vertex streams can occur for GS copy shaders).
2670 */
2671 static void si_llvm_emit_streamout(struct si_shader_context *ctx,
2672 struct si_shader_output_values *outputs,
2673 unsigned noutput, unsigned stream)
2674 {
2675 struct si_shader_selector *sel = ctx->shader->selector;
2676 struct pipe_stream_output_info *so = &sel->so;
2677 LLVMBuilderRef builder = ctx->ac.builder;
2678 int i;
2679
2680 /* Get bits [22:16], i.e. (so_param >> 16) & 127; */
2681 LLVMValueRef so_vtx_count =
2682 si_unpack_param(ctx, ctx->streamout_config, 16, 7);
2683
2684 LLVMValueRef tid = ac_get_thread_id(&ctx->ac);
2685
2686 /* can_emit = tid < so_vtx_count; */
2687 LLVMValueRef can_emit =
2688 LLVMBuildICmp(builder, LLVMIntULT, tid, so_vtx_count, "");
2689
2690 /* Emit the streamout code conditionally. This actually avoids
2691 * out-of-bounds buffer access. The hw tells us via the SGPR
2692 * (so_vtx_count) which threads are allowed to emit streamout data. */
2693 ac_build_ifcc(&ctx->ac, can_emit, 6501);
2694 {
2695 /* The buffer offset is computed as follows:
2696 * ByteOffset = streamout_offset[buffer_id]*4 +
2697 * (streamout_write_index + thread_id)*stride[buffer_id] +
2698 * attrib_offset
2699 */
2700
2701 LLVMValueRef so_write_index =
2702 ac_get_arg(&ctx->ac,
2703 ctx->streamout_write_index);
2704
2705 /* Compute (streamout_write_index + thread_id). */
2706 so_write_index = LLVMBuildAdd(builder, so_write_index, tid, "");
2707
2708 /* Load the descriptor and compute the write offset for each
2709 * enabled buffer. */
2710 LLVMValueRef so_write_offset[4] = {};
2711 LLVMValueRef so_buffers[4];
2712 LLVMValueRef buf_ptr = ac_get_arg(&ctx->ac,
2713 ctx->rw_buffers);
2714
2715 for (i = 0; i < 4; i++) {
2716 if (!so->stride[i])
2717 continue;
2718
2719 LLVMValueRef offset = LLVMConstInt(ctx->i32,
2720 SI_VS_STREAMOUT_BUF0 + i, 0);
2721
2722 so_buffers[i] = ac_build_load_to_sgpr(&ctx->ac, buf_ptr, offset);
2723
2724 LLVMValueRef so_offset = ac_get_arg(&ctx->ac,
2725 ctx->streamout_offset[i]);
2726 so_offset = LLVMBuildMul(builder, so_offset, LLVMConstInt(ctx->i32, 4, 0), "");
2727
2728 so_write_offset[i] = ac_build_imad(&ctx->ac, so_write_index,
2729 LLVMConstInt(ctx->i32, so->stride[i]*4, 0),
2730 so_offset);
2731 }
2732
2733 /* Write streamout data. */
2734 for (i = 0; i < so->num_outputs; i++) {
2735 unsigned reg = so->output[i].register_index;
2736
2737 if (reg >= noutput)
2738 continue;
2739
2740 if (stream != so->output[i].stream)
2741 continue;
2742
2743 si_emit_streamout_output(ctx, so_buffers, so_write_offset,
2744 &so->output[i], &outputs[reg]);
2745 }
2746 }
2747 ac_build_endif(&ctx->ac, 6501);
2748 }
2749
2750 static void si_export_param(struct si_shader_context *ctx, unsigned index,
2751 LLVMValueRef *values)
2752 {
2753 struct ac_export_args args;
2754
2755 si_llvm_init_export_args(ctx, values,
2756 V_008DFC_SQ_EXP_PARAM + index, &args);
2757 ac_build_export(&ctx->ac, &args);
2758 }
2759
2760 static void si_build_param_exports(struct si_shader_context *ctx,
2761 struct si_shader_output_values *outputs,
2762 unsigned noutput)
2763 {
2764 struct si_shader *shader = ctx->shader;
2765 unsigned param_count = 0;
2766
2767 for (unsigned i = 0; i < noutput; i++) {
2768 unsigned semantic_name = outputs[i].semantic_name;
2769 unsigned semantic_index = outputs[i].semantic_index;
2770
2771 if (outputs[i].vertex_stream[0] != 0 &&
2772 outputs[i].vertex_stream[1] != 0 &&
2773 outputs[i].vertex_stream[2] != 0 &&
2774 outputs[i].vertex_stream[3] != 0)
2775 continue;
2776
2777 switch (semantic_name) {
2778 case TGSI_SEMANTIC_LAYER:
2779 case TGSI_SEMANTIC_VIEWPORT_INDEX:
2780 case TGSI_SEMANTIC_CLIPDIST:
2781 case TGSI_SEMANTIC_COLOR:
2782 case TGSI_SEMANTIC_BCOLOR:
2783 case TGSI_SEMANTIC_PRIMID:
2784 case TGSI_SEMANTIC_FOG:
2785 case TGSI_SEMANTIC_TEXCOORD:
2786 case TGSI_SEMANTIC_GENERIC:
2787 break;
2788 default:
2789 continue;
2790 }
2791
2792 if ((semantic_name != TGSI_SEMANTIC_GENERIC ||
2793 semantic_index < SI_MAX_IO_GENERIC) &&
2794 shader->key.opt.kill_outputs &
2795 (1ull << si_shader_io_get_unique_index(semantic_name,
2796 semantic_index, true)))
2797 continue;
2798
2799 si_export_param(ctx, param_count, outputs[i].values);
2800
2801 assert(i < ARRAY_SIZE(shader->info.vs_output_param_offset));
2802 shader->info.vs_output_param_offset[i] = param_count++;
2803 }
2804
2805 shader->info.nr_param_exports = param_count;
2806 }
2807
2808 /**
2809 * Vertex color clamping.
2810 *
2811 * This uses a state constant loaded in a user data SGPR and
2812 * an IF statement is added that clamps all colors if the constant
2813 * is true.
2814 */
2815 static void si_vertex_color_clamping(struct si_shader_context *ctx,
2816 struct si_shader_output_values *outputs,
2817 unsigned noutput)
2818 {
2819 LLVMValueRef addr[SI_MAX_VS_OUTPUTS][4];
2820 bool has_colors = false;
2821
2822 /* Store original colors to alloca variables. */
2823 for (unsigned i = 0; i < noutput; i++) {
2824 if (outputs[i].semantic_name != TGSI_SEMANTIC_COLOR &&
2825 outputs[i].semantic_name != TGSI_SEMANTIC_BCOLOR)
2826 continue;
2827
2828 for (unsigned j = 0; j < 4; j++) {
2829 addr[i][j] = ac_build_alloca_undef(&ctx->ac, ctx->f32, "");
2830 LLVMBuildStore(ctx->ac.builder, outputs[i].values[j], addr[i][j]);
2831 }
2832 has_colors = true;
2833 }
2834
2835 if (!has_colors)
2836 return;
2837
2838 /* The state is in the first bit of the user SGPR. */
2839 LLVMValueRef cond = ac_get_arg(&ctx->ac, ctx->vs_state_bits);
2840 cond = LLVMBuildTrunc(ctx->ac.builder, cond, ctx->i1, "");
2841
2842 ac_build_ifcc(&ctx->ac, cond, 6502);
2843
2844 /* Store clamped colors to alloca variables within the conditional block. */
2845 for (unsigned i = 0; i < noutput; i++) {
2846 if (outputs[i].semantic_name != TGSI_SEMANTIC_COLOR &&
2847 outputs[i].semantic_name != TGSI_SEMANTIC_BCOLOR)
2848 continue;
2849
2850 for (unsigned j = 0; j < 4; j++) {
2851 LLVMBuildStore(ctx->ac.builder,
2852 ac_build_clamp(&ctx->ac, outputs[i].values[j]),
2853 addr[i][j]);
2854 }
2855 }
2856 ac_build_endif(&ctx->ac, 6502);
2857
2858 /* Load clamped colors */
2859 for (unsigned i = 0; i < noutput; i++) {
2860 if (outputs[i].semantic_name != TGSI_SEMANTIC_COLOR &&
2861 outputs[i].semantic_name != TGSI_SEMANTIC_BCOLOR)
2862 continue;
2863
2864 for (unsigned j = 0; j < 4; j++) {
2865 outputs[i].values[j] =
2866 LLVMBuildLoad(ctx->ac.builder, addr[i][j], "");
2867 }
2868 }
2869 }
2870
2871 /* Generate export instructions for hardware VS shader stage or NGG GS stage
2872 * (position and parameter data only).
2873 */
2874 void si_llvm_export_vs(struct si_shader_context *ctx,
2875 struct si_shader_output_values *outputs,
2876 unsigned noutput)
2877 {
2878 struct si_shader *shader = ctx->shader;
2879 struct ac_export_args pos_args[4] = {};
2880 LLVMValueRef psize_value = NULL, edgeflag_value = NULL, layer_value = NULL, viewport_index_value = NULL;
2881 unsigned pos_idx;
2882 int i;
2883
2884 si_vertex_color_clamping(ctx, outputs, noutput);
2885
2886 /* Build position exports. */
2887 for (i = 0; i < noutput; i++) {
2888 switch (outputs[i].semantic_name) {
2889 case TGSI_SEMANTIC_POSITION:
2890 si_llvm_init_export_args(ctx, outputs[i].values,
2891 V_008DFC_SQ_EXP_POS, &pos_args[0]);
2892 break;
2893 case TGSI_SEMANTIC_PSIZE:
2894 psize_value = outputs[i].values[0];
2895 break;
2896 case TGSI_SEMANTIC_LAYER:
2897 layer_value = outputs[i].values[0];
2898 break;
2899 case TGSI_SEMANTIC_VIEWPORT_INDEX:
2900 viewport_index_value = outputs[i].values[0];
2901 break;
2902 case TGSI_SEMANTIC_EDGEFLAG:
2903 edgeflag_value = outputs[i].values[0];
2904 break;
2905 case TGSI_SEMANTIC_CLIPDIST:
2906 if (!shader->key.opt.clip_disable) {
2907 unsigned index = 2 + outputs[i].semantic_index;
2908 si_llvm_init_export_args(ctx, outputs[i].values,
2909 V_008DFC_SQ_EXP_POS + index,
2910 &pos_args[index]);
2911 }
2912 break;
2913 case TGSI_SEMANTIC_CLIPVERTEX:
2914 if (!shader->key.opt.clip_disable) {
2915 si_llvm_emit_clipvertex(ctx, pos_args,
2916 outputs[i].values);
2917 }
2918 break;
2919 }
2920 }
2921
2922 /* We need to add the position output manually if it's missing. */
2923 if (!pos_args[0].out[0]) {
2924 pos_args[0].enabled_channels = 0xf; /* writemask */
2925 pos_args[0].valid_mask = 0; /* EXEC mask */
2926 pos_args[0].done = 0; /* last export? */
2927 pos_args[0].target = V_008DFC_SQ_EXP_POS;
2928 pos_args[0].compr = 0; /* COMPR flag */
2929 pos_args[0].out[0] = ctx->ac.f32_0; /* X */
2930 pos_args[0].out[1] = ctx->ac.f32_0; /* Y */
2931 pos_args[0].out[2] = ctx->ac.f32_0; /* Z */
2932 pos_args[0].out[3] = ctx->ac.f32_1; /* W */
2933 }
2934
2935 bool pos_writes_edgeflag = shader->selector->info.writes_edgeflag &&
2936 !shader->key.as_ngg;
2937
2938 /* Write the misc vector (point size, edgeflag, layer, viewport). */
2939 if (shader->selector->info.writes_psize ||
2940 pos_writes_edgeflag ||
2941 shader->selector->info.writes_viewport_index ||
2942 shader->selector->info.writes_layer) {
2943 pos_args[1].enabled_channels = shader->selector->info.writes_psize |
2944 (pos_writes_edgeflag << 1) |
2945 (shader->selector->info.writes_layer << 2);
2946
2947 pos_args[1].valid_mask = 0; /* EXEC mask */
2948 pos_args[1].done = 0; /* last export? */
2949 pos_args[1].target = V_008DFC_SQ_EXP_POS + 1;
2950 pos_args[1].compr = 0; /* COMPR flag */
2951 pos_args[1].out[0] = ctx->ac.f32_0; /* X */
2952 pos_args[1].out[1] = ctx->ac.f32_0; /* Y */
2953 pos_args[1].out[2] = ctx->ac.f32_0; /* Z */
2954 pos_args[1].out[3] = ctx->ac.f32_0; /* W */
2955
2956 if (shader->selector->info.writes_psize)
2957 pos_args[1].out[0] = psize_value;
2958
2959 if (pos_writes_edgeflag) {
2960 /* The output is a float, but the hw expects an integer
2961 * with the first bit containing the edge flag. */
2962 edgeflag_value = LLVMBuildFPToUI(ctx->ac.builder,
2963 edgeflag_value,
2964 ctx->i32, "");
2965 edgeflag_value = ac_build_umin(&ctx->ac,
2966 edgeflag_value,
2967 ctx->i32_1);
2968
2969 /* The LLVM intrinsic expects a float. */
2970 pos_args[1].out[1] = ac_to_float(&ctx->ac, edgeflag_value);
2971 }
2972
2973 if (ctx->screen->info.chip_class >= GFX9) {
2974 /* GFX9 has the layer in out.z[10:0] and the viewport
2975 * index in out.z[19:16].
2976 */
2977 if (shader->selector->info.writes_layer)
2978 pos_args[1].out[2] = layer_value;
2979
2980 if (shader->selector->info.writes_viewport_index) {
2981 LLVMValueRef v = viewport_index_value;
2982
2983 v = ac_to_integer(&ctx->ac, v);
2984 v = LLVMBuildShl(ctx->ac.builder, v,
2985 LLVMConstInt(ctx->i32, 16, 0), "");
2986 v = LLVMBuildOr(ctx->ac.builder, v,
2987 ac_to_integer(&ctx->ac, pos_args[1].out[2]), "");
2988 pos_args[1].out[2] = ac_to_float(&ctx->ac, v);
2989 pos_args[1].enabled_channels |= 1 << 2;
2990 }
2991 } else {
2992 if (shader->selector->info.writes_layer)
2993 pos_args[1].out[2] = layer_value;
2994
2995 if (shader->selector->info.writes_viewport_index) {
2996 pos_args[1].out[3] = viewport_index_value;
2997 pos_args[1].enabled_channels |= 1 << 3;
2998 }
2999 }
3000 }
3001
3002 for (i = 0; i < 4; i++)
3003 if (pos_args[i].out[0])
3004 shader->info.nr_pos_exports++;
3005
3006 /* Navi10-14 skip POS0 exports if EXEC=0 and DONE=0, causing a hang.
3007 * Setting valid_mask=1 prevents it and has no other effect.
3008 */
3009 if (ctx->screen->info.family == CHIP_NAVI10 ||
3010 ctx->screen->info.family == CHIP_NAVI12 ||
3011 ctx->screen->info.family == CHIP_NAVI14)
3012 pos_args[0].valid_mask = 1;
3013
3014 pos_idx = 0;
3015 for (i = 0; i < 4; i++) {
3016 if (!pos_args[i].out[0])
3017 continue;
3018
3019 /* Specify the target we are exporting */
3020 pos_args[i].target = V_008DFC_SQ_EXP_POS + pos_idx++;
3021
3022 if (pos_idx == shader->info.nr_pos_exports)
3023 /* Specify that this is the last export */
3024 pos_args[i].done = 1;
3025
3026 ac_build_export(&ctx->ac, &pos_args[i]);
3027 }
3028
3029 /* Build parameter exports. */
3030 si_build_param_exports(ctx, outputs, noutput);
3031 }
3032
3033 /**
3034 * Forward all outputs from the vertex shader to the TES. This is only used
3035 * for the fixed function TCS.
3036 */
3037 static void si_copy_tcs_inputs(struct lp_build_tgsi_context *bld_base)
3038 {
3039 struct si_shader_context *ctx = si_shader_context(bld_base);
3040 LLVMValueRef invocation_id, buffer, buffer_offset;
3041 LLVMValueRef lds_vertex_stride, lds_base;
3042 uint64_t inputs;
3043
3044 invocation_id = si_unpack_param(ctx, ctx->args.tcs_rel_ids, 8, 5);
3045 buffer = get_tess_ring_descriptor(ctx, TESS_OFFCHIP_RING_TCS);
3046 buffer_offset = ac_get_arg(&ctx->ac, ctx->tcs_offchip_offset);
3047
3048 lds_vertex_stride = get_tcs_in_vertex_dw_stride(ctx);
3049 lds_base = get_tcs_in_current_patch_offset(ctx);
3050 lds_base = ac_build_imad(&ctx->ac, invocation_id, lds_vertex_stride,
3051 lds_base);
3052
3053 inputs = ctx->shader->key.mono.u.ff_tcs_inputs_to_copy;
3054 while (inputs) {
3055 unsigned i = u_bit_scan64(&inputs);
3056
3057 LLVMValueRef lds_ptr = LLVMBuildAdd(ctx->ac.builder, lds_base,
3058 LLVMConstInt(ctx->i32, 4 * i, 0),
3059 "");
3060
3061 LLVMValueRef buffer_addr = get_tcs_tes_buffer_address(ctx,
3062 get_rel_patch_id(ctx),
3063 invocation_id,
3064 LLVMConstInt(ctx->i32, i, 0));
3065
3066 LLVMValueRef value = lshs_lds_load(bld_base, ctx->ac.i32, ~0, lds_ptr);
3067
3068 ac_build_buffer_store_dword(&ctx->ac, buffer, value, 4, buffer_addr,
3069 buffer_offset, 0, ac_glc);
3070 }
3071 }
3072
3073 static void si_write_tess_factors(struct lp_build_tgsi_context *bld_base,
3074 LLVMValueRef rel_patch_id,
3075 LLVMValueRef invocation_id,
3076 LLVMValueRef tcs_out_current_patch_data_offset,
3077 LLVMValueRef invoc0_tf_outer[4],
3078 LLVMValueRef invoc0_tf_inner[2])
3079 {
3080 struct si_shader_context *ctx = si_shader_context(bld_base);
3081 struct si_shader *shader = ctx->shader;
3082 unsigned tess_inner_index, tess_outer_index;
3083 LLVMValueRef lds_base, lds_inner, lds_outer, byteoffset, buffer;
3084 LLVMValueRef out[6], vec0, vec1, tf_base, inner[4], outer[4];
3085 unsigned stride, outer_comps, inner_comps, i, offset;
3086
3087 /* Add a barrier before loading tess factors from LDS. */
3088 if (!shader->key.part.tcs.epilog.invoc0_tess_factors_are_def)
3089 si_llvm_emit_barrier(NULL, bld_base, NULL);
3090
3091 /* Do this only for invocation 0, because the tess levels are per-patch,
3092 * not per-vertex.
3093 *
3094 * This can't jump, because invocation 0 executes this. It should
3095 * at least mask out the loads and stores for other invocations.
3096 */
3097 ac_build_ifcc(&ctx->ac,
3098 LLVMBuildICmp(ctx->ac.builder, LLVMIntEQ,
3099 invocation_id, ctx->i32_0, ""), 6503);
3100
3101 /* Determine the layout of one tess factor element in the buffer. */
3102 switch (shader->key.part.tcs.epilog.prim_mode) {
3103 case PIPE_PRIM_LINES:
3104 stride = 2; /* 2 dwords, 1 vec2 store */
3105 outer_comps = 2;
3106 inner_comps = 0;
3107 break;
3108 case PIPE_PRIM_TRIANGLES:
3109 stride = 4; /* 4 dwords, 1 vec4 store */
3110 outer_comps = 3;
3111 inner_comps = 1;
3112 break;
3113 case PIPE_PRIM_QUADS:
3114 stride = 6; /* 6 dwords, 2 stores (vec4 + vec2) */
3115 outer_comps = 4;
3116 inner_comps = 2;
3117 break;
3118 default:
3119 assert(0);
3120 return;
3121 }
3122
3123 for (i = 0; i < 4; i++) {
3124 inner[i] = LLVMGetUndef(ctx->i32);
3125 outer[i] = LLVMGetUndef(ctx->i32);
3126 }
3127
3128 if (shader->key.part.tcs.epilog.invoc0_tess_factors_are_def) {
3129 /* Tess factors are in VGPRs. */
3130 for (i = 0; i < outer_comps; i++)
3131 outer[i] = out[i] = invoc0_tf_outer[i];
3132 for (i = 0; i < inner_comps; i++)
3133 inner[i] = out[outer_comps+i] = invoc0_tf_inner[i];
3134 } else {
3135 /* Load tess_inner and tess_outer from LDS.
3136 * Any invocation can write them, so we can't get them from a temporary.
3137 */
3138 tess_inner_index = si_shader_io_get_unique_index_patch(TGSI_SEMANTIC_TESSINNER, 0);
3139 tess_outer_index = si_shader_io_get_unique_index_patch(TGSI_SEMANTIC_TESSOUTER, 0);
3140
3141 lds_base = tcs_out_current_patch_data_offset;
3142 lds_inner = LLVMBuildAdd(ctx->ac.builder, lds_base,
3143 LLVMConstInt(ctx->i32,
3144 tess_inner_index * 4, 0), "");
3145 lds_outer = LLVMBuildAdd(ctx->ac.builder, lds_base,
3146 LLVMConstInt(ctx->i32,
3147 tess_outer_index * 4, 0), "");
3148
3149 for (i = 0; i < outer_comps; i++) {
3150 outer[i] = out[i] =
3151 lshs_lds_load(bld_base, ctx->ac.i32, i, lds_outer);
3152 }
3153 for (i = 0; i < inner_comps; i++) {
3154 inner[i] = out[outer_comps+i] =
3155 lshs_lds_load(bld_base, ctx->ac.i32, i, lds_inner);
3156 }
3157 }
3158
3159 if (shader->key.part.tcs.epilog.prim_mode == PIPE_PRIM_LINES) {
3160 /* For isolines, the hardware expects tess factors in the
3161 * reverse order from what GLSL / TGSI specify.
3162 */
3163 LLVMValueRef tmp = out[0];
3164 out[0] = out[1];
3165 out[1] = tmp;
3166 }
3167
3168 /* Convert the outputs to vectors for stores. */
3169 vec0 = ac_build_gather_values(&ctx->ac, out, MIN2(stride, 4));
3170 vec1 = NULL;
3171
3172 if (stride > 4)
3173 vec1 = ac_build_gather_values(&ctx->ac, out+4, stride - 4);
3174
3175 /* Get the buffer. */
3176 buffer = get_tess_ring_descriptor(ctx, TCS_FACTOR_RING);
3177
3178 /* Get the offset. */
3179 tf_base = ac_get_arg(&ctx->ac,
3180 ctx->tcs_factor_offset);
3181 byteoffset = LLVMBuildMul(ctx->ac.builder, rel_patch_id,
3182 LLVMConstInt(ctx->i32, 4 * stride, 0), "");
3183
3184 ac_build_ifcc(&ctx->ac,
3185 LLVMBuildICmp(ctx->ac.builder, LLVMIntEQ,
3186 rel_patch_id, ctx->i32_0, ""), 6504);
3187
3188 /* Store the dynamic HS control word. */
3189 offset = 0;
3190 if (ctx->screen->info.chip_class <= GFX8) {
3191 ac_build_buffer_store_dword(&ctx->ac, buffer,
3192 LLVMConstInt(ctx->i32, 0x80000000, 0),
3193 1, ctx->i32_0, tf_base,
3194 offset, ac_glc);
3195 offset += 4;
3196 }
3197
3198 ac_build_endif(&ctx->ac, 6504);
3199
3200 /* Store the tessellation factors. */
3201 ac_build_buffer_store_dword(&ctx->ac, buffer, vec0,
3202 MIN2(stride, 4), byteoffset, tf_base,
3203 offset, ac_glc);
3204 offset += 16;
3205 if (vec1)
3206 ac_build_buffer_store_dword(&ctx->ac, buffer, vec1,
3207 stride - 4, byteoffset, tf_base,
3208 offset, ac_glc);
3209
3210 /* Store the tess factors into the offchip buffer if TES reads them. */
3211 if (shader->key.part.tcs.epilog.tes_reads_tess_factors) {
3212 LLVMValueRef buf, base, inner_vec, outer_vec, tf_outer_offset;
3213 LLVMValueRef tf_inner_offset;
3214 unsigned param_outer, param_inner;
3215
3216 buf = get_tess_ring_descriptor(ctx, TESS_OFFCHIP_RING_TCS);
3217 base = ac_get_arg(&ctx->ac, ctx->tcs_offchip_offset);
3218
3219 param_outer = si_shader_io_get_unique_index_patch(
3220 TGSI_SEMANTIC_TESSOUTER, 0);
3221 tf_outer_offset = get_tcs_tes_buffer_address(ctx, rel_patch_id, NULL,
3222 LLVMConstInt(ctx->i32, param_outer, 0));
3223
3224 unsigned outer_vec_size =
3225 ac_has_vec3_support(ctx->screen->info.chip_class, false) ?
3226 outer_comps : util_next_power_of_two(outer_comps);
3227 outer_vec = ac_build_gather_values(&ctx->ac, outer, outer_vec_size);
3228
3229 ac_build_buffer_store_dword(&ctx->ac, buf, outer_vec,
3230 outer_comps, tf_outer_offset,
3231 base, 0, ac_glc);
3232 if (inner_comps) {
3233 param_inner = si_shader_io_get_unique_index_patch(
3234 TGSI_SEMANTIC_TESSINNER, 0);
3235 tf_inner_offset = get_tcs_tes_buffer_address(ctx, rel_patch_id, NULL,
3236 LLVMConstInt(ctx->i32, param_inner, 0));
3237
3238 inner_vec = inner_comps == 1 ? inner[0] :
3239 ac_build_gather_values(&ctx->ac, inner, inner_comps);
3240 ac_build_buffer_store_dword(&ctx->ac, buf, inner_vec,
3241 inner_comps, tf_inner_offset,
3242 base, 0, ac_glc);
3243 }
3244 }
3245
3246 ac_build_endif(&ctx->ac, 6503);
3247 }
3248
3249 static LLVMValueRef
3250 si_insert_input_ret(struct si_shader_context *ctx, LLVMValueRef ret,
3251 struct ac_arg param, unsigned return_index)
3252 {
3253 return LLVMBuildInsertValue(ctx->ac.builder, ret,
3254 ac_get_arg(&ctx->ac, param),
3255 return_index, "");
3256 }
3257
3258 static LLVMValueRef
3259 si_insert_input_ret_float(struct si_shader_context *ctx, LLVMValueRef ret,
3260 struct ac_arg param, unsigned return_index)
3261 {
3262 LLVMBuilderRef builder = ctx->ac.builder;
3263 LLVMValueRef p = ac_get_arg(&ctx->ac, param);
3264
3265 return LLVMBuildInsertValue(builder, ret,
3266 ac_to_float(&ctx->ac, p),
3267 return_index, "");
3268 }
3269
3270 static LLVMValueRef
3271 si_insert_input_ptr(struct si_shader_context *ctx, LLVMValueRef ret,
3272 struct ac_arg param, unsigned return_index)
3273 {
3274 LLVMBuilderRef builder = ctx->ac.builder;
3275 LLVMValueRef ptr = ac_get_arg(&ctx->ac, param);
3276 ptr = LLVMBuildPtrToInt(builder, ptr, ctx->i32, "");
3277 return LLVMBuildInsertValue(builder, ret, ptr, return_index, "");
3278 }
3279
3280 /* This only writes the tessellation factor levels. */
3281 static void si_llvm_emit_tcs_epilogue(struct ac_shader_abi *abi,
3282 unsigned max_outputs,
3283 LLVMValueRef *addrs)
3284 {
3285 struct si_shader_context *ctx = si_shader_context_from_abi(abi);
3286 struct lp_build_tgsi_context *bld_base = &ctx->bld_base;
3287 LLVMBuilderRef builder = ctx->ac.builder;
3288 LLVMValueRef rel_patch_id, invocation_id, tf_lds_offset;
3289
3290 si_copy_tcs_inputs(bld_base);
3291
3292 rel_patch_id = get_rel_patch_id(ctx);
3293 invocation_id = si_unpack_param(ctx, ctx->args.tcs_rel_ids, 8, 5);
3294 tf_lds_offset = get_tcs_out_current_patch_data_offset(ctx);
3295
3296 if (ctx->screen->info.chip_class >= GFX9) {
3297 LLVMBasicBlockRef blocks[2] = {
3298 LLVMGetInsertBlock(builder),
3299 ctx->merged_wrap_if_entry_block
3300 };
3301 LLVMValueRef values[2];
3302
3303 ac_build_endif(&ctx->ac, ctx->merged_wrap_if_label);
3304
3305 values[0] = rel_patch_id;
3306 values[1] = LLVMGetUndef(ctx->i32);
3307 rel_patch_id = ac_build_phi(&ctx->ac, ctx->i32, 2, values, blocks);
3308
3309 values[0] = tf_lds_offset;
3310 values[1] = LLVMGetUndef(ctx->i32);
3311 tf_lds_offset = ac_build_phi(&ctx->ac, ctx->i32, 2, values, blocks);
3312
3313 values[0] = invocation_id;
3314 values[1] = ctx->i32_1; /* cause the epilog to skip threads */
3315 invocation_id = ac_build_phi(&ctx->ac, ctx->i32, 2, values, blocks);
3316 }
3317
3318 /* Return epilog parameters from this function. */
3319 LLVMValueRef ret = ctx->return_value;
3320 unsigned vgpr;
3321
3322 if (ctx->screen->info.chip_class >= GFX9) {
3323 ret = si_insert_input_ret(ctx, ret, ctx->tcs_offchip_layout,
3324 8 + GFX9_SGPR_TCS_OFFCHIP_LAYOUT);
3325 ret = si_insert_input_ret(ctx, ret, ctx->tcs_out_lds_layout,
3326 8 + GFX9_SGPR_TCS_OUT_LAYOUT);
3327 /* Tess offchip and tess factor offsets are at the beginning. */
3328 ret = si_insert_input_ret(ctx, ret, ctx->tcs_offchip_offset, 2);
3329 ret = si_insert_input_ret(ctx, ret, ctx->tcs_factor_offset, 4);
3330 vgpr = 8 + GFX9_SGPR_TCS_OUT_LAYOUT + 1;
3331 } else {
3332 ret = si_insert_input_ret(ctx, ret, ctx->tcs_offchip_layout,
3333 GFX6_SGPR_TCS_OFFCHIP_LAYOUT);
3334 ret = si_insert_input_ret(ctx, ret, ctx->tcs_out_lds_layout,
3335 GFX6_SGPR_TCS_OUT_LAYOUT);
3336 /* Tess offchip and tess factor offsets are after user SGPRs. */
3337 ret = si_insert_input_ret(ctx, ret, ctx->tcs_offchip_offset,
3338 GFX6_TCS_NUM_USER_SGPR);
3339 ret = si_insert_input_ret(ctx, ret, ctx->tcs_factor_offset,
3340 GFX6_TCS_NUM_USER_SGPR + 1);
3341 vgpr = GFX6_TCS_NUM_USER_SGPR + 2;
3342 }
3343
3344 /* VGPRs */
3345 rel_patch_id = ac_to_float(&ctx->ac, rel_patch_id);
3346 invocation_id = ac_to_float(&ctx->ac, invocation_id);
3347 tf_lds_offset = ac_to_float(&ctx->ac, tf_lds_offset);
3348
3349 /* Leave a hole corresponding to the two input VGPRs. This ensures that
3350 * the invocation_id output does not alias the tcs_rel_ids input,
3351 * which saves a V_MOV on gfx9.
3352 */
3353 vgpr += 2;
3354
3355 ret = LLVMBuildInsertValue(builder, ret, rel_patch_id, vgpr++, "");
3356 ret = LLVMBuildInsertValue(builder, ret, invocation_id, vgpr++, "");
3357
3358 if (ctx->shader->selector->tcs_info.tessfactors_are_def_in_all_invocs) {
3359 vgpr++; /* skip the tess factor LDS offset */
3360 for (unsigned i = 0; i < 6; i++) {
3361 LLVMValueRef value =
3362 LLVMBuildLoad(builder, ctx->invoc0_tess_factors[i], "");
3363 value = ac_to_float(&ctx->ac, value);
3364 ret = LLVMBuildInsertValue(builder, ret, value, vgpr++, "");
3365 }
3366 } else {
3367 ret = LLVMBuildInsertValue(builder, ret, tf_lds_offset, vgpr++, "");
3368 }
3369 ctx->return_value = ret;
3370 }
3371
3372 /* Pass TCS inputs from LS to TCS on GFX9. */
3373 static void si_set_ls_return_value_for_tcs(struct si_shader_context *ctx)
3374 {
3375 LLVMValueRef ret = ctx->return_value;
3376
3377 ret = si_insert_input_ptr(ctx, ret, ctx->other_const_and_shader_buffers, 0);
3378 ret = si_insert_input_ptr(ctx, ret, ctx->other_samplers_and_images, 1);
3379 ret = si_insert_input_ret(ctx, ret, ctx->tcs_offchip_offset, 2);
3380 ret = si_insert_input_ret(ctx, ret, ctx->merged_wave_info, 3);
3381 ret = si_insert_input_ret(ctx, ret, ctx->tcs_factor_offset, 4);
3382 ret = si_insert_input_ret(ctx, ret, ctx->merged_scratch_offset, 5);
3383
3384 ret = si_insert_input_ptr(ctx, ret, ctx->rw_buffers,
3385 8 + SI_SGPR_RW_BUFFERS);
3386 ret = si_insert_input_ptr(ctx, ret,
3387 ctx->bindless_samplers_and_images,
3388 8 + SI_SGPR_BINDLESS_SAMPLERS_AND_IMAGES);
3389
3390 ret = si_insert_input_ret(ctx, ret, ctx->vs_state_bits,
3391 8 + SI_SGPR_VS_STATE_BITS);
3392
3393 ret = si_insert_input_ret(ctx, ret, ctx->tcs_offchip_layout,
3394 8 + GFX9_SGPR_TCS_OFFCHIP_LAYOUT);
3395 ret = si_insert_input_ret(ctx, ret, ctx->tcs_out_lds_offsets,
3396 8 + GFX9_SGPR_TCS_OUT_OFFSETS);
3397 ret = si_insert_input_ret(ctx, ret, ctx->tcs_out_lds_layout,
3398 8 + GFX9_SGPR_TCS_OUT_LAYOUT);
3399
3400 unsigned vgpr = 8 + GFX9_TCS_NUM_USER_SGPR;
3401 ret = LLVMBuildInsertValue(ctx->ac.builder, ret,
3402 ac_to_float(&ctx->ac,
3403 ac_get_arg(&ctx->ac, ctx->args.tcs_patch_id)),
3404 vgpr++, "");
3405 ret = LLVMBuildInsertValue(ctx->ac.builder, ret,
3406 ac_to_float(&ctx->ac,
3407 ac_get_arg(&ctx->ac, ctx->args.tcs_rel_ids)),
3408 vgpr++, "");
3409 ctx->return_value = ret;
3410 }
3411
3412 /* Pass GS inputs from ES to GS on GFX9. */
3413 static void si_set_es_return_value_for_gs(struct si_shader_context *ctx)
3414 {
3415 LLVMValueRef ret = ctx->return_value;
3416
3417 ret = si_insert_input_ptr(ctx, ret, ctx->other_const_and_shader_buffers, 0);
3418 ret = si_insert_input_ptr(ctx, ret, ctx->other_samplers_and_images, 1);
3419 if (ctx->shader->key.as_ngg)
3420 ret = si_insert_input_ptr(ctx, ret, ctx->gs_tg_info, 2);
3421 else
3422 ret = si_insert_input_ret(ctx, ret, ctx->gs2vs_offset, 2);
3423 ret = si_insert_input_ret(ctx, ret, ctx->merged_wave_info, 3);
3424 ret = si_insert_input_ret(ctx, ret, ctx->merged_scratch_offset, 5);
3425
3426 ret = si_insert_input_ptr(ctx, ret, ctx->rw_buffers,
3427 8 + SI_SGPR_RW_BUFFERS);
3428 ret = si_insert_input_ptr(ctx, ret,
3429 ctx->bindless_samplers_and_images,
3430 8 + SI_SGPR_BINDLESS_SAMPLERS_AND_IMAGES);
3431 if (ctx->screen->use_ngg) {
3432 ret = si_insert_input_ptr(ctx, ret, ctx->vs_state_bits,
3433 8 + SI_SGPR_VS_STATE_BITS);
3434 }
3435
3436 unsigned vgpr;
3437 if (ctx->type == PIPE_SHADER_VERTEX)
3438 vgpr = 8 + GFX9_VSGS_NUM_USER_SGPR;
3439 else
3440 vgpr = 8 + GFX9_TESGS_NUM_USER_SGPR;
3441
3442 ret = si_insert_input_ret_float(ctx, ret, ctx->gs_vtx01_offset, vgpr++);
3443 ret = si_insert_input_ret_float(ctx, ret, ctx->gs_vtx23_offset, vgpr++);
3444 ret = si_insert_input_ret_float(ctx, ret, ctx->args.gs_prim_id, vgpr++);
3445 ret = si_insert_input_ret_float(ctx, ret, ctx->args.gs_invocation_id, vgpr++);
3446 ret = si_insert_input_ret_float(ctx, ret, ctx->gs_vtx45_offset, vgpr++);
3447 ctx->return_value = ret;
3448 }
3449
3450 static void si_llvm_emit_ls_epilogue(struct ac_shader_abi *abi,
3451 unsigned max_outputs,
3452 LLVMValueRef *addrs)
3453 {
3454 struct si_shader_context *ctx = si_shader_context_from_abi(abi);
3455 struct si_shader *shader = ctx->shader;
3456 struct tgsi_shader_info *info = &shader->selector->info;
3457 unsigned i, chan;
3458 LLVMValueRef vertex_id = ac_get_arg(&ctx->ac, ctx->rel_auto_id);
3459 LLVMValueRef vertex_dw_stride = get_tcs_in_vertex_dw_stride(ctx);
3460 LLVMValueRef base_dw_addr = LLVMBuildMul(ctx->ac.builder, vertex_id,
3461 vertex_dw_stride, "");
3462
3463 /* Write outputs to LDS. The next shader (TCS aka HS) will read
3464 * its inputs from it. */
3465 for (i = 0; i < info->num_outputs; i++) {
3466 unsigned name = info->output_semantic_name[i];
3467 unsigned index = info->output_semantic_index[i];
3468
3469 /* The ARB_shader_viewport_layer_array spec contains the
3470 * following issue:
3471 *
3472 * 2) What happens if gl_ViewportIndex or gl_Layer is
3473 * written in the vertex shader and a geometry shader is
3474 * present?
3475 *
3476 * RESOLVED: The value written by the last vertex processing
3477 * stage is used. If the last vertex processing stage
3478 * (vertex, tessellation evaluation or geometry) does not
3479 * statically assign to gl_ViewportIndex or gl_Layer, index
3480 * or layer zero is assumed.
3481 *
3482 * So writes to those outputs in VS-as-LS are simply ignored.
3483 */
3484 if (name == TGSI_SEMANTIC_LAYER ||
3485 name == TGSI_SEMANTIC_VIEWPORT_INDEX)
3486 continue;
3487
3488 int param = si_shader_io_get_unique_index(name, index, false);
3489 LLVMValueRef dw_addr = LLVMBuildAdd(ctx->ac.builder, base_dw_addr,
3490 LLVMConstInt(ctx->i32, param * 4, 0), "");
3491
3492 for (chan = 0; chan < 4; chan++) {
3493 if (!(info->output_usagemask[i] & (1 << chan)))
3494 continue;
3495
3496 lshs_lds_store(ctx, chan, dw_addr,
3497 LLVMBuildLoad(ctx->ac.builder, addrs[4 * i + chan], ""));
3498 }
3499 }
3500
3501 if (ctx->screen->info.chip_class >= GFX9)
3502 si_set_ls_return_value_for_tcs(ctx);
3503 }
3504
3505 static void si_llvm_emit_es_epilogue(struct ac_shader_abi *abi,
3506 unsigned max_outputs,
3507 LLVMValueRef *addrs)
3508 {
3509 struct si_shader_context *ctx = si_shader_context_from_abi(abi);
3510 struct si_shader *es = ctx->shader;
3511 struct tgsi_shader_info *info = &es->selector->info;
3512 LLVMValueRef lds_base = NULL;
3513 unsigned chan;
3514 int i;
3515
3516 if (ctx->screen->info.chip_class >= GFX9 && info->num_outputs) {
3517 unsigned itemsize_dw = es->selector->esgs_itemsize / 4;
3518 LLVMValueRef vertex_idx = ac_get_thread_id(&ctx->ac);
3519 LLVMValueRef wave_idx = si_unpack_param(ctx, ctx->merged_wave_info, 24, 4);
3520 vertex_idx = LLVMBuildOr(ctx->ac.builder, vertex_idx,
3521 LLVMBuildMul(ctx->ac.builder, wave_idx,
3522 LLVMConstInt(ctx->i32, ctx->ac.wave_size, false), ""), "");
3523 lds_base = LLVMBuildMul(ctx->ac.builder, vertex_idx,
3524 LLVMConstInt(ctx->i32, itemsize_dw, 0), "");
3525 }
3526
3527 for (i = 0; i < info->num_outputs; i++) {
3528 int param;
3529
3530 if (info->output_semantic_name[i] == TGSI_SEMANTIC_VIEWPORT_INDEX ||
3531 info->output_semantic_name[i] == TGSI_SEMANTIC_LAYER)
3532 continue;
3533
3534 param = si_shader_io_get_unique_index(info->output_semantic_name[i],
3535 info->output_semantic_index[i], false);
3536
3537 for (chan = 0; chan < 4; chan++) {
3538 if (!(info->output_usagemask[i] & (1 << chan)))
3539 continue;
3540
3541 LLVMValueRef out_val = LLVMBuildLoad(ctx->ac.builder, addrs[4 * i + chan], "");
3542 out_val = ac_to_integer(&ctx->ac, out_val);
3543
3544 /* GFX9 has the ESGS ring in LDS. */
3545 if (ctx->screen->info.chip_class >= GFX9) {
3546 LLVMValueRef idx = LLVMConstInt(ctx->i32, param * 4 + chan, false);
3547 idx = LLVMBuildAdd(ctx->ac.builder, lds_base, idx, "");
3548 ac_build_indexed_store(&ctx->ac, ctx->esgs_ring, idx, out_val);
3549 continue;
3550 }
3551
3552 ac_build_buffer_store_dword(&ctx->ac,
3553 ctx->esgs_ring,
3554 out_val, 1, NULL,
3555 ac_get_arg(&ctx->ac, ctx->es2gs_offset),
3556 (4 * param + chan) * 4,
3557 ac_glc | ac_slc | ac_swizzled);
3558 }
3559 }
3560
3561 if (ctx->screen->info.chip_class >= GFX9)
3562 si_set_es_return_value_for_gs(ctx);
3563 }
3564
3565 static LLVMValueRef si_get_gs_wave_id(struct si_shader_context *ctx)
3566 {
3567 if (ctx->screen->info.chip_class >= GFX9)
3568 return si_unpack_param(ctx, ctx->merged_wave_info, 16, 8);
3569 else
3570 return ac_get_arg(&ctx->ac, ctx->gs_wave_id);
3571 }
3572
3573 static void emit_gs_epilogue(struct si_shader_context *ctx)
3574 {
3575 if (ctx->shader->key.as_ngg) {
3576 gfx10_ngg_gs_emit_epilogue(ctx);
3577 return;
3578 }
3579
3580 if (ctx->screen->info.chip_class >= GFX10)
3581 LLVMBuildFence(ctx->ac.builder, LLVMAtomicOrderingRelease, false, "");
3582
3583 ac_build_sendmsg(&ctx->ac, AC_SENDMSG_GS_OP_NOP | AC_SENDMSG_GS_DONE,
3584 si_get_gs_wave_id(ctx));
3585
3586 if (ctx->screen->info.chip_class >= GFX9)
3587 ac_build_endif(&ctx->ac, ctx->merged_wrap_if_label);
3588 }
3589
3590 static void si_llvm_emit_gs_epilogue(struct ac_shader_abi *abi,
3591 unsigned max_outputs,
3592 LLVMValueRef *addrs)
3593 {
3594 struct si_shader_context *ctx = si_shader_context_from_abi(abi);
3595 struct tgsi_shader_info UNUSED *info = &ctx->shader->selector->info;
3596
3597 assert(info->num_outputs <= max_outputs);
3598
3599 emit_gs_epilogue(ctx);
3600 }
3601
3602 static void si_tgsi_emit_gs_epilogue(struct lp_build_tgsi_context *bld_base)
3603 {
3604 struct si_shader_context *ctx = si_shader_context(bld_base);
3605 emit_gs_epilogue(ctx);
3606 }
3607
3608 static void si_llvm_emit_vs_epilogue(struct ac_shader_abi *abi,
3609 unsigned max_outputs,
3610 LLVMValueRef *addrs)
3611 {
3612 struct si_shader_context *ctx = si_shader_context_from_abi(abi);
3613 struct tgsi_shader_info *info = &ctx->shader->selector->info;
3614 struct si_shader_output_values *outputs = NULL;
3615 int i,j;
3616
3617 assert(!ctx->shader->is_gs_copy_shader);
3618 assert(info->num_outputs <= max_outputs);
3619
3620 outputs = MALLOC((info->num_outputs + 1) * sizeof(outputs[0]));
3621
3622 for (i = 0; i < info->num_outputs; i++) {
3623 outputs[i].semantic_name = info->output_semantic_name[i];
3624 outputs[i].semantic_index = info->output_semantic_index[i];
3625
3626 for (j = 0; j < 4; j++) {
3627 outputs[i].values[j] =
3628 LLVMBuildLoad(ctx->ac.builder,
3629 addrs[4 * i + j],
3630 "");
3631 outputs[i].vertex_stream[j] =
3632 (info->output_streams[i] >> (2 * j)) & 3;
3633 }
3634 }
3635
3636 if (!ctx->screen->use_ngg_streamout &&
3637 ctx->shader->selector->so.num_outputs)
3638 si_llvm_emit_streamout(ctx, outputs, i, 0);
3639
3640 /* Export PrimitiveID. */
3641 if (ctx->shader->key.mono.u.vs_export_prim_id) {
3642 outputs[i].semantic_name = TGSI_SEMANTIC_PRIMID;
3643 outputs[i].semantic_index = 0;
3644 outputs[i].values[0] = ac_to_float(&ctx->ac, si_get_primitive_id(ctx, 0));
3645 for (j = 1; j < 4; j++)
3646 outputs[i].values[j] = LLVMConstReal(ctx->f32, 0);
3647
3648 memset(outputs[i].vertex_stream, 0,
3649 sizeof(outputs[i].vertex_stream));
3650 i++;
3651 }
3652
3653 si_llvm_export_vs(ctx, outputs, i);
3654 FREE(outputs);
3655 }
3656
3657 static void si_llvm_emit_prim_discard_cs_epilogue(struct ac_shader_abi *abi,
3658 unsigned max_outputs,
3659 LLVMValueRef *addrs)
3660 {
3661 struct si_shader_context *ctx = si_shader_context_from_abi(abi);
3662 struct tgsi_shader_info *info = &ctx->shader->selector->info;
3663 LLVMValueRef pos[4] = {};
3664
3665 assert(info->num_outputs <= max_outputs);
3666
3667 for (unsigned i = 0; i < info->num_outputs; i++) {
3668 if (info->output_semantic_name[i] != TGSI_SEMANTIC_POSITION)
3669 continue;
3670
3671 for (unsigned chan = 0; chan < 4; chan++)
3672 pos[chan] = LLVMBuildLoad(ctx->ac.builder, addrs[4 * i + chan], "");
3673 break;
3674 }
3675 assert(pos[0] != NULL);
3676
3677 /* Return the position output. */
3678 LLVMValueRef ret = ctx->return_value;
3679 for (unsigned chan = 0; chan < 4; chan++)
3680 ret = LLVMBuildInsertValue(ctx->ac.builder, ret, pos[chan], chan, "");
3681 ctx->return_value = ret;
3682 }
3683
3684 static void si_tgsi_emit_epilogue(struct lp_build_tgsi_context *bld_base)
3685 {
3686 struct si_shader_context *ctx = si_shader_context(bld_base);
3687
3688 ctx->abi.emit_outputs(&ctx->abi, RADEON_LLVM_MAX_OUTPUTS,
3689 &ctx->outputs[0][0]);
3690 }
3691
3692 struct si_ps_exports {
3693 unsigned num;
3694 struct ac_export_args args[10];
3695 };
3696
3697 static void si_export_mrt_z(struct lp_build_tgsi_context *bld_base,
3698 LLVMValueRef depth, LLVMValueRef stencil,
3699 LLVMValueRef samplemask, struct si_ps_exports *exp)
3700 {
3701 struct si_shader_context *ctx = si_shader_context(bld_base);
3702 struct ac_export_args args;
3703
3704 ac_export_mrt_z(&ctx->ac, depth, stencil, samplemask, &args);
3705
3706 memcpy(&exp->args[exp->num++], &args, sizeof(args));
3707 }
3708
3709 static void si_export_mrt_color(struct lp_build_tgsi_context *bld_base,
3710 LLVMValueRef *color, unsigned index,
3711 unsigned samplemask_param,
3712 bool is_last, struct si_ps_exports *exp)
3713 {
3714 struct si_shader_context *ctx = si_shader_context(bld_base);
3715 int i;
3716
3717 /* Clamp color */
3718 if (ctx->shader->key.part.ps.epilog.clamp_color)
3719 for (i = 0; i < 4; i++)
3720 color[i] = ac_build_clamp(&ctx->ac, color[i]);
3721
3722 /* Alpha to one */
3723 if (ctx->shader->key.part.ps.epilog.alpha_to_one)
3724 color[3] = ctx->ac.f32_1;
3725
3726 /* Alpha test */
3727 if (index == 0 &&
3728 ctx->shader->key.part.ps.epilog.alpha_func != PIPE_FUNC_ALWAYS)
3729 si_alpha_test(bld_base, color[3]);
3730
3731 /* Line & polygon smoothing */
3732 if (ctx->shader->key.part.ps.epilog.poly_line_smoothing)
3733 color[3] = si_scale_alpha_by_sample_mask(bld_base, color[3],
3734 samplemask_param);
3735
3736 /* If last_cbuf > 0, FS_COLOR0_WRITES_ALL_CBUFS is true. */
3737 if (ctx->shader->key.part.ps.epilog.last_cbuf > 0) {
3738 struct ac_export_args args[8];
3739 int c, last = -1;
3740
3741 /* Get the export arguments, also find out what the last one is. */
3742 for (c = 0; c <= ctx->shader->key.part.ps.epilog.last_cbuf; c++) {
3743 si_llvm_init_export_args(ctx, color,
3744 V_008DFC_SQ_EXP_MRT + c, &args[c]);
3745 if (args[c].enabled_channels)
3746 last = c;
3747 }
3748
3749 /* Emit all exports. */
3750 for (c = 0; c <= ctx->shader->key.part.ps.epilog.last_cbuf; c++) {
3751 if (is_last && last == c) {
3752 args[c].valid_mask = 1; /* whether the EXEC mask is valid */
3753 args[c].done = 1; /* DONE bit */
3754 } else if (!args[c].enabled_channels)
3755 continue; /* unnecessary NULL export */
3756
3757 memcpy(&exp->args[exp->num++], &args[c], sizeof(args[c]));
3758 }
3759 } else {
3760 struct ac_export_args args;
3761
3762 /* Export */
3763 si_llvm_init_export_args(ctx, color, V_008DFC_SQ_EXP_MRT + index,
3764 &args);
3765 if (is_last) {
3766 args.valid_mask = 1; /* whether the EXEC mask is valid */
3767 args.done = 1; /* DONE bit */
3768 } else if (!args.enabled_channels)
3769 return; /* unnecessary NULL export */
3770
3771 memcpy(&exp->args[exp->num++], &args, sizeof(args));
3772 }
3773 }
3774
3775 static void si_emit_ps_exports(struct si_shader_context *ctx,
3776 struct si_ps_exports *exp)
3777 {
3778 for (unsigned i = 0; i < exp->num; i++)
3779 ac_build_export(&ctx->ac, &exp->args[i]);
3780 }
3781
3782 /**
3783 * Return PS outputs in this order:
3784 *
3785 * v[0:3] = color0.xyzw
3786 * v[4:7] = color1.xyzw
3787 * ...
3788 * vN+0 = Depth
3789 * vN+1 = Stencil
3790 * vN+2 = SampleMask
3791 * vN+3 = SampleMaskIn (used for OpenGL smoothing)
3792 *
3793 * The alpha-ref SGPR is returned via its original location.
3794 */
3795 static void si_llvm_return_fs_outputs(struct ac_shader_abi *abi,
3796 unsigned max_outputs,
3797 LLVMValueRef *addrs)
3798 {
3799 struct si_shader_context *ctx = si_shader_context_from_abi(abi);
3800 struct si_shader *shader = ctx->shader;
3801 struct tgsi_shader_info *info = &shader->selector->info;
3802 LLVMBuilderRef builder = ctx->ac.builder;
3803 unsigned i, j, first_vgpr, vgpr;
3804
3805 LLVMValueRef color[8][4] = {};
3806 LLVMValueRef depth = NULL, stencil = NULL, samplemask = NULL;
3807 LLVMValueRef ret;
3808
3809 if (ctx->postponed_kill)
3810 ac_build_kill_if_false(&ctx->ac, LLVMBuildLoad(builder, ctx->postponed_kill, ""));
3811
3812 /* Read the output values. */
3813 for (i = 0; i < info->num_outputs; i++) {
3814 unsigned semantic_name = info->output_semantic_name[i];
3815 unsigned semantic_index = info->output_semantic_index[i];
3816
3817 switch (semantic_name) {
3818 case TGSI_SEMANTIC_COLOR:
3819 assert(semantic_index < 8);
3820 for (j = 0; j < 4; j++) {
3821 LLVMValueRef ptr = addrs[4 * i + j];
3822 LLVMValueRef result = LLVMBuildLoad(builder, ptr, "");
3823 color[semantic_index][j] = result;
3824 }
3825 break;
3826 case TGSI_SEMANTIC_POSITION:
3827 depth = LLVMBuildLoad(builder,
3828 addrs[4 * i + 2], "");
3829 break;
3830 case TGSI_SEMANTIC_STENCIL:
3831 stencil = LLVMBuildLoad(builder,
3832 addrs[4 * i + 1], "");
3833 break;
3834 case TGSI_SEMANTIC_SAMPLEMASK:
3835 samplemask = LLVMBuildLoad(builder,
3836 addrs[4 * i + 0], "");
3837 break;
3838 default:
3839 fprintf(stderr, "Warning: GFX6 unhandled fs output type:%d\n",
3840 semantic_name);
3841 }
3842 }
3843
3844 /* Fill the return structure. */
3845 ret = ctx->return_value;
3846
3847 /* Set SGPRs. */
3848 ret = LLVMBuildInsertValue(builder, ret,
3849 ac_to_integer(&ctx->ac,
3850 LLVMGetParam(ctx->main_fn,
3851 SI_PARAM_ALPHA_REF)),
3852 SI_SGPR_ALPHA_REF, "");
3853
3854 /* Set VGPRs */
3855 first_vgpr = vgpr = SI_SGPR_ALPHA_REF + 1;
3856 for (i = 0; i < ARRAY_SIZE(color); i++) {
3857 if (!color[i][0])
3858 continue;
3859
3860 for (j = 0; j < 4; j++)
3861 ret = LLVMBuildInsertValue(builder, ret, color[i][j], vgpr++, "");
3862 }
3863 if (depth)
3864 ret = LLVMBuildInsertValue(builder, ret, depth, vgpr++, "");
3865 if (stencil)
3866 ret = LLVMBuildInsertValue(builder, ret, stencil, vgpr++, "");
3867 if (samplemask)
3868 ret = LLVMBuildInsertValue(builder, ret, samplemask, vgpr++, "");
3869
3870 /* Add the input sample mask for smoothing at the end. */
3871 if (vgpr < first_vgpr + PS_EPILOG_SAMPLEMASK_MIN_LOC)
3872 vgpr = first_vgpr + PS_EPILOG_SAMPLEMASK_MIN_LOC;
3873 ret = LLVMBuildInsertValue(builder, ret,
3874 LLVMGetParam(ctx->main_fn,
3875 SI_PARAM_SAMPLE_COVERAGE), vgpr++, "");
3876
3877 ctx->return_value = ret;
3878 }
3879
3880 static void membar_emit(
3881 const struct lp_build_tgsi_action *action,
3882 struct lp_build_tgsi_context *bld_base,
3883 struct lp_build_emit_data *emit_data)
3884 {
3885 struct si_shader_context *ctx = si_shader_context(bld_base);
3886 LLVMValueRef src0 = lp_build_emit_fetch(bld_base, emit_data->inst, 0, 0);
3887 unsigned flags = LLVMConstIntGetZExtValue(src0);
3888 unsigned wait_flags = 0;
3889
3890 if (flags & TGSI_MEMBAR_THREAD_GROUP)
3891 wait_flags |= AC_WAIT_LGKM | AC_WAIT_VLOAD | AC_WAIT_VSTORE;
3892
3893 if (flags & (TGSI_MEMBAR_ATOMIC_BUFFER |
3894 TGSI_MEMBAR_SHADER_BUFFER |
3895 TGSI_MEMBAR_SHADER_IMAGE))
3896 wait_flags |= AC_WAIT_VLOAD | AC_WAIT_VSTORE;
3897
3898 if (flags & TGSI_MEMBAR_SHARED)
3899 wait_flags |= AC_WAIT_LGKM;
3900
3901 ac_build_waitcnt(&ctx->ac, wait_flags);
3902 }
3903
3904 static void clock_emit(
3905 const struct lp_build_tgsi_action *action,
3906 struct lp_build_tgsi_context *bld_base,
3907 struct lp_build_emit_data *emit_data)
3908 {
3909 struct si_shader_context *ctx = si_shader_context(bld_base);
3910 LLVMValueRef tmp = ac_build_shader_clock(&ctx->ac);
3911
3912 emit_data->output[0] =
3913 LLVMBuildExtractElement(ctx->ac.builder, tmp, ctx->i32_0, "");
3914 emit_data->output[1] =
3915 LLVMBuildExtractElement(ctx->ac.builder, tmp, ctx->i32_1, "");
3916 }
3917
3918 static void si_llvm_emit_ddxy(
3919 const struct lp_build_tgsi_action *action,
3920 struct lp_build_tgsi_context *bld_base,
3921 struct lp_build_emit_data *emit_data)
3922 {
3923 struct si_shader_context *ctx = si_shader_context(bld_base);
3924 unsigned opcode = emit_data->info->opcode;
3925 LLVMValueRef val;
3926 int idx;
3927 unsigned mask;
3928
3929 if (opcode == TGSI_OPCODE_DDX_FINE)
3930 mask = AC_TID_MASK_LEFT;
3931 else if (opcode == TGSI_OPCODE_DDY_FINE)
3932 mask = AC_TID_MASK_TOP;
3933 else
3934 mask = AC_TID_MASK_TOP_LEFT;
3935
3936 /* for DDX we want to next X pixel, DDY next Y pixel. */
3937 idx = (opcode == TGSI_OPCODE_DDX || opcode == TGSI_OPCODE_DDX_FINE) ? 1 : 2;
3938
3939 val = ac_to_integer(&ctx->ac, emit_data->args[0]);
3940 val = ac_build_ddxy(&ctx->ac, mask, idx, val);
3941 emit_data->output[emit_data->chan] = val;
3942 }
3943
3944 static void build_interp_intrinsic(const struct lp_build_tgsi_action *action,
3945 struct lp_build_tgsi_context *bld_base,
3946 struct lp_build_emit_data *emit_data)
3947 {
3948 struct si_shader_context *ctx = si_shader_context(bld_base);
3949 struct si_shader *shader = ctx->shader;
3950 const struct tgsi_shader_info *info = &shader->selector->info;
3951 LLVMValueRef interp_param;
3952 const struct tgsi_full_instruction *inst = emit_data->inst;
3953 const struct tgsi_full_src_register *input = &inst->Src[0];
3954 int input_base, input_array_size;
3955 int chan;
3956 int i;
3957 LLVMValueRef prim_mask = ac_get_arg(&ctx->ac, ctx->args.prim_mask);
3958 LLVMValueRef array_idx, offset_x = NULL, offset_y = NULL;
3959 int interp_param_idx;
3960 unsigned interp;
3961 unsigned location;
3962
3963 if (inst->Instruction.Opcode == TGSI_OPCODE_INTERP_OFFSET) {
3964 /* offset is in second src, first two channels */
3965 offset_x = lp_build_emit_fetch(bld_base, emit_data->inst, 1,
3966 TGSI_CHAN_X);
3967 offset_y = lp_build_emit_fetch(bld_base, emit_data->inst, 1,
3968 TGSI_CHAN_Y);
3969 } else if (inst->Instruction.Opcode == TGSI_OPCODE_INTERP_SAMPLE) {
3970 LLVMValueRef sample_position;
3971 LLVMValueRef sample_id;
3972 LLVMValueRef halfval = LLVMConstReal(ctx->f32, 0.5f);
3973
3974 /* fetch sample ID, then fetch its sample position,
3975 * and place into first two channels.
3976 */
3977 sample_id = lp_build_emit_fetch(bld_base,
3978 emit_data->inst, 1, TGSI_CHAN_X);
3979 sample_id = ac_to_integer(&ctx->ac, sample_id);
3980
3981 /* Section 8.13.2 (Interpolation Functions) of the OpenGL Shading
3982 * Language 4.50 spec says about interpolateAtSample:
3983 *
3984 * "Returns the value of the input interpolant variable at
3985 * the location of sample number sample. If multisample
3986 * buffers are not available, the input variable will be
3987 * evaluated at the center of the pixel. If sample sample
3988 * does not exist, the position used to interpolate the
3989 * input variable is undefined."
3990 *
3991 * This means that sample_id values outside of the valid are
3992 * in fact valid input, and the usual mechanism for loading the
3993 * sample position doesn't work.
3994 */
3995 if (ctx->shader->key.mono.u.ps.interpolate_at_sample_force_center) {
3996 LLVMValueRef center[4] = {
3997 LLVMConstReal(ctx->f32, 0.5),
3998 LLVMConstReal(ctx->f32, 0.5),
3999 ctx->ac.f32_0,
4000 ctx->ac.f32_0,
4001 };
4002
4003 sample_position = ac_build_gather_values(&ctx->ac, center, 4);
4004 } else {
4005 sample_position = load_sample_position(&ctx->abi, sample_id);
4006 }
4007
4008 offset_x = LLVMBuildExtractElement(ctx->ac.builder, sample_position,
4009 ctx->i32_0, "");
4010
4011 offset_x = LLVMBuildFSub(ctx->ac.builder, offset_x, halfval, "");
4012 offset_y = LLVMBuildExtractElement(ctx->ac.builder, sample_position,
4013 ctx->i32_1, "");
4014 offset_y = LLVMBuildFSub(ctx->ac.builder, offset_y, halfval, "");
4015 }
4016
4017 assert(input->Register.File == TGSI_FILE_INPUT);
4018
4019 if (input->Register.Indirect) {
4020 unsigned array_id = input->Indirect.ArrayID;
4021
4022 if (array_id) {
4023 input_base = info->input_array_first[array_id];
4024 input_array_size = info->input_array_last[array_id] - input_base + 1;
4025 } else {
4026 input_base = inst->Src[0].Register.Index;
4027 input_array_size = info->num_inputs - input_base;
4028 }
4029
4030 array_idx = si_get_indirect_index(ctx, &input->Indirect,
4031 1, input->Register.Index - input_base);
4032 } else {
4033 input_base = inst->Src[0].Register.Index;
4034 input_array_size = 1;
4035 array_idx = ctx->i32_0;
4036 }
4037
4038 interp = shader->selector->info.input_interpolate[input_base];
4039
4040 if (inst->Instruction.Opcode == TGSI_OPCODE_INTERP_OFFSET ||
4041 inst->Instruction.Opcode == TGSI_OPCODE_INTERP_SAMPLE)
4042 location = TGSI_INTERPOLATE_LOC_CENTER;
4043 else
4044 location = TGSI_INTERPOLATE_LOC_CENTROID;
4045
4046 interp_param_idx = lookup_interp_param_index(interp, location);
4047 if (interp_param_idx == -1)
4048 return;
4049 else if (interp_param_idx)
4050 interp_param = LLVMGetParam(ctx->main_fn, interp_param_idx);
4051 else
4052 interp_param = NULL;
4053
4054 if (inst->Instruction.Opcode == TGSI_OPCODE_INTERP_OFFSET ||
4055 inst->Instruction.Opcode == TGSI_OPCODE_INTERP_SAMPLE) {
4056 LLVMValueRef ij_out[2];
4057 LLVMValueRef ddxy_out = ac_build_ddxy_interp(&ctx->ac, interp_param);
4058
4059 /*
4060 * take the I then J parameters, and the DDX/Y for it, and
4061 * calculate the IJ inputs for the interpolator.
4062 * temp1 = ddx * offset/sample.x + I;
4063 * interp_param.I = ddy * offset/sample.y + temp1;
4064 * temp1 = ddx * offset/sample.x + J;
4065 * interp_param.J = ddy * offset/sample.y + temp1;
4066 */
4067 for (i = 0; i < 2; i++) {
4068 LLVMValueRef ix_ll = LLVMConstInt(ctx->i32, i, 0);
4069 LLVMValueRef iy_ll = LLVMConstInt(ctx->i32, i + 2, 0);
4070 LLVMValueRef ddx_el = LLVMBuildExtractElement(ctx->ac.builder,
4071 ddxy_out, ix_ll, "");
4072 LLVMValueRef ddy_el = LLVMBuildExtractElement(ctx->ac.builder,
4073 ddxy_out, iy_ll, "");
4074 LLVMValueRef interp_el = LLVMBuildExtractElement(ctx->ac.builder,
4075 interp_param, ix_ll, "");
4076 LLVMValueRef temp;
4077
4078 interp_el = ac_to_float(&ctx->ac, interp_el);
4079
4080 temp = ac_build_fmad(&ctx->ac, ddx_el, offset_x, interp_el);
4081 ij_out[i] = ac_build_fmad(&ctx->ac, ddy_el, offset_y, temp);
4082 }
4083 interp_param = ac_build_gather_values(&ctx->ac, ij_out, 2);
4084 }
4085
4086 if (interp_param)
4087 interp_param = ac_to_float(&ctx->ac, interp_param);
4088
4089 for (chan = 0; chan < 4; chan++) {
4090 LLVMValueRef gather = LLVMGetUndef(LLVMVectorType(ctx->f32, input_array_size));
4091 unsigned schan = tgsi_util_get_full_src_register_swizzle(&inst->Src[0], chan);
4092
4093 for (unsigned idx = 0; idx < input_array_size; ++idx) {
4094 LLVMValueRef v, i = NULL, j = NULL;
4095
4096 if (interp_param) {
4097 i = LLVMBuildExtractElement(
4098 ctx->ac.builder, interp_param, ctx->i32_0, "");
4099 j = LLVMBuildExtractElement(
4100 ctx->ac.builder, interp_param, ctx->i32_1, "");
4101 }
4102 v = si_build_fs_interp(ctx, input_base + idx, schan,
4103 prim_mask, i, j);
4104
4105 gather = LLVMBuildInsertElement(ctx->ac.builder,
4106 gather, v, LLVMConstInt(ctx->i32, idx, false), "");
4107 }
4108
4109 emit_data->output[chan] = LLVMBuildExtractElement(
4110 ctx->ac.builder, gather, array_idx, "");
4111 }
4112 }
4113
4114 static void vote_all_emit(
4115 const struct lp_build_tgsi_action *action,
4116 struct lp_build_tgsi_context *bld_base,
4117 struct lp_build_emit_data *emit_data)
4118 {
4119 struct si_shader_context *ctx = si_shader_context(bld_base);
4120
4121 LLVMValueRef tmp = ac_build_vote_all(&ctx->ac, emit_data->args[0]);
4122 emit_data->output[emit_data->chan] =
4123 LLVMBuildSExt(ctx->ac.builder, tmp, ctx->i32, "");
4124 }
4125
4126 static void vote_any_emit(
4127 const struct lp_build_tgsi_action *action,
4128 struct lp_build_tgsi_context *bld_base,
4129 struct lp_build_emit_data *emit_data)
4130 {
4131 struct si_shader_context *ctx = si_shader_context(bld_base);
4132
4133 LLVMValueRef tmp = ac_build_vote_any(&ctx->ac, emit_data->args[0]);
4134 emit_data->output[emit_data->chan] =
4135 LLVMBuildSExt(ctx->ac.builder, tmp, ctx->i32, "");
4136 }
4137
4138 static void vote_eq_emit(
4139 const struct lp_build_tgsi_action *action,
4140 struct lp_build_tgsi_context *bld_base,
4141 struct lp_build_emit_data *emit_data)
4142 {
4143 struct si_shader_context *ctx = si_shader_context(bld_base);
4144
4145 LLVMValueRef tmp = ac_build_vote_eq(&ctx->ac, emit_data->args[0]);
4146 emit_data->output[emit_data->chan] =
4147 LLVMBuildSExt(ctx->ac.builder, tmp, ctx->i32, "");
4148 }
4149
4150 static void ballot_emit(
4151 const struct lp_build_tgsi_action *action,
4152 struct lp_build_tgsi_context *bld_base,
4153 struct lp_build_emit_data *emit_data)
4154 {
4155 struct si_shader_context *ctx = si_shader_context(bld_base);
4156 LLVMBuilderRef builder = ctx->ac.builder;
4157 LLVMValueRef tmp;
4158
4159 tmp = lp_build_emit_fetch(bld_base, emit_data->inst, 0, TGSI_CHAN_X);
4160 tmp = ac_build_ballot(&ctx->ac, tmp);
4161
4162 emit_data->output[0] = LLVMBuildTrunc(builder, tmp, ctx->i32, "");
4163
4164 if (ctx->ac.wave_size == 32) {
4165 emit_data->output[1] = ctx->i32_0;
4166 } else {
4167 tmp = LLVMBuildLShr(builder, tmp, LLVMConstInt(ctx->i64, 32, 0), "");
4168 emit_data->output[1] = LLVMBuildTrunc(builder, tmp, ctx->i32, "");
4169 }
4170 }
4171
4172 static void read_lane_emit(
4173 const struct lp_build_tgsi_action *action,
4174 struct lp_build_tgsi_context *bld_base,
4175 struct lp_build_emit_data *emit_data)
4176 {
4177 struct si_shader_context *ctx = si_shader_context(bld_base);
4178
4179 if (emit_data->inst->Instruction.Opcode == TGSI_OPCODE_READ_INVOC) {
4180 emit_data->args[0] = lp_build_emit_fetch(bld_base, emit_data->inst,
4181 0, emit_data->src_chan);
4182
4183 /* Always read the source invocation (= lane) from the X channel. */
4184 emit_data->args[1] = lp_build_emit_fetch(bld_base, emit_data->inst,
4185 1, TGSI_CHAN_X);
4186 emit_data->arg_count = 2;
4187 }
4188
4189 /* We currently have no other way to prevent LLVM from lifting the icmp
4190 * calls to a dominating basic block.
4191 */
4192 ac_build_optimization_barrier(&ctx->ac, &emit_data->args[0]);
4193
4194 for (unsigned i = 0; i < emit_data->arg_count; ++i)
4195 emit_data->args[i] = ac_to_integer(&ctx->ac, emit_data->args[i]);
4196
4197 emit_data->output[emit_data->chan] =
4198 ac_build_intrinsic(&ctx->ac, action->intr_name,
4199 ctx->i32, emit_data->args, emit_data->arg_count,
4200 AC_FUNC_ATTR_READNONE |
4201 AC_FUNC_ATTR_CONVERGENT);
4202 }
4203
4204 static unsigned si_llvm_get_stream(struct lp_build_tgsi_context *bld_base,
4205 struct lp_build_emit_data *emit_data)
4206 {
4207 struct si_shader_context *ctx = si_shader_context(bld_base);
4208 struct tgsi_src_register src0 = emit_data->inst->Src[0].Register;
4209 LLVMValueRef imm;
4210 unsigned stream;
4211
4212 assert(src0.File == TGSI_FILE_IMMEDIATE);
4213
4214 imm = ctx->imms[src0.Index * TGSI_NUM_CHANNELS + src0.SwizzleX];
4215 stream = LLVMConstIntGetZExtValue(imm) & 0x3;
4216 return stream;
4217 }
4218
4219 /* Emit one vertex from the geometry shader */
4220 static void si_llvm_emit_vertex(struct ac_shader_abi *abi,
4221 unsigned stream,
4222 LLVMValueRef *addrs)
4223 {
4224 struct si_shader_context *ctx = si_shader_context_from_abi(abi);
4225
4226 if (ctx->shader->key.as_ngg) {
4227 gfx10_ngg_gs_emit_vertex(ctx, stream, addrs);
4228 return;
4229 }
4230
4231 struct tgsi_shader_info *info = &ctx->shader->selector->info;
4232 struct si_shader *shader = ctx->shader;
4233 LLVMValueRef soffset = ac_get_arg(&ctx->ac, ctx->gs2vs_offset);
4234 LLVMValueRef gs_next_vertex;
4235 LLVMValueRef can_emit;
4236 unsigned chan, offset;
4237 int i;
4238
4239 /* Write vertex attribute values to GSVS ring */
4240 gs_next_vertex = LLVMBuildLoad(ctx->ac.builder,
4241 ctx->gs_next_vertex[stream],
4242 "");
4243
4244 /* If this thread has already emitted the declared maximum number of
4245 * vertices, skip the write: excessive vertex emissions are not
4246 * supposed to have any effect.
4247 *
4248 * If the shader has no writes to memory, kill it instead. This skips
4249 * further memory loads and may allow LLVM to skip to the end
4250 * altogether.
4251 */
4252 can_emit = LLVMBuildICmp(ctx->ac.builder, LLVMIntULT, gs_next_vertex,
4253 LLVMConstInt(ctx->i32,
4254 shader->selector->gs_max_out_vertices, 0), "");
4255
4256 bool use_kill = !info->writes_memory;
4257 if (use_kill) {
4258 ac_build_kill_if_false(&ctx->ac, can_emit);
4259 } else {
4260 ac_build_ifcc(&ctx->ac, can_emit, 6505);
4261 }
4262
4263 offset = 0;
4264 for (i = 0; i < info->num_outputs; i++) {
4265 for (chan = 0; chan < 4; chan++) {
4266 if (!(info->output_usagemask[i] & (1 << chan)) ||
4267 ((info->output_streams[i] >> (2 * chan)) & 3) != stream)
4268 continue;
4269
4270 LLVMValueRef out_val = LLVMBuildLoad(ctx->ac.builder, addrs[4 * i + chan], "");
4271 LLVMValueRef voffset =
4272 LLVMConstInt(ctx->i32, offset *
4273 shader->selector->gs_max_out_vertices, 0);
4274 offset++;
4275
4276 voffset = LLVMBuildAdd(ctx->ac.builder, voffset, gs_next_vertex, "");
4277 voffset = LLVMBuildMul(ctx->ac.builder, voffset,
4278 LLVMConstInt(ctx->i32, 4, 0), "");
4279
4280 out_val = ac_to_integer(&ctx->ac, out_val);
4281
4282 ac_build_buffer_store_dword(&ctx->ac,
4283 ctx->gsvs_ring[stream],
4284 out_val, 1,
4285 voffset, soffset, 0,
4286 ac_glc | ac_slc | ac_swizzled);
4287 }
4288 }
4289
4290 gs_next_vertex = LLVMBuildAdd(ctx->ac.builder, gs_next_vertex, ctx->i32_1, "");
4291 LLVMBuildStore(ctx->ac.builder, gs_next_vertex, ctx->gs_next_vertex[stream]);
4292
4293 /* Signal vertex emission if vertex data was written. */
4294 if (offset) {
4295 ac_build_sendmsg(&ctx->ac, AC_SENDMSG_GS_OP_EMIT | AC_SENDMSG_GS | (stream << 8),
4296 si_get_gs_wave_id(ctx));
4297 }
4298
4299 if (!use_kill)
4300 ac_build_endif(&ctx->ac, 6505);
4301 }
4302
4303 /* Emit one vertex from the geometry shader */
4304 static void si_tgsi_emit_vertex(
4305 const struct lp_build_tgsi_action *action,
4306 struct lp_build_tgsi_context *bld_base,
4307 struct lp_build_emit_data *emit_data)
4308 {
4309 struct si_shader_context *ctx = si_shader_context(bld_base);
4310 unsigned stream = si_llvm_get_stream(bld_base, emit_data);
4311
4312 si_llvm_emit_vertex(&ctx->abi, stream, ctx->outputs[0]);
4313 }
4314
4315 /* Cut one primitive from the geometry shader */
4316 static void si_llvm_emit_primitive(struct ac_shader_abi *abi,
4317 unsigned stream)
4318 {
4319 struct si_shader_context *ctx = si_shader_context_from_abi(abi);
4320
4321 if (ctx->shader->key.as_ngg) {
4322 LLVMBuildStore(ctx->ac.builder, ctx->ac.i32_0, ctx->gs_curprim_verts[stream]);
4323 return;
4324 }
4325
4326 /* Signal primitive cut */
4327 ac_build_sendmsg(&ctx->ac, AC_SENDMSG_GS_OP_CUT | AC_SENDMSG_GS | (stream << 8),
4328 si_get_gs_wave_id(ctx));
4329 }
4330
4331 /* Cut one primitive from the geometry shader */
4332 static void si_tgsi_emit_primitive(
4333 const struct lp_build_tgsi_action *action,
4334 struct lp_build_tgsi_context *bld_base,
4335 struct lp_build_emit_data *emit_data)
4336 {
4337 struct si_shader_context *ctx = si_shader_context(bld_base);
4338
4339 si_llvm_emit_primitive(&ctx->abi, si_llvm_get_stream(bld_base, emit_data));
4340 }
4341
4342 static void si_llvm_emit_barrier(const struct lp_build_tgsi_action *action,
4343 struct lp_build_tgsi_context *bld_base,
4344 struct lp_build_emit_data *emit_data)
4345 {
4346 struct si_shader_context *ctx = si_shader_context(bld_base);
4347
4348 /* GFX6 only (thanks to a hw bug workaround):
4349 * The real barrier instruction isn’t needed, because an entire patch
4350 * always fits into a single wave.
4351 */
4352 if (ctx->screen->info.chip_class == GFX6 &&
4353 ctx->type == PIPE_SHADER_TESS_CTRL) {
4354 ac_build_waitcnt(&ctx->ac, AC_WAIT_LGKM | AC_WAIT_VLOAD | AC_WAIT_VSTORE);
4355 return;
4356 }
4357
4358 ac_build_s_barrier(&ctx->ac);
4359 }
4360
4361 void si_create_function(struct si_shader_context *ctx,
4362 const char *name,
4363 LLVMTypeRef *returns, unsigned num_returns,
4364 unsigned max_workgroup_size)
4365 {
4366 si_llvm_create_func(ctx, name, returns, num_returns);
4367 ctx->return_value = LLVMGetUndef(ctx->return_type);
4368
4369 if (ctx->screen->info.address32_hi) {
4370 ac_llvm_add_target_dep_function_attr(ctx->main_fn,
4371 "amdgpu-32bit-address-high-bits",
4372 ctx->screen->info.address32_hi);
4373 }
4374
4375 LLVMAddTargetDependentFunctionAttr(ctx->main_fn,
4376 "no-signed-zeros-fp-math",
4377 "true");
4378
4379 ac_llvm_set_workgroup_size(ctx->main_fn, max_workgroup_size);
4380 }
4381
4382 static void declare_streamout_params(struct si_shader_context *ctx,
4383 struct pipe_stream_output_info *so)
4384 {
4385 if (ctx->screen->use_ngg_streamout) {
4386 if (ctx->type == PIPE_SHADER_TESS_EVAL)
4387 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, NULL);
4388 return;
4389 }
4390
4391 /* Streamout SGPRs. */
4392 if (so->num_outputs) {
4393 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, &ctx->streamout_config);
4394 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, &ctx->streamout_write_index);
4395 } else if (ctx->type == PIPE_SHADER_TESS_EVAL) {
4396 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, NULL);
4397 }
4398
4399 /* A streamout buffer offset is loaded if the stride is non-zero. */
4400 for (int i = 0; i < 4; i++) {
4401 if (!so->stride[i])
4402 continue;
4403
4404 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, &ctx->streamout_offset[i]);
4405 }
4406 }
4407
4408 static unsigned si_get_max_workgroup_size(const struct si_shader *shader)
4409 {
4410 switch (shader->selector->type) {
4411 case PIPE_SHADER_VERTEX:
4412 case PIPE_SHADER_TESS_EVAL:
4413 return shader->key.as_ngg ? 128 : 0;
4414
4415 case PIPE_SHADER_TESS_CTRL:
4416 /* Return this so that LLVM doesn't remove s_barrier
4417 * instructions on chips where we use s_barrier. */
4418 return shader->selector->screen->info.chip_class >= GFX7 ? 128 : 0;
4419
4420 case PIPE_SHADER_GEOMETRY:
4421 return shader->selector->screen->info.chip_class >= GFX9 ? 128 : 0;
4422
4423 case PIPE_SHADER_COMPUTE:
4424 break; /* see below */
4425
4426 default:
4427 return 0;
4428 }
4429
4430 const unsigned *properties = shader->selector->info.properties;
4431 unsigned max_work_group_size =
4432 properties[TGSI_PROPERTY_CS_FIXED_BLOCK_WIDTH] *
4433 properties[TGSI_PROPERTY_CS_FIXED_BLOCK_HEIGHT] *
4434 properties[TGSI_PROPERTY_CS_FIXED_BLOCK_DEPTH];
4435
4436 if (!max_work_group_size) {
4437 /* This is a variable group size compute shader,
4438 * compile it for the maximum possible group size.
4439 */
4440 max_work_group_size = SI_MAX_VARIABLE_THREADS_PER_BLOCK;
4441 }
4442 return max_work_group_size;
4443 }
4444
4445 static void declare_const_and_shader_buffers(struct si_shader_context *ctx,
4446 bool assign_params)
4447 {
4448 enum ac_arg_type const_shader_buf_type;
4449
4450 if (ctx->shader->selector->info.const_buffers_declared == 1 &&
4451 ctx->shader->selector->info.shader_buffers_declared == 0)
4452 const_shader_buf_type = AC_ARG_CONST_FLOAT_PTR;
4453 else
4454 const_shader_buf_type = AC_ARG_CONST_DESC_PTR;
4455
4456 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, const_shader_buf_type,
4457 assign_params ? &ctx->const_and_shader_buffers :
4458 &ctx->other_const_and_shader_buffers);
4459 }
4460
4461 static void declare_samplers_and_images(struct si_shader_context *ctx,
4462 bool assign_params)
4463 {
4464 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_CONST_IMAGE_PTR,
4465 assign_params ? &ctx->samplers_and_images :
4466 &ctx->other_samplers_and_images);
4467 }
4468
4469 static void declare_per_stage_desc_pointers(struct si_shader_context *ctx,
4470 bool assign_params)
4471 {
4472 declare_const_and_shader_buffers(ctx, assign_params);
4473 declare_samplers_and_images(ctx, assign_params);
4474 }
4475
4476 static void declare_global_desc_pointers(struct si_shader_context *ctx)
4477 {
4478 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_CONST_DESC_PTR,
4479 &ctx->rw_buffers);
4480 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_CONST_IMAGE_PTR,
4481 &ctx->bindless_samplers_and_images);
4482 }
4483
4484 static void declare_vs_specific_input_sgprs(struct si_shader_context *ctx)
4485 {
4486 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, &ctx->vs_state_bits);
4487 if (!ctx->shader->is_gs_copy_shader) {
4488 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, &ctx->args.base_vertex);
4489 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, &ctx->args.start_instance);
4490 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, &ctx->args.draw_id);
4491 }
4492 }
4493
4494 static void declare_vs_input_vgprs(struct si_shader_context *ctx,
4495 unsigned *num_prolog_vgprs)
4496 {
4497 struct si_shader *shader = ctx->shader;
4498
4499 ac_add_arg(&ctx->args, AC_ARG_VGPR, 1, AC_ARG_INT, &ctx->args.vertex_id);
4500 if (shader->key.as_ls) {
4501 ac_add_arg(&ctx->args, AC_ARG_VGPR, 1, AC_ARG_INT, &ctx->rel_auto_id);
4502 if (ctx->screen->info.chip_class >= GFX10) {
4503 ac_add_arg(&ctx->args, AC_ARG_VGPR, 1, AC_ARG_INT, NULL); /* user VGPR */
4504 ac_add_arg(&ctx->args, AC_ARG_VGPR, 1, AC_ARG_INT, &ctx->args.instance_id);
4505 } else {
4506 ac_add_arg(&ctx->args, AC_ARG_VGPR, 1, AC_ARG_INT, &ctx->args.instance_id);
4507 ac_add_arg(&ctx->args, AC_ARG_VGPR, 1, AC_ARG_INT, NULL); /* unused */
4508 }
4509 } else if (ctx->screen->info.chip_class >= GFX10) {
4510 ac_add_arg(&ctx->args, AC_ARG_VGPR, 1, AC_ARG_INT, NULL); /* user VGPR */
4511 ac_add_arg(&ctx->args, AC_ARG_VGPR, 1, AC_ARG_INT,
4512 &ctx->vs_prim_id); /* user vgpr or PrimID (legacy) */
4513 ac_add_arg(&ctx->args, AC_ARG_VGPR, 1, AC_ARG_INT, &ctx->args.instance_id);
4514 } else {
4515 ac_add_arg(&ctx->args, AC_ARG_VGPR, 1, AC_ARG_INT, &ctx->args.instance_id);
4516 ac_add_arg(&ctx->args, AC_ARG_VGPR, 1, AC_ARG_INT, &ctx->vs_prim_id);
4517 ac_add_arg(&ctx->args, AC_ARG_VGPR, 1, AC_ARG_INT, NULL); /* unused */
4518 }
4519
4520 if (!shader->is_gs_copy_shader) {
4521 /* Vertex load indices. */
4522 ac_add_arg(&ctx->args, AC_ARG_VGPR, 1, AC_ARG_INT, &ctx->vertex_index0);
4523 for (unsigned i = 1; i < shader->selector->info.num_inputs; i++)
4524 ac_add_arg(&ctx->args, AC_ARG_VGPR, 1, AC_ARG_INT, NULL);
4525 *num_prolog_vgprs += shader->selector->info.num_inputs;
4526 }
4527 }
4528
4529 static void declare_vs_blit_inputs(struct si_shader_context *ctx,
4530 unsigned vs_blit_property)
4531 {
4532 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT,
4533 &ctx->vs_blit_inputs); /* i16 x1, y1 */
4534 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, NULL); /* i16 x1, y1 */
4535 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_FLOAT, NULL); /* depth */
4536
4537 if (vs_blit_property == SI_VS_BLIT_SGPRS_POS_COLOR) {
4538 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_FLOAT, NULL); /* color0 */
4539 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_FLOAT, NULL); /* color1 */
4540 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_FLOAT, NULL); /* color2 */
4541 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_FLOAT, NULL); /* color3 */
4542 } else if (vs_blit_property == SI_VS_BLIT_SGPRS_POS_TEXCOORD) {
4543 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_FLOAT, NULL); /* texcoord.x1 */
4544 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_FLOAT, NULL); /* texcoord.y1 */
4545 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_FLOAT, NULL); /* texcoord.x2 */
4546 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_FLOAT, NULL); /* texcoord.y2 */
4547 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_FLOAT, NULL); /* texcoord.z */
4548 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_FLOAT, NULL); /* texcoord.w */
4549 }
4550 }
4551
4552 static void declare_tes_input_vgprs(struct si_shader_context *ctx)
4553 {
4554 ac_add_arg(&ctx->args, AC_ARG_VGPR, 1, AC_ARG_FLOAT, &ctx->tes_u);
4555 ac_add_arg(&ctx->args, AC_ARG_VGPR, 1, AC_ARG_FLOAT, &ctx->tes_v);
4556 ac_add_arg(&ctx->args, AC_ARG_VGPR, 1, AC_ARG_INT, &ctx->tes_rel_patch_id);
4557 ac_add_arg(&ctx->args, AC_ARG_VGPR, 1, AC_ARG_INT, &ctx->args.tes_patch_id);
4558 }
4559
4560 enum {
4561 /* Convenient merged shader definitions. */
4562 SI_SHADER_MERGED_VERTEX_TESSCTRL = PIPE_SHADER_TYPES,
4563 SI_SHADER_MERGED_VERTEX_OR_TESSEVAL_GEOMETRY,
4564 };
4565
4566 static void add_arg_checked(struct ac_shader_args *args,
4567 enum ac_arg_regfile file,
4568 unsigned registers, enum ac_arg_type type,
4569 struct ac_arg *arg,
4570 unsigned idx)
4571 {
4572 assert(args->arg_count == idx);
4573 ac_add_arg(args, file, registers, type, arg);
4574 }
4575
4576 static void create_function(struct si_shader_context *ctx)
4577 {
4578 struct si_shader *shader = ctx->shader;
4579 LLVMTypeRef returns[16+32*4];
4580 unsigned i, num_return_sgprs;
4581 unsigned num_returns = 0;
4582 unsigned num_prolog_vgprs = 0;
4583 unsigned type = ctx->type;
4584 unsigned vs_blit_property =
4585 shader->selector->info.properties[TGSI_PROPERTY_VS_BLIT_SGPRS_AMD];
4586
4587 memset(&ctx->args, 0, sizeof(ctx->args));
4588
4589 /* Set MERGED shaders. */
4590 if (ctx->screen->info.chip_class >= GFX9) {
4591 if (shader->key.as_ls || type == PIPE_SHADER_TESS_CTRL)
4592 type = SI_SHADER_MERGED_VERTEX_TESSCTRL; /* LS or HS */
4593 else if (shader->key.as_es || shader->key.as_ngg || type == PIPE_SHADER_GEOMETRY)
4594 type = SI_SHADER_MERGED_VERTEX_OR_TESSEVAL_GEOMETRY;
4595 }
4596
4597 switch (type) {
4598 case PIPE_SHADER_VERTEX:
4599 declare_global_desc_pointers(ctx);
4600
4601 if (vs_blit_property) {
4602 declare_vs_blit_inputs(ctx, vs_blit_property);
4603
4604 /* VGPRs */
4605 declare_vs_input_vgprs(ctx, &num_prolog_vgprs);
4606 break;
4607 }
4608
4609 declare_per_stage_desc_pointers(ctx, true);
4610 declare_vs_specific_input_sgprs(ctx);
4611 if (!shader->is_gs_copy_shader) {
4612 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_CONST_DESC_PTR,
4613 &ctx->vertex_buffers);
4614 }
4615
4616 if (shader->key.as_es) {
4617 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT,
4618 &ctx->es2gs_offset);
4619 } else if (shader->key.as_ls) {
4620 /* no extra parameters */
4621 } else {
4622 /* The locations of the other parameters are assigned dynamically. */
4623 declare_streamout_params(ctx, &shader->selector->so);
4624 }
4625
4626 /* VGPRs */
4627 declare_vs_input_vgprs(ctx, &num_prolog_vgprs);
4628
4629 /* Return values */
4630 if (shader->key.opt.vs_as_prim_discard_cs) {
4631 for (i = 0; i < 4; i++)
4632 returns[num_returns++] = ctx->f32; /* VGPRs */
4633 }
4634 break;
4635
4636 case PIPE_SHADER_TESS_CTRL: /* GFX6-GFX8 */
4637 declare_global_desc_pointers(ctx);
4638 declare_per_stage_desc_pointers(ctx, true);
4639 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, &ctx->tcs_offchip_layout);
4640 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, &ctx->tcs_out_lds_offsets);
4641 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, &ctx->tcs_out_lds_layout);
4642 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, &ctx->vs_state_bits);
4643 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, &ctx->tcs_offchip_offset);
4644 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, &ctx->tcs_factor_offset);
4645
4646 /* VGPRs */
4647 ac_add_arg(&ctx->args, AC_ARG_VGPR, 1, AC_ARG_INT, &ctx->args.tcs_patch_id);
4648 ac_add_arg(&ctx->args, AC_ARG_VGPR, 1, AC_ARG_INT, &ctx->args.tcs_rel_ids);
4649
4650 /* param_tcs_offchip_offset and param_tcs_factor_offset are
4651 * placed after the user SGPRs.
4652 */
4653 for (i = 0; i < GFX6_TCS_NUM_USER_SGPR + 2; i++)
4654 returns[num_returns++] = ctx->i32; /* SGPRs */
4655 for (i = 0; i < 11; i++)
4656 returns[num_returns++] = ctx->f32; /* VGPRs */
4657 break;
4658
4659 case SI_SHADER_MERGED_VERTEX_TESSCTRL:
4660 /* Merged stages have 8 system SGPRs at the beginning. */
4661 /* SPI_SHADER_USER_DATA_ADDR_LO/HI_HS */
4662 declare_per_stage_desc_pointers(ctx,
4663 ctx->type == PIPE_SHADER_TESS_CTRL);
4664 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, &ctx->tcs_offchip_offset);
4665 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, &ctx->merged_wave_info);
4666 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, &ctx->tcs_factor_offset);
4667 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, &ctx->merged_scratch_offset);
4668 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, NULL); /* unused */
4669 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, NULL); /* unused */
4670
4671 declare_global_desc_pointers(ctx);
4672 declare_per_stage_desc_pointers(ctx,
4673 ctx->type == PIPE_SHADER_VERTEX);
4674 declare_vs_specific_input_sgprs(ctx);
4675
4676 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, &ctx->tcs_offchip_layout);
4677 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, &ctx->tcs_out_lds_offsets);
4678 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, &ctx->tcs_out_lds_layout);
4679 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_CONST_DESC_PTR, &ctx->vertex_buffers);
4680
4681 /* VGPRs (first TCS, then VS) */
4682 ac_add_arg(&ctx->args, AC_ARG_VGPR, 1, AC_ARG_INT, &ctx->args.tcs_patch_id);
4683 ac_add_arg(&ctx->args, AC_ARG_VGPR, 1, AC_ARG_INT, &ctx->args.tcs_rel_ids);
4684
4685 if (ctx->type == PIPE_SHADER_VERTEX) {
4686 declare_vs_input_vgprs(ctx, &num_prolog_vgprs);
4687
4688 /* LS return values are inputs to the TCS main shader part. */
4689 for (i = 0; i < 8 + GFX9_TCS_NUM_USER_SGPR; i++)
4690 returns[num_returns++] = ctx->i32; /* SGPRs */
4691 for (i = 0; i < 2; i++)
4692 returns[num_returns++] = ctx->f32; /* VGPRs */
4693 } else {
4694 /* TCS return values are inputs to the TCS epilog.
4695 *
4696 * param_tcs_offchip_offset, param_tcs_factor_offset,
4697 * param_tcs_offchip_layout, and param_rw_buffers
4698 * should be passed to the epilog.
4699 */
4700 for (i = 0; i <= 8 + GFX9_SGPR_TCS_OUT_LAYOUT; i++)
4701 returns[num_returns++] = ctx->i32; /* SGPRs */
4702 for (i = 0; i < 11; i++)
4703 returns[num_returns++] = ctx->f32; /* VGPRs */
4704 }
4705 break;
4706
4707 case SI_SHADER_MERGED_VERTEX_OR_TESSEVAL_GEOMETRY:
4708 /* Merged stages have 8 system SGPRs at the beginning. */
4709 /* SPI_SHADER_USER_DATA_ADDR_LO/HI_GS */
4710 declare_per_stage_desc_pointers(ctx,
4711 ctx->type == PIPE_SHADER_GEOMETRY);
4712
4713 if (ctx->shader->key.as_ngg)
4714 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, &ctx->gs_tg_info);
4715 else
4716 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, &ctx->gs2vs_offset);
4717
4718 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, &ctx->merged_wave_info);
4719 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, &ctx->tcs_offchip_offset);
4720 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, &ctx->merged_scratch_offset);
4721 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, NULL); /* unused (SPI_SHADER_PGM_LO/HI_GS << 8) */
4722 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, NULL); /* unused (SPI_SHADER_PGM_LO/HI_GS >> 24) */
4723
4724 declare_global_desc_pointers(ctx);
4725 if (ctx->type != PIPE_SHADER_VERTEX || !vs_blit_property) {
4726 declare_per_stage_desc_pointers(ctx,
4727 (ctx->type == PIPE_SHADER_VERTEX ||
4728 ctx->type == PIPE_SHADER_TESS_EVAL));
4729 }
4730
4731 if (ctx->type == PIPE_SHADER_VERTEX) {
4732 if (vs_blit_property)
4733 declare_vs_blit_inputs(ctx, vs_blit_property);
4734 else
4735 declare_vs_specific_input_sgprs(ctx);
4736 } else {
4737 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, &ctx->vs_state_bits);
4738 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, &ctx->tcs_offchip_layout);
4739 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, &ctx->tes_offchip_addr);
4740 /* Declare as many input SGPRs as the VS has. */
4741 }
4742
4743 if (ctx->type == PIPE_SHADER_VERTEX) {
4744 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_CONST_DESC_PTR,
4745 &ctx->vertex_buffers);
4746 }
4747
4748 /* VGPRs (first GS, then VS/TES) */
4749 ac_add_arg(&ctx->args, AC_ARG_VGPR, 1, AC_ARG_INT, &ctx->gs_vtx01_offset);
4750 ac_add_arg(&ctx->args, AC_ARG_VGPR, 1, AC_ARG_INT, &ctx->gs_vtx23_offset);
4751 ac_add_arg(&ctx->args, AC_ARG_VGPR, 1, AC_ARG_INT, &ctx->args.gs_prim_id);
4752 ac_add_arg(&ctx->args, AC_ARG_VGPR, 1, AC_ARG_INT, &ctx->args.gs_invocation_id);
4753 ac_add_arg(&ctx->args, AC_ARG_VGPR, 1, AC_ARG_INT, &ctx->gs_vtx45_offset);
4754
4755 if (ctx->type == PIPE_SHADER_VERTEX) {
4756 declare_vs_input_vgprs(ctx, &num_prolog_vgprs);
4757 } else if (ctx->type == PIPE_SHADER_TESS_EVAL) {
4758 declare_tes_input_vgprs(ctx);
4759 }
4760
4761 if (ctx->shader->key.as_es &&
4762 (ctx->type == PIPE_SHADER_VERTEX ||
4763 ctx->type == PIPE_SHADER_TESS_EVAL)) {
4764 unsigned num_user_sgprs;
4765
4766 if (ctx->type == PIPE_SHADER_VERTEX)
4767 num_user_sgprs = GFX9_VSGS_NUM_USER_SGPR;
4768 else
4769 num_user_sgprs = GFX9_TESGS_NUM_USER_SGPR;
4770
4771 /* ES return values are inputs to GS. */
4772 for (i = 0; i < 8 + num_user_sgprs; i++)
4773 returns[num_returns++] = ctx->i32; /* SGPRs */
4774 for (i = 0; i < 5; i++)
4775 returns[num_returns++] = ctx->f32; /* VGPRs */
4776 }
4777 break;
4778
4779 case PIPE_SHADER_TESS_EVAL:
4780 declare_global_desc_pointers(ctx);
4781 declare_per_stage_desc_pointers(ctx, true);
4782 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, &ctx->vs_state_bits);
4783 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, &ctx->tcs_offchip_layout);
4784 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, &ctx->tes_offchip_addr);
4785
4786 if (shader->key.as_es) {
4787 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, &ctx->tcs_offchip_offset);
4788 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, NULL);
4789 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, &ctx->es2gs_offset);
4790 } else {
4791 declare_streamout_params(ctx, &shader->selector->so);
4792 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, &ctx->tcs_offchip_offset);
4793 }
4794
4795 /* VGPRs */
4796 declare_tes_input_vgprs(ctx);
4797 break;
4798
4799 case PIPE_SHADER_GEOMETRY:
4800 declare_global_desc_pointers(ctx);
4801 declare_per_stage_desc_pointers(ctx, true);
4802 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, &ctx->gs2vs_offset);
4803 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, &ctx->gs_wave_id);
4804
4805 /* VGPRs */
4806 ac_add_arg(&ctx->args, AC_ARG_VGPR, 1, AC_ARG_INT, &ctx->gs_vtx_offset[0]);
4807 ac_add_arg(&ctx->args, AC_ARG_VGPR, 1, AC_ARG_INT, &ctx->gs_vtx_offset[1]);
4808 ac_add_arg(&ctx->args, AC_ARG_VGPR, 1, AC_ARG_INT, &ctx->args.gs_prim_id);
4809 ac_add_arg(&ctx->args, AC_ARG_VGPR, 1, AC_ARG_INT, &ctx->gs_vtx_offset[2]);
4810 ac_add_arg(&ctx->args, AC_ARG_VGPR, 1, AC_ARG_INT, &ctx->gs_vtx_offset[3]);
4811 ac_add_arg(&ctx->args, AC_ARG_VGPR, 1, AC_ARG_INT, &ctx->gs_vtx_offset[4]);
4812 ac_add_arg(&ctx->args, AC_ARG_VGPR, 1, AC_ARG_INT, &ctx->gs_vtx_offset[5]);
4813 ac_add_arg(&ctx->args, AC_ARG_VGPR, 1, AC_ARG_INT, &ctx->args.gs_invocation_id);
4814 break;
4815
4816 case PIPE_SHADER_FRAGMENT:
4817 declare_global_desc_pointers(ctx);
4818 declare_per_stage_desc_pointers(ctx, true);
4819 add_arg_checked(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, NULL,
4820 SI_PARAM_ALPHA_REF);
4821 add_arg_checked(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT,
4822 &ctx->args.prim_mask, SI_PARAM_PRIM_MASK);
4823
4824 add_arg_checked(&ctx->args, AC_ARG_VGPR, 2, AC_ARG_INT, &ctx->args.persp_sample,
4825 SI_PARAM_PERSP_SAMPLE);
4826 add_arg_checked(&ctx->args, AC_ARG_VGPR, 2, AC_ARG_INT,
4827 &ctx->args.persp_center, SI_PARAM_PERSP_CENTER);
4828 add_arg_checked(&ctx->args, AC_ARG_VGPR, 2, AC_ARG_INT,
4829 &ctx->args.persp_centroid, SI_PARAM_PERSP_CENTROID);
4830 add_arg_checked(&ctx->args, AC_ARG_VGPR, 3, AC_ARG_INT,
4831 NULL, SI_PARAM_PERSP_PULL_MODEL);
4832 add_arg_checked(&ctx->args, AC_ARG_VGPR, 2, AC_ARG_INT,
4833 &ctx->args.linear_sample, SI_PARAM_LINEAR_SAMPLE);
4834 add_arg_checked(&ctx->args, AC_ARG_VGPR, 2, AC_ARG_INT,
4835 &ctx->args.linear_center, SI_PARAM_LINEAR_CENTER);
4836 add_arg_checked(&ctx->args, AC_ARG_VGPR, 2, AC_ARG_INT,
4837 &ctx->args.linear_centroid, SI_PARAM_LINEAR_CENTROID);
4838 add_arg_checked(&ctx->args, AC_ARG_VGPR, 3, AC_ARG_FLOAT,
4839 NULL, SI_PARAM_LINE_STIPPLE_TEX);
4840 add_arg_checked(&ctx->args, AC_ARG_VGPR, 1, AC_ARG_FLOAT,
4841 &ctx->args.frag_pos[0], SI_PARAM_POS_X_FLOAT);
4842 add_arg_checked(&ctx->args, AC_ARG_VGPR, 1, AC_ARG_FLOAT,
4843 &ctx->args.frag_pos[1], SI_PARAM_POS_Y_FLOAT);
4844 add_arg_checked(&ctx->args, AC_ARG_VGPR, 1, AC_ARG_FLOAT,
4845 &ctx->args.frag_pos[2], SI_PARAM_POS_Z_FLOAT);
4846 add_arg_checked(&ctx->args, AC_ARG_VGPR, 1, AC_ARG_FLOAT,
4847 &ctx->args.frag_pos[3], SI_PARAM_POS_W_FLOAT);
4848 add_arg_checked(&ctx->args, AC_ARG_VGPR, 1, AC_ARG_INT,
4849 &ctx->args.front_face, SI_PARAM_FRONT_FACE);
4850 shader->info.face_vgpr_index = 20;
4851 add_arg_checked(&ctx->args, AC_ARG_VGPR, 1, AC_ARG_INT,
4852 &ctx->args.ancillary, SI_PARAM_ANCILLARY);
4853 shader->info.ancillary_vgpr_index = 21;
4854 add_arg_checked(&ctx->args, AC_ARG_VGPR, 1, AC_ARG_FLOAT,
4855 &ctx->args.sample_coverage, SI_PARAM_SAMPLE_COVERAGE);
4856 add_arg_checked(&ctx->args, AC_ARG_VGPR, 1, AC_ARG_INT,
4857 &ctx->pos_fixed_pt, SI_PARAM_POS_FIXED_PT);
4858
4859 /* Color inputs from the prolog. */
4860 if (shader->selector->info.colors_read) {
4861 unsigned num_color_elements =
4862 util_bitcount(shader->selector->info.colors_read);
4863
4864 for (i = 0; i < num_color_elements; i++)
4865 ac_add_arg(&ctx->args, AC_ARG_VGPR, 1, AC_ARG_FLOAT, NULL);
4866
4867 num_prolog_vgprs += num_color_elements;
4868 }
4869
4870 /* Outputs for the epilog. */
4871 num_return_sgprs = SI_SGPR_ALPHA_REF + 1;
4872 num_returns =
4873 num_return_sgprs +
4874 util_bitcount(shader->selector->info.colors_written) * 4 +
4875 shader->selector->info.writes_z +
4876 shader->selector->info.writes_stencil +
4877 shader->selector->info.writes_samplemask +
4878 1 /* SampleMaskIn */;
4879
4880 num_returns = MAX2(num_returns,
4881 num_return_sgprs +
4882 PS_EPILOG_SAMPLEMASK_MIN_LOC + 1);
4883
4884 for (i = 0; i < num_return_sgprs; i++)
4885 returns[i] = ctx->i32;
4886 for (; i < num_returns; i++)
4887 returns[i] = ctx->f32;
4888 break;
4889
4890 case PIPE_SHADER_COMPUTE:
4891 declare_global_desc_pointers(ctx);
4892 declare_per_stage_desc_pointers(ctx, true);
4893 if (shader->selector->info.uses_grid_size)
4894 ac_add_arg(&ctx->args, AC_ARG_SGPR, 3, AC_ARG_INT,
4895 &ctx->args.num_work_groups);
4896 if (shader->selector->info.uses_block_size &&
4897 shader->selector->info.properties[TGSI_PROPERTY_CS_FIXED_BLOCK_WIDTH] == 0)
4898 ac_add_arg(&ctx->args, AC_ARG_SGPR, 3, AC_ARG_INT, &ctx->block_size);
4899
4900 unsigned cs_user_data_dwords =
4901 shader->selector->info.properties[TGSI_PROPERTY_CS_USER_DATA_COMPONENTS_AMD];
4902 if (cs_user_data_dwords) {
4903 ac_add_arg(&ctx->args, AC_ARG_SGPR, cs_user_data_dwords, AC_ARG_INT,
4904 &ctx->cs_user_data);
4905 }
4906
4907 for (i = 0; i < 3; i++) {
4908 if (shader->selector->info.uses_block_id[i]) {
4909 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT,
4910 &ctx->args.workgroup_ids[i]);
4911 }
4912 }
4913
4914 ac_add_arg(&ctx->args, AC_ARG_VGPR, 3, AC_ARG_INT,
4915 &ctx->args.local_invocation_ids);
4916 break;
4917 default:
4918 assert(0 && "unimplemented shader");
4919 return;
4920 }
4921
4922 si_create_function(ctx, "main", returns, num_returns,
4923 si_get_max_workgroup_size(shader));
4924
4925 /* Reserve register locations for VGPR inputs the PS prolog may need. */
4926 if (ctx->type == PIPE_SHADER_FRAGMENT && !ctx->shader->is_monolithic) {
4927 ac_llvm_add_target_dep_function_attr(ctx->main_fn,
4928 "InitialPSInputAddr",
4929 S_0286D0_PERSP_SAMPLE_ENA(1) |
4930 S_0286D0_PERSP_CENTER_ENA(1) |
4931 S_0286D0_PERSP_CENTROID_ENA(1) |
4932 S_0286D0_LINEAR_SAMPLE_ENA(1) |
4933 S_0286D0_LINEAR_CENTER_ENA(1) |
4934 S_0286D0_LINEAR_CENTROID_ENA(1) |
4935 S_0286D0_FRONT_FACE_ENA(1) |
4936 S_0286D0_ANCILLARY_ENA(1) |
4937 S_0286D0_POS_FIXED_PT_ENA(1));
4938 }
4939
4940 shader->info.num_input_sgprs = ctx->args.num_sgprs_used;
4941 shader->info.num_input_vgprs = ctx->args.num_vgprs_used;
4942
4943 assert(shader->info.num_input_vgprs >= num_prolog_vgprs);
4944 shader->info.num_input_vgprs -= num_prolog_vgprs;
4945
4946 if (shader->key.as_ls || ctx->type == PIPE_SHADER_TESS_CTRL) {
4947 if (USE_LDS_SYMBOLS && LLVM_VERSION_MAJOR >= 9) {
4948 /* The LSHS size is not known until draw time, so we append it
4949 * at the end of whatever LDS use there may be in the rest of
4950 * the shader (currently none, unless LLVM decides to do its
4951 * own LDS-based lowering).
4952 */
4953 ctx->ac.lds = LLVMAddGlobalInAddressSpace(
4954 ctx->ac.module, LLVMArrayType(ctx->i32, 0),
4955 "__lds_end", AC_ADDR_SPACE_LDS);
4956 LLVMSetAlignment(ctx->ac.lds, 256);
4957 } else {
4958 ac_declare_lds_as_pointer(&ctx->ac);
4959 }
4960 }
4961
4962 /* Unlike radv, we override these arguments in the prolog, so to the
4963 * API shader they appear as normal arguments.
4964 */
4965 if (ctx->type == PIPE_SHADER_VERTEX) {
4966 ctx->abi.vertex_id = ac_get_arg(&ctx->ac, ctx->args.vertex_id);
4967 ctx->abi.instance_id = ac_get_arg(&ctx->ac, ctx->args.instance_id);
4968 } else if (ctx->type == PIPE_SHADER_FRAGMENT) {
4969 ctx->abi.persp_centroid = ac_get_arg(&ctx->ac, ctx->args.persp_centroid);
4970 ctx->abi.linear_centroid = ac_get_arg(&ctx->ac, ctx->args.linear_centroid);
4971 }
4972 }
4973
4974 /* Ensure that the esgs ring is declared.
4975 *
4976 * We declare it with 64KB alignment as a hint that the
4977 * pointer value will always be 0.
4978 */
4979 static void declare_esgs_ring(struct si_shader_context *ctx)
4980 {
4981 if (ctx->esgs_ring)
4982 return;
4983
4984 assert(!LLVMGetNamedGlobal(ctx->ac.module, "esgs_ring"));
4985
4986 ctx->esgs_ring = LLVMAddGlobalInAddressSpace(
4987 ctx->ac.module, LLVMArrayType(ctx->i32, 0),
4988 "esgs_ring",
4989 AC_ADDR_SPACE_LDS);
4990 LLVMSetLinkage(ctx->esgs_ring, LLVMExternalLinkage);
4991 LLVMSetAlignment(ctx->esgs_ring, 64 * 1024);
4992 }
4993
4994 /**
4995 * Load ESGS and GSVS ring buffer resource descriptors and save the variables
4996 * for later use.
4997 */
4998 static void preload_ring_buffers(struct si_shader_context *ctx)
4999 {
5000 LLVMBuilderRef builder = ctx->ac.builder;
5001
5002 LLVMValueRef buf_ptr = ac_get_arg(&ctx->ac, ctx->rw_buffers);
5003
5004 if (ctx->shader->key.as_es || ctx->type == PIPE_SHADER_GEOMETRY) {
5005 if (ctx->screen->info.chip_class <= GFX8) {
5006 unsigned ring =
5007 ctx->type == PIPE_SHADER_GEOMETRY ? SI_GS_RING_ESGS
5008 : SI_ES_RING_ESGS;
5009 LLVMValueRef offset = LLVMConstInt(ctx->i32, ring, 0);
5010
5011 ctx->esgs_ring =
5012 ac_build_load_to_sgpr(&ctx->ac, buf_ptr, offset);
5013 } else {
5014 if (USE_LDS_SYMBOLS && LLVM_VERSION_MAJOR >= 9) {
5015 /* Declare the ESGS ring as an explicit LDS symbol. */
5016 declare_esgs_ring(ctx);
5017 } else {
5018 ac_declare_lds_as_pointer(&ctx->ac);
5019 ctx->esgs_ring = ctx->ac.lds;
5020 }
5021 }
5022 }
5023
5024 if (ctx->shader->is_gs_copy_shader) {
5025 LLVMValueRef offset = LLVMConstInt(ctx->i32, SI_RING_GSVS, 0);
5026
5027 ctx->gsvs_ring[0] =
5028 ac_build_load_to_sgpr(&ctx->ac, buf_ptr, offset);
5029 } else if (ctx->type == PIPE_SHADER_GEOMETRY) {
5030 const struct si_shader_selector *sel = ctx->shader->selector;
5031 LLVMValueRef offset = LLVMConstInt(ctx->i32, SI_RING_GSVS, 0);
5032 LLVMValueRef base_ring;
5033
5034 base_ring = ac_build_load_to_sgpr(&ctx->ac, buf_ptr, offset);
5035
5036 /* The conceptual layout of the GSVS ring is
5037 * v0c0 .. vLv0 v0c1 .. vLc1 ..
5038 * but the real memory layout is swizzled across
5039 * threads:
5040 * t0v0c0 .. t15v0c0 t0v1c0 .. t15v1c0 ... t15vLcL
5041 * t16v0c0 ..
5042 * Override the buffer descriptor accordingly.
5043 */
5044 LLVMTypeRef v2i64 = LLVMVectorType(ctx->i64, 2);
5045 uint64_t stream_offset = 0;
5046
5047 for (unsigned stream = 0; stream < 4; ++stream) {
5048 unsigned num_components;
5049 unsigned stride;
5050 unsigned num_records;
5051 LLVMValueRef ring, tmp;
5052
5053 num_components = sel->info.num_stream_output_components[stream];
5054 if (!num_components)
5055 continue;
5056
5057 stride = 4 * num_components * sel->gs_max_out_vertices;
5058
5059 /* Limit on the stride field for <= GFX7. */
5060 assert(stride < (1 << 14));
5061
5062 num_records = ctx->ac.wave_size;
5063
5064 ring = LLVMBuildBitCast(builder, base_ring, v2i64, "");
5065 tmp = LLVMBuildExtractElement(builder, ring, ctx->i32_0, "");
5066 tmp = LLVMBuildAdd(builder, tmp,
5067 LLVMConstInt(ctx->i64,
5068 stream_offset, 0), "");
5069 stream_offset += stride * ctx->ac.wave_size;
5070
5071 ring = LLVMBuildInsertElement(builder, ring, tmp, ctx->i32_0, "");
5072 ring = LLVMBuildBitCast(builder, ring, ctx->v4i32, "");
5073 tmp = LLVMBuildExtractElement(builder, ring, ctx->i32_1, "");
5074 tmp = LLVMBuildOr(builder, tmp,
5075 LLVMConstInt(ctx->i32,
5076 S_008F04_STRIDE(stride) |
5077 S_008F04_SWIZZLE_ENABLE(1), 0), "");
5078 ring = LLVMBuildInsertElement(builder, ring, tmp, ctx->i32_1, "");
5079 ring = LLVMBuildInsertElement(builder, ring,
5080 LLVMConstInt(ctx->i32, num_records, 0),
5081 LLVMConstInt(ctx->i32, 2, 0), "");
5082
5083 uint32_t rsrc3 =
5084 S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) |
5085 S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
5086 S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) |
5087 S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W) |
5088 S_008F0C_INDEX_STRIDE(1) | /* index_stride = 16 (elements) */
5089 S_008F0C_ADD_TID_ENABLE(1);
5090
5091 if (ctx->ac.chip_class >= GFX10) {
5092 rsrc3 |= S_008F0C_FORMAT(V_008F0C_IMG_FORMAT_32_FLOAT) |
5093 S_008F0C_OOB_SELECT(2) |
5094 S_008F0C_RESOURCE_LEVEL(1);
5095 } else {
5096 rsrc3 |= S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
5097 S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32) |
5098 S_008F0C_ELEMENT_SIZE(1); /* element_size = 4 (bytes) */
5099 }
5100
5101 ring = LLVMBuildInsertElement(builder, ring,
5102 LLVMConstInt(ctx->i32, rsrc3, false),
5103 LLVMConstInt(ctx->i32, 3, 0), "");
5104
5105 ctx->gsvs_ring[stream] = ring;
5106 }
5107 } else if (ctx->type == PIPE_SHADER_TESS_EVAL) {
5108 ctx->tess_offchip_ring = get_tess_ring_descriptor(ctx, TESS_OFFCHIP_RING_TES);
5109 }
5110 }
5111
5112 static void si_llvm_emit_polygon_stipple(struct si_shader_context *ctx,
5113 LLVMValueRef param_rw_buffers,
5114 struct ac_arg param_pos_fixed_pt)
5115 {
5116 LLVMBuilderRef builder = ctx->ac.builder;
5117 LLVMValueRef slot, desc, offset, row, bit, address[2];
5118
5119 /* Use the fixed-point gl_FragCoord input.
5120 * Since the stipple pattern is 32x32 and it repeats, just get 5 bits
5121 * per coordinate to get the repeating effect.
5122 */
5123 address[0] = si_unpack_param(ctx, param_pos_fixed_pt, 0, 5);
5124 address[1] = si_unpack_param(ctx, param_pos_fixed_pt, 16, 5);
5125
5126 /* Load the buffer descriptor. */
5127 slot = LLVMConstInt(ctx->i32, SI_PS_CONST_POLY_STIPPLE, 0);
5128 desc = ac_build_load_to_sgpr(&ctx->ac, param_rw_buffers, slot);
5129
5130 /* The stipple pattern is 32x32, each row has 32 bits. */
5131 offset = LLVMBuildMul(builder, address[1],
5132 LLVMConstInt(ctx->i32, 4, 0), "");
5133 row = buffer_load_const(ctx, desc, offset);
5134 row = ac_to_integer(&ctx->ac, row);
5135 bit = LLVMBuildLShr(builder, row, address[0], "");
5136 bit = LLVMBuildTrunc(builder, bit, ctx->i1, "");
5137 ac_build_kill_if_false(&ctx->ac, bit);
5138 }
5139
5140 /* For the UMR disassembler. */
5141 #define DEBUGGER_END_OF_CODE_MARKER 0xbf9f0000 /* invalid instruction */
5142 #define DEBUGGER_NUM_MARKERS 5
5143
5144 static bool si_shader_binary_open(struct si_screen *screen,
5145 struct si_shader *shader,
5146 struct ac_rtld_binary *rtld)
5147 {
5148 const struct si_shader_selector *sel = shader->selector;
5149 const char *part_elfs[5];
5150 size_t part_sizes[5];
5151 unsigned num_parts = 0;
5152
5153 #define add_part(shader_or_part) \
5154 if (shader_or_part) { \
5155 part_elfs[num_parts] = (shader_or_part)->binary.elf_buffer; \
5156 part_sizes[num_parts] = (shader_or_part)->binary.elf_size; \
5157 num_parts++; \
5158 }
5159
5160 add_part(shader->prolog);
5161 add_part(shader->previous_stage);
5162 add_part(shader->prolog2);
5163 add_part(shader);
5164 add_part(shader->epilog);
5165
5166 #undef add_part
5167
5168 struct ac_rtld_symbol lds_symbols[2];
5169 unsigned num_lds_symbols = 0;
5170
5171 if (sel && screen->info.chip_class >= GFX9 && !shader->is_gs_copy_shader &&
5172 (sel->type == PIPE_SHADER_GEOMETRY || shader->key.as_ngg)) {
5173 /* We add this symbol even on LLVM <= 8 to ensure that
5174 * shader->config.lds_size is set correctly below.
5175 */
5176 struct ac_rtld_symbol *sym = &lds_symbols[num_lds_symbols++];
5177 sym->name = "esgs_ring";
5178 sym->size = shader->gs_info.esgs_ring_size;
5179 sym->align = 64 * 1024;
5180 }
5181
5182 if (shader->key.as_ngg && sel->type == PIPE_SHADER_GEOMETRY) {
5183 struct ac_rtld_symbol *sym = &lds_symbols[num_lds_symbols++];
5184 sym->name = "ngg_emit";
5185 sym->size = shader->ngg.ngg_emit_size * 4;
5186 sym->align = 4;
5187 }
5188
5189 bool ok = ac_rtld_open(rtld, (struct ac_rtld_open_info){
5190 .info = &screen->info,
5191 .options = {
5192 .halt_at_entry = screen->options.halt_shaders,
5193 },
5194 .shader_type = tgsi_processor_to_shader_stage(sel->type),
5195 .wave_size = si_get_shader_wave_size(shader),
5196 .num_parts = num_parts,
5197 .elf_ptrs = part_elfs,
5198 .elf_sizes = part_sizes,
5199 .num_shared_lds_symbols = num_lds_symbols,
5200 .shared_lds_symbols = lds_symbols });
5201
5202 if (rtld->lds_size > 0) {
5203 unsigned alloc_granularity = screen->info.chip_class >= GFX7 ? 512 : 256;
5204 shader->config.lds_size =
5205 align(rtld->lds_size, alloc_granularity) / alloc_granularity;
5206 }
5207
5208 return ok;
5209 }
5210
5211 static unsigned si_get_shader_binary_size(struct si_screen *screen, struct si_shader *shader)
5212 {
5213 struct ac_rtld_binary rtld;
5214 si_shader_binary_open(screen, shader, &rtld);
5215 return rtld.exec_size;
5216 }
5217
5218 static bool si_get_external_symbol(void *data, const char *name, uint64_t *value)
5219 {
5220 uint64_t *scratch_va = data;
5221
5222 if (!strcmp(scratch_rsrc_dword0_symbol, name)) {
5223 *value = (uint32_t)*scratch_va;
5224 return true;
5225 }
5226 if (!strcmp(scratch_rsrc_dword1_symbol, name)) {
5227 /* Enable scratch coalescing. */
5228 *value = S_008F04_BASE_ADDRESS_HI(*scratch_va >> 32) |
5229 S_008F04_SWIZZLE_ENABLE(1);
5230 return true;
5231 }
5232
5233 return false;
5234 }
5235
5236 bool si_shader_binary_upload(struct si_screen *sscreen, struct si_shader *shader,
5237 uint64_t scratch_va)
5238 {
5239 struct ac_rtld_binary binary;
5240 if (!si_shader_binary_open(sscreen, shader, &binary))
5241 return false;
5242
5243 si_resource_reference(&shader->bo, NULL);
5244 shader->bo = si_aligned_buffer_create(&sscreen->b,
5245 sscreen->info.cpdma_prefetch_writes_memory ?
5246 0 : SI_RESOURCE_FLAG_READ_ONLY,
5247 PIPE_USAGE_IMMUTABLE,
5248 align(binary.rx_size, SI_CPDMA_ALIGNMENT),
5249 256);
5250 if (!shader->bo)
5251 return false;
5252
5253 /* Upload. */
5254 struct ac_rtld_upload_info u = {};
5255 u.binary = &binary;
5256 u.get_external_symbol = si_get_external_symbol;
5257 u.cb_data = &scratch_va;
5258 u.rx_va = shader->bo->gpu_address;
5259 u.rx_ptr = sscreen->ws->buffer_map(shader->bo->buf, NULL,
5260 PIPE_TRANSFER_READ_WRITE |
5261 PIPE_TRANSFER_UNSYNCHRONIZED |
5262 RADEON_TRANSFER_TEMPORARY);
5263 if (!u.rx_ptr)
5264 return false;
5265
5266 bool ok = ac_rtld_upload(&u);
5267
5268 sscreen->ws->buffer_unmap(shader->bo->buf);
5269 ac_rtld_close(&binary);
5270
5271 return ok;
5272 }
5273
5274 static void si_shader_dump_disassembly(struct si_screen *screen,
5275 const struct si_shader_binary *binary,
5276 enum pipe_shader_type shader_type,
5277 unsigned wave_size,
5278 struct pipe_debug_callback *debug,
5279 const char *name, FILE *file)
5280 {
5281 struct ac_rtld_binary rtld_binary;
5282
5283 if (!ac_rtld_open(&rtld_binary, (struct ac_rtld_open_info){
5284 .info = &screen->info,
5285 .shader_type = tgsi_processor_to_shader_stage(shader_type),
5286 .wave_size = wave_size,
5287 .num_parts = 1,
5288 .elf_ptrs = &binary->elf_buffer,
5289 .elf_sizes = &binary->elf_size }))
5290 return;
5291
5292 const char *disasm;
5293 size_t nbytes;
5294
5295 if (!ac_rtld_get_section_by_name(&rtld_binary, ".AMDGPU.disasm", &disasm, &nbytes))
5296 goto out;
5297
5298 if (nbytes > INT_MAX)
5299 goto out;
5300
5301 if (debug && debug->debug_message) {
5302 /* Very long debug messages are cut off, so send the
5303 * disassembly one line at a time. This causes more
5304 * overhead, but on the plus side it simplifies
5305 * parsing of resulting logs.
5306 */
5307 pipe_debug_message(debug, SHADER_INFO,
5308 "Shader Disassembly Begin");
5309
5310 uint64_t line = 0;
5311 while (line < nbytes) {
5312 int count = nbytes - line;
5313 const char *nl = memchr(disasm + line, '\n', nbytes - line);
5314 if (nl)
5315 count = nl - (disasm + line);
5316
5317 if (count) {
5318 pipe_debug_message(debug, SHADER_INFO,
5319 "%.*s", count, disasm + line);
5320 }
5321
5322 line += count + 1;
5323 }
5324
5325 pipe_debug_message(debug, SHADER_INFO,
5326 "Shader Disassembly End");
5327 }
5328
5329 if (file) {
5330 fprintf(file, "Shader %s disassembly:\n", name);
5331 fprintf(file, "%*s", (int)nbytes, disasm);
5332 }
5333
5334 out:
5335 ac_rtld_close(&rtld_binary);
5336 }
5337
5338 static void si_calculate_max_simd_waves(struct si_shader *shader)
5339 {
5340 struct si_screen *sscreen = shader->selector->screen;
5341 struct ac_shader_config *conf = &shader->config;
5342 unsigned num_inputs = shader->selector->info.num_inputs;
5343 unsigned lds_increment = sscreen->info.chip_class >= GFX7 ? 512 : 256;
5344 unsigned lds_per_wave = 0;
5345 unsigned max_simd_waves;
5346
5347 max_simd_waves = sscreen->info.max_wave64_per_simd;
5348
5349 /* Compute LDS usage for PS. */
5350 switch (shader->selector->type) {
5351 case PIPE_SHADER_FRAGMENT:
5352 /* The minimum usage per wave is (num_inputs * 48). The maximum
5353 * usage is (num_inputs * 48 * 16).
5354 * We can get anything in between and it varies between waves.
5355 *
5356 * The 48 bytes per input for a single primitive is equal to
5357 * 4 bytes/component * 4 components/input * 3 points.
5358 *
5359 * Other stages don't know the size at compile time or don't
5360 * allocate LDS per wave, but instead they do it per thread group.
5361 */
5362 lds_per_wave = conf->lds_size * lds_increment +
5363 align(num_inputs * 48, lds_increment);
5364 break;
5365 case PIPE_SHADER_COMPUTE:
5366 if (shader->selector) {
5367 unsigned max_workgroup_size =
5368 si_get_max_workgroup_size(shader);
5369 lds_per_wave = (conf->lds_size * lds_increment) /
5370 DIV_ROUND_UP(max_workgroup_size,
5371 sscreen->compute_wave_size);
5372 }
5373 break;
5374 default:;
5375 }
5376
5377 /* Compute the per-SIMD wave counts. */
5378 if (conf->num_sgprs) {
5379 max_simd_waves =
5380 MIN2(max_simd_waves,
5381 sscreen->info.num_physical_sgprs_per_simd / conf->num_sgprs);
5382 }
5383
5384 if (conf->num_vgprs) {
5385 /* Always print wave limits as Wave64, so that we can compare
5386 * Wave32 and Wave64 with shader-db fairly. */
5387 unsigned max_vgprs = sscreen->info.num_physical_wave64_vgprs_per_simd;
5388 max_simd_waves = MIN2(max_simd_waves, max_vgprs / conf->num_vgprs);
5389 }
5390
5391 /* LDS is 64KB per CU (4 SIMDs) on GFX6-9, which is 16KB per SIMD (usage above
5392 * 16KB makes some SIMDs unoccupied).
5393 *
5394 * LDS is 128KB in WGP mode and 64KB in CU mode. Assume the WGP mode is used.
5395 */
5396 unsigned max_lds_size = sscreen->info.chip_class >= GFX10 ? 128*1024 : 64*1024;
5397 unsigned max_lds_per_simd = max_lds_size / 4;
5398 if (lds_per_wave)
5399 max_simd_waves = MIN2(max_simd_waves, max_lds_per_simd / lds_per_wave);
5400
5401 shader->info.max_simd_waves = max_simd_waves;
5402 }
5403
5404 void si_shader_dump_stats_for_shader_db(struct si_screen *screen,
5405 struct si_shader *shader,
5406 struct pipe_debug_callback *debug)
5407 {
5408 const struct ac_shader_config *conf = &shader->config;
5409
5410 if (screen->options.debug_disassembly)
5411 si_shader_dump_disassembly(screen, &shader->binary,
5412 shader->selector->type,
5413 si_get_shader_wave_size(shader),
5414 debug, "main", NULL);
5415
5416 pipe_debug_message(debug, SHADER_INFO,
5417 "Shader Stats: SGPRS: %d VGPRS: %d Code Size: %d "
5418 "LDS: %d Scratch: %d Max Waves: %d Spilled SGPRs: %d "
5419 "Spilled VGPRs: %d PrivMem VGPRs: %d",
5420 conf->num_sgprs, conf->num_vgprs,
5421 si_get_shader_binary_size(screen, shader),
5422 conf->lds_size, conf->scratch_bytes_per_wave,
5423 shader->info.max_simd_waves, conf->spilled_sgprs,
5424 conf->spilled_vgprs, shader->info.private_mem_vgprs);
5425 }
5426
5427 static void si_shader_dump_stats(struct si_screen *sscreen,
5428 struct si_shader *shader,
5429 FILE *file,
5430 bool check_debug_option)
5431 {
5432 const struct ac_shader_config *conf = &shader->config;
5433
5434 if (!check_debug_option ||
5435 si_can_dump_shader(sscreen, shader->selector->type)) {
5436 if (shader->selector->type == PIPE_SHADER_FRAGMENT) {
5437 fprintf(file, "*** SHADER CONFIG ***\n"
5438 "SPI_PS_INPUT_ADDR = 0x%04x\n"
5439 "SPI_PS_INPUT_ENA = 0x%04x\n",
5440 conf->spi_ps_input_addr, conf->spi_ps_input_ena);
5441 }
5442
5443 fprintf(file, "*** SHADER STATS ***\n"
5444 "SGPRS: %d\n"
5445 "VGPRS: %d\n"
5446 "Spilled SGPRs: %d\n"
5447 "Spilled VGPRs: %d\n"
5448 "Private memory VGPRs: %d\n"
5449 "Code Size: %d bytes\n"
5450 "LDS: %d blocks\n"
5451 "Scratch: %d bytes per wave\n"
5452 "Max Waves: %d\n"
5453 "********************\n\n\n",
5454 conf->num_sgprs, conf->num_vgprs,
5455 conf->spilled_sgprs, conf->spilled_vgprs,
5456 shader->info.private_mem_vgprs,
5457 si_get_shader_binary_size(sscreen, shader),
5458 conf->lds_size, conf->scratch_bytes_per_wave,
5459 shader->info.max_simd_waves);
5460 }
5461 }
5462
5463 const char *si_get_shader_name(const struct si_shader *shader)
5464 {
5465 switch (shader->selector->type) {
5466 case PIPE_SHADER_VERTEX:
5467 if (shader->key.as_es)
5468 return "Vertex Shader as ES";
5469 else if (shader->key.as_ls)
5470 return "Vertex Shader as LS";
5471 else if (shader->key.opt.vs_as_prim_discard_cs)
5472 return "Vertex Shader as Primitive Discard CS";
5473 else if (shader->key.as_ngg)
5474 return "Vertex Shader as ESGS";
5475 else
5476 return "Vertex Shader as VS";
5477 case PIPE_SHADER_TESS_CTRL:
5478 return "Tessellation Control Shader";
5479 case PIPE_SHADER_TESS_EVAL:
5480 if (shader->key.as_es)
5481 return "Tessellation Evaluation Shader as ES";
5482 else if (shader->key.as_ngg)
5483 return "Tessellation Evaluation Shader as ESGS";
5484 else
5485 return "Tessellation Evaluation Shader as VS";
5486 case PIPE_SHADER_GEOMETRY:
5487 if (shader->is_gs_copy_shader)
5488 return "GS Copy Shader as VS";
5489 else
5490 return "Geometry Shader";
5491 case PIPE_SHADER_FRAGMENT:
5492 return "Pixel Shader";
5493 case PIPE_SHADER_COMPUTE:
5494 return "Compute Shader";
5495 default:
5496 return "Unknown Shader";
5497 }
5498 }
5499
5500 void si_shader_dump(struct si_screen *sscreen, struct si_shader *shader,
5501 struct pipe_debug_callback *debug,
5502 FILE *file, bool check_debug_option)
5503 {
5504 enum pipe_shader_type shader_type = shader->selector->type;
5505
5506 if (!check_debug_option ||
5507 si_can_dump_shader(sscreen, shader_type))
5508 si_dump_shader_key(shader, file);
5509
5510 if (!check_debug_option && shader->binary.llvm_ir_string) {
5511 if (shader->previous_stage &&
5512 shader->previous_stage->binary.llvm_ir_string) {
5513 fprintf(file, "\n%s - previous stage - LLVM IR:\n\n",
5514 si_get_shader_name(shader));
5515 fprintf(file, "%s\n", shader->previous_stage->binary.llvm_ir_string);
5516 }
5517
5518 fprintf(file, "\n%s - main shader part - LLVM IR:\n\n",
5519 si_get_shader_name(shader));
5520 fprintf(file, "%s\n", shader->binary.llvm_ir_string);
5521 }
5522
5523 if (!check_debug_option ||
5524 (si_can_dump_shader(sscreen, shader_type) &&
5525 !(sscreen->debug_flags & DBG(NO_ASM)))) {
5526 unsigned wave_size = si_get_shader_wave_size(shader);
5527
5528 fprintf(file, "\n%s:\n", si_get_shader_name(shader));
5529
5530 if (shader->prolog)
5531 si_shader_dump_disassembly(sscreen, &shader->prolog->binary,
5532 shader_type, wave_size, debug, "prolog", file);
5533 if (shader->previous_stage)
5534 si_shader_dump_disassembly(sscreen, &shader->previous_stage->binary,
5535 shader_type, wave_size, debug, "previous stage", file);
5536 if (shader->prolog2)
5537 si_shader_dump_disassembly(sscreen, &shader->prolog2->binary,
5538 shader_type, wave_size, debug, "prolog2", file);
5539
5540 si_shader_dump_disassembly(sscreen, &shader->binary, shader_type,
5541 wave_size, debug, "main", file);
5542
5543 if (shader->epilog)
5544 si_shader_dump_disassembly(sscreen, &shader->epilog->binary,
5545 shader_type, wave_size, debug, "epilog", file);
5546 fprintf(file, "\n");
5547 }
5548
5549 si_shader_dump_stats(sscreen, shader, file, check_debug_option);
5550 }
5551
5552 static int si_compile_llvm(struct si_screen *sscreen,
5553 struct si_shader_binary *binary,
5554 struct ac_shader_config *conf,
5555 struct ac_llvm_compiler *compiler,
5556 LLVMModuleRef mod,
5557 struct pipe_debug_callback *debug,
5558 enum pipe_shader_type shader_type,
5559 unsigned wave_size,
5560 const char *name,
5561 bool less_optimized)
5562 {
5563 unsigned count = p_atomic_inc_return(&sscreen->num_compilations);
5564
5565 if (si_can_dump_shader(sscreen, shader_type)) {
5566 fprintf(stderr, "radeonsi: Compiling shader %d\n", count);
5567
5568 if (!(sscreen->debug_flags & (DBG(NO_IR) | DBG(PREOPT_IR)))) {
5569 fprintf(stderr, "%s LLVM IR:\n\n", name);
5570 ac_dump_module(mod);
5571 fprintf(stderr, "\n");
5572 }
5573 }
5574
5575 if (sscreen->record_llvm_ir) {
5576 char *ir = LLVMPrintModuleToString(mod);
5577 binary->llvm_ir_string = strdup(ir);
5578 LLVMDisposeMessage(ir);
5579 }
5580
5581 if (!si_replace_shader(count, binary)) {
5582 unsigned r = si_llvm_compile(mod, binary, compiler, debug,
5583 less_optimized, wave_size);
5584 if (r)
5585 return r;
5586 }
5587
5588 struct ac_rtld_binary rtld;
5589 if (!ac_rtld_open(&rtld, (struct ac_rtld_open_info){
5590 .info = &sscreen->info,
5591 .shader_type = tgsi_processor_to_shader_stage(shader_type),
5592 .wave_size = wave_size,
5593 .num_parts = 1,
5594 .elf_ptrs = &binary->elf_buffer,
5595 .elf_sizes = &binary->elf_size }))
5596 return -1;
5597
5598 bool ok = ac_rtld_read_config(&rtld, conf);
5599 ac_rtld_close(&rtld);
5600 if (!ok)
5601 return -1;
5602
5603 /* Enable 64-bit and 16-bit denormals, because there is no performance
5604 * cost.
5605 *
5606 * If denormals are enabled, all floating-point output modifiers are
5607 * ignored.
5608 *
5609 * Don't enable denormals for 32-bit floats, because:
5610 * - Floating-point output modifiers would be ignored by the hw.
5611 * - Some opcodes don't support denormals, such as v_mad_f32. We would
5612 * have to stop using those.
5613 * - GFX6 & GFX7 would be very slow.
5614 */
5615 conf->float_mode |= V_00B028_FP_64_DENORMS;
5616
5617 return 0;
5618 }
5619
5620 static void si_llvm_build_ret(struct si_shader_context *ctx, LLVMValueRef ret)
5621 {
5622 if (LLVMGetTypeKind(LLVMTypeOf(ret)) == LLVMVoidTypeKind)
5623 LLVMBuildRetVoid(ctx->ac.builder);
5624 else
5625 LLVMBuildRet(ctx->ac.builder, ret);
5626 }
5627
5628 /* Generate code for the hardware VS shader stage to go with a geometry shader */
5629 struct si_shader *
5630 si_generate_gs_copy_shader(struct si_screen *sscreen,
5631 struct ac_llvm_compiler *compiler,
5632 struct si_shader_selector *gs_selector,
5633 struct pipe_debug_callback *debug)
5634 {
5635 struct si_shader_context ctx;
5636 struct si_shader *shader;
5637 LLVMBuilderRef builder;
5638 struct si_shader_output_values outputs[SI_MAX_VS_OUTPUTS];
5639 struct tgsi_shader_info *gsinfo = &gs_selector->info;
5640 int i;
5641
5642
5643 shader = CALLOC_STRUCT(si_shader);
5644 if (!shader)
5645 return NULL;
5646
5647 /* We can leave the fence as permanently signaled because the GS copy
5648 * shader only becomes visible globally after it has been compiled. */
5649 util_queue_fence_init(&shader->ready);
5650
5651 shader->selector = gs_selector;
5652 shader->is_gs_copy_shader = true;
5653
5654 si_init_shader_ctx(&ctx, sscreen, compiler,
5655 si_get_wave_size(sscreen, PIPE_SHADER_VERTEX, false, false),
5656 false);
5657 ctx.shader = shader;
5658 ctx.type = PIPE_SHADER_VERTEX;
5659
5660 builder = ctx.ac.builder;
5661
5662 create_function(&ctx);
5663 preload_ring_buffers(&ctx);
5664
5665 LLVMValueRef voffset =
5666 LLVMBuildMul(ctx.ac.builder, ctx.abi.vertex_id,
5667 LLVMConstInt(ctx.i32, 4, 0), "");
5668
5669 /* Fetch the vertex stream ID.*/
5670 LLVMValueRef stream_id;
5671
5672 if (!sscreen->use_ngg_streamout && gs_selector->so.num_outputs)
5673 stream_id = si_unpack_param(&ctx, ctx.streamout_config, 24, 2);
5674 else
5675 stream_id = ctx.i32_0;
5676
5677 /* Fill in output information. */
5678 for (i = 0; i < gsinfo->num_outputs; ++i) {
5679 outputs[i].semantic_name = gsinfo->output_semantic_name[i];
5680 outputs[i].semantic_index = gsinfo->output_semantic_index[i];
5681
5682 for (int chan = 0; chan < 4; chan++) {
5683 outputs[i].vertex_stream[chan] =
5684 (gsinfo->output_streams[i] >> (2 * chan)) & 3;
5685 }
5686 }
5687
5688 LLVMBasicBlockRef end_bb;
5689 LLVMValueRef switch_inst;
5690
5691 end_bb = LLVMAppendBasicBlockInContext(ctx.ac.context, ctx.main_fn, "end");
5692 switch_inst = LLVMBuildSwitch(builder, stream_id, end_bb, 4);
5693
5694 for (int stream = 0; stream < 4; stream++) {
5695 LLVMBasicBlockRef bb;
5696 unsigned offset;
5697
5698 if (!gsinfo->num_stream_output_components[stream])
5699 continue;
5700
5701 if (stream > 0 && !gs_selector->so.num_outputs)
5702 continue;
5703
5704 bb = LLVMInsertBasicBlockInContext(ctx.ac.context, end_bb, "out");
5705 LLVMAddCase(switch_inst, LLVMConstInt(ctx.i32, stream, 0), bb);
5706 LLVMPositionBuilderAtEnd(builder, bb);
5707
5708 /* Fetch vertex data from GSVS ring */
5709 offset = 0;
5710 for (i = 0; i < gsinfo->num_outputs; ++i) {
5711 for (unsigned chan = 0; chan < 4; chan++) {
5712 if (!(gsinfo->output_usagemask[i] & (1 << chan)) ||
5713 outputs[i].vertex_stream[chan] != stream) {
5714 outputs[i].values[chan] = LLVMGetUndef(ctx.f32);
5715 continue;
5716 }
5717
5718 LLVMValueRef soffset = LLVMConstInt(ctx.i32,
5719 offset * gs_selector->gs_max_out_vertices * 16 * 4, 0);
5720 offset++;
5721
5722 outputs[i].values[chan] =
5723 ac_build_buffer_load(&ctx.ac,
5724 ctx.gsvs_ring[0], 1,
5725 ctx.i32_0, voffset,
5726 soffset, 0, ac_glc | ac_slc,
5727 true, false);
5728 }
5729 }
5730
5731 /* Streamout and exports. */
5732 if (!sscreen->use_ngg_streamout && gs_selector->so.num_outputs) {
5733 si_llvm_emit_streamout(&ctx, outputs,
5734 gsinfo->num_outputs,
5735 stream);
5736 }
5737
5738 if (stream == 0)
5739 si_llvm_export_vs(&ctx, outputs, gsinfo->num_outputs);
5740
5741 LLVMBuildBr(builder, end_bb);
5742 }
5743
5744 LLVMPositionBuilderAtEnd(builder, end_bb);
5745
5746 LLVMBuildRetVoid(ctx.ac.builder);
5747
5748 ctx.type = PIPE_SHADER_GEOMETRY; /* override for shader dumping */
5749 si_llvm_optimize_module(&ctx);
5750
5751 bool ok = false;
5752 if (si_compile_llvm(sscreen, &ctx.shader->binary,
5753 &ctx.shader->config, ctx.compiler,
5754 ctx.ac.module,
5755 debug, PIPE_SHADER_GEOMETRY, ctx.ac.wave_size,
5756 "GS Copy Shader", false) == 0) {
5757 if (si_can_dump_shader(sscreen, PIPE_SHADER_GEOMETRY))
5758 fprintf(stderr, "GS Copy Shader:\n");
5759 si_shader_dump(sscreen, ctx.shader, debug, stderr, true);
5760
5761 if (!ctx.shader->config.scratch_bytes_per_wave)
5762 ok = si_shader_binary_upload(sscreen, ctx.shader, 0);
5763 else
5764 ok = true;
5765 }
5766
5767 si_llvm_dispose(&ctx);
5768
5769 if (!ok) {
5770 FREE(shader);
5771 shader = NULL;
5772 } else {
5773 si_fix_resource_usage(sscreen, shader);
5774 }
5775 return shader;
5776 }
5777
5778 static void si_dump_shader_key_vs(const struct si_shader_key *key,
5779 const struct si_vs_prolog_bits *prolog,
5780 const char *prefix, FILE *f)
5781 {
5782 fprintf(f, " %s.instance_divisor_is_one = %u\n",
5783 prefix, prolog->instance_divisor_is_one);
5784 fprintf(f, " %s.instance_divisor_is_fetched = %u\n",
5785 prefix, prolog->instance_divisor_is_fetched);
5786 fprintf(f, " %s.unpack_instance_id_from_vertex_id = %u\n",
5787 prefix, prolog->unpack_instance_id_from_vertex_id);
5788 fprintf(f, " %s.ls_vgpr_fix = %u\n",
5789 prefix, prolog->ls_vgpr_fix);
5790
5791 fprintf(f, " mono.vs.fetch_opencode = %x\n", key->mono.vs_fetch_opencode);
5792 fprintf(f, " mono.vs.fix_fetch = {");
5793 for (int i = 0; i < SI_MAX_ATTRIBS; i++) {
5794 union si_vs_fix_fetch fix = key->mono.vs_fix_fetch[i];
5795 if (i)
5796 fprintf(f, ", ");
5797 if (!fix.bits)
5798 fprintf(f, "0");
5799 else
5800 fprintf(f, "%u.%u.%u.%u", fix.u.reverse, fix.u.log_size,
5801 fix.u.num_channels_m1, fix.u.format);
5802 }
5803 fprintf(f, "}\n");
5804 }
5805
5806 static void si_dump_shader_key(const struct si_shader *shader, FILE *f)
5807 {
5808 const struct si_shader_key *key = &shader->key;
5809 enum pipe_shader_type shader_type = shader->selector->type;
5810
5811 fprintf(f, "SHADER KEY\n");
5812
5813 switch (shader_type) {
5814 case PIPE_SHADER_VERTEX:
5815 si_dump_shader_key_vs(key, &key->part.vs.prolog,
5816 "part.vs.prolog", f);
5817 fprintf(f, " as_es = %u\n", key->as_es);
5818 fprintf(f, " as_ls = %u\n", key->as_ls);
5819 fprintf(f, " as_ngg = %u\n", key->as_ngg);
5820 fprintf(f, " mono.u.vs_export_prim_id = %u\n",
5821 key->mono.u.vs_export_prim_id);
5822 fprintf(f, " opt.vs_as_prim_discard_cs = %u\n",
5823 key->opt.vs_as_prim_discard_cs);
5824 fprintf(f, " opt.cs_prim_type = %s\n",
5825 tgsi_primitive_names[key->opt.cs_prim_type]);
5826 fprintf(f, " opt.cs_indexed = %u\n",
5827 key->opt.cs_indexed);
5828 fprintf(f, " opt.cs_instancing = %u\n",
5829 key->opt.cs_instancing);
5830 fprintf(f, " opt.cs_primitive_restart = %u\n",
5831 key->opt.cs_primitive_restart);
5832 fprintf(f, " opt.cs_provoking_vertex_first = %u\n",
5833 key->opt.cs_provoking_vertex_first);
5834 fprintf(f, " opt.cs_need_correct_orientation = %u\n",
5835 key->opt.cs_need_correct_orientation);
5836 fprintf(f, " opt.cs_cull_front = %u\n",
5837 key->opt.cs_cull_front);
5838 fprintf(f, " opt.cs_cull_back = %u\n",
5839 key->opt.cs_cull_back);
5840 fprintf(f, " opt.cs_cull_z = %u\n",
5841 key->opt.cs_cull_z);
5842 fprintf(f, " opt.cs_halfz_clip_space = %u\n",
5843 key->opt.cs_halfz_clip_space);
5844 break;
5845
5846 case PIPE_SHADER_TESS_CTRL:
5847 if (shader->selector->screen->info.chip_class >= GFX9) {
5848 si_dump_shader_key_vs(key, &key->part.tcs.ls_prolog,
5849 "part.tcs.ls_prolog", f);
5850 }
5851 fprintf(f, " part.tcs.epilog.prim_mode = %u\n", key->part.tcs.epilog.prim_mode);
5852 fprintf(f, " mono.u.ff_tcs_inputs_to_copy = 0x%"PRIx64"\n", key->mono.u.ff_tcs_inputs_to_copy);
5853 break;
5854
5855 case PIPE_SHADER_TESS_EVAL:
5856 fprintf(f, " as_es = %u\n", key->as_es);
5857 fprintf(f, " as_ngg = %u\n", key->as_ngg);
5858 fprintf(f, " mono.u.vs_export_prim_id = %u\n",
5859 key->mono.u.vs_export_prim_id);
5860 break;
5861
5862 case PIPE_SHADER_GEOMETRY:
5863 if (shader->is_gs_copy_shader)
5864 break;
5865
5866 if (shader->selector->screen->info.chip_class >= GFX9 &&
5867 key->part.gs.es->type == PIPE_SHADER_VERTEX) {
5868 si_dump_shader_key_vs(key, &key->part.gs.vs_prolog,
5869 "part.gs.vs_prolog", f);
5870 }
5871 fprintf(f, " part.gs.prolog.tri_strip_adj_fix = %u\n", key->part.gs.prolog.tri_strip_adj_fix);
5872 fprintf(f, " part.gs.prolog.gfx9_prev_is_vs = %u\n", key->part.gs.prolog.gfx9_prev_is_vs);
5873 fprintf(f, " as_ngg = %u\n", key->as_ngg);
5874 break;
5875
5876 case PIPE_SHADER_COMPUTE:
5877 break;
5878
5879 case PIPE_SHADER_FRAGMENT:
5880 fprintf(f, " part.ps.prolog.color_two_side = %u\n", key->part.ps.prolog.color_two_side);
5881 fprintf(f, " part.ps.prolog.flatshade_colors = %u\n", key->part.ps.prolog.flatshade_colors);
5882 fprintf(f, " part.ps.prolog.poly_stipple = %u\n", key->part.ps.prolog.poly_stipple);
5883 fprintf(f, " part.ps.prolog.force_persp_sample_interp = %u\n", key->part.ps.prolog.force_persp_sample_interp);
5884 fprintf(f, " part.ps.prolog.force_linear_sample_interp = %u\n", key->part.ps.prolog.force_linear_sample_interp);
5885 fprintf(f, " part.ps.prolog.force_persp_center_interp = %u\n", key->part.ps.prolog.force_persp_center_interp);
5886 fprintf(f, " part.ps.prolog.force_linear_center_interp = %u\n", key->part.ps.prolog.force_linear_center_interp);
5887 fprintf(f, " part.ps.prolog.bc_optimize_for_persp = %u\n", key->part.ps.prolog.bc_optimize_for_persp);
5888 fprintf(f, " part.ps.prolog.bc_optimize_for_linear = %u\n", key->part.ps.prolog.bc_optimize_for_linear);
5889 fprintf(f, " part.ps.prolog.samplemask_log_ps_iter = %u\n", key->part.ps.prolog.samplemask_log_ps_iter);
5890 fprintf(f, " part.ps.epilog.spi_shader_col_format = 0x%x\n", key->part.ps.epilog.spi_shader_col_format);
5891 fprintf(f, " part.ps.epilog.color_is_int8 = 0x%X\n", key->part.ps.epilog.color_is_int8);
5892 fprintf(f, " part.ps.epilog.color_is_int10 = 0x%X\n", key->part.ps.epilog.color_is_int10);
5893 fprintf(f, " part.ps.epilog.last_cbuf = %u\n", key->part.ps.epilog.last_cbuf);
5894 fprintf(f, " part.ps.epilog.alpha_func = %u\n", key->part.ps.epilog.alpha_func);
5895 fprintf(f, " part.ps.epilog.alpha_to_one = %u\n", key->part.ps.epilog.alpha_to_one);
5896 fprintf(f, " part.ps.epilog.poly_line_smoothing = %u\n", key->part.ps.epilog.poly_line_smoothing);
5897 fprintf(f, " part.ps.epilog.clamp_color = %u\n", key->part.ps.epilog.clamp_color);
5898 fprintf(f, " mono.u.ps.interpolate_at_sample_force_center = %u\n", key->mono.u.ps.interpolate_at_sample_force_center);
5899 fprintf(f, " mono.u.ps.fbfetch_msaa = %u\n", key->mono.u.ps.fbfetch_msaa);
5900 fprintf(f, " mono.u.ps.fbfetch_is_1D = %u\n", key->mono.u.ps.fbfetch_is_1D);
5901 fprintf(f, " mono.u.ps.fbfetch_layered = %u\n", key->mono.u.ps.fbfetch_layered);
5902 break;
5903
5904 default:
5905 assert(0);
5906 }
5907
5908 if ((shader_type == PIPE_SHADER_GEOMETRY ||
5909 shader_type == PIPE_SHADER_TESS_EVAL ||
5910 shader_type == PIPE_SHADER_VERTEX) &&
5911 !key->as_es && !key->as_ls) {
5912 fprintf(f, " opt.kill_outputs = 0x%"PRIx64"\n", key->opt.kill_outputs);
5913 fprintf(f, " opt.clip_disable = %u\n", key->opt.clip_disable);
5914 }
5915 }
5916
5917 static void si_init_shader_ctx(struct si_shader_context *ctx,
5918 struct si_screen *sscreen,
5919 struct ac_llvm_compiler *compiler,
5920 unsigned wave_size,
5921 bool nir)
5922 {
5923 struct lp_build_tgsi_context *bld_base;
5924
5925 si_llvm_context_init(ctx, sscreen, compiler, wave_size,
5926 nir ? 64 : wave_size);
5927
5928 bld_base = &ctx->bld_base;
5929 bld_base->emit_fetch_funcs[TGSI_FILE_CONSTANT] = fetch_constant;
5930
5931 bld_base->op_actions[TGSI_OPCODE_INTERP_CENTROID].emit = build_interp_intrinsic;
5932 bld_base->op_actions[TGSI_OPCODE_INTERP_SAMPLE].emit = build_interp_intrinsic;
5933 bld_base->op_actions[TGSI_OPCODE_INTERP_OFFSET].emit = build_interp_intrinsic;
5934
5935 bld_base->op_actions[TGSI_OPCODE_MEMBAR].emit = membar_emit;
5936
5937 bld_base->op_actions[TGSI_OPCODE_CLOCK].emit = clock_emit;
5938
5939 bld_base->op_actions[TGSI_OPCODE_DDX].emit = si_llvm_emit_ddxy;
5940 bld_base->op_actions[TGSI_OPCODE_DDY].emit = si_llvm_emit_ddxy;
5941 bld_base->op_actions[TGSI_OPCODE_DDX_FINE].emit = si_llvm_emit_ddxy;
5942 bld_base->op_actions[TGSI_OPCODE_DDY_FINE].emit = si_llvm_emit_ddxy;
5943
5944 bld_base->op_actions[TGSI_OPCODE_VOTE_ALL].emit = vote_all_emit;
5945 bld_base->op_actions[TGSI_OPCODE_VOTE_ANY].emit = vote_any_emit;
5946 bld_base->op_actions[TGSI_OPCODE_VOTE_EQ].emit = vote_eq_emit;
5947 bld_base->op_actions[TGSI_OPCODE_BALLOT].emit = ballot_emit;
5948 bld_base->op_actions[TGSI_OPCODE_READ_FIRST].intr_name = "llvm.amdgcn.readfirstlane";
5949 bld_base->op_actions[TGSI_OPCODE_READ_FIRST].emit = read_lane_emit;
5950 bld_base->op_actions[TGSI_OPCODE_READ_INVOC].intr_name = "llvm.amdgcn.readlane";
5951 bld_base->op_actions[TGSI_OPCODE_READ_INVOC].emit = read_lane_emit;
5952
5953 bld_base->op_actions[TGSI_OPCODE_EMIT].emit = si_tgsi_emit_vertex;
5954 bld_base->op_actions[TGSI_OPCODE_ENDPRIM].emit = si_tgsi_emit_primitive;
5955 bld_base->op_actions[TGSI_OPCODE_BARRIER].emit = si_llvm_emit_barrier;
5956 }
5957
5958 static void si_optimize_vs_outputs(struct si_shader_context *ctx)
5959 {
5960 struct si_shader *shader = ctx->shader;
5961 struct tgsi_shader_info *info = &shader->selector->info;
5962
5963 if ((ctx->type != PIPE_SHADER_VERTEX &&
5964 ctx->type != PIPE_SHADER_TESS_EVAL) ||
5965 shader->key.as_ls ||
5966 shader->key.as_es)
5967 return;
5968
5969 ac_optimize_vs_outputs(&ctx->ac,
5970 ctx->main_fn,
5971 shader->info.vs_output_param_offset,
5972 info->num_outputs,
5973 &shader->info.nr_param_exports);
5974 }
5975
5976 static void si_init_exec_from_input(struct si_shader_context *ctx,
5977 struct ac_arg param, unsigned bitoffset)
5978 {
5979 LLVMValueRef args[] = {
5980 ac_get_arg(&ctx->ac, param),
5981 LLVMConstInt(ctx->i32, bitoffset, 0),
5982 };
5983 ac_build_intrinsic(&ctx->ac,
5984 "llvm.amdgcn.init.exec.from.input",
5985 ctx->voidt, args, 2, AC_FUNC_ATTR_CONVERGENT);
5986 }
5987
5988 static bool si_vs_needs_prolog(const struct si_shader_selector *sel,
5989 const struct si_vs_prolog_bits *key)
5990 {
5991 /* VGPR initialization fixup for Vega10 and Raven is always done in the
5992 * VS prolog. */
5993 return sel->vs_needs_prolog || key->ls_vgpr_fix;
5994 }
5995
5996 static bool si_compile_tgsi_main(struct si_shader_context *ctx,
5997 struct nir_shader *nir, bool free_nir)
5998 {
5999 struct si_shader *shader = ctx->shader;
6000 struct si_shader_selector *sel = shader->selector;
6001 struct lp_build_tgsi_context *bld_base = &ctx->bld_base;
6002
6003 // TODO clean all this up!
6004 switch (ctx->type) {
6005 case PIPE_SHADER_VERTEX:
6006 ctx->load_input = declare_input_vs;
6007 if (shader->key.as_ls)
6008 ctx->abi.emit_outputs = si_llvm_emit_ls_epilogue;
6009 else if (shader->key.as_es)
6010 ctx->abi.emit_outputs = si_llvm_emit_es_epilogue;
6011 else if (shader->key.opt.vs_as_prim_discard_cs)
6012 ctx->abi.emit_outputs = si_llvm_emit_prim_discard_cs_epilogue;
6013 else if (shader->key.as_ngg)
6014 ctx->abi.emit_outputs = gfx10_emit_ngg_epilogue;
6015 else
6016 ctx->abi.emit_outputs = si_llvm_emit_vs_epilogue;
6017 bld_base->emit_epilogue = si_tgsi_emit_epilogue;
6018 ctx->abi.load_base_vertex = get_base_vertex;
6019 break;
6020 case PIPE_SHADER_TESS_CTRL:
6021 bld_base->emit_fetch_funcs[TGSI_FILE_INPUT] = fetch_input_tcs;
6022 ctx->abi.load_tess_varyings = si_nir_load_tcs_varyings;
6023 ctx->abi.load_tess_level = si_load_tess_level;
6024 bld_base->emit_fetch_funcs[TGSI_FILE_OUTPUT] = fetch_output_tcs;
6025 bld_base->emit_store = store_output_tcs;
6026 ctx->abi.store_tcs_outputs = si_nir_store_output_tcs;
6027 ctx->abi.emit_outputs = si_llvm_emit_tcs_epilogue;
6028 ctx->abi.load_patch_vertices_in = si_load_patch_vertices_in;
6029 bld_base->emit_epilogue = si_tgsi_emit_epilogue;
6030 break;
6031 case PIPE_SHADER_TESS_EVAL:
6032 bld_base->emit_fetch_funcs[TGSI_FILE_INPUT] = fetch_input_tes;
6033 ctx->abi.load_tess_varyings = si_nir_load_input_tes;
6034 ctx->abi.load_tess_coord = si_load_tess_coord;
6035 ctx->abi.load_tess_level = si_load_tess_level;
6036 ctx->abi.load_patch_vertices_in = si_load_patch_vertices_in;
6037 if (shader->key.as_es)
6038 ctx->abi.emit_outputs = si_llvm_emit_es_epilogue;
6039 else if (shader->key.as_ngg)
6040 ctx->abi.emit_outputs = gfx10_emit_ngg_epilogue;
6041 else
6042 ctx->abi.emit_outputs = si_llvm_emit_vs_epilogue;
6043 bld_base->emit_epilogue = si_tgsi_emit_epilogue;
6044 break;
6045 case PIPE_SHADER_GEOMETRY:
6046 bld_base->emit_fetch_funcs[TGSI_FILE_INPUT] = fetch_input_gs;
6047 ctx->abi.load_inputs = si_nir_load_input_gs;
6048 ctx->abi.emit_vertex = si_llvm_emit_vertex;
6049 ctx->abi.emit_primitive = si_llvm_emit_primitive;
6050 ctx->abi.emit_outputs = si_llvm_emit_gs_epilogue;
6051 bld_base->emit_epilogue = si_tgsi_emit_gs_epilogue;
6052 break;
6053 case PIPE_SHADER_FRAGMENT:
6054 ctx->load_input = declare_input_fs;
6055 ctx->abi.emit_outputs = si_llvm_return_fs_outputs;
6056 bld_base->emit_epilogue = si_tgsi_emit_epilogue;
6057 ctx->abi.load_sample_position = load_sample_position;
6058 ctx->abi.load_sample_mask_in = load_sample_mask_in;
6059 ctx->abi.emit_fbfetch = si_nir_emit_fbfetch;
6060 ctx->abi.emit_kill = si_llvm_emit_kill;
6061 break;
6062 case PIPE_SHADER_COMPUTE:
6063 ctx->abi.load_local_group_size = get_block_size;
6064 break;
6065 default:
6066 assert(!"Unsupported shader type");
6067 return false;
6068 }
6069
6070 ctx->abi.load_ubo = load_ubo;
6071 ctx->abi.load_ssbo = load_ssbo;
6072
6073 create_function(ctx);
6074 preload_ring_buffers(ctx);
6075
6076 if (ctx->type == PIPE_SHADER_TESS_CTRL &&
6077 sel->tcs_info.tessfactors_are_def_in_all_invocs) {
6078 for (unsigned i = 0; i < 6; i++) {
6079 ctx->invoc0_tess_factors[i] =
6080 ac_build_alloca_undef(&ctx->ac, ctx->i32, "");
6081 }
6082 }
6083
6084 if (ctx->type == PIPE_SHADER_GEOMETRY) {
6085 for (unsigned i = 0; i < 4; i++) {
6086 ctx->gs_next_vertex[i] =
6087 ac_build_alloca(&ctx->ac, ctx->i32, "");
6088 }
6089 if (shader->key.as_ngg) {
6090 for (unsigned i = 0; i < 4; ++i) {
6091 ctx->gs_curprim_verts[i] =
6092 ac_build_alloca(&ctx->ac, ctx->ac.i32, "");
6093 ctx->gs_generated_prims[i] =
6094 ac_build_alloca(&ctx->ac, ctx->ac.i32, "");
6095 }
6096
6097 unsigned scratch_size = 8;
6098 if (sel->so.num_outputs)
6099 scratch_size = 44;
6100
6101 LLVMTypeRef ai32 = LLVMArrayType(ctx->i32, scratch_size);
6102 ctx->gs_ngg_scratch = LLVMAddGlobalInAddressSpace(ctx->ac.module,
6103 ai32, "ngg_scratch", AC_ADDR_SPACE_LDS);
6104 LLVMSetInitializer(ctx->gs_ngg_scratch, LLVMGetUndef(ai32));
6105 LLVMSetAlignment(ctx->gs_ngg_scratch, 4);
6106
6107 ctx->gs_ngg_emit = LLVMAddGlobalInAddressSpace(ctx->ac.module,
6108 LLVMArrayType(ctx->i32, 0), "ngg_emit", AC_ADDR_SPACE_LDS);
6109 LLVMSetLinkage(ctx->gs_ngg_emit, LLVMExternalLinkage);
6110 LLVMSetAlignment(ctx->gs_ngg_emit, 4);
6111 }
6112 }
6113
6114 if (ctx->type != PIPE_SHADER_GEOMETRY &&
6115 (shader->key.as_ngg && !shader->key.as_es)) {
6116 /* Unconditionally declare scratch space base for streamout and
6117 * vertex compaction. Whether space is actually allocated is
6118 * determined during linking / PM4 creation.
6119 *
6120 * Add an extra dword per vertex to ensure an odd stride, which
6121 * avoids bank conflicts for SoA accesses.
6122 */
6123 declare_esgs_ring(ctx);
6124
6125 /* This is really only needed when streamout and / or vertex
6126 * compaction is enabled.
6127 */
6128 LLVMTypeRef asi32 = LLVMArrayType(ctx->i32, 8);
6129 ctx->gs_ngg_scratch = LLVMAddGlobalInAddressSpace(ctx->ac.module,
6130 asi32, "ngg_scratch", AC_ADDR_SPACE_LDS);
6131 LLVMSetInitializer(ctx->gs_ngg_scratch, LLVMGetUndef(asi32));
6132 LLVMSetAlignment(ctx->gs_ngg_scratch, 4);
6133 }
6134
6135 /* For GFX9 merged shaders:
6136 * - Set EXEC for the first shader. If the prolog is present, set
6137 * EXEC there instead.
6138 * - Add a barrier before the second shader.
6139 * - In the second shader, reset EXEC to ~0 and wrap the main part in
6140 * an if-statement. This is required for correctness in geometry
6141 * shaders, to ensure that empty GS waves do not send GS_EMIT and
6142 * GS_CUT messages.
6143 *
6144 * For monolithic merged shaders, the first shader is wrapped in an
6145 * if-block together with its prolog in si_build_wrapper_function.
6146 *
6147 * NGG vertex and tess eval shaders running as the last
6148 * vertex/geometry stage handle execution explicitly using
6149 * if-statements.
6150 */
6151 if (ctx->screen->info.chip_class >= GFX9) {
6152 if (!shader->is_monolithic &&
6153 sel->info.num_instructions > 1 && /* not empty shader */
6154 (shader->key.as_es || shader->key.as_ls) &&
6155 (ctx->type == PIPE_SHADER_TESS_EVAL ||
6156 (ctx->type == PIPE_SHADER_VERTEX &&
6157 !si_vs_needs_prolog(sel, &shader->key.part.vs.prolog)))) {
6158 si_init_exec_from_input(ctx,
6159 ctx->merged_wave_info, 0);
6160 } else if (ctx->type == PIPE_SHADER_TESS_CTRL ||
6161 ctx->type == PIPE_SHADER_GEOMETRY ||
6162 (shader->key.as_ngg && !shader->key.as_es)) {
6163 LLVMValueRef num_threads;
6164 bool nested_barrier;
6165
6166 if (!shader->is_monolithic ||
6167 (ctx->type == PIPE_SHADER_TESS_EVAL &&
6168 (shader->key.as_ngg && !shader->key.as_es)))
6169 ac_init_exec_full_mask(&ctx->ac);
6170
6171 if (ctx->type == PIPE_SHADER_TESS_CTRL ||
6172 ctx->type == PIPE_SHADER_GEOMETRY) {
6173 if (ctx->type == PIPE_SHADER_GEOMETRY && shader->key.as_ngg) {
6174 gfx10_ngg_gs_emit_prologue(ctx);
6175 nested_barrier = false;
6176 } else {
6177 nested_barrier = true;
6178 }
6179
6180 /* Number of patches / primitives */
6181 num_threads = si_unpack_param(ctx, ctx->merged_wave_info, 8, 8);
6182 } else {
6183 /* Number of vertices */
6184 num_threads = si_unpack_param(ctx, ctx->merged_wave_info, 0, 8);
6185 nested_barrier = false;
6186 }
6187
6188 LLVMValueRef ena =
6189 LLVMBuildICmp(ctx->ac.builder, LLVMIntULT,
6190 ac_get_thread_id(&ctx->ac), num_threads, "");
6191
6192 ctx->merged_wrap_if_entry_block = LLVMGetInsertBlock(ctx->ac.builder);
6193 ctx->merged_wrap_if_label = 11500;
6194 ac_build_ifcc(&ctx->ac, ena, ctx->merged_wrap_if_label);
6195
6196 if (nested_barrier) {
6197 /* Execute a barrier before the second shader in
6198 * a merged shader.
6199 *
6200 * Execute the barrier inside the conditional block,
6201 * so that empty waves can jump directly to s_endpgm,
6202 * which will also signal the barrier.
6203 *
6204 * This is possible in gfx9, because an empty wave
6205 * for the second shader does not participate in
6206 * the epilogue. With NGG, empty waves may still
6207 * be required to export data (e.g. GS output vertices),
6208 * so we cannot let them exit early.
6209 *
6210 * If the shader is TCS and the TCS epilog is present
6211 * and contains a barrier, it will wait there and then
6212 * reach s_endpgm.
6213 */
6214 si_llvm_emit_barrier(NULL, bld_base, NULL);
6215 }
6216 }
6217 }
6218
6219 if (sel->force_correct_derivs_after_kill) {
6220 ctx->postponed_kill = ac_build_alloca_undef(&ctx->ac, ctx->i1, "");
6221 /* true = don't kill. */
6222 LLVMBuildStore(ctx->ac.builder, ctx->i1true,
6223 ctx->postponed_kill);
6224 }
6225
6226 if (sel->tokens) {
6227 if (!lp_build_tgsi_llvm(bld_base, sel->tokens)) {
6228 fprintf(stderr, "Failed to translate shader from TGSI to LLVM\n");
6229 return false;
6230 }
6231 } else {
6232 bool success = si_nir_build_llvm(ctx, nir);
6233 if (free_nir)
6234 ralloc_free(nir);
6235 if (!success) {
6236 fprintf(stderr, "Failed to translate shader from NIR to LLVM\n");
6237 return false;
6238 }
6239 }
6240
6241 si_llvm_build_ret(ctx, ctx->return_value);
6242 return true;
6243 }
6244
6245 /**
6246 * Compute the VS prolog key, which contains all the information needed to
6247 * build the VS prolog function, and set shader->info bits where needed.
6248 *
6249 * \param info Shader info of the vertex shader.
6250 * \param num_input_sgprs Number of input SGPRs for the vertex shader.
6251 * \param prolog_key Key of the VS prolog
6252 * \param shader_out The vertex shader, or the next shader if merging LS+HS or ES+GS.
6253 * \param key Output shader part key.
6254 */
6255 static void si_get_vs_prolog_key(const struct tgsi_shader_info *info,
6256 unsigned num_input_sgprs,
6257 const struct si_vs_prolog_bits *prolog_key,
6258 struct si_shader *shader_out,
6259 union si_shader_part_key *key)
6260 {
6261 memset(key, 0, sizeof(*key));
6262 key->vs_prolog.states = *prolog_key;
6263 key->vs_prolog.num_input_sgprs = num_input_sgprs;
6264 key->vs_prolog.last_input = MAX2(1, info->num_inputs) - 1;
6265 key->vs_prolog.as_ls = shader_out->key.as_ls;
6266 key->vs_prolog.as_es = shader_out->key.as_es;
6267 key->vs_prolog.as_ngg = shader_out->key.as_ngg;
6268
6269 if (shader_out->selector->type == PIPE_SHADER_TESS_CTRL) {
6270 key->vs_prolog.as_ls = 1;
6271 key->vs_prolog.num_merged_next_stage_vgprs = 2;
6272 } else if (shader_out->selector->type == PIPE_SHADER_GEOMETRY) {
6273 key->vs_prolog.as_es = 1;
6274 key->vs_prolog.num_merged_next_stage_vgprs = 5;
6275 } else if (shader_out->key.as_ngg) {
6276 key->vs_prolog.num_merged_next_stage_vgprs = 5;
6277 }
6278
6279 /* Enable loading the InstanceID VGPR. */
6280 uint16_t input_mask = u_bit_consecutive(0, info->num_inputs);
6281
6282 if ((key->vs_prolog.states.instance_divisor_is_one |
6283 key->vs_prolog.states.instance_divisor_is_fetched) & input_mask)
6284 shader_out->info.uses_instanceid = true;
6285 }
6286
6287 /**
6288 * Compute the PS prolog key, which contains all the information needed to
6289 * build the PS prolog function, and set related bits in shader->config.
6290 */
6291 static void si_get_ps_prolog_key(struct si_shader *shader,
6292 union si_shader_part_key *key,
6293 bool separate_prolog)
6294 {
6295 struct tgsi_shader_info *info = &shader->selector->info;
6296
6297 memset(key, 0, sizeof(*key));
6298 key->ps_prolog.states = shader->key.part.ps.prolog;
6299 key->ps_prolog.colors_read = info->colors_read;
6300 key->ps_prolog.num_input_sgprs = shader->info.num_input_sgprs;
6301 key->ps_prolog.num_input_vgprs = shader->info.num_input_vgprs;
6302 key->ps_prolog.wqm = info->uses_derivatives &&
6303 (key->ps_prolog.colors_read ||
6304 key->ps_prolog.states.force_persp_sample_interp ||
6305 key->ps_prolog.states.force_linear_sample_interp ||
6306 key->ps_prolog.states.force_persp_center_interp ||
6307 key->ps_prolog.states.force_linear_center_interp ||
6308 key->ps_prolog.states.bc_optimize_for_persp ||
6309 key->ps_prolog.states.bc_optimize_for_linear);
6310 key->ps_prolog.ancillary_vgpr_index = shader->info.ancillary_vgpr_index;
6311
6312 if (info->colors_read) {
6313 unsigned *color = shader->selector->color_attr_index;
6314
6315 if (shader->key.part.ps.prolog.color_two_side) {
6316 /* BCOLORs are stored after the last input. */
6317 key->ps_prolog.num_interp_inputs = info->num_inputs;
6318 key->ps_prolog.face_vgpr_index = shader->info.face_vgpr_index;
6319 if (separate_prolog)
6320 shader->config.spi_ps_input_ena |= S_0286CC_FRONT_FACE_ENA(1);
6321 }
6322
6323 for (unsigned i = 0; i < 2; i++) {
6324 unsigned interp = info->input_interpolate[color[i]];
6325 unsigned location = info->input_interpolate_loc[color[i]];
6326
6327 if (!(info->colors_read & (0xf << i*4)))
6328 continue;
6329
6330 key->ps_prolog.color_attr_index[i] = color[i];
6331
6332 if (shader->key.part.ps.prolog.flatshade_colors &&
6333 interp == TGSI_INTERPOLATE_COLOR)
6334 interp = TGSI_INTERPOLATE_CONSTANT;
6335
6336 switch (interp) {
6337 case TGSI_INTERPOLATE_CONSTANT:
6338 key->ps_prolog.color_interp_vgpr_index[i] = -1;
6339 break;
6340 case TGSI_INTERPOLATE_PERSPECTIVE:
6341 case TGSI_INTERPOLATE_COLOR:
6342 /* Force the interpolation location for colors here. */
6343 if (shader->key.part.ps.prolog.force_persp_sample_interp)
6344 location = TGSI_INTERPOLATE_LOC_SAMPLE;
6345 if (shader->key.part.ps.prolog.force_persp_center_interp)
6346 location = TGSI_INTERPOLATE_LOC_CENTER;
6347
6348 switch (location) {
6349 case TGSI_INTERPOLATE_LOC_SAMPLE:
6350 key->ps_prolog.color_interp_vgpr_index[i] = 0;
6351 if (separate_prolog) {
6352 shader->config.spi_ps_input_ena |=
6353 S_0286CC_PERSP_SAMPLE_ENA(1);
6354 }
6355 break;
6356 case TGSI_INTERPOLATE_LOC_CENTER:
6357 key->ps_prolog.color_interp_vgpr_index[i] = 2;
6358 if (separate_prolog) {
6359 shader->config.spi_ps_input_ena |=
6360 S_0286CC_PERSP_CENTER_ENA(1);
6361 }
6362 break;
6363 case TGSI_INTERPOLATE_LOC_CENTROID:
6364 key->ps_prolog.color_interp_vgpr_index[i] = 4;
6365 if (separate_prolog) {
6366 shader->config.spi_ps_input_ena |=
6367 S_0286CC_PERSP_CENTROID_ENA(1);
6368 }
6369 break;
6370 default:
6371 assert(0);
6372 }
6373 break;
6374 case TGSI_INTERPOLATE_LINEAR:
6375 /* Force the interpolation location for colors here. */
6376 if (shader->key.part.ps.prolog.force_linear_sample_interp)
6377 location = TGSI_INTERPOLATE_LOC_SAMPLE;
6378 if (shader->key.part.ps.prolog.force_linear_center_interp)
6379 location = TGSI_INTERPOLATE_LOC_CENTER;
6380
6381 /* The VGPR assignment for non-monolithic shaders
6382 * works because InitialPSInputAddr is set on the
6383 * main shader and PERSP_PULL_MODEL is never used.
6384 */
6385 switch (location) {
6386 case TGSI_INTERPOLATE_LOC_SAMPLE:
6387 key->ps_prolog.color_interp_vgpr_index[i] =
6388 separate_prolog ? 6 : 9;
6389 if (separate_prolog) {
6390 shader->config.spi_ps_input_ena |=
6391 S_0286CC_LINEAR_SAMPLE_ENA(1);
6392 }
6393 break;
6394 case TGSI_INTERPOLATE_LOC_CENTER:
6395 key->ps_prolog.color_interp_vgpr_index[i] =
6396 separate_prolog ? 8 : 11;
6397 if (separate_prolog) {
6398 shader->config.spi_ps_input_ena |=
6399 S_0286CC_LINEAR_CENTER_ENA(1);
6400 }
6401 break;
6402 case TGSI_INTERPOLATE_LOC_CENTROID:
6403 key->ps_prolog.color_interp_vgpr_index[i] =
6404 separate_prolog ? 10 : 13;
6405 if (separate_prolog) {
6406 shader->config.spi_ps_input_ena |=
6407 S_0286CC_LINEAR_CENTROID_ENA(1);
6408 }
6409 break;
6410 default:
6411 assert(0);
6412 }
6413 break;
6414 default:
6415 assert(0);
6416 }
6417 }
6418 }
6419 }
6420
6421 /**
6422 * Check whether a PS prolog is required based on the key.
6423 */
6424 static bool si_need_ps_prolog(const union si_shader_part_key *key)
6425 {
6426 return key->ps_prolog.colors_read ||
6427 key->ps_prolog.states.force_persp_sample_interp ||
6428 key->ps_prolog.states.force_linear_sample_interp ||
6429 key->ps_prolog.states.force_persp_center_interp ||
6430 key->ps_prolog.states.force_linear_center_interp ||
6431 key->ps_prolog.states.bc_optimize_for_persp ||
6432 key->ps_prolog.states.bc_optimize_for_linear ||
6433 key->ps_prolog.states.poly_stipple ||
6434 key->ps_prolog.states.samplemask_log_ps_iter;
6435 }
6436
6437 /**
6438 * Compute the PS epilog key, which contains all the information needed to
6439 * build the PS epilog function.
6440 */
6441 static void si_get_ps_epilog_key(struct si_shader *shader,
6442 union si_shader_part_key *key)
6443 {
6444 struct tgsi_shader_info *info = &shader->selector->info;
6445 memset(key, 0, sizeof(*key));
6446 key->ps_epilog.colors_written = info->colors_written;
6447 key->ps_epilog.writes_z = info->writes_z;
6448 key->ps_epilog.writes_stencil = info->writes_stencil;
6449 key->ps_epilog.writes_samplemask = info->writes_samplemask;
6450 key->ps_epilog.states = shader->key.part.ps.epilog;
6451 }
6452
6453 /**
6454 * Build the GS prolog function. Rotate the input vertices for triangle strips
6455 * with adjacency.
6456 */
6457 static void si_build_gs_prolog_function(struct si_shader_context *ctx,
6458 union si_shader_part_key *key)
6459 {
6460 unsigned num_sgprs, num_vgprs;
6461 LLVMBuilderRef builder = ctx->ac.builder;
6462 LLVMTypeRef returns[48];
6463 LLVMValueRef func, ret;
6464
6465 memset(&ctx->args, 0, sizeof(ctx->args));
6466
6467 if (ctx->screen->info.chip_class >= GFX9) {
6468 if (key->gs_prolog.states.gfx9_prev_is_vs)
6469 num_sgprs = 8 + GFX9_VSGS_NUM_USER_SGPR;
6470 else
6471 num_sgprs = 8 + GFX9_TESGS_NUM_USER_SGPR;
6472 num_vgprs = 5; /* ES inputs are not needed by GS */
6473 } else {
6474 num_sgprs = GFX6_GS_NUM_USER_SGPR + 2;
6475 num_vgprs = 8;
6476 }
6477
6478 for (unsigned i = 0; i < num_sgprs; ++i) {
6479 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, NULL);
6480 returns[i] = ctx->i32;
6481 }
6482
6483 for (unsigned i = 0; i < num_vgprs; ++i) {
6484 ac_add_arg(&ctx->args, AC_ARG_VGPR, 1, AC_ARG_INT, NULL);
6485 returns[num_sgprs + i] = ctx->f32;
6486 }
6487
6488 /* Create the function. */
6489 si_create_function(ctx, "gs_prolog", returns, num_sgprs + num_vgprs,
6490 0);
6491 func = ctx->main_fn;
6492
6493 /* Set the full EXEC mask for the prolog, because we are only fiddling
6494 * with registers here. The main shader part will set the correct EXEC
6495 * mask.
6496 */
6497 if (ctx->screen->info.chip_class >= GFX9 && !key->gs_prolog.is_monolithic)
6498 ac_init_exec_full_mask(&ctx->ac);
6499
6500 /* Copy inputs to outputs. This should be no-op, as the registers match,
6501 * but it will prevent the compiler from overwriting them unintentionally.
6502 */
6503 ret = ctx->return_value;
6504 for (unsigned i = 0; i < num_sgprs; i++) {
6505 LLVMValueRef p = LLVMGetParam(func, i);
6506 ret = LLVMBuildInsertValue(builder, ret, p, i, "");
6507 }
6508 for (unsigned i = 0; i < num_vgprs; i++) {
6509 LLVMValueRef p = LLVMGetParam(func, num_sgprs + i);
6510 p = ac_to_float(&ctx->ac, p);
6511 ret = LLVMBuildInsertValue(builder, ret, p, num_sgprs + i, "");
6512 }
6513
6514 if (key->gs_prolog.states.tri_strip_adj_fix) {
6515 /* Remap the input vertices for every other primitive. */
6516 const struct ac_arg gfx6_vtx_params[6] = {
6517 { .used = true, .arg_index = num_sgprs },
6518 { .used = true, .arg_index = num_sgprs + 1 },
6519 { .used = true, .arg_index = num_sgprs + 3 },
6520 { .used = true, .arg_index = num_sgprs + 4 },
6521 { .used = true, .arg_index = num_sgprs + 5 },
6522 { .used = true, .arg_index = num_sgprs + 6 },
6523 };
6524 const struct ac_arg gfx9_vtx_params[3] = {
6525 { .used = true, .arg_index = num_sgprs },
6526 { .used = true, .arg_index = num_sgprs + 1 },
6527 { .used = true, .arg_index = num_sgprs + 4 },
6528 };
6529 LLVMValueRef vtx_in[6], vtx_out[6];
6530 LLVMValueRef prim_id, rotate;
6531
6532 if (ctx->screen->info.chip_class >= GFX9) {
6533 for (unsigned i = 0; i < 3; i++) {
6534 vtx_in[i*2] = si_unpack_param(ctx, gfx9_vtx_params[i], 0, 16);
6535 vtx_in[i*2+1] = si_unpack_param(ctx, gfx9_vtx_params[i], 16, 16);
6536 }
6537 } else {
6538 for (unsigned i = 0; i < 6; i++)
6539 vtx_in[i] = ac_get_arg(&ctx->ac, gfx6_vtx_params[i]);
6540 }
6541
6542 prim_id = LLVMGetParam(func, num_sgprs + 2);
6543 rotate = LLVMBuildTrunc(builder, prim_id, ctx->i1, "");
6544
6545 for (unsigned i = 0; i < 6; ++i) {
6546 LLVMValueRef base, rotated;
6547 base = vtx_in[i];
6548 rotated = vtx_in[(i + 4) % 6];
6549 vtx_out[i] = LLVMBuildSelect(builder, rotate, rotated, base, "");
6550 }
6551
6552 if (ctx->screen->info.chip_class >= GFX9) {
6553 for (unsigned i = 0; i < 3; i++) {
6554 LLVMValueRef hi, out;
6555
6556 hi = LLVMBuildShl(builder, vtx_out[i*2+1],
6557 LLVMConstInt(ctx->i32, 16, 0), "");
6558 out = LLVMBuildOr(builder, vtx_out[i*2], hi, "");
6559 out = ac_to_float(&ctx->ac, out);
6560 ret = LLVMBuildInsertValue(builder, ret, out,
6561 gfx9_vtx_params[i].arg_index, "");
6562 }
6563 } else {
6564 for (unsigned i = 0; i < 6; i++) {
6565 LLVMValueRef out;
6566
6567 out = ac_to_float(&ctx->ac, vtx_out[i]);
6568 ret = LLVMBuildInsertValue(builder, ret, out,
6569 gfx6_vtx_params[i].arg_index, "");
6570 }
6571 }
6572 }
6573
6574 LLVMBuildRet(builder, ret);
6575 }
6576
6577 /**
6578 * Given a list of shader part functions, build a wrapper function that
6579 * runs them in sequence to form a monolithic shader.
6580 */
6581 static void si_build_wrapper_function(struct si_shader_context *ctx,
6582 LLVMValueRef *parts,
6583 unsigned num_parts,
6584 unsigned main_part,
6585 unsigned next_shader_first_part)
6586 {
6587 LLVMBuilderRef builder = ctx->ac.builder;
6588 /* PS epilog has one arg per color component; gfx9 merged shader
6589 * prologs need to forward 32 user SGPRs.
6590 */
6591 LLVMValueRef initial[64], out[64];
6592 LLVMTypeRef function_type;
6593 unsigned num_first_params;
6594 unsigned num_out, initial_num_out;
6595 ASSERTED unsigned num_out_sgpr; /* used in debug checks */
6596 ASSERTED unsigned initial_num_out_sgpr; /* used in debug checks */
6597 unsigned num_sgprs, num_vgprs;
6598 unsigned gprs;
6599
6600 memset(&ctx->args, 0, sizeof(ctx->args));
6601
6602 for (unsigned i = 0; i < num_parts; ++i) {
6603 ac_add_function_attr(ctx->ac.context, parts[i], -1,
6604 AC_FUNC_ATTR_ALWAYSINLINE);
6605 LLVMSetLinkage(parts[i], LLVMPrivateLinkage);
6606 }
6607
6608 /* The parameters of the wrapper function correspond to those of the
6609 * first part in terms of SGPRs and VGPRs, but we use the types of the
6610 * main part to get the right types. This is relevant for the
6611 * dereferenceable attribute on descriptor table pointers.
6612 */
6613 num_sgprs = 0;
6614 num_vgprs = 0;
6615
6616 function_type = LLVMGetElementType(LLVMTypeOf(parts[0]));
6617 num_first_params = LLVMCountParamTypes(function_type);
6618
6619 for (unsigned i = 0; i < num_first_params; ++i) {
6620 LLVMValueRef param = LLVMGetParam(parts[0], i);
6621
6622 if (ac_is_sgpr_param(param)) {
6623 assert(num_vgprs == 0);
6624 num_sgprs += ac_get_type_size(LLVMTypeOf(param)) / 4;
6625 } else {
6626 num_vgprs += ac_get_type_size(LLVMTypeOf(param)) / 4;
6627 }
6628 }
6629
6630 gprs = 0;
6631 while (gprs < num_sgprs + num_vgprs) {
6632 LLVMValueRef param = LLVMGetParam(parts[main_part], ctx->args.arg_count);
6633 LLVMTypeRef type = LLVMTypeOf(param);
6634 unsigned size = ac_get_type_size(type) / 4;
6635
6636 /* This is going to get casted anyways, so we don't have to
6637 * have the exact same type. But we do have to preserve the
6638 * pointer-ness so that LLVM knows about it.
6639 */
6640 enum ac_arg_type arg_type = AC_ARG_INT;
6641 if (LLVMGetTypeKind(type) == LLVMPointerTypeKind) {
6642 arg_type = AC_ARG_CONST_PTR;
6643 }
6644
6645 ac_add_arg(&ctx->args, gprs < num_sgprs ? AC_ARG_SGPR : AC_ARG_VGPR,
6646 size, arg_type, NULL);
6647
6648 assert(ac_is_sgpr_param(param) == (gprs < num_sgprs));
6649 assert(gprs + size <= num_sgprs + num_vgprs &&
6650 (gprs >= num_sgprs || gprs + size <= num_sgprs));
6651
6652 gprs += size;
6653 }
6654
6655 /* Prepare the return type. */
6656 unsigned num_returns = 0;
6657 LLVMTypeRef returns[32], last_func_type, return_type;
6658
6659 last_func_type = LLVMGetElementType(LLVMTypeOf(parts[num_parts - 1]));
6660 return_type = LLVMGetReturnType(last_func_type);
6661
6662 switch (LLVMGetTypeKind(return_type)) {
6663 case LLVMStructTypeKind:
6664 num_returns = LLVMCountStructElementTypes(return_type);
6665 assert(num_returns <= ARRAY_SIZE(returns));
6666 LLVMGetStructElementTypes(return_type, returns);
6667 break;
6668 case LLVMVoidTypeKind:
6669 break;
6670 default:
6671 unreachable("unexpected type");
6672 }
6673
6674 si_create_function(ctx, "wrapper", returns, num_returns,
6675 si_get_max_workgroup_size(ctx->shader));
6676
6677 if (is_merged_shader(ctx))
6678 ac_init_exec_full_mask(&ctx->ac);
6679
6680 /* Record the arguments of the function as if they were an output of
6681 * a previous part.
6682 */
6683 num_out = 0;
6684 num_out_sgpr = 0;
6685
6686 for (unsigned i = 0; i < ctx->args.arg_count; ++i) {
6687 LLVMValueRef param = LLVMGetParam(ctx->main_fn, i);
6688 LLVMTypeRef param_type = LLVMTypeOf(param);
6689 LLVMTypeRef out_type = ctx->args.args[i].file == AC_ARG_SGPR ? ctx->i32 : ctx->f32;
6690 unsigned size = ac_get_type_size(param_type) / 4;
6691
6692 if (size == 1) {
6693 if (LLVMGetTypeKind(param_type) == LLVMPointerTypeKind) {
6694 param = LLVMBuildPtrToInt(builder, param, ctx->i32, "");
6695 param_type = ctx->i32;
6696 }
6697
6698 if (param_type != out_type)
6699 param = LLVMBuildBitCast(builder, param, out_type, "");
6700 out[num_out++] = param;
6701 } else {
6702 LLVMTypeRef vector_type = LLVMVectorType(out_type, size);
6703
6704 if (LLVMGetTypeKind(param_type) == LLVMPointerTypeKind) {
6705 param = LLVMBuildPtrToInt(builder, param, ctx->i64, "");
6706 param_type = ctx->i64;
6707 }
6708
6709 if (param_type != vector_type)
6710 param = LLVMBuildBitCast(builder, param, vector_type, "");
6711
6712 for (unsigned j = 0; j < size; ++j)
6713 out[num_out++] = LLVMBuildExtractElement(
6714 builder, param, LLVMConstInt(ctx->i32, j, 0), "");
6715 }
6716
6717 if (ctx->args.args[i].file == AC_ARG_SGPR)
6718 num_out_sgpr = num_out;
6719 }
6720
6721 memcpy(initial, out, sizeof(out));
6722 initial_num_out = num_out;
6723 initial_num_out_sgpr = num_out_sgpr;
6724
6725 /* Now chain the parts. */
6726 LLVMValueRef ret = NULL;
6727 for (unsigned part = 0; part < num_parts; ++part) {
6728 LLVMValueRef in[48];
6729 LLVMTypeRef ret_type;
6730 unsigned out_idx = 0;
6731 unsigned num_params = LLVMCountParams(parts[part]);
6732
6733 /* Merged shaders are executed conditionally depending
6734 * on the number of enabled threads passed in the input SGPRs. */
6735 if (is_multi_part_shader(ctx) && part == 0) {
6736 LLVMValueRef ena, count = initial[3];
6737
6738 count = LLVMBuildAnd(builder, count,
6739 LLVMConstInt(ctx->i32, 0x7f, 0), "");
6740 ena = LLVMBuildICmp(builder, LLVMIntULT,
6741 ac_get_thread_id(&ctx->ac), count, "");
6742 ac_build_ifcc(&ctx->ac, ena, 6506);
6743 }
6744
6745 /* Derive arguments for the next part from outputs of the
6746 * previous one.
6747 */
6748 for (unsigned param_idx = 0; param_idx < num_params; ++param_idx) {
6749 LLVMValueRef param;
6750 LLVMTypeRef param_type;
6751 bool is_sgpr;
6752 unsigned param_size;
6753 LLVMValueRef arg = NULL;
6754
6755 param = LLVMGetParam(parts[part], param_idx);
6756 param_type = LLVMTypeOf(param);
6757 param_size = ac_get_type_size(param_type) / 4;
6758 is_sgpr = ac_is_sgpr_param(param);
6759
6760 if (is_sgpr) {
6761 ac_add_function_attr(ctx->ac.context, parts[part],
6762 param_idx + 1, AC_FUNC_ATTR_INREG);
6763 } else if (out_idx < num_out_sgpr) {
6764 /* Skip returned SGPRs the current part doesn't
6765 * declare on the input. */
6766 out_idx = num_out_sgpr;
6767 }
6768
6769 assert(out_idx + param_size <= (is_sgpr ? num_out_sgpr : num_out));
6770
6771 if (param_size == 1)
6772 arg = out[out_idx];
6773 else
6774 arg = ac_build_gather_values(&ctx->ac, &out[out_idx], param_size);
6775
6776 if (LLVMTypeOf(arg) != param_type) {
6777 if (LLVMGetTypeKind(param_type) == LLVMPointerTypeKind) {
6778 if (LLVMGetPointerAddressSpace(param_type) ==
6779 AC_ADDR_SPACE_CONST_32BIT) {
6780 arg = LLVMBuildBitCast(builder, arg, ctx->i32, "");
6781 arg = LLVMBuildIntToPtr(builder, arg, param_type, "");
6782 } else {
6783 arg = LLVMBuildBitCast(builder, arg, ctx->i64, "");
6784 arg = LLVMBuildIntToPtr(builder, arg, param_type, "");
6785 }
6786 } else {
6787 arg = LLVMBuildBitCast(builder, arg, param_type, "");
6788 }
6789 }
6790
6791 in[param_idx] = arg;
6792 out_idx += param_size;
6793 }
6794
6795 ret = ac_build_call(&ctx->ac, parts[part], in, num_params);
6796
6797 if (is_multi_part_shader(ctx) &&
6798 part + 1 == next_shader_first_part) {
6799 ac_build_endif(&ctx->ac, 6506);
6800
6801 /* The second half of the merged shader should use
6802 * the inputs from the toplevel (wrapper) function,
6803 * not the return value from the last call.
6804 *
6805 * That's because the last call was executed condi-
6806 * tionally, so we can't consume it in the main
6807 * block.
6808 */
6809 memcpy(out, initial, sizeof(initial));
6810 num_out = initial_num_out;
6811 num_out_sgpr = initial_num_out_sgpr;
6812 continue;
6813 }
6814
6815 /* Extract the returned GPRs. */
6816 ret_type = LLVMTypeOf(ret);
6817 num_out = 0;
6818 num_out_sgpr = 0;
6819
6820 if (LLVMGetTypeKind(ret_type) != LLVMVoidTypeKind) {
6821 assert(LLVMGetTypeKind(ret_type) == LLVMStructTypeKind);
6822
6823 unsigned ret_size = LLVMCountStructElementTypes(ret_type);
6824
6825 for (unsigned i = 0; i < ret_size; ++i) {
6826 LLVMValueRef val =
6827 LLVMBuildExtractValue(builder, ret, i, "");
6828
6829 assert(num_out < ARRAY_SIZE(out));
6830 out[num_out++] = val;
6831
6832 if (LLVMTypeOf(val) == ctx->i32) {
6833 assert(num_out_sgpr + 1 == num_out);
6834 num_out_sgpr = num_out;
6835 }
6836 }
6837 }
6838 }
6839
6840 /* Return the value from the last part. */
6841 if (LLVMGetTypeKind(LLVMTypeOf(ret)) == LLVMVoidTypeKind)
6842 LLVMBuildRetVoid(builder);
6843 else
6844 LLVMBuildRet(builder, ret);
6845 }
6846
6847 static bool si_should_optimize_less(struct ac_llvm_compiler *compiler,
6848 struct si_shader_selector *sel)
6849 {
6850 if (!compiler->low_opt_passes)
6851 return false;
6852
6853 /* Assume a slow CPU. */
6854 assert(!sel->screen->info.has_dedicated_vram &&
6855 sel->screen->info.chip_class <= GFX8);
6856
6857 /* For a crazy dEQP test containing 2597 memory opcodes, mostly
6858 * buffer stores. */
6859 return sel->type == PIPE_SHADER_COMPUTE &&
6860 sel->info.num_memory_instructions > 1000;
6861 }
6862
6863 static struct nir_shader *get_nir_shader(struct si_shader_selector *sel,
6864 bool *free_nir)
6865 {
6866 *free_nir = false;
6867
6868 if (sel->nir) {
6869 return sel->nir;
6870 } else if (sel->nir_binary) {
6871 struct pipe_screen *screen = &sel->screen->b;
6872 const void *options =
6873 screen->get_compiler_options(screen, PIPE_SHADER_IR_NIR,
6874 sel->type);
6875
6876 struct blob_reader blob_reader;
6877 blob_reader_init(&blob_reader, sel->nir_binary, sel->nir_size);
6878 *free_nir = true;
6879 return nir_deserialize(NULL, options, &blob_reader);
6880 }
6881 return NULL;
6882 }
6883
6884 int si_compile_tgsi_shader(struct si_screen *sscreen,
6885 struct ac_llvm_compiler *compiler,
6886 struct si_shader *shader,
6887 struct pipe_debug_callback *debug)
6888 {
6889 struct si_shader_selector *sel = shader->selector;
6890 struct si_shader_context ctx;
6891 bool free_nir;
6892 struct nir_shader *nir = get_nir_shader(sel, &free_nir);
6893 int r = -1;
6894
6895 /* Dump TGSI code before doing TGSI->LLVM conversion in case the
6896 * conversion fails. */
6897 if (si_can_dump_shader(sscreen, sel->type) &&
6898 !(sscreen->debug_flags & DBG(NO_TGSI))) {
6899 if (sel->tokens)
6900 tgsi_dump(sel->tokens, 0);
6901 else
6902 nir_print_shader(nir, stderr);
6903 si_dump_streamout(&sel->so);
6904 }
6905
6906 si_init_shader_ctx(&ctx, sscreen, compiler, si_get_shader_wave_size(shader),
6907 nir != NULL);
6908 si_llvm_context_set_ir(&ctx, shader, nir);
6909
6910 memset(shader->info.vs_output_param_offset, AC_EXP_PARAM_UNDEFINED,
6911 sizeof(shader->info.vs_output_param_offset));
6912
6913 shader->info.uses_instanceid = sel->info.uses_instanceid;
6914
6915 if (!si_compile_tgsi_main(&ctx, nir, free_nir)) {
6916 si_llvm_dispose(&ctx);
6917 return -1;
6918 }
6919
6920 if (shader->is_monolithic && ctx.type == PIPE_SHADER_VERTEX) {
6921 LLVMValueRef parts[2];
6922 bool need_prolog = sel->vs_needs_prolog;
6923
6924 parts[1] = ctx.main_fn;
6925
6926 if (need_prolog) {
6927 union si_shader_part_key prolog_key;
6928 si_get_vs_prolog_key(&sel->info,
6929 shader->info.num_input_sgprs,
6930 &shader->key.part.vs.prolog,
6931 shader, &prolog_key);
6932 si_build_vs_prolog_function(&ctx, &prolog_key);
6933 parts[0] = ctx.main_fn;
6934 }
6935
6936 si_build_wrapper_function(&ctx, parts + !need_prolog,
6937 1 + need_prolog, need_prolog, 0);
6938
6939 if (ctx.shader->key.opt.vs_as_prim_discard_cs)
6940 si_build_prim_discard_compute_shader(&ctx);
6941 } else if (shader->is_monolithic && ctx.type == PIPE_SHADER_TESS_CTRL) {
6942 if (sscreen->info.chip_class >= GFX9) {
6943 struct si_shader_selector *ls = shader->key.part.tcs.ls;
6944 LLVMValueRef parts[4];
6945 bool vs_needs_prolog =
6946 si_vs_needs_prolog(ls, &shader->key.part.tcs.ls_prolog);
6947
6948 /* TCS main part */
6949 parts[2] = ctx.main_fn;
6950
6951 /* TCS epilog */
6952 union si_shader_part_key tcs_epilog_key;
6953 memset(&tcs_epilog_key, 0, sizeof(tcs_epilog_key));
6954 tcs_epilog_key.tcs_epilog.states = shader->key.part.tcs.epilog;
6955 si_build_tcs_epilog_function(&ctx, &tcs_epilog_key);
6956 parts[3] = ctx.main_fn;
6957
6958 /* VS as LS main part */
6959 nir = get_nir_shader(ls, &free_nir);
6960 struct si_shader shader_ls = {};
6961 shader_ls.selector = ls;
6962 shader_ls.key.as_ls = 1;
6963 shader_ls.key.mono = shader->key.mono;
6964 shader_ls.key.opt = shader->key.opt;
6965 shader_ls.is_monolithic = true;
6966 si_llvm_context_set_ir(&ctx, &shader_ls, nir);
6967
6968 if (!si_compile_tgsi_main(&ctx, nir, free_nir)) {
6969 si_llvm_dispose(&ctx);
6970 return -1;
6971 }
6972 shader->info.uses_instanceid |= ls->info.uses_instanceid;
6973 parts[1] = ctx.main_fn;
6974
6975 /* LS prolog */
6976 if (vs_needs_prolog) {
6977 union si_shader_part_key vs_prolog_key;
6978 si_get_vs_prolog_key(&ls->info,
6979 shader_ls.info.num_input_sgprs,
6980 &shader->key.part.tcs.ls_prolog,
6981 shader, &vs_prolog_key);
6982 vs_prolog_key.vs_prolog.is_monolithic = true;
6983 si_build_vs_prolog_function(&ctx, &vs_prolog_key);
6984 parts[0] = ctx.main_fn;
6985 }
6986
6987 /* Reset the shader context. */
6988 ctx.shader = shader;
6989 ctx.type = PIPE_SHADER_TESS_CTRL;
6990
6991 si_build_wrapper_function(&ctx,
6992 parts + !vs_needs_prolog,
6993 4 - !vs_needs_prolog, vs_needs_prolog,
6994 vs_needs_prolog ? 2 : 1);
6995 } else {
6996 LLVMValueRef parts[2];
6997 union si_shader_part_key epilog_key;
6998
6999 parts[0] = ctx.main_fn;
7000
7001 memset(&epilog_key, 0, sizeof(epilog_key));
7002 epilog_key.tcs_epilog.states = shader->key.part.tcs.epilog;
7003 si_build_tcs_epilog_function(&ctx, &epilog_key);
7004 parts[1] = ctx.main_fn;
7005
7006 si_build_wrapper_function(&ctx, parts, 2, 0, 0);
7007 }
7008 } else if (shader->is_monolithic && ctx.type == PIPE_SHADER_GEOMETRY) {
7009 if (ctx.screen->info.chip_class >= GFX9) {
7010 struct si_shader_selector *es = shader->key.part.gs.es;
7011 LLVMValueRef es_prolog = NULL;
7012 LLVMValueRef es_main = NULL;
7013 LLVMValueRef gs_prolog = NULL;
7014 LLVMValueRef gs_main = ctx.main_fn;
7015
7016 /* GS prolog */
7017 union si_shader_part_key gs_prolog_key;
7018 memset(&gs_prolog_key, 0, sizeof(gs_prolog_key));
7019 gs_prolog_key.gs_prolog.states = shader->key.part.gs.prolog;
7020 gs_prolog_key.gs_prolog.is_monolithic = true;
7021 gs_prolog_key.gs_prolog.as_ngg = shader->key.as_ngg;
7022 si_build_gs_prolog_function(&ctx, &gs_prolog_key);
7023 gs_prolog = ctx.main_fn;
7024
7025 /* ES main part */
7026 nir = get_nir_shader(es, &free_nir);
7027 struct si_shader shader_es = {};
7028 shader_es.selector = es;
7029 shader_es.key.as_es = 1;
7030 shader_es.key.as_ngg = shader->key.as_ngg;
7031 shader_es.key.mono = shader->key.mono;
7032 shader_es.key.opt = shader->key.opt;
7033 shader_es.is_monolithic = true;
7034 si_llvm_context_set_ir(&ctx, &shader_es, nir);
7035
7036 if (!si_compile_tgsi_main(&ctx, nir, free_nir)) {
7037 si_llvm_dispose(&ctx);
7038 return -1;
7039 }
7040 shader->info.uses_instanceid |= es->info.uses_instanceid;
7041 es_main = ctx.main_fn;
7042
7043 /* ES prolog */
7044 if (es->vs_needs_prolog) {
7045 union si_shader_part_key vs_prolog_key;
7046 si_get_vs_prolog_key(&es->info,
7047 shader_es.info.num_input_sgprs,
7048 &shader->key.part.gs.vs_prolog,
7049 shader, &vs_prolog_key);
7050 vs_prolog_key.vs_prolog.is_monolithic = true;
7051 si_build_vs_prolog_function(&ctx, &vs_prolog_key);
7052 es_prolog = ctx.main_fn;
7053 }
7054
7055 /* Reset the shader context. */
7056 ctx.shader = shader;
7057 ctx.type = PIPE_SHADER_GEOMETRY;
7058
7059 /* Prepare the array of shader parts. */
7060 LLVMValueRef parts[4];
7061 unsigned num_parts = 0, main_part, next_first_part;
7062
7063 if (es_prolog)
7064 parts[num_parts++] = es_prolog;
7065
7066 parts[main_part = num_parts++] = es_main;
7067 parts[next_first_part = num_parts++] = gs_prolog;
7068 parts[num_parts++] = gs_main;
7069
7070 si_build_wrapper_function(&ctx, parts, num_parts,
7071 main_part, next_first_part);
7072 } else {
7073 LLVMValueRef parts[2];
7074 union si_shader_part_key prolog_key;
7075
7076 parts[1] = ctx.main_fn;
7077
7078 memset(&prolog_key, 0, sizeof(prolog_key));
7079 prolog_key.gs_prolog.states = shader->key.part.gs.prolog;
7080 si_build_gs_prolog_function(&ctx, &prolog_key);
7081 parts[0] = ctx.main_fn;
7082
7083 si_build_wrapper_function(&ctx, parts, 2, 1, 0);
7084 }
7085 } else if (shader->is_monolithic && ctx.type == PIPE_SHADER_FRAGMENT) {
7086 LLVMValueRef parts[3];
7087 union si_shader_part_key prolog_key;
7088 union si_shader_part_key epilog_key;
7089 bool need_prolog;
7090
7091 si_get_ps_prolog_key(shader, &prolog_key, false);
7092 need_prolog = si_need_ps_prolog(&prolog_key);
7093
7094 parts[need_prolog ? 1 : 0] = ctx.main_fn;
7095
7096 if (need_prolog) {
7097 si_build_ps_prolog_function(&ctx, &prolog_key);
7098 parts[0] = ctx.main_fn;
7099 }
7100
7101 si_get_ps_epilog_key(shader, &epilog_key);
7102 si_build_ps_epilog_function(&ctx, &epilog_key);
7103 parts[need_prolog ? 2 : 1] = ctx.main_fn;
7104
7105 si_build_wrapper_function(&ctx, parts, need_prolog ? 3 : 2,
7106 need_prolog ? 1 : 0, 0);
7107 }
7108
7109 si_llvm_optimize_module(&ctx);
7110
7111 /* Post-optimization transformations and analysis. */
7112 si_optimize_vs_outputs(&ctx);
7113
7114 if ((debug && debug->debug_message) ||
7115 si_can_dump_shader(sscreen, ctx.type)) {
7116 ctx.shader->info.private_mem_vgprs =
7117 ac_count_scratch_private_memory(ctx.main_fn);
7118 }
7119
7120 /* Make sure the input is a pointer and not integer followed by inttoptr. */
7121 assert(LLVMGetTypeKind(LLVMTypeOf(LLVMGetParam(ctx.main_fn, 0))) ==
7122 LLVMPointerTypeKind);
7123
7124 /* Compile to bytecode. */
7125 r = si_compile_llvm(sscreen, &shader->binary, &shader->config, compiler,
7126 ctx.ac.module, debug, ctx.type, ctx.ac.wave_size,
7127 si_get_shader_name(shader),
7128 si_should_optimize_less(compiler, shader->selector));
7129 si_llvm_dispose(&ctx);
7130 if (r) {
7131 fprintf(stderr, "LLVM failed to compile shader\n");
7132 return r;
7133 }
7134
7135 /* Validate SGPR and VGPR usage for compute to detect compiler bugs.
7136 * LLVM 3.9svn has this bug.
7137 */
7138 if (sel->type == PIPE_SHADER_COMPUTE) {
7139 unsigned wave_size = sscreen->compute_wave_size;
7140 unsigned max_vgprs = sscreen->info.num_physical_wave64_vgprs_per_simd *
7141 (wave_size == 32 ? 2 : 1);
7142 unsigned max_sgprs = sscreen->info.num_physical_sgprs_per_simd;
7143 unsigned max_sgprs_per_wave = 128;
7144 unsigned simds_per_tg = 4; /* assuming WGP mode on gfx10 */
7145 unsigned threads_per_tg = si_get_max_workgroup_size(shader);
7146 unsigned waves_per_tg = DIV_ROUND_UP(threads_per_tg, wave_size);
7147 unsigned waves_per_simd = DIV_ROUND_UP(waves_per_tg, simds_per_tg);
7148
7149 max_vgprs = max_vgprs / waves_per_simd;
7150 max_sgprs = MIN2(max_sgprs / waves_per_simd, max_sgprs_per_wave);
7151
7152 if (shader->config.num_sgprs > max_sgprs ||
7153 shader->config.num_vgprs > max_vgprs) {
7154 fprintf(stderr, "LLVM failed to compile a shader correctly: "
7155 "SGPR:VGPR usage is %u:%u, but the hw limit is %u:%u\n",
7156 shader->config.num_sgprs, shader->config.num_vgprs,
7157 max_sgprs, max_vgprs);
7158
7159 /* Just terminate the process, because dependent
7160 * shaders can hang due to bad input data, but use
7161 * the env var to allow shader-db to work.
7162 */
7163 if (!debug_get_bool_option("SI_PASS_BAD_SHADERS", false))
7164 abort();
7165 }
7166 }
7167
7168 /* Add the scratch offset to input SGPRs. */
7169 if (shader->config.scratch_bytes_per_wave && !is_merged_shader(&ctx))
7170 shader->info.num_input_sgprs += 1; /* scratch byte offset */
7171
7172 /* Calculate the number of fragment input VGPRs. */
7173 if (ctx.type == PIPE_SHADER_FRAGMENT) {
7174 shader->info.num_input_vgprs = ac_get_fs_input_vgpr_cnt(&shader->config,
7175 &shader->info.face_vgpr_index,
7176 &shader->info.ancillary_vgpr_index);
7177 }
7178
7179 si_calculate_max_simd_waves(shader);
7180 si_shader_dump_stats_for_shader_db(sscreen, shader, debug);
7181 return 0;
7182 }
7183
7184 /**
7185 * Create, compile and return a shader part (prolog or epilog).
7186 *
7187 * \param sscreen screen
7188 * \param list list of shader parts of the same category
7189 * \param type shader type
7190 * \param key shader part key
7191 * \param prolog whether the part being requested is a prolog
7192 * \param tm LLVM target machine
7193 * \param debug debug callback
7194 * \param build the callback responsible for building the main function
7195 * \return non-NULL on success
7196 */
7197 static struct si_shader_part *
7198 si_get_shader_part(struct si_screen *sscreen,
7199 struct si_shader_part **list,
7200 enum pipe_shader_type type,
7201 bool prolog,
7202 union si_shader_part_key *key,
7203 struct ac_llvm_compiler *compiler,
7204 struct pipe_debug_callback *debug,
7205 void (*build)(struct si_shader_context *,
7206 union si_shader_part_key *),
7207 const char *name)
7208 {
7209 struct si_shader_part *result;
7210
7211 simple_mtx_lock(&sscreen->shader_parts_mutex);
7212
7213 /* Find existing. */
7214 for (result = *list; result; result = result->next) {
7215 if (memcmp(&result->key, key, sizeof(*key)) == 0) {
7216 simple_mtx_unlock(&sscreen->shader_parts_mutex);
7217 return result;
7218 }
7219 }
7220
7221 /* Compile a new one. */
7222 result = CALLOC_STRUCT(si_shader_part);
7223 result->key = *key;
7224
7225 struct si_shader shader = {};
7226
7227 switch (type) {
7228 case PIPE_SHADER_VERTEX:
7229 shader.key.as_ls = key->vs_prolog.as_ls;
7230 shader.key.as_es = key->vs_prolog.as_es;
7231 shader.key.as_ngg = key->vs_prolog.as_ngg;
7232 break;
7233 case PIPE_SHADER_TESS_CTRL:
7234 assert(!prolog);
7235 shader.key.part.tcs.epilog = key->tcs_epilog.states;
7236 break;
7237 case PIPE_SHADER_GEOMETRY:
7238 assert(prolog);
7239 shader.key.as_ngg = key->gs_prolog.as_ngg;
7240 break;
7241 case PIPE_SHADER_FRAGMENT:
7242 if (prolog)
7243 shader.key.part.ps.prolog = key->ps_prolog.states;
7244 else
7245 shader.key.part.ps.epilog = key->ps_epilog.states;
7246 break;
7247 default:
7248 unreachable("bad shader part");
7249 }
7250
7251 struct si_shader_context ctx;
7252 si_init_shader_ctx(&ctx, sscreen, compiler,
7253 si_get_wave_size(sscreen, type, shader.key.as_ngg,
7254 shader.key.as_es),
7255 false);
7256 ctx.shader = &shader;
7257 ctx.type = type;
7258
7259 build(&ctx, key);
7260
7261 /* Compile. */
7262 si_llvm_optimize_module(&ctx);
7263
7264 if (si_compile_llvm(sscreen, &result->binary, &result->config, compiler,
7265 ctx.ac.module, debug, ctx.type, ctx.ac.wave_size,
7266 name, false)) {
7267 FREE(result);
7268 result = NULL;
7269 goto out;
7270 }
7271
7272 result->next = *list;
7273 *list = result;
7274
7275 out:
7276 si_llvm_dispose(&ctx);
7277 simple_mtx_unlock(&sscreen->shader_parts_mutex);
7278 return result;
7279 }
7280
7281 static LLVMValueRef si_prolog_get_rw_buffers(struct si_shader_context *ctx)
7282 {
7283 LLVMValueRef ptr[2], list;
7284 bool merged_shader = is_merged_shader(ctx);
7285
7286 ptr[0] = LLVMGetParam(ctx->main_fn, (merged_shader ? 8 : 0) + SI_SGPR_RW_BUFFERS);
7287 list = LLVMBuildIntToPtr(ctx->ac.builder, ptr[0],
7288 ac_array_in_const32_addr_space(ctx->v4i32), "");
7289 return list;
7290 }
7291
7292 /**
7293 * Build the vertex shader prolog function.
7294 *
7295 * The inputs are the same as VS (a lot of SGPRs and 4 VGPR system values).
7296 * All inputs are returned unmodified. The vertex load indices are
7297 * stored after them, which will be used by the API VS for fetching inputs.
7298 *
7299 * For example, the expected outputs for instance_divisors[] = {0, 1, 2} are:
7300 * input_v0,
7301 * input_v1,
7302 * input_v2,
7303 * input_v3,
7304 * (VertexID + BaseVertex),
7305 * (InstanceID + StartInstance),
7306 * (InstanceID / 2 + StartInstance)
7307 */
7308 static void si_build_vs_prolog_function(struct si_shader_context *ctx,
7309 union si_shader_part_key *key)
7310 {
7311 LLVMTypeRef *returns;
7312 LLVMValueRef ret, func;
7313 int num_returns, i;
7314 unsigned first_vs_vgpr = key->vs_prolog.num_merged_next_stage_vgprs;
7315 unsigned num_input_vgprs = key->vs_prolog.num_merged_next_stage_vgprs + 4;
7316 struct ac_arg input_sgpr_param[key->vs_prolog.num_input_sgprs];
7317 struct ac_arg input_vgpr_param[9];
7318 LLVMValueRef input_vgprs[9];
7319 unsigned num_all_input_regs = key->vs_prolog.num_input_sgprs +
7320 num_input_vgprs;
7321 unsigned user_sgpr_base = key->vs_prolog.num_merged_next_stage_vgprs ? 8 : 0;
7322
7323 memset(&ctx->args, 0, sizeof(ctx->args));
7324
7325 /* 4 preloaded VGPRs + vertex load indices as prolog outputs */
7326 returns = alloca((num_all_input_regs + key->vs_prolog.last_input + 1) *
7327 sizeof(LLVMTypeRef));
7328 num_returns = 0;
7329
7330 /* Declare input and output SGPRs. */
7331 for (i = 0; i < key->vs_prolog.num_input_sgprs; i++) {
7332 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT,
7333 &input_sgpr_param[i]);
7334 returns[num_returns++] = ctx->i32;
7335 }
7336
7337 struct ac_arg merged_wave_info = input_sgpr_param[3];
7338
7339 /* Preloaded VGPRs (outputs must be floats) */
7340 for (i = 0; i < num_input_vgprs; i++) {
7341 ac_add_arg(&ctx->args, AC_ARG_VGPR, 1, AC_ARG_INT, &input_vgpr_param[i]);
7342 returns[num_returns++] = ctx->f32;
7343 }
7344
7345 /* Vertex load indices. */
7346 for (i = 0; i <= key->vs_prolog.last_input; i++)
7347 returns[num_returns++] = ctx->f32;
7348
7349 /* Create the function. */
7350 si_create_function(ctx, "vs_prolog", returns, num_returns, 0);
7351 func = ctx->main_fn;
7352
7353 for (i = 0; i < num_input_vgprs; i++) {
7354 input_vgprs[i] = ac_get_arg(&ctx->ac, input_vgpr_param[i]);
7355 }
7356
7357 if (key->vs_prolog.num_merged_next_stage_vgprs) {
7358 if (!key->vs_prolog.is_monolithic)
7359 si_init_exec_from_input(ctx, merged_wave_info, 0);
7360
7361 if (key->vs_prolog.as_ls &&
7362 ctx->screen->info.has_ls_vgpr_init_bug) {
7363 /* If there are no HS threads, SPI loads the LS VGPRs
7364 * starting at VGPR 0. Shift them back to where they
7365 * belong.
7366 */
7367 LLVMValueRef has_hs_threads =
7368 LLVMBuildICmp(ctx->ac.builder, LLVMIntNE,
7369 si_unpack_param(ctx, input_sgpr_param[3], 8, 8),
7370 ctx->i32_0, "");
7371
7372 for (i = 4; i > 0; --i) {
7373 input_vgprs[i + 1] =
7374 LLVMBuildSelect(ctx->ac.builder, has_hs_threads,
7375 input_vgprs[i + 1],
7376 input_vgprs[i - 1], "");
7377 }
7378 }
7379 }
7380
7381 unsigned vertex_id_vgpr = first_vs_vgpr;
7382 unsigned instance_id_vgpr =
7383 ctx->screen->info.chip_class >= GFX10 ?
7384 first_vs_vgpr + 3 :
7385 first_vs_vgpr + (key->vs_prolog.as_ls ? 2 : 1);
7386
7387 ctx->abi.vertex_id = input_vgprs[vertex_id_vgpr];
7388 ctx->abi.instance_id = input_vgprs[instance_id_vgpr];
7389
7390 /* InstanceID = VertexID >> 16;
7391 * VertexID = VertexID & 0xffff;
7392 */
7393 if (key->vs_prolog.states.unpack_instance_id_from_vertex_id) {
7394 ctx->abi.instance_id = LLVMBuildLShr(ctx->ac.builder, ctx->abi.vertex_id,
7395 LLVMConstInt(ctx->i32, 16, 0), "");
7396 ctx->abi.vertex_id = LLVMBuildAnd(ctx->ac.builder, ctx->abi.vertex_id,
7397 LLVMConstInt(ctx->i32, 0xffff, 0), "");
7398 }
7399
7400 /* Copy inputs to outputs. This should be no-op, as the registers match,
7401 * but it will prevent the compiler from overwriting them unintentionally.
7402 */
7403 ret = ctx->return_value;
7404 for (i = 0; i < key->vs_prolog.num_input_sgprs; i++) {
7405 LLVMValueRef p = LLVMGetParam(func, i);
7406 ret = LLVMBuildInsertValue(ctx->ac.builder, ret, p, i, "");
7407 }
7408 for (i = 0; i < num_input_vgprs; i++) {
7409 LLVMValueRef p = input_vgprs[i];
7410
7411 if (i == vertex_id_vgpr)
7412 p = ctx->abi.vertex_id;
7413 else if (i == instance_id_vgpr)
7414 p = ctx->abi.instance_id;
7415
7416 p = ac_to_float(&ctx->ac, p);
7417 ret = LLVMBuildInsertValue(ctx->ac.builder, ret, p,
7418 key->vs_prolog.num_input_sgprs + i, "");
7419 }
7420
7421 LLVMValueRef original_ret = ret;
7422 bool wrapped = false;
7423 LLVMBasicBlockRef if_entry_block = NULL;
7424
7425 if (key->vs_prolog.is_monolithic && key->vs_prolog.as_ngg) {
7426 LLVMValueRef num_threads;
7427 LLVMValueRef ena;
7428
7429 num_threads = si_unpack_param(ctx, merged_wave_info, 0, 8);
7430 ena = LLVMBuildICmp(ctx->ac.builder, LLVMIntULT,
7431 ac_get_thread_id(&ctx->ac), num_threads, "");
7432 if_entry_block = LLVMGetInsertBlock(ctx->ac.builder);
7433 ac_build_ifcc(&ctx->ac, ena, 11501);
7434 wrapped = true;
7435 }
7436
7437 /* Compute vertex load indices from instance divisors. */
7438 LLVMValueRef instance_divisor_constbuf = NULL;
7439
7440 if (key->vs_prolog.states.instance_divisor_is_fetched) {
7441 LLVMValueRef list = si_prolog_get_rw_buffers(ctx);
7442 LLVMValueRef buf_index =
7443 LLVMConstInt(ctx->i32, SI_VS_CONST_INSTANCE_DIVISORS, 0);
7444 instance_divisor_constbuf =
7445 ac_build_load_to_sgpr(&ctx->ac, list, buf_index);
7446 }
7447
7448 for (i = 0; i <= key->vs_prolog.last_input; i++) {
7449 bool divisor_is_one =
7450 key->vs_prolog.states.instance_divisor_is_one & (1u << i);
7451 bool divisor_is_fetched =
7452 key->vs_prolog.states.instance_divisor_is_fetched & (1u << i);
7453 LLVMValueRef index = NULL;
7454
7455 if (divisor_is_one) {
7456 index = ctx->abi.instance_id;
7457 } else if (divisor_is_fetched) {
7458 LLVMValueRef udiv_factors[4];
7459
7460 for (unsigned j = 0; j < 4; j++) {
7461 udiv_factors[j] =
7462 buffer_load_const(ctx, instance_divisor_constbuf,
7463 LLVMConstInt(ctx->i32, i*16 + j*4, 0));
7464 udiv_factors[j] = ac_to_integer(&ctx->ac, udiv_factors[j]);
7465 }
7466 /* The faster NUW version doesn't work when InstanceID == UINT_MAX.
7467 * Such InstanceID might not be achievable in a reasonable time though.
7468 */
7469 index = ac_build_fast_udiv_nuw(&ctx->ac, ctx->abi.instance_id,
7470 udiv_factors[0], udiv_factors[1],
7471 udiv_factors[2], udiv_factors[3]);
7472 }
7473
7474 if (divisor_is_one || divisor_is_fetched) {
7475 /* Add StartInstance. */
7476 index = LLVMBuildAdd(ctx->ac.builder, index,
7477 LLVMGetParam(ctx->main_fn, user_sgpr_base +
7478 SI_SGPR_START_INSTANCE), "");
7479 } else {
7480 /* VertexID + BaseVertex */
7481 index = LLVMBuildAdd(ctx->ac.builder,
7482 ctx->abi.vertex_id,
7483 LLVMGetParam(func, user_sgpr_base +
7484 SI_SGPR_BASE_VERTEX), "");
7485 }
7486
7487 index = ac_to_float(&ctx->ac, index);
7488 ret = LLVMBuildInsertValue(ctx->ac.builder, ret, index,
7489 ctx->args.arg_count + i, "");
7490 }
7491
7492 if (wrapped) {
7493 LLVMBasicBlockRef bbs[2] = {
7494 LLVMGetInsertBlock(ctx->ac.builder),
7495 if_entry_block,
7496 };
7497 ac_build_endif(&ctx->ac, 11501);
7498
7499 LLVMValueRef values[2] = {
7500 ret,
7501 original_ret
7502 };
7503 ret = ac_build_phi(&ctx->ac, LLVMTypeOf(ret), 2, values, bbs);
7504 }
7505
7506 si_llvm_build_ret(ctx, ret);
7507 }
7508
7509 static bool si_get_vs_prolog(struct si_screen *sscreen,
7510 struct ac_llvm_compiler *compiler,
7511 struct si_shader *shader,
7512 struct pipe_debug_callback *debug,
7513 struct si_shader *main_part,
7514 const struct si_vs_prolog_bits *key)
7515 {
7516 struct si_shader_selector *vs = main_part->selector;
7517
7518 if (!si_vs_needs_prolog(vs, key))
7519 return true;
7520
7521 /* Get the prolog. */
7522 union si_shader_part_key prolog_key;
7523 si_get_vs_prolog_key(&vs->info, main_part->info.num_input_sgprs,
7524 key, shader, &prolog_key);
7525
7526 shader->prolog =
7527 si_get_shader_part(sscreen, &sscreen->vs_prologs,
7528 PIPE_SHADER_VERTEX, true, &prolog_key, compiler,
7529 debug, si_build_vs_prolog_function,
7530 "Vertex Shader Prolog");
7531 return shader->prolog != NULL;
7532 }
7533
7534 /**
7535 * Select and compile (or reuse) vertex shader parts (prolog & epilog).
7536 */
7537 static bool si_shader_select_vs_parts(struct si_screen *sscreen,
7538 struct ac_llvm_compiler *compiler,
7539 struct si_shader *shader,
7540 struct pipe_debug_callback *debug)
7541 {
7542 return si_get_vs_prolog(sscreen, compiler, shader, debug, shader,
7543 &shader->key.part.vs.prolog);
7544 }
7545
7546 /**
7547 * Compile the TCS epilog function. This writes tesselation factors to memory
7548 * based on the output primitive type of the tesselator (determined by TES).
7549 */
7550 static void si_build_tcs_epilog_function(struct si_shader_context *ctx,
7551 union si_shader_part_key *key)
7552 {
7553 struct lp_build_tgsi_context *bld_base = &ctx->bld_base;
7554
7555 memset(&ctx->args, 0, sizeof(ctx->args));
7556
7557 if (ctx->screen->info.chip_class >= GFX9) {
7558 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, NULL);
7559 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, NULL);
7560 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT,
7561 &ctx->tcs_offchip_offset);
7562 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, NULL); /* wave info */
7563 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT,
7564 &ctx->tcs_factor_offset);
7565 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, NULL);
7566 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, NULL);
7567 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, NULL);
7568 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, NULL);
7569 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, NULL);
7570 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, NULL);
7571 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, NULL);
7572 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, NULL);
7573 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, NULL);
7574 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, NULL);
7575 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, NULL);
7576 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT,
7577 &ctx->tcs_offchip_layout);
7578 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, NULL);
7579 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT,
7580 &ctx->tcs_out_lds_layout);
7581 } else {
7582 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, NULL);
7583 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, NULL);
7584 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, NULL);
7585 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, NULL);
7586 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT,
7587 &ctx->tcs_offchip_layout);
7588 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, NULL);
7589 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT,
7590 &ctx->tcs_out_lds_layout);
7591 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, NULL);
7592 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT,
7593 &ctx->tcs_offchip_offset);
7594 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT,
7595 &ctx->tcs_factor_offset);
7596 }
7597
7598 ac_add_arg(&ctx->args, AC_ARG_VGPR, 1, AC_ARG_INT, NULL); /* VGPR gap */
7599 ac_add_arg(&ctx->args, AC_ARG_VGPR, 1, AC_ARG_INT, NULL); /* VGPR gap */
7600 struct ac_arg rel_patch_id; /* patch index within the wave (REL_PATCH_ID) */
7601 ac_add_arg(&ctx->args, AC_ARG_VGPR, 1, AC_ARG_INT, &rel_patch_id);
7602 struct ac_arg invocation_id; /* invocation ID within the patch */
7603 ac_add_arg(&ctx->args, AC_ARG_VGPR, 1, AC_ARG_INT, &invocation_id);
7604 struct ac_arg tcs_out_current_patch_data_offset; /* LDS offset where tess factors should be loaded from */
7605 ac_add_arg(&ctx->args, AC_ARG_VGPR, 1, AC_ARG_INT,
7606 &tcs_out_current_patch_data_offset);
7607
7608 struct ac_arg tess_factors[6];
7609 for (unsigned i = 0; i < 6; i++)
7610 ac_add_arg(&ctx->args, AC_ARG_VGPR, 1, AC_ARG_INT, &tess_factors[i]);
7611
7612 /* Create the function. */
7613 si_create_function(ctx, "tcs_epilog", NULL, 0,
7614 ctx->screen->info.chip_class >= GFX7 ? 128 : 0);
7615 ac_declare_lds_as_pointer(&ctx->ac);
7616
7617 LLVMValueRef invoc0_tess_factors[6];
7618 for (unsigned i = 0; i < 6; i++)
7619 invoc0_tess_factors[i] = ac_get_arg(&ctx->ac, tess_factors[i]);
7620
7621 si_write_tess_factors(bld_base,
7622 ac_get_arg(&ctx->ac, rel_patch_id),
7623 ac_get_arg(&ctx->ac, invocation_id),
7624 ac_get_arg(&ctx->ac, tcs_out_current_patch_data_offset),
7625 invoc0_tess_factors, invoc0_tess_factors + 4);
7626
7627 LLVMBuildRetVoid(ctx->ac.builder);
7628 }
7629
7630 /**
7631 * Select and compile (or reuse) TCS parts (epilog).
7632 */
7633 static bool si_shader_select_tcs_parts(struct si_screen *sscreen,
7634 struct ac_llvm_compiler *compiler,
7635 struct si_shader *shader,
7636 struct pipe_debug_callback *debug)
7637 {
7638 if (sscreen->info.chip_class >= GFX9) {
7639 struct si_shader *ls_main_part =
7640 shader->key.part.tcs.ls->main_shader_part_ls;
7641
7642 if (!si_get_vs_prolog(sscreen, compiler, shader, debug, ls_main_part,
7643 &shader->key.part.tcs.ls_prolog))
7644 return false;
7645
7646 shader->previous_stage = ls_main_part;
7647 }
7648
7649 /* Get the epilog. */
7650 union si_shader_part_key epilog_key;
7651 memset(&epilog_key, 0, sizeof(epilog_key));
7652 epilog_key.tcs_epilog.states = shader->key.part.tcs.epilog;
7653
7654 shader->epilog = si_get_shader_part(sscreen, &sscreen->tcs_epilogs,
7655 PIPE_SHADER_TESS_CTRL, false,
7656 &epilog_key, compiler, debug,
7657 si_build_tcs_epilog_function,
7658 "Tessellation Control Shader Epilog");
7659 return shader->epilog != NULL;
7660 }
7661
7662 /**
7663 * Select and compile (or reuse) GS parts (prolog).
7664 */
7665 static bool si_shader_select_gs_parts(struct si_screen *sscreen,
7666 struct ac_llvm_compiler *compiler,
7667 struct si_shader *shader,
7668 struct pipe_debug_callback *debug)
7669 {
7670 if (sscreen->info.chip_class >= GFX9) {
7671 struct si_shader *es_main_part;
7672 enum pipe_shader_type es_type = shader->key.part.gs.es->type;
7673
7674 if (shader->key.as_ngg)
7675 es_main_part = shader->key.part.gs.es->main_shader_part_ngg_es;
7676 else
7677 es_main_part = shader->key.part.gs.es->main_shader_part_es;
7678
7679 if (es_type == PIPE_SHADER_VERTEX &&
7680 !si_get_vs_prolog(sscreen, compiler, shader, debug, es_main_part,
7681 &shader->key.part.gs.vs_prolog))
7682 return false;
7683
7684 shader->previous_stage = es_main_part;
7685 }
7686
7687 if (!shader->key.part.gs.prolog.tri_strip_adj_fix)
7688 return true;
7689
7690 union si_shader_part_key prolog_key;
7691 memset(&prolog_key, 0, sizeof(prolog_key));
7692 prolog_key.gs_prolog.states = shader->key.part.gs.prolog;
7693 prolog_key.gs_prolog.as_ngg = shader->key.as_ngg;
7694
7695 shader->prolog2 = si_get_shader_part(sscreen, &sscreen->gs_prologs,
7696 PIPE_SHADER_GEOMETRY, true,
7697 &prolog_key, compiler, debug,
7698 si_build_gs_prolog_function,
7699 "Geometry Shader Prolog");
7700 return shader->prolog2 != NULL;
7701 }
7702
7703 /**
7704 * Build the pixel shader prolog function. This handles:
7705 * - two-side color selection and interpolation
7706 * - overriding interpolation parameters for the API PS
7707 * - polygon stippling
7708 *
7709 * All preloaded SGPRs and VGPRs are passed through unmodified unless they are
7710 * overriden by other states. (e.g. per-sample interpolation)
7711 * Interpolated colors are stored after the preloaded VGPRs.
7712 */
7713 static void si_build_ps_prolog_function(struct si_shader_context *ctx,
7714 union si_shader_part_key *key)
7715 {
7716 LLVMValueRef ret, func;
7717 int num_returns, i, num_color_channels;
7718
7719 assert(si_need_ps_prolog(key));
7720
7721 memset(&ctx->args, 0, sizeof(ctx->args));
7722
7723 /* Declare inputs. */
7724 LLVMTypeRef return_types[AC_MAX_ARGS];
7725 num_returns = 0;
7726 num_color_channels = util_bitcount(key->ps_prolog.colors_read);
7727 assert(key->ps_prolog.num_input_sgprs +
7728 key->ps_prolog.num_input_vgprs +
7729 num_color_channels <= AC_MAX_ARGS);
7730 for (i = 0; i < key->ps_prolog.num_input_sgprs; i++) {
7731 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, NULL);
7732 return_types[num_returns++] = ctx->i32;
7733
7734 }
7735
7736 struct ac_arg pos_fixed_pt;
7737 struct ac_arg ancillary;
7738 struct ac_arg param_sample_mask;
7739 for (i = 0; i < key->ps_prolog.num_input_vgprs; i++) {
7740 struct ac_arg *arg = NULL;
7741 if (i == key->ps_prolog.ancillary_vgpr_index) {
7742 arg = &ancillary;
7743 } else if (i == key->ps_prolog.ancillary_vgpr_index + 1) {
7744 arg = &param_sample_mask;
7745 } else if (i == key->ps_prolog.num_input_vgprs - 1) {
7746 /* POS_FIXED_PT is always last. */
7747 arg = &pos_fixed_pt;
7748 }
7749 ac_add_arg(&ctx->args, AC_ARG_VGPR, 1, AC_ARG_FLOAT, arg);
7750 return_types[num_returns++] = ctx->f32;
7751 }
7752
7753 /* Declare outputs (same as inputs + add colors if needed) */
7754 for (i = 0; i < num_color_channels; i++)
7755 return_types[num_returns++] = ctx->f32;
7756
7757 /* Create the function. */
7758 si_create_function(ctx, "ps_prolog", return_types, num_returns, 0);
7759 func = ctx->main_fn;
7760
7761 /* Copy inputs to outputs. This should be no-op, as the registers match,
7762 * but it will prevent the compiler from overwriting them unintentionally.
7763 */
7764 ret = ctx->return_value;
7765 for (i = 0; i < ctx->args.arg_count; i++) {
7766 LLVMValueRef p = LLVMGetParam(func, i);
7767 ret = LLVMBuildInsertValue(ctx->ac.builder, ret, p, i, "");
7768 }
7769
7770 /* Polygon stippling. */
7771 if (key->ps_prolog.states.poly_stipple) {
7772 LLVMValueRef list = si_prolog_get_rw_buffers(ctx);
7773
7774 si_llvm_emit_polygon_stipple(ctx, list, pos_fixed_pt);
7775 }
7776
7777 if (key->ps_prolog.states.bc_optimize_for_persp ||
7778 key->ps_prolog.states.bc_optimize_for_linear) {
7779 unsigned i, base = key->ps_prolog.num_input_sgprs;
7780 LLVMValueRef center[2], centroid[2], tmp, bc_optimize;
7781
7782 /* The shader should do: if (PRIM_MASK[31]) CENTROID = CENTER;
7783 * The hw doesn't compute CENTROID if the whole wave only
7784 * contains fully-covered quads.
7785 *
7786 * PRIM_MASK is after user SGPRs.
7787 */
7788 bc_optimize = LLVMGetParam(func, SI_PS_NUM_USER_SGPR);
7789 bc_optimize = LLVMBuildLShr(ctx->ac.builder, bc_optimize,
7790 LLVMConstInt(ctx->i32, 31, 0), "");
7791 bc_optimize = LLVMBuildTrunc(ctx->ac.builder, bc_optimize,
7792 ctx->i1, "");
7793
7794 if (key->ps_prolog.states.bc_optimize_for_persp) {
7795 /* Read PERSP_CENTER. */
7796 for (i = 0; i < 2; i++)
7797 center[i] = LLVMGetParam(func, base + 2 + i);
7798 /* Read PERSP_CENTROID. */
7799 for (i = 0; i < 2; i++)
7800 centroid[i] = LLVMGetParam(func, base + 4 + i);
7801 /* Select PERSP_CENTROID. */
7802 for (i = 0; i < 2; i++) {
7803 tmp = LLVMBuildSelect(ctx->ac.builder, bc_optimize,
7804 center[i], centroid[i], "");
7805 ret = LLVMBuildInsertValue(ctx->ac.builder, ret,
7806 tmp, base + 4 + i, "");
7807 }
7808 }
7809 if (key->ps_prolog.states.bc_optimize_for_linear) {
7810 /* Read LINEAR_CENTER. */
7811 for (i = 0; i < 2; i++)
7812 center[i] = LLVMGetParam(func, base + 8 + i);
7813 /* Read LINEAR_CENTROID. */
7814 for (i = 0; i < 2; i++)
7815 centroid[i] = LLVMGetParam(func, base + 10 + i);
7816 /* Select LINEAR_CENTROID. */
7817 for (i = 0; i < 2; i++) {
7818 tmp = LLVMBuildSelect(ctx->ac.builder, bc_optimize,
7819 center[i], centroid[i], "");
7820 ret = LLVMBuildInsertValue(ctx->ac.builder, ret,
7821 tmp, base + 10 + i, "");
7822 }
7823 }
7824 }
7825
7826 /* Force per-sample interpolation. */
7827 if (key->ps_prolog.states.force_persp_sample_interp) {
7828 unsigned i, base = key->ps_prolog.num_input_sgprs;
7829 LLVMValueRef persp_sample[2];
7830
7831 /* Read PERSP_SAMPLE. */
7832 for (i = 0; i < 2; i++)
7833 persp_sample[i] = LLVMGetParam(func, base + i);
7834 /* Overwrite PERSP_CENTER. */
7835 for (i = 0; i < 2; i++)
7836 ret = LLVMBuildInsertValue(ctx->ac.builder, ret,
7837 persp_sample[i], base + 2 + i, "");
7838 /* Overwrite PERSP_CENTROID. */
7839 for (i = 0; i < 2; i++)
7840 ret = LLVMBuildInsertValue(ctx->ac.builder, ret,
7841 persp_sample[i], base + 4 + i, "");
7842 }
7843 if (key->ps_prolog.states.force_linear_sample_interp) {
7844 unsigned i, base = key->ps_prolog.num_input_sgprs;
7845 LLVMValueRef linear_sample[2];
7846
7847 /* Read LINEAR_SAMPLE. */
7848 for (i = 0; i < 2; i++)
7849 linear_sample[i] = LLVMGetParam(func, base + 6 + i);
7850 /* Overwrite LINEAR_CENTER. */
7851 for (i = 0; i < 2; i++)
7852 ret = LLVMBuildInsertValue(ctx->ac.builder, ret,
7853 linear_sample[i], base + 8 + i, "");
7854 /* Overwrite LINEAR_CENTROID. */
7855 for (i = 0; i < 2; i++)
7856 ret = LLVMBuildInsertValue(ctx->ac.builder, ret,
7857 linear_sample[i], base + 10 + i, "");
7858 }
7859
7860 /* Force center interpolation. */
7861 if (key->ps_prolog.states.force_persp_center_interp) {
7862 unsigned i, base = key->ps_prolog.num_input_sgprs;
7863 LLVMValueRef persp_center[2];
7864
7865 /* Read PERSP_CENTER. */
7866 for (i = 0; i < 2; i++)
7867 persp_center[i] = LLVMGetParam(func, base + 2 + i);
7868 /* Overwrite PERSP_SAMPLE. */
7869 for (i = 0; i < 2; i++)
7870 ret = LLVMBuildInsertValue(ctx->ac.builder, ret,
7871 persp_center[i], base + i, "");
7872 /* Overwrite PERSP_CENTROID. */
7873 for (i = 0; i < 2; i++)
7874 ret = LLVMBuildInsertValue(ctx->ac.builder, ret,
7875 persp_center[i], base + 4 + i, "");
7876 }
7877 if (key->ps_prolog.states.force_linear_center_interp) {
7878 unsigned i, base = key->ps_prolog.num_input_sgprs;
7879 LLVMValueRef linear_center[2];
7880
7881 /* Read LINEAR_CENTER. */
7882 for (i = 0; i < 2; i++)
7883 linear_center[i] = LLVMGetParam(func, base + 8 + i);
7884 /* Overwrite LINEAR_SAMPLE. */
7885 for (i = 0; i < 2; i++)
7886 ret = LLVMBuildInsertValue(ctx->ac.builder, ret,
7887 linear_center[i], base + 6 + i, "");
7888 /* Overwrite LINEAR_CENTROID. */
7889 for (i = 0; i < 2; i++)
7890 ret = LLVMBuildInsertValue(ctx->ac.builder, ret,
7891 linear_center[i], base + 10 + i, "");
7892 }
7893
7894 /* Interpolate colors. */
7895 unsigned color_out_idx = 0;
7896 for (i = 0; i < 2; i++) {
7897 unsigned writemask = (key->ps_prolog.colors_read >> (i * 4)) & 0xf;
7898 unsigned face_vgpr = key->ps_prolog.num_input_sgprs +
7899 key->ps_prolog.face_vgpr_index;
7900 LLVMValueRef interp[2], color[4];
7901 LLVMValueRef interp_ij = NULL, prim_mask = NULL, face = NULL;
7902
7903 if (!writemask)
7904 continue;
7905
7906 /* If the interpolation qualifier is not CONSTANT (-1). */
7907 if (key->ps_prolog.color_interp_vgpr_index[i] != -1) {
7908 unsigned interp_vgpr = key->ps_prolog.num_input_sgprs +
7909 key->ps_prolog.color_interp_vgpr_index[i];
7910
7911 /* Get the (i,j) updated by bc_optimize handling. */
7912 interp[0] = LLVMBuildExtractValue(ctx->ac.builder, ret,
7913 interp_vgpr, "");
7914 interp[1] = LLVMBuildExtractValue(ctx->ac.builder, ret,
7915 interp_vgpr + 1, "");
7916 interp_ij = ac_build_gather_values(&ctx->ac, interp, 2);
7917 }
7918
7919 /* Use the absolute location of the input. */
7920 prim_mask = LLVMGetParam(func, SI_PS_NUM_USER_SGPR);
7921
7922 if (key->ps_prolog.states.color_two_side) {
7923 face = LLVMGetParam(func, face_vgpr);
7924 face = ac_to_integer(&ctx->ac, face);
7925 }
7926
7927 interp_fs_input(ctx,
7928 key->ps_prolog.color_attr_index[i],
7929 TGSI_SEMANTIC_COLOR, i,
7930 key->ps_prolog.num_interp_inputs,
7931 key->ps_prolog.colors_read, interp_ij,
7932 prim_mask, face, color);
7933
7934 while (writemask) {
7935 unsigned chan = u_bit_scan(&writemask);
7936 ret = LLVMBuildInsertValue(ctx->ac.builder, ret, color[chan],
7937 ctx->args.arg_count + color_out_idx++, "");
7938 }
7939 }
7940
7941 /* Section 15.2.2 (Shader Inputs) of the OpenGL 4.5 (Core Profile) spec
7942 * says:
7943 *
7944 * "When per-sample shading is active due to the use of a fragment
7945 * input qualified by sample or due to the use of the gl_SampleID
7946 * or gl_SamplePosition variables, only the bit for the current
7947 * sample is set in gl_SampleMaskIn. When state specifies multiple
7948 * fragment shader invocations for a given fragment, the sample
7949 * mask for any single fragment shader invocation may specify a
7950 * subset of the covered samples for the fragment. In this case,
7951 * the bit corresponding to each covered sample will be set in
7952 * exactly one fragment shader invocation."
7953 *
7954 * The samplemask loaded by hardware is always the coverage of the
7955 * entire pixel/fragment, so mask bits out based on the sample ID.
7956 */
7957 if (key->ps_prolog.states.samplemask_log_ps_iter) {
7958 /* The bit pattern matches that used by fixed function fragment
7959 * processing. */
7960 static const uint16_t ps_iter_masks[] = {
7961 0xffff, /* not used */
7962 0x5555,
7963 0x1111,
7964 0x0101,
7965 0x0001,
7966 };
7967 assert(key->ps_prolog.states.samplemask_log_ps_iter < ARRAY_SIZE(ps_iter_masks));
7968
7969 uint32_t ps_iter_mask = ps_iter_masks[key->ps_prolog.states.samplemask_log_ps_iter];
7970 LLVMValueRef sampleid = si_unpack_param(ctx, ancillary, 8, 4);
7971 LLVMValueRef samplemask = ac_get_arg(&ctx->ac, param_sample_mask);
7972
7973 samplemask = ac_to_integer(&ctx->ac, samplemask);
7974 samplemask = LLVMBuildAnd(
7975 ctx->ac.builder,
7976 samplemask,
7977 LLVMBuildShl(ctx->ac.builder,
7978 LLVMConstInt(ctx->i32, ps_iter_mask, false),
7979 sampleid, ""),
7980 "");
7981 samplemask = ac_to_float(&ctx->ac, samplemask);
7982
7983 ret = LLVMBuildInsertValue(ctx->ac.builder, ret, samplemask,
7984 param_sample_mask.arg_index, "");
7985 }
7986
7987 /* Tell LLVM to insert WQM instruction sequence when needed. */
7988 if (key->ps_prolog.wqm) {
7989 LLVMAddTargetDependentFunctionAttr(func,
7990 "amdgpu-ps-wqm-outputs", "");
7991 }
7992
7993 si_llvm_build_ret(ctx, ret);
7994 }
7995
7996 /**
7997 * Build the pixel shader epilog function. This handles everything that must be
7998 * emulated for pixel shader exports. (alpha-test, format conversions, etc)
7999 */
8000 static void si_build_ps_epilog_function(struct si_shader_context *ctx,
8001 union si_shader_part_key *key)
8002 {
8003 struct lp_build_tgsi_context *bld_base = &ctx->bld_base;
8004 LLVMValueRef depth = NULL, stencil = NULL, samplemask = NULL;
8005 int i;
8006 struct si_ps_exports exp = {};
8007
8008 memset(&ctx->args, 0, sizeof(ctx->args));
8009
8010 /* Declare input SGPRs. */
8011 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, &ctx->rw_buffers);
8012 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT,
8013 &ctx->bindless_samplers_and_images);
8014 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT,
8015 &ctx->const_and_shader_buffers);
8016 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT,
8017 &ctx->samplers_and_images);
8018 add_arg_checked(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_FLOAT,
8019 NULL, SI_PARAM_ALPHA_REF);
8020
8021 /* Declare input VGPRs. */
8022 unsigned required_num_params =
8023 ctx->args.num_sgprs_used +
8024 util_bitcount(key->ps_epilog.colors_written) * 4 +
8025 key->ps_epilog.writes_z +
8026 key->ps_epilog.writes_stencil +
8027 key->ps_epilog.writes_samplemask;
8028
8029 required_num_params = MAX2(required_num_params,
8030 ctx->args.num_sgprs_used + PS_EPILOG_SAMPLEMASK_MIN_LOC + 1);
8031
8032 while (ctx->args.arg_count < required_num_params)
8033 ac_add_arg(&ctx->args, AC_ARG_VGPR, 1, AC_ARG_FLOAT, NULL);
8034
8035 /* Create the function. */
8036 si_create_function(ctx, "ps_epilog", NULL, 0, 0);
8037 /* Disable elimination of unused inputs. */
8038 ac_llvm_add_target_dep_function_attr(ctx->main_fn,
8039 "InitialPSInputAddr", 0xffffff);
8040
8041 /* Process colors. */
8042 unsigned vgpr = ctx->args.num_sgprs_used;
8043 unsigned colors_written = key->ps_epilog.colors_written;
8044 int last_color_export = -1;
8045
8046 /* Find the last color export. */
8047 if (!key->ps_epilog.writes_z &&
8048 !key->ps_epilog.writes_stencil &&
8049 !key->ps_epilog.writes_samplemask) {
8050 unsigned spi_format = key->ps_epilog.states.spi_shader_col_format;
8051
8052 /* If last_cbuf > 0, FS_COLOR0_WRITES_ALL_CBUFS is true. */
8053 if (colors_written == 0x1 && key->ps_epilog.states.last_cbuf > 0) {
8054 /* Just set this if any of the colorbuffers are enabled. */
8055 if (spi_format &
8056 ((1ull << (4 * (key->ps_epilog.states.last_cbuf + 1))) - 1))
8057 last_color_export = 0;
8058 } else {
8059 for (i = 0; i < 8; i++)
8060 if (colors_written & (1 << i) &&
8061 (spi_format >> (i * 4)) & 0xf)
8062 last_color_export = i;
8063 }
8064 }
8065
8066 while (colors_written) {
8067 LLVMValueRef color[4];
8068 int mrt = u_bit_scan(&colors_written);
8069
8070 for (i = 0; i < 4; i++)
8071 color[i] = LLVMGetParam(ctx->main_fn, vgpr++);
8072
8073 si_export_mrt_color(bld_base, color, mrt,
8074 ctx->args.arg_count - 1,
8075 mrt == last_color_export, &exp);
8076 }
8077
8078 /* Process depth, stencil, samplemask. */
8079 if (key->ps_epilog.writes_z)
8080 depth = LLVMGetParam(ctx->main_fn, vgpr++);
8081 if (key->ps_epilog.writes_stencil)
8082 stencil = LLVMGetParam(ctx->main_fn, vgpr++);
8083 if (key->ps_epilog.writes_samplemask)
8084 samplemask = LLVMGetParam(ctx->main_fn, vgpr++);
8085
8086 if (depth || stencil || samplemask)
8087 si_export_mrt_z(bld_base, depth, stencil, samplemask, &exp);
8088 else if (last_color_export == -1)
8089 ac_build_export_null(&ctx->ac);
8090
8091 if (exp.num)
8092 si_emit_ps_exports(ctx, &exp);
8093
8094 /* Compile. */
8095 LLVMBuildRetVoid(ctx->ac.builder);
8096 }
8097
8098 /**
8099 * Select and compile (or reuse) pixel shader parts (prolog & epilog).
8100 */
8101 static bool si_shader_select_ps_parts(struct si_screen *sscreen,
8102 struct ac_llvm_compiler *compiler,
8103 struct si_shader *shader,
8104 struct pipe_debug_callback *debug)
8105 {
8106 union si_shader_part_key prolog_key;
8107 union si_shader_part_key epilog_key;
8108
8109 /* Get the prolog. */
8110 si_get_ps_prolog_key(shader, &prolog_key, true);
8111
8112 /* The prolog is a no-op if these aren't set. */
8113 if (si_need_ps_prolog(&prolog_key)) {
8114 shader->prolog =
8115 si_get_shader_part(sscreen, &sscreen->ps_prologs,
8116 PIPE_SHADER_FRAGMENT, true,
8117 &prolog_key, compiler, debug,
8118 si_build_ps_prolog_function,
8119 "Fragment Shader Prolog");
8120 if (!shader->prolog)
8121 return false;
8122 }
8123
8124 /* Get the epilog. */
8125 si_get_ps_epilog_key(shader, &epilog_key);
8126
8127 shader->epilog =
8128 si_get_shader_part(sscreen, &sscreen->ps_epilogs,
8129 PIPE_SHADER_FRAGMENT, false,
8130 &epilog_key, compiler, debug,
8131 si_build_ps_epilog_function,
8132 "Fragment Shader Epilog");
8133 if (!shader->epilog)
8134 return false;
8135
8136 /* Enable POS_FIXED_PT if polygon stippling is enabled. */
8137 if (shader->key.part.ps.prolog.poly_stipple) {
8138 shader->config.spi_ps_input_ena |= S_0286CC_POS_FIXED_PT_ENA(1);
8139 assert(G_0286CC_POS_FIXED_PT_ENA(shader->config.spi_ps_input_addr));
8140 }
8141
8142 /* Set up the enable bits for per-sample shading if needed. */
8143 if (shader->key.part.ps.prolog.force_persp_sample_interp &&
8144 (G_0286CC_PERSP_CENTER_ENA(shader->config.spi_ps_input_ena) ||
8145 G_0286CC_PERSP_CENTROID_ENA(shader->config.spi_ps_input_ena))) {
8146 shader->config.spi_ps_input_ena &= C_0286CC_PERSP_CENTER_ENA;
8147 shader->config.spi_ps_input_ena &= C_0286CC_PERSP_CENTROID_ENA;
8148 shader->config.spi_ps_input_ena |= S_0286CC_PERSP_SAMPLE_ENA(1);
8149 }
8150 if (shader->key.part.ps.prolog.force_linear_sample_interp &&
8151 (G_0286CC_LINEAR_CENTER_ENA(shader->config.spi_ps_input_ena) ||
8152 G_0286CC_LINEAR_CENTROID_ENA(shader->config.spi_ps_input_ena))) {
8153 shader->config.spi_ps_input_ena &= C_0286CC_LINEAR_CENTER_ENA;
8154 shader->config.spi_ps_input_ena &= C_0286CC_LINEAR_CENTROID_ENA;
8155 shader->config.spi_ps_input_ena |= S_0286CC_LINEAR_SAMPLE_ENA(1);
8156 }
8157 if (shader->key.part.ps.prolog.force_persp_center_interp &&
8158 (G_0286CC_PERSP_SAMPLE_ENA(shader->config.spi_ps_input_ena) ||
8159 G_0286CC_PERSP_CENTROID_ENA(shader->config.spi_ps_input_ena))) {
8160 shader->config.spi_ps_input_ena &= C_0286CC_PERSP_SAMPLE_ENA;
8161 shader->config.spi_ps_input_ena &= C_0286CC_PERSP_CENTROID_ENA;
8162 shader->config.spi_ps_input_ena |= S_0286CC_PERSP_CENTER_ENA(1);
8163 }
8164 if (shader->key.part.ps.prolog.force_linear_center_interp &&
8165 (G_0286CC_LINEAR_SAMPLE_ENA(shader->config.spi_ps_input_ena) ||
8166 G_0286CC_LINEAR_CENTROID_ENA(shader->config.spi_ps_input_ena))) {
8167 shader->config.spi_ps_input_ena &= C_0286CC_LINEAR_SAMPLE_ENA;
8168 shader->config.spi_ps_input_ena &= C_0286CC_LINEAR_CENTROID_ENA;
8169 shader->config.spi_ps_input_ena |= S_0286CC_LINEAR_CENTER_ENA(1);
8170 }
8171
8172 /* POW_W_FLOAT requires that one of the perspective weights is enabled. */
8173 if (G_0286CC_POS_W_FLOAT_ENA(shader->config.spi_ps_input_ena) &&
8174 !(shader->config.spi_ps_input_ena & 0xf)) {
8175 shader->config.spi_ps_input_ena |= S_0286CC_PERSP_CENTER_ENA(1);
8176 assert(G_0286CC_PERSP_CENTER_ENA(shader->config.spi_ps_input_addr));
8177 }
8178
8179 /* At least one pair of interpolation weights must be enabled. */
8180 if (!(shader->config.spi_ps_input_ena & 0x7f)) {
8181 shader->config.spi_ps_input_ena |= S_0286CC_LINEAR_CENTER_ENA(1);
8182 assert(G_0286CC_LINEAR_CENTER_ENA(shader->config.spi_ps_input_addr));
8183 }
8184
8185 /* Samplemask fixup requires the sample ID. */
8186 if (shader->key.part.ps.prolog.samplemask_log_ps_iter) {
8187 shader->config.spi_ps_input_ena |= S_0286CC_ANCILLARY_ENA(1);
8188 assert(G_0286CC_ANCILLARY_ENA(shader->config.spi_ps_input_addr));
8189 }
8190
8191 /* The sample mask input is always enabled, because the API shader always
8192 * passes it through to the epilog. Disable it here if it's unused.
8193 */
8194 if (!shader->key.part.ps.epilog.poly_line_smoothing &&
8195 !shader->selector->info.reads_samplemask)
8196 shader->config.spi_ps_input_ena &= C_0286CC_SAMPLE_COVERAGE_ENA;
8197
8198 return true;
8199 }
8200
8201 void si_multiwave_lds_size_workaround(struct si_screen *sscreen,
8202 unsigned *lds_size)
8203 {
8204 /* If tessellation is all offchip and on-chip GS isn't used, this
8205 * workaround is not needed.
8206 */
8207 return;
8208
8209 /* SPI barrier management bug:
8210 * Make sure we have at least 4k of LDS in use to avoid the bug.
8211 * It applies to workgroup sizes of more than one wavefront.
8212 */
8213 if (sscreen->info.family == CHIP_BONAIRE ||
8214 sscreen->info.family == CHIP_KABINI)
8215 *lds_size = MAX2(*lds_size, 8);
8216 }
8217
8218 static void si_fix_resource_usage(struct si_screen *sscreen,
8219 struct si_shader *shader)
8220 {
8221 unsigned min_sgprs = shader->info.num_input_sgprs + 2; /* VCC */
8222
8223 shader->config.num_sgprs = MAX2(shader->config.num_sgprs, min_sgprs);
8224
8225 if (shader->selector->type == PIPE_SHADER_COMPUTE &&
8226 si_get_max_workgroup_size(shader) > sscreen->compute_wave_size) {
8227 si_multiwave_lds_size_workaround(sscreen,
8228 &shader->config.lds_size);
8229 }
8230 }
8231
8232 bool si_shader_create(struct si_screen *sscreen, struct ac_llvm_compiler *compiler,
8233 struct si_shader *shader,
8234 struct pipe_debug_callback *debug)
8235 {
8236 struct si_shader_selector *sel = shader->selector;
8237 struct si_shader *mainp = *si_get_main_shader_part(sel, &shader->key);
8238 int r;
8239
8240 /* LS, ES, VS are compiled on demand if the main part hasn't been
8241 * compiled for that stage.
8242 *
8243 * GS are compiled on demand if the main part hasn't been compiled
8244 * for the chosen NGG-ness.
8245 *
8246 * Vertex shaders are compiled on demand when a vertex fetch
8247 * workaround must be applied.
8248 */
8249 if (shader->is_monolithic) {
8250 /* Monolithic shader (compiled as a whole, has many variants,
8251 * may take a long time to compile).
8252 */
8253 r = si_compile_tgsi_shader(sscreen, compiler, shader, debug);
8254 if (r)
8255 return false;
8256 } else {
8257 /* The shader consists of several parts:
8258 *
8259 * - the middle part is the user shader, it has 1 variant only
8260 * and it was compiled during the creation of the shader
8261 * selector
8262 * - the prolog part is inserted at the beginning
8263 * - the epilog part is inserted at the end
8264 *
8265 * The prolog and epilog have many (but simple) variants.
8266 *
8267 * Starting with gfx9, geometry and tessellation control
8268 * shaders also contain the prolog and user shader parts of
8269 * the previous shader stage.
8270 */
8271
8272 if (!mainp)
8273 return false;
8274
8275 /* Copy the compiled TGSI shader data over. */
8276 shader->is_binary_shared = true;
8277 shader->binary = mainp->binary;
8278 shader->config = mainp->config;
8279 shader->info.num_input_sgprs = mainp->info.num_input_sgprs;
8280 shader->info.num_input_vgprs = mainp->info.num_input_vgprs;
8281 shader->info.face_vgpr_index = mainp->info.face_vgpr_index;
8282 shader->info.ancillary_vgpr_index = mainp->info.ancillary_vgpr_index;
8283 memcpy(shader->info.vs_output_param_offset,
8284 mainp->info.vs_output_param_offset,
8285 sizeof(mainp->info.vs_output_param_offset));
8286 shader->info.uses_instanceid = mainp->info.uses_instanceid;
8287 shader->info.nr_pos_exports = mainp->info.nr_pos_exports;
8288 shader->info.nr_param_exports = mainp->info.nr_param_exports;
8289
8290 /* Select prologs and/or epilogs. */
8291 switch (sel->type) {
8292 case PIPE_SHADER_VERTEX:
8293 if (!si_shader_select_vs_parts(sscreen, compiler, shader, debug))
8294 return false;
8295 break;
8296 case PIPE_SHADER_TESS_CTRL:
8297 if (!si_shader_select_tcs_parts(sscreen, compiler, shader, debug))
8298 return false;
8299 break;
8300 case PIPE_SHADER_TESS_EVAL:
8301 break;
8302 case PIPE_SHADER_GEOMETRY:
8303 if (!si_shader_select_gs_parts(sscreen, compiler, shader, debug))
8304 return false;
8305 break;
8306 case PIPE_SHADER_FRAGMENT:
8307 if (!si_shader_select_ps_parts(sscreen, compiler, shader, debug))
8308 return false;
8309
8310 /* Make sure we have at least as many VGPRs as there
8311 * are allocated inputs.
8312 */
8313 shader->config.num_vgprs = MAX2(shader->config.num_vgprs,
8314 shader->info.num_input_vgprs);
8315 break;
8316 default:;
8317 }
8318
8319 /* Update SGPR and VGPR counts. */
8320 if (shader->prolog) {
8321 shader->config.num_sgprs = MAX2(shader->config.num_sgprs,
8322 shader->prolog->config.num_sgprs);
8323 shader->config.num_vgprs = MAX2(shader->config.num_vgprs,
8324 shader->prolog->config.num_vgprs);
8325 }
8326 if (shader->previous_stage) {
8327 shader->config.num_sgprs = MAX2(shader->config.num_sgprs,
8328 shader->previous_stage->config.num_sgprs);
8329 shader->config.num_vgprs = MAX2(shader->config.num_vgprs,
8330 shader->previous_stage->config.num_vgprs);
8331 shader->config.spilled_sgprs =
8332 MAX2(shader->config.spilled_sgprs,
8333 shader->previous_stage->config.spilled_sgprs);
8334 shader->config.spilled_vgprs =
8335 MAX2(shader->config.spilled_vgprs,
8336 shader->previous_stage->config.spilled_vgprs);
8337 shader->info.private_mem_vgprs =
8338 MAX2(shader->info.private_mem_vgprs,
8339 shader->previous_stage->info.private_mem_vgprs);
8340 shader->config.scratch_bytes_per_wave =
8341 MAX2(shader->config.scratch_bytes_per_wave,
8342 shader->previous_stage->config.scratch_bytes_per_wave);
8343 shader->info.uses_instanceid |=
8344 shader->previous_stage->info.uses_instanceid;
8345 }
8346 if (shader->prolog2) {
8347 shader->config.num_sgprs = MAX2(shader->config.num_sgprs,
8348 shader->prolog2->config.num_sgprs);
8349 shader->config.num_vgprs = MAX2(shader->config.num_vgprs,
8350 shader->prolog2->config.num_vgprs);
8351 }
8352 if (shader->epilog) {
8353 shader->config.num_sgprs = MAX2(shader->config.num_sgprs,
8354 shader->epilog->config.num_sgprs);
8355 shader->config.num_vgprs = MAX2(shader->config.num_vgprs,
8356 shader->epilog->config.num_vgprs);
8357 }
8358 si_calculate_max_simd_waves(shader);
8359 }
8360
8361 if (shader->key.as_ngg) {
8362 assert(!shader->key.as_es && !shader->key.as_ls);
8363 gfx10_ngg_calculate_subgroup_info(shader);
8364 } else if (sscreen->info.chip_class >= GFX9 && sel->type == PIPE_SHADER_GEOMETRY) {
8365 gfx9_get_gs_info(shader->previous_stage_sel, sel, &shader->gs_info);
8366 }
8367
8368 si_fix_resource_usage(sscreen, shader);
8369 si_shader_dump(sscreen, shader, debug, stderr, true);
8370
8371 /* Upload. */
8372 if (!si_shader_binary_upload(sscreen, shader, 0)) {
8373 fprintf(stderr, "LLVM failed to upload shader\n");
8374 return false;
8375 }
8376
8377 return true;
8378 }
8379
8380 void si_shader_destroy(struct si_shader *shader)
8381 {
8382 if (shader->scratch_bo)
8383 si_resource_reference(&shader->scratch_bo, NULL);
8384
8385 si_resource_reference(&shader->bo, NULL);
8386
8387 if (!shader->is_binary_shared)
8388 si_shader_binary_clean(&shader->binary);
8389
8390 free(shader->shader_log);
8391 }