projects / mesa.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
history | raw | HEAD
radeonsi: fix color inputs/outputs for GS and tess
[mesa.git] / src / gallium / drivers / radeonsi / si_shader.c
1 /*
2 * Copyright 2012 Advanced Micro Devices, Inc.
3 * All Rights Reserved.
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * on the rights to use, copy, modify, merge, publish, distribute, sub
9 * license, and/or sell copies of the Software, and to permit persons to whom
10 * the Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice (including the next
13 * paragraph) shall be included in all copies or substantial portions of the
14 * Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
20 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
21 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
22 * USE OR OTHER DEALINGS IN THE SOFTWARE.
23 */
24
25 #include "gallivm/lp_bld_const.h"
26 #include "gallivm/lp_bld_gather.h"
27 #include "gallivm/lp_bld_intr.h"
28 #include "gallivm/lp_bld_logic.h"
29 #include "gallivm/lp_bld_arit.h"
30 #include "gallivm/lp_bld_flow.h"
31 #include "gallivm/lp_bld_misc.h"
32 #include "util/u_memory.h"
33 #include "util/u_string.h"
34 #include "tgsi/tgsi_build.h"
35 #include "tgsi/tgsi_util.h"
36 #include "tgsi/tgsi_dump.h"
37
38 #include "ac_binary.h"
39 #include "ac_llvm_util.h"
40 #include "ac_exp_param.h"
41 #include "ac_shader_util.h"
42 #include "si_shader_internal.h"
43 #include "si_pipe.h"
44 #include "sid.h"
45
46 #include "compiler/nir/nir.h"
47
48 static const char *scratch_rsrc_dword0_symbol =
49 "SCRATCH_RSRC_DWORD0";
50
51 static const char *scratch_rsrc_dword1_symbol =
52 "SCRATCH_RSRC_DWORD1";
53
54 struct si_shader_output_values
55 {
56 LLVMValueRef values[4];
57 unsigned semantic_name;
58 unsigned semantic_index;
59 ubyte vertex_stream[4];
60 };
61
62 /**
63 * Used to collect types and other info about arguments of the LLVM function
64 * before the function is created.
65 */
66 struct si_function_info {
67 LLVMTypeRef types[100];
68 LLVMValueRef *assign[100];
69 unsigned num_sgpr_params;
70 unsigned num_params;
71 };
72
73 enum si_arg_regfile {
74 ARG_SGPR,
75 ARG_VGPR
76 };
77
78 static void si_init_shader_ctx(struct si_shader_context *ctx,
79 struct si_screen *sscreen,
80 struct si_compiler *compiler);
81
82 static void si_llvm_emit_barrier(const struct lp_build_tgsi_action *action,
83 struct lp_build_tgsi_context *bld_base,
84 struct lp_build_emit_data *emit_data);
85
86 static void si_dump_shader_key(unsigned processor, const struct si_shader *shader,
87 FILE *f);
88
89 static void si_build_vs_prolog_function(struct si_shader_context *ctx,
90 union si_shader_part_key *key);
91 static void si_build_tcs_epilog_function(struct si_shader_context *ctx,
92 union si_shader_part_key *key);
93 static void si_build_ps_prolog_function(struct si_shader_context *ctx,
94 union si_shader_part_key *key);
95 static void si_build_ps_epilog_function(struct si_shader_context *ctx,
96 union si_shader_part_key *key);
97
98 /* Ideally pass the sample mask input to the PS epilog as v14, which
99 * is its usual location, so that the shader doesn't have to add v_mov.
100 */
101 #define PS_EPILOG_SAMPLEMASK_MIN_LOC 14
102
103 static bool llvm_type_is_64bit(struct si_shader_context *ctx,
104 LLVMTypeRef type)
105 {
106 if (type == ctx->ac.i64 || type == ctx->ac.f64)
107 return true;
108
109 return false;
110 }
111
112 static bool is_merged_shader(struct si_shader *shader)
113 {
114 if (shader->selector->screen->info.chip_class <= VI)
115 return false;
116
117 return shader->key.as_ls ||
118 shader->key.as_es ||
119 shader->selector->type == PIPE_SHADER_TESS_CTRL ||
120 shader->selector->type == PIPE_SHADER_GEOMETRY;
121 }
122
123 static void si_init_function_info(struct si_function_info *fninfo)
124 {
125 fninfo->num_params = 0;
126 fninfo->num_sgpr_params = 0;
127 }
128
129 static unsigned add_arg_assign(struct si_function_info *fninfo,
130 enum si_arg_regfile regfile, LLVMTypeRef type,
131 LLVMValueRef *assign)
132 {
133 assert(regfile != ARG_SGPR || fninfo->num_sgpr_params == fninfo->num_params);
134
135 unsigned idx = fninfo->num_params++;
136 assert(idx < ARRAY_SIZE(fninfo->types));
137
138 if (regfile == ARG_SGPR)
139 fninfo->num_sgpr_params = fninfo->num_params;
140
141 fninfo->types[idx] = type;
142 fninfo->assign[idx] = assign;
143 return idx;
144 }
145
146 static unsigned add_arg(struct si_function_info *fninfo,
147 enum si_arg_regfile regfile, LLVMTypeRef type)
148 {
149 return add_arg_assign(fninfo, regfile, type, NULL);
150 }
151
152 static void add_arg_assign_checked(struct si_function_info *fninfo,
153 enum si_arg_regfile regfile, LLVMTypeRef type,
154 LLVMValueRef *assign, unsigned idx)
155 {
156 MAYBE_UNUSED unsigned actual = add_arg_assign(fninfo, regfile, type, assign);
157 assert(actual == idx);
158 }
159
160 static void add_arg_checked(struct si_function_info *fninfo,
161 enum si_arg_regfile regfile, LLVMTypeRef type,
162 unsigned idx)
163 {
164 add_arg_assign_checked(fninfo, regfile, type, NULL, idx);
165 }
166
167 /**
168 * Returns a unique index for a per-patch semantic name and index. The index
169 * must be less than 32, so that a 32-bit bitmask of used inputs or outputs
170 * can be calculated.
171 */
172 unsigned si_shader_io_get_unique_index_patch(unsigned semantic_name, unsigned index)
173 {
174 switch (semantic_name) {
175 case TGSI_SEMANTIC_TESSOUTER:
176 return 0;
177 case TGSI_SEMANTIC_TESSINNER:
178 return 1;
179 case TGSI_SEMANTIC_PATCH:
180 assert(index < 30);
181 return 2 + index;
182
183 default:
184 assert(!"invalid semantic name");
185 return 0;
186 }
187 }
188
189 /**
190 * Returns a unique index for a semantic name and index. The index must be
191 * less than 64, so that a 64-bit bitmask of used inputs or outputs can be
192 * calculated.
193 */
194 unsigned si_shader_io_get_unique_index(unsigned semantic_name, unsigned index,
195 unsigned is_varying)
196 {
197 switch (semantic_name) {
198 case TGSI_SEMANTIC_POSITION:
199 return 0;
200 case TGSI_SEMANTIC_GENERIC:
201 /* Since some shader stages use the the highest used IO index
202 * to determine the size to allocate for inputs/outputs
203 * (in LDS, tess and GS rings). GENERIC should be placed right
204 * after POSITION to make that size as small as possible.
205 */
206 if (index < SI_MAX_IO_GENERIC)
207 return 1 + index;
208
209 assert(!"invalid generic index");
210 return 0;
211 case TGSI_SEMANTIC_PSIZE:
212 return SI_MAX_IO_GENERIC + 1;
213 case TGSI_SEMANTIC_CLIPDIST:
214 assert(index <= 1);
215 return SI_MAX_IO_GENERIC + 2 + index;
216 case TGSI_SEMANTIC_FOG:
217 return SI_MAX_IO_GENERIC + 4;
218 case TGSI_SEMANTIC_LAYER:
219 return SI_MAX_IO_GENERIC + 5;
220 case TGSI_SEMANTIC_VIEWPORT_INDEX:
221 return SI_MAX_IO_GENERIC + 6;
222 case TGSI_SEMANTIC_PRIMID:
223 return SI_MAX_IO_GENERIC + 7;
224 case TGSI_SEMANTIC_COLOR:
225 assert(index < 2);
226 return SI_MAX_IO_GENERIC + 8 + index;
227 case TGSI_SEMANTIC_BCOLOR:
228 assert(index < 2);
229 /* If it's a varying, COLOR and BCOLOR alias. */
230 if (is_varying)
231 return SI_MAX_IO_GENERIC + 8 + index;
232 else
233 return SI_MAX_IO_GENERIC + 10 + index;
234 case TGSI_SEMANTIC_TEXCOORD:
235 assert(index < 8);
236 STATIC_ASSERT(SI_MAX_IO_GENERIC + 12 + 8 <= 64);
237 return SI_MAX_IO_GENERIC + 12 + index;
238 default:
239 assert(!"invalid semantic name");
240 return 0;
241 }
242 }
243
244 /**
245 * Get the value of a shader input parameter and extract a bitfield.
246 */
247 static LLVMValueRef unpack_llvm_param(struct si_shader_context *ctx,
248 LLVMValueRef value, unsigned rshift,
249 unsigned bitwidth)
250 {
251 if (LLVMGetTypeKind(LLVMTypeOf(value)) == LLVMFloatTypeKind)
252 value = ac_to_integer(&ctx->ac, value);
253
254 if (rshift)
255 value = LLVMBuildLShr(ctx->ac.builder, value,
256 LLVMConstInt(ctx->i32, rshift, 0), "");
257
258 if (rshift + bitwidth < 32) {
259 unsigned mask = (1 << bitwidth) - 1;
260 value = LLVMBuildAnd(ctx->ac.builder, value,
261 LLVMConstInt(ctx->i32, mask, 0), "");
262 }
263
264 return value;
265 }
266
267 LLVMValueRef si_unpack_param(struct si_shader_context *ctx,
268 unsigned param, unsigned rshift,
269 unsigned bitwidth)
270 {
271 LLVMValueRef value = LLVMGetParam(ctx->main_fn, param);
272
273 return unpack_llvm_param(ctx, value, rshift, bitwidth);
274 }
275
276 static LLVMValueRef get_rel_patch_id(struct si_shader_context *ctx)
277 {
278 switch (ctx->type) {
279 case PIPE_SHADER_TESS_CTRL:
280 return unpack_llvm_param(ctx, ctx->abi.tcs_rel_ids, 0, 8);
281
282 case PIPE_SHADER_TESS_EVAL:
283 return LLVMGetParam(ctx->main_fn,
284 ctx->param_tes_rel_patch_id);
285
286 default:
287 assert(0);
288 return NULL;
289 }
290 }
291
292 /* Tessellation shaders pass outputs to the next shader using LDS.
293 *
294 * LS outputs = TCS inputs
295 * TCS outputs = TES inputs
296 *
297 * The LDS layout is:
298 * - TCS inputs for patch 0
299 * - TCS inputs for patch 1
300 * - TCS inputs for patch 2 = get_tcs_in_current_patch_offset (if RelPatchID==2)
301 * - ...
302 * - TCS outputs for patch 0 = get_tcs_out_patch0_offset
303 * - Per-patch TCS outputs for patch 0 = get_tcs_out_patch0_patch_data_offset
304 * - TCS outputs for patch 1
305 * - Per-patch TCS outputs for patch 1
306 * - TCS outputs for patch 2 = get_tcs_out_current_patch_offset (if RelPatchID==2)
307 * - Per-patch TCS outputs for patch 2 = get_tcs_out_current_patch_data_offset (if RelPatchID==2)
308 * - ...
309 *
310 * All three shaders VS(LS), TCS, TES share the same LDS space.
311 */
312
313 static LLVMValueRef
314 get_tcs_in_patch_stride(struct si_shader_context *ctx)
315 {
316 return si_unpack_param(ctx, ctx->param_vs_state_bits, 8, 13);
317 }
318
319 static unsigned get_tcs_out_vertex_dw_stride_constant(struct si_shader_context *ctx)
320 {
321 assert(ctx->type == PIPE_SHADER_TESS_CTRL);
322
323 if (ctx->shader->key.mono.u.ff_tcs_inputs_to_copy)
324 return util_last_bit64(ctx->shader->key.mono.u.ff_tcs_inputs_to_copy) * 4;
325
326 return util_last_bit64(ctx->shader->selector->outputs_written) * 4;
327 }
328
329 static LLVMValueRef get_tcs_out_vertex_dw_stride(struct si_shader_context *ctx)
330 {
331 unsigned stride = get_tcs_out_vertex_dw_stride_constant(ctx);
332
333 return LLVMConstInt(ctx->i32, stride, 0);
334 }
335
336 static LLVMValueRef get_tcs_out_patch_stride(struct si_shader_context *ctx)
337 {
338 if (ctx->shader->key.mono.u.ff_tcs_inputs_to_copy)
339 return si_unpack_param(ctx, ctx->param_tcs_out_lds_layout, 0, 13);
340
341 const struct tgsi_shader_info *info = &ctx->shader->selector->info;
342 unsigned tcs_out_vertices = info->properties[TGSI_PROPERTY_TCS_VERTICES_OUT];
343 unsigned vertex_dw_stride = get_tcs_out_vertex_dw_stride_constant(ctx);
344 unsigned num_patch_outputs = util_last_bit64(ctx->shader->selector->patch_outputs_written);
345 unsigned patch_dw_stride = tcs_out_vertices * vertex_dw_stride +
346 num_patch_outputs * 4;
347 return LLVMConstInt(ctx->i32, patch_dw_stride, 0);
348 }
349
350 static LLVMValueRef
351 get_tcs_out_patch0_offset(struct si_shader_context *ctx)
352 {
353 return lp_build_mul_imm(&ctx->bld_base.uint_bld,
354 si_unpack_param(ctx,
355 ctx->param_tcs_out_lds_offsets,
356 0, 16),
357 4);
358 }
359
360 static LLVMValueRef
361 get_tcs_out_patch0_patch_data_offset(struct si_shader_context *ctx)
362 {
363 return lp_build_mul_imm(&ctx->bld_base.uint_bld,
364 si_unpack_param(ctx,
365 ctx->param_tcs_out_lds_offsets,
366 16, 16),
367 4);
368 }
369
370 static LLVMValueRef
371 get_tcs_in_current_patch_offset(struct si_shader_context *ctx)
372 {
373 LLVMValueRef patch_stride = get_tcs_in_patch_stride(ctx);
374 LLVMValueRef rel_patch_id = get_rel_patch_id(ctx);
375
376 return LLVMBuildMul(ctx->ac.builder, patch_stride, rel_patch_id, "");
377 }
378
379 static LLVMValueRef
380 get_tcs_out_current_patch_offset(struct si_shader_context *ctx)
381 {
382 LLVMValueRef patch0_offset = get_tcs_out_patch0_offset(ctx);
383 LLVMValueRef patch_stride = get_tcs_out_patch_stride(ctx);
384 LLVMValueRef rel_patch_id = get_rel_patch_id(ctx);
385
386 return LLVMBuildAdd(ctx->ac.builder, patch0_offset,
387 LLVMBuildMul(ctx->ac.builder, patch_stride,
388 rel_patch_id, ""),
389 "");
390 }
391
392 static LLVMValueRef
393 get_tcs_out_current_patch_data_offset(struct si_shader_context *ctx)
394 {
395 LLVMValueRef patch0_patch_data_offset =
396 get_tcs_out_patch0_patch_data_offset(ctx);
397 LLVMValueRef patch_stride = get_tcs_out_patch_stride(ctx);
398 LLVMValueRef rel_patch_id = get_rel_patch_id(ctx);
399
400 return LLVMBuildAdd(ctx->ac.builder, patch0_patch_data_offset,
401 LLVMBuildMul(ctx->ac.builder, patch_stride,
402 rel_patch_id, ""),
403 "");
404 }
405
406 static LLVMValueRef get_num_tcs_out_vertices(struct si_shader_context *ctx)
407 {
408 unsigned tcs_out_vertices =
409 ctx->shader->selector ?
410 ctx->shader->selector->info.properties[TGSI_PROPERTY_TCS_VERTICES_OUT] : 0;
411
412 /* If !tcs_out_vertices, it's either the fixed-func TCS or the TCS epilog. */
413 if (ctx->type == PIPE_SHADER_TESS_CTRL && tcs_out_vertices)
414 return LLVMConstInt(ctx->i32, tcs_out_vertices, 0);
415
416 return si_unpack_param(ctx, ctx->param_tcs_offchip_layout, 6, 6);
417 }
418
419 static LLVMValueRef get_tcs_in_vertex_dw_stride(struct si_shader_context *ctx)
420 {
421 unsigned stride;
422
423 switch (ctx->type) {
424 case PIPE_SHADER_VERTEX:
425 stride = util_last_bit64(ctx->shader->selector->outputs_written);
426 return LLVMConstInt(ctx->i32, stride * 4, 0);
427
428 case PIPE_SHADER_TESS_CTRL:
429 if (ctx->screen->info.chip_class >= GFX9 &&
430 ctx->shader->is_monolithic) {
431 stride = util_last_bit64(ctx->shader->key.part.tcs.ls->outputs_written);
432 return LLVMConstInt(ctx->i32, stride * 4, 0);
433 }
434 return si_unpack_param(ctx, ctx->param_vs_state_bits, 24, 8);
435
436 default:
437 assert(0);
438 return NULL;
439 }
440 }
441
442 static LLVMValueRef get_instance_index_for_fetch(
443 struct si_shader_context *ctx,
444 unsigned param_start_instance, LLVMValueRef divisor)
445 {
446 LLVMValueRef result = ctx->abi.instance_id;
447
448 /* The division must be done before START_INSTANCE is added. */
449 if (divisor != ctx->i32_1)
450 result = LLVMBuildUDiv(ctx->ac.builder, result, divisor, "");
451
452 return LLVMBuildAdd(ctx->ac.builder, result,
453 LLVMGetParam(ctx->main_fn, param_start_instance), "");
454 }
455
456 /* Bitcast <4 x float> to <2 x double>, extract the component, and convert
457 * to float. */
458 static LLVMValueRef extract_double_to_float(struct si_shader_context *ctx,
459 LLVMValueRef vec4,
460 unsigned double_index)
461 {
462 LLVMBuilderRef builder = ctx->ac.builder;
463 LLVMTypeRef f64 = LLVMDoubleTypeInContext(ctx->ac.context);
464 LLVMValueRef dvec2 = LLVMBuildBitCast(builder, vec4,
465 LLVMVectorType(f64, 2), "");
466 LLVMValueRef index = LLVMConstInt(ctx->i32, double_index, 0);
467 LLVMValueRef value = LLVMBuildExtractElement(builder, dvec2, index, "");
468 return LLVMBuildFPTrunc(builder, value, ctx->f32, "");
469 }
470
471 static LLVMValueRef unpack_sint16(struct si_shader_context *ctx,
472 LLVMValueRef i32, unsigned index)
473 {
474 assert(index <= 1);
475
476 if (index == 1)
477 return LLVMBuildAShr(ctx->ac.builder, i32,
478 LLVMConstInt(ctx->i32, 16, 0), "");
479
480 return LLVMBuildSExt(ctx->ac.builder,
481 LLVMBuildTrunc(ctx->ac.builder, i32,
482 ctx->ac.i16, ""),
483 ctx->i32, "");
484 }
485
486 void si_llvm_load_input_vs(
487 struct si_shader_context *ctx,
488 unsigned input_index,
489 LLVMValueRef out[4])
490 {
491 const struct tgsi_shader_info *info = &ctx->shader->selector->info;
492 unsigned vs_blit_property = info->properties[TGSI_PROPERTY_VS_BLIT_SGPRS];
493
494 if (vs_blit_property) {
495 LLVMValueRef vertex_id = ctx->abi.vertex_id;
496 LLVMValueRef sel_x1 = LLVMBuildICmp(ctx->ac.builder,
497 LLVMIntULE, vertex_id,
498 ctx->i32_1, "");
499 /* Use LLVMIntNE, because we have 3 vertices and only
500 * the middle one should use y2.
501 */
502 LLVMValueRef sel_y1 = LLVMBuildICmp(ctx->ac.builder,
503 LLVMIntNE, vertex_id,
504 ctx->i32_1, "");
505
506 if (input_index == 0) {
507 /* Position: */
508 LLVMValueRef x1y1 = LLVMGetParam(ctx->main_fn,
509 ctx->param_vs_blit_inputs);
510 LLVMValueRef x2y2 = LLVMGetParam(ctx->main_fn,
511 ctx->param_vs_blit_inputs + 1);
512
513 LLVMValueRef x1 = unpack_sint16(ctx, x1y1, 0);
514 LLVMValueRef y1 = unpack_sint16(ctx, x1y1, 1);
515 LLVMValueRef x2 = unpack_sint16(ctx, x2y2, 0);
516 LLVMValueRef y2 = unpack_sint16(ctx, x2y2, 1);
517
518 LLVMValueRef x = LLVMBuildSelect(ctx->ac.builder, sel_x1,
519 x1, x2, "");
520 LLVMValueRef y = LLVMBuildSelect(ctx->ac.builder, sel_y1,
521 y1, y2, "");
522
523 out[0] = LLVMBuildSIToFP(ctx->ac.builder, x, ctx->f32, "");
524 out[1] = LLVMBuildSIToFP(ctx->ac.builder, y, ctx->f32, "");
525 out[2] = LLVMGetParam(ctx->main_fn,
526 ctx->param_vs_blit_inputs + 2);
527 out[3] = ctx->ac.f32_1;
528 return;
529 }
530
531 /* Color or texture coordinates: */
532 assert(input_index == 1);
533
534 if (vs_blit_property == SI_VS_BLIT_SGPRS_POS_COLOR) {
535 for (int i = 0; i < 4; i++) {
536 out[i] = LLVMGetParam(ctx->main_fn,
537 ctx->param_vs_blit_inputs + 3 + i);
538 }
539 } else {
540 assert(vs_blit_property == SI_VS_BLIT_SGPRS_POS_TEXCOORD);
541 LLVMValueRef x1 = LLVMGetParam(ctx->main_fn,
542 ctx->param_vs_blit_inputs + 3);
543 LLVMValueRef y1 = LLVMGetParam(ctx->main_fn,
544 ctx->param_vs_blit_inputs + 4);
545 LLVMValueRef x2 = LLVMGetParam(ctx->main_fn,
546 ctx->param_vs_blit_inputs + 5);
547 LLVMValueRef y2 = LLVMGetParam(ctx->main_fn,
548 ctx->param_vs_blit_inputs + 6);
549
550 out[0] = LLVMBuildSelect(ctx->ac.builder, sel_x1,
551 x1, x2, "");
552 out[1] = LLVMBuildSelect(ctx->ac.builder, sel_y1,
553 y1, y2, "");
554 out[2] = LLVMGetParam(ctx->main_fn,
555 ctx->param_vs_blit_inputs + 7);
556 out[3] = LLVMGetParam(ctx->main_fn,
557 ctx->param_vs_blit_inputs + 8);
558 }
559 return;
560 }
561
562 unsigned chan;
563 unsigned fix_fetch;
564 unsigned num_fetches;
565 unsigned fetch_stride;
566 unsigned num_channels;
567
568 LLVMValueRef t_list_ptr;
569 LLVMValueRef t_offset;
570 LLVMValueRef t_list;
571 LLVMValueRef vertex_index;
572 LLVMValueRef input[3];
573
574 /* Load the T list */
575 t_list_ptr = LLVMGetParam(ctx->main_fn, ctx->param_vertex_buffers);
576
577 t_offset = LLVMConstInt(ctx->i32, input_index, 0);
578
579 t_list = ac_build_load_to_sgpr(&ctx->ac, t_list_ptr, t_offset);
580
581 vertex_index = LLVMGetParam(ctx->main_fn,
582 ctx->param_vertex_index0 +
583 input_index);
584
585 fix_fetch = ctx->shader->key.mono.vs_fix_fetch[input_index];
586
587 /* Do multiple loads for special formats. */
588 switch (fix_fetch) {
589 case SI_FIX_FETCH_RGB_64_FLOAT:
590 num_fetches = 3; /* 3 2-dword loads */
591 fetch_stride = 8;
592 num_channels = 2;
593 break;
594 case SI_FIX_FETCH_RGBA_64_FLOAT:
595 num_fetches = 2; /* 2 4-dword loads */
596 fetch_stride = 16;
597 num_channels = 4;
598 break;
599 case SI_FIX_FETCH_RGB_8:
600 case SI_FIX_FETCH_RGB_8_INT:
601 num_fetches = 3;
602 fetch_stride = 1;
603 num_channels = 1;
604 break;
605 case SI_FIX_FETCH_RGB_16:
606 case SI_FIX_FETCH_RGB_16_INT:
607 num_fetches = 3;
608 fetch_stride = 2;
609 num_channels = 1;
610 break;
611 default:
612 num_fetches = 1;
613 fetch_stride = 0;
614 num_channels = util_last_bit(info->input_usage_mask[input_index]);
615 }
616
617 for (unsigned i = 0; i < num_fetches; i++) {
618 LLVMValueRef voffset = LLVMConstInt(ctx->i32, fetch_stride * i, 0);
619
620 input[i] = ac_build_buffer_load_format(&ctx->ac, t_list,
621 vertex_index, voffset,
622 num_channels, false, true);
623 input[i] = ac_build_expand_to_vec4(&ctx->ac, input[i], num_channels);
624 }
625
626 /* Break up the vec4 into individual components */
627 for (chan = 0; chan < 4; chan++) {
628 LLVMValueRef llvm_chan = LLVMConstInt(ctx->i32, chan, 0);
629 out[chan] = LLVMBuildExtractElement(ctx->ac.builder,
630 input[0], llvm_chan, "");
631 }
632
633 switch (fix_fetch) {
634 case SI_FIX_FETCH_A2_SNORM:
635 case SI_FIX_FETCH_A2_SSCALED:
636 case SI_FIX_FETCH_A2_SINT: {
637 /* The hardware returns an unsigned value; convert it to a
638 * signed one.
639 */
640 LLVMValueRef tmp = out[3];
641 LLVMValueRef c30 = LLVMConstInt(ctx->i32, 30, 0);
642
643 /* First, recover the sign-extended signed integer value. */
644 if (fix_fetch == SI_FIX_FETCH_A2_SSCALED)
645 tmp = LLVMBuildFPToUI(ctx->ac.builder, tmp, ctx->i32, "");
646 else
647 tmp = ac_to_integer(&ctx->ac, tmp);
648
649 /* For the integer-like cases, do a natural sign extension.
650 *
651 * For the SNORM case, the values are 0.0, 0.333, 0.666, 1.0
652 * and happen to contain 0, 1, 2, 3 as the two LSBs of the
653 * exponent.
654 */
655 tmp = LLVMBuildShl(ctx->ac.builder, tmp,
656 fix_fetch == SI_FIX_FETCH_A2_SNORM ?
657 LLVMConstInt(ctx->i32, 7, 0) : c30, "");
658 tmp = LLVMBuildAShr(ctx->ac.builder, tmp, c30, "");
659
660 /* Convert back to the right type. */
661 if (fix_fetch == SI_FIX_FETCH_A2_SNORM) {
662 LLVMValueRef clamp;
663 LLVMValueRef neg_one = LLVMConstReal(ctx->f32, -1.0);
664 tmp = LLVMBuildSIToFP(ctx->ac.builder, tmp, ctx->f32, "");
665 clamp = LLVMBuildFCmp(ctx->ac.builder, LLVMRealULT, tmp, neg_one, "");
666 tmp = LLVMBuildSelect(ctx->ac.builder, clamp, neg_one, tmp, "");
667 } else if (fix_fetch == SI_FIX_FETCH_A2_SSCALED) {
668 tmp = LLVMBuildSIToFP(ctx->ac.builder, tmp, ctx->f32, "");
669 }
670
671 out[3] = tmp;
672 break;
673 }
674 case SI_FIX_FETCH_RGBA_32_UNORM:
675 case SI_FIX_FETCH_RGBX_32_UNORM:
676 for (chan = 0; chan < 4; chan++) {
677 out[chan] = ac_to_integer(&ctx->ac, out[chan]);
678 out[chan] = LLVMBuildUIToFP(ctx->ac.builder,
679 out[chan], ctx->f32, "");
680 out[chan] = LLVMBuildFMul(ctx->ac.builder, out[chan],
681 LLVMConstReal(ctx->f32, 1.0 / UINT_MAX), "");
682 }
683 /* RGBX UINT returns 1 in alpha, which would be rounded to 0 by normalizing. */
684 if (fix_fetch == SI_FIX_FETCH_RGBX_32_UNORM)
685 out[3] = LLVMConstReal(ctx->f32, 1);
686 break;
687 case SI_FIX_FETCH_RGBA_32_SNORM:
688 case SI_FIX_FETCH_RGBX_32_SNORM:
689 case SI_FIX_FETCH_RGBA_32_FIXED:
690 case SI_FIX_FETCH_RGBX_32_FIXED: {
691 double scale;
692 if (fix_fetch >= SI_FIX_FETCH_RGBA_32_FIXED)
693 scale = 1.0 / 0x10000;
694 else
695 scale = 1.0 / INT_MAX;
696
697 for (chan = 0; chan < 4; chan++) {
698 out[chan] = ac_to_integer(&ctx->ac, out[chan]);
699 out[chan] = LLVMBuildSIToFP(ctx->ac.builder,
700 out[chan], ctx->f32, "");
701 out[chan] = LLVMBuildFMul(ctx->ac.builder, out[chan],
702 LLVMConstReal(ctx->f32, scale), "");
703 }
704 /* RGBX SINT returns 1 in alpha, which would be rounded to 0 by normalizing. */
705 if (fix_fetch == SI_FIX_FETCH_RGBX_32_SNORM ||
706 fix_fetch == SI_FIX_FETCH_RGBX_32_FIXED)
707 out[3] = LLVMConstReal(ctx->f32, 1);
708 break;
709 }
710 case SI_FIX_FETCH_RGBA_32_USCALED:
711 for (chan = 0; chan < 4; chan++) {
712 out[chan] = ac_to_integer(&ctx->ac, out[chan]);
713 out[chan] = LLVMBuildUIToFP(ctx->ac.builder,
714 out[chan], ctx->f32, "");
715 }
716 break;
717 case SI_FIX_FETCH_RGBA_32_SSCALED:
718 for (chan = 0; chan < 4; chan++) {
719 out[chan] = ac_to_integer(&ctx->ac, out[chan]);
720 out[chan] = LLVMBuildSIToFP(ctx->ac.builder,
721 out[chan], ctx->f32, "");
722 }
723 break;
724 case SI_FIX_FETCH_RG_64_FLOAT:
725 for (chan = 0; chan < 2; chan++)
726 out[chan] = extract_double_to_float(ctx, input[0], chan);
727
728 out[2] = LLVMConstReal(ctx->f32, 0);
729 out[3] = LLVMConstReal(ctx->f32, 1);
730 break;
731 case SI_FIX_FETCH_RGB_64_FLOAT:
732 for (chan = 0; chan < 3; chan++)
733 out[chan] = extract_double_to_float(ctx, input[chan], 0);
734
735 out[3] = LLVMConstReal(ctx->f32, 1);
736 break;
737 case SI_FIX_FETCH_RGBA_64_FLOAT:
738 for (chan = 0; chan < 4; chan++) {
739 out[chan] = extract_double_to_float(ctx, input[chan / 2],
740 chan % 2);
741 }
742 break;
743 case SI_FIX_FETCH_RGB_8:
744 case SI_FIX_FETCH_RGB_8_INT:
745 case SI_FIX_FETCH_RGB_16:
746 case SI_FIX_FETCH_RGB_16_INT:
747 for (chan = 0; chan < 3; chan++) {
748 out[chan] = LLVMBuildExtractElement(ctx->ac.builder,
749 input[chan],
750 ctx->i32_0, "");
751 }
752 if (fix_fetch == SI_FIX_FETCH_RGB_8 ||
753 fix_fetch == SI_FIX_FETCH_RGB_16) {
754 out[3] = LLVMConstReal(ctx->f32, 1);
755 } else {
756 out[3] = ac_to_float(&ctx->ac, ctx->i32_1);
757 }
758 break;
759 }
760 }
761
762 static void declare_input_vs(
763 struct si_shader_context *ctx,
764 unsigned input_index,
765 const struct tgsi_full_declaration *decl,
766 LLVMValueRef out[4])
767 {
768 si_llvm_load_input_vs(ctx, input_index, out);
769 }
770
771 static LLVMValueRef get_primitive_id(struct si_shader_context *ctx,
772 unsigned swizzle)
773 {
774 if (swizzle > 0)
775 return ctx->i32_0;
776
777 switch (ctx->type) {
778 case PIPE_SHADER_VERTEX:
779 return LLVMGetParam(ctx->main_fn,
780 ctx->param_vs_prim_id);
781 case PIPE_SHADER_TESS_CTRL:
782 return ctx->abi.tcs_patch_id;
783 case PIPE_SHADER_TESS_EVAL:
784 return ctx->abi.tes_patch_id;
785 case PIPE_SHADER_GEOMETRY:
786 return ctx->abi.gs_prim_id;
787 default:
788 assert(0);
789 return ctx->i32_0;
790 }
791 }
792
793 /**
794 * Return the value of tgsi_ind_register for indexing.
795 * This is the indirect index with the constant offset added to it.
796 */
797 LLVMValueRef si_get_indirect_index(struct si_shader_context *ctx,
798 const struct tgsi_ind_register *ind,
799 unsigned addr_mul,
800 int rel_index)
801 {
802 LLVMValueRef result;
803
804 if (ind->File == TGSI_FILE_ADDRESS) {
805 result = ctx->addrs[ind->Index][ind->Swizzle];
806 result = LLVMBuildLoad(ctx->ac.builder, result, "");
807 } else {
808 struct tgsi_full_src_register src = {};
809
810 src.Register.File = ind->File;
811 src.Register.Index = ind->Index;
812
813 /* Set the second index to 0 for constants. */
814 if (ind->File == TGSI_FILE_CONSTANT)
815 src.Register.Dimension = 1;
816
817 result = ctx->bld_base.emit_fetch_funcs[ind->File](&ctx->bld_base, &src,
818 TGSI_TYPE_SIGNED,
819 ind->Swizzle);
820 result = ac_to_integer(&ctx->ac, result);
821 }
822
823 if (addr_mul != 1)
824 result = LLVMBuildMul(ctx->ac.builder, result,
825 LLVMConstInt(ctx->i32, addr_mul, 0), "");
826 result = LLVMBuildAdd(ctx->ac.builder, result,
827 LLVMConstInt(ctx->i32, rel_index, 0), "");
828 return result;
829 }
830
831 /**
832 * Like si_get_indirect_index, but restricts the return value to a (possibly
833 * undefined) value inside [0..num).
834 */
835 LLVMValueRef si_get_bounded_indirect_index(struct si_shader_context *ctx,
836 const struct tgsi_ind_register *ind,
837 int rel_index, unsigned num)
838 {
839 LLVMValueRef result = si_get_indirect_index(ctx, ind, 1, rel_index);
840
841 return si_llvm_bound_index(ctx, result, num);
842 }
843
844 static LLVMValueRef get_dw_address_from_generic_indices(struct si_shader_context *ctx,
845 LLVMValueRef vertex_dw_stride,
846 LLVMValueRef base_addr,
847 LLVMValueRef vertex_index,
848 LLVMValueRef param_index,
849 unsigned input_index,
850 ubyte *name,
851 ubyte *index,
852 bool is_patch)
853 {
854 if (vertex_dw_stride) {
855 base_addr = LLVMBuildAdd(ctx->ac.builder, base_addr,
856 LLVMBuildMul(ctx->ac.builder, vertex_index,
857 vertex_dw_stride, ""), "");
858 }
859
860 if (param_index) {
861 base_addr = LLVMBuildAdd(ctx->ac.builder, base_addr,
862 LLVMBuildMul(ctx->ac.builder, param_index,
863 LLVMConstInt(ctx->i32, 4, 0), ""), "");
864 }
865
866 int param = is_patch ?
867 si_shader_io_get_unique_index_patch(name[input_index],
868 index[input_index]) :
869 si_shader_io_get_unique_index(name[input_index],
870 index[input_index], false);
871
872 /* Add the base address of the element. */
873 return LLVMBuildAdd(ctx->ac.builder, base_addr,
874 LLVMConstInt(ctx->i32, param * 4, 0), "");
875 }
876
877 /**
878 * Calculate a dword address given an input or output register and a stride.
879 */
880 static LLVMValueRef get_dw_address(struct si_shader_context *ctx,
881 const struct tgsi_full_dst_register *dst,
882 const struct tgsi_full_src_register *src,
883 LLVMValueRef vertex_dw_stride,
884 LLVMValueRef base_addr)
885 {
886 struct tgsi_shader_info *info = &ctx->shader->selector->info;
887 ubyte *name, *index, *array_first;
888 int input_index;
889 struct tgsi_full_dst_register reg;
890 LLVMValueRef vertex_index = NULL;
891 LLVMValueRef ind_index = NULL;
892
893 /* Set the register description. The address computation is the same
894 * for sources and destinations. */
895 if (src) {
896 reg.Register.File = src->Register.File;
897 reg.Register.Index = src->Register.Index;
898 reg.Register.Indirect = src->Register.Indirect;
899 reg.Register.Dimension = src->Register.Dimension;
900 reg.Indirect = src->Indirect;
901 reg.Dimension = src->Dimension;
902 reg.DimIndirect = src->DimIndirect;
903 } else
904 reg = *dst;
905
906 /* If the register is 2-dimensional (e.g. an array of vertices
907 * in a primitive), calculate the base address of the vertex. */
908 if (reg.Register.Dimension) {
909 if (reg.Dimension.Indirect)
910 vertex_index = si_get_indirect_index(ctx, &reg.DimIndirect,
911 1, reg.Dimension.Index);
912 else
913 vertex_index = LLVMConstInt(ctx->i32, reg.Dimension.Index, 0);
914 }
915
916 /* Get information about the register. */
917 if (reg.Register.File == TGSI_FILE_INPUT) {
918 name = info->input_semantic_name;
919 index = info->input_semantic_index;
920 array_first = info->input_array_first;
921 } else if (reg.Register.File == TGSI_FILE_OUTPUT) {
922 name = info->output_semantic_name;
923 index = info->output_semantic_index;
924 array_first = info->output_array_first;
925 } else {
926 assert(0);
927 return NULL;
928 }
929
930 if (reg.Register.Indirect) {
931 /* Add the relative address of the element. */
932 if (reg.Indirect.ArrayID)
933 input_index = array_first[reg.Indirect.ArrayID];
934 else
935 input_index = reg.Register.Index;
936
937 ind_index = si_get_indirect_index(ctx, &reg.Indirect,
938 1, reg.Register.Index - input_index);
939 } else {
940 input_index = reg.Register.Index;
941 }
942
943 return get_dw_address_from_generic_indices(ctx, vertex_dw_stride,
944 base_addr, vertex_index,
945 ind_index, input_index,
946 name, index,
947 !reg.Register.Dimension);
948 }
949
950 /* The offchip buffer layout for TCS->TES is
951 *
952 * - attribute 0 of patch 0 vertex 0
953 * - attribute 0 of patch 0 vertex 1
954 * - attribute 0 of patch 0 vertex 2
955 * ...
956 * - attribute 0 of patch 1 vertex 0
957 * - attribute 0 of patch 1 vertex 1
958 * ...
959 * - attribute 1 of patch 0 vertex 0
960 * - attribute 1 of patch 0 vertex 1
961 * ...
962 * - per patch attribute 0 of patch 0
963 * - per patch attribute 0 of patch 1
964 * ...
965 *
966 * Note that every attribute has 4 components.
967 */
968 static LLVMValueRef get_tcs_tes_buffer_address(struct si_shader_context *ctx,
969 LLVMValueRef rel_patch_id,
970 LLVMValueRef vertex_index,
971 LLVMValueRef param_index)
972 {
973 LLVMValueRef base_addr, vertices_per_patch, num_patches, total_vertices;
974 LLVMValueRef param_stride, constant16;
975
976 vertices_per_patch = get_num_tcs_out_vertices(ctx);
977 num_patches = si_unpack_param(ctx, ctx->param_tcs_offchip_layout, 0, 6);
978 total_vertices = LLVMBuildMul(ctx->ac.builder, vertices_per_patch,
979 num_patches, "");
980
981 constant16 = LLVMConstInt(ctx->i32, 16, 0);
982 if (vertex_index) {
983 base_addr = LLVMBuildMul(ctx->ac.builder, rel_patch_id,
984 vertices_per_patch, "");
985
986 base_addr = LLVMBuildAdd(ctx->ac.builder, base_addr,
987 vertex_index, "");
988
989 param_stride = total_vertices;
990 } else {
991 base_addr = rel_patch_id;
992 param_stride = num_patches;
993 }
994
995 base_addr = LLVMBuildAdd(ctx->ac.builder, base_addr,
996 LLVMBuildMul(ctx->ac.builder, param_index,
997 param_stride, ""), "");
998
999 base_addr = LLVMBuildMul(ctx->ac.builder, base_addr, constant16, "");
1000
1001 if (!vertex_index) {
1002 LLVMValueRef patch_data_offset =
1003 si_unpack_param(ctx, ctx->param_tcs_offchip_layout, 12, 20);
1004
1005 base_addr = LLVMBuildAdd(ctx->ac.builder, base_addr,
1006 patch_data_offset, "");
1007 }
1008 return base_addr;
1009 }
1010
1011 /* This is a generic helper that can be shared by the NIR and TGSI backends */
1012 static LLVMValueRef get_tcs_tes_buffer_address_from_generic_indices(
1013 struct si_shader_context *ctx,
1014 LLVMValueRef vertex_index,
1015 LLVMValueRef param_index,
1016 unsigned param_base,
1017 ubyte *name,
1018 ubyte *index,
1019 bool is_patch)
1020 {
1021 unsigned param_index_base;
1022
1023 param_index_base = is_patch ?
1024 si_shader_io_get_unique_index_patch(name[param_base], index[param_base]) :
1025 si_shader_io_get_unique_index(name[param_base], index[param_base], false);
1026
1027 if (param_index) {
1028 param_index = LLVMBuildAdd(ctx->ac.builder, param_index,
1029 LLVMConstInt(ctx->i32, param_index_base, 0),
1030 "");
1031 } else {
1032 param_index = LLVMConstInt(ctx->i32, param_index_base, 0);
1033 }
1034
1035 return get_tcs_tes_buffer_address(ctx, get_rel_patch_id(ctx),
1036 vertex_index, param_index);
1037 }
1038
1039 static LLVMValueRef get_tcs_tes_buffer_address_from_reg(
1040 struct si_shader_context *ctx,
1041 const struct tgsi_full_dst_register *dst,
1042 const struct tgsi_full_src_register *src)
1043 {
1044 struct tgsi_shader_info *info = &ctx->shader->selector->info;
1045 ubyte *name, *index, *array_first;
1046 struct tgsi_full_src_register reg;
1047 LLVMValueRef vertex_index = NULL;
1048 LLVMValueRef param_index = NULL;
1049 unsigned param_base;
1050
1051 reg = src ? *src : tgsi_full_src_register_from_dst(dst);
1052
1053 if (reg.Register.Dimension) {
1054
1055 if (reg.Dimension.Indirect)
1056 vertex_index = si_get_indirect_index(ctx, &reg.DimIndirect,
1057 1, reg.Dimension.Index);
1058 else
1059 vertex_index = LLVMConstInt(ctx->i32, reg.Dimension.Index, 0);
1060 }
1061
1062 /* Get information about the register. */
1063 if (reg.Register.File == TGSI_FILE_INPUT) {
1064 name = info->input_semantic_name;
1065 index = info->input_semantic_index;
1066 array_first = info->input_array_first;
1067 } else if (reg.Register.File == TGSI_FILE_OUTPUT) {
1068 name = info->output_semantic_name;
1069 index = info->output_semantic_index;
1070 array_first = info->output_array_first;
1071 } else {
1072 assert(0);
1073 return NULL;
1074 }
1075
1076 if (reg.Register.Indirect) {
1077 if (reg.Indirect.ArrayID)
1078 param_base = array_first[reg.Indirect.ArrayID];
1079 else
1080 param_base = reg.Register.Index;
1081
1082 param_index = si_get_indirect_index(ctx, &reg.Indirect,
1083 1, reg.Register.Index - param_base);
1084
1085 } else {
1086 param_base = reg.Register.Index;
1087 }
1088
1089 return get_tcs_tes_buffer_address_from_generic_indices(ctx, vertex_index,
1090 param_index, param_base,
1091 name, index, !reg.Register.Dimension);
1092 }
1093
1094 static LLVMValueRef buffer_load(struct lp_build_tgsi_context *bld_base,
1095 LLVMTypeRef type, unsigned swizzle,
1096 LLVMValueRef buffer, LLVMValueRef offset,
1097 LLVMValueRef base, bool can_speculate)
1098 {
1099 struct si_shader_context *ctx = si_shader_context(bld_base);
1100 LLVMValueRef value, value2;
1101 LLVMTypeRef vec_type = LLVMVectorType(type, 4);
1102
1103 if (swizzle == ~0) {
1104 value = ac_build_buffer_load(&ctx->ac, buffer, 4, NULL, base, offset,
1105 0, 1, 0, can_speculate, false);
1106
1107 return LLVMBuildBitCast(ctx->ac.builder, value, vec_type, "");
1108 }
1109
1110 if (!llvm_type_is_64bit(ctx, type)) {
1111 value = ac_build_buffer_load(&ctx->ac, buffer, 4, NULL, base, offset,
1112 0, 1, 0, can_speculate, false);
1113
1114 value = LLVMBuildBitCast(ctx->ac.builder, value, vec_type, "");
1115 return LLVMBuildExtractElement(ctx->ac.builder, value,
1116 LLVMConstInt(ctx->i32, swizzle, 0), "");
1117 }
1118
1119 value = ac_build_buffer_load(&ctx->ac, buffer, 1, NULL, base, offset,
1120 swizzle * 4, 1, 0, can_speculate, false);
1121
1122 value2 = ac_build_buffer_load(&ctx->ac, buffer, 1, NULL, base, offset,
1123 swizzle * 4 + 4, 1, 0, can_speculate, false);
1124
1125 return si_llvm_emit_fetch_64bit(bld_base, type, value, value2);
1126 }
1127
1128 /**
1129 * Load from LDS.
1130 *
1131 * \param type output value type
1132 * \param swizzle offset (typically 0..3); it can be ~0, which loads a vec4
1133 * \param dw_addr address in dwords
1134 */
1135 static LLVMValueRef lds_load(struct lp_build_tgsi_context *bld_base,
1136 LLVMTypeRef type, unsigned swizzle,
1137 LLVMValueRef dw_addr)
1138 {
1139 struct si_shader_context *ctx = si_shader_context(bld_base);
1140 LLVMValueRef value;
1141
1142 if (swizzle == ~0) {
1143 LLVMValueRef values[TGSI_NUM_CHANNELS];
1144
1145 for (unsigned chan = 0; chan < TGSI_NUM_CHANNELS; chan++)
1146 values[chan] = lds_load(bld_base, type, chan, dw_addr);
1147
1148 return lp_build_gather_values(&ctx->gallivm, values,
1149 TGSI_NUM_CHANNELS);
1150 }
1151
1152 /* Split 64-bit loads. */
1153 if (llvm_type_is_64bit(ctx, type)) {
1154 LLVMValueRef lo, hi;
1155
1156 lo = lds_load(bld_base, ctx->i32, swizzle, dw_addr);
1157 hi = lds_load(bld_base, ctx->i32, swizzle + 1, dw_addr);
1158 return si_llvm_emit_fetch_64bit(bld_base, type, lo, hi);
1159 }
1160
1161 dw_addr = lp_build_add(&bld_base->uint_bld, dw_addr,
1162 LLVMConstInt(ctx->i32, swizzle, 0));
1163
1164 value = ac_lds_load(&ctx->ac, dw_addr);
1165
1166 return LLVMBuildBitCast(ctx->ac.builder, value, type, "");
1167 }
1168
1169 /**
1170 * Store to LDS.
1171 *
1172 * \param swizzle offset (typically 0..3)
1173 * \param dw_addr address in dwords
1174 * \param value value to store
1175 */
1176 static void lds_store(struct si_shader_context *ctx,
1177 unsigned dw_offset_imm, LLVMValueRef dw_addr,
1178 LLVMValueRef value)
1179 {
1180 dw_addr = lp_build_add(&ctx->bld_base.uint_bld, dw_addr,
1181 LLVMConstInt(ctx->i32, dw_offset_imm, 0));
1182
1183 ac_lds_store(&ctx->ac, dw_addr, value);
1184 }
1185
1186 enum si_tess_ring {
1187 TCS_FACTOR_RING,
1188 TESS_OFFCHIP_RING_TCS,
1189 TESS_OFFCHIP_RING_TES,
1190 };
1191
1192 static LLVMValueRef get_tess_ring_descriptor(struct si_shader_context *ctx,
1193 enum si_tess_ring ring)
1194 {
1195 LLVMBuilderRef builder = ctx->ac.builder;
1196 unsigned param = ring == TESS_OFFCHIP_RING_TES ? ctx->param_tes_offchip_addr :
1197 ctx->param_tcs_out_lds_layout;
1198 LLVMValueRef addr = LLVMGetParam(ctx->main_fn, param);
1199
1200 /* TCS only receives high 13 bits of the address. */
1201 if (ring == TESS_OFFCHIP_RING_TCS || ring == TCS_FACTOR_RING) {
1202 addr = LLVMBuildAnd(builder, addr,
1203 LLVMConstInt(ctx->i32, 0xfff80000, 0), "");
1204 }
1205
1206 if (ring == TCS_FACTOR_RING) {
1207 unsigned tf_offset = ctx->screen->tess_offchip_ring_size;
1208 addr = LLVMBuildAdd(builder, addr,
1209 LLVMConstInt(ctx->i32, tf_offset, 0), "");
1210 }
1211
1212 LLVMValueRef desc[4];
1213 desc[0] = addr;
1214 desc[1] = LLVMConstInt(ctx->i32,
1215 S_008F04_BASE_ADDRESS_HI(ctx->screen->info.address32_hi), 0);
1216 desc[2] = LLVMConstInt(ctx->i32, 0xffffffff, 0);
1217 desc[3] = LLVMConstInt(ctx->i32,
1218 S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) |
1219 S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
1220 S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) |
1221 S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W) |
1222 S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
1223 S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32), 0);
1224
1225 return ac_build_gather_values(&ctx->ac, desc, 4);
1226 }
1227
1228 static LLVMValueRef fetch_input_tcs(
1229 struct lp_build_tgsi_context *bld_base,
1230 const struct tgsi_full_src_register *reg,
1231 enum tgsi_opcode_type type, unsigned swizzle)
1232 {
1233 struct si_shader_context *ctx = si_shader_context(bld_base);
1234 LLVMValueRef dw_addr, stride;
1235
1236 stride = get_tcs_in_vertex_dw_stride(ctx);
1237 dw_addr = get_tcs_in_current_patch_offset(ctx);
1238 dw_addr = get_dw_address(ctx, NULL, reg, stride, dw_addr);
1239
1240 return lds_load(bld_base, tgsi2llvmtype(bld_base, type), swizzle, dw_addr);
1241 }
1242
1243 static LLVMValueRef si_nir_load_tcs_varyings(struct ac_shader_abi *abi,
1244 LLVMTypeRef type,
1245 LLVMValueRef vertex_index,
1246 LLVMValueRef param_index,
1247 unsigned const_index,
1248 unsigned location,
1249 unsigned driver_location,
1250 unsigned component,
1251 unsigned num_components,
1252 bool is_patch,
1253 bool is_compact,
1254 bool load_input)
1255 {
1256 struct si_shader_context *ctx = si_shader_context_from_abi(abi);
1257 struct tgsi_shader_info *info = &ctx->shader->selector->info;
1258 struct lp_build_tgsi_context *bld_base = &ctx->bld_base;
1259 LLVMValueRef dw_addr, stride;
1260
1261 driver_location = driver_location / 4;
1262
1263 if (load_input) {
1264 stride = get_tcs_in_vertex_dw_stride(ctx);
1265 dw_addr = get_tcs_in_current_patch_offset(ctx);
1266 } else {
1267 if (is_patch) {
1268 stride = NULL;
1269 dw_addr = get_tcs_out_current_patch_data_offset(ctx);
1270 } else {
1271 stride = get_tcs_out_vertex_dw_stride(ctx);
1272 dw_addr = get_tcs_out_current_patch_offset(ctx);
1273 }
1274 }
1275
1276 if (param_index) {
1277 /* Add the constant index to the indirect index */
1278 param_index = LLVMBuildAdd(ctx->ac.builder, param_index,
1279 LLVMConstInt(ctx->i32, const_index, 0), "");
1280 } else {
1281 param_index = LLVMConstInt(ctx->i32, const_index, 0);
1282 }
1283
1284 ubyte *names;
1285 ubyte *indices;
1286 if (load_input) {
1287 names = info->input_semantic_name;
1288 indices = info->input_semantic_index;
1289 } else {
1290 names = info->output_semantic_name;
1291 indices = info->output_semantic_index;
1292 }
1293
1294 dw_addr = get_dw_address_from_generic_indices(ctx, stride, dw_addr,
1295 vertex_index, param_index,
1296 driver_location,
1297 names, indices,
1298 is_patch);
1299
1300 LLVMValueRef value[4];
1301 for (unsigned i = 0; i < num_components; i++) {
1302 unsigned offset = i;
1303 if (llvm_type_is_64bit(ctx, type))
1304 offset *= 2;
1305
1306 offset += component;
1307 value[i + component] = lds_load(bld_base, type, offset, dw_addr);
1308 }
1309
1310 return ac_build_varying_gather_values(&ctx->ac, value, num_components, component);
1311 }
1312
1313 static LLVMValueRef fetch_output_tcs(
1314 struct lp_build_tgsi_context *bld_base,
1315 const struct tgsi_full_src_register *reg,
1316 enum tgsi_opcode_type type, unsigned swizzle)
1317 {
1318 struct si_shader_context *ctx = si_shader_context(bld_base);
1319 LLVMValueRef dw_addr, stride;
1320
1321 if (reg->Register.Dimension) {
1322 stride = get_tcs_out_vertex_dw_stride(ctx);
1323 dw_addr = get_tcs_out_current_patch_offset(ctx);
1324 dw_addr = get_dw_address(ctx, NULL, reg, stride, dw_addr);
1325 } else {
1326 dw_addr = get_tcs_out_current_patch_data_offset(ctx);
1327 dw_addr = get_dw_address(ctx, NULL, reg, NULL, dw_addr);
1328 }
1329
1330 return lds_load(bld_base, tgsi2llvmtype(bld_base, type), swizzle, dw_addr);
1331 }
1332
1333 static LLVMValueRef fetch_input_tes(
1334 struct lp_build_tgsi_context *bld_base,
1335 const struct tgsi_full_src_register *reg,
1336 enum tgsi_opcode_type type, unsigned swizzle)
1337 {
1338 struct si_shader_context *ctx = si_shader_context(bld_base);
1339 LLVMValueRef base, addr;
1340
1341 base = LLVMGetParam(ctx->main_fn, ctx->param_tcs_offchip_offset);
1342 addr = get_tcs_tes_buffer_address_from_reg(ctx, NULL, reg);
1343
1344 return buffer_load(bld_base, tgsi2llvmtype(bld_base, type), swizzle,
1345 ctx->tess_offchip_ring, base, addr, true);
1346 }
1347
1348 LLVMValueRef si_nir_load_input_tes(struct ac_shader_abi *abi,
1349 LLVMTypeRef type,
1350 LLVMValueRef vertex_index,
1351 LLVMValueRef param_index,
1352 unsigned const_index,
1353 unsigned location,
1354 unsigned driver_location,
1355 unsigned component,
1356 unsigned num_components,
1357 bool is_patch,
1358 bool is_compact,
1359 bool load_input)
1360 {
1361 struct si_shader_context *ctx = si_shader_context_from_abi(abi);
1362 struct tgsi_shader_info *info = &ctx->shader->selector->info;
1363 LLVMValueRef base, addr;
1364
1365 driver_location = driver_location / 4;
1366
1367 base = LLVMGetParam(ctx->main_fn, ctx->param_tcs_offchip_offset);
1368
1369 if (param_index) {
1370 /* Add the constant index to the indirect index */
1371 param_index = LLVMBuildAdd(ctx->ac.builder, param_index,
1372 LLVMConstInt(ctx->i32, const_index, 0), "");
1373 } else {
1374 param_index = LLVMConstInt(ctx->i32, const_index, 0);
1375 }
1376
1377 addr = get_tcs_tes_buffer_address_from_generic_indices(ctx, vertex_index,
1378 param_index, driver_location,
1379 info->input_semantic_name,
1380 info->input_semantic_index,
1381 is_patch);
1382
1383 /* TODO: This will generate rather ordinary llvm code, although it
1384 * should be easy for the optimiser to fix up. In future we might want
1385 * to refactor buffer_load(), but for now this maximises code sharing
1386 * between the NIR and TGSI backends.
1387 */
1388 LLVMValueRef value[4];
1389 for (unsigned i = 0; i < num_components; i++) {
1390 unsigned offset = i;
1391 if (llvm_type_is_64bit(ctx, type))
1392 offset *= 2;
1393
1394 offset += component;
1395 value[i + component] = buffer_load(&ctx->bld_base, type, offset,
1396 ctx->tess_offchip_ring, base, addr, true);
1397 }
1398
1399 return ac_build_varying_gather_values(&ctx->ac, value, num_components, component);
1400 }
1401
1402 static void store_output_tcs(struct lp_build_tgsi_context *bld_base,
1403 const struct tgsi_full_instruction *inst,
1404 const struct tgsi_opcode_info *info,
1405 unsigned index,
1406 LLVMValueRef dst[4])
1407 {
1408 struct si_shader_context *ctx = si_shader_context(bld_base);
1409 const struct tgsi_full_dst_register *reg = &inst->Dst[index];
1410 const struct tgsi_shader_info *sh_info = &ctx->shader->selector->info;
1411 unsigned chan_index;
1412 LLVMValueRef dw_addr, stride;
1413 LLVMValueRef buffer, base, buf_addr;
1414 LLVMValueRef values[4];
1415 bool skip_lds_store;
1416 bool is_tess_factor = false, is_tess_inner = false;
1417
1418 /* Only handle per-patch and per-vertex outputs here.
1419 * Vectors will be lowered to scalars and this function will be called again.
1420 */
1421 if (reg->Register.File != TGSI_FILE_OUTPUT ||
1422 (dst[0] && LLVMGetTypeKind(LLVMTypeOf(dst[0])) == LLVMVectorTypeKind)) {
1423 si_llvm_emit_store(bld_base, inst, info, index, dst);
1424 return;
1425 }
1426
1427 if (reg->Register.Dimension) {
1428 stride = get_tcs_out_vertex_dw_stride(ctx);
1429 dw_addr = get_tcs_out_current_patch_offset(ctx);
1430 dw_addr = get_dw_address(ctx, reg, NULL, stride, dw_addr);
1431 skip_lds_store = !sh_info->reads_pervertex_outputs;
1432 } else {
1433 dw_addr = get_tcs_out_current_patch_data_offset(ctx);
1434 dw_addr = get_dw_address(ctx, reg, NULL, NULL, dw_addr);
1435 skip_lds_store = !sh_info->reads_perpatch_outputs;
1436
1437 if (!reg->Register.Indirect) {
1438 int name = sh_info->output_semantic_name[reg->Register.Index];
1439
1440 /* Always write tess factors into LDS for the TCS epilog. */
1441 if (name == TGSI_SEMANTIC_TESSINNER ||
1442 name == TGSI_SEMANTIC_TESSOUTER) {
1443 /* The epilog doesn't read LDS if invocation 0 defines tess factors. */
1444 skip_lds_store = !sh_info->reads_tessfactor_outputs &&
1445 ctx->shader->selector->tcs_info.tessfactors_are_def_in_all_invocs;
1446 is_tess_factor = true;
1447 is_tess_inner = name == TGSI_SEMANTIC_TESSINNER;
1448 }
1449 }
1450 }
1451
1452 buffer = get_tess_ring_descriptor(ctx, TESS_OFFCHIP_RING_TCS);
1453
1454 base = LLVMGetParam(ctx->main_fn, ctx->param_tcs_offchip_offset);
1455 buf_addr = get_tcs_tes_buffer_address_from_reg(ctx, reg, NULL);
1456
1457 uint32_t writemask = reg->Register.WriteMask;
1458 while (writemask) {
1459 chan_index = u_bit_scan(&writemask);
1460 LLVMValueRef value = dst[chan_index];
1461
1462 if (inst->Instruction.Saturate)
1463 value = ac_build_clamp(&ctx->ac, value);
1464
1465 /* Skip LDS stores if there is no LDS read of this output. */
1466 if (!skip_lds_store)
1467 lds_store(ctx, chan_index, dw_addr, value);
1468
1469 value = ac_to_integer(&ctx->ac, value);
1470 values[chan_index] = value;
1471
1472 if (reg->Register.WriteMask != 0xF && !is_tess_factor) {
1473 ac_build_buffer_store_dword(&ctx->ac, buffer, value, 1,
1474 buf_addr, base,
1475 4 * chan_index, 1, 0, true, false);
1476 }
1477
1478 /* Write tess factors into VGPRs for the epilog. */
1479 if (is_tess_factor &&
1480 ctx->shader->selector->tcs_info.tessfactors_are_def_in_all_invocs) {
1481 if (!is_tess_inner) {
1482 LLVMBuildStore(ctx->ac.builder, value, /* outer */
1483 ctx->invoc0_tess_factors[chan_index]);
1484 } else if (chan_index < 2) {
1485 LLVMBuildStore(ctx->ac.builder, value, /* inner */
1486 ctx->invoc0_tess_factors[4 + chan_index]);
1487 }
1488 }
1489 }
1490
1491 if (reg->Register.WriteMask == 0xF && !is_tess_factor) {
1492 LLVMValueRef value = lp_build_gather_values(&ctx->gallivm,
1493 values, 4);
1494 ac_build_buffer_store_dword(&ctx->ac, buffer, value, 4, buf_addr,
1495 base, 0, 1, 0, true, false);
1496 }
1497 }
1498
1499 static void si_nir_store_output_tcs(struct ac_shader_abi *abi,
1500 const struct nir_variable *var,
1501 LLVMValueRef vertex_index,
1502 LLVMValueRef param_index,
1503 unsigned const_index,
1504 LLVMValueRef src,
1505 unsigned writemask)
1506 {
1507 struct si_shader_context *ctx = si_shader_context_from_abi(abi);
1508 struct tgsi_shader_info *info = &ctx->shader->selector->info;
1509 const unsigned component = var->data.location_frac;
1510 const bool is_patch = var->data.patch;
1511 unsigned driver_location = var->data.driver_location;
1512 LLVMValueRef dw_addr, stride;
1513 LLVMValueRef buffer, base, addr;
1514 LLVMValueRef values[4];
1515 bool skip_lds_store;
1516 bool is_tess_factor = false, is_tess_inner = false;
1517
1518 driver_location = driver_location / 4;
1519
1520 if (param_index) {
1521 /* Add the constant index to the indirect index */
1522 param_index = LLVMBuildAdd(ctx->ac.builder, param_index,
1523 LLVMConstInt(ctx->i32, const_index, 0), "");
1524 } else {
1525 if (const_index != 0)
1526 param_index = LLVMConstInt(ctx->i32, const_index, 0);
1527 }
1528
1529 if (!is_patch) {
1530 stride = get_tcs_out_vertex_dw_stride(ctx);
1531 dw_addr = get_tcs_out_current_patch_offset(ctx);
1532 dw_addr = get_dw_address_from_generic_indices(ctx, stride, dw_addr,
1533 vertex_index, param_index,
1534 driver_location,
1535 info->output_semantic_name,
1536 info->output_semantic_index,
1537 is_patch);
1538
1539 skip_lds_store = !info->reads_pervertex_outputs;
1540 } else {
1541 dw_addr = get_tcs_out_current_patch_data_offset(ctx);
1542 dw_addr = get_dw_address_from_generic_indices(ctx, NULL, dw_addr,
1543 vertex_index, param_index,
1544 driver_location,
1545 info->output_semantic_name,
1546 info->output_semantic_index,
1547 is_patch);
1548
1549 skip_lds_store = !info->reads_perpatch_outputs;
1550
1551 if (!param_index) {
1552 int name = info->output_semantic_name[driver_location];
1553
1554 /* Always write tess factors into LDS for the TCS epilog. */
1555 if (name == TGSI_SEMANTIC_TESSINNER ||
1556 name == TGSI_SEMANTIC_TESSOUTER) {
1557 /* The epilog doesn't read LDS if invocation 0 defines tess factors. */
1558 skip_lds_store = !info->reads_tessfactor_outputs &&
1559 ctx->shader->selector->tcs_info.tessfactors_are_def_in_all_invocs;
1560 is_tess_factor = true;
1561 is_tess_inner = name == TGSI_SEMANTIC_TESSINNER;
1562 }
1563 }
1564 }
1565
1566 buffer = get_tess_ring_descriptor(ctx, TESS_OFFCHIP_RING_TCS);
1567
1568 base = LLVMGetParam(ctx->main_fn, ctx->param_tcs_offchip_offset);
1569
1570 addr = get_tcs_tes_buffer_address_from_generic_indices(ctx, vertex_index,
1571 param_index, driver_location,
1572 info->output_semantic_name,
1573 info->output_semantic_index,
1574 is_patch);
1575
1576 for (unsigned chan = 0; chan < 4; chan++) {
1577 if (!(writemask & (1 << chan)))
1578 continue;
1579 LLVMValueRef value = ac_llvm_extract_elem(&ctx->ac, src, chan - component);
1580
1581 /* Skip LDS stores if there is no LDS read of this output. */
1582 if (!skip_lds_store)
1583 lds_store(ctx, chan, dw_addr, value);
1584
1585 value = ac_to_integer(&ctx->ac, value);
1586 values[chan] = value;
1587
1588 if (writemask != 0xF && !is_tess_factor) {
1589 ac_build_buffer_store_dword(&ctx->ac, buffer, value, 1,
1590 addr, base,
1591 4 * chan, 1, 0, true, false);
1592 }
1593
1594 /* Write tess factors into VGPRs for the epilog. */
1595 if (is_tess_factor &&
1596 ctx->shader->selector->tcs_info.tessfactors_are_def_in_all_invocs) {
1597 if (!is_tess_inner) {
1598 LLVMBuildStore(ctx->ac.builder, value, /* outer */
1599 ctx->invoc0_tess_factors[chan]);
1600 } else if (chan < 2) {
1601 LLVMBuildStore(ctx->ac.builder, value, /* inner */
1602 ctx->invoc0_tess_factors[4 + chan]);
1603 }
1604 }
1605 }
1606
1607 if (writemask == 0xF && !is_tess_factor) {
1608 LLVMValueRef value = lp_build_gather_values(&ctx->gallivm,
1609 values, 4);
1610 ac_build_buffer_store_dword(&ctx->ac, buffer, value, 4, addr,
1611 base, 0, 1, 0, true, false);
1612 }
1613 }
1614
1615 LLVMValueRef si_llvm_load_input_gs(struct ac_shader_abi *abi,
1616 unsigned input_index,
1617 unsigned vtx_offset_param,
1618 LLVMTypeRef type,
1619 unsigned swizzle)
1620 {
1621 struct si_shader_context *ctx = si_shader_context_from_abi(abi);
1622 struct lp_build_tgsi_context *bld_base = &ctx->bld_base;
1623 struct si_shader *shader = ctx->shader;
1624 struct lp_build_context *uint = &ctx->bld_base.uint_bld;
1625 LLVMValueRef vtx_offset, soffset;
1626 struct tgsi_shader_info *info = &shader->selector->info;
1627 unsigned semantic_name = info->input_semantic_name[input_index];
1628 unsigned semantic_index = info->input_semantic_index[input_index];
1629 unsigned param;
1630 LLVMValueRef value;
1631
1632 param = si_shader_io_get_unique_index(semantic_name, semantic_index, false);
1633
1634 /* GFX9 has the ESGS ring in LDS. */
1635 if (ctx->screen->info.chip_class >= GFX9) {
1636 unsigned index = vtx_offset_param;
1637
1638 switch (index / 2) {
1639 case 0:
1640 vtx_offset = si_unpack_param(ctx, ctx->param_gs_vtx01_offset,
1641 index % 2 ? 16 : 0, 16);
1642 break;
1643 case 1:
1644 vtx_offset = si_unpack_param(ctx, ctx->param_gs_vtx23_offset,
1645 index % 2 ? 16 : 0, 16);
1646 break;
1647 case 2:
1648 vtx_offset = si_unpack_param(ctx, ctx->param_gs_vtx45_offset,
1649 index % 2 ? 16 : 0, 16);
1650 break;
1651 default:
1652 assert(0);
1653 return NULL;
1654 }
1655
1656 vtx_offset = LLVMBuildAdd(ctx->ac.builder, vtx_offset,
1657 LLVMConstInt(ctx->i32, param * 4, 0), "");
1658 return lds_load(bld_base, type, swizzle, vtx_offset);
1659 }
1660
1661 /* GFX6: input load from the ESGS ring in memory. */
1662 if (swizzle == ~0) {
1663 LLVMValueRef values[TGSI_NUM_CHANNELS];
1664 unsigned chan;
1665 for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) {
1666 values[chan] = si_llvm_load_input_gs(abi, input_index, vtx_offset_param,
1667 type, chan);
1668 }
1669 return lp_build_gather_values(&ctx->gallivm, values,
1670 TGSI_NUM_CHANNELS);
1671 }
1672
1673 /* Get the vertex offset parameter on GFX6. */
1674 LLVMValueRef gs_vtx_offset = ctx->gs_vtx_offset[vtx_offset_param];
1675
1676 vtx_offset = lp_build_mul_imm(uint, gs_vtx_offset, 4);
1677
1678 soffset = LLVMConstInt(ctx->i32, (param * 4 + swizzle) * 256, 0);
1679
1680 value = ac_build_buffer_load(&ctx->ac, ctx->esgs_ring, 1, ctx->i32_0,
1681 vtx_offset, soffset, 0, 1, 0, true, false);
1682 if (llvm_type_is_64bit(ctx, type)) {
1683 LLVMValueRef value2;
1684 soffset = LLVMConstInt(ctx->i32, (param * 4 + swizzle + 1) * 256, 0);
1685
1686 value2 = ac_build_buffer_load(&ctx->ac, ctx->esgs_ring, 1,
1687 ctx->i32_0, vtx_offset, soffset,
1688 0, 1, 0, true, false);
1689 return si_llvm_emit_fetch_64bit(bld_base, type, value, value2);
1690 }
1691 return LLVMBuildBitCast(ctx->ac.builder, value, type, "");
1692 }
1693
1694 static LLVMValueRef si_nir_load_input_gs(struct ac_shader_abi *abi,
1695 unsigned location,
1696 unsigned driver_location,
1697 unsigned component,
1698 unsigned num_components,
1699 unsigned vertex_index,
1700 unsigned const_index,
1701 LLVMTypeRef type)
1702 {
1703 struct si_shader_context *ctx = si_shader_context_from_abi(abi);
1704
1705 LLVMValueRef value[4];
1706 for (unsigned i = 0; i < num_components; i++) {
1707 unsigned offset = i;
1708 if (llvm_type_is_64bit(ctx, type))
1709 offset *= 2;
1710
1711 offset += component;
1712 value[i + component] = si_llvm_load_input_gs(&ctx->abi, driver_location / 4,
1713 vertex_index, type, offset);
1714 }
1715
1716 return ac_build_varying_gather_values(&ctx->ac, value, num_components, component);
1717 }
1718
1719 static LLVMValueRef fetch_input_gs(
1720 struct lp_build_tgsi_context *bld_base,
1721 const struct tgsi_full_src_register *reg,
1722 enum tgsi_opcode_type type,
1723 unsigned swizzle)
1724 {
1725 struct si_shader_context *ctx = si_shader_context(bld_base);
1726 struct tgsi_shader_info *info = &ctx->shader->selector->info;
1727
1728 unsigned semantic_name = info->input_semantic_name[reg->Register.Index];
1729 if (swizzle != ~0 && semantic_name == TGSI_SEMANTIC_PRIMID)
1730 return get_primitive_id(ctx, swizzle);
1731
1732 if (!reg->Register.Dimension)
1733 return NULL;
1734
1735 return si_llvm_load_input_gs(&ctx->abi, reg->Register.Index,
1736 reg->Dimension.Index,
1737 tgsi2llvmtype(bld_base, type),
1738 swizzle);
1739 }
1740
1741 static int lookup_interp_param_index(unsigned interpolate, unsigned location)
1742 {
1743 switch (interpolate) {
1744 case TGSI_INTERPOLATE_CONSTANT:
1745 return 0;
1746
1747 case TGSI_INTERPOLATE_LINEAR:
1748 if (location == TGSI_INTERPOLATE_LOC_SAMPLE)
1749 return SI_PARAM_LINEAR_SAMPLE;
1750 else if (location == TGSI_INTERPOLATE_LOC_CENTROID)
1751 return SI_PARAM_LINEAR_CENTROID;
1752 else
1753 return SI_PARAM_LINEAR_CENTER;
1754 break;
1755 case TGSI_INTERPOLATE_COLOR:
1756 case TGSI_INTERPOLATE_PERSPECTIVE:
1757 if (location == TGSI_INTERPOLATE_LOC_SAMPLE)
1758 return SI_PARAM_PERSP_SAMPLE;
1759 else if (location == TGSI_INTERPOLATE_LOC_CENTROID)
1760 return SI_PARAM_PERSP_CENTROID;
1761 else
1762 return SI_PARAM_PERSP_CENTER;
1763 break;
1764 default:
1765 fprintf(stderr, "Warning: Unhandled interpolation mode.\n");
1766 return -1;
1767 }
1768 }
1769
1770 static LLVMValueRef si_build_fs_interp(struct si_shader_context *ctx,
1771 unsigned attr_index, unsigned chan,
1772 LLVMValueRef prim_mask,
1773 LLVMValueRef i, LLVMValueRef j)
1774 {
1775 if (i || j) {
1776 return ac_build_fs_interp(&ctx->ac,
1777 LLVMConstInt(ctx->i32, chan, 0),
1778 LLVMConstInt(ctx->i32, attr_index, 0),
1779 prim_mask, i, j);
1780 }
1781 return ac_build_fs_interp_mov(&ctx->ac,
1782 LLVMConstInt(ctx->i32, 2, 0), /* P0 */
1783 LLVMConstInt(ctx->i32, chan, 0),
1784 LLVMConstInt(ctx->i32, attr_index, 0),
1785 prim_mask);
1786 }
1787
1788 /**
1789 * Interpolate a fragment shader input.
1790 *
1791 * @param ctx context
1792 * @param input_index index of the input in hardware
1793 * @param semantic_name TGSI_SEMANTIC_*
1794 * @param semantic_index semantic index
1795 * @param num_interp_inputs number of all interpolated inputs (= BCOLOR offset)
1796 * @param colors_read_mask color components read (4 bits for each color, 8 bits in total)
1797 * @param interp_param interpolation weights (i,j)
1798 * @param prim_mask SI_PARAM_PRIM_MASK
1799 * @param face SI_PARAM_FRONT_FACE
1800 * @param result the return value (4 components)
1801 */
1802 static void interp_fs_input(struct si_shader_context *ctx,
1803 unsigned input_index,
1804 unsigned semantic_name,
1805 unsigned semantic_index,
1806 unsigned num_interp_inputs,
1807 unsigned colors_read_mask,
1808 LLVMValueRef interp_param,
1809 LLVMValueRef prim_mask,
1810 LLVMValueRef face,
1811 LLVMValueRef result[4])
1812 {
1813 LLVMValueRef i = NULL, j = NULL;
1814 unsigned chan;
1815
1816 /* fs.constant returns the param from the middle vertex, so it's not
1817 * really useful for flat shading. It's meant to be used for custom
1818 * interpolation (but the intrinsic can't fetch from the other two
1819 * vertices).
1820 *
1821 * Luckily, it doesn't matter, because we rely on the FLAT_SHADE state
1822 * to do the right thing. The only reason we use fs.constant is that
1823 * fs.interp cannot be used on integers, because they can be equal
1824 * to NaN.
1825 *
1826 * When interp is false we will use fs.constant or for newer llvm,
1827 * amdgcn.interp.mov.
1828 */
1829 bool interp = interp_param != NULL;
1830
1831 if (interp) {
1832 interp_param = LLVMBuildBitCast(ctx->ac.builder, interp_param,
1833 LLVMVectorType(ctx->f32, 2), "");
1834
1835 i = LLVMBuildExtractElement(ctx->ac.builder, interp_param,
1836 ctx->i32_0, "");
1837 j = LLVMBuildExtractElement(ctx->ac.builder, interp_param,
1838 ctx->i32_1, "");
1839 }
1840
1841 if (semantic_name == TGSI_SEMANTIC_COLOR &&
1842 ctx->shader->key.part.ps.prolog.color_two_side) {
1843 LLVMValueRef is_face_positive;
1844
1845 /* If BCOLOR0 is used, BCOLOR1 is at offset "num_inputs + 1",
1846 * otherwise it's at offset "num_inputs".
1847 */
1848 unsigned back_attr_offset = num_interp_inputs;
1849 if (semantic_index == 1 && colors_read_mask & 0xf)
1850 back_attr_offset += 1;
1851
1852 is_face_positive = LLVMBuildICmp(ctx->ac.builder, LLVMIntNE,
1853 face, ctx->i32_0, "");
1854
1855 for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) {
1856 LLVMValueRef front, back;
1857
1858 front = si_build_fs_interp(ctx,
1859 input_index, chan,
1860 prim_mask, i, j);
1861 back = si_build_fs_interp(ctx,
1862 back_attr_offset, chan,
1863 prim_mask, i, j);
1864
1865 result[chan] = LLVMBuildSelect(ctx->ac.builder,
1866 is_face_positive,
1867 front,
1868 back,
1869 "");
1870 }
1871 } else if (semantic_name == TGSI_SEMANTIC_FOG) {
1872 result[0] = si_build_fs_interp(ctx, input_index,
1873 0, prim_mask, i, j);
1874 result[1] =
1875 result[2] = LLVMConstReal(ctx->f32, 0.0f);
1876 result[3] = LLVMConstReal(ctx->f32, 1.0f);
1877 } else {
1878 for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) {
1879 result[chan] = si_build_fs_interp(ctx,
1880 input_index, chan,
1881 prim_mask, i, j);
1882 }
1883 }
1884 }
1885
1886 void si_llvm_load_input_fs(
1887 struct si_shader_context *ctx,
1888 unsigned input_index,
1889 LLVMValueRef out[4])
1890 {
1891 struct lp_build_context *base = &ctx->bld_base.base;
1892 struct si_shader *shader = ctx->shader;
1893 struct tgsi_shader_info *info = &shader->selector->info;
1894 LLVMValueRef main_fn = ctx->main_fn;
1895 LLVMValueRef interp_param = NULL;
1896 int interp_param_idx;
1897 enum tgsi_semantic semantic_name = info->input_semantic_name[input_index];
1898 unsigned semantic_index = info->input_semantic_index[input_index];
1899 enum tgsi_interpolate_mode interp_mode = info->input_interpolate[input_index];
1900 enum tgsi_interpolate_loc interp_loc = info->input_interpolate_loc[input_index];
1901
1902 /* Get colors from input VGPRs (set by the prolog). */
1903 if (semantic_name == TGSI_SEMANTIC_COLOR) {
1904 unsigned colors_read = shader->selector->info.colors_read;
1905 unsigned mask = colors_read >> (semantic_index * 4);
1906 unsigned offset = SI_PARAM_POS_FIXED_PT + 1 +
1907 (semantic_index ? util_bitcount(colors_read & 0xf) : 0);
1908
1909 out[0] = mask & 0x1 ? LLVMGetParam(main_fn, offset++) : base->undef;
1910 out[1] = mask & 0x2 ? LLVMGetParam(main_fn, offset++) : base->undef;
1911 out[2] = mask & 0x4 ? LLVMGetParam(main_fn, offset++) : base->undef;
1912 out[3] = mask & 0x8 ? LLVMGetParam(main_fn, offset++) : base->undef;
1913 return;
1914 }
1915
1916 interp_param_idx = lookup_interp_param_index(interp_mode, interp_loc);
1917 if (interp_param_idx == -1)
1918 return;
1919 else if (interp_param_idx) {
1920 interp_param = LLVMGetParam(ctx->main_fn, interp_param_idx);
1921 }
1922
1923 interp_fs_input(ctx, input_index, semantic_name,
1924 semantic_index, 0, /* this param is unused */
1925 shader->selector->info.colors_read, interp_param,
1926 ctx->abi.prim_mask,
1927 LLVMGetParam(main_fn, SI_PARAM_FRONT_FACE),
1928 &out[0]);
1929 }
1930
1931 static void declare_input_fs(
1932 struct si_shader_context *ctx,
1933 unsigned input_index,
1934 const struct tgsi_full_declaration *decl,
1935 LLVMValueRef out[4])
1936 {
1937 si_llvm_load_input_fs(ctx, input_index, out);
1938 }
1939
1940 LLVMValueRef si_get_sample_id(struct si_shader_context *ctx)
1941 {
1942 return si_unpack_param(ctx, SI_PARAM_ANCILLARY, 8, 4);
1943 }
1944
1945 static LLVMValueRef get_base_vertex(struct ac_shader_abi *abi)
1946 {
1947 struct si_shader_context *ctx = si_shader_context_from_abi(abi);
1948
1949 /* For non-indexed draws, the base vertex set by the driver
1950 * (for direct draws) or the CP (for indirect draws) is the
1951 * first vertex ID, but GLSL expects 0 to be returned.
1952 */
1953 LLVMValueRef vs_state = LLVMGetParam(ctx->main_fn,
1954 ctx->param_vs_state_bits);
1955 LLVMValueRef indexed;
1956
1957 indexed = LLVMBuildLShr(ctx->ac.builder, vs_state, ctx->i32_1, "");
1958 indexed = LLVMBuildTrunc(ctx->ac.builder, indexed, ctx->i1, "");
1959
1960 return LLVMBuildSelect(ctx->ac.builder, indexed, ctx->abi.base_vertex,
1961 ctx->i32_0, "");
1962 }
1963
1964 static LLVMValueRef get_block_size(struct ac_shader_abi *abi)
1965 {
1966 struct si_shader_context *ctx = si_shader_context_from_abi(abi);
1967
1968 LLVMValueRef values[3];
1969 LLVMValueRef result;
1970 unsigned i;
1971 unsigned *properties = ctx->shader->selector->info.properties;
1972
1973 if (properties[TGSI_PROPERTY_CS_FIXED_BLOCK_WIDTH] != 0) {
1974 unsigned sizes[3] = {
1975 properties[TGSI_PROPERTY_CS_FIXED_BLOCK_WIDTH],
1976 properties[TGSI_PROPERTY_CS_FIXED_BLOCK_HEIGHT],
1977 properties[TGSI_PROPERTY_CS_FIXED_BLOCK_DEPTH]
1978 };
1979
1980 for (i = 0; i < 3; ++i)
1981 values[i] = LLVMConstInt(ctx->i32, sizes[i], 0);
1982
1983 result = lp_build_gather_values(&ctx->gallivm, values, 3);
1984 } else {
1985 result = LLVMGetParam(ctx->main_fn, ctx->param_block_size);
1986 }
1987
1988 return result;
1989 }
1990
1991 /**
1992 * Load a dword from a constant buffer.
1993 */
1994 static LLVMValueRef buffer_load_const(struct si_shader_context *ctx,
1995 LLVMValueRef resource,
1996 LLVMValueRef offset)
1997 {
1998 return ac_build_buffer_load(&ctx->ac, resource, 1, NULL, offset, NULL,
1999 0, 0, 0, true, true);
2000 }
2001
2002 static LLVMValueRef load_sample_position(struct ac_shader_abi *abi, LLVMValueRef sample_id)
2003 {
2004 struct si_shader_context *ctx = si_shader_context_from_abi(abi);
2005 struct lp_build_context *uint_bld = &ctx->bld_base.uint_bld;
2006 LLVMValueRef desc = LLVMGetParam(ctx->main_fn, ctx->param_rw_buffers);
2007 LLVMValueRef buf_index = LLVMConstInt(ctx->i32, SI_PS_CONST_SAMPLE_POSITIONS, 0);
2008 LLVMValueRef resource = ac_build_load_to_sgpr(&ctx->ac, desc, buf_index);
2009
2010 /* offset = sample_id * 8 (8 = 2 floats containing samplepos.xy) */
2011 LLVMValueRef offset0 = lp_build_mul_imm(uint_bld, sample_id, 8);
2012 LLVMValueRef offset1 = LLVMBuildAdd(ctx->ac.builder, offset0, LLVMConstInt(ctx->i32, 4, 0), "");
2013
2014 LLVMValueRef pos[4] = {
2015 buffer_load_const(ctx, resource, offset0),
2016 buffer_load_const(ctx, resource, offset1),
2017 LLVMConstReal(ctx->f32, 0),
2018 LLVMConstReal(ctx->f32, 0)
2019 };
2020
2021 return lp_build_gather_values(&ctx->gallivm, pos, 4);
2022 }
2023
2024 static LLVMValueRef load_sample_mask_in(struct ac_shader_abi *abi)
2025 {
2026 struct si_shader_context *ctx = si_shader_context_from_abi(abi);
2027 return ac_to_integer(&ctx->ac, abi->sample_coverage);
2028 }
2029
2030 static LLVMValueRef si_load_tess_coord(struct ac_shader_abi *abi)
2031 {
2032 struct si_shader_context *ctx = si_shader_context_from_abi(abi);
2033 struct lp_build_context *bld = &ctx->bld_base.base;
2034
2035 LLVMValueRef coord[4] = {
2036 LLVMGetParam(ctx->main_fn, ctx->param_tes_u),
2037 LLVMGetParam(ctx->main_fn, ctx->param_tes_v),
2038 ctx->ac.f32_0,
2039 ctx->ac.f32_0
2040 };
2041
2042 /* For triangles, the vector should be (u, v, 1-u-v). */
2043 if (ctx->shader->selector->info.properties[TGSI_PROPERTY_TES_PRIM_MODE] ==
2044 PIPE_PRIM_TRIANGLES)
2045 coord[2] = lp_build_sub(bld, ctx->ac.f32_1,
2046 lp_build_add(bld, coord[0], coord[1]));
2047
2048 return lp_build_gather_values(&ctx->gallivm, coord, 4);
2049 }
2050
2051 static LLVMValueRef load_tess_level(struct si_shader_context *ctx,
2052 unsigned semantic_name)
2053 {
2054 LLVMValueRef base, addr;
2055
2056 int param = si_shader_io_get_unique_index_patch(semantic_name, 0);
2057
2058 base = LLVMGetParam(ctx->main_fn, ctx->param_tcs_offchip_offset);
2059 addr = get_tcs_tes_buffer_address(ctx, get_rel_patch_id(ctx), NULL,
2060 LLVMConstInt(ctx->i32, param, 0));
2061
2062 return buffer_load(&ctx->bld_base, ctx->f32,
2063 ~0, ctx->tess_offchip_ring, base, addr, true);
2064
2065 }
2066
2067 static LLVMValueRef si_load_tess_level(struct ac_shader_abi *abi,
2068 unsigned varying_id)
2069 {
2070 struct si_shader_context *ctx = si_shader_context_from_abi(abi);
2071 unsigned semantic_name;
2072
2073 switch (varying_id) {
2074 case VARYING_SLOT_TESS_LEVEL_INNER:
2075 semantic_name = TGSI_SEMANTIC_TESSINNER;
2076 break;
2077 case VARYING_SLOT_TESS_LEVEL_OUTER:
2078 semantic_name = TGSI_SEMANTIC_TESSOUTER;
2079 break;
2080 default:
2081 unreachable("unknown tess level");
2082 }
2083
2084 return load_tess_level(ctx, semantic_name);
2085
2086 }
2087
2088 static LLVMValueRef si_load_patch_vertices_in(struct ac_shader_abi *abi)
2089 {
2090 struct si_shader_context *ctx = si_shader_context_from_abi(abi);
2091 if (ctx->type == PIPE_SHADER_TESS_CTRL)
2092 return si_unpack_param(ctx, ctx->param_tcs_out_lds_layout, 13, 6);
2093 else if (ctx->type == PIPE_SHADER_TESS_EVAL)
2094 return get_num_tcs_out_vertices(ctx);
2095 else
2096 unreachable("invalid shader stage for TGSI_SEMANTIC_VERTICESIN");
2097 }
2098
2099 void si_load_system_value(struct si_shader_context *ctx,
2100 unsigned index,
2101 const struct tgsi_full_declaration *decl)
2102 {
2103 LLVMValueRef value = 0;
2104
2105 assert(index < RADEON_LLVM_MAX_SYSTEM_VALUES);
2106
2107 switch (decl->Semantic.Name) {
2108 case TGSI_SEMANTIC_INSTANCEID:
2109 value = ctx->abi.instance_id;
2110 break;
2111
2112 case TGSI_SEMANTIC_VERTEXID:
2113 value = LLVMBuildAdd(ctx->ac.builder,
2114 ctx->abi.vertex_id,
2115 ctx->abi.base_vertex, "");
2116 break;
2117
2118 case TGSI_SEMANTIC_VERTEXID_NOBASE:
2119 /* Unused. Clarify the meaning in indexed vs. non-indexed
2120 * draws if this is ever used again. */
2121 assert(false);
2122 break;
2123
2124 case TGSI_SEMANTIC_BASEVERTEX:
2125 value = get_base_vertex(&ctx->abi);
2126 break;
2127
2128 case TGSI_SEMANTIC_BASEINSTANCE:
2129 value = ctx->abi.start_instance;
2130 break;
2131
2132 case TGSI_SEMANTIC_DRAWID:
2133 value = ctx->abi.draw_id;
2134 break;
2135
2136 case TGSI_SEMANTIC_INVOCATIONID:
2137 if (ctx->type == PIPE_SHADER_TESS_CTRL)
2138 value = unpack_llvm_param(ctx, ctx->abi.tcs_rel_ids, 8, 5);
2139 else if (ctx->type == PIPE_SHADER_GEOMETRY)
2140 value = ctx->abi.gs_invocation_id;
2141 else
2142 assert(!"INVOCATIONID not implemented");
2143 break;
2144
2145 case TGSI_SEMANTIC_POSITION:
2146 {
2147 LLVMValueRef pos[4] = {
2148 LLVMGetParam(ctx->main_fn, SI_PARAM_POS_X_FLOAT),
2149 LLVMGetParam(ctx->main_fn, SI_PARAM_POS_Y_FLOAT),
2150 LLVMGetParam(ctx->main_fn, SI_PARAM_POS_Z_FLOAT),
2151 lp_build_emit_llvm_unary(&ctx->bld_base, TGSI_OPCODE_RCP,
2152 LLVMGetParam(ctx->main_fn,
2153 SI_PARAM_POS_W_FLOAT)),
2154 };
2155 value = lp_build_gather_values(&ctx->gallivm, pos, 4);
2156 break;
2157 }
2158
2159 case TGSI_SEMANTIC_FACE:
2160 value = ctx->abi.front_face;
2161 break;
2162
2163 case TGSI_SEMANTIC_SAMPLEID:
2164 value = si_get_sample_id(ctx);
2165 break;
2166
2167 case TGSI_SEMANTIC_SAMPLEPOS: {
2168 LLVMValueRef pos[4] = {
2169 LLVMGetParam(ctx->main_fn, SI_PARAM_POS_X_FLOAT),
2170 LLVMGetParam(ctx->main_fn, SI_PARAM_POS_Y_FLOAT),
2171 LLVMConstReal(ctx->f32, 0),
2172 LLVMConstReal(ctx->f32, 0)
2173 };
2174 pos[0] = lp_build_emit_llvm_unary(&ctx->bld_base,
2175 TGSI_OPCODE_FRC, pos[0]);
2176 pos[1] = lp_build_emit_llvm_unary(&ctx->bld_base,
2177 TGSI_OPCODE_FRC, pos[1]);
2178 value = lp_build_gather_values(&ctx->gallivm, pos, 4);
2179 break;
2180 }
2181
2182 case TGSI_SEMANTIC_SAMPLEMASK:
2183 /* This can only occur with the OpenGL Core profile, which
2184 * doesn't support smoothing.
2185 */
2186 value = LLVMGetParam(ctx->main_fn, SI_PARAM_SAMPLE_COVERAGE);
2187 break;
2188
2189 case TGSI_SEMANTIC_TESSCOORD:
2190 value = si_load_tess_coord(&ctx->abi);
2191 break;
2192
2193 case TGSI_SEMANTIC_VERTICESIN:
2194 value = si_load_patch_vertices_in(&ctx->abi);
2195 break;
2196
2197 case TGSI_SEMANTIC_TESSINNER:
2198 case TGSI_SEMANTIC_TESSOUTER:
2199 value = load_tess_level(ctx, decl->Semantic.Name);
2200 break;
2201
2202 case TGSI_SEMANTIC_DEFAULT_TESSOUTER_SI:
2203 case TGSI_SEMANTIC_DEFAULT_TESSINNER_SI:
2204 {
2205 LLVMValueRef buf, slot, val[4];
2206 int i, offset;
2207
2208 slot = LLVMConstInt(ctx->i32, SI_HS_CONST_DEFAULT_TESS_LEVELS, 0);
2209 buf = LLVMGetParam(ctx->main_fn, ctx->param_rw_buffers);
2210 buf = ac_build_load_to_sgpr(&ctx->ac, buf, slot);
2211 offset = decl->Semantic.Name == TGSI_SEMANTIC_DEFAULT_TESSINNER_SI ? 4 : 0;
2212
2213 for (i = 0; i < 4; i++)
2214 val[i] = buffer_load_const(ctx, buf,
2215 LLVMConstInt(ctx->i32, (offset + i) * 4, 0));
2216 value = lp_build_gather_values(&ctx->gallivm, val, 4);
2217 break;
2218 }
2219
2220 case TGSI_SEMANTIC_PRIMID:
2221 value = get_primitive_id(ctx, 0);
2222 break;
2223
2224 case TGSI_SEMANTIC_GRID_SIZE:
2225 value = ctx->abi.num_work_groups;
2226 break;
2227
2228 case TGSI_SEMANTIC_BLOCK_SIZE:
2229 value = get_block_size(&ctx->abi);
2230 break;
2231
2232 case TGSI_SEMANTIC_BLOCK_ID:
2233 {
2234 LLVMValueRef values[3];
2235
2236 for (int i = 0; i < 3; i++) {
2237 values[i] = ctx->i32_0;
2238 if (ctx->abi.workgroup_ids[i]) {
2239 values[i] = ctx->abi.workgroup_ids[i];
2240 }
2241 }
2242 value = lp_build_gather_values(&ctx->gallivm, values, 3);
2243 break;
2244 }
2245
2246 case TGSI_SEMANTIC_THREAD_ID:
2247 value = ctx->abi.local_invocation_ids;
2248 break;
2249
2250 case TGSI_SEMANTIC_HELPER_INVOCATION:
2251 value = lp_build_intrinsic(ctx->ac.builder,
2252 "llvm.amdgcn.ps.live",
2253 ctx->i1, NULL, 0,
2254 LP_FUNC_ATTR_READNONE);
2255 value = LLVMBuildNot(ctx->ac.builder, value, "");
2256 value = LLVMBuildSExt(ctx->ac.builder, value, ctx->i32, "");
2257 break;
2258
2259 case TGSI_SEMANTIC_SUBGROUP_SIZE:
2260 value = LLVMConstInt(ctx->i32, 64, 0);
2261 break;
2262
2263 case TGSI_SEMANTIC_SUBGROUP_INVOCATION:
2264 value = ac_get_thread_id(&ctx->ac);
2265 break;
2266
2267 case TGSI_SEMANTIC_SUBGROUP_EQ_MASK:
2268 {
2269 LLVMValueRef id = ac_get_thread_id(&ctx->ac);
2270 id = LLVMBuildZExt(ctx->ac.builder, id, ctx->i64, "");
2271 value = LLVMBuildShl(ctx->ac.builder, LLVMConstInt(ctx->i64, 1, 0), id, "");
2272 value = LLVMBuildBitCast(ctx->ac.builder, value, ctx->v2i32, "");
2273 break;
2274 }
2275
2276 case TGSI_SEMANTIC_SUBGROUP_GE_MASK:
2277 case TGSI_SEMANTIC_SUBGROUP_GT_MASK:
2278 case TGSI_SEMANTIC_SUBGROUP_LE_MASK:
2279 case TGSI_SEMANTIC_SUBGROUP_LT_MASK:
2280 {
2281 LLVMValueRef id = ac_get_thread_id(&ctx->ac);
2282 if (decl->Semantic.Name == TGSI_SEMANTIC_SUBGROUP_GT_MASK ||
2283 decl->Semantic.Name == TGSI_SEMANTIC_SUBGROUP_LE_MASK) {
2284 /* All bits set except LSB */
2285 value = LLVMConstInt(ctx->i64, -2, 0);
2286 } else {
2287 /* All bits set */
2288 value = LLVMConstInt(ctx->i64, -1, 0);
2289 }
2290 id = LLVMBuildZExt(ctx->ac.builder, id, ctx->i64, "");
2291 value = LLVMBuildShl(ctx->ac.builder, value, id, "");
2292 if (decl->Semantic.Name == TGSI_SEMANTIC_SUBGROUP_LE_MASK ||
2293 decl->Semantic.Name == TGSI_SEMANTIC_SUBGROUP_LT_MASK)
2294 value = LLVMBuildNot(ctx->ac.builder, value, "");
2295 value = LLVMBuildBitCast(ctx->ac.builder, value, ctx->v2i32, "");
2296 break;
2297 }
2298
2299 default:
2300 assert(!"unknown system value");
2301 return;
2302 }
2303
2304 ctx->system_values[index] = value;
2305 }
2306
2307 void si_declare_compute_memory(struct si_shader_context *ctx)
2308 {
2309 struct si_shader_selector *sel = ctx->shader->selector;
2310
2311 LLVMTypeRef i8p = LLVMPointerType(ctx->i8, AC_LOCAL_ADDR_SPACE);
2312 LLVMValueRef var;
2313
2314 assert(!ctx->ac.lds);
2315
2316 var = LLVMAddGlobalInAddressSpace(ctx->ac.module,
2317 LLVMArrayType(ctx->i8, sel->local_size),
2318 "compute_lds",
2319 AC_LOCAL_ADDR_SPACE);
2320 LLVMSetAlignment(var, 4);
2321
2322 ctx->ac.lds = LLVMBuildBitCast(ctx->ac.builder, var, i8p, "");
2323 }
2324
2325 void si_tgsi_declare_compute_memory(struct si_shader_context *ctx,
2326 const struct tgsi_full_declaration *decl)
2327 {
2328 assert(decl->Declaration.MemType == TGSI_MEMORY_TYPE_SHARED);
2329 assert(decl->Range.First == decl->Range.Last);
2330
2331 si_declare_compute_memory(ctx);
2332 }
2333
2334 static LLVMValueRef load_const_buffer_desc_fast_path(struct si_shader_context *ctx)
2335 {
2336 LLVMValueRef ptr =
2337 LLVMGetParam(ctx->main_fn, ctx->param_const_and_shader_buffers);
2338 struct si_shader_selector *sel = ctx->shader->selector;
2339
2340 /* Do the bounds checking with a descriptor, because
2341 * doing computation and manual bounds checking of 64-bit
2342 * addresses generates horrible VALU code with very high
2343 * VGPR usage and very low SIMD occupancy.
2344 */
2345 ptr = LLVMBuildPtrToInt(ctx->ac.builder, ptr, ctx->ac.intptr, "");
2346
2347 LLVMValueRef desc0, desc1;
2348 if (HAVE_32BIT_POINTERS) {
2349 desc0 = ptr;
2350 desc1 = LLVMConstInt(ctx->i32,
2351 S_008F04_BASE_ADDRESS_HI(ctx->screen->info.address32_hi), 0);
2352 } else {
2353 ptr = LLVMBuildBitCast(ctx->ac.builder, ptr, ctx->v2i32, "");
2354 desc0 = LLVMBuildExtractElement(ctx->ac.builder, ptr, ctx->i32_0, "");
2355 desc1 = LLVMBuildExtractElement(ctx->ac.builder, ptr, ctx->i32_1, "");
2356 /* Mask out all bits except BASE_ADDRESS_HI. */
2357 desc1 = LLVMBuildAnd(ctx->ac.builder, desc1,
2358 LLVMConstInt(ctx->i32, ~C_008F04_BASE_ADDRESS_HI, 0), "");
2359 }
2360
2361 LLVMValueRef desc_elems[] = {
2362 desc0,
2363 desc1,
2364 LLVMConstInt(ctx->i32, (sel->info.const_file_max[0] + 1) * 16, 0),
2365 LLVMConstInt(ctx->i32,
2366 S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) |
2367 S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
2368 S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) |
2369 S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W) |
2370 S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
2371 S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32), 0)
2372 };
2373
2374 return ac_build_gather_values(&ctx->ac, desc_elems, 4);
2375 }
2376
2377 static LLVMValueRef load_const_buffer_desc(struct si_shader_context *ctx, int i)
2378 {
2379 LLVMValueRef list_ptr = LLVMGetParam(ctx->main_fn,
2380 ctx->param_const_and_shader_buffers);
2381
2382 return ac_build_load_to_sgpr(&ctx->ac, list_ptr,
2383 LLVMConstInt(ctx->i32, si_get_constbuf_slot(i), 0));
2384 }
2385
2386 static LLVMValueRef load_ubo(struct ac_shader_abi *abi, LLVMValueRef index)
2387 {
2388 struct si_shader_context *ctx = si_shader_context_from_abi(abi);
2389 struct si_shader_selector *sel = ctx->shader->selector;
2390
2391 LLVMValueRef ptr = LLVMGetParam(ctx->main_fn, ctx->param_const_and_shader_buffers);
2392
2393 if (sel->info.const_buffers_declared == 1 &&
2394 sel->info.shader_buffers_declared == 0) {
2395 return load_const_buffer_desc_fast_path(ctx);
2396 }
2397
2398 index = si_llvm_bound_index(ctx, index, ctx->num_const_buffers);
2399 index = LLVMBuildAdd(ctx->ac.builder, index,
2400 LLVMConstInt(ctx->i32, SI_NUM_SHADER_BUFFERS, 0), "");
2401
2402 return ac_build_load_to_sgpr(&ctx->ac, ptr, index);
2403 }
2404
2405 static LLVMValueRef
2406 load_ssbo(struct ac_shader_abi *abi, LLVMValueRef index, bool write)
2407 {
2408 struct si_shader_context *ctx = si_shader_context_from_abi(abi);
2409 LLVMValueRef rsrc_ptr = LLVMGetParam(ctx->main_fn,
2410 ctx->param_const_and_shader_buffers);
2411
2412 index = si_llvm_bound_index(ctx, index, ctx->num_shader_buffers);
2413 index = LLVMBuildSub(ctx->ac.builder,
2414 LLVMConstInt(ctx->i32, SI_NUM_SHADER_BUFFERS - 1, 0),
2415 index, "");
2416
2417 return ac_build_load_to_sgpr(&ctx->ac, rsrc_ptr, index);
2418 }
2419
2420 static LLVMValueRef fetch_constant(
2421 struct lp_build_tgsi_context *bld_base,
2422 const struct tgsi_full_src_register *reg,
2423 enum tgsi_opcode_type type,
2424 unsigned swizzle)
2425 {
2426 struct si_shader_context *ctx = si_shader_context(bld_base);
2427 struct si_shader_selector *sel = ctx->shader->selector;
2428 const struct tgsi_ind_register *ireg = &reg->Indirect;
2429 unsigned buf, idx;
2430
2431 LLVMValueRef addr, bufp;
2432
2433 if (swizzle == LP_CHAN_ALL) {
2434 unsigned chan;
2435 LLVMValueRef values[4];
2436 for (chan = 0; chan < TGSI_NUM_CHANNELS; ++chan)
2437 values[chan] = fetch_constant(bld_base, reg, type, chan);
2438
2439 return lp_build_gather_values(&ctx->gallivm, values, 4);
2440 }
2441
2442 /* Split 64-bit loads. */
2443 if (tgsi_type_is_64bit(type)) {
2444 LLVMValueRef lo, hi;
2445
2446 lo = fetch_constant(bld_base, reg, TGSI_TYPE_UNSIGNED, swizzle);
2447 hi = fetch_constant(bld_base, reg, TGSI_TYPE_UNSIGNED, swizzle + 1);
2448 return si_llvm_emit_fetch_64bit(bld_base, tgsi2llvmtype(bld_base, type),
2449 lo, hi);
2450 }
2451
2452 idx = reg->Register.Index * 4 + swizzle;
2453 if (reg->Register.Indirect) {
2454 addr = si_get_indirect_index(ctx, ireg, 16, idx * 4);
2455 } else {
2456 addr = LLVMConstInt(ctx->i32, idx * 4, 0);
2457 }
2458
2459 /* Fast path when user data SGPRs point to constant buffer 0 directly. */
2460 if (sel->info.const_buffers_declared == 1 &&
2461 sel->info.shader_buffers_declared == 0) {
2462
2463 /* This enables use of s_load_dword and flat_load_dword for const buffer 0
2464 * loads, and up to x4 load opcode merging. However, it leads to horrible
2465 * code reducing SIMD wave occupancy from 8 to 2 in many cases.
2466 *
2467 * Using s_buffer_load_dword (x1) seems to be the best option right now.
2468 *
2469 * LLVM 5.0 on SI doesn't insert a required s_nop between SALU setting
2470 * a descriptor and s_buffer_load_dword using it, so we can't expand
2471 * the pointer into a full descriptor like below. We have to use
2472 * s_load_dword instead. The only case when LLVM 5.0 would select
2473 * s_buffer_load_dword (that we have to prevent) is when we use use
2474 * a literal offset where we don't need bounds checking.
2475 */
2476 if (ctx->screen->info.chip_class == SI && HAVE_LLVM < 0x0600 &&
2477 !reg->Register.Indirect) {
2478 LLVMValueRef ptr =
2479 LLVMGetParam(ctx->main_fn, ctx->param_const_and_shader_buffers);
2480
2481 addr = LLVMBuildLShr(ctx->ac.builder, addr, LLVMConstInt(ctx->i32, 2, 0), "");
2482 LLVMValueRef result = ac_build_load_invariant(&ctx->ac, ptr, addr);
2483 return bitcast(bld_base, type, result);
2484 }
2485
2486 LLVMValueRef desc = load_const_buffer_desc_fast_path(ctx);
2487 LLVMValueRef result = buffer_load_const(ctx, desc, addr);
2488 return bitcast(bld_base, type, result);
2489 }
2490
2491 assert(reg->Register.Dimension);
2492 buf = reg->Dimension.Index;
2493
2494 if (reg->Dimension.Indirect) {
2495 LLVMValueRef ptr = LLVMGetParam(ctx->main_fn, ctx->param_const_and_shader_buffers);
2496 LLVMValueRef index;
2497 index = si_get_bounded_indirect_index(ctx, &reg->DimIndirect,
2498 reg->Dimension.Index,
2499 ctx->num_const_buffers);
2500 index = LLVMBuildAdd(ctx->ac.builder, index,
2501 LLVMConstInt(ctx->i32, SI_NUM_SHADER_BUFFERS, 0), "");
2502 bufp = ac_build_load_to_sgpr(&ctx->ac, ptr, index);
2503 } else
2504 bufp = load_const_buffer_desc(ctx, buf);
2505
2506 return bitcast(bld_base, type, buffer_load_const(ctx, bufp, addr));
2507 }
2508
2509 /* Initialize arguments for the shader export intrinsic */
2510 static void si_llvm_init_export_args(struct si_shader_context *ctx,
2511 LLVMValueRef *values,
2512 unsigned target,
2513 struct ac_export_args *args)
2514 {
2515 LLVMValueRef f32undef = LLVMGetUndef(ctx->ac.f32);
2516 unsigned spi_shader_col_format = V_028714_SPI_SHADER_32_ABGR;
2517 unsigned chan;
2518 bool is_int8, is_int10;
2519
2520 /* Default is 0xf. Adjusted below depending on the format. */
2521 args->enabled_channels = 0xf; /* writemask */
2522
2523 /* Specify whether the EXEC mask represents the valid mask */
2524 args->valid_mask = 0;
2525
2526 /* Specify whether this is the last export */
2527 args->done = 0;
2528
2529 /* Specify the target we are exporting */
2530 args->target = target;
2531
2532 if (ctx->type == PIPE_SHADER_FRAGMENT) {
2533 const struct si_shader_key *key = &ctx->shader->key;
2534 unsigned col_formats = key->part.ps.epilog.spi_shader_col_format;
2535 int cbuf = target - V_008DFC_SQ_EXP_MRT;
2536
2537 assert(cbuf >= 0 && cbuf < 8);
2538 spi_shader_col_format = (col_formats >> (cbuf * 4)) & 0xf;
2539 is_int8 = (key->part.ps.epilog.color_is_int8 >> cbuf) & 0x1;
2540 is_int10 = (key->part.ps.epilog.color_is_int10 >> cbuf) & 0x1;
2541 }
2542
2543 args->compr = false;
2544 args->out[0] = f32undef;
2545 args->out[1] = f32undef;
2546 args->out[2] = f32undef;
2547 args->out[3] = f32undef;
2548
2549 LLVMValueRef (*packf)(struct ac_llvm_context *ctx, LLVMValueRef args[2]) = NULL;
2550 LLVMValueRef (*packi)(struct ac_llvm_context *ctx, LLVMValueRef args[2],
2551 unsigned bits, bool hi) = NULL;
2552
2553 switch (spi_shader_col_format) {
2554 case V_028714_SPI_SHADER_ZERO:
2555 args->enabled_channels = 0; /* writemask */
2556 args->target = V_008DFC_SQ_EXP_NULL;
2557 break;
2558
2559 case V_028714_SPI_SHADER_32_R:
2560 args->enabled_channels = 1; /* writemask */
2561 args->out[0] = values[0];
2562 break;
2563
2564 case V_028714_SPI_SHADER_32_GR:
2565 args->enabled_channels = 0x3; /* writemask */
2566 args->out[0] = values[0];
2567 args->out[1] = values[1];
2568 break;
2569
2570 case V_028714_SPI_SHADER_32_AR:
2571 args->enabled_channels = 0x9; /* writemask */
2572 args->out[0] = values[0];
2573 args->out[3] = values[3];
2574 break;
2575
2576 case V_028714_SPI_SHADER_FP16_ABGR:
2577 packf = ac_build_cvt_pkrtz_f16;
2578 break;
2579
2580 case V_028714_SPI_SHADER_UNORM16_ABGR:
2581 packf = ac_build_cvt_pknorm_u16;
2582 break;
2583
2584 case V_028714_SPI_SHADER_SNORM16_ABGR:
2585 packf = ac_build_cvt_pknorm_i16;
2586 break;
2587
2588 case V_028714_SPI_SHADER_UINT16_ABGR:
2589 packi = ac_build_cvt_pk_u16;
2590 break;
2591
2592 case V_028714_SPI_SHADER_SINT16_ABGR:
2593 packi = ac_build_cvt_pk_i16;
2594 break;
2595
2596 case V_028714_SPI_SHADER_32_ABGR:
2597 memcpy(&args->out[0], values, sizeof(values[0]) * 4);
2598 break;
2599 }
2600
2601 /* Pack f16 or norm_i16/u16. */
2602 if (packf) {
2603 for (chan = 0; chan < 2; chan++) {
2604 LLVMValueRef pack_args[2] = {
2605 values[2 * chan],
2606 values[2 * chan + 1]
2607 };
2608 LLVMValueRef packed;
2609
2610 packed = packf(&ctx->ac, pack_args);
2611 args->out[chan] = ac_to_float(&ctx->ac, packed);
2612 }
2613 args->compr = 1; /* COMPR flag */
2614 }
2615 /* Pack i16/u16. */
2616 if (packi) {
2617 for (chan = 0; chan < 2; chan++) {
2618 LLVMValueRef pack_args[2] = {
2619 ac_to_integer(&ctx->ac, values[2 * chan]),
2620 ac_to_integer(&ctx->ac, values[2 * chan + 1])
2621 };
2622 LLVMValueRef packed;
2623
2624 packed = packi(&ctx->ac, pack_args,
2625 is_int8 ? 8 : is_int10 ? 10 : 16,
2626 chan == 1);
2627 args->out[chan] = ac_to_float(&ctx->ac, packed);
2628 }
2629 args->compr = 1; /* COMPR flag */
2630 }
2631 }
2632
2633 static void si_alpha_test(struct lp_build_tgsi_context *bld_base,
2634 LLVMValueRef alpha)
2635 {
2636 struct si_shader_context *ctx = si_shader_context(bld_base);
2637
2638 if (ctx->shader->key.part.ps.epilog.alpha_func != PIPE_FUNC_NEVER) {
2639 static LLVMRealPredicate cond_map[PIPE_FUNC_ALWAYS + 1] = {
2640 [PIPE_FUNC_LESS] = LLVMRealOLT,
2641 [PIPE_FUNC_EQUAL] = LLVMRealOEQ,
2642 [PIPE_FUNC_LEQUAL] = LLVMRealOLE,
2643 [PIPE_FUNC_GREATER] = LLVMRealOGT,
2644 [PIPE_FUNC_NOTEQUAL] = LLVMRealONE,
2645 [PIPE_FUNC_GEQUAL] = LLVMRealOGE,
2646 };
2647 LLVMRealPredicate cond = cond_map[ctx->shader->key.part.ps.epilog.alpha_func];
2648 assert(cond);
2649
2650 LLVMValueRef alpha_ref = LLVMGetParam(ctx->main_fn,
2651 SI_PARAM_ALPHA_REF);
2652 LLVMValueRef alpha_pass =
2653 LLVMBuildFCmp(ctx->ac.builder, cond, alpha, alpha_ref, "");
2654 ac_build_kill_if_false(&ctx->ac, alpha_pass);
2655 } else {
2656 ac_build_kill_if_false(&ctx->ac, LLVMConstInt(ctx->i1, 0, 0));
2657 }
2658 }
2659
2660 static LLVMValueRef si_scale_alpha_by_sample_mask(struct lp_build_tgsi_context *bld_base,
2661 LLVMValueRef alpha,
2662 unsigned samplemask_param)
2663 {
2664 struct si_shader_context *ctx = si_shader_context(bld_base);
2665 LLVMValueRef coverage;
2666
2667 /* alpha = alpha * popcount(coverage) / SI_NUM_SMOOTH_AA_SAMPLES */
2668 coverage = LLVMGetParam(ctx->main_fn,
2669 samplemask_param);
2670 coverage = ac_to_integer(&ctx->ac, coverage);
2671
2672 coverage = lp_build_intrinsic(ctx->ac.builder, "llvm.ctpop.i32",
2673 ctx->i32,
2674 &coverage, 1, LP_FUNC_ATTR_READNONE);
2675
2676 coverage = LLVMBuildUIToFP(ctx->ac.builder, coverage,
2677 ctx->f32, "");
2678
2679 coverage = LLVMBuildFMul(ctx->ac.builder, coverage,
2680 LLVMConstReal(ctx->f32,
2681 1.0 / SI_NUM_SMOOTH_AA_SAMPLES), "");
2682
2683 return LLVMBuildFMul(ctx->ac.builder, alpha, coverage, "");
2684 }
2685
2686 static void si_llvm_emit_clipvertex(struct si_shader_context *ctx,
2687 struct ac_export_args *pos, LLVMValueRef *out_elts)
2688 {
2689 unsigned reg_index;
2690 unsigned chan;
2691 unsigned const_chan;
2692 LLVMValueRef base_elt;
2693 LLVMValueRef ptr = LLVMGetParam(ctx->main_fn, ctx->param_rw_buffers);
2694 LLVMValueRef constbuf_index = LLVMConstInt(ctx->i32,
2695 SI_VS_CONST_CLIP_PLANES, 0);
2696 LLVMValueRef const_resource = ac_build_load_to_sgpr(&ctx->ac, ptr, constbuf_index);
2697
2698 for (reg_index = 0; reg_index < 2; reg_index ++) {
2699 struct ac_export_args *args = &pos[2 + reg_index];
2700
2701 args->out[0] =
2702 args->out[1] =
2703 args->out[2] =
2704 args->out[3] = LLVMConstReal(ctx->f32, 0.0f);
2705
2706 /* Compute dot products of position and user clip plane vectors */
2707 for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) {
2708 for (const_chan = 0; const_chan < TGSI_NUM_CHANNELS; const_chan++) {
2709 LLVMValueRef addr =
2710 LLVMConstInt(ctx->i32, ((reg_index * 4 + chan) * 4 +
2711 const_chan) * 4, 0);
2712 base_elt = buffer_load_const(ctx, const_resource,
2713 addr);
2714 args->out[chan] =
2715 lp_build_add(&ctx->bld_base.base, args->out[chan],
2716 lp_build_mul(&ctx->bld_base.base, base_elt,
2717 out_elts[const_chan]));
2718 }
2719 }
2720
2721 args->enabled_channels = 0xf;
2722 args->valid_mask = 0;
2723 args->done = 0;
2724 args->target = V_008DFC_SQ_EXP_POS + 2 + reg_index;
2725 args->compr = 0;
2726 }
2727 }
2728
2729 static void si_dump_streamout(struct pipe_stream_output_info *so)
2730 {
2731 unsigned i;
2732
2733 if (so->num_outputs)
2734 fprintf(stderr, "STREAMOUT\n");
2735
2736 for (i = 0; i < so->num_outputs; i++) {
2737 unsigned mask = ((1 << so->output[i].num_components) - 1) <<
2738 so->output[i].start_component;
2739 fprintf(stderr, " %i: BUF%i[%i..%i] <- OUT[%i].%s%s%s%s\n",
2740 i, so->output[i].output_buffer,
2741 so->output[i].dst_offset, so->output[i].dst_offset + so->output[i].num_components - 1,
2742 so->output[i].register_index,
2743 mask & 1 ? "x" : "",
2744 mask & 2 ? "y" : "",
2745 mask & 4 ? "z" : "",
2746 mask & 8 ? "w" : "");
2747 }
2748 }
2749
2750 static void emit_streamout_output(struct si_shader_context *ctx,
2751 LLVMValueRef const *so_buffers,
2752 LLVMValueRef const *so_write_offsets,
2753 struct pipe_stream_output *stream_out,
2754 struct si_shader_output_values *shader_out)
2755 {
2756 unsigned buf_idx = stream_out->output_buffer;
2757 unsigned start = stream_out->start_component;
2758 unsigned num_comps = stream_out->num_components;
2759 LLVMValueRef out[4];
2760
2761 assert(num_comps && num_comps <= 4);
2762 if (!num_comps || num_comps > 4)
2763 return;
2764
2765 /* Load the output as int. */
2766 for (int j = 0; j < num_comps; j++) {
2767 assert(stream_out->stream == shader_out->vertex_stream[start + j]);
2768
2769 out[j] = ac_to_integer(&ctx->ac, shader_out->values[start + j]);
2770 }
2771
2772 /* Pack the output. */
2773 LLVMValueRef vdata = NULL;
2774
2775 switch (num_comps) {
2776 case 1: /* as i32 */
2777 vdata = out[0];
2778 break;
2779 case 2: /* as v2i32 */
2780 case 3: /* as v4i32 (aligned to 4) */
2781 case 4: /* as v4i32 */
2782 vdata = LLVMGetUndef(LLVMVectorType(ctx->i32, util_next_power_of_two(num_comps)));
2783 for (int j = 0; j < num_comps; j++) {
2784 vdata = LLVMBuildInsertElement(ctx->ac.builder, vdata, out[j],
2785 LLVMConstInt(ctx->i32, j, 0), "");
2786 }
2787 break;
2788 }
2789
2790 ac_build_buffer_store_dword(&ctx->ac, so_buffers[buf_idx],
2791 vdata, num_comps,
2792 so_write_offsets[buf_idx],
2793 ctx->i32_0,
2794 stream_out->dst_offset * 4, 1, 1, true, false);
2795 }
2796
2797 /**
2798 * Write streamout data to buffers for vertex stream @p stream (different
2799 * vertex streams can occur for GS copy shaders).
2800 */
2801 static void si_llvm_emit_streamout(struct si_shader_context *ctx,
2802 struct si_shader_output_values *outputs,
2803 unsigned noutput, unsigned stream)
2804 {
2805 struct si_shader_selector *sel = ctx->shader->selector;
2806 struct pipe_stream_output_info *so = &sel->so;
2807 LLVMBuilderRef builder = ctx->ac.builder;
2808 int i;
2809 struct lp_build_if_state if_ctx;
2810
2811 /* Get bits [22:16], i.e. (so_param >> 16) & 127; */
2812 LLVMValueRef so_vtx_count =
2813 si_unpack_param(ctx, ctx->param_streamout_config, 16, 7);
2814
2815 LLVMValueRef tid = ac_get_thread_id(&ctx->ac);
2816
2817 /* can_emit = tid < so_vtx_count; */
2818 LLVMValueRef can_emit =
2819 LLVMBuildICmp(builder, LLVMIntULT, tid, so_vtx_count, "");
2820
2821 /* Emit the streamout code conditionally. This actually avoids
2822 * out-of-bounds buffer access. The hw tells us via the SGPR
2823 * (so_vtx_count) which threads are allowed to emit streamout data. */
2824 lp_build_if(&if_ctx, &ctx->gallivm, can_emit);
2825 {
2826 /* The buffer offset is computed as follows:
2827 * ByteOffset = streamout_offset[buffer_id]*4 +
2828 * (streamout_write_index + thread_id)*stride[buffer_id] +
2829 * attrib_offset
2830 */
2831
2832 LLVMValueRef so_write_index =
2833 LLVMGetParam(ctx->main_fn,
2834 ctx->param_streamout_write_index);
2835
2836 /* Compute (streamout_write_index + thread_id). */
2837 so_write_index = LLVMBuildAdd(builder, so_write_index, tid, "");
2838
2839 /* Load the descriptor and compute the write offset for each
2840 * enabled buffer. */
2841 LLVMValueRef so_write_offset[4] = {};
2842 LLVMValueRef so_buffers[4];
2843 LLVMValueRef buf_ptr = LLVMGetParam(ctx->main_fn,
2844 ctx->param_rw_buffers);
2845
2846 for (i = 0; i < 4; i++) {
2847 if (!so->stride[i])
2848 continue;
2849
2850 LLVMValueRef offset = LLVMConstInt(ctx->i32,
2851 SI_VS_STREAMOUT_BUF0 + i, 0);
2852
2853 so_buffers[i] = ac_build_load_to_sgpr(&ctx->ac, buf_ptr, offset);
2854
2855 LLVMValueRef so_offset = LLVMGetParam(ctx->main_fn,
2856 ctx->param_streamout_offset[i]);
2857 so_offset = LLVMBuildMul(builder, so_offset, LLVMConstInt(ctx->i32, 4, 0), "");
2858
2859 so_write_offset[i] = LLVMBuildMul(builder, so_write_index,
2860 LLVMConstInt(ctx->i32, so->stride[i]*4, 0), "");
2861 so_write_offset[i] = LLVMBuildAdd(builder, so_write_offset[i], so_offset, "");
2862 }
2863
2864 /* Write streamout data. */
2865 for (i = 0; i < so->num_outputs; i++) {
2866 unsigned reg = so->output[i].register_index;
2867
2868 if (reg >= noutput)
2869 continue;
2870
2871 if (stream != so->output[i].stream)
2872 continue;
2873
2874 emit_streamout_output(ctx, so_buffers, so_write_offset,
2875 &so->output[i], &outputs[reg]);
2876 }
2877 }
2878 lp_build_endif(&if_ctx);
2879 }
2880
2881 static void si_export_param(struct si_shader_context *ctx, unsigned index,
2882 LLVMValueRef *values)
2883 {
2884 struct ac_export_args args;
2885
2886 si_llvm_init_export_args(ctx, values,
2887 V_008DFC_SQ_EXP_PARAM + index, &args);
2888 ac_build_export(&ctx->ac, &args);
2889 }
2890
2891 static void si_build_param_exports(struct si_shader_context *ctx,
2892 struct si_shader_output_values *outputs,
2893 unsigned noutput)
2894 {
2895 struct si_shader *shader = ctx->shader;
2896 unsigned param_count = 0;
2897
2898 for (unsigned i = 0; i < noutput; i++) {
2899 unsigned semantic_name = outputs[i].semantic_name;
2900 unsigned semantic_index = outputs[i].semantic_index;
2901
2902 if (outputs[i].vertex_stream[0] != 0 &&
2903 outputs[i].vertex_stream[1] != 0 &&
2904 outputs[i].vertex_stream[2] != 0 &&
2905 outputs[i].vertex_stream[3] != 0)
2906 continue;
2907
2908 switch (semantic_name) {
2909 case TGSI_SEMANTIC_LAYER:
2910 case TGSI_SEMANTIC_VIEWPORT_INDEX:
2911 case TGSI_SEMANTIC_CLIPDIST:
2912 case TGSI_SEMANTIC_COLOR:
2913 case TGSI_SEMANTIC_BCOLOR:
2914 case TGSI_SEMANTIC_PRIMID:
2915 case TGSI_SEMANTIC_FOG:
2916 case TGSI_SEMANTIC_TEXCOORD:
2917 case TGSI_SEMANTIC_GENERIC:
2918 break;
2919 default:
2920 continue;
2921 }
2922
2923 if ((semantic_name != TGSI_SEMANTIC_GENERIC ||
2924 semantic_index < SI_MAX_IO_GENERIC) &&
2925 shader->key.opt.kill_outputs &
2926 (1ull << si_shader_io_get_unique_index(semantic_name,
2927 semantic_index, true)))
2928 continue;
2929
2930 si_export_param(ctx, param_count, outputs[i].values);
2931
2932 assert(i < ARRAY_SIZE(shader->info.vs_output_param_offset));
2933 shader->info.vs_output_param_offset[i] = param_count++;
2934 }
2935
2936 shader->info.nr_param_exports = param_count;
2937 }
2938
2939 /* Generate export instructions for hardware VS shader stage */
2940 static void si_llvm_export_vs(struct si_shader_context *ctx,
2941 struct si_shader_output_values *outputs,
2942 unsigned noutput)
2943 {
2944 struct si_shader *shader = ctx->shader;
2945 struct ac_export_args pos_args[4] = {};
2946 LLVMValueRef psize_value = NULL, edgeflag_value = NULL, layer_value = NULL, viewport_index_value = NULL;
2947 unsigned pos_idx;
2948 int i;
2949
2950 /* Build position exports. */
2951 for (i = 0; i < noutput; i++) {
2952 switch (outputs[i].semantic_name) {
2953 case TGSI_SEMANTIC_POSITION:
2954 si_llvm_init_export_args(ctx, outputs[i].values,
2955 V_008DFC_SQ_EXP_POS, &pos_args[0]);
2956 break;
2957 case TGSI_SEMANTIC_PSIZE:
2958 psize_value = outputs[i].values[0];
2959 break;
2960 case TGSI_SEMANTIC_LAYER:
2961 layer_value = outputs[i].values[0];
2962 break;
2963 case TGSI_SEMANTIC_VIEWPORT_INDEX:
2964 viewport_index_value = outputs[i].values[0];
2965 break;
2966 case TGSI_SEMANTIC_EDGEFLAG:
2967 edgeflag_value = outputs[i].values[0];
2968 break;
2969 case TGSI_SEMANTIC_CLIPDIST:
2970 if (!shader->key.opt.clip_disable) {
2971 unsigned index = 2 + outputs[i].semantic_index;
2972 si_llvm_init_export_args(ctx, outputs[i].values,
2973 V_008DFC_SQ_EXP_POS + index,
2974 &pos_args[index]);
2975 }
2976 break;
2977 case TGSI_SEMANTIC_CLIPVERTEX:
2978 if (!shader->key.opt.clip_disable) {
2979 si_llvm_emit_clipvertex(ctx, pos_args,
2980 outputs[i].values);
2981 }
2982 break;
2983 }
2984 }
2985
2986 /* We need to add the position output manually if it's missing. */
2987 if (!pos_args[0].out[0]) {
2988 pos_args[0].enabled_channels = 0xf; /* writemask */
2989 pos_args[0].valid_mask = 0; /* EXEC mask */
2990 pos_args[0].done = 0; /* last export? */
2991 pos_args[0].target = V_008DFC_SQ_EXP_POS;
2992 pos_args[0].compr = 0; /* COMPR flag */
2993 pos_args[0].out[0] = ctx->ac.f32_0; /* X */
2994 pos_args[0].out[1] = ctx->ac.f32_0; /* Y */
2995 pos_args[0].out[2] = ctx->ac.f32_0; /* Z */
2996 pos_args[0].out[3] = ctx->ac.f32_1; /* W */
2997 }
2998
2999 /* Write the misc vector (point size, edgeflag, layer, viewport). */
3000 if (shader->selector->info.writes_psize ||
3001 shader->selector->info.writes_edgeflag ||
3002 shader->selector->info.writes_viewport_index ||
3003 shader->selector->info.writes_layer) {
3004 pos_args[1].enabled_channels = shader->selector->info.writes_psize |
3005 (shader->selector->info.writes_edgeflag << 1) |
3006 (shader->selector->info.writes_layer << 2);
3007
3008 pos_args[1].valid_mask = 0; /* EXEC mask */
3009 pos_args[1].done = 0; /* last export? */
3010 pos_args[1].target = V_008DFC_SQ_EXP_POS + 1;
3011 pos_args[1].compr = 0; /* COMPR flag */
3012 pos_args[1].out[0] = ctx->ac.f32_0; /* X */
3013 pos_args[1].out[1] = ctx->ac.f32_0; /* Y */
3014 pos_args[1].out[2] = ctx->ac.f32_0; /* Z */
3015 pos_args[1].out[3] = ctx->ac.f32_0; /* W */
3016
3017 if (shader->selector->info.writes_psize)
3018 pos_args[1].out[0] = psize_value;
3019
3020 if (shader->selector->info.writes_edgeflag) {
3021 /* The output is a float, but the hw expects an integer
3022 * with the first bit containing the edge flag. */
3023 edgeflag_value = LLVMBuildFPToUI(ctx->ac.builder,
3024 edgeflag_value,
3025 ctx->i32, "");
3026 edgeflag_value = ac_build_umin(&ctx->ac,
3027 edgeflag_value,
3028 ctx->i32_1);
3029
3030 /* The LLVM intrinsic expects a float. */
3031 pos_args[1].out[1] = ac_to_float(&ctx->ac, edgeflag_value);
3032 }
3033
3034 if (ctx->screen->info.chip_class >= GFX9) {
3035 /* GFX9 has the layer in out.z[10:0] and the viewport
3036 * index in out.z[19:16].
3037 */
3038 if (shader->selector->info.writes_layer)
3039 pos_args[1].out[2] = layer_value;
3040
3041 if (shader->selector->info.writes_viewport_index) {
3042 LLVMValueRef v = viewport_index_value;
3043
3044 v = ac_to_integer(&ctx->ac, v);
3045 v = LLVMBuildShl(ctx->ac.builder, v,
3046 LLVMConstInt(ctx->i32, 16, 0), "");
3047 v = LLVMBuildOr(ctx->ac.builder, v,
3048 ac_to_integer(&ctx->ac, pos_args[1].out[2]), "");
3049 pos_args[1].out[2] = ac_to_float(&ctx->ac, v);
3050 pos_args[1].enabled_channels |= 1 << 2;
3051 }
3052 } else {
3053 if (shader->selector->info.writes_layer)
3054 pos_args[1].out[2] = layer_value;
3055
3056 if (shader->selector->info.writes_viewport_index) {
3057 pos_args[1].out[3] = viewport_index_value;
3058 pos_args[1].enabled_channels |= 1 << 3;
3059 }
3060 }
3061 }
3062
3063 for (i = 0; i < 4; i++)
3064 if (pos_args[i].out[0])
3065 shader->info.nr_pos_exports++;
3066
3067 pos_idx = 0;
3068 for (i = 0; i < 4; i++) {
3069 if (!pos_args[i].out[0])
3070 continue;
3071
3072 /* Specify the target we are exporting */
3073 pos_args[i].target = V_008DFC_SQ_EXP_POS + pos_idx++;
3074
3075 if (pos_idx == shader->info.nr_pos_exports)
3076 /* Specify that this is the last export */
3077 pos_args[i].done = 1;
3078
3079 ac_build_export(&ctx->ac, &pos_args[i]);
3080 }
3081
3082 /* Build parameter exports. */
3083 si_build_param_exports(ctx, outputs, noutput);
3084 }
3085
3086 /**
3087 * Forward all outputs from the vertex shader to the TES. This is only used
3088 * for the fixed function TCS.
3089 */
3090 static void si_copy_tcs_inputs(struct lp_build_tgsi_context *bld_base)
3091 {
3092 struct si_shader_context *ctx = si_shader_context(bld_base);
3093 LLVMValueRef invocation_id, buffer, buffer_offset;
3094 LLVMValueRef lds_vertex_stride, lds_vertex_offset, lds_base;
3095 uint64_t inputs;
3096
3097 invocation_id = unpack_llvm_param(ctx, ctx->abi.tcs_rel_ids, 8, 5);
3098 buffer = get_tess_ring_descriptor(ctx, TESS_OFFCHIP_RING_TCS);
3099 buffer_offset = LLVMGetParam(ctx->main_fn, ctx->param_tcs_offchip_offset);
3100
3101 lds_vertex_stride = get_tcs_in_vertex_dw_stride(ctx);
3102 lds_vertex_offset = LLVMBuildMul(ctx->ac.builder, invocation_id,
3103 lds_vertex_stride, "");
3104 lds_base = get_tcs_in_current_patch_offset(ctx);
3105 lds_base = LLVMBuildAdd(ctx->ac.builder, lds_base, lds_vertex_offset, "");
3106
3107 inputs = ctx->shader->key.mono.u.ff_tcs_inputs_to_copy;
3108 while (inputs) {
3109 unsigned i = u_bit_scan64(&inputs);
3110
3111 LLVMValueRef lds_ptr = LLVMBuildAdd(ctx->ac.builder, lds_base,
3112 LLVMConstInt(ctx->i32, 4 * i, 0),
3113 "");
3114
3115 LLVMValueRef buffer_addr = get_tcs_tes_buffer_address(ctx,
3116 get_rel_patch_id(ctx),
3117 invocation_id,
3118 LLVMConstInt(ctx->i32, i, 0));
3119
3120 LLVMValueRef value = lds_load(bld_base, ctx->ac.i32, ~0,
3121 lds_ptr);
3122
3123 ac_build_buffer_store_dword(&ctx->ac, buffer, value, 4, buffer_addr,
3124 buffer_offset, 0, 1, 0, true, false);
3125 }
3126 }
3127
3128 static void si_write_tess_factors(struct lp_build_tgsi_context *bld_base,
3129 LLVMValueRef rel_patch_id,
3130 LLVMValueRef invocation_id,
3131 LLVMValueRef tcs_out_current_patch_data_offset,
3132 LLVMValueRef invoc0_tf_outer[4],
3133 LLVMValueRef invoc0_tf_inner[2])
3134 {
3135 struct si_shader_context *ctx = si_shader_context(bld_base);
3136 struct si_shader *shader = ctx->shader;
3137 unsigned tess_inner_index, tess_outer_index;
3138 LLVMValueRef lds_base, lds_inner, lds_outer, byteoffset, buffer;
3139 LLVMValueRef out[6], vec0, vec1, tf_base, inner[4], outer[4];
3140 unsigned stride, outer_comps, inner_comps, i, offset;
3141 struct lp_build_if_state if_ctx, inner_if_ctx;
3142
3143 /* Add a barrier before loading tess factors from LDS. */
3144 if (!shader->key.part.tcs.epilog.invoc0_tess_factors_are_def)
3145 si_llvm_emit_barrier(NULL, bld_base, NULL);
3146
3147 /* Do this only for invocation 0, because the tess levels are per-patch,
3148 * not per-vertex.
3149 *
3150 * This can't jump, because invocation 0 executes this. It should
3151 * at least mask out the loads and stores for other invocations.
3152 */
3153 lp_build_if(&if_ctx, &ctx->gallivm,
3154 LLVMBuildICmp(ctx->ac.builder, LLVMIntEQ,
3155 invocation_id, ctx->i32_0, ""));
3156
3157 /* Determine the layout of one tess factor element in the buffer. */
3158 switch (shader->key.part.tcs.epilog.prim_mode) {
3159 case PIPE_PRIM_LINES:
3160 stride = 2; /* 2 dwords, 1 vec2 store */
3161 outer_comps = 2;
3162 inner_comps = 0;
3163 break;
3164 case PIPE_PRIM_TRIANGLES:
3165 stride = 4; /* 4 dwords, 1 vec4 store */
3166 outer_comps = 3;
3167 inner_comps = 1;
3168 break;
3169 case PIPE_PRIM_QUADS:
3170 stride = 6; /* 6 dwords, 2 stores (vec4 + vec2) */
3171 outer_comps = 4;
3172 inner_comps = 2;
3173 break;
3174 default:
3175 assert(0);
3176 return;
3177 }
3178
3179 for (i = 0; i < 4; i++) {
3180 inner[i] = LLVMGetUndef(ctx->i32);
3181 outer[i] = LLVMGetUndef(ctx->i32);
3182 }
3183
3184 if (shader->key.part.tcs.epilog.invoc0_tess_factors_are_def) {
3185 /* Tess factors are in VGPRs. */
3186 for (i = 0; i < outer_comps; i++)
3187 outer[i] = out[i] = invoc0_tf_outer[i];
3188 for (i = 0; i < inner_comps; i++)
3189 inner[i] = out[outer_comps+i] = invoc0_tf_inner[i];
3190 } else {
3191 /* Load tess_inner and tess_outer from LDS.
3192 * Any invocation can write them, so we can't get them from a temporary.
3193 */
3194 tess_inner_index = si_shader_io_get_unique_index_patch(TGSI_SEMANTIC_TESSINNER, 0);
3195 tess_outer_index = si_shader_io_get_unique_index_patch(TGSI_SEMANTIC_TESSOUTER, 0);
3196
3197 lds_base = tcs_out_current_patch_data_offset;
3198 lds_inner = LLVMBuildAdd(ctx->ac.builder, lds_base,
3199 LLVMConstInt(ctx->i32,
3200 tess_inner_index * 4, 0), "");
3201 lds_outer = LLVMBuildAdd(ctx->ac.builder, lds_base,
3202 LLVMConstInt(ctx->i32,
3203 tess_outer_index * 4, 0), "");
3204
3205 for (i = 0; i < outer_comps; i++) {
3206 outer[i] = out[i] =
3207 lds_load(bld_base, ctx->ac.i32, i, lds_outer);
3208 }
3209 for (i = 0; i < inner_comps; i++) {
3210 inner[i] = out[outer_comps+i] =
3211 lds_load(bld_base, ctx->ac.i32, i, lds_inner);
3212 }
3213 }
3214
3215 if (shader->key.part.tcs.epilog.prim_mode == PIPE_PRIM_LINES) {
3216 /* For isolines, the hardware expects tess factors in the
3217 * reverse order from what GLSL / TGSI specify.
3218 */
3219 LLVMValueRef tmp = out[0];
3220 out[0] = out[1];
3221 out[1] = tmp;
3222 }
3223
3224 /* Convert the outputs to vectors for stores. */
3225 vec0 = lp_build_gather_values(&ctx->gallivm, out, MIN2(stride, 4));
3226 vec1 = NULL;
3227
3228 if (stride > 4)
3229 vec1 = lp_build_gather_values(&ctx->gallivm, out+4, stride - 4);
3230
3231 /* Get the buffer. */
3232 buffer = get_tess_ring_descriptor(ctx, TCS_FACTOR_RING);
3233
3234 /* Get the offset. */
3235 tf_base = LLVMGetParam(ctx->main_fn,
3236 ctx->param_tcs_factor_offset);
3237 byteoffset = LLVMBuildMul(ctx->ac.builder, rel_patch_id,
3238 LLVMConstInt(ctx->i32, 4 * stride, 0), "");
3239
3240 lp_build_if(&inner_if_ctx, &ctx->gallivm,
3241 LLVMBuildICmp(ctx->ac.builder, LLVMIntEQ,
3242 rel_patch_id, ctx->i32_0, ""));
3243
3244 /* Store the dynamic HS control word. */
3245 offset = 0;
3246 if (ctx->screen->info.chip_class <= VI) {
3247 ac_build_buffer_store_dword(&ctx->ac, buffer,
3248 LLVMConstInt(ctx->i32, 0x80000000, 0),
3249 1, ctx->i32_0, tf_base,
3250 offset, 1, 0, true, false);
3251 offset += 4;
3252 }
3253
3254 lp_build_endif(&inner_if_ctx);
3255
3256 /* Store the tessellation factors. */
3257 ac_build_buffer_store_dword(&ctx->ac, buffer, vec0,
3258 MIN2(stride, 4), byteoffset, tf_base,
3259 offset, 1, 0, true, false);
3260 offset += 16;
3261 if (vec1)
3262 ac_build_buffer_store_dword(&ctx->ac, buffer, vec1,
3263 stride - 4, byteoffset, tf_base,
3264 offset, 1, 0, true, false);
3265
3266 /* Store the tess factors into the offchip buffer if TES reads them. */
3267 if (shader->key.part.tcs.epilog.tes_reads_tess_factors) {
3268 LLVMValueRef buf, base, inner_vec, outer_vec, tf_outer_offset;
3269 LLVMValueRef tf_inner_offset;
3270 unsigned param_outer, param_inner;
3271
3272 buf = get_tess_ring_descriptor(ctx, TESS_OFFCHIP_RING_TCS);
3273 base = LLVMGetParam(ctx->main_fn, ctx->param_tcs_offchip_offset);
3274
3275 param_outer = si_shader_io_get_unique_index_patch(
3276 TGSI_SEMANTIC_TESSOUTER, 0);
3277 tf_outer_offset = get_tcs_tes_buffer_address(ctx, rel_patch_id, NULL,
3278 LLVMConstInt(ctx->i32, param_outer, 0));
3279
3280 outer_vec = lp_build_gather_values(&ctx->gallivm, outer,
3281 util_next_power_of_two(outer_comps));
3282
3283 ac_build_buffer_store_dword(&ctx->ac, buf, outer_vec,
3284 outer_comps, tf_outer_offset,
3285 base, 0, 1, 0, true, false);
3286 if (inner_comps) {
3287 param_inner = si_shader_io_get_unique_index_patch(
3288 TGSI_SEMANTIC_TESSINNER, 0);
3289 tf_inner_offset = get_tcs_tes_buffer_address(ctx, rel_patch_id, NULL,
3290 LLVMConstInt(ctx->i32, param_inner, 0));
3291
3292 inner_vec = inner_comps == 1 ? inner[0] :
3293 lp_build_gather_values(&ctx->gallivm, inner, inner_comps);
3294 ac_build_buffer_store_dword(&ctx->ac, buf, inner_vec,
3295 inner_comps, tf_inner_offset,
3296 base, 0, 1, 0, true, false);
3297 }
3298 }
3299
3300 lp_build_endif(&if_ctx);
3301 }
3302
3303 static LLVMValueRef
3304 si_insert_input_ret(struct si_shader_context *ctx, LLVMValueRef ret,
3305 unsigned param, unsigned return_index)
3306 {
3307 return LLVMBuildInsertValue(ctx->ac.builder, ret,
3308 LLVMGetParam(ctx->main_fn, param),
3309 return_index, "");
3310 }
3311
3312 static LLVMValueRef
3313 si_insert_input_ret_float(struct si_shader_context *ctx, LLVMValueRef ret,
3314 unsigned param, unsigned return_index)
3315 {
3316 LLVMBuilderRef builder = ctx->ac.builder;
3317 LLVMValueRef p = LLVMGetParam(ctx->main_fn, param);
3318
3319 return LLVMBuildInsertValue(builder, ret,
3320 ac_to_float(&ctx->ac, p),
3321 return_index, "");
3322 }
3323
3324 static LLVMValueRef
3325 si_insert_input_ptr(struct si_shader_context *ctx, LLVMValueRef ret,
3326 unsigned param, unsigned return_index)
3327 {
3328 LLVMBuilderRef builder = ctx->ac.builder;
3329 LLVMValueRef ptr, lo, hi;
3330
3331 if (HAVE_32BIT_POINTERS) {
3332 ptr = LLVMGetParam(ctx->main_fn, param);
3333 ptr = LLVMBuildPtrToInt(builder, ptr, ctx->i32, "");
3334 return LLVMBuildInsertValue(builder, ret, ptr, return_index, "");
3335 }
3336
3337 ptr = LLVMGetParam(ctx->main_fn, param);
3338 ptr = LLVMBuildPtrToInt(builder, ptr, ctx->i64, "");
3339 ptr = LLVMBuildBitCast(builder, ptr, ctx->v2i32, "");
3340 lo = LLVMBuildExtractElement(builder, ptr, ctx->i32_0, "");
3341 hi = LLVMBuildExtractElement(builder, ptr, ctx->i32_1, "");
3342 ret = LLVMBuildInsertValue(builder, ret, lo, return_index, "");
3343 return LLVMBuildInsertValue(builder, ret, hi, return_index + 1, "");
3344 }
3345
3346 /* This only writes the tessellation factor levels. */
3347 static void si_llvm_emit_tcs_epilogue(struct ac_shader_abi *abi,
3348 unsigned max_outputs,
3349 LLVMValueRef *addrs)
3350 {
3351 struct si_shader_context *ctx = si_shader_context_from_abi(abi);
3352 struct lp_build_tgsi_context *bld_base = &ctx->bld_base;
3353 LLVMBuilderRef builder = ctx->ac.builder;
3354 LLVMValueRef rel_patch_id, invocation_id, tf_lds_offset;
3355
3356 si_copy_tcs_inputs(bld_base);
3357
3358 rel_patch_id = get_rel_patch_id(ctx);
3359 invocation_id = unpack_llvm_param(ctx, ctx->abi.tcs_rel_ids, 8, 5);
3360 tf_lds_offset = get_tcs_out_current_patch_data_offset(ctx);
3361
3362 if (ctx->screen->info.chip_class >= GFX9) {
3363 LLVMBasicBlockRef blocks[2] = {
3364 LLVMGetInsertBlock(builder),
3365 ctx->merged_wrap_if_state.entry_block
3366 };
3367 LLVMValueRef values[2];
3368
3369 lp_build_endif(&ctx->merged_wrap_if_state);
3370
3371 values[0] = rel_patch_id;
3372 values[1] = LLVMGetUndef(ctx->i32);
3373 rel_patch_id = ac_build_phi(&ctx->ac, ctx->i32, 2, values, blocks);
3374
3375 values[0] = tf_lds_offset;
3376 values[1] = LLVMGetUndef(ctx->i32);
3377 tf_lds_offset = ac_build_phi(&ctx->ac, ctx->i32, 2, values, blocks);
3378
3379 values[0] = invocation_id;
3380 values[1] = ctx->i32_1; /* cause the epilog to skip threads */
3381 invocation_id = ac_build_phi(&ctx->ac, ctx->i32, 2, values, blocks);
3382 }
3383
3384 /* Return epilog parameters from this function. */
3385 LLVMValueRef ret = ctx->return_value;
3386 unsigned vgpr;
3387
3388 if (ctx->screen->info.chip_class >= GFX9) {
3389 ret = si_insert_input_ret(ctx, ret, ctx->param_tcs_offchip_layout,
3390 8 + GFX9_SGPR_TCS_OFFCHIP_LAYOUT);
3391 ret = si_insert_input_ret(ctx, ret, ctx->param_tcs_out_lds_layout,
3392 8 + GFX9_SGPR_TCS_OUT_LAYOUT);
3393 /* Tess offchip and tess factor offsets are at the beginning. */
3394 ret = si_insert_input_ret(ctx, ret, ctx->param_tcs_offchip_offset, 2);
3395 ret = si_insert_input_ret(ctx, ret, ctx->param_tcs_factor_offset, 4);
3396 vgpr = 8 + GFX9_SGPR_TCS_OUT_LAYOUT + 1;
3397 } else {
3398 ret = si_insert_input_ret(ctx, ret, ctx->param_tcs_offchip_layout,
3399 GFX6_SGPR_TCS_OFFCHIP_LAYOUT);
3400 ret = si_insert_input_ret(ctx, ret, ctx->param_tcs_out_lds_layout,
3401 GFX6_SGPR_TCS_OUT_LAYOUT);
3402 /* Tess offchip and tess factor offsets are after user SGPRs. */
3403 ret = si_insert_input_ret(ctx, ret, ctx->param_tcs_offchip_offset,
3404 GFX6_TCS_NUM_USER_SGPR);
3405 ret = si_insert_input_ret(ctx, ret, ctx->param_tcs_factor_offset,
3406 GFX6_TCS_NUM_USER_SGPR + 1);
3407 vgpr = GFX6_TCS_NUM_USER_SGPR + 2;
3408 }
3409
3410 /* VGPRs */
3411 rel_patch_id = ac_to_float(&ctx->ac, rel_patch_id);
3412 invocation_id = ac_to_float(&ctx->ac, invocation_id);
3413 tf_lds_offset = ac_to_float(&ctx->ac, tf_lds_offset);
3414
3415 /* Leave a hole corresponding to the two input VGPRs. This ensures that
3416 * the invocation_id output does not alias the tcs_rel_ids input,
3417 * which saves a V_MOV on gfx9.
3418 */
3419 vgpr += 2;
3420
3421 ret = LLVMBuildInsertValue(builder, ret, rel_patch_id, vgpr++, "");
3422 ret = LLVMBuildInsertValue(builder, ret, invocation_id, vgpr++, "");
3423
3424 if (ctx->shader->selector->tcs_info.tessfactors_are_def_in_all_invocs) {
3425 vgpr++; /* skip the tess factor LDS offset */
3426 for (unsigned i = 0; i < 6; i++) {
3427 LLVMValueRef value =
3428 LLVMBuildLoad(builder, ctx->invoc0_tess_factors[i], "");
3429 value = ac_to_float(&ctx->ac, value);
3430 ret = LLVMBuildInsertValue(builder, ret, value, vgpr++, "");
3431 }
3432 } else {
3433 ret = LLVMBuildInsertValue(builder, ret, tf_lds_offset, vgpr++, "");
3434 }
3435 ctx->return_value = ret;
3436 }
3437
3438 /* Pass TCS inputs from LS to TCS on GFX9. */
3439 static void si_set_ls_return_value_for_tcs(struct si_shader_context *ctx)
3440 {
3441 LLVMValueRef ret = ctx->return_value;
3442
3443 ret = si_insert_input_ptr(ctx, ret, 0, 0);
3444 if (HAVE_32BIT_POINTERS)
3445 ret = si_insert_input_ptr(ctx, ret, 1, 1);
3446 ret = si_insert_input_ret(ctx, ret, ctx->param_tcs_offchip_offset, 2);
3447 ret = si_insert_input_ret(ctx, ret, ctx->param_merged_wave_info, 3);
3448 ret = si_insert_input_ret(ctx, ret, ctx->param_tcs_factor_offset, 4);
3449 ret = si_insert_input_ret(ctx, ret, ctx->param_merged_scratch_offset, 5);
3450
3451 ret = si_insert_input_ptr(ctx, ret, ctx->param_rw_buffers,
3452 8 + SI_SGPR_RW_BUFFERS);
3453 ret = si_insert_input_ptr(ctx, ret,
3454 ctx->param_bindless_samplers_and_images,
3455 8 + SI_SGPR_BINDLESS_SAMPLERS_AND_IMAGES);
3456
3457 ret = si_insert_input_ret(ctx, ret, ctx->param_vs_state_bits,
3458 8 + SI_SGPR_VS_STATE_BITS);
3459
3460 #if !HAVE_32BIT_POINTERS
3461 ret = si_insert_input_ptr(ctx, ret, ctx->param_vs_state_bits + 1,
3462 8 + GFX9_SGPR_2ND_SAMPLERS_AND_IMAGES);
3463 #endif
3464
3465 ret = si_insert_input_ret(ctx, ret, ctx->param_tcs_offchip_layout,
3466 8 + GFX9_SGPR_TCS_OFFCHIP_LAYOUT);
3467 ret = si_insert_input_ret(ctx, ret, ctx->param_tcs_out_lds_offsets,
3468 8 + GFX9_SGPR_TCS_OUT_OFFSETS);
3469 ret = si_insert_input_ret(ctx, ret, ctx->param_tcs_out_lds_layout,
3470 8 + GFX9_SGPR_TCS_OUT_LAYOUT);
3471
3472 unsigned vgpr = 8 + GFX9_TCS_NUM_USER_SGPR;
3473 ret = LLVMBuildInsertValue(ctx->ac.builder, ret,
3474 ac_to_float(&ctx->ac, ctx->abi.tcs_patch_id),
3475 vgpr++, "");
3476 ret = LLVMBuildInsertValue(ctx->ac.builder, ret,
3477 ac_to_float(&ctx->ac, ctx->abi.tcs_rel_ids),
3478 vgpr++, "");
3479 ctx->return_value = ret;
3480 }
3481
3482 /* Pass GS inputs from ES to GS on GFX9. */
3483 static void si_set_es_return_value_for_gs(struct si_shader_context *ctx)
3484 {
3485 LLVMValueRef ret = ctx->return_value;
3486
3487 ret = si_insert_input_ptr(ctx, ret, 0, 0);
3488 if (HAVE_32BIT_POINTERS)
3489 ret = si_insert_input_ptr(ctx, ret, 1, 1);
3490 ret = si_insert_input_ret(ctx, ret, ctx->param_gs2vs_offset, 2);
3491 ret = si_insert_input_ret(ctx, ret, ctx->param_merged_wave_info, 3);
3492 ret = si_insert_input_ret(ctx, ret, ctx->param_merged_scratch_offset, 5);
3493
3494 ret = si_insert_input_ptr(ctx, ret, ctx->param_rw_buffers,
3495 8 + SI_SGPR_RW_BUFFERS);
3496 ret = si_insert_input_ptr(ctx, ret,
3497 ctx->param_bindless_samplers_and_images,
3498 8 + SI_SGPR_BINDLESS_SAMPLERS_AND_IMAGES);
3499
3500 #if !HAVE_32BIT_POINTERS
3501 ret = si_insert_input_ptr(ctx, ret, ctx->param_vs_state_bits + 1,
3502 8 + GFX9_SGPR_2ND_SAMPLERS_AND_IMAGES);
3503 #endif
3504
3505 unsigned vgpr;
3506 if (ctx->type == PIPE_SHADER_VERTEX)
3507 vgpr = 8 + GFX9_VSGS_NUM_USER_SGPR;
3508 else
3509 vgpr = 8 + GFX9_TESGS_NUM_USER_SGPR;
3510
3511 for (unsigned i = 0; i < 5; i++) {
3512 unsigned param = ctx->param_gs_vtx01_offset + i;
3513 ret = si_insert_input_ret_float(ctx, ret, param, vgpr++);
3514 }
3515 ctx->return_value = ret;
3516 }
3517
3518 static void si_llvm_emit_ls_epilogue(struct ac_shader_abi *abi,
3519 unsigned max_outputs,
3520 LLVMValueRef *addrs)
3521 {
3522 struct si_shader_context *ctx = si_shader_context_from_abi(abi);
3523 struct si_shader *shader = ctx->shader;
3524 struct tgsi_shader_info *info = &shader->selector->info;
3525 unsigned i, chan;
3526 LLVMValueRef vertex_id = LLVMGetParam(ctx->main_fn,
3527 ctx->param_rel_auto_id);
3528 LLVMValueRef vertex_dw_stride = get_tcs_in_vertex_dw_stride(ctx);
3529 LLVMValueRef base_dw_addr = LLVMBuildMul(ctx->ac.builder, vertex_id,
3530 vertex_dw_stride, "");
3531
3532 /* Write outputs to LDS. The next shader (TCS aka HS) will read
3533 * its inputs from it. */
3534 for (i = 0; i < info->num_outputs; i++) {
3535 unsigned name = info->output_semantic_name[i];
3536 unsigned index = info->output_semantic_index[i];
3537
3538 /* The ARB_shader_viewport_layer_array spec contains the
3539 * following issue:
3540 *
3541 * 2) What happens if gl_ViewportIndex or gl_Layer is
3542 * written in the vertex shader and a geometry shader is
3543 * present?
3544 *
3545 * RESOLVED: The value written by the last vertex processing
3546 * stage is used. If the last vertex processing stage
3547 * (vertex, tessellation evaluation or geometry) does not
3548 * statically assign to gl_ViewportIndex or gl_Layer, index
3549 * or layer zero is assumed.
3550 *
3551 * So writes to those outputs in VS-as-LS are simply ignored.
3552 */
3553 if (name == TGSI_SEMANTIC_LAYER ||
3554 name == TGSI_SEMANTIC_VIEWPORT_INDEX)
3555 continue;
3556
3557 int param = si_shader_io_get_unique_index(name, index, false);
3558 LLVMValueRef dw_addr = LLVMBuildAdd(ctx->ac.builder, base_dw_addr,
3559 LLVMConstInt(ctx->i32, param * 4, 0), "");
3560
3561 for (chan = 0; chan < 4; chan++) {
3562 if (!(info->output_usagemask[i] & (1 << chan)))
3563 continue;
3564
3565 lds_store(ctx, chan, dw_addr,
3566 LLVMBuildLoad(ctx->ac.builder, addrs[4 * i + chan], ""));
3567 }
3568 }
3569
3570 if (ctx->screen->info.chip_class >= GFX9)
3571 si_set_ls_return_value_for_tcs(ctx);
3572 }
3573
3574 static void si_llvm_emit_es_epilogue(struct ac_shader_abi *abi,
3575 unsigned max_outputs,
3576 LLVMValueRef *addrs)
3577 {
3578 struct si_shader_context *ctx = si_shader_context_from_abi(abi);
3579 struct si_shader *es = ctx->shader;
3580 struct tgsi_shader_info *info = &es->selector->info;
3581 LLVMValueRef soffset = LLVMGetParam(ctx->main_fn,
3582 ctx->param_es2gs_offset);
3583 LLVMValueRef lds_base = NULL;
3584 unsigned chan;
3585 int i;
3586
3587 if (ctx->screen->info.chip_class >= GFX9 && info->num_outputs) {
3588 unsigned itemsize_dw = es->selector->esgs_itemsize / 4;
3589 LLVMValueRef vertex_idx = ac_get_thread_id(&ctx->ac);
3590 LLVMValueRef wave_idx = si_unpack_param(ctx, ctx->param_merged_wave_info, 24, 4);
3591 vertex_idx = LLVMBuildOr(ctx->ac.builder, vertex_idx,
3592 LLVMBuildMul(ctx->ac.builder, wave_idx,
3593 LLVMConstInt(ctx->i32, 64, false), ""), "");
3594 lds_base = LLVMBuildMul(ctx->ac.builder, vertex_idx,
3595 LLVMConstInt(ctx->i32, itemsize_dw, 0), "");
3596 }
3597
3598 for (i = 0; i < info->num_outputs; i++) {
3599 int param;
3600
3601 if (info->output_semantic_name[i] == TGSI_SEMANTIC_VIEWPORT_INDEX ||
3602 info->output_semantic_name[i] == TGSI_SEMANTIC_LAYER)
3603 continue;
3604
3605 param = si_shader_io_get_unique_index(info->output_semantic_name[i],
3606 info->output_semantic_index[i], false);
3607
3608 for (chan = 0; chan < 4; chan++) {
3609 if (!(info->output_usagemask[i] & (1 << chan)))
3610 continue;
3611
3612 LLVMValueRef out_val = LLVMBuildLoad(ctx->ac.builder, addrs[4 * i + chan], "");
3613 out_val = ac_to_integer(&ctx->ac, out_val);
3614
3615 /* GFX9 has the ESGS ring in LDS. */
3616 if (ctx->screen->info.chip_class >= GFX9) {
3617 lds_store(ctx, param * 4 + chan, lds_base, out_val);
3618 continue;
3619 }
3620
3621 ac_build_buffer_store_dword(&ctx->ac,
3622 ctx->esgs_ring,
3623 out_val, 1, NULL, soffset,
3624 (4 * param + chan) * 4,
3625 1, 1, true, true);
3626 }
3627 }
3628
3629 if (ctx->screen->info.chip_class >= GFX9)
3630 si_set_es_return_value_for_gs(ctx);
3631 }
3632
3633 static LLVMValueRef si_get_gs_wave_id(struct si_shader_context *ctx)
3634 {
3635 if (ctx->screen->info.chip_class >= GFX9)
3636 return si_unpack_param(ctx, ctx->param_merged_wave_info, 16, 8);
3637 else
3638 return LLVMGetParam(ctx->main_fn, ctx->param_gs_wave_id);
3639 }
3640
3641 static void emit_gs_epilogue(struct si_shader_context *ctx)
3642 {
3643 ac_build_sendmsg(&ctx->ac, AC_SENDMSG_GS_OP_NOP | AC_SENDMSG_GS_DONE,
3644 si_get_gs_wave_id(ctx));
3645
3646 if (ctx->screen->info.chip_class >= GFX9)
3647 lp_build_endif(&ctx->merged_wrap_if_state);
3648 }
3649
3650 static void si_llvm_emit_gs_epilogue(struct ac_shader_abi *abi,
3651 unsigned max_outputs,
3652 LLVMValueRef *addrs)
3653 {
3654 struct si_shader_context *ctx = si_shader_context_from_abi(abi);
3655 struct tgsi_shader_info UNUSED *info = &ctx->shader->selector->info;
3656
3657 assert(info->num_outputs <= max_outputs);
3658
3659 emit_gs_epilogue(ctx);
3660 }
3661
3662 static void si_tgsi_emit_gs_epilogue(struct lp_build_tgsi_context *bld_base)
3663 {
3664 struct si_shader_context *ctx = si_shader_context(bld_base);
3665 emit_gs_epilogue(ctx);
3666 }
3667
3668 static void si_llvm_emit_vs_epilogue(struct ac_shader_abi *abi,
3669 unsigned max_outputs,
3670 LLVMValueRef *addrs)
3671 {
3672 struct si_shader_context *ctx = si_shader_context_from_abi(abi);
3673 struct tgsi_shader_info *info = &ctx->shader->selector->info;
3674 struct si_shader_output_values *outputs = NULL;
3675 int i,j;
3676
3677 assert(!ctx->shader->is_gs_copy_shader);
3678 assert(info->num_outputs <= max_outputs);
3679
3680 outputs = MALLOC((info->num_outputs + 1) * sizeof(outputs[0]));
3681
3682 /* Vertex color clamping.
3683 *
3684 * This uses a state constant loaded in a user data SGPR and
3685 * an IF statement is added that clamps all colors if the constant
3686 * is true.
3687 */
3688 if (ctx->type == PIPE_SHADER_VERTEX) {
3689 struct lp_build_if_state if_ctx;
3690 LLVMValueRef cond = NULL;
3691 LLVMValueRef addr, val;
3692
3693 for (i = 0; i < info->num_outputs; i++) {
3694 if (info->output_semantic_name[i] != TGSI_SEMANTIC_COLOR &&
3695 info->output_semantic_name[i] != TGSI_SEMANTIC_BCOLOR)
3696 continue;
3697
3698 /* We've found a color. */
3699 if (!cond) {
3700 /* The state is in the first bit of the user SGPR. */
3701 cond = LLVMGetParam(ctx->main_fn,
3702 ctx->param_vs_state_bits);
3703 cond = LLVMBuildTrunc(ctx->ac.builder, cond,
3704 ctx->i1, "");
3705 lp_build_if(&if_ctx, &ctx->gallivm, cond);
3706 }
3707
3708 for (j = 0; j < 4; j++) {
3709 addr = addrs[4 * i + j];
3710 val = LLVMBuildLoad(ctx->ac.builder, addr, "");
3711 val = ac_build_clamp(&ctx->ac, val);
3712 LLVMBuildStore(ctx->ac.builder, val, addr);
3713 }
3714 }
3715
3716 if (cond)
3717 lp_build_endif(&if_ctx);
3718 }
3719
3720 for (i = 0; i < info->num_outputs; i++) {
3721 outputs[i].semantic_name = info->output_semantic_name[i];
3722 outputs[i].semantic_index = info->output_semantic_index[i];
3723
3724 for (j = 0; j < 4; j++) {
3725 outputs[i].values[j] =
3726 LLVMBuildLoad(ctx->ac.builder,
3727 addrs[4 * i + j],
3728 "");
3729 outputs[i].vertex_stream[j] =
3730 (info->output_streams[i] >> (2 * j)) & 3;
3731 }
3732 }
3733
3734 if (ctx->shader->selector->so.num_outputs)
3735 si_llvm_emit_streamout(ctx, outputs, i, 0);
3736
3737 /* Export PrimitiveID. */
3738 if (ctx->shader->key.mono.u.vs_export_prim_id) {
3739 outputs[i].semantic_name = TGSI_SEMANTIC_PRIMID;
3740 outputs[i].semantic_index = 0;
3741 outputs[i].values[0] = ac_to_float(&ctx->ac, get_primitive_id(ctx, 0));
3742 for (j = 1; j < 4; j++)
3743 outputs[i].values[j] = LLVMConstReal(ctx->f32, 0);
3744
3745 memset(outputs[i].vertex_stream, 0,
3746 sizeof(outputs[i].vertex_stream));
3747 i++;
3748 }
3749
3750 si_llvm_export_vs(ctx, outputs, i);
3751 FREE(outputs);
3752 }
3753
3754 static void si_tgsi_emit_epilogue(struct lp_build_tgsi_context *bld_base)
3755 {
3756 struct si_shader_context *ctx = si_shader_context(bld_base);
3757
3758 ctx->abi.emit_outputs(&ctx->abi, RADEON_LLVM_MAX_OUTPUTS,
3759 &ctx->outputs[0][0]);
3760 }
3761
3762 struct si_ps_exports {
3763 unsigned num;
3764 struct ac_export_args args[10];
3765 };
3766
3767 static void si_export_mrt_z(struct lp_build_tgsi_context *bld_base,
3768 LLVMValueRef depth, LLVMValueRef stencil,
3769 LLVMValueRef samplemask, struct si_ps_exports *exp)
3770 {
3771 struct si_shader_context *ctx = si_shader_context(bld_base);
3772 struct ac_export_args args;
3773
3774 ac_export_mrt_z(&ctx->ac, depth, stencil, samplemask, &args);
3775
3776 memcpy(&exp->args[exp->num++], &args, sizeof(args));
3777 }
3778
3779 static void si_export_mrt_color(struct lp_build_tgsi_context *bld_base,
3780 LLVMValueRef *color, unsigned index,
3781 unsigned samplemask_param,
3782 bool is_last, struct si_ps_exports *exp)
3783 {
3784 struct si_shader_context *ctx = si_shader_context(bld_base);
3785 int i;
3786
3787 /* Clamp color */
3788 if (ctx->shader->key.part.ps.epilog.clamp_color)
3789 for (i = 0; i < 4; i++)
3790 color[i] = ac_build_clamp(&ctx->ac, color[i]);
3791
3792 /* Alpha to one */
3793 if (ctx->shader->key.part.ps.epilog.alpha_to_one)
3794 color[3] = ctx->ac.f32_1;
3795
3796 /* Alpha test */
3797 if (index == 0 &&
3798 ctx->shader->key.part.ps.epilog.alpha_func != PIPE_FUNC_ALWAYS)
3799 si_alpha_test(bld_base, color[3]);
3800
3801 /* Line & polygon smoothing */
3802 if (ctx->shader->key.part.ps.epilog.poly_line_smoothing)
3803 color[3] = si_scale_alpha_by_sample_mask(bld_base, color[3],
3804 samplemask_param);
3805
3806 /* If last_cbuf > 0, FS_COLOR0_WRITES_ALL_CBUFS is true. */
3807 if (ctx->shader->key.part.ps.epilog.last_cbuf > 0) {
3808 struct ac_export_args args[8];
3809 int c, last = -1;
3810
3811 /* Get the export arguments, also find out what the last one is. */
3812 for (c = 0; c <= ctx->shader->key.part.ps.epilog.last_cbuf; c++) {
3813 si_llvm_init_export_args(ctx, color,
3814 V_008DFC_SQ_EXP_MRT + c, &args[c]);
3815 if (args[c].enabled_channels)
3816 last = c;
3817 }
3818
3819 /* Emit all exports. */
3820 for (c = 0; c <= ctx->shader->key.part.ps.epilog.last_cbuf; c++) {
3821 if (is_last && last == c) {
3822 args[c].valid_mask = 1; /* whether the EXEC mask is valid */
3823 args[c].done = 1; /* DONE bit */
3824 } else if (!args[c].enabled_channels)
3825 continue; /* unnecessary NULL export */
3826
3827 memcpy(&exp->args[exp->num++], &args[c], sizeof(args[c]));
3828 }
3829 } else {
3830 struct ac_export_args args;
3831
3832 /* Export */
3833 si_llvm_init_export_args(ctx, color, V_008DFC_SQ_EXP_MRT + index,
3834 &args);
3835 if (is_last) {
3836 args.valid_mask = 1; /* whether the EXEC mask is valid */
3837 args.done = 1; /* DONE bit */
3838 } else if (!args.enabled_channels)
3839 return; /* unnecessary NULL export */
3840
3841 memcpy(&exp->args[exp->num++], &args, sizeof(args));
3842 }
3843 }
3844
3845 static void si_emit_ps_exports(struct si_shader_context *ctx,
3846 struct si_ps_exports *exp)
3847 {
3848 for (unsigned i = 0; i < exp->num; i++)
3849 ac_build_export(&ctx->ac, &exp->args[i]);
3850 }
3851
3852 /**
3853 * Return PS outputs in this order:
3854 *
3855 * v[0:3] = color0.xyzw
3856 * v[4:7] = color1.xyzw
3857 * ...
3858 * vN+0 = Depth
3859 * vN+1 = Stencil
3860 * vN+2 = SampleMask
3861 * vN+3 = SampleMaskIn (used for OpenGL smoothing)
3862 *
3863 * The alpha-ref SGPR is returned via its original location.
3864 */
3865 static void si_llvm_return_fs_outputs(struct ac_shader_abi *abi,
3866 unsigned max_outputs,
3867 LLVMValueRef *addrs)
3868 {
3869 struct si_shader_context *ctx = si_shader_context_from_abi(abi);
3870 struct si_shader *shader = ctx->shader;
3871 struct tgsi_shader_info *info = &shader->selector->info;
3872 LLVMBuilderRef builder = ctx->ac.builder;
3873 unsigned i, j, first_vgpr, vgpr;
3874
3875 LLVMValueRef color[8][4] = {};
3876 LLVMValueRef depth = NULL, stencil = NULL, samplemask = NULL;
3877 LLVMValueRef ret;
3878
3879 if (ctx->postponed_kill)
3880 ac_build_kill_if_false(&ctx->ac, LLVMBuildLoad(builder, ctx->postponed_kill, ""));
3881
3882 /* Read the output values. */
3883 for (i = 0; i < info->num_outputs; i++) {
3884 unsigned semantic_name = info->output_semantic_name[i];
3885 unsigned semantic_index = info->output_semantic_index[i];
3886
3887 switch (semantic_name) {
3888 case TGSI_SEMANTIC_COLOR:
3889 assert(semantic_index < 8);
3890 for (j = 0; j < 4; j++) {
3891 LLVMValueRef ptr = addrs[4 * i + j];
3892 LLVMValueRef result = LLVMBuildLoad(builder, ptr, "");
3893 color[semantic_index][j] = result;
3894 }
3895 break;
3896 case TGSI_SEMANTIC_POSITION:
3897 depth = LLVMBuildLoad(builder,
3898 addrs[4 * i + 2], "");
3899 break;
3900 case TGSI_SEMANTIC_STENCIL:
3901 stencil = LLVMBuildLoad(builder,
3902 addrs[4 * i + 1], "");
3903 break;
3904 case TGSI_SEMANTIC_SAMPLEMASK:
3905 samplemask = LLVMBuildLoad(builder,
3906 addrs[4 * i + 0], "");
3907 break;
3908 default:
3909 fprintf(stderr, "Warning: SI unhandled fs output type:%d\n",
3910 semantic_name);
3911 }
3912 }
3913
3914 /* Fill the return structure. */
3915 ret = ctx->return_value;
3916
3917 /* Set SGPRs. */
3918 ret = LLVMBuildInsertValue(builder, ret,
3919 ac_to_integer(&ctx->ac,
3920 LLVMGetParam(ctx->main_fn,
3921 SI_PARAM_ALPHA_REF)),
3922 SI_SGPR_ALPHA_REF, "");
3923
3924 /* Set VGPRs */
3925 first_vgpr = vgpr = SI_SGPR_ALPHA_REF + 1;
3926 for (i = 0; i < ARRAY_SIZE(color); i++) {
3927 if (!color[i][0])
3928 continue;
3929
3930 for (j = 0; j < 4; j++)
3931 ret = LLVMBuildInsertValue(builder, ret, color[i][j], vgpr++, "");
3932 }
3933 if (depth)
3934 ret = LLVMBuildInsertValue(builder, ret, depth, vgpr++, "");
3935 if (stencil)
3936 ret = LLVMBuildInsertValue(builder, ret, stencil, vgpr++, "");
3937 if (samplemask)
3938 ret = LLVMBuildInsertValue(builder, ret, samplemask, vgpr++, "");
3939
3940 /* Add the input sample mask for smoothing at the end. */
3941 if (vgpr < first_vgpr + PS_EPILOG_SAMPLEMASK_MIN_LOC)
3942 vgpr = first_vgpr + PS_EPILOG_SAMPLEMASK_MIN_LOC;
3943 ret = LLVMBuildInsertValue(builder, ret,
3944 LLVMGetParam(ctx->main_fn,
3945 SI_PARAM_SAMPLE_COVERAGE), vgpr++, "");
3946
3947 ctx->return_value = ret;
3948 }
3949
3950 static void membar_emit(
3951 const struct lp_build_tgsi_action *action,
3952 struct lp_build_tgsi_context *bld_base,
3953 struct lp_build_emit_data *emit_data)
3954 {
3955 struct si_shader_context *ctx = si_shader_context(bld_base);
3956 LLVMValueRef src0 = lp_build_emit_fetch(bld_base, emit_data->inst, 0, 0);
3957 unsigned flags = LLVMConstIntGetZExtValue(src0);
3958 unsigned waitcnt = NOOP_WAITCNT;
3959
3960 if (flags & TGSI_MEMBAR_THREAD_GROUP)
3961 waitcnt &= VM_CNT & LGKM_CNT;
3962
3963 if (flags & (TGSI_MEMBAR_ATOMIC_BUFFER |
3964 TGSI_MEMBAR_SHADER_BUFFER |
3965 TGSI_MEMBAR_SHADER_IMAGE))
3966 waitcnt &= VM_CNT;
3967
3968 if (flags & TGSI_MEMBAR_SHARED)
3969 waitcnt &= LGKM_CNT;
3970
3971 if (waitcnt != NOOP_WAITCNT)
3972 ac_build_waitcnt(&ctx->ac, waitcnt);
3973 }
3974
3975 static void clock_emit(
3976 const struct lp_build_tgsi_action *action,
3977 struct lp_build_tgsi_context *bld_base,
3978 struct lp_build_emit_data *emit_data)
3979 {
3980 struct si_shader_context *ctx = si_shader_context(bld_base);
3981 LLVMValueRef tmp = ac_build_shader_clock(&ctx->ac);
3982
3983 emit_data->output[0] =
3984 LLVMBuildExtractElement(ctx->ac.builder, tmp, ctx->i32_0, "");
3985 emit_data->output[1] =
3986 LLVMBuildExtractElement(ctx->ac.builder, tmp, ctx->i32_1, "");
3987 }
3988
3989 static void si_llvm_emit_ddxy(
3990 const struct lp_build_tgsi_action *action,
3991 struct lp_build_tgsi_context *bld_base,
3992 struct lp_build_emit_data *emit_data)
3993 {
3994 struct si_shader_context *ctx = si_shader_context(bld_base);
3995 unsigned opcode = emit_data->info->opcode;
3996 LLVMValueRef val;
3997 int idx;
3998 unsigned mask;
3999
4000 if (opcode == TGSI_OPCODE_DDX_FINE)
4001 mask = AC_TID_MASK_LEFT;
4002 else if (opcode == TGSI_OPCODE_DDY_FINE)
4003 mask = AC_TID_MASK_TOP;
4004 else
4005 mask = AC_TID_MASK_TOP_LEFT;
4006
4007 /* for DDX we want to next X pixel, DDY next Y pixel. */
4008 idx = (opcode == TGSI_OPCODE_DDX || opcode == TGSI_OPCODE_DDX_FINE) ? 1 : 2;
4009
4010 val = ac_to_integer(&ctx->ac, emit_data->args[0]);
4011 val = ac_build_ddxy(&ctx->ac, mask, idx, val);
4012 emit_data->output[emit_data->chan] = val;
4013 }
4014
4015 /*
4016 * this takes an I,J coordinate pair,
4017 * and works out the X and Y derivatives.
4018 * it returns DDX(I), DDX(J), DDY(I), DDY(J).
4019 */
4020 static LLVMValueRef si_llvm_emit_ddxy_interp(
4021 struct lp_build_tgsi_context *bld_base,
4022 LLVMValueRef interp_ij)
4023 {
4024 struct si_shader_context *ctx = si_shader_context(bld_base);
4025 LLVMValueRef result[4], a;
4026 unsigned i;
4027
4028 for (i = 0; i < 2; i++) {
4029 a = LLVMBuildExtractElement(ctx->ac.builder, interp_ij,
4030 LLVMConstInt(ctx->i32, i, 0), "");
4031 result[i] = lp_build_emit_llvm_unary(bld_base, TGSI_OPCODE_DDX, a);
4032 result[2+i] = lp_build_emit_llvm_unary(bld_base, TGSI_OPCODE_DDY, a);
4033 }
4034
4035 return lp_build_gather_values(&ctx->gallivm, result, 4);
4036 }
4037
4038 static void interp_fetch_args(
4039 struct lp_build_tgsi_context *bld_base,
4040 struct lp_build_emit_data *emit_data)
4041 {
4042 struct si_shader_context *ctx = si_shader_context(bld_base);
4043 const struct tgsi_full_instruction *inst = emit_data->inst;
4044
4045 if (inst->Instruction.Opcode == TGSI_OPCODE_INTERP_OFFSET) {
4046 /* offset is in second src, first two channels */
4047 emit_data->args[0] = lp_build_emit_fetch(bld_base,
4048 emit_data->inst, 1,
4049 TGSI_CHAN_X);
4050 emit_data->args[1] = lp_build_emit_fetch(bld_base,
4051 emit_data->inst, 1,
4052 TGSI_CHAN_Y);
4053 emit_data->arg_count = 2;
4054 } else if (inst->Instruction.Opcode == TGSI_OPCODE_INTERP_SAMPLE) {
4055 LLVMValueRef sample_position;
4056 LLVMValueRef sample_id;
4057 LLVMValueRef halfval = LLVMConstReal(ctx->f32, 0.5f);
4058
4059 /* fetch sample ID, then fetch its sample position,
4060 * and place into first two channels.
4061 */
4062 sample_id = lp_build_emit_fetch(bld_base,
4063 emit_data->inst, 1, TGSI_CHAN_X);
4064 sample_id = ac_to_integer(&ctx->ac, sample_id);
4065
4066 /* Section 8.13.2 (Interpolation Functions) of the OpenGL Shading
4067 * Language 4.50 spec says about interpolateAtSample:
4068 *
4069 * "Returns the value of the input interpolant variable at
4070 * the location of sample number sample. If multisample
4071 * buffers are not available, the input variable will be
4072 * evaluated at the center of the pixel. If sample sample
4073 * does not exist, the position used to interpolate the
4074 * input variable is undefined."
4075 *
4076 * This means that sample_id values outside of the valid are
4077 * in fact valid input, and the usual mechanism for loading the
4078 * sample position doesn't work.
4079 */
4080 if (ctx->shader->key.mono.u.ps.interpolate_at_sample_force_center) {
4081 LLVMValueRef center[4] = {
4082 LLVMConstReal(ctx->f32, 0.5),
4083 LLVMConstReal(ctx->f32, 0.5),
4084 ctx->ac.f32_0,
4085 ctx->ac.f32_0,
4086 };
4087
4088 sample_position = lp_build_gather_values(&ctx->gallivm, center, 4);
4089 } else {
4090 sample_position = load_sample_position(&ctx->abi, sample_id);
4091 }
4092
4093 emit_data->args[0] = LLVMBuildExtractElement(ctx->ac.builder,
4094 sample_position,
4095 ctx->i32_0, "");
4096
4097 emit_data->args[0] = LLVMBuildFSub(ctx->ac.builder, emit_data->args[0], halfval, "");
4098 emit_data->args[1] = LLVMBuildExtractElement(ctx->ac.builder,
4099 sample_position,
4100 ctx->i32_1, "");
4101 emit_data->args[1] = LLVMBuildFSub(ctx->ac.builder, emit_data->args[1], halfval, "");
4102 emit_data->arg_count = 2;
4103 }
4104 }
4105
4106 static void build_interp_intrinsic(const struct lp_build_tgsi_action *action,
4107 struct lp_build_tgsi_context *bld_base,
4108 struct lp_build_emit_data *emit_data)
4109 {
4110 struct si_shader_context *ctx = si_shader_context(bld_base);
4111 struct si_shader *shader = ctx->shader;
4112 const struct tgsi_shader_info *info = &shader->selector->info;
4113 LLVMValueRef interp_param;
4114 const struct tgsi_full_instruction *inst = emit_data->inst;
4115 const struct tgsi_full_src_register *input = &inst->Src[0];
4116 int input_base, input_array_size;
4117 int chan;
4118 int i;
4119 LLVMValueRef prim_mask = ctx->abi.prim_mask;
4120 LLVMValueRef array_idx;
4121 int interp_param_idx;
4122 unsigned interp;
4123 unsigned location;
4124
4125 assert(input->Register.File == TGSI_FILE_INPUT);
4126
4127 if (input->Register.Indirect) {
4128 unsigned array_id = input->Indirect.ArrayID;
4129
4130 if (array_id) {
4131 input_base = info->input_array_first[array_id];
4132 input_array_size = info->input_array_last[array_id] - input_base + 1;
4133 } else {
4134 input_base = inst->Src[0].Register.Index;
4135 input_array_size = info->num_inputs - input_base;
4136 }
4137
4138 array_idx = si_get_indirect_index(ctx, &input->Indirect,
4139 1, input->Register.Index - input_base);
4140 } else {
4141 input_base = inst->Src[0].Register.Index;
4142 input_array_size = 1;
4143 array_idx = ctx->i32_0;
4144 }
4145
4146 interp = shader->selector->info.input_interpolate[input_base];
4147
4148 if (inst->Instruction.Opcode == TGSI_OPCODE_INTERP_OFFSET ||
4149 inst->Instruction.Opcode == TGSI_OPCODE_INTERP_SAMPLE)
4150 location = TGSI_INTERPOLATE_LOC_CENTER;
4151 else
4152 location = TGSI_INTERPOLATE_LOC_CENTROID;
4153
4154 interp_param_idx = lookup_interp_param_index(interp, location);
4155 if (interp_param_idx == -1)
4156 return;
4157 else if (interp_param_idx)
4158 interp_param = LLVMGetParam(ctx->main_fn, interp_param_idx);
4159 else
4160 interp_param = NULL;
4161
4162 if (inst->Instruction.Opcode == TGSI_OPCODE_INTERP_OFFSET ||
4163 inst->Instruction.Opcode == TGSI_OPCODE_INTERP_SAMPLE) {
4164 LLVMValueRef ij_out[2];
4165 LLVMValueRef ddxy_out = si_llvm_emit_ddxy_interp(bld_base, interp_param);
4166
4167 /*
4168 * take the I then J parameters, and the DDX/Y for it, and
4169 * calculate the IJ inputs for the interpolator.
4170 * temp1 = ddx * offset/sample.x + I;
4171 * interp_param.I = ddy * offset/sample.y + temp1;
4172 * temp1 = ddx * offset/sample.x + J;
4173 * interp_param.J = ddy * offset/sample.y + temp1;
4174 */
4175 for (i = 0; i < 2; i++) {
4176 LLVMValueRef ix_ll = LLVMConstInt(ctx->i32, i, 0);
4177 LLVMValueRef iy_ll = LLVMConstInt(ctx->i32, i + 2, 0);
4178 LLVMValueRef ddx_el = LLVMBuildExtractElement(ctx->ac.builder,
4179 ddxy_out, ix_ll, "");
4180 LLVMValueRef ddy_el = LLVMBuildExtractElement(ctx->ac.builder,
4181 ddxy_out, iy_ll, "");
4182 LLVMValueRef interp_el = LLVMBuildExtractElement(ctx->ac.builder,
4183 interp_param, ix_ll, "");
4184 LLVMValueRef temp1, temp2;
4185
4186 interp_el = ac_to_float(&ctx->ac, interp_el);
4187
4188 temp1 = LLVMBuildFMul(ctx->ac.builder, ddx_el, emit_data->args[0], "");
4189
4190 temp1 = LLVMBuildFAdd(ctx->ac.builder, temp1, interp_el, "");
4191
4192 temp2 = LLVMBuildFMul(ctx->ac.builder, ddy_el, emit_data->args[1], "");
4193
4194 ij_out[i] = LLVMBuildFAdd(ctx->ac.builder, temp2, temp1, "");
4195 }
4196 interp_param = lp_build_gather_values(&ctx->gallivm, ij_out, 2);
4197 }
4198
4199 if (interp_param)
4200 interp_param = ac_to_float(&ctx->ac, interp_param);
4201
4202 for (chan = 0; chan < 4; chan++) {
4203 LLVMValueRef gather = LLVMGetUndef(LLVMVectorType(ctx->f32, input_array_size));
4204 unsigned schan = tgsi_util_get_full_src_register_swizzle(&inst->Src[0], chan);
4205
4206 for (unsigned idx = 0; idx < input_array_size; ++idx) {
4207 LLVMValueRef v, i = NULL, j = NULL;
4208
4209 if (interp_param) {
4210 i = LLVMBuildExtractElement(
4211 ctx->ac.builder, interp_param, ctx->i32_0, "");
4212 j = LLVMBuildExtractElement(
4213 ctx->ac.builder, interp_param, ctx->i32_1, "");
4214 }
4215 v = si_build_fs_interp(ctx, input_base + idx, schan,
4216 prim_mask, i, j);
4217
4218 gather = LLVMBuildInsertElement(ctx->ac.builder,
4219 gather, v, LLVMConstInt(ctx->i32, idx, false), "");
4220 }
4221
4222 emit_data->output[chan] = LLVMBuildExtractElement(
4223 ctx->ac.builder, gather, array_idx, "");
4224 }
4225 }
4226
4227 static void vote_all_emit(
4228 const struct lp_build_tgsi_action *action,
4229 struct lp_build_tgsi_context *bld_base,
4230 struct lp_build_emit_data *emit_data)
4231 {
4232 struct si_shader_context *ctx = si_shader_context(bld_base);
4233
4234 LLVMValueRef tmp = ac_build_vote_all(&ctx->ac, emit_data->args[0]);
4235 emit_data->output[emit_data->chan] =
4236 LLVMBuildSExt(ctx->ac.builder, tmp, ctx->i32, "");
4237 }
4238
4239 static void vote_any_emit(
4240 const struct lp_build_tgsi_action *action,
4241 struct lp_build_tgsi_context *bld_base,
4242 struct lp_build_emit_data *emit_data)
4243 {
4244 struct si_shader_context *ctx = si_shader_context(bld_base);
4245
4246 LLVMValueRef tmp = ac_build_vote_any(&ctx->ac, emit_data->args[0]);
4247 emit_data->output[emit_data->chan] =
4248 LLVMBuildSExt(ctx->ac.builder, tmp, ctx->i32, "");
4249 }
4250
4251 static void vote_eq_emit(
4252 const struct lp_build_tgsi_action *action,
4253 struct lp_build_tgsi_context *bld_base,
4254 struct lp_build_emit_data *emit_data)
4255 {
4256 struct si_shader_context *ctx = si_shader_context(bld_base);
4257
4258 LLVMValueRef tmp = ac_build_vote_eq(&ctx->ac, emit_data->args[0]);
4259 emit_data->output[emit_data->chan] =
4260 LLVMBuildSExt(ctx->ac.builder, tmp, ctx->i32, "");
4261 }
4262
4263 static void ballot_emit(
4264 const struct lp_build_tgsi_action *action,
4265 struct lp_build_tgsi_context *bld_base,
4266 struct lp_build_emit_data *emit_data)
4267 {
4268 struct si_shader_context *ctx = si_shader_context(bld_base);
4269 LLVMBuilderRef builder = ctx->ac.builder;
4270 LLVMValueRef tmp;
4271
4272 tmp = lp_build_emit_fetch(bld_base, emit_data->inst, 0, TGSI_CHAN_X);
4273 tmp = ac_build_ballot(&ctx->ac, tmp);
4274 tmp = LLVMBuildBitCast(builder, tmp, ctx->v2i32, "");
4275
4276 emit_data->output[0] = LLVMBuildExtractElement(builder, tmp, ctx->i32_0, "");
4277 emit_data->output[1] = LLVMBuildExtractElement(builder, tmp, ctx->i32_1, "");
4278 }
4279
4280 static void read_invoc_fetch_args(
4281 struct lp_build_tgsi_context *bld_base,
4282 struct lp_build_emit_data *emit_data)
4283 {
4284 emit_data->args[0] = lp_build_emit_fetch(bld_base, emit_data->inst,
4285 0, emit_data->src_chan);
4286
4287 /* Always read the source invocation (= lane) from the X channel. */
4288 emit_data->args[1] = lp_build_emit_fetch(bld_base, emit_data->inst,
4289 1, TGSI_CHAN_X);
4290 emit_data->arg_count = 2;
4291 }
4292
4293 static void read_lane_emit(
4294 const struct lp_build_tgsi_action *action,
4295 struct lp_build_tgsi_context *bld_base,
4296 struct lp_build_emit_data *emit_data)
4297 {
4298 struct si_shader_context *ctx = si_shader_context(bld_base);
4299
4300 /* We currently have no other way to prevent LLVM from lifting the icmp
4301 * calls to a dominating basic block.
4302 */
4303 ac_build_optimization_barrier(&ctx->ac, &emit_data->args[0]);
4304
4305 for (unsigned i = 0; i < emit_data->arg_count; ++i)
4306 emit_data->args[i] = ac_to_integer(&ctx->ac, emit_data->args[i]);
4307
4308 emit_data->output[emit_data->chan] =
4309 ac_build_intrinsic(&ctx->ac, action->intr_name,
4310 ctx->i32, emit_data->args, emit_data->arg_count,
4311 AC_FUNC_ATTR_READNONE |
4312 AC_FUNC_ATTR_CONVERGENT);
4313 }
4314
4315 static unsigned si_llvm_get_stream(struct lp_build_tgsi_context *bld_base,
4316 struct lp_build_emit_data *emit_data)
4317 {
4318 struct si_shader_context *ctx = si_shader_context(bld_base);
4319 struct tgsi_src_register src0 = emit_data->inst->Src[0].Register;
4320 LLVMValueRef imm;
4321 unsigned stream;
4322
4323 assert(src0.File == TGSI_FILE_IMMEDIATE);
4324
4325 imm = ctx->imms[src0.Index * TGSI_NUM_CHANNELS + src0.SwizzleX];
4326 stream = LLVMConstIntGetZExtValue(imm) & 0x3;
4327 return stream;
4328 }
4329
4330 /* Emit one vertex from the geometry shader */
4331 static void si_llvm_emit_vertex(struct ac_shader_abi *abi,
4332 unsigned stream,
4333 LLVMValueRef *addrs)
4334 {
4335 struct si_shader_context *ctx = si_shader_context_from_abi(abi);
4336 struct tgsi_shader_info *info = &ctx->shader->selector->info;
4337 struct lp_build_context *uint = &ctx->bld_base.uint_bld;
4338 struct si_shader *shader = ctx->shader;
4339 struct lp_build_if_state if_state;
4340 LLVMValueRef soffset = LLVMGetParam(ctx->main_fn,
4341 ctx->param_gs2vs_offset);
4342 LLVMValueRef gs_next_vertex;
4343 LLVMValueRef can_emit;
4344 unsigned chan, offset;
4345 int i;
4346
4347 /* Write vertex attribute values to GSVS ring */
4348 gs_next_vertex = LLVMBuildLoad(ctx->ac.builder,
4349 ctx->gs_next_vertex[stream],
4350 "");
4351
4352 /* If this thread has already emitted the declared maximum number of
4353 * vertices, skip the write: excessive vertex emissions are not
4354 * supposed to have any effect.
4355 *
4356 * If the shader has no writes to memory, kill it instead. This skips
4357 * further memory loads and may allow LLVM to skip to the end
4358 * altogether.
4359 */
4360 can_emit = LLVMBuildICmp(ctx->ac.builder, LLVMIntULT, gs_next_vertex,
4361 LLVMConstInt(ctx->i32,
4362 shader->selector->gs_max_out_vertices, 0), "");
4363
4364 bool use_kill = !info->writes_memory;
4365 if (use_kill) {
4366 ac_build_kill_if_false(&ctx->ac, can_emit);
4367 } else {
4368 lp_build_if(&if_state, &ctx->gallivm, can_emit);
4369 }
4370
4371 offset = 0;
4372 for (i = 0; i < info->num_outputs; i++) {
4373 for (chan = 0; chan < 4; chan++) {
4374 if (!(info->output_usagemask[i] & (1 << chan)) ||
4375 ((info->output_streams[i] >> (2 * chan)) & 3) != stream)
4376 continue;
4377
4378 LLVMValueRef out_val = LLVMBuildLoad(ctx->ac.builder, addrs[4 * i + chan], "");
4379 LLVMValueRef voffset =
4380 LLVMConstInt(ctx->i32, offset *
4381 shader->selector->gs_max_out_vertices, 0);
4382 offset++;
4383
4384 voffset = lp_build_add(uint, voffset, gs_next_vertex);
4385 voffset = lp_build_mul_imm(uint, voffset, 4);
4386
4387 out_val = ac_to_integer(&ctx->ac, out_val);
4388
4389 ac_build_buffer_store_dword(&ctx->ac,
4390 ctx->gsvs_ring[stream],
4391 out_val, 1,
4392 voffset, soffset, 0,
4393 1, 1, true, true);
4394 }
4395 }
4396
4397 gs_next_vertex = lp_build_add(uint, gs_next_vertex,
4398 ctx->i32_1);
4399
4400 LLVMBuildStore(ctx->ac.builder, gs_next_vertex, ctx->gs_next_vertex[stream]);
4401
4402 /* Signal vertex emission */
4403 ac_build_sendmsg(&ctx->ac, AC_SENDMSG_GS_OP_EMIT | AC_SENDMSG_GS | (stream << 8),
4404 si_get_gs_wave_id(ctx));
4405 if (!use_kill)
4406 lp_build_endif(&if_state);
4407 }
4408
4409 /* Emit one vertex from the geometry shader */
4410 static void si_tgsi_emit_vertex(
4411 const struct lp_build_tgsi_action *action,
4412 struct lp_build_tgsi_context *bld_base,
4413 struct lp_build_emit_data *emit_data)
4414 {
4415 struct si_shader_context *ctx = si_shader_context(bld_base);
4416 unsigned stream = si_llvm_get_stream(bld_base, emit_data);
4417
4418 si_llvm_emit_vertex(&ctx->abi, stream, ctx->outputs[0]);
4419 }
4420
4421 /* Cut one primitive from the geometry shader */
4422 static void si_llvm_emit_primitive(struct ac_shader_abi *abi,
4423 unsigned stream)
4424 {
4425 struct si_shader_context *ctx = si_shader_context_from_abi(abi);
4426
4427 /* Signal primitive cut */
4428 ac_build_sendmsg(&ctx->ac, AC_SENDMSG_GS_OP_CUT | AC_SENDMSG_GS | (stream << 8),
4429 si_get_gs_wave_id(ctx));
4430 }
4431
4432 /* Cut one primitive from the geometry shader */
4433 static void si_tgsi_emit_primitive(
4434 const struct lp_build_tgsi_action *action,
4435 struct lp_build_tgsi_context *bld_base,
4436 struct lp_build_emit_data *emit_data)
4437 {
4438 struct si_shader_context *ctx = si_shader_context(bld_base);
4439
4440 si_llvm_emit_primitive(&ctx->abi, si_llvm_get_stream(bld_base, emit_data));
4441 }
4442
4443 static void si_llvm_emit_barrier(const struct lp_build_tgsi_action *action,
4444 struct lp_build_tgsi_context *bld_base,
4445 struct lp_build_emit_data *emit_data)
4446 {
4447 struct si_shader_context *ctx = si_shader_context(bld_base);
4448
4449 /* SI only (thanks to a hw bug workaround):
4450 * The real barrier instruction isn’t needed, because an entire patch
4451 * always fits into a single wave.
4452 */
4453 if (ctx->screen->info.chip_class == SI &&
4454 ctx->type == PIPE_SHADER_TESS_CTRL) {
4455 ac_build_waitcnt(&ctx->ac, LGKM_CNT & VM_CNT);
4456 return;
4457 }
4458
4459 lp_build_intrinsic(ctx->ac.builder,
4460 "llvm.amdgcn.s.barrier",
4461 ctx->voidt, NULL, 0, LP_FUNC_ATTR_CONVERGENT);
4462 }
4463
4464 static const struct lp_build_tgsi_action interp_action = {
4465 .fetch_args = interp_fetch_args,
4466 .emit = build_interp_intrinsic,
4467 };
4468
4469 static void si_create_function(struct si_shader_context *ctx,
4470 const char *name,
4471 LLVMTypeRef *returns, unsigned num_returns,
4472 struct si_function_info *fninfo,
4473 unsigned max_workgroup_size)
4474 {
4475 int i;
4476
4477 si_llvm_create_func(ctx, name, returns, num_returns,
4478 fninfo->types, fninfo->num_params);
4479 ctx->return_value = LLVMGetUndef(ctx->return_type);
4480
4481 for (i = 0; i < fninfo->num_sgpr_params; ++i) {
4482 LLVMValueRef P = LLVMGetParam(ctx->main_fn, i);
4483
4484 /* The combination of:
4485 * - noalias
4486 * - dereferenceable
4487 * - invariant.load
4488 * allows the optimization passes to move loads and reduces
4489 * SGPR spilling significantly.
4490 */
4491 lp_add_function_attr(ctx->main_fn, i + 1, LP_FUNC_ATTR_INREG);
4492
4493 if (LLVMGetTypeKind(LLVMTypeOf(P)) == LLVMPointerTypeKind) {
4494 lp_add_function_attr(ctx->main_fn, i + 1, LP_FUNC_ATTR_NOALIAS);
4495 ac_add_attr_dereferenceable(P, UINT64_MAX);
4496 }
4497 }
4498
4499 for (i = 0; i < fninfo->num_params; ++i) {
4500 if (fninfo->assign[i])
4501 *fninfo->assign[i] = LLVMGetParam(ctx->main_fn, i);
4502 }
4503
4504 if (ctx->screen->info.address32_hi) {
4505 ac_llvm_add_target_dep_function_attr(ctx->main_fn,
4506 "amdgpu-32bit-address-high-bits",
4507 ctx->screen->info.address32_hi);
4508 }
4509
4510 if (max_workgroup_size) {
4511 ac_llvm_add_target_dep_function_attr(ctx->main_fn,
4512 "amdgpu-max-work-group-size",
4513 max_workgroup_size);
4514 }
4515 LLVMAddTargetDependentFunctionAttr(ctx->main_fn,
4516 "no-signed-zeros-fp-math",
4517 "true");
4518
4519 if (ctx->screen->debug_flags & DBG(UNSAFE_MATH)) {
4520 /* These were copied from some LLVM test. */
4521 LLVMAddTargetDependentFunctionAttr(ctx->main_fn,
4522 "less-precise-fpmad",
4523 "true");
4524 LLVMAddTargetDependentFunctionAttr(ctx->main_fn,
4525 "no-infs-fp-math",
4526 "true");
4527 LLVMAddTargetDependentFunctionAttr(ctx->main_fn,
4528 "no-nans-fp-math",
4529 "true");
4530 LLVMAddTargetDependentFunctionAttr(ctx->main_fn,
4531 "unsafe-fp-math",
4532 "true");
4533 }
4534 }
4535
4536 static void declare_streamout_params(struct si_shader_context *ctx,
4537 struct pipe_stream_output_info *so,
4538 struct si_function_info *fninfo)
4539 {
4540 int i;
4541
4542 /* Streamout SGPRs. */
4543 if (so->num_outputs) {
4544 if (ctx->type != PIPE_SHADER_TESS_EVAL)
4545 ctx->param_streamout_config = add_arg(fninfo, ARG_SGPR, ctx->ac.i32);
4546 else
4547 ctx->param_streamout_config = fninfo->num_params - 1;
4548
4549 ctx->param_streamout_write_index = add_arg(fninfo, ARG_SGPR, ctx->ac.i32);
4550 }
4551 /* A streamout buffer offset is loaded if the stride is non-zero. */
4552 for (i = 0; i < 4; i++) {
4553 if (!so->stride[i])
4554 continue;
4555
4556 ctx->param_streamout_offset[i] = add_arg(fninfo, ARG_SGPR, ctx->ac.i32);
4557 }
4558 }
4559
4560 static unsigned si_get_max_workgroup_size(const struct si_shader *shader)
4561 {
4562 switch (shader->selector->type) {
4563 case PIPE_SHADER_TESS_CTRL:
4564 /* Return this so that LLVM doesn't remove s_barrier
4565 * instructions on chips where we use s_barrier. */
4566 return shader->selector->screen->info.chip_class >= CIK ? 128 : 64;
4567
4568 case PIPE_SHADER_GEOMETRY:
4569 return shader->selector->screen->info.chip_class >= GFX9 ? 128 : 64;
4570
4571 case PIPE_SHADER_COMPUTE:
4572 break; /* see below */
4573
4574 default:
4575 return 0;
4576 }
4577
4578 const unsigned *properties = shader->selector->info.properties;
4579 unsigned max_work_group_size =
4580 properties[TGSI_PROPERTY_CS_FIXED_BLOCK_WIDTH] *
4581 properties[TGSI_PROPERTY_CS_FIXED_BLOCK_HEIGHT] *
4582 properties[TGSI_PROPERTY_CS_FIXED_BLOCK_DEPTH];
4583
4584 if (!max_work_group_size) {
4585 /* This is a variable group size compute shader,
4586 * compile it for the maximum possible group size.
4587 */
4588 max_work_group_size = SI_MAX_VARIABLE_THREADS_PER_BLOCK;
4589 }
4590 return max_work_group_size;
4591 }
4592
4593 static void declare_const_and_shader_buffers(struct si_shader_context *ctx,
4594 struct si_function_info *fninfo,
4595 bool assign_params)
4596 {
4597 LLVMTypeRef const_shader_buf_type;
4598
4599 if (ctx->shader->selector->info.const_buffers_declared == 1 &&
4600 ctx->shader->selector->info.shader_buffers_declared == 0)
4601 const_shader_buf_type = ctx->f32;
4602 else
4603 const_shader_buf_type = ctx->v4i32;
4604
4605 unsigned const_and_shader_buffers =
4606 add_arg(fninfo, ARG_SGPR,
4607 ac_array_in_const32_addr_space(const_shader_buf_type));
4608
4609 if (assign_params)
4610 ctx->param_const_and_shader_buffers = const_and_shader_buffers;
4611 }
4612
4613 static void declare_samplers_and_images(struct si_shader_context *ctx,
4614 struct si_function_info *fninfo,
4615 bool assign_params)
4616 {
4617 unsigned samplers_and_images =
4618 add_arg(fninfo, ARG_SGPR,
4619 ac_array_in_const32_addr_space(ctx->v8i32));
4620
4621 if (assign_params)
4622 ctx->param_samplers_and_images = samplers_and_images;
4623 }
4624
4625 static void declare_per_stage_desc_pointers(struct si_shader_context *ctx,
4626 struct si_function_info *fninfo,
4627 bool assign_params)
4628 {
4629 declare_const_and_shader_buffers(ctx, fninfo, assign_params);
4630 declare_samplers_and_images(ctx, fninfo, assign_params);
4631 }
4632
4633 static void declare_global_desc_pointers(struct si_shader_context *ctx,
4634 struct si_function_info *fninfo)
4635 {
4636 ctx->param_rw_buffers = add_arg(fninfo, ARG_SGPR,
4637 ac_array_in_const32_addr_space(ctx->v4i32));
4638 ctx->param_bindless_samplers_and_images = add_arg(fninfo, ARG_SGPR,
4639 ac_array_in_const32_addr_space(ctx->v8i32));
4640 }
4641
4642 static void declare_vs_specific_input_sgprs(struct si_shader_context *ctx,
4643 struct si_function_info *fninfo)
4644 {
4645 add_arg_assign(fninfo, ARG_SGPR, ctx->i32, &ctx->abi.base_vertex);
4646 add_arg_assign(fninfo, ARG_SGPR, ctx->i32, &ctx->abi.start_instance);
4647 add_arg_assign(fninfo, ARG_SGPR, ctx->i32, &ctx->abi.draw_id);
4648 ctx->param_vs_state_bits = add_arg(fninfo, ARG_SGPR, ctx->i32);
4649 }
4650
4651 static void declare_vs_input_vgprs(struct si_shader_context *ctx,
4652 struct si_function_info *fninfo,
4653 unsigned *num_prolog_vgprs)
4654 {
4655 struct si_shader *shader = ctx->shader;
4656
4657 add_arg_assign(fninfo, ARG_VGPR, ctx->i32, &ctx->abi.vertex_id);
4658 if (shader->key.as_ls) {
4659 ctx->param_rel_auto_id = add_arg(fninfo, ARG_VGPR, ctx->i32);
4660 add_arg_assign(fninfo, ARG_VGPR, ctx->i32, &ctx->abi.instance_id);
4661 } else {
4662 add_arg_assign(fninfo, ARG_VGPR, ctx->i32, &ctx->abi.instance_id);
4663 ctx->param_vs_prim_id = add_arg(fninfo, ARG_VGPR, ctx->i32);
4664 }
4665 add_arg(fninfo, ARG_VGPR, ctx->i32); /* unused */
4666
4667 if (!shader->is_gs_copy_shader) {
4668 /* Vertex load indices. */
4669 ctx->param_vertex_index0 = fninfo->num_params;
4670 for (unsigned i = 0; i < shader->selector->info.num_inputs; i++)
4671 add_arg(fninfo, ARG_VGPR, ctx->i32);
4672 *num_prolog_vgprs += shader->selector->info.num_inputs;
4673 }
4674 }
4675
4676 static void declare_tes_input_vgprs(struct si_shader_context *ctx,
4677 struct si_function_info *fninfo)
4678 {
4679 ctx->param_tes_u = add_arg(fninfo, ARG_VGPR, ctx->f32);
4680 ctx->param_tes_v = add_arg(fninfo, ARG_VGPR, ctx->f32);
4681 ctx->param_tes_rel_patch_id = add_arg(fninfo, ARG_VGPR, ctx->i32);
4682 add_arg_assign(fninfo, ARG_VGPR, ctx->i32, &ctx->abi.tes_patch_id);
4683 }
4684
4685 enum {
4686 /* Convenient merged shader definitions. */
4687 SI_SHADER_MERGED_VERTEX_TESSCTRL = PIPE_SHADER_TYPES,
4688 SI_SHADER_MERGED_VERTEX_OR_TESSEVAL_GEOMETRY,
4689 };
4690
4691 static void create_function(struct si_shader_context *ctx)
4692 {
4693 struct si_shader *shader = ctx->shader;
4694 struct si_function_info fninfo;
4695 LLVMTypeRef returns[16+32*4];
4696 unsigned i, num_return_sgprs;
4697 unsigned num_returns = 0;
4698 unsigned num_prolog_vgprs = 0;
4699 unsigned type = ctx->type;
4700 unsigned vs_blit_property =
4701 shader->selector->info.properties[TGSI_PROPERTY_VS_BLIT_SGPRS];
4702
4703 si_init_function_info(&fninfo);
4704
4705 /* Set MERGED shaders. */
4706 if (ctx->screen->info.chip_class >= GFX9) {
4707 if (shader->key.as_ls || type == PIPE_SHADER_TESS_CTRL)
4708 type = SI_SHADER_MERGED_VERTEX_TESSCTRL; /* LS or HS */
4709 else if (shader->key.as_es || type == PIPE_SHADER_GEOMETRY)
4710 type = SI_SHADER_MERGED_VERTEX_OR_TESSEVAL_GEOMETRY;
4711 }
4712
4713 LLVMTypeRef v3i32 = LLVMVectorType(ctx->i32, 3);
4714
4715 switch (type) {
4716 case PIPE_SHADER_VERTEX:
4717 declare_global_desc_pointers(ctx, &fninfo);
4718
4719 if (vs_blit_property) {
4720 ctx->param_vs_blit_inputs = fninfo.num_params;
4721 add_arg(&fninfo, ARG_SGPR, ctx->i32); /* i16 x1, y1 */
4722 add_arg(&fninfo, ARG_SGPR, ctx->i32); /* i16 x2, y2 */
4723 add_arg(&fninfo, ARG_SGPR, ctx->f32); /* depth */
4724
4725 if (vs_blit_property == SI_VS_BLIT_SGPRS_POS_COLOR) {
4726 add_arg(&fninfo, ARG_SGPR, ctx->f32); /* color0 */
4727 add_arg(&fninfo, ARG_SGPR, ctx->f32); /* color1 */
4728 add_arg(&fninfo, ARG_SGPR, ctx->f32); /* color2 */
4729 add_arg(&fninfo, ARG_SGPR, ctx->f32); /* color3 */
4730 } else if (vs_blit_property == SI_VS_BLIT_SGPRS_POS_TEXCOORD) {
4731 add_arg(&fninfo, ARG_SGPR, ctx->f32); /* texcoord.x1 */
4732 add_arg(&fninfo, ARG_SGPR, ctx->f32); /* texcoord.y1 */
4733 add_arg(&fninfo, ARG_SGPR, ctx->f32); /* texcoord.x2 */
4734 add_arg(&fninfo, ARG_SGPR, ctx->f32); /* texcoord.y2 */
4735 add_arg(&fninfo, ARG_SGPR, ctx->f32); /* texcoord.z */
4736 add_arg(&fninfo, ARG_SGPR, ctx->f32); /* texcoord.w */
4737 }
4738
4739 /* VGPRs */
4740 declare_vs_input_vgprs(ctx, &fninfo, &num_prolog_vgprs);
4741 break;
4742 }
4743
4744 declare_per_stage_desc_pointers(ctx, &fninfo, true);
4745 declare_vs_specific_input_sgprs(ctx, &fninfo);
4746 ctx->param_vertex_buffers = add_arg(&fninfo, ARG_SGPR,
4747 ac_array_in_const32_addr_space(ctx->v4i32));
4748
4749 if (shader->key.as_es) {
4750 ctx->param_es2gs_offset = add_arg(&fninfo, ARG_SGPR, ctx->i32);
4751 } else if (shader->key.as_ls) {
4752 /* no extra parameters */
4753 } else {
4754 if (shader->is_gs_copy_shader) {
4755 fninfo.num_params = ctx->param_rw_buffers + 1;
4756 fninfo.num_sgpr_params = fninfo.num_params;
4757 }
4758
4759 /* The locations of the other parameters are assigned dynamically. */
4760 declare_streamout_params(ctx, &shader->selector->so,
4761 &fninfo);
4762 }
4763
4764 /* VGPRs */
4765 declare_vs_input_vgprs(ctx, &fninfo, &num_prolog_vgprs);
4766 break;
4767
4768 case PIPE_SHADER_TESS_CTRL: /* SI-CI-VI */
4769 declare_global_desc_pointers(ctx, &fninfo);
4770 declare_per_stage_desc_pointers(ctx, &fninfo, true);
4771 ctx->param_tcs_offchip_layout = add_arg(&fninfo, ARG_SGPR, ctx->i32);
4772 ctx->param_tcs_out_lds_offsets = add_arg(&fninfo, ARG_SGPR, ctx->i32);
4773 ctx->param_tcs_out_lds_layout = add_arg(&fninfo, ARG_SGPR, ctx->i32);
4774 ctx->param_vs_state_bits = add_arg(&fninfo, ARG_SGPR, ctx->i32);
4775 ctx->param_tcs_offchip_offset = add_arg(&fninfo, ARG_SGPR, ctx->i32);
4776 ctx->param_tcs_factor_offset = add_arg(&fninfo, ARG_SGPR, ctx->i32);
4777
4778 /* VGPRs */
4779 add_arg_assign(&fninfo, ARG_VGPR, ctx->i32, &ctx->abi.tcs_patch_id);
4780 add_arg_assign(&fninfo, ARG_VGPR, ctx->i32, &ctx->abi.tcs_rel_ids);
4781
4782 /* param_tcs_offchip_offset and param_tcs_factor_offset are
4783 * placed after the user SGPRs.
4784 */
4785 for (i = 0; i < GFX6_TCS_NUM_USER_SGPR + 2; i++)
4786 returns[num_returns++] = ctx->i32; /* SGPRs */
4787 for (i = 0; i < 11; i++)
4788 returns[num_returns++] = ctx->f32; /* VGPRs */
4789 break;
4790
4791 case SI_SHADER_MERGED_VERTEX_TESSCTRL:
4792 /* Merged stages have 8 system SGPRs at the beginning. */
4793 /* SPI_SHADER_USER_DATA_ADDR_LO/HI_HS */
4794 if (HAVE_32BIT_POINTERS) {
4795 declare_per_stage_desc_pointers(ctx, &fninfo,
4796 ctx->type == PIPE_SHADER_TESS_CTRL);
4797 } else {
4798 declare_const_and_shader_buffers(ctx, &fninfo,
4799 ctx->type == PIPE_SHADER_TESS_CTRL);
4800 }
4801 ctx->param_tcs_offchip_offset = add_arg(&fninfo, ARG_SGPR, ctx->i32);
4802 ctx->param_merged_wave_info = add_arg(&fninfo, ARG_SGPR, ctx->i32);
4803 ctx->param_tcs_factor_offset = add_arg(&fninfo, ARG_SGPR, ctx->i32);
4804 ctx->param_merged_scratch_offset = add_arg(&fninfo, ARG_SGPR, ctx->i32);
4805 add_arg(&fninfo, ARG_SGPR, ctx->i32); /* unused */
4806 add_arg(&fninfo, ARG_SGPR, ctx->i32); /* unused */
4807
4808 declare_global_desc_pointers(ctx, &fninfo);
4809 declare_per_stage_desc_pointers(ctx, &fninfo,
4810 ctx->type == PIPE_SHADER_VERTEX);
4811 declare_vs_specific_input_sgprs(ctx, &fninfo);
4812
4813 if (!HAVE_32BIT_POINTERS) {
4814 declare_samplers_and_images(ctx, &fninfo,
4815 ctx->type == PIPE_SHADER_TESS_CTRL);
4816 }
4817 ctx->param_tcs_offchip_layout = add_arg(&fninfo, ARG_SGPR, ctx->i32);
4818 ctx->param_tcs_out_lds_offsets = add_arg(&fninfo, ARG_SGPR, ctx->i32);
4819 ctx->param_tcs_out_lds_layout = add_arg(&fninfo, ARG_SGPR, ctx->i32);
4820 if (!HAVE_32BIT_POINTERS) /* Align to 2 dwords. */
4821 add_arg(&fninfo, ARG_SGPR, ctx->i32); /* unused */
4822 ctx->param_vertex_buffers = add_arg(&fninfo, ARG_SGPR,
4823 ac_array_in_const32_addr_space(ctx->v4i32));
4824
4825 /* VGPRs (first TCS, then VS) */
4826 add_arg_assign(&fninfo, ARG_VGPR, ctx->i32, &ctx->abi.tcs_patch_id);
4827 add_arg_assign(&fninfo, ARG_VGPR, ctx->i32, &ctx->abi.tcs_rel_ids);
4828
4829 if (ctx->type == PIPE_SHADER_VERTEX) {
4830 declare_vs_input_vgprs(ctx, &fninfo,
4831 &num_prolog_vgprs);
4832
4833 /* LS return values are inputs to the TCS main shader part. */
4834 for (i = 0; i < 8 + GFX9_TCS_NUM_USER_SGPR; i++)
4835 returns[num_returns++] = ctx->i32; /* SGPRs */
4836 for (i = 0; i < 2; i++)
4837 returns[num_returns++] = ctx->f32; /* VGPRs */
4838 } else {
4839 /* TCS return values are inputs to the TCS epilog.
4840 *
4841 * param_tcs_offchip_offset, param_tcs_factor_offset,
4842 * param_tcs_offchip_layout, and param_rw_buffers
4843 * should be passed to the epilog.
4844 */
4845 for (i = 0; i <= 8 + GFX9_SGPR_TCS_OUT_LAYOUT; i++)
4846 returns[num_returns++] = ctx->i32; /* SGPRs */
4847 for (i = 0; i < 11; i++)
4848 returns[num_returns++] = ctx->f32; /* VGPRs */
4849 }
4850 break;
4851
4852 case SI_SHADER_MERGED_VERTEX_OR_TESSEVAL_GEOMETRY:
4853 /* Merged stages have 8 system SGPRs at the beginning. */
4854 /* SPI_SHADER_USER_DATA_ADDR_LO/HI_GS */
4855 if (HAVE_32BIT_POINTERS) {
4856 declare_per_stage_desc_pointers(ctx, &fninfo,
4857 ctx->type == PIPE_SHADER_GEOMETRY);
4858 } else {
4859 declare_const_and_shader_buffers(ctx, &fninfo,
4860 ctx->type == PIPE_SHADER_GEOMETRY);
4861 }
4862 ctx->param_gs2vs_offset = add_arg(&fninfo, ARG_SGPR, ctx->i32);
4863 ctx->param_merged_wave_info = add_arg(&fninfo, ARG_SGPR, ctx->i32);
4864 ctx->param_tcs_offchip_offset = add_arg(&fninfo, ARG_SGPR, ctx->i32);
4865 ctx->param_merged_scratch_offset = add_arg(&fninfo, ARG_SGPR, ctx->i32);
4866 add_arg(&fninfo, ARG_SGPR, ctx->i32); /* unused (SPI_SHADER_PGM_LO/HI_GS << 8) */
4867 add_arg(&fninfo, ARG_SGPR, ctx->i32); /* unused (SPI_SHADER_PGM_LO/HI_GS >> 24) */
4868
4869 declare_global_desc_pointers(ctx, &fninfo);
4870 declare_per_stage_desc_pointers(ctx, &fninfo,
4871 (ctx->type == PIPE_SHADER_VERTEX ||
4872 ctx->type == PIPE_SHADER_TESS_EVAL));
4873 if (ctx->type == PIPE_SHADER_VERTEX) {
4874 declare_vs_specific_input_sgprs(ctx, &fninfo);
4875 } else {
4876 ctx->param_tcs_offchip_layout = add_arg(&fninfo, ARG_SGPR, ctx->i32);
4877 ctx->param_tes_offchip_addr = add_arg(&fninfo, ARG_SGPR, ctx->i32);
4878 if (!HAVE_32BIT_POINTERS) {
4879 /* Declare as many input SGPRs as the VS has. */
4880 add_arg(&fninfo, ARG_SGPR, ctx->i32); /* unused */
4881 ctx->param_vs_state_bits = add_arg(&fninfo, ARG_SGPR, ctx->i32); /* unused */
4882 }
4883 }
4884
4885 if (!HAVE_32BIT_POINTERS) {
4886 declare_samplers_and_images(ctx, &fninfo,
4887 ctx->type == PIPE_SHADER_GEOMETRY);
4888 }
4889 if (ctx->type == PIPE_SHADER_VERTEX) {
4890 ctx->param_vertex_buffers = add_arg(&fninfo, ARG_SGPR,
4891 ac_array_in_const32_addr_space(ctx->v4i32));
4892 }
4893
4894 /* VGPRs (first GS, then VS/TES) */
4895 ctx->param_gs_vtx01_offset = add_arg(&fninfo, ARG_VGPR, ctx->i32);
4896 ctx->param_gs_vtx23_offset = add_arg(&fninfo, ARG_VGPR, ctx->i32);
4897 add_arg_assign(&fninfo, ARG_VGPR, ctx->i32, &ctx->abi.gs_prim_id);
4898 add_arg_assign(&fninfo, ARG_VGPR, ctx->i32, &ctx->abi.gs_invocation_id);
4899 ctx->param_gs_vtx45_offset = add_arg(&fninfo, ARG_VGPR, ctx->i32);
4900
4901 if (ctx->type == PIPE_SHADER_VERTEX) {
4902 declare_vs_input_vgprs(ctx, &fninfo,
4903 &num_prolog_vgprs);
4904 } else if (ctx->type == PIPE_SHADER_TESS_EVAL) {
4905 declare_tes_input_vgprs(ctx, &fninfo);
4906 }
4907
4908 if (ctx->type == PIPE_SHADER_VERTEX ||
4909 ctx->type == PIPE_SHADER_TESS_EVAL) {
4910 unsigned num_user_sgprs;
4911
4912 if (ctx->type == PIPE_SHADER_VERTEX)
4913 num_user_sgprs = GFX9_VSGS_NUM_USER_SGPR;
4914 else
4915 num_user_sgprs = GFX9_TESGS_NUM_USER_SGPR;
4916
4917 /* ES return values are inputs to GS. */
4918 for (i = 0; i < 8 + num_user_sgprs; i++)
4919 returns[num_returns++] = ctx->i32; /* SGPRs */
4920 for (i = 0; i < 5; i++)
4921 returns[num_returns++] = ctx->f32; /* VGPRs */
4922 }
4923 break;
4924
4925 case PIPE_SHADER_TESS_EVAL:
4926 declare_global_desc_pointers(ctx, &fninfo);
4927 declare_per_stage_desc_pointers(ctx, &fninfo, true);
4928 ctx->param_tcs_offchip_layout = add_arg(&fninfo, ARG_SGPR, ctx->i32);
4929 ctx->param_tes_offchip_addr = add_arg(&fninfo, ARG_SGPR, ctx->i32);
4930
4931 if (shader->key.as_es) {
4932 ctx->param_tcs_offchip_offset = add_arg(&fninfo, ARG_SGPR, ctx->i32);
4933 add_arg(&fninfo, ARG_SGPR, ctx->i32);
4934 ctx->param_es2gs_offset = add_arg(&fninfo, ARG_SGPR, ctx->i32);
4935 } else {
4936 add_arg(&fninfo, ARG_SGPR, ctx->i32);
4937 declare_streamout_params(ctx, &shader->selector->so,
4938 &fninfo);
4939 ctx->param_tcs_offchip_offset = add_arg(&fninfo, ARG_SGPR, ctx->i32);
4940 }
4941
4942 /* VGPRs */
4943 declare_tes_input_vgprs(ctx, &fninfo);
4944 break;
4945
4946 case PIPE_SHADER_GEOMETRY:
4947 declare_global_desc_pointers(ctx, &fninfo);
4948 declare_per_stage_desc_pointers(ctx, &fninfo, true);
4949 ctx->param_gs2vs_offset = add_arg(&fninfo, ARG_SGPR, ctx->i32);
4950 ctx->param_gs_wave_id = add_arg(&fninfo, ARG_SGPR, ctx->i32);
4951
4952 /* VGPRs */
4953 add_arg_assign(&fninfo, ARG_VGPR, ctx->i32, &ctx->gs_vtx_offset[0]);
4954 add_arg_assign(&fninfo, ARG_VGPR, ctx->i32, &ctx->gs_vtx_offset[1]);
4955 add_arg_assign(&fninfo, ARG_VGPR, ctx->i32, &ctx->abi.gs_prim_id);
4956 add_arg_assign(&fninfo, ARG_VGPR, ctx->i32, &ctx->gs_vtx_offset[2]);
4957 add_arg_assign(&fninfo, ARG_VGPR, ctx->i32, &ctx->gs_vtx_offset[3]);
4958 add_arg_assign(&fninfo, ARG_VGPR, ctx->i32, &ctx->gs_vtx_offset[4]);
4959 add_arg_assign(&fninfo, ARG_VGPR, ctx->i32, &ctx->gs_vtx_offset[5]);
4960 add_arg_assign(&fninfo, ARG_VGPR, ctx->i32, &ctx->abi.gs_invocation_id);
4961 break;
4962
4963 case PIPE_SHADER_FRAGMENT:
4964 declare_global_desc_pointers(ctx, &fninfo);
4965 declare_per_stage_desc_pointers(ctx, &fninfo, true);
4966 add_arg_checked(&fninfo, ARG_SGPR, ctx->f32, SI_PARAM_ALPHA_REF);
4967 add_arg_assign_checked(&fninfo, ARG_SGPR, ctx->i32,
4968 &ctx->abi.prim_mask, SI_PARAM_PRIM_MASK);
4969
4970 add_arg_checked(&fninfo, ARG_VGPR, ctx->v2i32, SI_PARAM_PERSP_SAMPLE);
4971 add_arg_checked(&fninfo, ARG_VGPR, ctx->v2i32, SI_PARAM_PERSP_CENTER);
4972 add_arg_checked(&fninfo, ARG_VGPR, ctx->v2i32, SI_PARAM_PERSP_CENTROID);
4973 add_arg_checked(&fninfo, ARG_VGPR, v3i32, SI_PARAM_PERSP_PULL_MODEL);
4974 add_arg_checked(&fninfo, ARG_VGPR, ctx->v2i32, SI_PARAM_LINEAR_SAMPLE);
4975 add_arg_checked(&fninfo, ARG_VGPR, ctx->v2i32, SI_PARAM_LINEAR_CENTER);
4976 add_arg_checked(&fninfo, ARG_VGPR, ctx->v2i32, SI_PARAM_LINEAR_CENTROID);
4977 add_arg_checked(&fninfo, ARG_VGPR, ctx->f32, SI_PARAM_LINE_STIPPLE_TEX);
4978 add_arg_assign_checked(&fninfo, ARG_VGPR, ctx->f32,
4979 &ctx->abi.frag_pos[0], SI_PARAM_POS_X_FLOAT);
4980 add_arg_assign_checked(&fninfo, ARG_VGPR, ctx->f32,
4981 &ctx->abi.frag_pos[1], SI_PARAM_POS_Y_FLOAT);
4982 add_arg_assign_checked(&fninfo, ARG_VGPR, ctx->f32,
4983 &ctx->abi.frag_pos[2], SI_PARAM_POS_Z_FLOAT);
4984 add_arg_assign_checked(&fninfo, ARG_VGPR, ctx->f32,
4985 &ctx->abi.frag_pos[3], SI_PARAM_POS_W_FLOAT);
4986 add_arg_assign_checked(&fninfo, ARG_VGPR, ctx->i32,
4987 &ctx->abi.front_face, SI_PARAM_FRONT_FACE);
4988 shader->info.face_vgpr_index = 20;
4989 add_arg_assign_checked(&fninfo, ARG_VGPR, ctx->i32,
4990 &ctx->abi.ancillary, SI_PARAM_ANCILLARY);
4991 shader->info.ancillary_vgpr_index = 21;
4992 add_arg_assign_checked(&fninfo, ARG_VGPR, ctx->f32,
4993 &ctx->abi.sample_coverage, SI_PARAM_SAMPLE_COVERAGE);
4994 add_arg_checked(&fninfo, ARG_VGPR, ctx->i32, SI_PARAM_POS_FIXED_PT);
4995
4996 /* Color inputs from the prolog. */
4997 if (shader->selector->info.colors_read) {
4998 unsigned num_color_elements =
4999 util_bitcount(shader->selector->info.colors_read);
5000
5001 assert(fninfo.num_params + num_color_elements <= ARRAY_SIZE(fninfo.types));
5002 for (i = 0; i < num_color_elements; i++)
5003 add_arg(&fninfo, ARG_VGPR, ctx->f32);
5004
5005 num_prolog_vgprs += num_color_elements;
5006 }
5007
5008 /* Outputs for the epilog. */
5009 num_return_sgprs = SI_SGPR_ALPHA_REF + 1;
5010 num_returns =
5011 num_return_sgprs +
5012 util_bitcount(shader->selector->info.colors_written) * 4 +
5013 shader->selector->info.writes_z +
5014 shader->selector->info.writes_stencil +
5015 shader->selector->info.writes_samplemask +
5016 1 /* SampleMaskIn */;
5017
5018 num_returns = MAX2(num_returns,
5019 num_return_sgprs +
5020 PS_EPILOG_SAMPLEMASK_MIN_LOC + 1);
5021
5022 for (i = 0; i < num_return_sgprs; i++)
5023 returns[i] = ctx->i32;
5024 for (; i < num_returns; i++)
5025 returns[i] = ctx->f32;
5026 break;
5027
5028 case PIPE_SHADER_COMPUTE:
5029 declare_global_desc_pointers(ctx, &fninfo);
5030 declare_per_stage_desc_pointers(ctx, &fninfo, true);
5031 if (shader->selector->info.uses_grid_size)
5032 add_arg_assign(&fninfo, ARG_SGPR, v3i32, &ctx->abi.num_work_groups);
5033 if (shader->selector->info.uses_block_size)
5034 ctx->param_block_size = add_arg(&fninfo, ARG_SGPR, v3i32);
5035
5036 for (i = 0; i < 3; i++) {
5037 ctx->abi.workgroup_ids[i] = NULL;
5038 if (shader->selector->info.uses_block_id[i])
5039 add_arg_assign(&fninfo, ARG_SGPR, ctx->i32, &ctx->abi.workgroup_ids[i]);
5040 }
5041
5042 add_arg_assign(&fninfo, ARG_VGPR, v3i32, &ctx->abi.local_invocation_ids);
5043 break;
5044 default:
5045 assert(0 && "unimplemented shader");
5046 return;
5047 }
5048
5049 si_create_function(ctx, "main", returns, num_returns, &fninfo,
5050 si_get_max_workgroup_size(shader));
5051
5052 /* Reserve register locations for VGPR inputs the PS prolog may need. */
5053 if (ctx->type == PIPE_SHADER_FRAGMENT &&
5054 ctx->separate_prolog) {
5055 ac_llvm_add_target_dep_function_attr(ctx->main_fn,
5056 "InitialPSInputAddr",
5057 S_0286D0_PERSP_SAMPLE_ENA(1) |
5058 S_0286D0_PERSP_CENTER_ENA(1) |
5059 S_0286D0_PERSP_CENTROID_ENA(1) |
5060 S_0286D0_LINEAR_SAMPLE_ENA(1) |
5061 S_0286D0_LINEAR_CENTER_ENA(1) |
5062 S_0286D0_LINEAR_CENTROID_ENA(1) |
5063 S_0286D0_FRONT_FACE_ENA(1) |
5064 S_0286D0_ANCILLARY_ENA(1) |
5065 S_0286D0_POS_FIXED_PT_ENA(1));
5066 }
5067
5068 shader->info.num_input_sgprs = 0;
5069 shader->info.num_input_vgprs = 0;
5070
5071 for (i = 0; i < fninfo.num_sgpr_params; ++i)
5072 shader->info.num_input_sgprs += ac_get_type_size(fninfo.types[i]) / 4;
5073
5074 for (; i < fninfo.num_params; ++i)
5075 shader->info.num_input_vgprs += ac_get_type_size(fninfo.types[i]) / 4;
5076
5077 assert(shader->info.num_input_vgprs >= num_prolog_vgprs);
5078 shader->info.num_input_vgprs -= num_prolog_vgprs;
5079
5080 if (shader->key.as_ls ||
5081 ctx->type == PIPE_SHADER_TESS_CTRL ||
5082 /* GFX9 has the ESGS ring buffer in LDS. */
5083 type == SI_SHADER_MERGED_VERTEX_OR_TESSEVAL_GEOMETRY)
5084 ac_declare_lds_as_pointer(&ctx->ac);
5085 }
5086
5087 /**
5088 * Load ESGS and GSVS ring buffer resource descriptors and save the variables
5089 * for later use.
5090 */
5091 static void preload_ring_buffers(struct si_shader_context *ctx)
5092 {
5093 LLVMBuilderRef builder = ctx->ac.builder;
5094
5095 LLVMValueRef buf_ptr = LLVMGetParam(ctx->main_fn,
5096 ctx->param_rw_buffers);
5097
5098 if (ctx->screen->info.chip_class <= VI &&
5099 (ctx->shader->key.as_es || ctx->type == PIPE_SHADER_GEOMETRY)) {
5100 unsigned ring =
5101 ctx->type == PIPE_SHADER_GEOMETRY ? SI_GS_RING_ESGS
5102 : SI_ES_RING_ESGS;
5103 LLVMValueRef offset = LLVMConstInt(ctx->i32, ring, 0);
5104
5105 ctx->esgs_ring =
5106 ac_build_load_to_sgpr(&ctx->ac, buf_ptr, offset);
5107 }
5108
5109 if (ctx->shader->is_gs_copy_shader) {
5110 LLVMValueRef offset = LLVMConstInt(ctx->i32, SI_RING_GSVS, 0);
5111
5112 ctx->gsvs_ring[0] =
5113 ac_build_load_to_sgpr(&ctx->ac, buf_ptr, offset);
5114 } else if (ctx->type == PIPE_SHADER_GEOMETRY) {
5115 const struct si_shader_selector *sel = ctx->shader->selector;
5116 LLVMValueRef offset = LLVMConstInt(ctx->i32, SI_RING_GSVS, 0);
5117 LLVMValueRef base_ring;
5118
5119 base_ring = ac_build_load_to_sgpr(&ctx->ac, buf_ptr, offset);
5120
5121 /* The conceptual layout of the GSVS ring is
5122 * v0c0 .. vLv0 v0c1 .. vLc1 ..
5123 * but the real memory layout is swizzled across
5124 * threads:
5125 * t0v0c0 .. t15v0c0 t0v1c0 .. t15v1c0 ... t15vLcL
5126 * t16v0c0 ..
5127 * Override the buffer descriptor accordingly.
5128 */
5129 LLVMTypeRef v2i64 = LLVMVectorType(ctx->i64, 2);
5130 uint64_t stream_offset = 0;
5131
5132 for (unsigned stream = 0; stream < 4; ++stream) {
5133 unsigned num_components;
5134 unsigned stride;
5135 unsigned num_records;
5136 LLVMValueRef ring, tmp;
5137
5138 num_components = sel->info.num_stream_output_components[stream];
5139 if (!num_components)
5140 continue;
5141
5142 stride = 4 * num_components * sel->gs_max_out_vertices;
5143
5144 /* Limit on the stride field for <= CIK. */
5145 assert(stride < (1 << 14));
5146
5147 num_records = 64;
5148
5149 ring = LLVMBuildBitCast(builder, base_ring, v2i64, "");
5150 tmp = LLVMBuildExtractElement(builder, ring, ctx->i32_0, "");
5151 tmp = LLVMBuildAdd(builder, tmp,
5152 LLVMConstInt(ctx->i64,
5153 stream_offset, 0), "");
5154 stream_offset += stride * 64;
5155
5156 ring = LLVMBuildInsertElement(builder, ring, tmp, ctx->i32_0, "");
5157 ring = LLVMBuildBitCast(builder, ring, ctx->v4i32, "");
5158 tmp = LLVMBuildExtractElement(builder, ring, ctx->i32_1, "");
5159 tmp = LLVMBuildOr(builder, tmp,
5160 LLVMConstInt(ctx->i32,
5161 S_008F04_STRIDE(stride) |
5162 S_008F04_SWIZZLE_ENABLE(1), 0), "");
5163 ring = LLVMBuildInsertElement(builder, ring, tmp, ctx->i32_1, "");
5164 ring = LLVMBuildInsertElement(builder, ring,
5165 LLVMConstInt(ctx->i32, num_records, 0),
5166 LLVMConstInt(ctx->i32, 2, 0), "");
5167 ring = LLVMBuildInsertElement(builder, ring,
5168 LLVMConstInt(ctx->i32,
5169 S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) |
5170 S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
5171 S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) |
5172 S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W) |
5173 S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
5174 S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32) |
5175 S_008F0C_ELEMENT_SIZE(1) | /* element_size = 4 (bytes) */
5176 S_008F0C_INDEX_STRIDE(1) | /* index_stride = 16 (elements) */
5177 S_008F0C_ADD_TID_ENABLE(1),
5178 0),
5179 LLVMConstInt(ctx->i32, 3, 0), "");
5180
5181 ctx->gsvs_ring[stream] = ring;
5182 }
5183 } else if (ctx->type == PIPE_SHADER_TESS_EVAL) {
5184 ctx->tess_offchip_ring = get_tess_ring_descriptor(ctx, TESS_OFFCHIP_RING_TES);
5185 }
5186 }
5187
5188 static void si_llvm_emit_polygon_stipple(struct si_shader_context *ctx,
5189 LLVMValueRef param_rw_buffers,
5190 unsigned param_pos_fixed_pt)
5191 {
5192 LLVMBuilderRef builder = ctx->ac.builder;
5193 LLVMValueRef slot, desc, offset, row, bit, address[2];
5194
5195 /* Use the fixed-point gl_FragCoord input.
5196 * Since the stipple pattern is 32x32 and it repeats, just get 5 bits
5197 * per coordinate to get the repeating effect.
5198 */
5199 address[0] = si_unpack_param(ctx, param_pos_fixed_pt, 0, 5);
5200 address[1] = si_unpack_param(ctx, param_pos_fixed_pt, 16, 5);
5201
5202 /* Load the buffer descriptor. */
5203 slot = LLVMConstInt(ctx->i32, SI_PS_CONST_POLY_STIPPLE, 0);
5204 desc = ac_build_load_to_sgpr(&ctx->ac, param_rw_buffers, slot);
5205
5206 /* The stipple pattern is 32x32, each row has 32 bits. */
5207 offset = LLVMBuildMul(builder, address[1],
5208 LLVMConstInt(ctx->i32, 4, 0), "");
5209 row = buffer_load_const(ctx, desc, offset);
5210 row = ac_to_integer(&ctx->ac, row);
5211 bit = LLVMBuildLShr(builder, row, address[0], "");
5212 bit = LLVMBuildTrunc(builder, bit, ctx->i1, "");
5213 ac_build_kill_if_false(&ctx->ac, bit);
5214 }
5215
5216 void si_shader_binary_read_config(struct ac_shader_binary *binary,
5217 struct si_shader_config *conf,
5218 unsigned symbol_offset)
5219 {
5220 unsigned i;
5221 const unsigned char *config =
5222 ac_shader_binary_config_start(binary, symbol_offset);
5223 bool really_needs_scratch = false;
5224
5225 /* LLVM adds SGPR spills to the scratch size.
5226 * Find out if we really need the scratch buffer.
5227 */
5228 for (i = 0; i < binary->reloc_count; i++) {
5229 const struct ac_shader_reloc *reloc = &binary->relocs[i];
5230
5231 if (!strcmp(scratch_rsrc_dword0_symbol, reloc->name) ||
5232 !strcmp(scratch_rsrc_dword1_symbol, reloc->name)) {
5233 really_needs_scratch = true;
5234 break;
5235 }
5236 }
5237
5238 /* XXX: We may be able to emit some of these values directly rather than
5239 * extracting fields to be emitted later.
5240 */
5241
5242 for (i = 0; i < binary->config_size_per_symbol; i+= 8) {
5243 unsigned reg = util_le32_to_cpu(*(uint32_t*)(config + i));
5244 unsigned value = util_le32_to_cpu(*(uint32_t*)(config + i + 4));
5245 switch (reg) {
5246 case R_00B028_SPI_SHADER_PGM_RSRC1_PS:
5247 case R_00B128_SPI_SHADER_PGM_RSRC1_VS:
5248 case R_00B228_SPI_SHADER_PGM_RSRC1_GS:
5249 case R_00B428_SPI_SHADER_PGM_RSRC1_HS:
5250 case R_00B848_COMPUTE_PGM_RSRC1:
5251 conf->num_sgprs = MAX2(conf->num_sgprs, (G_00B028_SGPRS(value) + 1) * 8);
5252 conf->num_vgprs = MAX2(conf->num_vgprs, (G_00B028_VGPRS(value) + 1) * 4);
5253 conf->float_mode = G_00B028_FLOAT_MODE(value);
5254 conf->rsrc1 = value;
5255 break;
5256 case R_00B02C_SPI_SHADER_PGM_RSRC2_PS:
5257 conf->lds_size = MAX2(conf->lds_size, G_00B02C_EXTRA_LDS_SIZE(value));
5258 break;
5259 case R_00B84C_COMPUTE_PGM_RSRC2:
5260 conf->lds_size = MAX2(conf->lds_size, G_00B84C_LDS_SIZE(value));
5261 conf->rsrc2 = value;
5262 break;
5263 case R_0286CC_SPI_PS_INPUT_ENA:
5264 conf->spi_ps_input_ena = value;
5265 break;
5266 case R_0286D0_SPI_PS_INPUT_ADDR:
5267 conf->spi_ps_input_addr = value;
5268 break;
5269 case R_0286E8_SPI_TMPRING_SIZE:
5270 case R_00B860_COMPUTE_TMPRING_SIZE:
5271 /* WAVESIZE is in units of 256 dwords. */
5272 if (really_needs_scratch)
5273 conf->scratch_bytes_per_wave =
5274 G_00B860_WAVESIZE(value) * 256 * 4;
5275 break;
5276 case 0x4: /* SPILLED_SGPRS */
5277 conf->spilled_sgprs = value;
5278 break;
5279 case 0x8: /* SPILLED_VGPRS */
5280 conf->spilled_vgprs = value;
5281 break;
5282 default:
5283 {
5284 static bool printed;
5285
5286 if (!printed) {
5287 fprintf(stderr, "Warning: LLVM emitted unknown "
5288 "config register: 0x%x\n", reg);
5289 printed = true;
5290 }
5291 }
5292 break;
5293 }
5294 }
5295
5296 if (!conf->spi_ps_input_addr)
5297 conf->spi_ps_input_addr = conf->spi_ps_input_ena;
5298 }
5299
5300 void si_shader_apply_scratch_relocs(struct si_shader *shader,
5301 uint64_t scratch_va)
5302 {
5303 unsigned i;
5304 uint32_t scratch_rsrc_dword0 = scratch_va;
5305 uint32_t scratch_rsrc_dword1 =
5306 S_008F04_BASE_ADDRESS_HI(scratch_va >> 32);
5307
5308 /* Enable scratch coalescing. */
5309 scratch_rsrc_dword1 |= S_008F04_SWIZZLE_ENABLE(1);
5310
5311 for (i = 0 ; i < shader->binary.reloc_count; i++) {
5312 const struct ac_shader_reloc *reloc =
5313 &shader->binary.relocs[i];
5314 if (!strcmp(scratch_rsrc_dword0_symbol, reloc->name)) {
5315 util_memcpy_cpu_to_le32(shader->binary.code + reloc->offset,
5316 &scratch_rsrc_dword0, 4);
5317 } else if (!strcmp(scratch_rsrc_dword1_symbol, reloc->name)) {
5318 util_memcpy_cpu_to_le32(shader->binary.code + reloc->offset,
5319 &scratch_rsrc_dword1, 4);
5320 }
5321 }
5322 }
5323
5324 /* For the UMR disassembler. */
5325 #define DEBUGGER_END_OF_CODE_MARKER 0xbf9f0000 /* invalid instruction */
5326 #define DEBUGGER_NUM_MARKERS 5
5327
5328 static unsigned si_get_shader_binary_size(const struct si_shader *shader)
5329 {
5330 unsigned size = shader->binary.code_size;
5331
5332 if (shader->prolog)
5333 size += shader->prolog->binary.code_size;
5334 if (shader->previous_stage)
5335 size += shader->previous_stage->binary.code_size;
5336 if (shader->prolog2)
5337 size += shader->prolog2->binary.code_size;
5338 if (shader->epilog)
5339 size += shader->epilog->binary.code_size;
5340 return size + DEBUGGER_NUM_MARKERS * 4;
5341 }
5342
5343 int si_shader_binary_upload(struct si_screen *sscreen, struct si_shader *shader)
5344 {
5345 const struct ac_shader_binary *prolog =
5346 shader->prolog ? &shader->prolog->binary : NULL;
5347 const struct ac_shader_binary *previous_stage =
5348 shader->previous_stage ? &shader->previous_stage->binary : NULL;
5349 const struct ac_shader_binary *prolog2 =
5350 shader->prolog2 ? &shader->prolog2->binary : NULL;
5351 const struct ac_shader_binary *epilog =
5352 shader->epilog ? &shader->epilog->binary : NULL;
5353 const struct ac_shader_binary *mainb = &shader->binary;
5354 unsigned bo_size = si_get_shader_binary_size(shader) +
5355 (!epilog ? mainb->rodata_size : 0);
5356 unsigned char *ptr;
5357
5358 assert(!prolog || !prolog->rodata_size);
5359 assert(!previous_stage || !previous_stage->rodata_size);
5360 assert(!prolog2 || !prolog2->rodata_size);
5361 assert((!prolog && !previous_stage && !prolog2 && !epilog) ||
5362 !mainb->rodata_size);
5363 assert(!epilog || !epilog->rodata_size);
5364
5365 r600_resource_reference(&shader->bo, NULL);
5366 shader->bo = si_aligned_buffer_create(&sscreen->b,
5367 sscreen->cpdma_prefetch_writes_memory ?
5368 0 : SI_RESOURCE_FLAG_READ_ONLY,
5369 PIPE_USAGE_IMMUTABLE,
5370 align(bo_size, SI_CPDMA_ALIGNMENT),
5371 256);
5372 if (!shader->bo)
5373 return -ENOMEM;
5374
5375 /* Upload. */
5376 ptr = sscreen->ws->buffer_map(shader->bo->buf, NULL,
5377 PIPE_TRANSFER_READ_WRITE |
5378 PIPE_TRANSFER_UNSYNCHRONIZED);
5379
5380 /* Don't use util_memcpy_cpu_to_le32. LLVM binaries are
5381 * endian-independent. */
5382 if (prolog) {
5383 memcpy(ptr, prolog->code, prolog->code_size);
5384 ptr += prolog->code_size;
5385 }
5386 if (previous_stage) {
5387 memcpy(ptr, previous_stage->code, previous_stage->code_size);
5388 ptr += previous_stage->code_size;
5389 }
5390 if (prolog2) {
5391 memcpy(ptr, prolog2->code, prolog2->code_size);
5392 ptr += prolog2->code_size;
5393 }
5394
5395 memcpy(ptr, mainb->code, mainb->code_size);
5396 ptr += mainb->code_size;
5397
5398 if (epilog) {
5399 memcpy(ptr, epilog->code, epilog->code_size);
5400 ptr += epilog->code_size;
5401 } else if (mainb->rodata_size > 0) {
5402 memcpy(ptr, mainb->rodata, mainb->rodata_size);
5403 ptr += mainb->rodata_size;
5404 }
5405
5406 /* Add end-of-code markers for the UMR disassembler. */
5407 uint32_t *ptr32 = (uint32_t*)ptr;
5408 for (unsigned i = 0; i < DEBUGGER_NUM_MARKERS; i++)
5409 ptr32[i] = DEBUGGER_END_OF_CODE_MARKER;
5410
5411 sscreen->ws->buffer_unmap(shader->bo->buf);
5412 return 0;
5413 }
5414
5415 static void si_shader_dump_disassembly(const struct ac_shader_binary *binary,
5416 struct pipe_debug_callback *debug,
5417 const char *name, FILE *file)
5418 {
5419 char *line, *p;
5420 unsigned i, count;
5421
5422 if (binary->disasm_string) {
5423 fprintf(file, "Shader %s disassembly:\n", name);
5424 fprintf(file, "%s", binary->disasm_string);
5425
5426 if (debug && debug->debug_message) {
5427 /* Very long debug messages are cut off, so send the
5428 * disassembly one line at a time. This causes more
5429 * overhead, but on the plus side it simplifies
5430 * parsing of resulting logs.
5431 */
5432 pipe_debug_message(debug, SHADER_INFO,
5433 "Shader Disassembly Begin");
5434
5435 line = binary->disasm_string;
5436 while (*line) {
5437 p = util_strchrnul(line, '\n');
5438 count = p - line;
5439
5440 if (count) {
5441 pipe_debug_message(debug, SHADER_INFO,
5442 "%.*s", count, line);
5443 }
5444
5445 if (!*p)
5446 break;
5447 line = p + 1;
5448 }
5449
5450 pipe_debug_message(debug, SHADER_INFO,
5451 "Shader Disassembly End");
5452 }
5453 } else {
5454 fprintf(file, "Shader %s binary:\n", name);
5455 for (i = 0; i < binary->code_size; i += 4) {
5456 fprintf(file, "@0x%x: %02x%02x%02x%02x\n", i,
5457 binary->code[i + 3], binary->code[i + 2],
5458 binary->code[i + 1], binary->code[i]);
5459 }
5460 }
5461 }
5462
5463 static void si_calculate_max_simd_waves(struct si_shader *shader)
5464 {
5465 struct si_screen *sscreen = shader->selector->screen;
5466 struct si_shader_config *conf = &shader->config;
5467 unsigned num_inputs = shader->selector->info.num_inputs;
5468 unsigned lds_increment = sscreen->info.chip_class >= CIK ? 512 : 256;
5469 unsigned lds_per_wave = 0;
5470 unsigned max_simd_waves;
5471
5472 max_simd_waves = ac_get_max_simd_waves(sscreen->info.family);
5473
5474 /* Compute LDS usage for PS. */
5475 switch (shader->selector->type) {
5476 case PIPE_SHADER_FRAGMENT:
5477 /* The minimum usage per wave is (num_inputs * 48). The maximum
5478 * usage is (num_inputs * 48 * 16).
5479 * We can get anything in between and it varies between waves.
5480 *
5481 * The 48 bytes per input for a single primitive is equal to
5482 * 4 bytes/component * 4 components/input * 3 points.
5483 *
5484 * Other stages don't know the size at compile time or don't
5485 * allocate LDS per wave, but instead they do it per thread group.
5486 */
5487 lds_per_wave = conf->lds_size * lds_increment +
5488 align(num_inputs * 48, lds_increment);
5489 break;
5490 case PIPE_SHADER_COMPUTE:
5491 if (shader->selector) {
5492 unsigned max_workgroup_size =
5493 si_get_max_workgroup_size(shader);
5494 lds_per_wave = (conf->lds_size * lds_increment) /
5495 DIV_ROUND_UP(max_workgroup_size, 64);
5496 }
5497 break;
5498 }
5499
5500 /* Compute the per-SIMD wave counts. */
5501 if (conf->num_sgprs) {
5502 if (sscreen->info.chip_class >= VI)
5503 max_simd_waves = MIN2(max_simd_waves, 800 / conf->num_sgprs);
5504 else
5505 max_simd_waves = MIN2(max_simd_waves, 512 / conf->num_sgprs);
5506 }
5507
5508 if (conf->num_vgprs)
5509 max_simd_waves = MIN2(max_simd_waves, 256 / conf->num_vgprs);
5510
5511 /* LDS is 64KB per CU (4 SIMDs), which is 16KB per SIMD (usage above
5512 * 16KB makes some SIMDs unoccupied). */
5513 if (lds_per_wave)
5514 max_simd_waves = MIN2(max_simd_waves, 16384 / lds_per_wave);
5515
5516 conf->max_simd_waves = max_simd_waves;
5517 }
5518
5519 void si_shader_dump_stats_for_shader_db(const struct si_shader *shader,
5520 struct pipe_debug_callback *debug)
5521 {
5522 const struct si_shader_config *conf = &shader->config;
5523
5524 pipe_debug_message(debug, SHADER_INFO,
5525 "Shader Stats: SGPRS: %d VGPRS: %d Code Size: %d "
5526 "LDS: %d Scratch: %d Max Waves: %d Spilled SGPRs: %d "
5527 "Spilled VGPRs: %d PrivMem VGPRs: %d",
5528 conf->num_sgprs, conf->num_vgprs,
5529 si_get_shader_binary_size(shader),
5530 conf->lds_size, conf->scratch_bytes_per_wave,
5531 conf->max_simd_waves, conf->spilled_sgprs,
5532 conf->spilled_vgprs, conf->private_mem_vgprs);
5533 }
5534
5535 static void si_shader_dump_stats(struct si_screen *sscreen,
5536 const struct si_shader *shader,
5537 unsigned processor,
5538 FILE *file,
5539 bool check_debug_option)
5540 {
5541 const struct si_shader_config *conf = &shader->config;
5542
5543 if (!check_debug_option ||
5544 si_can_dump_shader(sscreen, processor)) {
5545 if (processor == PIPE_SHADER_FRAGMENT) {
5546 fprintf(file, "*** SHADER CONFIG ***\n"
5547 "SPI_PS_INPUT_ADDR = 0x%04x\n"
5548 "SPI_PS_INPUT_ENA = 0x%04x\n",
5549 conf->spi_ps_input_addr, conf->spi_ps_input_ena);
5550 }
5551
5552 fprintf(file, "*** SHADER STATS ***\n"
5553 "SGPRS: %d\n"
5554 "VGPRS: %d\n"
5555 "Spilled SGPRs: %d\n"
5556 "Spilled VGPRs: %d\n"
5557 "Private memory VGPRs: %d\n"
5558 "Code Size: %d bytes\n"
5559 "LDS: %d blocks\n"
5560 "Scratch: %d bytes per wave\n"
5561 "Max Waves: %d\n"
5562 "********************\n\n\n",
5563 conf->num_sgprs, conf->num_vgprs,
5564 conf->spilled_sgprs, conf->spilled_vgprs,
5565 conf->private_mem_vgprs,
5566 si_get_shader_binary_size(shader),
5567 conf->lds_size, conf->scratch_bytes_per_wave,
5568 conf->max_simd_waves);
5569 }
5570 }
5571
5572 const char *si_get_shader_name(const struct si_shader *shader, unsigned processor)
5573 {
5574 switch (processor) {
5575 case PIPE_SHADER_VERTEX:
5576 if (shader->key.as_es)
5577 return "Vertex Shader as ES";
5578 else if (shader->key.as_ls)
5579 return "Vertex Shader as LS";
5580 else
5581 return "Vertex Shader as VS";
5582 case PIPE_SHADER_TESS_CTRL:
5583 return "Tessellation Control Shader";
5584 case PIPE_SHADER_TESS_EVAL:
5585 if (shader->key.as_es)
5586 return "Tessellation Evaluation Shader as ES";
5587 else
5588 return "Tessellation Evaluation Shader as VS";
5589 case PIPE_SHADER_GEOMETRY:
5590 if (shader->is_gs_copy_shader)
5591 return "GS Copy Shader as VS";
5592 else
5593 return "Geometry Shader";
5594 case PIPE_SHADER_FRAGMENT:
5595 return "Pixel Shader";
5596 case PIPE_SHADER_COMPUTE:
5597 return "Compute Shader";
5598 default:
5599 return "Unknown Shader";
5600 }
5601 }
5602
5603 void si_shader_dump(struct si_screen *sscreen, const struct si_shader *shader,
5604 struct pipe_debug_callback *debug, unsigned processor,
5605 FILE *file, bool check_debug_option)
5606 {
5607 if (!check_debug_option ||
5608 si_can_dump_shader(sscreen, processor))
5609 si_dump_shader_key(processor, shader, file);
5610
5611 if (!check_debug_option && shader->binary.llvm_ir_string) {
5612 if (shader->previous_stage &&
5613 shader->previous_stage->binary.llvm_ir_string) {
5614 fprintf(file, "\n%s - previous stage - LLVM IR:\n\n",
5615 si_get_shader_name(shader, processor));
5616 fprintf(file, "%s\n", shader->previous_stage->binary.llvm_ir_string);
5617 }
5618
5619 fprintf(file, "\n%s - main shader part - LLVM IR:\n\n",
5620 si_get_shader_name(shader, processor));
5621 fprintf(file, "%s\n", shader->binary.llvm_ir_string);
5622 }
5623
5624 if (!check_debug_option ||
5625 (si_can_dump_shader(sscreen, processor) &&
5626 !(sscreen->debug_flags & DBG(NO_ASM)))) {
5627 fprintf(file, "\n%s:\n", si_get_shader_name(shader, processor));
5628
5629 if (shader->prolog)
5630 si_shader_dump_disassembly(&shader->prolog->binary,
5631 debug, "prolog", file);
5632 if (shader->previous_stage)
5633 si_shader_dump_disassembly(&shader->previous_stage->binary,
5634 debug, "previous stage", file);
5635 if (shader->prolog2)
5636 si_shader_dump_disassembly(&shader->prolog2->binary,
5637 debug, "prolog2", file);
5638
5639 si_shader_dump_disassembly(&shader->binary, debug, "main", file);
5640
5641 if (shader->epilog)
5642 si_shader_dump_disassembly(&shader->epilog->binary,
5643 debug, "epilog", file);
5644 fprintf(file, "\n");
5645 }
5646
5647 si_shader_dump_stats(sscreen, shader, processor, file,
5648 check_debug_option);
5649 }
5650
5651 static int si_compile_llvm(struct si_screen *sscreen,
5652 struct ac_shader_binary *binary,
5653 struct si_shader_config *conf,
5654 struct si_compiler *compiler,
5655 LLVMModuleRef mod,
5656 struct pipe_debug_callback *debug,
5657 unsigned processor,
5658 const char *name)
5659 {
5660 int r = 0;
5661 unsigned count = p_atomic_inc_return(&sscreen->num_compilations);
5662
5663 if (si_can_dump_shader(sscreen, processor)) {
5664 fprintf(stderr, "radeonsi: Compiling shader %d\n", count);
5665
5666 if (!(sscreen->debug_flags & (DBG(NO_IR) | DBG(PREOPT_IR)))) {
5667 fprintf(stderr, "%s LLVM IR:\n\n", name);
5668 ac_dump_module(mod);
5669 fprintf(stderr, "\n");
5670 }
5671 }
5672
5673 if (sscreen->record_llvm_ir) {
5674 char *ir = LLVMPrintModuleToString(mod);
5675 binary->llvm_ir_string = strdup(ir);
5676 LLVMDisposeMessage(ir);
5677 }
5678
5679 if (!si_replace_shader(count, binary)) {
5680 r = si_llvm_compile(mod, binary, compiler, debug);
5681 if (r)
5682 return r;
5683 }
5684
5685 si_shader_binary_read_config(binary, conf, 0);
5686
5687 /* Enable 64-bit and 16-bit denormals, because there is no performance
5688 * cost.
5689 *
5690 * If denormals are enabled, all floating-point output modifiers are
5691 * ignored.
5692 *
5693 * Don't enable denormals for 32-bit floats, because:
5694 * - Floating-point output modifiers would be ignored by the hw.
5695 * - Some opcodes don't support denormals, such as v_mad_f32. We would
5696 * have to stop using those.
5697 * - SI & CI would be very slow.
5698 */
5699 conf->float_mode |= V_00B028_FP_64_DENORMS;
5700
5701 FREE(binary->config);
5702 FREE(binary->global_symbol_offsets);
5703 binary->config = NULL;
5704 binary->global_symbol_offsets = NULL;
5705
5706 /* Some shaders can't have rodata because their binaries can be
5707 * concatenated.
5708 */
5709 if (binary->rodata_size &&
5710 (processor == PIPE_SHADER_VERTEX ||
5711 processor == PIPE_SHADER_TESS_CTRL ||
5712 processor == PIPE_SHADER_TESS_EVAL ||
5713 processor == PIPE_SHADER_FRAGMENT)) {
5714 fprintf(stderr, "radeonsi: The shader can't have rodata.");
5715 return -EINVAL;
5716 }
5717
5718 return r;
5719 }
5720
5721 static void si_llvm_build_ret(struct si_shader_context *ctx, LLVMValueRef ret)
5722 {
5723 if (LLVMGetTypeKind(LLVMTypeOf(ret)) == LLVMVoidTypeKind)
5724 LLVMBuildRetVoid(ctx->ac.builder);
5725 else
5726 LLVMBuildRet(ctx->ac.builder, ret);
5727 }
5728
5729 /* Generate code for the hardware VS shader stage to go with a geometry shader */
5730 struct si_shader *
5731 si_generate_gs_copy_shader(struct si_screen *sscreen,
5732 struct si_compiler *compiler,
5733 struct si_shader_selector *gs_selector,
5734 struct pipe_debug_callback *debug)
5735 {
5736 struct si_shader_context ctx;
5737 struct si_shader *shader;
5738 LLVMBuilderRef builder;
5739 struct lp_build_tgsi_context *bld_base = &ctx.bld_base;
5740 struct lp_build_context *uint = &bld_base->uint_bld;
5741 struct si_shader_output_values *outputs;
5742 struct tgsi_shader_info *gsinfo = &gs_selector->info;
5743 int i, r;
5744
5745 outputs = MALLOC(gsinfo->num_outputs * sizeof(outputs[0]));
5746
5747 if (!outputs)
5748 return NULL;
5749
5750 shader = CALLOC_STRUCT(si_shader);
5751 if (!shader) {
5752 FREE(outputs);
5753 return NULL;
5754 }
5755
5756 /* We can leave the fence as permanently signaled because the GS copy
5757 * shader only becomes visible globally after it has been compiled. */
5758 util_queue_fence_init(&shader->ready);
5759
5760 shader->selector = gs_selector;
5761 shader->is_gs_copy_shader = true;
5762
5763 si_init_shader_ctx(&ctx, sscreen, compiler);
5764 ctx.shader = shader;
5765 ctx.type = PIPE_SHADER_VERTEX;
5766
5767 builder = ctx.ac.builder;
5768
5769 create_function(&ctx);
5770 preload_ring_buffers(&ctx);
5771
5772 LLVMValueRef voffset =
5773 lp_build_mul_imm(uint, ctx.abi.vertex_id, 4);
5774
5775 /* Fetch the vertex stream ID.*/
5776 LLVMValueRef stream_id;
5777
5778 if (gs_selector->so.num_outputs)
5779 stream_id = si_unpack_param(&ctx, ctx.param_streamout_config, 24, 2);
5780 else
5781 stream_id = ctx.i32_0;
5782
5783 /* Fill in output information. */
5784 for (i = 0; i < gsinfo->num_outputs; ++i) {
5785 outputs[i].semantic_name = gsinfo->output_semantic_name[i];
5786 outputs[i].semantic_index = gsinfo->output_semantic_index[i];
5787
5788 for (int chan = 0; chan < 4; chan++) {
5789 outputs[i].vertex_stream[chan] =
5790 (gsinfo->output_streams[i] >> (2 * chan)) & 3;
5791 }
5792 }
5793
5794 LLVMBasicBlockRef end_bb;
5795 LLVMValueRef switch_inst;
5796
5797 end_bb = LLVMAppendBasicBlockInContext(ctx.ac.context, ctx.main_fn, "end");
5798 switch_inst = LLVMBuildSwitch(builder, stream_id, end_bb, 4);
5799
5800 for (int stream = 0; stream < 4; stream++) {
5801 LLVMBasicBlockRef bb;
5802 unsigned offset;
5803
5804 if (!gsinfo->num_stream_output_components[stream])
5805 continue;
5806
5807 if (stream > 0 && !gs_selector->so.num_outputs)
5808 continue;
5809
5810 bb = LLVMInsertBasicBlockInContext(ctx.ac.context, end_bb, "out");
5811 LLVMAddCase(switch_inst, LLVMConstInt(ctx.i32, stream, 0), bb);
5812 LLVMPositionBuilderAtEnd(builder, bb);
5813
5814 /* Fetch vertex data from GSVS ring */
5815 offset = 0;
5816 for (i = 0; i < gsinfo->num_outputs; ++i) {
5817 for (unsigned chan = 0; chan < 4; chan++) {
5818 if (!(gsinfo->output_usagemask[i] & (1 << chan)) ||
5819 outputs[i].vertex_stream[chan] != stream) {
5820 outputs[i].values[chan] = ctx.bld_base.base.undef;
5821 continue;
5822 }
5823
5824 LLVMValueRef soffset = LLVMConstInt(ctx.i32,
5825 offset * gs_selector->gs_max_out_vertices * 16 * 4, 0);
5826 offset++;
5827
5828 outputs[i].values[chan] =
5829 ac_build_buffer_load(&ctx.ac,
5830 ctx.gsvs_ring[0], 1,
5831 ctx.i32_0, voffset,
5832 soffset, 0, 1, 1,
5833 true, false);
5834 }
5835 }
5836
5837 /* Streamout and exports. */
5838 if (gs_selector->so.num_outputs) {
5839 si_llvm_emit_streamout(&ctx, outputs,
5840 gsinfo->num_outputs,
5841 stream);
5842 }
5843
5844 if (stream == 0)
5845 si_llvm_export_vs(&ctx, outputs, gsinfo->num_outputs);
5846
5847 LLVMBuildBr(builder, end_bb);
5848 }
5849
5850 LLVMPositionBuilderAtEnd(builder, end_bb);
5851
5852 LLVMBuildRetVoid(ctx.ac.builder);
5853
5854 ctx.type = PIPE_SHADER_GEOMETRY; /* override for shader dumping */
5855 si_llvm_optimize_module(&ctx);
5856
5857 r = si_compile_llvm(sscreen, &ctx.shader->binary,
5858 &ctx.shader->config, ctx.compiler,
5859 ctx.gallivm.module,
5860 debug, PIPE_SHADER_GEOMETRY,
5861 "GS Copy Shader");
5862 if (!r) {
5863 if (si_can_dump_shader(sscreen, PIPE_SHADER_GEOMETRY))
5864 fprintf(stderr, "GS Copy Shader:\n");
5865 si_shader_dump(sscreen, ctx.shader, debug,
5866 PIPE_SHADER_GEOMETRY, stderr, true);
5867 r = si_shader_binary_upload(sscreen, ctx.shader);
5868 }
5869
5870 si_llvm_dispose(&ctx);
5871
5872 FREE(outputs);
5873
5874 if (r != 0) {
5875 FREE(shader);
5876 shader = NULL;
5877 }
5878 return shader;
5879 }
5880
5881 static void si_dump_shader_key_vs(const struct si_shader_key *key,
5882 const struct si_vs_prolog_bits *prolog,
5883 const char *prefix, FILE *f)
5884 {
5885 fprintf(f, " %s.instance_divisor_is_one = %u\n",
5886 prefix, prolog->instance_divisor_is_one);
5887 fprintf(f, " %s.instance_divisor_is_fetched = %u\n",
5888 prefix, prolog->instance_divisor_is_fetched);
5889 fprintf(f, " %s.ls_vgpr_fix = %u\n",
5890 prefix, prolog->ls_vgpr_fix);
5891
5892 fprintf(f, " mono.vs.fix_fetch = {");
5893 for (int i = 0; i < SI_MAX_ATTRIBS; i++)
5894 fprintf(f, !i ? "%u" : ", %u", key->mono.vs_fix_fetch[i]);
5895 fprintf(f, "}\n");
5896 }
5897
5898 static void si_dump_shader_key(unsigned processor, const struct si_shader *shader,
5899 FILE *f)
5900 {
5901 const struct si_shader_key *key = &shader->key;
5902
5903 fprintf(f, "SHADER KEY\n");
5904
5905 switch (processor) {
5906 case PIPE_SHADER_VERTEX:
5907 si_dump_shader_key_vs(key, &key->part.vs.prolog,
5908 "part.vs.prolog", f);
5909 fprintf(f, " as_es = %u\n", key->as_es);
5910 fprintf(f, " as_ls = %u\n", key->as_ls);
5911 fprintf(f, " mono.u.vs_export_prim_id = %u\n",
5912 key->mono.u.vs_export_prim_id);
5913 break;
5914
5915 case PIPE_SHADER_TESS_CTRL:
5916 if (shader->selector->screen->info.chip_class >= GFX9) {
5917 si_dump_shader_key_vs(key, &key->part.tcs.ls_prolog,
5918 "part.tcs.ls_prolog", f);
5919 }
5920 fprintf(f, " part.tcs.epilog.prim_mode = %u\n", key->part.tcs.epilog.prim_mode);
5921 fprintf(f, " mono.u.ff_tcs_inputs_to_copy = 0x%"PRIx64"\n", key->mono.u.ff_tcs_inputs_to_copy);
5922 break;
5923
5924 case PIPE_SHADER_TESS_EVAL:
5925 fprintf(f, " as_es = %u\n", key->as_es);
5926 fprintf(f, " mono.u.vs_export_prim_id = %u\n",
5927 key->mono.u.vs_export_prim_id);
5928 break;
5929
5930 case PIPE_SHADER_GEOMETRY:
5931 if (shader->is_gs_copy_shader)
5932 break;
5933
5934 if (shader->selector->screen->info.chip_class >= GFX9 &&
5935 key->part.gs.es->type == PIPE_SHADER_VERTEX) {
5936 si_dump_shader_key_vs(key, &key->part.gs.vs_prolog,
5937 "part.gs.vs_prolog", f);
5938 }
5939 fprintf(f, " part.gs.prolog.tri_strip_adj_fix = %u\n", key->part.gs.prolog.tri_strip_adj_fix);
5940 break;
5941
5942 case PIPE_SHADER_COMPUTE:
5943 break;
5944
5945 case PIPE_SHADER_FRAGMENT:
5946 fprintf(f, " part.ps.prolog.color_two_side = %u\n", key->part.ps.prolog.color_two_side);
5947 fprintf(f, " part.ps.prolog.flatshade_colors = %u\n", key->part.ps.prolog.flatshade_colors);
5948 fprintf(f, " part.ps.prolog.poly_stipple = %u\n", key->part.ps.prolog.poly_stipple);
5949 fprintf(f, " part.ps.prolog.force_persp_sample_interp = %u\n", key->part.ps.prolog.force_persp_sample_interp);
5950 fprintf(f, " part.ps.prolog.force_linear_sample_interp = %u\n", key->part.ps.prolog.force_linear_sample_interp);
5951 fprintf(f, " part.ps.prolog.force_persp_center_interp = %u\n", key->part.ps.prolog.force_persp_center_interp);
5952 fprintf(f, " part.ps.prolog.force_linear_center_interp = %u\n", key->part.ps.prolog.force_linear_center_interp);
5953 fprintf(f, " part.ps.prolog.bc_optimize_for_persp = %u\n", key->part.ps.prolog.bc_optimize_for_persp);
5954 fprintf(f, " part.ps.prolog.bc_optimize_for_linear = %u\n", key->part.ps.prolog.bc_optimize_for_linear);
5955 fprintf(f, " part.ps.epilog.spi_shader_col_format = 0x%x\n", key->part.ps.epilog.spi_shader_col_format);
5956 fprintf(f, " part.ps.epilog.color_is_int8 = 0x%X\n", key->part.ps.epilog.color_is_int8);
5957 fprintf(f, " part.ps.epilog.color_is_int10 = 0x%X\n", key->part.ps.epilog.color_is_int10);
5958 fprintf(f, " part.ps.epilog.last_cbuf = %u\n", key->part.ps.epilog.last_cbuf);
5959 fprintf(f, " part.ps.epilog.alpha_func = %u\n", key->part.ps.epilog.alpha_func);
5960 fprintf(f, " part.ps.epilog.alpha_to_one = %u\n", key->part.ps.epilog.alpha_to_one);
5961 fprintf(f, " part.ps.epilog.poly_line_smoothing = %u\n", key->part.ps.epilog.poly_line_smoothing);
5962 fprintf(f, " part.ps.epilog.clamp_color = %u\n", key->part.ps.epilog.clamp_color);
5963 break;
5964
5965 default:
5966 assert(0);
5967 }
5968
5969 if ((processor == PIPE_SHADER_GEOMETRY ||
5970 processor == PIPE_SHADER_TESS_EVAL ||
5971 processor == PIPE_SHADER_VERTEX) &&
5972 !key->as_es && !key->as_ls) {
5973 fprintf(f, " opt.kill_outputs = 0x%"PRIx64"\n", key->opt.kill_outputs);
5974 fprintf(f, " opt.clip_disable = %u\n", key->opt.clip_disable);
5975 }
5976 }
5977
5978 static void si_init_shader_ctx(struct si_shader_context *ctx,
5979 struct si_screen *sscreen,
5980 struct si_compiler *compiler)
5981 {
5982 struct lp_build_tgsi_context *bld_base;
5983
5984 si_llvm_context_init(ctx, sscreen, compiler);
5985
5986 bld_base = &ctx->bld_base;
5987 bld_base->emit_fetch_funcs[TGSI_FILE_CONSTANT] = fetch_constant;
5988
5989 bld_base->op_actions[TGSI_OPCODE_INTERP_CENTROID] = interp_action;
5990 bld_base->op_actions[TGSI_OPCODE_INTERP_SAMPLE] = interp_action;
5991 bld_base->op_actions[TGSI_OPCODE_INTERP_OFFSET] = interp_action;
5992
5993 bld_base->op_actions[TGSI_OPCODE_MEMBAR].emit = membar_emit;
5994
5995 bld_base->op_actions[TGSI_OPCODE_CLOCK].emit = clock_emit;
5996
5997 bld_base->op_actions[TGSI_OPCODE_DDX].emit = si_llvm_emit_ddxy;
5998 bld_base->op_actions[TGSI_OPCODE_DDY].emit = si_llvm_emit_ddxy;
5999 bld_base->op_actions[TGSI_OPCODE_DDX_FINE].emit = si_llvm_emit_ddxy;
6000 bld_base->op_actions[TGSI_OPCODE_DDY_FINE].emit = si_llvm_emit_ddxy;
6001
6002 bld_base->op_actions[TGSI_OPCODE_VOTE_ALL].emit = vote_all_emit;
6003 bld_base->op_actions[TGSI_OPCODE_VOTE_ANY].emit = vote_any_emit;
6004 bld_base->op_actions[TGSI_OPCODE_VOTE_EQ].emit = vote_eq_emit;
6005 bld_base->op_actions[TGSI_OPCODE_BALLOT].emit = ballot_emit;
6006 bld_base->op_actions[TGSI_OPCODE_READ_FIRST].intr_name = "llvm.amdgcn.readfirstlane";
6007 bld_base->op_actions[TGSI_OPCODE_READ_FIRST].emit = read_lane_emit;
6008 bld_base->op_actions[TGSI_OPCODE_READ_INVOC].intr_name = "llvm.amdgcn.readlane";
6009 bld_base->op_actions[TGSI_OPCODE_READ_INVOC].fetch_args = read_invoc_fetch_args;
6010 bld_base->op_actions[TGSI_OPCODE_READ_INVOC].emit = read_lane_emit;
6011
6012 bld_base->op_actions[TGSI_OPCODE_EMIT].emit = si_tgsi_emit_vertex;
6013 bld_base->op_actions[TGSI_OPCODE_ENDPRIM].emit = si_tgsi_emit_primitive;
6014 bld_base->op_actions[TGSI_OPCODE_BARRIER].emit = si_llvm_emit_barrier;
6015 }
6016
6017 static void si_optimize_vs_outputs(struct si_shader_context *ctx)
6018 {
6019 struct si_shader *shader = ctx->shader;
6020 struct tgsi_shader_info *info = &shader->selector->info;
6021
6022 if ((ctx->type != PIPE_SHADER_VERTEX &&
6023 ctx->type != PIPE_SHADER_TESS_EVAL) ||
6024 shader->key.as_ls ||
6025 shader->key.as_es)
6026 return;
6027
6028 ac_optimize_vs_outputs(&ctx->ac,
6029 ctx->main_fn,
6030 shader->info.vs_output_param_offset,
6031 info->num_outputs,
6032 &shader->info.nr_param_exports);
6033 }
6034
6035 static void si_init_exec_from_input(struct si_shader_context *ctx,
6036 unsigned param, unsigned bitoffset)
6037 {
6038 LLVMValueRef args[] = {
6039 LLVMGetParam(ctx->main_fn, param),
6040 LLVMConstInt(ctx->i32, bitoffset, 0),
6041 };
6042 lp_build_intrinsic(ctx->ac.builder,
6043 "llvm.amdgcn.init.exec.from.input",
6044 ctx->voidt, args, 2, LP_FUNC_ATTR_CONVERGENT);
6045 }
6046
6047 static bool si_vs_needs_prolog(const struct si_shader_selector *sel,
6048 const struct si_vs_prolog_bits *key)
6049 {
6050 /* VGPR initialization fixup for Vega10 and Raven is always done in the
6051 * VS prolog. */
6052 return sel->vs_needs_prolog || key->ls_vgpr_fix;
6053 }
6054
6055 static bool si_compile_tgsi_main(struct si_shader_context *ctx,
6056 bool is_monolithic)
6057 {
6058 struct si_shader *shader = ctx->shader;
6059 struct si_shader_selector *sel = shader->selector;
6060 struct lp_build_tgsi_context *bld_base = &ctx->bld_base;
6061
6062 // TODO clean all this up!
6063 switch (ctx->type) {
6064 case PIPE_SHADER_VERTEX:
6065 ctx->load_input = declare_input_vs;
6066 if (shader->key.as_ls)
6067 ctx->abi.emit_outputs = si_llvm_emit_ls_epilogue;
6068 else if (shader->key.as_es)
6069 ctx->abi.emit_outputs = si_llvm_emit_es_epilogue;
6070 else
6071 ctx->abi.emit_outputs = si_llvm_emit_vs_epilogue;
6072 bld_base->emit_epilogue = si_tgsi_emit_epilogue;
6073 ctx->abi.load_base_vertex = get_base_vertex;
6074 break;
6075 case PIPE_SHADER_TESS_CTRL:
6076 bld_base->emit_fetch_funcs[TGSI_FILE_INPUT] = fetch_input_tcs;
6077 ctx->abi.load_tess_varyings = si_nir_load_tcs_varyings;
6078 bld_base->emit_fetch_funcs[TGSI_FILE_OUTPUT] = fetch_output_tcs;
6079 bld_base->emit_store = store_output_tcs;
6080 ctx->abi.store_tcs_outputs = si_nir_store_output_tcs;
6081 ctx->abi.emit_outputs = si_llvm_emit_tcs_epilogue;
6082 ctx->abi.load_patch_vertices_in = si_load_patch_vertices_in;
6083 bld_base->emit_epilogue = si_tgsi_emit_epilogue;
6084 break;
6085 case PIPE_SHADER_TESS_EVAL:
6086 bld_base->emit_fetch_funcs[TGSI_FILE_INPUT] = fetch_input_tes;
6087 ctx->abi.load_tess_varyings = si_nir_load_input_tes;
6088 ctx->abi.load_tess_coord = si_load_tess_coord;
6089 ctx->abi.load_tess_level = si_load_tess_level;
6090 ctx->abi.load_patch_vertices_in = si_load_patch_vertices_in;
6091 if (shader->key.as_es)
6092 ctx->abi.emit_outputs = si_llvm_emit_es_epilogue;
6093 else
6094 ctx->abi.emit_outputs = si_llvm_emit_vs_epilogue;
6095 bld_base->emit_epilogue = si_tgsi_emit_epilogue;
6096 break;
6097 case PIPE_SHADER_GEOMETRY:
6098 bld_base->emit_fetch_funcs[TGSI_FILE_INPUT] = fetch_input_gs;
6099 ctx->abi.load_inputs = si_nir_load_input_gs;
6100 ctx->abi.emit_vertex = si_llvm_emit_vertex;
6101 ctx->abi.emit_primitive = si_llvm_emit_primitive;
6102 ctx->abi.emit_outputs = si_llvm_emit_gs_epilogue;
6103 bld_base->emit_epilogue = si_tgsi_emit_gs_epilogue;
6104 break;
6105 case PIPE_SHADER_FRAGMENT:
6106 ctx->load_input = declare_input_fs;
6107 ctx->abi.emit_outputs = si_llvm_return_fs_outputs;
6108 bld_base->emit_epilogue = si_tgsi_emit_epilogue;
6109 ctx->abi.lookup_interp_param = si_nir_lookup_interp_param;
6110 ctx->abi.load_sample_position = load_sample_position;
6111 ctx->abi.load_sample_mask_in = load_sample_mask_in;
6112 ctx->abi.emit_kill = si_llvm_emit_kill;
6113 break;
6114 case PIPE_SHADER_COMPUTE:
6115 ctx->abi.load_local_group_size = get_block_size;
6116 break;
6117 default:
6118 assert(!"Unsupported shader type");
6119 return false;
6120 }
6121
6122 ctx->abi.load_ubo = load_ubo;
6123 ctx->abi.load_ssbo = load_ssbo;
6124
6125 create_function(ctx);
6126 preload_ring_buffers(ctx);
6127
6128 /* For GFX9 merged shaders:
6129 * - Set EXEC for the first shader. If the prolog is present, set
6130 * EXEC there instead.
6131 * - Add a barrier before the second shader.
6132 * - In the second shader, reset EXEC to ~0 and wrap the main part in
6133 * an if-statement. This is required for correctness in geometry
6134 * shaders, to ensure that empty GS waves do not send GS_EMIT and
6135 * GS_CUT messages.
6136 *
6137 * For monolithic merged shaders, the first shader is wrapped in an
6138 * if-block together with its prolog in si_build_wrapper_function.
6139 */
6140 if (ctx->screen->info.chip_class >= GFX9) {
6141 if (!is_monolithic &&
6142 sel->info.num_instructions > 1 && /* not empty shader */
6143 (shader->key.as_es || shader->key.as_ls) &&
6144 (ctx->type == PIPE_SHADER_TESS_EVAL ||
6145 (ctx->type == PIPE_SHADER_VERTEX &&
6146 !si_vs_needs_prolog(sel, &shader->key.part.vs.prolog)))) {
6147 si_init_exec_from_input(ctx,
6148 ctx->param_merged_wave_info, 0);
6149 } else if (ctx->type == PIPE_SHADER_TESS_CTRL ||
6150 ctx->type == PIPE_SHADER_GEOMETRY) {
6151 if (!is_monolithic)
6152 ac_init_exec_full_mask(&ctx->ac);
6153
6154 /* The barrier must execute for all shaders in a
6155 * threadgroup.
6156 */
6157 si_llvm_emit_barrier(NULL, bld_base, NULL);
6158
6159 LLVMValueRef num_threads = si_unpack_param(ctx, ctx->param_merged_wave_info, 8, 8);
6160 LLVMValueRef ena =
6161 LLVMBuildICmp(ctx->ac.builder, LLVMIntULT,
6162 ac_get_thread_id(&ctx->ac), num_threads, "");
6163 lp_build_if(&ctx->merged_wrap_if_state, &ctx->gallivm, ena);
6164 }
6165 }
6166
6167 if (ctx->type == PIPE_SHADER_TESS_CTRL &&
6168 sel->tcs_info.tessfactors_are_def_in_all_invocs) {
6169 for (unsigned i = 0; i < 6; i++) {
6170 ctx->invoc0_tess_factors[i] =
6171 lp_build_alloca_undef(&ctx->gallivm, ctx->i32, "");
6172 }
6173 }
6174
6175 if (ctx->type == PIPE_SHADER_GEOMETRY) {
6176 int i;
6177 for (i = 0; i < 4; i++) {
6178 ctx->gs_next_vertex[i] =
6179 lp_build_alloca(&ctx->gallivm,
6180 ctx->i32, "");
6181 }
6182 }
6183
6184 if (sel->force_correct_derivs_after_kill) {
6185 ctx->postponed_kill = lp_build_alloca_undef(&ctx->gallivm, ctx->i1, "");
6186 /* true = don't kill. */
6187 LLVMBuildStore(ctx->ac.builder, LLVMConstInt(ctx->i1, 1, 0),
6188 ctx->postponed_kill);
6189 }
6190
6191 if (sel->tokens) {
6192 if (!lp_build_tgsi_llvm(bld_base, sel->tokens)) {
6193 fprintf(stderr, "Failed to translate shader from TGSI to LLVM\n");
6194 return false;
6195 }
6196 } else {
6197 if (!si_nir_build_llvm(ctx, sel->nir)) {
6198 fprintf(stderr, "Failed to translate shader from NIR to LLVM\n");
6199 return false;
6200 }
6201 }
6202
6203 si_llvm_build_ret(ctx, ctx->return_value);
6204 return true;
6205 }
6206
6207 /**
6208 * Compute the VS prolog key, which contains all the information needed to
6209 * build the VS prolog function, and set shader->info bits where needed.
6210 *
6211 * \param info Shader info of the vertex shader.
6212 * \param num_input_sgprs Number of input SGPRs for the vertex shader.
6213 * \param prolog_key Key of the VS prolog
6214 * \param shader_out The vertex shader, or the next shader if merging LS+HS or ES+GS.
6215 * \param key Output shader part key.
6216 */
6217 static void si_get_vs_prolog_key(const struct tgsi_shader_info *info,
6218 unsigned num_input_sgprs,
6219 const struct si_vs_prolog_bits *prolog_key,
6220 struct si_shader *shader_out,
6221 union si_shader_part_key *key)
6222 {
6223 memset(key, 0, sizeof(*key));
6224 key->vs_prolog.states = *prolog_key;
6225 key->vs_prolog.num_input_sgprs = num_input_sgprs;
6226 key->vs_prolog.last_input = MAX2(1, info->num_inputs) - 1;
6227 key->vs_prolog.as_ls = shader_out->key.as_ls;
6228 key->vs_prolog.as_es = shader_out->key.as_es;
6229
6230 if (shader_out->selector->type == PIPE_SHADER_TESS_CTRL) {
6231 key->vs_prolog.as_ls = 1;
6232 key->vs_prolog.num_merged_next_stage_vgprs = 2;
6233 } else if (shader_out->selector->type == PIPE_SHADER_GEOMETRY) {
6234 key->vs_prolog.as_es = 1;
6235 key->vs_prolog.num_merged_next_stage_vgprs = 5;
6236 }
6237
6238 /* Enable loading the InstanceID VGPR. */
6239 uint16_t input_mask = u_bit_consecutive(0, info->num_inputs);
6240
6241 if ((key->vs_prolog.states.instance_divisor_is_one |
6242 key->vs_prolog.states.instance_divisor_is_fetched) & input_mask)
6243 shader_out->info.uses_instanceid = true;
6244 }
6245
6246 /**
6247 * Compute the PS prolog key, which contains all the information needed to
6248 * build the PS prolog function, and set related bits in shader->config.
6249 */
6250 static void si_get_ps_prolog_key(struct si_shader *shader,
6251 union si_shader_part_key *key,
6252 bool separate_prolog)
6253 {
6254 struct tgsi_shader_info *info = &shader->selector->info;
6255
6256 memset(key, 0, sizeof(*key));
6257 key->ps_prolog.states = shader->key.part.ps.prolog;
6258 key->ps_prolog.colors_read = info->colors_read;
6259 key->ps_prolog.num_input_sgprs = shader->info.num_input_sgprs;
6260 key->ps_prolog.num_input_vgprs = shader->info.num_input_vgprs;
6261 key->ps_prolog.wqm = info->uses_derivatives &&
6262 (key->ps_prolog.colors_read ||
6263 key->ps_prolog.states.force_persp_sample_interp ||
6264 key->ps_prolog.states.force_linear_sample_interp ||
6265 key->ps_prolog.states.force_persp_center_interp ||
6266 key->ps_prolog.states.force_linear_center_interp ||
6267 key->ps_prolog.states.bc_optimize_for_persp ||
6268 key->ps_prolog.states.bc_optimize_for_linear);
6269 key->ps_prolog.ancillary_vgpr_index = shader->info.ancillary_vgpr_index;
6270
6271 if (info->colors_read) {
6272 unsigned *color = shader->selector->color_attr_index;
6273
6274 if (shader->key.part.ps.prolog.color_two_side) {
6275 /* BCOLORs are stored after the last input. */
6276 key->ps_prolog.num_interp_inputs = info->num_inputs;
6277 key->ps_prolog.face_vgpr_index = shader->info.face_vgpr_index;
6278 shader->config.spi_ps_input_ena |= S_0286CC_FRONT_FACE_ENA(1);
6279 }
6280
6281 for (unsigned i = 0; i < 2; i++) {
6282 unsigned interp = info->input_interpolate[color[i]];
6283 unsigned location = info->input_interpolate_loc[color[i]];
6284
6285 if (!(info->colors_read & (0xf << i*4)))
6286 continue;
6287
6288 key->ps_prolog.color_attr_index[i] = color[i];
6289
6290 if (shader->key.part.ps.prolog.flatshade_colors &&
6291 interp == TGSI_INTERPOLATE_COLOR)
6292 interp = TGSI_INTERPOLATE_CONSTANT;
6293
6294 switch (interp) {
6295 case TGSI_INTERPOLATE_CONSTANT:
6296 key->ps_prolog.color_interp_vgpr_index[i] = -1;
6297 break;
6298 case TGSI_INTERPOLATE_PERSPECTIVE:
6299 case TGSI_INTERPOLATE_COLOR:
6300 /* Force the interpolation location for colors here. */
6301 if (shader->key.part.ps.prolog.force_persp_sample_interp)
6302 location = TGSI_INTERPOLATE_LOC_SAMPLE;
6303 if (shader->key.part.ps.prolog.force_persp_center_interp)
6304 location = TGSI_INTERPOLATE_LOC_CENTER;
6305
6306 switch (location) {
6307 case TGSI_INTERPOLATE_LOC_SAMPLE:
6308 key->ps_prolog.color_interp_vgpr_index[i] = 0;
6309 shader->config.spi_ps_input_ena |=
6310 S_0286CC_PERSP_SAMPLE_ENA(1);
6311 break;
6312 case TGSI_INTERPOLATE_LOC_CENTER:
6313 key->ps_prolog.color_interp_vgpr_index[i] = 2;
6314 shader->config.spi_ps_input_ena |=
6315 S_0286CC_PERSP_CENTER_ENA(1);
6316 break;
6317 case TGSI_INTERPOLATE_LOC_CENTROID:
6318 key->ps_prolog.color_interp_vgpr_index[i] = 4;
6319 shader->config.spi_ps_input_ena |=
6320 S_0286CC_PERSP_CENTROID_ENA(1);
6321 break;
6322 default:
6323 assert(0);
6324 }
6325 break;
6326 case TGSI_INTERPOLATE_LINEAR:
6327 /* Force the interpolation location for colors here. */
6328 if (shader->key.part.ps.prolog.force_linear_sample_interp)
6329 location = TGSI_INTERPOLATE_LOC_SAMPLE;
6330 if (shader->key.part.ps.prolog.force_linear_center_interp)
6331 location = TGSI_INTERPOLATE_LOC_CENTER;
6332
6333 /* The VGPR assignment for non-monolithic shaders
6334 * works because InitialPSInputAddr is set on the
6335 * main shader and PERSP_PULL_MODEL is never used.
6336 */
6337 switch (location) {
6338 case TGSI_INTERPOLATE_LOC_SAMPLE:
6339 key->ps_prolog.color_interp_vgpr_index[i] =
6340 separate_prolog ? 6 : 9;
6341 shader->config.spi_ps_input_ena |=
6342 S_0286CC_LINEAR_SAMPLE_ENA(1);
6343 break;
6344 case TGSI_INTERPOLATE_LOC_CENTER:
6345 key->ps_prolog.color_interp_vgpr_index[i] =
6346 separate_prolog ? 8 : 11;
6347 shader->config.spi_ps_input_ena |=
6348 S_0286CC_LINEAR_CENTER_ENA(1);
6349 break;
6350 case TGSI_INTERPOLATE_LOC_CENTROID:
6351 key->ps_prolog.color_interp_vgpr_index[i] =
6352 separate_prolog ? 10 : 13;
6353 shader->config.spi_ps_input_ena |=
6354 S_0286CC_LINEAR_CENTROID_ENA(1);
6355 break;
6356 default:
6357 assert(0);
6358 }
6359 break;
6360 default:
6361 assert(0);
6362 }
6363 }
6364 }
6365 }
6366
6367 /**
6368 * Check whether a PS prolog is required based on the key.
6369 */
6370 static bool si_need_ps_prolog(const union si_shader_part_key *key)
6371 {
6372 return key->ps_prolog.colors_read ||
6373 key->ps_prolog.states.force_persp_sample_interp ||
6374 key->ps_prolog.states.force_linear_sample_interp ||
6375 key->ps_prolog.states.force_persp_center_interp ||
6376 key->ps_prolog.states.force_linear_center_interp ||
6377 key->ps_prolog.states.bc_optimize_for_persp ||
6378 key->ps_prolog.states.bc_optimize_for_linear ||
6379 key->ps_prolog.states.poly_stipple ||
6380 key->ps_prolog.states.samplemask_log_ps_iter;
6381 }
6382
6383 /**
6384 * Compute the PS epilog key, which contains all the information needed to
6385 * build the PS epilog function.
6386 */
6387 static void si_get_ps_epilog_key(struct si_shader *shader,
6388 union si_shader_part_key *key)
6389 {
6390 struct tgsi_shader_info *info = &shader->selector->info;
6391 memset(key, 0, sizeof(*key));
6392 key->ps_epilog.colors_written = info->colors_written;
6393 key->ps_epilog.writes_z = info->writes_z;
6394 key->ps_epilog.writes_stencil = info->writes_stencil;
6395 key->ps_epilog.writes_samplemask = info->writes_samplemask;
6396 key->ps_epilog.states = shader->key.part.ps.epilog;
6397 }
6398
6399 /**
6400 * Build the GS prolog function. Rotate the input vertices for triangle strips
6401 * with adjacency.
6402 */
6403 static void si_build_gs_prolog_function(struct si_shader_context *ctx,
6404 union si_shader_part_key *key)
6405 {
6406 unsigned num_sgprs, num_vgprs;
6407 struct si_function_info fninfo;
6408 LLVMBuilderRef builder = ctx->ac.builder;
6409 LLVMTypeRef returns[48];
6410 LLVMValueRef func, ret;
6411
6412 si_init_function_info(&fninfo);
6413
6414 if (ctx->screen->info.chip_class >= GFX9) {
6415 if (key->gs_prolog.states.gfx9_prev_is_vs)
6416 num_sgprs = 8 + GFX9_VSGS_NUM_USER_SGPR;
6417 else
6418 num_sgprs = 8 + GFX9_TESGS_NUM_USER_SGPR;
6419 num_vgprs = 5; /* ES inputs are not needed by GS */
6420 } else {
6421 num_sgprs = GFX6_GS_NUM_USER_SGPR + 2;
6422 num_vgprs = 8;
6423 }
6424
6425 for (unsigned i = 0; i < num_sgprs; ++i) {
6426 add_arg(&fninfo, ARG_SGPR, ctx->i32);
6427 returns[i] = ctx->i32;
6428 }
6429
6430 for (unsigned i = 0; i < num_vgprs; ++i) {
6431 add_arg(&fninfo, ARG_VGPR, ctx->i32);
6432 returns[num_sgprs + i] = ctx->f32;
6433 }
6434
6435 /* Create the function. */
6436 si_create_function(ctx, "gs_prolog", returns, num_sgprs + num_vgprs,
6437 &fninfo, 0);
6438 func = ctx->main_fn;
6439
6440 /* Set the full EXEC mask for the prolog, because we are only fiddling
6441 * with registers here. The main shader part will set the correct EXEC
6442 * mask.
6443 */
6444 if (ctx->screen->info.chip_class >= GFX9 && !key->gs_prolog.is_monolithic)
6445 ac_init_exec_full_mask(&ctx->ac);
6446
6447 /* Copy inputs to outputs. This should be no-op, as the registers match,
6448 * but it will prevent the compiler from overwriting them unintentionally.
6449 */
6450 ret = ctx->return_value;
6451 for (unsigned i = 0; i < num_sgprs; i++) {
6452 LLVMValueRef p = LLVMGetParam(func, i);
6453 ret = LLVMBuildInsertValue(builder, ret, p, i, "");
6454 }
6455 for (unsigned i = 0; i < num_vgprs; i++) {
6456 LLVMValueRef p = LLVMGetParam(func, num_sgprs + i);
6457 p = ac_to_float(&ctx->ac, p);
6458 ret = LLVMBuildInsertValue(builder, ret, p, num_sgprs + i, "");
6459 }
6460
6461 if (key->gs_prolog.states.tri_strip_adj_fix) {
6462 /* Remap the input vertices for every other primitive. */
6463 const unsigned gfx6_vtx_params[6] = {
6464 num_sgprs,
6465 num_sgprs + 1,
6466 num_sgprs + 3,
6467 num_sgprs + 4,
6468 num_sgprs + 5,
6469 num_sgprs + 6
6470 };
6471 const unsigned gfx9_vtx_params[3] = {
6472 num_sgprs,
6473 num_sgprs + 1,
6474 num_sgprs + 4,
6475 };
6476 LLVMValueRef vtx_in[6], vtx_out[6];
6477 LLVMValueRef prim_id, rotate;
6478
6479 if (ctx->screen->info.chip_class >= GFX9) {
6480 for (unsigned i = 0; i < 3; i++) {
6481 vtx_in[i*2] = si_unpack_param(ctx, gfx9_vtx_params[i], 0, 16);
6482 vtx_in[i*2+1] = si_unpack_param(ctx, gfx9_vtx_params[i], 16, 16);
6483 }
6484 } else {
6485 for (unsigned i = 0; i < 6; i++)
6486 vtx_in[i] = LLVMGetParam(func, gfx6_vtx_params[i]);
6487 }
6488
6489 prim_id = LLVMGetParam(func, num_sgprs + 2);
6490 rotate = LLVMBuildTrunc(builder, prim_id, ctx->i1, "");
6491
6492 for (unsigned i = 0; i < 6; ++i) {
6493 LLVMValueRef base, rotated;
6494 base = vtx_in[i];
6495 rotated = vtx_in[(i + 4) % 6];
6496 vtx_out[i] = LLVMBuildSelect(builder, rotate, rotated, base, "");
6497 }
6498
6499 if (ctx->screen->info.chip_class >= GFX9) {
6500 for (unsigned i = 0; i < 3; i++) {
6501 LLVMValueRef hi, out;
6502
6503 hi = LLVMBuildShl(builder, vtx_out[i*2+1],
6504 LLVMConstInt(ctx->i32, 16, 0), "");
6505 out = LLVMBuildOr(builder, vtx_out[i*2], hi, "");
6506 out = ac_to_float(&ctx->ac, out);
6507 ret = LLVMBuildInsertValue(builder, ret, out,
6508 gfx9_vtx_params[i], "");
6509 }
6510 } else {
6511 for (unsigned i = 0; i < 6; i++) {
6512 LLVMValueRef out;
6513
6514 out = ac_to_float(&ctx->ac, vtx_out[i]);
6515 ret = LLVMBuildInsertValue(builder, ret, out,
6516 gfx6_vtx_params[i], "");
6517 }
6518 }
6519 }
6520
6521 LLVMBuildRet(builder, ret);
6522 }
6523
6524 /**
6525 * Given a list of shader part functions, build a wrapper function that
6526 * runs them in sequence to form a monolithic shader.
6527 */
6528 static void si_build_wrapper_function(struct si_shader_context *ctx,
6529 LLVMValueRef *parts,
6530 unsigned num_parts,
6531 unsigned main_part,
6532 unsigned next_shader_first_part)
6533 {
6534 LLVMBuilderRef builder = ctx->ac.builder;
6535 /* PS epilog has one arg per color component; gfx9 merged shader
6536 * prologs need to forward 32 user SGPRs.
6537 */
6538 struct si_function_info fninfo;
6539 LLVMValueRef initial[64], out[64];
6540 LLVMTypeRef function_type;
6541 unsigned num_first_params;
6542 unsigned num_out, initial_num_out;
6543 MAYBE_UNUSED unsigned num_out_sgpr; /* used in debug checks */
6544 MAYBE_UNUSED unsigned initial_num_out_sgpr; /* used in debug checks */
6545 unsigned num_sgprs, num_vgprs;
6546 unsigned gprs;
6547 struct lp_build_if_state if_state;
6548
6549 si_init_function_info(&fninfo);
6550
6551 for (unsigned i = 0; i < num_parts; ++i) {
6552 lp_add_function_attr(parts[i], -1, LP_FUNC_ATTR_ALWAYSINLINE);
6553 LLVMSetLinkage(parts[i], LLVMPrivateLinkage);
6554 }
6555
6556 /* The parameters of the wrapper function correspond to those of the
6557 * first part in terms of SGPRs and VGPRs, but we use the types of the
6558 * main part to get the right types. This is relevant for the
6559 * dereferenceable attribute on descriptor table pointers.
6560 */
6561 num_sgprs = 0;
6562 num_vgprs = 0;
6563
6564 function_type = LLVMGetElementType(LLVMTypeOf(parts[0]));
6565 num_first_params = LLVMCountParamTypes(function_type);
6566
6567 for (unsigned i = 0; i < num_first_params; ++i) {
6568 LLVMValueRef param = LLVMGetParam(parts[0], i);
6569
6570 if (ac_is_sgpr_param(param)) {
6571 assert(num_vgprs == 0);
6572 num_sgprs += ac_get_type_size(LLVMTypeOf(param)) / 4;
6573 } else {
6574 num_vgprs += ac_get_type_size(LLVMTypeOf(param)) / 4;
6575 }
6576 }
6577
6578 gprs = 0;
6579 while (gprs < num_sgprs + num_vgprs) {
6580 LLVMValueRef param = LLVMGetParam(parts[main_part], fninfo.num_params);
6581 LLVMTypeRef type = LLVMTypeOf(param);
6582 unsigned size = ac_get_type_size(type) / 4;
6583
6584 add_arg(&fninfo, gprs < num_sgprs ? ARG_SGPR : ARG_VGPR, type);
6585
6586 assert(ac_is_sgpr_param(param) == (gprs < num_sgprs));
6587 assert(gprs + size <= num_sgprs + num_vgprs &&
6588 (gprs >= num_sgprs || gprs + size <= num_sgprs));
6589
6590 gprs += size;
6591 }
6592
6593 si_create_function(ctx, "wrapper", NULL, 0, &fninfo,
6594 si_get_max_workgroup_size(ctx->shader));
6595
6596 if (is_merged_shader(ctx->shader))
6597 ac_init_exec_full_mask(&ctx->ac);
6598
6599 /* Record the arguments of the function as if they were an output of
6600 * a previous part.
6601 */
6602 num_out = 0;
6603 num_out_sgpr = 0;
6604
6605 for (unsigned i = 0; i < fninfo.num_params; ++i) {
6606 LLVMValueRef param = LLVMGetParam(ctx->main_fn, i);
6607 LLVMTypeRef param_type = LLVMTypeOf(param);
6608 LLVMTypeRef out_type = i < fninfo.num_sgpr_params ? ctx->i32 : ctx->f32;
6609 unsigned size = ac_get_type_size(param_type) / 4;
6610
6611 if (size == 1) {
6612 if (LLVMGetTypeKind(param_type) == LLVMPointerTypeKind) {
6613 param = LLVMBuildPtrToInt(builder, param, ctx->i32, "");
6614 param_type = ctx->i32;
6615 }
6616
6617 if (param_type != out_type)
6618 param = LLVMBuildBitCast(builder, param, out_type, "");
6619 out[num_out++] = param;
6620 } else {
6621 LLVMTypeRef vector_type = LLVMVectorType(out_type, size);
6622
6623 if (LLVMGetTypeKind(param_type) == LLVMPointerTypeKind) {
6624 param = LLVMBuildPtrToInt(builder, param, ctx->i64, "");
6625 param_type = ctx->i64;
6626 }
6627
6628 if (param_type != vector_type)
6629 param = LLVMBuildBitCast(builder, param, vector_type, "");
6630
6631 for (unsigned j = 0; j < size; ++j)
6632 out[num_out++] = LLVMBuildExtractElement(
6633 builder, param, LLVMConstInt(ctx->i32, j, 0), "");
6634 }
6635
6636 if (i < fninfo.num_sgpr_params)
6637 num_out_sgpr = num_out;
6638 }
6639
6640 memcpy(initial, out, sizeof(out));
6641 initial_num_out = num_out;
6642 initial_num_out_sgpr = num_out_sgpr;
6643
6644 /* Now chain the parts. */
6645 for (unsigned part = 0; part < num_parts; ++part) {
6646 LLVMValueRef in[48];
6647 LLVMValueRef ret;
6648 LLVMTypeRef ret_type;
6649 unsigned out_idx = 0;
6650 unsigned num_params = LLVMCountParams(parts[part]);
6651
6652 /* Merged shaders are executed conditionally depending
6653 * on the number of enabled threads passed in the input SGPRs. */
6654 if (is_merged_shader(ctx->shader) && part == 0) {
6655 LLVMValueRef ena, count = initial[3];
6656
6657 count = LLVMBuildAnd(builder, count,
6658 LLVMConstInt(ctx->i32, 0x7f, 0), "");
6659 ena = LLVMBuildICmp(builder, LLVMIntULT,
6660 ac_get_thread_id(&ctx->ac), count, "");
6661 lp_build_if(&if_state, &ctx->gallivm, ena);
6662 }
6663
6664 /* Derive arguments for the next part from outputs of the
6665 * previous one.
6666 */
6667 for (unsigned param_idx = 0; param_idx < num_params; ++param_idx) {
6668 LLVMValueRef param;
6669 LLVMTypeRef param_type;
6670 bool is_sgpr;
6671 unsigned param_size;
6672 LLVMValueRef arg = NULL;
6673
6674 param = LLVMGetParam(parts[part], param_idx);
6675 param_type = LLVMTypeOf(param);
6676 param_size = ac_get_type_size(param_type) / 4;
6677 is_sgpr = ac_is_sgpr_param(param);
6678
6679 if (is_sgpr)
6680 lp_add_function_attr(parts[part], param_idx + 1, LP_FUNC_ATTR_INREG);
6681 else if (out_idx < num_out_sgpr) {
6682 /* Skip returned SGPRs the current part doesn't
6683 * declare on the input. */
6684 out_idx = num_out_sgpr;
6685 }
6686
6687 assert(out_idx + param_size <= (is_sgpr ? num_out_sgpr : num_out));
6688
6689 if (param_size == 1)
6690 arg = out[out_idx];
6691 else
6692 arg = lp_build_gather_values(&ctx->gallivm, &out[out_idx], param_size);
6693
6694 if (LLVMTypeOf(arg) != param_type) {
6695 if (LLVMGetTypeKind(param_type) == LLVMPointerTypeKind) {
6696 if (LLVMGetPointerAddressSpace(param_type) ==
6697 AC_CONST_32BIT_ADDR_SPACE) {
6698 arg = LLVMBuildBitCast(builder, arg, ctx->i32, "");
6699 arg = LLVMBuildIntToPtr(builder, arg, param_type, "");
6700 } else {
6701 arg = LLVMBuildBitCast(builder, arg, ctx->i64, "");
6702 arg = LLVMBuildIntToPtr(builder, arg, param_type, "");
6703 }
6704 } else {
6705 arg = LLVMBuildBitCast(builder, arg, param_type, "");
6706 }
6707 }
6708
6709 in[param_idx] = arg;
6710 out_idx += param_size;
6711 }
6712
6713 ret = LLVMBuildCall(builder, parts[part], in, num_params, "");
6714
6715 if (is_merged_shader(ctx->shader) &&
6716 part + 1 == next_shader_first_part) {
6717 lp_build_endif(&if_state);
6718
6719 /* The second half of the merged shader should use
6720 * the inputs from the toplevel (wrapper) function,
6721 * not the return value from the last call.
6722 *
6723 * That's because the last call was executed condi-
6724 * tionally, so we can't consume it in the main
6725 * block.
6726 */
6727 memcpy(out, initial, sizeof(initial));
6728 num_out = initial_num_out;
6729 num_out_sgpr = initial_num_out_sgpr;
6730 continue;
6731 }
6732
6733 /* Extract the returned GPRs. */
6734 ret_type = LLVMTypeOf(ret);
6735 num_out = 0;
6736 num_out_sgpr = 0;
6737
6738 if (LLVMGetTypeKind(ret_type) != LLVMVoidTypeKind) {
6739 assert(LLVMGetTypeKind(ret_type) == LLVMStructTypeKind);
6740
6741 unsigned ret_size = LLVMCountStructElementTypes(ret_type);
6742
6743 for (unsigned i = 0; i < ret_size; ++i) {
6744 LLVMValueRef val =
6745 LLVMBuildExtractValue(builder, ret, i, "");
6746
6747 assert(num_out < ARRAY_SIZE(out));
6748 out[num_out++] = val;
6749
6750 if (LLVMTypeOf(val) == ctx->i32) {
6751 assert(num_out_sgpr + 1 == num_out);
6752 num_out_sgpr = num_out;
6753 }
6754 }
6755 }
6756 }
6757
6758 LLVMBuildRetVoid(builder);
6759 }
6760
6761 int si_compile_tgsi_shader(struct si_screen *sscreen,
6762 struct si_compiler *compiler,
6763 struct si_shader *shader,
6764 bool is_monolithic,
6765 struct pipe_debug_callback *debug)
6766 {
6767 struct si_shader_selector *sel = shader->selector;
6768 struct si_shader_context ctx;
6769 int r = -1;
6770
6771 /* Dump TGSI code before doing TGSI->LLVM conversion in case the
6772 * conversion fails. */
6773 if (si_can_dump_shader(sscreen, sel->info.processor) &&
6774 !(sscreen->debug_flags & DBG(NO_TGSI))) {
6775 if (sel->tokens)
6776 tgsi_dump(sel->tokens, 0);
6777 else
6778 nir_print_shader(sel->nir, stderr);
6779 si_dump_streamout(&sel->so);
6780 }
6781
6782 si_init_shader_ctx(&ctx, sscreen, compiler);
6783 si_llvm_context_set_tgsi(&ctx, shader);
6784 ctx.separate_prolog = !is_monolithic;
6785
6786 memset(shader->info.vs_output_param_offset, AC_EXP_PARAM_UNDEFINED,
6787 sizeof(shader->info.vs_output_param_offset));
6788
6789 shader->info.uses_instanceid = sel->info.uses_instanceid;
6790
6791 if (!si_compile_tgsi_main(&ctx, is_monolithic)) {
6792 si_llvm_dispose(&ctx);
6793 return -1;
6794 }
6795
6796 if (is_monolithic && ctx.type == PIPE_SHADER_VERTEX) {
6797 LLVMValueRef parts[2];
6798 bool need_prolog = sel->vs_needs_prolog;
6799
6800 parts[1] = ctx.main_fn;
6801
6802 if (need_prolog) {
6803 union si_shader_part_key prolog_key;
6804 si_get_vs_prolog_key(&sel->info,
6805 shader->info.num_input_sgprs,
6806 &shader->key.part.vs.prolog,
6807 shader, &prolog_key);
6808 si_build_vs_prolog_function(&ctx, &prolog_key);
6809 parts[0] = ctx.main_fn;
6810 }
6811
6812 si_build_wrapper_function(&ctx, parts + !need_prolog,
6813 1 + need_prolog, need_prolog, 0);
6814 } else if (is_monolithic && ctx.type == PIPE_SHADER_TESS_CTRL) {
6815 if (sscreen->info.chip_class >= GFX9) {
6816 struct si_shader_selector *ls = shader->key.part.tcs.ls;
6817 LLVMValueRef parts[4];
6818 bool vs_needs_prolog =
6819 si_vs_needs_prolog(ls, &shader->key.part.tcs.ls_prolog);
6820
6821 /* TCS main part */
6822 parts[2] = ctx.main_fn;
6823
6824 /* TCS epilog */
6825 union si_shader_part_key tcs_epilog_key;
6826 memset(&tcs_epilog_key, 0, sizeof(tcs_epilog_key));
6827 tcs_epilog_key.tcs_epilog.states = shader->key.part.tcs.epilog;
6828 si_build_tcs_epilog_function(&ctx, &tcs_epilog_key);
6829 parts[3] = ctx.main_fn;
6830
6831 /* VS as LS main part */
6832 struct si_shader shader_ls = {};
6833 shader_ls.selector = ls;
6834 shader_ls.key.as_ls = 1;
6835 shader_ls.key.mono = shader->key.mono;
6836 shader_ls.key.opt = shader->key.opt;
6837 si_llvm_context_set_tgsi(&ctx, &shader_ls);
6838
6839 if (!si_compile_tgsi_main(&ctx, true)) {
6840 si_llvm_dispose(&ctx);
6841 return -1;
6842 }
6843 shader->info.uses_instanceid |= ls->info.uses_instanceid;
6844 parts[1] = ctx.main_fn;
6845
6846 /* LS prolog */
6847 if (vs_needs_prolog) {
6848 union si_shader_part_key vs_prolog_key;
6849 si_get_vs_prolog_key(&ls->info,
6850 shader_ls.info.num_input_sgprs,
6851 &shader->key.part.tcs.ls_prolog,
6852 shader, &vs_prolog_key);
6853 vs_prolog_key.vs_prolog.is_monolithic = true;
6854 si_build_vs_prolog_function(&ctx, &vs_prolog_key);
6855 parts[0] = ctx.main_fn;
6856 }
6857
6858 /* Reset the shader context. */
6859 ctx.shader = shader;
6860 ctx.type = PIPE_SHADER_TESS_CTRL;
6861
6862 si_build_wrapper_function(&ctx,
6863 parts + !vs_needs_prolog,
6864 4 - !vs_needs_prolog, vs_needs_prolog,
6865 vs_needs_prolog ? 2 : 1);
6866 } else {
6867 LLVMValueRef parts[2];
6868 union si_shader_part_key epilog_key;
6869
6870 parts[0] = ctx.main_fn;
6871
6872 memset(&epilog_key, 0, sizeof(epilog_key));
6873 epilog_key.tcs_epilog.states = shader->key.part.tcs.epilog;
6874 si_build_tcs_epilog_function(&ctx, &epilog_key);
6875 parts[1] = ctx.main_fn;
6876
6877 si_build_wrapper_function(&ctx, parts, 2, 0, 0);
6878 }
6879 } else if (is_monolithic && ctx.type == PIPE_SHADER_GEOMETRY) {
6880 if (ctx.screen->info.chip_class >= GFX9) {
6881 struct si_shader_selector *es = shader->key.part.gs.es;
6882 LLVMValueRef es_prolog = NULL;
6883 LLVMValueRef es_main = NULL;
6884 LLVMValueRef gs_prolog = NULL;
6885 LLVMValueRef gs_main = ctx.main_fn;
6886
6887 /* GS prolog */
6888 union si_shader_part_key gs_prolog_key;
6889 memset(&gs_prolog_key, 0, sizeof(gs_prolog_key));
6890 gs_prolog_key.gs_prolog.states = shader->key.part.gs.prolog;
6891 gs_prolog_key.gs_prolog.is_monolithic = true;
6892 si_build_gs_prolog_function(&ctx, &gs_prolog_key);
6893 gs_prolog = ctx.main_fn;
6894
6895 /* ES main part */
6896 struct si_shader shader_es = {};
6897 shader_es.selector = es;
6898 shader_es.key.as_es = 1;
6899 shader_es.key.mono = shader->key.mono;
6900 shader_es.key.opt = shader->key.opt;
6901 si_llvm_context_set_tgsi(&ctx, &shader_es);
6902
6903 if (!si_compile_tgsi_main(&ctx, true)) {
6904 si_llvm_dispose(&ctx);
6905 return -1;
6906 }
6907 shader->info.uses_instanceid |= es->info.uses_instanceid;
6908 es_main = ctx.main_fn;
6909
6910 /* ES prolog */
6911 if (es->vs_needs_prolog) {
6912 union si_shader_part_key vs_prolog_key;
6913 si_get_vs_prolog_key(&es->info,
6914 shader_es.info.num_input_sgprs,
6915 &shader->key.part.gs.vs_prolog,
6916 shader, &vs_prolog_key);
6917 vs_prolog_key.vs_prolog.is_monolithic = true;
6918 si_build_vs_prolog_function(&ctx, &vs_prolog_key);
6919 es_prolog = ctx.main_fn;
6920 }
6921
6922 /* Reset the shader context. */
6923 ctx.shader = shader;
6924 ctx.type = PIPE_SHADER_GEOMETRY;
6925
6926 /* Prepare the array of shader parts. */
6927 LLVMValueRef parts[4];
6928 unsigned num_parts = 0, main_part, next_first_part;
6929
6930 if (es_prolog)
6931 parts[num_parts++] = es_prolog;
6932
6933 parts[main_part = num_parts++] = es_main;
6934 parts[next_first_part = num_parts++] = gs_prolog;
6935 parts[num_parts++] = gs_main;
6936
6937 si_build_wrapper_function(&ctx, parts, num_parts,
6938 main_part, next_first_part);
6939 } else {
6940 LLVMValueRef parts[2];
6941 union si_shader_part_key prolog_key;
6942
6943 parts[1] = ctx.main_fn;
6944
6945 memset(&prolog_key, 0, sizeof(prolog_key));
6946 prolog_key.gs_prolog.states = shader->key.part.gs.prolog;
6947 si_build_gs_prolog_function(&ctx, &prolog_key);
6948 parts[0] = ctx.main_fn;
6949
6950 si_build_wrapper_function(&ctx, parts, 2, 1, 0);
6951 }
6952 } else if (is_monolithic && ctx.type == PIPE_SHADER_FRAGMENT) {
6953 LLVMValueRef parts[3];
6954 union si_shader_part_key prolog_key;
6955 union si_shader_part_key epilog_key;
6956 bool need_prolog;
6957
6958 si_get_ps_prolog_key(shader, &prolog_key, false);
6959 need_prolog = si_need_ps_prolog(&prolog_key);
6960
6961 parts[need_prolog ? 1 : 0] = ctx.main_fn;
6962
6963 if (need_prolog) {
6964 si_build_ps_prolog_function(&ctx, &prolog_key);
6965 parts[0] = ctx.main_fn;
6966 }
6967
6968 si_get_ps_epilog_key(shader, &epilog_key);
6969 si_build_ps_epilog_function(&ctx, &epilog_key);
6970 parts[need_prolog ? 2 : 1] = ctx.main_fn;
6971
6972 si_build_wrapper_function(&ctx, parts, need_prolog ? 3 : 2,
6973 need_prolog ? 1 : 0, 0);
6974 }
6975
6976 si_llvm_optimize_module(&ctx);
6977
6978 /* Post-optimization transformations and analysis. */
6979 si_optimize_vs_outputs(&ctx);
6980
6981 if ((debug && debug->debug_message) ||
6982 si_can_dump_shader(sscreen, ctx.type)) {
6983 ctx.shader->config.private_mem_vgprs =
6984 ac_count_scratch_private_memory(ctx.main_fn);
6985 }
6986
6987 /* Make sure the input is a pointer and not integer followed by inttoptr. */
6988 assert(LLVMGetTypeKind(LLVMTypeOf(LLVMGetParam(ctx.main_fn, 0))) ==
6989 LLVMPointerTypeKind);
6990
6991 /* Compile to bytecode. */
6992 r = si_compile_llvm(sscreen, &shader->binary, &shader->config, compiler,
6993 ctx.gallivm.module, debug, ctx.type, "TGSI shader");
6994 si_llvm_dispose(&ctx);
6995 if (r) {
6996 fprintf(stderr, "LLVM failed to compile shader\n");
6997 return r;
6998 }
6999
7000 /* Validate SGPR and VGPR usage for compute to detect compiler bugs.
7001 * LLVM 3.9svn has this bug.
7002 */
7003 if (sel->type == PIPE_SHADER_COMPUTE) {
7004 unsigned wave_size = 64;
7005 unsigned max_vgprs = 256;
7006 unsigned max_sgprs = sscreen->info.chip_class >= VI ? 800 : 512;
7007 unsigned max_sgprs_per_wave = 128;
7008 unsigned max_block_threads = si_get_max_workgroup_size(shader);
7009 unsigned min_waves_per_cu = DIV_ROUND_UP(max_block_threads, wave_size);
7010 unsigned min_waves_per_simd = DIV_ROUND_UP(min_waves_per_cu, 4);
7011
7012 max_vgprs = max_vgprs / min_waves_per_simd;
7013 max_sgprs = MIN2(max_sgprs / min_waves_per_simd, max_sgprs_per_wave);
7014
7015 if (shader->config.num_sgprs > max_sgprs ||
7016 shader->config.num_vgprs > max_vgprs) {
7017 fprintf(stderr, "LLVM failed to compile a shader correctly: "
7018 "SGPR:VGPR usage is %u:%u, but the hw limit is %u:%u\n",
7019 shader->config.num_sgprs, shader->config.num_vgprs,
7020 max_sgprs, max_vgprs);
7021
7022 /* Just terminate the process, because dependent
7023 * shaders can hang due to bad input data, but use
7024 * the env var to allow shader-db to work.
7025 */
7026 if (!debug_get_bool_option("SI_PASS_BAD_SHADERS", false))
7027 abort();
7028 }
7029 }
7030
7031 /* Add the scratch offset to input SGPRs. */
7032 if (shader->config.scratch_bytes_per_wave && !is_merged_shader(shader))
7033 shader->info.num_input_sgprs += 1; /* scratch byte offset */
7034
7035 /* Calculate the number of fragment input VGPRs. */
7036 if (ctx.type == PIPE_SHADER_FRAGMENT) {
7037 shader->info.num_input_vgprs = 0;
7038 shader->info.face_vgpr_index = -1;
7039 shader->info.ancillary_vgpr_index = -1;
7040
7041 if (G_0286CC_PERSP_SAMPLE_ENA(shader->config.spi_ps_input_addr))
7042 shader->info.num_input_vgprs += 2;
7043 if (G_0286CC_PERSP_CENTER_ENA(shader->config.spi_ps_input_addr))
7044 shader->info.num_input_vgprs += 2;
7045 if (G_0286CC_PERSP_CENTROID_ENA(shader->config.spi_ps_input_addr))
7046 shader->info.num_input_vgprs += 2;
7047 if (G_0286CC_PERSP_PULL_MODEL_ENA(shader->config.spi_ps_input_addr))
7048 shader->info.num_input_vgprs += 3;
7049 if (G_0286CC_LINEAR_SAMPLE_ENA(shader->config.spi_ps_input_addr))
7050 shader->info.num_input_vgprs += 2;
7051 if (G_0286CC_LINEAR_CENTER_ENA(shader->config.spi_ps_input_addr))
7052 shader->info.num_input_vgprs += 2;
7053 if (G_0286CC_LINEAR_CENTROID_ENA(shader->config.spi_ps_input_addr))
7054 shader->info.num_input_vgprs += 2;
7055 if (G_0286CC_LINE_STIPPLE_TEX_ENA(shader->config.spi_ps_input_addr))
7056 shader->info.num_input_vgprs += 1;
7057 if (G_0286CC_POS_X_FLOAT_ENA(shader->config.spi_ps_input_addr))
7058 shader->info.num_input_vgprs += 1;
7059 if (G_0286CC_POS_Y_FLOAT_ENA(shader->config.spi_ps_input_addr))
7060 shader->info.num_input_vgprs += 1;
7061 if (G_0286CC_POS_Z_FLOAT_ENA(shader->config.spi_ps_input_addr))
7062 shader->info.num_input_vgprs += 1;
7063 if (G_0286CC_POS_W_FLOAT_ENA(shader->config.spi_ps_input_addr))
7064 shader->info.num_input_vgprs += 1;
7065 if (G_0286CC_FRONT_FACE_ENA(shader->config.spi_ps_input_addr)) {
7066 shader->info.face_vgpr_index = shader->info.num_input_vgprs;
7067 shader->info.num_input_vgprs += 1;
7068 }
7069 if (G_0286CC_ANCILLARY_ENA(shader->config.spi_ps_input_addr)) {
7070 shader->info.ancillary_vgpr_index = shader->info.num_input_vgprs;
7071 shader->info.num_input_vgprs += 1;
7072 }
7073 if (G_0286CC_SAMPLE_COVERAGE_ENA(shader->config.spi_ps_input_addr))
7074 shader->info.num_input_vgprs += 1;
7075 if (G_0286CC_POS_FIXED_PT_ENA(shader->config.spi_ps_input_addr))
7076 shader->info.num_input_vgprs += 1;
7077 }
7078
7079 si_calculate_max_simd_waves(shader);
7080 si_shader_dump_stats_for_shader_db(shader, debug);
7081 return 0;
7082 }
7083
7084 /**
7085 * Create, compile and return a shader part (prolog or epilog).
7086 *
7087 * \param sscreen screen
7088 * \param list list of shader parts of the same category
7089 * \param type shader type
7090 * \param key shader part key
7091 * \param prolog whether the part being requested is a prolog
7092 * \param tm LLVM target machine
7093 * \param debug debug callback
7094 * \param build the callback responsible for building the main function
7095 * \return non-NULL on success
7096 */
7097 static struct si_shader_part *
7098 si_get_shader_part(struct si_screen *sscreen,
7099 struct si_shader_part **list,
7100 enum pipe_shader_type type,
7101 bool prolog,
7102 union si_shader_part_key *key,
7103 struct si_compiler *compiler,
7104 struct pipe_debug_callback *debug,
7105 void (*build)(struct si_shader_context *,
7106 union si_shader_part_key *),
7107 const char *name)
7108 {
7109 struct si_shader_part *result;
7110
7111 mtx_lock(&sscreen->shader_parts_mutex);
7112
7113 /* Find existing. */
7114 for (result = *list; result; result = result->next) {
7115 if (memcmp(&result->key, key, sizeof(*key)) == 0) {
7116 mtx_unlock(&sscreen->shader_parts_mutex);
7117 return result;
7118 }
7119 }
7120
7121 /* Compile a new one. */
7122 result = CALLOC_STRUCT(si_shader_part);
7123 result->key = *key;
7124
7125 struct si_shader shader = {};
7126 struct si_shader_context ctx;
7127
7128 si_init_shader_ctx(&ctx, sscreen, compiler);
7129 ctx.shader = &shader;
7130 ctx.type = type;
7131
7132 switch (type) {
7133 case PIPE_SHADER_VERTEX:
7134 shader.key.as_ls = key->vs_prolog.as_ls;
7135 shader.key.as_es = key->vs_prolog.as_es;
7136 break;
7137 case PIPE_SHADER_TESS_CTRL:
7138 assert(!prolog);
7139 shader.key.part.tcs.epilog = key->tcs_epilog.states;
7140 break;
7141 case PIPE_SHADER_GEOMETRY:
7142 assert(prolog);
7143 break;
7144 case PIPE_SHADER_FRAGMENT:
7145 if (prolog)
7146 shader.key.part.ps.prolog = key->ps_prolog.states;
7147 else
7148 shader.key.part.ps.epilog = key->ps_epilog.states;
7149 break;
7150 default:
7151 unreachable("bad shader part");
7152 }
7153
7154 build(&ctx, key);
7155
7156 /* Compile. */
7157 si_llvm_optimize_module(&ctx);
7158
7159 if (si_compile_llvm(sscreen, &result->binary, &result->config, compiler,
7160 ctx.ac.module, debug, ctx.type, name)) {
7161 FREE(result);
7162 result = NULL;
7163 goto out;
7164 }
7165
7166 result->next = *list;
7167 *list = result;
7168
7169 out:
7170 si_llvm_dispose(&ctx);
7171 mtx_unlock(&sscreen->shader_parts_mutex);
7172 return result;
7173 }
7174
7175 static LLVMValueRef si_prolog_get_rw_buffers(struct si_shader_context *ctx)
7176 {
7177 LLVMValueRef ptr[2], list;
7178 bool is_merged_shader =
7179 ctx->screen->info.chip_class >= GFX9 &&
7180 (ctx->type == PIPE_SHADER_TESS_CTRL ||
7181 ctx->type == PIPE_SHADER_GEOMETRY ||
7182 ctx->shader->key.as_ls || ctx->shader->key.as_es);
7183
7184 if (HAVE_32BIT_POINTERS) {
7185 ptr[0] = LLVMGetParam(ctx->main_fn, (is_merged_shader ? 8 : 0) + SI_SGPR_RW_BUFFERS);
7186 list = LLVMBuildIntToPtr(ctx->ac.builder, ptr[0],
7187 ac_array_in_const32_addr_space(ctx->v4i32), "");
7188 return list;
7189 }
7190
7191 /* Get the pointer to rw buffers. */
7192 ptr[0] = LLVMGetParam(ctx->main_fn, (is_merged_shader ? 8 : 0) + SI_SGPR_RW_BUFFERS);
7193 ptr[1] = LLVMGetParam(ctx->main_fn, (is_merged_shader ? 8 : 0) + SI_SGPR_RW_BUFFERS + 1);
7194 list = lp_build_gather_values(&ctx->gallivm, ptr, 2);
7195 list = LLVMBuildBitCast(ctx->ac.builder, list, ctx->i64, "");
7196 list = LLVMBuildIntToPtr(ctx->ac.builder, list,
7197 ac_array_in_const_addr_space(ctx->v4i32), "");
7198 return list;
7199 }
7200
7201 /**
7202 * Build the vertex shader prolog function.
7203 *
7204 * The inputs are the same as VS (a lot of SGPRs and 4 VGPR system values).
7205 * All inputs are returned unmodified. The vertex load indices are
7206 * stored after them, which will be used by the API VS for fetching inputs.
7207 *
7208 * For example, the expected outputs for instance_divisors[] = {0, 1, 2} are:
7209 * input_v0,
7210 * input_v1,
7211 * input_v2,
7212 * input_v3,
7213 * (VertexID + BaseVertex),
7214 * (InstanceID + StartInstance),
7215 * (InstanceID / 2 + StartInstance)
7216 */
7217 static void si_build_vs_prolog_function(struct si_shader_context *ctx,
7218 union si_shader_part_key *key)
7219 {
7220 struct si_function_info fninfo;
7221 LLVMTypeRef *returns;
7222 LLVMValueRef ret, func;
7223 int num_returns, i;
7224 unsigned first_vs_vgpr = key->vs_prolog.num_merged_next_stage_vgprs;
7225 unsigned num_input_vgprs = key->vs_prolog.num_merged_next_stage_vgprs + 4;
7226 LLVMValueRef input_vgprs[9];
7227 unsigned num_all_input_regs = key->vs_prolog.num_input_sgprs +
7228 num_input_vgprs;
7229 unsigned user_sgpr_base = key->vs_prolog.num_merged_next_stage_vgprs ? 8 : 0;
7230
7231 si_init_function_info(&fninfo);
7232
7233 /* 4 preloaded VGPRs + vertex load indices as prolog outputs */
7234 returns = alloca((num_all_input_regs + key->vs_prolog.last_input + 1) *
7235 sizeof(LLVMTypeRef));
7236 num_returns = 0;
7237
7238 /* Declare input and output SGPRs. */
7239 for (i = 0; i < key->vs_prolog.num_input_sgprs; i++) {
7240 add_arg(&fninfo, ARG_SGPR, ctx->i32);
7241 returns[num_returns++] = ctx->i32;
7242 }
7243
7244 /* Preloaded VGPRs (outputs must be floats) */
7245 for (i = 0; i < num_input_vgprs; i++) {
7246 add_arg_assign(&fninfo, ARG_VGPR, ctx->i32, &input_vgprs[i]);
7247 returns[num_returns++] = ctx->f32;
7248 }
7249
7250 /* Vertex load indices. */
7251 for (i = 0; i <= key->vs_prolog.last_input; i++)
7252 returns[num_returns++] = ctx->f32;
7253
7254 /* Create the function. */
7255 si_create_function(ctx, "vs_prolog", returns, num_returns, &fninfo, 0);
7256 func = ctx->main_fn;
7257
7258 if (key->vs_prolog.num_merged_next_stage_vgprs) {
7259 if (!key->vs_prolog.is_monolithic)
7260 si_init_exec_from_input(ctx, 3, 0);
7261
7262 if (key->vs_prolog.as_ls &&
7263 ctx->screen->has_ls_vgpr_init_bug) {
7264 /* If there are no HS threads, SPI loads the LS VGPRs
7265 * starting at VGPR 0. Shift them back to where they
7266 * belong.
7267 */
7268 LLVMValueRef has_hs_threads =
7269 LLVMBuildICmp(ctx->ac.builder, LLVMIntNE,
7270 si_unpack_param(ctx, 3, 8, 8),
7271 ctx->i32_0, "");
7272
7273 for (i = 4; i > 0; --i) {
7274 input_vgprs[i + 1] =
7275 LLVMBuildSelect(ctx->ac.builder, has_hs_threads,
7276 input_vgprs[i + 1],
7277 input_vgprs[i - 1], "");
7278 }
7279 }
7280 }
7281
7282 ctx->abi.vertex_id = input_vgprs[first_vs_vgpr];
7283 ctx->abi.instance_id = input_vgprs[first_vs_vgpr + (key->vs_prolog.as_ls ? 2 : 1)];
7284
7285 /* Copy inputs to outputs. This should be no-op, as the registers match,
7286 * but it will prevent the compiler from overwriting them unintentionally.
7287 */
7288 ret = ctx->return_value;
7289 for (i = 0; i < key->vs_prolog.num_input_sgprs; i++) {
7290 LLVMValueRef p = LLVMGetParam(func, i);
7291 ret = LLVMBuildInsertValue(ctx->ac.builder, ret, p, i, "");
7292 }
7293 for (i = 0; i < num_input_vgprs; i++) {
7294 LLVMValueRef p = input_vgprs[i];
7295 p = ac_to_float(&ctx->ac, p);
7296 ret = LLVMBuildInsertValue(ctx->ac.builder, ret, p,
7297 key->vs_prolog.num_input_sgprs + i, "");
7298 }
7299
7300 /* Compute vertex load indices from instance divisors. */
7301 LLVMValueRef instance_divisor_constbuf = NULL;
7302
7303 if (key->vs_prolog.states.instance_divisor_is_fetched) {
7304 LLVMValueRef list = si_prolog_get_rw_buffers(ctx);
7305 LLVMValueRef buf_index =
7306 LLVMConstInt(ctx->i32, SI_VS_CONST_INSTANCE_DIVISORS, 0);
7307 instance_divisor_constbuf =
7308 ac_build_load_to_sgpr(&ctx->ac, list, buf_index);
7309 }
7310
7311 for (i = 0; i <= key->vs_prolog.last_input; i++) {
7312 bool divisor_is_one =
7313 key->vs_prolog.states.instance_divisor_is_one & (1u << i);
7314 bool divisor_is_fetched =
7315 key->vs_prolog.states.instance_divisor_is_fetched & (1u << i);
7316 LLVMValueRef index;
7317
7318 if (divisor_is_one || divisor_is_fetched) {
7319 LLVMValueRef divisor = ctx->i32_1;
7320
7321 if (divisor_is_fetched) {
7322 divisor = buffer_load_const(ctx, instance_divisor_constbuf,
7323 LLVMConstInt(ctx->i32, i * 4, 0));
7324 divisor = ac_to_integer(&ctx->ac, divisor);
7325 }
7326
7327 /* InstanceID / Divisor + StartInstance */
7328 index = get_instance_index_for_fetch(ctx,
7329 user_sgpr_base +
7330 SI_SGPR_START_INSTANCE,
7331 divisor);
7332 } else {
7333 /* VertexID + BaseVertex */
7334 index = LLVMBuildAdd(ctx->ac.builder,
7335 ctx->abi.vertex_id,
7336 LLVMGetParam(func, user_sgpr_base +
7337 SI_SGPR_BASE_VERTEX), "");
7338 }
7339
7340 index = ac_to_float(&ctx->ac, index);
7341 ret = LLVMBuildInsertValue(ctx->ac.builder, ret, index,
7342 fninfo.num_params + i, "");
7343 }
7344
7345 si_llvm_build_ret(ctx, ret);
7346 }
7347
7348 static bool si_get_vs_prolog(struct si_screen *sscreen,
7349 struct si_compiler *compiler,
7350 struct si_shader *shader,
7351 struct pipe_debug_callback *debug,
7352 struct si_shader *main_part,
7353 const struct si_vs_prolog_bits *key)
7354 {
7355 struct si_shader_selector *vs = main_part->selector;
7356
7357 if (!si_vs_needs_prolog(vs, key))
7358 return true;
7359
7360 /* Get the prolog. */
7361 union si_shader_part_key prolog_key;
7362 si_get_vs_prolog_key(&vs->info, main_part->info.num_input_sgprs,
7363 key, shader, &prolog_key);
7364
7365 shader->prolog =
7366 si_get_shader_part(sscreen, &sscreen->vs_prologs,
7367 PIPE_SHADER_VERTEX, true, &prolog_key, compiler,
7368 debug, si_build_vs_prolog_function,
7369 "Vertex Shader Prolog");
7370 return shader->prolog != NULL;
7371 }
7372
7373 /**
7374 * Select and compile (or reuse) vertex shader parts (prolog & epilog).
7375 */
7376 static bool si_shader_select_vs_parts(struct si_screen *sscreen,
7377 struct si_compiler *compiler,
7378 struct si_shader *shader,
7379 struct pipe_debug_callback *debug)
7380 {
7381 return si_get_vs_prolog(sscreen, compiler, shader, debug, shader,
7382 &shader->key.part.vs.prolog);
7383 }
7384
7385 /**
7386 * Compile the TCS epilog function. This writes tesselation factors to memory
7387 * based on the output primitive type of the tesselator (determined by TES).
7388 */
7389 static void si_build_tcs_epilog_function(struct si_shader_context *ctx,
7390 union si_shader_part_key *key)
7391 {
7392 struct lp_build_tgsi_context *bld_base = &ctx->bld_base;
7393 struct si_function_info fninfo;
7394 LLVMValueRef func;
7395
7396 si_init_function_info(&fninfo);
7397
7398 if (ctx->screen->info.chip_class >= GFX9) {
7399 add_arg(&fninfo, ARG_SGPR, ctx->i32);
7400 add_arg(&fninfo, ARG_SGPR, ctx->i32);
7401 ctx->param_tcs_offchip_offset = add_arg(&fninfo, ARG_SGPR, ctx->i32);
7402 add_arg(&fninfo, ARG_SGPR, ctx->i32); /* wave info */
7403 ctx->param_tcs_factor_offset = add_arg(&fninfo, ARG_SGPR, ctx->i32);
7404 add_arg(&fninfo, ARG_SGPR, ctx->i32);
7405 add_arg(&fninfo, ARG_SGPR, ctx->i32);
7406 add_arg(&fninfo, ARG_SGPR, ctx->i32);
7407 add_arg(&fninfo, ARG_SGPR, ctx->ac.intptr);
7408 add_arg(&fninfo, ARG_SGPR, ctx->ac.intptr);
7409 add_arg(&fninfo, ARG_SGPR, ctx->ac.intptr);
7410 add_arg(&fninfo, ARG_SGPR, ctx->ac.intptr);
7411 add_arg(&fninfo, ARG_SGPR, ctx->i32);
7412 add_arg(&fninfo, ARG_SGPR, ctx->i32);
7413 add_arg(&fninfo, ARG_SGPR, ctx->i32);
7414 add_arg(&fninfo, ARG_SGPR, ctx->i32);
7415 if (!HAVE_32BIT_POINTERS)
7416 add_arg(&fninfo, ARG_SGPR, ctx->ac.intptr);
7417 ctx->param_tcs_offchip_layout = add_arg(&fninfo, ARG_SGPR, ctx->i32);
7418 add_arg(&fninfo, ARG_SGPR, ctx->i32);
7419 ctx->param_tcs_out_lds_layout = add_arg(&fninfo, ARG_SGPR, ctx->i32);
7420 } else {
7421 add_arg(&fninfo, ARG_SGPR, ctx->ac.intptr);
7422 add_arg(&fninfo, ARG_SGPR, ctx->ac.intptr);
7423 add_arg(&fninfo, ARG_SGPR, ctx->ac.intptr);
7424 add_arg(&fninfo, ARG_SGPR, ctx->ac.intptr);
7425 ctx->param_tcs_offchip_layout = add_arg(&fninfo, ARG_SGPR, ctx->i32);
7426 add_arg(&fninfo, ARG_SGPR, ctx->i32);
7427 ctx->param_tcs_out_lds_layout = add_arg(&fninfo, ARG_SGPR, ctx->i32);
7428 add_arg(&fninfo, ARG_SGPR, ctx->i32);
7429 ctx->param_tcs_offchip_offset = add_arg(&fninfo, ARG_SGPR, ctx->i32);
7430 ctx->param_tcs_factor_offset = add_arg(&fninfo, ARG_SGPR, ctx->i32);
7431 }
7432
7433 add_arg(&fninfo, ARG_VGPR, ctx->i32); /* VGPR gap */
7434 add_arg(&fninfo, ARG_VGPR, ctx->i32); /* VGPR gap */
7435 unsigned tess_factors_idx =
7436 add_arg(&fninfo, ARG_VGPR, ctx->i32); /* patch index within the wave (REL_PATCH_ID) */
7437 add_arg(&fninfo, ARG_VGPR, ctx->i32); /* invocation ID within the patch */
7438 add_arg(&fninfo, ARG_VGPR, ctx->i32); /* LDS offset where tess factors should be loaded from */
7439
7440 for (unsigned i = 0; i < 6; i++)
7441 add_arg(&fninfo, ARG_VGPR, ctx->i32); /* tess factors */
7442
7443 /* Create the function. */
7444 si_create_function(ctx, "tcs_epilog", NULL, 0, &fninfo,
7445 ctx->screen->info.chip_class >= CIK ? 128 : 64);
7446 ac_declare_lds_as_pointer(&ctx->ac);
7447 func = ctx->main_fn;
7448
7449 LLVMValueRef invoc0_tess_factors[6];
7450 for (unsigned i = 0; i < 6; i++)
7451 invoc0_tess_factors[i] = LLVMGetParam(func, tess_factors_idx + 3 + i);
7452
7453 si_write_tess_factors(bld_base,
7454 LLVMGetParam(func, tess_factors_idx),
7455 LLVMGetParam(func, tess_factors_idx + 1),
7456 LLVMGetParam(func, tess_factors_idx + 2),
7457 invoc0_tess_factors, invoc0_tess_factors + 4);
7458
7459 LLVMBuildRetVoid(ctx->ac.builder);
7460 }
7461
7462 /**
7463 * Select and compile (or reuse) TCS parts (epilog).
7464 */
7465 static bool si_shader_select_tcs_parts(struct si_screen *sscreen,
7466 struct si_compiler *compiler,
7467 struct si_shader *shader,
7468 struct pipe_debug_callback *debug)
7469 {
7470 if (sscreen->info.chip_class >= GFX9) {
7471 struct si_shader *ls_main_part =
7472 shader->key.part.tcs.ls->main_shader_part_ls;
7473
7474 if (!si_get_vs_prolog(sscreen, compiler, shader, debug, ls_main_part,
7475 &shader->key.part.tcs.ls_prolog))
7476 return false;
7477
7478 shader->previous_stage = ls_main_part;
7479 }
7480
7481 /* Get the epilog. */
7482 union si_shader_part_key epilog_key;
7483 memset(&epilog_key, 0, sizeof(epilog_key));
7484 epilog_key.tcs_epilog.states = shader->key.part.tcs.epilog;
7485
7486 shader->epilog = si_get_shader_part(sscreen, &sscreen->tcs_epilogs,
7487 PIPE_SHADER_TESS_CTRL, false,
7488 &epilog_key, compiler, debug,
7489 si_build_tcs_epilog_function,
7490 "Tessellation Control Shader Epilog");
7491 return shader->epilog != NULL;
7492 }
7493
7494 /**
7495 * Select and compile (or reuse) GS parts (prolog).
7496 */
7497 static bool si_shader_select_gs_parts(struct si_screen *sscreen,
7498 struct si_compiler *compiler,
7499 struct si_shader *shader,
7500 struct pipe_debug_callback *debug)
7501 {
7502 if (sscreen->info.chip_class >= GFX9) {
7503 struct si_shader *es_main_part =
7504 shader->key.part.gs.es->main_shader_part_es;
7505
7506 if (shader->key.part.gs.es->type == PIPE_SHADER_VERTEX &&
7507 !si_get_vs_prolog(sscreen, compiler, shader, debug, es_main_part,
7508 &shader->key.part.gs.vs_prolog))
7509 return false;
7510
7511 shader->previous_stage = es_main_part;
7512 }
7513
7514 if (!shader->key.part.gs.prolog.tri_strip_adj_fix)
7515 return true;
7516
7517 union si_shader_part_key prolog_key;
7518 memset(&prolog_key, 0, sizeof(prolog_key));
7519 prolog_key.gs_prolog.states = shader->key.part.gs.prolog;
7520
7521 shader->prolog2 = si_get_shader_part(sscreen, &sscreen->gs_prologs,
7522 PIPE_SHADER_GEOMETRY, true,
7523 &prolog_key, compiler, debug,
7524 si_build_gs_prolog_function,
7525 "Geometry Shader Prolog");
7526 return shader->prolog2 != NULL;
7527 }
7528
7529 /**
7530 * Build the pixel shader prolog function. This handles:
7531 * - two-side color selection and interpolation
7532 * - overriding interpolation parameters for the API PS
7533 * - polygon stippling
7534 *
7535 * All preloaded SGPRs and VGPRs are passed through unmodified unless they are
7536 * overriden by other states. (e.g. per-sample interpolation)
7537 * Interpolated colors are stored after the preloaded VGPRs.
7538 */
7539 static void si_build_ps_prolog_function(struct si_shader_context *ctx,
7540 union si_shader_part_key *key)
7541 {
7542 struct si_function_info fninfo;
7543 LLVMValueRef ret, func;
7544 int num_returns, i, num_color_channels;
7545
7546 assert(si_need_ps_prolog(key));
7547
7548 si_init_function_info(&fninfo);
7549
7550 /* Declare inputs. */
7551 for (i = 0; i < key->ps_prolog.num_input_sgprs; i++)
7552 add_arg(&fninfo, ARG_SGPR, ctx->i32);
7553
7554 for (i = 0; i < key->ps_prolog.num_input_vgprs; i++)
7555 add_arg(&fninfo, ARG_VGPR, ctx->f32);
7556
7557 /* Declare outputs (same as inputs + add colors if needed) */
7558 num_returns = fninfo.num_params;
7559 num_color_channels = util_bitcount(key->ps_prolog.colors_read);
7560 for (i = 0; i < num_color_channels; i++)
7561 fninfo.types[num_returns++] = ctx->f32;
7562
7563 /* Create the function. */
7564 si_create_function(ctx, "ps_prolog", fninfo.types, num_returns,
7565 &fninfo, 0);
7566 func = ctx->main_fn;
7567
7568 /* Copy inputs to outputs. This should be no-op, as the registers match,
7569 * but it will prevent the compiler from overwriting them unintentionally.
7570 */
7571 ret = ctx->return_value;
7572 for (i = 0; i < fninfo.num_params; i++) {
7573 LLVMValueRef p = LLVMGetParam(func, i);
7574 ret = LLVMBuildInsertValue(ctx->ac.builder, ret, p, i, "");
7575 }
7576
7577 /* Polygon stippling. */
7578 if (key->ps_prolog.states.poly_stipple) {
7579 /* POS_FIXED_PT is always last. */
7580 unsigned pos = key->ps_prolog.num_input_sgprs +
7581 key->ps_prolog.num_input_vgprs - 1;
7582 LLVMValueRef list = si_prolog_get_rw_buffers(ctx);
7583
7584 si_llvm_emit_polygon_stipple(ctx, list, pos);
7585 }
7586
7587 if (key->ps_prolog.states.bc_optimize_for_persp ||
7588 key->ps_prolog.states.bc_optimize_for_linear) {
7589 unsigned i, base = key->ps_prolog.num_input_sgprs;
7590 LLVMValueRef center[2], centroid[2], tmp, bc_optimize;
7591
7592 /* The shader should do: if (PRIM_MASK[31]) CENTROID = CENTER;
7593 * The hw doesn't compute CENTROID if the whole wave only
7594 * contains fully-covered quads.
7595 *
7596 * PRIM_MASK is after user SGPRs.
7597 */
7598 bc_optimize = LLVMGetParam(func, SI_PS_NUM_USER_SGPR);
7599 bc_optimize = LLVMBuildLShr(ctx->ac.builder, bc_optimize,
7600 LLVMConstInt(ctx->i32, 31, 0), "");
7601 bc_optimize = LLVMBuildTrunc(ctx->ac.builder, bc_optimize,
7602 ctx->i1, "");
7603
7604 if (key->ps_prolog.states.bc_optimize_for_persp) {
7605 /* Read PERSP_CENTER. */
7606 for (i = 0; i < 2; i++)
7607 center[i] = LLVMGetParam(func, base + 2 + i);
7608 /* Read PERSP_CENTROID. */
7609 for (i = 0; i < 2; i++)
7610 centroid[i] = LLVMGetParam(func, base + 4 + i);
7611 /* Select PERSP_CENTROID. */
7612 for (i = 0; i < 2; i++) {
7613 tmp = LLVMBuildSelect(ctx->ac.builder, bc_optimize,
7614 center[i], centroid[i], "");
7615 ret = LLVMBuildInsertValue(ctx->ac.builder, ret,
7616 tmp, base + 4 + i, "");
7617 }
7618 }
7619 if (key->ps_prolog.states.bc_optimize_for_linear) {
7620 /* Read LINEAR_CENTER. */
7621 for (i = 0; i < 2; i++)
7622 center[i] = LLVMGetParam(func, base + 8 + i);
7623 /* Read LINEAR_CENTROID. */
7624 for (i = 0; i < 2; i++)
7625 centroid[i] = LLVMGetParam(func, base + 10 + i);
7626 /* Select LINEAR_CENTROID. */
7627 for (i = 0; i < 2; i++) {
7628 tmp = LLVMBuildSelect(ctx->ac.builder, bc_optimize,
7629 center[i], centroid[i], "");
7630 ret = LLVMBuildInsertValue(ctx->ac.builder, ret,
7631 tmp, base + 10 + i, "");
7632 }
7633 }
7634 }
7635
7636 /* Force per-sample interpolation. */
7637 if (key->ps_prolog.states.force_persp_sample_interp) {
7638 unsigned i, base = key->ps_prolog.num_input_sgprs;
7639 LLVMValueRef persp_sample[2];
7640
7641 /* Read PERSP_SAMPLE. */
7642 for (i = 0; i < 2; i++)
7643 persp_sample[i] = LLVMGetParam(func, base + i);
7644 /* Overwrite PERSP_CENTER. */
7645 for (i = 0; i < 2; i++)
7646 ret = LLVMBuildInsertValue(ctx->ac.builder, ret,
7647 persp_sample[i], base + 2 + i, "");
7648 /* Overwrite PERSP_CENTROID. */
7649 for (i = 0; i < 2; i++)
7650 ret = LLVMBuildInsertValue(ctx->ac.builder, ret,
7651 persp_sample[i], base + 4 + i, "");
7652 }
7653 if (key->ps_prolog.states.force_linear_sample_interp) {
7654 unsigned i, base = key->ps_prolog.num_input_sgprs;
7655 LLVMValueRef linear_sample[2];
7656
7657 /* Read LINEAR_SAMPLE. */
7658 for (i = 0; i < 2; i++)
7659 linear_sample[i] = LLVMGetParam(func, base + 6 + i);
7660 /* Overwrite LINEAR_CENTER. */
7661 for (i = 0; i < 2; i++)
7662 ret = LLVMBuildInsertValue(ctx->ac.builder, ret,
7663 linear_sample[i], base + 8 + i, "");
7664 /* Overwrite LINEAR_CENTROID. */
7665 for (i = 0; i < 2; i++)
7666 ret = LLVMBuildInsertValue(ctx->ac.builder, ret,
7667 linear_sample[i], base + 10 + i, "");
7668 }
7669
7670 /* Force center interpolation. */
7671 if (key->ps_prolog.states.force_persp_center_interp) {
7672 unsigned i, base = key->ps_prolog.num_input_sgprs;
7673 LLVMValueRef persp_center[2];
7674
7675 /* Read PERSP_CENTER. */
7676 for (i = 0; i < 2; i++)
7677 persp_center[i] = LLVMGetParam(func, base + 2 + i);
7678 /* Overwrite PERSP_SAMPLE. */
7679 for (i = 0; i < 2; i++)
7680 ret = LLVMBuildInsertValue(ctx->ac.builder, ret,
7681 persp_center[i], base + i, "");
7682 /* Overwrite PERSP_CENTROID. */
7683 for (i = 0; i < 2; i++)
7684 ret = LLVMBuildInsertValue(ctx->ac.builder, ret,
7685 persp_center[i], base + 4 + i, "");
7686 }
7687 if (key->ps_prolog.states.force_linear_center_interp) {
7688 unsigned i, base = key->ps_prolog.num_input_sgprs;
7689 LLVMValueRef linear_center[2];
7690
7691 /* Read LINEAR_CENTER. */
7692 for (i = 0; i < 2; i++)
7693 linear_center[i] = LLVMGetParam(func, base + 8 + i);
7694 /* Overwrite LINEAR_SAMPLE. */
7695 for (i = 0; i < 2; i++)
7696 ret = LLVMBuildInsertValue(ctx->ac.builder, ret,
7697 linear_center[i], base + 6 + i, "");
7698 /* Overwrite LINEAR_CENTROID. */
7699 for (i = 0; i < 2; i++)
7700 ret = LLVMBuildInsertValue(ctx->ac.builder, ret,
7701 linear_center[i], base + 10 + i, "");
7702 }
7703
7704 /* Interpolate colors. */
7705 unsigned color_out_idx = 0;
7706 for (i = 0; i < 2; i++) {
7707 unsigned writemask = (key->ps_prolog.colors_read >> (i * 4)) & 0xf;
7708 unsigned face_vgpr = key->ps_prolog.num_input_sgprs +
7709 key->ps_prolog.face_vgpr_index;
7710 LLVMValueRef interp[2], color[4];
7711 LLVMValueRef interp_ij = NULL, prim_mask = NULL, face = NULL;
7712
7713 if (!writemask)
7714 continue;
7715
7716 /* If the interpolation qualifier is not CONSTANT (-1). */
7717 if (key->ps_prolog.color_interp_vgpr_index[i] != -1) {
7718 unsigned interp_vgpr = key->ps_prolog.num_input_sgprs +
7719 key->ps_prolog.color_interp_vgpr_index[i];
7720
7721 /* Get the (i,j) updated by bc_optimize handling. */
7722 interp[0] = LLVMBuildExtractValue(ctx->ac.builder, ret,
7723 interp_vgpr, "");
7724 interp[1] = LLVMBuildExtractValue(ctx->ac.builder, ret,
7725 interp_vgpr + 1, "");
7726 interp_ij = lp_build_gather_values(&ctx->gallivm, interp, 2);
7727 }
7728
7729 /* Use the absolute location of the input. */
7730 prim_mask = LLVMGetParam(func, SI_PS_NUM_USER_SGPR);
7731
7732 if (key->ps_prolog.states.color_two_side) {
7733 face = LLVMGetParam(func, face_vgpr);
7734 face = ac_to_integer(&ctx->ac, face);
7735 }
7736
7737 interp_fs_input(ctx,
7738 key->ps_prolog.color_attr_index[i],
7739 TGSI_SEMANTIC_COLOR, i,
7740 key->ps_prolog.num_interp_inputs,
7741 key->ps_prolog.colors_read, interp_ij,
7742 prim_mask, face, color);
7743
7744 while (writemask) {
7745 unsigned chan = u_bit_scan(&writemask);
7746 ret = LLVMBuildInsertValue(ctx->ac.builder, ret, color[chan],
7747 fninfo.num_params + color_out_idx++, "");
7748 }
7749 }
7750
7751 /* Section 15.2.2 (Shader Inputs) of the OpenGL 4.5 (Core Profile) spec
7752 * says:
7753 *
7754 * "When per-sample shading is active due to the use of a fragment
7755 * input qualified by sample or due to the use of the gl_SampleID
7756 * or gl_SamplePosition variables, only the bit for the current
7757 * sample is set in gl_SampleMaskIn. When state specifies multiple
7758 * fragment shader invocations for a given fragment, the sample
7759 * mask for any single fragment shader invocation may specify a
7760 * subset of the covered samples for the fragment. In this case,
7761 * the bit corresponding to each covered sample will be set in
7762 * exactly one fragment shader invocation."
7763 *
7764 * The samplemask loaded by hardware is always the coverage of the
7765 * entire pixel/fragment, so mask bits out based on the sample ID.
7766 */
7767 if (key->ps_prolog.states.samplemask_log_ps_iter) {
7768 /* The bit pattern matches that used by fixed function fragment
7769 * processing. */
7770 static const uint16_t ps_iter_masks[] = {
7771 0xffff, /* not used */
7772 0x5555,
7773 0x1111,
7774 0x0101,
7775 0x0001,
7776 };
7777 assert(key->ps_prolog.states.samplemask_log_ps_iter < ARRAY_SIZE(ps_iter_masks));
7778
7779 uint32_t ps_iter_mask = ps_iter_masks[key->ps_prolog.states.samplemask_log_ps_iter];
7780 unsigned ancillary_vgpr = key->ps_prolog.num_input_sgprs +
7781 key->ps_prolog.ancillary_vgpr_index;
7782 LLVMValueRef sampleid = si_unpack_param(ctx, ancillary_vgpr, 8, 4);
7783 LLVMValueRef samplemask = LLVMGetParam(func, ancillary_vgpr + 1);
7784
7785 samplemask = ac_to_integer(&ctx->ac, samplemask);
7786 samplemask = LLVMBuildAnd(
7787 ctx->ac.builder,
7788 samplemask,
7789 LLVMBuildShl(ctx->ac.builder,
7790 LLVMConstInt(ctx->i32, ps_iter_mask, false),
7791 sampleid, ""),
7792 "");
7793 samplemask = ac_to_float(&ctx->ac, samplemask);
7794
7795 ret = LLVMBuildInsertValue(ctx->ac.builder, ret, samplemask,
7796 ancillary_vgpr + 1, "");
7797 }
7798
7799 /* Tell LLVM to insert WQM instruction sequence when needed. */
7800 if (key->ps_prolog.wqm) {
7801 LLVMAddTargetDependentFunctionAttr(func,
7802 "amdgpu-ps-wqm-outputs", "");
7803 }
7804
7805 si_llvm_build_ret(ctx, ret);
7806 }
7807
7808 /**
7809 * Build the pixel shader epilog function. This handles everything that must be
7810 * emulated for pixel shader exports. (alpha-test, format conversions, etc)
7811 */
7812 static void si_build_ps_epilog_function(struct si_shader_context *ctx,
7813 union si_shader_part_key *key)
7814 {
7815 struct lp_build_tgsi_context *bld_base = &ctx->bld_base;
7816 struct si_function_info fninfo;
7817 LLVMValueRef depth = NULL, stencil = NULL, samplemask = NULL;
7818 int i;
7819 struct si_ps_exports exp = {};
7820
7821 si_init_function_info(&fninfo);
7822
7823 /* Declare input SGPRs. */
7824 ctx->param_rw_buffers = add_arg(&fninfo, ARG_SGPR, ctx->ac.intptr);
7825 ctx->param_bindless_samplers_and_images = add_arg(&fninfo, ARG_SGPR, ctx->ac.intptr);
7826 ctx->param_const_and_shader_buffers = add_arg(&fninfo, ARG_SGPR, ctx->ac.intptr);
7827 ctx->param_samplers_and_images = add_arg(&fninfo, ARG_SGPR, ctx->ac.intptr);
7828 add_arg_checked(&fninfo, ARG_SGPR, ctx->f32, SI_PARAM_ALPHA_REF);
7829
7830 /* Declare input VGPRs. */
7831 unsigned required_num_params =
7832 fninfo.num_sgpr_params +
7833 util_bitcount(key->ps_epilog.colors_written) * 4 +
7834 key->ps_epilog.writes_z +
7835 key->ps_epilog.writes_stencil +
7836 key->ps_epilog.writes_samplemask;
7837
7838 required_num_params = MAX2(required_num_params,
7839 fninfo.num_sgpr_params + PS_EPILOG_SAMPLEMASK_MIN_LOC + 1);
7840
7841 while (fninfo.num_params < required_num_params)
7842 add_arg(&fninfo, ARG_VGPR, ctx->f32);
7843
7844 /* Create the function. */
7845 si_create_function(ctx, "ps_epilog", NULL, 0, &fninfo, 0);
7846 /* Disable elimination of unused inputs. */
7847 ac_llvm_add_target_dep_function_attr(ctx->main_fn,
7848 "InitialPSInputAddr", 0xffffff);
7849
7850 /* Process colors. */
7851 unsigned vgpr = fninfo.num_sgpr_params;
7852 unsigned colors_written = key->ps_epilog.colors_written;
7853 int last_color_export = -1;
7854
7855 /* Find the last color export. */
7856 if (!key->ps_epilog.writes_z &&
7857 !key->ps_epilog.writes_stencil &&
7858 !key->ps_epilog.writes_samplemask) {
7859 unsigned spi_format = key->ps_epilog.states.spi_shader_col_format;
7860
7861 /* If last_cbuf > 0, FS_COLOR0_WRITES_ALL_CBUFS is true. */
7862 if (colors_written == 0x1 && key->ps_epilog.states.last_cbuf > 0) {
7863 /* Just set this if any of the colorbuffers are enabled. */
7864 if (spi_format &
7865 ((1ull << (4 * (key->ps_epilog.states.last_cbuf + 1))) - 1))
7866 last_color_export = 0;
7867 } else {
7868 for (i = 0; i < 8; i++)
7869 if (colors_written & (1 << i) &&
7870 (spi_format >> (i * 4)) & 0xf)
7871 last_color_export = i;
7872 }
7873 }
7874
7875 while (colors_written) {
7876 LLVMValueRef color[4];
7877 int mrt = u_bit_scan(&colors_written);
7878
7879 for (i = 0; i < 4; i++)
7880 color[i] = LLVMGetParam(ctx->main_fn, vgpr++);
7881
7882 si_export_mrt_color(bld_base, color, mrt,
7883 fninfo.num_params - 1,
7884 mrt == last_color_export, &exp);
7885 }
7886
7887 /* Process depth, stencil, samplemask. */
7888 if (key->ps_epilog.writes_z)
7889 depth = LLVMGetParam(ctx->main_fn, vgpr++);
7890 if (key->ps_epilog.writes_stencil)
7891 stencil = LLVMGetParam(ctx->main_fn, vgpr++);
7892 if (key->ps_epilog.writes_samplemask)
7893 samplemask = LLVMGetParam(ctx->main_fn, vgpr++);
7894
7895 if (depth || stencil || samplemask)
7896 si_export_mrt_z(bld_base, depth, stencil, samplemask, &exp);
7897 else if (last_color_export == -1)
7898 ac_build_export_null(&ctx->ac);
7899
7900 if (exp.num)
7901 si_emit_ps_exports(ctx, &exp);
7902
7903 /* Compile. */
7904 LLVMBuildRetVoid(ctx->ac.builder);
7905 }
7906
7907 /**
7908 * Select and compile (or reuse) pixel shader parts (prolog & epilog).
7909 */
7910 static bool si_shader_select_ps_parts(struct si_screen *sscreen,
7911 struct si_compiler *compiler,
7912 struct si_shader *shader,
7913 struct pipe_debug_callback *debug)
7914 {
7915 union si_shader_part_key prolog_key;
7916 union si_shader_part_key epilog_key;
7917
7918 /* Get the prolog. */
7919 si_get_ps_prolog_key(shader, &prolog_key, true);
7920
7921 /* The prolog is a no-op if these aren't set. */
7922 if (si_need_ps_prolog(&prolog_key)) {
7923 shader->prolog =
7924 si_get_shader_part(sscreen, &sscreen->ps_prologs,
7925 PIPE_SHADER_FRAGMENT, true,
7926 &prolog_key, compiler, debug,
7927 si_build_ps_prolog_function,
7928 "Fragment Shader Prolog");
7929 if (!shader->prolog)
7930 return false;
7931 }
7932
7933 /* Get the epilog. */
7934 si_get_ps_epilog_key(shader, &epilog_key);
7935
7936 shader->epilog =
7937 si_get_shader_part(sscreen, &sscreen->ps_epilogs,
7938 PIPE_SHADER_FRAGMENT, false,
7939 &epilog_key, compiler, debug,
7940 si_build_ps_epilog_function,
7941 "Fragment Shader Epilog");
7942 if (!shader->epilog)
7943 return false;
7944
7945 /* Enable POS_FIXED_PT if polygon stippling is enabled. */
7946 if (shader->key.part.ps.prolog.poly_stipple) {
7947 shader->config.spi_ps_input_ena |= S_0286CC_POS_FIXED_PT_ENA(1);
7948 assert(G_0286CC_POS_FIXED_PT_ENA(shader->config.spi_ps_input_addr));
7949 }
7950
7951 /* Set up the enable bits for per-sample shading if needed. */
7952 if (shader->key.part.ps.prolog.force_persp_sample_interp &&
7953 (G_0286CC_PERSP_CENTER_ENA(shader->config.spi_ps_input_ena) ||
7954 G_0286CC_PERSP_CENTROID_ENA(shader->config.spi_ps_input_ena))) {
7955 shader->config.spi_ps_input_ena &= C_0286CC_PERSP_CENTER_ENA;
7956 shader->config.spi_ps_input_ena &= C_0286CC_PERSP_CENTROID_ENA;
7957 shader->config.spi_ps_input_ena |= S_0286CC_PERSP_SAMPLE_ENA(1);
7958 }
7959 if (shader->key.part.ps.prolog.force_linear_sample_interp &&
7960 (G_0286CC_LINEAR_CENTER_ENA(shader->config.spi_ps_input_ena) ||
7961 G_0286CC_LINEAR_CENTROID_ENA(shader->config.spi_ps_input_ena))) {
7962 shader->config.spi_ps_input_ena &= C_0286CC_LINEAR_CENTER_ENA;
7963 shader->config.spi_ps_input_ena &= C_0286CC_LINEAR_CENTROID_ENA;
7964 shader->config.spi_ps_input_ena |= S_0286CC_LINEAR_SAMPLE_ENA(1);
7965 }
7966 if (shader->key.part.ps.prolog.force_persp_center_interp &&
7967 (G_0286CC_PERSP_SAMPLE_ENA(shader->config.spi_ps_input_ena) ||
7968 G_0286CC_PERSP_CENTROID_ENA(shader->config.spi_ps_input_ena))) {
7969 shader->config.spi_ps_input_ena &= C_0286CC_PERSP_SAMPLE_ENA;
7970 shader->config.spi_ps_input_ena &= C_0286CC_PERSP_CENTROID_ENA;
7971 shader->config.spi_ps_input_ena |= S_0286CC_PERSP_CENTER_ENA(1);
7972 }
7973 if (shader->key.part.ps.prolog.force_linear_center_interp &&
7974 (G_0286CC_LINEAR_SAMPLE_ENA(shader->config.spi_ps_input_ena) ||
7975 G_0286CC_LINEAR_CENTROID_ENA(shader->config.spi_ps_input_ena))) {
7976 shader->config.spi_ps_input_ena &= C_0286CC_LINEAR_SAMPLE_ENA;
7977 shader->config.spi_ps_input_ena &= C_0286CC_LINEAR_CENTROID_ENA;
7978 shader->config.spi_ps_input_ena |= S_0286CC_LINEAR_CENTER_ENA(1);
7979 }
7980
7981 /* POW_W_FLOAT requires that one of the perspective weights is enabled. */
7982 if (G_0286CC_POS_W_FLOAT_ENA(shader->config.spi_ps_input_ena) &&
7983 !(shader->config.spi_ps_input_ena & 0xf)) {
7984 shader->config.spi_ps_input_ena |= S_0286CC_PERSP_CENTER_ENA(1);
7985 assert(G_0286CC_PERSP_CENTER_ENA(shader->config.spi_ps_input_addr));
7986 }
7987
7988 /* At least one pair of interpolation weights must be enabled. */
7989 if (!(shader->config.spi_ps_input_ena & 0x7f)) {
7990 shader->config.spi_ps_input_ena |= S_0286CC_LINEAR_CENTER_ENA(1);
7991 assert(G_0286CC_LINEAR_CENTER_ENA(shader->config.spi_ps_input_addr));
7992 }
7993
7994 /* Samplemask fixup requires the sample ID. */
7995 if (shader->key.part.ps.prolog.samplemask_log_ps_iter) {
7996 shader->config.spi_ps_input_ena |= S_0286CC_ANCILLARY_ENA(1);
7997 assert(G_0286CC_ANCILLARY_ENA(shader->config.spi_ps_input_addr));
7998 }
7999
8000 /* The sample mask input is always enabled, because the API shader always
8001 * passes it through to the epilog. Disable it here if it's unused.
8002 */
8003 if (!shader->key.part.ps.epilog.poly_line_smoothing &&
8004 !shader->selector->info.reads_samplemask)
8005 shader->config.spi_ps_input_ena &= C_0286CC_SAMPLE_COVERAGE_ENA;
8006
8007 return true;
8008 }
8009
8010 void si_multiwave_lds_size_workaround(struct si_screen *sscreen,
8011 unsigned *lds_size)
8012 {
8013 /* If tessellation is all offchip and on-chip GS isn't used, this
8014 * workaround is not needed.
8015 */
8016 return;
8017
8018 /* SPI barrier management bug:
8019 * Make sure we have at least 4k of LDS in use to avoid the bug.
8020 * It applies to workgroup sizes of more than one wavefront.
8021 */
8022 if (sscreen->info.family == CHIP_BONAIRE ||
8023 sscreen->info.family == CHIP_KABINI ||
8024 sscreen->info.family == CHIP_MULLINS)
8025 *lds_size = MAX2(*lds_size, 8);
8026 }
8027
8028 static void si_fix_resource_usage(struct si_screen *sscreen,
8029 struct si_shader *shader)
8030 {
8031 unsigned min_sgprs = shader->info.num_input_sgprs + 2; /* VCC */
8032
8033 shader->config.num_sgprs = MAX2(shader->config.num_sgprs, min_sgprs);
8034
8035 if (shader->selector->type == PIPE_SHADER_COMPUTE &&
8036 si_get_max_workgroup_size(shader) > 64) {
8037 si_multiwave_lds_size_workaround(sscreen,
8038 &shader->config.lds_size);
8039 }
8040 }
8041
8042 int si_shader_create(struct si_screen *sscreen, struct si_compiler *compiler,
8043 struct si_shader *shader,
8044 struct pipe_debug_callback *debug)
8045 {
8046 struct si_shader_selector *sel = shader->selector;
8047 struct si_shader *mainp = *si_get_main_shader_part(sel, &shader->key);
8048 int r;
8049
8050 /* LS, ES, VS are compiled on demand if the main part hasn't been
8051 * compiled for that stage.
8052 *
8053 * Vertex shaders are compiled on demand when a vertex fetch
8054 * workaround must be applied.
8055 */
8056 if (shader->is_monolithic) {
8057 /* Monolithic shader (compiled as a whole, has many variants,
8058 * may take a long time to compile).
8059 */
8060 r = si_compile_tgsi_shader(sscreen, compiler, shader, true, debug);
8061 if (r)
8062 return r;
8063 } else {
8064 /* The shader consists of several parts:
8065 *
8066 * - the middle part is the user shader, it has 1 variant only
8067 * and it was compiled during the creation of the shader
8068 * selector
8069 * - the prolog part is inserted at the beginning
8070 * - the epilog part is inserted at the end
8071 *
8072 * The prolog and epilog have many (but simple) variants.
8073 *
8074 * Starting with gfx9, geometry and tessellation control
8075 * shaders also contain the prolog and user shader parts of
8076 * the previous shader stage.
8077 */
8078
8079 if (!mainp)
8080 return -1;
8081
8082 /* Copy the compiled TGSI shader data over. */
8083 shader->is_binary_shared = true;
8084 shader->binary = mainp->binary;
8085 shader->config = mainp->config;
8086 shader->info.num_input_sgprs = mainp->info.num_input_sgprs;
8087 shader->info.num_input_vgprs = mainp->info.num_input_vgprs;
8088 shader->info.face_vgpr_index = mainp->info.face_vgpr_index;
8089 shader->info.ancillary_vgpr_index = mainp->info.ancillary_vgpr_index;
8090 memcpy(shader->info.vs_output_param_offset,
8091 mainp->info.vs_output_param_offset,
8092 sizeof(mainp->info.vs_output_param_offset));
8093 shader->info.uses_instanceid = mainp->info.uses_instanceid;
8094 shader->info.nr_pos_exports = mainp->info.nr_pos_exports;
8095 shader->info.nr_param_exports = mainp->info.nr_param_exports;
8096
8097 /* Select prologs and/or epilogs. */
8098 switch (sel->type) {
8099 case PIPE_SHADER_VERTEX:
8100 if (!si_shader_select_vs_parts(sscreen, compiler, shader, debug))
8101 return -1;
8102 break;
8103 case PIPE_SHADER_TESS_CTRL:
8104 if (!si_shader_select_tcs_parts(sscreen, compiler, shader, debug))
8105 return -1;
8106 break;
8107 case PIPE_SHADER_TESS_EVAL:
8108 break;
8109 case PIPE_SHADER_GEOMETRY:
8110 if (!si_shader_select_gs_parts(sscreen, compiler, shader, debug))
8111 return -1;
8112 break;
8113 case PIPE_SHADER_FRAGMENT:
8114 if (!si_shader_select_ps_parts(sscreen, compiler, shader, debug))
8115 return -1;
8116
8117 /* Make sure we have at least as many VGPRs as there
8118 * are allocated inputs.
8119 */
8120 shader->config.num_vgprs = MAX2(shader->config.num_vgprs,
8121 shader->info.num_input_vgprs);
8122 break;
8123 }
8124
8125 /* Update SGPR and VGPR counts. */
8126 if (shader->prolog) {
8127 shader->config.num_sgprs = MAX2(shader->config.num_sgprs,
8128 shader->prolog->config.num_sgprs);
8129 shader->config.num_vgprs = MAX2(shader->config.num_vgprs,
8130 shader->prolog->config.num_vgprs);
8131 }
8132 if (shader->previous_stage) {
8133 shader->config.num_sgprs = MAX2(shader->config.num_sgprs,
8134 shader->previous_stage->config.num_sgprs);
8135 shader->config.num_vgprs = MAX2(shader->config.num_vgprs,
8136 shader->previous_stage->config.num_vgprs);
8137 shader->config.spilled_sgprs =
8138 MAX2(shader->config.spilled_sgprs,
8139 shader->previous_stage->config.spilled_sgprs);
8140 shader->config.spilled_vgprs =
8141 MAX2(shader->config.spilled_vgprs,
8142 shader->previous_stage->config.spilled_vgprs);
8143 shader->config.private_mem_vgprs =
8144 MAX2(shader->config.private_mem_vgprs,
8145 shader->previous_stage->config.private_mem_vgprs);
8146 shader->config.scratch_bytes_per_wave =
8147 MAX2(shader->config.scratch_bytes_per_wave,
8148 shader->previous_stage->config.scratch_bytes_per_wave);
8149 shader->info.uses_instanceid |=
8150 shader->previous_stage->info.uses_instanceid;
8151 }
8152 if (shader->prolog2) {
8153 shader->config.num_sgprs = MAX2(shader->config.num_sgprs,
8154 shader->prolog2->config.num_sgprs);
8155 shader->config.num_vgprs = MAX2(shader->config.num_vgprs,
8156 shader->prolog2->config.num_vgprs);
8157 }
8158 if (shader->epilog) {
8159 shader->config.num_sgprs = MAX2(shader->config.num_sgprs,
8160 shader->epilog->config.num_sgprs);
8161 shader->config.num_vgprs = MAX2(shader->config.num_vgprs,
8162 shader->epilog->config.num_vgprs);
8163 }
8164 si_calculate_max_simd_waves(shader);
8165 }
8166
8167 si_fix_resource_usage(sscreen, shader);
8168 si_shader_dump(sscreen, shader, debug, sel->info.processor,
8169 stderr, true);
8170
8171 /* Upload. */
8172 r = si_shader_binary_upload(sscreen, shader);
8173 if (r) {
8174 fprintf(stderr, "LLVM failed to upload shader\n");
8175 return r;
8176 }
8177
8178 return 0;
8179 }
8180
8181 void si_shader_destroy(struct si_shader *shader)
8182 {
8183 if (shader->scratch_bo)
8184 r600_resource_reference(&shader->scratch_bo, NULL);
8185
8186 r600_resource_reference(&shader->bo, NULL);
8187
8188 if (!shader->is_binary_shared)
8189 ac_shader_binary_clean(&shader->binary);
8190
8191 free(shader->shader_log);
8192 }