projects / mesa.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
history | raw | HEAD
radeonsi: Lower TGSI_OPCODE_LOAD down to LLVM op (v3)
[mesa.git] / src / gallium / drivers / radeonsi / si_shader.c
1 /*
2 * Copyright 2012 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * on the rights to use, copy, modify, merge, publish, distribute, sub
8 * license, and/or sell copies of the Software, and to permit persons to whom
9 * the Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
19 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
20 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
21 * USE OR OTHER DEALINGS IN THE SOFTWARE.
22 *
23 * Authors:
24 * Tom Stellard <thomas.stellard@amd.com>
25 * Michel Dänzer <michel.daenzer@amd.com>
26 * Christian König <christian.koenig@amd.com>
27 */
28
29 #include "gallivm/lp_bld_const.h"
30 #include "gallivm/lp_bld_gather.h"
31 #include "gallivm/lp_bld_intr.h"
32 #include "gallivm/lp_bld_logic.h"
33 #include "gallivm/lp_bld_arit.h"
34 #include "gallivm/lp_bld_bitarit.h"
35 #include "gallivm/lp_bld_flow.h"
36 #include "radeon/r600_cs.h"
37 #include "radeon/radeon_llvm.h"
38 #include "radeon/radeon_elf_util.h"
39 #include "radeon/radeon_llvm_emit.h"
40 #include "util/u_memory.h"
41 #include "util/u_pstipple.h"
42 #include "tgsi/tgsi_parse.h"
43 #include "tgsi/tgsi_util.h"
44 #include "tgsi/tgsi_dump.h"
45
46 #include "si_pipe.h"
47 #include "si_shader.h"
48 #include "sid.h"
49
50 #include <errno.h>
51
52 static const char *scratch_rsrc_dword0_symbol =
53 "SCRATCH_RSRC_DWORD0";
54
55 static const char *scratch_rsrc_dword1_symbol =
56 "SCRATCH_RSRC_DWORD1";
57
58 struct si_shader_output_values
59 {
60 LLVMValueRef values[4];
61 unsigned name;
62 unsigned sid;
63 };
64
65 struct si_shader_context
66 {
67 struct radeon_llvm_context radeon_bld;
68 struct si_shader *shader;
69 struct si_screen *screen;
70
71 unsigned type; /* TGSI_PROCESSOR_* specifies the type of shader. */
72 bool is_gs_copy_shader;
73
74 /* Whether to generate the optimized shader variant compiled as a whole
75 * (without a prolog and epilog)
76 */
77 bool is_monolithic;
78
79 int param_streamout_config;
80 int param_streamout_write_index;
81 int param_streamout_offset[4];
82 int param_vertex_id;
83 int param_rel_auto_id;
84 int param_vs_prim_id;
85 int param_instance_id;
86 int param_vertex_index0;
87 int param_tes_u;
88 int param_tes_v;
89 int param_tes_rel_patch_id;
90 int param_tes_patch_id;
91 int param_es2gs_offset;
92
93 LLVMTargetMachineRef tm;
94
95 LLVMValueRef const_md;
96 LLVMValueRef const_buffers[SI_NUM_CONST_BUFFERS];
97 LLVMValueRef lds;
98 LLVMValueRef *constants[SI_NUM_CONST_BUFFERS];
99 LLVMValueRef sampler_views[SI_NUM_SAMPLERS];
100 LLVMValueRef sampler_states[SI_NUM_SAMPLERS];
101 LLVMValueRef fmasks[SI_NUM_USER_SAMPLERS];
102 LLVMValueRef images[SI_NUM_IMAGES];
103 LLVMValueRef so_buffers[4];
104 LLVMValueRef esgs_ring;
105 LLVMValueRef gsvs_ring[4];
106 LLVMValueRef gs_next_vertex[4];
107 LLVMValueRef return_value;
108
109 LLVMTypeRef voidt;
110 LLVMTypeRef i1;
111 LLVMTypeRef i8;
112 LLVMTypeRef i32;
113 LLVMTypeRef i64;
114 LLVMTypeRef i128;
115 LLVMTypeRef f32;
116 LLVMTypeRef v16i8;
117 LLVMTypeRef v2i32;
118 LLVMTypeRef v4i32;
119 LLVMTypeRef v4f32;
120 LLVMTypeRef v8i32;
121 };
122
123 static struct si_shader_context *si_shader_context(
124 struct lp_build_tgsi_context *bld_base)
125 {
126 return (struct si_shader_context *)bld_base;
127 }
128
129 static void si_init_shader_ctx(struct si_shader_context *ctx,
130 struct si_screen *sscreen,
131 struct si_shader *shader,
132 LLVMTargetMachineRef tm);
133
134 /* Ideally pass the sample mask input to the PS epilog as v13, which
135 * is its usual location, so that the shader doesn't have to add v_mov.
136 */
137 #define PS_EPILOG_SAMPLEMASK_MIN_LOC 13
138
139 /* The VS location of the PrimitiveID input is the same in the epilog,
140 * so that the main shader part doesn't have to move it.
141 */
142 #define VS_EPILOG_PRIMID_LOC 2
143
144 #define PERSPECTIVE_BASE 0
145 #define LINEAR_BASE 9
146
147 #define SAMPLE_OFFSET 0
148 #define CENTER_OFFSET 2
149 #define CENTROID_OFSET 4
150
151 #define USE_SGPR_MAX_SUFFIX_LEN 5
152 #define CONST_ADDR_SPACE 2
153 #define LOCAL_ADDR_SPACE 3
154 #define USER_SGPR_ADDR_SPACE 8
155
156
157 #define SENDMSG_GS 2
158 #define SENDMSG_GS_DONE 3
159
160 #define SENDMSG_GS_OP_NOP (0 << 4)
161 #define SENDMSG_GS_OP_CUT (1 << 4)
162 #define SENDMSG_GS_OP_EMIT (2 << 4)
163 #define SENDMSG_GS_OP_EMIT_CUT (3 << 4)
164
165 /**
166 * Returns a unique index for a semantic name and index. The index must be
167 * less than 64, so that a 64-bit bitmask of used inputs or outputs can be
168 * calculated.
169 */
170 unsigned si_shader_io_get_unique_index(unsigned semantic_name, unsigned index)
171 {
172 switch (semantic_name) {
173 case TGSI_SEMANTIC_POSITION:
174 return 0;
175 case TGSI_SEMANTIC_PSIZE:
176 return 1;
177 case TGSI_SEMANTIC_CLIPDIST:
178 assert(index <= 1);
179 return 2 + index;
180 case TGSI_SEMANTIC_GENERIC:
181 if (index <= 63-4)
182 return 4 + index;
183 else
184 /* same explanation as in the default statement,
185 * the only user hitting this is st/nine.
186 */
187 return 0;
188
189 /* patch indices are completely separate and thus start from 0 */
190 case TGSI_SEMANTIC_TESSOUTER:
191 return 0;
192 case TGSI_SEMANTIC_TESSINNER:
193 return 1;
194 case TGSI_SEMANTIC_PATCH:
195 return 2 + index;
196
197 default:
198 /* Don't fail here. The result of this function is only used
199 * for LS, TCS, TES, and GS, where legacy GL semantics can't
200 * occur, but this function is called for all vertex shaders
201 * before it's known whether LS will be compiled or not.
202 */
203 return 0;
204 }
205 }
206
207 /**
208 * Get the value of a shader input parameter and extract a bitfield.
209 */
210 static LLVMValueRef unpack_param(struct si_shader_context *ctx,
211 unsigned param, unsigned rshift,
212 unsigned bitwidth)
213 {
214 struct gallivm_state *gallivm = &ctx->radeon_bld.gallivm;
215 LLVMValueRef value = LLVMGetParam(ctx->radeon_bld.main_fn,
216 param);
217
218 if (LLVMGetTypeKind(LLVMTypeOf(value)) == LLVMFloatTypeKind)
219 value = bitcast(&ctx->radeon_bld.soa.bld_base,
220 TGSI_TYPE_UNSIGNED, value);
221
222 if (rshift)
223 value = LLVMBuildLShr(gallivm->builder, value,
224 lp_build_const_int32(gallivm, rshift), "");
225
226 if (rshift + bitwidth < 32) {
227 unsigned mask = (1 << bitwidth) - 1;
228 value = LLVMBuildAnd(gallivm->builder, value,
229 lp_build_const_int32(gallivm, mask), "");
230 }
231
232 return value;
233 }
234
235 static LLVMValueRef get_rel_patch_id(struct si_shader_context *ctx)
236 {
237 switch (ctx->type) {
238 case TGSI_PROCESSOR_TESS_CTRL:
239 return unpack_param(ctx, SI_PARAM_REL_IDS, 0, 8);
240
241 case TGSI_PROCESSOR_TESS_EVAL:
242 return LLVMGetParam(ctx->radeon_bld.main_fn,
243 ctx->param_tes_rel_patch_id);
244
245 default:
246 assert(0);
247 return NULL;
248 }
249 }
250
251 /* Tessellation shaders pass outputs to the next shader using LDS.
252 *
253 * LS outputs = TCS inputs
254 * TCS outputs = TES inputs
255 *
256 * The LDS layout is:
257 * - TCS inputs for patch 0
258 * - TCS inputs for patch 1
259 * - TCS inputs for patch 2 = get_tcs_in_current_patch_offset (if RelPatchID==2)
260 * - ...
261 * - TCS outputs for patch 0 = get_tcs_out_patch0_offset
262 * - Per-patch TCS outputs for patch 0 = get_tcs_out_patch0_patch_data_offset
263 * - TCS outputs for patch 1
264 * - Per-patch TCS outputs for patch 1
265 * - TCS outputs for patch 2 = get_tcs_out_current_patch_offset (if RelPatchID==2)
266 * - Per-patch TCS outputs for patch 2 = get_tcs_out_current_patch_data_offset (if RelPatchID==2)
267 * - ...
268 *
269 * All three shaders VS(LS), TCS, TES share the same LDS space.
270 */
271
272 static LLVMValueRef
273 get_tcs_in_patch_stride(struct si_shader_context *ctx)
274 {
275 if (ctx->type == TGSI_PROCESSOR_VERTEX)
276 return unpack_param(ctx, SI_PARAM_LS_OUT_LAYOUT, 0, 13);
277 else if (ctx->type == TGSI_PROCESSOR_TESS_CTRL)
278 return unpack_param(ctx, SI_PARAM_TCS_IN_LAYOUT, 0, 13);
279 else {
280 assert(0);
281 return NULL;
282 }
283 }
284
285 static LLVMValueRef
286 get_tcs_out_patch_stride(struct si_shader_context *ctx)
287 {
288 return unpack_param(ctx, SI_PARAM_TCS_OUT_LAYOUT, 0, 13);
289 }
290
291 static LLVMValueRef
292 get_tcs_out_patch0_offset(struct si_shader_context *ctx)
293 {
294 return lp_build_mul_imm(&ctx->radeon_bld.soa.bld_base.uint_bld,
295 unpack_param(ctx,
296 SI_PARAM_TCS_OUT_OFFSETS,
297 0, 16),
298 4);
299 }
300
301 static LLVMValueRef
302 get_tcs_out_patch0_patch_data_offset(struct si_shader_context *ctx)
303 {
304 return lp_build_mul_imm(&ctx->radeon_bld.soa.bld_base.uint_bld,
305 unpack_param(ctx,
306 SI_PARAM_TCS_OUT_OFFSETS,
307 16, 16),
308 4);
309 }
310
311 static LLVMValueRef
312 get_tcs_in_current_patch_offset(struct si_shader_context *ctx)
313 {
314 struct gallivm_state *gallivm = &ctx->radeon_bld.gallivm;
315 LLVMValueRef patch_stride = get_tcs_in_patch_stride(ctx);
316 LLVMValueRef rel_patch_id = get_rel_patch_id(ctx);
317
318 return LLVMBuildMul(gallivm->builder, patch_stride, rel_patch_id, "");
319 }
320
321 static LLVMValueRef
322 get_tcs_out_current_patch_offset(struct si_shader_context *ctx)
323 {
324 struct gallivm_state *gallivm = &ctx->radeon_bld.gallivm;
325 LLVMValueRef patch0_offset = get_tcs_out_patch0_offset(ctx);
326 LLVMValueRef patch_stride = get_tcs_out_patch_stride(ctx);
327 LLVMValueRef rel_patch_id = get_rel_patch_id(ctx);
328
329 return LLVMBuildAdd(gallivm->builder, patch0_offset,
330 LLVMBuildMul(gallivm->builder, patch_stride,
331 rel_patch_id, ""),
332 "");
333 }
334
335 static LLVMValueRef
336 get_tcs_out_current_patch_data_offset(struct si_shader_context *ctx)
337 {
338 struct gallivm_state *gallivm = &ctx->radeon_bld.gallivm;
339 LLVMValueRef patch0_patch_data_offset =
340 get_tcs_out_patch0_patch_data_offset(ctx);
341 LLVMValueRef patch_stride = get_tcs_out_patch_stride(ctx);
342 LLVMValueRef rel_patch_id = get_rel_patch_id(ctx);
343
344 return LLVMBuildAdd(gallivm->builder, patch0_patch_data_offset,
345 LLVMBuildMul(gallivm->builder, patch_stride,
346 rel_patch_id, ""),
347 "");
348 }
349
350 static void build_indexed_store(struct si_shader_context *ctx,
351 LLVMValueRef base_ptr, LLVMValueRef index,
352 LLVMValueRef value)
353 {
354 struct lp_build_tgsi_context *bld_base = &ctx->radeon_bld.soa.bld_base;
355 struct gallivm_state *gallivm = bld_base->base.gallivm;
356 LLVMValueRef indices[2], pointer;
357
358 indices[0] = bld_base->uint_bld.zero;
359 indices[1] = index;
360
361 pointer = LLVMBuildGEP(gallivm->builder, base_ptr, indices, 2, "");
362 LLVMBuildStore(gallivm->builder, value, pointer);
363 }
364
365 /**
366 * Build an LLVM bytecode indexed load using LLVMBuildGEP + LLVMBuildLoad.
367 * It's equivalent to doing a load from &base_ptr[index].
368 *
369 * \param base_ptr Where the array starts.
370 * \param index The element index into the array.
371 */
372 static LLVMValueRef build_indexed_load(struct si_shader_context *ctx,
373 LLVMValueRef base_ptr, LLVMValueRef index)
374 {
375 struct lp_build_tgsi_context *bld_base = &ctx->radeon_bld.soa.bld_base;
376 struct gallivm_state *gallivm = bld_base->base.gallivm;
377 LLVMValueRef indices[2], pointer;
378
379 indices[0] = bld_base->uint_bld.zero;
380 indices[1] = index;
381
382 pointer = LLVMBuildGEP(gallivm->builder, base_ptr, indices, 2, "");
383 return LLVMBuildLoad(gallivm->builder, pointer, "");
384 }
385
386 /**
387 * Do a load from &base_ptr[index], but also add a flag that it's loading
388 * a constant.
389 */
390 static LLVMValueRef build_indexed_load_const(
391 struct si_shader_context *ctx,
392 LLVMValueRef base_ptr, LLVMValueRef index)
393 {
394 LLVMValueRef result = build_indexed_load(ctx, base_ptr, index);
395 LLVMSetMetadata(result, 1, ctx->const_md);
396 return result;
397 }
398
399 static LLVMValueRef get_instance_index_for_fetch(
400 struct radeon_llvm_context *radeon_bld,
401 unsigned param_start_instance, unsigned divisor)
402 {
403 struct si_shader_context *ctx =
404 si_shader_context(&radeon_bld->soa.bld_base);
405 struct gallivm_state *gallivm = radeon_bld->soa.bld_base.base.gallivm;
406
407 LLVMValueRef result = LLVMGetParam(radeon_bld->main_fn,
408 ctx->param_instance_id);
409
410 /* The division must be done before START_INSTANCE is added. */
411 if (divisor > 1)
412 result = LLVMBuildUDiv(gallivm->builder, result,
413 lp_build_const_int32(gallivm, divisor), "");
414
415 return LLVMBuildAdd(gallivm->builder, result,
416 LLVMGetParam(radeon_bld->main_fn, param_start_instance), "");
417 }
418
419 static void declare_input_vs(
420 struct radeon_llvm_context *radeon_bld,
421 unsigned input_index,
422 const struct tgsi_full_declaration *decl)
423 {
424 struct lp_build_context *base = &radeon_bld->soa.bld_base.base;
425 struct gallivm_state *gallivm = base->gallivm;
426 struct si_shader_context *ctx =
427 si_shader_context(&radeon_bld->soa.bld_base);
428 unsigned divisor =
429 ctx->shader->key.vs.prolog.instance_divisors[input_index];
430
431 unsigned chan;
432
433 LLVMValueRef t_list_ptr;
434 LLVMValueRef t_offset;
435 LLVMValueRef t_list;
436 LLVMValueRef attribute_offset;
437 LLVMValueRef buffer_index;
438 LLVMValueRef args[3];
439 LLVMValueRef input;
440
441 /* Load the T list */
442 t_list_ptr = LLVMGetParam(ctx->radeon_bld.main_fn, SI_PARAM_VERTEX_BUFFERS);
443
444 t_offset = lp_build_const_int32(gallivm, input_index);
445
446 t_list = build_indexed_load_const(ctx, t_list_ptr, t_offset);
447
448 /* Build the attribute offset */
449 attribute_offset = lp_build_const_int32(gallivm, 0);
450
451 if (!ctx->is_monolithic) {
452 buffer_index = LLVMGetParam(radeon_bld->main_fn,
453 ctx->param_vertex_index0 +
454 input_index);
455 } else if (divisor) {
456 /* Build index from instance ID, start instance and divisor */
457 ctx->shader->info.uses_instanceid = true;
458 buffer_index = get_instance_index_for_fetch(&ctx->radeon_bld,
459 SI_PARAM_START_INSTANCE,
460 divisor);
461 } else {
462 /* Load the buffer index for vertices. */
463 LLVMValueRef vertex_id = LLVMGetParam(ctx->radeon_bld.main_fn,
464 ctx->param_vertex_id);
465 LLVMValueRef base_vertex = LLVMGetParam(radeon_bld->main_fn,
466 SI_PARAM_BASE_VERTEX);
467 buffer_index = LLVMBuildAdd(gallivm->builder, base_vertex, vertex_id, "");
468 }
469
470 args[0] = t_list;
471 args[1] = attribute_offset;
472 args[2] = buffer_index;
473 input = lp_build_intrinsic(gallivm->builder,
474 "llvm.SI.vs.load.input", ctx->v4f32, args, 3,
475 LLVMReadNoneAttribute | LLVMNoUnwindAttribute);
476
477 /* Break up the vec4 into individual components */
478 for (chan = 0; chan < 4; chan++) {
479 LLVMValueRef llvm_chan = lp_build_const_int32(gallivm, chan);
480 /* XXX: Use a helper function for this. There is one in
481 * tgsi_llvm.c. */
482 ctx->radeon_bld.inputs[radeon_llvm_reg_index_soa(input_index, chan)] =
483 LLVMBuildExtractElement(gallivm->builder,
484 input, llvm_chan, "");
485 }
486 }
487
488 static LLVMValueRef get_primitive_id(struct lp_build_tgsi_context *bld_base,
489 unsigned swizzle)
490 {
491 struct si_shader_context *ctx = si_shader_context(bld_base);
492
493 if (swizzle > 0)
494 return bld_base->uint_bld.zero;
495
496 switch (ctx->type) {
497 case TGSI_PROCESSOR_VERTEX:
498 return LLVMGetParam(ctx->radeon_bld.main_fn,
499 ctx->param_vs_prim_id);
500 case TGSI_PROCESSOR_TESS_CTRL:
501 return LLVMGetParam(ctx->radeon_bld.main_fn,
502 SI_PARAM_PATCH_ID);
503 case TGSI_PROCESSOR_TESS_EVAL:
504 return LLVMGetParam(ctx->radeon_bld.main_fn,
505 ctx->param_tes_patch_id);
506 case TGSI_PROCESSOR_GEOMETRY:
507 return LLVMGetParam(ctx->radeon_bld.main_fn,
508 SI_PARAM_PRIMITIVE_ID);
509 default:
510 assert(0);
511 return bld_base->uint_bld.zero;
512 }
513 }
514
515 /**
516 * Return the value of tgsi_ind_register for indexing.
517 * This is the indirect index with the constant offset added to it.
518 */
519 static LLVMValueRef get_indirect_index(struct si_shader_context *ctx,
520 const struct tgsi_ind_register *ind,
521 int rel_index)
522 {
523 struct gallivm_state *gallivm = ctx->radeon_bld.soa.bld_base.base.gallivm;
524 LLVMValueRef result;
525
526 result = ctx->radeon_bld.soa.addr[ind->Index][ind->Swizzle];
527 result = LLVMBuildLoad(gallivm->builder, result, "");
528 result = LLVMBuildAdd(gallivm->builder, result,
529 lp_build_const_int32(gallivm, rel_index), "");
530 return result;
531 }
532
533 /**
534 * Calculate a dword address given an input or output register and a stride.
535 */
536 static LLVMValueRef get_dw_address(struct si_shader_context *ctx,
537 const struct tgsi_full_dst_register *dst,
538 const struct tgsi_full_src_register *src,
539 LLVMValueRef vertex_dw_stride,
540 LLVMValueRef base_addr)
541 {
542 struct gallivm_state *gallivm = ctx->radeon_bld.soa.bld_base.base.gallivm;
543 struct tgsi_shader_info *info = &ctx->shader->selector->info;
544 ubyte *name, *index, *array_first;
545 int first, param;
546 struct tgsi_full_dst_register reg;
547
548 /* Set the register description. The address computation is the same
549 * for sources and destinations. */
550 if (src) {
551 reg.Register.File = src->Register.File;
552 reg.Register.Index = src->Register.Index;
553 reg.Register.Indirect = src->Register.Indirect;
554 reg.Register.Dimension = src->Register.Dimension;
555 reg.Indirect = src->Indirect;
556 reg.Dimension = src->Dimension;
557 reg.DimIndirect = src->DimIndirect;
558 } else
559 reg = *dst;
560
561 /* If the register is 2-dimensional (e.g. an array of vertices
562 * in a primitive), calculate the base address of the vertex. */
563 if (reg.Register.Dimension) {
564 LLVMValueRef index;
565
566 if (reg.Dimension.Indirect)
567 index = get_indirect_index(ctx, &reg.DimIndirect,
568 reg.Dimension.Index);
569 else
570 index = lp_build_const_int32(gallivm, reg.Dimension.Index);
571
572 base_addr = LLVMBuildAdd(gallivm->builder, base_addr,
573 LLVMBuildMul(gallivm->builder, index,
574 vertex_dw_stride, ""), "");
575 }
576
577 /* Get information about the register. */
578 if (reg.Register.File == TGSI_FILE_INPUT) {
579 name = info->input_semantic_name;
580 index = info->input_semantic_index;
581 array_first = info->input_array_first;
582 } else if (reg.Register.File == TGSI_FILE_OUTPUT) {
583 name = info->output_semantic_name;
584 index = info->output_semantic_index;
585 array_first = info->output_array_first;
586 } else {
587 assert(0);
588 return NULL;
589 }
590
591 if (reg.Register.Indirect) {
592 /* Add the relative address of the element. */
593 LLVMValueRef ind_index;
594
595 if (reg.Indirect.ArrayID)
596 first = array_first[reg.Indirect.ArrayID];
597 else
598 first = reg.Register.Index;
599
600 ind_index = get_indirect_index(ctx, &reg.Indirect,
601 reg.Register.Index - first);
602
603 base_addr = LLVMBuildAdd(gallivm->builder, base_addr,
604 LLVMBuildMul(gallivm->builder, ind_index,
605 lp_build_const_int32(gallivm, 4), ""), "");
606
607 param = si_shader_io_get_unique_index(name[first], index[first]);
608 } else {
609 param = si_shader_io_get_unique_index(name[reg.Register.Index],
610 index[reg.Register.Index]);
611 }
612
613 /* Add the base address of the element. */
614 return LLVMBuildAdd(gallivm->builder, base_addr,
615 lp_build_const_int32(gallivm, param * 4), "");
616 }
617
618 /**
619 * Load from LDS.
620 *
621 * \param type output value type
622 * \param swizzle offset (typically 0..3); it can be ~0, which loads a vec4
623 * \param dw_addr address in dwords
624 */
625 static LLVMValueRef lds_load(struct lp_build_tgsi_context *bld_base,
626 enum tgsi_opcode_type type, unsigned swizzle,
627 LLVMValueRef dw_addr)
628 {
629 struct si_shader_context *ctx = si_shader_context(bld_base);
630 struct gallivm_state *gallivm = bld_base->base.gallivm;
631 LLVMValueRef value;
632
633 if (swizzle == ~0) {
634 LLVMValueRef values[TGSI_NUM_CHANNELS];
635
636 for (unsigned chan = 0; chan < TGSI_NUM_CHANNELS; chan++)
637 values[chan] = lds_load(bld_base, type, chan, dw_addr);
638
639 return lp_build_gather_values(bld_base->base.gallivm, values,
640 TGSI_NUM_CHANNELS);
641 }
642
643 dw_addr = lp_build_add(&bld_base->uint_bld, dw_addr,
644 lp_build_const_int32(gallivm, swizzle));
645
646 value = build_indexed_load(ctx, ctx->lds, dw_addr);
647 if (type == TGSI_TYPE_DOUBLE) {
648 LLVMValueRef value2;
649 dw_addr = lp_build_add(&bld_base->uint_bld, dw_addr,
650 lp_build_const_int32(gallivm, swizzle + 1));
651 value2 = build_indexed_load(ctx, ctx->lds, dw_addr);
652 return radeon_llvm_emit_fetch_double(bld_base, value, value2);
653 }
654
655 return LLVMBuildBitCast(gallivm->builder, value,
656 tgsi2llvmtype(bld_base, type), "");
657 }
658
659 /**
660 * Store to LDS.
661 *
662 * \param swizzle offset (typically 0..3)
663 * \param dw_addr address in dwords
664 * \param value value to store
665 */
666 static void lds_store(struct lp_build_tgsi_context *bld_base,
667 unsigned swizzle, LLVMValueRef dw_addr,
668 LLVMValueRef value)
669 {
670 struct si_shader_context *ctx = si_shader_context(bld_base);
671 struct gallivm_state *gallivm = bld_base->base.gallivm;
672
673 dw_addr = lp_build_add(&bld_base->uint_bld, dw_addr,
674 lp_build_const_int32(gallivm, swizzle));
675
676 value = LLVMBuildBitCast(gallivm->builder, value, ctx->i32, "");
677 build_indexed_store(ctx, ctx->lds,
678 dw_addr, value);
679 }
680
681 static LLVMValueRef fetch_input_tcs(
682 struct lp_build_tgsi_context *bld_base,
683 const struct tgsi_full_src_register *reg,
684 enum tgsi_opcode_type type, unsigned swizzle)
685 {
686 struct si_shader_context *ctx = si_shader_context(bld_base);
687 LLVMValueRef dw_addr, stride;
688
689 stride = unpack_param(ctx, SI_PARAM_TCS_IN_LAYOUT, 13, 8);
690 dw_addr = get_tcs_in_current_patch_offset(ctx);
691 dw_addr = get_dw_address(ctx, NULL, reg, stride, dw_addr);
692
693 return lds_load(bld_base, type, swizzle, dw_addr);
694 }
695
696 static LLVMValueRef fetch_output_tcs(
697 struct lp_build_tgsi_context *bld_base,
698 const struct tgsi_full_src_register *reg,
699 enum tgsi_opcode_type type, unsigned swizzle)
700 {
701 struct si_shader_context *ctx = si_shader_context(bld_base);
702 LLVMValueRef dw_addr, stride;
703
704 if (reg->Register.Dimension) {
705 stride = unpack_param(ctx, SI_PARAM_TCS_OUT_LAYOUT, 13, 8);
706 dw_addr = get_tcs_out_current_patch_offset(ctx);
707 dw_addr = get_dw_address(ctx, NULL, reg, stride, dw_addr);
708 } else {
709 dw_addr = get_tcs_out_current_patch_data_offset(ctx);
710 dw_addr = get_dw_address(ctx, NULL, reg, NULL, dw_addr);
711 }
712
713 return lds_load(bld_base, type, swizzle, dw_addr);
714 }
715
716 static LLVMValueRef fetch_input_tes(
717 struct lp_build_tgsi_context *bld_base,
718 const struct tgsi_full_src_register *reg,
719 enum tgsi_opcode_type type, unsigned swizzle)
720 {
721 struct si_shader_context *ctx = si_shader_context(bld_base);
722 LLVMValueRef dw_addr, stride;
723
724 if (reg->Register.Dimension) {
725 stride = unpack_param(ctx, SI_PARAM_TCS_OUT_LAYOUT, 13, 8);
726 dw_addr = get_tcs_out_current_patch_offset(ctx);
727 dw_addr = get_dw_address(ctx, NULL, reg, stride, dw_addr);
728 } else {
729 dw_addr = get_tcs_out_current_patch_data_offset(ctx);
730 dw_addr = get_dw_address(ctx, NULL, reg, NULL, dw_addr);
731 }
732
733 return lds_load(bld_base, type, swizzle, dw_addr);
734 }
735
736 static void store_output_tcs(struct lp_build_tgsi_context *bld_base,
737 const struct tgsi_full_instruction *inst,
738 const struct tgsi_opcode_info *info,
739 LLVMValueRef dst[4])
740 {
741 struct si_shader_context *ctx = si_shader_context(bld_base);
742 const struct tgsi_full_dst_register *reg = &inst->Dst[0];
743 unsigned chan_index;
744 LLVMValueRef dw_addr, stride;
745
746 /* Only handle per-patch and per-vertex outputs here.
747 * Vectors will be lowered to scalars and this function will be called again.
748 */
749 if (reg->Register.File != TGSI_FILE_OUTPUT ||
750 (dst[0] && LLVMGetTypeKind(LLVMTypeOf(dst[0])) == LLVMVectorTypeKind)) {
751 radeon_llvm_emit_store(bld_base, inst, info, dst);
752 return;
753 }
754
755 if (reg->Register.Dimension) {
756 stride = unpack_param(ctx, SI_PARAM_TCS_OUT_LAYOUT, 13, 8);
757 dw_addr = get_tcs_out_current_patch_offset(ctx);
758 dw_addr = get_dw_address(ctx, reg, NULL, stride, dw_addr);
759 } else {
760 dw_addr = get_tcs_out_current_patch_data_offset(ctx);
761 dw_addr = get_dw_address(ctx, reg, NULL, NULL, dw_addr);
762 }
763
764 TGSI_FOR_EACH_DST0_ENABLED_CHANNEL(inst, chan_index) {
765 LLVMValueRef value = dst[chan_index];
766
767 if (inst->Instruction.Saturate)
768 value = radeon_llvm_saturate(bld_base, value);
769
770 lds_store(bld_base, chan_index, dw_addr, value);
771 }
772 }
773
774 static LLVMValueRef fetch_input_gs(
775 struct lp_build_tgsi_context *bld_base,
776 const struct tgsi_full_src_register *reg,
777 enum tgsi_opcode_type type,
778 unsigned swizzle)
779 {
780 struct lp_build_context *base = &bld_base->base;
781 struct si_shader_context *ctx = si_shader_context(bld_base);
782 struct si_shader *shader = ctx->shader;
783 struct lp_build_context *uint = &ctx->radeon_bld.soa.bld_base.uint_bld;
784 struct gallivm_state *gallivm = base->gallivm;
785 LLVMValueRef vtx_offset;
786 LLVMValueRef args[9];
787 unsigned vtx_offset_param;
788 struct tgsi_shader_info *info = &shader->selector->info;
789 unsigned semantic_name = info->input_semantic_name[reg->Register.Index];
790 unsigned semantic_index = info->input_semantic_index[reg->Register.Index];
791 unsigned param;
792 LLVMValueRef value;
793
794 if (swizzle != ~0 && semantic_name == TGSI_SEMANTIC_PRIMID)
795 return get_primitive_id(bld_base, swizzle);
796
797 if (!reg->Register.Dimension)
798 return NULL;
799
800 if (swizzle == ~0) {
801 LLVMValueRef values[TGSI_NUM_CHANNELS];
802 unsigned chan;
803 for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) {
804 values[chan] = fetch_input_gs(bld_base, reg, type, chan);
805 }
806 return lp_build_gather_values(bld_base->base.gallivm, values,
807 TGSI_NUM_CHANNELS);
808 }
809
810 /* Get the vertex offset parameter */
811 vtx_offset_param = reg->Dimension.Index;
812 if (vtx_offset_param < 2) {
813 vtx_offset_param += SI_PARAM_VTX0_OFFSET;
814 } else {
815 assert(vtx_offset_param < 6);
816 vtx_offset_param += SI_PARAM_VTX2_OFFSET - 2;
817 }
818 vtx_offset = lp_build_mul_imm(uint,
819 LLVMGetParam(ctx->radeon_bld.main_fn,
820 vtx_offset_param),
821 4);
822
823 param = si_shader_io_get_unique_index(semantic_name, semantic_index);
824 args[0] = ctx->esgs_ring;
825 args[1] = vtx_offset;
826 args[2] = lp_build_const_int32(gallivm, (param * 4 + swizzle) * 256);
827 args[3] = uint->zero;
828 args[4] = uint->one; /* OFFEN */
829 args[5] = uint->zero; /* IDXEN */
830 args[6] = uint->one; /* GLC */
831 args[7] = uint->zero; /* SLC */
832 args[8] = uint->zero; /* TFE */
833
834 value = lp_build_intrinsic(gallivm->builder,
835 "llvm.SI.buffer.load.dword.i32.i32",
836 ctx->i32, args, 9,
837 LLVMReadOnlyAttribute | LLVMNoUnwindAttribute);
838 if (type == TGSI_TYPE_DOUBLE) {
839 LLVMValueRef value2;
840 args[2] = lp_build_const_int32(gallivm, (param * 4 + swizzle + 1) * 256);
841 value2 = lp_build_intrinsic(gallivm->builder,
842 "llvm.SI.buffer.load.dword.i32.i32",
843 ctx->i32, args, 9,
844 LLVMReadOnlyAttribute | LLVMNoUnwindAttribute);
845 return radeon_llvm_emit_fetch_double(bld_base,
846 value, value2);
847 }
848 return LLVMBuildBitCast(gallivm->builder,
849 value,
850 tgsi2llvmtype(bld_base, type), "");
851 }
852
853 static int lookup_interp_param_index(unsigned interpolate, unsigned location)
854 {
855 switch (interpolate) {
856 case TGSI_INTERPOLATE_CONSTANT:
857 return 0;
858
859 case TGSI_INTERPOLATE_LINEAR:
860 if (location == TGSI_INTERPOLATE_LOC_SAMPLE)
861 return SI_PARAM_LINEAR_SAMPLE;
862 else if (location == TGSI_INTERPOLATE_LOC_CENTROID)
863 return SI_PARAM_LINEAR_CENTROID;
864 else
865 return SI_PARAM_LINEAR_CENTER;
866 break;
867 case TGSI_INTERPOLATE_COLOR:
868 case TGSI_INTERPOLATE_PERSPECTIVE:
869 if (location == TGSI_INTERPOLATE_LOC_SAMPLE)
870 return SI_PARAM_PERSP_SAMPLE;
871 else if (location == TGSI_INTERPOLATE_LOC_CENTROID)
872 return SI_PARAM_PERSP_CENTROID;
873 else
874 return SI_PARAM_PERSP_CENTER;
875 break;
876 default:
877 fprintf(stderr, "Warning: Unhandled interpolation mode.\n");
878 return -1;
879 }
880 }
881
882 /* This shouldn't be used by explicit INTERP opcodes. */
883 static unsigned select_interp_param(struct si_shader_context *ctx,
884 unsigned param)
885 {
886 if (!ctx->shader->key.ps.prolog.force_persample_interp ||
887 !ctx->is_monolithic)
888 return param;
889
890 /* If the shader doesn't use center/centroid, just return the parameter.
891 *
892 * If the shader only uses one set of (i,j), "si_emit_spi_ps_input" can
893 * switch between center/centroid and sample without shader changes.
894 */
895 switch (param) {
896 case SI_PARAM_PERSP_CENTROID:
897 case SI_PARAM_PERSP_CENTER:
898 return SI_PARAM_PERSP_SAMPLE;
899
900 case SI_PARAM_LINEAR_CENTROID:
901 case SI_PARAM_LINEAR_CENTER:
902 return SI_PARAM_LINEAR_SAMPLE;
903
904 default:
905 return param;
906 }
907 }
908
909 /**
910 * Interpolate a fragment shader input.
911 *
912 * @param ctx context
913 * @param input_index index of the input in hardware
914 * @param semantic_name TGSI_SEMANTIC_*
915 * @param semantic_index semantic index
916 * @param num_interp_inputs number of all interpolated inputs (= BCOLOR offset)
917 * @param colors_read_mask color components read (4 bits for each color, 8 bits in total)
918 * @param interp_param interpolation weights (i,j)
919 * @param prim_mask SI_PARAM_PRIM_MASK
920 * @param face SI_PARAM_FRONT_FACE
921 * @param result the return value (4 components)
922 */
923 static void interp_fs_input(struct si_shader_context *ctx,
924 unsigned input_index,
925 unsigned semantic_name,
926 unsigned semantic_index,
927 unsigned num_interp_inputs,
928 unsigned colors_read_mask,
929 LLVMValueRef interp_param,
930 LLVMValueRef prim_mask,
931 LLVMValueRef face,
932 LLVMValueRef result[4])
933 {
934 struct lp_build_context *base = &ctx->radeon_bld.soa.bld_base.base;
935 struct lp_build_context *uint = &ctx->radeon_bld.soa.bld_base.uint_bld;
936 struct gallivm_state *gallivm = base->gallivm;
937 const char *intr_name;
938 LLVMValueRef attr_number;
939
940 unsigned chan;
941
942 attr_number = lp_build_const_int32(gallivm, input_index);
943
944 /* fs.constant returns the param from the middle vertex, so it's not
945 * really useful for flat shading. It's meant to be used for custom
946 * interpolation (but the intrinsic can't fetch from the other two
947 * vertices).
948 *
949 * Luckily, it doesn't matter, because we rely on the FLAT_SHADE state
950 * to do the right thing. The only reason we use fs.constant is that
951 * fs.interp cannot be used on integers, because they can be equal
952 * to NaN.
953 */
954 intr_name = interp_param ? "llvm.SI.fs.interp" : "llvm.SI.fs.constant";
955
956 if (semantic_name == TGSI_SEMANTIC_COLOR &&
957 ctx->shader->key.ps.prolog.color_two_side) {
958 LLVMValueRef args[4];
959 LLVMValueRef is_face_positive;
960 LLVMValueRef back_attr_number;
961
962 /* If BCOLOR0 is used, BCOLOR1 is at offset "num_inputs + 1",
963 * otherwise it's at offset "num_inputs".
964 */
965 unsigned back_attr_offset = num_interp_inputs;
966 if (semantic_index == 1 && colors_read_mask & 0xf)
967 back_attr_offset += 1;
968
969 back_attr_number = lp_build_const_int32(gallivm, back_attr_offset);
970
971 is_face_positive = LLVMBuildICmp(gallivm->builder, LLVMIntNE,
972 face, uint->zero, "");
973
974 args[2] = prim_mask;
975 args[3] = interp_param;
976 for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) {
977 LLVMValueRef llvm_chan = lp_build_const_int32(gallivm, chan);
978 LLVMValueRef front, back;
979
980 args[0] = llvm_chan;
981 args[1] = attr_number;
982 front = lp_build_intrinsic(gallivm->builder, intr_name,
983 ctx->f32, args, args[3] ? 4 : 3,
984 LLVMReadNoneAttribute | LLVMNoUnwindAttribute);
985
986 args[1] = back_attr_number;
987 back = lp_build_intrinsic(gallivm->builder, intr_name,
988 ctx->f32, args, args[3] ? 4 : 3,
989 LLVMReadNoneAttribute | LLVMNoUnwindAttribute);
990
991 result[chan] = LLVMBuildSelect(gallivm->builder,
992 is_face_positive,
993 front,
994 back,
995 "");
996 }
997 } else if (semantic_name == TGSI_SEMANTIC_FOG) {
998 LLVMValueRef args[4];
999
1000 args[0] = uint->zero;
1001 args[1] = attr_number;
1002 args[2] = prim_mask;
1003 args[3] = interp_param;
1004 result[0] = lp_build_intrinsic(gallivm->builder, intr_name,
1005 ctx->f32, args, args[3] ? 4 : 3,
1006 LLVMReadNoneAttribute | LLVMNoUnwindAttribute);
1007 result[1] =
1008 result[2] = lp_build_const_float(gallivm, 0.0f);
1009 result[3] = lp_build_const_float(gallivm, 1.0f);
1010 } else {
1011 for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) {
1012 LLVMValueRef args[4];
1013 LLVMValueRef llvm_chan = lp_build_const_int32(gallivm, chan);
1014
1015 args[0] = llvm_chan;
1016 args[1] = attr_number;
1017 args[2] = prim_mask;
1018 args[3] = interp_param;
1019 result[chan] = lp_build_intrinsic(gallivm->builder, intr_name,
1020 ctx->f32, args, args[3] ? 4 : 3,
1021 LLVMReadNoneAttribute | LLVMNoUnwindAttribute);
1022 }
1023 }
1024 }
1025
1026 static void declare_input_fs(
1027 struct radeon_llvm_context *radeon_bld,
1028 unsigned input_index,
1029 const struct tgsi_full_declaration *decl)
1030 {
1031 struct lp_build_context *base = &radeon_bld->soa.bld_base.base;
1032 struct si_shader_context *ctx =
1033 si_shader_context(&radeon_bld->soa.bld_base);
1034 struct si_shader *shader = ctx->shader;
1035 LLVMValueRef main_fn = radeon_bld->main_fn;
1036 LLVMValueRef interp_param = NULL;
1037 int interp_param_idx;
1038
1039 /* Get colors from input VGPRs (set by the prolog). */
1040 if (!ctx->is_monolithic &&
1041 decl->Semantic.Name == TGSI_SEMANTIC_COLOR) {
1042 unsigned i = decl->Semantic.Index;
1043 unsigned colors_read = shader->selector->info.colors_read;
1044 unsigned mask = colors_read >> (i * 4);
1045 unsigned offset = SI_PARAM_POS_FIXED_PT + 1 +
1046 (i ? util_bitcount(colors_read & 0xf) : 0);
1047
1048 radeon_bld->inputs[radeon_llvm_reg_index_soa(input_index, 0)] =
1049 mask & 0x1 ? LLVMGetParam(main_fn, offset++) : base->undef;
1050 radeon_bld->inputs[radeon_llvm_reg_index_soa(input_index, 1)] =
1051 mask & 0x2 ? LLVMGetParam(main_fn, offset++) : base->undef;
1052 radeon_bld->inputs[radeon_llvm_reg_index_soa(input_index, 2)] =
1053 mask & 0x4 ? LLVMGetParam(main_fn, offset++) : base->undef;
1054 radeon_bld->inputs[radeon_llvm_reg_index_soa(input_index, 3)] =
1055 mask & 0x8 ? LLVMGetParam(main_fn, offset++) : base->undef;
1056 return;
1057 }
1058
1059 interp_param_idx = lookup_interp_param_index(decl->Interp.Interpolate,
1060 decl->Interp.Location);
1061 if (interp_param_idx == -1)
1062 return;
1063 else if (interp_param_idx) {
1064 interp_param_idx = select_interp_param(ctx,
1065 interp_param_idx);
1066 interp_param = LLVMGetParam(main_fn, interp_param_idx);
1067 }
1068
1069 interp_fs_input(ctx, input_index, decl->Semantic.Name,
1070 decl->Semantic.Index, shader->selector->info.num_inputs,
1071 shader->selector->info.colors_read, interp_param,
1072 LLVMGetParam(main_fn, SI_PARAM_PRIM_MASK),
1073 LLVMGetParam(main_fn, SI_PARAM_FRONT_FACE),
1074 &radeon_bld->inputs[radeon_llvm_reg_index_soa(input_index, 0)]);
1075 }
1076
1077 static LLVMValueRef get_sample_id(struct radeon_llvm_context *radeon_bld)
1078 {
1079 return unpack_param(si_shader_context(&radeon_bld->soa.bld_base),
1080 SI_PARAM_ANCILLARY, 8, 4);
1081 }
1082
1083 /**
1084 * Load a dword from a constant buffer.
1085 */
1086 static LLVMValueRef buffer_load_const(LLVMBuilderRef builder, LLVMValueRef resource,
1087 LLVMValueRef offset, LLVMTypeRef return_type)
1088 {
1089 LLVMValueRef args[2] = {resource, offset};
1090
1091 return lp_build_intrinsic(builder, "llvm.SI.load.const", return_type, args, 2,
1092 LLVMReadNoneAttribute | LLVMNoUnwindAttribute);
1093 }
1094
1095 static LLVMValueRef load_sample_position(struct radeon_llvm_context *radeon_bld, LLVMValueRef sample_id)
1096 {
1097 struct si_shader_context *ctx =
1098 si_shader_context(&radeon_bld->soa.bld_base);
1099 struct lp_build_context *uint_bld = &radeon_bld->soa.bld_base.uint_bld;
1100 struct gallivm_state *gallivm = &radeon_bld->gallivm;
1101 LLVMBuilderRef builder = gallivm->builder;
1102 LLVMValueRef desc = LLVMGetParam(ctx->radeon_bld.main_fn, SI_PARAM_CONST_BUFFERS);
1103 LLVMValueRef buf_index = lp_build_const_int32(gallivm, SI_DRIVER_STATE_CONST_BUF);
1104 LLVMValueRef resource = build_indexed_load_const(ctx, desc, buf_index);
1105
1106 /* offset = sample_id * 8 (8 = 2 floats containing samplepos.xy) */
1107 LLVMValueRef offset0 = lp_build_mul_imm(uint_bld, sample_id, 8);
1108 LLVMValueRef offset1 = LLVMBuildAdd(builder, offset0, lp_build_const_int32(gallivm, 4), "");
1109
1110 LLVMValueRef pos[4] = {
1111 buffer_load_const(builder, resource, offset0, ctx->f32),
1112 buffer_load_const(builder, resource, offset1, ctx->f32),
1113 lp_build_const_float(gallivm, 0),
1114 lp_build_const_float(gallivm, 0)
1115 };
1116
1117 return lp_build_gather_values(gallivm, pos, 4);
1118 }
1119
1120 static void declare_system_value(
1121 struct radeon_llvm_context *radeon_bld,
1122 unsigned index,
1123 const struct tgsi_full_declaration *decl)
1124 {
1125 struct si_shader_context *ctx =
1126 si_shader_context(&radeon_bld->soa.bld_base);
1127 struct lp_build_context *bld = &radeon_bld->soa.bld_base.base;
1128 struct gallivm_state *gallivm = &radeon_bld->gallivm;
1129 LLVMValueRef value = 0;
1130
1131 switch (decl->Semantic.Name) {
1132 case TGSI_SEMANTIC_INSTANCEID:
1133 value = LLVMGetParam(radeon_bld->main_fn,
1134 ctx->param_instance_id);
1135 break;
1136
1137 case TGSI_SEMANTIC_VERTEXID:
1138 value = LLVMBuildAdd(gallivm->builder,
1139 LLVMGetParam(radeon_bld->main_fn,
1140 ctx->param_vertex_id),
1141 LLVMGetParam(radeon_bld->main_fn,
1142 SI_PARAM_BASE_VERTEX), "");
1143 break;
1144
1145 case TGSI_SEMANTIC_VERTEXID_NOBASE:
1146 value = LLVMGetParam(radeon_bld->main_fn,
1147 ctx->param_vertex_id);
1148 break;
1149
1150 case TGSI_SEMANTIC_BASEVERTEX:
1151 value = LLVMGetParam(radeon_bld->main_fn,
1152 SI_PARAM_BASE_VERTEX);
1153 break;
1154
1155 case TGSI_SEMANTIC_INVOCATIONID:
1156 if (ctx->type == TGSI_PROCESSOR_TESS_CTRL)
1157 value = unpack_param(ctx, SI_PARAM_REL_IDS, 8, 5);
1158 else if (ctx->type == TGSI_PROCESSOR_GEOMETRY)
1159 value = LLVMGetParam(radeon_bld->main_fn,
1160 SI_PARAM_GS_INSTANCE_ID);
1161 else
1162 assert(!"INVOCATIONID not implemented");
1163 break;
1164
1165 case TGSI_SEMANTIC_POSITION:
1166 {
1167 LLVMValueRef pos[4] = {
1168 LLVMGetParam(radeon_bld->main_fn, SI_PARAM_POS_X_FLOAT),
1169 LLVMGetParam(radeon_bld->main_fn, SI_PARAM_POS_Y_FLOAT),
1170 LLVMGetParam(radeon_bld->main_fn, SI_PARAM_POS_Z_FLOAT),
1171 lp_build_emit_llvm_unary(&radeon_bld->soa.bld_base, TGSI_OPCODE_RCP,
1172 LLVMGetParam(radeon_bld->main_fn,
1173 SI_PARAM_POS_W_FLOAT)),
1174 };
1175 value = lp_build_gather_values(gallivm, pos, 4);
1176 break;
1177 }
1178
1179 case TGSI_SEMANTIC_FACE:
1180 value = LLVMGetParam(radeon_bld->main_fn, SI_PARAM_FRONT_FACE);
1181 break;
1182
1183 case TGSI_SEMANTIC_SAMPLEID:
1184 value = get_sample_id(radeon_bld);
1185 break;
1186
1187 case TGSI_SEMANTIC_SAMPLEPOS: {
1188 LLVMValueRef pos[4] = {
1189 LLVMGetParam(radeon_bld->main_fn, SI_PARAM_POS_X_FLOAT),
1190 LLVMGetParam(radeon_bld->main_fn, SI_PARAM_POS_Y_FLOAT),
1191 lp_build_const_float(gallivm, 0),
1192 lp_build_const_float(gallivm, 0)
1193 };
1194 pos[0] = lp_build_emit_llvm_unary(&radeon_bld->soa.bld_base,
1195 TGSI_OPCODE_FRC, pos[0]);
1196 pos[1] = lp_build_emit_llvm_unary(&radeon_bld->soa.bld_base,
1197 TGSI_OPCODE_FRC, pos[1]);
1198 value = lp_build_gather_values(gallivm, pos, 4);
1199 break;
1200 }
1201
1202 case TGSI_SEMANTIC_SAMPLEMASK:
1203 /* This can only occur with the OpenGL Core profile, which
1204 * doesn't support smoothing.
1205 */
1206 value = LLVMGetParam(radeon_bld->main_fn, SI_PARAM_SAMPLE_COVERAGE);
1207 break;
1208
1209 case TGSI_SEMANTIC_TESSCOORD:
1210 {
1211 LLVMValueRef coord[4] = {
1212 LLVMGetParam(radeon_bld->main_fn, ctx->param_tes_u),
1213 LLVMGetParam(radeon_bld->main_fn, ctx->param_tes_v),
1214 bld->zero,
1215 bld->zero
1216 };
1217
1218 /* For triangles, the vector should be (u, v, 1-u-v). */
1219 if (ctx->shader->selector->info.properties[TGSI_PROPERTY_TES_PRIM_MODE] ==
1220 PIPE_PRIM_TRIANGLES)
1221 coord[2] = lp_build_sub(bld, bld->one,
1222 lp_build_add(bld, coord[0], coord[1]));
1223
1224 value = lp_build_gather_values(gallivm, coord, 4);
1225 break;
1226 }
1227
1228 case TGSI_SEMANTIC_VERTICESIN:
1229 value = unpack_param(ctx, SI_PARAM_TCS_OUT_LAYOUT, 26, 6);
1230 break;
1231
1232 case TGSI_SEMANTIC_TESSINNER:
1233 case TGSI_SEMANTIC_TESSOUTER:
1234 {
1235 LLVMValueRef dw_addr;
1236 int param = si_shader_io_get_unique_index(decl->Semantic.Name, 0);
1237
1238 dw_addr = get_tcs_out_current_patch_data_offset(ctx);
1239 dw_addr = LLVMBuildAdd(gallivm->builder, dw_addr,
1240 lp_build_const_int32(gallivm, param * 4), "");
1241
1242 value = lds_load(&radeon_bld->soa.bld_base, TGSI_TYPE_FLOAT,
1243 ~0, dw_addr);
1244 break;
1245 }
1246
1247 case TGSI_SEMANTIC_PRIMID:
1248 value = get_primitive_id(&radeon_bld->soa.bld_base, 0);
1249 break;
1250
1251 default:
1252 assert(!"unknown system value");
1253 return;
1254 }
1255
1256 radeon_bld->system_values[index] = value;
1257 }
1258
1259 static LLVMValueRef fetch_constant(
1260 struct lp_build_tgsi_context *bld_base,
1261 const struct tgsi_full_src_register *reg,
1262 enum tgsi_opcode_type type,
1263 unsigned swizzle)
1264 {
1265 struct si_shader_context *ctx = si_shader_context(bld_base);
1266 struct lp_build_context *base = &bld_base->base;
1267 const struct tgsi_ind_register *ireg = &reg->Indirect;
1268 unsigned buf, idx;
1269
1270 LLVMValueRef addr, bufp;
1271 LLVMValueRef result;
1272
1273 if (swizzle == LP_CHAN_ALL) {
1274 unsigned chan;
1275 LLVMValueRef values[4];
1276 for (chan = 0; chan < TGSI_NUM_CHANNELS; ++chan)
1277 values[chan] = fetch_constant(bld_base, reg, type, chan);
1278
1279 return lp_build_gather_values(bld_base->base.gallivm, values, 4);
1280 }
1281
1282 buf = reg->Register.Dimension ? reg->Dimension.Index : 0;
1283 idx = reg->Register.Index * 4 + swizzle;
1284
1285 if (!reg->Register.Indirect && !reg->Dimension.Indirect) {
1286 if (type != TGSI_TYPE_DOUBLE)
1287 return bitcast(bld_base, type, ctx->constants[buf][idx]);
1288 else {
1289 return radeon_llvm_emit_fetch_double(bld_base,
1290 ctx->constants[buf][idx],
1291 ctx->constants[buf][idx + 1]);
1292 }
1293 }
1294
1295 if (reg->Register.Dimension && reg->Dimension.Indirect) {
1296 LLVMValueRef ptr = LLVMGetParam(ctx->radeon_bld.main_fn, SI_PARAM_CONST_BUFFERS);
1297 LLVMValueRef index;
1298 index = get_indirect_index(ctx, &reg->DimIndirect,
1299 reg->Dimension.Index);
1300 bufp = build_indexed_load_const(ctx, ptr, index);
1301 } else
1302 bufp = ctx->const_buffers[buf];
1303
1304 addr = ctx->radeon_bld.soa.addr[ireg->Index][ireg->Swizzle];
1305 addr = LLVMBuildLoad(base->gallivm->builder, addr, "load addr reg");
1306 addr = lp_build_mul_imm(&bld_base->uint_bld, addr, 16);
1307 addr = lp_build_add(&bld_base->uint_bld, addr,
1308 lp_build_const_int32(base->gallivm, idx * 4));
1309
1310 result = buffer_load_const(base->gallivm->builder, bufp,
1311 addr, ctx->f32);
1312
1313 if (type != TGSI_TYPE_DOUBLE)
1314 result = bitcast(bld_base, type, result);
1315 else {
1316 LLVMValueRef addr2, result2;
1317 addr2 = ctx->radeon_bld.soa.addr[ireg->Index][ireg->Swizzle + 1];
1318 addr2 = LLVMBuildLoad(base->gallivm->builder, addr2, "load addr reg2");
1319 addr2 = lp_build_mul_imm(&bld_base->uint_bld, addr2, 16);
1320 addr2 = lp_build_add(&bld_base->uint_bld, addr2,
1321 lp_build_const_int32(base->gallivm, idx * 4));
1322
1323 result2 = buffer_load_const(base->gallivm->builder, ctx->const_buffers[buf],
1324 addr2, ctx->f32);
1325
1326 result = radeon_llvm_emit_fetch_double(bld_base,
1327 result, result2);
1328 }
1329 return result;
1330 }
1331
1332 /* Upper 16 bits must be zero. */
1333 static LLVMValueRef si_llvm_pack_two_int16(struct gallivm_state *gallivm,
1334 LLVMValueRef val[2])
1335 {
1336 return LLVMBuildOr(gallivm->builder, val[0],
1337 LLVMBuildShl(gallivm->builder, val[1],
1338 lp_build_const_int32(gallivm, 16),
1339 ""), "");
1340 }
1341
1342 /* Upper 16 bits are ignored and will be dropped. */
1343 static LLVMValueRef si_llvm_pack_two_int32_as_int16(struct gallivm_state *gallivm,
1344 LLVMValueRef val[2])
1345 {
1346 LLVMValueRef v[2] = {
1347 LLVMBuildAnd(gallivm->builder, val[0],
1348 lp_build_const_int32(gallivm, 0xffff), ""),
1349 val[1],
1350 };
1351 return si_llvm_pack_two_int16(gallivm, v);
1352 }
1353
1354 /* Initialize arguments for the shader export intrinsic */
1355 static void si_llvm_init_export_args(struct lp_build_tgsi_context *bld_base,
1356 LLVMValueRef *values,
1357 unsigned target,
1358 LLVMValueRef *args)
1359 {
1360 struct si_shader_context *ctx = si_shader_context(bld_base);
1361 struct lp_build_context *uint =
1362 &ctx->radeon_bld.soa.bld_base.uint_bld;
1363 struct lp_build_context *base = &bld_base->base;
1364 struct gallivm_state *gallivm = base->gallivm;
1365 LLVMBuilderRef builder = base->gallivm->builder;
1366 LLVMValueRef val[4];
1367 unsigned spi_shader_col_format = V_028714_SPI_SHADER_32_ABGR;
1368 unsigned chan;
1369 bool is_int8;
1370
1371 /* Default is 0xf. Adjusted below depending on the format. */
1372 args[0] = lp_build_const_int32(base->gallivm, 0xf); /* writemask */
1373
1374 /* Specify whether the EXEC mask represents the valid mask */
1375 args[1] = uint->zero;
1376
1377 /* Specify whether this is the last export */
1378 args[2] = uint->zero;
1379
1380 /* Specify the target we are exporting */
1381 args[3] = lp_build_const_int32(base->gallivm, target);
1382
1383 if (ctx->type == TGSI_PROCESSOR_FRAGMENT) {
1384 const union si_shader_key *key = &ctx->shader->key;
1385 unsigned col_formats = key->ps.epilog.spi_shader_col_format;
1386 int cbuf = target - V_008DFC_SQ_EXP_MRT;
1387
1388 assert(cbuf >= 0 && cbuf < 8);
1389 spi_shader_col_format = (col_formats >> (cbuf * 4)) & 0xf;
1390 is_int8 = (key->ps.epilog.color_is_int8 >> cbuf) & 0x1;
1391 }
1392
1393 args[4] = uint->zero; /* COMPR flag */
1394 args[5] = base->undef;
1395 args[6] = base->undef;
1396 args[7] = base->undef;
1397 args[8] = base->undef;
1398
1399 switch (spi_shader_col_format) {
1400 case V_028714_SPI_SHADER_ZERO:
1401 args[0] = uint->zero; /* writemask */
1402 args[3] = lp_build_const_int32(base->gallivm, V_008DFC_SQ_EXP_NULL);
1403 break;
1404
1405 case V_028714_SPI_SHADER_32_R:
1406 args[0] = uint->one; /* writemask */
1407 args[5] = values[0];
1408 break;
1409
1410 case V_028714_SPI_SHADER_32_GR:
1411 args[0] = lp_build_const_int32(base->gallivm, 0x3); /* writemask */
1412 args[5] = values[0];
1413 args[6] = values[1];
1414 break;
1415
1416 case V_028714_SPI_SHADER_32_AR:
1417 args[0] = lp_build_const_int32(base->gallivm, 0x9); /* writemask */
1418 args[5] = values[0];
1419 args[8] = values[3];
1420 break;
1421
1422 case V_028714_SPI_SHADER_FP16_ABGR:
1423 args[4] = uint->one; /* COMPR flag */
1424
1425 for (chan = 0; chan < 2; chan++) {
1426 LLVMValueRef pack_args[2] = {
1427 values[2 * chan],
1428 values[2 * chan + 1]
1429 };
1430 LLVMValueRef packed;
1431
1432 packed = lp_build_intrinsic(base->gallivm->builder,
1433 "llvm.SI.packf16",
1434 ctx->i32, pack_args, 2,
1435 LLVMReadNoneAttribute | LLVMNoUnwindAttribute);
1436 args[chan + 5] =
1437 LLVMBuildBitCast(base->gallivm->builder,
1438 packed, ctx->f32, "");
1439 }
1440 break;
1441
1442 case V_028714_SPI_SHADER_UNORM16_ABGR:
1443 for (chan = 0; chan < 4; chan++) {
1444 val[chan] = radeon_llvm_saturate(bld_base, values[chan]);
1445 val[chan] = LLVMBuildFMul(builder, val[chan],
1446 lp_build_const_float(gallivm, 65535), "");
1447 val[chan] = LLVMBuildFAdd(builder, val[chan],
1448 lp_build_const_float(gallivm, 0.5), "");
1449 val[chan] = LLVMBuildFPToUI(builder, val[chan],
1450 ctx->i32, "");
1451 }
1452
1453 args[4] = uint->one; /* COMPR flag */
1454 args[5] = bitcast(bld_base, TGSI_TYPE_FLOAT,
1455 si_llvm_pack_two_int16(gallivm, val));
1456 args[6] = bitcast(bld_base, TGSI_TYPE_FLOAT,
1457 si_llvm_pack_two_int16(gallivm, val+2));
1458 break;
1459
1460 case V_028714_SPI_SHADER_SNORM16_ABGR:
1461 for (chan = 0; chan < 4; chan++) {
1462 /* Clamp between [-1, 1]. */
1463 val[chan] = lp_build_emit_llvm_binary(bld_base, TGSI_OPCODE_MIN,
1464 values[chan],
1465 lp_build_const_float(gallivm, 1));
1466 val[chan] = lp_build_emit_llvm_binary(bld_base, TGSI_OPCODE_MAX,
1467 val[chan],
1468 lp_build_const_float(gallivm, -1));
1469 /* Convert to a signed integer in [-32767, 32767]. */
1470 val[chan] = LLVMBuildFMul(builder, val[chan],
1471 lp_build_const_float(gallivm, 32767), "");
1472 /* If positive, add 0.5, else add -0.5. */
1473 val[chan] = LLVMBuildFAdd(builder, val[chan],
1474 LLVMBuildSelect(builder,
1475 LLVMBuildFCmp(builder, LLVMRealOGE,
1476 val[chan], base->zero, ""),
1477 lp_build_const_float(gallivm, 0.5),
1478 lp_build_const_float(gallivm, -0.5), ""), "");
1479 val[chan] = LLVMBuildFPToSI(builder, val[chan], ctx->i32, "");
1480 }
1481
1482 args[4] = uint->one; /* COMPR flag */
1483 args[5] = bitcast(bld_base, TGSI_TYPE_FLOAT,
1484 si_llvm_pack_two_int32_as_int16(gallivm, val));
1485 args[6] = bitcast(bld_base, TGSI_TYPE_FLOAT,
1486 si_llvm_pack_two_int32_as_int16(gallivm, val+2));
1487 break;
1488
1489 case V_028714_SPI_SHADER_UINT16_ABGR: {
1490 LLVMValueRef max = lp_build_const_int32(gallivm, is_int8 ?
1491 255 : 65535);
1492 /* Clamp. */
1493 for (chan = 0; chan < 4; chan++) {
1494 val[chan] = bitcast(bld_base, TGSI_TYPE_UNSIGNED, values[chan]);
1495 val[chan] = lp_build_emit_llvm_binary(bld_base, TGSI_OPCODE_UMIN,
1496 val[chan], max);
1497 }
1498
1499 args[4] = uint->one; /* COMPR flag */
1500 args[5] = bitcast(bld_base, TGSI_TYPE_FLOAT,
1501 si_llvm_pack_two_int16(gallivm, val));
1502 args[6] = bitcast(bld_base, TGSI_TYPE_FLOAT,
1503 si_llvm_pack_two_int16(gallivm, val+2));
1504 break;
1505 }
1506
1507 case V_028714_SPI_SHADER_SINT16_ABGR: {
1508 LLVMValueRef max = lp_build_const_int32(gallivm, is_int8 ?
1509 127 : 32767);
1510 LLVMValueRef min = lp_build_const_int32(gallivm, is_int8 ?
1511 -128 : -32768);
1512 /* Clamp. */
1513 for (chan = 0; chan < 4; chan++) {
1514 val[chan] = bitcast(bld_base, TGSI_TYPE_UNSIGNED, values[chan]);
1515 val[chan] = lp_build_emit_llvm_binary(bld_base,
1516 TGSI_OPCODE_IMIN,
1517 val[chan], max);
1518 val[chan] = lp_build_emit_llvm_binary(bld_base,
1519 TGSI_OPCODE_IMAX,
1520 val[chan], min);
1521 }
1522
1523 args[4] = uint->one; /* COMPR flag */
1524 args[5] = bitcast(bld_base, TGSI_TYPE_FLOAT,
1525 si_llvm_pack_two_int32_as_int16(gallivm, val));
1526 args[6] = bitcast(bld_base, TGSI_TYPE_FLOAT,
1527 si_llvm_pack_two_int32_as_int16(gallivm, val+2));
1528 break;
1529 }
1530
1531 case V_028714_SPI_SHADER_32_ABGR:
1532 memcpy(&args[5], values, sizeof(values[0]) * 4);
1533 break;
1534 }
1535 }
1536
1537 static void si_alpha_test(struct lp_build_tgsi_context *bld_base,
1538 LLVMValueRef alpha)
1539 {
1540 struct si_shader_context *ctx = si_shader_context(bld_base);
1541 struct gallivm_state *gallivm = bld_base->base.gallivm;
1542
1543 if (ctx->shader->key.ps.epilog.alpha_func != PIPE_FUNC_NEVER) {
1544 LLVMValueRef alpha_ref = LLVMGetParam(ctx->radeon_bld.main_fn,
1545 SI_PARAM_ALPHA_REF);
1546
1547 LLVMValueRef alpha_pass =
1548 lp_build_cmp(&bld_base->base,
1549 ctx->shader->key.ps.epilog.alpha_func,
1550 alpha, alpha_ref);
1551 LLVMValueRef arg =
1552 lp_build_select(&bld_base->base,
1553 alpha_pass,
1554 lp_build_const_float(gallivm, 1.0f),
1555 lp_build_const_float(gallivm, -1.0f));
1556
1557 lp_build_intrinsic(gallivm->builder, "llvm.AMDGPU.kill",
1558 ctx->voidt, &arg, 1, 0);
1559 } else {
1560 lp_build_intrinsic(gallivm->builder, "llvm.AMDGPU.kilp",
1561 ctx->voidt, NULL, 0, 0);
1562 }
1563 }
1564
1565 static LLVMValueRef si_scale_alpha_by_sample_mask(struct lp_build_tgsi_context *bld_base,
1566 LLVMValueRef alpha,
1567 unsigned samplemask_param)
1568 {
1569 struct si_shader_context *ctx = si_shader_context(bld_base);
1570 struct gallivm_state *gallivm = bld_base->base.gallivm;
1571 LLVMValueRef coverage;
1572
1573 /* alpha = alpha * popcount(coverage) / SI_NUM_SMOOTH_AA_SAMPLES */
1574 coverage = LLVMGetParam(ctx->radeon_bld.main_fn,
1575 samplemask_param);
1576 coverage = bitcast(bld_base, TGSI_TYPE_SIGNED, coverage);
1577
1578 coverage = lp_build_intrinsic(gallivm->builder, "llvm.ctpop.i32",
1579 ctx->i32,
1580 &coverage, 1, LLVMReadNoneAttribute);
1581
1582 coverage = LLVMBuildUIToFP(gallivm->builder, coverage,
1583 ctx->f32, "");
1584
1585 coverage = LLVMBuildFMul(gallivm->builder, coverage,
1586 lp_build_const_float(gallivm,
1587 1.0 / SI_NUM_SMOOTH_AA_SAMPLES), "");
1588
1589 return LLVMBuildFMul(gallivm->builder, alpha, coverage, "");
1590 }
1591
1592 static void si_llvm_emit_clipvertex(struct lp_build_tgsi_context *bld_base,
1593 LLVMValueRef (*pos)[9], LLVMValueRef *out_elts)
1594 {
1595 struct si_shader_context *ctx = si_shader_context(bld_base);
1596 struct lp_build_context *base = &bld_base->base;
1597 struct lp_build_context *uint = &ctx->radeon_bld.soa.bld_base.uint_bld;
1598 unsigned reg_index;
1599 unsigned chan;
1600 unsigned const_chan;
1601 LLVMValueRef base_elt;
1602 LLVMValueRef ptr = LLVMGetParam(ctx->radeon_bld.main_fn, SI_PARAM_CONST_BUFFERS);
1603 LLVMValueRef constbuf_index = lp_build_const_int32(base->gallivm, SI_DRIVER_STATE_CONST_BUF);
1604 LLVMValueRef const_resource = build_indexed_load_const(ctx, ptr, constbuf_index);
1605
1606 for (reg_index = 0; reg_index < 2; reg_index ++) {
1607 LLVMValueRef *args = pos[2 + reg_index];
1608
1609 args[5] =
1610 args[6] =
1611 args[7] =
1612 args[8] = lp_build_const_float(base->gallivm, 0.0f);
1613
1614 /* Compute dot products of position and user clip plane vectors */
1615 for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) {
1616 for (const_chan = 0; const_chan < TGSI_NUM_CHANNELS; const_chan++) {
1617 args[1] = lp_build_const_int32(base->gallivm,
1618 ((reg_index * 4 + chan) * 4 +
1619 const_chan) * 4);
1620 base_elt = buffer_load_const(base->gallivm->builder, const_resource,
1621 args[1], ctx->f32);
1622 args[5 + chan] =
1623 lp_build_add(base, args[5 + chan],
1624 lp_build_mul(base, base_elt,
1625 out_elts[const_chan]));
1626 }
1627 }
1628
1629 args[0] = lp_build_const_int32(base->gallivm, 0xf);
1630 args[1] = uint->zero;
1631 args[2] = uint->zero;
1632 args[3] = lp_build_const_int32(base->gallivm,
1633 V_008DFC_SQ_EXP_POS + 2 + reg_index);
1634 args[4] = uint->zero;
1635 }
1636 }
1637
1638 static void si_dump_streamout(struct pipe_stream_output_info *so)
1639 {
1640 unsigned i;
1641
1642 if (so->num_outputs)
1643 fprintf(stderr, "STREAMOUT\n");
1644
1645 for (i = 0; i < so->num_outputs; i++) {
1646 unsigned mask = ((1 << so->output[i].num_components) - 1) <<
1647 so->output[i].start_component;
1648 fprintf(stderr, " %i: BUF%i[%i..%i] <- OUT[%i].%s%s%s%s\n",
1649 i, so->output[i].output_buffer,
1650 so->output[i].dst_offset, so->output[i].dst_offset + so->output[i].num_components - 1,
1651 so->output[i].register_index,
1652 mask & 1 ? "x" : "",
1653 mask & 2 ? "y" : "",
1654 mask & 4 ? "z" : "",
1655 mask & 8 ? "w" : "");
1656 }
1657 }
1658
1659 /* TBUFFER_STORE_FORMAT_{X,XY,XYZ,XYZW} <- the suffix is selected by num_channels=1..4.
1660 * The type of vdata must be one of i32 (num_channels=1), v2i32 (num_channels=2),
1661 * or v4i32 (num_channels=3,4). */
1662 static void build_tbuffer_store(struct si_shader_context *ctx,
1663 LLVMValueRef rsrc,
1664 LLVMValueRef vdata,
1665 unsigned num_channels,
1666 LLVMValueRef vaddr,
1667 LLVMValueRef soffset,
1668 unsigned inst_offset,
1669 unsigned dfmt,
1670 unsigned nfmt,
1671 unsigned offen,
1672 unsigned idxen,
1673 unsigned glc,
1674 unsigned slc,
1675 unsigned tfe)
1676 {
1677 struct gallivm_state *gallivm = &ctx->radeon_bld.gallivm;
1678 LLVMValueRef args[] = {
1679 rsrc,
1680 vdata,
1681 LLVMConstInt(ctx->i32, num_channels, 0),
1682 vaddr,
1683 soffset,
1684 LLVMConstInt(ctx->i32, inst_offset, 0),
1685 LLVMConstInt(ctx->i32, dfmt, 0),
1686 LLVMConstInt(ctx->i32, nfmt, 0),
1687 LLVMConstInt(ctx->i32, offen, 0),
1688 LLVMConstInt(ctx->i32, idxen, 0),
1689 LLVMConstInt(ctx->i32, glc, 0),
1690 LLVMConstInt(ctx->i32, slc, 0),
1691 LLVMConstInt(ctx->i32, tfe, 0)
1692 };
1693
1694 /* The instruction offset field has 12 bits */
1695 assert(offen || inst_offset < (1 << 12));
1696
1697 /* The intrinsic is overloaded, we need to add a type suffix for overloading to work. */
1698 unsigned func = CLAMP(num_channels, 1, 3) - 1;
1699 const char *types[] = {"i32", "v2i32", "v4i32"};
1700 char name[256];
1701 snprintf(name, sizeof(name), "llvm.SI.tbuffer.store.%s", types[func]);
1702
1703 lp_build_intrinsic(gallivm->builder, name, ctx->voidt,
1704 args, Elements(args), 0);
1705 }
1706
1707 static void build_tbuffer_store_dwords(struct si_shader_context *ctx,
1708 LLVMValueRef rsrc,
1709 LLVMValueRef vdata,
1710 unsigned num_channels,
1711 LLVMValueRef vaddr,
1712 LLVMValueRef soffset,
1713 unsigned inst_offset)
1714 {
1715 static unsigned dfmt[] = {
1716 V_008F0C_BUF_DATA_FORMAT_32,
1717 V_008F0C_BUF_DATA_FORMAT_32_32,
1718 V_008F0C_BUF_DATA_FORMAT_32_32_32,
1719 V_008F0C_BUF_DATA_FORMAT_32_32_32_32
1720 };
1721 assert(num_channels >= 1 && num_channels <= 4);
1722
1723 build_tbuffer_store(ctx, rsrc, vdata, num_channels, vaddr, soffset,
1724 inst_offset, dfmt[num_channels-1],
1725 V_008F0C_BUF_NUM_FORMAT_UINT, 1, 0, 1, 1, 0);
1726 }
1727
1728 /* On SI, the vertex shader is responsible for writing streamout data
1729 * to buffers. */
1730 static void si_llvm_emit_streamout(struct si_shader_context *ctx,
1731 struct si_shader_output_values *outputs,
1732 unsigned noutput)
1733 {
1734 struct pipe_stream_output_info *so = &ctx->shader->selector->so;
1735 struct gallivm_state *gallivm = &ctx->radeon_bld.gallivm;
1736 LLVMBuilderRef builder = gallivm->builder;
1737 int i, j;
1738 struct lp_build_if_state if_ctx;
1739
1740 /* Get bits [22:16], i.e. (so_param >> 16) & 127; */
1741 LLVMValueRef so_vtx_count =
1742 unpack_param(ctx, ctx->param_streamout_config, 16, 7);
1743
1744 LLVMValueRef tid = lp_build_intrinsic(builder, "llvm.SI.tid", ctx->i32,
1745 NULL, 0, LLVMReadNoneAttribute);
1746
1747 /* can_emit = tid < so_vtx_count; */
1748 LLVMValueRef can_emit =
1749 LLVMBuildICmp(builder, LLVMIntULT, tid, so_vtx_count, "");
1750
1751 LLVMValueRef stream_id =
1752 unpack_param(ctx, ctx->param_streamout_config, 24, 2);
1753
1754 /* Emit the streamout code conditionally. This actually avoids
1755 * out-of-bounds buffer access. The hw tells us via the SGPR
1756 * (so_vtx_count) which threads are allowed to emit streamout data. */
1757 lp_build_if(&if_ctx, gallivm, can_emit);
1758 {
1759 /* The buffer offset is computed as follows:
1760 * ByteOffset = streamout_offset[buffer_id]*4 +
1761 * (streamout_write_index + thread_id)*stride[buffer_id] +
1762 * attrib_offset
1763 */
1764
1765 LLVMValueRef so_write_index =
1766 LLVMGetParam(ctx->radeon_bld.main_fn,
1767 ctx->param_streamout_write_index);
1768
1769 /* Compute (streamout_write_index + thread_id). */
1770 so_write_index = LLVMBuildAdd(builder, so_write_index, tid, "");
1771
1772 /* Compute the write offset for each enabled buffer. */
1773 LLVMValueRef so_write_offset[4] = {};
1774 for (i = 0; i < 4; i++) {
1775 if (!so->stride[i])
1776 continue;
1777
1778 LLVMValueRef so_offset = LLVMGetParam(ctx->radeon_bld.main_fn,
1779 ctx->param_streamout_offset[i]);
1780 so_offset = LLVMBuildMul(builder, so_offset, LLVMConstInt(ctx->i32, 4, 0), "");
1781
1782 so_write_offset[i] = LLVMBuildMul(builder, so_write_index,
1783 LLVMConstInt(ctx->i32, so->stride[i]*4, 0), "");
1784 so_write_offset[i] = LLVMBuildAdd(builder, so_write_offset[i], so_offset, "");
1785 }
1786
1787 /* Write streamout data. */
1788 for (i = 0; i < so->num_outputs; i++) {
1789 unsigned buf_idx = so->output[i].output_buffer;
1790 unsigned reg = so->output[i].register_index;
1791 unsigned start = so->output[i].start_component;
1792 unsigned num_comps = so->output[i].num_components;
1793 unsigned stream = so->output[i].stream;
1794 LLVMValueRef out[4];
1795 struct lp_build_if_state if_ctx_stream;
1796
1797 assert(num_comps && num_comps <= 4);
1798 if (!num_comps || num_comps > 4)
1799 continue;
1800
1801 if (reg >= noutput)
1802 continue;
1803
1804 /* Load the output as int. */
1805 for (j = 0; j < num_comps; j++) {
1806 out[j] = LLVMBuildBitCast(builder,
1807 outputs[reg].values[start+j],
1808 ctx->i32, "");
1809 }
1810
1811 /* Pack the output. */
1812 LLVMValueRef vdata = NULL;
1813
1814 switch (num_comps) {
1815 case 1: /* as i32 */
1816 vdata = out[0];
1817 break;
1818 case 2: /* as v2i32 */
1819 case 3: /* as v4i32 (aligned to 4) */
1820 case 4: /* as v4i32 */
1821 vdata = LLVMGetUndef(LLVMVectorType(ctx->i32, util_next_power_of_two(num_comps)));
1822 for (j = 0; j < num_comps; j++) {
1823 vdata = LLVMBuildInsertElement(builder, vdata, out[j],
1824 LLVMConstInt(ctx->i32, j, 0), "");
1825 }
1826 break;
1827 }
1828
1829 LLVMValueRef can_emit_stream =
1830 LLVMBuildICmp(builder, LLVMIntEQ,
1831 stream_id,
1832 lp_build_const_int32(gallivm, stream), "");
1833
1834 lp_build_if(&if_ctx_stream, gallivm, can_emit_stream);
1835 build_tbuffer_store_dwords(ctx, ctx->so_buffers[buf_idx],
1836 vdata, num_comps,
1837 so_write_offset[buf_idx],
1838 LLVMConstInt(ctx->i32, 0, 0),
1839 so->output[i].dst_offset*4);
1840 lp_build_endif(&if_ctx_stream);
1841 }
1842 }
1843 lp_build_endif(&if_ctx);
1844 }
1845
1846
1847 /* Generate export instructions for hardware VS shader stage */
1848 static void si_llvm_export_vs(struct lp_build_tgsi_context *bld_base,
1849 struct si_shader_output_values *outputs,
1850 unsigned noutput)
1851 {
1852 struct si_shader_context *ctx = si_shader_context(bld_base);
1853 struct si_shader *shader = ctx->shader;
1854 struct lp_build_context *base = &bld_base->base;
1855 struct lp_build_context *uint =
1856 &ctx->radeon_bld.soa.bld_base.uint_bld;
1857 LLVMValueRef args[9];
1858 LLVMValueRef pos_args[4][9] = { { 0 } };
1859 LLVMValueRef psize_value = NULL, edgeflag_value = NULL, layer_value = NULL, viewport_index_value = NULL;
1860 unsigned semantic_name, semantic_index;
1861 unsigned target;
1862 unsigned param_count = 0;
1863 unsigned pos_idx;
1864 int i;
1865
1866 if (outputs && ctx->shader->selector->so.num_outputs) {
1867 si_llvm_emit_streamout(ctx, outputs, noutput);
1868 }
1869
1870 for (i = 0; i < noutput; i++) {
1871 semantic_name = outputs[i].name;
1872 semantic_index = outputs[i].sid;
1873
1874 handle_semantic:
1875 /* Select the correct target */
1876 switch(semantic_name) {
1877 case TGSI_SEMANTIC_PSIZE:
1878 psize_value = outputs[i].values[0];
1879 continue;
1880 case TGSI_SEMANTIC_EDGEFLAG:
1881 edgeflag_value = outputs[i].values[0];
1882 continue;
1883 case TGSI_SEMANTIC_LAYER:
1884 layer_value = outputs[i].values[0];
1885 semantic_name = TGSI_SEMANTIC_GENERIC;
1886 goto handle_semantic;
1887 case TGSI_SEMANTIC_VIEWPORT_INDEX:
1888 viewport_index_value = outputs[i].values[0];
1889 semantic_name = TGSI_SEMANTIC_GENERIC;
1890 goto handle_semantic;
1891 case TGSI_SEMANTIC_POSITION:
1892 target = V_008DFC_SQ_EXP_POS;
1893 break;
1894 case TGSI_SEMANTIC_COLOR:
1895 case TGSI_SEMANTIC_BCOLOR:
1896 target = V_008DFC_SQ_EXP_PARAM + param_count;
1897 assert(i < ARRAY_SIZE(shader->info.vs_output_param_offset));
1898 shader->info.vs_output_param_offset[i] = param_count;
1899 param_count++;
1900 break;
1901 case TGSI_SEMANTIC_CLIPDIST:
1902 target = V_008DFC_SQ_EXP_POS + 2 + semantic_index;
1903 break;
1904 case TGSI_SEMANTIC_CLIPVERTEX:
1905 si_llvm_emit_clipvertex(bld_base, pos_args, outputs[i].values);
1906 continue;
1907 case TGSI_SEMANTIC_PRIMID:
1908 case TGSI_SEMANTIC_FOG:
1909 case TGSI_SEMANTIC_TEXCOORD:
1910 case TGSI_SEMANTIC_GENERIC:
1911 target = V_008DFC_SQ_EXP_PARAM + param_count;
1912 assert(i < ARRAY_SIZE(shader->info.vs_output_param_offset));
1913 shader->info.vs_output_param_offset[i] = param_count;
1914 param_count++;
1915 break;
1916 default:
1917 target = 0;
1918 fprintf(stderr,
1919 "Warning: SI unhandled vs output type:%d\n",
1920 semantic_name);
1921 }
1922
1923 si_llvm_init_export_args(bld_base, outputs[i].values, target, args);
1924
1925 if (target >= V_008DFC_SQ_EXP_POS &&
1926 target <= (V_008DFC_SQ_EXP_POS + 3)) {
1927 memcpy(pos_args[target - V_008DFC_SQ_EXP_POS],
1928 args, sizeof(args));
1929 } else {
1930 lp_build_intrinsic(base->gallivm->builder,
1931 "llvm.SI.export", ctx->voidt,
1932 args, 9, 0);
1933 }
1934
1935 if (semantic_name == TGSI_SEMANTIC_CLIPDIST) {
1936 semantic_name = TGSI_SEMANTIC_GENERIC;
1937 goto handle_semantic;
1938 }
1939 }
1940
1941 shader->info.nr_param_exports = param_count;
1942
1943 /* We need to add the position output manually if it's missing. */
1944 if (!pos_args[0][0]) {
1945 pos_args[0][0] = lp_build_const_int32(base->gallivm, 0xf); /* writemask */
1946 pos_args[0][1] = uint->zero; /* EXEC mask */
1947 pos_args[0][2] = uint->zero; /* last export? */
1948 pos_args[0][3] = lp_build_const_int32(base->gallivm, V_008DFC_SQ_EXP_POS);
1949 pos_args[0][4] = uint->zero; /* COMPR flag */
1950 pos_args[0][5] = base->zero; /* X */
1951 pos_args[0][6] = base->zero; /* Y */
1952 pos_args[0][7] = base->zero; /* Z */
1953 pos_args[0][8] = base->one; /* W */
1954 }
1955
1956 /* Write the misc vector (point size, edgeflag, layer, viewport). */
1957 if (shader->selector->info.writes_psize ||
1958 shader->selector->info.writes_edgeflag ||
1959 shader->selector->info.writes_viewport_index ||
1960 shader->selector->info.writes_layer) {
1961 pos_args[1][0] = lp_build_const_int32(base->gallivm, /* writemask */
1962 shader->selector->info.writes_psize |
1963 (shader->selector->info.writes_edgeflag << 1) |
1964 (shader->selector->info.writes_layer << 2) |
1965 (shader->selector->info.writes_viewport_index << 3));
1966 pos_args[1][1] = uint->zero; /* EXEC mask */
1967 pos_args[1][2] = uint->zero; /* last export? */
1968 pos_args[1][3] = lp_build_const_int32(base->gallivm, V_008DFC_SQ_EXP_POS + 1);
1969 pos_args[1][4] = uint->zero; /* COMPR flag */
1970 pos_args[1][5] = base->zero; /* X */
1971 pos_args[1][6] = base->zero; /* Y */
1972 pos_args[1][7] = base->zero; /* Z */
1973 pos_args[1][8] = base->zero; /* W */
1974
1975 if (shader->selector->info.writes_psize)
1976 pos_args[1][5] = psize_value;
1977
1978 if (shader->selector->info.writes_edgeflag) {
1979 /* The output is a float, but the hw expects an integer
1980 * with the first bit containing the edge flag. */
1981 edgeflag_value = LLVMBuildFPToUI(base->gallivm->builder,
1982 edgeflag_value,
1983 ctx->i32, "");
1984 edgeflag_value = lp_build_min(&bld_base->int_bld,
1985 edgeflag_value,
1986 bld_base->int_bld.one);
1987
1988 /* The LLVM intrinsic expects a float. */
1989 pos_args[1][6] = LLVMBuildBitCast(base->gallivm->builder,
1990 edgeflag_value,
1991 ctx->f32, "");
1992 }
1993
1994 if (shader->selector->info.writes_layer)
1995 pos_args[1][7] = layer_value;
1996
1997 if (shader->selector->info.writes_viewport_index)
1998 pos_args[1][8] = viewport_index_value;
1999 }
2000
2001 for (i = 0; i < 4; i++)
2002 if (pos_args[i][0])
2003 shader->info.nr_pos_exports++;
2004
2005 pos_idx = 0;
2006 for (i = 0; i < 4; i++) {
2007 if (!pos_args[i][0])
2008 continue;
2009
2010 /* Specify the target we are exporting */
2011 pos_args[i][3] = lp_build_const_int32(base->gallivm, V_008DFC_SQ_EXP_POS + pos_idx++);
2012
2013 if (pos_idx == shader->info.nr_pos_exports)
2014 /* Specify that this is the last export */
2015 pos_args[i][2] = uint->one;
2016
2017 lp_build_intrinsic(base->gallivm->builder, "llvm.SI.export",
2018 ctx->voidt, pos_args[i], 9, 0);
2019 }
2020 }
2021
2022 static void si_write_tess_factors(struct lp_build_tgsi_context *bld_base,
2023 LLVMValueRef rel_patch_id,
2024 LLVMValueRef invocation_id,
2025 LLVMValueRef tcs_out_current_patch_data_offset)
2026 {
2027 struct si_shader_context *ctx = si_shader_context(bld_base);
2028 struct gallivm_state *gallivm = bld_base->base.gallivm;
2029 struct si_shader *shader = ctx->shader;
2030 unsigned tess_inner_index, tess_outer_index;
2031 LLVMValueRef lds_base, lds_inner, lds_outer, byteoffset, buffer;
2032 LLVMValueRef out[6], vec0, vec1, rw_buffers, tf_base;
2033 unsigned stride, outer_comps, inner_comps, i;
2034 struct lp_build_if_state if_ctx;
2035
2036 /* Do this only for invocation 0, because the tess levels are per-patch,
2037 * not per-vertex.
2038 *
2039 * This can't jump, because invocation 0 executes this. It should
2040 * at least mask out the loads and stores for other invocations.
2041 */
2042 lp_build_if(&if_ctx, gallivm,
2043 LLVMBuildICmp(gallivm->builder, LLVMIntEQ,
2044 invocation_id, bld_base->uint_bld.zero, ""));
2045
2046 /* Determine the layout of one tess factor element in the buffer. */
2047 switch (shader->key.tcs.epilog.prim_mode) {
2048 case PIPE_PRIM_LINES:
2049 stride = 2; /* 2 dwords, 1 vec2 store */
2050 outer_comps = 2;
2051 inner_comps = 0;
2052 break;
2053 case PIPE_PRIM_TRIANGLES:
2054 stride = 4; /* 4 dwords, 1 vec4 store */
2055 outer_comps = 3;
2056 inner_comps = 1;
2057 break;
2058 case PIPE_PRIM_QUADS:
2059 stride = 6; /* 6 dwords, 2 stores (vec4 + vec2) */
2060 outer_comps = 4;
2061 inner_comps = 2;
2062 break;
2063 default:
2064 assert(0);
2065 return;
2066 }
2067
2068 /* Load tess_inner and tess_outer from LDS.
2069 * Any invocation can write them, so we can't get them from a temporary.
2070 */
2071 tess_inner_index = si_shader_io_get_unique_index(TGSI_SEMANTIC_TESSINNER, 0);
2072 tess_outer_index = si_shader_io_get_unique_index(TGSI_SEMANTIC_TESSOUTER, 0);
2073
2074 lds_base = tcs_out_current_patch_data_offset;
2075 lds_inner = LLVMBuildAdd(gallivm->builder, lds_base,
2076 lp_build_const_int32(gallivm,
2077 tess_inner_index * 4), "");
2078 lds_outer = LLVMBuildAdd(gallivm->builder, lds_base,
2079 lp_build_const_int32(gallivm,
2080 tess_outer_index * 4), "");
2081
2082 for (i = 0; i < outer_comps; i++)
2083 out[i] = lds_load(bld_base, TGSI_TYPE_SIGNED, i, lds_outer);
2084 for (i = 0; i < inner_comps; i++)
2085 out[outer_comps+i] = lds_load(bld_base, TGSI_TYPE_SIGNED, i, lds_inner);
2086
2087 /* Convert the outputs to vectors for stores. */
2088 vec0 = lp_build_gather_values(gallivm, out, MIN2(stride, 4));
2089 vec1 = NULL;
2090
2091 if (stride > 4)
2092 vec1 = lp_build_gather_values(gallivm, out+4, stride - 4);
2093
2094 /* Get the buffer. */
2095 rw_buffers = LLVMGetParam(ctx->radeon_bld.main_fn,
2096 SI_PARAM_RW_BUFFERS);
2097 buffer = build_indexed_load_const(ctx, rw_buffers,
2098 lp_build_const_int32(gallivm, SI_RING_TESS_FACTOR));
2099
2100 /* Get the offset. */
2101 tf_base = LLVMGetParam(ctx->radeon_bld.main_fn,
2102 SI_PARAM_TESS_FACTOR_OFFSET);
2103 byteoffset = LLVMBuildMul(gallivm->builder, rel_patch_id,
2104 lp_build_const_int32(gallivm, 4 * stride), "");
2105
2106 /* Store the outputs. */
2107 build_tbuffer_store_dwords(ctx, buffer, vec0,
2108 MIN2(stride, 4), byteoffset, tf_base, 0);
2109 if (vec1)
2110 build_tbuffer_store_dwords(ctx, buffer, vec1,
2111 stride - 4, byteoffset, tf_base, 16);
2112 lp_build_endif(&if_ctx);
2113 }
2114
2115 /* This only writes the tessellation factor levels. */
2116 static void si_llvm_emit_tcs_epilogue(struct lp_build_tgsi_context *bld_base)
2117 {
2118 struct si_shader_context *ctx = si_shader_context(bld_base);
2119 LLVMValueRef rel_patch_id, invocation_id, tf_lds_offset;
2120
2121 rel_patch_id = get_rel_patch_id(ctx);
2122 invocation_id = unpack_param(ctx, SI_PARAM_REL_IDS, 8, 5);
2123 tf_lds_offset = get_tcs_out_current_patch_data_offset(ctx);
2124
2125 if (!ctx->is_monolithic) {
2126 /* Return epilog parameters from this function. */
2127 LLVMBuilderRef builder = bld_base->base.gallivm->builder;
2128 LLVMValueRef ret = ctx->return_value;
2129 LLVMValueRef rw_buffers, rw0, rw1, tf_soffset;
2130 unsigned vgpr;
2131
2132 /* RW_BUFFERS pointer */
2133 rw_buffers = LLVMGetParam(ctx->radeon_bld.main_fn,
2134 SI_PARAM_RW_BUFFERS);
2135 rw_buffers = LLVMBuildPtrToInt(builder, rw_buffers, ctx->i64, "");
2136 rw_buffers = LLVMBuildBitCast(builder, rw_buffers, ctx->v2i32, "");
2137 rw0 = LLVMBuildExtractElement(builder, rw_buffers,
2138 bld_base->uint_bld.zero, "");
2139 rw1 = LLVMBuildExtractElement(builder, rw_buffers,
2140 bld_base->uint_bld.one, "");
2141 ret = LLVMBuildInsertValue(builder, ret, rw0, 0, "");
2142 ret = LLVMBuildInsertValue(builder, ret, rw1, 1, "");
2143
2144 /* Tess factor buffer soffset is after user SGPRs. */
2145 tf_soffset = LLVMGetParam(ctx->radeon_bld.main_fn,
2146 SI_PARAM_TESS_FACTOR_OFFSET);
2147 ret = LLVMBuildInsertValue(builder, ret, tf_soffset,
2148 SI_TCS_NUM_USER_SGPR, "");
2149
2150 /* VGPRs */
2151 rel_patch_id = bitcast(bld_base, TGSI_TYPE_FLOAT, rel_patch_id);
2152 invocation_id = bitcast(bld_base, TGSI_TYPE_FLOAT, invocation_id);
2153 tf_lds_offset = bitcast(bld_base, TGSI_TYPE_FLOAT, tf_lds_offset);
2154
2155 vgpr = SI_TCS_NUM_USER_SGPR + 1;
2156 ret = LLVMBuildInsertValue(builder, ret, rel_patch_id, vgpr++, "");
2157 ret = LLVMBuildInsertValue(builder, ret, invocation_id, vgpr++, "");
2158 ret = LLVMBuildInsertValue(builder, ret, tf_lds_offset, vgpr++, "");
2159 ctx->return_value = ret;
2160 return;
2161 }
2162
2163 si_write_tess_factors(bld_base, rel_patch_id, invocation_id, tf_lds_offset);
2164 }
2165
2166 static void si_llvm_emit_ls_epilogue(struct lp_build_tgsi_context *bld_base)
2167 {
2168 struct si_shader_context *ctx = si_shader_context(bld_base);
2169 struct si_shader *shader = ctx->shader;
2170 struct tgsi_shader_info *info = &shader->selector->info;
2171 struct gallivm_state *gallivm = bld_base->base.gallivm;
2172 unsigned i, chan;
2173 LLVMValueRef vertex_id = LLVMGetParam(ctx->radeon_bld.main_fn,
2174 ctx->param_rel_auto_id);
2175 LLVMValueRef vertex_dw_stride =
2176 unpack_param(ctx, SI_PARAM_LS_OUT_LAYOUT, 13, 8);
2177 LLVMValueRef base_dw_addr = LLVMBuildMul(gallivm->builder, vertex_id,
2178 vertex_dw_stride, "");
2179
2180 /* Write outputs to LDS. The next shader (TCS aka HS) will read
2181 * its inputs from it. */
2182 for (i = 0; i < info->num_outputs; i++) {
2183 LLVMValueRef *out_ptr = ctx->radeon_bld.soa.outputs[i];
2184 unsigned name = info->output_semantic_name[i];
2185 unsigned index = info->output_semantic_index[i];
2186 int param = si_shader_io_get_unique_index(name, index);
2187 LLVMValueRef dw_addr = LLVMBuildAdd(gallivm->builder, base_dw_addr,
2188 lp_build_const_int32(gallivm, param * 4), "");
2189
2190 for (chan = 0; chan < 4; chan++) {
2191 lds_store(bld_base, chan, dw_addr,
2192 LLVMBuildLoad(gallivm->builder, out_ptr[chan], ""));
2193 }
2194 }
2195 }
2196
2197 static void si_llvm_emit_es_epilogue(struct lp_build_tgsi_context *bld_base)
2198 {
2199 struct si_shader_context *ctx = si_shader_context(bld_base);
2200 struct gallivm_state *gallivm = bld_base->base.gallivm;
2201 struct si_shader *es = ctx->shader;
2202 struct tgsi_shader_info *info = &es->selector->info;
2203 LLVMValueRef soffset = LLVMGetParam(ctx->radeon_bld.main_fn,
2204 ctx->param_es2gs_offset);
2205 unsigned chan;
2206 int i;
2207
2208 for (i = 0; i < info->num_outputs; i++) {
2209 LLVMValueRef *out_ptr =
2210 ctx->radeon_bld.soa.outputs[i];
2211 int param_index;
2212
2213 if (info->output_semantic_name[i] == TGSI_SEMANTIC_VIEWPORT_INDEX ||
2214 info->output_semantic_name[i] == TGSI_SEMANTIC_LAYER)
2215 continue;
2216
2217 param_index = si_shader_io_get_unique_index(info->output_semantic_name[i],
2218 info->output_semantic_index[i]);
2219
2220 for (chan = 0; chan < 4; chan++) {
2221 LLVMValueRef out_val = LLVMBuildLoad(gallivm->builder, out_ptr[chan], "");
2222 out_val = LLVMBuildBitCast(gallivm->builder, out_val, ctx->i32, "");
2223
2224 build_tbuffer_store(ctx,
2225 ctx->esgs_ring,
2226 out_val, 1,
2227 LLVMGetUndef(ctx->i32), soffset,
2228 (4 * param_index + chan) * 4,
2229 V_008F0C_BUF_DATA_FORMAT_32,
2230 V_008F0C_BUF_NUM_FORMAT_UINT,
2231 0, 0, 1, 1, 0);
2232 }
2233 }
2234 }
2235
2236 static void si_llvm_emit_gs_epilogue(struct lp_build_tgsi_context *bld_base)
2237 {
2238 struct si_shader_context *ctx = si_shader_context(bld_base);
2239 struct gallivm_state *gallivm = bld_base->base.gallivm;
2240 LLVMValueRef args[2];
2241
2242 args[0] = lp_build_const_int32(gallivm, SENDMSG_GS_OP_NOP | SENDMSG_GS_DONE);
2243 args[1] = LLVMGetParam(ctx->radeon_bld.main_fn, SI_PARAM_GS_WAVE_ID);
2244 lp_build_intrinsic(gallivm->builder, "llvm.SI.sendmsg",
2245 ctx->voidt, args, 2, LLVMNoUnwindAttribute);
2246 }
2247
2248 static void si_llvm_emit_vs_epilogue(struct lp_build_tgsi_context *bld_base)
2249 {
2250 struct si_shader_context *ctx = si_shader_context(bld_base);
2251 struct gallivm_state *gallivm = bld_base->base.gallivm;
2252 struct tgsi_shader_info *info = &ctx->shader->selector->info;
2253 struct si_shader_output_values *outputs = NULL;
2254 int i,j;
2255
2256 assert(!ctx->is_gs_copy_shader);
2257
2258 outputs = MALLOC((info->num_outputs + 1) * sizeof(outputs[0]));
2259
2260 /* Vertex color clamping.
2261 *
2262 * This uses a state constant loaded in a user data SGPR and
2263 * an IF statement is added that clamps all colors if the constant
2264 * is true.
2265 */
2266 if (ctx->type == TGSI_PROCESSOR_VERTEX) {
2267 struct lp_build_if_state if_ctx;
2268 LLVMValueRef cond = NULL;
2269 LLVMValueRef addr, val;
2270
2271 for (i = 0; i < info->num_outputs; i++) {
2272 if (info->output_semantic_name[i] != TGSI_SEMANTIC_COLOR &&
2273 info->output_semantic_name[i] != TGSI_SEMANTIC_BCOLOR)
2274 continue;
2275
2276 /* We've found a color. */
2277 if (!cond) {
2278 /* The state is in the first bit of the user SGPR. */
2279 cond = LLVMGetParam(ctx->radeon_bld.main_fn,
2280 SI_PARAM_VS_STATE_BITS);
2281 cond = LLVMBuildTrunc(gallivm->builder, cond,
2282 ctx->i1, "");
2283 lp_build_if(&if_ctx, gallivm, cond);
2284 }
2285
2286 for (j = 0; j < 4; j++) {
2287 addr = ctx->radeon_bld.soa.outputs[i][j];
2288 val = LLVMBuildLoad(gallivm->builder, addr, "");
2289 val = radeon_llvm_saturate(bld_base, val);
2290 LLVMBuildStore(gallivm->builder, val, addr);
2291 }
2292 }
2293
2294 if (cond)
2295 lp_build_endif(&if_ctx);
2296 }
2297
2298 for (i = 0; i < info->num_outputs; i++) {
2299 outputs[i].name = info->output_semantic_name[i];
2300 outputs[i].sid = info->output_semantic_index[i];
2301
2302 for (j = 0; j < 4; j++)
2303 outputs[i].values[j] =
2304 LLVMBuildLoad(gallivm->builder,
2305 ctx->radeon_bld.soa.outputs[i][j],
2306 "");
2307 }
2308
2309 if (ctx->is_monolithic) {
2310 /* Export PrimitiveID when PS needs it. */
2311 if (si_vs_exports_prim_id(ctx->shader)) {
2312 outputs[i].name = TGSI_SEMANTIC_PRIMID;
2313 outputs[i].sid = 0;
2314 outputs[i].values[0] = bitcast(bld_base, TGSI_TYPE_FLOAT,
2315 get_primitive_id(bld_base, 0));
2316 outputs[i].values[1] = bld_base->base.undef;
2317 outputs[i].values[2] = bld_base->base.undef;
2318 outputs[i].values[3] = bld_base->base.undef;
2319 i++;
2320 }
2321 } else {
2322 /* Return the primitive ID from the LLVM function. */
2323 ctx->return_value =
2324 LLVMBuildInsertValue(gallivm->builder,
2325 ctx->return_value,
2326 bitcast(bld_base, TGSI_TYPE_FLOAT,
2327 get_primitive_id(bld_base, 0)),
2328 VS_EPILOG_PRIMID_LOC, "");
2329 }
2330
2331 si_llvm_export_vs(bld_base, outputs, i);
2332 FREE(outputs);
2333 }
2334
2335 static void si_export_mrt_z(struct lp_build_tgsi_context *bld_base,
2336 LLVMValueRef depth, LLVMValueRef stencil,
2337 LLVMValueRef samplemask)
2338 {
2339 struct si_shader_context *ctx = si_shader_context(bld_base);
2340 struct lp_build_context *base = &bld_base->base;
2341 struct lp_build_context *uint = &bld_base->uint_bld;
2342 LLVMValueRef args[9];
2343 unsigned mask = 0;
2344
2345 assert(depth || stencil || samplemask);
2346
2347 args[1] = uint->one; /* whether the EXEC mask is valid */
2348 args[2] = uint->one; /* DONE bit */
2349
2350 /* Specify the target we are exporting */
2351 args[3] = lp_build_const_int32(base->gallivm, V_008DFC_SQ_EXP_MRTZ);
2352
2353 args[4] = uint->zero; /* COMP flag */
2354 args[5] = base->undef; /* R, depth */
2355 args[6] = base->undef; /* G, stencil test value[0:7], stencil op value[8:15] */
2356 args[7] = base->undef; /* B, sample mask */
2357 args[8] = base->undef; /* A, alpha to mask */
2358
2359 if (depth) {
2360 args[5] = depth;
2361 mask |= 0x1;
2362 }
2363
2364 if (stencil) {
2365 args[6] = stencil;
2366 mask |= 0x2;
2367 }
2368
2369 if (samplemask) {
2370 args[7] = samplemask;
2371 mask |= 0x4;
2372 }
2373
2374 /* SI (except OLAND) has a bug that it only looks
2375 * at the X writemask component. */
2376 if (ctx->screen->b.chip_class == SI &&
2377 ctx->screen->b.family != CHIP_OLAND)
2378 mask |= 0x1;
2379
2380 /* Specify which components to enable */
2381 args[0] = lp_build_const_int32(base->gallivm, mask);
2382
2383 lp_build_intrinsic(base->gallivm->builder, "llvm.SI.export",
2384 ctx->voidt, args, 9, 0);
2385 }
2386
2387 static void si_export_mrt_color(struct lp_build_tgsi_context *bld_base,
2388 LLVMValueRef *color, unsigned index,
2389 unsigned samplemask_param,
2390 bool is_last)
2391 {
2392 struct si_shader_context *ctx = si_shader_context(bld_base);
2393 struct lp_build_context *base = &bld_base->base;
2394 int i;
2395
2396 /* Clamp color */
2397 if (ctx->shader->key.ps.epilog.clamp_color)
2398 for (i = 0; i < 4; i++)
2399 color[i] = radeon_llvm_saturate(bld_base, color[i]);
2400
2401 /* Alpha to one */
2402 if (ctx->shader->key.ps.epilog.alpha_to_one)
2403 color[3] = base->one;
2404
2405 /* Alpha test */
2406 if (index == 0 &&
2407 ctx->shader->key.ps.epilog.alpha_func != PIPE_FUNC_ALWAYS)
2408 si_alpha_test(bld_base, color[3]);
2409
2410 /* Line & polygon smoothing */
2411 if (ctx->shader->key.ps.epilog.poly_line_smoothing)
2412 color[3] = si_scale_alpha_by_sample_mask(bld_base, color[3],
2413 samplemask_param);
2414
2415 /* If last_cbuf > 0, FS_COLOR0_WRITES_ALL_CBUFS is true. */
2416 if (ctx->shader->key.ps.epilog.last_cbuf > 0) {
2417 LLVMValueRef args[8][9];
2418 int c, last = -1;
2419
2420 /* Get the export arguments, also find out what the last one is. */
2421 for (c = 0; c <= ctx->shader->key.ps.epilog.last_cbuf; c++) {
2422 si_llvm_init_export_args(bld_base, color,
2423 V_008DFC_SQ_EXP_MRT + c, args[c]);
2424 if (args[c][0] != bld_base->uint_bld.zero)
2425 last = c;
2426 }
2427
2428 /* Emit all exports. */
2429 for (c = 0; c <= ctx->shader->key.ps.epilog.last_cbuf; c++) {
2430 if (is_last && last == c) {
2431 args[c][1] = bld_base->uint_bld.one; /* whether the EXEC mask is valid */
2432 args[c][2] = bld_base->uint_bld.one; /* DONE bit */
2433 } else if (args[c][0] == bld_base->uint_bld.zero)
2434 continue; /* unnecessary NULL export */
2435
2436 lp_build_intrinsic(base->gallivm->builder, "llvm.SI.export",
2437 ctx->voidt, args[c], 9, 0);
2438 }
2439 } else {
2440 LLVMValueRef args[9];
2441
2442 /* Export */
2443 si_llvm_init_export_args(bld_base, color, V_008DFC_SQ_EXP_MRT + index,
2444 args);
2445 if (is_last) {
2446 args[1] = bld_base->uint_bld.one; /* whether the EXEC mask is valid */
2447 args[2] = bld_base->uint_bld.one; /* DONE bit */
2448 } else if (args[0] == bld_base->uint_bld.zero)
2449 return; /* unnecessary NULL export */
2450
2451 lp_build_intrinsic(base->gallivm->builder, "llvm.SI.export",
2452 ctx->voidt, args, 9, 0);
2453 }
2454 }
2455
2456 static void si_export_null(struct lp_build_tgsi_context *bld_base)
2457 {
2458 struct si_shader_context *ctx = si_shader_context(bld_base);
2459 struct lp_build_context *base = &bld_base->base;
2460 struct lp_build_context *uint = &bld_base->uint_bld;
2461 LLVMValueRef args[9];
2462
2463 args[0] = lp_build_const_int32(base->gallivm, 0x0); /* enabled channels */
2464 args[1] = uint->one; /* whether the EXEC mask is valid */
2465 args[2] = uint->one; /* DONE bit */
2466 args[3] = lp_build_const_int32(base->gallivm, V_008DFC_SQ_EXP_NULL);
2467 args[4] = uint->zero; /* COMPR flag (0 = 32-bit export) */
2468 args[5] = uint->undef; /* R */
2469 args[6] = uint->undef; /* G */
2470 args[7] = uint->undef; /* B */
2471 args[8] = uint->undef; /* A */
2472
2473 lp_build_intrinsic(base->gallivm->builder, "llvm.SI.export",
2474 ctx->voidt, args, 9, 0);
2475 }
2476
2477 static void si_llvm_emit_fs_epilogue(struct lp_build_tgsi_context *bld_base)
2478 {
2479 struct si_shader_context *ctx = si_shader_context(bld_base);
2480 struct si_shader *shader = ctx->shader;
2481 struct lp_build_context *base = &bld_base->base;
2482 struct tgsi_shader_info *info = &shader->selector->info;
2483 LLVMBuilderRef builder = base->gallivm->builder;
2484 LLVMValueRef depth = NULL, stencil = NULL, samplemask = NULL;
2485 int last_color_export = -1;
2486 int i;
2487
2488 /* Determine the last export. If MRTZ is present, it's always last.
2489 * Otherwise, find the last color export.
2490 */
2491 if (!info->writes_z && !info->writes_stencil && !info->writes_samplemask) {
2492 unsigned spi_format = shader->key.ps.epilog.spi_shader_col_format;
2493
2494 /* Don't export NULL and return if alpha-test is enabled. */
2495 if (shader->key.ps.epilog.alpha_func != PIPE_FUNC_ALWAYS &&
2496 shader->key.ps.epilog.alpha_func != PIPE_FUNC_NEVER &&
2497 (spi_format & 0xf) == 0)
2498 spi_format |= V_028714_SPI_SHADER_32_AR;
2499
2500 for (i = 0; i < info->num_outputs; i++) {
2501 unsigned index = info->output_semantic_index[i];
2502
2503 if (info->output_semantic_name[i] != TGSI_SEMANTIC_COLOR)
2504 continue;
2505
2506 /* If last_cbuf > 0, FS_COLOR0_WRITES_ALL_CBUFS is true. */
2507 if (shader->key.ps.epilog.last_cbuf > 0) {
2508 /* Just set this if any of the colorbuffers are enabled. */
2509 if (spi_format &
2510 ((1llu << (4 * (shader->key.ps.epilog.last_cbuf + 1))) - 1))
2511 last_color_export = i;
2512 continue;
2513 }
2514
2515 if ((spi_format >> (index * 4)) & 0xf)
2516 last_color_export = i;
2517 }
2518
2519 /* If there are no outputs, export NULL. */
2520 if (last_color_export == -1) {
2521 si_export_null(bld_base);
2522 return;
2523 }
2524 }
2525
2526 for (i = 0; i < info->num_outputs; i++) {
2527 unsigned semantic_name = info->output_semantic_name[i];
2528 unsigned semantic_index = info->output_semantic_index[i];
2529 unsigned j;
2530 LLVMValueRef color[4] = {};
2531
2532 /* Select the correct target */
2533 switch (semantic_name) {
2534 case TGSI_SEMANTIC_POSITION:
2535 depth = LLVMBuildLoad(builder,
2536 ctx->radeon_bld.soa.outputs[i][2], "");
2537 break;
2538 case TGSI_SEMANTIC_STENCIL:
2539 stencil = LLVMBuildLoad(builder,
2540 ctx->radeon_bld.soa.outputs[i][1], "");
2541 break;
2542 case TGSI_SEMANTIC_SAMPLEMASK:
2543 samplemask = LLVMBuildLoad(builder,
2544 ctx->radeon_bld.soa.outputs[i][0], "");
2545 break;
2546 case TGSI_SEMANTIC_COLOR:
2547 for (j = 0; j < 4; j++)
2548 color[j] = LLVMBuildLoad(builder,
2549 ctx->radeon_bld.soa.outputs[i][j], "");
2550
2551 si_export_mrt_color(bld_base, color, semantic_index,
2552 SI_PARAM_SAMPLE_COVERAGE,
2553 last_color_export == i);
2554 break;
2555 default:
2556 fprintf(stderr,
2557 "Warning: SI unhandled fs output type:%d\n",
2558 semantic_name);
2559 }
2560 }
2561
2562 if (depth || stencil || samplemask)
2563 si_export_mrt_z(bld_base, depth, stencil, samplemask);
2564 }
2565
2566 /**
2567 * Return PS outputs in this order:
2568 *
2569 * v[0:3] = color0.xyzw
2570 * v[4:7] = color1.xyzw
2571 * ...
2572 * vN+0 = Depth
2573 * vN+1 = Stencil
2574 * vN+2 = SampleMask
2575 * vN+3 = SampleMaskIn (used for OpenGL smoothing)
2576 *
2577 * The alpha-ref SGPR is returned via its original location.
2578 */
2579 static void si_llvm_return_fs_outputs(struct lp_build_tgsi_context *bld_base)
2580 {
2581 struct si_shader_context *ctx = si_shader_context(bld_base);
2582 struct si_shader *shader = ctx->shader;
2583 struct lp_build_context *base = &bld_base->base;
2584 struct tgsi_shader_info *info = &shader->selector->info;
2585 LLVMBuilderRef builder = base->gallivm->builder;
2586 unsigned i, j, first_vgpr, vgpr;
2587
2588 LLVMValueRef color[8][4] = {};
2589 LLVMValueRef depth = NULL, stencil = NULL, samplemask = NULL;
2590 LLVMValueRef ret;
2591
2592 /* Read the output values. */
2593 for (i = 0; i < info->num_outputs; i++) {
2594 unsigned semantic_name = info->output_semantic_name[i];
2595 unsigned semantic_index = info->output_semantic_index[i];
2596
2597 switch (semantic_name) {
2598 case TGSI_SEMANTIC_COLOR:
2599 assert(semantic_index < 8);
2600 for (j = 0; j < 4; j++) {
2601 LLVMValueRef ptr = ctx->radeon_bld.soa.outputs[i][j];
2602 LLVMValueRef result = LLVMBuildLoad(builder, ptr, "");
2603 color[semantic_index][j] = result;
2604 }
2605 break;
2606 case TGSI_SEMANTIC_POSITION:
2607 depth = LLVMBuildLoad(builder,
2608 ctx->radeon_bld.soa.outputs[i][2], "");
2609 break;
2610 case TGSI_SEMANTIC_STENCIL:
2611 stencil = LLVMBuildLoad(builder,
2612 ctx->radeon_bld.soa.outputs[i][1], "");
2613 break;
2614 case TGSI_SEMANTIC_SAMPLEMASK:
2615 samplemask = LLVMBuildLoad(builder,
2616 ctx->radeon_bld.soa.outputs[i][0], "");
2617 break;
2618 default:
2619 fprintf(stderr, "Warning: SI unhandled fs output type:%d\n",
2620 semantic_name);
2621 }
2622 }
2623
2624 /* Fill the return structure. */
2625 ret = ctx->return_value;
2626
2627 /* Set SGPRs. */
2628 ret = LLVMBuildInsertValue(builder, ret,
2629 bitcast(bld_base, TGSI_TYPE_SIGNED,
2630 LLVMGetParam(ctx->radeon_bld.main_fn,
2631 SI_PARAM_ALPHA_REF)),
2632 SI_SGPR_ALPHA_REF, "");
2633
2634 /* Set VGPRs */
2635 first_vgpr = vgpr = SI_SGPR_ALPHA_REF + 1;
2636 for (i = 0; i < ARRAY_SIZE(color); i++) {
2637 if (!color[i][0])
2638 continue;
2639
2640 for (j = 0; j < 4; j++)
2641 ret = LLVMBuildInsertValue(builder, ret, color[i][j], vgpr++, "");
2642 }
2643 if (depth)
2644 ret = LLVMBuildInsertValue(builder, ret, depth, vgpr++, "");
2645 if (stencil)
2646 ret = LLVMBuildInsertValue(builder, ret, stencil, vgpr++, "");
2647 if (samplemask)
2648 ret = LLVMBuildInsertValue(builder, ret, samplemask, vgpr++, "");
2649
2650 /* Add the input sample mask for smoothing at the end. */
2651 if (vgpr < first_vgpr + PS_EPILOG_SAMPLEMASK_MIN_LOC)
2652 vgpr = first_vgpr + PS_EPILOG_SAMPLEMASK_MIN_LOC;
2653 ret = LLVMBuildInsertValue(builder, ret,
2654 LLVMGetParam(ctx->radeon_bld.main_fn,
2655 SI_PARAM_SAMPLE_COVERAGE), vgpr++, "");
2656
2657 ctx->return_value = ret;
2658 }
2659
2660 /**
2661 * Given a v8i32 resource descriptor for a buffer, extract the size of the
2662 * buffer in number of elements and return it as an i32.
2663 */
2664 static LLVMValueRef get_buffer_size(
2665 struct lp_build_tgsi_context *bld_base,
2666 LLVMValueRef descriptor)
2667 {
2668 struct si_shader_context *ctx = si_shader_context(bld_base);
2669 struct gallivm_state *gallivm = bld_base->base.gallivm;
2670 LLVMBuilderRef builder = gallivm->builder;
2671 LLVMValueRef size =
2672 LLVMBuildExtractElement(builder, descriptor,
2673 lp_build_const_int32(gallivm, 6), "");
2674
2675 if (ctx->screen->b.chip_class >= VI) {
2676 /* On VI, the descriptor contains the size in bytes,
2677 * but TXQ must return the size in elements.
2678 * The stride is always non-zero for resources using TXQ.
2679 */
2680 LLVMValueRef stride =
2681 LLVMBuildExtractElement(builder, descriptor,
2682 lp_build_const_int32(gallivm, 5), "");
2683 stride = LLVMBuildLShr(builder, stride,
2684 lp_build_const_int32(gallivm, 16), "");
2685 stride = LLVMBuildAnd(builder, stride,
2686 lp_build_const_int32(gallivm, 0x3FFF), "");
2687
2688 size = LLVMBuildUDiv(builder, size, stride, "");
2689 }
2690
2691 return size;
2692 }
2693
2694 /**
2695 * Given the i32 or vNi32 \p type, generate the textual name (e.g. for use with
2696 * intrinsic names).
2697 */
2698 static void build_int_type_name(
2699 LLVMTypeRef type,
2700 char *buf, unsigned bufsize)
2701 {
2702 assert(bufsize >= 6);
2703
2704 if (LLVMGetTypeKind(type) == LLVMVectorTypeKind)
2705 snprintf(buf, bufsize, "v%ui32",
2706 LLVMGetVectorSize(type));
2707 else
2708 strcpy(buf, "i32");
2709 }
2710
2711 static void build_tex_intrinsic(const struct lp_build_tgsi_action *action,
2712 struct lp_build_tgsi_context *bld_base,
2713 struct lp_build_emit_data *emit_data);
2714
2715 static bool tgsi_is_array_sampler(unsigned target)
2716 {
2717 return target == TGSI_TEXTURE_1D_ARRAY ||
2718 target == TGSI_TEXTURE_SHADOW1D_ARRAY ||
2719 target == TGSI_TEXTURE_2D_ARRAY ||
2720 target == TGSI_TEXTURE_SHADOW2D_ARRAY ||
2721 target == TGSI_TEXTURE_CUBE_ARRAY ||
2722 target == TGSI_TEXTURE_SHADOWCUBE_ARRAY ||
2723 target == TGSI_TEXTURE_2D_ARRAY_MSAA;
2724 }
2725
2726 static bool tgsi_is_array_image(unsigned target)
2727 {
2728 return target == TGSI_TEXTURE_3D ||
2729 target == TGSI_TEXTURE_CUBE ||
2730 target == TGSI_TEXTURE_1D_ARRAY ||
2731 target == TGSI_TEXTURE_2D_ARRAY ||
2732 target == TGSI_TEXTURE_CUBE_ARRAY ||
2733 target == TGSI_TEXTURE_2D_ARRAY_MSAA;
2734 }
2735
2736 /**
2737 * Load the resource descriptor for \p image.
2738 */
2739 static void
2740 image_fetch_rsrc(
2741 struct lp_build_tgsi_context *bld_base,
2742 const struct tgsi_full_src_register *image,
2743 LLVMValueRef *rsrc)
2744 {
2745 struct si_shader_context *ctx = si_shader_context(bld_base);
2746
2747 assert(image->Register.File == TGSI_FILE_IMAGE);
2748
2749 if (!image->Register.Indirect) {
2750 /* Fast path: use preloaded resources */
2751 *rsrc = ctx->images[image->Register.Index];
2752 } else {
2753 /* Indexing and manual load */
2754 LLVMValueRef ind_index;
2755 LLVMValueRef rsrc_ptr;
2756
2757 ind_index = get_indirect_index(ctx, &image->Indirect, image->Register.Index);
2758
2759 rsrc_ptr = LLVMGetParam(ctx->radeon_bld.main_fn, SI_PARAM_IMAGES);
2760 *rsrc = build_indexed_load_const(ctx, rsrc_ptr, ind_index);
2761 }
2762 }
2763
2764 static LLVMValueRef image_fetch_coords(
2765 struct lp_build_tgsi_context *bld_base,
2766 const struct tgsi_full_instruction *inst)
2767 {
2768 struct gallivm_state *gallivm = bld_base->base.gallivm;
2769 LLVMBuilderRef builder = gallivm->builder;
2770 unsigned target = inst->Memory.Texture;
2771 int sample;
2772 unsigned num_coords = tgsi_util_get_texture_coord_dim(target, &sample);
2773 LLVMValueRef coords[4];
2774 LLVMValueRef tmp;
2775 int chan;
2776
2777 for (chan = 0; chan < num_coords; ++chan) {
2778 tmp = lp_build_emit_fetch(bld_base, inst, 1, chan);
2779 tmp = LLVMBuildBitCast(builder, tmp, bld_base->uint_bld.elem_type, "");
2780 coords[chan] = tmp;
2781 }
2782
2783 if (num_coords == 1)
2784 return coords[0];
2785
2786 if (num_coords == 3) {
2787 /* LLVM has difficulties lowering 3-element vectors. */
2788 coords[3] = bld_base->uint_bld.undef;
2789 num_coords = 4;
2790 }
2791
2792 return lp_build_gather_values(gallivm, coords, num_coords);
2793 }
2794
2795 /**
2796 * Append the extra mode bits that are used by image load and store.
2797 */
2798 static void image_append_args(
2799 struct si_shader_context *ctx,
2800 struct lp_build_emit_data * emit_data,
2801 unsigned target)
2802 {
2803 LLVMValueRef i1false = LLVMConstInt(ctx->i1, 0, 0);
2804 LLVMValueRef i1true = LLVMConstInt(ctx->i1, 1, 0);
2805
2806 emit_data->args[emit_data->arg_count++] = i1false; /* r128 */
2807 emit_data->args[emit_data->arg_count++] =
2808 tgsi_is_array_image(target) ? i1true : i1false; /* da */
2809 emit_data->args[emit_data->arg_count++] = i1false; /* glc */
2810 emit_data->args[emit_data->arg_count++] = i1false; /* slc */
2811 }
2812
2813 /**
2814 * Append the resource and indexing arguments for buffer intrinsics.
2815 *
2816 * \param rsrc the 256 bit resource
2817 * \param index index into the buffer
2818 */
2819 static void buffer_append_args(
2820 struct si_shader_context *ctx,
2821 struct lp_build_emit_data *emit_data,
2822 LLVMValueRef rsrc,
2823 LLVMValueRef index)
2824 {
2825 struct gallivm_state *gallivm = &ctx->radeon_bld.gallivm;
2826 struct lp_build_tgsi_context *bld_base = &ctx->radeon_bld.soa.bld_base;
2827 LLVMTypeRef v2i128 = LLVMVectorType(ctx->i128, 2);
2828 LLVMValueRef i1false = LLVMConstInt(ctx->i1, 0, 0);
2829
2830 rsrc = LLVMBuildBitCast(gallivm->builder, rsrc, v2i128, "");
2831 rsrc = LLVMBuildExtractElement(gallivm->builder, rsrc, bld_base->uint_bld.one, "");
2832 rsrc = LLVMBuildBitCast(gallivm->builder, rsrc, ctx->v4i32, "");
2833
2834 emit_data->args[emit_data->arg_count++] = rsrc;
2835 emit_data->args[emit_data->arg_count++] = index; /* vindex */
2836 emit_data->args[emit_data->arg_count++] = bld_base->uint_bld.zero; /* voffset */
2837 emit_data->args[emit_data->arg_count++] = i1false; /* glc */
2838 emit_data->args[emit_data->arg_count++] = i1false; /* slc */
2839 }
2840
2841 static void load_fetch_args(
2842 struct lp_build_tgsi_context * bld_base,
2843 struct lp_build_emit_data * emit_data)
2844 {
2845 struct si_shader_context *ctx = si_shader_context(bld_base);
2846 struct gallivm_state *gallivm = bld_base->base.gallivm;
2847 const struct tgsi_full_instruction * inst = emit_data->inst;
2848 unsigned target = inst->Memory.Texture;
2849 LLVMValueRef coords;
2850 LLVMValueRef rsrc;
2851
2852 emit_data->dst_type = LLVMVectorType(bld_base->base.elem_type, 4);
2853
2854 image_fetch_rsrc(bld_base, &inst->Src[0], &rsrc);
2855 coords = image_fetch_coords(bld_base, inst);
2856
2857 if (target == TGSI_TEXTURE_BUFFER) {
2858 buffer_append_args(ctx, emit_data, rsrc, coords);
2859 } else {
2860 emit_data->args[0] = coords;
2861 emit_data->args[1] = rsrc;
2862 emit_data->args[2] = lp_build_const_int32(gallivm, 15); /* dmask */
2863 emit_data->arg_count = 3;
2864
2865 image_append_args(ctx, emit_data, target);
2866 }
2867 }
2868
2869 static void load_emit(
2870 const struct lp_build_tgsi_action *action,
2871 struct lp_build_tgsi_context *bld_base,
2872 struct lp_build_emit_data *emit_data)
2873 {
2874 struct gallivm_state *gallivm = bld_base->base.gallivm;
2875 LLVMBuilderRef builder = gallivm->builder;
2876 const struct tgsi_full_instruction * inst = emit_data->inst;
2877 unsigned target = inst->Memory.Texture;
2878 char intrinsic_name[32];
2879 char coords_type[8];
2880
2881 if (target == TGSI_TEXTURE_BUFFER) {
2882 emit_data->output[emit_data->chan] = lp_build_intrinsic(
2883 builder, "llvm.amdgcn.buffer.load.format.v4f32", emit_data->dst_type,
2884 emit_data->args, emit_data->arg_count,
2885 LLVMReadOnlyAttribute | LLVMNoUnwindAttribute);
2886 } else {
2887 build_int_type_name(LLVMTypeOf(emit_data->args[0]),
2888 coords_type, sizeof(coords_type));
2889
2890 snprintf(intrinsic_name, sizeof(intrinsic_name),
2891 "llvm.amdgcn.image.load.%s", coords_type);
2892
2893 emit_data->output[emit_data->chan] =
2894 lp_build_intrinsic(
2895 builder, intrinsic_name, emit_data->dst_type,
2896 emit_data->args, emit_data->arg_count,
2897 LLVMReadOnlyAttribute | LLVMNoUnwindAttribute);
2898 }
2899 }
2900
2901 static void resq_fetch_args(
2902 struct lp_build_tgsi_context * bld_base,
2903 struct lp_build_emit_data * emit_data)
2904 {
2905 struct gallivm_state *gallivm = bld_base->base.gallivm;
2906 const struct tgsi_full_instruction *inst = emit_data->inst;
2907 const struct tgsi_full_src_register *reg = &inst->Src[0];
2908 unsigned tex_target = inst->Memory.Texture;
2909
2910 emit_data->dst_type = LLVMVectorType(bld_base->base.elem_type, 4);
2911
2912 if (tex_target == TGSI_TEXTURE_BUFFER) {
2913 image_fetch_rsrc(bld_base, reg, &emit_data->args[0]);
2914 emit_data->arg_count = 1;
2915 } else {
2916 emit_data->args[0] = bld_base->uint_bld.zero; /* mip level */
2917 image_fetch_rsrc(bld_base, reg, &emit_data->args[1]);
2918 emit_data->args[2] = lp_build_const_int32(gallivm, 15); /* dmask */
2919 emit_data->args[3] = bld_base->uint_bld.zero; /* unorm */
2920 emit_data->args[4] = bld_base->uint_bld.zero; /* r128 */
2921 emit_data->args[5] = tgsi_is_array_image(tex_target) ?
2922 bld_base->uint_bld.one : bld_base->uint_bld.zero; /* da */
2923 emit_data->args[6] = bld_base->uint_bld.zero; /* glc */
2924 emit_data->args[7] = bld_base->uint_bld.zero; /* slc */
2925 emit_data->args[8] = bld_base->uint_bld.zero; /* tfe */
2926 emit_data->args[9] = bld_base->uint_bld.zero; /* lwe */
2927 emit_data->arg_count = 10;
2928 }
2929 }
2930
2931 static void resq_emit(
2932 const struct lp_build_tgsi_action *action,
2933 struct lp_build_tgsi_context *bld_base,
2934 struct lp_build_emit_data *emit_data)
2935 {
2936 struct gallivm_state *gallivm = bld_base->base.gallivm;
2937 LLVMBuilderRef builder = gallivm->builder;
2938 const struct tgsi_full_instruction *inst = emit_data->inst;
2939 unsigned target = inst->Memory.Texture;
2940 LLVMValueRef out;
2941
2942 if (target == TGSI_TEXTURE_BUFFER) {
2943 out = get_buffer_size(bld_base, emit_data->args[0]);
2944 } else {
2945 out = lp_build_intrinsic(
2946 builder, "llvm.SI.getresinfo.i32", emit_data->dst_type,
2947 emit_data->args, emit_data->arg_count,
2948 LLVMReadNoneAttribute | LLVMNoUnwindAttribute);
2949
2950 /* Divide the number of layers by 6 to get the number of cubes. */
2951 if (target == TGSI_TEXTURE_CUBE_ARRAY) {
2952 LLVMValueRef imm2 = lp_build_const_int32(gallivm, 2);
2953 LLVMValueRef imm6 = lp_build_const_int32(gallivm, 6);
2954
2955 LLVMValueRef z = LLVMBuildExtractElement(builder, out, imm2, "");
2956 z = LLVMBuildBitCast(builder, z, bld_base->uint_bld.elem_type, "");
2957 z = LLVMBuildSDiv(builder, z, imm6, "");
2958 z = LLVMBuildBitCast(builder, z, bld_base->base.elem_type, "");
2959 out = LLVMBuildInsertElement(builder, out, z, imm2, "");
2960 }
2961 }
2962
2963 emit_data->output[emit_data->chan] = out;
2964 }
2965
2966 static void set_tex_fetch_args(struct si_shader_context *ctx,
2967 struct lp_build_emit_data *emit_data,
2968 unsigned opcode, unsigned target,
2969 LLVMValueRef res_ptr, LLVMValueRef samp_ptr,
2970 LLVMValueRef *param, unsigned count,
2971 unsigned dmask)
2972 {
2973 struct gallivm_state *gallivm = &ctx->radeon_bld.gallivm;
2974 unsigned num_args;
2975 unsigned is_rect = target == TGSI_TEXTURE_RECT;
2976
2977 /* Pad to power of two vector */
2978 while (count < util_next_power_of_two(count))
2979 param[count++] = LLVMGetUndef(ctx->i32);
2980
2981 /* Texture coordinates. */
2982 if (count > 1)
2983 emit_data->args[0] = lp_build_gather_values(gallivm, param, count);
2984 else
2985 emit_data->args[0] = param[0];
2986
2987 /* Resource. */
2988 emit_data->args[1] = res_ptr;
2989 num_args = 2;
2990
2991 if (opcode == TGSI_OPCODE_TXF || opcode == TGSI_OPCODE_TXQ)
2992 emit_data->dst_type = ctx->v4i32;
2993 else {
2994 emit_data->dst_type = ctx->v4f32;
2995
2996 emit_data->args[num_args++] = samp_ptr;
2997 }
2998
2999 emit_data->args[num_args++] = lp_build_const_int32(gallivm, dmask);
3000 emit_data->args[num_args++] = lp_build_const_int32(gallivm, is_rect); /* unorm */
3001 emit_data->args[num_args++] = lp_build_const_int32(gallivm, 0); /* r128 */
3002 emit_data->args[num_args++] = lp_build_const_int32(gallivm,
3003 tgsi_is_array_sampler(target)); /* da */
3004 emit_data->args[num_args++] = lp_build_const_int32(gallivm, 0); /* glc */
3005 emit_data->args[num_args++] = lp_build_const_int32(gallivm, 0); /* slc */
3006 emit_data->args[num_args++] = lp_build_const_int32(gallivm, 0); /* tfe */
3007 emit_data->args[num_args++] = lp_build_const_int32(gallivm, 0); /* lwe */
3008
3009 emit_data->arg_count = num_args;
3010 }
3011
3012 static const struct lp_build_tgsi_action tex_action;
3013
3014 enum desc_type {
3015 DESC_IMAGE,
3016 DESC_FMASK,
3017 DESC_SAMPLER
3018 };
3019
3020 static LLVMTypeRef const_array(LLVMTypeRef elem_type, int num_elements)
3021 {
3022 return LLVMPointerType(LLVMArrayType(elem_type, num_elements),
3023 CONST_ADDR_SPACE);
3024 }
3025
3026 /**
3027 * Load an image view, fmask view. or sampler state descriptor.
3028 */
3029 static LLVMValueRef get_sampler_desc_custom(struct si_shader_context *ctx,
3030 LLVMValueRef list, LLVMValueRef index,
3031 enum desc_type type)
3032 {
3033 struct gallivm_state *gallivm = &ctx->radeon_bld.gallivm;
3034 LLVMBuilderRef builder = gallivm->builder;
3035
3036 switch (type) {
3037 case DESC_IMAGE:
3038 /* The image is at [0:7]. */
3039 index = LLVMBuildMul(builder, index, LLVMConstInt(ctx->i32, 2, 0), "");
3040 break;
3041 case DESC_FMASK:
3042 /* The FMASK is at [8:15]. */
3043 index = LLVMBuildMul(builder, index, LLVMConstInt(ctx->i32, 2, 0), "");
3044 index = LLVMBuildAdd(builder, index, LLVMConstInt(ctx->i32, 1, 0), "");
3045 break;
3046 case DESC_SAMPLER:
3047 /* The sampler state is at [12:15]. */
3048 index = LLVMBuildMul(builder, index, LLVMConstInt(ctx->i32, 4, 0), "");
3049 index = LLVMBuildAdd(builder, index, LLVMConstInt(ctx->i32, 3, 0), "");
3050 list = LLVMBuildPointerCast(builder, list,
3051 const_array(ctx->v4i32, 0), "");
3052 break;
3053 }
3054
3055 return build_indexed_load_const(ctx, list, index);
3056 }
3057
3058 static LLVMValueRef get_sampler_desc(struct si_shader_context *ctx,
3059 LLVMValueRef index, enum desc_type type)
3060 {
3061 LLVMValueRef list = LLVMGetParam(ctx->radeon_bld.main_fn,
3062 SI_PARAM_SAMPLERS);
3063
3064 return get_sampler_desc_custom(ctx, list, index, type);
3065 }
3066
3067 static void tex_fetch_ptrs(
3068 struct lp_build_tgsi_context *bld_base,
3069 struct lp_build_emit_data *emit_data,
3070 LLVMValueRef *res_ptr, LLVMValueRef *samp_ptr, LLVMValueRef *fmask_ptr)
3071 {
3072 struct si_shader_context *ctx = si_shader_context(bld_base);
3073 const struct tgsi_full_instruction *inst = emit_data->inst;
3074 unsigned target = inst->Texture.Texture;
3075 unsigned sampler_src;
3076 unsigned sampler_index;
3077
3078 sampler_src = emit_data->inst->Instruction.NumSrcRegs - 1;
3079 sampler_index = emit_data->inst->Src[sampler_src].Register.Index;
3080
3081 if (emit_data->inst->Src[sampler_src].Register.Indirect) {
3082 const struct tgsi_full_src_register *reg = &emit_data->inst->Src[sampler_src];
3083 LLVMValueRef ind_index;
3084
3085 ind_index = get_indirect_index(ctx, &reg->Indirect, reg->Register.Index);
3086
3087 *res_ptr = get_sampler_desc(ctx, ind_index, DESC_IMAGE);
3088
3089 if (target == TGSI_TEXTURE_2D_MSAA ||
3090 target == TGSI_TEXTURE_2D_ARRAY_MSAA) {
3091 *samp_ptr = NULL;
3092 *fmask_ptr = get_sampler_desc(ctx, ind_index, DESC_FMASK);
3093 } else {
3094 *samp_ptr = get_sampler_desc(ctx, ind_index, DESC_SAMPLER);
3095 *fmask_ptr = NULL;
3096 }
3097 } else {
3098 *res_ptr = ctx->sampler_views[sampler_index];
3099 *samp_ptr = ctx->sampler_states[sampler_index];
3100 *fmask_ptr = ctx->fmasks[sampler_index];
3101 }
3102 }
3103
3104 static void tex_fetch_args(
3105 struct lp_build_tgsi_context *bld_base,
3106 struct lp_build_emit_data *emit_data)
3107 {
3108 struct si_shader_context *ctx = si_shader_context(bld_base);
3109 struct gallivm_state *gallivm = bld_base->base.gallivm;
3110 LLVMBuilderRef builder = gallivm->builder;
3111 const struct tgsi_full_instruction *inst = emit_data->inst;
3112 unsigned opcode = inst->Instruction.Opcode;
3113 unsigned target = inst->Texture.Texture;
3114 LLVMValueRef coords[5], derivs[6];
3115 LLVMValueRef address[16];
3116 int ref_pos;
3117 unsigned num_coords = tgsi_util_get_texture_coord_dim(target, &ref_pos);
3118 unsigned count = 0;
3119 unsigned chan;
3120 unsigned num_deriv_channels = 0;
3121 bool has_offset = inst->Texture.NumOffsets > 0;
3122 LLVMValueRef res_ptr, samp_ptr, fmask_ptr = NULL;
3123 unsigned dmask = 0xf;
3124
3125 tex_fetch_ptrs(bld_base, emit_data, &res_ptr, &samp_ptr, &fmask_ptr);
3126
3127 if (opcode == TGSI_OPCODE_TXQ) {
3128 if (target == TGSI_TEXTURE_BUFFER) {
3129 /* Read the size from the buffer descriptor directly. */
3130 LLVMValueRef res = LLVMBuildBitCast(builder, res_ptr, ctx->v8i32, "");
3131 emit_data->args[0] = get_buffer_size(bld_base, res);
3132 return;
3133 }
3134
3135 /* Textures - set the mip level. */
3136 address[count++] = lp_build_emit_fetch(bld_base, inst, 0, TGSI_CHAN_X);
3137
3138 set_tex_fetch_args(ctx, emit_data, opcode, target, res_ptr,
3139 NULL, address, count, 0xf);
3140 return;
3141 }
3142
3143 if (target == TGSI_TEXTURE_BUFFER) {
3144 LLVMTypeRef v2i128 = LLVMVectorType(ctx->i128, 2);
3145
3146 /* Bitcast and truncate v8i32 to v16i8. */
3147 LLVMValueRef res = res_ptr;
3148 res = LLVMBuildBitCast(gallivm->builder, res, v2i128, "");
3149 res = LLVMBuildExtractElement(gallivm->builder, res, bld_base->uint_bld.one, "");
3150 res = LLVMBuildBitCast(gallivm->builder, res, ctx->v16i8, "");
3151
3152 emit_data->dst_type = ctx->v4f32;
3153 emit_data->args[0] = res;
3154 emit_data->args[1] = bld_base->uint_bld.zero;
3155 emit_data->args[2] = lp_build_emit_fetch(bld_base, emit_data->inst, 0, TGSI_CHAN_X);
3156 emit_data->arg_count = 3;
3157 return;
3158 }
3159
3160 /* Fetch and project texture coordinates */
3161 coords[3] = lp_build_emit_fetch(bld_base, emit_data->inst, 0, TGSI_CHAN_W);
3162 for (chan = 0; chan < 3; chan++ ) {
3163 coords[chan] = lp_build_emit_fetch(bld_base,
3164 emit_data->inst, 0,
3165 chan);
3166 if (opcode == TGSI_OPCODE_TXP)
3167 coords[chan] = lp_build_emit_llvm_binary(bld_base,
3168 TGSI_OPCODE_DIV,
3169 coords[chan],
3170 coords[3]);
3171 }
3172
3173 if (opcode == TGSI_OPCODE_TXP)
3174 coords[3] = bld_base->base.one;
3175
3176 /* Pack offsets. */
3177 if (has_offset && opcode != TGSI_OPCODE_TXF) {
3178 /* The offsets are six-bit signed integers packed like this:
3179 * X=[5:0], Y=[13:8], and Z=[21:16].
3180 */
3181 LLVMValueRef offset[3], pack;
3182
3183 assert(inst->Texture.NumOffsets == 1);
3184
3185 for (chan = 0; chan < 3; chan++) {
3186 offset[chan] = lp_build_emit_fetch_texoffset(bld_base,
3187 emit_data->inst, 0, chan);
3188 offset[chan] = LLVMBuildAnd(gallivm->builder, offset[chan],
3189 lp_build_const_int32(gallivm, 0x3f), "");
3190 if (chan)
3191 offset[chan] = LLVMBuildShl(gallivm->builder, offset[chan],
3192 lp_build_const_int32(gallivm, chan*8), "");
3193 }
3194
3195 pack = LLVMBuildOr(gallivm->builder, offset[0], offset[1], "");
3196 pack = LLVMBuildOr(gallivm->builder, pack, offset[2], "");
3197 address[count++] = pack;
3198 }
3199
3200 /* Pack LOD bias value */
3201 if (opcode == TGSI_OPCODE_TXB)
3202 address[count++] = coords[3];
3203 if (opcode == TGSI_OPCODE_TXB2)
3204 address[count++] = lp_build_emit_fetch(bld_base, inst, 1, TGSI_CHAN_X);
3205
3206 /* Pack depth comparison value */
3207 if (tgsi_is_shadow_target(target) && opcode != TGSI_OPCODE_LODQ) {
3208 if (target == TGSI_TEXTURE_SHADOWCUBE_ARRAY) {
3209 address[count++] = lp_build_emit_fetch(bld_base, inst, 1, TGSI_CHAN_X);
3210 } else {
3211 assert(ref_pos >= 0);
3212 address[count++] = coords[ref_pos];
3213 }
3214 }
3215
3216 /* Pack user derivatives */
3217 if (opcode == TGSI_OPCODE_TXD) {
3218 int param, num_src_deriv_channels;
3219
3220 switch (target) {
3221 case TGSI_TEXTURE_3D:
3222 num_src_deriv_channels = 3;
3223 num_deriv_channels = 3;
3224 break;
3225 case TGSI_TEXTURE_2D:
3226 case TGSI_TEXTURE_SHADOW2D:
3227 case TGSI_TEXTURE_RECT:
3228 case TGSI_TEXTURE_SHADOWRECT:
3229 case TGSI_TEXTURE_2D_ARRAY:
3230 case TGSI_TEXTURE_SHADOW2D_ARRAY:
3231 num_src_deriv_channels = 2;
3232 num_deriv_channels = 2;
3233 break;
3234 case TGSI_TEXTURE_CUBE:
3235 case TGSI_TEXTURE_SHADOWCUBE:
3236 case TGSI_TEXTURE_CUBE_ARRAY:
3237 case TGSI_TEXTURE_SHADOWCUBE_ARRAY:
3238 /* Cube derivatives will be converted to 2D. */
3239 num_src_deriv_channels = 3;
3240 num_deriv_channels = 2;
3241 break;
3242 case TGSI_TEXTURE_1D:
3243 case TGSI_TEXTURE_SHADOW1D:
3244 case TGSI_TEXTURE_1D_ARRAY:
3245 case TGSI_TEXTURE_SHADOW1D_ARRAY:
3246 num_src_deriv_channels = 1;
3247 num_deriv_channels = 1;
3248 break;
3249 default:
3250 unreachable("invalid target");
3251 }
3252
3253 for (param = 0; param < 2; param++)
3254 for (chan = 0; chan < num_src_deriv_channels; chan++)
3255 derivs[param * num_src_deriv_channels + chan] =
3256 lp_build_emit_fetch(bld_base, inst, param+1, chan);
3257 }
3258
3259 if (target == TGSI_TEXTURE_CUBE ||
3260 target == TGSI_TEXTURE_CUBE_ARRAY ||
3261 target == TGSI_TEXTURE_SHADOWCUBE ||
3262 target == TGSI_TEXTURE_SHADOWCUBE_ARRAY)
3263 radeon_llvm_emit_prepare_cube_coords(bld_base, emit_data, coords, derivs);
3264
3265 if (opcode == TGSI_OPCODE_TXD)
3266 for (int i = 0; i < num_deriv_channels * 2; i++)
3267 address[count++] = derivs[i];
3268
3269 /* Pack texture coordinates */
3270 address[count++] = coords[0];
3271 if (num_coords > 1)
3272 address[count++] = coords[1];
3273 if (num_coords > 2)
3274 address[count++] = coords[2];
3275
3276 /* Pack LOD or sample index */
3277 if (opcode == TGSI_OPCODE_TXL || opcode == TGSI_OPCODE_TXF)
3278 address[count++] = coords[3];
3279 else if (opcode == TGSI_OPCODE_TXL2)
3280 address[count++] = lp_build_emit_fetch(bld_base, inst, 1, TGSI_CHAN_X);
3281
3282 if (count > 16) {
3283 assert(!"Cannot handle more than 16 texture address parameters");
3284 count = 16;
3285 }
3286
3287 for (chan = 0; chan < count; chan++ ) {
3288 address[chan] = LLVMBuildBitCast(gallivm->builder,
3289 address[chan], ctx->i32, "");
3290 }
3291
3292 /* Adjust the sample index according to FMASK.
3293 *
3294 * For uncompressed MSAA surfaces, FMASK should return 0x76543210,
3295 * which is the identity mapping. Each nibble says which physical sample
3296 * should be fetched to get that sample.
3297 *
3298 * For example, 0x11111100 means there are only 2 samples stored and
3299 * the second sample covers 3/4 of the pixel. When reading samples 0
3300 * and 1, return physical sample 0 (determined by the first two 0s
3301 * in FMASK), otherwise return physical sample 1.
3302 *
3303 * The sample index should be adjusted as follows:
3304 * sample_index = (fmask >> (sample_index * 4)) & 0xF;
3305 */
3306 if (target == TGSI_TEXTURE_2D_MSAA ||
3307 target == TGSI_TEXTURE_2D_ARRAY_MSAA) {
3308 struct lp_build_context *uint_bld = &bld_base->uint_bld;
3309 struct lp_build_emit_data txf_emit_data = *emit_data;
3310 LLVMValueRef txf_address[4];
3311 unsigned txf_count = count;
3312 struct tgsi_full_instruction inst = {};
3313
3314 memcpy(txf_address, address, sizeof(txf_address));
3315
3316 if (target == TGSI_TEXTURE_2D_MSAA) {
3317 txf_address[2] = bld_base->uint_bld.zero;
3318 }
3319 txf_address[3] = bld_base->uint_bld.zero;
3320
3321 /* Read FMASK using TXF. */
3322 inst.Instruction.Opcode = TGSI_OPCODE_TXF;
3323 inst.Texture.Texture = target;
3324 txf_emit_data.inst = &inst;
3325 txf_emit_data.chan = 0;
3326 set_tex_fetch_args(ctx, &txf_emit_data, TGSI_OPCODE_TXF,
3327 target, fmask_ptr, NULL,
3328 txf_address, txf_count, 0xf);
3329 build_tex_intrinsic(&tex_action, bld_base, &txf_emit_data);
3330
3331 /* Initialize some constants. */
3332 LLVMValueRef four = LLVMConstInt(ctx->i32, 4, 0);
3333 LLVMValueRef F = LLVMConstInt(ctx->i32, 0xF, 0);
3334
3335 /* Apply the formula. */
3336 LLVMValueRef fmask =
3337 LLVMBuildExtractElement(gallivm->builder,
3338 txf_emit_data.output[0],
3339 uint_bld->zero, "");
3340
3341 unsigned sample_chan = target == TGSI_TEXTURE_2D_MSAA ? 2 : 3;
3342
3343 LLVMValueRef sample_index4 =
3344 LLVMBuildMul(gallivm->builder, address[sample_chan], four, "");
3345
3346 LLVMValueRef shifted_fmask =
3347 LLVMBuildLShr(gallivm->builder, fmask, sample_index4, "");
3348
3349 LLVMValueRef final_sample =
3350 LLVMBuildAnd(gallivm->builder, shifted_fmask, F, "");
3351
3352 /* Don't rewrite the sample index if WORD1.DATA_FORMAT of the FMASK
3353 * resource descriptor is 0 (invalid),
3354 */
3355 LLVMValueRef fmask_desc =
3356 LLVMBuildBitCast(gallivm->builder, fmask_ptr,
3357 ctx->v8i32, "");
3358
3359 LLVMValueRef fmask_word1 =
3360 LLVMBuildExtractElement(gallivm->builder, fmask_desc,
3361 uint_bld->one, "");
3362
3363 LLVMValueRef word1_is_nonzero =
3364 LLVMBuildICmp(gallivm->builder, LLVMIntNE,
3365 fmask_word1, uint_bld->zero, "");
3366
3367 /* Replace the MSAA sample index. */
3368 address[sample_chan] =
3369 LLVMBuildSelect(gallivm->builder, word1_is_nonzero,
3370 final_sample, address[sample_chan], "");
3371 }
3372
3373 if (opcode == TGSI_OPCODE_TXF) {
3374 /* add tex offsets */
3375 if (inst->Texture.NumOffsets) {
3376 struct lp_build_context *uint_bld = &bld_base->uint_bld;
3377 struct lp_build_tgsi_soa_context *bld = lp_soa_context(bld_base);
3378 const struct tgsi_texture_offset *off = inst->TexOffsets;
3379
3380 assert(inst->Texture.NumOffsets == 1);
3381
3382 switch (target) {
3383 case TGSI_TEXTURE_3D:
3384 address[2] = lp_build_add(uint_bld, address[2],
3385 bld->immediates[off->Index][off->SwizzleZ]);
3386 /* fall through */
3387 case TGSI_TEXTURE_2D:
3388 case TGSI_TEXTURE_SHADOW2D:
3389 case TGSI_TEXTURE_RECT:
3390 case TGSI_TEXTURE_SHADOWRECT:
3391 case TGSI_TEXTURE_2D_ARRAY:
3392 case TGSI_TEXTURE_SHADOW2D_ARRAY:
3393 address[1] =
3394 lp_build_add(uint_bld, address[1],
3395 bld->immediates[off->Index][off->SwizzleY]);
3396 /* fall through */
3397 case TGSI_TEXTURE_1D:
3398 case TGSI_TEXTURE_SHADOW1D:
3399 case TGSI_TEXTURE_1D_ARRAY:
3400 case TGSI_TEXTURE_SHADOW1D_ARRAY:
3401 address[0] =
3402 lp_build_add(uint_bld, address[0],
3403 bld->immediates[off->Index][off->SwizzleX]);
3404 break;
3405 /* texture offsets do not apply to other texture targets */
3406 }
3407 }
3408 }
3409
3410 if (opcode == TGSI_OPCODE_TG4) {
3411 unsigned gather_comp = 0;
3412
3413 /* DMASK was repurposed for GATHER4. 4 components are always
3414 * returned and DMASK works like a swizzle - it selects
3415 * the component to fetch. The only valid DMASK values are
3416 * 1=red, 2=green, 4=blue, 8=alpha. (e.g. 1 returns
3417 * (red,red,red,red) etc.) The ISA document doesn't mention
3418 * this.
3419 */
3420
3421 /* Get the component index from src1.x for Gather4. */
3422 if (!tgsi_is_shadow_target(target)) {
3423 LLVMValueRef (*imms)[4] = lp_soa_context(bld_base)->immediates;
3424 LLVMValueRef comp_imm;
3425 struct tgsi_src_register src1 = inst->Src[1].Register;
3426
3427 assert(src1.File == TGSI_FILE_IMMEDIATE);
3428
3429 comp_imm = imms[src1.Index][src1.SwizzleX];
3430 gather_comp = LLVMConstIntGetZExtValue(comp_imm);
3431 gather_comp = CLAMP(gather_comp, 0, 3);
3432 }
3433
3434 dmask = 1 << gather_comp;
3435 }
3436
3437 set_tex_fetch_args(ctx, emit_data, opcode, target, res_ptr,
3438 samp_ptr, address, count, dmask);
3439 }
3440
3441 static void build_tex_intrinsic(const struct lp_build_tgsi_action *action,
3442 struct lp_build_tgsi_context *bld_base,
3443 struct lp_build_emit_data *emit_data)
3444 {
3445 struct lp_build_context *base = &bld_base->base;
3446 unsigned opcode = emit_data->inst->Instruction.Opcode;
3447 unsigned target = emit_data->inst->Texture.Texture;
3448 char intr_name[127];
3449 bool has_offset = emit_data->inst->Texture.NumOffsets > 0;
3450 bool is_shadow = tgsi_is_shadow_target(target);
3451 char type[64];
3452 const char *name = "llvm.SI.image.sample";
3453 const char *infix = "";
3454
3455 if (opcode == TGSI_OPCODE_TXQ && target == TGSI_TEXTURE_BUFFER) {
3456 /* Just return the buffer size. */
3457 emit_data->output[emit_data->chan] = emit_data->args[0];
3458 return;
3459 }
3460
3461 if (target == TGSI_TEXTURE_BUFFER) {
3462 emit_data->output[emit_data->chan] = lp_build_intrinsic(
3463 base->gallivm->builder,
3464 "llvm.SI.vs.load.input", emit_data->dst_type,
3465 emit_data->args, emit_data->arg_count,
3466 LLVMReadNoneAttribute | LLVMNoUnwindAttribute);
3467 return;
3468 }
3469
3470 switch (opcode) {
3471 case TGSI_OPCODE_TXF:
3472 name = target == TGSI_TEXTURE_2D_MSAA ||
3473 target == TGSI_TEXTURE_2D_ARRAY_MSAA ?
3474 "llvm.SI.image.load" :
3475 "llvm.SI.image.load.mip";
3476 is_shadow = false;
3477 has_offset = false;
3478 break;
3479 case TGSI_OPCODE_TXQ:
3480 name = "llvm.SI.getresinfo";
3481 is_shadow = false;
3482 has_offset = false;
3483 break;
3484 case TGSI_OPCODE_LODQ:
3485 name = "llvm.SI.getlod";
3486 is_shadow = false;
3487 has_offset = false;
3488 break;
3489 case TGSI_OPCODE_TEX:
3490 case TGSI_OPCODE_TEX2:
3491 case TGSI_OPCODE_TXP:
3492 break;
3493 case TGSI_OPCODE_TXB:
3494 case TGSI_OPCODE_TXB2:
3495 infix = ".b";
3496 break;
3497 case TGSI_OPCODE_TXL:
3498 case TGSI_OPCODE_TXL2:
3499 infix = ".l";
3500 break;
3501 case TGSI_OPCODE_TXD:
3502 infix = ".d";
3503 break;
3504 case TGSI_OPCODE_TG4:
3505 name = "llvm.SI.gather4";
3506 break;
3507 default:
3508 assert(0);
3509 return;
3510 }
3511
3512 /* Add the type and suffixes .c, .o if needed. */
3513 build_int_type_name(LLVMTypeOf(emit_data->args[0]), type, sizeof(type));
3514 sprintf(intr_name, "%s%s%s%s.%s",
3515 name, is_shadow ? ".c" : "", infix,
3516 has_offset ? ".o" : "", type);
3517
3518 emit_data->output[emit_data->chan] = lp_build_intrinsic(
3519 base->gallivm->builder, intr_name, emit_data->dst_type,
3520 emit_data->args, emit_data->arg_count,
3521 LLVMReadNoneAttribute | LLVMNoUnwindAttribute);
3522
3523 /* Divide the number of layers by 6 to get the number of cubes. */
3524 if (opcode == TGSI_OPCODE_TXQ &&
3525 (target == TGSI_TEXTURE_CUBE_ARRAY ||
3526 target == TGSI_TEXTURE_SHADOWCUBE_ARRAY)) {
3527 LLVMBuilderRef builder = bld_base->base.gallivm->builder;
3528 LLVMValueRef two = lp_build_const_int32(bld_base->base.gallivm, 2);
3529 LLVMValueRef six = lp_build_const_int32(bld_base->base.gallivm, 6);
3530
3531 LLVMValueRef v4 = emit_data->output[emit_data->chan];
3532 LLVMValueRef z = LLVMBuildExtractElement(builder, v4, two, "");
3533 z = LLVMBuildSDiv(builder, z, six, "");
3534
3535 emit_data->output[emit_data->chan] =
3536 LLVMBuildInsertElement(builder, v4, z, two, "");
3537 }
3538 }
3539
3540 static void si_llvm_emit_txqs(
3541 const struct lp_build_tgsi_action *action,
3542 struct lp_build_tgsi_context *bld_base,
3543 struct lp_build_emit_data *emit_data)
3544 {
3545 struct si_shader_context *ctx = si_shader_context(bld_base);
3546 struct gallivm_state *gallivm = bld_base->base.gallivm;
3547 LLVMBuilderRef builder = gallivm->builder;
3548 LLVMValueRef res, samples;
3549 LLVMValueRef res_ptr, samp_ptr, fmask_ptr = NULL;
3550
3551 tex_fetch_ptrs(bld_base, emit_data, &res_ptr, &samp_ptr, &fmask_ptr);
3552
3553
3554 /* Read the samples from the descriptor directly. */
3555 res = LLVMBuildBitCast(builder, res_ptr, ctx->v8i32, "");
3556 samples = LLVMBuildExtractElement(
3557 builder, res,
3558 lp_build_const_int32(gallivm, 3), "");
3559 samples = LLVMBuildLShr(builder, samples,
3560 lp_build_const_int32(gallivm, 16), "");
3561 samples = LLVMBuildAnd(builder, samples,
3562 lp_build_const_int32(gallivm, 0xf), "");
3563 samples = LLVMBuildShl(builder, lp_build_const_int32(gallivm, 1),
3564 samples, "");
3565
3566 emit_data->output[emit_data->chan] = samples;
3567 }
3568
3569 /*
3570 * SI implements derivatives using the local data store (LDS)
3571 * All writes to the LDS happen in all executing threads at
3572 * the same time. TID is the Thread ID for the current
3573 * thread and is a value between 0 and 63, representing
3574 * the thread's position in the wavefront.
3575 *
3576 * For the pixel shader threads are grouped into quads of four pixels.
3577 * The TIDs of the pixels of a quad are:
3578 *
3579 * +------+------+
3580 * |4n + 0|4n + 1|
3581 * +------+------+
3582 * |4n + 2|4n + 3|
3583 * +------+------+
3584 *
3585 * So, masking the TID with 0xfffffffc yields the TID of the top left pixel
3586 * of the quad, masking with 0xfffffffd yields the TID of the top pixel of
3587 * the current pixel's column, and masking with 0xfffffffe yields the TID
3588 * of the left pixel of the current pixel's row.
3589 *
3590 * Adding 1 yields the TID of the pixel to the right of the left pixel, and
3591 * adding 2 yields the TID of the pixel below the top pixel.
3592 */
3593 /* masks for thread ID. */
3594 #define TID_MASK_TOP_LEFT 0xfffffffc
3595 #define TID_MASK_TOP 0xfffffffd
3596 #define TID_MASK_LEFT 0xfffffffe
3597
3598 static void si_llvm_emit_ddxy(
3599 const struct lp_build_tgsi_action *action,
3600 struct lp_build_tgsi_context *bld_base,
3601 struct lp_build_emit_data *emit_data)
3602 {
3603 struct si_shader_context *ctx = si_shader_context(bld_base);
3604 struct gallivm_state *gallivm = bld_base->base.gallivm;
3605 const struct tgsi_full_instruction *inst = emit_data->inst;
3606 unsigned opcode = inst->Instruction.Opcode;
3607 LLVMValueRef indices[2];
3608 LLVMValueRef store_ptr, load_ptr0, load_ptr1;
3609 LLVMValueRef tl, trbl, result[4];
3610 unsigned swizzle[4];
3611 unsigned c;
3612 int idx;
3613 unsigned mask;
3614
3615 indices[0] = bld_base->uint_bld.zero;
3616 indices[1] = lp_build_intrinsic(gallivm->builder, "llvm.SI.tid", ctx->i32,
3617 NULL, 0, LLVMReadNoneAttribute);
3618 store_ptr = LLVMBuildGEP(gallivm->builder, ctx->lds,
3619 indices, 2, "");
3620
3621 if (opcode == TGSI_OPCODE_DDX_FINE)
3622 mask = TID_MASK_LEFT;
3623 else if (opcode == TGSI_OPCODE_DDY_FINE)
3624 mask = TID_MASK_TOP;
3625 else
3626 mask = TID_MASK_TOP_LEFT;
3627
3628 indices[1] = LLVMBuildAnd(gallivm->builder, indices[1],
3629 lp_build_const_int32(gallivm, mask), "");
3630 load_ptr0 = LLVMBuildGEP(gallivm->builder, ctx->lds,
3631 indices, 2, "");
3632
3633 /* for DDX we want to next X pixel, DDY next Y pixel. */
3634 idx = (opcode == TGSI_OPCODE_DDX || opcode == TGSI_OPCODE_DDX_FINE) ? 1 : 2;
3635 indices[1] = LLVMBuildAdd(gallivm->builder, indices[1],
3636 lp_build_const_int32(gallivm, idx), "");
3637 load_ptr1 = LLVMBuildGEP(gallivm->builder, ctx->lds,
3638 indices, 2, "");
3639
3640 for (c = 0; c < 4; ++c) {
3641 unsigned i;
3642
3643 swizzle[c] = tgsi_util_get_full_src_register_swizzle(&inst->Src[0], c);
3644 for (i = 0; i < c; ++i) {
3645 if (swizzle[i] == swizzle[c]) {
3646 result[c] = result[i];
3647 break;
3648 }
3649 }
3650 if (i != c)
3651 continue;
3652
3653 LLVMBuildStore(gallivm->builder,
3654 LLVMBuildBitCast(gallivm->builder,
3655 lp_build_emit_fetch(bld_base, inst, 0, c),
3656 ctx->i32, ""),
3657 store_ptr);
3658
3659 tl = LLVMBuildLoad(gallivm->builder, load_ptr0, "");
3660 tl = LLVMBuildBitCast(gallivm->builder, tl, ctx->f32, "");
3661
3662 trbl = LLVMBuildLoad(gallivm->builder, load_ptr1, "");
3663 trbl = LLVMBuildBitCast(gallivm->builder, trbl, ctx->f32, "");
3664
3665 result[c] = LLVMBuildFSub(gallivm->builder, trbl, tl, "");
3666 }
3667
3668 emit_data->output[0] = lp_build_gather_values(gallivm, result, 4);
3669 }
3670
3671 /*
3672 * this takes an I,J coordinate pair,
3673 * and works out the X and Y derivatives.
3674 * it returns DDX(I), DDX(J), DDY(I), DDY(J).
3675 */
3676 static LLVMValueRef si_llvm_emit_ddxy_interp(
3677 struct lp_build_tgsi_context *bld_base,
3678 LLVMValueRef interp_ij)
3679 {
3680 struct si_shader_context *ctx = si_shader_context(bld_base);
3681 struct gallivm_state *gallivm = bld_base->base.gallivm;
3682 LLVMValueRef indices[2];
3683 LLVMValueRef store_ptr, load_ptr_x, load_ptr_y, load_ptr_ddx, load_ptr_ddy, temp, temp2;
3684 LLVMValueRef tl, tr, bl, result[4];
3685 unsigned c;
3686
3687 indices[0] = bld_base->uint_bld.zero;
3688 indices[1] = lp_build_intrinsic(gallivm->builder, "llvm.SI.tid", ctx->i32,
3689 NULL, 0, LLVMReadNoneAttribute);
3690 store_ptr = LLVMBuildGEP(gallivm->builder, ctx->lds,
3691 indices, 2, "");
3692
3693 temp = LLVMBuildAnd(gallivm->builder, indices[1],
3694 lp_build_const_int32(gallivm, TID_MASK_LEFT), "");
3695
3696 temp2 = LLVMBuildAnd(gallivm->builder, indices[1],
3697 lp_build_const_int32(gallivm, TID_MASK_TOP), "");
3698
3699 indices[1] = temp;
3700 load_ptr_x = LLVMBuildGEP(gallivm->builder, ctx->lds,
3701 indices, 2, "");
3702
3703 indices[1] = temp2;
3704 load_ptr_y = LLVMBuildGEP(gallivm->builder, ctx->lds,
3705 indices, 2, "");
3706
3707 indices[1] = LLVMBuildAdd(gallivm->builder, temp,
3708 lp_build_const_int32(gallivm, 1), "");
3709 load_ptr_ddx = LLVMBuildGEP(gallivm->builder, ctx->lds,
3710 indices, 2, "");
3711
3712 indices[1] = LLVMBuildAdd(gallivm->builder, temp2,
3713 lp_build_const_int32(gallivm, 2), "");
3714 load_ptr_ddy = LLVMBuildGEP(gallivm->builder, ctx->lds,
3715 indices, 2, "");
3716
3717 for (c = 0; c < 2; ++c) {
3718 LLVMValueRef store_val;
3719 LLVMValueRef c_ll = lp_build_const_int32(gallivm, c);
3720
3721 store_val = LLVMBuildExtractElement(gallivm->builder,
3722 interp_ij, c_ll, "");
3723 LLVMBuildStore(gallivm->builder,
3724 store_val,
3725 store_ptr);
3726
3727 tl = LLVMBuildLoad(gallivm->builder, load_ptr_x, "");
3728 tl = LLVMBuildBitCast(gallivm->builder, tl, ctx->f32, "");
3729
3730 tr = LLVMBuildLoad(gallivm->builder, load_ptr_ddx, "");
3731 tr = LLVMBuildBitCast(gallivm->builder, tr, ctx->f32, "");
3732
3733 result[c] = LLVMBuildFSub(gallivm->builder, tr, tl, "");
3734
3735 tl = LLVMBuildLoad(gallivm->builder, load_ptr_y, "");
3736 tl = LLVMBuildBitCast(gallivm->builder, tl, ctx->f32, "");
3737
3738 bl = LLVMBuildLoad(gallivm->builder, load_ptr_ddy, "");
3739 bl = LLVMBuildBitCast(gallivm->builder, bl, ctx->f32, "");
3740
3741 result[c + 2] = LLVMBuildFSub(gallivm->builder, bl, tl, "");
3742 }
3743
3744 return lp_build_gather_values(gallivm, result, 4);
3745 }
3746
3747 static void interp_fetch_args(
3748 struct lp_build_tgsi_context *bld_base,
3749 struct lp_build_emit_data *emit_data)
3750 {
3751 struct si_shader_context *ctx = si_shader_context(bld_base);
3752 struct gallivm_state *gallivm = bld_base->base.gallivm;
3753 const struct tgsi_full_instruction *inst = emit_data->inst;
3754
3755 if (inst->Instruction.Opcode == TGSI_OPCODE_INTERP_OFFSET) {
3756 /* offset is in second src, first two channels */
3757 emit_data->args[0] = lp_build_emit_fetch(bld_base,
3758 emit_data->inst, 1,
3759 TGSI_CHAN_X);
3760 emit_data->args[1] = lp_build_emit_fetch(bld_base,
3761 emit_data->inst, 1,
3762 TGSI_CHAN_Y);
3763 emit_data->arg_count = 2;
3764 } else if (inst->Instruction.Opcode == TGSI_OPCODE_INTERP_SAMPLE) {
3765 LLVMValueRef sample_position;
3766 LLVMValueRef sample_id;
3767 LLVMValueRef halfval = lp_build_const_float(gallivm, 0.5f);
3768
3769 /* fetch sample ID, then fetch its sample position,
3770 * and place into first two channels.
3771 */
3772 sample_id = lp_build_emit_fetch(bld_base,
3773 emit_data->inst, 1, TGSI_CHAN_X);
3774 sample_id = LLVMBuildBitCast(gallivm->builder, sample_id,
3775 ctx->i32, "");
3776 sample_position = load_sample_position(&ctx->radeon_bld, sample_id);
3777
3778 emit_data->args[0] = LLVMBuildExtractElement(gallivm->builder,
3779 sample_position,
3780 lp_build_const_int32(gallivm, 0), "");
3781
3782 emit_data->args[0] = LLVMBuildFSub(gallivm->builder, emit_data->args[0], halfval, "");
3783 emit_data->args[1] = LLVMBuildExtractElement(gallivm->builder,
3784 sample_position,
3785 lp_build_const_int32(gallivm, 1), "");
3786 emit_data->args[1] = LLVMBuildFSub(gallivm->builder, emit_data->args[1], halfval, "");
3787 emit_data->arg_count = 2;
3788 }
3789 }
3790
3791 static void build_interp_intrinsic(const struct lp_build_tgsi_action *action,
3792 struct lp_build_tgsi_context *bld_base,
3793 struct lp_build_emit_data *emit_data)
3794 {
3795 struct si_shader_context *ctx = si_shader_context(bld_base);
3796 struct si_shader *shader = ctx->shader;
3797 struct gallivm_state *gallivm = bld_base->base.gallivm;
3798 LLVMValueRef interp_param;
3799 const struct tgsi_full_instruction *inst = emit_data->inst;
3800 const char *intr_name;
3801 int input_index = inst->Src[0].Register.Index;
3802 int chan;
3803 int i;
3804 LLVMValueRef attr_number;
3805 LLVMValueRef params = LLVMGetParam(ctx->radeon_bld.main_fn, SI_PARAM_PRIM_MASK);
3806 int interp_param_idx;
3807 unsigned interp = shader->selector->info.input_interpolate[input_index];
3808 unsigned location;
3809
3810 assert(inst->Src[0].Register.File == TGSI_FILE_INPUT);
3811
3812 if (inst->Instruction.Opcode == TGSI_OPCODE_INTERP_OFFSET ||
3813 inst->Instruction.Opcode == TGSI_OPCODE_INTERP_SAMPLE)
3814 location = TGSI_INTERPOLATE_LOC_CENTER;
3815 else
3816 location = TGSI_INTERPOLATE_LOC_CENTROID;
3817
3818 interp_param_idx = lookup_interp_param_index(interp, location);
3819 if (interp_param_idx == -1)
3820 return;
3821 else if (interp_param_idx)
3822 interp_param = LLVMGetParam(ctx->radeon_bld.main_fn, interp_param_idx);
3823 else
3824 interp_param = NULL;
3825
3826 attr_number = lp_build_const_int32(gallivm, input_index);
3827
3828 if (inst->Instruction.Opcode == TGSI_OPCODE_INTERP_OFFSET ||
3829 inst->Instruction.Opcode == TGSI_OPCODE_INTERP_SAMPLE) {
3830 LLVMValueRef ij_out[2];
3831 LLVMValueRef ddxy_out = si_llvm_emit_ddxy_interp(bld_base, interp_param);
3832
3833 /*
3834 * take the I then J parameters, and the DDX/Y for it, and
3835 * calculate the IJ inputs for the interpolator.
3836 * temp1 = ddx * offset/sample.x + I;
3837 * interp_param.I = ddy * offset/sample.y + temp1;
3838 * temp1 = ddx * offset/sample.x + J;
3839 * interp_param.J = ddy * offset/sample.y + temp1;
3840 */
3841 for (i = 0; i < 2; i++) {
3842 LLVMValueRef ix_ll = lp_build_const_int32(gallivm, i);
3843 LLVMValueRef iy_ll = lp_build_const_int32(gallivm, i + 2);
3844 LLVMValueRef ddx_el = LLVMBuildExtractElement(gallivm->builder,
3845 ddxy_out, ix_ll, "");
3846 LLVMValueRef ddy_el = LLVMBuildExtractElement(gallivm->builder,
3847 ddxy_out, iy_ll, "");
3848 LLVMValueRef interp_el = LLVMBuildExtractElement(gallivm->builder,
3849 interp_param, ix_ll, "");
3850 LLVMValueRef temp1, temp2;
3851
3852 interp_el = LLVMBuildBitCast(gallivm->builder, interp_el,
3853 ctx->f32, "");
3854
3855 temp1 = LLVMBuildFMul(gallivm->builder, ddx_el, emit_data->args[0], "");
3856
3857 temp1 = LLVMBuildFAdd(gallivm->builder, temp1, interp_el, "");
3858
3859 temp2 = LLVMBuildFMul(gallivm->builder, ddy_el, emit_data->args[1], "");
3860
3861 temp2 = LLVMBuildFAdd(gallivm->builder, temp2, temp1, "");
3862
3863 ij_out[i] = LLVMBuildBitCast(gallivm->builder,
3864 temp2, ctx->i32, "");
3865 }
3866 interp_param = lp_build_gather_values(bld_base->base.gallivm, ij_out, 2);
3867 }
3868
3869 intr_name = interp_param ? "llvm.SI.fs.interp" : "llvm.SI.fs.constant";
3870 for (chan = 0; chan < 2; chan++) {
3871 LLVMValueRef args[4];
3872 LLVMValueRef llvm_chan;
3873 unsigned schan;
3874
3875 schan = tgsi_util_get_full_src_register_swizzle(&inst->Src[0], chan);
3876 llvm_chan = lp_build_const_int32(gallivm, schan);
3877
3878 args[0] = llvm_chan;
3879 args[1] = attr_number;
3880 args[2] = params;
3881 args[3] = interp_param;
3882
3883 emit_data->output[chan] =
3884 lp_build_intrinsic(gallivm->builder, intr_name,
3885 ctx->f32, args, args[3] ? 4 : 3,
3886 LLVMReadNoneAttribute | LLVMNoUnwindAttribute);
3887 }
3888 }
3889
3890 static unsigned si_llvm_get_stream(struct lp_build_tgsi_context *bld_base,
3891 struct lp_build_emit_data *emit_data)
3892 {
3893 LLVMValueRef (*imms)[4] = lp_soa_context(bld_base)->immediates;
3894 struct tgsi_src_register src0 = emit_data->inst->Src[0].Register;
3895 unsigned stream;
3896
3897 assert(src0.File == TGSI_FILE_IMMEDIATE);
3898
3899 stream = LLVMConstIntGetZExtValue(imms[src0.Index][src0.SwizzleX]) & 0x3;
3900 return stream;
3901 }
3902
3903 /* Emit one vertex from the geometry shader */
3904 static void si_llvm_emit_vertex(
3905 const struct lp_build_tgsi_action *action,
3906 struct lp_build_tgsi_context *bld_base,
3907 struct lp_build_emit_data *emit_data)
3908 {
3909 struct si_shader_context *ctx = si_shader_context(bld_base);
3910 struct lp_build_context *uint = &bld_base->uint_bld;
3911 struct si_shader *shader = ctx->shader;
3912 struct tgsi_shader_info *info = &shader->selector->info;
3913 struct gallivm_state *gallivm = bld_base->base.gallivm;
3914 LLVMValueRef soffset = LLVMGetParam(ctx->radeon_bld.main_fn,
3915 SI_PARAM_GS2VS_OFFSET);
3916 LLVMValueRef gs_next_vertex;
3917 LLVMValueRef can_emit, kill;
3918 LLVMValueRef args[2];
3919 unsigned chan;
3920 int i;
3921 unsigned stream;
3922
3923 stream = si_llvm_get_stream(bld_base, emit_data);
3924
3925 /* Write vertex attribute values to GSVS ring */
3926 gs_next_vertex = LLVMBuildLoad(gallivm->builder,
3927 ctx->gs_next_vertex[stream],
3928 "");
3929
3930 /* If this thread has already emitted the declared maximum number of
3931 * vertices, kill it: excessive vertex emissions are not supposed to
3932 * have any effect, and GS threads have no externally observable
3933 * effects other than emitting vertices.
3934 */
3935 can_emit = LLVMBuildICmp(gallivm->builder, LLVMIntULE, gs_next_vertex,
3936 lp_build_const_int32(gallivm,
3937 shader->selector->gs_max_out_vertices), "");
3938 kill = lp_build_select(&bld_base->base, can_emit,
3939 lp_build_const_float(gallivm, 1.0f),
3940 lp_build_const_float(gallivm, -1.0f));
3941
3942 lp_build_intrinsic(gallivm->builder, "llvm.AMDGPU.kill",
3943 ctx->voidt, &kill, 1, 0);
3944
3945 for (i = 0; i < info->num_outputs; i++) {
3946 LLVMValueRef *out_ptr =
3947 ctx->radeon_bld.soa.outputs[i];
3948
3949 for (chan = 0; chan < 4; chan++) {
3950 LLVMValueRef out_val = LLVMBuildLoad(gallivm->builder, out_ptr[chan], "");
3951 LLVMValueRef voffset =
3952 lp_build_const_int32(gallivm, (i * 4 + chan) *
3953 shader->selector->gs_max_out_vertices);
3954
3955 voffset = lp_build_add(uint, voffset, gs_next_vertex);
3956 voffset = lp_build_mul_imm(uint, voffset, 4);
3957
3958 out_val = LLVMBuildBitCast(gallivm->builder, out_val, ctx->i32, "");
3959
3960 build_tbuffer_store(ctx,
3961 ctx->gsvs_ring[stream],
3962 out_val, 1,
3963 voffset, soffset, 0,
3964 V_008F0C_BUF_DATA_FORMAT_32,
3965 V_008F0C_BUF_NUM_FORMAT_UINT,
3966 1, 0, 1, 1, 0);
3967 }
3968 }
3969 gs_next_vertex = lp_build_add(uint, gs_next_vertex,
3970 lp_build_const_int32(gallivm, 1));
3971
3972 LLVMBuildStore(gallivm->builder, gs_next_vertex, ctx->gs_next_vertex[stream]);
3973
3974 /* Signal vertex emission */
3975 args[0] = lp_build_const_int32(gallivm, SENDMSG_GS_OP_EMIT | SENDMSG_GS | (stream << 8));
3976 args[1] = LLVMGetParam(ctx->radeon_bld.main_fn, SI_PARAM_GS_WAVE_ID);
3977 lp_build_intrinsic(gallivm->builder, "llvm.SI.sendmsg",
3978 ctx->voidt, args, 2, LLVMNoUnwindAttribute);
3979 }
3980
3981 /* Cut one primitive from the geometry shader */
3982 static void si_llvm_emit_primitive(
3983 const struct lp_build_tgsi_action *action,
3984 struct lp_build_tgsi_context *bld_base,
3985 struct lp_build_emit_data *emit_data)
3986 {
3987 struct si_shader_context *ctx = si_shader_context(bld_base);
3988 struct gallivm_state *gallivm = bld_base->base.gallivm;
3989 LLVMValueRef args[2];
3990 unsigned stream;
3991
3992 /* Signal primitive cut */
3993 stream = si_llvm_get_stream(bld_base, emit_data);
3994 args[0] = lp_build_const_int32(gallivm, SENDMSG_GS_OP_CUT | SENDMSG_GS | (stream << 8));
3995 args[1] = LLVMGetParam(ctx->radeon_bld.main_fn, SI_PARAM_GS_WAVE_ID);
3996 lp_build_intrinsic(gallivm->builder, "llvm.SI.sendmsg",
3997 ctx->voidt, args, 2, LLVMNoUnwindAttribute);
3998 }
3999
4000 static void si_llvm_emit_barrier(const struct lp_build_tgsi_action *action,
4001 struct lp_build_tgsi_context *bld_base,
4002 struct lp_build_emit_data *emit_data)
4003 {
4004 struct si_shader_context *ctx = si_shader_context(bld_base);
4005 struct gallivm_state *gallivm = bld_base->base.gallivm;
4006
4007 lp_build_intrinsic(gallivm->builder,
4008 HAVE_LLVM >= 0x0309 ? "llvm.amdgcn.s.barrier"
4009 : "llvm.AMDGPU.barrier.local",
4010 ctx->voidt, NULL, 0, LLVMNoUnwindAttribute);
4011 }
4012
4013 static const struct lp_build_tgsi_action tex_action = {
4014 .fetch_args = tex_fetch_args,
4015 .emit = build_tex_intrinsic,
4016 };
4017
4018 static const struct lp_build_tgsi_action interp_action = {
4019 .fetch_args = interp_fetch_args,
4020 .emit = build_interp_intrinsic,
4021 };
4022
4023 static void si_create_function(struct si_shader_context *ctx,
4024 LLVMTypeRef *returns, unsigned num_returns,
4025 LLVMTypeRef *params, unsigned num_params,
4026 int last_array_pointer, int last_sgpr)
4027 {
4028 int i;
4029
4030 radeon_llvm_create_func(&ctx->radeon_bld, returns, num_returns,
4031 params, num_params);
4032 radeon_llvm_shader_type(ctx->radeon_bld.main_fn, ctx->type);
4033 ctx->return_value = LLVMGetUndef(ctx->radeon_bld.return_type);
4034
4035 for (i = 0; i <= last_sgpr; ++i) {
4036 LLVMValueRef P = LLVMGetParam(ctx->radeon_bld.main_fn, i);
4037
4038 /* We tell llvm that array inputs are passed by value to allow Sinking pass
4039 * to move load. Inputs are constant so this is fine. */
4040 if (i <= last_array_pointer)
4041 LLVMAddAttribute(P, LLVMByValAttribute);
4042 else
4043 LLVMAddAttribute(P, LLVMInRegAttribute);
4044 }
4045 }
4046
4047 static void create_meta_data(struct si_shader_context *ctx)
4048 {
4049 struct gallivm_state *gallivm = ctx->radeon_bld.soa.bld_base.base.gallivm;
4050 LLVMValueRef args[3];
4051
4052 args[0] = LLVMMDStringInContext(gallivm->context, "const", 5);
4053 args[1] = 0;
4054 args[2] = lp_build_const_int32(gallivm, 1);
4055
4056 ctx->const_md = LLVMMDNodeInContext(gallivm->context, args, 3);
4057 }
4058
4059 static void declare_streamout_params(struct si_shader_context *ctx,
4060 struct pipe_stream_output_info *so,
4061 LLVMTypeRef *params, LLVMTypeRef i32,
4062 unsigned *num_params)
4063 {
4064 int i;
4065
4066 /* Streamout SGPRs. */
4067 if (so->num_outputs) {
4068 params[ctx->param_streamout_config = (*num_params)++] = i32;
4069 params[ctx->param_streamout_write_index = (*num_params)++] = i32;
4070 }
4071 /* A streamout buffer offset is loaded if the stride is non-zero. */
4072 for (i = 0; i < 4; i++) {
4073 if (!so->stride[i])
4074 continue;
4075
4076 params[ctx->param_streamout_offset[i] = (*num_params)++] = i32;
4077 }
4078 }
4079
4080 static unsigned llvm_get_type_size(LLVMTypeRef type)
4081 {
4082 LLVMTypeKind kind = LLVMGetTypeKind(type);
4083
4084 switch (kind) {
4085 case LLVMIntegerTypeKind:
4086 return LLVMGetIntTypeWidth(type) / 8;
4087 case LLVMFloatTypeKind:
4088 return 4;
4089 case LLVMPointerTypeKind:
4090 return 8;
4091 case LLVMVectorTypeKind:
4092 return LLVMGetVectorSize(type) *
4093 llvm_get_type_size(LLVMGetElementType(type));
4094 default:
4095 assert(0);
4096 return 0;
4097 }
4098 }
4099
4100 static void declare_tess_lds(struct si_shader_context *ctx)
4101 {
4102 struct gallivm_state *gallivm = &ctx->radeon_bld.gallivm;
4103 LLVMTypeRef i32 = ctx->radeon_bld.soa.bld_base.uint_bld.elem_type;
4104
4105 /* This is the upper bound, maximum is 32 inputs times 32 vertices */
4106 unsigned vertex_data_dw_size = 32*32*4;
4107 unsigned patch_data_dw_size = 32*4;
4108 /* The formula is: TCS inputs + TCS outputs + TCS patch outputs. */
4109 unsigned patch_dw_size = vertex_data_dw_size*2 + patch_data_dw_size;
4110 unsigned lds_dwords = patch_dw_size;
4111
4112 /* The actual size is computed outside of the shader to reduce
4113 * the number of shader variants. */
4114 ctx->lds =
4115 LLVMAddGlobalInAddressSpace(gallivm->module,
4116 LLVMArrayType(i32, lds_dwords),
4117 "tess_lds",
4118 LOCAL_ADDR_SPACE);
4119 }
4120
4121 static void create_function(struct si_shader_context *ctx)
4122 {
4123 struct lp_build_tgsi_context *bld_base = &ctx->radeon_bld.soa.bld_base;
4124 struct gallivm_state *gallivm = bld_base->base.gallivm;
4125 struct si_shader *shader = ctx->shader;
4126 LLVMTypeRef params[SI_NUM_PARAMS + SI_NUM_VERTEX_BUFFERS], v3i32;
4127 LLVMTypeRef returns[16+32*4];
4128 unsigned i, last_array_pointer, last_sgpr, num_params, num_return_sgprs;
4129 unsigned num_returns = 0;
4130
4131 v3i32 = LLVMVectorType(ctx->i32, 3);
4132
4133 params[SI_PARAM_RW_BUFFERS] = const_array(ctx->v16i8, SI_NUM_RW_BUFFERS);
4134 params[SI_PARAM_CONST_BUFFERS] = const_array(ctx->v16i8, SI_NUM_CONST_BUFFERS);
4135 params[SI_PARAM_SAMPLERS] = const_array(ctx->v8i32, SI_NUM_SAMPLERS);
4136 params[SI_PARAM_IMAGES] = const_array(ctx->v8i32, SI_NUM_IMAGES);
4137 last_array_pointer = SI_PARAM_IMAGES;
4138
4139 switch (ctx->type) {
4140 case TGSI_PROCESSOR_VERTEX:
4141 params[SI_PARAM_VERTEX_BUFFERS] = const_array(ctx->v16i8, SI_NUM_VERTEX_BUFFERS);
4142 last_array_pointer = SI_PARAM_VERTEX_BUFFERS;
4143 params[SI_PARAM_BASE_VERTEX] = ctx->i32;
4144 params[SI_PARAM_START_INSTANCE] = ctx->i32;
4145 num_params = SI_PARAM_START_INSTANCE+1;
4146
4147 if (shader->key.vs.as_es) {
4148 params[ctx->param_es2gs_offset = num_params++] = ctx->i32;
4149 } else if (shader->key.vs.as_ls) {
4150 params[SI_PARAM_LS_OUT_LAYOUT] = ctx->i32;
4151 num_params = SI_PARAM_LS_OUT_LAYOUT+1;
4152 } else {
4153 if (ctx->is_gs_copy_shader) {
4154 last_array_pointer = SI_PARAM_CONST_BUFFERS;
4155 num_params = SI_PARAM_CONST_BUFFERS+1;
4156 } else {
4157 params[SI_PARAM_VS_STATE_BITS] = ctx->i32;
4158 num_params = SI_PARAM_VS_STATE_BITS+1;
4159 }
4160
4161 /* The locations of the other parameters are assigned dynamically. */
4162 declare_streamout_params(ctx, &shader->selector->so,
4163 params, ctx->i32, &num_params);
4164 }
4165
4166 last_sgpr = num_params-1;
4167
4168 /* VGPRs */
4169 params[ctx->param_vertex_id = num_params++] = ctx->i32;
4170 params[ctx->param_rel_auto_id = num_params++] = ctx->i32;
4171 params[ctx->param_vs_prim_id = num_params++] = ctx->i32;
4172 params[ctx->param_instance_id = num_params++] = ctx->i32;
4173
4174 if (!ctx->is_monolithic &&
4175 !ctx->is_gs_copy_shader) {
4176 /* Vertex load indices. */
4177 ctx->param_vertex_index0 = num_params;
4178
4179 for (i = 0; i < shader->selector->info.num_inputs; i++)
4180 params[num_params++] = ctx->i32;
4181
4182 /* PrimitiveID output. */
4183 if (!shader->key.vs.as_es && !shader->key.vs.as_ls)
4184 for (i = 0; i <= VS_EPILOG_PRIMID_LOC; i++)
4185 returns[num_returns++] = ctx->f32;
4186 }
4187 break;
4188
4189 case TGSI_PROCESSOR_TESS_CTRL:
4190 params[SI_PARAM_TCS_OUT_OFFSETS] = ctx->i32;
4191 params[SI_PARAM_TCS_OUT_LAYOUT] = ctx->i32;
4192 params[SI_PARAM_TCS_IN_LAYOUT] = ctx->i32;
4193 params[SI_PARAM_TESS_FACTOR_OFFSET] = ctx->i32;
4194 last_sgpr = SI_PARAM_TESS_FACTOR_OFFSET;
4195
4196 /* VGPRs */
4197 params[SI_PARAM_PATCH_ID] = ctx->i32;
4198 params[SI_PARAM_REL_IDS] = ctx->i32;
4199 num_params = SI_PARAM_REL_IDS+1;
4200
4201 if (!ctx->is_monolithic) {
4202 /* PARAM_TESS_FACTOR_OFFSET is after user SGPRs. */
4203 for (i = 0; i <= SI_TCS_NUM_USER_SGPR; i++)
4204 returns[num_returns++] = ctx->i32; /* SGPRs */
4205
4206 for (i = 0; i < 3; i++)
4207 returns[num_returns++] = ctx->f32; /* VGPRs */
4208 }
4209 break;
4210
4211 case TGSI_PROCESSOR_TESS_EVAL:
4212 params[SI_PARAM_TCS_OUT_OFFSETS] = ctx->i32;
4213 params[SI_PARAM_TCS_OUT_LAYOUT] = ctx->i32;
4214 num_params = SI_PARAM_TCS_OUT_LAYOUT+1;
4215
4216 if (shader->key.tes.as_es) {
4217 params[ctx->param_es2gs_offset = num_params++] = ctx->i32;
4218 } else {
4219 declare_streamout_params(ctx, &shader->selector->so,
4220 params, ctx->i32, &num_params);
4221 }
4222 last_sgpr = num_params - 1;
4223
4224 /* VGPRs */
4225 params[ctx->param_tes_u = num_params++] = ctx->f32;
4226 params[ctx->param_tes_v = num_params++] = ctx->f32;
4227 params[ctx->param_tes_rel_patch_id = num_params++] = ctx->i32;
4228 params[ctx->param_tes_patch_id = num_params++] = ctx->i32;
4229
4230 /* PrimitiveID output. */
4231 if (!ctx->is_monolithic && !shader->key.tes.as_es)
4232 for (i = 0; i <= VS_EPILOG_PRIMID_LOC; i++)
4233 returns[num_returns++] = ctx->f32;
4234 break;
4235
4236 case TGSI_PROCESSOR_GEOMETRY:
4237 params[SI_PARAM_GS2VS_OFFSET] = ctx->i32;
4238 params[SI_PARAM_GS_WAVE_ID] = ctx->i32;
4239 last_sgpr = SI_PARAM_GS_WAVE_ID;
4240
4241 /* VGPRs */
4242 params[SI_PARAM_VTX0_OFFSET] = ctx->i32;
4243 params[SI_PARAM_VTX1_OFFSET] = ctx->i32;
4244 params[SI_PARAM_PRIMITIVE_ID] = ctx->i32;
4245 params[SI_PARAM_VTX2_OFFSET] = ctx->i32;
4246 params[SI_PARAM_VTX3_OFFSET] = ctx->i32;
4247 params[SI_PARAM_VTX4_OFFSET] = ctx->i32;
4248 params[SI_PARAM_VTX5_OFFSET] = ctx->i32;
4249 params[SI_PARAM_GS_INSTANCE_ID] = ctx->i32;
4250 num_params = SI_PARAM_GS_INSTANCE_ID+1;
4251 break;
4252
4253 case TGSI_PROCESSOR_FRAGMENT:
4254 params[SI_PARAM_ALPHA_REF] = ctx->f32;
4255 params[SI_PARAM_PRIM_MASK] = ctx->i32;
4256 last_sgpr = SI_PARAM_PRIM_MASK;
4257 params[SI_PARAM_PERSP_SAMPLE] = ctx->v2i32;
4258 params[SI_PARAM_PERSP_CENTER] = ctx->v2i32;
4259 params[SI_PARAM_PERSP_CENTROID] = ctx->v2i32;
4260 params[SI_PARAM_PERSP_PULL_MODEL] = v3i32;
4261 params[SI_PARAM_LINEAR_SAMPLE] = ctx->v2i32;
4262 params[SI_PARAM_LINEAR_CENTER] = ctx->v2i32;
4263 params[SI_PARAM_LINEAR_CENTROID] = ctx->v2i32;
4264 params[SI_PARAM_LINE_STIPPLE_TEX] = ctx->f32;
4265 params[SI_PARAM_POS_X_FLOAT] = ctx->f32;
4266 params[SI_PARAM_POS_Y_FLOAT] = ctx->f32;
4267 params[SI_PARAM_POS_Z_FLOAT] = ctx->f32;
4268 params[SI_PARAM_POS_W_FLOAT] = ctx->f32;
4269 params[SI_PARAM_FRONT_FACE] = ctx->i32;
4270 params[SI_PARAM_ANCILLARY] = ctx->i32;
4271 params[SI_PARAM_SAMPLE_COVERAGE] = ctx->f32;
4272 params[SI_PARAM_POS_FIXED_PT] = ctx->i32;
4273 num_params = SI_PARAM_POS_FIXED_PT+1;
4274
4275 if (!ctx->is_monolithic) {
4276 /* Color inputs from the prolog. */
4277 if (shader->selector->info.colors_read) {
4278 unsigned num_color_elements =
4279 util_bitcount(shader->selector->info.colors_read);
4280
4281 assert(num_params + num_color_elements <= ARRAY_SIZE(params));
4282 for (i = 0; i < num_color_elements; i++)
4283 params[num_params++] = ctx->f32;
4284 }
4285
4286 /* Outputs for the epilog. */
4287 num_return_sgprs = SI_SGPR_ALPHA_REF + 1;
4288 num_returns =
4289 num_return_sgprs +
4290 util_bitcount(shader->selector->info.colors_written) * 4 +
4291 shader->selector->info.writes_z +
4292 shader->selector->info.writes_stencil +
4293 shader->selector->info.writes_samplemask +
4294 1 /* SampleMaskIn */;
4295
4296 num_returns = MAX2(num_returns,
4297 num_return_sgprs +
4298 PS_EPILOG_SAMPLEMASK_MIN_LOC + 1);
4299
4300 for (i = 0; i < num_return_sgprs; i++)
4301 returns[i] = ctx->i32;
4302 for (; i < num_returns; i++)
4303 returns[i] = ctx->f32;
4304 }
4305 break;
4306
4307 default:
4308 assert(0 && "unimplemented shader");
4309 return;
4310 }
4311
4312 assert(num_params <= Elements(params));
4313
4314 si_create_function(ctx, returns, num_returns, params,
4315 num_params, last_array_pointer, last_sgpr);
4316
4317 /* Reserve register locations for VGPR inputs the PS prolog may need. */
4318 if (ctx->type == TGSI_PROCESSOR_FRAGMENT &&
4319 !ctx->is_monolithic) {
4320 radeon_llvm_add_attribute(ctx->radeon_bld.main_fn,
4321 "InitialPSInputAddr",
4322 S_0286D0_PERSP_SAMPLE_ENA(1) |
4323 S_0286D0_PERSP_CENTER_ENA(1) |
4324 S_0286D0_PERSP_CENTROID_ENA(1) |
4325 S_0286D0_LINEAR_SAMPLE_ENA(1) |
4326 S_0286D0_LINEAR_CENTER_ENA(1) |
4327 S_0286D0_LINEAR_CENTROID_ENA(1) |
4328 S_0286D0_FRONT_FACE_ENA(1) |
4329 S_0286D0_POS_FIXED_PT_ENA(1));
4330 }
4331
4332 shader->info.num_input_sgprs = 0;
4333 shader->info.num_input_vgprs = 0;
4334
4335 for (i = 0; i <= last_sgpr; ++i)
4336 shader->info.num_input_sgprs += llvm_get_type_size(params[i]) / 4;
4337
4338 /* Unused fragment shader inputs are eliminated by the compiler,
4339 * so we don't know yet how many there will be.
4340 */
4341 if (ctx->type != TGSI_PROCESSOR_FRAGMENT)
4342 for (; i < num_params; ++i)
4343 shader->info.num_input_vgprs += llvm_get_type_size(params[i]) / 4;
4344
4345 if (bld_base->info &&
4346 (bld_base->info->opcode_count[TGSI_OPCODE_DDX] > 0 ||
4347 bld_base->info->opcode_count[TGSI_OPCODE_DDY] > 0 ||
4348 bld_base->info->opcode_count[TGSI_OPCODE_DDX_FINE] > 0 ||
4349 bld_base->info->opcode_count[TGSI_OPCODE_DDY_FINE] > 0 ||
4350 bld_base->info->opcode_count[TGSI_OPCODE_INTERP_OFFSET] > 0 ||
4351 bld_base->info->opcode_count[TGSI_OPCODE_INTERP_SAMPLE] > 0))
4352 ctx->lds =
4353 LLVMAddGlobalInAddressSpace(gallivm->module,
4354 LLVMArrayType(ctx->i32, 64),
4355 "ddxy_lds",
4356 LOCAL_ADDR_SPACE);
4357
4358 if ((ctx->type == TGSI_PROCESSOR_VERTEX && shader->key.vs.as_ls) ||
4359 ctx->type == TGSI_PROCESSOR_TESS_CTRL ||
4360 ctx->type == TGSI_PROCESSOR_TESS_EVAL)
4361 declare_tess_lds(ctx);
4362 }
4363
4364 static void preload_constants(struct si_shader_context *ctx)
4365 {
4366 struct lp_build_tgsi_context *bld_base = &ctx->radeon_bld.soa.bld_base;
4367 struct gallivm_state *gallivm = bld_base->base.gallivm;
4368 const struct tgsi_shader_info *info = bld_base->info;
4369 unsigned buf;
4370 LLVMValueRef ptr = LLVMGetParam(ctx->radeon_bld.main_fn, SI_PARAM_CONST_BUFFERS);
4371
4372 for (buf = 0; buf < SI_NUM_CONST_BUFFERS; buf++) {
4373 unsigned i, num_const = info->const_file_max[buf] + 1;
4374
4375 if (num_const == 0)
4376 continue;
4377
4378 /* Allocate space for the constant values */
4379 ctx->constants[buf] = CALLOC(num_const * 4, sizeof(LLVMValueRef));
4380
4381 /* Load the resource descriptor */
4382 ctx->const_buffers[buf] =
4383 build_indexed_load_const(ctx, ptr, lp_build_const_int32(gallivm, buf));
4384
4385 /* Load the constants, we rely on the code sinking to do the rest */
4386 for (i = 0; i < num_const * 4; ++i) {
4387 ctx->constants[buf][i] =
4388 buffer_load_const(gallivm->builder,
4389 ctx->const_buffers[buf],
4390 lp_build_const_int32(gallivm, i * 4),
4391 ctx->f32);
4392 }
4393 }
4394 }
4395
4396 static void preload_samplers(struct si_shader_context *ctx)
4397 {
4398 struct lp_build_tgsi_context *bld_base = &ctx->radeon_bld.soa.bld_base;
4399 struct gallivm_state *gallivm = bld_base->base.gallivm;
4400 const struct tgsi_shader_info *info = bld_base->info;
4401 unsigned i, num_samplers = info->file_max[TGSI_FILE_SAMPLER] + 1;
4402 LLVMValueRef offset;
4403
4404 if (num_samplers == 0)
4405 return;
4406
4407 /* Load the resources and samplers, we rely on the code sinking to do the rest */
4408 for (i = 0; i < num_samplers; ++i) {
4409 /* Resource */
4410 offset = lp_build_const_int32(gallivm, i);
4411 ctx->sampler_views[i] =
4412 get_sampler_desc(ctx, offset, DESC_IMAGE);
4413
4414 /* FMASK resource */
4415 if (info->is_msaa_sampler[i])
4416 ctx->fmasks[i] =
4417 get_sampler_desc(ctx, offset, DESC_FMASK);
4418 else
4419 ctx->sampler_states[i] =
4420 get_sampler_desc(ctx, offset, DESC_SAMPLER);
4421 }
4422 }
4423
4424 static void preload_images(struct si_shader_context *ctx)
4425 {
4426 struct lp_build_tgsi_context *bld_base = &ctx->radeon_bld.soa.bld_base;
4427 struct gallivm_state *gallivm = bld_base->base.gallivm;
4428 unsigned num_images = bld_base->info->file_max[TGSI_FILE_IMAGE] + 1;
4429 LLVMValueRef res_ptr;
4430 unsigned i;
4431
4432 if (num_images == 0)
4433 return;
4434
4435 res_ptr = LLVMGetParam(ctx->radeon_bld.main_fn, SI_PARAM_IMAGES);
4436
4437 for (i = 0; i < num_images; ++i) {
4438 /* Rely on LLVM to shrink the load for buffer resources. */
4439 ctx->images[i] =
4440 build_indexed_load_const(ctx, res_ptr,
4441 lp_build_const_int32(gallivm, i));
4442 }
4443 }
4444
4445 static void preload_streamout_buffers(struct si_shader_context *ctx)
4446 {
4447 struct lp_build_tgsi_context *bld_base = &ctx->radeon_bld.soa.bld_base;
4448 struct gallivm_state *gallivm = bld_base->base.gallivm;
4449 unsigned i;
4450
4451 /* Streamout can only be used if the shader is compiled as VS. */
4452 if (!ctx->shader->selector->so.num_outputs ||
4453 (ctx->type == TGSI_PROCESSOR_VERTEX &&
4454 (ctx->shader->key.vs.as_es ||
4455 ctx->shader->key.vs.as_ls)) ||
4456 (ctx->type == TGSI_PROCESSOR_TESS_EVAL &&
4457 ctx->shader->key.tes.as_es))
4458 return;
4459
4460 LLVMValueRef buf_ptr = LLVMGetParam(ctx->radeon_bld.main_fn,
4461 SI_PARAM_RW_BUFFERS);
4462
4463 /* Load the resources, we rely on the code sinking to do the rest */
4464 for (i = 0; i < 4; ++i) {
4465 if (ctx->shader->selector->so.stride[i]) {
4466 LLVMValueRef offset = lp_build_const_int32(gallivm,
4467 SI_SO_BUF_OFFSET + i);
4468
4469 ctx->so_buffers[i] = build_indexed_load_const(ctx, buf_ptr, offset);
4470 }
4471 }
4472 }
4473
4474 /**
4475 * Load ESGS and GSVS ring buffer resource descriptors and save the variables
4476 * for later use.
4477 */
4478 static void preload_ring_buffers(struct si_shader_context *ctx)
4479 {
4480 struct gallivm_state *gallivm =
4481 ctx->radeon_bld.soa.bld_base.base.gallivm;
4482
4483 LLVMValueRef buf_ptr = LLVMGetParam(ctx->radeon_bld.main_fn,
4484 SI_PARAM_RW_BUFFERS);
4485
4486 if ((ctx->type == TGSI_PROCESSOR_VERTEX &&
4487 ctx->shader->key.vs.as_es) ||
4488 (ctx->type == TGSI_PROCESSOR_TESS_EVAL &&
4489 ctx->shader->key.tes.as_es) ||
4490 ctx->type == TGSI_PROCESSOR_GEOMETRY) {
4491 LLVMValueRef offset = lp_build_const_int32(gallivm, SI_RING_ESGS);
4492
4493 ctx->esgs_ring =
4494 build_indexed_load_const(ctx, buf_ptr, offset);
4495 }
4496
4497 if (ctx->is_gs_copy_shader) {
4498 LLVMValueRef offset = lp_build_const_int32(gallivm, SI_RING_GSVS);
4499
4500 ctx->gsvs_ring[0] =
4501 build_indexed_load_const(ctx, buf_ptr, offset);
4502 }
4503 if (ctx->type == TGSI_PROCESSOR_GEOMETRY) {
4504 int i;
4505 for (i = 0; i < 4; i++) {
4506 LLVMValueRef offset = lp_build_const_int32(gallivm, SI_RING_GSVS + i);
4507
4508 ctx->gsvs_ring[i] =
4509 build_indexed_load_const(ctx, buf_ptr, offset);
4510 }
4511 }
4512 }
4513
4514 static void si_llvm_emit_polygon_stipple(struct si_shader_context *ctx,
4515 LLVMValueRef param_sampler_views,
4516 unsigned param_pos_fixed_pt)
4517 {
4518 struct lp_build_tgsi_context *bld_base =
4519 &ctx->radeon_bld.soa.bld_base;
4520 struct gallivm_state *gallivm = bld_base->base.gallivm;
4521 struct lp_build_emit_data result = {};
4522 struct tgsi_full_instruction inst = {};
4523 LLVMValueRef desc, sampler_index, address[2], pix;
4524
4525 /* Use the fixed-point gl_FragCoord input.
4526 * Since the stipple pattern is 32x32 and it repeats, just get 5 bits
4527 * per coordinate to get the repeating effect.
4528 */
4529 address[0] = unpack_param(ctx, param_pos_fixed_pt, 0, 5);
4530 address[1] = unpack_param(ctx, param_pos_fixed_pt, 16, 5);
4531
4532 /* Load the sampler view descriptor. */
4533 sampler_index = lp_build_const_int32(gallivm, SI_POLY_STIPPLE_SAMPLER);
4534 desc = get_sampler_desc_custom(ctx, param_sampler_views,
4535 sampler_index, DESC_IMAGE);
4536
4537 /* Load the texel. */
4538 inst.Instruction.Opcode = TGSI_OPCODE_TXF;
4539 inst.Texture.Texture = TGSI_TEXTURE_2D_MSAA; /* = use load, not load_mip */
4540 result.inst = &inst;
4541 set_tex_fetch_args(ctx, &result, TGSI_OPCODE_TXF,
4542 inst.Texture.Texture,
4543 desc, NULL, address, ARRAY_SIZE(address), 0xf);
4544 build_tex_intrinsic(&tex_action, bld_base, &result);
4545
4546 /* Kill the thread accordingly. */
4547 pix = LLVMBuildExtractElement(gallivm->builder, result.output[0],
4548 lp_build_const_int32(gallivm, 3), "");
4549 pix = bitcast(bld_base, TGSI_TYPE_FLOAT, pix);
4550 pix = LLVMBuildFNeg(gallivm->builder, pix, "");
4551
4552 lp_build_intrinsic(gallivm->builder, "llvm.AMDGPU.kill",
4553 LLVMVoidTypeInContext(gallivm->context),
4554 &pix, 1, 0);
4555 }
4556
4557 void si_shader_binary_read_config(struct radeon_shader_binary *binary,
4558 struct si_shader_config *conf,
4559 unsigned symbol_offset)
4560 {
4561 unsigned i;
4562 const unsigned char *config =
4563 radeon_shader_binary_config_start(binary, symbol_offset);
4564
4565 /* XXX: We may be able to emit some of these values directly rather than
4566 * extracting fields to be emitted later.
4567 */
4568
4569 for (i = 0; i < binary->config_size_per_symbol; i+= 8) {
4570 unsigned reg = util_le32_to_cpu(*(uint32_t*)(config + i));
4571 unsigned value = util_le32_to_cpu(*(uint32_t*)(config + i + 4));
4572 switch (reg) {
4573 case R_00B028_SPI_SHADER_PGM_RSRC1_PS:
4574 case R_00B128_SPI_SHADER_PGM_RSRC1_VS:
4575 case R_00B228_SPI_SHADER_PGM_RSRC1_GS:
4576 case R_00B848_COMPUTE_PGM_RSRC1:
4577 conf->num_sgprs = MAX2(conf->num_sgprs, (G_00B028_SGPRS(value) + 1) * 8);
4578 conf->num_vgprs = MAX2(conf->num_vgprs, (G_00B028_VGPRS(value) + 1) * 4);
4579 conf->float_mode = G_00B028_FLOAT_MODE(value);
4580 conf->rsrc1 = value;
4581 break;
4582 case R_00B02C_SPI_SHADER_PGM_RSRC2_PS:
4583 conf->lds_size = MAX2(conf->lds_size, G_00B02C_EXTRA_LDS_SIZE(value));
4584 break;
4585 case R_00B84C_COMPUTE_PGM_RSRC2:
4586 conf->lds_size = MAX2(conf->lds_size, G_00B84C_LDS_SIZE(value));
4587 conf->rsrc2 = value;
4588 break;
4589 case R_0286CC_SPI_PS_INPUT_ENA:
4590 conf->spi_ps_input_ena = value;
4591 break;
4592 case R_0286D0_SPI_PS_INPUT_ADDR:
4593 conf->spi_ps_input_addr = value;
4594 break;
4595 case R_0286E8_SPI_TMPRING_SIZE:
4596 case R_00B860_COMPUTE_TMPRING_SIZE:
4597 /* WAVESIZE is in units of 256 dwords. */
4598 conf->scratch_bytes_per_wave =
4599 G_00B860_WAVESIZE(value) * 256 * 4 * 1;
4600 break;
4601 default:
4602 {
4603 static bool printed;
4604
4605 if (!printed) {
4606 fprintf(stderr, "Warning: LLVM emitted unknown "
4607 "config register: 0x%x\n", reg);
4608 printed = true;
4609 }
4610 }
4611 break;
4612 }
4613
4614 if (!conf->spi_ps_input_addr)
4615 conf->spi_ps_input_addr = conf->spi_ps_input_ena;
4616 }
4617 }
4618
4619 void si_shader_apply_scratch_relocs(struct si_context *sctx,
4620 struct si_shader *shader,
4621 uint64_t scratch_va)
4622 {
4623 unsigned i;
4624 uint32_t scratch_rsrc_dword0 = scratch_va;
4625 uint32_t scratch_rsrc_dword1 =
4626 S_008F04_BASE_ADDRESS_HI(scratch_va >> 32)
4627 | S_008F04_STRIDE(shader->config.scratch_bytes_per_wave / 64);
4628
4629 for (i = 0 ; i < shader->binary.reloc_count; i++) {
4630 const struct radeon_shader_reloc *reloc =
4631 &shader->binary.relocs[i];
4632 if (!strcmp(scratch_rsrc_dword0_symbol, reloc->name)) {
4633 util_memcpy_cpu_to_le32(shader->binary.code + reloc->offset,
4634 &scratch_rsrc_dword0, 4);
4635 } else if (!strcmp(scratch_rsrc_dword1_symbol, reloc->name)) {
4636 util_memcpy_cpu_to_le32(shader->binary.code + reloc->offset,
4637 &scratch_rsrc_dword1, 4);
4638 }
4639 }
4640 }
4641
4642 static unsigned si_get_shader_binary_size(struct si_shader *shader)
4643 {
4644 unsigned size = shader->binary.code_size;
4645
4646 if (shader->prolog)
4647 size += shader->prolog->binary.code_size;
4648 if (shader->epilog)
4649 size += shader->epilog->binary.code_size;
4650 return size;
4651 }
4652
4653 int si_shader_binary_upload(struct si_screen *sscreen, struct si_shader *shader)
4654 {
4655 const struct radeon_shader_binary *prolog =
4656 shader->prolog ? &shader->prolog->binary : NULL;
4657 const struct radeon_shader_binary *epilog =
4658 shader->epilog ? &shader->epilog->binary : NULL;
4659 const struct radeon_shader_binary *mainb = &shader->binary;
4660 unsigned bo_size = si_get_shader_binary_size(shader) +
4661 (!epilog ? mainb->rodata_size : 0);
4662 unsigned char *ptr;
4663
4664 assert(!prolog || !prolog->rodata_size);
4665 assert((!prolog && !epilog) || !mainb->rodata_size);
4666 assert(!epilog || !epilog->rodata_size);
4667
4668 r600_resource_reference(&shader->bo, NULL);
4669 shader->bo = si_resource_create_custom(&sscreen->b.b,
4670 PIPE_USAGE_IMMUTABLE,
4671 bo_size);
4672 if (!shader->bo)
4673 return -ENOMEM;
4674
4675 /* Upload. */
4676 ptr = sscreen->b.ws->buffer_map(shader->bo->buf, NULL,
4677 PIPE_TRANSFER_READ_WRITE);
4678
4679 if (prolog) {
4680 util_memcpy_cpu_to_le32(ptr, prolog->code, prolog->code_size);
4681 ptr += prolog->code_size;
4682 }
4683
4684 util_memcpy_cpu_to_le32(ptr, mainb->code, mainb->code_size);
4685 ptr += mainb->code_size;
4686
4687 if (epilog)
4688 util_memcpy_cpu_to_le32(ptr, epilog->code, epilog->code_size);
4689 else if (mainb->rodata_size > 0)
4690 util_memcpy_cpu_to_le32(ptr, mainb->rodata, mainb->rodata_size);
4691
4692 sscreen->b.ws->buffer_unmap(shader->bo->buf);
4693 return 0;
4694 }
4695
4696 static void si_shader_dump_disassembly(const struct radeon_shader_binary *binary,
4697 struct pipe_debug_callback *debug,
4698 const char *name, FILE *file)
4699 {
4700 char *line, *p;
4701 unsigned i, count;
4702
4703 if (binary->disasm_string) {
4704 fprintf(file, "Shader %s disassembly:\n", name);
4705 fprintf(file, "%s", binary->disasm_string);
4706
4707 if (debug && debug->debug_message) {
4708 /* Very long debug messages are cut off, so send the
4709 * disassembly one line at a time. This causes more
4710 * overhead, but on the plus side it simplifies
4711 * parsing of resulting logs.
4712 */
4713 pipe_debug_message(debug, SHADER_INFO,
4714 "Shader Disassembly Begin");
4715
4716 line = binary->disasm_string;
4717 while (*line) {
4718 p = strchrnul(line, '\n');
4719 count = p - line;
4720
4721 if (count) {
4722 pipe_debug_message(debug, SHADER_INFO,
4723 "%.*s", count, line);
4724 }
4725
4726 if (!*p)
4727 break;
4728 line = p + 1;
4729 }
4730
4731 pipe_debug_message(debug, SHADER_INFO,
4732 "Shader Disassembly End");
4733 }
4734 } else {
4735 fprintf(file, "Shader %s binary:\n", name);
4736 for (i = 0; i < binary->code_size; i += 4) {
4737 fprintf(file, "@0x%x: %02x%02x%02x%02x\n", i,
4738 binary->code[i + 3], binary->code[i + 2],
4739 binary->code[i + 1], binary->code[i]);
4740 }
4741 }
4742 }
4743
4744 static void si_shader_dump_stats(struct si_screen *sscreen,
4745 struct si_shader_config *conf,
4746 unsigned num_inputs,
4747 unsigned code_size,
4748 struct pipe_debug_callback *debug,
4749 unsigned processor,
4750 FILE *file)
4751 {
4752 unsigned lds_increment = sscreen->b.chip_class >= CIK ? 512 : 256;
4753 unsigned lds_per_wave = 0;
4754 unsigned max_simd_waves = 10;
4755
4756 /* Compute LDS usage for PS. */
4757 if (processor == TGSI_PROCESSOR_FRAGMENT) {
4758 /* The minimum usage per wave is (num_inputs * 36). The maximum
4759 * usage is (num_inputs * 36 * 16).
4760 * We can get anything in between and it varies between waves.
4761 *
4762 * Other stages don't know the size at compile time or don't
4763 * allocate LDS per wave, but instead they do it per thread group.
4764 */
4765 lds_per_wave = conf->lds_size * lds_increment +
4766 align(num_inputs * 36, lds_increment);
4767 }
4768
4769 /* Compute the per-SIMD wave counts. */
4770 if (conf->num_sgprs) {
4771 if (sscreen->b.chip_class >= VI)
4772 max_simd_waves = MIN2(max_simd_waves, 800 / conf->num_sgprs);
4773 else
4774 max_simd_waves = MIN2(max_simd_waves, 512 / conf->num_sgprs);
4775 }
4776
4777 if (conf->num_vgprs)
4778 max_simd_waves = MIN2(max_simd_waves, 256 / conf->num_vgprs);
4779
4780 /* LDS is 64KB per CU (4 SIMDs), divided into 16KB blocks per SIMD
4781 * that PS can use.
4782 */
4783 if (lds_per_wave)
4784 max_simd_waves = MIN2(max_simd_waves, 16384 / lds_per_wave);
4785
4786 if (file != stderr ||
4787 r600_can_dump_shader(&sscreen->b, processor)) {
4788 if (processor == TGSI_PROCESSOR_FRAGMENT) {
4789 fprintf(file, "*** SHADER CONFIG ***\n"
4790 "SPI_PS_INPUT_ADDR = 0x%04x\n"
4791 "SPI_PS_INPUT_ENA = 0x%04x\n",
4792 conf->spi_ps_input_addr, conf->spi_ps_input_ena);
4793 }
4794
4795 fprintf(file, "*** SHADER STATS ***\n"
4796 "SGPRS: %d\n"
4797 "VGPRS: %d\n"
4798 "Code Size: %d bytes\n"
4799 "LDS: %d blocks\n"
4800 "Scratch: %d bytes per wave\n"
4801 "Max Waves: %d\n"
4802 "********************\n",
4803 conf->num_sgprs, conf->num_vgprs, code_size,
4804 conf->lds_size, conf->scratch_bytes_per_wave,
4805 max_simd_waves);
4806 }
4807
4808 pipe_debug_message(debug, SHADER_INFO,
4809 "Shader Stats: SGPRS: %d VGPRS: %d Code Size: %d "
4810 "LDS: %d Scratch: %d Max Waves: %d",
4811 conf->num_sgprs, conf->num_vgprs, code_size,
4812 conf->lds_size, conf->scratch_bytes_per_wave,
4813 max_simd_waves);
4814 }
4815
4816 static const char *si_get_shader_name(struct si_shader *shader,
4817 unsigned processor)
4818 {
4819 switch (processor) {
4820 case TGSI_PROCESSOR_VERTEX:
4821 if (shader->key.vs.as_es)
4822 return "Vertex Shader as ES";
4823 else if (shader->key.vs.as_ls)
4824 return "Vertex Shader as LS";
4825 else
4826 return "Vertex Shader as VS";
4827 case TGSI_PROCESSOR_TESS_CTRL:
4828 return "Tessellation Control Shader";
4829 case TGSI_PROCESSOR_TESS_EVAL:
4830 if (shader->key.tes.as_es)
4831 return "Tessellation Evaluation Shader as ES";
4832 else
4833 return "Tessellation Evaluation Shader as VS";
4834 case TGSI_PROCESSOR_GEOMETRY:
4835 if (shader->gs_copy_shader == NULL)
4836 return "GS Copy Shader as VS";
4837 else
4838 return "Geometry Shader";
4839 case TGSI_PROCESSOR_FRAGMENT:
4840 return "Pixel Shader";
4841 case TGSI_PROCESSOR_COMPUTE:
4842 return "Compute Shader";
4843 default:
4844 return "Unknown Shader";
4845 }
4846 }
4847
4848 void si_shader_dump(struct si_screen *sscreen, struct si_shader *shader,
4849 struct pipe_debug_callback *debug, unsigned processor,
4850 FILE *file)
4851 {
4852 if (file != stderr ||
4853 (r600_can_dump_shader(&sscreen->b, processor) &&
4854 !(sscreen->b.debug_flags & DBG_NO_ASM))) {
4855 fprintf(file, "\n%s:\n", si_get_shader_name(shader, processor));
4856
4857 if (shader->prolog)
4858 si_shader_dump_disassembly(&shader->prolog->binary,
4859 debug, "prolog", file);
4860
4861 si_shader_dump_disassembly(&shader->binary, debug, "main", file);
4862
4863 if (shader->epilog)
4864 si_shader_dump_disassembly(&shader->epilog->binary,
4865 debug, "epilog", file);
4866 fprintf(file, "\n");
4867 }
4868
4869 si_shader_dump_stats(sscreen, &shader->config,
4870 shader->selector ? shader->selector->info.num_inputs : 0,
4871 si_get_shader_binary_size(shader), debug, processor,
4872 file);
4873 }
4874
4875 int si_compile_llvm(struct si_screen *sscreen,
4876 struct radeon_shader_binary *binary,
4877 struct si_shader_config *conf,
4878 LLVMTargetMachineRef tm,
4879 LLVMModuleRef mod,
4880 struct pipe_debug_callback *debug,
4881 unsigned processor,
4882 const char *name)
4883 {
4884 int r = 0;
4885 unsigned count = p_atomic_inc_return(&sscreen->b.num_compilations);
4886
4887 if (r600_can_dump_shader(&sscreen->b, processor)) {
4888 fprintf(stderr, "radeonsi: Compiling shader %d\n", count);
4889
4890 if (!(sscreen->b.debug_flags & (DBG_NO_IR | DBG_PREOPT_IR))) {
4891 fprintf(stderr, "%s LLVM IR:\n\n", name);
4892 LLVMDumpModule(mod);
4893 fprintf(stderr, "\n");
4894 }
4895 }
4896
4897 if (!si_replace_shader(count, binary)) {
4898 r = radeon_llvm_compile(mod, binary,
4899 r600_get_llvm_processor_name(sscreen->b.family), tm,
4900 debug);
4901 if (r)
4902 return r;
4903 }
4904
4905 si_shader_binary_read_config(binary, conf, 0);
4906
4907 /* Enable 64-bit and 16-bit denormals, because there is no performance
4908 * cost.
4909 *
4910 * If denormals are enabled, all floating-point output modifiers are
4911 * ignored.
4912 *
4913 * Don't enable denormals for 32-bit floats, because:
4914 * - Floating-point output modifiers would be ignored by the hw.
4915 * - Some opcodes don't support denormals, such as v_mad_f32. We would
4916 * have to stop using those.
4917 * - SI & CI would be very slow.
4918 */
4919 conf->float_mode |= V_00B028_FP_64_DENORMS;
4920
4921 FREE(binary->config);
4922 FREE(binary->global_symbol_offsets);
4923 binary->config = NULL;
4924 binary->global_symbol_offsets = NULL;
4925
4926 /* Some shaders can't have rodata because their binaries can be
4927 * concatenated.
4928 */
4929 if (binary->rodata_size &&
4930 (processor == TGSI_PROCESSOR_VERTEX ||
4931 processor == TGSI_PROCESSOR_TESS_CTRL ||
4932 processor == TGSI_PROCESSOR_TESS_EVAL ||
4933 processor == TGSI_PROCESSOR_FRAGMENT)) {
4934 fprintf(stderr, "radeonsi: The shader can't have rodata.");
4935 return -EINVAL;
4936 }
4937
4938 return r;
4939 }
4940
4941 /* Generate code for the hardware VS shader stage to go with a geometry shader */
4942 static int si_generate_gs_copy_shader(struct si_screen *sscreen,
4943 struct si_shader_context *ctx,
4944 struct si_shader *gs,
4945 struct pipe_debug_callback *debug)
4946 {
4947 struct gallivm_state *gallivm = &ctx->radeon_bld.gallivm;
4948 struct lp_build_tgsi_context *bld_base = &ctx->radeon_bld.soa.bld_base;
4949 struct lp_build_context *uint = &bld_base->uint_bld;
4950 struct si_shader_output_values *outputs;
4951 struct tgsi_shader_info *gsinfo = &gs->selector->info;
4952 LLVMValueRef args[9];
4953 int i, r;
4954
4955 outputs = MALLOC(gsinfo->num_outputs * sizeof(outputs[0]));
4956
4957 si_init_shader_ctx(ctx, sscreen, ctx->shader, ctx->tm);
4958 ctx->type = TGSI_PROCESSOR_VERTEX;
4959 ctx->is_gs_copy_shader = true;
4960
4961 create_meta_data(ctx);
4962 create_function(ctx);
4963 preload_streamout_buffers(ctx);
4964 preload_ring_buffers(ctx);
4965
4966 args[0] = ctx->gsvs_ring[0];
4967 args[1] = lp_build_mul_imm(uint,
4968 LLVMGetParam(ctx->radeon_bld.main_fn,
4969 ctx->param_vertex_id),
4970 4);
4971 args[3] = uint->zero;
4972 args[4] = uint->one; /* OFFEN */
4973 args[5] = uint->zero; /* IDXEN */
4974 args[6] = uint->one; /* GLC */
4975 args[7] = uint->one; /* SLC */
4976 args[8] = uint->zero; /* TFE */
4977
4978 /* Fetch vertex data from GSVS ring */
4979 for (i = 0; i < gsinfo->num_outputs; ++i) {
4980 unsigned chan;
4981
4982 outputs[i].name = gsinfo->output_semantic_name[i];
4983 outputs[i].sid = gsinfo->output_semantic_index[i];
4984
4985 for (chan = 0; chan < 4; chan++) {
4986 args[2] = lp_build_const_int32(gallivm,
4987 (i * 4 + chan) *
4988 gs->selector->gs_max_out_vertices * 16 * 4);
4989
4990 outputs[i].values[chan] =
4991 LLVMBuildBitCast(gallivm->builder,
4992 lp_build_intrinsic(gallivm->builder,
4993 "llvm.SI.buffer.load.dword.i32.i32",
4994 ctx->i32, args, 9,
4995 LLVMReadOnlyAttribute | LLVMNoUnwindAttribute),
4996 ctx->f32, "");
4997 }
4998 }
4999
5000 si_llvm_export_vs(bld_base, outputs, gsinfo->num_outputs);
5001
5002 LLVMBuildRet(gallivm->builder, ctx->return_value);
5003
5004 /* Dump LLVM IR before any optimization passes */
5005 if (sscreen->b.debug_flags & DBG_PREOPT_IR &&
5006 r600_can_dump_shader(&sscreen->b, TGSI_PROCESSOR_GEOMETRY))
5007 LLVMDumpModule(bld_base->base.gallivm->module);
5008
5009 radeon_llvm_finalize_module(&ctx->radeon_bld);
5010
5011 r = si_compile_llvm(sscreen, &ctx->shader->binary,
5012 &ctx->shader->config, ctx->tm,
5013 bld_base->base.gallivm->module,
5014 debug, TGSI_PROCESSOR_GEOMETRY,
5015 "GS Copy Shader");
5016 if (!r) {
5017 if (r600_can_dump_shader(&sscreen->b, TGSI_PROCESSOR_GEOMETRY))
5018 fprintf(stderr, "GS Copy Shader:\n");
5019 si_shader_dump(sscreen, ctx->shader, debug,
5020 TGSI_PROCESSOR_GEOMETRY, stderr);
5021 r = si_shader_binary_upload(sscreen, ctx->shader);
5022 }
5023
5024 radeon_llvm_dispose(&ctx->radeon_bld);
5025
5026 FREE(outputs);
5027 return r;
5028 }
5029
5030 void si_dump_shader_key(unsigned shader, union si_shader_key *key, FILE *f)
5031 {
5032 int i;
5033
5034 fprintf(f, "SHADER KEY\n");
5035
5036 switch (shader) {
5037 case PIPE_SHADER_VERTEX:
5038 fprintf(f, " instance_divisors = {");
5039 for (i = 0; i < Elements(key->vs.prolog.instance_divisors); i++)
5040 fprintf(f, !i ? "%u" : ", %u",
5041 key->vs.prolog.instance_divisors[i]);
5042 fprintf(f, "}\n");
5043 fprintf(f, " as_es = %u\n", key->vs.as_es);
5044 fprintf(f, " as_ls = %u\n", key->vs.as_ls);
5045 fprintf(f, " export_prim_id = %u\n", key->vs.epilog.export_prim_id);
5046 break;
5047
5048 case PIPE_SHADER_TESS_CTRL:
5049 fprintf(f, " prim_mode = %u\n", key->tcs.epilog.prim_mode);
5050 break;
5051
5052 case PIPE_SHADER_TESS_EVAL:
5053 fprintf(f, " as_es = %u\n", key->tes.as_es);
5054 fprintf(f, " export_prim_id = %u\n", key->tes.epilog.export_prim_id);
5055 break;
5056
5057 case PIPE_SHADER_GEOMETRY:
5058 break;
5059
5060 case PIPE_SHADER_FRAGMENT:
5061 fprintf(f, " prolog.color_two_side = %u\n", key->ps.prolog.color_two_side);
5062 fprintf(f, " prolog.poly_stipple = %u\n", key->ps.prolog.poly_stipple);
5063 fprintf(f, " prolog.force_persample_interp = %u\n", key->ps.prolog.force_persample_interp);
5064 fprintf(f, " epilog.spi_shader_col_format = 0x%x\n", key->ps.epilog.spi_shader_col_format);
5065 fprintf(f, " epilog.color_is_int8 = 0x%X\n", key->ps.epilog.color_is_int8);
5066 fprintf(f, " epilog.last_cbuf = %u\n", key->ps.epilog.last_cbuf);
5067 fprintf(f, " epilog.alpha_func = %u\n", key->ps.epilog.alpha_func);
5068 fprintf(f, " epilog.alpha_to_one = %u\n", key->ps.epilog.alpha_to_one);
5069 fprintf(f, " epilog.poly_line_smoothing = %u\n", key->ps.epilog.poly_line_smoothing);
5070 fprintf(f, " epilog.clamp_color = %u\n", key->ps.epilog.clamp_color);
5071 break;
5072
5073 default:
5074 assert(0);
5075 }
5076 }
5077
5078 static void si_init_shader_ctx(struct si_shader_context *ctx,
5079 struct si_screen *sscreen,
5080 struct si_shader *shader,
5081 LLVMTargetMachineRef tm)
5082 {
5083 struct lp_build_tgsi_context *bld_base;
5084
5085 memset(ctx, 0, sizeof(*ctx));
5086 radeon_llvm_context_init(&ctx->radeon_bld, "amdgcn--");
5087 ctx->tm = tm;
5088 ctx->screen = sscreen;
5089 if (shader && shader->selector)
5090 ctx->type = shader->selector->info.processor;
5091 else
5092 ctx->type = -1;
5093 ctx->shader = shader;
5094
5095 ctx->voidt = LLVMVoidTypeInContext(ctx->radeon_bld.gallivm.context);
5096 ctx->i1 = LLVMInt1TypeInContext(ctx->radeon_bld.gallivm.context);
5097 ctx->i8 = LLVMInt8TypeInContext(ctx->radeon_bld.gallivm.context);
5098 ctx->i32 = LLVMInt32TypeInContext(ctx->radeon_bld.gallivm.context);
5099 ctx->i64 = LLVMInt64TypeInContext(ctx->radeon_bld.gallivm.context);
5100 ctx->i128 = LLVMIntTypeInContext(ctx->radeon_bld.gallivm.context, 128);
5101 ctx->f32 = LLVMFloatTypeInContext(ctx->radeon_bld.gallivm.context);
5102 ctx->v16i8 = LLVMVectorType(ctx->i8, 16);
5103 ctx->v2i32 = LLVMVectorType(ctx->i32, 2);
5104 ctx->v4i32 = LLVMVectorType(ctx->i32, 4);
5105 ctx->v4f32 = LLVMVectorType(ctx->f32, 4);
5106 ctx->v8i32 = LLVMVectorType(ctx->i32, 8);
5107
5108 bld_base = &ctx->radeon_bld.soa.bld_base;
5109 if (shader && shader->selector)
5110 bld_base->info = &shader->selector->info;
5111 bld_base->emit_fetch_funcs[TGSI_FILE_CONSTANT] = fetch_constant;
5112
5113 bld_base->op_actions[TGSI_OPCODE_INTERP_CENTROID] = interp_action;
5114 bld_base->op_actions[TGSI_OPCODE_INTERP_SAMPLE] = interp_action;
5115 bld_base->op_actions[TGSI_OPCODE_INTERP_OFFSET] = interp_action;
5116
5117 bld_base->op_actions[TGSI_OPCODE_TEX] = tex_action;
5118 bld_base->op_actions[TGSI_OPCODE_TEX2] = tex_action;
5119 bld_base->op_actions[TGSI_OPCODE_TXB] = tex_action;
5120 bld_base->op_actions[TGSI_OPCODE_TXB2] = tex_action;
5121 bld_base->op_actions[TGSI_OPCODE_TXD] = tex_action;
5122 bld_base->op_actions[TGSI_OPCODE_TXF] = tex_action;
5123 bld_base->op_actions[TGSI_OPCODE_TXL] = tex_action;
5124 bld_base->op_actions[TGSI_OPCODE_TXL2] = tex_action;
5125 bld_base->op_actions[TGSI_OPCODE_TXP] = tex_action;
5126 bld_base->op_actions[TGSI_OPCODE_TXQ] = tex_action;
5127 bld_base->op_actions[TGSI_OPCODE_TG4] = tex_action;
5128 bld_base->op_actions[TGSI_OPCODE_LODQ] = tex_action;
5129 bld_base->op_actions[TGSI_OPCODE_TXQS].emit = si_llvm_emit_txqs;
5130
5131 bld_base->op_actions[TGSI_OPCODE_LOAD].fetch_args = load_fetch_args;
5132 bld_base->op_actions[TGSI_OPCODE_LOAD].emit = load_emit;
5133 bld_base->op_actions[TGSI_OPCODE_RESQ].fetch_args = resq_fetch_args;
5134 bld_base->op_actions[TGSI_OPCODE_RESQ].emit = resq_emit;
5135
5136 bld_base->op_actions[TGSI_OPCODE_DDX].emit = si_llvm_emit_ddxy;
5137 bld_base->op_actions[TGSI_OPCODE_DDY].emit = si_llvm_emit_ddxy;
5138 bld_base->op_actions[TGSI_OPCODE_DDX_FINE].emit = si_llvm_emit_ddxy;
5139 bld_base->op_actions[TGSI_OPCODE_DDY_FINE].emit = si_llvm_emit_ddxy;
5140
5141 bld_base->op_actions[TGSI_OPCODE_EMIT].emit = si_llvm_emit_vertex;
5142 bld_base->op_actions[TGSI_OPCODE_ENDPRIM].emit = si_llvm_emit_primitive;
5143 bld_base->op_actions[TGSI_OPCODE_BARRIER].emit = si_llvm_emit_barrier;
5144
5145 bld_base->op_actions[TGSI_OPCODE_MAX].emit = build_tgsi_intrinsic_nomem;
5146 bld_base->op_actions[TGSI_OPCODE_MAX].intr_name = "llvm.maxnum.f32";
5147 bld_base->op_actions[TGSI_OPCODE_MIN].emit = build_tgsi_intrinsic_nomem;
5148 bld_base->op_actions[TGSI_OPCODE_MIN].intr_name = "llvm.minnum.f32";
5149 }
5150
5151 int si_compile_tgsi_shader(struct si_screen *sscreen,
5152 LLVMTargetMachineRef tm,
5153 struct si_shader *shader,
5154 bool is_monolithic,
5155 struct pipe_debug_callback *debug)
5156 {
5157 struct si_shader_selector *sel = shader->selector;
5158 struct si_shader_context ctx;
5159 struct lp_build_tgsi_context *bld_base;
5160 LLVMModuleRef mod;
5161 int r = 0;
5162
5163 /* Dump TGSI code before doing TGSI->LLVM conversion in case the
5164 * conversion fails. */
5165 if (r600_can_dump_shader(&sscreen->b, sel->info.processor) &&
5166 !(sscreen->b.debug_flags & DBG_NO_TGSI)) {
5167 si_dump_shader_key(sel->type, &shader->key, stderr);
5168 tgsi_dump(sel->tokens, 0);
5169 si_dump_streamout(&sel->so);
5170 }
5171
5172 si_init_shader_ctx(&ctx, sscreen, shader, tm);
5173 ctx.is_monolithic = is_monolithic;
5174
5175 shader->info.uses_instanceid = sel->info.uses_instanceid;
5176
5177 bld_base = &ctx.radeon_bld.soa.bld_base;
5178 ctx.radeon_bld.load_system_value = declare_system_value;
5179
5180 switch (ctx.type) {
5181 case TGSI_PROCESSOR_VERTEX:
5182 ctx.radeon_bld.load_input = declare_input_vs;
5183 if (shader->key.vs.as_ls)
5184 bld_base->emit_epilogue = si_llvm_emit_ls_epilogue;
5185 else if (shader->key.vs.as_es)
5186 bld_base->emit_epilogue = si_llvm_emit_es_epilogue;
5187 else
5188 bld_base->emit_epilogue = si_llvm_emit_vs_epilogue;
5189 break;
5190 case TGSI_PROCESSOR_TESS_CTRL:
5191 bld_base->emit_fetch_funcs[TGSI_FILE_INPUT] = fetch_input_tcs;
5192 bld_base->emit_fetch_funcs[TGSI_FILE_OUTPUT] = fetch_output_tcs;
5193 bld_base->emit_store = store_output_tcs;
5194 bld_base->emit_epilogue = si_llvm_emit_tcs_epilogue;
5195 break;
5196 case TGSI_PROCESSOR_TESS_EVAL:
5197 bld_base->emit_fetch_funcs[TGSI_FILE_INPUT] = fetch_input_tes;
5198 if (shader->key.tes.as_es)
5199 bld_base->emit_epilogue = si_llvm_emit_es_epilogue;
5200 else
5201 bld_base->emit_epilogue = si_llvm_emit_vs_epilogue;
5202 break;
5203 case TGSI_PROCESSOR_GEOMETRY:
5204 bld_base->emit_fetch_funcs[TGSI_FILE_INPUT] = fetch_input_gs;
5205 bld_base->emit_epilogue = si_llvm_emit_gs_epilogue;
5206 break;
5207 case TGSI_PROCESSOR_FRAGMENT:
5208 ctx.radeon_bld.load_input = declare_input_fs;
5209 if (is_monolithic)
5210 bld_base->emit_epilogue = si_llvm_emit_fs_epilogue;
5211 else
5212 bld_base->emit_epilogue = si_llvm_return_fs_outputs;
5213 break;
5214 default:
5215 assert(!"Unsupported shader type");
5216 return -1;
5217 }
5218
5219 create_meta_data(&ctx);
5220 create_function(&ctx);
5221 preload_constants(&ctx);
5222 preload_samplers(&ctx);
5223 preload_images(&ctx);
5224 preload_streamout_buffers(&ctx);
5225 preload_ring_buffers(&ctx);
5226
5227 if (ctx.is_monolithic && sel->type == PIPE_SHADER_FRAGMENT &&
5228 shader->key.ps.prolog.poly_stipple) {
5229 LLVMValueRef views = LLVMGetParam(ctx.radeon_bld.main_fn,
5230 SI_PARAM_SAMPLERS);
5231 si_llvm_emit_polygon_stipple(&ctx, views,
5232 SI_PARAM_POS_FIXED_PT);
5233 }
5234
5235 if (ctx.type == TGSI_PROCESSOR_GEOMETRY) {
5236 int i;
5237 for (i = 0; i < 4; i++) {
5238 ctx.gs_next_vertex[i] =
5239 lp_build_alloca(bld_base->base.gallivm,
5240 ctx.i32, "");
5241 }
5242 }
5243
5244 if (!lp_build_tgsi_llvm(bld_base, sel->tokens)) {
5245 fprintf(stderr, "Failed to translate shader from TGSI to LLVM\n");
5246 goto out;
5247 }
5248
5249 LLVMBuildRet(bld_base->base.gallivm->builder, ctx.return_value);
5250 mod = bld_base->base.gallivm->module;
5251
5252 /* Dump LLVM IR before any optimization passes */
5253 if (sscreen->b.debug_flags & DBG_PREOPT_IR &&
5254 r600_can_dump_shader(&sscreen->b, ctx.type))
5255 LLVMDumpModule(mod);
5256
5257 radeon_llvm_finalize_module(&ctx.radeon_bld);
5258
5259 r = si_compile_llvm(sscreen, &shader->binary, &shader->config, tm,
5260 mod, debug, ctx.type, "TGSI shader");
5261 if (r) {
5262 fprintf(stderr, "LLVM failed to compile shader\n");
5263 goto out;
5264 }
5265
5266 radeon_llvm_dispose(&ctx.radeon_bld);
5267
5268 /* Calculate the number of fragment input VGPRs. */
5269 if (ctx.type == TGSI_PROCESSOR_FRAGMENT) {
5270 shader->info.num_input_vgprs = 0;
5271 shader->info.face_vgpr_index = -1;
5272
5273 if (G_0286CC_PERSP_SAMPLE_ENA(shader->config.spi_ps_input_addr))
5274 shader->info.num_input_vgprs += 2;
5275 if (G_0286CC_PERSP_CENTER_ENA(shader->config.spi_ps_input_addr))
5276 shader->info.num_input_vgprs += 2;
5277 if (G_0286CC_PERSP_CENTROID_ENA(shader->config.spi_ps_input_addr))
5278 shader->info.num_input_vgprs += 2;
5279 if (G_0286CC_PERSP_PULL_MODEL_ENA(shader->config.spi_ps_input_addr))
5280 shader->info.num_input_vgprs += 3;
5281 if (G_0286CC_LINEAR_SAMPLE_ENA(shader->config.spi_ps_input_addr))
5282 shader->info.num_input_vgprs += 2;
5283 if (G_0286CC_LINEAR_CENTER_ENA(shader->config.spi_ps_input_addr))
5284 shader->info.num_input_vgprs += 2;
5285 if (G_0286CC_LINEAR_CENTROID_ENA(shader->config.spi_ps_input_addr))
5286 shader->info.num_input_vgprs += 2;
5287 if (G_0286CC_LINE_STIPPLE_TEX_ENA(shader->config.spi_ps_input_addr))
5288 shader->info.num_input_vgprs += 1;
5289 if (G_0286CC_POS_X_FLOAT_ENA(shader->config.spi_ps_input_addr))
5290 shader->info.num_input_vgprs += 1;
5291 if (G_0286CC_POS_Y_FLOAT_ENA(shader->config.spi_ps_input_addr))
5292 shader->info.num_input_vgprs += 1;
5293 if (G_0286CC_POS_Z_FLOAT_ENA(shader->config.spi_ps_input_addr))
5294 shader->info.num_input_vgprs += 1;
5295 if (G_0286CC_POS_W_FLOAT_ENA(shader->config.spi_ps_input_addr))
5296 shader->info.num_input_vgprs += 1;
5297 if (G_0286CC_FRONT_FACE_ENA(shader->config.spi_ps_input_addr)) {
5298 shader->info.face_vgpr_index = shader->info.num_input_vgprs;
5299 shader->info.num_input_vgprs += 1;
5300 }
5301 if (G_0286CC_ANCILLARY_ENA(shader->config.spi_ps_input_addr))
5302 shader->info.num_input_vgprs += 1;
5303 if (G_0286CC_SAMPLE_COVERAGE_ENA(shader->config.spi_ps_input_addr))
5304 shader->info.num_input_vgprs += 1;
5305 if (G_0286CC_POS_FIXED_PT_ENA(shader->config.spi_ps_input_addr))
5306 shader->info.num_input_vgprs += 1;
5307 }
5308
5309 if (ctx.type == TGSI_PROCESSOR_GEOMETRY) {
5310 shader->gs_copy_shader = CALLOC_STRUCT(si_shader);
5311 shader->gs_copy_shader->selector = shader->selector;
5312 ctx.shader = shader->gs_copy_shader;
5313 if ((r = si_generate_gs_copy_shader(sscreen, &ctx,
5314 shader, debug))) {
5315 free(shader->gs_copy_shader);
5316 shader->gs_copy_shader = NULL;
5317 goto out;
5318 }
5319 }
5320
5321 out:
5322 for (int i = 0; i < SI_NUM_CONST_BUFFERS; i++)
5323 FREE(ctx.constants[i]);
5324 return r;
5325 }
5326
5327 /**
5328 * Create, compile and return a shader part (prolog or epilog).
5329 *
5330 * \param sscreen screen
5331 * \param list list of shader parts of the same category
5332 * \param key shader part key
5333 * \param tm LLVM target machine
5334 * \param debug debug callback
5335 * \param compile the callback responsible for compilation
5336 * \return non-NULL on success
5337 */
5338 static struct si_shader_part *
5339 si_get_shader_part(struct si_screen *sscreen,
5340 struct si_shader_part **list,
5341 union si_shader_part_key *key,
5342 LLVMTargetMachineRef tm,
5343 struct pipe_debug_callback *debug,
5344 bool (*compile)(struct si_screen *,
5345 LLVMTargetMachineRef,
5346 struct pipe_debug_callback *,
5347 struct si_shader_part *))
5348 {
5349 struct si_shader_part *result;
5350
5351 pipe_mutex_lock(sscreen->shader_parts_mutex);
5352
5353 /* Find existing. */
5354 for (result = *list; result; result = result->next) {
5355 if (memcmp(&result->key, key, sizeof(*key)) == 0) {
5356 pipe_mutex_unlock(sscreen->shader_parts_mutex);
5357 return result;
5358 }
5359 }
5360
5361 /* Compile a new one. */
5362 result = CALLOC_STRUCT(si_shader_part);
5363 result->key = *key;
5364 if (!compile(sscreen, tm, debug, result)) {
5365 FREE(result);
5366 pipe_mutex_unlock(sscreen->shader_parts_mutex);
5367 return NULL;
5368 }
5369
5370 result->next = *list;
5371 *list = result;
5372 pipe_mutex_unlock(sscreen->shader_parts_mutex);
5373 return result;
5374 }
5375
5376 /**
5377 * Create a vertex shader prolog.
5378 *
5379 * The inputs are the same as VS (a lot of SGPRs and 4 VGPR system values).
5380 * All inputs are returned unmodified. The vertex load indices are
5381 * stored after them, which will used by the API VS for fetching inputs.
5382 *
5383 * For example, the expected outputs for instance_divisors[] = {0, 1, 2} are:
5384 * input_v0,
5385 * input_v1,
5386 * input_v2,
5387 * input_v3,
5388 * (VertexID + BaseVertex),
5389 * (InstanceID + StartInstance),
5390 * (InstanceID / 2 + StartInstance)
5391 */
5392 static bool si_compile_vs_prolog(struct si_screen *sscreen,
5393 LLVMTargetMachineRef tm,
5394 struct pipe_debug_callback *debug,
5395 struct si_shader_part *out)
5396 {
5397 union si_shader_part_key *key = &out->key;
5398 struct si_shader shader = {};
5399 struct si_shader_context ctx;
5400 struct gallivm_state *gallivm = &ctx.radeon_bld.gallivm;
5401 LLVMTypeRef *params, *returns;
5402 LLVMValueRef ret, func;
5403 int last_sgpr, num_params, num_returns, i;
5404 bool status = true;
5405
5406 si_init_shader_ctx(&ctx, sscreen, &shader, tm);
5407 ctx.type = TGSI_PROCESSOR_VERTEX;
5408 ctx.param_vertex_id = key->vs_prolog.num_input_sgprs;
5409 ctx.param_instance_id = key->vs_prolog.num_input_sgprs + 3;
5410
5411 /* 4 preloaded VGPRs + vertex load indices as prolog outputs */
5412 params = alloca((key->vs_prolog.num_input_sgprs + 4) *
5413 sizeof(LLVMTypeRef));
5414 returns = alloca((key->vs_prolog.num_input_sgprs + 4 +
5415 key->vs_prolog.last_input + 1) *
5416 sizeof(LLVMTypeRef));
5417 num_params = 0;
5418 num_returns = 0;
5419
5420 /* Declare input and output SGPRs. */
5421 num_params = 0;
5422 for (i = 0; i < key->vs_prolog.num_input_sgprs; i++) {
5423 params[num_params++] = ctx.i32;
5424 returns[num_returns++] = ctx.i32;
5425 }
5426 last_sgpr = num_params - 1;
5427
5428 /* 4 preloaded VGPRs (outputs must be floats) */
5429 for (i = 0; i < 4; i++) {
5430 params[num_params++] = ctx.i32;
5431 returns[num_returns++] = ctx.f32;
5432 }
5433
5434 /* Vertex load indices. */
5435 for (i = 0; i <= key->vs_prolog.last_input; i++)
5436 returns[num_returns++] = ctx.f32;
5437
5438 /* Create the function. */
5439 si_create_function(&ctx, returns, num_returns, params,
5440 num_params, -1, last_sgpr);
5441 func = ctx.radeon_bld.main_fn;
5442
5443 /* Copy inputs to outputs. This should be no-op, as the registers match,
5444 * but it will prevent the compiler from overwriting them unintentionally.
5445 */
5446 ret = ctx.return_value;
5447 for (i = 0; i < key->vs_prolog.num_input_sgprs; i++) {
5448 LLVMValueRef p = LLVMGetParam(func, i);
5449 ret = LLVMBuildInsertValue(gallivm->builder, ret, p, i, "");
5450 }
5451 for (i = num_params - 4; i < num_params; i++) {
5452 LLVMValueRef p = LLVMGetParam(func, i);
5453 p = LLVMBuildBitCast(gallivm->builder, p, ctx.f32, "");
5454 ret = LLVMBuildInsertValue(gallivm->builder, ret, p, i, "");
5455 }
5456
5457 /* Compute vertex load indices from instance divisors. */
5458 for (i = 0; i <= key->vs_prolog.last_input; i++) {
5459 unsigned divisor = key->vs_prolog.states.instance_divisors[i];
5460 LLVMValueRef index;
5461
5462 if (divisor) {
5463 /* InstanceID / Divisor + StartInstance */
5464 index = get_instance_index_for_fetch(&ctx.radeon_bld,
5465 SI_SGPR_START_INSTANCE,
5466 divisor);
5467 } else {
5468 /* VertexID + BaseVertex */
5469 index = LLVMBuildAdd(gallivm->builder,
5470 LLVMGetParam(func, ctx.param_vertex_id),
5471 LLVMGetParam(func, SI_SGPR_BASE_VERTEX), "");
5472 }
5473
5474 index = LLVMBuildBitCast(gallivm->builder, index, ctx.f32, "");
5475 ret = LLVMBuildInsertValue(gallivm->builder, ret, index,
5476 num_params++, "");
5477 }
5478
5479 /* Compile. */
5480 LLVMBuildRet(gallivm->builder, ret);
5481 radeon_llvm_finalize_module(&ctx.radeon_bld);
5482
5483 if (si_compile_llvm(sscreen, &out->binary, &out->config, tm,
5484 gallivm->module, debug, ctx.type,
5485 "Vertex Shader Prolog"))
5486 status = false;
5487
5488 radeon_llvm_dispose(&ctx.radeon_bld);
5489 return status;
5490 }
5491
5492 /**
5493 * Compile the vertex shader epilog. This is also used by the tessellation
5494 * evaluation shader compiled as VS.
5495 *
5496 * The input is PrimitiveID.
5497 *
5498 * If PrimitiveID is required by the pixel shader, export it.
5499 * Otherwise, do nothing.
5500 */
5501 static bool si_compile_vs_epilog(struct si_screen *sscreen,
5502 LLVMTargetMachineRef tm,
5503 struct pipe_debug_callback *debug,
5504 struct si_shader_part *out)
5505 {
5506 union si_shader_part_key *key = &out->key;
5507 struct si_shader_context ctx;
5508 struct gallivm_state *gallivm = &ctx.radeon_bld.gallivm;
5509 struct lp_build_tgsi_context *bld_base = &ctx.radeon_bld.soa.bld_base;
5510 LLVMTypeRef params[5];
5511 int num_params, i;
5512 bool status = true;
5513
5514 si_init_shader_ctx(&ctx, sscreen, NULL, tm);
5515 ctx.type = TGSI_PROCESSOR_VERTEX;
5516
5517 /* Declare input VGPRs. */
5518 num_params = key->vs_epilog.states.export_prim_id ?
5519 (VS_EPILOG_PRIMID_LOC + 1) : 0;
5520 assert(num_params <= ARRAY_SIZE(params));
5521
5522 for (i = 0; i < num_params; i++)
5523 params[i] = ctx.f32;
5524
5525 /* Create the function. */
5526 si_create_function(&ctx, NULL, 0, params, num_params,
5527 -1, -1);
5528
5529 /* Emit exports. */
5530 if (key->vs_epilog.states.export_prim_id) {
5531 struct lp_build_context *base = &bld_base->base;
5532 struct lp_build_context *uint = &bld_base->uint_bld;
5533 LLVMValueRef args[9];
5534
5535 args[0] = lp_build_const_int32(base->gallivm, 0x0); /* enabled channels */
5536 args[1] = uint->zero; /* whether the EXEC mask is valid */
5537 args[2] = uint->zero; /* DONE bit */
5538 args[3] = lp_build_const_int32(base->gallivm, V_008DFC_SQ_EXP_PARAM +
5539 key->vs_epilog.prim_id_param_offset);
5540 args[4] = uint->zero; /* COMPR flag (0 = 32-bit export) */
5541 args[5] = LLVMGetParam(ctx.radeon_bld.main_fn,
5542 VS_EPILOG_PRIMID_LOC); /* X */
5543 args[6] = uint->undef; /* Y */
5544 args[7] = uint->undef; /* Z */
5545 args[8] = uint->undef; /* W */
5546
5547 lp_build_intrinsic(base->gallivm->builder, "llvm.SI.export",
5548 LLVMVoidTypeInContext(base->gallivm->context),
5549 args, 9, 0);
5550 }
5551
5552 /* Compile. */
5553 LLVMBuildRet(gallivm->builder, ctx.return_value);
5554 radeon_llvm_finalize_module(&ctx.radeon_bld);
5555
5556 if (si_compile_llvm(sscreen, &out->binary, &out->config, tm,
5557 gallivm->module, debug, ctx.type,
5558 "Vertex Shader Epilog"))
5559 status = false;
5560
5561 radeon_llvm_dispose(&ctx.radeon_bld);
5562 return status;
5563 }
5564
5565 /**
5566 * Create & compile a vertex shader epilog. This a helper used by VS and TES.
5567 */
5568 static bool si_get_vs_epilog(struct si_screen *sscreen,
5569 LLVMTargetMachineRef tm,
5570 struct si_shader *shader,
5571 struct pipe_debug_callback *debug,
5572 struct si_vs_epilog_bits *states)
5573 {
5574 union si_shader_part_key epilog_key;
5575
5576 memset(&epilog_key, 0, sizeof(epilog_key));
5577 epilog_key.vs_epilog.states = *states;
5578
5579 /* Set up the PrimitiveID output. */
5580 if (shader->key.vs.epilog.export_prim_id) {
5581 unsigned index = shader->selector->info.num_outputs;
5582 unsigned offset = shader->info.nr_param_exports++;
5583
5584 epilog_key.vs_epilog.prim_id_param_offset = offset;
5585 assert(index < ARRAY_SIZE(shader->info.vs_output_param_offset));
5586 shader->info.vs_output_param_offset[index] = offset;
5587 }
5588
5589 shader->epilog = si_get_shader_part(sscreen, &sscreen->vs_epilogs,
5590 &epilog_key, tm, debug,
5591 si_compile_vs_epilog);
5592 return shader->epilog != NULL;
5593 }
5594
5595 /**
5596 * Select and compile (or reuse) vertex shader parts (prolog & epilog).
5597 */
5598 static bool si_shader_select_vs_parts(struct si_screen *sscreen,
5599 LLVMTargetMachineRef tm,
5600 struct si_shader *shader,
5601 struct pipe_debug_callback *debug)
5602 {
5603 struct tgsi_shader_info *info = &shader->selector->info;
5604 union si_shader_part_key prolog_key;
5605 unsigned i;
5606
5607 /* Get the prolog. */
5608 memset(&prolog_key, 0, sizeof(prolog_key));
5609 prolog_key.vs_prolog.states = shader->key.vs.prolog;
5610 prolog_key.vs_prolog.num_input_sgprs = shader->info.num_input_sgprs;
5611 prolog_key.vs_prolog.last_input = MAX2(1, info->num_inputs) - 1;
5612
5613 /* The prolog is a no-op if there are no inputs. */
5614 if (info->num_inputs) {
5615 shader->prolog =
5616 si_get_shader_part(sscreen, &sscreen->vs_prologs,
5617 &prolog_key, tm, debug,
5618 si_compile_vs_prolog);
5619 if (!shader->prolog)
5620 return false;
5621 }
5622
5623 /* Get the epilog. */
5624 if (!shader->key.vs.as_es && !shader->key.vs.as_ls &&
5625 !si_get_vs_epilog(sscreen, tm, shader, debug,
5626 &shader->key.vs.epilog))
5627 return false;
5628
5629 /* Set the instanceID flag. */
5630 for (i = 0; i < info->num_inputs; i++)
5631 if (prolog_key.vs_prolog.states.instance_divisors[i])
5632 shader->info.uses_instanceid = true;
5633
5634 return true;
5635 }
5636
5637 /**
5638 * Select and compile (or reuse) TES parts (epilog).
5639 */
5640 static bool si_shader_select_tes_parts(struct si_screen *sscreen,
5641 LLVMTargetMachineRef tm,
5642 struct si_shader *shader,
5643 struct pipe_debug_callback *debug)
5644 {
5645 if (shader->key.tes.as_es)
5646 return true;
5647
5648 /* TES compiled as VS. */
5649 return si_get_vs_epilog(sscreen, tm, shader, debug,
5650 &shader->key.tes.epilog);
5651 }
5652
5653 /**
5654 * Compile the TCS epilog. This writes tesselation factors to memory based on
5655 * the output primitive type of the tesselator (determined by TES).
5656 */
5657 static bool si_compile_tcs_epilog(struct si_screen *sscreen,
5658 LLVMTargetMachineRef tm,
5659 struct pipe_debug_callback *debug,
5660 struct si_shader_part *out)
5661 {
5662 union si_shader_part_key *key = &out->key;
5663 struct si_shader shader = {};
5664 struct si_shader_context ctx;
5665 struct gallivm_state *gallivm = &ctx.radeon_bld.gallivm;
5666 struct lp_build_tgsi_context *bld_base = &ctx.radeon_bld.soa.bld_base;
5667 LLVMTypeRef params[16];
5668 LLVMValueRef func;
5669 int last_array_pointer, last_sgpr, num_params;
5670 bool status = true;
5671
5672 si_init_shader_ctx(&ctx, sscreen, &shader, tm);
5673 ctx.type = TGSI_PROCESSOR_TESS_CTRL;
5674 shader.key.tcs.epilog = key->tcs_epilog.states;
5675
5676 /* Declare inputs. Only RW_BUFFERS and TESS_FACTOR_OFFSET are used. */
5677 params[SI_PARAM_RW_BUFFERS] = const_array(ctx.v16i8, SI_NUM_RW_BUFFERS);
5678 last_array_pointer = SI_PARAM_RW_BUFFERS;
5679 params[SI_PARAM_CONST_BUFFERS] = ctx.i64;
5680 params[SI_PARAM_SAMPLERS] = ctx.i64;
5681 params[SI_PARAM_IMAGES] = ctx.i64;
5682 params[SI_PARAM_TCS_OUT_OFFSETS] = ctx.i32;
5683 params[SI_PARAM_TCS_OUT_LAYOUT] = ctx.i32;
5684 params[SI_PARAM_TCS_IN_LAYOUT] = ctx.i32;
5685 params[SI_PARAM_TESS_FACTOR_OFFSET] = ctx.i32;
5686 last_sgpr = SI_PARAM_TESS_FACTOR_OFFSET;
5687 num_params = last_sgpr + 1;
5688
5689 params[num_params++] = ctx.i32; /* patch index within the wave (REL_PATCH_ID) */
5690 params[num_params++] = ctx.i32; /* invocation ID within the patch */
5691 params[num_params++] = ctx.i32; /* LDS offset where tess factors should be loaded from */
5692
5693 /* Create the function. */
5694 si_create_function(&ctx, NULL, 0, params, num_params,
5695 last_array_pointer, last_sgpr);
5696 declare_tess_lds(&ctx);
5697 func = ctx.radeon_bld.main_fn;
5698
5699 si_write_tess_factors(bld_base,
5700 LLVMGetParam(func, last_sgpr + 1),
5701 LLVMGetParam(func, last_sgpr + 2),
5702 LLVMGetParam(func, last_sgpr + 3));
5703
5704 /* Compile. */
5705 LLVMBuildRet(gallivm->builder, ctx.return_value);
5706 radeon_llvm_finalize_module(&ctx.radeon_bld);
5707
5708 if (si_compile_llvm(sscreen, &out->binary, &out->config, tm,
5709 gallivm->module, debug, ctx.type,
5710 "Tessellation Control Shader Epilog"))
5711 status = false;
5712
5713 radeon_llvm_dispose(&ctx.radeon_bld);
5714 return status;
5715 }
5716
5717 /**
5718 * Select and compile (or reuse) TCS parts (epilog).
5719 */
5720 static bool si_shader_select_tcs_parts(struct si_screen *sscreen,
5721 LLVMTargetMachineRef tm,
5722 struct si_shader *shader,
5723 struct pipe_debug_callback *debug)
5724 {
5725 union si_shader_part_key epilog_key;
5726
5727 /* Get the epilog. */
5728 memset(&epilog_key, 0, sizeof(epilog_key));
5729 epilog_key.tcs_epilog.states = shader->key.tcs.epilog;
5730
5731 shader->epilog = si_get_shader_part(sscreen, &sscreen->tcs_epilogs,
5732 &epilog_key, tm, debug,
5733 si_compile_tcs_epilog);
5734 return shader->epilog != NULL;
5735 }
5736
5737 /**
5738 * Compile the pixel shader prolog. This handles:
5739 * - two-side color selection and interpolation
5740 * - overriding interpolation parameters for the API PS
5741 * - polygon stippling
5742 *
5743 * All preloaded SGPRs and VGPRs are passed through unmodified unless they are
5744 * overriden by other states. (e.g. per-sample interpolation)
5745 * Interpolated colors are stored after the preloaded VGPRs.
5746 */
5747 static bool si_compile_ps_prolog(struct si_screen *sscreen,
5748 LLVMTargetMachineRef tm,
5749 struct pipe_debug_callback *debug,
5750 struct si_shader_part *out)
5751 {
5752 union si_shader_part_key *key = &out->key;
5753 struct si_shader shader = {};
5754 struct si_shader_context ctx;
5755 struct gallivm_state *gallivm = &ctx.radeon_bld.gallivm;
5756 LLVMTypeRef *params;
5757 LLVMValueRef ret, func;
5758 int last_sgpr, num_params, num_returns, i, num_color_channels;
5759 bool status = true;
5760
5761 si_init_shader_ctx(&ctx, sscreen, &shader, tm);
5762 ctx.type = TGSI_PROCESSOR_FRAGMENT;
5763 shader.key.ps.prolog = key->ps_prolog.states;
5764
5765 /* Number of inputs + 8 color elements. */
5766 params = alloca((key->ps_prolog.num_input_sgprs +
5767 key->ps_prolog.num_input_vgprs + 8) *
5768 sizeof(LLVMTypeRef));
5769
5770 /* Declare inputs. */
5771 num_params = 0;
5772 for (i = 0; i < key->ps_prolog.num_input_sgprs; i++)
5773 params[num_params++] = ctx.i32;
5774 last_sgpr = num_params - 1;
5775
5776 for (i = 0; i < key->ps_prolog.num_input_vgprs; i++)
5777 params[num_params++] = ctx.f32;
5778
5779 /* Declare outputs (same as inputs + add colors if needed) */
5780 num_returns = num_params;
5781 num_color_channels = util_bitcount(key->ps_prolog.colors_read);
5782 for (i = 0; i < num_color_channels; i++)
5783 params[num_returns++] = ctx.f32;
5784
5785 /* Create the function. */
5786 si_create_function(&ctx, params, num_returns, params,
5787 num_params, -1, last_sgpr);
5788 func = ctx.radeon_bld.main_fn;
5789
5790 /* Copy inputs to outputs. This should be no-op, as the registers match,
5791 * but it will prevent the compiler from overwriting them unintentionally.
5792 */
5793 ret = ctx.return_value;
5794 for (i = 0; i < num_params; i++) {
5795 LLVMValueRef p = LLVMGetParam(func, i);
5796 ret = LLVMBuildInsertValue(gallivm->builder, ret, p, i, "");
5797 }
5798
5799 /* Polygon stippling. */
5800 if (key->ps_prolog.states.poly_stipple) {
5801 /* POS_FIXED_PT is always last. */
5802 unsigned pos = key->ps_prolog.num_input_sgprs +
5803 key->ps_prolog.num_input_vgprs - 1;
5804 LLVMValueRef ptr[2], views;
5805
5806 /* Get the pointer to sampler views. */
5807 ptr[0] = LLVMGetParam(func, SI_SGPR_SAMPLERS);
5808 ptr[1] = LLVMGetParam(func, SI_SGPR_SAMPLERS+1);
5809 views = lp_build_gather_values(gallivm, ptr, 2);
5810 views = LLVMBuildBitCast(gallivm->builder, views, ctx.i64, "");
5811 views = LLVMBuildIntToPtr(gallivm->builder, views,
5812 const_array(ctx.v8i32, SI_NUM_SAMPLERS), "");
5813
5814 si_llvm_emit_polygon_stipple(&ctx, views, pos);
5815 }
5816
5817 /* Interpolate colors. */
5818 for (i = 0; i < 2; i++) {
5819 unsigned writemask = (key->ps_prolog.colors_read >> (i * 4)) & 0xf;
5820 unsigned face_vgpr = key->ps_prolog.num_input_sgprs +
5821 key->ps_prolog.face_vgpr_index;
5822 LLVMValueRef interp[2], color[4];
5823 LLVMValueRef interp_ij = NULL, prim_mask = NULL, face = NULL;
5824
5825 if (!writemask)
5826 continue;
5827
5828 /* If the interpolation qualifier is not CONSTANT (-1). */
5829 if (key->ps_prolog.color_interp_vgpr_index[i] != -1) {
5830 unsigned interp_vgpr = key->ps_prolog.num_input_sgprs +
5831 key->ps_prolog.color_interp_vgpr_index[i];
5832
5833 interp[0] = LLVMGetParam(func, interp_vgpr);
5834 interp[1] = LLVMGetParam(func, interp_vgpr + 1);
5835 interp_ij = lp_build_gather_values(gallivm, interp, 2);
5836 interp_ij = LLVMBuildBitCast(gallivm->builder, interp_ij,
5837 ctx.v2i32, "");
5838 }
5839
5840 /* Use the absolute location of the input. */
5841 prim_mask = LLVMGetParam(func, SI_PS_NUM_USER_SGPR);
5842
5843 if (key->ps_prolog.states.color_two_side) {
5844 face = LLVMGetParam(func, face_vgpr);
5845 face = LLVMBuildBitCast(gallivm->builder, face, ctx.i32, "");
5846 }
5847
5848 interp_fs_input(&ctx,
5849 key->ps_prolog.color_attr_index[i],
5850 TGSI_SEMANTIC_COLOR, i,
5851 key->ps_prolog.num_interp_inputs,
5852 key->ps_prolog.colors_read, interp_ij,
5853 prim_mask, face, color);
5854
5855 while (writemask) {
5856 unsigned chan = u_bit_scan(&writemask);
5857 ret = LLVMBuildInsertValue(gallivm->builder, ret, color[chan],
5858 num_params++, "");
5859 }
5860 }
5861
5862 /* Force per-sample interpolation. */
5863 if (key->ps_prolog.states.force_persample_interp) {
5864 unsigned i, base = key->ps_prolog.num_input_sgprs;
5865 LLVMValueRef persp_sample[2], linear_sample[2];
5866
5867 /* Read PERSP_SAMPLE. */
5868 for (i = 0; i < 2; i++)
5869 persp_sample[i] = LLVMGetParam(func, base + i);
5870 /* Overwrite PERSP_CENTER. */
5871 for (i = 0; i < 2; i++)
5872 ret = LLVMBuildInsertValue(gallivm->builder, ret,
5873 persp_sample[i], base + 2 + i, "");
5874 /* Overwrite PERSP_CENTROID. */
5875 for (i = 0; i < 2; i++)
5876 ret = LLVMBuildInsertValue(gallivm->builder, ret,
5877 persp_sample[i], base + 4 + i, "");
5878 /* Read LINEAR_SAMPLE. */
5879 for (i = 0; i < 2; i++)
5880 linear_sample[i] = LLVMGetParam(func, base + 6 + i);
5881 /* Overwrite LINEAR_CENTER. */
5882 for (i = 0; i < 2; i++)
5883 ret = LLVMBuildInsertValue(gallivm->builder, ret,
5884 linear_sample[i], base + 8 + i, "");
5885 /* Overwrite LINEAR_CENTROID. */
5886 for (i = 0; i < 2; i++)
5887 ret = LLVMBuildInsertValue(gallivm->builder, ret,
5888 linear_sample[i], base + 10 + i, "");
5889 }
5890
5891 /* Compile. */
5892 LLVMBuildRet(gallivm->builder, ret);
5893 radeon_llvm_finalize_module(&ctx.radeon_bld);
5894
5895 if (si_compile_llvm(sscreen, &out->binary, &out->config, tm,
5896 gallivm->module, debug, ctx.type,
5897 "Fragment Shader Prolog"))
5898 status = false;
5899
5900 radeon_llvm_dispose(&ctx.radeon_bld);
5901 return status;
5902 }
5903
5904 /**
5905 * Compile the pixel shader epilog. This handles everything that must be
5906 * emulated for pixel shader exports. (alpha-test, format conversions, etc)
5907 */
5908 static bool si_compile_ps_epilog(struct si_screen *sscreen,
5909 LLVMTargetMachineRef tm,
5910 struct pipe_debug_callback *debug,
5911 struct si_shader_part *out)
5912 {
5913 union si_shader_part_key *key = &out->key;
5914 struct si_shader shader = {};
5915 struct si_shader_context ctx;
5916 struct gallivm_state *gallivm = &ctx.radeon_bld.gallivm;
5917 struct lp_build_tgsi_context *bld_base = &ctx.radeon_bld.soa.bld_base;
5918 LLVMTypeRef params[16+8*4+3];
5919 LLVMValueRef depth = NULL, stencil = NULL, samplemask = NULL;
5920 int last_array_pointer, last_sgpr, num_params, i;
5921 bool status = true;
5922
5923 si_init_shader_ctx(&ctx, sscreen, &shader, tm);
5924 ctx.type = TGSI_PROCESSOR_FRAGMENT;
5925 shader.key.ps.epilog = key->ps_epilog.states;
5926
5927 /* Declare input SGPRs. */
5928 params[SI_PARAM_RW_BUFFERS] = ctx.i64;
5929 params[SI_PARAM_CONST_BUFFERS] = ctx.i64;
5930 params[SI_PARAM_SAMPLERS] = ctx.i64;
5931 params[SI_PARAM_IMAGES] = ctx.i64;
5932 params[SI_PARAM_ALPHA_REF] = ctx.f32;
5933 last_array_pointer = -1;
5934 last_sgpr = SI_PARAM_ALPHA_REF;
5935
5936 /* Declare input VGPRs. */
5937 num_params = (last_sgpr + 1) +
5938 util_bitcount(key->ps_epilog.colors_written) * 4 +
5939 key->ps_epilog.writes_z +
5940 key->ps_epilog.writes_stencil +
5941 key->ps_epilog.writes_samplemask;
5942
5943 num_params = MAX2(num_params,
5944 last_sgpr + 1 + PS_EPILOG_SAMPLEMASK_MIN_LOC + 1);
5945
5946 assert(num_params <= ARRAY_SIZE(params));
5947
5948 for (i = last_sgpr + 1; i < num_params; i++)
5949 params[i] = ctx.f32;
5950
5951 /* Create the function. */
5952 si_create_function(&ctx, NULL, 0, params, num_params,
5953 last_array_pointer, last_sgpr);
5954 /* Disable elimination of unused inputs. */
5955 radeon_llvm_add_attribute(ctx.radeon_bld.main_fn,
5956 "InitialPSInputAddr", 0xffffff);
5957
5958 /* Process colors. */
5959 unsigned vgpr = last_sgpr + 1;
5960 unsigned colors_written = key->ps_epilog.colors_written;
5961 int last_color_export = -1;
5962
5963 /* Find the last color export. */
5964 if (!key->ps_epilog.writes_z &&
5965 !key->ps_epilog.writes_stencil &&
5966 !key->ps_epilog.writes_samplemask) {
5967 unsigned spi_format = key->ps_epilog.states.spi_shader_col_format;
5968
5969 /* If last_cbuf > 0, FS_COLOR0_WRITES_ALL_CBUFS is true. */
5970 if (colors_written == 0x1 && key->ps_epilog.states.last_cbuf > 0) {
5971 /* Just set this if any of the colorbuffers are enabled. */
5972 if (spi_format &
5973 ((1llu << (4 * (key->ps_epilog.states.last_cbuf + 1))) - 1))
5974 last_color_export = 0;
5975 } else {
5976 for (i = 0; i < 8; i++)
5977 if (colors_written & (1 << i) &&
5978 (spi_format >> (i * 4)) & 0xf)
5979 last_color_export = i;
5980 }
5981 }
5982
5983 while (colors_written) {
5984 LLVMValueRef color[4];
5985 int mrt = u_bit_scan(&colors_written);
5986
5987 for (i = 0; i < 4; i++)
5988 color[i] = LLVMGetParam(ctx.radeon_bld.main_fn, vgpr++);
5989
5990 si_export_mrt_color(bld_base, color, mrt,
5991 num_params - 1,
5992 mrt == last_color_export);
5993 }
5994
5995 /* Process depth, stencil, samplemask. */
5996 if (key->ps_epilog.writes_z)
5997 depth = LLVMGetParam(ctx.radeon_bld.main_fn, vgpr++);
5998 if (key->ps_epilog.writes_stencil)
5999 stencil = LLVMGetParam(ctx.radeon_bld.main_fn, vgpr++);
6000 if (key->ps_epilog.writes_samplemask)
6001 samplemask = LLVMGetParam(ctx.radeon_bld.main_fn, vgpr++);
6002
6003 if (depth || stencil || samplemask)
6004 si_export_mrt_z(bld_base, depth, stencil, samplemask);
6005 else if (last_color_export == -1)
6006 si_export_null(bld_base);
6007
6008 /* Compile. */
6009 LLVMBuildRetVoid(gallivm->builder);
6010 radeon_llvm_finalize_module(&ctx.radeon_bld);
6011
6012 if (si_compile_llvm(sscreen, &out->binary, &out->config, tm,
6013 gallivm->module, debug, ctx.type,
6014 "Fragment Shader Epilog"))
6015 status = false;
6016
6017 radeon_llvm_dispose(&ctx.radeon_bld);
6018 return status;
6019 }
6020
6021 /**
6022 * Select and compile (or reuse) pixel shader parts (prolog & epilog).
6023 */
6024 static bool si_shader_select_ps_parts(struct si_screen *sscreen,
6025 LLVMTargetMachineRef tm,
6026 struct si_shader *shader,
6027 struct pipe_debug_callback *debug)
6028 {
6029 struct tgsi_shader_info *info = &shader->selector->info;
6030 union si_shader_part_key prolog_key;
6031 union si_shader_part_key epilog_key;
6032 unsigned i;
6033
6034 /* Get the prolog. */
6035 memset(&prolog_key, 0, sizeof(prolog_key));
6036 prolog_key.ps_prolog.states = shader->key.ps.prolog;
6037 prolog_key.ps_prolog.colors_read = info->colors_read;
6038 prolog_key.ps_prolog.num_input_sgprs = shader->info.num_input_sgprs;
6039 prolog_key.ps_prolog.num_input_vgprs = shader->info.num_input_vgprs;
6040
6041 if (info->colors_read) {
6042 unsigned *color = shader->selector->color_attr_index;
6043
6044 if (shader->key.ps.prolog.color_two_side) {
6045 /* BCOLORs are stored after the last input. */
6046 prolog_key.ps_prolog.num_interp_inputs = info->num_inputs;
6047 prolog_key.ps_prolog.face_vgpr_index = shader->info.face_vgpr_index;
6048 shader->config.spi_ps_input_ena |= S_0286CC_FRONT_FACE_ENA(1);
6049 }
6050
6051 for (i = 0; i < 2; i++) {
6052 unsigned location = info->input_interpolate_loc[color[i]];
6053
6054 if (!(info->colors_read & (0xf << i*4)))
6055 continue;
6056
6057 prolog_key.ps_prolog.color_attr_index[i] = color[i];
6058
6059 /* Force per-sample interpolation for the colors here. */
6060 if (shader->key.ps.prolog.force_persample_interp)
6061 location = TGSI_INTERPOLATE_LOC_SAMPLE;
6062
6063 switch (info->input_interpolate[color[i]]) {
6064 case TGSI_INTERPOLATE_CONSTANT:
6065 prolog_key.ps_prolog.color_interp_vgpr_index[i] = -1;
6066 break;
6067 case TGSI_INTERPOLATE_PERSPECTIVE:
6068 case TGSI_INTERPOLATE_COLOR:
6069 switch (location) {
6070 case TGSI_INTERPOLATE_LOC_SAMPLE:
6071 prolog_key.ps_prolog.color_interp_vgpr_index[i] = 0;
6072 shader->config.spi_ps_input_ena |=
6073 S_0286CC_PERSP_SAMPLE_ENA(1);
6074 break;
6075 case TGSI_INTERPOLATE_LOC_CENTER:
6076 prolog_key.ps_prolog.color_interp_vgpr_index[i] = 2;
6077 shader->config.spi_ps_input_ena |=
6078 S_0286CC_PERSP_CENTER_ENA(1);
6079 break;
6080 case TGSI_INTERPOLATE_LOC_CENTROID:
6081 prolog_key.ps_prolog.color_interp_vgpr_index[i] = 4;
6082 shader->config.spi_ps_input_ena |=
6083 S_0286CC_PERSP_CENTROID_ENA(1);
6084 break;
6085 default:
6086 assert(0);
6087 }
6088 break;
6089 case TGSI_INTERPOLATE_LINEAR:
6090 switch (location) {
6091 case TGSI_INTERPOLATE_LOC_SAMPLE:
6092 prolog_key.ps_prolog.color_interp_vgpr_index[i] = 6;
6093 shader->config.spi_ps_input_ena |=
6094 S_0286CC_LINEAR_SAMPLE_ENA(1);
6095 break;
6096 case TGSI_INTERPOLATE_LOC_CENTER:
6097 prolog_key.ps_prolog.color_interp_vgpr_index[i] = 8;
6098 shader->config.spi_ps_input_ena |=
6099 S_0286CC_LINEAR_CENTER_ENA(1);
6100 break;
6101 case TGSI_INTERPOLATE_LOC_CENTROID:
6102 prolog_key.ps_prolog.color_interp_vgpr_index[i] = 10;
6103 shader->config.spi_ps_input_ena |=
6104 S_0286CC_LINEAR_CENTROID_ENA(1);
6105 break;
6106 default:
6107 assert(0);
6108 }
6109 break;
6110 default:
6111 assert(0);
6112 }
6113 }
6114 }
6115
6116 /* The prolog is a no-op if these aren't set. */
6117 if (prolog_key.ps_prolog.colors_read ||
6118 prolog_key.ps_prolog.states.force_persample_interp ||
6119 prolog_key.ps_prolog.states.poly_stipple) {
6120 shader->prolog =
6121 si_get_shader_part(sscreen, &sscreen->ps_prologs,
6122 &prolog_key, tm, debug,
6123 si_compile_ps_prolog);
6124 if (!shader->prolog)
6125 return false;
6126 }
6127
6128 /* Get the epilog. */
6129 memset(&epilog_key, 0, sizeof(epilog_key));
6130 epilog_key.ps_epilog.colors_written = info->colors_written;
6131 epilog_key.ps_epilog.writes_z = info->writes_z;
6132 epilog_key.ps_epilog.writes_stencil = info->writes_stencil;
6133 epilog_key.ps_epilog.writes_samplemask = info->writes_samplemask;
6134 epilog_key.ps_epilog.states = shader->key.ps.epilog;
6135
6136 shader->epilog =
6137 si_get_shader_part(sscreen, &sscreen->ps_epilogs,
6138 &epilog_key, tm, debug,
6139 si_compile_ps_epilog);
6140 if (!shader->epilog)
6141 return false;
6142
6143 /* Enable POS_FIXED_PT if polygon stippling is enabled. */
6144 if (shader->key.ps.prolog.poly_stipple) {
6145 shader->config.spi_ps_input_ena |= S_0286CC_POS_FIXED_PT_ENA(1);
6146 assert(G_0286CC_POS_FIXED_PT_ENA(shader->config.spi_ps_input_addr));
6147 }
6148
6149 /* Set up the enable bits for per-sample shading if needed. */
6150 if (shader->key.ps.prolog.force_persample_interp) {
6151 if (G_0286CC_PERSP_CENTER_ENA(shader->config.spi_ps_input_ena) ||
6152 G_0286CC_PERSP_CENTROID_ENA(shader->config.spi_ps_input_ena)) {
6153 shader->config.spi_ps_input_ena &= C_0286CC_PERSP_CENTER_ENA;
6154 shader->config.spi_ps_input_ena &= C_0286CC_PERSP_CENTROID_ENA;
6155 shader->config.spi_ps_input_ena |= S_0286CC_PERSP_SAMPLE_ENA(1);
6156 }
6157 if (G_0286CC_LINEAR_CENTER_ENA(shader->config.spi_ps_input_ena) ||
6158 G_0286CC_LINEAR_CENTROID_ENA(shader->config.spi_ps_input_ena)) {
6159 shader->config.spi_ps_input_ena &= C_0286CC_LINEAR_CENTER_ENA;
6160 shader->config.spi_ps_input_ena &= C_0286CC_LINEAR_CENTROID_ENA;
6161 shader->config.spi_ps_input_ena |= S_0286CC_LINEAR_SAMPLE_ENA(1);
6162 }
6163 }
6164
6165 /* POW_W_FLOAT requires that one of the perspective weights is enabled. */
6166 if (G_0286CC_POS_W_FLOAT_ENA(shader->config.spi_ps_input_ena) &&
6167 !(shader->config.spi_ps_input_ena & 0xf)) {
6168 shader->config.spi_ps_input_ena |= S_0286CC_PERSP_CENTER_ENA(1);
6169 assert(G_0286CC_PERSP_CENTER_ENA(shader->config.spi_ps_input_addr));
6170 }
6171
6172 /* At least one pair of interpolation weights must be enabled. */
6173 if (!(shader->config.spi_ps_input_ena & 0x7f)) {
6174 shader->config.spi_ps_input_ena |= S_0286CC_LINEAR_CENTER_ENA(1);
6175 assert(G_0286CC_LINEAR_CENTER_ENA(shader->config.spi_ps_input_addr));
6176 }
6177
6178 /* The sample mask input is always enabled, because the API shader always
6179 * passes it through to the epilog. Disable it here if it's unused.
6180 */
6181 if (!shader->key.ps.epilog.poly_line_smoothing &&
6182 !shader->selector->info.reads_samplemask)
6183 shader->config.spi_ps_input_ena &= C_0286CC_SAMPLE_COVERAGE_ENA;
6184
6185 return true;
6186 }
6187
6188 int si_shader_create(struct si_screen *sscreen, LLVMTargetMachineRef tm,
6189 struct si_shader *shader,
6190 struct pipe_debug_callback *debug)
6191 {
6192 struct si_shader *mainp = shader->selector->main_shader_part;
6193 int r;
6194
6195 /* LS, ES, VS are compiled on demand if the main part hasn't been
6196 * compiled for that stage.
6197 */
6198 if (!mainp ||
6199 (shader->selector->type == PIPE_SHADER_VERTEX &&
6200 (shader->key.vs.as_es != mainp->key.vs.as_es ||
6201 shader->key.vs.as_ls != mainp->key.vs.as_ls)) ||
6202 (shader->selector->type == PIPE_SHADER_TESS_EVAL &&
6203 shader->key.tes.as_es != mainp->key.tes.as_es)) {
6204 /* Monolithic shader (compiled as a whole, has many variants,
6205 * may take a long time to compile).
6206 */
6207 r = si_compile_tgsi_shader(sscreen, tm, shader, true, debug);
6208 if (r)
6209 return r;
6210 } else {
6211 /* The shader consists of 2-3 parts:
6212 *
6213 * - the middle part is the user shader, it has 1 variant only
6214 * and it was compiled during the creation of the shader
6215 * selector
6216 * - the prolog part is inserted at the beginning
6217 * - the epilog part is inserted at the end
6218 *
6219 * The prolog and epilog have many (but simple) variants.
6220 */
6221
6222 /* Copy the compiled TGSI shader data over. */
6223 shader->is_binary_shared = true;
6224 shader->binary = mainp->binary;
6225 shader->config = mainp->config;
6226 shader->info.num_input_sgprs = mainp->info.num_input_sgprs;
6227 shader->info.num_input_vgprs = mainp->info.num_input_vgprs;
6228 shader->info.face_vgpr_index = mainp->info.face_vgpr_index;
6229 memcpy(shader->info.vs_output_param_offset,
6230 mainp->info.vs_output_param_offset,
6231 sizeof(mainp->info.vs_output_param_offset));
6232 shader->info.uses_instanceid = mainp->info.uses_instanceid;
6233 shader->info.nr_pos_exports = mainp->info.nr_pos_exports;
6234 shader->info.nr_param_exports = mainp->info.nr_param_exports;
6235
6236 /* Select prologs and/or epilogs. */
6237 switch (shader->selector->type) {
6238 case PIPE_SHADER_VERTEX:
6239 if (!si_shader_select_vs_parts(sscreen, tm, shader, debug))
6240 return -1;
6241 break;
6242 case PIPE_SHADER_TESS_CTRL:
6243 if (!si_shader_select_tcs_parts(sscreen, tm, shader, debug))
6244 return -1;
6245 break;
6246 case PIPE_SHADER_TESS_EVAL:
6247 if (!si_shader_select_tes_parts(sscreen, tm, shader, debug))
6248 return -1;
6249 break;
6250 case PIPE_SHADER_FRAGMENT:
6251 if (!si_shader_select_ps_parts(sscreen, tm, shader, debug))
6252 return -1;
6253
6254 /* Make sure we have at least as many VGPRs as there
6255 * are allocated inputs.
6256 */
6257 shader->config.num_vgprs = MAX2(shader->config.num_vgprs,
6258 shader->info.num_input_vgprs);
6259 break;
6260 }
6261
6262 /* Update SGPR and VGPR counts. */
6263 if (shader->prolog) {
6264 shader->config.num_sgprs = MAX2(shader->config.num_sgprs,
6265 shader->prolog->config.num_sgprs);
6266 shader->config.num_vgprs = MAX2(shader->config.num_vgprs,
6267 shader->prolog->config.num_vgprs);
6268 }
6269 if (shader->epilog) {
6270 shader->config.num_sgprs = MAX2(shader->config.num_sgprs,
6271 shader->epilog->config.num_sgprs);
6272 shader->config.num_vgprs = MAX2(shader->config.num_vgprs,
6273 shader->epilog->config.num_vgprs);
6274 }
6275 }
6276
6277 si_shader_dump(sscreen, shader, debug, shader->selector->info.processor,
6278 stderr);
6279
6280 /* Upload. */
6281 r = si_shader_binary_upload(sscreen, shader);
6282 if (r) {
6283 fprintf(stderr, "LLVM failed to upload shader\n");
6284 return r;
6285 }
6286
6287 return 0;
6288 }
6289
6290 void si_shader_destroy(struct si_shader *shader)
6291 {
6292 if (shader->gs_copy_shader) {
6293 si_shader_destroy(shader->gs_copy_shader);
6294 FREE(shader->gs_copy_shader);
6295 }
6296
6297 if (shader->scratch_bo)
6298 r600_resource_reference(&shader->scratch_bo, NULL);
6299
6300 r600_resource_reference(&shader->bo, NULL);
6301
6302 if (!shader->is_binary_shared)
6303 radeon_shader_binary_clean(&shader->binary);
6304 }