projects / mesa.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
history | raw | HEAD
tgsi: change tgsi_shader_info::properties to a one-dimensional array
[mesa.git] / src / gallium / drivers / radeonsi / si_shader.c
1 /*
2 * Copyright 2012 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * on the rights to use, copy, modify, merge, publish, distribute, sub
8 * license, and/or sell copies of the Software, and to permit persons to whom
9 * the Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
19 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
20 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
21 * USE OR OTHER DEALINGS IN THE SOFTWARE.
22 *
23 * Authors:
24 * Tom Stellard <thomas.stellard@amd.com>
25 * Michel Dänzer <michel.daenzer@amd.com>
26 * Christian König <christian.koenig@amd.com>
27 */
28
29 #include "gallivm/lp_bld_const.h"
30 #include "gallivm/lp_bld_gather.h"
31 #include "gallivm/lp_bld_intr.h"
32 #include "gallivm/lp_bld_logic.h"
33 #include "gallivm/lp_bld_arit.h"
34 #include "gallivm/lp_bld_flow.h"
35 #include "radeon/radeon_llvm.h"
36 #include "radeon/radeon_llvm_emit.h"
37 #include "util/u_memory.h"
38 #include "tgsi/tgsi_parse.h"
39 #include "tgsi/tgsi_util.h"
40 #include "tgsi/tgsi_dump.h"
41
42 #include "si_pipe.h"
43 #include "si_shader.h"
44 #include "sid.h"
45
46 #include <errno.h>
47
48 struct si_shader_output_values
49 {
50 LLVMValueRef values[4];
51 unsigned name;
52 unsigned index;
53 unsigned sid;
54 unsigned usage;
55 };
56
57 struct si_shader_context
58 {
59 struct radeon_llvm_context radeon_bld;
60 struct tgsi_parse_context parse;
61 struct tgsi_token * tokens;
62 struct si_shader *shader;
63 unsigned type; /* TGSI_PROCESSOR_* specifies the type of shader. */
64 int param_streamout_config;
65 int param_streamout_write_index;
66 int param_streamout_offset[4];
67 int param_vertex_id;
68 int param_instance_id;
69 LLVMValueRef const_md;
70 LLVMValueRef const_resource[SI_NUM_CONST_BUFFERS];
71 LLVMValueRef ddxy_lds;
72 LLVMValueRef *constants[SI_NUM_CONST_BUFFERS];
73 LLVMValueRef *resources;
74 LLVMValueRef *samplers;
75 LLVMValueRef so_buffers[4];
76 LLVMValueRef gs_next_vertex;
77 };
78
79 static struct si_shader_context * si_shader_context(
80 struct lp_build_tgsi_context * bld_base)
81 {
82 return (struct si_shader_context *)bld_base;
83 }
84
85
86 #define PERSPECTIVE_BASE 0
87 #define LINEAR_BASE 9
88
89 #define SAMPLE_OFFSET 0
90 #define CENTER_OFFSET 2
91 #define CENTROID_OFSET 4
92
93 #define USE_SGPR_MAX_SUFFIX_LEN 5
94 #define CONST_ADDR_SPACE 2
95 #define LOCAL_ADDR_SPACE 3
96 #define USER_SGPR_ADDR_SPACE 8
97
98
99 #define SENDMSG_GS 2
100 #define SENDMSG_GS_DONE 3
101
102 #define SENDMSG_GS_OP_NOP (0 << 4)
103 #define SENDMSG_GS_OP_CUT (1 << 4)
104 #define SENDMSG_GS_OP_EMIT (2 << 4)
105 #define SENDMSG_GS_OP_EMIT_CUT (3 << 4)
106
107 /**
108 * Returns a unique index for a semantic name and index. The index must be
109 * less than 64, so that a 64-bit bitmask of used inputs or outputs can be
110 * calculated.
111 */
112 unsigned si_shader_io_get_unique_index(unsigned semantic_name, unsigned index)
113 {
114 switch (semantic_name) {
115 case TGSI_SEMANTIC_POSITION:
116 return 0;
117 case TGSI_SEMANTIC_PSIZE:
118 return 1;
119 case TGSI_SEMANTIC_CLIPDIST:
120 assert(index <= 1);
121 return 2 + index;
122 case TGSI_SEMANTIC_CLIPVERTEX:
123 return 4;
124 case TGSI_SEMANTIC_COLOR:
125 assert(index <= 1);
126 return 5 + index;
127 case TGSI_SEMANTIC_BCOLOR:
128 assert(index <= 1);
129 return 7 + index;
130 case TGSI_SEMANTIC_FOG:
131 return 9;
132 case TGSI_SEMANTIC_EDGEFLAG:
133 return 10;
134 case TGSI_SEMANTIC_GENERIC:
135 assert(index <= 63-11);
136 return 11 + index;
137 default:
138 assert(0);
139 return 63;
140 }
141 }
142
143 /**
144 * Given a semantic name and index of a parameter and a mask of used parameters
145 * (inputs or outputs), return the index of the parameter in the list of all
146 * used parameters.
147 *
148 * For example, assume this list of parameters:
149 * POSITION, PSIZE, GENERIC0, GENERIC2
150 * which has the mask:
151 * 11000000000101
152 * Then:
153 * querying POSITION returns 0,
154 * querying PSIZE returns 1,
155 * querying GENERIC0 returns 2,
156 * querying GENERIC2 returns 3.
157 *
158 * Which can be used as an offset to a parameter buffer in units of vec4s.
159 */
160 static int get_param_index(unsigned semantic_name, unsigned index,
161 uint64_t mask)
162 {
163 unsigned unique_index = si_shader_io_get_unique_index(semantic_name, index);
164 int i, param_index = 0;
165
166 /* If not present... */
167 if (!((1llu << unique_index) & mask))
168 return -1;
169
170 for (i = 0; mask; i++) {
171 uint64_t bit = 1llu << i;
172
173 if (bit & mask) {
174 if (i == unique_index)
175 return param_index;
176
177 mask &= ~bit;
178 param_index++;
179 }
180 }
181
182 assert(!"unreachable");
183 return -1;
184 }
185
186 /**
187 * Build an LLVM bytecode indexed load using LLVMBuildGEP + LLVMBuildLoad
188 *
189 * @param offset The offset parameter specifies the number of
190 * elements to offset, not the number of bytes or dwords. An element is the
191 * the type pointed to by the base_ptr parameter (e.g. int is the element of
192 * an int* pointer)
193 *
194 * When LLVM lowers the load instruction, it will convert the element offset
195 * into a dword offset automatically.
196 *
197 */
198 static LLVMValueRef build_indexed_load(
199 struct si_shader_context * si_shader_ctx,
200 LLVMValueRef base_ptr,
201 LLVMValueRef offset)
202 {
203 struct lp_build_context * base = &si_shader_ctx->radeon_bld.soa.bld_base.base;
204
205 LLVMValueRef indices[2] = {
206 LLVMConstInt(LLVMInt64TypeInContext(base->gallivm->context), 0, false),
207 offset
208 };
209 LLVMValueRef computed_ptr = LLVMBuildGEP(
210 base->gallivm->builder, base_ptr, indices, 2, "");
211
212 LLVMValueRef result = LLVMBuildLoad(base->gallivm->builder, computed_ptr, "");
213 LLVMSetMetadata(result, 1, si_shader_ctx->const_md);
214 return result;
215 }
216
217 static LLVMValueRef get_instance_index_for_fetch(
218 struct radeon_llvm_context * radeon_bld,
219 unsigned divisor)
220 {
221 struct si_shader_context *si_shader_ctx =
222 si_shader_context(&radeon_bld->soa.bld_base);
223 struct gallivm_state * gallivm = radeon_bld->soa.bld_base.base.gallivm;
224
225 LLVMValueRef result = LLVMGetParam(radeon_bld->main_fn,
226 si_shader_ctx->param_instance_id);
227 result = LLVMBuildAdd(gallivm->builder, result, LLVMGetParam(
228 radeon_bld->main_fn, SI_PARAM_START_INSTANCE), "");
229
230 if (divisor > 1)
231 result = LLVMBuildUDiv(gallivm->builder, result,
232 lp_build_const_int32(gallivm, divisor), "");
233
234 return result;
235 }
236
237 static int si_store_shader_io_attribs(struct si_shader *shader,
238 const struct tgsi_full_declaration *d)
239 {
240 int i = -1;
241
242 switch (d->Declaration.File) {
243 case TGSI_FILE_INPUT:
244 i = shader->ninput++;
245 assert(i < Elements(shader->input));
246 shader->input[i].name = d->Semantic.Name;
247 shader->input[i].sid = d->Semantic.Index;
248 shader->input[i].index = d->Range.First;
249 shader->input[i].interpolate = d->Interp.Interpolate;
250 shader->input[i].centroid = d->Interp.Location == TGSI_INTERPOLATE_LOC_CENTROID;
251 return -1;
252
253 case TGSI_FILE_OUTPUT:
254 i = shader->noutput++;
255 assert(i < Elements(shader->output));
256 shader->output[i].name = d->Semantic.Name;
257 shader->output[i].sid = d->Semantic.Index;
258 shader->output[i].index = d->Range.First;
259 shader->output[i].usage = d->Declaration.UsageMask;
260 break;
261 }
262
263 return i;
264 }
265
266 static void declare_input_vs(
267 struct radeon_llvm_context *radeon_bld,
268 unsigned input_index,
269 const struct tgsi_full_declaration *decl)
270 {
271 struct lp_build_context *base = &radeon_bld->soa.bld_base.base;
272 struct gallivm_state *gallivm = base->gallivm;
273 struct si_shader_context *si_shader_ctx =
274 si_shader_context(&radeon_bld->soa.bld_base);
275 unsigned divisor = si_shader_ctx->shader->key.vs.instance_divisors[input_index];
276
277 unsigned chan;
278
279 LLVMValueRef t_list_ptr;
280 LLVMValueRef t_offset;
281 LLVMValueRef t_list;
282 LLVMValueRef attribute_offset;
283 LLVMValueRef buffer_index;
284 LLVMValueRef args[3];
285 LLVMTypeRef vec4_type;
286 LLVMValueRef input;
287
288 /* Load the T list */
289 t_list_ptr = LLVMGetParam(si_shader_ctx->radeon_bld.main_fn, SI_PARAM_VERTEX_BUFFER);
290
291 t_offset = lp_build_const_int32(gallivm, input_index);
292
293 t_list = build_indexed_load(si_shader_ctx, t_list_ptr, t_offset);
294
295 /* Build the attribute offset */
296 attribute_offset = lp_build_const_int32(gallivm, 0);
297
298 if (divisor) {
299 /* Build index from instance ID, start instance and divisor */
300 si_shader_ctx->shader->uses_instanceid = true;
301 buffer_index = get_instance_index_for_fetch(&si_shader_ctx->radeon_bld, divisor);
302 } else {
303 /* Load the buffer index for vertices. */
304 LLVMValueRef vertex_id = LLVMGetParam(si_shader_ctx->radeon_bld.main_fn,
305 si_shader_ctx->param_vertex_id);
306 LLVMValueRef base_vertex = LLVMGetParam(radeon_bld->main_fn,
307 SI_PARAM_BASE_VERTEX);
308 buffer_index = LLVMBuildAdd(gallivm->builder, base_vertex, vertex_id, "");
309 }
310
311 vec4_type = LLVMVectorType(base->elem_type, 4);
312 args[0] = t_list;
313 args[1] = attribute_offset;
314 args[2] = buffer_index;
315 input = build_intrinsic(gallivm->builder,
316 "llvm.SI.vs.load.input", vec4_type, args, 3,
317 LLVMReadNoneAttribute | LLVMNoUnwindAttribute);
318
319 /* Break up the vec4 into individual components */
320 for (chan = 0; chan < 4; chan++) {
321 LLVMValueRef llvm_chan = lp_build_const_int32(gallivm, chan);
322 /* XXX: Use a helper function for this. There is one in
323 * tgsi_llvm.c. */
324 si_shader_ctx->radeon_bld.inputs[radeon_llvm_reg_index_soa(input_index, chan)] =
325 LLVMBuildExtractElement(gallivm->builder,
326 input, llvm_chan, "");
327 }
328 }
329
330 static void declare_input_gs(
331 struct radeon_llvm_context *radeon_bld,
332 unsigned input_index,
333 const struct tgsi_full_declaration *decl)
334 {
335 struct si_shader_context *si_shader_ctx =
336 si_shader_context(&radeon_bld->soa.bld_base);
337 struct si_shader *shader = si_shader_ctx->shader;
338
339 si_store_shader_io_attribs(shader, decl);
340 }
341
342 static LLVMValueRef fetch_input_gs(
343 struct lp_build_tgsi_context *bld_base,
344 const struct tgsi_full_src_register *reg,
345 enum tgsi_opcode_type type,
346 unsigned swizzle)
347 {
348 struct lp_build_context *base = &bld_base->base;
349 struct si_shader_context *si_shader_ctx = si_shader_context(bld_base);
350 struct si_shader *shader = si_shader_ctx->shader;
351 struct lp_build_context *uint = &si_shader_ctx->radeon_bld.soa.bld_base.uint_bld;
352 struct gallivm_state *gallivm = base->gallivm;
353 LLVMTypeRef i32 = LLVMInt32TypeInContext(gallivm->context);
354 LLVMValueRef vtx_offset;
355 LLVMValueRef t_list_ptr;
356 LLVMValueRef t_list;
357 LLVMValueRef args[9];
358 unsigned vtx_offset_param;
359 struct si_shader_input *input = &shader->input[reg->Register.Index];
360
361 if (swizzle != ~0 &&
362 shader->input[reg->Register.Index].name == TGSI_SEMANTIC_PRIMID) {
363 if (swizzle == 0)
364 return LLVMGetParam(si_shader_ctx->radeon_bld.main_fn,
365 SI_PARAM_PRIMITIVE_ID);
366 else
367 return uint->zero;
368 }
369
370 if (!reg->Register.Dimension)
371 return NULL;
372
373 if (swizzle == ~0) {
374 LLVMValueRef values[TGSI_NUM_CHANNELS];
375 unsigned chan;
376 for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) {
377 values[chan] = fetch_input_gs(bld_base, reg, type, chan);
378 }
379 return lp_build_gather_values(bld_base->base.gallivm, values,
380 TGSI_NUM_CHANNELS);
381 }
382
383 /* Get the vertex offset parameter */
384 vtx_offset_param = reg->Dimension.Index;
385 if (vtx_offset_param < 2) {
386 vtx_offset_param += SI_PARAM_VTX0_OFFSET;
387 } else {
388 assert(vtx_offset_param < 6);
389 vtx_offset_param += SI_PARAM_VTX2_OFFSET - 2;
390 }
391 vtx_offset = lp_build_mul_imm(uint,
392 LLVMGetParam(si_shader_ctx->radeon_bld.main_fn,
393 vtx_offset_param),
394 4);
395
396 /* Load the ESGS ring resource descriptor */
397 t_list_ptr = LLVMGetParam(si_shader_ctx->radeon_bld.main_fn,
398 SI_PARAM_RW_BUFFERS);
399 t_list = build_indexed_load(si_shader_ctx, t_list_ptr,
400 lp_build_const_int32(gallivm, SI_RING_ESGS));
401
402 args[0] = t_list;
403 args[1] = vtx_offset;
404 args[2] = lp_build_const_int32(gallivm,
405 (get_param_index(input->name, input->sid,
406 shader->selector->gs_used_inputs) * 4 +
407 swizzle) * 256);
408 args[3] = uint->zero;
409 args[4] = uint->one; /* OFFEN */
410 args[5] = uint->zero; /* IDXEN */
411 args[6] = uint->one; /* GLC */
412 args[7] = uint->zero; /* SLC */
413 args[8] = uint->zero; /* TFE */
414
415 return LLVMBuildBitCast(gallivm->builder,
416 build_intrinsic(gallivm->builder,
417 "llvm.SI.buffer.load.dword.i32.i32",
418 i32, args, 9,
419 LLVMReadOnlyAttribute | LLVMNoUnwindAttribute),
420 tgsi2llvmtype(bld_base, type), "");
421 }
422
423 static void declare_input_fs(
424 struct radeon_llvm_context *radeon_bld,
425 unsigned input_index,
426 const struct tgsi_full_declaration *decl)
427 {
428 struct lp_build_context *base = &radeon_bld->soa.bld_base.base;
429 struct si_shader_context *si_shader_ctx =
430 si_shader_context(&radeon_bld->soa.bld_base);
431 struct si_shader *shader = si_shader_ctx->shader;
432 struct lp_build_context *uint = &radeon_bld->soa.bld_base.uint_bld;
433 struct gallivm_state *gallivm = base->gallivm;
434 LLVMTypeRef input_type = LLVMFloatTypeInContext(gallivm->context);
435 LLVMValueRef main_fn = radeon_bld->main_fn;
436
437 LLVMValueRef interp_param;
438 const char * intr_name;
439
440 /* This value is:
441 * [15:0] NewPrimMask (Bit mask for each quad. It is set it the
442 * quad begins a new primitive. Bit 0 always needs
443 * to be unset)
444 * [32:16] ParamOffset
445 *
446 */
447 LLVMValueRef params = LLVMGetParam(main_fn, SI_PARAM_PRIM_MASK);
448 LLVMValueRef attr_number;
449
450 unsigned chan;
451
452 if (decl->Semantic.Name == TGSI_SEMANTIC_POSITION) {
453 for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) {
454 unsigned soa_index =
455 radeon_llvm_reg_index_soa(input_index, chan);
456 radeon_bld->inputs[soa_index] =
457 LLVMGetParam(main_fn, SI_PARAM_POS_X_FLOAT + chan);
458
459 if (chan == 3)
460 /* RCP for fragcoord.w */
461 radeon_bld->inputs[soa_index] =
462 LLVMBuildFDiv(gallivm->builder,
463 lp_build_const_float(gallivm, 1.0f),
464 radeon_bld->inputs[soa_index],
465 "");
466 }
467 return;
468 }
469
470 if (decl->Semantic.Name == TGSI_SEMANTIC_FACE) {
471 LLVMValueRef face, is_face_positive;
472
473 face = LLVMGetParam(main_fn, SI_PARAM_FRONT_FACE);
474
475 is_face_positive = LLVMBuildFCmp(gallivm->builder,
476 LLVMRealUGT, face,
477 lp_build_const_float(gallivm, 0.0f),
478 "");
479
480 radeon_bld->inputs[radeon_llvm_reg_index_soa(input_index, 0)] =
481 LLVMBuildSelect(gallivm->builder,
482 is_face_positive,
483 lp_build_const_float(gallivm, 1.0f),
484 lp_build_const_float(gallivm, 0.0f),
485 "");
486 radeon_bld->inputs[radeon_llvm_reg_index_soa(input_index, 1)] =
487 radeon_bld->inputs[radeon_llvm_reg_index_soa(input_index, 2)] =
488 lp_build_const_float(gallivm, 0.0f);
489 radeon_bld->inputs[radeon_llvm_reg_index_soa(input_index, 3)] =
490 lp_build_const_float(gallivm, 1.0f);
491
492 return;
493 }
494
495 shader->input[input_index].param_offset = shader->nparam++;
496 attr_number = lp_build_const_int32(gallivm,
497 shader->input[input_index].param_offset);
498
499 switch (decl->Interp.Interpolate) {
500 case TGSI_INTERPOLATE_CONSTANT:
501 interp_param = 0;
502 break;
503 case TGSI_INTERPOLATE_LINEAR:
504 if (decl->Interp.Location == TGSI_INTERPOLATE_LOC_SAMPLE)
505 interp_param = LLVMGetParam(main_fn, SI_PARAM_LINEAR_SAMPLE);
506 else if (decl->Interp.Location == TGSI_INTERPOLATE_LOC_CENTROID)
507 interp_param = LLVMGetParam(main_fn, SI_PARAM_LINEAR_CENTROID);
508 else
509 interp_param = LLVMGetParam(main_fn, SI_PARAM_LINEAR_CENTER);
510 break;
511 case TGSI_INTERPOLATE_COLOR:
512 if (si_shader_ctx->shader->key.ps.flatshade) {
513 interp_param = 0;
514 break;
515 }
516 /* fall through to perspective */
517 case TGSI_INTERPOLATE_PERSPECTIVE:
518 if (decl->Interp.Location == TGSI_INTERPOLATE_LOC_SAMPLE)
519 interp_param = LLVMGetParam(main_fn, SI_PARAM_PERSP_SAMPLE);
520 else if (decl->Interp.Location == TGSI_INTERPOLATE_LOC_CENTROID)
521 interp_param = LLVMGetParam(main_fn, SI_PARAM_PERSP_CENTROID);
522 else
523 interp_param = LLVMGetParam(main_fn, SI_PARAM_PERSP_CENTER);
524 break;
525 default:
526 fprintf(stderr, "Warning: Unhandled interpolation mode.\n");
527 return;
528 }
529
530 intr_name = interp_param ? "llvm.SI.fs.interp" : "llvm.SI.fs.constant";
531
532 /* XXX: Could there be more than TGSI_NUM_CHANNELS (4) ? */
533 if (decl->Semantic.Name == TGSI_SEMANTIC_COLOR &&
534 si_shader_ctx->shader->key.ps.color_two_side) {
535 LLVMValueRef args[4];
536 LLVMValueRef face, is_face_positive;
537 LLVMValueRef back_attr_number =
538 lp_build_const_int32(gallivm,
539 shader->input[input_index].param_offset + 1);
540
541 face = LLVMGetParam(main_fn, SI_PARAM_FRONT_FACE);
542
543 is_face_positive = LLVMBuildFCmp(gallivm->builder,
544 LLVMRealUGT, face,
545 lp_build_const_float(gallivm, 0.0f),
546 "");
547
548 args[2] = params;
549 args[3] = interp_param;
550 for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) {
551 LLVMValueRef llvm_chan = lp_build_const_int32(gallivm, chan);
552 unsigned soa_index = radeon_llvm_reg_index_soa(input_index, chan);
553 LLVMValueRef front, back;
554
555 args[0] = llvm_chan;
556 args[1] = attr_number;
557 front = build_intrinsic(gallivm->builder, intr_name,
558 input_type, args, args[3] ? 4 : 3,
559 LLVMReadNoneAttribute | LLVMNoUnwindAttribute);
560
561 args[1] = back_attr_number;
562 back = build_intrinsic(gallivm->builder, intr_name,
563 input_type, args, args[3] ? 4 : 3,
564 LLVMReadNoneAttribute | LLVMNoUnwindAttribute);
565
566 radeon_bld->inputs[soa_index] =
567 LLVMBuildSelect(gallivm->builder,
568 is_face_positive,
569 front,
570 back,
571 "");
572 }
573
574 shader->nparam++;
575 } else if (decl->Semantic.Name == TGSI_SEMANTIC_FOG) {
576 LLVMValueRef args[4];
577
578 args[0] = uint->zero;
579 args[1] = attr_number;
580 args[2] = params;
581 args[3] = interp_param;
582 radeon_bld->inputs[radeon_llvm_reg_index_soa(input_index, 0)] =
583 build_intrinsic(gallivm->builder, intr_name,
584 input_type, args, args[3] ? 4 : 3,
585 LLVMReadNoneAttribute | LLVMNoUnwindAttribute);
586 radeon_bld->inputs[radeon_llvm_reg_index_soa(input_index, 1)] =
587 radeon_bld->inputs[radeon_llvm_reg_index_soa(input_index, 2)] =
588 lp_build_const_float(gallivm, 0.0f);
589 radeon_bld->inputs[radeon_llvm_reg_index_soa(input_index, 3)] =
590 lp_build_const_float(gallivm, 1.0f);
591 } else {
592 for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) {
593 LLVMValueRef args[4];
594 LLVMValueRef llvm_chan = lp_build_const_int32(gallivm, chan);
595 unsigned soa_index = radeon_llvm_reg_index_soa(input_index, chan);
596 args[0] = llvm_chan;
597 args[1] = attr_number;
598 args[2] = params;
599 args[3] = interp_param;
600 radeon_bld->inputs[soa_index] =
601 build_intrinsic(gallivm->builder, intr_name,
602 input_type, args, args[3] ? 4 : 3,
603 LLVMReadNoneAttribute | LLVMNoUnwindAttribute);
604 }
605 }
606 }
607
608 static LLVMValueRef get_sample_id(struct radeon_llvm_context *radeon_bld)
609 {
610 struct gallivm_state *gallivm = &radeon_bld->gallivm;
611 LLVMValueRef value = LLVMGetParam(radeon_bld->main_fn,
612 SI_PARAM_ANCILLARY);
613 value = LLVMBuildLShr(gallivm->builder, value,
614 lp_build_const_int32(gallivm, 8), "");
615 value = LLVMBuildAnd(gallivm->builder, value,
616 lp_build_const_int32(gallivm, 0xf), "");
617 return value;
618 }
619
620 static LLVMValueRef load_const(LLVMBuilderRef builder, LLVMValueRef resource,
621 LLVMValueRef offset, LLVMTypeRef return_type)
622 {
623 LLVMValueRef args[2] = {resource, offset};
624
625 return build_intrinsic(builder, "llvm.SI.load.const", return_type, args, 2,
626 LLVMReadNoneAttribute | LLVMNoUnwindAttribute);
627 }
628
629 static void declare_system_value(
630 struct radeon_llvm_context * radeon_bld,
631 unsigned index,
632 const struct tgsi_full_declaration *decl)
633 {
634 struct si_shader_context *si_shader_ctx =
635 si_shader_context(&radeon_bld->soa.bld_base);
636 struct lp_build_context *uint_bld = &radeon_bld->soa.bld_base.uint_bld;
637 struct gallivm_state *gallivm = &radeon_bld->gallivm;
638 LLVMValueRef value = 0;
639
640 switch (decl->Semantic.Name) {
641 case TGSI_SEMANTIC_INSTANCEID:
642 value = LLVMGetParam(radeon_bld->main_fn,
643 si_shader_ctx->param_instance_id);
644 break;
645
646 case TGSI_SEMANTIC_VERTEXID:
647 value = LLVMGetParam(radeon_bld->main_fn,
648 si_shader_ctx->param_vertex_id);
649 break;
650
651 case TGSI_SEMANTIC_SAMPLEID:
652 value = get_sample_id(radeon_bld);
653 break;
654
655 case TGSI_SEMANTIC_SAMPLEPOS:
656 {
657 LLVMBuilderRef builder = gallivm->builder;
658 LLVMValueRef desc = LLVMGetParam(si_shader_ctx->radeon_bld.main_fn, SI_PARAM_CONST);
659 LLVMValueRef buf_index = lp_build_const_int32(gallivm, SI_DRIVER_STATE_CONST_BUF);
660 LLVMValueRef resource = build_indexed_load(si_shader_ctx, desc, buf_index);
661
662 /* offset = sample_id * 8 (8 = 2 floats containing samplepos.xy) */
663 LLVMValueRef offset0 = lp_build_mul_imm(uint_bld, get_sample_id(radeon_bld), 8);
664 LLVMValueRef offset1 = LLVMBuildAdd(builder, offset0, lp_build_const_int32(gallivm, 4), "");
665
666 LLVMValueRef pos[4] = {
667 load_const(builder, resource, offset0, radeon_bld->soa.bld_base.base.elem_type),
668 load_const(builder, resource, offset1, radeon_bld->soa.bld_base.base.elem_type),
669 lp_build_const_float(gallivm, 0),
670 lp_build_const_float(gallivm, 0)
671 };
672 value = lp_build_gather_values(gallivm, pos, 4);
673 break;
674 }
675
676 default:
677 assert(!"unknown system value");
678 return;
679 }
680
681 radeon_bld->system_values[index] = value;
682 }
683
684 static LLVMValueRef fetch_constant(
685 struct lp_build_tgsi_context * bld_base,
686 const struct tgsi_full_src_register *reg,
687 enum tgsi_opcode_type type,
688 unsigned swizzle)
689 {
690 struct si_shader_context *si_shader_ctx = si_shader_context(bld_base);
691 struct lp_build_context * base = &bld_base->base;
692 const struct tgsi_ind_register *ireg = &reg->Indirect;
693 unsigned buf, idx;
694
695 LLVMValueRef addr;
696 LLVMValueRef result;
697
698 if (swizzle == LP_CHAN_ALL) {
699 unsigned chan;
700 LLVMValueRef values[4];
701 for (chan = 0; chan < TGSI_NUM_CHANNELS; ++chan)
702 values[chan] = fetch_constant(bld_base, reg, type, chan);
703
704 return lp_build_gather_values(bld_base->base.gallivm, values, 4);
705 }
706
707 buf = reg->Register.Dimension ? reg->Dimension.Index : 0;
708 idx = reg->Register.Index * 4 + swizzle;
709
710 if (!reg->Register.Indirect)
711 return bitcast(bld_base, type, si_shader_ctx->constants[buf][idx]);
712
713 addr = si_shader_ctx->radeon_bld.soa.addr[ireg->Index][ireg->Swizzle];
714 addr = LLVMBuildLoad(base->gallivm->builder, addr, "load addr reg");
715 addr = lp_build_mul_imm(&bld_base->uint_bld, addr, 16);
716 addr = lp_build_add(&bld_base->uint_bld, addr,
717 lp_build_const_int32(base->gallivm, idx * 4));
718
719 result = load_const(base->gallivm->builder, si_shader_ctx->const_resource[buf],
720 addr, base->elem_type);
721
722 return bitcast(bld_base, type, result);
723 }
724
725 /* Initialize arguments for the shader export intrinsic */
726 static void si_llvm_init_export_args(struct lp_build_tgsi_context *bld_base,
727 LLVMValueRef *values,
728 unsigned target,
729 LLVMValueRef *args)
730 {
731 struct si_shader_context *si_shader_ctx = si_shader_context(bld_base);
732 struct lp_build_context *uint =
733 &si_shader_ctx->radeon_bld.soa.bld_base.uint_bld;
734 struct lp_build_context *base = &bld_base->base;
735 unsigned compressed = 0;
736 unsigned chan;
737
738 if (si_shader_ctx->type == TGSI_PROCESSOR_FRAGMENT) {
739 int cbuf = target - V_008DFC_SQ_EXP_MRT;
740
741 if (cbuf >= 0 && cbuf < 8) {
742 compressed = (si_shader_ctx->shader->key.ps.export_16bpc >> cbuf) & 0x1;
743
744 if (compressed)
745 si_shader_ctx->shader->spi_shader_col_format |=
746 V_028714_SPI_SHADER_FP16_ABGR << (4 * cbuf);
747 else
748 si_shader_ctx->shader->spi_shader_col_format |=
749 V_028714_SPI_SHADER_32_ABGR << (4 * cbuf);
750
751 si_shader_ctx->shader->cb_shader_mask |= 0xf << (4 * cbuf);
752 }
753 }
754
755 if (compressed) {
756 /* Pixel shader needs to pack output values before export */
757 for (chan = 0; chan < 2; chan++ ) {
758 args[0] = values[2 * chan];
759 args[1] = values[2 * chan + 1];
760 args[chan + 5] =
761 build_intrinsic(base->gallivm->builder,
762 "llvm.SI.packf16",
763 LLVMInt32TypeInContext(base->gallivm->context),
764 args, 2,
765 LLVMReadNoneAttribute | LLVMNoUnwindAttribute);
766 args[chan + 7] = args[chan + 5] =
767 LLVMBuildBitCast(base->gallivm->builder,
768 args[chan + 5],
769 LLVMFloatTypeInContext(base->gallivm->context),
770 "");
771 }
772
773 /* Set COMPR flag */
774 args[4] = uint->one;
775 } else {
776 for (chan = 0; chan < 4; chan++ )
777 /* +5 because the first output value will be
778 * the 6th argument to the intrinsic. */
779 args[chan + 5] = values[chan];
780
781 /* Clear COMPR flag */
782 args[4] = uint->zero;
783 }
784
785 /* XXX: This controls which components of the output
786 * registers actually get exported. (e.g bit 0 means export
787 * X component, bit 1 means export Y component, etc.) I'm
788 * hard coding this to 0xf for now. In the future, we might
789 * want to do something else. */
790 args[0] = lp_build_const_int32(base->gallivm, 0xf);
791
792 /* Specify whether the EXEC mask represents the valid mask */
793 args[1] = uint->zero;
794
795 /* Specify whether this is the last export */
796 args[2] = uint->zero;
797
798 /* Specify the target we are exporting */
799 args[3] = lp_build_const_int32(base->gallivm, target);
800
801 /* XXX: We probably need to keep track of the output
802 * values, so we know what we are passing to the next
803 * stage. */
804 }
805
806 /* Load from output pointers and initialize arguments for the shader export intrinsic */
807 static void si_llvm_init_export_args_load(struct lp_build_tgsi_context *bld_base,
808 LLVMValueRef *out_ptr,
809 unsigned target,
810 LLVMValueRef *args)
811 {
812 struct gallivm_state *gallivm = bld_base->base.gallivm;
813 LLVMValueRef values[4];
814 int i;
815
816 for (i = 0; i < 4; i++)
817 values[i] = LLVMBuildLoad(gallivm->builder, out_ptr[i], "");
818
819 si_llvm_init_export_args(bld_base, values, target, args);
820 }
821
822 static void si_alpha_test(struct lp_build_tgsi_context *bld_base,
823 LLVMValueRef *out_ptr)
824 {
825 struct si_shader_context *si_shader_ctx = si_shader_context(bld_base);
826 struct gallivm_state *gallivm = bld_base->base.gallivm;
827
828 if (si_shader_ctx->shader->key.ps.alpha_func != PIPE_FUNC_NEVER) {
829 LLVMValueRef alpha_ref = LLVMGetParam(si_shader_ctx->radeon_bld.main_fn,
830 SI_PARAM_ALPHA_REF);
831
832 LLVMValueRef alpha_pass =
833 lp_build_cmp(&bld_base->base,
834 si_shader_ctx->shader->key.ps.alpha_func,
835 LLVMBuildLoad(gallivm->builder, out_ptr[3], ""),
836 alpha_ref);
837 LLVMValueRef arg =
838 lp_build_select(&bld_base->base,
839 alpha_pass,
840 lp_build_const_float(gallivm, 1.0f),
841 lp_build_const_float(gallivm, -1.0f));
842
843 build_intrinsic(gallivm->builder,
844 "llvm.AMDGPU.kill",
845 LLVMVoidTypeInContext(gallivm->context),
846 &arg, 1, 0);
847 } else {
848 build_intrinsic(gallivm->builder,
849 "llvm.AMDGPU.kilp",
850 LLVMVoidTypeInContext(gallivm->context),
851 NULL, 0, 0);
852 }
853
854 si_shader_ctx->shader->db_shader_control |= S_02880C_KILL_ENABLE(1);
855 }
856
857 static void si_llvm_emit_clipvertex(struct lp_build_tgsi_context * bld_base,
858 LLVMValueRef (*pos)[9], LLVMValueRef *out_elts)
859 {
860 struct si_shader_context *si_shader_ctx = si_shader_context(bld_base);
861 struct si_shader *shader = si_shader_ctx->shader;
862 struct lp_build_context *base = &bld_base->base;
863 struct lp_build_context *uint = &si_shader_ctx->radeon_bld.soa.bld_base.uint_bld;
864 unsigned reg_index;
865 unsigned chan;
866 unsigned const_chan;
867 LLVMValueRef base_elt;
868 LLVMValueRef ptr = LLVMGetParam(si_shader_ctx->radeon_bld.main_fn, SI_PARAM_CONST);
869 LLVMValueRef constbuf_index = lp_build_const_int32(base->gallivm, SI_DRIVER_STATE_CONST_BUF);
870 LLVMValueRef const_resource = build_indexed_load(si_shader_ctx, ptr, constbuf_index);
871
872 for (reg_index = 0; reg_index < 2; reg_index ++) {
873 LLVMValueRef *args = pos[2 + reg_index];
874
875 if (!(shader->key.vs.ucps_enabled & (1 << reg_index)))
876 continue;
877
878 shader->clip_dist_write |= 0xf << (4 * reg_index);
879
880 args[5] =
881 args[6] =
882 args[7] =
883 args[8] = lp_build_const_float(base->gallivm, 0.0f);
884
885 /* Compute dot products of position and user clip plane vectors */
886 for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) {
887 for (const_chan = 0; const_chan < TGSI_NUM_CHANNELS; const_chan++) {
888 args[1] = lp_build_const_int32(base->gallivm,
889 ((reg_index * 4 + chan) * 4 +
890 const_chan) * 4);
891 base_elt = load_const(base->gallivm->builder, const_resource,
892 args[1], base->elem_type);
893 args[5 + chan] =
894 lp_build_add(base, args[5 + chan],
895 lp_build_mul(base, base_elt,
896 out_elts[const_chan]));
897 }
898 }
899
900 args[0] = lp_build_const_int32(base->gallivm, 0xf);
901 args[1] = uint->zero;
902 args[2] = uint->zero;
903 args[3] = lp_build_const_int32(base->gallivm,
904 V_008DFC_SQ_EXP_POS + 2 + reg_index);
905 args[4] = uint->zero;
906 }
907 }
908
909 static void si_dump_streamout(struct pipe_stream_output_info *so)
910 {
911 unsigned i;
912
913 if (so->num_outputs)
914 fprintf(stderr, "STREAMOUT\n");
915
916 for (i = 0; i < so->num_outputs; i++) {
917 unsigned mask = ((1 << so->output[i].num_components) - 1) <<
918 so->output[i].start_component;
919 fprintf(stderr, " %i: BUF%i[%i..%i] <- OUT[%i].%s%s%s%s\n",
920 i, so->output[i].output_buffer,
921 so->output[i].dst_offset, so->output[i].dst_offset + so->output[i].num_components - 1,
922 so->output[i].register_index,
923 mask & 1 ? "x" : "",
924 mask & 2 ? "y" : "",
925 mask & 4 ? "z" : "",
926 mask & 8 ? "w" : "");
927 }
928 }
929
930 /* TBUFFER_STORE_FORMAT_{X,XY,XYZ,XYZW} <- the suffix is selected by num_channels=1..4.
931 * The type of vdata must be one of i32 (num_channels=1), v2i32 (num_channels=2),
932 * or v4i32 (num_channels=3,4). */
933 static void build_tbuffer_store(struct si_shader_context *shader,
934 LLVMValueRef rsrc,
935 LLVMValueRef vdata,
936 unsigned num_channels,
937 LLVMValueRef vaddr,
938 LLVMValueRef soffset,
939 unsigned inst_offset,
940 unsigned dfmt,
941 unsigned nfmt,
942 unsigned offen,
943 unsigned idxen,
944 unsigned glc,
945 unsigned slc,
946 unsigned tfe)
947 {
948 struct gallivm_state *gallivm = &shader->radeon_bld.gallivm;
949 LLVMTypeRef i32 = LLVMInt32TypeInContext(gallivm->context);
950 LLVMValueRef args[] = {
951 rsrc,
952 vdata,
953 LLVMConstInt(i32, num_channels, 0),
954 vaddr,
955 soffset,
956 LLVMConstInt(i32, inst_offset, 0),
957 LLVMConstInt(i32, dfmt, 0),
958 LLVMConstInt(i32, nfmt, 0),
959 LLVMConstInt(i32, offen, 0),
960 LLVMConstInt(i32, idxen, 0),
961 LLVMConstInt(i32, glc, 0),
962 LLVMConstInt(i32, slc, 0),
963 LLVMConstInt(i32, tfe, 0)
964 };
965
966 /* The instruction offset field has 12 bits */
967 assert(offen || inst_offset < (1 << 12));
968
969 /* The intrinsic is overloaded, we need to add a type suffix for overloading to work. */
970 unsigned func = CLAMP(num_channels, 1, 3) - 1;
971 const char *types[] = {"i32", "v2i32", "v4i32"};
972 char name[256];
973 snprintf(name, sizeof(name), "llvm.SI.tbuffer.store.%s", types[func]);
974
975 lp_build_intrinsic(gallivm->builder, name,
976 LLVMVoidTypeInContext(gallivm->context),
977 args, Elements(args));
978 }
979
980 static void build_streamout_store(struct si_shader_context *shader,
981 LLVMValueRef rsrc,
982 LLVMValueRef vdata,
983 unsigned num_channels,
984 LLVMValueRef vaddr,
985 LLVMValueRef soffset,
986 unsigned inst_offset)
987 {
988 static unsigned dfmt[] = {
989 V_008F0C_BUF_DATA_FORMAT_32,
990 V_008F0C_BUF_DATA_FORMAT_32_32,
991 V_008F0C_BUF_DATA_FORMAT_32_32_32,
992 V_008F0C_BUF_DATA_FORMAT_32_32_32_32
993 };
994 assert(num_channels >= 1 && num_channels <= 4);
995
996 build_tbuffer_store(shader, rsrc, vdata, num_channels, vaddr, soffset,
997 inst_offset, dfmt[num_channels-1],
998 V_008F0C_BUF_NUM_FORMAT_UINT, 1, 0, 1, 1, 0);
999 }
1000
1001 /* On SI, the vertex shader is responsible for writing streamout data
1002 * to buffers. */
1003 static void si_llvm_emit_streamout(struct si_shader_context *shader,
1004 struct si_shader_output_values *outputs,
1005 unsigned noutput)
1006 {
1007 struct pipe_stream_output_info *so = &shader->shader->selector->so;
1008 struct gallivm_state *gallivm = &shader->radeon_bld.gallivm;
1009 LLVMBuilderRef builder = gallivm->builder;
1010 int i, j;
1011 struct lp_build_if_state if_ctx;
1012
1013 LLVMTypeRef i32 = LLVMInt32TypeInContext(gallivm->context);
1014
1015 LLVMValueRef so_param =
1016 LLVMGetParam(shader->radeon_bld.main_fn,
1017 shader->param_streamout_config);
1018
1019 /* Get bits [22:16], i.e. (so_param >> 16) & 127; */
1020 LLVMValueRef so_vtx_count =
1021 LLVMBuildAnd(builder,
1022 LLVMBuildLShr(builder, so_param,
1023 LLVMConstInt(i32, 16, 0), ""),
1024 LLVMConstInt(i32, 127, 0), "");
1025
1026 LLVMValueRef tid = build_intrinsic(builder, "llvm.SI.tid", i32,
1027 NULL, 0, LLVMReadNoneAttribute);
1028
1029 /* can_emit = tid < so_vtx_count; */
1030 LLVMValueRef can_emit =
1031 LLVMBuildICmp(builder, LLVMIntULT, tid, so_vtx_count, "");
1032
1033 /* Emit the streamout code conditionally. This actually avoids
1034 * out-of-bounds buffer access. The hw tells us via the SGPR
1035 * (so_vtx_count) which threads are allowed to emit streamout data. */
1036 lp_build_if(&if_ctx, gallivm, can_emit);
1037 {
1038 /* The buffer offset is computed as follows:
1039 * ByteOffset = streamout_offset[buffer_id]*4 +
1040 * (streamout_write_index + thread_id)*stride[buffer_id] +
1041 * attrib_offset
1042 */
1043
1044 LLVMValueRef so_write_index =
1045 LLVMGetParam(shader->radeon_bld.main_fn,
1046 shader->param_streamout_write_index);
1047
1048 /* Compute (streamout_write_index + thread_id). */
1049 so_write_index = LLVMBuildAdd(builder, so_write_index, tid, "");
1050
1051 /* Compute the write offset for each enabled buffer. */
1052 LLVMValueRef so_write_offset[4] = {};
1053 for (i = 0; i < 4; i++) {
1054 if (!so->stride[i])
1055 continue;
1056
1057 LLVMValueRef so_offset = LLVMGetParam(shader->radeon_bld.main_fn,
1058 shader->param_streamout_offset[i]);
1059 so_offset = LLVMBuildMul(builder, so_offset, LLVMConstInt(i32, 4, 0), "");
1060
1061 so_write_offset[i] = LLVMBuildMul(builder, so_write_index,
1062 LLVMConstInt(i32, so->stride[i]*4, 0), "");
1063 so_write_offset[i] = LLVMBuildAdd(builder, so_write_offset[i], so_offset, "");
1064 }
1065
1066 /* Write streamout data. */
1067 for (i = 0; i < so->num_outputs; i++) {
1068 unsigned buf_idx = so->output[i].output_buffer;
1069 unsigned reg = so->output[i].register_index;
1070 unsigned start = so->output[i].start_component;
1071 unsigned num_comps = so->output[i].num_components;
1072 LLVMValueRef out[4];
1073
1074 assert(num_comps && num_comps <= 4);
1075 if (!num_comps || num_comps > 4)
1076 continue;
1077
1078 /* Load the output as int. */
1079 for (j = 0; j < num_comps; j++) {
1080 unsigned outidx = 0;
1081
1082 while (outidx < noutput && outputs[outidx].index != reg)
1083 outidx++;
1084
1085 if (outidx < noutput)
1086 out[j] = LLVMBuildBitCast(builder,
1087 outputs[outidx].values[start+j],
1088 i32, "");
1089 else
1090 out[j] = NULL;
1091 }
1092
1093 if (!out[0])
1094 continue;
1095
1096 /* Pack the output. */
1097 LLVMValueRef vdata = NULL;
1098
1099 switch (num_comps) {
1100 case 1: /* as i32 */
1101 vdata = out[0];
1102 break;
1103 case 2: /* as v2i32 */
1104 case 3: /* as v4i32 (aligned to 4) */
1105 case 4: /* as v4i32 */
1106 vdata = LLVMGetUndef(LLVMVectorType(i32, util_next_power_of_two(num_comps)));
1107 for (j = 0; j < num_comps; j++) {
1108 vdata = LLVMBuildInsertElement(builder, vdata, out[j],
1109 LLVMConstInt(i32, j, 0), "");
1110 }
1111 break;
1112 }
1113
1114 build_streamout_store(shader, shader->so_buffers[buf_idx],
1115 vdata, num_comps,
1116 so_write_offset[buf_idx],
1117 LLVMConstInt(i32, 0, 0),
1118 so->output[i].dst_offset*4);
1119 }
1120 }
1121 lp_build_endif(&if_ctx);
1122 }
1123
1124
1125 /* Generate export instructions for hardware VS shader stage */
1126 static void si_llvm_export_vs(struct lp_build_tgsi_context *bld_base,
1127 struct si_shader_output_values *outputs,
1128 unsigned noutput)
1129 {
1130 struct si_shader_context * si_shader_ctx = si_shader_context(bld_base);
1131 struct si_shader * shader = si_shader_ctx->shader;
1132 struct lp_build_context * base = &bld_base->base;
1133 struct lp_build_context * uint =
1134 &si_shader_ctx->radeon_bld.soa.bld_base.uint_bld;
1135 LLVMValueRef args[9];
1136 LLVMValueRef pos_args[4][9] = { { 0 } };
1137 LLVMValueRef psize_value = NULL, edgeflag_value = NULL, layer_value = NULL;
1138 unsigned semantic_name, semantic_index, semantic_usage;
1139 unsigned target;
1140 unsigned param_count = 0;
1141 unsigned pos_idx;
1142 int i;
1143
1144 if (outputs && si_shader_ctx->shader->selector->so.num_outputs) {
1145 si_llvm_emit_streamout(si_shader_ctx, outputs, noutput);
1146 }
1147
1148 for (i = 0; i < noutput; i++) {
1149 semantic_name = outputs[i].name;
1150 semantic_index = outputs[i].sid;
1151 semantic_usage = outputs[i].usage;
1152
1153 handle_semantic:
1154 /* Select the correct target */
1155 switch(semantic_name) {
1156 case TGSI_SEMANTIC_PSIZE:
1157 shader->vs_out_misc_write = true;
1158 shader->vs_out_point_size = true;
1159 psize_value = outputs[i].values[0];
1160 continue;
1161 case TGSI_SEMANTIC_EDGEFLAG:
1162 shader->vs_out_misc_write = true;
1163 shader->vs_out_edgeflag = true;
1164 edgeflag_value = outputs[i].values[0];
1165 continue;
1166 case TGSI_SEMANTIC_LAYER:
1167 shader->vs_out_misc_write = true;
1168 shader->vs_out_layer = true;
1169 layer_value = outputs[i].values[0];
1170 continue;
1171 case TGSI_SEMANTIC_POSITION:
1172 target = V_008DFC_SQ_EXP_POS;
1173 break;
1174 case TGSI_SEMANTIC_COLOR:
1175 case TGSI_SEMANTIC_BCOLOR:
1176 target = V_008DFC_SQ_EXP_PARAM + param_count;
1177 shader->output[i].param_offset = param_count;
1178 param_count++;
1179 break;
1180 case TGSI_SEMANTIC_CLIPDIST:
1181 if (!(si_shader_ctx->shader->key.vs.ucps_enabled &
1182 (1 << semantic_index)))
1183 continue;
1184 shader->clip_dist_write |=
1185 semantic_usage << (semantic_index << 2);
1186 target = V_008DFC_SQ_EXP_POS + 2 + semantic_index;
1187 break;
1188 case TGSI_SEMANTIC_CLIPVERTEX:
1189 si_llvm_emit_clipvertex(bld_base, pos_args, outputs[i].values);
1190 continue;
1191 case TGSI_SEMANTIC_PRIMID:
1192 case TGSI_SEMANTIC_FOG:
1193 case TGSI_SEMANTIC_GENERIC:
1194 target = V_008DFC_SQ_EXP_PARAM + param_count;
1195 shader->output[i].param_offset = param_count;
1196 param_count++;
1197 break;
1198 default:
1199 target = 0;
1200 fprintf(stderr,
1201 "Warning: SI unhandled vs output type:%d\n",
1202 semantic_name);
1203 }
1204
1205 si_llvm_init_export_args(bld_base, outputs[i].values, target, args);
1206
1207 if (target >= V_008DFC_SQ_EXP_POS &&
1208 target <= (V_008DFC_SQ_EXP_POS + 3)) {
1209 memcpy(pos_args[target - V_008DFC_SQ_EXP_POS],
1210 args, sizeof(args));
1211 } else {
1212 lp_build_intrinsic(base->gallivm->builder,
1213 "llvm.SI.export",
1214 LLVMVoidTypeInContext(base->gallivm->context),
1215 args, 9);
1216 }
1217
1218 if (semantic_name == TGSI_SEMANTIC_CLIPDIST) {
1219 semantic_name = TGSI_SEMANTIC_GENERIC;
1220 goto handle_semantic;
1221 }
1222 }
1223
1224 /* We need to add the position output manually if it's missing. */
1225 if (!pos_args[0][0]) {
1226 pos_args[0][0] = lp_build_const_int32(base->gallivm, 0xf); /* writemask */
1227 pos_args[0][1] = uint->zero; /* EXEC mask */
1228 pos_args[0][2] = uint->zero; /* last export? */
1229 pos_args[0][3] = lp_build_const_int32(base->gallivm, V_008DFC_SQ_EXP_POS);
1230 pos_args[0][4] = uint->zero; /* COMPR flag */
1231 pos_args[0][5] = base->zero; /* X */
1232 pos_args[0][6] = base->zero; /* Y */
1233 pos_args[0][7] = base->zero; /* Z */
1234 pos_args[0][8] = base->one; /* W */
1235 }
1236
1237 /* Write the misc vector (point size, edgeflag, layer, viewport). */
1238 if (shader->vs_out_misc_write) {
1239 pos_args[1][0] = lp_build_const_int32(base->gallivm, /* writemask */
1240 shader->vs_out_point_size |
1241 (shader->vs_out_edgeflag << 1) |
1242 (shader->vs_out_layer << 2));
1243 pos_args[1][1] = uint->zero; /* EXEC mask */
1244 pos_args[1][2] = uint->zero; /* last export? */
1245 pos_args[1][3] = lp_build_const_int32(base->gallivm, V_008DFC_SQ_EXP_POS + 1);
1246 pos_args[1][4] = uint->zero; /* COMPR flag */
1247 pos_args[1][5] = base->zero; /* X */
1248 pos_args[1][6] = base->zero; /* Y */
1249 pos_args[1][7] = base->zero; /* Z */
1250 pos_args[1][8] = base->zero; /* W */
1251
1252 if (shader->vs_out_point_size)
1253 pos_args[1][5] = psize_value;
1254
1255 if (shader->vs_out_edgeflag) {
1256 /* The output is a float, but the hw expects an integer
1257 * with the first bit containing the edge flag. */
1258 edgeflag_value = LLVMBuildFPToUI(base->gallivm->builder,
1259 edgeflag_value,
1260 bld_base->uint_bld.elem_type, "");
1261 edgeflag_value = lp_build_min(&bld_base->int_bld,
1262 edgeflag_value,
1263 bld_base->int_bld.one);
1264
1265 /* The LLVM intrinsic expects a float. */
1266 pos_args[1][6] = LLVMBuildBitCast(base->gallivm->builder,
1267 edgeflag_value,
1268 base->elem_type, "");
1269 }
1270
1271 if (shader->vs_out_layer)
1272 pos_args[1][7] = layer_value;
1273 }
1274
1275 for (i = 0; i < 4; i++)
1276 if (pos_args[i][0])
1277 shader->nr_pos_exports++;
1278
1279 pos_idx = 0;
1280 for (i = 0; i < 4; i++) {
1281 if (!pos_args[i][0])
1282 continue;
1283
1284 /* Specify the target we are exporting */
1285 pos_args[i][3] = lp_build_const_int32(base->gallivm, V_008DFC_SQ_EXP_POS + pos_idx++);
1286
1287 if (pos_idx == shader->nr_pos_exports)
1288 /* Specify that this is the last export */
1289 pos_args[i][2] = uint->one;
1290
1291 lp_build_intrinsic(base->gallivm->builder,
1292 "llvm.SI.export",
1293 LLVMVoidTypeInContext(base->gallivm->context),
1294 pos_args[i], 9);
1295 }
1296 }
1297
1298 static void si_llvm_emit_es_epilogue(struct lp_build_tgsi_context * bld_base)
1299 {
1300 struct si_shader_context *si_shader_ctx = si_shader_context(bld_base);
1301 struct gallivm_state *gallivm = bld_base->base.gallivm;
1302 struct si_shader *es = si_shader_ctx->shader;
1303 struct tgsi_parse_context *parse = &si_shader_ctx->parse;
1304 LLVMTypeRef i32 = LLVMInt32TypeInContext(gallivm->context);
1305 LLVMValueRef soffset = LLVMGetParam(si_shader_ctx->radeon_bld.main_fn,
1306 SI_PARAM_ES2GS_OFFSET);
1307 LLVMValueRef t_list_ptr;
1308 LLVMValueRef t_list;
1309 unsigned chan;
1310 int i;
1311
1312 while (!tgsi_parse_end_of_tokens(parse)) {
1313 struct tgsi_full_declaration *d =
1314 &parse->FullToken.FullDeclaration;
1315
1316 tgsi_parse_token(parse);
1317
1318 if (parse->FullToken.Token.Type != TGSI_TOKEN_TYPE_DECLARATION)
1319 continue;
1320
1321 si_store_shader_io_attribs(es, d);
1322 }
1323
1324 /* Load the ESGS ring resource descriptor */
1325 t_list_ptr = LLVMGetParam(si_shader_ctx->radeon_bld.main_fn,
1326 SI_PARAM_RW_BUFFERS);
1327 t_list = build_indexed_load(si_shader_ctx, t_list_ptr,
1328 lp_build_const_int32(gallivm, SI_RING_ESGS));
1329
1330 for (i = 0; i < es->noutput; i++) {
1331 LLVMValueRef *out_ptr =
1332 si_shader_ctx->radeon_bld.soa.outputs[es->output[i].index];
1333 int param_index = get_param_index(es->output[i].name,
1334 es->output[i].sid,
1335 es->key.vs.gs_used_inputs);
1336
1337 if (param_index < 0)
1338 continue;
1339
1340 for (chan = 0; chan < 4; chan++) {
1341 LLVMValueRef out_val = LLVMBuildLoad(gallivm->builder, out_ptr[chan], "");
1342 out_val = LLVMBuildBitCast(gallivm->builder, out_val, i32, "");
1343
1344 build_tbuffer_store(si_shader_ctx, t_list, out_val, 1,
1345 LLVMGetUndef(i32), soffset,
1346 (4 * param_index + chan) * 4,
1347 V_008F0C_BUF_DATA_FORMAT_32,
1348 V_008F0C_BUF_NUM_FORMAT_UINT,
1349 0, 0, 1, 1, 0);
1350 }
1351 }
1352 }
1353
1354 static void si_llvm_emit_gs_epilogue(struct lp_build_tgsi_context *bld_base)
1355 {
1356 struct si_shader_context *si_shader_ctx = si_shader_context(bld_base);
1357 struct gallivm_state *gallivm = bld_base->base.gallivm;
1358 LLVMValueRef args[2];
1359
1360 args[0] = lp_build_const_int32(gallivm, SENDMSG_GS_OP_NOP | SENDMSG_GS_DONE);
1361 args[1] = LLVMGetParam(si_shader_ctx->radeon_bld.main_fn, SI_PARAM_GS_WAVE_ID);
1362 build_intrinsic(gallivm->builder, "llvm.SI.sendmsg",
1363 LLVMVoidTypeInContext(gallivm->context), args, 2,
1364 LLVMNoUnwindAttribute);
1365 }
1366
1367 static void si_llvm_emit_vs_epilogue(struct lp_build_tgsi_context * bld_base)
1368 {
1369 struct si_shader_context *si_shader_ctx = si_shader_context(bld_base);
1370 struct gallivm_state *gallivm = bld_base->base.gallivm;
1371 struct si_shader *shader = si_shader_ctx->shader;
1372 struct tgsi_parse_context *parse = &si_shader_ctx->parse;
1373 struct si_shader_output_values *outputs = NULL;
1374 unsigned noutput = 0;
1375 int i;
1376
1377 while (!tgsi_parse_end_of_tokens(parse)) {
1378 struct tgsi_full_declaration *d =
1379 &parse->FullToken.FullDeclaration;
1380 unsigned index;
1381
1382 tgsi_parse_token(parse);
1383
1384 if (parse->FullToken.Token.Type != TGSI_TOKEN_TYPE_DECLARATION)
1385 continue;
1386
1387 i = si_store_shader_io_attribs(shader, d);
1388 if (i < 0)
1389 continue;
1390
1391 outputs = REALLOC(outputs, noutput * sizeof(outputs[0]),
1392 (noutput + 1) * sizeof(outputs[0]));
1393 for (index = d->Range.First; index <= d->Range.Last; index++) {
1394 outputs[noutput].index = index;
1395 outputs[noutput].name = d->Semantic.Name;
1396 outputs[noutput].sid = d->Semantic.Index;
1397 outputs[noutput].usage = d->Declaration.UsageMask;
1398
1399 for (i = 0; i < 4; i++)
1400 outputs[noutput].values[i] =
1401 LLVMBuildLoad(gallivm->builder,
1402 si_shader_ctx->radeon_bld.soa.outputs[index][i],
1403 "");
1404 }
1405 noutput++;
1406 }
1407
1408 si_llvm_export_vs(bld_base, outputs, noutput);
1409 FREE(outputs);
1410 }
1411
1412 static void si_llvm_emit_fs_epilogue(struct lp_build_tgsi_context * bld_base)
1413 {
1414 struct si_shader_context * si_shader_ctx = si_shader_context(bld_base);
1415 struct si_shader * shader = si_shader_ctx->shader;
1416 struct lp_build_context * base = &bld_base->base;
1417 struct lp_build_context * uint =
1418 &si_shader_ctx->radeon_bld.soa.bld_base.uint_bld;
1419 struct tgsi_parse_context *parse = &si_shader_ctx->parse;
1420 LLVMValueRef args[9];
1421 LLVMValueRef last_args[9] = { 0 };
1422 unsigned semantic_name;
1423 int depth_index = -1, stencil_index = -1, samplemask_index = -1;
1424 int i;
1425
1426 while (!tgsi_parse_end_of_tokens(parse)) {
1427 struct tgsi_full_declaration *d =
1428 &parse->FullToken.FullDeclaration;
1429 unsigned target;
1430 unsigned index;
1431
1432 tgsi_parse_token(parse);
1433
1434 if (parse->FullToken.Token.Type != TGSI_TOKEN_TYPE_DECLARATION)
1435 continue;
1436
1437 i = si_store_shader_io_attribs(shader, d);
1438 if (i < 0)
1439 continue;
1440
1441 semantic_name = d->Semantic.Name;
1442 for (index = d->Range.First; index <= d->Range.Last; index++) {
1443 /* Select the correct target */
1444 switch(semantic_name) {
1445 case TGSI_SEMANTIC_POSITION:
1446 depth_index = index;
1447 continue;
1448 case TGSI_SEMANTIC_STENCIL:
1449 stencil_index = index;
1450 continue;
1451 case TGSI_SEMANTIC_SAMPLEMASK:
1452 samplemask_index = index;
1453 continue;
1454 case TGSI_SEMANTIC_COLOR:
1455 target = V_008DFC_SQ_EXP_MRT + d->Semantic.Index;
1456 if (si_shader_ctx->shader->key.ps.alpha_to_one)
1457 LLVMBuildStore(bld_base->base.gallivm->builder,
1458 bld_base->base.one,
1459 si_shader_ctx->radeon_bld.soa.outputs[index][3]);
1460
1461 if (d->Semantic.Index == 0 &&
1462 si_shader_ctx->shader->key.ps.alpha_func != PIPE_FUNC_ALWAYS)
1463 si_alpha_test(bld_base,
1464 si_shader_ctx->radeon_bld.soa.outputs[index]);
1465 break;
1466 default:
1467 target = 0;
1468 fprintf(stderr,
1469 "Warning: SI unhandled fs output type:%d\n",
1470 semantic_name);
1471 }
1472
1473 si_llvm_init_export_args_load(bld_base,
1474 si_shader_ctx->radeon_bld.soa.outputs[index],
1475 target, args);
1476
1477 if (semantic_name == TGSI_SEMANTIC_COLOR) {
1478 /* If there is an export instruction waiting to be emitted, do so now. */
1479 if (last_args[0]) {
1480 lp_build_intrinsic(base->gallivm->builder,
1481 "llvm.SI.export",
1482 LLVMVoidTypeInContext(base->gallivm->context),
1483 last_args, 9);
1484 }
1485
1486 /* This instruction will be emitted at the end of the shader. */
1487 memcpy(last_args, args, sizeof(args));
1488
1489 /* Handle FS_COLOR0_WRITES_ALL_CBUFS. */
1490 if (shader->selector->info.properties[TGSI_PROPERTY_FS_COLOR0_WRITES_ALL_CBUFS] &&
1491 shader->output[i].sid == 0 &&
1492 si_shader_ctx->shader->key.ps.nr_cbufs > 1) {
1493 for (int c = 1; c < si_shader_ctx->shader->key.ps.nr_cbufs; c++) {
1494 si_llvm_init_export_args_load(bld_base,
1495 si_shader_ctx->radeon_bld.soa.outputs[index],
1496 V_008DFC_SQ_EXP_MRT + c, args);
1497 lp_build_intrinsic(base->gallivm->builder,
1498 "llvm.SI.export",
1499 LLVMVoidTypeInContext(base->gallivm->context),
1500 args, 9);
1501 }
1502 }
1503 } else {
1504 lp_build_intrinsic(base->gallivm->builder,
1505 "llvm.SI.export",
1506 LLVMVoidTypeInContext(base->gallivm->context),
1507 args, 9);
1508 }
1509 }
1510 }
1511
1512 if (depth_index >= 0 || stencil_index >= 0 || samplemask_index >= 0) {
1513 LLVMValueRef out_ptr;
1514 unsigned mask = 0;
1515
1516 /* Specify the target we are exporting */
1517 args[3] = lp_build_const_int32(base->gallivm, V_008DFC_SQ_EXP_MRTZ);
1518
1519 args[5] = base->zero; /* R, depth */
1520 args[6] = base->zero; /* G, stencil test value[0:7], stencil op value[8:15] */
1521 args[7] = base->zero; /* B, sample mask */
1522 args[8] = base->zero; /* A, alpha to mask */
1523
1524 if (depth_index >= 0) {
1525 out_ptr = si_shader_ctx->radeon_bld.soa.outputs[depth_index][2];
1526 args[5] = LLVMBuildLoad(base->gallivm->builder, out_ptr, "");
1527 mask |= 0x1;
1528 si_shader_ctx->shader->db_shader_control |= S_02880C_Z_EXPORT_ENABLE(1);
1529 }
1530
1531 if (stencil_index >= 0) {
1532 out_ptr = si_shader_ctx->radeon_bld.soa.outputs[stencil_index][1];
1533 args[6] = LLVMBuildLoad(base->gallivm->builder, out_ptr, "");
1534 /* Only setting the stencil component bit (0x2) here
1535 * breaks some stencil piglit tests
1536 */
1537 mask |= 0x3;
1538 si_shader_ctx->shader->db_shader_control |=
1539 S_02880C_STENCIL_TEST_VAL_EXPORT_ENABLE(1);
1540 }
1541
1542 if (samplemask_index >= 0) {
1543 out_ptr = si_shader_ctx->radeon_bld.soa.outputs[samplemask_index][0];
1544 args[7] = LLVMBuildLoad(base->gallivm->builder, out_ptr, "");
1545 mask |= 0xf; /* Set all components. */
1546 si_shader_ctx->shader->db_shader_control |= S_02880C_MASK_EXPORT_ENABLE(1);
1547 }
1548
1549 if (samplemask_index >= 0)
1550 si_shader_ctx->shader->spi_shader_z_format = V_028710_SPI_SHADER_32_ABGR;
1551 else if (stencil_index >= 0)
1552 si_shader_ctx->shader->spi_shader_z_format = V_028710_SPI_SHADER_32_GR;
1553 else
1554 si_shader_ctx->shader->spi_shader_z_format = V_028710_SPI_SHADER_32_R;
1555
1556 /* Specify which components to enable */
1557 args[0] = lp_build_const_int32(base->gallivm, mask);
1558
1559 args[1] =
1560 args[2] =
1561 args[4] = uint->zero;
1562
1563 if (last_args[0])
1564 lp_build_intrinsic(base->gallivm->builder,
1565 "llvm.SI.export",
1566 LLVMVoidTypeInContext(base->gallivm->context),
1567 args, 9);
1568 else
1569 memcpy(last_args, args, sizeof(args));
1570 }
1571
1572 if (!last_args[0]) {
1573 /* Specify which components to enable */
1574 last_args[0] = lp_build_const_int32(base->gallivm, 0x0);
1575
1576 /* Specify the target we are exporting */
1577 last_args[3] = lp_build_const_int32(base->gallivm, V_008DFC_SQ_EXP_MRT);
1578
1579 /* Set COMPR flag to zero to export data as 32-bit */
1580 last_args[4] = uint->zero;
1581
1582 /* dummy bits */
1583 last_args[5]= uint->zero;
1584 last_args[6]= uint->zero;
1585 last_args[7]= uint->zero;
1586 last_args[8]= uint->zero;
1587 }
1588
1589 /* Specify whether the EXEC mask represents the valid mask */
1590 last_args[1] = uint->one;
1591
1592 /* Specify that this is the last export */
1593 last_args[2] = lp_build_const_int32(base->gallivm, 1);
1594
1595 lp_build_intrinsic(base->gallivm->builder,
1596 "llvm.SI.export",
1597 LLVMVoidTypeInContext(base->gallivm->context),
1598 last_args, 9);
1599 }
1600
1601 static void build_tex_intrinsic(const struct lp_build_tgsi_action * action,
1602 struct lp_build_tgsi_context * bld_base,
1603 struct lp_build_emit_data * emit_data);
1604
1605 static bool tgsi_is_shadow_sampler(unsigned target)
1606 {
1607 return target == TGSI_TEXTURE_SHADOW1D ||
1608 target == TGSI_TEXTURE_SHADOW1D_ARRAY ||
1609 target == TGSI_TEXTURE_SHADOW2D ||
1610 target == TGSI_TEXTURE_SHADOW2D_ARRAY ||
1611 target == TGSI_TEXTURE_SHADOWCUBE ||
1612 target == TGSI_TEXTURE_SHADOWCUBE_ARRAY ||
1613 target == TGSI_TEXTURE_SHADOWRECT;
1614 }
1615
1616 static const struct lp_build_tgsi_action tex_action;
1617
1618 static void tex_fetch_args(
1619 struct lp_build_tgsi_context * bld_base,
1620 struct lp_build_emit_data * emit_data)
1621 {
1622 struct si_shader_context *si_shader_ctx = si_shader_context(bld_base);
1623 struct gallivm_state *gallivm = bld_base->base.gallivm;
1624 const struct tgsi_full_instruction * inst = emit_data->inst;
1625 unsigned opcode = inst->Instruction.Opcode;
1626 unsigned target = inst->Texture.Texture;
1627 LLVMValueRef coords[4];
1628 LLVMValueRef address[16];
1629 int ref_pos;
1630 unsigned num_coords = tgsi_util_get_texture_coord_dim(target, &ref_pos);
1631 unsigned count = 0;
1632 unsigned chan;
1633 unsigned sampler_src = emit_data->inst->Instruction.NumSrcRegs - 1;
1634 unsigned sampler_index = emit_data->inst->Src[sampler_src].Register.Index;
1635 bool has_offset = HAVE_LLVM >= 0x0305 ? inst->Texture.NumOffsets > 0 : false;
1636
1637 if (target == TGSI_TEXTURE_BUFFER) {
1638 LLVMTypeRef i128 = LLVMIntTypeInContext(gallivm->context, 128);
1639 LLVMTypeRef v2i128 = LLVMVectorType(i128, 2);
1640 LLVMTypeRef i8 = LLVMInt8TypeInContext(gallivm->context);
1641 LLVMTypeRef v16i8 = LLVMVectorType(i8, 16);
1642
1643 /* Bitcast and truncate v8i32 to v16i8. */
1644 LLVMValueRef res = si_shader_ctx->resources[sampler_index];
1645 res = LLVMBuildBitCast(gallivm->builder, res, v2i128, "");
1646 res = LLVMBuildExtractElement(gallivm->builder, res, bld_base->uint_bld.zero, "");
1647 res = LLVMBuildBitCast(gallivm->builder, res, v16i8, "");
1648
1649 emit_data->dst_type = LLVMVectorType(bld_base->base.elem_type, 4);
1650 emit_data->args[0] = res;
1651 emit_data->args[1] = bld_base->uint_bld.zero;
1652 emit_data->args[2] = lp_build_emit_fetch(bld_base, emit_data->inst, 0, 0);
1653 emit_data->arg_count = 3;
1654 return;
1655 }
1656
1657 /* Fetch and project texture coordinates */
1658 coords[3] = lp_build_emit_fetch(bld_base, emit_data->inst, 0, TGSI_CHAN_W);
1659 for (chan = 0; chan < 3; chan++ ) {
1660 coords[chan] = lp_build_emit_fetch(bld_base,
1661 emit_data->inst, 0,
1662 chan);
1663 if (opcode == TGSI_OPCODE_TXP)
1664 coords[chan] = lp_build_emit_llvm_binary(bld_base,
1665 TGSI_OPCODE_DIV,
1666 coords[chan],
1667 coords[3]);
1668 }
1669
1670 if (opcode == TGSI_OPCODE_TXP)
1671 coords[3] = bld_base->base.one;
1672
1673 /* Pack offsets. */
1674 if (has_offset && opcode != TGSI_OPCODE_TXF) {
1675 /* The offsets are six-bit signed integers packed like this:
1676 * X=[5:0], Y=[13:8], and Z=[21:16].
1677 */
1678 LLVMValueRef offset[3], pack;
1679
1680 assert(inst->Texture.NumOffsets == 1);
1681
1682 for (chan = 0; chan < 3; chan++) {
1683 offset[chan] = lp_build_emit_fetch_texoffset(bld_base,
1684 emit_data->inst, 0, chan);
1685 offset[chan] = LLVMBuildAnd(gallivm->builder, offset[chan],
1686 lp_build_const_int32(gallivm, 0x3f), "");
1687 if (chan)
1688 offset[chan] = LLVMBuildShl(gallivm->builder, offset[chan],
1689 lp_build_const_int32(gallivm, chan*8), "");
1690 }
1691
1692 pack = LLVMBuildOr(gallivm->builder, offset[0], offset[1], "");
1693 pack = LLVMBuildOr(gallivm->builder, pack, offset[2], "");
1694 address[count++] = pack;
1695 }
1696
1697 /* Pack LOD bias value */
1698 if (opcode == TGSI_OPCODE_TXB)
1699 address[count++] = coords[3];
1700 if (opcode == TGSI_OPCODE_TXB2)
1701 address[count++] = lp_build_emit_fetch(bld_base, inst, 1, 0);
1702
1703 /* Pack depth comparison value */
1704 if (tgsi_is_shadow_sampler(target) && opcode != TGSI_OPCODE_LODQ) {
1705 if (target == TGSI_TEXTURE_SHADOWCUBE_ARRAY) {
1706 address[count++] = lp_build_emit_fetch(bld_base, inst, 1, 0);
1707 } else {
1708 assert(ref_pos >= 0);
1709 address[count++] = coords[ref_pos];
1710 }
1711 }
1712
1713 if (target == TGSI_TEXTURE_CUBE ||
1714 target == TGSI_TEXTURE_CUBE_ARRAY ||
1715 target == TGSI_TEXTURE_SHADOWCUBE ||
1716 target == TGSI_TEXTURE_SHADOWCUBE_ARRAY)
1717 radeon_llvm_emit_prepare_cube_coords(bld_base, emit_data, coords);
1718
1719 /* Pack user derivatives */
1720 if (opcode == TGSI_OPCODE_TXD) {
1721 int num_deriv_channels, param;
1722
1723 switch (target) {
1724 case TGSI_TEXTURE_3D:
1725 num_deriv_channels = 3;
1726 break;
1727 case TGSI_TEXTURE_2D:
1728 case TGSI_TEXTURE_SHADOW2D:
1729 case TGSI_TEXTURE_RECT:
1730 case TGSI_TEXTURE_SHADOWRECT:
1731 case TGSI_TEXTURE_2D_ARRAY:
1732 case TGSI_TEXTURE_SHADOW2D_ARRAY:
1733 case TGSI_TEXTURE_CUBE:
1734 case TGSI_TEXTURE_SHADOWCUBE:
1735 case TGSI_TEXTURE_CUBE_ARRAY:
1736 case TGSI_TEXTURE_SHADOWCUBE_ARRAY:
1737 num_deriv_channels = 2;
1738 break;
1739 case TGSI_TEXTURE_1D:
1740 case TGSI_TEXTURE_SHADOW1D:
1741 case TGSI_TEXTURE_1D_ARRAY:
1742 case TGSI_TEXTURE_SHADOW1D_ARRAY:
1743 num_deriv_channels = 1;
1744 break;
1745 default:
1746 assert(0); /* no other targets are valid here */
1747 }
1748
1749 for (param = 1; param <= 2; param++)
1750 for (chan = 0; chan < num_deriv_channels; chan++)
1751 address[count++] = lp_build_emit_fetch(bld_base, inst, param, chan);
1752 }
1753
1754 /* Pack texture coordinates */
1755 address[count++] = coords[0];
1756 if (num_coords > 1)
1757 address[count++] = coords[1];
1758 if (num_coords > 2)
1759 address[count++] = coords[2];
1760
1761 /* Pack LOD or sample index */
1762 if (opcode == TGSI_OPCODE_TXL || opcode == TGSI_OPCODE_TXF)
1763 address[count++] = coords[3];
1764 else if (opcode == TGSI_OPCODE_TXL2)
1765 address[count++] = lp_build_emit_fetch(bld_base, inst, 1, 0);
1766
1767 if (count > 16) {
1768 assert(!"Cannot handle more than 16 texture address parameters");
1769 count = 16;
1770 }
1771
1772 for (chan = 0; chan < count; chan++ ) {
1773 address[chan] = LLVMBuildBitCast(gallivm->builder,
1774 address[chan],
1775 LLVMInt32TypeInContext(gallivm->context),
1776 "");
1777 }
1778
1779 /* Adjust the sample index according to FMASK.
1780 *
1781 * For uncompressed MSAA surfaces, FMASK should return 0x76543210,
1782 * which is the identity mapping. Each nibble says which physical sample
1783 * should be fetched to get that sample.
1784 *
1785 * For example, 0x11111100 means there are only 2 samples stored and
1786 * the second sample covers 3/4 of the pixel. When reading samples 0
1787 * and 1, return physical sample 0 (determined by the first two 0s
1788 * in FMASK), otherwise return physical sample 1.
1789 *
1790 * The sample index should be adjusted as follows:
1791 * sample_index = (fmask >> (sample_index * 4)) & 0xF;
1792 */
1793 if (target == TGSI_TEXTURE_2D_MSAA ||
1794 target == TGSI_TEXTURE_2D_ARRAY_MSAA) {
1795 struct lp_build_context *uint_bld = &bld_base->uint_bld;
1796 struct lp_build_emit_data txf_emit_data = *emit_data;
1797 LLVMValueRef txf_address[4];
1798 unsigned txf_count = count;
1799 struct tgsi_full_instruction inst = {};
1800
1801 memcpy(txf_address, address, sizeof(txf_address));
1802
1803 if (target == TGSI_TEXTURE_2D_MSAA) {
1804 txf_address[2] = bld_base->uint_bld.zero;
1805 }
1806 txf_address[3] = bld_base->uint_bld.zero;
1807
1808 /* Pad to a power-of-two size. */
1809 while (txf_count < util_next_power_of_two(txf_count))
1810 txf_address[txf_count++] = LLVMGetUndef(LLVMInt32TypeInContext(gallivm->context));
1811
1812 /* Read FMASK using TXF. */
1813 inst.Instruction.Opcode = TGSI_OPCODE_TXF;
1814 inst.Texture.Texture = target == TGSI_TEXTURE_2D_MSAA ? TGSI_TEXTURE_2D : TGSI_TEXTURE_2D_ARRAY;
1815 txf_emit_data.inst = &inst;
1816 txf_emit_data.chan = 0;
1817 txf_emit_data.dst_type = LLVMVectorType(
1818 LLVMInt32TypeInContext(gallivm->context), 4);
1819 txf_emit_data.args[0] = lp_build_gather_values(gallivm, txf_address, txf_count);
1820 txf_emit_data.args[1] = si_shader_ctx->resources[SI_FMASK_TEX_OFFSET + sampler_index];
1821 txf_emit_data.args[2] = lp_build_const_int32(gallivm, inst.Texture.Texture);
1822 txf_emit_data.arg_count = 3;
1823
1824 build_tex_intrinsic(&tex_action, bld_base, &txf_emit_data);
1825
1826 /* Initialize some constants. */
1827 LLVMValueRef four = LLVMConstInt(uint_bld->elem_type, 4, 0);
1828 LLVMValueRef F = LLVMConstInt(uint_bld->elem_type, 0xF, 0);
1829
1830 /* Apply the formula. */
1831 LLVMValueRef fmask =
1832 LLVMBuildExtractElement(gallivm->builder,
1833 txf_emit_data.output[0],
1834 uint_bld->zero, "");
1835
1836 unsigned sample_chan = target == TGSI_TEXTURE_2D_MSAA ? 2 : 3;
1837
1838 LLVMValueRef sample_index4 =
1839 LLVMBuildMul(gallivm->builder, address[sample_chan], four, "");
1840
1841 LLVMValueRef shifted_fmask =
1842 LLVMBuildLShr(gallivm->builder, fmask, sample_index4, "");
1843
1844 LLVMValueRef final_sample =
1845 LLVMBuildAnd(gallivm->builder, shifted_fmask, F, "");
1846
1847 /* Don't rewrite the sample index if WORD1.DATA_FORMAT of the FMASK
1848 * resource descriptor is 0 (invalid),
1849 */
1850 LLVMValueRef fmask_desc =
1851 LLVMBuildBitCast(gallivm->builder,
1852 si_shader_ctx->resources[SI_FMASK_TEX_OFFSET + sampler_index],
1853 LLVMVectorType(uint_bld->elem_type, 8), "");
1854
1855 LLVMValueRef fmask_word1 =
1856 LLVMBuildExtractElement(gallivm->builder, fmask_desc,
1857 uint_bld->one, "");
1858
1859 LLVMValueRef word1_is_nonzero =
1860 LLVMBuildICmp(gallivm->builder, LLVMIntNE,
1861 fmask_word1, uint_bld->zero, "");
1862
1863 /* Replace the MSAA sample index. */
1864 address[sample_chan] =
1865 LLVMBuildSelect(gallivm->builder, word1_is_nonzero,
1866 final_sample, address[sample_chan], "");
1867 }
1868
1869 /* Resource */
1870 emit_data->args[1] = si_shader_ctx->resources[sampler_index];
1871
1872 if (opcode == TGSI_OPCODE_TXF) {
1873 /* add tex offsets */
1874 if (inst->Texture.NumOffsets) {
1875 struct lp_build_context *uint_bld = &bld_base->uint_bld;
1876 struct lp_build_tgsi_soa_context *bld = lp_soa_context(bld_base);
1877 const struct tgsi_texture_offset * off = inst->TexOffsets;
1878
1879 assert(inst->Texture.NumOffsets == 1);
1880
1881 switch (target) {
1882 case TGSI_TEXTURE_3D:
1883 address[2] = lp_build_add(uint_bld, address[2],
1884 bld->immediates[off->Index][off->SwizzleZ]);
1885 /* fall through */
1886 case TGSI_TEXTURE_2D:
1887 case TGSI_TEXTURE_SHADOW2D:
1888 case TGSI_TEXTURE_RECT:
1889 case TGSI_TEXTURE_SHADOWRECT:
1890 case TGSI_TEXTURE_2D_ARRAY:
1891 case TGSI_TEXTURE_SHADOW2D_ARRAY:
1892 address[1] =
1893 lp_build_add(uint_bld, address[1],
1894 bld->immediates[off->Index][off->SwizzleY]);
1895 /* fall through */
1896 case TGSI_TEXTURE_1D:
1897 case TGSI_TEXTURE_SHADOW1D:
1898 case TGSI_TEXTURE_1D_ARRAY:
1899 case TGSI_TEXTURE_SHADOW1D_ARRAY:
1900 address[0] =
1901 lp_build_add(uint_bld, address[0],
1902 bld->immediates[off->Index][off->SwizzleX]);
1903 break;
1904 /* texture offsets do not apply to other texture targets */
1905 }
1906 }
1907
1908 emit_data->args[2] = lp_build_const_int32(gallivm, target);
1909 emit_data->arg_count = 3;
1910
1911 emit_data->dst_type = LLVMVectorType(
1912 LLVMInt32TypeInContext(gallivm->context),
1913 4);
1914 } else if (opcode == TGSI_OPCODE_TG4 ||
1915 opcode == TGSI_OPCODE_LODQ ||
1916 has_offset) {
1917 unsigned is_array = target == TGSI_TEXTURE_1D_ARRAY ||
1918 target == TGSI_TEXTURE_SHADOW1D_ARRAY ||
1919 target == TGSI_TEXTURE_2D_ARRAY ||
1920 target == TGSI_TEXTURE_SHADOW2D_ARRAY ||
1921 target == TGSI_TEXTURE_CUBE_ARRAY ||
1922 target == TGSI_TEXTURE_SHADOWCUBE_ARRAY;
1923 unsigned is_rect = target == TGSI_TEXTURE_RECT;
1924 unsigned dmask = 0xf;
1925
1926 if (opcode == TGSI_OPCODE_TG4) {
1927 unsigned gather_comp = 0;
1928
1929 /* DMASK was repurposed for GATHER4. 4 components are always
1930 * returned and DMASK works like a swizzle - it selects
1931 * the component to fetch. The only valid DMASK values are
1932 * 1=red, 2=green, 4=blue, 8=alpha. (e.g. 1 returns
1933 * (red,red,red,red) etc.) The ISA document doesn't mention
1934 * this.
1935 */
1936
1937 /* Get the component index from src1.x for Gather4. */
1938 if (!tgsi_is_shadow_sampler(target)) {
1939 LLVMValueRef (*imms)[4] = lp_soa_context(bld_base)->immediates;
1940 LLVMValueRef comp_imm;
1941 struct tgsi_src_register src1 = inst->Src[1].Register;
1942
1943 assert(src1.File == TGSI_FILE_IMMEDIATE);
1944
1945 comp_imm = imms[src1.Index][src1.SwizzleX];
1946 gather_comp = LLVMConstIntGetZExtValue(comp_imm);
1947 gather_comp = CLAMP(gather_comp, 0, 3);
1948 }
1949
1950 dmask = 1 << gather_comp;
1951 }
1952
1953 emit_data->args[2] = si_shader_ctx->samplers[sampler_index];
1954 emit_data->args[3] = lp_build_const_int32(gallivm, dmask);
1955 emit_data->args[4] = lp_build_const_int32(gallivm, is_rect); /* unorm */
1956 emit_data->args[5] = lp_build_const_int32(gallivm, 0); /* r128 */
1957 emit_data->args[6] = lp_build_const_int32(gallivm, is_array); /* da */
1958 emit_data->args[7] = lp_build_const_int32(gallivm, 0); /* glc */
1959 emit_data->args[8] = lp_build_const_int32(gallivm, 0); /* slc */
1960 emit_data->args[9] = lp_build_const_int32(gallivm, 0); /* tfe */
1961 emit_data->args[10] = lp_build_const_int32(gallivm, 0); /* lwe */
1962
1963 emit_data->arg_count = 11;
1964
1965 emit_data->dst_type = LLVMVectorType(
1966 LLVMFloatTypeInContext(gallivm->context),
1967 4);
1968 } else {
1969 emit_data->args[2] = si_shader_ctx->samplers[sampler_index];
1970 emit_data->args[3] = lp_build_const_int32(gallivm, target);
1971 emit_data->arg_count = 4;
1972
1973 emit_data->dst_type = LLVMVectorType(
1974 LLVMFloatTypeInContext(gallivm->context),
1975 4);
1976 }
1977
1978 /* The fetch opcode has been converted to a 2D array fetch.
1979 * This simplifies the LLVM backend. */
1980 if (target == TGSI_TEXTURE_CUBE_ARRAY)
1981 target = TGSI_TEXTURE_2D_ARRAY;
1982 else if (target == TGSI_TEXTURE_SHADOWCUBE_ARRAY)
1983 target = TGSI_TEXTURE_SHADOW2D_ARRAY;
1984
1985 /* Pad to power of two vector */
1986 while (count < util_next_power_of_two(count))
1987 address[count++] = LLVMGetUndef(LLVMInt32TypeInContext(gallivm->context));
1988
1989 emit_data->args[0] = lp_build_gather_values(gallivm, address, count);
1990 }
1991
1992 static void build_tex_intrinsic(const struct lp_build_tgsi_action * action,
1993 struct lp_build_tgsi_context * bld_base,
1994 struct lp_build_emit_data * emit_data)
1995 {
1996 struct lp_build_context * base = &bld_base->base;
1997 unsigned opcode = emit_data->inst->Instruction.Opcode;
1998 unsigned target = emit_data->inst->Texture.Texture;
1999 char intr_name[127];
2000 bool has_offset = HAVE_LLVM >= 0x0305 ?
2001 emit_data->inst->Texture.NumOffsets > 0 : false;
2002
2003 if (target == TGSI_TEXTURE_BUFFER) {
2004 emit_data->output[emit_data->chan] = build_intrinsic(
2005 base->gallivm->builder,
2006 "llvm.SI.vs.load.input", emit_data->dst_type,
2007 emit_data->args, emit_data->arg_count,
2008 LLVMReadNoneAttribute | LLVMNoUnwindAttribute);
2009 return;
2010 }
2011
2012 if (opcode == TGSI_OPCODE_TG4 ||
2013 opcode == TGSI_OPCODE_LODQ ||
2014 (opcode != TGSI_OPCODE_TXF && has_offset)) {
2015 bool is_shadow = tgsi_is_shadow_sampler(target);
2016 const char *name = "llvm.SI.image.sample";
2017 const char *infix = "";
2018
2019 switch (opcode) {
2020 case TGSI_OPCODE_TEX:
2021 case TGSI_OPCODE_TEX2:
2022 case TGSI_OPCODE_TXP:
2023 break;
2024 case TGSI_OPCODE_TXB:
2025 case TGSI_OPCODE_TXB2:
2026 infix = ".b";
2027 break;
2028 case TGSI_OPCODE_TXL:
2029 case TGSI_OPCODE_TXL2:
2030 infix = ".l";
2031 break;
2032 case TGSI_OPCODE_TXD:
2033 infix = ".d";
2034 break;
2035 case TGSI_OPCODE_TG4:
2036 name = "llvm.SI.gather4";
2037 break;
2038 case TGSI_OPCODE_LODQ:
2039 name = "llvm.SI.getlod";
2040 is_shadow = false;
2041 has_offset = false;
2042 break;
2043 default:
2044 assert(0);
2045 return;
2046 }
2047
2048 /* Add the type and suffixes .c, .o if needed. */
2049 sprintf(intr_name, "%s%s%s%s.v%ui32", name,
2050 is_shadow ? ".c" : "", infix, has_offset ? ".o" : "",
2051 LLVMGetVectorSize(LLVMTypeOf(emit_data->args[0])));
2052
2053 emit_data->output[emit_data->chan] = build_intrinsic(
2054 base->gallivm->builder, intr_name, emit_data->dst_type,
2055 emit_data->args, emit_data->arg_count,
2056 LLVMReadNoneAttribute | LLVMNoUnwindAttribute);
2057 } else {
2058 LLVMTypeRef i8, v16i8, v32i8;
2059 const char *name;
2060
2061 switch (opcode) {
2062 case TGSI_OPCODE_TEX:
2063 case TGSI_OPCODE_TEX2:
2064 case TGSI_OPCODE_TXP:
2065 name = "llvm.SI.sample";
2066 break;
2067 case TGSI_OPCODE_TXB:
2068 case TGSI_OPCODE_TXB2:
2069 name = "llvm.SI.sampleb";
2070 break;
2071 case TGSI_OPCODE_TXD:
2072 name = "llvm.SI.sampled";
2073 break;
2074 case TGSI_OPCODE_TXF:
2075 name = "llvm.SI.imageload";
2076 break;
2077 case TGSI_OPCODE_TXL:
2078 case TGSI_OPCODE_TXL2:
2079 name = "llvm.SI.samplel";
2080 break;
2081 default:
2082 assert(0);
2083 return;
2084 }
2085
2086 i8 = LLVMInt8TypeInContext(base->gallivm->context);
2087 v16i8 = LLVMVectorType(i8, 16);
2088 v32i8 = LLVMVectorType(i8, 32);
2089
2090 emit_data->args[1] = LLVMBuildBitCast(base->gallivm->builder,
2091 emit_data->args[1], v32i8, "");
2092 if (opcode != TGSI_OPCODE_TXF) {
2093 emit_data->args[2] = LLVMBuildBitCast(base->gallivm->builder,
2094 emit_data->args[2], v16i8, "");
2095 }
2096
2097 sprintf(intr_name, "%s.v%ui32", name,
2098 LLVMGetVectorSize(LLVMTypeOf(emit_data->args[0])));
2099
2100 emit_data->output[emit_data->chan] = build_intrinsic(
2101 base->gallivm->builder, intr_name, emit_data->dst_type,
2102 emit_data->args, emit_data->arg_count,
2103 LLVMReadNoneAttribute | LLVMNoUnwindAttribute);
2104 }
2105 }
2106
2107 static void txq_fetch_args(
2108 struct lp_build_tgsi_context * bld_base,
2109 struct lp_build_emit_data * emit_data)
2110 {
2111 struct si_shader_context *si_shader_ctx = si_shader_context(bld_base);
2112 const struct tgsi_full_instruction *inst = emit_data->inst;
2113 struct gallivm_state *gallivm = bld_base->base.gallivm;
2114 unsigned target = inst->Texture.Texture;
2115
2116 if (target == TGSI_TEXTURE_BUFFER) {
2117 LLVMTypeRef i32 = LLVMInt32TypeInContext(gallivm->context);
2118 LLVMTypeRef v8i32 = LLVMVectorType(i32, 8);
2119
2120 /* Read the size from the buffer descriptor directly. */
2121 LLVMValueRef size = si_shader_ctx->resources[inst->Src[1].Register.Index];
2122 size = LLVMBuildBitCast(gallivm->builder, size, v8i32, "");
2123 size = LLVMBuildExtractElement(gallivm->builder, size,
2124 lp_build_const_int32(gallivm, 2), "");
2125 emit_data->args[0] = size;
2126 return;
2127 }
2128
2129 /* Mip level */
2130 emit_data->args[0] = lp_build_emit_fetch(bld_base, inst, 0, TGSI_CHAN_X);
2131
2132 /* Resource */
2133 emit_data->args[1] = si_shader_ctx->resources[inst->Src[1].Register.Index];
2134
2135 /* Texture target */
2136 if (target == TGSI_TEXTURE_CUBE_ARRAY ||
2137 target == TGSI_TEXTURE_SHADOWCUBE_ARRAY)
2138 target = TGSI_TEXTURE_2D_ARRAY;
2139
2140 emit_data->args[2] = lp_build_const_int32(bld_base->base.gallivm,
2141 target);
2142
2143 emit_data->arg_count = 3;
2144
2145 emit_data->dst_type = LLVMVectorType(
2146 LLVMInt32TypeInContext(bld_base->base.gallivm->context),
2147 4);
2148 }
2149
2150 static void build_txq_intrinsic(const struct lp_build_tgsi_action * action,
2151 struct lp_build_tgsi_context * bld_base,
2152 struct lp_build_emit_data * emit_data)
2153 {
2154 unsigned target = emit_data->inst->Texture.Texture;
2155
2156 if (target == TGSI_TEXTURE_BUFFER) {
2157 /* Just return the buffer size. */
2158 emit_data->output[emit_data->chan] = emit_data->args[0];
2159 return;
2160 }
2161
2162 build_tgsi_intrinsic_nomem(action, bld_base, emit_data);
2163
2164 /* Divide the number of layers by 6 to get the number of cubes. */
2165 if (target == TGSI_TEXTURE_CUBE_ARRAY ||
2166 target == TGSI_TEXTURE_SHADOWCUBE_ARRAY) {
2167 LLVMBuilderRef builder = bld_base->base.gallivm->builder;
2168 LLVMValueRef two = lp_build_const_int32(bld_base->base.gallivm, 2);
2169 LLVMValueRef six = lp_build_const_int32(bld_base->base.gallivm, 6);
2170
2171 LLVMValueRef v4 = emit_data->output[emit_data->chan];
2172 LLVMValueRef z = LLVMBuildExtractElement(builder, v4, two, "");
2173 z = LLVMBuildSDiv(builder, z, six, "");
2174
2175 emit_data->output[emit_data->chan] =
2176 LLVMBuildInsertElement(builder, v4, z, two, "");
2177 }
2178 }
2179
2180 static void si_llvm_emit_ddxy(
2181 const struct lp_build_tgsi_action * action,
2182 struct lp_build_tgsi_context * bld_base,
2183 struct lp_build_emit_data * emit_data)
2184 {
2185 struct si_shader_context *si_shader_ctx = si_shader_context(bld_base);
2186 struct gallivm_state *gallivm = bld_base->base.gallivm;
2187 struct lp_build_context * base = &bld_base->base;
2188 const struct tgsi_full_instruction *inst = emit_data->inst;
2189 unsigned opcode = inst->Instruction.Opcode;
2190 LLVMValueRef indices[2];
2191 LLVMValueRef store_ptr, load_ptr0, load_ptr1;
2192 LLVMValueRef tl, trbl, result[4];
2193 LLVMTypeRef i32;
2194 unsigned swizzle[4];
2195 unsigned c;
2196
2197 i32 = LLVMInt32TypeInContext(gallivm->context);
2198
2199 indices[0] = bld_base->uint_bld.zero;
2200 indices[1] = build_intrinsic(gallivm->builder, "llvm.SI.tid", i32,
2201 NULL, 0, LLVMReadNoneAttribute);
2202 store_ptr = LLVMBuildGEP(gallivm->builder, si_shader_ctx->ddxy_lds,
2203 indices, 2, "");
2204
2205 indices[1] = LLVMBuildAnd(gallivm->builder, indices[1],
2206 lp_build_const_int32(gallivm, 0xfffffffc), "");
2207 load_ptr0 = LLVMBuildGEP(gallivm->builder, si_shader_ctx->ddxy_lds,
2208 indices, 2, "");
2209
2210 indices[1] = LLVMBuildAdd(gallivm->builder, indices[1],
2211 lp_build_const_int32(gallivm,
2212 opcode == TGSI_OPCODE_DDX ? 1 : 2),
2213 "");
2214 load_ptr1 = LLVMBuildGEP(gallivm->builder, si_shader_ctx->ddxy_lds,
2215 indices, 2, "");
2216
2217 for (c = 0; c < 4; ++c) {
2218 unsigned i;
2219
2220 swizzle[c] = tgsi_util_get_full_src_register_swizzle(&inst->Src[0], c);
2221 for (i = 0; i < c; ++i) {
2222 if (swizzle[i] == swizzle[c]) {
2223 result[c] = result[i];
2224 break;
2225 }
2226 }
2227 if (i != c)
2228 continue;
2229
2230 LLVMBuildStore(gallivm->builder,
2231 LLVMBuildBitCast(gallivm->builder,
2232 lp_build_emit_fetch(bld_base, inst, 0, c),
2233 i32, ""),
2234 store_ptr);
2235
2236 tl = LLVMBuildLoad(gallivm->builder, load_ptr0, "");
2237 tl = LLVMBuildBitCast(gallivm->builder, tl, base->elem_type, "");
2238
2239 trbl = LLVMBuildLoad(gallivm->builder, load_ptr1, "");
2240 trbl = LLVMBuildBitCast(gallivm->builder, trbl, base->elem_type, "");
2241
2242 result[c] = LLVMBuildFSub(gallivm->builder, trbl, tl, "");
2243 }
2244
2245 emit_data->output[0] = lp_build_gather_values(gallivm, result, 4);
2246 }
2247
2248 /* Emit one vertex from the geometry shader */
2249 static void si_llvm_emit_vertex(
2250 const struct lp_build_tgsi_action *action,
2251 struct lp_build_tgsi_context *bld_base,
2252 struct lp_build_emit_data *emit_data)
2253 {
2254 struct si_shader_context *si_shader_ctx = si_shader_context(bld_base);
2255 struct lp_build_context *uint = &bld_base->uint_bld;
2256 struct si_shader *shader = si_shader_ctx->shader;
2257 struct gallivm_state *gallivm = bld_base->base.gallivm;
2258 LLVMTypeRef i32 = LLVMInt32TypeInContext(gallivm->context);
2259 LLVMValueRef soffset = LLVMGetParam(si_shader_ctx->radeon_bld.main_fn,
2260 SI_PARAM_GS2VS_OFFSET);
2261 LLVMValueRef gs_next_vertex;
2262 LLVMValueRef can_emit, kill;
2263 LLVMValueRef t_list_ptr;
2264 LLVMValueRef t_list;
2265 LLVMValueRef args[2];
2266 unsigned chan;
2267 int i;
2268
2269 /* Load the GSVS ring resource descriptor */
2270 t_list_ptr = LLVMGetParam(si_shader_ctx->radeon_bld.main_fn,
2271 SI_PARAM_RW_BUFFERS);
2272 t_list = build_indexed_load(si_shader_ctx, t_list_ptr,
2273 lp_build_const_int32(gallivm, SI_RING_GSVS));
2274
2275 if (shader->noutput == 0) {
2276 struct tgsi_parse_context *parse = &si_shader_ctx->parse;
2277
2278 while (!tgsi_parse_end_of_tokens(parse)) {
2279 tgsi_parse_token(parse);
2280
2281 if (parse->FullToken.Token.Type == TGSI_TOKEN_TYPE_DECLARATION) {
2282 struct tgsi_full_declaration *d = &parse->FullToken.FullDeclaration;
2283
2284 if (d->Declaration.File == TGSI_FILE_OUTPUT)
2285 si_store_shader_io_attribs(shader, d);
2286 }
2287 }
2288 }
2289
2290 /* Write vertex attribute values to GSVS ring */
2291 gs_next_vertex = LLVMBuildLoad(gallivm->builder, si_shader_ctx->gs_next_vertex, "");
2292
2293 /* If this thread has already emitted the declared maximum number of
2294 * vertices, kill it: excessive vertex emissions are not supposed to
2295 * have any effect, and GS threads have no externally observable
2296 * effects other than emitting vertices.
2297 */
2298 can_emit = LLVMBuildICmp(gallivm->builder, LLVMIntULE, gs_next_vertex,
2299 lp_build_const_int32(gallivm,
2300 shader->selector->gs_max_out_vertices), "");
2301 kill = lp_build_select(&bld_base->base, can_emit,
2302 lp_build_const_float(gallivm, 1.0f),
2303 lp_build_const_float(gallivm, -1.0f));
2304 build_intrinsic(gallivm->builder, "llvm.AMDGPU.kill",
2305 LLVMVoidTypeInContext(gallivm->context), &kill, 1, 0);
2306
2307 for (i = 0; i < shader->noutput; i++) {
2308 LLVMValueRef *out_ptr =
2309 si_shader_ctx->radeon_bld.soa.outputs[shader->output[i].index];
2310
2311 for (chan = 0; chan < 4; chan++) {
2312 LLVMValueRef out_val = LLVMBuildLoad(gallivm->builder, out_ptr[chan], "");
2313 LLVMValueRef voffset =
2314 lp_build_const_int32(gallivm, (i * 4 + chan) *
2315 shader->selector->gs_max_out_vertices);
2316
2317 voffset = lp_build_add(uint, voffset, gs_next_vertex);
2318 voffset = lp_build_mul_imm(uint, voffset, 4);
2319
2320 out_val = LLVMBuildBitCast(gallivm->builder, out_val, i32, "");
2321
2322 build_tbuffer_store(si_shader_ctx, t_list, out_val, 1,
2323 voffset, soffset, 0,
2324 V_008F0C_BUF_DATA_FORMAT_32,
2325 V_008F0C_BUF_NUM_FORMAT_UINT,
2326 1, 0, 1, 1, 0);
2327 }
2328 }
2329 gs_next_vertex = lp_build_add(uint, gs_next_vertex,
2330 lp_build_const_int32(gallivm, 1));
2331 LLVMBuildStore(gallivm->builder, gs_next_vertex, si_shader_ctx->gs_next_vertex);
2332
2333 /* Signal vertex emission */
2334 args[0] = lp_build_const_int32(gallivm, SENDMSG_GS_OP_EMIT | SENDMSG_GS);
2335 args[1] = LLVMGetParam(si_shader_ctx->radeon_bld.main_fn, SI_PARAM_GS_WAVE_ID);
2336 build_intrinsic(gallivm->builder, "llvm.SI.sendmsg",
2337 LLVMVoidTypeInContext(gallivm->context), args, 2,
2338 LLVMNoUnwindAttribute);
2339 }
2340
2341 /* Cut one primitive from the geometry shader */
2342 static void si_llvm_emit_primitive(
2343 const struct lp_build_tgsi_action *action,
2344 struct lp_build_tgsi_context *bld_base,
2345 struct lp_build_emit_data *emit_data)
2346 {
2347 struct si_shader_context *si_shader_ctx = si_shader_context(bld_base);
2348 struct gallivm_state *gallivm = bld_base->base.gallivm;
2349 LLVMValueRef args[2];
2350
2351 /* Signal primitive cut */
2352 args[0] = lp_build_const_int32(gallivm, SENDMSG_GS_OP_CUT | SENDMSG_GS);
2353 args[1] = LLVMGetParam(si_shader_ctx->radeon_bld.main_fn, SI_PARAM_GS_WAVE_ID);
2354 build_intrinsic(gallivm->builder, "llvm.SI.sendmsg",
2355 LLVMVoidTypeInContext(gallivm->context), args, 2,
2356 LLVMNoUnwindAttribute);
2357 }
2358
2359 static const struct lp_build_tgsi_action tex_action = {
2360 .fetch_args = tex_fetch_args,
2361 .emit = build_tex_intrinsic,
2362 };
2363
2364 static const struct lp_build_tgsi_action txq_action = {
2365 .fetch_args = txq_fetch_args,
2366 .emit = build_txq_intrinsic,
2367 .intr_name = "llvm.SI.resinfo"
2368 };
2369
2370 static void create_meta_data(struct si_shader_context *si_shader_ctx)
2371 {
2372 struct gallivm_state *gallivm = si_shader_ctx->radeon_bld.soa.bld_base.base.gallivm;
2373 LLVMValueRef args[3];
2374
2375 args[0] = LLVMMDStringInContext(gallivm->context, "const", 5);
2376 args[1] = 0;
2377 args[2] = lp_build_const_int32(gallivm, 1);
2378
2379 si_shader_ctx->const_md = LLVMMDNodeInContext(gallivm->context, args, 3);
2380 }
2381
2382 static LLVMTypeRef const_array(LLVMTypeRef elem_type, int num_elements)
2383 {
2384 return LLVMPointerType(LLVMArrayType(elem_type, num_elements),
2385 CONST_ADDR_SPACE);
2386 }
2387
2388 static void create_function(struct si_shader_context *si_shader_ctx)
2389 {
2390 struct lp_build_tgsi_context *bld_base = &si_shader_ctx->radeon_bld.soa.bld_base;
2391 struct gallivm_state *gallivm = bld_base->base.gallivm;
2392 struct si_shader *shader = si_shader_ctx->shader;
2393 LLVMTypeRef params[SI_NUM_PARAMS], f32, i8, i32, v2i32, v3i32, v16i8, v4i32, v8i32;
2394 unsigned i, last_array_pointer, last_sgpr, num_params;
2395
2396 i8 = LLVMInt8TypeInContext(gallivm->context);
2397 i32 = LLVMInt32TypeInContext(gallivm->context);
2398 f32 = LLVMFloatTypeInContext(gallivm->context);
2399 v2i32 = LLVMVectorType(i32, 2);
2400 v3i32 = LLVMVectorType(i32, 3);
2401 v4i32 = LLVMVectorType(i32, 4);
2402 v8i32 = LLVMVectorType(i32, 8);
2403 v16i8 = LLVMVectorType(i8, 16);
2404
2405 params[SI_PARAM_RW_BUFFERS] = const_array(v16i8, SI_NUM_RW_BUFFERS);
2406 params[SI_PARAM_CONST] = const_array(v16i8, SI_NUM_CONST_BUFFERS);
2407 params[SI_PARAM_SAMPLER] = const_array(v4i32, SI_NUM_SAMPLER_STATES);
2408 params[SI_PARAM_RESOURCE] = const_array(v8i32, SI_NUM_SAMPLER_VIEWS);
2409 last_array_pointer = SI_PARAM_RESOURCE;
2410
2411 switch (si_shader_ctx->type) {
2412 case TGSI_PROCESSOR_VERTEX:
2413 params[SI_PARAM_VERTEX_BUFFER] = const_array(v16i8, SI_NUM_VERTEX_BUFFERS);
2414 last_array_pointer = SI_PARAM_VERTEX_BUFFER;
2415 params[SI_PARAM_BASE_VERTEX] = i32;
2416 params[SI_PARAM_START_INSTANCE] = i32;
2417 num_params = SI_PARAM_START_INSTANCE+1;
2418
2419 if (shader->key.vs.as_es) {
2420 params[SI_PARAM_ES2GS_OFFSET] = i32;
2421 num_params++;
2422 } else {
2423 if (shader->is_gs_copy_shader) {
2424 last_array_pointer = SI_PARAM_CONST;
2425 num_params = SI_PARAM_CONST+1;
2426 }
2427
2428 /* The locations of the other parameters are assigned dynamically. */
2429
2430 /* Streamout SGPRs. */
2431 if (shader->selector->so.num_outputs) {
2432 params[si_shader_ctx->param_streamout_config = num_params++] = i32;
2433 params[si_shader_ctx->param_streamout_write_index = num_params++] = i32;
2434 }
2435 /* A streamout buffer offset is loaded if the stride is non-zero. */
2436 for (i = 0; i < 4; i++) {
2437 if (!shader->selector->so.stride[i])
2438 continue;
2439
2440 params[si_shader_ctx->param_streamout_offset[i] = num_params++] = i32;
2441 }
2442 }
2443
2444 last_sgpr = num_params-1;
2445
2446 /* VGPRs */
2447 params[si_shader_ctx->param_vertex_id = num_params++] = i32;
2448 params[num_params++] = i32; /* unused*/
2449 params[num_params++] = i32; /* unused */
2450 params[si_shader_ctx->param_instance_id = num_params++] = i32;
2451 break;
2452
2453 case TGSI_PROCESSOR_GEOMETRY:
2454 params[SI_PARAM_GS2VS_OFFSET] = i32;
2455 params[SI_PARAM_GS_WAVE_ID] = i32;
2456 last_sgpr = SI_PARAM_GS_WAVE_ID;
2457
2458 /* VGPRs */
2459 params[SI_PARAM_VTX0_OFFSET] = i32;
2460 params[SI_PARAM_VTX1_OFFSET] = i32;
2461 params[SI_PARAM_PRIMITIVE_ID] = i32;
2462 params[SI_PARAM_VTX2_OFFSET] = i32;
2463 params[SI_PARAM_VTX3_OFFSET] = i32;
2464 params[SI_PARAM_VTX4_OFFSET] = i32;
2465 params[SI_PARAM_VTX5_OFFSET] = i32;
2466 params[SI_PARAM_GS_INSTANCE_ID] = i32;
2467 num_params = SI_PARAM_GS_INSTANCE_ID+1;
2468 break;
2469
2470 case TGSI_PROCESSOR_FRAGMENT:
2471 params[SI_PARAM_ALPHA_REF] = f32;
2472 params[SI_PARAM_PRIM_MASK] = i32;
2473 last_sgpr = SI_PARAM_PRIM_MASK;
2474 params[SI_PARAM_PERSP_SAMPLE] = v2i32;
2475 params[SI_PARAM_PERSP_CENTER] = v2i32;
2476 params[SI_PARAM_PERSP_CENTROID] = v2i32;
2477 params[SI_PARAM_PERSP_PULL_MODEL] = v3i32;
2478 params[SI_PARAM_LINEAR_SAMPLE] = v2i32;
2479 params[SI_PARAM_LINEAR_CENTER] = v2i32;
2480 params[SI_PARAM_LINEAR_CENTROID] = v2i32;
2481 params[SI_PARAM_LINE_STIPPLE_TEX] = f32;
2482 params[SI_PARAM_POS_X_FLOAT] = f32;
2483 params[SI_PARAM_POS_Y_FLOAT] = f32;
2484 params[SI_PARAM_POS_Z_FLOAT] = f32;
2485 params[SI_PARAM_POS_W_FLOAT] = f32;
2486 params[SI_PARAM_FRONT_FACE] = f32;
2487 params[SI_PARAM_ANCILLARY] = i32;
2488 params[SI_PARAM_SAMPLE_COVERAGE] = f32;
2489 params[SI_PARAM_POS_FIXED_PT] = f32;
2490 num_params = SI_PARAM_POS_FIXED_PT+1;
2491 break;
2492
2493 default:
2494 assert(0 && "unimplemented shader");
2495 return;
2496 }
2497
2498 assert(num_params <= Elements(params));
2499 radeon_llvm_create_func(&si_shader_ctx->radeon_bld, params, num_params);
2500 radeon_llvm_shader_type(si_shader_ctx->radeon_bld.main_fn, si_shader_ctx->type);
2501
2502 for (i = 0; i <= last_sgpr; ++i) {
2503 LLVMValueRef P = LLVMGetParam(si_shader_ctx->radeon_bld.main_fn, i);
2504
2505 /* We tell llvm that array inputs are passed by value to allow Sinking pass
2506 * to move load. Inputs are constant so this is fine. */
2507 if (i <= last_array_pointer)
2508 LLVMAddAttribute(P, LLVMByValAttribute);
2509 else
2510 LLVMAddAttribute(P, LLVMInRegAttribute);
2511 }
2512
2513 if (bld_base->info &&
2514 (bld_base->info->opcode_count[TGSI_OPCODE_DDX] > 0 ||
2515 bld_base->info->opcode_count[TGSI_OPCODE_DDY] > 0))
2516 si_shader_ctx->ddxy_lds =
2517 LLVMAddGlobalInAddressSpace(gallivm->module,
2518 LLVMArrayType(i32, 64),
2519 "ddxy_lds",
2520 LOCAL_ADDR_SPACE);
2521 }
2522
2523 static void preload_constants(struct si_shader_context *si_shader_ctx)
2524 {
2525 struct lp_build_tgsi_context * bld_base = &si_shader_ctx->radeon_bld.soa.bld_base;
2526 struct gallivm_state * gallivm = bld_base->base.gallivm;
2527 const struct tgsi_shader_info * info = bld_base->info;
2528 unsigned buf;
2529 LLVMValueRef ptr = LLVMGetParam(si_shader_ctx->radeon_bld.main_fn, SI_PARAM_CONST);
2530
2531 for (buf = 0; buf < SI_NUM_CONST_BUFFERS; buf++) {
2532 unsigned i, num_const = info->const_file_max[buf] + 1;
2533
2534 if (num_const == 0)
2535 continue;
2536
2537 /* Allocate space for the constant values */
2538 si_shader_ctx->constants[buf] = CALLOC(num_const * 4, sizeof(LLVMValueRef));
2539
2540 /* Load the resource descriptor */
2541 si_shader_ctx->const_resource[buf] =
2542 build_indexed_load(si_shader_ctx, ptr, lp_build_const_int32(gallivm, buf));
2543
2544 /* Load the constants, we rely on the code sinking to do the rest */
2545 for (i = 0; i < num_const * 4; ++i) {
2546 si_shader_ctx->constants[buf][i] =
2547 load_const(gallivm->builder,
2548 si_shader_ctx->const_resource[buf],
2549 lp_build_const_int32(gallivm, i * 4),
2550 bld_base->base.elem_type);
2551 }
2552 }
2553 }
2554
2555 static void preload_samplers(struct si_shader_context *si_shader_ctx)
2556 {
2557 struct lp_build_tgsi_context * bld_base = &si_shader_ctx->radeon_bld.soa.bld_base;
2558 struct gallivm_state * gallivm = bld_base->base.gallivm;
2559 const struct tgsi_shader_info * info = bld_base->info;
2560
2561 unsigned i, num_samplers = info->file_max[TGSI_FILE_SAMPLER] + 1;
2562
2563 LLVMValueRef res_ptr, samp_ptr;
2564 LLVMValueRef offset;
2565
2566 if (num_samplers == 0)
2567 return;
2568
2569 /* Allocate space for the values */
2570 si_shader_ctx->resources = CALLOC(SI_NUM_SAMPLER_VIEWS, sizeof(LLVMValueRef));
2571 si_shader_ctx->samplers = CALLOC(num_samplers, sizeof(LLVMValueRef));
2572
2573 res_ptr = LLVMGetParam(si_shader_ctx->radeon_bld.main_fn, SI_PARAM_RESOURCE);
2574 samp_ptr = LLVMGetParam(si_shader_ctx->radeon_bld.main_fn, SI_PARAM_SAMPLER);
2575
2576 /* Load the resources and samplers, we rely on the code sinking to do the rest */
2577 for (i = 0; i < num_samplers; ++i) {
2578 /* Resource */
2579 offset = lp_build_const_int32(gallivm, i);
2580 si_shader_ctx->resources[i] = build_indexed_load(si_shader_ctx, res_ptr, offset);
2581
2582 /* Sampler */
2583 offset = lp_build_const_int32(gallivm, i);
2584 si_shader_ctx->samplers[i] = build_indexed_load(si_shader_ctx, samp_ptr, offset);
2585
2586 /* FMASK resource */
2587 if (info->is_msaa_sampler[i]) {
2588 offset = lp_build_const_int32(gallivm, SI_FMASK_TEX_OFFSET + i);
2589 si_shader_ctx->resources[SI_FMASK_TEX_OFFSET + i] =
2590 build_indexed_load(si_shader_ctx, res_ptr, offset);
2591 }
2592 }
2593 }
2594
2595 static void preload_streamout_buffers(struct si_shader_context *si_shader_ctx)
2596 {
2597 struct lp_build_tgsi_context * bld_base = &si_shader_ctx->radeon_bld.soa.bld_base;
2598 struct gallivm_state * gallivm = bld_base->base.gallivm;
2599 unsigned i;
2600
2601 if (si_shader_ctx->type != TGSI_PROCESSOR_VERTEX ||
2602 si_shader_ctx->shader->key.vs.as_es ||
2603 !si_shader_ctx->shader->selector->so.num_outputs)
2604 return;
2605
2606 LLVMValueRef buf_ptr = LLVMGetParam(si_shader_ctx->radeon_bld.main_fn,
2607 SI_PARAM_RW_BUFFERS);
2608
2609 /* Load the resources, we rely on the code sinking to do the rest */
2610 for (i = 0; i < 4; ++i) {
2611 if (si_shader_ctx->shader->selector->so.stride[i]) {
2612 LLVMValueRef offset = lp_build_const_int32(gallivm,
2613 SI_SO_BUF_OFFSET + i);
2614
2615 si_shader_ctx->so_buffers[i] = build_indexed_load(si_shader_ctx, buf_ptr, offset);
2616 }
2617 }
2618 }
2619
2620 int si_compile_llvm(struct si_screen *sscreen, struct si_shader *shader,
2621 LLVMModuleRef mod)
2622 {
2623 unsigned r; /* llvm_compile result */
2624 unsigned i;
2625 unsigned char *ptr;
2626 struct radeon_shader_binary binary;
2627 bool dump = r600_can_dump_shader(&sscreen->b,
2628 shader->selector ? shader->selector->tokens : NULL);
2629 const char * gpu_family = r600_get_llvm_processor_name(sscreen->b.family);
2630 unsigned code_size;
2631
2632 /* Use LLVM to compile shader */
2633 memset(&binary, 0, sizeof(binary));
2634 r = radeon_llvm_compile(mod, &binary, gpu_family, dump);
2635
2636 /* Output binary dump if rscreen->debug_flags are set */
2637 if (dump && ! binary.disassembled) {
2638 fprintf(stderr, "SI CODE:\n");
2639 for (i = 0; i < binary.code_size; i+=4 ) {
2640 fprintf(stderr, "%02x%02x%02x%02x\n", binary.code[i + 3],
2641 binary.code[i + 2], binary.code[i + 1],
2642 binary.code[i]);
2643 }
2644 }
2645
2646 /* XXX: We may be able to emit some of these values directly rather than
2647 * extracting fields to be emitted later.
2648 */
2649 /* Parse config data in compiled binary */
2650 for (i = 0; i < binary.config_size; i+= 8) {
2651 unsigned reg = util_le32_to_cpu(*(uint32_t*)(binary.config + i));
2652 unsigned value = util_le32_to_cpu(*(uint32_t*)(binary.config + i + 4));
2653 switch (reg) {
2654 case R_00B028_SPI_SHADER_PGM_RSRC1_PS:
2655 case R_00B128_SPI_SHADER_PGM_RSRC1_VS:
2656 case R_00B228_SPI_SHADER_PGM_RSRC1_GS:
2657 case R_00B848_COMPUTE_PGM_RSRC1:
2658 shader->num_sgprs = (G_00B028_SGPRS(value) + 1) * 8;
2659 shader->num_vgprs = (G_00B028_VGPRS(value) + 1) * 4;
2660 break;
2661 case R_00B02C_SPI_SHADER_PGM_RSRC2_PS:
2662 shader->lds_size = G_00B02C_EXTRA_LDS_SIZE(value);
2663 break;
2664 case R_00B84C_COMPUTE_PGM_RSRC2:
2665 shader->lds_size = G_00B84C_LDS_SIZE(value);
2666 break;
2667 case R_0286CC_SPI_PS_INPUT_ENA:
2668 shader->spi_ps_input_ena = value;
2669 break;
2670 case R_00B860_COMPUTE_TMPRING_SIZE:
2671 /* WAVESIZE is in units of 256 dwords. */
2672 shader->scratch_bytes_per_wave =
2673 G_00B860_WAVESIZE(value) * 256 * 4 * 1;
2674 break;
2675 default:
2676 fprintf(stderr, "Warning: Compiler emitted unknown "
2677 "config register: 0x%x\n", reg);
2678 break;
2679 }
2680 }
2681
2682 /* copy new shader */
2683 code_size = binary.code_size + binary.rodata_size;
2684 r600_resource_reference(&shader->bo, NULL);
2685 shader->bo = si_resource_create_custom(&sscreen->b.b, PIPE_USAGE_IMMUTABLE,
2686 code_size);
2687 if (shader->bo == NULL) {
2688 return -ENOMEM;
2689 }
2690
2691 ptr = sscreen->b.ws->buffer_map(shader->bo->cs_buf, NULL, PIPE_TRANSFER_WRITE);
2692 util_memcpy_cpu_to_le32(ptr, binary.code, binary.code_size);
2693 if (binary.rodata_size > 0) {
2694 ptr += binary.code_size;
2695 util_memcpy_cpu_to_le32(ptr, binary.rodata, binary.rodata_size);
2696 }
2697
2698 sscreen->b.ws->buffer_unmap(shader->bo->cs_buf);
2699
2700 free(binary.code);
2701 free(binary.config);
2702 free(binary.rodata);
2703
2704 return r;
2705 }
2706
2707 /* Generate code for the hardware VS shader stage to go with a geometry shader */
2708 static int si_generate_gs_copy_shader(struct si_screen *sscreen,
2709 struct si_shader_context *si_shader_ctx,
2710 struct si_shader *gs, bool dump)
2711 {
2712 struct gallivm_state *gallivm = &si_shader_ctx->radeon_bld.gallivm;
2713 struct lp_build_tgsi_context *bld_base = &si_shader_ctx->radeon_bld.soa.bld_base;
2714 struct lp_build_context *base = &bld_base->base;
2715 struct lp_build_context *uint = &bld_base->uint_bld;
2716 struct si_shader *shader = si_shader_ctx->shader;
2717 struct si_shader_output_values *outputs;
2718 LLVMValueRef t_list_ptr, t_list;
2719 LLVMValueRef args[9];
2720 int i, r;
2721
2722 outputs = MALLOC(gs->noutput * sizeof(outputs[0]));
2723
2724 si_shader_ctx->type = TGSI_PROCESSOR_VERTEX;
2725 shader->is_gs_copy_shader = true;
2726
2727 radeon_llvm_context_init(&si_shader_ctx->radeon_bld);
2728
2729 create_meta_data(si_shader_ctx);
2730 create_function(si_shader_ctx);
2731 preload_streamout_buffers(si_shader_ctx);
2732
2733 /* Load the GSVS ring resource descriptor */
2734 t_list_ptr = LLVMGetParam(si_shader_ctx->radeon_bld.main_fn,
2735 SI_PARAM_RW_BUFFERS);
2736 t_list = build_indexed_load(si_shader_ctx, t_list_ptr,
2737 lp_build_const_int32(gallivm, SI_RING_GSVS));
2738
2739 args[0] = t_list;
2740 args[1] = lp_build_mul_imm(uint,
2741 LLVMGetParam(si_shader_ctx->radeon_bld.main_fn,
2742 si_shader_ctx->param_vertex_id),
2743 4);
2744 args[3] = uint->zero;
2745 args[4] = uint->one; /* OFFEN */
2746 args[5] = uint->zero; /* IDXEN */
2747 args[6] = uint->one; /* GLC */
2748 args[7] = uint->one; /* SLC */
2749 args[8] = uint->zero; /* TFE */
2750
2751 /* Fetch vertex data from GSVS ring */
2752 for (i = 0; i < gs->noutput; ++i) {
2753 struct si_shader_output *out = gs->output + i;
2754 unsigned chan;
2755
2756 shader->output[i] = *out;
2757
2758 outputs[i].name = out->name;
2759 outputs[i].index = out->index;
2760 outputs[i].sid = out->sid;
2761 outputs[i].usage = out->usage;
2762
2763 for (chan = 0; chan < 4; chan++) {
2764 args[2] = lp_build_const_int32(gallivm,
2765 (i * 4 + chan) *
2766 gs->selector->gs_max_out_vertices * 16 * 4);
2767
2768 outputs[i].values[chan] =
2769 LLVMBuildBitCast(gallivm->builder,
2770 build_intrinsic(gallivm->builder,
2771 "llvm.SI.buffer.load.dword.i32.i32",
2772 LLVMInt32TypeInContext(gallivm->context),
2773 args, 9,
2774 LLVMReadOnlyAttribute | LLVMNoUnwindAttribute),
2775 base->elem_type, "");
2776 }
2777 }
2778 shader->noutput = gs->noutput;
2779
2780 si_llvm_export_vs(bld_base, outputs, gs->noutput);
2781
2782 radeon_llvm_finalize_module(&si_shader_ctx->radeon_bld);
2783
2784 if (dump)
2785 fprintf(stderr, "Copy Vertex Shader for Geometry Shader:\n\n");
2786
2787 r = si_compile_llvm(sscreen, si_shader_ctx->shader,
2788 bld_base->base.gallivm->module);
2789
2790 radeon_llvm_dispose(&si_shader_ctx->radeon_bld);
2791
2792 FREE(outputs);
2793 return r;
2794 }
2795
2796 int si_shader_create(struct si_screen *sscreen, struct si_shader *shader)
2797 {
2798 struct si_shader_selector *sel = shader->selector;
2799 struct si_shader_context si_shader_ctx;
2800 struct lp_build_tgsi_context * bld_base;
2801 LLVMModuleRef mod;
2802 int r = 0;
2803 bool dump = r600_can_dump_shader(&sscreen->b, sel->tokens);
2804
2805 /* Dump TGSI code before doing TGSI->LLVM conversion in case the
2806 * conversion fails. */
2807 if (dump) {
2808 tgsi_dump(sel->tokens, 0);
2809 si_dump_streamout(&sel->so);
2810 }
2811
2812 assert(shader->noutput == 0);
2813 assert(shader->nparam == 0);
2814 assert(shader->ninput == 0);
2815
2816 memset(&si_shader_ctx, 0, sizeof(si_shader_ctx));
2817 radeon_llvm_context_init(&si_shader_ctx.radeon_bld);
2818 bld_base = &si_shader_ctx.radeon_bld.soa.bld_base;
2819
2820 if (sel->info.uses_kill)
2821 shader->db_shader_control |= S_02880C_KILL_ENABLE(1);
2822
2823 shader->uses_instanceid = sel->info.uses_instanceid;
2824 bld_base->info = &sel->info;
2825 bld_base->emit_fetch_funcs[TGSI_FILE_CONSTANT] = fetch_constant;
2826
2827 bld_base->op_actions[TGSI_OPCODE_TEX] = tex_action;
2828 bld_base->op_actions[TGSI_OPCODE_TEX2] = tex_action;
2829 bld_base->op_actions[TGSI_OPCODE_TXB] = tex_action;
2830 bld_base->op_actions[TGSI_OPCODE_TXB2] = tex_action;
2831 bld_base->op_actions[TGSI_OPCODE_TXD] = tex_action;
2832 bld_base->op_actions[TGSI_OPCODE_TXF] = tex_action;
2833 bld_base->op_actions[TGSI_OPCODE_TXL] = tex_action;
2834 bld_base->op_actions[TGSI_OPCODE_TXL2] = tex_action;
2835 bld_base->op_actions[TGSI_OPCODE_TXP] = tex_action;
2836 bld_base->op_actions[TGSI_OPCODE_TXQ] = txq_action;
2837 bld_base->op_actions[TGSI_OPCODE_TG4] = tex_action;
2838 bld_base->op_actions[TGSI_OPCODE_LODQ] = tex_action;
2839
2840 bld_base->op_actions[TGSI_OPCODE_DDX].emit = si_llvm_emit_ddxy;
2841 bld_base->op_actions[TGSI_OPCODE_DDY].emit = si_llvm_emit_ddxy;
2842
2843 bld_base->op_actions[TGSI_OPCODE_EMIT].emit = si_llvm_emit_vertex;
2844 bld_base->op_actions[TGSI_OPCODE_ENDPRIM].emit = si_llvm_emit_primitive;
2845
2846 si_shader_ctx.radeon_bld.load_system_value = declare_system_value;
2847 si_shader_ctx.tokens = sel->tokens;
2848 tgsi_parse_init(&si_shader_ctx.parse, si_shader_ctx.tokens);
2849 si_shader_ctx.shader = shader;
2850 si_shader_ctx.type = si_shader_ctx.parse.FullHeader.Processor.Processor;
2851
2852 switch (si_shader_ctx.type) {
2853 case TGSI_PROCESSOR_VERTEX:
2854 si_shader_ctx.radeon_bld.load_input = declare_input_vs;
2855 if (shader->key.vs.as_es) {
2856 bld_base->emit_epilogue = si_llvm_emit_es_epilogue;
2857 } else {
2858 bld_base->emit_epilogue = si_llvm_emit_vs_epilogue;
2859 }
2860 break;
2861 case TGSI_PROCESSOR_GEOMETRY:
2862 si_shader_ctx.radeon_bld.load_input = declare_input_gs;
2863 bld_base->emit_fetch_funcs[TGSI_FILE_INPUT] = fetch_input_gs;
2864 bld_base->emit_epilogue = si_llvm_emit_gs_epilogue;
2865 break;
2866 case TGSI_PROCESSOR_FRAGMENT:
2867 si_shader_ctx.radeon_bld.load_input = declare_input_fs;
2868 bld_base->emit_epilogue = si_llvm_emit_fs_epilogue;
2869
2870 switch (sel->info.properties[TGSI_PROPERTY_FS_DEPTH_LAYOUT]) {
2871 case TGSI_FS_DEPTH_LAYOUT_GREATER:
2872 shader->db_shader_control |=
2873 S_02880C_CONSERVATIVE_Z_EXPORT(V_02880C_EXPORT_GREATER_THAN_Z);
2874 break;
2875 case TGSI_FS_DEPTH_LAYOUT_LESS:
2876 shader->db_shader_control |=
2877 S_02880C_CONSERVATIVE_Z_EXPORT(V_02880C_EXPORT_LESS_THAN_Z);
2878 break;
2879 }
2880 break;
2881 default:
2882 assert(!"Unsupported shader type");
2883 return -1;
2884 }
2885
2886 create_meta_data(&si_shader_ctx);
2887 create_function(&si_shader_ctx);
2888 preload_constants(&si_shader_ctx);
2889 preload_samplers(&si_shader_ctx);
2890 preload_streamout_buffers(&si_shader_ctx);
2891
2892 if (si_shader_ctx.type == TGSI_PROCESSOR_GEOMETRY) {
2893 si_shader_ctx.gs_next_vertex =
2894 lp_build_alloca(bld_base->base.gallivm,
2895 bld_base->uint_bld.elem_type, "");
2896 }
2897
2898 if (!lp_build_tgsi_llvm(bld_base, sel->tokens)) {
2899 fprintf(stderr, "Failed to translate shader from TGSI to LLVM\n");
2900 goto out;
2901 }
2902
2903 radeon_llvm_finalize_module(&si_shader_ctx.radeon_bld);
2904
2905 mod = bld_base->base.gallivm->module;
2906 r = si_compile_llvm(sscreen, shader, mod);
2907 if (r) {
2908 fprintf(stderr, "LLVM failed to compile shader\n");
2909 goto out;
2910 }
2911
2912 radeon_llvm_dispose(&si_shader_ctx.radeon_bld);
2913
2914 if (si_shader_ctx.type == TGSI_PROCESSOR_GEOMETRY) {
2915 shader->gs_copy_shader = CALLOC_STRUCT(si_shader);
2916 shader->gs_copy_shader->selector = shader->selector;
2917 shader->gs_copy_shader->key = shader->key;
2918 si_shader_ctx.shader = shader->gs_copy_shader;
2919 if ((r = si_generate_gs_copy_shader(sscreen, &si_shader_ctx,
2920 shader, dump))) {
2921 free(shader->gs_copy_shader);
2922 shader->gs_copy_shader = NULL;
2923 goto out;
2924 }
2925 }
2926
2927 tgsi_parse_free(&si_shader_ctx.parse);
2928
2929 out:
2930 for (int i = 0; i < SI_NUM_CONST_BUFFERS; i++)
2931 FREE(si_shader_ctx.constants[i]);
2932 FREE(si_shader_ctx.resources);
2933 FREE(si_shader_ctx.samplers);
2934
2935 return r;
2936 }
2937
2938 void si_shader_destroy(struct pipe_context *ctx, struct si_shader *shader)
2939 {
2940 if (shader->gs_copy_shader)
2941 si_shader_destroy(ctx, shader->gs_copy_shader);
2942
2943 r600_resource_reference(&shader->bo, NULL);
2944 r600_resource_reference(&shader->scratch_bo, NULL);
2945 }