projects / mesa.git / blob
? search:


8759c38cabbf840b496e8a149541a8bf26be44dc
[mesa.git] / src / gallium / auxiliary / draw / draw_llvm.c
1 /**************************************************************************
2 *
3 * Copyright 2010 VMware, Inc.
4 * All Rights Reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sub license, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
13 *
14 * The above copyright notice and this permission notice (including the
15 * next paragraph) shall be included in all copies or substantial portions
16 * of the Software.
17 *
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
19 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
20 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
21 * IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR
22 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
23 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
24 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
25 *
26 **************************************************************************/
27
28 #include "draw_llvm.h"
29
30 #include "draw_context.h"
31 #include "draw_vs.h"
32
33 #include "gallivm/lp_bld_arit.h"
34 #include "gallivm/lp_bld_struct.h"
35 #include "gallivm/lp_bld_type.h"
36 #include "gallivm/lp_bld_flow.h"
37 #include "gallivm/lp_bld_debug.h"
38 #include "gallivm/lp_bld_tgsi.h"
39 #include "gallivm/lp_bld_printf.h"
40 #include "gallivm/lp_bld_intr.h"
41 #include "gallivm/lp_bld_init.h"
42
43 #include "tgsi/tgsi_exec.h"
44 #include "tgsi/tgsi_dump.h"
45
46 #include "util/u_cpu_detect.h"
47 #include "util/u_pointer.h"
48 #include "util/u_string.h"
49
50 #include <llvm-c/Transforms/Scalar.h>
51
52 #define DEBUG_STORE 0
53
54 /* generates the draw jit function */
55 static void
56 draw_llvm_generate(struct draw_llvm *llvm, struct draw_llvm_variant *var);
57 static void
58 draw_llvm_generate_elts(struct draw_llvm *llvm, struct draw_llvm_variant *var);
59
60 static void
61 init_globals(struct draw_llvm *llvm)
62 {
63 LLVMTypeRef texture_type;
64
65 /* struct draw_jit_texture */
66 {
67 LLVMTypeRef elem_types[DRAW_JIT_TEXTURE_NUM_FIELDS];
68
69 elem_types[DRAW_JIT_TEXTURE_WIDTH] = LLVMInt32Type();
70 elem_types[DRAW_JIT_TEXTURE_HEIGHT] = LLVMInt32Type();
71 elem_types[DRAW_JIT_TEXTURE_DEPTH] = LLVMInt32Type();
72 elem_types[DRAW_JIT_TEXTURE_LAST_LEVEL] = LLVMInt32Type();
73 elem_types[DRAW_JIT_TEXTURE_ROW_STRIDE] =
74 LLVMArrayType(LLVMInt32Type(), DRAW_MAX_TEXTURE_LEVELS);
75 elem_types[DRAW_JIT_TEXTURE_IMG_STRIDE] =
76 LLVMArrayType(LLVMInt32Type(), DRAW_MAX_TEXTURE_LEVELS);
77 elem_types[DRAW_JIT_TEXTURE_DATA] =
78 LLVMArrayType(LLVMPointerType(LLVMInt8Type(), 0),
79 DRAW_MAX_TEXTURE_LEVELS);
80
81 texture_type = LLVMStructType(elem_types, Elements(elem_types), 0);
82
83 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, width,
84 llvm->target, texture_type,
85 DRAW_JIT_TEXTURE_WIDTH);
86 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, height,
87 llvm->target, texture_type,
88 DRAW_JIT_TEXTURE_HEIGHT);
89 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, depth,
90 llvm->target, texture_type,
91 DRAW_JIT_TEXTURE_DEPTH);
92 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, last_level,
93 llvm->target, texture_type,
94 DRAW_JIT_TEXTURE_LAST_LEVEL);
95 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, row_stride,
96 llvm->target, texture_type,
97 DRAW_JIT_TEXTURE_ROW_STRIDE);
98 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, img_stride,
99 llvm->target, texture_type,
100 DRAW_JIT_TEXTURE_IMG_STRIDE);
101 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, data,
102 llvm->target, texture_type,
103 DRAW_JIT_TEXTURE_DATA);
104 LP_CHECK_STRUCT_SIZE(struct draw_jit_texture,
105 llvm->target, texture_type);
106
107 LLVMAddTypeName(llvm->module, "texture", texture_type);
108 }
109
110
111 /* struct draw_jit_context */
112 {
113 LLVMTypeRef elem_types[3];
114 LLVMTypeRef context_type;
115
116 elem_types[0] = LLVMPointerType(LLVMFloatType(), 0); /* vs_constants */
117 elem_types[1] = LLVMPointerType(LLVMFloatType(), 0); /* vs_constants */
118 elem_types[2] = LLVMArrayType(texture_type,
119 PIPE_MAX_VERTEX_SAMPLERS); /* textures */
120
121 context_type = LLVMStructType(elem_types, Elements(elem_types), 0);
122
123 LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, vs_constants,
124 llvm->target, context_type, 0);
125 LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, gs_constants,
126 llvm->target, context_type, 1);
127 LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, textures,
128 llvm->target, context_type,
129 DRAW_JIT_CTX_TEXTURES);
130 LP_CHECK_STRUCT_SIZE(struct draw_jit_context,
131 llvm->target, context_type);
132
133 LLVMAddTypeName(llvm->module, "draw_jit_context", context_type);
134
135 llvm->context_ptr_type = LLVMPointerType(context_type, 0);
136 }
137 {
138 LLVMTypeRef buffer_ptr = LLVMPointerType(LLVMIntType(8), 0);
139 llvm->buffer_ptr_type = LLVMPointerType(buffer_ptr, 0);
140 }
141 /* struct pipe_vertex_buffer */
142 {
143 LLVMTypeRef elem_types[4];
144 LLVMTypeRef vb_type;
145
146 elem_types[0] = LLVMInt32Type();
147 elem_types[1] = LLVMInt32Type();
148 elem_types[2] = LLVMInt32Type();
149 elem_types[3] = LLVMPointerType(LLVMOpaqueType(), 0); /* vs_constants */
150
151 vb_type = LLVMStructType(elem_types, Elements(elem_types), 0);
152
153 LP_CHECK_MEMBER_OFFSET(struct pipe_vertex_buffer, stride,
154 llvm->target, vb_type, 0);
155 LP_CHECK_MEMBER_OFFSET(struct pipe_vertex_buffer, buffer_offset,
156 llvm->target, vb_type, 2);
157 LP_CHECK_STRUCT_SIZE(struct pipe_vertex_buffer,
158 llvm->target, vb_type);
159
160 LLVMAddTypeName(llvm->module, "pipe_vertex_buffer", vb_type);
161
162 llvm->vb_ptr_type = LLVMPointerType(vb_type, 0);
163 }
164 }
165
166 static LLVMTypeRef
167 create_vertex_header(struct draw_llvm *llvm, int data_elems)
168 {
169 /* struct vertex_header */
170 LLVMTypeRef elem_types[3];
171 LLVMTypeRef vertex_header;
172 char struct_name[24];
173
174 util_snprintf(struct_name, 23, "vertex_header%d", data_elems);
175
176 elem_types[0] = LLVMIntType(32);
177 elem_types[1] = LLVMArrayType(LLVMFloatType(), 4);
178 elem_types[2] = LLVMArrayType(elem_types[1], data_elems);
179
180 vertex_header = LLVMStructType(elem_types, Elements(elem_types), 0);
181
182 /* these are bit-fields and we can't take address of them
183 LP_CHECK_MEMBER_OFFSET(struct vertex_header, clipmask,
184 llvm->target, vertex_header,
185 DRAW_JIT_VERTEX_CLIPMASK);
186 LP_CHECK_MEMBER_OFFSET(struct vertex_header, edgeflag,
187 llvm->target, vertex_header,
188 DRAW_JIT_VERTEX_EDGEFLAG);
189 LP_CHECK_MEMBER_OFFSET(struct vertex_header, pad,
190 llvm->target, vertex_header,
191 DRAW_JIT_VERTEX_PAD);
192 LP_CHECK_MEMBER_OFFSET(struct vertex_header, vertex_id,
193 llvm->target, vertex_header,
194 DRAW_JIT_VERTEX_VERTEX_ID);
195 */
196 LP_CHECK_MEMBER_OFFSET(struct vertex_header, clip,
197 llvm->target, vertex_header,
198 DRAW_JIT_VERTEX_CLIP);
199 LP_CHECK_MEMBER_OFFSET(struct vertex_header, data,
200 llvm->target, vertex_header,
201 DRAW_JIT_VERTEX_DATA);
202
203 LLVMAddTypeName(llvm->module, struct_name, vertex_header);
204
205 return LLVMPointerType(vertex_header, 0);
206 }
207
208 struct draw_llvm *
209 draw_llvm_create(struct draw_context *draw)
210 {
211 struct draw_llvm *llvm;
212
213 llvm = CALLOC_STRUCT( draw_llvm );
214 if (!llvm)
215 return NULL;
216
217 llvm->draw = draw;
218 llvm->engine = draw->engine;
219
220 debug_assert(llvm->engine);
221
222 llvm->module = LLVMModuleCreateWithName("draw_llvm");
223 llvm->provider = LLVMCreateModuleProviderForExistingModule(llvm->module);
224
225 LLVMAddModuleProvider(llvm->engine, llvm->provider);
226
227 llvm->target = LLVMGetExecutionEngineTargetData(llvm->engine);
228
229 llvm->pass = LLVMCreateFunctionPassManager(llvm->provider);
230 LLVMAddTargetData(llvm->target, llvm->pass);
231
232 if ((gallivm_debug & GALLIVM_DEBUG_NO_OPT) == 0) {
233 /* These are the passes currently listed in llvm-c/Transforms/Scalar.h,
234 * but there are more on SVN. */
235 /* TODO: Add more passes */
236
237 LLVMAddCFGSimplificationPass(llvm->pass);
238
239 if (HAVE_LLVM >= 0x207 && sizeof(void*) == 4) {
240 /* For LLVM >= 2.7 and 32-bit build, use this order of passes to
241 * avoid generating bad code.
242 * Test with piglit glsl-vs-sqrt-zero test.
243 */
244 LLVMAddConstantPropagationPass(llvm->pass);
245 LLVMAddPromoteMemoryToRegisterPass(llvm->pass);
246 }
247 else {
248 LLVMAddPromoteMemoryToRegisterPass(llvm->pass);
249 LLVMAddConstantPropagationPass(llvm->pass);
250 }
251
252 if(util_cpu_caps.has_sse4_1) {
253 /* FIXME: There is a bug in this pass, whereby the combination of fptosi
254 * and sitofp (necessary for trunc/floor/ceil/round implementation)
255 * somehow becomes invalid code.
256 */
257 LLVMAddInstructionCombiningPass(llvm->pass);
258 }
259 LLVMAddGVNPass(llvm->pass);
260 } else {
261 /* We need at least this pass to prevent the backends to fail in
262 * unexpected ways.
263 */
264 LLVMAddPromoteMemoryToRegisterPass(llvm->pass);
265 }
266
267 init_globals(llvm);
268
269 if (gallivm_debug & GALLIVM_DEBUG_IR) {
270 LLVMDumpModule(llvm->module);
271 }
272
273 llvm->nr_variants = 0;
274 make_empty_list(&llvm->vs_variants_list);
275
276 return llvm;
277 }
278
279 void
280 draw_llvm_destroy(struct draw_llvm *llvm)
281 {
282 LLVMDisposePassManager(llvm->pass);
283
284 FREE(llvm);
285 }
286
287 struct draw_llvm_variant *
288 draw_llvm_create_variant(struct draw_llvm *llvm,
289 unsigned num_inputs,
290 const struct draw_llvm_variant_key *key)
291 {
292 struct draw_llvm_variant *variant;
293 struct llvm_vertex_shader *shader =
294 llvm_vertex_shader(llvm->draw->vs.vertex_shader);
295
296 variant = MALLOC(sizeof *variant +
297 shader->variant_key_size -
298 sizeof variant->key);
299 if (variant == NULL)
300 return NULL;
301
302 variant->llvm = llvm;
303
304 memcpy(&variant->key, key, shader->variant_key_size);
305
306 llvm->vertex_header_ptr_type = create_vertex_header(llvm, num_inputs);
307
308 draw_llvm_generate(llvm, variant);
309 draw_llvm_generate_elts(llvm, variant);
310
311 variant->shader = shader;
312 variant->list_item_global.base = variant;
313 variant->list_item_local.base = variant;
314 /*variant->no = */shader->variants_created++;
315 variant->list_item_global.base = variant;
316
317 return variant;
318 }
319
320 static void
321 generate_vs(struct draw_llvm *llvm,
322 LLVMBuilderRef builder,
323 LLVMValueRef (*outputs)[NUM_CHANNELS],
324 const LLVMValueRef (*inputs)[NUM_CHANNELS],
325 LLVMValueRef context_ptr,
326 struct lp_build_sampler_soa *draw_sampler)
327 {
328 const struct tgsi_token *tokens = llvm->draw->vs.vertex_shader->state.tokens;
329 struct lp_type vs_type;
330 LLVMValueRef consts_ptr = draw_jit_context_vs_constants(builder, context_ptr);
331 struct lp_build_sampler_soa *sampler = 0;
332
333 memset(&vs_type, 0, sizeof vs_type);
334 vs_type.floating = TRUE; /* floating point values */
335 vs_type.sign = TRUE; /* values are signed */
336 vs_type.norm = FALSE; /* values are not limited to [0,1] or [-1,1] */
337 vs_type.width = 32; /* 32-bit float */
338 vs_type.length = 4; /* 4 elements per vector */
339 #if 0
340 num_vs = 4; /* number of vertices per block */
341 #endif
342
343 if (gallivm_debug & GALLIVM_DEBUG_IR) {
344 tgsi_dump(tokens, 0);
345 }
346
347 if (llvm->draw->num_sampler_views &&
348 llvm->draw->num_samplers)
349 sampler = draw_sampler;
350
351 lp_build_tgsi_soa(builder,
352 tokens,
353 vs_type,
354 NULL /*struct lp_build_mask_context *mask*/,
355 consts_ptr,
356 NULL /*pos*/,
357 inputs,
358 outputs,
359 sampler,
360 &llvm->draw->vs.vertex_shader->info);
361 }
362
363 #if DEBUG_STORE
364 static void print_vectorf(LLVMBuilderRef builder,
365 LLVMValueRef vec)
366 {
367 LLVMValueRef val[4];
368 val[0] = LLVMBuildExtractElement(builder, vec,
369 LLVMConstInt(LLVMInt32Type(), 0, 0), "");
370 val[1] = LLVMBuildExtractElement(builder, vec,
371 LLVMConstInt(LLVMInt32Type(), 1, 0), "");
372 val[2] = LLVMBuildExtractElement(builder, vec,
373 LLVMConstInt(LLVMInt32Type(), 2, 0), "");
374 val[3] = LLVMBuildExtractElement(builder, vec,
375 LLVMConstInt(LLVMInt32Type(), 3, 0), "");
376 lp_build_printf(builder, "vector = [%f, %f, %f, %f]\n",
377 val[0], val[1], val[2], val[3]);
378 }
379 #endif
380
381 static void
382 generate_fetch(LLVMBuilderRef builder,
383 LLVMValueRef vbuffers_ptr,
384 LLVMValueRef *res,
385 struct pipe_vertex_element *velem,
386 LLVMValueRef vbuf,
387 LLVMValueRef index,
388 LLVMValueRef instance_id)
389 {
390 LLVMValueRef indices = LLVMConstInt(LLVMInt64Type(), velem->vertex_buffer_index, 0);
391 LLVMValueRef vbuffer_ptr = LLVMBuildGEP(builder, vbuffers_ptr,
392 &indices, 1, "");
393 LLVMValueRef vb_stride = draw_jit_vbuffer_stride(builder, vbuf);
394 LLVMValueRef vb_max_index = draw_jit_vbuffer_max_index(builder, vbuf);
395 LLVMValueRef vb_buffer_offset = draw_jit_vbuffer_offset(builder, vbuf);
396 LLVMValueRef cond;
397 LLVMValueRef stride;
398
399 if (velem->instance_divisor) {
400 /* array index = instance_id / instance_divisor */
401 index = LLVMBuildUDiv(builder, instance_id,
402 LLVMConstInt(LLVMInt32Type(), velem->instance_divisor, 0),
403 "instance_divisor");
404 }
405
406 /* limit index to min(inex, vb_max_index) */
407 cond = LLVMBuildICmp(builder, LLVMIntULE, index, vb_max_index, "");
408 index = LLVMBuildSelect(builder, cond, index, vb_max_index, "");
409
410 stride = LLVMBuildMul(builder, vb_stride, index, "");
411
412 vbuffer_ptr = LLVMBuildLoad(builder, vbuffer_ptr, "vbuffer");
413
414 stride = LLVMBuildAdd(builder, stride,
415 vb_buffer_offset,
416 "");
417 stride = LLVMBuildAdd(builder, stride,
418 LLVMConstInt(LLVMInt32Type(), velem->src_offset, 0),
419 "");
420
421 /*lp_build_printf(builder, "vbuf index = %d, stride is %d\n", indices, stride);*/
422 vbuffer_ptr = LLVMBuildGEP(builder, vbuffer_ptr, &stride, 1, "");
423
424 *res = draw_llvm_translate_from(builder, vbuffer_ptr, velem->src_format);
425 }
426
427 static LLVMValueRef
428 aos_to_soa(LLVMBuilderRef builder,
429 LLVMValueRef val0,
430 LLVMValueRef val1,
431 LLVMValueRef val2,
432 LLVMValueRef val3,
433 LLVMValueRef channel)
434 {
435 LLVMValueRef ex, res;
436
437 ex = LLVMBuildExtractElement(builder, val0,
438 channel, "");
439 res = LLVMBuildInsertElement(builder,
440 LLVMConstNull(LLVMTypeOf(val0)),
441 ex,
442 LLVMConstInt(LLVMInt32Type(), 0, 0),
443 "");
444
445 ex = LLVMBuildExtractElement(builder, val1,
446 channel, "");
447 res = LLVMBuildInsertElement(builder,
448 res, ex,
449 LLVMConstInt(LLVMInt32Type(), 1, 0),
450 "");
451
452 ex = LLVMBuildExtractElement(builder, val2,
453 channel, "");
454 res = LLVMBuildInsertElement(builder,
455 res, ex,
456 LLVMConstInt(LLVMInt32Type(), 2, 0),
457 "");
458
459 ex = LLVMBuildExtractElement(builder, val3,
460 channel, "");
461 res = LLVMBuildInsertElement(builder,
462 res, ex,
463 LLVMConstInt(LLVMInt32Type(), 3, 0),
464 "");
465
466 return res;
467 }
468
469 static void
470 soa_to_aos(LLVMBuilderRef builder,
471 LLVMValueRef soa[NUM_CHANNELS],
472 LLVMValueRef aos[NUM_CHANNELS])
473 {
474 LLVMValueRef comp;
475 int i = 0;
476
477 debug_assert(NUM_CHANNELS == 4);
478
479 aos[0] = LLVMConstNull(LLVMTypeOf(soa[0]));
480 aos[1] = aos[2] = aos[3] = aos[0];
481
482 for (i = 0; i < NUM_CHANNELS; ++i) {
483 LLVMValueRef channel = LLVMConstInt(LLVMInt32Type(), i, 0);
484
485 comp = LLVMBuildExtractElement(builder, soa[i],
486 LLVMConstInt(LLVMInt32Type(), 0, 0), "");
487 aos[0] = LLVMBuildInsertElement(builder, aos[0], comp, channel, "");
488
489 comp = LLVMBuildExtractElement(builder, soa[i],
490 LLVMConstInt(LLVMInt32Type(), 1, 0), "");
491 aos[1] = LLVMBuildInsertElement(builder, aos[1], comp, channel, "");
492
493 comp = LLVMBuildExtractElement(builder, soa[i],
494 LLVMConstInt(LLVMInt32Type(), 2, 0), "");
495 aos[2] = LLVMBuildInsertElement(builder, aos[2], comp, channel, "");
496
497 comp = LLVMBuildExtractElement(builder, soa[i],
498 LLVMConstInt(LLVMInt32Type(), 3, 0), "");
499 aos[3] = LLVMBuildInsertElement(builder, aos[3], comp, channel, "");
500
501 }
502 }
503
504 static void
505 convert_to_soa(LLVMBuilderRef builder,
506 LLVMValueRef (*aos)[NUM_CHANNELS],
507 LLVMValueRef (*soa)[NUM_CHANNELS],
508 int num_attribs)
509 {
510 int i;
511
512 debug_assert(NUM_CHANNELS == 4);
513
514 for (i = 0; i < num_attribs; ++i) {
515 LLVMValueRef val0 = aos[i][0];
516 LLVMValueRef val1 = aos[i][1];
517 LLVMValueRef val2 = aos[i][2];
518 LLVMValueRef val3 = aos[i][3];
519
520 soa[i][0] = aos_to_soa(builder, val0, val1, val2, val3,
521 LLVMConstInt(LLVMInt32Type(), 0, 0));
522 soa[i][1] = aos_to_soa(builder, val0, val1, val2, val3,
523 LLVMConstInt(LLVMInt32Type(), 1, 0));
524 soa[i][2] = aos_to_soa(builder, val0, val1, val2, val3,
525 LLVMConstInt(LLVMInt32Type(), 2, 0));
526 soa[i][3] = aos_to_soa(builder, val0, val1, val2, val3,
527 LLVMConstInt(LLVMInt32Type(), 3, 0));
528 }
529 }
530
531 static void
532 store_aos(LLVMBuilderRef builder,
533 LLVMValueRef io_ptr,
534 LLVMValueRef index,
535 LLVMValueRef value)
536 {
537 LLVMValueRef id_ptr = draw_jit_header_id(builder, io_ptr);
538 LLVMValueRef data_ptr = draw_jit_header_data(builder, io_ptr);
539 LLVMValueRef indices[3];
540
541 indices[0] = LLVMConstInt(LLVMInt32Type(), 0, 0);
542 indices[1] = index;
543 indices[2] = LLVMConstInt(LLVMInt32Type(), 0, 0);
544
545 /* undefined vertex */
546 LLVMBuildStore(builder, LLVMConstInt(LLVMInt32Type(),
547 0xffff, 0), id_ptr);
548
549 #if DEBUG_STORE
550 lp_build_printf(builder, " ---- %p storing attribute %d (io = %p)\n", data_ptr, index, io_ptr);
551 #endif
552 #if 0
553 /*lp_build_printf(builder, " ---- %p storing at %d (%p) ", io_ptr, index, data_ptr);
554 print_vectorf(builder, value);*/
555 data_ptr = LLVMBuildBitCast(builder, data_ptr,
556 LLVMPointerType(LLVMArrayType(LLVMVectorType(LLVMFloatType(), 4), 0), 0),
557 "datavec");
558 data_ptr = LLVMBuildGEP(builder, data_ptr, indices, 2, "");
559
560 LLVMBuildStore(builder, value, data_ptr);
561 #else
562 {
563 LLVMValueRef x, y, z, w;
564 LLVMValueRef idx0, idx1, idx2, idx3;
565 LLVMValueRef gep0, gep1, gep2, gep3;
566 data_ptr = LLVMBuildGEP(builder, data_ptr, indices, 3, "");
567
568 idx0 = LLVMConstInt(LLVMInt32Type(), 0, 0);
569 idx1 = LLVMConstInt(LLVMInt32Type(), 1, 0);
570 idx2 = LLVMConstInt(LLVMInt32Type(), 2, 0);
571 idx3 = LLVMConstInt(LLVMInt32Type(), 3, 0);
572
573 x = LLVMBuildExtractElement(builder, value,
574 idx0, "");
575 y = LLVMBuildExtractElement(builder, value,
576 idx1, "");
577 z = LLVMBuildExtractElement(builder, value,
578 idx2, "");
579 w = LLVMBuildExtractElement(builder, value,
580 idx3, "");
581
582 gep0 = LLVMBuildGEP(builder, data_ptr, &idx0, 1, "");
583 gep1 = LLVMBuildGEP(builder, data_ptr, &idx1, 1, "");
584 gep2 = LLVMBuildGEP(builder, data_ptr, &idx2, 1, "");
585 gep3 = LLVMBuildGEP(builder, data_ptr, &idx3, 1, "");
586
587 /*lp_build_printf(builder, "##### x = %f (%p), y = %f (%p), z = %f (%p), w = %f (%p)\n",
588 x, gep0, y, gep1, z, gep2, w, gep3);*/
589 LLVMBuildStore(builder, x, gep0);
590 LLVMBuildStore(builder, y, gep1);
591 LLVMBuildStore(builder, z, gep2);
592 LLVMBuildStore(builder, w, gep3);
593 }
594 #endif
595 }
596
597 static void
598 store_aos_array(LLVMBuilderRef builder,
599 LLVMValueRef io_ptr,
600 LLVMValueRef aos[NUM_CHANNELS],
601 int attrib,
602 int num_outputs)
603 {
604 LLVMValueRef attr_index = LLVMConstInt(LLVMInt32Type(), attrib, 0);
605 LLVMValueRef ind0 = LLVMConstInt(LLVMInt32Type(), 0, 0);
606 LLVMValueRef ind1 = LLVMConstInt(LLVMInt32Type(), 1, 0);
607 LLVMValueRef ind2 = LLVMConstInt(LLVMInt32Type(), 2, 0);
608 LLVMValueRef ind3 = LLVMConstInt(LLVMInt32Type(), 3, 0);
609 LLVMValueRef io0_ptr, io1_ptr, io2_ptr, io3_ptr;
610
611 debug_assert(NUM_CHANNELS == 4);
612
613 io0_ptr = LLVMBuildGEP(builder, io_ptr,
614 &ind0, 1, "");
615 io1_ptr = LLVMBuildGEP(builder, io_ptr,
616 &ind1, 1, "");
617 io2_ptr = LLVMBuildGEP(builder, io_ptr,
618 &ind2, 1, "");
619 io3_ptr = LLVMBuildGEP(builder, io_ptr,
620 &ind3, 1, "");
621
622 #if DEBUG_STORE
623 lp_build_printf(builder, " io = %p, indexes[%d, %d, %d, %d]\n",
624 io_ptr, ind0, ind1, ind2, ind3);
625 #endif
626
627 store_aos(builder, io0_ptr, attr_index, aos[0]);
628 store_aos(builder, io1_ptr, attr_index, aos[1]);
629 store_aos(builder, io2_ptr, attr_index, aos[2]);
630 store_aos(builder, io3_ptr, attr_index, aos[3]);
631 }
632
633 static void
634 convert_to_aos(LLVMBuilderRef builder,
635 LLVMValueRef io,
636 LLVMValueRef (*outputs)[NUM_CHANNELS],
637 int num_outputs,
638 int max_vertices)
639 {
640 unsigned chan, attrib;
641
642 #if DEBUG_STORE
643 lp_build_printf(builder, " # storing begin\n");
644 #endif
645 for (attrib = 0; attrib < num_outputs; ++attrib) {
646 LLVMValueRef soa[4];
647 LLVMValueRef aos[4];
648 for(chan = 0; chan < NUM_CHANNELS; ++chan) {
649 if(outputs[attrib][chan]) {
650 LLVMValueRef out = LLVMBuildLoad(builder, outputs[attrib][chan], "");
651 lp_build_name(out, "output%u.%c", attrib, "xyzw"[chan]);
652 /*lp_build_printf(builder, "output %d : %d ",
653 LLVMConstInt(LLVMInt32Type(), attrib, 0),
654 LLVMConstInt(LLVMInt32Type(), chan, 0));
655 print_vectorf(builder, out);*/
656 soa[chan] = out;
657 } else
658 soa[chan] = 0;
659 }
660 soa_to_aos(builder, soa, aos);
661 store_aos_array(builder,
662 io,
663 aos,
664 attrib,
665 num_outputs);
666 }
667 #if DEBUG_STORE
668 lp_build_printf(builder, " # storing end\n");
669 #endif
670 }
671
672 static void
673 draw_llvm_generate(struct draw_llvm *llvm, struct draw_llvm_variant *variant)
674 {
675 LLVMTypeRef arg_types[8];
676 LLVMTypeRef func_type;
677 LLVMValueRef context_ptr;
678 LLVMBasicBlockRef block;
679 LLVMBuilderRef builder;
680 LLVMValueRef start, end, count, stride, step, io_itr;
681 LLVMValueRef io_ptr, vbuffers_ptr, vb_ptr;
682 LLVMValueRef instance_id;
683 struct draw_context *draw = llvm->draw;
684 unsigned i, j;
685 struct lp_build_context bld;
686 struct lp_build_loop_state lp_loop;
687 const int max_vertices = 4;
688 LLVMValueRef outputs[PIPE_MAX_SHADER_OUTPUTS][NUM_CHANNELS];
689 void *code;
690 struct lp_build_sampler_soa *sampler = 0;
691
692 arg_types[0] = llvm->context_ptr_type; /* context */
693 arg_types[1] = llvm->vertex_header_ptr_type; /* vertex_header */
694 arg_types[2] = llvm->buffer_ptr_type; /* vbuffers */
695 arg_types[3] = LLVMInt32Type(); /* start */
696 arg_types[4] = LLVMInt32Type(); /* count */
697 arg_types[5] = LLVMInt32Type(); /* stride */
698 arg_types[6] = llvm->vb_ptr_type; /* pipe_vertex_buffer's */
699 arg_types[7] = LLVMInt32Type(); /* instance_id */
700
701 func_type = LLVMFunctionType(LLVMVoidType(), arg_types, Elements(arg_types), 0);
702
703 variant->function = LLVMAddFunction(llvm->module, "draw_llvm_shader", func_type);
704 LLVMSetFunctionCallConv(variant->function, LLVMCCallConv);
705 for(i = 0; i < Elements(arg_types); ++i)
706 if(LLVMGetTypeKind(arg_types[i]) == LLVMPointerTypeKind)
707 LLVMAddAttribute(LLVMGetParam(variant->function, i), LLVMNoAliasAttribute);
708
709 context_ptr = LLVMGetParam(variant->function, 0);
710 io_ptr = LLVMGetParam(variant->function, 1);
711 vbuffers_ptr = LLVMGetParam(variant->function, 2);
712 start = LLVMGetParam(variant->function, 3);
713 count = LLVMGetParam(variant->function, 4);
714 stride = LLVMGetParam(variant->function, 5);
715 vb_ptr = LLVMGetParam(variant->function, 6);
716 instance_id = LLVMGetParam(variant->function, 7);
717
718 lp_build_name(context_ptr, "context");
719 lp_build_name(io_ptr, "io");
720 lp_build_name(vbuffers_ptr, "vbuffers");
721 lp_build_name(start, "start");
722 lp_build_name(count, "count");
723 lp_build_name(stride, "stride");
724 lp_build_name(vb_ptr, "vb");
725 lp_build_name(instance_id, "instance_id");
726
727 /*
728 * Function body
729 */
730
731 block = LLVMAppendBasicBlock(variant->function, "entry");
732 builder = LLVMCreateBuilder();
733 LLVMPositionBuilderAtEnd(builder, block);
734
735 lp_build_context_init(&bld, builder, lp_type_int(32));
736
737 end = lp_build_add(&bld, start, count);
738
739 step = LLVMConstInt(LLVMInt32Type(), max_vertices, 0);
740
741 /* code generated texture sampling */
742 sampler = draw_llvm_sampler_soa_create(
743 draw_llvm_variant_key_samplers(&variant->key),
744 context_ptr);
745
746 #if DEBUG_STORE
747 lp_build_printf(builder, "start = %d, end = %d, step = %d\n",
748 start, end, step);
749 #endif
750 lp_build_loop_begin(builder, start, &lp_loop);
751 {
752 LLVMValueRef inputs[PIPE_MAX_SHADER_INPUTS][NUM_CHANNELS];
753 LLVMValueRef aos_attribs[PIPE_MAX_SHADER_INPUTS][NUM_CHANNELS] = { { 0 } };
754 LLVMValueRef io;
755 const LLVMValueRef (*ptr_aos)[NUM_CHANNELS];
756
757 io_itr = LLVMBuildSub(builder, lp_loop.counter, start, "");
758 io = LLVMBuildGEP(builder, io_ptr, &io_itr, 1, "");
759 #if DEBUG_STORE
760 lp_build_printf(builder, " --- io %d = %p, loop counter %d\n",
761 io_itr, io, lp_loop.counter);
762 #endif
763 for (i = 0; i < NUM_CHANNELS; ++i) {
764 LLVMValueRef true_index = LLVMBuildAdd(
765 builder,
766 lp_loop.counter,
767 LLVMConstInt(LLVMInt32Type(), i, 0), "");
768 for (j = 0; j < draw->pt.nr_vertex_elements; ++j) {
769 struct pipe_vertex_element *velem = &draw->pt.vertex_element[j];
770 LLVMValueRef vb_index = LLVMConstInt(LLVMInt32Type(),
771 velem->vertex_buffer_index,
772 0);
773 LLVMValueRef vb = LLVMBuildGEP(builder, vb_ptr,
774 &vb_index, 1, "");
775 generate_fetch(builder, vbuffers_ptr,
776 &aos_attribs[j][i], velem, vb, true_index,
777 instance_id);
778 }
779 }
780 convert_to_soa(builder, aos_attribs, inputs,
781 draw->pt.nr_vertex_elements);
782
783 ptr_aos = (const LLVMValueRef (*)[NUM_CHANNELS]) inputs;
784 generate_vs(llvm,
785 builder,
786 outputs,
787 ptr_aos,
788 context_ptr,
789 sampler);
790
791 convert_to_aos(builder, io, outputs,
792 draw->vs.vertex_shader->info.num_outputs,
793 max_vertices);
794 }
795 lp_build_loop_end_cond(builder, end, step, LLVMIntUGE, &lp_loop);
796
797 sampler->destroy(sampler);
798
799 #ifdef PIPE_ARCH_X86
800 /* Avoid corrupting the FPU stack on 32bit OSes. */
801 lp_build_intrinsic(builder, "llvm.x86.mmx.emms", LLVMVoidType(), NULL, 0);
802 #endif
803
804 LLVMBuildRetVoid(builder);
805
806 LLVMDisposeBuilder(builder);
807
808 /*
809 * Translate the LLVM IR into machine code.
810 */
811 #ifdef DEBUG
812 if(LLVMVerifyFunction(variant->function, LLVMPrintMessageAction)) {
813 lp_debug_dump_value(variant->function);
814 assert(0);
815 }
816 #endif
817
818 LLVMRunFunctionPassManager(llvm->pass, variant->function);
819
820 if (gallivm_debug & GALLIVM_DEBUG_IR) {
821 lp_debug_dump_value(variant->function);
822 debug_printf("\n");
823 }
824
825 code = LLVMGetPointerToGlobal(llvm->draw->engine, variant->function);
826 variant->jit_func = (draw_jit_vert_func)pointer_to_func(code);
827
828 if (gallivm_debug & GALLIVM_DEBUG_ASM) {
829 lp_disassemble(code);
830 }
831 lp_func_delete_body(variant->function);
832 }
833
834
835 static void
836 draw_llvm_generate_elts(struct draw_llvm *llvm, struct draw_llvm_variant *variant)
837 {
838 LLVMTypeRef arg_types[8];
839 LLVMTypeRef func_type;
840 LLVMValueRef context_ptr;
841 LLVMBasicBlockRef block;
842 LLVMBuilderRef builder;
843 LLVMValueRef fetch_elts, fetch_count, stride, step, io_itr;
844 LLVMValueRef io_ptr, vbuffers_ptr, vb_ptr;
845 LLVMValueRef instance_id;
846 struct draw_context *draw = llvm->draw;
847 unsigned i, j;
848 struct lp_build_context bld;
849 struct lp_build_loop_state lp_loop;
850 const int max_vertices = 4;
851 LLVMValueRef outputs[PIPE_MAX_SHADER_OUTPUTS][NUM_CHANNELS];
852 LLVMValueRef fetch_max;
853 void *code;
854 struct lp_build_sampler_soa *sampler = 0;
855
856 arg_types[0] = llvm->context_ptr_type; /* context */
857 arg_types[1] = llvm->vertex_header_ptr_type; /* vertex_header */
858 arg_types[2] = llvm->buffer_ptr_type; /* vbuffers */
859 arg_types[3] = LLVMPointerType(LLVMInt32Type(), 0); /* fetch_elts * */
860 arg_types[4] = LLVMInt32Type(); /* fetch_count */
861 arg_types[5] = LLVMInt32Type(); /* stride */
862 arg_types[6] = llvm->vb_ptr_type; /* pipe_vertex_buffer's */
863 arg_types[7] = LLVMInt32Type(); /* instance_id */
864
865 func_type = LLVMFunctionType(LLVMVoidType(), arg_types, Elements(arg_types), 0);
866
867 variant->function_elts = LLVMAddFunction(llvm->module, "draw_llvm_shader_elts",
868 func_type);
869 LLVMSetFunctionCallConv(variant->function_elts, LLVMCCallConv);
870 for(i = 0; i < Elements(arg_types); ++i)
871 if(LLVMGetTypeKind(arg_types[i]) == LLVMPointerTypeKind)
872 LLVMAddAttribute(LLVMGetParam(variant->function_elts, i),
873 LLVMNoAliasAttribute);
874
875 context_ptr = LLVMGetParam(variant->function_elts, 0);
876 io_ptr = LLVMGetParam(variant->function_elts, 1);
877 vbuffers_ptr = LLVMGetParam(variant->function_elts, 2);
878 fetch_elts = LLVMGetParam(variant->function_elts, 3);
879 fetch_count = LLVMGetParam(variant->function_elts, 4);
880 stride = LLVMGetParam(variant->function_elts, 5);
881 vb_ptr = LLVMGetParam(variant->function_elts, 6);
882 instance_id = LLVMGetParam(variant->function_elts, 7);
883
884 lp_build_name(context_ptr, "context");
885 lp_build_name(io_ptr, "io");
886 lp_build_name(vbuffers_ptr, "vbuffers");
887 lp_build_name(fetch_elts, "fetch_elts");
888 lp_build_name(fetch_count, "fetch_count");
889 lp_build_name(stride, "stride");
890 lp_build_name(vb_ptr, "vb");
891 lp_build_name(instance_id, "instance_id");
892
893 /*
894 * Function body
895 */
896
897 block = LLVMAppendBasicBlock(variant->function_elts, "entry");
898 builder = LLVMCreateBuilder();
899 LLVMPositionBuilderAtEnd(builder, block);
900
901 lp_build_context_init(&bld, builder, lp_type_int(32));
902
903 step = LLVMConstInt(LLVMInt32Type(), max_vertices, 0);
904
905 /* code generated texture sampling */
906 sampler = draw_llvm_sampler_soa_create(
907 draw_llvm_variant_key_samplers(&variant->key),
908 context_ptr);
909
910 fetch_max = LLVMBuildSub(builder, fetch_count,
911 LLVMConstInt(LLVMInt32Type(), 1, 0),
912 "fetch_max");
913
914 lp_build_loop_begin(builder, LLVMConstInt(LLVMInt32Type(), 0, 0), &lp_loop);
915 {
916 LLVMValueRef inputs[PIPE_MAX_SHADER_INPUTS][NUM_CHANNELS];
917 LLVMValueRef aos_attribs[PIPE_MAX_SHADER_INPUTS][NUM_CHANNELS] = { { 0 } };
918 LLVMValueRef io;
919 const LLVMValueRef (*ptr_aos)[NUM_CHANNELS];
920
921 io_itr = lp_loop.counter;
922 io = LLVMBuildGEP(builder, io_ptr, &io_itr, 1, "");
923 #if DEBUG_STORE
924 lp_build_printf(builder, " --- io %d = %p, loop counter %d\n",
925 io_itr, io, lp_loop.counter);
926 #endif
927 for (i = 0; i < NUM_CHANNELS; ++i) {
928 LLVMValueRef true_index = LLVMBuildAdd(
929 builder,
930 lp_loop.counter,
931 LLVMConstInt(LLVMInt32Type(), i, 0), "");
932 LLVMValueRef fetch_ptr;
933
934 /* make sure we're not out of bounds which can happen
935 * if fetch_count % 4 != 0, because on the last iteration
936 * a few of the 4 vertex fetches will be out of bounds */
937 true_index = lp_build_min(&bld, true_index, fetch_max);
938
939 fetch_ptr = LLVMBuildGEP(builder, fetch_elts,
940 &true_index, 1, "");
941 true_index = LLVMBuildLoad(builder, fetch_ptr, "fetch_elt");
942 for (j = 0; j < draw->pt.nr_vertex_elements; ++j) {
943 struct pipe_vertex_element *velem = &draw->pt.vertex_element[j];
944 LLVMValueRef vb_index = LLVMConstInt(LLVMInt32Type(),
945 velem->vertex_buffer_index,
946 0);
947 LLVMValueRef vb = LLVMBuildGEP(builder, vb_ptr,
948 &vb_index, 1, "");
949 generate_fetch(builder, vbuffers_ptr,
950 &aos_attribs[j][i], velem, vb, true_index,
951 instance_id);
952 }
953 }
954 convert_to_soa(builder, aos_attribs, inputs,
955 draw->pt.nr_vertex_elements);
956
957 ptr_aos = (const LLVMValueRef (*)[NUM_CHANNELS]) inputs;
958 generate_vs(llvm,
959 builder,
960 outputs,
961 ptr_aos,
962 context_ptr,
963 sampler);
964
965 convert_to_aos(builder, io, outputs,
966 draw->vs.vertex_shader->info.num_outputs,
967 max_vertices);
968 }
969 lp_build_loop_end_cond(builder, fetch_count, step, LLVMIntUGE, &lp_loop);
970
971 sampler->destroy(sampler);
972
973 #ifdef PIPE_ARCH_X86
974 /* Avoid corrupting the FPU stack on 32bit OSes. */
975 lp_build_intrinsic(builder, "llvm.x86.mmx.emms", LLVMVoidType(), NULL, 0);
976 #endif
977
978 LLVMBuildRetVoid(builder);
979
980 LLVMDisposeBuilder(builder);
981
982 /*
983 * Translate the LLVM IR into machine code.
984 */
985 #ifdef DEBUG
986 if(LLVMVerifyFunction(variant->function_elts, LLVMPrintMessageAction)) {
987 lp_debug_dump_value(variant->function_elts);
988 assert(0);
989 }
990 #endif
991
992 LLVMRunFunctionPassManager(llvm->pass, variant->function_elts);
993
994 if (gallivm_debug & GALLIVM_DEBUG_IR) {
995 lp_debug_dump_value(variant->function_elts);
996 debug_printf("\n");
997 }
998
999 code = LLVMGetPointerToGlobal(llvm->draw->engine, variant->function_elts);
1000 variant->jit_func_elts = (draw_jit_vert_func_elts)pointer_to_func(code);
1001
1002 if (gallivm_debug & GALLIVM_DEBUG_ASM) {
1003 lp_disassemble(code);
1004 }
1005 lp_func_delete_body(variant->function_elts);
1006 }
1007
1008
1009 struct draw_llvm_variant_key *
1010 draw_llvm_make_variant_key(struct draw_llvm *llvm, char *store)
1011 {
1012 unsigned i;
1013 struct draw_llvm_variant_key *key;
1014 struct lp_sampler_static_state *sampler;
1015
1016 key = (struct draw_llvm_variant_key *)store;
1017
1018 /* Presumably all variants of the shader should have the same
1019 * number of vertex elements - ie the number of shader inputs.
1020 */
1021 key->nr_vertex_elements = llvm->draw->pt.nr_vertex_elements;
1022
1023 /* All variants of this shader will have the same value for
1024 * nr_samplers. Not yet trying to compact away holes in the
1025 * sampler array.
1026 */
1027 key->nr_samplers = llvm->draw->vs.vertex_shader->info.file_max[TGSI_FILE_SAMPLER] + 1;
1028
1029 sampler = draw_llvm_variant_key_samplers(key);
1030
1031 memcpy(key->vertex_element,
1032 llvm->draw->pt.vertex_element,
1033 sizeof(struct pipe_vertex_element) * key->nr_vertex_elements);
1034
1035 memset(sampler, 0, key->nr_samplers * sizeof *sampler);
1036
1037 for (i = 0 ; i < key->nr_samplers; i++) {
1038 lp_sampler_static_state(&sampler[i],
1039 llvm->draw->sampler_views[i],
1040 llvm->draw->samplers[i]);
1041 }
1042
1043 return key;
1044 }
1045
1046 void
1047 draw_llvm_set_mapped_texture(struct draw_context *draw,
1048 unsigned sampler_idx,
1049 uint32_t width, uint32_t height, uint32_t depth,
1050 uint32_t last_level,
1051 uint32_t row_stride[DRAW_MAX_TEXTURE_LEVELS],
1052 uint32_t img_stride[DRAW_MAX_TEXTURE_LEVELS],
1053 const void *data[DRAW_MAX_TEXTURE_LEVELS])
1054 {
1055 unsigned j;
1056 struct draw_jit_texture *jit_tex;
1057
1058 assert(sampler_idx < PIPE_MAX_VERTEX_SAMPLERS);
1059
1060
1061 jit_tex = &draw->llvm->jit_context.textures[sampler_idx];
1062
1063 jit_tex->width = width;
1064 jit_tex->height = height;
1065 jit_tex->depth = depth;
1066 jit_tex->last_level = last_level;
1067
1068 for (j = 0; j <= last_level; j++) {
1069 jit_tex->data[j] = data[j];
1070 jit_tex->row_stride[j] = row_stride[j];
1071 jit_tex->img_stride[j] = img_stride[j];
1072 }
1073 }
1074
1075 void
1076 draw_llvm_destroy_variant(struct draw_llvm_variant *variant)
1077 {
1078 struct draw_llvm *llvm = variant->llvm;
1079 struct draw_context *draw = llvm->draw;
1080
1081 if (variant->function_elts) {
1082 if (variant->function_elts)
1083 LLVMFreeMachineCodeForFunction(draw->engine,
1084 variant->function_elts);
1085 LLVMDeleteFunction(variant->function_elts);
1086 }
1087
1088 if (variant->function) {
1089 if (variant->function)
1090 LLVMFreeMachineCodeForFunction(draw->engine,
1091 variant->function);
1092 LLVMDeleteFunction(variant->function);
1093 }
1094
1095 remove_from_list(&variant->list_item_local);
1096 variant->shader->variants_cached--;
1097 remove_from_list(&variant->list_item_global);
1098 llvm->nr_variants--;
1099 FREE(variant);
1100 }