projects / mesa.git / blob
? search:


2c234285b5e0b968b201d754ca08bd38c7dae889
[mesa.git] / src / gallium / auxiliary / draw / draw_llvm.c
1 #include "draw_llvm.h"
2
3 #include "draw_context.h"
4 #include "draw_vs.h"
5
6 #include "gallivm/lp_bld_arit.h"
7 #include "gallivm/lp_bld_struct.h"
8 #include "gallivm/lp_bld_type.h"
9 #include "gallivm/lp_bld_flow.h"
10 #include "gallivm/lp_bld_debug.h"
11 #include "gallivm/lp_bld_tgsi.h"
12 #include "gallivm/lp_bld_printf.h"
13
14 #include "tgsi/tgsi_exec.h"
15
16 #include "util/u_cpu_detect.h"
17 #include "util/u_string.h"
18
19 #include <llvm-c/Transforms/Scalar.h>
20
21 #define DEBUG_STORE 0
22
23
24 /* generates the draw jit function */
25 static void
26 draw_llvm_generate(struct draw_llvm *llvm, struct draw_llvm_variant *var);
27 static void
28 draw_llvm_generate_elts(struct draw_llvm *llvm, struct draw_llvm_variant *var);
29
30 static void
31 init_globals(struct draw_llvm *llvm)
32 {
33 LLVMTypeRef texture_type;
34
35 /* struct draw_jit_texture */
36 {
37 LLVMTypeRef elem_types[4];
38
39 elem_types[DRAW_JIT_TEXTURE_WIDTH] = LLVMInt32Type();
40 elem_types[DRAW_JIT_TEXTURE_HEIGHT] = LLVMInt32Type();
41 elem_types[DRAW_JIT_TEXTURE_STRIDE] = LLVMInt32Type();
42 elem_types[DRAW_JIT_TEXTURE_DATA] = LLVMPointerType(LLVMInt8Type(), 0);
43
44 texture_type = LLVMStructType(elem_types, Elements(elem_types), 0);
45
46 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, width,
47 llvm->target, texture_type,
48 DRAW_JIT_TEXTURE_WIDTH);
49 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, height,
50 llvm->target, texture_type,
51 DRAW_JIT_TEXTURE_HEIGHT);
52 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, stride,
53 llvm->target, texture_type,
54 DRAW_JIT_TEXTURE_STRIDE);
55 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, data,
56 llvm->target, texture_type,
57 DRAW_JIT_TEXTURE_DATA);
58 LP_CHECK_STRUCT_SIZE(struct draw_jit_texture,
59 llvm->target, texture_type);
60
61 LLVMAddTypeName(llvm->module, "texture", texture_type);
62 }
63
64
65 /* struct draw_jit_context */
66 {
67 LLVMTypeRef elem_types[3];
68 LLVMTypeRef context_type;
69
70 elem_types[0] = LLVMPointerType(LLVMFloatType(), 0); /* vs_constants */
71 elem_types[1] = LLVMPointerType(LLVMFloatType(), 0); /* vs_constants */
72 elem_types[2] = LLVMArrayType(texture_type, PIPE_MAX_SAMPLERS); /* textures */
73
74 context_type = LLVMStructType(elem_types, Elements(elem_types), 0);
75
76 LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, vs_constants,
77 llvm->target, context_type, 0);
78 LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, gs_constants,
79 llvm->target, context_type, 1);
80 LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, textures,
81 llvm->target, context_type,
82 DRAW_JIT_CONTEXT_TEXTURES_INDEX);
83 LP_CHECK_STRUCT_SIZE(struct draw_jit_context,
84 llvm->target, context_type);
85
86 LLVMAddTypeName(llvm->module, "draw_jit_context", context_type);
87
88 llvm->context_ptr_type = LLVMPointerType(context_type, 0);
89 }
90 {
91 LLVMTypeRef buffer_ptr = LLVMPointerType(LLVMIntType(8), 0);
92 llvm->buffer_ptr_type = LLVMPointerType(buffer_ptr, 0);
93 }
94 /* struct pipe_vertex_buffer */
95 {
96 LLVMTypeRef elem_types[4];
97 LLVMTypeRef vb_type;
98
99 elem_types[0] = LLVMInt32Type();
100 elem_types[1] = LLVMInt32Type();
101 elem_types[2] = LLVMInt32Type();
102 elem_types[3] = LLVMPointerType(LLVMOpaqueType(), 0); /* vs_constants */
103
104 vb_type = LLVMStructType(elem_types, Elements(elem_types), 0);
105
106 LP_CHECK_MEMBER_OFFSET(struct pipe_vertex_buffer, stride,
107 llvm->target, vb_type, 0);
108 LP_CHECK_MEMBER_OFFSET(struct pipe_vertex_buffer, buffer_offset,
109 llvm->target, vb_type, 2);
110 LP_CHECK_STRUCT_SIZE(struct pipe_vertex_buffer,
111 llvm->target, vb_type);
112
113 LLVMAddTypeName(llvm->module, "pipe_vertex_buffer", vb_type);
114
115 llvm->vb_ptr_type = LLVMPointerType(vb_type, 0);
116 }
117 }
118
119 static LLVMTypeRef
120 create_vertex_header(struct draw_llvm *llvm, int data_elems)
121 {
122 /* struct vertex_header */
123 LLVMTypeRef elem_types[3];
124 LLVMTypeRef vertex_header;
125 char struct_name[24];
126
127 util_snprintf(struct_name, 23, "vertex_header%d", data_elems);
128
129 elem_types[0] = LLVMIntType(32);
130 elem_types[1] = LLVMArrayType(LLVMFloatType(), 4);
131 elem_types[2] = LLVMArrayType(elem_types[1], data_elems);
132
133 vertex_header = LLVMStructType(elem_types, Elements(elem_types), 0);
134
135 /* these are bit-fields and we can't take address of them
136 LP_CHECK_MEMBER_OFFSET(struct vertex_header, clipmask,
137 llvm->target, vertex_header,
138 DRAW_JIT_VERTEX_CLIPMASK);
139 LP_CHECK_MEMBER_OFFSET(struct vertex_header, edgeflag,
140 llvm->target, vertex_header,
141 DRAW_JIT_VERTEX_EDGEFLAG);
142 LP_CHECK_MEMBER_OFFSET(struct vertex_header, pad,
143 llvm->target, vertex_header,
144 DRAW_JIT_VERTEX_PAD);
145 LP_CHECK_MEMBER_OFFSET(struct vertex_header, vertex_id,
146 llvm->target, vertex_header,
147 DRAW_JIT_VERTEX_VERTEX_ID);
148 */
149 LP_CHECK_MEMBER_OFFSET(struct vertex_header, clip,
150 llvm->target, vertex_header,
151 DRAW_JIT_VERTEX_CLIP);
152 LP_CHECK_MEMBER_OFFSET(struct vertex_header, data,
153 llvm->target, vertex_header,
154 DRAW_JIT_VERTEX_DATA);
155
156 LLVMAddTypeName(llvm->module, struct_name, vertex_header);
157
158 return LLVMPointerType(vertex_header, 0);
159 }
160
161 struct draw_llvm *
162 draw_llvm_create(struct draw_context *draw)
163 {
164 struct draw_llvm *llvm = CALLOC_STRUCT( draw_llvm );
165
166 util_cpu_detect();
167
168 llvm->draw = draw;
169 llvm->engine = draw->engine;
170
171 debug_assert(llvm->engine);
172
173 llvm->module = LLVMModuleCreateWithName("draw_llvm");
174 llvm->provider = LLVMCreateModuleProviderForExistingModule(llvm->module);
175
176 LLVMAddModuleProvider(llvm->engine, llvm->provider);
177
178 llvm->target = LLVMGetExecutionEngineTargetData(llvm->engine);
179
180 llvm->pass = LLVMCreateFunctionPassManager(llvm->provider);
181 LLVMAddTargetData(llvm->target, llvm->pass);
182 /* These are the passes currently listed in llvm-c/Transforms/Scalar.h,
183 * but there are more on SVN. */
184 /* TODO: Add more passes */
185 LLVMAddConstantPropagationPass(llvm->pass);
186 if(util_cpu_caps.has_sse4_1) {
187 /* FIXME: There is a bug in this pass, whereby the combination of fptosi
188 * and sitofp (necessary for trunc/floor/ceil/round implementation)
189 * somehow becomes invalid code.
190 */
191 LLVMAddInstructionCombiningPass(llvm->pass);
192 }
193 LLVMAddPromoteMemoryToRegisterPass(llvm->pass);
194 LLVMAddGVNPass(llvm->pass);
195 LLVMAddCFGSimplificationPass(llvm->pass);
196
197 init_globals(llvm);
198
199
200 #if 0
201 LLVMDumpModule(llvm->module);
202 #endif
203
204 return llvm;
205 }
206
207 void
208 draw_llvm_destroy(struct draw_llvm *llvm)
209 {
210 LLVMDisposePassManager(llvm->pass);
211
212 FREE(llvm);
213 }
214
215 struct draw_llvm_variant *
216 draw_llvm_prepare(struct draw_llvm *llvm, int num_inputs)
217 {
218 struct draw_llvm_variant *variant = MALLOC(sizeof(struct draw_llvm_variant));
219
220 draw_llvm_make_variant_key(llvm, &variant->key);
221
222 llvm->vertex_header_ptr_type = create_vertex_header(llvm, num_inputs);
223
224 draw_llvm_generate(llvm, variant);
225 draw_llvm_generate_elts(llvm, variant);
226
227 return variant;
228 }
229
230 static void
231 generate_vs(struct draw_llvm *llvm,
232 LLVMBuilderRef builder,
233 LLVMValueRef (*outputs)[NUM_CHANNELS],
234 const LLVMValueRef (*inputs)[NUM_CHANNELS],
235 LLVMValueRef context_ptr)
236 {
237 const struct tgsi_token *tokens = llvm->draw->vs.vertex_shader->state.tokens;
238 struct lp_type vs_type;
239 LLVMValueRef consts_ptr = draw_jit_context_vs_constants(builder, context_ptr);
240
241 memset(&vs_type, 0, sizeof vs_type);
242 vs_type.floating = TRUE; /* floating point values */
243 vs_type.sign = TRUE; /* values are signed */
244 vs_type.norm = FALSE; /* values are not limited to [0,1] or [-1,1] */
245 vs_type.width = 32; /* 32-bit float */
246 vs_type.length = 4; /* 4 elements per vector */
247 #if 0
248 num_vs = 4; /* number of vertices per block */
249 #endif
250
251 /*tgsi_dump(tokens, 0);*/
252 lp_build_tgsi_soa(builder,
253 tokens,
254 vs_type,
255 NULL /*struct lp_build_mask_context *mask*/,
256 consts_ptr,
257 NULL /*pos*/,
258 inputs,
259 outputs,
260 NULL/*sampler*/,
261 &llvm->draw->vs.vertex_shader->info);
262 }
263
264 #if DEBUG_STORE
265 static void print_vectorf(LLVMBuilderRef builder,
266 LLVMValueRef vec)
267 {
268 LLVMValueRef val[4];
269 val[0] = LLVMBuildExtractElement(builder, vec,
270 LLVMConstInt(LLVMInt32Type(), 0, 0), "");
271 val[1] = LLVMBuildExtractElement(builder, vec,
272 LLVMConstInt(LLVMInt32Type(), 1, 0), "");
273 val[2] = LLVMBuildExtractElement(builder, vec,
274 LLVMConstInt(LLVMInt32Type(), 2, 0), "");
275 val[3] = LLVMBuildExtractElement(builder, vec,
276 LLVMConstInt(LLVMInt32Type(), 3, 0), "");
277 lp_build_printf(builder, "vector = [%f, %f, %f, %f]\n",
278 val[0], val[1], val[2], val[3]);
279 }
280 #endif
281
282 static void
283 generate_fetch(LLVMBuilderRef builder,
284 LLVMValueRef vbuffers_ptr,
285 LLVMValueRef *res,
286 struct pipe_vertex_element *velem,
287 LLVMValueRef vbuf,
288 LLVMValueRef index)
289 {
290 LLVMValueRef indices = LLVMConstInt(LLVMInt64Type(), velem->vertex_buffer_index, 0);
291 LLVMValueRef vbuffer_ptr = LLVMBuildGEP(builder, vbuffers_ptr,
292 &indices, 1, "");
293 LLVMValueRef vb_stride = draw_jit_vbuffer_stride(builder, vbuf);
294 LLVMValueRef vb_max_index = draw_jit_vbuffer_max_index(builder, vbuf);
295 LLVMValueRef vb_buffer_offset = draw_jit_vbuffer_offset(builder, vbuf);
296 LLVMValueRef cond;
297 LLVMValueRef stride;
298
299 cond = LLVMBuildICmp(builder, LLVMIntULE, index, vb_max_index, "");
300
301 index = LLVMBuildSelect(builder, cond, index, vb_max_index, "");
302
303 stride = LLVMBuildMul(builder, vb_stride, index, "");
304
305 vbuffer_ptr = LLVMBuildLoad(builder, vbuffer_ptr, "vbuffer");
306
307 stride = LLVMBuildAdd(builder, stride,
308 vb_buffer_offset,
309 "");
310 stride = LLVMBuildAdd(builder, stride,
311 LLVMConstInt(LLVMInt32Type(), velem->src_offset, 0),
312 "");
313
314 /*lp_build_printf(builder, "vbuf index = %d, stride is %d\n", indices, stride);*/
315 vbuffer_ptr = LLVMBuildGEP(builder, vbuffer_ptr, &stride, 1, "");
316
317 *res = draw_llvm_translate_from(builder, vbuffer_ptr, velem->src_format);
318 }
319
320 static LLVMValueRef
321 aos_to_soa(LLVMBuilderRef builder,
322 LLVMValueRef val0,
323 LLVMValueRef val1,
324 LLVMValueRef val2,
325 LLVMValueRef val3,
326 LLVMValueRef channel)
327 {
328 LLVMValueRef ex, res;
329
330 ex = LLVMBuildExtractElement(builder, val0,
331 channel, "");
332 res = LLVMBuildInsertElement(builder,
333 LLVMConstNull(LLVMTypeOf(val0)),
334 ex,
335 LLVMConstInt(LLVMInt32Type(), 0, 0),
336 "");
337
338 ex = LLVMBuildExtractElement(builder, val1,
339 channel, "");
340 res = LLVMBuildInsertElement(builder,
341 res, ex,
342 LLVMConstInt(LLVMInt32Type(), 1, 0),
343 "");
344
345 ex = LLVMBuildExtractElement(builder, val2,
346 channel, "");
347 res = LLVMBuildInsertElement(builder,
348 res, ex,
349 LLVMConstInt(LLVMInt32Type(), 2, 0),
350 "");
351
352 ex = LLVMBuildExtractElement(builder, val3,
353 channel, "");
354 res = LLVMBuildInsertElement(builder,
355 res, ex,
356 LLVMConstInt(LLVMInt32Type(), 3, 0),
357 "");
358
359 return res;
360 }
361
362 static void
363 soa_to_aos(LLVMBuilderRef builder,
364 LLVMValueRef soa[NUM_CHANNELS],
365 LLVMValueRef aos[NUM_CHANNELS])
366 {
367 LLVMValueRef comp;
368 int i = 0;
369
370 debug_assert(NUM_CHANNELS == 4);
371
372 aos[0] = LLVMConstNull(LLVMTypeOf(soa[0]));
373 aos[1] = aos[2] = aos[3] = aos[0];
374
375 for (i = 0; i < NUM_CHANNELS; ++i) {
376 LLVMValueRef channel = LLVMConstInt(LLVMInt32Type(), i, 0);
377
378 comp = LLVMBuildExtractElement(builder, soa[i],
379 LLVMConstInt(LLVMInt32Type(), 0, 0), "");
380 aos[0] = LLVMBuildInsertElement(builder, aos[0], comp, channel, "");
381
382 comp = LLVMBuildExtractElement(builder, soa[i],
383 LLVMConstInt(LLVMInt32Type(), 1, 0), "");
384 aos[1] = LLVMBuildInsertElement(builder, aos[1], comp, channel, "");
385
386 comp = LLVMBuildExtractElement(builder, soa[i],
387 LLVMConstInt(LLVMInt32Type(), 2, 0), "");
388 aos[2] = LLVMBuildInsertElement(builder, aos[2], comp, channel, "");
389
390 comp = LLVMBuildExtractElement(builder, soa[i],
391 LLVMConstInt(LLVMInt32Type(), 3, 0), "");
392 aos[3] = LLVMBuildInsertElement(builder, aos[3], comp, channel, "");
393
394 }
395 }
396
397 static void
398 convert_to_soa(LLVMBuilderRef builder,
399 LLVMValueRef (*aos)[NUM_CHANNELS],
400 LLVMValueRef (*soa)[NUM_CHANNELS],
401 int num_attribs)
402 {
403 int i;
404
405 debug_assert(NUM_CHANNELS == 4);
406
407 for (i = 0; i < num_attribs; ++i) {
408 LLVMValueRef val0 = aos[i][0];
409 LLVMValueRef val1 = aos[i][1];
410 LLVMValueRef val2 = aos[i][2];
411 LLVMValueRef val3 = aos[i][3];
412
413 soa[i][0] = aos_to_soa(builder, val0, val1, val2, val3,
414 LLVMConstInt(LLVMInt32Type(), 0, 0));
415 soa[i][1] = aos_to_soa(builder, val0, val1, val2, val3,
416 LLVMConstInt(LLVMInt32Type(), 1, 0));
417 soa[i][2] = aos_to_soa(builder, val0, val1, val2, val3,
418 LLVMConstInt(LLVMInt32Type(), 2, 0));
419 soa[i][3] = aos_to_soa(builder, val0, val1, val2, val3,
420 LLVMConstInt(LLVMInt32Type(), 3, 0));
421 }
422 }
423
424 static void
425 store_aos(LLVMBuilderRef builder,
426 LLVMValueRef io_ptr,
427 LLVMValueRef index,
428 LLVMValueRef value)
429 {
430 LLVMValueRef id_ptr = draw_jit_header_id(builder, io_ptr);
431 LLVMValueRef data_ptr = draw_jit_header_data(builder, io_ptr);
432 LLVMValueRef indices[3];
433
434 indices[0] = LLVMConstInt(LLVMInt32Type(), 0, 0);
435 indices[1] = index;
436 indices[2] = LLVMConstInt(LLVMInt32Type(), 0, 0);
437
438 /* undefined vertex */
439 LLVMBuildStore(builder, LLVMConstInt(LLVMInt32Type(),
440 0xffff, 0), id_ptr);
441
442 #if DEBUG_STORE
443 lp_build_printf(builder, " ---- %p storing attribute %d (io = %p)\n", data_ptr, index, io_ptr);
444 #endif
445 #if 0
446 /*lp_build_printf(builder, " ---- %p storing at %d (%p) ", io_ptr, index, data_ptr);
447 print_vectorf(builder, value);*/
448 data_ptr = LLVMBuildBitCast(builder, data_ptr,
449 LLVMPointerType(LLVMArrayType(LLVMVectorType(LLVMFloatType(), 4), 0), 0),
450 "datavec");
451 data_ptr = LLVMBuildGEP(builder, data_ptr, indices, 2, "");
452
453 LLVMBuildStore(builder, value, data_ptr);
454 #else
455 {
456 LLVMValueRef x, y, z, w;
457 LLVMValueRef idx0, idx1, idx2, idx3;
458 LLVMValueRef gep0, gep1, gep2, gep3;
459 data_ptr = LLVMBuildGEP(builder, data_ptr, indices, 3, "");
460
461 idx0 = LLVMConstInt(LLVMInt32Type(), 0, 0);
462 idx1 = LLVMConstInt(LLVMInt32Type(), 1, 0);
463 idx2 = LLVMConstInt(LLVMInt32Type(), 2, 0);
464 idx3 = LLVMConstInt(LLVMInt32Type(), 3, 0);
465
466 x = LLVMBuildExtractElement(builder, value,
467 idx0, "");
468 y = LLVMBuildExtractElement(builder, value,
469 idx1, "");
470 z = LLVMBuildExtractElement(builder, value,
471 idx2, "");
472 w = LLVMBuildExtractElement(builder, value,
473 idx3, "");
474
475 gep0 = LLVMBuildGEP(builder, data_ptr, &idx0, 1, "");
476 gep1 = LLVMBuildGEP(builder, data_ptr, &idx1, 1, "");
477 gep2 = LLVMBuildGEP(builder, data_ptr, &idx2, 1, "");
478 gep3 = LLVMBuildGEP(builder, data_ptr, &idx3, 1, "");
479
480 /*lp_build_printf(builder, "##### x = %f (%p), y = %f (%p), z = %f (%p), w = %f (%p)\n",
481 x, gep0, y, gep1, z, gep2, w, gep3);*/
482 LLVMBuildStore(builder, x, gep0);
483 LLVMBuildStore(builder, y, gep1);
484 LLVMBuildStore(builder, z, gep2);
485 LLVMBuildStore(builder, w, gep3);
486 }
487 #endif
488 }
489
490 static void
491 store_aos_array(LLVMBuilderRef builder,
492 LLVMValueRef io_ptr,
493 LLVMValueRef aos[NUM_CHANNELS],
494 int attrib,
495 int num_outputs)
496 {
497 LLVMValueRef attr_index = LLVMConstInt(LLVMInt32Type(), attrib, 0);
498 LLVMValueRef ind0 = LLVMConstInt(LLVMInt32Type(), 0, 0);
499 LLVMValueRef ind1 = LLVMConstInt(LLVMInt32Type(), 1, 0);
500 LLVMValueRef ind2 = LLVMConstInt(LLVMInt32Type(), 2, 0);
501 LLVMValueRef ind3 = LLVMConstInt(LLVMInt32Type(), 3, 0);
502 LLVMValueRef io0_ptr, io1_ptr, io2_ptr, io3_ptr;
503
504 debug_assert(NUM_CHANNELS == 4);
505
506 io0_ptr = LLVMBuildGEP(builder, io_ptr,
507 &ind0, 1, "");
508 io1_ptr = LLVMBuildGEP(builder, io_ptr,
509 &ind1, 1, "");
510 io2_ptr = LLVMBuildGEP(builder, io_ptr,
511 &ind2, 1, "");
512 io3_ptr = LLVMBuildGEP(builder, io_ptr,
513 &ind3, 1, "");
514
515 #if DEBUG_STORE
516 lp_build_printf(builder, " io = %p, indexes[%d, %d, %d, %d]\n",
517 io_ptr, ind0, ind1, ind2, ind3);
518 #endif
519
520 store_aos(builder, io0_ptr, attr_index, aos[0]);
521 store_aos(builder, io1_ptr, attr_index, aos[1]);
522 store_aos(builder, io2_ptr, attr_index, aos[2]);
523 store_aos(builder, io3_ptr, attr_index, aos[3]);
524 }
525
526 static void
527 convert_to_aos(LLVMBuilderRef builder,
528 LLVMValueRef io,
529 LLVMValueRef (*outputs)[NUM_CHANNELS],
530 int num_outputs,
531 int max_vertices)
532 {
533 unsigned chan, attrib;
534
535 #if DEBUG_STORE
536 lp_build_printf(builder, " # storing begin\n");
537 #endif
538 for (attrib = 0; attrib < num_outputs; ++attrib) {
539 LLVMValueRef soa[4];
540 LLVMValueRef aos[4];
541 for(chan = 0; chan < NUM_CHANNELS; ++chan) {
542 if(outputs[attrib][chan]) {
543 LLVMValueRef out = LLVMBuildLoad(builder, outputs[attrib][chan], "");
544 lp_build_name(out, "output%u.%c", attrib, "xyzw"[chan]);
545 /*lp_build_printf(builder, "output %d : %d ",
546 LLVMConstInt(LLVMInt32Type(), attrib, 0),
547 LLVMConstInt(LLVMInt32Type(), chan, 0));
548 print_vectorf(builder, out);*/
549 soa[chan] = out;
550 } else
551 soa[chan] = 0;
552 }
553 soa_to_aos(builder, soa, aos);
554 store_aos_array(builder,
555 io,
556 aos,
557 attrib,
558 num_outputs);
559 }
560 #if DEBUG_STORE
561 lp_build_printf(builder, " # storing end\n");
562 #endif
563 }
564
565 static void
566 draw_llvm_generate(struct draw_llvm *llvm, struct draw_llvm_variant *variant)
567 {
568 LLVMTypeRef arg_types[7];
569 LLVMTypeRef func_type;
570 LLVMValueRef context_ptr;
571 LLVMBasicBlockRef block;
572 LLVMBuilderRef builder;
573 LLVMValueRef start, end, count, stride, step, io_itr;
574 LLVMValueRef io_ptr, vbuffers_ptr, vb_ptr;
575 struct draw_context *draw = llvm->draw;
576 unsigned i, j;
577 struct lp_build_context bld;
578 struct lp_build_loop_state lp_loop;
579 struct lp_type vs_type = lp_type_float_vec(32);
580 const int max_vertices = 4;
581 LLVMValueRef outputs[PIPE_MAX_SHADER_OUTPUTS][NUM_CHANNELS];
582
583 arg_types[0] = llvm->context_ptr_type; /* context */
584 arg_types[1] = llvm->vertex_header_ptr_type; /* vertex_header */
585 arg_types[2] = llvm->buffer_ptr_type; /* vbuffers */
586 arg_types[3] = LLVMInt32Type(); /* start */
587 arg_types[4] = LLVMInt32Type(); /* count */
588 arg_types[5] = LLVMInt32Type(); /* stride */
589 arg_types[6] = llvm->vb_ptr_type; /* pipe_vertex_buffer's */
590
591 func_type = LLVMFunctionType(LLVMVoidType(), arg_types, Elements(arg_types), 0);
592
593 variant->function = LLVMAddFunction(llvm->module, "draw_llvm_shader", func_type);
594 LLVMSetFunctionCallConv(variant->function, LLVMCCallConv);
595 for(i = 0; i < Elements(arg_types); ++i)
596 if(LLVMGetTypeKind(arg_types[i]) == LLVMPointerTypeKind)
597 LLVMAddAttribute(LLVMGetParam(variant->function, i), LLVMNoAliasAttribute);
598
599 context_ptr = LLVMGetParam(variant->function, 0);
600 io_ptr = LLVMGetParam(variant->function, 1);
601 vbuffers_ptr = LLVMGetParam(variant->function, 2);
602 start = LLVMGetParam(variant->function, 3);
603 count = LLVMGetParam(variant->function, 4);
604 stride = LLVMGetParam(variant->function, 5);
605 vb_ptr = LLVMGetParam(variant->function, 6);
606
607 lp_build_name(context_ptr, "context");
608 lp_build_name(io_ptr, "io");
609 lp_build_name(vbuffers_ptr, "vbuffers");
610 lp_build_name(start, "start");
611 lp_build_name(count, "count");
612 lp_build_name(stride, "stride");
613 lp_build_name(vb_ptr, "vb");
614
615 /*
616 * Function body
617 */
618
619 block = LLVMAppendBasicBlock(variant->function, "entry");
620 builder = LLVMCreateBuilder();
621 LLVMPositionBuilderAtEnd(builder, block);
622
623 lp_build_context_init(&bld, builder, vs_type);
624
625 end = lp_build_add(&bld, start, count);
626
627 step = LLVMConstInt(LLVMInt32Type(), max_vertices, 0);
628
629 #if DEBUG_STORE
630 lp_build_printf(builder, "start = %d, end = %d, step = %d\n",
631 start, end, step);
632 #endif
633 lp_build_loop_begin(builder, start, &lp_loop);
634 {
635 LLVMValueRef inputs[PIPE_MAX_SHADER_INPUTS][NUM_CHANNELS];
636 LLVMValueRef aos_attribs[PIPE_MAX_SHADER_INPUTS][NUM_CHANNELS] = { { 0 } };
637 LLVMValueRef io;
638 const LLVMValueRef (*ptr_aos)[NUM_CHANNELS];
639
640 io_itr = LLVMBuildSub(builder, lp_loop.counter, start, "");
641 io = LLVMBuildGEP(builder, io_ptr, &io_itr, 1, "");
642 #if DEBUG_STORE
643 lp_build_printf(builder, " --- io %d = %p, loop counter %d\n",
644 io_itr, io, lp_loop.counter);
645 #endif
646 for (i = 0; i < NUM_CHANNELS; ++i) {
647 LLVMValueRef true_index = LLVMBuildAdd(
648 builder,
649 lp_loop.counter,
650 LLVMConstInt(LLVMInt32Type(), i, 0), "");
651 for (j = 0; j < draw->pt.nr_vertex_elements; ++j) {
652 struct pipe_vertex_element *velem = &draw->pt.vertex_element[j];
653 LLVMValueRef vb_index = LLVMConstInt(LLVMInt32Type(),
654 velem->vertex_buffer_index,
655 0);
656 LLVMValueRef vb = LLVMBuildGEP(builder, vb_ptr,
657 &vb_index, 1, "");
658 generate_fetch(builder, vbuffers_ptr,
659 &aos_attribs[j][i], velem, vb, true_index);
660 }
661 }
662 convert_to_soa(builder, aos_attribs, inputs,
663 draw->pt.nr_vertex_elements);
664
665 ptr_aos = (const LLVMValueRef (*)[NUM_CHANNELS]) inputs;
666 generate_vs(llvm,
667 builder,
668 outputs,
669 ptr_aos,
670 context_ptr);
671
672 convert_to_aos(builder, io, outputs,
673 draw->vs.vertex_shader->info.num_outputs,
674 max_vertices);
675 }
676 lp_build_loop_end_cond(builder, end, step, LLVMIntUGE, &lp_loop);
677
678 LLVMBuildRetVoid(builder);
679
680 LLVMDisposeBuilder(builder);
681
682 /*
683 * Translate the LLVM IR into machine code.
684 */
685 #ifdef DEBUG
686 if(LLVMVerifyFunction(variant->function, LLVMPrintMessageAction)) {
687 LLVMDumpValue(variant->function);
688 assert(0);
689 }
690 #endif
691
692 LLVMRunFunctionPassManager(llvm->pass, variant->function);
693
694 if (0) {
695 LLVMDumpValue(variant->function);
696 debug_printf("\n");
697 }
698 variant->jit_func = (draw_jit_vert_func)LLVMGetPointerToGlobal(llvm->draw->engine, variant->function);
699
700 if (0)
701 lp_disassemble(variant->jit_func);
702 }
703
704
705 static void
706 draw_llvm_generate_elts(struct draw_llvm *llvm, struct draw_llvm_variant *variant)
707 {
708 LLVMTypeRef arg_types[7];
709 LLVMTypeRef func_type;
710 LLVMValueRef context_ptr;
711 LLVMBasicBlockRef block;
712 LLVMBuilderRef builder;
713 LLVMValueRef fetch_elts, fetch_count, stride, step, io_itr;
714 LLVMValueRef io_ptr, vbuffers_ptr, vb_ptr;
715 struct draw_context *draw = llvm->draw;
716 unsigned i, j;
717 struct lp_build_context bld;
718 struct lp_build_context bld_int;
719 struct lp_build_loop_state lp_loop;
720 struct lp_type vs_type = lp_type_float_vec(32);
721 const int max_vertices = 4;
722 LLVMValueRef outputs[PIPE_MAX_SHADER_OUTPUTS][NUM_CHANNELS];
723 LLVMValueRef fetch_max;
724
725 arg_types[0] = llvm->context_ptr_type; /* context */
726 arg_types[1] = llvm->vertex_header_ptr_type; /* vertex_header */
727 arg_types[2] = llvm->buffer_ptr_type; /* vbuffers */
728 arg_types[3] = LLVMPointerType(LLVMInt32Type(), 0); /* fetch_elts * */
729 arg_types[4] = LLVMInt32Type(); /* fetch_count */
730 arg_types[5] = LLVMInt32Type(); /* stride */
731 arg_types[6] = llvm->vb_ptr_type; /* pipe_vertex_buffer's */
732
733 func_type = LLVMFunctionType(LLVMVoidType(), arg_types, Elements(arg_types), 0);
734
735 variant->function_elts = LLVMAddFunction(llvm->module, "draw_llvm_shader_elts", func_type);
736 LLVMSetFunctionCallConv(variant->function_elts, LLVMCCallConv);
737 for(i = 0; i < Elements(arg_types); ++i)
738 if(LLVMGetTypeKind(arg_types[i]) == LLVMPointerTypeKind)
739 LLVMAddAttribute(LLVMGetParam(variant->function_elts, i), LLVMNoAliasAttribute);
740
741 context_ptr = LLVMGetParam(variant->function_elts, 0);
742 io_ptr = LLVMGetParam(variant->function_elts, 1);
743 vbuffers_ptr = LLVMGetParam(variant->function_elts, 2);
744 fetch_elts = LLVMGetParam(variant->function_elts, 3);
745 fetch_count = LLVMGetParam(variant->function_elts, 4);
746 stride = LLVMGetParam(variant->function_elts, 5);
747 vb_ptr = LLVMGetParam(variant->function_elts, 6);
748
749 lp_build_name(context_ptr, "context");
750 lp_build_name(io_ptr, "io");
751 lp_build_name(vbuffers_ptr, "vbuffers");
752 lp_build_name(fetch_elts, "fetch_elts");
753 lp_build_name(fetch_count, "fetch_count");
754 lp_build_name(stride, "stride");
755 lp_build_name(vb_ptr, "vb");
756
757 /*
758 * Function body
759 */
760
761 block = LLVMAppendBasicBlock(variant->function_elts, "entry");
762 builder = LLVMCreateBuilder();
763 LLVMPositionBuilderAtEnd(builder, block);
764
765 lp_build_context_init(&bld, builder, vs_type);
766 lp_build_context_init(&bld_int, builder, lp_type_int(32));
767
768 step = LLVMConstInt(LLVMInt32Type(), max_vertices, 0);
769
770 fetch_max = LLVMBuildSub(builder, fetch_count,
771 LLVMConstInt(LLVMInt32Type(), 1, 0),
772 "fetch_max");
773
774 lp_build_loop_begin(builder, LLVMConstInt(LLVMInt32Type(), 0, 0), &lp_loop);
775 {
776 LLVMValueRef inputs[PIPE_MAX_SHADER_INPUTS][NUM_CHANNELS];
777 LLVMValueRef aos_attribs[PIPE_MAX_SHADER_INPUTS][NUM_CHANNELS] = { { 0 } };
778 LLVMValueRef io;
779 const LLVMValueRef (*ptr_aos)[NUM_CHANNELS];
780
781 io_itr = lp_loop.counter;
782 io = LLVMBuildGEP(builder, io_ptr, &io_itr, 1, "");
783 #if DEBUG_STORE
784 lp_build_printf(builder, " --- io %d = %p, loop counter %d\n",
785 io_itr, io, lp_loop.counter);
786 #endif
787 for (i = 0; i < NUM_CHANNELS; ++i) {
788 LLVMValueRef true_index = LLVMBuildAdd(
789 builder,
790 lp_loop.counter,
791 LLVMConstInt(LLVMInt32Type(), i, 0), "");
792 LLVMValueRef fetch_ptr;
793
794 /* make sure we're not out of bounds which can happen
795 * if fetch_count % 4 != 0, because on the last iteration
796 * a few of the 4 vertex fetches will be out of bounds */
797 true_index = lp_build_min(&bld_int, true_index, fetch_max);
798
799 fetch_ptr = LLVMBuildGEP(builder, fetch_elts,
800 &true_index, 1, "");
801 true_index = LLVMBuildLoad(builder, fetch_ptr, "fetch_elt");
802 for (j = 0; j < draw->pt.nr_vertex_elements; ++j) {
803 struct pipe_vertex_element *velem = &draw->pt.vertex_element[j];
804 LLVMValueRef vb_index = LLVMConstInt(LLVMInt32Type(),
805 velem->vertex_buffer_index,
806 0);
807 LLVMValueRef vb = LLVMBuildGEP(builder, vb_ptr,
808 &vb_index, 1, "");
809 generate_fetch(builder, vbuffers_ptr,
810 &aos_attribs[j][i], velem, vb, true_index);
811 }
812 }
813 convert_to_soa(builder, aos_attribs, inputs,
814 draw->pt.nr_vertex_elements);
815
816 ptr_aos = (const LLVMValueRef (*)[NUM_CHANNELS]) inputs;
817 generate_vs(llvm,
818 builder,
819 outputs,
820 ptr_aos,
821 context_ptr);
822
823 convert_to_aos(builder, io, outputs,
824 draw->vs.vertex_shader->info.num_outputs,
825 max_vertices);
826 }
827 lp_build_loop_end_cond(builder, fetch_count, step, LLVMIntUGE, &lp_loop);
828
829 LLVMBuildRetVoid(builder);
830
831 LLVMDisposeBuilder(builder);
832
833 /*
834 * Translate the LLVM IR into machine code.
835 */
836 #ifdef DEBUG
837 if(LLVMVerifyFunction(variant->function_elts, LLVMPrintMessageAction)) {
838 LLVMDumpValue(variant->function_elts);
839 assert(0);
840 }
841 #endif
842
843 LLVMRunFunctionPassManager(llvm->pass, variant->function_elts);
844
845 if (0) {
846 LLVMDumpValue(variant->function_elts);
847 debug_printf("\n");
848 }
849 variant->jit_func_elts = (draw_jit_vert_func_elts)LLVMGetPointerToGlobal(
850 llvm->draw->engine, variant->function_elts);
851
852 if (0)
853 lp_disassemble(variant->jit_func_elts);
854 }
855
856 void
857 draw_llvm_make_variant_key(struct draw_llvm *llvm,
858 struct draw_llvm_variant_key *key)
859 {
860 memset(key, 0, sizeof(struct draw_llvm_variant_key));
861
862 key->nr_vertex_elements = llvm->draw->pt.nr_vertex_elements;
863
864 memcpy(key->vertex_element,
865 llvm->draw->pt.vertex_element,
866 sizeof(struct pipe_vertex_element) * key->nr_vertex_elements);
867
868 memcpy(&key->vs,
869 &llvm->draw->vs.vertex_shader->state,
870 sizeof(struct pipe_shader_state));
871 }