3 #include "draw_context.h"
6 #include "gallivm/lp_bld_arit.h"
7 #include "gallivm/lp_bld_struct.h"
8 #include "gallivm/lp_bld_type.h"
9 #include "gallivm/lp_bld_flow.h"
10 #include "gallivm/lp_bld_debug.h"
11 #include "gallivm/lp_bld_tgsi.h"
12 #include "gallivm/lp_bld_printf.h"
14 #include "tgsi/tgsi_exec.h"
16 #include "util/u_cpu_detect.h"
17 #include "util/u_string.h"
19 #include <llvm-c/Transforms/Scalar.h>
25 static INLINE draw_jit_vert_func_elts
26 voidptr_to_draw_vert_func_elts(void *v
)
30 draw_jit_vert_func_elts f
;
32 assert(sizeof(u
.v
) == sizeof(u
.f
));
39 static INLINE draw_jit_vert_func
40 voidptr_to_draw_jit_vert_func(void *v
)
46 assert(sizeof(u
.v
) == sizeof(u
.f
));
52 /* generates the draw jit function */
54 draw_llvm_generate(struct draw_llvm
*llvm
, struct draw_llvm_variant
*var
);
56 draw_llvm_generate_elts(struct draw_llvm
*llvm
, struct draw_llvm_variant
*var
);
59 init_globals(struct draw_llvm
*llvm
)
61 LLVMTypeRef texture_type
;
63 /* struct draw_jit_texture */
65 LLVMTypeRef elem_types
[4];
67 elem_types
[DRAW_JIT_TEXTURE_WIDTH
] = LLVMInt32Type();
68 elem_types
[DRAW_JIT_TEXTURE_HEIGHT
] = LLVMInt32Type();
69 elem_types
[DRAW_JIT_TEXTURE_STRIDE
] = LLVMInt32Type();
70 elem_types
[DRAW_JIT_TEXTURE_DATA
] = LLVMPointerType(LLVMInt8Type(), 0);
72 texture_type
= LLVMStructType(elem_types
, Elements(elem_types
), 0);
74 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture
, width
,
75 llvm
->target
, texture_type
,
76 DRAW_JIT_TEXTURE_WIDTH
);
77 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture
, height
,
78 llvm
->target
, texture_type
,
79 DRAW_JIT_TEXTURE_HEIGHT
);
80 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture
, stride
,
81 llvm
->target
, texture_type
,
82 DRAW_JIT_TEXTURE_STRIDE
);
83 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture
, data
,
84 llvm
->target
, texture_type
,
85 DRAW_JIT_TEXTURE_DATA
);
86 LP_CHECK_STRUCT_SIZE(struct draw_jit_texture
,
87 llvm
->target
, texture_type
);
89 LLVMAddTypeName(llvm
->module
, "texture", texture_type
);
93 /* struct draw_jit_context */
95 LLVMTypeRef elem_types
[3];
96 LLVMTypeRef context_type
;
98 elem_types
[0] = LLVMPointerType(LLVMFloatType(), 0); /* vs_constants */
99 elem_types
[1] = LLVMPointerType(LLVMFloatType(), 0); /* vs_constants */
100 elem_types
[2] = LLVMArrayType(texture_type
, PIPE_MAX_SAMPLERS
); /* textures */
102 context_type
= LLVMStructType(elem_types
, Elements(elem_types
), 0);
104 LP_CHECK_MEMBER_OFFSET(struct draw_jit_context
, vs_constants
,
105 llvm
->target
, context_type
, 0);
106 LP_CHECK_MEMBER_OFFSET(struct draw_jit_context
, gs_constants
,
107 llvm
->target
, context_type
, 1);
108 LP_CHECK_MEMBER_OFFSET(struct draw_jit_context
, textures
,
109 llvm
->target
, context_type
,
110 DRAW_JIT_CONTEXT_TEXTURES_INDEX
);
111 LP_CHECK_STRUCT_SIZE(struct draw_jit_context
,
112 llvm
->target
, context_type
);
114 LLVMAddTypeName(llvm
->module
, "draw_jit_context", context_type
);
116 llvm
->context_ptr_type
= LLVMPointerType(context_type
, 0);
119 LLVMTypeRef buffer_ptr
= LLVMPointerType(LLVMIntType(8), 0);
120 llvm
->buffer_ptr_type
= LLVMPointerType(buffer_ptr
, 0);
122 /* struct pipe_vertex_buffer */
124 LLVMTypeRef elem_types
[4];
127 elem_types
[0] = LLVMInt32Type();
128 elem_types
[1] = LLVMInt32Type();
129 elem_types
[2] = LLVMInt32Type();
130 elem_types
[3] = LLVMPointerType(LLVMOpaqueType(), 0); /* vs_constants */
132 vb_type
= LLVMStructType(elem_types
, Elements(elem_types
), 0);
134 LP_CHECK_MEMBER_OFFSET(struct pipe_vertex_buffer
, stride
,
135 llvm
->target
, vb_type
, 0);
136 LP_CHECK_MEMBER_OFFSET(struct pipe_vertex_buffer
, buffer_offset
,
137 llvm
->target
, vb_type
, 2);
138 LP_CHECK_STRUCT_SIZE(struct pipe_vertex_buffer
,
139 llvm
->target
, vb_type
);
141 LLVMAddTypeName(llvm
->module
, "pipe_vertex_buffer", vb_type
);
143 llvm
->vb_ptr_type
= LLVMPointerType(vb_type
, 0);
148 create_vertex_header(struct draw_llvm
*llvm
, int data_elems
)
150 /* struct vertex_header */
151 LLVMTypeRef elem_types
[3];
152 LLVMTypeRef vertex_header
;
153 char struct_name
[24];
155 util_snprintf(struct_name
, 23, "vertex_header%d", data_elems
);
157 elem_types
[0] = LLVMIntType(32);
158 elem_types
[1] = LLVMArrayType(LLVMFloatType(), 4);
159 elem_types
[2] = LLVMArrayType(elem_types
[1], data_elems
);
161 vertex_header
= LLVMStructType(elem_types
, Elements(elem_types
), 0);
163 /* these are bit-fields and we can't take address of them
164 LP_CHECK_MEMBER_OFFSET(struct vertex_header, clipmask,
165 llvm->target, vertex_header,
166 DRAW_JIT_VERTEX_CLIPMASK);
167 LP_CHECK_MEMBER_OFFSET(struct vertex_header, edgeflag,
168 llvm->target, vertex_header,
169 DRAW_JIT_VERTEX_EDGEFLAG);
170 LP_CHECK_MEMBER_OFFSET(struct vertex_header, pad,
171 llvm->target, vertex_header,
172 DRAW_JIT_VERTEX_PAD);
173 LP_CHECK_MEMBER_OFFSET(struct vertex_header, vertex_id,
174 llvm->target, vertex_header,
175 DRAW_JIT_VERTEX_VERTEX_ID);
177 LP_CHECK_MEMBER_OFFSET(struct vertex_header
, clip
,
178 llvm
->target
, vertex_header
,
179 DRAW_JIT_VERTEX_CLIP
);
180 LP_CHECK_MEMBER_OFFSET(struct vertex_header
, data
,
181 llvm
->target
, vertex_header
,
182 DRAW_JIT_VERTEX_DATA
);
184 LLVMAddTypeName(llvm
->module
, struct_name
, vertex_header
);
186 return LLVMPointerType(vertex_header
, 0);
190 draw_llvm_create(struct draw_context
*draw
)
192 struct draw_llvm
*llvm
;
196 /* require SSE2 due to LLVM PR6960. */
197 if (!util_cpu_caps
.has_sse2
)
201 llvm
= CALLOC_STRUCT( draw_llvm
);
204 llvm
->engine
= draw
->engine
;
206 debug_assert(llvm
->engine
);
208 llvm
->module
= LLVMModuleCreateWithName("draw_llvm");
209 llvm
->provider
= LLVMCreateModuleProviderForExistingModule(llvm
->module
);
211 LLVMAddModuleProvider(llvm
->engine
, llvm
->provider
);
213 llvm
->target
= LLVMGetExecutionEngineTargetData(llvm
->engine
);
215 llvm
->pass
= LLVMCreateFunctionPassManager(llvm
->provider
);
216 LLVMAddTargetData(llvm
->target
, llvm
->pass
);
217 /* These are the passes currently listed in llvm-c/Transforms/Scalar.h,
218 * but there are more on SVN. */
219 /* TODO: Add more passes */
220 LLVMAddCFGSimplificationPass(llvm
->pass
);
221 LLVMAddPromoteMemoryToRegisterPass(llvm
->pass
);
222 LLVMAddConstantPropagationPass(llvm
->pass
);
223 if(util_cpu_caps
.has_sse4_1
) {
224 /* FIXME: There is a bug in this pass, whereby the combination of fptosi
225 * and sitofp (necessary for trunc/floor/ceil/round implementation)
226 * somehow becomes invalid code.
228 LLVMAddInstructionCombiningPass(llvm
->pass
);
230 LLVMAddGVNPass(llvm
->pass
);
236 LLVMDumpModule(lp_build_module
);
243 draw_llvm_destroy(struct draw_llvm
*llvm
)
245 LLVMDisposePassManager(llvm
->pass
);
250 struct draw_llvm_variant
*
251 draw_llvm_prepare(struct draw_llvm
*llvm
, int num_inputs
)
253 struct draw_llvm_variant
*variant
= MALLOC(sizeof(struct draw_llvm_variant
));
255 draw_llvm_make_variant_key(llvm
, &variant
->key
);
257 llvm
->vertex_header_ptr_type
= create_vertex_header(llvm
, num_inputs
);
259 draw_llvm_generate(llvm
, variant
);
260 draw_llvm_generate_elts(llvm
, variant
);
266 generate_vs(struct draw_llvm
*llvm
,
267 LLVMBuilderRef builder
,
268 LLVMValueRef (*outputs
)[NUM_CHANNELS
],
269 const LLVMValueRef (*inputs
)[NUM_CHANNELS
],
270 LLVMValueRef context_ptr
)
272 const struct tgsi_token
*tokens
= llvm
->draw
->vs
.vertex_shader
->state
.tokens
;
273 struct lp_type vs_type
;
274 LLVMValueRef consts_ptr
= draw_jit_context_vs_constants(builder
, context_ptr
);
276 memset(&vs_type
, 0, sizeof vs_type
);
277 vs_type
.floating
= TRUE
; /* floating point values */
278 vs_type
.sign
= TRUE
; /* values are signed */
279 vs_type
.norm
= FALSE
; /* values are not limited to [0,1] or [-1,1] */
280 vs_type
.width
= 32; /* 32-bit float */
281 vs_type
.length
= 4; /* 4 elements per vector */
283 num_vs
= 4; /* number of vertices per block */
286 /*tgsi_dump(tokens, 0);*/
287 lp_build_tgsi_soa(builder
,
290 NULL
/*struct lp_build_mask_context *mask*/,
296 &llvm
->draw
->vs
.vertex_shader
->info
);
300 static void print_vectorf(LLVMBuilderRef builder
,
304 val
[0] = LLVMBuildExtractElement(builder
, vec
,
305 LLVMConstInt(LLVMInt32Type(), 0, 0), "");
306 val
[1] = LLVMBuildExtractElement(builder
, vec
,
307 LLVMConstInt(LLVMInt32Type(), 1, 0), "");
308 val
[2] = LLVMBuildExtractElement(builder
, vec
,
309 LLVMConstInt(LLVMInt32Type(), 2, 0), "");
310 val
[3] = LLVMBuildExtractElement(builder
, vec
,
311 LLVMConstInt(LLVMInt32Type(), 3, 0), "");
312 lp_build_printf(builder
, "vector = [%f, %f, %f, %f]\n",
313 val
[0], val
[1], val
[2], val
[3]);
318 generate_fetch(LLVMBuilderRef builder
,
319 LLVMValueRef vbuffers_ptr
,
321 struct pipe_vertex_element
*velem
,
325 LLVMValueRef indices
= LLVMConstInt(LLVMInt64Type(), velem
->vertex_buffer_index
, 0);
326 LLVMValueRef vbuffer_ptr
= LLVMBuildGEP(builder
, vbuffers_ptr
,
328 LLVMValueRef vb_stride
= draw_jit_vbuffer_stride(builder
, vbuf
);
329 LLVMValueRef vb_max_index
= draw_jit_vbuffer_max_index(builder
, vbuf
);
330 LLVMValueRef vb_buffer_offset
= draw_jit_vbuffer_offset(builder
, vbuf
);
334 cond
= LLVMBuildICmp(builder
, LLVMIntULE
, index
, vb_max_index
, "");
336 index
= LLVMBuildSelect(builder
, cond
, index
, vb_max_index
, "");
338 stride
= LLVMBuildMul(builder
, vb_stride
, index
, "");
340 vbuffer_ptr
= LLVMBuildLoad(builder
, vbuffer_ptr
, "vbuffer");
342 stride
= LLVMBuildAdd(builder
, stride
,
345 stride
= LLVMBuildAdd(builder
, stride
,
346 LLVMConstInt(LLVMInt32Type(), velem
->src_offset
, 0),
349 /*lp_build_printf(builder, "vbuf index = %d, stride is %d\n", indices, stride);*/
350 vbuffer_ptr
= LLVMBuildGEP(builder
, vbuffer_ptr
, &stride
, 1, "");
352 *res
= draw_llvm_translate_from(builder
, vbuffer_ptr
, velem
->src_format
);
356 aos_to_soa(LLVMBuilderRef builder
,
361 LLVMValueRef channel
)
363 LLVMValueRef ex
, res
;
365 ex
= LLVMBuildExtractElement(builder
, val0
,
367 res
= LLVMBuildInsertElement(builder
,
368 LLVMConstNull(LLVMTypeOf(val0
)),
370 LLVMConstInt(LLVMInt32Type(), 0, 0),
373 ex
= LLVMBuildExtractElement(builder
, val1
,
375 res
= LLVMBuildInsertElement(builder
,
377 LLVMConstInt(LLVMInt32Type(), 1, 0),
380 ex
= LLVMBuildExtractElement(builder
, val2
,
382 res
= LLVMBuildInsertElement(builder
,
384 LLVMConstInt(LLVMInt32Type(), 2, 0),
387 ex
= LLVMBuildExtractElement(builder
, val3
,
389 res
= LLVMBuildInsertElement(builder
,
391 LLVMConstInt(LLVMInt32Type(), 3, 0),
398 soa_to_aos(LLVMBuilderRef builder
,
399 LLVMValueRef soa
[NUM_CHANNELS
],
400 LLVMValueRef aos
[NUM_CHANNELS
])
405 debug_assert(NUM_CHANNELS
== 4);
407 aos
[0] = LLVMConstNull(LLVMTypeOf(soa
[0]));
408 aos
[1] = aos
[2] = aos
[3] = aos
[0];
410 for (i
= 0; i
< NUM_CHANNELS
; ++i
) {
411 LLVMValueRef channel
= LLVMConstInt(LLVMInt32Type(), i
, 0);
413 comp
= LLVMBuildExtractElement(builder
, soa
[i
],
414 LLVMConstInt(LLVMInt32Type(), 0, 0), "");
415 aos
[0] = LLVMBuildInsertElement(builder
, aos
[0], comp
, channel
, "");
417 comp
= LLVMBuildExtractElement(builder
, soa
[i
],
418 LLVMConstInt(LLVMInt32Type(), 1, 0), "");
419 aos
[1] = LLVMBuildInsertElement(builder
, aos
[1], comp
, channel
, "");
421 comp
= LLVMBuildExtractElement(builder
, soa
[i
],
422 LLVMConstInt(LLVMInt32Type(), 2, 0), "");
423 aos
[2] = LLVMBuildInsertElement(builder
, aos
[2], comp
, channel
, "");
425 comp
= LLVMBuildExtractElement(builder
, soa
[i
],
426 LLVMConstInt(LLVMInt32Type(), 3, 0), "");
427 aos
[3] = LLVMBuildInsertElement(builder
, aos
[3], comp
, channel
, "");
433 convert_to_soa(LLVMBuilderRef builder
,
434 LLVMValueRef (*aos
)[NUM_CHANNELS
],
435 LLVMValueRef (*soa
)[NUM_CHANNELS
],
440 debug_assert(NUM_CHANNELS
== 4);
442 for (i
= 0; i
< num_attribs
; ++i
) {
443 LLVMValueRef val0
= aos
[i
][0];
444 LLVMValueRef val1
= aos
[i
][1];
445 LLVMValueRef val2
= aos
[i
][2];
446 LLVMValueRef val3
= aos
[i
][3];
448 soa
[i
][0] = aos_to_soa(builder
, val0
, val1
, val2
, val3
,
449 LLVMConstInt(LLVMInt32Type(), 0, 0));
450 soa
[i
][1] = aos_to_soa(builder
, val0
, val1
, val2
, val3
,
451 LLVMConstInt(LLVMInt32Type(), 1, 0));
452 soa
[i
][2] = aos_to_soa(builder
, val0
, val1
, val2
, val3
,
453 LLVMConstInt(LLVMInt32Type(), 2, 0));
454 soa
[i
][3] = aos_to_soa(builder
, val0
, val1
, val2
, val3
,
455 LLVMConstInt(LLVMInt32Type(), 3, 0));
460 store_aos(LLVMBuilderRef builder
,
465 LLVMValueRef id_ptr
= draw_jit_header_id(builder
, io_ptr
);
466 LLVMValueRef data_ptr
= draw_jit_header_data(builder
, io_ptr
);
467 LLVMValueRef indices
[3];
469 indices
[0] = LLVMConstInt(LLVMInt32Type(), 0, 0);
471 indices
[2] = LLVMConstInt(LLVMInt32Type(), 0, 0);
473 /* undefined vertex */
474 LLVMBuildStore(builder
, LLVMConstInt(LLVMInt32Type(),
478 lp_build_printf(builder
, " ---- %p storing attribute %d (io = %p)\n", data_ptr
, index
, io_ptr
);
481 /*lp_build_printf(builder, " ---- %p storing at %d (%p) ", io_ptr, index, data_ptr);
482 print_vectorf(builder, value);*/
483 data_ptr
= LLVMBuildBitCast(builder
, data_ptr
,
484 LLVMPointerType(LLVMArrayType(LLVMVectorType(LLVMFloatType(), 4), 0), 0),
486 data_ptr
= LLVMBuildGEP(builder
, data_ptr
, indices
, 2, "");
488 LLVMBuildStore(builder
, value
, data_ptr
);
491 LLVMValueRef x
, y
, z
, w
;
492 LLVMValueRef idx0
, idx1
, idx2
, idx3
;
493 LLVMValueRef gep0
, gep1
, gep2
, gep3
;
494 data_ptr
= LLVMBuildGEP(builder
, data_ptr
, indices
, 3, "");
496 idx0
= LLVMConstInt(LLVMInt32Type(), 0, 0);
497 idx1
= LLVMConstInt(LLVMInt32Type(), 1, 0);
498 idx2
= LLVMConstInt(LLVMInt32Type(), 2, 0);
499 idx3
= LLVMConstInt(LLVMInt32Type(), 3, 0);
501 x
= LLVMBuildExtractElement(builder
, value
,
503 y
= LLVMBuildExtractElement(builder
, value
,
505 z
= LLVMBuildExtractElement(builder
, value
,
507 w
= LLVMBuildExtractElement(builder
, value
,
510 gep0
= LLVMBuildGEP(builder
, data_ptr
, &idx0
, 1, "");
511 gep1
= LLVMBuildGEP(builder
, data_ptr
, &idx1
, 1, "");
512 gep2
= LLVMBuildGEP(builder
, data_ptr
, &idx2
, 1, "");
513 gep3
= LLVMBuildGEP(builder
, data_ptr
, &idx3
, 1, "");
515 /*lp_build_printf(builder, "##### x = %f (%p), y = %f (%p), z = %f (%p), w = %f (%p)\n",
516 x, gep0, y, gep1, z, gep2, w, gep3);*/
517 LLVMBuildStore(builder
, x
, gep0
);
518 LLVMBuildStore(builder
, y
, gep1
);
519 LLVMBuildStore(builder
, z
, gep2
);
520 LLVMBuildStore(builder
, w
, gep3
);
526 store_aos_array(LLVMBuilderRef builder
,
528 LLVMValueRef aos
[NUM_CHANNELS
],
532 LLVMValueRef attr_index
= LLVMConstInt(LLVMInt32Type(), attrib
, 0);
533 LLVMValueRef ind0
= LLVMConstInt(LLVMInt32Type(), 0, 0);
534 LLVMValueRef ind1
= LLVMConstInt(LLVMInt32Type(), 1, 0);
535 LLVMValueRef ind2
= LLVMConstInt(LLVMInt32Type(), 2, 0);
536 LLVMValueRef ind3
= LLVMConstInt(LLVMInt32Type(), 3, 0);
537 LLVMValueRef io0_ptr
, io1_ptr
, io2_ptr
, io3_ptr
;
539 debug_assert(NUM_CHANNELS
== 4);
541 io0_ptr
= LLVMBuildGEP(builder
, io_ptr
,
543 io1_ptr
= LLVMBuildGEP(builder
, io_ptr
,
545 io2_ptr
= LLVMBuildGEP(builder
, io_ptr
,
547 io3_ptr
= LLVMBuildGEP(builder
, io_ptr
,
551 lp_build_printf(builder
, " io = %p, indexes[%d, %d, %d, %d]\n",
552 io_ptr
, ind0
, ind1
, ind2
, ind3
);
555 store_aos(builder
, io0_ptr
, attr_index
, aos
[0]);
556 store_aos(builder
, io1_ptr
, attr_index
, aos
[1]);
557 store_aos(builder
, io2_ptr
, attr_index
, aos
[2]);
558 store_aos(builder
, io3_ptr
, attr_index
, aos
[3]);
562 convert_to_aos(LLVMBuilderRef builder
,
564 LLVMValueRef (*outputs
)[NUM_CHANNELS
],
568 unsigned chan
, attrib
;
571 lp_build_printf(builder
, " # storing begin\n");
573 for (attrib
= 0; attrib
< num_outputs
; ++attrib
) {
576 for(chan
= 0; chan
< NUM_CHANNELS
; ++chan
) {
577 if(outputs
[attrib
][chan
]) {
578 LLVMValueRef out
= LLVMBuildLoad(builder
, outputs
[attrib
][chan
], "");
579 lp_build_name(out
, "output%u.%c", attrib
, "xyzw"[chan
]);
580 /*lp_build_printf(builder, "output %d : %d ",
581 LLVMConstInt(LLVMInt32Type(), attrib, 0),
582 LLVMConstInt(LLVMInt32Type(), chan, 0));
583 print_vectorf(builder, out);*/
588 soa_to_aos(builder
, soa
, aos
);
589 store_aos_array(builder
,
596 lp_build_printf(builder
, " # storing end\n");
601 draw_llvm_generate(struct draw_llvm
*llvm
, struct draw_llvm_variant
*variant
)
603 LLVMTypeRef arg_types
[7];
604 LLVMTypeRef func_type
;
605 LLVMValueRef context_ptr
;
606 LLVMBasicBlockRef block
;
607 LLVMBuilderRef builder
;
608 LLVMValueRef start
, end
, count
, stride
, step
, io_itr
;
609 LLVMValueRef io_ptr
, vbuffers_ptr
, vb_ptr
;
610 struct draw_context
*draw
= llvm
->draw
;
612 struct lp_build_context bld
;
613 struct lp_build_loop_state lp_loop
;
614 struct lp_type vs_type
= lp_type_float_vec(32);
615 const int max_vertices
= 4;
616 LLVMValueRef outputs
[PIPE_MAX_SHADER_OUTPUTS
][NUM_CHANNELS
];
619 arg_types
[0] = llvm
->context_ptr_type
; /* context */
620 arg_types
[1] = llvm
->vertex_header_ptr_type
; /* vertex_header */
621 arg_types
[2] = llvm
->buffer_ptr_type
; /* vbuffers */
622 arg_types
[3] = LLVMInt32Type(); /* start */
623 arg_types
[4] = LLVMInt32Type(); /* count */
624 arg_types
[5] = LLVMInt32Type(); /* stride */
625 arg_types
[6] = llvm
->vb_ptr_type
; /* pipe_vertex_buffer's */
627 func_type
= LLVMFunctionType(LLVMVoidType(), arg_types
, Elements(arg_types
), 0);
629 variant
->function
= LLVMAddFunction(llvm
->module
, "draw_llvm_shader", func_type
);
630 LLVMSetFunctionCallConv(variant
->function
, LLVMCCallConv
);
631 for(i
= 0; i
< Elements(arg_types
); ++i
)
632 if(LLVMGetTypeKind(arg_types
[i
]) == LLVMPointerTypeKind
)
633 LLVMAddAttribute(LLVMGetParam(variant
->function
, i
), LLVMNoAliasAttribute
);
635 context_ptr
= LLVMGetParam(variant
->function
, 0);
636 io_ptr
= LLVMGetParam(variant
->function
, 1);
637 vbuffers_ptr
= LLVMGetParam(variant
->function
, 2);
638 start
= LLVMGetParam(variant
->function
, 3);
639 count
= LLVMGetParam(variant
->function
, 4);
640 stride
= LLVMGetParam(variant
->function
, 5);
641 vb_ptr
= LLVMGetParam(variant
->function
, 6);
643 lp_build_name(context_ptr
, "context");
644 lp_build_name(io_ptr
, "io");
645 lp_build_name(vbuffers_ptr
, "vbuffers");
646 lp_build_name(start
, "start");
647 lp_build_name(count
, "count");
648 lp_build_name(stride
, "stride");
649 lp_build_name(vb_ptr
, "vb");
655 block
= LLVMAppendBasicBlock(variant
->function
, "entry");
656 builder
= LLVMCreateBuilder();
657 LLVMPositionBuilderAtEnd(builder
, block
);
659 lp_build_context_init(&bld
, builder
, vs_type
);
661 end
= lp_build_add(&bld
, start
, count
);
663 step
= LLVMConstInt(LLVMInt32Type(), max_vertices
, 0);
666 lp_build_printf(builder
, "start = %d, end = %d, step = %d\n",
669 lp_build_loop_begin(builder
, start
, &lp_loop
);
671 LLVMValueRef inputs
[PIPE_MAX_SHADER_INPUTS
][NUM_CHANNELS
];
672 LLVMValueRef aos_attribs
[PIPE_MAX_SHADER_INPUTS
][NUM_CHANNELS
] = { { 0 } };
674 const LLVMValueRef (*ptr_aos
)[NUM_CHANNELS
];
676 io_itr
= LLVMBuildSub(builder
, lp_loop
.counter
, start
, "");
677 io
= LLVMBuildGEP(builder
, io_ptr
, &io_itr
, 1, "");
679 lp_build_printf(builder
, " --- io %d = %p, loop counter %d\n",
680 io_itr
, io
, lp_loop
.counter
);
682 for (i
= 0; i
< NUM_CHANNELS
; ++i
) {
683 LLVMValueRef true_index
= LLVMBuildAdd(
686 LLVMConstInt(LLVMInt32Type(), i
, 0), "");
687 for (j
= 0; j
< draw
->pt
.nr_vertex_elements
; ++j
) {
688 struct pipe_vertex_element
*velem
= &draw
->pt
.vertex_element
[j
];
689 LLVMValueRef vb_index
= LLVMConstInt(LLVMInt32Type(),
690 velem
->vertex_buffer_index
,
692 LLVMValueRef vb
= LLVMBuildGEP(builder
, vb_ptr
,
694 generate_fetch(builder
, vbuffers_ptr
,
695 &aos_attribs
[j
][i
], velem
, vb
, true_index
);
698 convert_to_soa(builder
, aos_attribs
, inputs
,
699 draw
->pt
.nr_vertex_elements
);
701 ptr_aos
= (const LLVMValueRef (*)[NUM_CHANNELS
]) inputs
;
708 convert_to_aos(builder
, io
, outputs
,
709 draw
->vs
.vertex_shader
->info
.num_outputs
,
712 lp_build_loop_end_cond(builder
, end
, step
, LLVMIntUGE
, &lp_loop
);
714 LLVMBuildRetVoid(builder
);
716 LLVMDisposeBuilder(builder
);
719 * Translate the LLVM IR into machine code.
722 if(LLVMVerifyFunction(variant
->function
, LLVMPrintMessageAction
)) {
723 lp_debug_dump_value(variant
->function
);
728 LLVMRunFunctionPassManager(llvm
->pass
, variant
->function
);
731 lp_debug_dump_value(variant
->function
);
735 code
= LLVMGetPointerToGlobal(llvm
->draw
->engine
, variant
->function
);
736 variant
->jit_func
= voidptr_to_draw_jit_vert_func(code
);
739 lp_disassemble(code
);
744 draw_llvm_generate_elts(struct draw_llvm
*llvm
, struct draw_llvm_variant
*variant
)
746 LLVMTypeRef arg_types
[7];
747 LLVMTypeRef func_type
;
748 LLVMValueRef context_ptr
;
749 LLVMBasicBlockRef block
;
750 LLVMBuilderRef builder
;
751 LLVMValueRef fetch_elts
, fetch_count
, stride
, step
, io_itr
;
752 LLVMValueRef io_ptr
, vbuffers_ptr
, vb_ptr
;
753 struct draw_context
*draw
= llvm
->draw
;
755 struct lp_build_context bld
;
756 struct lp_build_context bld_int
;
757 struct lp_build_loop_state lp_loop
;
758 struct lp_type vs_type
= lp_type_float_vec(32);
759 const int max_vertices
= 4;
760 LLVMValueRef outputs
[PIPE_MAX_SHADER_OUTPUTS
][NUM_CHANNELS
];
761 LLVMValueRef fetch_max
;
764 arg_types
[0] = llvm
->context_ptr_type
; /* context */
765 arg_types
[1] = llvm
->vertex_header_ptr_type
; /* vertex_header */
766 arg_types
[2] = llvm
->buffer_ptr_type
; /* vbuffers */
767 arg_types
[3] = LLVMPointerType(LLVMInt32Type(), 0); /* fetch_elts * */
768 arg_types
[4] = LLVMInt32Type(); /* fetch_count */
769 arg_types
[5] = LLVMInt32Type(); /* stride */
770 arg_types
[6] = llvm
->vb_ptr_type
; /* pipe_vertex_buffer's */
772 func_type
= LLVMFunctionType(LLVMVoidType(), arg_types
, Elements(arg_types
), 0);
774 variant
->function_elts
= LLVMAddFunction(llvm
->module
, "draw_llvm_shader_elts", func_type
);
775 LLVMSetFunctionCallConv(variant
->function_elts
, LLVMCCallConv
);
776 for(i
= 0; i
< Elements(arg_types
); ++i
)
777 if(LLVMGetTypeKind(arg_types
[i
]) == LLVMPointerTypeKind
)
778 LLVMAddAttribute(LLVMGetParam(variant
->function_elts
, i
), LLVMNoAliasAttribute
);
780 context_ptr
= LLVMGetParam(variant
->function_elts
, 0);
781 io_ptr
= LLVMGetParam(variant
->function_elts
, 1);
782 vbuffers_ptr
= LLVMGetParam(variant
->function_elts
, 2);
783 fetch_elts
= LLVMGetParam(variant
->function_elts
, 3);
784 fetch_count
= LLVMGetParam(variant
->function_elts
, 4);
785 stride
= LLVMGetParam(variant
->function_elts
, 5);
786 vb_ptr
= LLVMGetParam(variant
->function_elts
, 6);
788 lp_build_name(context_ptr
, "context");
789 lp_build_name(io_ptr
, "io");
790 lp_build_name(vbuffers_ptr
, "vbuffers");
791 lp_build_name(fetch_elts
, "fetch_elts");
792 lp_build_name(fetch_count
, "fetch_count");
793 lp_build_name(stride
, "stride");
794 lp_build_name(vb_ptr
, "vb");
800 block
= LLVMAppendBasicBlock(variant
->function_elts
, "entry");
801 builder
= LLVMCreateBuilder();
802 LLVMPositionBuilderAtEnd(builder
, block
);
804 lp_build_context_init(&bld
, builder
, vs_type
);
805 lp_build_context_init(&bld_int
, builder
, lp_type_int(32));
807 step
= LLVMConstInt(LLVMInt32Type(), max_vertices
, 0);
809 fetch_max
= LLVMBuildSub(builder
, fetch_count
,
810 LLVMConstInt(LLVMInt32Type(), 1, 0),
813 lp_build_loop_begin(builder
, LLVMConstInt(LLVMInt32Type(), 0, 0), &lp_loop
);
815 LLVMValueRef inputs
[PIPE_MAX_SHADER_INPUTS
][NUM_CHANNELS
];
816 LLVMValueRef aos_attribs
[PIPE_MAX_SHADER_INPUTS
][NUM_CHANNELS
] = { { 0 } };
818 const LLVMValueRef (*ptr_aos
)[NUM_CHANNELS
];
820 io_itr
= lp_loop
.counter
;
821 io
= LLVMBuildGEP(builder
, io_ptr
, &io_itr
, 1, "");
823 lp_build_printf(builder
, " --- io %d = %p, loop counter %d\n",
824 io_itr
, io
, lp_loop
.counter
);
826 for (i
= 0; i
< NUM_CHANNELS
; ++i
) {
827 LLVMValueRef true_index
= LLVMBuildAdd(
830 LLVMConstInt(LLVMInt32Type(), i
, 0), "");
831 LLVMValueRef fetch_ptr
;
833 /* make sure we're not out of bounds which can happen
834 * if fetch_count % 4 != 0, because on the last iteration
835 * a few of the 4 vertex fetches will be out of bounds */
836 true_index
= lp_build_min(&bld_int
, true_index
, fetch_max
);
838 fetch_ptr
= LLVMBuildGEP(builder
, fetch_elts
,
840 true_index
= LLVMBuildLoad(builder
, fetch_ptr
, "fetch_elt");
841 for (j
= 0; j
< draw
->pt
.nr_vertex_elements
; ++j
) {
842 struct pipe_vertex_element
*velem
= &draw
->pt
.vertex_element
[j
];
843 LLVMValueRef vb_index
= LLVMConstInt(LLVMInt32Type(),
844 velem
->vertex_buffer_index
,
846 LLVMValueRef vb
= LLVMBuildGEP(builder
, vb_ptr
,
848 generate_fetch(builder
, vbuffers_ptr
,
849 &aos_attribs
[j
][i
], velem
, vb
, true_index
);
852 convert_to_soa(builder
, aos_attribs
, inputs
,
853 draw
->pt
.nr_vertex_elements
);
855 ptr_aos
= (const LLVMValueRef (*)[NUM_CHANNELS
]) inputs
;
862 convert_to_aos(builder
, io
, outputs
,
863 draw
->vs
.vertex_shader
->info
.num_outputs
,
866 lp_build_loop_end_cond(builder
, fetch_count
, step
, LLVMIntUGE
, &lp_loop
);
868 LLVMBuildRetVoid(builder
);
870 LLVMDisposeBuilder(builder
);
873 * Translate the LLVM IR into machine code.
876 if(LLVMVerifyFunction(variant
->function_elts
, LLVMPrintMessageAction
)) {
877 lp_debug_dump_value(variant
->function_elts
);
882 LLVMRunFunctionPassManager(llvm
->pass
, variant
->function_elts
);
885 lp_debug_dump_value(variant
->function_elts
);
889 code
= LLVMGetPointerToGlobal(llvm
->draw
->engine
, variant
->function_elts
);
890 variant
->jit_func_elts
= voidptr_to_draw_vert_func_elts(code
);
893 lp_disassemble(code
);
897 draw_llvm_make_variant_key(struct draw_llvm
*llvm
,
898 struct draw_llvm_variant_key
*key
)
900 memset(key
, 0, sizeof(struct draw_llvm_variant_key
));
902 key
->nr_vertex_elements
= llvm
->draw
->pt
.nr_vertex_elements
;
904 memcpy(key
->vertex_element
,
905 llvm
->draw
->pt
.vertex_element
,
906 sizeof(struct pipe_vertex_element
) * key
->nr_vertex_elements
);
909 &llvm
->draw
->vs
.vertex_shader
->state
,
910 sizeof(struct pipe_shader_state
));