projects / mesa.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
history | raw | HEAD
st/xa: Fix crosscompile builds with nonstandard ld locations
[mesa.git] / src / gallium / auxiliary / draw / draw_llvm.c
1 /**************************************************************************
2 *
3 * Copyright 2010 VMware, Inc.
4 * All Rights Reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sub license, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
13 *
14 * The above copyright notice and this permission notice (including the
15 * next paragraph) shall be included in all copies or substantial portions
16 * of the Software.
17 *
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
19 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
20 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
21 * IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR
22 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
23 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
24 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
25 *
26 **************************************************************************/
27
28 #include "draw_llvm.h"
29
30 #include "draw_context.h"
31 #include "draw_vs.h"
32
33 #include "gallivm/lp_bld_arit.h"
34 #include "gallivm/lp_bld_logic.h"
35 #include "gallivm/lp_bld_const.h"
36 #include "gallivm/lp_bld_swizzle.h"
37 #include "gallivm/lp_bld_struct.h"
38 #include "gallivm/lp_bld_type.h"
39 #include "gallivm/lp_bld_flow.h"
40 #include "gallivm/lp_bld_debug.h"
41 #include "gallivm/lp_bld_tgsi.h"
42 #include "gallivm/lp_bld_printf.h"
43 #include "gallivm/lp_bld_intr.h"
44 #include "gallivm/lp_bld_init.h"
45 #include "gallivm/lp_bld_type.h"
46
47 #include "tgsi/tgsi_exec.h"
48 #include "tgsi/tgsi_dump.h"
49
50 #include "util/u_math.h"
51 #include "util/u_pointer.h"
52 #include "util/u_string.h"
53 #include "util/u_simple_list.h"
54
55
56 #define DEBUG_STORE 0
57
58
59 /**
60 * This function is called by the gallivm "garbage collector" when
61 * the LLVM global data structures are freed. We must free all LLVM-related
62 * data. Specifically, all JIT'd shader variants.
63 */
64 static void
65 draw_llvm_garbage_collect_callback(void *cb_data)
66 {
67 struct draw_llvm *llvm = (struct draw_llvm *) cb_data;
68 struct draw_llvm_variant_list_item *li;
69
70 /* free all shader variants */
71 li = first_elem(&llvm->vs_variants_list);
72 while (!at_end(&llvm->vs_variants_list, li)) {
73 struct draw_llvm_variant_list_item *next = next_elem(li);
74 draw_llvm_destroy_variant(li->base);
75 li = next;
76 }
77
78 /* Null-out these pointers so they get remade next time they're needed.
79 * See the accessor functions below.
80 */
81 llvm->context_ptr_type = NULL;
82 llvm->buffer_ptr_type = NULL;
83 llvm->vb_ptr_type = NULL;
84 llvm->vertex_header_ptr_type = NULL;
85 }
86
87
88 static void
89 draw_llvm_generate(struct draw_llvm *llvm, struct draw_llvm_variant *var);
90
91 static void
92 draw_llvm_generate_elts(struct draw_llvm *llvm, struct draw_llvm_variant *var);
93
94
95 /**
96 * Create LLVM type for struct draw_jit_texture
97 */
98 static LLVMTypeRef
99 create_jit_texture_type(struct gallivm_state *gallivm)
100 {
101 LLVMTargetDataRef target = gallivm->target;
102 LLVMTypeRef texture_type;
103 LLVMTypeRef elem_types[DRAW_JIT_TEXTURE_NUM_FIELDS];
104 LLVMTypeRef int32_type = LLVMInt32TypeInContext(gallivm->context);
105
106 elem_types[DRAW_JIT_TEXTURE_WIDTH] =
107 elem_types[DRAW_JIT_TEXTURE_HEIGHT] =
108 elem_types[DRAW_JIT_TEXTURE_DEPTH] =
109 elem_types[DRAW_JIT_TEXTURE_FIRST_LEVEL] =
110 elem_types[DRAW_JIT_TEXTURE_LAST_LEVEL] = int32_type;
111 elem_types[DRAW_JIT_TEXTURE_ROW_STRIDE] =
112 elem_types[DRAW_JIT_TEXTURE_IMG_STRIDE] =
113 LLVMArrayType(int32_type, PIPE_MAX_TEXTURE_LEVELS);
114 elem_types[DRAW_JIT_TEXTURE_DATA] =
115 LLVMArrayType(LLVMPointerType(LLVMInt8TypeInContext(gallivm->context), 0),
116 PIPE_MAX_TEXTURE_LEVELS);
117 elem_types[DRAW_JIT_TEXTURE_MIN_LOD] =
118 elem_types[DRAW_JIT_TEXTURE_MAX_LOD] =
119 elem_types[DRAW_JIT_TEXTURE_LOD_BIAS] = LLVMFloatTypeInContext(gallivm->context);
120 elem_types[DRAW_JIT_TEXTURE_BORDER_COLOR] =
121 LLVMArrayType(LLVMFloatTypeInContext(gallivm->context), 4);
122
123 texture_type = LLVMStructTypeInContext(gallivm->context, elem_types,
124 Elements(elem_types), 0);
125
126 /* Make sure the target's struct layout cache doesn't return
127 * stale/invalid data.
128 */
129 LLVMInvalidateStructLayout(gallivm->target, texture_type);
130
131 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, width,
132 target, texture_type,
133 DRAW_JIT_TEXTURE_WIDTH);
134 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, height,
135 target, texture_type,
136 DRAW_JIT_TEXTURE_HEIGHT);
137 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, depth,
138 target, texture_type,
139 DRAW_JIT_TEXTURE_DEPTH);
140 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, first_level,
141 target, texture_type,
142 DRAW_JIT_TEXTURE_FIRST_LEVEL);
143 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, last_level,
144 target, texture_type,
145 DRAW_JIT_TEXTURE_LAST_LEVEL);
146 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, row_stride,
147 target, texture_type,
148 DRAW_JIT_TEXTURE_ROW_STRIDE);
149 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, img_stride,
150 target, texture_type,
151 DRAW_JIT_TEXTURE_IMG_STRIDE);
152 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, data,
153 target, texture_type,
154 DRAW_JIT_TEXTURE_DATA);
155 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, min_lod,
156 target, texture_type,
157 DRAW_JIT_TEXTURE_MIN_LOD);
158 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, max_lod,
159 target, texture_type,
160 DRAW_JIT_TEXTURE_MAX_LOD);
161 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, lod_bias,
162 target, texture_type,
163 DRAW_JIT_TEXTURE_LOD_BIAS);
164 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, border_color,
165 target, texture_type,
166 DRAW_JIT_TEXTURE_BORDER_COLOR);
167
168 LP_CHECK_STRUCT_SIZE(struct draw_jit_texture, target, texture_type);
169
170 return texture_type;
171 }
172
173
174 /**
175 * Create LLVM type for struct draw_jit_texture
176 */
177 static LLVMTypeRef
178 create_jit_context_type(struct gallivm_state *gallivm,
179 LLVMTypeRef texture_type)
180 {
181 LLVMTargetDataRef target = gallivm->target;
182 LLVMTypeRef float_type = LLVMFloatTypeInContext(gallivm->context);
183 LLVMTypeRef elem_types[5];
184 LLVMTypeRef context_type;
185
186 elem_types[0] = LLVMPointerType(float_type, 0); /* vs_constants */
187 elem_types[1] = LLVMPointerType(float_type, 0); /* gs_constants */
188 elem_types[2] = LLVMPointerType(LLVMArrayType(LLVMArrayType(float_type, 4), 12), 0); /* planes */
189 elem_types[3] = LLVMPointerType(float_type, 0); /* viewport */
190 elem_types[4] = LLVMArrayType(texture_type,
191 PIPE_MAX_VERTEX_SAMPLERS); /* textures */
192
193 context_type = LLVMStructTypeInContext(gallivm->context, elem_types,
194 Elements(elem_types), 0);
195
196 LLVMInvalidateStructLayout(gallivm->target, context_type);
197
198 LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, vs_constants,
199 target, context_type, 0);
200 LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, gs_constants,
201 target, context_type, 1);
202 LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, planes,
203 target, context_type, 2);
204 LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, textures,
205 target, context_type,
206 DRAW_JIT_CTX_TEXTURES);
207 LP_CHECK_STRUCT_SIZE(struct draw_jit_context,
208 target, context_type);
209
210 return context_type;
211 }
212
213
214 /**
215 * Create LLVM type for struct pipe_vertex_buffer
216 */
217 static LLVMTypeRef
218 create_jit_vertex_buffer_type(struct gallivm_state *gallivm)
219 {
220 LLVMTargetDataRef target = gallivm->target;
221 LLVMTypeRef elem_types[3];
222 LLVMTypeRef vb_type;
223
224 elem_types[0] =
225 elem_types[1] = LLVMInt32TypeInContext(gallivm->context);
226 elem_types[2] = LLVMPointerType(LLVMInt8TypeInContext(gallivm->context), 0); /* vs_constants */
227
228 vb_type = LLVMStructTypeInContext(gallivm->context, elem_types,
229 Elements(elem_types), 0);
230
231 LLVMInvalidateStructLayout(gallivm->target, vb_type);
232
233 LP_CHECK_MEMBER_OFFSET(struct pipe_vertex_buffer, stride,
234 target, vb_type, 0);
235 LP_CHECK_MEMBER_OFFSET(struct pipe_vertex_buffer, buffer_offset,
236 target, vb_type, 1);
237
238 LP_CHECK_STRUCT_SIZE(struct pipe_vertex_buffer, target, vb_type);
239
240 return vb_type;
241 }
242
243
244 /**
245 * Create LLVM type for struct vertex_header;
246 */
247 static LLVMTypeRef
248 create_jit_vertex_header(struct gallivm_state *gallivm, int data_elems)
249 {
250 LLVMTargetDataRef target = gallivm->target;
251 LLVMTypeRef elem_types[3];
252 LLVMTypeRef vertex_header;
253 char struct_name[24];
254
255 util_snprintf(struct_name, 23, "vertex_header%d", data_elems);
256
257 elem_types[0] = LLVMIntTypeInContext(gallivm->context, 32);
258 elem_types[1] = LLVMArrayType(LLVMFloatTypeInContext(gallivm->context), 4);
259 elem_types[2] = LLVMArrayType(elem_types[1], data_elems);
260
261 vertex_header = LLVMStructTypeInContext(gallivm->context, elem_types,
262 Elements(elem_types), 0);
263
264 LLVMInvalidateStructLayout(gallivm->target, vertex_header);
265
266 /* these are bit-fields and we can't take address of them
267 LP_CHECK_MEMBER_OFFSET(struct vertex_header, clipmask,
268 target, vertex_header,
269 DRAW_JIT_VERTEX_CLIPMASK);
270 LP_CHECK_MEMBER_OFFSET(struct vertex_header, edgeflag,
271 target, vertex_header,
272 DRAW_JIT_VERTEX_EDGEFLAG);
273 LP_CHECK_MEMBER_OFFSET(struct vertex_header, pad,
274 target, vertex_header,
275 DRAW_JIT_VERTEX_PAD);
276 LP_CHECK_MEMBER_OFFSET(struct vertex_header, vertex_id,
277 target, vertex_header,
278 DRAW_JIT_VERTEX_VERTEX_ID);
279 */
280 LP_CHECK_MEMBER_OFFSET(struct vertex_header, clip,
281 target, vertex_header,
282 DRAW_JIT_VERTEX_CLIP);
283 LP_CHECK_MEMBER_OFFSET(struct vertex_header, data,
284 target, vertex_header,
285 DRAW_JIT_VERTEX_DATA);
286
287 LLVMAddTypeName(gallivm->module, struct_name, vertex_header);
288
289 return vertex_header;
290 }
291
292
293 /**
294 * Create LLVM types for various structures.
295 */
296 static void
297 create_jit_types(struct draw_llvm *llvm)
298 {
299 struct gallivm_state *gallivm = llvm->gallivm;
300 LLVMTypeRef texture_type, context_type, buffer_type, vb_type;
301
302 texture_type = create_jit_texture_type(gallivm);
303 LLVMAddTypeName(gallivm->module, "texture", texture_type);
304
305 context_type = create_jit_context_type(gallivm, texture_type);
306 LLVMAddTypeName(gallivm->module, "draw_jit_context", context_type);
307 llvm->context_ptr_type = LLVMPointerType(context_type, 0);
308
309 buffer_type = LLVMPointerType(LLVMIntTypeInContext(gallivm->context, 8), 0);
310 LLVMAddTypeName(gallivm->module, "buffer", buffer_type);
311 llvm->buffer_ptr_type = LLVMPointerType(buffer_type, 0);
312
313 vb_type = create_jit_vertex_buffer_type(gallivm);
314 LLVMAddTypeName(gallivm->module, "pipe_vertex_buffer", vb_type);
315 llvm->vb_ptr_type = LLVMPointerType(vb_type, 0);
316 }
317
318
319 static LLVMTypeRef
320 get_context_ptr_type(struct draw_llvm *llvm)
321 {
322 if (!llvm->context_ptr_type)
323 create_jit_types(llvm);
324 return llvm->context_ptr_type;
325 }
326
327
328 static LLVMTypeRef
329 get_buffer_ptr_type(struct draw_llvm *llvm)
330 {
331 if (!llvm->buffer_ptr_type)
332 create_jit_types(llvm);
333 return llvm->buffer_ptr_type;
334 }
335
336
337 static LLVMTypeRef
338 get_vb_ptr_type(struct draw_llvm *llvm)
339 {
340 if (!llvm->vb_ptr_type)
341 create_jit_types(llvm);
342 return llvm->vb_ptr_type;
343 }
344
345 static LLVMTypeRef
346 get_vertex_header_ptr_type(struct draw_llvm *llvm)
347 {
348 if (!llvm->vertex_header_ptr_type)
349 create_jit_types(llvm);
350 return llvm->vertex_header_ptr_type;
351 }
352
353
354 /**
355 * Create per-context LLVM info.
356 */
357 struct draw_llvm *
358 draw_llvm_create(struct draw_context *draw, struct gallivm_state *gallivm)
359 {
360 struct draw_llvm *llvm;
361
362 llvm = CALLOC_STRUCT( draw_llvm );
363 if (!llvm)
364 return NULL;
365
366 lp_build_init();
367
368 llvm->draw = draw;
369 llvm->gallivm = gallivm;
370
371 if (gallivm_debug & GALLIVM_DEBUG_IR) {
372 LLVMDumpModule(llvm->gallivm->module);
373 }
374
375 llvm->nr_variants = 0;
376 make_empty_list(&llvm->vs_variants_list);
377
378 gallivm_register_garbage_collector_callback(
379 draw_llvm_garbage_collect_callback, llvm);
380
381 return llvm;
382 }
383
384
385 /**
386 * Free per-context LLVM info.
387 */
388 void
389 draw_llvm_destroy(struct draw_llvm *llvm)
390 {
391 gallivm_remove_garbage_collector_callback(
392 draw_llvm_garbage_collect_callback, llvm);
393
394 /* XXX free other draw_llvm data? */
395 FREE(llvm);
396 }
397
398
399 /**
400 * Create LLVM-generated code for a vertex shader.
401 */
402 struct draw_llvm_variant *
403 draw_llvm_create_variant(struct draw_llvm *llvm,
404 unsigned num_inputs,
405 const struct draw_llvm_variant_key *key)
406 {
407 struct draw_llvm_variant *variant;
408 struct llvm_vertex_shader *shader =
409 llvm_vertex_shader(llvm->draw->vs.vertex_shader);
410 LLVMTypeRef vertex_header;
411
412 variant = MALLOC(sizeof *variant +
413 shader->variant_key_size -
414 sizeof variant->key);
415 if (variant == NULL)
416 return NULL;
417
418 variant->llvm = llvm;
419
420 memcpy(&variant->key, key, shader->variant_key_size);
421
422 vertex_header = create_jit_vertex_header(llvm->gallivm, num_inputs);
423
424 llvm->vertex_header_ptr_type = LLVMPointerType(vertex_header, 0);
425
426 draw_llvm_generate(llvm, variant);
427 draw_llvm_generate_elts(llvm, variant);
428
429 variant->shader = shader;
430 variant->list_item_global.base = variant;
431 variant->list_item_local.base = variant;
432 /*variant->no = */shader->variants_created++;
433 variant->list_item_global.base = variant;
434
435 return variant;
436 }
437
438
439 static void
440 generate_vs(struct draw_llvm *llvm,
441 LLVMBuilderRef builder,
442 LLVMValueRef (*outputs)[NUM_CHANNELS],
443 const LLVMValueRef (*inputs)[NUM_CHANNELS],
444 LLVMValueRef system_values_array,
445 LLVMValueRef context_ptr,
446 struct lp_build_sampler_soa *draw_sampler,
447 boolean clamp_vertex_color)
448 {
449 const struct tgsi_token *tokens = llvm->draw->vs.vertex_shader->state.tokens;
450 struct lp_type vs_type;
451 LLVMValueRef consts_ptr = draw_jit_context_vs_constants(llvm->gallivm, context_ptr);
452 struct lp_build_sampler_soa *sampler = 0;
453
454 memset(&vs_type, 0, sizeof vs_type);
455 vs_type.floating = TRUE; /* floating point values */
456 vs_type.sign = TRUE; /* values are signed */
457 vs_type.norm = FALSE; /* values are not limited to [0,1] or [-1,1] */
458 vs_type.width = 32; /* 32-bit float */
459 vs_type.length = 4; /* 4 elements per vector */
460 #if 0
461 num_vs = 4; /* number of vertices per block */
462 #endif
463
464 if (gallivm_debug & GALLIVM_DEBUG_IR) {
465 tgsi_dump(tokens, 0);
466 }
467
468 if (llvm->draw->num_sampler_views && llvm->draw->num_samplers)
469 sampler = draw_sampler;
470
471 lp_build_tgsi_soa(llvm->gallivm,
472 tokens,
473 vs_type,
474 NULL /*struct lp_build_mask_context *mask*/,
475 consts_ptr,
476 system_values_array,
477 NULL /*pos*/,
478 inputs,
479 outputs,
480 sampler,
481 &llvm->draw->vs.vertex_shader->info);
482
483 if (clamp_vertex_color) {
484 LLVMValueRef out;
485 unsigned chan, attrib;
486 struct lp_build_context bld;
487 struct tgsi_shader_info* info = &llvm->draw->vs.vertex_shader->info;
488 lp_build_context_init(&bld, llvm->gallivm, vs_type);
489
490 for (attrib = 0; attrib < info->num_outputs; ++attrib) {
491 for (chan = 0; chan < NUM_CHANNELS; ++chan) {
492 if (outputs[attrib][chan]) {
493 switch (info->output_semantic_name[attrib]) {
494 case TGSI_SEMANTIC_COLOR:
495 case TGSI_SEMANTIC_BCOLOR:
496 out = LLVMBuildLoad(builder, outputs[attrib][chan], "");
497 out = lp_build_clamp(&bld, out, bld.zero, bld.one);
498 LLVMBuildStore(builder, out, outputs[attrib][chan]);
499 break;
500 }
501 }
502 }
503 }
504 }
505 }
506
507
508 #if DEBUG_STORE
509 static void print_vectorf(LLVMBuilderRef builder,
510 LLVMValueRef vec)
511 {
512 LLVMValueRef val[4];
513 val[0] = LLVMBuildExtractElement(builder, vec,
514 lp_build_const_int32(gallivm, 0), "");
515 val[1] = LLVMBuildExtractElement(builder, vec,
516 lp_build_const_int32(gallivm, 1), "");
517 val[2] = LLVMBuildExtractElement(builder, vec,
518 lp_build_const_int32(gallivm, 2), "");
519 val[3] = LLVMBuildExtractElement(builder, vec,
520 lp_build_const_int32(gallivm, 3), "");
521 lp_build_printf(builder, "vector = [%f, %f, %f, %f]\n",
522 val[0], val[1], val[2], val[3]);
523 }
524 #endif
525
526
527 static void
528 generate_fetch(struct gallivm_state *gallivm,
529 LLVMValueRef vbuffers_ptr,
530 LLVMValueRef *res,
531 struct pipe_vertex_element *velem,
532 LLVMValueRef vbuf,
533 LLVMValueRef index,
534 LLVMValueRef instance_id)
535 {
536 LLVMBuilderRef builder = gallivm->builder;
537 LLVMValueRef indices =
538 LLVMConstInt(LLVMInt64TypeInContext(gallivm->context),
539 velem->vertex_buffer_index, 0);
540 LLVMValueRef vbuffer_ptr = LLVMBuildGEP(builder, vbuffers_ptr,
541 &indices, 1, "");
542 LLVMValueRef vb_stride = draw_jit_vbuffer_stride(gallivm, vbuf);
543 LLVMValueRef vb_buffer_offset = draw_jit_vbuffer_offset(gallivm, vbuf);
544 LLVMValueRef stride;
545
546 if (velem->instance_divisor) {
547 /* array index = instance_id / instance_divisor */
548 index = LLVMBuildUDiv(builder, instance_id,
549 lp_build_const_int32(gallivm, velem->instance_divisor),
550 "instance_divisor");
551 }
552
553 stride = LLVMBuildMul(builder, vb_stride, index, "");
554
555 vbuffer_ptr = LLVMBuildLoad(builder, vbuffer_ptr, "vbuffer");
556
557 stride = LLVMBuildAdd(builder, stride,
558 vb_buffer_offset,
559 "");
560 stride = LLVMBuildAdd(builder, stride,
561 lp_build_const_int32(gallivm, velem->src_offset),
562 "");
563
564 /*lp_build_printf(builder, "vbuf index = %d, stride is %d\n", indices, stride);*/
565 vbuffer_ptr = LLVMBuildGEP(builder, vbuffer_ptr, &stride, 1, "");
566
567 *res = draw_llvm_translate_from(gallivm, vbuffer_ptr, velem->src_format);
568 }
569
570
571 static LLVMValueRef
572 aos_to_soa(struct gallivm_state *gallivm,
573 LLVMValueRef val0,
574 LLVMValueRef val1,
575 LLVMValueRef val2,
576 LLVMValueRef val3,
577 LLVMValueRef channel)
578 {
579 LLVMBuilderRef builder = gallivm->builder;
580 LLVMValueRef ex, res;
581
582 ex = LLVMBuildExtractElement(builder, val0,
583 channel, "");
584 res = LLVMBuildInsertElement(builder,
585 LLVMConstNull(LLVMTypeOf(val0)),
586 ex,
587 lp_build_const_int32(gallivm, 0),
588 "");
589
590 ex = LLVMBuildExtractElement(builder, val1,
591 channel, "");
592 res = LLVMBuildInsertElement(builder,
593 res, ex,
594 lp_build_const_int32(gallivm, 1),
595 "");
596
597 ex = LLVMBuildExtractElement(builder, val2,
598 channel, "");
599 res = LLVMBuildInsertElement(builder,
600 res, ex,
601 lp_build_const_int32(gallivm, 2),
602 "");
603
604 ex = LLVMBuildExtractElement(builder, val3,
605 channel, "");
606 res = LLVMBuildInsertElement(builder,
607 res, ex,
608 lp_build_const_int32(gallivm, 3),
609 "");
610
611 return res;
612 }
613
614
615 static void
616 soa_to_aos(struct gallivm_state *gallivm,
617 LLVMValueRef soa[NUM_CHANNELS],
618 LLVMValueRef aos[NUM_CHANNELS])
619 {
620 LLVMBuilderRef builder = gallivm->builder;
621 LLVMValueRef comp;
622 int i = 0;
623
624 debug_assert(NUM_CHANNELS == 4);
625
626 aos[0] = LLVMConstNull(LLVMTypeOf(soa[0]));
627 aos[1] = aos[2] = aos[3] = aos[0];
628
629 for (i = 0; i < NUM_CHANNELS; ++i) {
630 LLVMValueRef channel = lp_build_const_int32(gallivm, i);
631
632 comp = LLVMBuildExtractElement(builder, soa[i],
633 lp_build_const_int32(gallivm, 0), "");
634 aos[0] = LLVMBuildInsertElement(builder, aos[0], comp, channel, "");
635
636 comp = LLVMBuildExtractElement(builder, soa[i],
637 lp_build_const_int32(gallivm, 1), "");
638 aos[1] = LLVMBuildInsertElement(builder, aos[1], comp, channel, "");
639
640 comp = LLVMBuildExtractElement(builder, soa[i],
641 lp_build_const_int32(gallivm, 2), "");
642 aos[2] = LLVMBuildInsertElement(builder, aos[2], comp, channel, "");
643
644 comp = LLVMBuildExtractElement(builder, soa[i],
645 lp_build_const_int32(gallivm, 3), "");
646 aos[3] = LLVMBuildInsertElement(builder, aos[3], comp, channel, "");
647
648 }
649 }
650
651
652 static void
653 convert_to_soa(struct gallivm_state *gallivm,
654 LLVMValueRef (*aos)[NUM_CHANNELS],
655 LLVMValueRef (*soa)[NUM_CHANNELS],
656 int num_attribs)
657 {
658 int i;
659
660 debug_assert(NUM_CHANNELS == 4);
661
662 for (i = 0; i < num_attribs; ++i) {
663 LLVMValueRef val0 = aos[i][0];
664 LLVMValueRef val1 = aos[i][1];
665 LLVMValueRef val2 = aos[i][2];
666 LLVMValueRef val3 = aos[i][3];
667
668 soa[i][0] = aos_to_soa(gallivm, val0, val1, val2, val3,
669 lp_build_const_int32(gallivm, 0));
670 soa[i][1] = aos_to_soa(gallivm, val0, val1, val2, val3,
671 lp_build_const_int32(gallivm, 1));
672 soa[i][2] = aos_to_soa(gallivm, val0, val1, val2, val3,
673 lp_build_const_int32(gallivm, 2));
674 soa[i][3] = aos_to_soa(gallivm, val0, val1, val2, val3,
675 lp_build_const_int32(gallivm, 3));
676 }
677 }
678
679
680 static void
681 store_aos(struct gallivm_state *gallivm,
682 LLVMValueRef io_ptr,
683 LLVMValueRef index,
684 LLVMValueRef value,
685 LLVMValueRef clipmask)
686 {
687 LLVMBuilderRef builder = gallivm->builder;
688 LLVMValueRef id_ptr = draw_jit_header_id(gallivm, io_ptr);
689 LLVMValueRef data_ptr = draw_jit_header_data(gallivm, io_ptr);
690 LLVMValueRef indices[3];
691 LLVMValueRef val, shift;
692
693 indices[0] = lp_build_const_int32(gallivm, 0);
694 indices[1] = index;
695 indices[2] = lp_build_const_int32(gallivm, 0);
696
697 /* initialize vertex id:16 = 0xffff, pad:3 = 0, edgeflag:1 = 1 */
698 val = lp_build_const_int32(gallivm, 0xffff1);
699 shift = lp_build_const_int32(gallivm, 12);
700 val = LLVMBuildShl(builder, val, shift, "");
701 /* add clipmask:12 */
702 val = LLVMBuildOr(builder, val, clipmask, "");
703
704 /* store vertex header */
705 LLVMBuildStore(builder, val, id_ptr);
706
707
708 #if DEBUG_STORE
709 lp_build_printf(builder, " ---- %p storing attribute %d (io = %p)\n", data_ptr, index, io_ptr);
710 #endif
711 #if 0
712 /*lp_build_printf(builder, " ---- %p storing at %d (%p) ", io_ptr, index, data_ptr);
713 print_vectorf(builder, value);*/
714 data_ptr = LLVMBuildBitCast(builder, data_ptr,
715 LLVMPointerType(LLVMArrayType(LLVMVectorType(LLVMFloatTypeInContext(gallivm->context), 4), 0), 0),
716 "datavec");
717 data_ptr = LLVMBuildGEP(builder, data_ptr, indices, 2, "");
718
719 LLVMBuildStore(builder, value, data_ptr);
720 #else
721 {
722 LLVMValueRef x, y, z, w;
723 LLVMValueRef idx0, idx1, idx2, idx3;
724 LLVMValueRef gep0, gep1, gep2, gep3;
725 data_ptr = LLVMBuildGEP(builder, data_ptr, indices, 3, "");
726
727 idx0 = lp_build_const_int32(gallivm, 0);
728 idx1 = lp_build_const_int32(gallivm, 1);
729 idx2 = lp_build_const_int32(gallivm, 2);
730 idx3 = lp_build_const_int32(gallivm, 3);
731
732 x = LLVMBuildExtractElement(builder, value,
733 idx0, "");
734 y = LLVMBuildExtractElement(builder, value,
735 idx1, "");
736 z = LLVMBuildExtractElement(builder, value,
737 idx2, "");
738 w = LLVMBuildExtractElement(builder, value,
739 idx3, "");
740
741 gep0 = LLVMBuildGEP(builder, data_ptr, &idx0, 1, "");
742 gep1 = LLVMBuildGEP(builder, data_ptr, &idx1, 1, "");
743 gep2 = LLVMBuildGEP(builder, data_ptr, &idx2, 1, "");
744 gep3 = LLVMBuildGEP(builder, data_ptr, &idx3, 1, "");
745
746 /*lp_build_printf(builder, "##### x = %f (%p), y = %f (%p), z = %f (%p), w = %f (%p)\n",
747 x, gep0, y, gep1, z, gep2, w, gep3);*/
748 LLVMBuildStore(builder, x, gep0);
749 LLVMBuildStore(builder, y, gep1);
750 LLVMBuildStore(builder, z, gep2);
751 LLVMBuildStore(builder, w, gep3);
752 }
753 #endif
754 }
755
756
757 static void
758 store_aos_array(struct gallivm_state *gallivm,
759 LLVMValueRef io_ptr,
760 LLVMValueRef aos[NUM_CHANNELS],
761 int attrib,
762 int num_outputs,
763 LLVMValueRef clipmask)
764 {
765 LLVMBuilderRef builder = gallivm->builder;
766 LLVMValueRef attr_index = lp_build_const_int32(gallivm, attrib);
767 LLVMValueRef ind0 = lp_build_const_int32(gallivm, 0);
768 LLVMValueRef ind1 = lp_build_const_int32(gallivm, 1);
769 LLVMValueRef ind2 = lp_build_const_int32(gallivm, 2);
770 LLVMValueRef ind3 = lp_build_const_int32(gallivm, 3);
771 LLVMValueRef io0_ptr, io1_ptr, io2_ptr, io3_ptr;
772 LLVMValueRef clipmask0, clipmask1, clipmask2, clipmask3;
773
774 debug_assert(NUM_CHANNELS == 4);
775
776 io0_ptr = LLVMBuildGEP(builder, io_ptr,
777 &ind0, 1, "");
778 io1_ptr = LLVMBuildGEP(builder, io_ptr,
779 &ind1, 1, "");
780 io2_ptr = LLVMBuildGEP(builder, io_ptr,
781 &ind2, 1, "");
782 io3_ptr = LLVMBuildGEP(builder, io_ptr,
783 &ind3, 1, "");
784
785 clipmask0 = LLVMBuildExtractElement(builder, clipmask,
786 ind0, "");
787 clipmask1 = LLVMBuildExtractElement(builder, clipmask,
788 ind1, "");
789 clipmask2 = LLVMBuildExtractElement(builder, clipmask,
790 ind2, "");
791 clipmask3 = LLVMBuildExtractElement(builder, clipmask,
792 ind3, "");
793
794 #if DEBUG_STORE
795 lp_build_printf(builder, "io = %p, indexes[%d, %d, %d, %d]\n, clipmask0 = %x, clipmask1 = %x, clipmask2 = %x, clipmask3 = %x\n",
796 io_ptr, ind0, ind1, ind2, ind3, clipmask0, clipmask1, clipmask2, clipmask3);
797 #endif
798 /* store for each of the 4 vertices */
799 store_aos(gallivm, io0_ptr, attr_index, aos[0], clipmask0);
800 store_aos(gallivm, io1_ptr, attr_index, aos[1], clipmask1);
801 store_aos(gallivm, io2_ptr, attr_index, aos[2], clipmask2);
802 store_aos(gallivm, io3_ptr, attr_index, aos[3], clipmask3);
803 }
804
805
806 static void
807 convert_to_aos(struct gallivm_state *gallivm,
808 LLVMValueRef io,
809 LLVMValueRef (*outputs)[NUM_CHANNELS],
810 LLVMValueRef clipmask,
811 int num_outputs,
812 int max_vertices)
813 {
814 LLVMBuilderRef builder = gallivm->builder;
815 unsigned chan, attrib;
816
817 #if DEBUG_STORE
818 lp_build_printf(builder, " # storing begin\n");
819 #endif
820 for (attrib = 0; attrib < num_outputs; ++attrib) {
821 LLVMValueRef soa[4];
822 LLVMValueRef aos[4];
823 for (chan = 0; chan < NUM_CHANNELS; ++chan) {
824 if (outputs[attrib][chan]) {
825 LLVMValueRef out = LLVMBuildLoad(builder, outputs[attrib][chan], "");
826 lp_build_name(out, "output%u.%c", attrib, "xyzw"[chan]);
827 /*lp_build_printf(builder, "output %d : %d ",
828 LLVMConstInt(LLVMInt32Type(), attrib, 0),
829 LLVMConstInt(LLVMInt32Type(), chan, 0));
830 print_vectorf(builder, out);*/
831 soa[chan] = out;
832 }
833 else {
834 soa[chan] = 0;
835 }
836 }
837 soa_to_aos(gallivm, soa, aos);
838 store_aos_array(gallivm,
839 io,
840 aos,
841 attrib,
842 num_outputs,
843 clipmask);
844 }
845 #if DEBUG_STORE
846 lp_build_printf(builder, " # storing end\n");
847 #endif
848 }
849
850
851 /**
852 * Stores original vertex positions in clip coordinates
853 * There is probably a more efficient way to do this, 4 floats at once
854 * rather than extracting each element one by one.
855 */
856 static void
857 store_clip(struct gallivm_state *gallivm,
858 LLVMValueRef io_ptr,
859 LLVMValueRef (*outputs)[NUM_CHANNELS])
860 {
861 LLVMBuilderRef builder = gallivm->builder;
862 LLVMValueRef out[4];
863 LLVMValueRef indices[2];
864 LLVMValueRef io0_ptr, io1_ptr, io2_ptr, io3_ptr;
865 LLVMValueRef clip_ptr0, clip_ptr1, clip_ptr2, clip_ptr3;
866 LLVMValueRef clip0_ptr, clip1_ptr, clip2_ptr, clip3_ptr;
867 LLVMValueRef out0elem, out1elem, out2elem, out3elem;
868 int i;
869
870 LLVMValueRef ind0 = lp_build_const_int32(gallivm, 0);
871 LLVMValueRef ind1 = lp_build_const_int32(gallivm, 1);
872 LLVMValueRef ind2 = lp_build_const_int32(gallivm, 2);
873 LLVMValueRef ind3 = lp_build_const_int32(gallivm, 3);
874
875 indices[0] =
876 indices[1] = lp_build_const_int32(gallivm, 0);
877
878 out[0] = LLVMBuildLoad(builder, outputs[0][0], ""); /*x0 x1 x2 x3*/
879 out[1] = LLVMBuildLoad(builder, outputs[0][1], ""); /*y0 y1 y2 y3*/
880 out[2] = LLVMBuildLoad(builder, outputs[0][2], ""); /*z0 z1 z2 z3*/
881 out[3] = LLVMBuildLoad(builder, outputs[0][3], ""); /*w0 w1 w2 w3*/
882
883 io0_ptr = LLVMBuildGEP(builder, io_ptr, &ind0, 1, "");
884 io1_ptr = LLVMBuildGEP(builder, io_ptr, &ind1, 1, "");
885 io2_ptr = LLVMBuildGEP(builder, io_ptr, &ind2, 1, "");
886 io3_ptr = LLVMBuildGEP(builder, io_ptr, &ind3, 1, "");
887
888 clip_ptr0 = draw_jit_header_clip(gallivm, io0_ptr);
889 clip_ptr1 = draw_jit_header_clip(gallivm, io1_ptr);
890 clip_ptr2 = draw_jit_header_clip(gallivm, io2_ptr);
891 clip_ptr3 = draw_jit_header_clip(gallivm, io3_ptr);
892
893 for (i = 0; i<4; i++) {
894 clip0_ptr = LLVMBuildGEP(builder, clip_ptr0, indices, 2, ""); /* x0 */
895 clip1_ptr = LLVMBuildGEP(builder, clip_ptr1, indices, 2, ""); /* x1 */
896 clip2_ptr = LLVMBuildGEP(builder, clip_ptr2, indices, 2, ""); /* x2 */
897 clip3_ptr = LLVMBuildGEP(builder, clip_ptr3, indices, 2, ""); /* x3 */
898
899 out0elem = LLVMBuildExtractElement(builder, out[i], ind0, ""); /* x0 */
900 out1elem = LLVMBuildExtractElement(builder, out[i], ind1, ""); /* x1 */
901 out2elem = LLVMBuildExtractElement(builder, out[i], ind2, ""); /* x2 */
902 out3elem = LLVMBuildExtractElement(builder, out[i], ind3, ""); /* x3 */
903
904 LLVMBuildStore(builder, out0elem, clip0_ptr);
905 LLVMBuildStore(builder, out1elem, clip1_ptr);
906 LLVMBuildStore(builder, out2elem, clip2_ptr);
907 LLVMBuildStore(builder, out3elem, clip3_ptr);
908
909 indices[1]= LLVMBuildAdd(builder, indices[1], ind1, "");
910 }
911
912 }
913
914
915 /**
916 * Equivalent of _mm_set1_ps(a)
917 */
918 static LLVMValueRef
919 vec4f_from_scalar(struct gallivm_state *gallivm,
920 LLVMValueRef a,
921 const char *name)
922 {
923 LLVMTypeRef float_type = LLVMFloatTypeInContext(gallivm->context);
924 LLVMValueRef res = LLVMGetUndef(LLVMVectorType(float_type, 4));
925 int i;
926
927 for (i = 0; i < 4; ++i) {
928 LLVMValueRef index = lp_build_const_int32(gallivm, i);
929 res = LLVMBuildInsertElement(gallivm->builder, res, a,
930 index, i == 3 ? name : "");
931 }
932
933 return res;
934 }
935
936
937 /**
938 * Transforms the outputs for viewport mapping
939 */
940 static void
941 generate_viewport(struct draw_llvm *llvm,
942 LLVMBuilderRef builder,
943 LLVMValueRef (*outputs)[NUM_CHANNELS],
944 LLVMValueRef context_ptr)
945 {
946 int i;
947 struct gallivm_state *gallivm = llvm->gallivm;
948 struct lp_type f32_type = lp_type_float_vec(32);
949 LLVMValueRef out3 = LLVMBuildLoad(builder, outputs[0][3], ""); /*w0 w1 w2 w3*/
950 LLVMValueRef const1 = lp_build_const_vec(gallivm, f32_type, 1.0); /*1.0 1.0 1.0 1.0*/
951 LLVMValueRef vp_ptr = draw_jit_context_viewport(gallivm, context_ptr);
952
953 /* for 1/w convention*/
954 out3 = LLVMBuildFDiv(builder, const1, out3, "");
955 LLVMBuildStore(builder, out3, outputs[0][3]);
956
957 /* Viewport Mapping */
958 for (i=0; i<3; i++) {
959 LLVMValueRef out = LLVMBuildLoad(builder, outputs[0][i], ""); /*x0 x1 x2 x3*/
960 LLVMValueRef scale;
961 LLVMValueRef trans;
962 LLVMValueRef scale_i;
963 LLVMValueRef trans_i;
964 LLVMValueRef index;
965
966 index = lp_build_const_int32(gallivm, i);
967 scale_i = LLVMBuildGEP(builder, vp_ptr, &index, 1, "");
968
969 index = lp_build_const_int32(gallivm, i+4);
970 trans_i = LLVMBuildGEP(builder, vp_ptr, &index, 1, "");
971
972 scale = vec4f_from_scalar(gallivm, LLVMBuildLoad(builder, scale_i, ""), "scale");
973 trans = vec4f_from_scalar(gallivm, LLVMBuildLoad(builder, trans_i, ""), "trans");
974
975 /* divide by w */
976 out = LLVMBuildFMul(builder, out, out3, "");
977 /* mult by scale */
978 out = LLVMBuildFMul(builder, out, scale, "");
979 /* add translation */
980 out = LLVMBuildFAdd(builder, out, trans, "");
981
982 /* store transformed outputs */
983 LLVMBuildStore(builder, out, outputs[0][i]);
984 }
985
986 }
987
988
989 /**
990 * Returns clipmask as 4xi32 bitmask for the 4 vertices
991 */
992 static LLVMValueRef
993 generate_clipmask(struct gallivm_state *gallivm,
994 LLVMValueRef (*outputs)[NUM_CHANNELS],
995 boolean clip_xy,
996 boolean clip_z,
997 boolean clip_user,
998 boolean clip_halfz,
999 unsigned nr,
1000 LLVMValueRef context_ptr)
1001 {
1002 LLVMBuilderRef builder = gallivm->builder;
1003 LLVMValueRef mask; /* stores the <4xi32> clipmasks */
1004 LLVMValueRef test, temp;
1005 LLVMValueRef zero, shift;
1006 LLVMValueRef pos_x, pos_y, pos_z, pos_w;
1007 LLVMValueRef plane1, planes, plane_ptr, sum;
1008 unsigned i;
1009 struct lp_type f32_type = lp_type_float_vec(32);
1010
1011 mask = lp_build_const_int_vec(gallivm, lp_type_int_vec(32), 0);
1012 temp = lp_build_const_int_vec(gallivm, lp_type_int_vec(32), 0);
1013 zero = lp_build_const_vec(gallivm, f32_type, 0); /* 0.0f 0.0f 0.0f 0.0f */
1014 shift = lp_build_const_int_vec(gallivm, lp_type_int_vec(32), 1); /* 1 1 1 1 */
1015
1016 /* Assuming position stored at output[0] */
1017 pos_x = LLVMBuildLoad(builder, outputs[0][0], ""); /*x0 x1 x2 x3*/
1018 pos_y = LLVMBuildLoad(builder, outputs[0][1], ""); /*y0 y1 y2 y3*/
1019 pos_z = LLVMBuildLoad(builder, outputs[0][2], ""); /*z0 z1 z2 z3*/
1020 pos_w = LLVMBuildLoad(builder, outputs[0][3], ""); /*w0 w1 w2 w3*/
1021
1022 /* Cliptest, for hardwired planes */
1023 if (clip_xy) {
1024 /* plane 1 */
1025 test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, pos_x , pos_w);
1026 temp = shift;
1027 test = LLVMBuildAnd(builder, test, temp, "");
1028 mask = test;
1029
1030 /* plane 2 */
1031 test = LLVMBuildFAdd(builder, pos_x, pos_w, "");
1032 test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, zero, test);
1033 temp = LLVMBuildShl(builder, temp, shift, "");
1034 test = LLVMBuildAnd(builder, test, temp, "");
1035 mask = LLVMBuildOr(builder, mask, test, "");
1036
1037 /* plane 3 */
1038 test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, pos_y, pos_w);
1039 temp = LLVMBuildShl(builder, temp, shift, "");
1040 test = LLVMBuildAnd(builder, test, temp, "");
1041 mask = LLVMBuildOr(builder, mask, test, "");
1042
1043 /* plane 4 */
1044 test = LLVMBuildFAdd(builder, pos_y, pos_w, "");
1045 test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, zero, test);
1046 temp = LLVMBuildShl(builder, temp, shift, "");
1047 test = LLVMBuildAnd(builder, test, temp, "");
1048 mask = LLVMBuildOr(builder, mask, test, "");
1049 }
1050
1051 if (clip_z) {
1052 temp = lp_build_const_int_vec(gallivm, lp_type_int_vec(32), 16);
1053 if (clip_halfz) {
1054 /* plane 5 */
1055 test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, zero, pos_z);
1056 test = LLVMBuildAnd(builder, test, temp, "");
1057 mask = LLVMBuildOr(builder, mask, test, "");
1058 }
1059 else {
1060 /* plane 5 */
1061 test = LLVMBuildFAdd(builder, pos_z, pos_w, "");
1062 test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, zero, test);
1063 test = LLVMBuildAnd(builder, test, temp, "");
1064 mask = LLVMBuildOr(builder, mask, test, "");
1065 }
1066 /* plane 6 */
1067 test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, pos_z, pos_w);
1068 temp = LLVMBuildShl(builder, temp, shift, "");
1069 test = LLVMBuildAnd(builder, test, temp, "");
1070 mask = LLVMBuildOr(builder, mask, test, "");
1071 }
1072
1073 if (clip_user) {
1074 LLVMValueRef planes_ptr = draw_jit_context_planes(gallivm, context_ptr);
1075 LLVMValueRef indices[3];
1076 temp = lp_build_const_int_vec(gallivm, lp_type_int_vec(32), 32);
1077
1078 /* userclip planes */
1079 for (i = 6; i < nr; i++) {
1080 indices[0] = lp_build_const_int32(gallivm, 0);
1081 indices[1] = lp_build_const_int32(gallivm, i);
1082
1083 indices[2] = lp_build_const_int32(gallivm, 0);
1084 plane_ptr = LLVMBuildGEP(builder, planes_ptr, indices, 3, "");
1085 plane1 = LLVMBuildLoad(builder, plane_ptr, "plane_x");
1086 planes = vec4f_from_scalar(gallivm, plane1, "plane4_x");
1087 sum = LLVMBuildFMul(builder, planes, pos_x, "");
1088
1089 indices[2] = lp_build_const_int32(gallivm, 1);
1090 plane_ptr = LLVMBuildGEP(builder, planes_ptr, indices, 3, "");
1091 plane1 = LLVMBuildLoad(builder, plane_ptr, "plane_y");
1092 planes = vec4f_from_scalar(gallivm, plane1, "plane4_y");
1093 test = LLVMBuildFMul(builder, planes, pos_y, "");
1094 sum = LLVMBuildFAdd(builder, sum, test, "");
1095
1096 indices[2] = lp_build_const_int32(gallivm, 2);
1097 plane_ptr = LLVMBuildGEP(builder, planes_ptr, indices, 3, "");
1098 plane1 = LLVMBuildLoad(builder, plane_ptr, "plane_z");
1099 planes = vec4f_from_scalar(gallivm, plane1, "plane4_z");
1100 test = LLVMBuildFMul(builder, planes, pos_z, "");
1101 sum = LLVMBuildFAdd(builder, sum, test, "");
1102
1103 indices[2] = lp_build_const_int32(gallivm, 3);
1104 plane_ptr = LLVMBuildGEP(builder, planes_ptr, indices, 3, "");
1105 plane1 = LLVMBuildLoad(builder, plane_ptr, "plane_w");
1106 planes = vec4f_from_scalar(gallivm, plane1, "plane4_w");
1107 test = LLVMBuildFMul(builder, planes, pos_w, "");
1108 sum = LLVMBuildFAdd(builder, sum, test, "");
1109
1110 test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, zero, sum);
1111 temp = LLVMBuildShl(builder, temp, shift, "");
1112 test = LLVMBuildAnd(builder, test, temp, "");
1113 mask = LLVMBuildOr(builder, mask, test, "");
1114 }
1115 }
1116 return mask;
1117 }
1118
1119
1120 /**
1121 * Returns boolean if any clipping has occurred
1122 * Used zero/non-zero i32 value to represent boolean
1123 */
1124 static void
1125 clipmask_bool(struct gallivm_state *gallivm,
1126 LLVMValueRef clipmask,
1127 LLVMValueRef ret_ptr)
1128 {
1129 LLVMBuilderRef builder = gallivm->builder;
1130 LLVMValueRef ret = LLVMBuildLoad(builder, ret_ptr, "");
1131 LLVMValueRef temp;
1132 int i;
1133
1134 for (i=0; i<4; i++) {
1135 temp = LLVMBuildExtractElement(builder, clipmask,
1136 lp_build_const_int32(gallivm, i) , "");
1137 ret = LLVMBuildOr(builder, ret, temp, "");
1138 }
1139
1140 LLVMBuildStore(builder, ret, ret_ptr);
1141 }
1142
1143
1144 static void
1145 draw_llvm_generate(struct draw_llvm *llvm, struct draw_llvm_variant *variant)
1146 {
1147 struct gallivm_state *gallivm = llvm->gallivm;
1148 LLVMContextRef context = gallivm->context;
1149 LLVMTypeRef int32_type = LLVMInt32TypeInContext(context);
1150 LLVMTypeRef arg_types[8];
1151 LLVMTypeRef func_type;
1152 LLVMValueRef context_ptr;
1153 LLVMBasicBlockRef block;
1154 LLVMBuilderRef builder;
1155 LLVMValueRef start, end, count, stride, step, io_itr;
1156 LLVMValueRef io_ptr, vbuffers_ptr, vb_ptr;
1157 LLVMValueRef instance_id;
1158 LLVMValueRef system_values_array;
1159 struct draw_context *draw = llvm->draw;
1160 const struct tgsi_shader_info *vs_info = &draw->vs.vertex_shader->info;
1161 unsigned i, j;
1162 struct lp_build_context bld;
1163 struct lp_build_loop_state lp_loop;
1164 const int max_vertices = 4;
1165 LLVMValueRef outputs[PIPE_MAX_SHADER_OUTPUTS][NUM_CHANNELS];
1166 void *code;
1167 struct lp_build_sampler_soa *sampler = 0;
1168 LLVMValueRef ret, ret_ptr;
1169 boolean bypass_viewport = variant->key.bypass_viewport;
1170 boolean enable_cliptest = variant->key.clip_xy ||
1171 variant->key.clip_z ||
1172 variant->key.clip_user;
1173
1174 arg_types[0] = get_context_ptr_type(llvm); /* context */
1175 arg_types[1] = get_vertex_header_ptr_type(llvm); /* vertex_header */
1176 arg_types[2] = get_buffer_ptr_type(llvm); /* vbuffers */
1177 arg_types[3] = int32_type; /* start */
1178 arg_types[4] = int32_type; /* count */
1179 arg_types[5] = int32_type; /* stride */
1180 arg_types[6] = get_vb_ptr_type(llvm); /* pipe_vertex_buffer's */
1181 arg_types[7] = int32_type; /* instance_id */
1182
1183 func_type = LLVMFunctionType(int32_type, arg_types, Elements(arg_types), 0);
1184
1185 variant->function = LLVMAddFunction(gallivm->module, "draw_llvm_shader",
1186 func_type);
1187 LLVMSetFunctionCallConv(variant->function, LLVMCCallConv);
1188 for (i = 0; i < Elements(arg_types); ++i)
1189 if (LLVMGetTypeKind(arg_types[i]) == LLVMPointerTypeKind)
1190 LLVMAddAttribute(LLVMGetParam(variant->function, i), LLVMNoAliasAttribute);
1191
1192 context_ptr = LLVMGetParam(variant->function, 0);
1193 io_ptr = LLVMGetParam(variant->function, 1);
1194 vbuffers_ptr = LLVMGetParam(variant->function, 2);
1195 start = LLVMGetParam(variant->function, 3);
1196 count = LLVMGetParam(variant->function, 4);
1197 stride = LLVMGetParam(variant->function, 5);
1198 vb_ptr = LLVMGetParam(variant->function, 6);
1199 instance_id = LLVMGetParam(variant->function, 7);
1200
1201 lp_build_name(context_ptr, "context");
1202 lp_build_name(io_ptr, "io");
1203 lp_build_name(vbuffers_ptr, "vbuffers");
1204 lp_build_name(start, "start");
1205 lp_build_name(count, "count");
1206 lp_build_name(stride, "stride");
1207 lp_build_name(vb_ptr, "vb");
1208 lp_build_name(instance_id, "instance_id");
1209
1210 /*
1211 * Function body
1212 */
1213
1214 block = LLVMAppendBasicBlockInContext(gallivm->context, variant->function, "entry");
1215 builder = gallivm->builder;
1216 assert(builder);
1217 LLVMPositionBuilderAtEnd(builder, block);
1218
1219 lp_build_context_init(&bld, llvm->gallivm, lp_type_int(32));
1220
1221 system_values_array = lp_build_system_values_array(gallivm, vs_info,
1222 instance_id, NULL);
1223
1224 end = lp_build_add(&bld, start, count);
1225
1226 step = lp_build_const_int32(gallivm, max_vertices);
1227
1228 /* function will return non-zero i32 value if any clipped vertices */
1229 ret_ptr = lp_build_alloca(gallivm, int32_type, "");
1230 LLVMBuildStore(builder, lp_build_const_int32(gallivm, 0), ret_ptr);
1231
1232 /* code generated texture sampling */
1233 sampler = draw_llvm_sampler_soa_create(
1234 draw_llvm_variant_key_samplers(&variant->key),
1235 context_ptr);
1236
1237 #if DEBUG_STORE
1238 lp_build_printf(builder, "start = %d, end = %d, step = %d\n",
1239 start, end, step);
1240 #endif
1241 lp_build_loop_begin(&lp_loop, llvm->gallivm, start);
1242 {
1243 LLVMValueRef inputs[PIPE_MAX_SHADER_INPUTS][NUM_CHANNELS];
1244 LLVMValueRef aos_attribs[PIPE_MAX_SHADER_INPUTS][NUM_CHANNELS] = { { 0 } };
1245 LLVMValueRef io;
1246 LLVMValueRef clipmask; /* holds the clipmask value */
1247 const LLVMValueRef (*ptr_aos)[NUM_CHANNELS];
1248
1249 io_itr = LLVMBuildSub(builder, lp_loop.counter, start, "");
1250 io = LLVMBuildGEP(builder, io_ptr, &io_itr, 1, "");
1251 #if DEBUG_STORE
1252 lp_build_printf(builder, " --- io %d = %p, loop counter %d\n",
1253 io_itr, io, lp_loop.counter);
1254 #endif
1255 for (i = 0; i < NUM_CHANNELS; ++i) {
1256 LLVMValueRef true_index = LLVMBuildAdd(
1257 builder,
1258 lp_loop.counter,
1259 lp_build_const_int32(gallivm, i), "");
1260 for (j = 0; j < draw->pt.nr_vertex_elements; ++j) {
1261 struct pipe_vertex_element *velem = &draw->pt.vertex_element[j];
1262 LLVMValueRef vb_index = lp_build_const_int32(gallivm, velem->vertex_buffer_index);
1263 LLVMValueRef vb = LLVMBuildGEP(builder, vb_ptr,
1264 &vb_index, 1, "");
1265 generate_fetch(llvm->gallivm, vbuffers_ptr,
1266 &aos_attribs[j][i], velem, vb, true_index,
1267 instance_id);
1268 }
1269 }
1270 convert_to_soa(gallivm, aos_attribs, inputs,
1271 draw->pt.nr_vertex_elements);
1272
1273 ptr_aos = (const LLVMValueRef (*)[NUM_CHANNELS]) inputs;
1274 generate_vs(llvm,
1275 builder,
1276 outputs,
1277 ptr_aos,
1278 system_values_array,
1279 context_ptr,
1280 sampler,
1281 variant->key.clamp_vertex_color);
1282
1283 /* store original positions in clip before further manipulation */
1284 store_clip(gallivm, io, outputs);
1285
1286 /* do cliptest */
1287 if (enable_cliptest) {
1288 /* allocate clipmask, assign it integer type */
1289 clipmask = generate_clipmask(gallivm, outputs,
1290 variant->key.clip_xy,
1291 variant->key.clip_z,
1292 variant->key.clip_user,
1293 variant->key.clip_halfz,
1294 variant->key.nr_planes,
1295 context_ptr);
1296 /* return clipping boolean value for function */
1297 clipmask_bool(gallivm, clipmask, ret_ptr);
1298 }
1299 else {
1300 clipmask = lp_build_const_int_vec(gallivm, lp_type_int_vec(32), 0);
1301 }
1302
1303 /* do viewport mapping */
1304 if (!bypass_viewport) {
1305 generate_viewport(llvm, builder, outputs, context_ptr);
1306 }
1307
1308 /* store clipmask in vertex header and positions in data */
1309 convert_to_aos(gallivm, io, outputs, clipmask,
1310 vs_info->num_outputs, max_vertices);
1311 }
1312
1313 lp_build_loop_end_cond(&lp_loop, end, step, LLVMIntUGE);
1314
1315 sampler->destroy(sampler);
1316
1317 ret = LLVMBuildLoad(builder, ret_ptr,"");
1318 LLVMBuildRet(builder, ret);
1319
1320 /*
1321 * Translate the LLVM IR into machine code.
1322 */
1323 #ifdef DEBUG
1324 if (LLVMVerifyFunction(variant->function, LLVMPrintMessageAction)) {
1325 lp_debug_dump_value(variant->function);
1326 assert(0);
1327 }
1328 #endif
1329
1330 LLVMRunFunctionPassManager(gallivm->passmgr, variant->function);
1331
1332 if (gallivm_debug & GALLIVM_DEBUG_IR) {
1333 lp_debug_dump_value(variant->function);
1334 debug_printf("\n");
1335 }
1336
1337 code = LLVMGetPointerToGlobal(gallivm->engine, variant->function);
1338 variant->jit_func = (draw_jit_vert_func)pointer_to_func(code);
1339
1340 if (gallivm_debug & GALLIVM_DEBUG_ASM) {
1341 lp_disassemble(code);
1342 }
1343 lp_func_delete_body(variant->function);
1344 }
1345
1346
1347 static void
1348 draw_llvm_generate_elts(struct draw_llvm *llvm, struct draw_llvm_variant *variant)
1349 {
1350 struct gallivm_state *gallivm = llvm->gallivm;
1351 LLVMContextRef context = gallivm->context;
1352 LLVMTypeRef int32_type = LLVMInt32TypeInContext(context);
1353 LLVMTypeRef arg_types[8];
1354 LLVMTypeRef func_type;
1355 LLVMValueRef context_ptr;
1356 LLVMBasicBlockRef block;
1357 LLVMBuilderRef builder;
1358 LLVMValueRef fetch_elts, fetch_count, stride, step, io_itr;
1359 LLVMValueRef io_ptr, vbuffers_ptr, vb_ptr;
1360 LLVMValueRef instance_id;
1361 LLVMValueRef system_values_array;
1362 struct draw_context *draw = llvm->draw;
1363 const struct tgsi_shader_info *vs_info = &draw->vs.vertex_shader->info;
1364 unsigned i, j;
1365 struct lp_build_context bld;
1366 struct lp_build_loop_state lp_loop;
1367 const int max_vertices = 4;
1368 LLVMValueRef outputs[PIPE_MAX_SHADER_OUTPUTS][NUM_CHANNELS];
1369 LLVMValueRef fetch_max;
1370 void *code;
1371 struct lp_build_sampler_soa *sampler = 0;
1372 LLVMValueRef ret, ret_ptr;
1373 boolean bypass_viewport = variant->key.bypass_viewport;
1374 boolean enable_cliptest = variant->key.clip_xy ||
1375 variant->key.clip_z ||
1376 variant->key.clip_user;
1377
1378 arg_types[0] = get_context_ptr_type(llvm); /* context */
1379 arg_types[1] = get_vertex_header_ptr_type(llvm); /* vertex_header */
1380 arg_types[2] = get_buffer_ptr_type(llvm); /* vbuffers */
1381 arg_types[3] = LLVMPointerType(int32_type, 0); /* fetch_elts * */
1382 arg_types[4] = int32_type; /* fetch_count */
1383 arg_types[5] = int32_type; /* stride */
1384 arg_types[6] = get_vb_ptr_type(llvm); /* pipe_vertex_buffer's */
1385 arg_types[7] = int32_type; /* instance_id */
1386
1387 func_type = LLVMFunctionType(int32_type, arg_types, Elements(arg_types), 0);
1388
1389 variant->function_elts = LLVMAddFunction(gallivm->module, "draw_llvm_shader_elts", func_type);
1390 LLVMSetFunctionCallConv(variant->function_elts, LLVMCCallConv);
1391 for (i = 0; i < Elements(arg_types); ++i)
1392 if (LLVMGetTypeKind(arg_types[i]) == LLVMPointerTypeKind)
1393 LLVMAddAttribute(LLVMGetParam(variant->function_elts, i),
1394 LLVMNoAliasAttribute);
1395
1396 context_ptr = LLVMGetParam(variant->function_elts, 0);
1397 io_ptr = LLVMGetParam(variant->function_elts, 1);
1398 vbuffers_ptr = LLVMGetParam(variant->function_elts, 2);
1399 fetch_elts = LLVMGetParam(variant->function_elts, 3);
1400 fetch_count = LLVMGetParam(variant->function_elts, 4);
1401 stride = LLVMGetParam(variant->function_elts, 5);
1402 vb_ptr = LLVMGetParam(variant->function_elts, 6);
1403 instance_id = LLVMGetParam(variant->function_elts, 7);
1404
1405 lp_build_name(context_ptr, "context");
1406 lp_build_name(io_ptr, "io");
1407 lp_build_name(vbuffers_ptr, "vbuffers");
1408 lp_build_name(fetch_elts, "fetch_elts");
1409 lp_build_name(fetch_count, "fetch_count");
1410 lp_build_name(stride, "stride");
1411 lp_build_name(vb_ptr, "vb");
1412 lp_build_name(instance_id, "instance_id");
1413
1414 /*
1415 * Function body
1416 */
1417
1418 block = LLVMAppendBasicBlockInContext(gallivm->context, variant->function_elts, "entry");
1419 builder = gallivm->builder;
1420 LLVMPositionBuilderAtEnd(builder, block);
1421
1422 lp_build_context_init(&bld, gallivm, lp_type_int(32));
1423
1424 system_values_array = lp_build_system_values_array(gallivm, vs_info,
1425 instance_id, NULL);
1426
1427
1428 step = lp_build_const_int32(gallivm, max_vertices);
1429
1430 /* code generated texture sampling */
1431 sampler = draw_llvm_sampler_soa_create(
1432 draw_llvm_variant_key_samplers(&variant->key),
1433 context_ptr);
1434
1435 fetch_max = LLVMBuildSub(builder, fetch_count,
1436 lp_build_const_int32(gallivm, 1),
1437 "fetch_max");
1438
1439 /* function returns non-zero i32 value if any clipped vertices */
1440 ret_ptr = lp_build_alloca(gallivm, int32_type, "");
1441 LLVMBuildStore(builder, lp_build_const_int32(gallivm, 0), ret_ptr);
1442
1443 lp_build_loop_begin(&lp_loop, gallivm, lp_build_const_int32(gallivm, 0));
1444 {
1445 LLVMValueRef inputs[PIPE_MAX_SHADER_INPUTS][NUM_CHANNELS];
1446 LLVMValueRef aos_attribs[PIPE_MAX_SHADER_INPUTS][NUM_CHANNELS] = { { 0 } };
1447 LLVMValueRef io;
1448 LLVMValueRef clipmask; /* holds the clipmask value */
1449 const LLVMValueRef (*ptr_aos)[NUM_CHANNELS];
1450
1451 io_itr = lp_loop.counter;
1452 io = LLVMBuildGEP(builder, io_ptr, &io_itr, 1, "");
1453 #if DEBUG_STORE
1454 lp_build_printf(builder, " --- io %d = %p, loop counter %d\n",
1455 io_itr, io, lp_loop.counter);
1456 #endif
1457 for (i = 0; i < NUM_CHANNELS; ++i) {
1458 LLVMValueRef true_index = LLVMBuildAdd(
1459 builder,
1460 lp_loop.counter,
1461 lp_build_const_int32(gallivm, i), "");
1462 LLVMValueRef fetch_ptr;
1463
1464 /* make sure we're not out of bounds which can happen
1465 * if fetch_count % 4 != 0, because on the last iteration
1466 * a few of the 4 vertex fetches will be out of bounds */
1467 true_index = lp_build_min(&bld, true_index, fetch_max);
1468
1469 fetch_ptr = LLVMBuildGEP(builder, fetch_elts,
1470 &true_index, 1, "");
1471 true_index = LLVMBuildLoad(builder, fetch_ptr, "fetch_elt");
1472 for (j = 0; j < draw->pt.nr_vertex_elements; ++j) {
1473 struct pipe_vertex_element *velem = &draw->pt.vertex_element[j];
1474 LLVMValueRef vb_index = lp_build_const_int32(gallivm, velem->vertex_buffer_index);
1475 LLVMValueRef vb = LLVMBuildGEP(builder, vb_ptr,
1476 &vb_index, 1, "");
1477 generate_fetch(gallivm, vbuffers_ptr,
1478 &aos_attribs[j][i], velem, vb, true_index,
1479 instance_id);
1480 }
1481 }
1482 convert_to_soa(gallivm, aos_attribs, inputs,
1483 draw->pt.nr_vertex_elements);
1484
1485 ptr_aos = (const LLVMValueRef (*)[NUM_CHANNELS]) inputs;
1486 generate_vs(llvm,
1487 builder,
1488 outputs,
1489 ptr_aos,
1490 system_values_array,
1491 context_ptr,
1492 sampler,
1493 variant->key.clamp_vertex_color);
1494
1495 /* store original positions in clip before further manipulation */
1496 store_clip(gallivm, io, outputs);
1497
1498 /* do cliptest */
1499 if (enable_cliptest) {
1500 /* allocate clipmask, assign it integer type */
1501 clipmask = generate_clipmask(gallivm, outputs,
1502 variant->key.clip_xy,
1503 variant->key.clip_z,
1504 variant->key.clip_user,
1505 variant->key.clip_halfz,
1506 variant->key.nr_planes,
1507 context_ptr);
1508 /* return clipping boolean value for function */
1509 clipmask_bool(gallivm, clipmask, ret_ptr);
1510 }
1511 else {
1512 clipmask = lp_build_const_int_vec(gallivm, lp_type_int_vec(32), 0);
1513 }
1514
1515 /* do viewport mapping */
1516 if (!bypass_viewport) {
1517 generate_viewport(llvm, builder, outputs, context_ptr);
1518 }
1519
1520 /* store clipmask in vertex header,
1521 * original positions in clip
1522 * and transformed positions in data
1523 */
1524 convert_to_aos(gallivm, io, outputs, clipmask,
1525 vs_info->num_outputs, max_vertices);
1526 }
1527
1528 lp_build_loop_end_cond(&lp_loop, fetch_count, step, LLVMIntUGE);
1529
1530 sampler->destroy(sampler);
1531
1532 ret = LLVMBuildLoad(builder, ret_ptr,"");
1533 LLVMBuildRet(builder, ret);
1534
1535 /*
1536 * Translate the LLVM IR into machine code.
1537 */
1538 #ifdef DEBUG
1539 if (LLVMVerifyFunction(variant->function_elts, LLVMPrintMessageAction)) {
1540 lp_debug_dump_value(variant->function_elts);
1541 assert(0);
1542 }
1543 #endif
1544
1545 LLVMRunFunctionPassManager(gallivm->passmgr, variant->function_elts);
1546
1547 if (gallivm_debug & GALLIVM_DEBUG_IR) {
1548 lp_debug_dump_value(variant->function_elts);
1549 debug_printf("\n");
1550 }
1551
1552 code = LLVMGetPointerToGlobal(gallivm->engine, variant->function_elts);
1553 variant->jit_func_elts = (draw_jit_vert_func_elts)pointer_to_func(code);
1554
1555 if (gallivm_debug & GALLIVM_DEBUG_ASM) {
1556 lp_disassemble(code);
1557 }
1558 lp_func_delete_body(variant->function_elts);
1559 }
1560
1561
1562 struct draw_llvm_variant_key *
1563 draw_llvm_make_variant_key(struct draw_llvm *llvm, char *store)
1564 {
1565 unsigned i;
1566 struct draw_llvm_variant_key *key;
1567 struct lp_sampler_static_state *sampler;
1568
1569 key = (struct draw_llvm_variant_key *)store;
1570
1571 key->clamp_vertex_color = llvm->draw->rasterizer->clamp_vertex_color; /**/
1572
1573 /* Presumably all variants of the shader should have the same
1574 * number of vertex elements - ie the number of shader inputs.
1575 */
1576 key->nr_vertex_elements = llvm->draw->pt.nr_vertex_elements;
1577
1578 /* will have to rig this up properly later */
1579 key->clip_xy = llvm->draw->clip_xy;
1580 key->clip_z = llvm->draw->clip_z;
1581 key->clip_user = llvm->draw->clip_user;
1582 key->bypass_viewport = llvm->draw->identity_viewport;
1583 key->clip_halfz = !llvm->draw->rasterizer->gl_rasterization_rules;
1584 key->need_edgeflags = (llvm->draw->vs.edgeflag_output ? TRUE : FALSE);
1585 key->nr_planes = llvm->draw->nr_planes;
1586 key->pad = 0;
1587
1588 /* All variants of this shader will have the same value for
1589 * nr_samplers. Not yet trying to compact away holes in the
1590 * sampler array.
1591 */
1592 key->nr_samplers = llvm->draw->vs.vertex_shader->info.file_max[TGSI_FILE_SAMPLER] + 1;
1593
1594 sampler = draw_llvm_variant_key_samplers(key);
1595
1596 memcpy(key->vertex_element,
1597 llvm->draw->pt.vertex_element,
1598 sizeof(struct pipe_vertex_element) * key->nr_vertex_elements);
1599
1600 memset(sampler, 0, key->nr_samplers * sizeof *sampler);
1601
1602 for (i = 0 ; i < key->nr_samplers; i++) {
1603 lp_sampler_static_state(&sampler[i],
1604 llvm->draw->sampler_views[i],
1605 llvm->draw->samplers[i]);
1606 }
1607
1608 return key;
1609 }
1610
1611
1612 void
1613 draw_llvm_set_mapped_texture(struct draw_context *draw,
1614 unsigned sampler_idx,
1615 uint32_t width, uint32_t height, uint32_t depth,
1616 uint32_t first_level, uint32_t last_level,
1617 uint32_t row_stride[PIPE_MAX_TEXTURE_LEVELS],
1618 uint32_t img_stride[PIPE_MAX_TEXTURE_LEVELS],
1619 const void *data[PIPE_MAX_TEXTURE_LEVELS])
1620 {
1621 unsigned j;
1622 struct draw_jit_texture *jit_tex;
1623
1624 assert(sampler_idx < PIPE_MAX_VERTEX_SAMPLERS);
1625
1626 jit_tex = &draw->llvm->jit_context.textures[sampler_idx];
1627
1628 jit_tex->width = width;
1629 jit_tex->height = height;
1630 jit_tex->depth = depth;
1631 jit_tex->first_level = first_level;
1632 jit_tex->last_level = last_level;
1633
1634 for (j = first_level; j <= last_level; j++) {
1635 jit_tex->data[j] = data[j];
1636 jit_tex->row_stride[j] = row_stride[j];
1637 jit_tex->img_stride[j] = img_stride[j];
1638 }
1639 }
1640
1641
1642 void
1643 draw_llvm_set_sampler_state(struct draw_context *draw)
1644 {
1645 unsigned i;
1646
1647 for (i = 0; i < draw->num_samplers; i++) {
1648 struct draw_jit_texture *jit_tex = &draw->llvm->jit_context.textures[i];
1649
1650 if (draw->samplers[i]) {
1651 jit_tex->min_lod = draw->samplers[i]->min_lod;
1652 jit_tex->max_lod = draw->samplers[i]->max_lod;
1653 jit_tex->lod_bias = draw->samplers[i]->lod_bias;
1654 COPY_4V(jit_tex->border_color, draw->samplers[i]->border_color);
1655 }
1656 }
1657 }
1658
1659
1660 void
1661 draw_llvm_destroy_variant(struct draw_llvm_variant *variant)
1662 {
1663 struct draw_llvm *llvm = variant->llvm;
1664
1665 if (variant->function_elts) {
1666 LLVMFreeMachineCodeForFunction(llvm->gallivm->engine,
1667 variant->function_elts);
1668 LLVMDeleteFunction(variant->function_elts);
1669 }
1670
1671 if (variant->function) {
1672 LLVMFreeMachineCodeForFunction(llvm->gallivm->engine,
1673 variant->function);
1674 LLVMDeleteFunction(variant->function);
1675 }
1676
1677 remove_from_list(&variant->list_item_local);
1678 variant->shader->variants_cached--;
1679 remove_from_list(&variant->list_item_global);
1680 llvm->nr_variants--;
1681 FREE(variant);
1682 }