projects / mesa.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
history | raw | HEAD
Squash-merge branch 'gallium-clip-state'
[mesa.git] / src / gallium / auxiliary / draw / draw_llvm.c
1 /**************************************************************************
2 *
3 * Copyright 2010 VMware, Inc.
4 * All Rights Reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sub license, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
13 *
14 * The above copyright notice and this permission notice (including the
15 * next paragraph) shall be included in all copies or substantial portions
16 * of the Software.
17 *
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
19 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
20 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
21 * IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR
22 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
23 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
24 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
25 *
26 **************************************************************************/
27
28 #include "draw_llvm.h"
29
30 #include "draw_context.h"
31 #include "draw_vs.h"
32
33 #include "gallivm/lp_bld_arit.h"
34 #include "gallivm/lp_bld_logic.h"
35 #include "gallivm/lp_bld_const.h"
36 #include "gallivm/lp_bld_swizzle.h"
37 #include "gallivm/lp_bld_struct.h"
38 #include "gallivm/lp_bld_type.h"
39 #include "gallivm/lp_bld_flow.h"
40 #include "gallivm/lp_bld_debug.h"
41 #include "gallivm/lp_bld_tgsi.h"
42 #include "gallivm/lp_bld_printf.h"
43 #include "gallivm/lp_bld_intr.h"
44 #include "gallivm/lp_bld_init.h"
45 #include "gallivm/lp_bld_type.h"
46
47 #include "tgsi/tgsi_exec.h"
48 #include "tgsi/tgsi_dump.h"
49
50 #include "util/u_math.h"
51 #include "util/u_pointer.h"
52 #include "util/u_string.h"
53 #include "util/u_simple_list.h"
54
55
56 #define DEBUG_STORE 0
57
58
59 /**
60 * This function is called by the gallivm "garbage collector" when
61 * the LLVM global data structures are freed. We must free all LLVM-related
62 * data. Specifically, all JIT'd shader variants.
63 */
64 static void
65 draw_llvm_garbage_collect_callback(void *cb_data)
66 {
67 struct draw_llvm *llvm = (struct draw_llvm *) cb_data;
68 struct draw_llvm_variant_list_item *li;
69
70 /* free all shader variants */
71 li = first_elem(&llvm->vs_variants_list);
72 while (!at_end(&llvm->vs_variants_list, li)) {
73 struct draw_llvm_variant_list_item *next = next_elem(li);
74 draw_llvm_destroy_variant(li->base);
75 li = next;
76 }
77
78 /* Null-out these pointers so they get remade next time they're needed.
79 * See the accessor functions below.
80 */
81 llvm->context_ptr_type = NULL;
82 llvm->buffer_ptr_type = NULL;
83 llvm->vb_ptr_type = NULL;
84 llvm->vertex_header_ptr_type = NULL;
85 }
86
87
88 static void
89 draw_llvm_generate(struct draw_llvm *llvm, struct draw_llvm_variant *var,
90 boolean elts);
91
92
93 /**
94 * Create LLVM type for struct draw_jit_texture
95 */
96 static LLVMTypeRef
97 create_jit_texture_type(struct gallivm_state *gallivm, const char *struct_name)
98 {
99 LLVMTargetDataRef target = gallivm->target;
100 LLVMTypeRef texture_type;
101 LLVMTypeRef elem_types[DRAW_JIT_TEXTURE_NUM_FIELDS];
102 LLVMTypeRef int32_type = LLVMInt32TypeInContext(gallivm->context);
103
104 elem_types[DRAW_JIT_TEXTURE_WIDTH] =
105 elem_types[DRAW_JIT_TEXTURE_HEIGHT] =
106 elem_types[DRAW_JIT_TEXTURE_DEPTH] =
107 elem_types[DRAW_JIT_TEXTURE_FIRST_LEVEL] =
108 elem_types[DRAW_JIT_TEXTURE_LAST_LEVEL] = int32_type;
109 elem_types[DRAW_JIT_TEXTURE_ROW_STRIDE] =
110 elem_types[DRAW_JIT_TEXTURE_IMG_STRIDE] =
111 LLVMArrayType(int32_type, PIPE_MAX_TEXTURE_LEVELS);
112 elem_types[DRAW_JIT_TEXTURE_DATA] =
113 LLVMArrayType(LLVMPointerType(LLVMInt8TypeInContext(gallivm->context), 0),
114 PIPE_MAX_TEXTURE_LEVELS);
115 elem_types[DRAW_JIT_TEXTURE_MIN_LOD] =
116 elem_types[DRAW_JIT_TEXTURE_MAX_LOD] =
117 elem_types[DRAW_JIT_TEXTURE_LOD_BIAS] = LLVMFloatTypeInContext(gallivm->context);
118 elem_types[DRAW_JIT_TEXTURE_BORDER_COLOR] =
119 LLVMArrayType(LLVMFloatTypeInContext(gallivm->context), 4);
120
121 #if HAVE_LLVM >= 0x0300
122 texture_type = LLVMStructCreateNamed(gallivm->context, struct_name);
123 LLVMStructSetBody(texture_type, elem_types,
124 Elements(elem_types), 0);
125 #else
126 texture_type = LLVMStructTypeInContext(gallivm->context, elem_types,
127 Elements(elem_types), 0);
128
129 LLVMAddTypeName(gallivm->module, struct_name, texture_type);
130
131 /* Make sure the target's struct layout cache doesn't return
132 * stale/invalid data.
133 */
134 LLVMInvalidateStructLayout(gallivm->target, texture_type);
135 #endif
136
137 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, width,
138 target, texture_type,
139 DRAW_JIT_TEXTURE_WIDTH);
140 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, height,
141 target, texture_type,
142 DRAW_JIT_TEXTURE_HEIGHT);
143 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, depth,
144 target, texture_type,
145 DRAW_JIT_TEXTURE_DEPTH);
146 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, first_level,
147 target, texture_type,
148 DRAW_JIT_TEXTURE_FIRST_LEVEL);
149 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, last_level,
150 target, texture_type,
151 DRAW_JIT_TEXTURE_LAST_LEVEL);
152 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, row_stride,
153 target, texture_type,
154 DRAW_JIT_TEXTURE_ROW_STRIDE);
155 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, img_stride,
156 target, texture_type,
157 DRAW_JIT_TEXTURE_IMG_STRIDE);
158 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, data,
159 target, texture_type,
160 DRAW_JIT_TEXTURE_DATA);
161 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, min_lod,
162 target, texture_type,
163 DRAW_JIT_TEXTURE_MIN_LOD);
164 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, max_lod,
165 target, texture_type,
166 DRAW_JIT_TEXTURE_MAX_LOD);
167 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, lod_bias,
168 target, texture_type,
169 DRAW_JIT_TEXTURE_LOD_BIAS);
170 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, border_color,
171 target, texture_type,
172 DRAW_JIT_TEXTURE_BORDER_COLOR);
173
174 LP_CHECK_STRUCT_SIZE(struct draw_jit_texture, target, texture_type);
175
176 return texture_type;
177 }
178
179
180 /**
181 * Create LLVM type for struct draw_jit_texture
182 */
183 static LLVMTypeRef
184 create_jit_context_type(struct gallivm_state *gallivm,
185 LLVMTypeRef texture_type, const char *struct_name)
186 {
187 LLVMTargetDataRef target = gallivm->target;
188 LLVMTypeRef float_type = LLVMFloatTypeInContext(gallivm->context);
189 LLVMTypeRef elem_types[5];
190 LLVMTypeRef context_type;
191
192 elem_types[0] = LLVMPointerType(float_type, 0); /* vs_constants */
193 elem_types[1] = LLVMPointerType(float_type, 0); /* gs_constants */
194 elem_types[2] = LLVMPointerType(LLVMArrayType(LLVMArrayType(float_type, 4),
195 DRAW_TOTAL_CLIP_PLANES), 0);
196 elem_types[3] = LLVMPointerType(float_type, 0); /* viewport */
197 elem_types[4] = LLVMArrayType(texture_type,
198 PIPE_MAX_VERTEX_SAMPLERS); /* textures */
199 #if HAVE_LLVM >= 0x0300
200 context_type = LLVMStructCreateNamed(gallivm->context, struct_name);
201 LLVMStructSetBody(context_type, elem_types,
202 Elements(elem_types), 0);
203 #else
204 context_type = LLVMStructTypeInContext(gallivm->context, elem_types,
205 Elements(elem_types), 0);
206 LLVMAddTypeName(gallivm->module, struct_name, context_type);
207
208 LLVMInvalidateStructLayout(gallivm->target, context_type);
209 #endif
210
211 LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, vs_constants,
212 target, context_type, 0);
213 LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, gs_constants,
214 target, context_type, 1);
215 LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, planes,
216 target, context_type, 2);
217 LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, textures,
218 target, context_type,
219 DRAW_JIT_CTX_TEXTURES);
220 LP_CHECK_STRUCT_SIZE(struct draw_jit_context,
221 target, context_type);
222
223 return context_type;
224 }
225
226
227 /**
228 * Create LLVM type for struct pipe_vertex_buffer
229 */
230 static LLVMTypeRef
231 create_jit_vertex_buffer_type(struct gallivm_state *gallivm, const char *struct_name)
232 {
233 LLVMTargetDataRef target = gallivm->target;
234 LLVMTypeRef elem_types[3];
235 LLVMTypeRef vb_type;
236
237 elem_types[0] =
238 elem_types[1] = LLVMInt32TypeInContext(gallivm->context);
239 elem_types[2] = LLVMPointerType(LLVMInt8TypeInContext(gallivm->context), 0); /* vs_constants */
240
241 #if HAVE_LLVM >= 0x0300
242 vb_type = LLVMStructCreateNamed(gallivm->context, struct_name);
243 LLVMStructSetBody(vb_type, elem_types,
244 Elements(elem_types), 0);
245 #else
246 vb_type = LLVMStructTypeInContext(gallivm->context, elem_types,
247 Elements(elem_types), 0);
248 LLVMAddTypeName(gallivm->module, struct_name, vb_type);
249
250 LLVMInvalidateStructLayout(gallivm->target, vb_type);
251 #endif
252
253 LP_CHECK_MEMBER_OFFSET(struct pipe_vertex_buffer, stride,
254 target, vb_type, 0);
255 LP_CHECK_MEMBER_OFFSET(struct pipe_vertex_buffer, buffer_offset,
256 target, vb_type, 1);
257
258 LP_CHECK_STRUCT_SIZE(struct pipe_vertex_buffer, target, vb_type);
259
260 return vb_type;
261 }
262
263
264 /**
265 * Create LLVM type for struct vertex_header;
266 */
267 static LLVMTypeRef
268 create_jit_vertex_header(struct gallivm_state *gallivm, int data_elems)
269 {
270 LLVMTargetDataRef target = gallivm->target;
271 LLVMTypeRef elem_types[3];
272 LLVMTypeRef vertex_header;
273 char struct_name[24];
274
275 util_snprintf(struct_name, 23, "vertex_header%d", data_elems);
276
277 elem_types[0] = LLVMIntTypeInContext(gallivm->context, 32);
278 elem_types[1] = LLVMArrayType(LLVMFloatTypeInContext(gallivm->context), 4);
279 elem_types[2] = LLVMArrayType(elem_types[1], data_elems);
280
281 #if HAVE_LLVM >= 0x0300
282 vertex_header = LLVMStructCreateNamed(gallivm->context, struct_name);
283 LLVMStructSetBody(vertex_header, elem_types,
284 Elements(elem_types), 0);
285 #else
286 vertex_header = LLVMStructTypeInContext(gallivm->context, elem_types,
287 Elements(elem_types), 0);
288 LLVMAddTypeName(gallivm->module, struct_name, vertex_header);
289
290 LLVMInvalidateStructLayout(gallivm->target, vertex_header);
291 #endif
292
293 /* these are bit-fields and we can't take address of them
294 LP_CHECK_MEMBER_OFFSET(struct vertex_header, clipmask,
295 target, vertex_header,
296 DRAW_JIT_VERTEX_CLIPMASK);
297 LP_CHECK_MEMBER_OFFSET(struct vertex_header, edgeflag,
298 target, vertex_header,
299 DRAW_JIT_VERTEX_EDGEFLAG);
300 LP_CHECK_MEMBER_OFFSET(struct vertex_header, pad,
301 target, vertex_header,
302 DRAW_JIT_VERTEX_PAD);
303 LP_CHECK_MEMBER_OFFSET(struct vertex_header, vertex_id,
304 target, vertex_header,
305 DRAW_JIT_VERTEX_VERTEX_ID);
306 */
307 LP_CHECK_MEMBER_OFFSET(struct vertex_header, clip,
308 target, vertex_header,
309 DRAW_JIT_VERTEX_CLIP);
310 LP_CHECK_MEMBER_OFFSET(struct vertex_header, data,
311 target, vertex_header,
312 DRAW_JIT_VERTEX_DATA);
313
314 return vertex_header;
315 }
316
317
318 /**
319 * Create LLVM types for various structures.
320 */
321 static void
322 create_jit_types(struct draw_llvm *llvm)
323 {
324 struct gallivm_state *gallivm = llvm->gallivm;
325 LLVMTypeRef texture_type, context_type, buffer_type, vb_type;
326
327 texture_type = create_jit_texture_type(gallivm, "texture");
328
329 context_type = create_jit_context_type(gallivm, texture_type, "draw_jit_context");
330 llvm->context_ptr_type = LLVMPointerType(context_type, 0);
331
332 buffer_type = LLVMPointerType(LLVMIntTypeInContext(gallivm->context, 8), 0);
333 llvm->buffer_ptr_type = LLVMPointerType(buffer_type, 0);
334
335 vb_type = create_jit_vertex_buffer_type(gallivm, "pipe_vertex_buffer");
336 llvm->vb_ptr_type = LLVMPointerType(vb_type, 0);
337 }
338
339
340 static LLVMTypeRef
341 get_context_ptr_type(struct draw_llvm *llvm)
342 {
343 if (!llvm->context_ptr_type)
344 create_jit_types(llvm);
345 return llvm->context_ptr_type;
346 }
347
348
349 static LLVMTypeRef
350 get_buffer_ptr_type(struct draw_llvm *llvm)
351 {
352 if (!llvm->buffer_ptr_type)
353 create_jit_types(llvm);
354 return llvm->buffer_ptr_type;
355 }
356
357
358 static LLVMTypeRef
359 get_vb_ptr_type(struct draw_llvm *llvm)
360 {
361 if (!llvm->vb_ptr_type)
362 create_jit_types(llvm);
363 return llvm->vb_ptr_type;
364 }
365
366 static LLVMTypeRef
367 get_vertex_header_ptr_type(struct draw_llvm *llvm)
368 {
369 if (!llvm->vertex_header_ptr_type)
370 create_jit_types(llvm);
371 return llvm->vertex_header_ptr_type;
372 }
373
374
375 /**
376 * Create per-context LLVM info.
377 */
378 struct draw_llvm *
379 draw_llvm_create(struct draw_context *draw, struct gallivm_state *gallivm)
380 {
381 struct draw_llvm *llvm;
382
383 llvm = CALLOC_STRUCT( draw_llvm );
384 if (!llvm)
385 return NULL;
386
387 lp_build_init();
388
389 llvm->draw = draw;
390 llvm->gallivm = gallivm;
391
392 if (gallivm_debug & GALLIVM_DEBUG_IR) {
393 LLVMDumpModule(llvm->gallivm->module);
394 }
395
396 llvm->nr_variants = 0;
397 make_empty_list(&llvm->vs_variants_list);
398
399 gallivm_register_garbage_collector_callback(
400 draw_llvm_garbage_collect_callback, llvm);
401
402 return llvm;
403 }
404
405
406 /**
407 * Free per-context LLVM info.
408 */
409 void
410 draw_llvm_destroy(struct draw_llvm *llvm)
411 {
412 gallivm_remove_garbage_collector_callback(
413 draw_llvm_garbage_collect_callback, llvm);
414
415 /* XXX free other draw_llvm data? */
416 FREE(llvm);
417 }
418
419
420 /**
421 * Create LLVM-generated code for a vertex shader.
422 */
423 struct draw_llvm_variant *
424 draw_llvm_create_variant(struct draw_llvm *llvm,
425 unsigned num_inputs,
426 const struct draw_llvm_variant_key *key)
427 {
428 struct draw_llvm_variant *variant;
429 struct llvm_vertex_shader *shader =
430 llvm_vertex_shader(llvm->draw->vs.vertex_shader);
431 LLVMTypeRef vertex_header;
432
433 variant = MALLOC(sizeof *variant +
434 shader->variant_key_size -
435 sizeof variant->key);
436 if (variant == NULL)
437 return NULL;
438
439 variant->llvm = llvm;
440
441 memcpy(&variant->key, key, shader->variant_key_size);
442
443 vertex_header = create_jit_vertex_header(llvm->gallivm, num_inputs);
444
445 llvm->vertex_header_ptr_type = LLVMPointerType(vertex_header, 0);
446
447 draw_llvm_generate(llvm, variant, FALSE); /* linear */
448 draw_llvm_generate(llvm, variant, TRUE); /* elts */
449
450 variant->shader = shader;
451 variant->list_item_global.base = variant;
452 variant->list_item_local.base = variant;
453 /*variant->no = */shader->variants_created++;
454 variant->list_item_global.base = variant;
455
456 return variant;
457 }
458
459
460 static void
461 generate_vs(struct draw_llvm *llvm,
462 LLVMBuilderRef builder,
463 LLVMValueRef (*outputs)[NUM_CHANNELS],
464 const LLVMValueRef (*inputs)[NUM_CHANNELS],
465 LLVMValueRef system_values_array,
466 LLVMValueRef context_ptr,
467 struct lp_build_sampler_soa *draw_sampler,
468 boolean clamp_vertex_color)
469 {
470 const struct tgsi_token *tokens = llvm->draw->vs.vertex_shader->state.tokens;
471 struct lp_type vs_type;
472 LLVMValueRef consts_ptr = draw_jit_context_vs_constants(llvm->gallivm, context_ptr);
473 struct lp_build_sampler_soa *sampler = 0;
474
475 memset(&vs_type, 0, sizeof vs_type);
476 vs_type.floating = TRUE; /* floating point values */
477 vs_type.sign = TRUE; /* values are signed */
478 vs_type.norm = FALSE; /* values are not limited to [0,1] or [-1,1] */
479 vs_type.width = 32; /* 32-bit float */
480 vs_type.length = 4; /* 4 elements per vector */
481 #if 0
482 num_vs = 4; /* number of vertices per block */
483 #endif
484
485 if (gallivm_debug & GALLIVM_DEBUG_IR) {
486 tgsi_dump(tokens, 0);
487 }
488
489 if (llvm->draw->num_sampler_views && llvm->draw->num_samplers)
490 sampler = draw_sampler;
491
492 lp_build_tgsi_soa(llvm->gallivm,
493 tokens,
494 vs_type,
495 NULL /*struct lp_build_mask_context *mask*/,
496 consts_ptr,
497 system_values_array,
498 NULL /*pos*/,
499 inputs,
500 outputs,
501 sampler,
502 &llvm->draw->vs.vertex_shader->info);
503
504 if (clamp_vertex_color) {
505 LLVMValueRef out;
506 unsigned chan, attrib;
507 struct lp_build_context bld;
508 struct tgsi_shader_info* info = &llvm->draw->vs.vertex_shader->info;
509 lp_build_context_init(&bld, llvm->gallivm, vs_type);
510
511 for (attrib = 0; attrib < info->num_outputs; ++attrib) {
512 for (chan = 0; chan < NUM_CHANNELS; ++chan) {
513 if (outputs[attrib][chan]) {
514 switch (info->output_semantic_name[attrib]) {
515 case TGSI_SEMANTIC_COLOR:
516 case TGSI_SEMANTIC_BCOLOR:
517 out = LLVMBuildLoad(builder, outputs[attrib][chan], "");
518 out = lp_build_clamp(&bld, out, bld.zero, bld.one);
519 LLVMBuildStore(builder, out, outputs[attrib][chan]);
520 break;
521 }
522 }
523 }
524 }
525 }
526 }
527
528
529 #if DEBUG_STORE
530 static void print_vectorf(LLVMBuilderRef builder,
531 LLVMValueRef vec)
532 {
533 LLVMValueRef val[4];
534 val[0] = LLVMBuildExtractElement(builder, vec,
535 lp_build_const_int32(gallivm, 0), "");
536 val[1] = LLVMBuildExtractElement(builder, vec,
537 lp_build_const_int32(gallivm, 1), "");
538 val[2] = LLVMBuildExtractElement(builder, vec,
539 lp_build_const_int32(gallivm, 2), "");
540 val[3] = LLVMBuildExtractElement(builder, vec,
541 lp_build_const_int32(gallivm, 3), "");
542 lp_build_printf(builder, "vector = [%f, %f, %f, %f]\n",
543 val[0], val[1], val[2], val[3]);
544 }
545 #endif
546
547
548 static void
549 generate_fetch(struct gallivm_state *gallivm,
550 LLVMValueRef vbuffers_ptr,
551 LLVMValueRef *res,
552 struct pipe_vertex_element *velem,
553 LLVMValueRef vbuf,
554 LLVMValueRef index,
555 LLVMValueRef instance_id)
556 {
557 LLVMBuilderRef builder = gallivm->builder;
558 LLVMValueRef indices =
559 LLVMConstInt(LLVMInt64TypeInContext(gallivm->context),
560 velem->vertex_buffer_index, 0);
561 LLVMValueRef vbuffer_ptr = LLVMBuildGEP(builder, vbuffers_ptr,
562 &indices, 1, "");
563 LLVMValueRef vb_stride = draw_jit_vbuffer_stride(gallivm, vbuf);
564 LLVMValueRef vb_buffer_offset = draw_jit_vbuffer_offset(gallivm, vbuf);
565 LLVMValueRef stride;
566
567 if (velem->instance_divisor) {
568 /* array index = instance_id / instance_divisor */
569 index = LLVMBuildUDiv(builder, instance_id,
570 lp_build_const_int32(gallivm, velem->instance_divisor),
571 "instance_divisor");
572 }
573
574 stride = LLVMBuildMul(builder, vb_stride, index, "");
575
576 vbuffer_ptr = LLVMBuildLoad(builder, vbuffer_ptr, "vbuffer");
577
578 stride = LLVMBuildAdd(builder, stride,
579 vb_buffer_offset,
580 "");
581 stride = LLVMBuildAdd(builder, stride,
582 lp_build_const_int32(gallivm, velem->src_offset),
583 "");
584
585 /*lp_build_printf(builder, "vbuf index = %d, stride is %d\n", indices, stride);*/
586 vbuffer_ptr = LLVMBuildGEP(builder, vbuffer_ptr, &stride, 1, "");
587
588 *res = draw_llvm_translate_from(gallivm, vbuffer_ptr, velem->src_format);
589 }
590
591
592 static LLVMValueRef
593 aos_to_soa(struct gallivm_state *gallivm,
594 LLVMValueRef val0,
595 LLVMValueRef val1,
596 LLVMValueRef val2,
597 LLVMValueRef val3,
598 LLVMValueRef channel)
599 {
600 LLVMBuilderRef builder = gallivm->builder;
601 LLVMValueRef ex, res;
602
603 ex = LLVMBuildExtractElement(builder, val0,
604 channel, "");
605 res = LLVMBuildInsertElement(builder,
606 LLVMConstNull(LLVMTypeOf(val0)),
607 ex,
608 lp_build_const_int32(gallivm, 0),
609 "");
610
611 ex = LLVMBuildExtractElement(builder, val1,
612 channel, "");
613 res = LLVMBuildInsertElement(builder,
614 res, ex,
615 lp_build_const_int32(gallivm, 1),
616 "");
617
618 ex = LLVMBuildExtractElement(builder, val2,
619 channel, "");
620 res = LLVMBuildInsertElement(builder,
621 res, ex,
622 lp_build_const_int32(gallivm, 2),
623 "");
624
625 ex = LLVMBuildExtractElement(builder, val3,
626 channel, "");
627 res = LLVMBuildInsertElement(builder,
628 res, ex,
629 lp_build_const_int32(gallivm, 3),
630 "");
631
632 return res;
633 }
634
635
636 static void
637 soa_to_aos(struct gallivm_state *gallivm,
638 LLVMValueRef soa[NUM_CHANNELS],
639 LLVMValueRef aos[NUM_CHANNELS])
640 {
641 LLVMBuilderRef builder = gallivm->builder;
642 LLVMValueRef comp;
643 int i = 0;
644
645 debug_assert(NUM_CHANNELS == 4);
646
647 aos[0] = LLVMConstNull(LLVMTypeOf(soa[0]));
648 aos[1] = aos[2] = aos[3] = aos[0];
649
650 for (i = 0; i < NUM_CHANNELS; ++i) {
651 LLVMValueRef channel = lp_build_const_int32(gallivm, i);
652
653 comp = LLVMBuildExtractElement(builder, soa[i],
654 lp_build_const_int32(gallivm, 0), "");
655 aos[0] = LLVMBuildInsertElement(builder, aos[0], comp, channel, "");
656
657 comp = LLVMBuildExtractElement(builder, soa[i],
658 lp_build_const_int32(gallivm, 1), "");
659 aos[1] = LLVMBuildInsertElement(builder, aos[1], comp, channel, "");
660
661 comp = LLVMBuildExtractElement(builder, soa[i],
662 lp_build_const_int32(gallivm, 2), "");
663 aos[2] = LLVMBuildInsertElement(builder, aos[2], comp, channel, "");
664
665 comp = LLVMBuildExtractElement(builder, soa[i],
666 lp_build_const_int32(gallivm, 3), "");
667 aos[3] = LLVMBuildInsertElement(builder, aos[3], comp, channel, "");
668
669 }
670 }
671
672
673 static void
674 convert_to_soa(struct gallivm_state *gallivm,
675 LLVMValueRef (*aos)[NUM_CHANNELS],
676 LLVMValueRef (*soa)[NUM_CHANNELS],
677 int num_attribs)
678 {
679 int i;
680
681 debug_assert(NUM_CHANNELS == 4);
682
683 for (i = 0; i < num_attribs; ++i) {
684 LLVMValueRef val0 = aos[i][0];
685 LLVMValueRef val1 = aos[i][1];
686 LLVMValueRef val2 = aos[i][2];
687 LLVMValueRef val3 = aos[i][3];
688
689 soa[i][0] = aos_to_soa(gallivm, val0, val1, val2, val3,
690 lp_build_const_int32(gallivm, 0));
691 soa[i][1] = aos_to_soa(gallivm, val0, val1, val2, val3,
692 lp_build_const_int32(gallivm, 1));
693 soa[i][2] = aos_to_soa(gallivm, val0, val1, val2, val3,
694 lp_build_const_int32(gallivm, 2));
695 soa[i][3] = aos_to_soa(gallivm, val0, val1, val2, val3,
696 lp_build_const_int32(gallivm, 3));
697 }
698 }
699
700
701 static void
702 store_aos(struct gallivm_state *gallivm,
703 LLVMValueRef io_ptr,
704 LLVMValueRef index,
705 LLVMValueRef value,
706 LLVMValueRef clipmask)
707 {
708 LLVMBuilderRef builder = gallivm->builder;
709 LLVMValueRef id_ptr = draw_jit_header_id(gallivm, io_ptr);
710 LLVMValueRef data_ptr = draw_jit_header_data(gallivm, io_ptr);
711 LLVMValueRef indices[3];
712 LLVMValueRef val;
713 int vertex_id_pad_edgeflag;
714
715 indices[0] = lp_build_const_int32(gallivm, 0);
716 indices[1] = index;
717 indices[2] = lp_build_const_int32(gallivm, 0);
718
719 /* If this assertion fails, it means we need to update the bit twidding
720 * code here. See struct vertex_header in draw_private.h.
721 */
722 assert(DRAW_TOTAL_CLIP_PLANES==14);
723 /* initialize vertex id:16 = 0xffff, pad:1 = 0, edgeflag:1 = 1 */
724 vertex_id_pad_edgeflag = (0xffff << 16) | (1 << DRAW_TOTAL_CLIP_PLANES);
725 val = lp_build_const_int32(gallivm, vertex_id_pad_edgeflag);
726 /* OR with the clipmask */
727 val = LLVMBuildOr(builder, val, clipmask, "");
728
729 /* store vertex header */
730 LLVMBuildStore(builder, val, id_ptr);
731
732
733 #if DEBUG_STORE
734 lp_build_printf(builder, " ---- %p storing attribute %d (io = %p)\n", data_ptr, index, io_ptr);
735 #endif
736 #if 0
737 /*lp_build_printf(builder, " ---- %p storing at %d (%p) ", io_ptr, index, data_ptr);
738 print_vectorf(builder, value);*/
739 data_ptr = LLVMBuildBitCast(builder, data_ptr,
740 LLVMPointerType(LLVMArrayType(LLVMVectorType(LLVMFloatTypeInContext(gallivm->context), 4), 0), 0),
741 "datavec");
742 data_ptr = LLVMBuildGEP(builder, data_ptr, indices, 2, "");
743
744 LLVMBuildStore(builder, value, data_ptr);
745 #else
746 {
747 LLVMValueRef x, y, z, w;
748 LLVMValueRef idx0, idx1, idx2, idx3;
749 LLVMValueRef gep0, gep1, gep2, gep3;
750 data_ptr = LLVMBuildGEP(builder, data_ptr, indices, 3, "");
751
752 idx0 = lp_build_const_int32(gallivm, 0);
753 idx1 = lp_build_const_int32(gallivm, 1);
754 idx2 = lp_build_const_int32(gallivm, 2);
755 idx3 = lp_build_const_int32(gallivm, 3);
756
757 x = LLVMBuildExtractElement(builder, value,
758 idx0, "");
759 y = LLVMBuildExtractElement(builder, value,
760 idx1, "");
761 z = LLVMBuildExtractElement(builder, value,
762 idx2, "");
763 w = LLVMBuildExtractElement(builder, value,
764 idx3, "");
765
766 gep0 = LLVMBuildGEP(builder, data_ptr, &idx0, 1, "");
767 gep1 = LLVMBuildGEP(builder, data_ptr, &idx1, 1, "");
768 gep2 = LLVMBuildGEP(builder, data_ptr, &idx2, 1, "");
769 gep3 = LLVMBuildGEP(builder, data_ptr, &idx3, 1, "");
770
771 /*lp_build_printf(builder, "##### x = %f (%p), y = %f (%p), z = %f (%p), w = %f (%p)\n",
772 x, gep0, y, gep1, z, gep2, w, gep3);*/
773 LLVMBuildStore(builder, x, gep0);
774 LLVMBuildStore(builder, y, gep1);
775 LLVMBuildStore(builder, z, gep2);
776 LLVMBuildStore(builder, w, gep3);
777 }
778 #endif
779 }
780
781
782 static void
783 store_aos_array(struct gallivm_state *gallivm,
784 LLVMValueRef io_ptr,
785 LLVMValueRef aos[NUM_CHANNELS],
786 int attrib,
787 int num_outputs,
788 LLVMValueRef clipmask)
789 {
790 LLVMBuilderRef builder = gallivm->builder;
791 LLVMValueRef attr_index = lp_build_const_int32(gallivm, attrib);
792 LLVMValueRef ind0 = lp_build_const_int32(gallivm, 0);
793 LLVMValueRef ind1 = lp_build_const_int32(gallivm, 1);
794 LLVMValueRef ind2 = lp_build_const_int32(gallivm, 2);
795 LLVMValueRef ind3 = lp_build_const_int32(gallivm, 3);
796 LLVMValueRef io0_ptr, io1_ptr, io2_ptr, io3_ptr;
797 LLVMValueRef clipmask0, clipmask1, clipmask2, clipmask3;
798
799 debug_assert(NUM_CHANNELS == 4);
800
801 io0_ptr = LLVMBuildGEP(builder, io_ptr,
802 &ind0, 1, "");
803 io1_ptr = LLVMBuildGEP(builder, io_ptr,
804 &ind1, 1, "");
805 io2_ptr = LLVMBuildGEP(builder, io_ptr,
806 &ind2, 1, "");
807 io3_ptr = LLVMBuildGEP(builder, io_ptr,
808 &ind3, 1, "");
809
810 clipmask0 = LLVMBuildExtractElement(builder, clipmask,
811 ind0, "");
812 clipmask1 = LLVMBuildExtractElement(builder, clipmask,
813 ind1, "");
814 clipmask2 = LLVMBuildExtractElement(builder, clipmask,
815 ind2, "");
816 clipmask3 = LLVMBuildExtractElement(builder, clipmask,
817 ind3, "");
818
819 #if DEBUG_STORE
820 lp_build_printf(builder, "io = %p, indexes[%d, %d, %d, %d]\n, clipmask0 = %x, clipmask1 = %x, clipmask2 = %x, clipmask3 = %x\n",
821 io_ptr, ind0, ind1, ind2, ind3, clipmask0, clipmask1, clipmask2, clipmask3);
822 #endif
823 /* store for each of the 4 vertices */
824 store_aos(gallivm, io0_ptr, attr_index, aos[0], clipmask0);
825 store_aos(gallivm, io1_ptr, attr_index, aos[1], clipmask1);
826 store_aos(gallivm, io2_ptr, attr_index, aos[2], clipmask2);
827 store_aos(gallivm, io3_ptr, attr_index, aos[3], clipmask3);
828 }
829
830
831 static void
832 convert_to_aos(struct gallivm_state *gallivm,
833 LLVMValueRef io,
834 LLVMValueRef (*outputs)[NUM_CHANNELS],
835 LLVMValueRef clipmask,
836 int num_outputs,
837 int max_vertices)
838 {
839 LLVMBuilderRef builder = gallivm->builder;
840 unsigned chan, attrib;
841
842 #if DEBUG_STORE
843 lp_build_printf(builder, " # storing begin\n");
844 #endif
845 for (attrib = 0; attrib < num_outputs; ++attrib) {
846 LLVMValueRef soa[4];
847 LLVMValueRef aos[4];
848 for (chan = 0; chan < NUM_CHANNELS; ++chan) {
849 if (outputs[attrib][chan]) {
850 LLVMValueRef out = LLVMBuildLoad(builder, outputs[attrib][chan], "");
851 lp_build_name(out, "output%u.%c", attrib, "xyzw"[chan]);
852 /*lp_build_printf(builder, "output %d : %d ",
853 LLVMConstInt(LLVMInt32Type(), attrib, 0),
854 LLVMConstInt(LLVMInt32Type(), chan, 0));
855 print_vectorf(builder, out);*/
856 soa[chan] = out;
857 }
858 else {
859 soa[chan] = 0;
860 }
861 }
862 soa_to_aos(gallivm, soa, aos);
863 store_aos_array(gallivm,
864 io,
865 aos,
866 attrib,
867 num_outputs,
868 clipmask);
869 }
870 #if DEBUG_STORE
871 lp_build_printf(builder, " # storing end\n");
872 #endif
873 }
874
875
876 /**
877 * Stores original vertex positions in clip coordinates
878 * There is probably a more efficient way to do this, 4 floats at once
879 * rather than extracting each element one by one.
880 */
881 static void
882 store_clip(struct gallivm_state *gallivm,
883 LLVMValueRef io_ptr,
884 LLVMValueRef (*outputs)[NUM_CHANNELS])
885 {
886 LLVMBuilderRef builder = gallivm->builder;
887 LLVMValueRef out[4];
888 LLVMValueRef indices[2];
889 LLVMValueRef io0_ptr, io1_ptr, io2_ptr, io3_ptr;
890 LLVMValueRef clip_ptr0, clip_ptr1, clip_ptr2, clip_ptr3;
891 LLVMValueRef clip0_ptr, clip1_ptr, clip2_ptr, clip3_ptr;
892 LLVMValueRef out0elem, out1elem, out2elem, out3elem;
893 int i;
894
895 LLVMValueRef ind0 = lp_build_const_int32(gallivm, 0);
896 LLVMValueRef ind1 = lp_build_const_int32(gallivm, 1);
897 LLVMValueRef ind2 = lp_build_const_int32(gallivm, 2);
898 LLVMValueRef ind3 = lp_build_const_int32(gallivm, 3);
899
900 indices[0] =
901 indices[1] = lp_build_const_int32(gallivm, 0);
902
903 out[0] = LLVMBuildLoad(builder, outputs[0][0], ""); /*x0 x1 x2 x3*/
904 out[1] = LLVMBuildLoad(builder, outputs[0][1], ""); /*y0 y1 y2 y3*/
905 out[2] = LLVMBuildLoad(builder, outputs[0][2], ""); /*z0 z1 z2 z3*/
906 out[3] = LLVMBuildLoad(builder, outputs[0][3], ""); /*w0 w1 w2 w3*/
907
908 io0_ptr = LLVMBuildGEP(builder, io_ptr, &ind0, 1, "");
909 io1_ptr = LLVMBuildGEP(builder, io_ptr, &ind1, 1, "");
910 io2_ptr = LLVMBuildGEP(builder, io_ptr, &ind2, 1, "");
911 io3_ptr = LLVMBuildGEP(builder, io_ptr, &ind3, 1, "");
912
913 clip_ptr0 = draw_jit_header_clip(gallivm, io0_ptr);
914 clip_ptr1 = draw_jit_header_clip(gallivm, io1_ptr);
915 clip_ptr2 = draw_jit_header_clip(gallivm, io2_ptr);
916 clip_ptr3 = draw_jit_header_clip(gallivm, io3_ptr);
917
918 for (i = 0; i<4; i++) {
919 clip0_ptr = LLVMBuildGEP(builder, clip_ptr0, indices, 2, ""); /* x0 */
920 clip1_ptr = LLVMBuildGEP(builder, clip_ptr1, indices, 2, ""); /* x1 */
921 clip2_ptr = LLVMBuildGEP(builder, clip_ptr2, indices, 2, ""); /* x2 */
922 clip3_ptr = LLVMBuildGEP(builder, clip_ptr3, indices, 2, ""); /* x3 */
923
924 out0elem = LLVMBuildExtractElement(builder, out[i], ind0, ""); /* x0 */
925 out1elem = LLVMBuildExtractElement(builder, out[i], ind1, ""); /* x1 */
926 out2elem = LLVMBuildExtractElement(builder, out[i], ind2, ""); /* x2 */
927 out3elem = LLVMBuildExtractElement(builder, out[i], ind3, ""); /* x3 */
928
929 LLVMBuildStore(builder, out0elem, clip0_ptr);
930 LLVMBuildStore(builder, out1elem, clip1_ptr);
931 LLVMBuildStore(builder, out2elem, clip2_ptr);
932 LLVMBuildStore(builder, out3elem, clip3_ptr);
933
934 indices[1]= LLVMBuildAdd(builder, indices[1], ind1, "");
935 }
936
937 }
938
939
940 /**
941 * Equivalent of _mm_set1_ps(a)
942 */
943 static LLVMValueRef
944 vec4f_from_scalar(struct gallivm_state *gallivm,
945 LLVMValueRef a,
946 const char *name)
947 {
948 LLVMTypeRef float_type = LLVMFloatTypeInContext(gallivm->context);
949 LLVMValueRef res = LLVMGetUndef(LLVMVectorType(float_type, 4));
950 int i;
951
952 for (i = 0; i < 4; ++i) {
953 LLVMValueRef index = lp_build_const_int32(gallivm, i);
954 res = LLVMBuildInsertElement(gallivm->builder, res, a,
955 index, i == 3 ? name : "");
956 }
957
958 return res;
959 }
960
961
962 /**
963 * Transforms the outputs for viewport mapping
964 */
965 static void
966 generate_viewport(struct draw_llvm *llvm,
967 LLVMBuilderRef builder,
968 LLVMValueRef (*outputs)[NUM_CHANNELS],
969 LLVMValueRef context_ptr)
970 {
971 int i;
972 struct gallivm_state *gallivm = llvm->gallivm;
973 struct lp_type f32_type = lp_type_float_vec(32);
974 LLVMValueRef out3 = LLVMBuildLoad(builder, outputs[0][3], ""); /*w0 w1 w2 w3*/
975 LLVMValueRef const1 = lp_build_const_vec(gallivm, f32_type, 1.0); /*1.0 1.0 1.0 1.0*/
976 LLVMValueRef vp_ptr = draw_jit_context_viewport(gallivm, context_ptr);
977
978 /* for 1/w convention*/
979 out3 = LLVMBuildFDiv(builder, const1, out3, "");
980 LLVMBuildStore(builder, out3, outputs[0][3]);
981
982 /* Viewport Mapping */
983 for (i=0; i<3; i++) {
984 LLVMValueRef out = LLVMBuildLoad(builder, outputs[0][i], ""); /*x0 x1 x2 x3*/
985 LLVMValueRef scale;
986 LLVMValueRef trans;
987 LLVMValueRef scale_i;
988 LLVMValueRef trans_i;
989 LLVMValueRef index;
990
991 index = lp_build_const_int32(gallivm, i);
992 scale_i = LLVMBuildGEP(builder, vp_ptr, &index, 1, "");
993
994 index = lp_build_const_int32(gallivm, i+4);
995 trans_i = LLVMBuildGEP(builder, vp_ptr, &index, 1, "");
996
997 scale = vec4f_from_scalar(gallivm, LLVMBuildLoad(builder, scale_i, ""), "scale");
998 trans = vec4f_from_scalar(gallivm, LLVMBuildLoad(builder, trans_i, ""), "trans");
999
1000 /* divide by w */
1001 out = LLVMBuildFMul(builder, out, out3, "");
1002 /* mult by scale */
1003 out = LLVMBuildFMul(builder, out, scale, "");
1004 /* add translation */
1005 out = LLVMBuildFAdd(builder, out, trans, "");
1006
1007 /* store transformed outputs */
1008 LLVMBuildStore(builder, out, outputs[0][i]);
1009 }
1010
1011 }
1012
1013
1014 /**
1015 * Returns clipmask as 4xi32 bitmask for the 4 vertices
1016 */
1017 static LLVMValueRef
1018 generate_clipmask(struct gallivm_state *gallivm,
1019 LLVMValueRef (*outputs)[NUM_CHANNELS],
1020 boolean clip_xy,
1021 boolean clip_z,
1022 boolean clip_user,
1023 boolean clip_halfz,
1024 unsigned ucp_enable,
1025 LLVMValueRef context_ptr)
1026 {
1027 LLVMBuilderRef builder = gallivm->builder;
1028 LLVMValueRef mask; /* stores the <4xi32> clipmasks */
1029 LLVMValueRef test, temp;
1030 LLVMValueRef zero, shift;
1031 LLVMValueRef pos_x, pos_y, pos_z, pos_w;
1032 LLVMValueRef plane1, planes, plane_ptr, sum;
1033 struct lp_type f32_type = lp_type_float_vec(32);
1034
1035 mask = lp_build_const_int_vec(gallivm, lp_type_int_vec(32), 0);
1036 temp = lp_build_const_int_vec(gallivm, lp_type_int_vec(32), 0);
1037 zero = lp_build_const_vec(gallivm, f32_type, 0); /* 0.0f 0.0f 0.0f 0.0f */
1038 shift = lp_build_const_int_vec(gallivm, lp_type_int_vec(32), 1); /* 1 1 1 1 */
1039
1040 /* Assuming position stored at output[0] */
1041 pos_x = LLVMBuildLoad(builder, outputs[0][0], ""); /*x0 x1 x2 x3*/
1042 pos_y = LLVMBuildLoad(builder, outputs[0][1], ""); /*y0 y1 y2 y3*/
1043 pos_z = LLVMBuildLoad(builder, outputs[0][2], ""); /*z0 z1 z2 z3*/
1044 pos_w = LLVMBuildLoad(builder, outputs[0][3], ""); /*w0 w1 w2 w3*/
1045
1046 /* Cliptest, for hardwired planes */
1047 if (clip_xy) {
1048 /* plane 1 */
1049 test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, pos_x , pos_w);
1050 temp = shift;
1051 test = LLVMBuildAnd(builder, test, temp, "");
1052 mask = test;
1053
1054 /* plane 2 */
1055 test = LLVMBuildFAdd(builder, pos_x, pos_w, "");
1056 test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, zero, test);
1057 temp = LLVMBuildShl(builder, temp, shift, "");
1058 test = LLVMBuildAnd(builder, test, temp, "");
1059 mask = LLVMBuildOr(builder, mask, test, "");
1060
1061 /* plane 3 */
1062 test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, pos_y, pos_w);
1063 temp = LLVMBuildShl(builder, temp, shift, "");
1064 test = LLVMBuildAnd(builder, test, temp, "");
1065 mask = LLVMBuildOr(builder, mask, test, "");
1066
1067 /* plane 4 */
1068 test = LLVMBuildFAdd(builder, pos_y, pos_w, "");
1069 test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, zero, test);
1070 temp = LLVMBuildShl(builder, temp, shift, "");
1071 test = LLVMBuildAnd(builder, test, temp, "");
1072 mask = LLVMBuildOr(builder, mask, test, "");
1073 }
1074
1075 if (clip_z) {
1076 temp = lp_build_const_int_vec(gallivm, lp_type_int_vec(32), 16);
1077 if (clip_halfz) {
1078 /* plane 5 */
1079 test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, zero, pos_z);
1080 test = LLVMBuildAnd(builder, test, temp, "");
1081 mask = LLVMBuildOr(builder, mask, test, "");
1082 }
1083 else {
1084 /* plane 5 */
1085 test = LLVMBuildFAdd(builder, pos_z, pos_w, "");
1086 test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, zero, test);
1087 test = LLVMBuildAnd(builder, test, temp, "");
1088 mask = LLVMBuildOr(builder, mask, test, "");
1089 }
1090 /* plane 6 */
1091 test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, pos_z, pos_w);
1092 temp = LLVMBuildShl(builder, temp, shift, "");
1093 test = LLVMBuildAnd(builder, test, temp, "");
1094 mask = LLVMBuildOr(builder, mask, test, "");
1095 }
1096
1097 if (clip_user) {
1098 LLVMValueRef planes_ptr = draw_jit_context_planes(gallivm, context_ptr);
1099 LLVMValueRef indices[3];
1100
1101 /* userclip planes */
1102 while (ucp_enable) {
1103 unsigned plane_idx = ffs(ucp_enable)-1;
1104 ucp_enable &= ~(1 << plane_idx);
1105 plane_idx += 6;
1106
1107 indices[0] = lp_build_const_int32(gallivm, 0);
1108 indices[1] = lp_build_const_int32(gallivm, plane_idx);
1109
1110 indices[2] = lp_build_const_int32(gallivm, 0);
1111 plane_ptr = LLVMBuildGEP(builder, planes_ptr, indices, 3, "");
1112 plane1 = LLVMBuildLoad(builder, plane_ptr, "plane_x");
1113 planes = vec4f_from_scalar(gallivm, plane1, "plane4_x");
1114 sum = LLVMBuildFMul(builder, planes, pos_x, "");
1115
1116 indices[2] = lp_build_const_int32(gallivm, 1);
1117 plane_ptr = LLVMBuildGEP(builder, planes_ptr, indices, 3, "");
1118 plane1 = LLVMBuildLoad(builder, plane_ptr, "plane_y");
1119 planes = vec4f_from_scalar(gallivm, plane1, "plane4_y");
1120 test = LLVMBuildFMul(builder, planes, pos_y, "");
1121 sum = LLVMBuildFAdd(builder, sum, test, "");
1122
1123 indices[2] = lp_build_const_int32(gallivm, 2);
1124 plane_ptr = LLVMBuildGEP(builder, planes_ptr, indices, 3, "");
1125 plane1 = LLVMBuildLoad(builder, plane_ptr, "plane_z");
1126 planes = vec4f_from_scalar(gallivm, plane1, "plane4_z");
1127 test = LLVMBuildFMul(builder, planes, pos_z, "");
1128 sum = LLVMBuildFAdd(builder, sum, test, "");
1129
1130 indices[2] = lp_build_const_int32(gallivm, 3);
1131 plane_ptr = LLVMBuildGEP(builder, planes_ptr, indices, 3, "");
1132 plane1 = LLVMBuildLoad(builder, plane_ptr, "plane_w");
1133 planes = vec4f_from_scalar(gallivm, plane1, "plane4_w");
1134 test = LLVMBuildFMul(builder, planes, pos_w, "");
1135 sum = LLVMBuildFAdd(builder, sum, test, "");
1136
1137 test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, zero, sum);
1138 temp = lp_build_const_int_vec(gallivm, lp_type_int_vec(32), 1 << plane_idx);
1139 test = LLVMBuildAnd(builder, test, temp, "");
1140 mask = LLVMBuildOr(builder, mask, test, "");
1141 }
1142 }
1143 return mask;
1144 }
1145
1146
1147 /**
1148 * Returns boolean if any clipping has occurred
1149 * Used zero/non-zero i32 value to represent boolean
1150 */
1151 static void
1152 clipmask_bool(struct gallivm_state *gallivm,
1153 LLVMValueRef clipmask,
1154 LLVMValueRef ret_ptr)
1155 {
1156 LLVMBuilderRef builder = gallivm->builder;
1157 LLVMValueRef ret = LLVMBuildLoad(builder, ret_ptr, "");
1158 LLVMValueRef temp;
1159 int i;
1160
1161 for (i=0; i<4; i++) {
1162 temp = LLVMBuildExtractElement(builder, clipmask,
1163 lp_build_const_int32(gallivm, i) , "");
1164 ret = LLVMBuildOr(builder, ret, temp, "");
1165 }
1166
1167 LLVMBuildStore(builder, ret, ret_ptr);
1168 }
1169
1170
1171 static void
1172 draw_llvm_generate(struct draw_llvm *llvm, struct draw_llvm_variant *variant,
1173 boolean elts)
1174 {
1175 struct gallivm_state *gallivm = llvm->gallivm;
1176 LLVMContextRef context = gallivm->context;
1177 LLVMTypeRef int32_type = LLVMInt32TypeInContext(context);
1178 LLVMTypeRef arg_types[8];
1179 LLVMTypeRef func_type;
1180 LLVMValueRef context_ptr;
1181 LLVMBasicBlockRef block;
1182 LLVMBuilderRef builder;
1183 LLVMValueRef end, start;
1184 LLVMValueRef count, fetch_elts, fetch_count;
1185 LLVMValueRef stride, step, io_itr;
1186 LLVMValueRef io_ptr, vbuffers_ptr, vb_ptr;
1187 LLVMValueRef instance_id;
1188 LLVMValueRef system_values_array;
1189 LLVMValueRef zero = lp_build_const_int32(gallivm, 0);
1190 LLVMValueRef one = lp_build_const_int32(gallivm, 1);
1191 struct draw_context *draw = llvm->draw;
1192 const struct tgsi_shader_info *vs_info = &draw->vs.vertex_shader->info;
1193 unsigned i, j;
1194 struct lp_build_context bld;
1195 struct lp_build_loop_state lp_loop;
1196 const int max_vertices = 4;
1197 LLVMValueRef outputs[PIPE_MAX_SHADER_OUTPUTS][NUM_CHANNELS];
1198 LLVMValueRef fetch_max;
1199 void *code;
1200 struct lp_build_sampler_soa *sampler = 0;
1201 LLVMValueRef ret, ret_ptr;
1202 const boolean bypass_viewport = variant->key.bypass_viewport;
1203 const boolean enable_cliptest = variant->key.clip_xy ||
1204 variant->key.clip_z ||
1205 variant->key.clip_user;
1206 LLVMValueRef variant_func;
1207
1208 arg_types[0] = get_context_ptr_type(llvm); /* context */
1209 arg_types[1] = get_vertex_header_ptr_type(llvm); /* vertex_header */
1210 arg_types[2] = get_buffer_ptr_type(llvm); /* vbuffers */
1211 if (elts)
1212 arg_types[3] = LLVMPointerType(int32_type, 0);/* fetch_elts * */
1213 else
1214 arg_types[3] = int32_type; /* start */
1215 arg_types[4] = int32_type; /* fetch_count / count */
1216 arg_types[5] = int32_type; /* stride */
1217 arg_types[6] = get_vb_ptr_type(llvm); /* pipe_vertex_buffer's */
1218 arg_types[7] = int32_type; /* instance_id */
1219
1220 func_type = LLVMFunctionType(int32_type, arg_types, Elements(arg_types), 0);
1221
1222 variant_func = LLVMAddFunction(gallivm->module,
1223 elts ? "draw_llvm_shader_elts" : "draw_llvm_shader",
1224 func_type);
1225
1226 if (elts)
1227 variant->function_elts = variant_func;
1228 else
1229 variant->function = variant_func;
1230
1231 LLVMSetFunctionCallConv(variant_func, LLVMCCallConv);
1232 for (i = 0; i < Elements(arg_types); ++i)
1233 if (LLVMGetTypeKind(arg_types[i]) == LLVMPointerTypeKind)
1234 LLVMAddAttribute(LLVMGetParam(variant_func, i),
1235 LLVMNoAliasAttribute);
1236
1237 context_ptr = LLVMGetParam(variant_func, 0);
1238 io_ptr = LLVMGetParam(variant_func, 1);
1239 vbuffers_ptr = LLVMGetParam(variant_func, 2);
1240 stride = LLVMGetParam(variant_func, 5);
1241 vb_ptr = LLVMGetParam(variant_func, 6);
1242 instance_id = LLVMGetParam(variant_func, 7);
1243
1244 lp_build_name(context_ptr, "context");
1245 lp_build_name(io_ptr, "io");
1246 lp_build_name(vbuffers_ptr, "vbuffers");
1247 lp_build_name(stride, "stride");
1248 lp_build_name(vb_ptr, "vb");
1249 lp_build_name(instance_id, "instance_id");
1250
1251 if (elts) {
1252 fetch_elts = LLVMGetParam(variant_func, 3);
1253 fetch_count = LLVMGetParam(variant_func, 4);
1254 lp_build_name(fetch_elts, "fetch_elts");
1255 lp_build_name(fetch_count, "fetch_count");
1256 start = count = NULL;
1257 }
1258 else {
1259 start = LLVMGetParam(variant_func, 3);
1260 count = LLVMGetParam(variant_func, 4);
1261 lp_build_name(start, "start");
1262 lp_build_name(count, "count");
1263 fetch_elts = fetch_count = NULL;
1264 }
1265
1266 /*
1267 * Function body
1268 */
1269
1270 block = LLVMAppendBasicBlockInContext(gallivm->context, variant_func, "entry");
1271 builder = gallivm->builder;
1272 LLVMPositionBuilderAtEnd(builder, block);
1273
1274 lp_build_context_init(&bld, gallivm, lp_type_int(32));
1275
1276 system_values_array = lp_build_system_values_array(gallivm, vs_info,
1277 instance_id, NULL);
1278
1279 /* function will return non-zero i32 value if any clipped vertices */
1280 ret_ptr = lp_build_alloca(gallivm, int32_type, "");
1281 LLVMBuildStore(builder, zero, ret_ptr);
1282
1283 /* code generated texture sampling */
1284 sampler = draw_llvm_sampler_soa_create(
1285 draw_llvm_variant_key_samplers(&variant->key),
1286 context_ptr);
1287
1288 if (elts) {
1289 start = zero;
1290 end = fetch_count;
1291 }
1292 else {
1293 end = lp_build_add(&bld, start, count);
1294 }
1295
1296 step = lp_build_const_int32(gallivm, max_vertices);
1297
1298 fetch_max = LLVMBuildSub(builder, end, one, "fetch_max");
1299
1300 lp_build_loop_begin(&lp_loop, gallivm, start);
1301 {
1302 LLVMValueRef inputs[PIPE_MAX_SHADER_INPUTS][NUM_CHANNELS];
1303 LLVMValueRef aos_attribs[PIPE_MAX_SHADER_INPUTS][NUM_CHANNELS] = { { 0 } };
1304 LLVMValueRef io;
1305 LLVMValueRef clipmask; /* holds the clipmask value */
1306 const LLVMValueRef (*ptr_aos)[NUM_CHANNELS];
1307
1308 if (elts)
1309 io_itr = lp_loop.counter;
1310 else
1311 io_itr = LLVMBuildSub(builder, lp_loop.counter, start, "");
1312
1313 io = LLVMBuildGEP(builder, io_ptr, &io_itr, 1, "");
1314 #if DEBUG_STORE
1315 lp_build_printf(builder, " --- io %d = %p, loop counter %d\n",
1316 io_itr, io, lp_loop.counter);
1317 #endif
1318 for (i = 0; i < NUM_CHANNELS; ++i) {
1319 LLVMValueRef true_index =
1320 LLVMBuildAdd(builder,
1321 lp_loop.counter,
1322 lp_build_const_int32(gallivm, i), "");
1323
1324 /* make sure we're not out of bounds which can happen
1325 * if fetch_count % 4 != 0, because on the last iteration
1326 * a few of the 4 vertex fetches will be out of bounds */
1327 true_index = lp_build_min(&bld, true_index, fetch_max);
1328
1329 if (elts) {
1330 LLVMValueRef fetch_ptr;
1331 fetch_ptr = LLVMBuildGEP(builder, fetch_elts,
1332 &true_index, 1, "");
1333 true_index = LLVMBuildLoad(builder, fetch_ptr, "fetch_elt");
1334 }
1335
1336 for (j = 0; j < draw->pt.nr_vertex_elements; ++j) {
1337 struct pipe_vertex_element *velem = &draw->pt.vertex_element[j];
1338 LLVMValueRef vb_index =
1339 lp_build_const_int32(gallivm, velem->vertex_buffer_index);
1340 LLVMValueRef vb = LLVMBuildGEP(builder, vb_ptr, &vb_index, 1, "");
1341 generate_fetch(gallivm, vbuffers_ptr,
1342 &aos_attribs[j][i], velem, vb, true_index,
1343 instance_id);
1344 }
1345 }
1346 convert_to_soa(gallivm, aos_attribs, inputs,
1347 draw->pt.nr_vertex_elements);
1348
1349 ptr_aos = (const LLVMValueRef (*)[NUM_CHANNELS]) inputs;
1350 generate_vs(llvm,
1351 builder,
1352 outputs,
1353 ptr_aos,
1354 system_values_array,
1355 context_ptr,
1356 sampler,
1357 variant->key.clamp_vertex_color);
1358
1359 /* store original positions in clip before further manipulation */
1360 store_clip(gallivm, io, outputs);
1361
1362 /* do cliptest */
1363 if (enable_cliptest) {
1364 /* allocate clipmask, assign it integer type */
1365 clipmask = generate_clipmask(gallivm, outputs,
1366 variant->key.clip_xy,
1367 variant->key.clip_z,
1368 variant->key.clip_user,
1369 variant->key.clip_halfz,
1370 variant->key.ucp_enable,
1371 context_ptr);
1372 /* return clipping boolean value for function */
1373 clipmask_bool(gallivm, clipmask, ret_ptr);
1374 }
1375 else {
1376 clipmask = lp_build_const_int_vec(gallivm, lp_type_int_vec(32), 0);
1377 }
1378
1379 /* do viewport mapping */
1380 if (!bypass_viewport) {
1381 generate_viewport(llvm, builder, outputs, context_ptr);
1382 }
1383
1384 /* store clipmask in vertex header,
1385 * original positions in clip
1386 * and transformed positions in data
1387 */
1388 convert_to_aos(gallivm, io, outputs, clipmask,
1389 vs_info->num_outputs, max_vertices);
1390 }
1391
1392 lp_build_loop_end_cond(&lp_loop, end, step, LLVMIntUGE);
1393
1394 sampler->destroy(sampler);
1395
1396 ret = LLVMBuildLoad(builder, ret_ptr, "");
1397 LLVMBuildRet(builder, ret);
1398
1399 /*
1400 * Translate the LLVM IR into machine code.
1401 */
1402 #ifdef DEBUG
1403 if (LLVMVerifyFunction(variant_func, LLVMPrintMessageAction)) {
1404 lp_debug_dump_value(variant_func);
1405 assert(0);
1406 }
1407 #endif
1408
1409 LLVMRunFunctionPassManager(gallivm->passmgr, variant_func);
1410
1411 if (gallivm_debug & GALLIVM_DEBUG_IR) {
1412 lp_debug_dump_value(variant_func);
1413 debug_printf("\n");
1414 }
1415
1416 code = LLVMGetPointerToGlobal(gallivm->engine, variant_func);
1417 if (elts)
1418 variant->jit_func_elts = (draw_jit_vert_func_elts) pointer_to_func(code);
1419 else
1420 variant->jit_func = (draw_jit_vert_func) pointer_to_func(code);
1421
1422 if (gallivm_debug & GALLIVM_DEBUG_ASM) {
1423 lp_disassemble(code);
1424 }
1425 lp_func_delete_body(variant_func);
1426 }
1427
1428
1429 struct draw_llvm_variant_key *
1430 draw_llvm_make_variant_key(struct draw_llvm *llvm, char *store)
1431 {
1432 unsigned i;
1433 struct draw_llvm_variant_key *key;
1434 struct lp_sampler_static_state *sampler;
1435
1436 key = (struct draw_llvm_variant_key *)store;
1437
1438 key->clamp_vertex_color = llvm->draw->rasterizer->clamp_vertex_color; /**/
1439
1440 /* Presumably all variants of the shader should have the same
1441 * number of vertex elements - ie the number of shader inputs.
1442 */
1443 key->nr_vertex_elements = llvm->draw->pt.nr_vertex_elements;
1444
1445 /* will have to rig this up properly later */
1446 key->clip_xy = llvm->draw->clip_xy;
1447 key->clip_z = llvm->draw->clip_z;
1448 key->clip_user = llvm->draw->clip_user;
1449 key->bypass_viewport = llvm->draw->identity_viewport;
1450 key->clip_halfz = !llvm->draw->rasterizer->gl_rasterization_rules;
1451 key->need_edgeflags = (llvm->draw->vs.edgeflag_output ? TRUE : FALSE);
1452 key->ucp_enable = llvm->draw->rasterizer->clip_plane_enable;
1453 key->pad = 0;
1454
1455 /* All variants of this shader will have the same value for
1456 * nr_samplers. Not yet trying to compact away holes in the
1457 * sampler array.
1458 */
1459 key->nr_samplers = llvm->draw->vs.vertex_shader->info.file_max[TGSI_FILE_SAMPLER] + 1;
1460
1461 sampler = draw_llvm_variant_key_samplers(key);
1462
1463 memcpy(key->vertex_element,
1464 llvm->draw->pt.vertex_element,
1465 sizeof(struct pipe_vertex_element) * key->nr_vertex_elements);
1466
1467 memset(sampler, 0, key->nr_samplers * sizeof *sampler);
1468
1469 for (i = 0 ; i < key->nr_samplers; i++) {
1470 lp_sampler_static_state(&sampler[i],
1471 llvm->draw->sampler_views[i],
1472 llvm->draw->samplers[i]);
1473 }
1474
1475 return key;
1476 }
1477
1478
1479 void
1480 draw_llvm_set_mapped_texture(struct draw_context *draw,
1481 unsigned sampler_idx,
1482 uint32_t width, uint32_t height, uint32_t depth,
1483 uint32_t first_level, uint32_t last_level,
1484 uint32_t row_stride[PIPE_MAX_TEXTURE_LEVELS],
1485 uint32_t img_stride[PIPE_MAX_TEXTURE_LEVELS],
1486 const void *data[PIPE_MAX_TEXTURE_LEVELS])
1487 {
1488 unsigned j;
1489 struct draw_jit_texture *jit_tex;
1490
1491 assert(sampler_idx < PIPE_MAX_VERTEX_SAMPLERS);
1492
1493 jit_tex = &draw->llvm->jit_context.textures[sampler_idx];
1494
1495 jit_tex->width = width;
1496 jit_tex->height = height;
1497 jit_tex->depth = depth;
1498 jit_tex->first_level = first_level;
1499 jit_tex->last_level = last_level;
1500
1501 for (j = first_level; j <= last_level; j++) {
1502 jit_tex->data[j] = data[j];
1503 jit_tex->row_stride[j] = row_stride[j];
1504 jit_tex->img_stride[j] = img_stride[j];
1505 }
1506 }
1507
1508
1509 void
1510 draw_llvm_set_sampler_state(struct draw_context *draw)
1511 {
1512 unsigned i;
1513
1514 for (i = 0; i < draw->num_samplers; i++) {
1515 struct draw_jit_texture *jit_tex = &draw->llvm->jit_context.textures[i];
1516
1517 if (draw->samplers[i]) {
1518 jit_tex->min_lod = draw->samplers[i]->min_lod;
1519 jit_tex->max_lod = draw->samplers[i]->max_lod;
1520 jit_tex->lod_bias = draw->samplers[i]->lod_bias;
1521 COPY_4V(jit_tex->border_color, draw->samplers[i]->border_color.f);
1522 }
1523 }
1524 }
1525
1526
1527 void
1528 draw_llvm_destroy_variant(struct draw_llvm_variant *variant)
1529 {
1530 struct draw_llvm *llvm = variant->llvm;
1531
1532 if (variant->function_elts) {
1533 LLVMFreeMachineCodeForFunction(llvm->gallivm->engine,
1534 variant->function_elts);
1535 LLVMDeleteFunction(variant->function_elts);
1536 }
1537
1538 if (variant->function) {
1539 LLVMFreeMachineCodeForFunction(llvm->gallivm->engine,
1540 variant->function);
1541 LLVMDeleteFunction(variant->function);
1542 }
1543
1544 remove_from_list(&variant->list_item_local);
1545 variant->shader->variants_cached--;
1546 remove_from_list(&variant->list_item_global);
1547 llvm->nr_variants--;
1548 FREE(variant);
1549 }