projects / mesa.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
history | raw | HEAD
Merge branch 'draw-instanced'
[mesa.git] / src / gallium / auxiliary / draw / draw_llvm.c
1 /**************************************************************************
2 *
3 * Copyright 2010 VMware, Inc.
4 * All Rights Reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sub license, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
13 *
14 * The above copyright notice and this permission notice (including the
15 * next paragraph) shall be included in all copies or substantial portions
16 * of the Software.
17 *
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
19 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
20 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
21 * IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR
22 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
23 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
24 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
25 *
26 **************************************************************************/
27
28 #include "draw_llvm.h"
29
30 #include "draw_context.h"
31 #include "draw_vs.h"
32
33 #include "gallivm/lp_bld_arit.h"
34 #include "gallivm/lp_bld_logic.h"
35 #include "gallivm/lp_bld_const.h"
36 #include "gallivm/lp_bld_swizzle.h"
37 #include "gallivm/lp_bld_struct.h"
38 #include "gallivm/lp_bld_type.h"
39 #include "gallivm/lp_bld_flow.h"
40 #include "gallivm/lp_bld_debug.h"
41 #include "gallivm/lp_bld_tgsi.h"
42 #include "gallivm/lp_bld_printf.h"
43 #include "gallivm/lp_bld_intr.h"
44 #include "gallivm/lp_bld_init.h"
45 #include "gallivm/lp_bld_type.h"
46
47 #include "tgsi/tgsi_exec.h"
48 #include "tgsi/tgsi_dump.h"
49
50 #include "util/u_math.h"
51 #include "util/u_pointer.h"
52 #include "util/u_string.h"
53 #include "util/u_simple_list.h"
54
55
56 #define DEBUG_STORE 0
57
58
59 /**
60 * This function is called by the gallivm "garbage collector" when
61 * the LLVM global data structures are freed. We must free all LLVM-related
62 * data. Specifically, all JIT'd shader variants.
63 */
64 static void
65 draw_llvm_garbage_collect_callback(void *cb_data)
66 {
67 struct draw_llvm *llvm = (struct draw_llvm *) cb_data;
68 struct draw_llvm_variant_list_item *li;
69
70 /* free all shader variants */
71 li = first_elem(&llvm->vs_variants_list);
72 while (!at_end(&llvm->vs_variants_list, li)) {
73 struct draw_llvm_variant_list_item *next = next_elem(li);
74 draw_llvm_destroy_variant(li->base);
75 li = next;
76 }
77
78 /* Null-out these pointers so they get remade next time they're needed.
79 * See the accessor functions below.
80 */
81 llvm->context_ptr_type = NULL;
82 llvm->buffer_ptr_type = NULL;
83 llvm->vb_ptr_type = NULL;
84 llvm->vertex_header_ptr_type = NULL;
85 }
86
87
88 static void
89 draw_llvm_generate(struct draw_llvm *llvm, struct draw_llvm_variant *var);
90
91 static void
92 draw_llvm_generate_elts(struct draw_llvm *llvm, struct draw_llvm_variant *var);
93
94
95 /**
96 * Create LLVM type for struct draw_jit_texture
97 */
98 static LLVMTypeRef
99 create_jit_texture_type(struct gallivm_state *gallivm)
100 {
101 LLVMTargetDataRef target = gallivm->target;
102 LLVMTypeRef texture_type;
103 LLVMTypeRef elem_types[DRAW_JIT_TEXTURE_NUM_FIELDS];
104 LLVMTypeRef int32_type = LLVMInt32TypeInContext(gallivm->context);
105
106 elem_types[DRAW_JIT_TEXTURE_WIDTH] =
107 elem_types[DRAW_JIT_TEXTURE_HEIGHT] =
108 elem_types[DRAW_JIT_TEXTURE_DEPTH] =
109 elem_types[DRAW_JIT_TEXTURE_LAST_LEVEL] = int32_type;
110 elem_types[DRAW_JIT_TEXTURE_ROW_STRIDE] =
111 elem_types[DRAW_JIT_TEXTURE_IMG_STRIDE] =
112 LLVMArrayType(int32_type, PIPE_MAX_TEXTURE_LEVELS);
113 elem_types[DRAW_JIT_TEXTURE_DATA] =
114 LLVMArrayType(LLVMPointerType(LLVMInt8TypeInContext(gallivm->context), 0),
115 PIPE_MAX_TEXTURE_LEVELS);
116 elem_types[DRAW_JIT_TEXTURE_MIN_LOD] =
117 elem_types[DRAW_JIT_TEXTURE_MAX_LOD] =
118 elem_types[DRAW_JIT_TEXTURE_LOD_BIAS] = LLVMFloatTypeInContext(gallivm->context);
119 elem_types[DRAW_JIT_TEXTURE_BORDER_COLOR] =
120 LLVMArrayType(LLVMFloatTypeInContext(gallivm->context), 4);
121
122 texture_type = LLVMStructTypeInContext(gallivm->context, elem_types,
123 Elements(elem_types), 0);
124
125 /* Make sure the target's struct layout cache doesn't return
126 * stale/invalid data.
127 */
128 LLVMInvalidateStructLayout(gallivm->target, texture_type);
129
130 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, width,
131 target, texture_type,
132 DRAW_JIT_TEXTURE_WIDTH);
133 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, height,
134 target, texture_type,
135 DRAW_JIT_TEXTURE_HEIGHT);
136 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, depth,
137 target, texture_type,
138 DRAW_JIT_TEXTURE_DEPTH);
139 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, last_level,
140 target, texture_type,
141 DRAW_JIT_TEXTURE_LAST_LEVEL);
142 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, row_stride,
143 target, texture_type,
144 DRAW_JIT_TEXTURE_ROW_STRIDE);
145 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, img_stride,
146 target, texture_type,
147 DRAW_JIT_TEXTURE_IMG_STRIDE);
148 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, data,
149 target, texture_type,
150 DRAW_JIT_TEXTURE_DATA);
151 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, min_lod,
152 target, texture_type,
153 DRAW_JIT_TEXTURE_MIN_LOD);
154 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, max_lod,
155 target, texture_type,
156 DRAW_JIT_TEXTURE_MAX_LOD);
157 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, lod_bias,
158 target, texture_type,
159 DRAW_JIT_TEXTURE_LOD_BIAS);
160 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, border_color,
161 target, texture_type,
162 DRAW_JIT_TEXTURE_BORDER_COLOR);
163
164 LP_CHECK_STRUCT_SIZE(struct draw_jit_texture, target, texture_type);
165
166 return texture_type;
167 }
168
169
170 /**
171 * Create LLVM type for struct draw_jit_texture
172 */
173 static LLVMTypeRef
174 create_jit_context_type(struct gallivm_state *gallivm,
175 LLVMTypeRef texture_type)
176 {
177 LLVMTargetDataRef target = gallivm->target;
178 LLVMTypeRef float_type = LLVMFloatTypeInContext(gallivm->context);
179 LLVMTypeRef elem_types[5];
180 LLVMTypeRef context_type;
181
182 elem_types[0] = LLVMPointerType(float_type, 0); /* vs_constants */
183 elem_types[1] = LLVMPointerType(float_type, 0); /* gs_constants */
184 elem_types[2] = LLVMPointerType(LLVMArrayType(LLVMArrayType(float_type, 4), 12), 0); /* planes */
185 elem_types[3] = LLVMPointerType(float_type, 0); /* viewport */
186 elem_types[4] = LLVMArrayType(texture_type,
187 PIPE_MAX_VERTEX_SAMPLERS); /* textures */
188
189 context_type = LLVMStructTypeInContext(gallivm->context, elem_types,
190 Elements(elem_types), 0);
191
192 LLVMInvalidateStructLayout(gallivm->target, context_type);
193
194 LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, vs_constants,
195 target, context_type, 0);
196 LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, gs_constants,
197 target, context_type, 1);
198 LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, planes,
199 target, context_type, 2);
200 LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, textures,
201 target, context_type,
202 DRAW_JIT_CTX_TEXTURES);
203 LP_CHECK_STRUCT_SIZE(struct draw_jit_context,
204 target, context_type);
205
206 return context_type;
207 }
208
209
210 /**
211 * Create LLVM type for struct pipe_vertex_buffer
212 */
213 static LLVMTypeRef
214 create_jit_vertex_buffer_type(struct gallivm_state *gallivm)
215 {
216 LLVMTargetDataRef target = gallivm->target;
217 LLVMTypeRef elem_types[4];
218 LLVMTypeRef vb_type;
219
220 elem_types[0] =
221 elem_types[1] =
222 elem_types[2] = LLVMInt32TypeInContext(gallivm->context);
223 elem_types[3] = LLVMPointerType(LLVMOpaqueTypeInContext(gallivm->context), 0); /* vs_constants */
224
225 vb_type = LLVMStructTypeInContext(gallivm->context, elem_types,
226 Elements(elem_types), 0);
227
228 LLVMInvalidateStructLayout(gallivm->target, vb_type);
229
230 LP_CHECK_MEMBER_OFFSET(struct pipe_vertex_buffer, stride,
231 target, vb_type, 0);
232 LP_CHECK_MEMBER_OFFSET(struct pipe_vertex_buffer, buffer_offset,
233 target, vb_type, 2);
234
235 LP_CHECK_STRUCT_SIZE(struct pipe_vertex_buffer, target, vb_type);
236
237 return vb_type;
238 }
239
240
241 /**
242 * Create LLVM type for struct vertex_header;
243 */
244 static LLVMTypeRef
245 create_jit_vertex_header(struct gallivm_state *gallivm, int data_elems)
246 {
247 LLVMTargetDataRef target = gallivm->target;
248 LLVMTypeRef elem_types[3];
249 LLVMTypeRef vertex_header;
250 char struct_name[24];
251
252 util_snprintf(struct_name, 23, "vertex_header%d", data_elems);
253
254 elem_types[0] = LLVMIntTypeInContext(gallivm->context, 32);
255 elem_types[1] = LLVMArrayType(LLVMFloatTypeInContext(gallivm->context), 4);
256 elem_types[2] = LLVMArrayType(elem_types[1], data_elems);
257
258 vertex_header = LLVMStructTypeInContext(gallivm->context, elem_types,
259 Elements(elem_types), 0);
260
261 LLVMInvalidateStructLayout(gallivm->target, vertex_header);
262
263 /* these are bit-fields and we can't take address of them
264 LP_CHECK_MEMBER_OFFSET(struct vertex_header, clipmask,
265 target, vertex_header,
266 DRAW_JIT_VERTEX_CLIPMASK);
267 LP_CHECK_MEMBER_OFFSET(struct vertex_header, edgeflag,
268 target, vertex_header,
269 DRAW_JIT_VERTEX_EDGEFLAG);
270 LP_CHECK_MEMBER_OFFSET(struct vertex_header, pad,
271 target, vertex_header,
272 DRAW_JIT_VERTEX_PAD);
273 LP_CHECK_MEMBER_OFFSET(struct vertex_header, vertex_id,
274 target, vertex_header,
275 DRAW_JIT_VERTEX_VERTEX_ID);
276 */
277 LP_CHECK_MEMBER_OFFSET(struct vertex_header, clip,
278 target, vertex_header,
279 DRAW_JIT_VERTEX_CLIP);
280 LP_CHECK_MEMBER_OFFSET(struct vertex_header, data,
281 target, vertex_header,
282 DRAW_JIT_VERTEX_DATA);
283
284 LLVMAddTypeName(gallivm->module, struct_name, vertex_header);
285
286 return vertex_header;
287 }
288
289
290 /**
291 * Create LLVM types for various structures.
292 */
293 static void
294 create_jit_types(struct draw_llvm *llvm)
295 {
296 struct gallivm_state *gallivm = llvm->gallivm;
297 LLVMTypeRef texture_type, context_type, buffer_type, vb_type;
298
299 texture_type = create_jit_texture_type(gallivm);
300 LLVMAddTypeName(gallivm->module, "texture", texture_type);
301
302 context_type = create_jit_context_type(gallivm, texture_type);
303 LLVMAddTypeName(gallivm->module, "draw_jit_context", context_type);
304 llvm->context_ptr_type = LLVMPointerType(context_type, 0);
305
306 buffer_type = LLVMPointerType(LLVMIntTypeInContext(gallivm->context, 8), 0);
307 LLVMAddTypeName(gallivm->module, "buffer", buffer_type);
308 llvm->buffer_ptr_type = LLVMPointerType(buffer_type, 0);
309
310 vb_type = create_jit_vertex_buffer_type(gallivm);
311 LLVMAddTypeName(gallivm->module, "pipe_vertex_buffer", vb_type);
312 llvm->vb_ptr_type = LLVMPointerType(vb_type, 0);
313 }
314
315
316 static LLVMTypeRef
317 get_context_ptr_type(struct draw_llvm *llvm)
318 {
319 if (!llvm->context_ptr_type)
320 create_jit_types(llvm);
321 return llvm->context_ptr_type;
322 }
323
324
325 static LLVMTypeRef
326 get_buffer_ptr_type(struct draw_llvm *llvm)
327 {
328 if (!llvm->buffer_ptr_type)
329 create_jit_types(llvm);
330 return llvm->buffer_ptr_type;
331 }
332
333
334 static LLVMTypeRef
335 get_vb_ptr_type(struct draw_llvm *llvm)
336 {
337 if (!llvm->vb_ptr_type)
338 create_jit_types(llvm);
339 return llvm->vb_ptr_type;
340 }
341
342 static LLVMTypeRef
343 get_vertex_header_ptr_type(struct draw_llvm *llvm)
344 {
345 if (!llvm->vertex_header_ptr_type)
346 create_jit_types(llvm);
347 return llvm->vertex_header_ptr_type;
348 }
349
350
351 /**
352 * Create per-context LLVM info.
353 */
354 struct draw_llvm *
355 draw_llvm_create(struct draw_context *draw, struct gallivm_state *gallivm)
356 {
357 struct draw_llvm *llvm;
358
359 llvm = CALLOC_STRUCT( draw_llvm );
360 if (!llvm)
361 return NULL;
362
363 lp_build_init();
364
365 llvm->draw = draw;
366 llvm->gallivm = gallivm;
367
368 if (gallivm_debug & GALLIVM_DEBUG_IR) {
369 LLVMDumpModule(llvm->gallivm->module);
370 }
371
372 llvm->nr_variants = 0;
373 make_empty_list(&llvm->vs_variants_list);
374
375 gallivm_register_garbage_collector_callback(
376 draw_llvm_garbage_collect_callback, llvm);
377
378 return llvm;
379 }
380
381
382 /**
383 * Free per-context LLVM info.
384 */
385 void
386 draw_llvm_destroy(struct draw_llvm *llvm)
387 {
388 gallivm_remove_garbage_collector_callback(
389 draw_llvm_garbage_collect_callback, llvm);
390
391 /* XXX free other draw_llvm data? */
392 FREE(llvm);
393 }
394
395
396 /**
397 * Create LLVM-generated code for a vertex shader.
398 */
399 struct draw_llvm_variant *
400 draw_llvm_create_variant(struct draw_llvm *llvm,
401 unsigned num_inputs,
402 const struct draw_llvm_variant_key *key)
403 {
404 struct draw_llvm_variant *variant;
405 struct llvm_vertex_shader *shader =
406 llvm_vertex_shader(llvm->draw->vs.vertex_shader);
407 LLVMTypeRef vertex_header;
408
409 variant = MALLOC(sizeof *variant +
410 shader->variant_key_size -
411 sizeof variant->key);
412 if (variant == NULL)
413 return NULL;
414
415 variant->llvm = llvm;
416
417 memcpy(&variant->key, key, shader->variant_key_size);
418
419 vertex_header = create_jit_vertex_header(llvm->gallivm, num_inputs);
420
421 llvm->vertex_header_ptr_type = LLVMPointerType(vertex_header, 0);
422
423 draw_llvm_generate(llvm, variant);
424 draw_llvm_generate_elts(llvm, variant);
425
426 variant->shader = shader;
427 variant->list_item_global.base = variant;
428 variant->list_item_local.base = variant;
429 /*variant->no = */shader->variants_created++;
430 variant->list_item_global.base = variant;
431
432 return variant;
433 }
434
435 static void
436 generate_vs(struct draw_llvm *llvm,
437 LLVMBuilderRef builder,
438 LLVMValueRef (*outputs)[NUM_CHANNELS],
439 const LLVMValueRef (*inputs)[NUM_CHANNELS],
440 LLVMValueRef system_values_array,
441 LLVMValueRef context_ptr,
442 struct lp_build_sampler_soa *draw_sampler)
443 {
444 const struct tgsi_token *tokens = llvm->draw->vs.vertex_shader->state.tokens;
445 struct lp_type vs_type;
446 LLVMValueRef consts_ptr = draw_jit_context_vs_constants(llvm->gallivm, context_ptr);
447 struct lp_build_sampler_soa *sampler = 0;
448
449 memset(&vs_type, 0, sizeof vs_type);
450 vs_type.floating = TRUE; /* floating point values */
451 vs_type.sign = TRUE; /* values are signed */
452 vs_type.norm = FALSE; /* values are not limited to [0,1] or [-1,1] */
453 vs_type.width = 32; /* 32-bit float */
454 vs_type.length = 4; /* 4 elements per vector */
455 #if 0
456 num_vs = 4; /* number of vertices per block */
457 #endif
458
459 if (gallivm_debug & GALLIVM_DEBUG_IR) {
460 tgsi_dump(tokens, 0);
461 }
462
463 if (llvm->draw->num_sampler_views &&
464 llvm->draw->num_samplers)
465 sampler = draw_sampler;
466
467 lp_build_tgsi_soa(llvm->gallivm,
468 tokens,
469 vs_type,
470 NULL /*struct lp_build_mask_context *mask*/,
471 consts_ptr,
472 system_values_array,
473 NULL /*pos*/,
474 inputs,
475 outputs,
476 sampler,
477 &llvm->draw->vs.vertex_shader->info);
478 }
479
480 #if DEBUG_STORE
481 static void print_vectorf(LLVMBuilderRef builder,
482 LLVMValueRef vec)
483 {
484 LLVMValueRef val[4];
485 val[0] = LLVMBuildExtractElement(builder, vec,
486 lp_build_const_int32(gallivm, 0), "");
487 val[1] = LLVMBuildExtractElement(builder, vec,
488 lp_build_const_int32(gallivm, 1), "");
489 val[2] = LLVMBuildExtractElement(builder, vec,
490 lp_build_const_int32(gallivm, 2), "");
491 val[3] = LLVMBuildExtractElement(builder, vec,
492 lp_build_const_int32(gallivm, 3), "");
493 lp_build_printf(builder, "vector = [%f, %f, %f, %f]\n",
494 val[0], val[1], val[2], val[3]);
495 }
496 #endif
497
498 static void
499 generate_fetch(struct gallivm_state *gallivm,
500 LLVMValueRef vbuffers_ptr,
501 LLVMValueRef *res,
502 struct pipe_vertex_element *velem,
503 LLVMValueRef vbuf,
504 LLVMValueRef index,
505 LLVMValueRef instance_id)
506 {
507 LLVMBuilderRef builder = gallivm->builder;
508 LLVMValueRef indices =
509 LLVMConstInt(LLVMInt64TypeInContext(gallivm->context),
510 velem->vertex_buffer_index, 0);
511 LLVMValueRef vbuffer_ptr = LLVMBuildGEP(builder, vbuffers_ptr,
512 &indices, 1, "");
513 LLVMValueRef vb_stride = draw_jit_vbuffer_stride(gallivm, vbuf);
514 LLVMValueRef vb_max_index = draw_jit_vbuffer_max_index(gallivm, vbuf);
515 LLVMValueRef vb_buffer_offset = draw_jit_vbuffer_offset(gallivm, vbuf);
516 LLVMValueRef cond;
517 LLVMValueRef stride;
518
519 if (velem->instance_divisor) {
520 /* array index = instance_id / instance_divisor */
521 index = LLVMBuildUDiv(builder, instance_id,
522 lp_build_const_int32(gallivm, velem->instance_divisor),
523 "instance_divisor");
524 }
525
526 /* limit index to min(index, vb_max_index) */
527 cond = LLVMBuildICmp(builder, LLVMIntULE, index, vb_max_index, "");
528 index = LLVMBuildSelect(builder, cond, index, vb_max_index, "");
529
530 stride = LLVMBuildMul(builder, vb_stride, index, "");
531
532 vbuffer_ptr = LLVMBuildLoad(builder, vbuffer_ptr, "vbuffer");
533
534 stride = LLVMBuildAdd(builder, stride,
535 vb_buffer_offset,
536 "");
537 stride = LLVMBuildAdd(builder, stride,
538 lp_build_const_int32(gallivm, velem->src_offset),
539 "");
540
541 /*lp_build_printf(builder, "vbuf index = %d, stride is %d\n", indices, stride);*/
542 vbuffer_ptr = LLVMBuildGEP(builder, vbuffer_ptr, &stride, 1, "");
543
544 *res = draw_llvm_translate_from(gallivm, vbuffer_ptr, velem->src_format);
545 }
546
547 static LLVMValueRef
548 aos_to_soa(struct gallivm_state *gallivm,
549 LLVMValueRef val0,
550 LLVMValueRef val1,
551 LLVMValueRef val2,
552 LLVMValueRef val3,
553 LLVMValueRef channel)
554 {
555 LLVMBuilderRef builder = gallivm->builder;
556 LLVMValueRef ex, res;
557
558 ex = LLVMBuildExtractElement(builder, val0,
559 channel, "");
560 res = LLVMBuildInsertElement(builder,
561 LLVMConstNull(LLVMTypeOf(val0)),
562 ex,
563 lp_build_const_int32(gallivm, 0),
564 "");
565
566 ex = LLVMBuildExtractElement(builder, val1,
567 channel, "");
568 res = LLVMBuildInsertElement(builder,
569 res, ex,
570 lp_build_const_int32(gallivm, 1),
571 "");
572
573 ex = LLVMBuildExtractElement(builder, val2,
574 channel, "");
575 res = LLVMBuildInsertElement(builder,
576 res, ex,
577 lp_build_const_int32(gallivm, 2),
578 "");
579
580 ex = LLVMBuildExtractElement(builder, val3,
581 channel, "");
582 res = LLVMBuildInsertElement(builder,
583 res, ex,
584 lp_build_const_int32(gallivm, 3),
585 "");
586
587 return res;
588 }
589
590 static void
591 soa_to_aos(struct gallivm_state *gallivm,
592 LLVMValueRef soa[NUM_CHANNELS],
593 LLVMValueRef aos[NUM_CHANNELS])
594 {
595 LLVMBuilderRef builder = gallivm->builder;
596 LLVMValueRef comp;
597 int i = 0;
598
599 debug_assert(NUM_CHANNELS == 4);
600
601 aos[0] = LLVMConstNull(LLVMTypeOf(soa[0]));
602 aos[1] = aos[2] = aos[3] = aos[0];
603
604 for (i = 0; i < NUM_CHANNELS; ++i) {
605 LLVMValueRef channel = lp_build_const_int32(gallivm, i);
606
607 comp = LLVMBuildExtractElement(builder, soa[i],
608 lp_build_const_int32(gallivm, 0), "");
609 aos[0] = LLVMBuildInsertElement(builder, aos[0], comp, channel, "");
610
611 comp = LLVMBuildExtractElement(builder, soa[i],
612 lp_build_const_int32(gallivm, 1), "");
613 aos[1] = LLVMBuildInsertElement(builder, aos[1], comp, channel, "");
614
615 comp = LLVMBuildExtractElement(builder, soa[i],
616 lp_build_const_int32(gallivm, 2), "");
617 aos[2] = LLVMBuildInsertElement(builder, aos[2], comp, channel, "");
618
619 comp = LLVMBuildExtractElement(builder, soa[i],
620 lp_build_const_int32(gallivm, 3), "");
621 aos[3] = LLVMBuildInsertElement(builder, aos[3], comp, channel, "");
622
623 }
624 }
625
626 static void
627 convert_to_soa(struct gallivm_state *gallivm,
628 LLVMValueRef (*aos)[NUM_CHANNELS],
629 LLVMValueRef (*soa)[NUM_CHANNELS],
630 int num_attribs)
631 {
632 int i;
633
634 debug_assert(NUM_CHANNELS == 4);
635
636 for (i = 0; i < num_attribs; ++i) {
637 LLVMValueRef val0 = aos[i][0];
638 LLVMValueRef val1 = aos[i][1];
639 LLVMValueRef val2 = aos[i][2];
640 LLVMValueRef val3 = aos[i][3];
641
642 soa[i][0] = aos_to_soa(gallivm, val0, val1, val2, val3,
643 lp_build_const_int32(gallivm, 0));
644 soa[i][1] = aos_to_soa(gallivm, val0, val1, val2, val3,
645 lp_build_const_int32(gallivm, 1));
646 soa[i][2] = aos_to_soa(gallivm, val0, val1, val2, val3,
647 lp_build_const_int32(gallivm, 2));
648 soa[i][3] = aos_to_soa(gallivm, val0, val1, val2, val3,
649 lp_build_const_int32(gallivm, 3));
650 }
651 }
652
653 static void
654 store_aos(struct gallivm_state *gallivm,
655 LLVMValueRef io_ptr,
656 LLVMValueRef index,
657 LLVMValueRef value,
658 LLVMValueRef clipmask)
659 {
660 LLVMBuilderRef builder = gallivm->builder;
661 LLVMValueRef id_ptr = draw_jit_header_id(gallivm, io_ptr);
662 LLVMValueRef data_ptr = draw_jit_header_data(gallivm, io_ptr);
663 LLVMValueRef indices[3];
664 LLVMValueRef val, shift;
665
666 indices[0] = lp_build_const_int32(gallivm, 0);
667 indices[1] = index;
668 indices[2] = lp_build_const_int32(gallivm, 0);
669
670 /* initialize vertex id:16 = 0xffff, pad:3 = 0, edgeflag:1 = 1 */
671 val = lp_build_const_int32(gallivm, 0xffff1);
672 shift = lp_build_const_int32(gallivm, 12);
673 val = LLVMBuildShl(builder, val, shift, "");
674 /* add clipmask:12 */
675 val = LLVMBuildOr(builder, val, clipmask, "");
676
677 /* store vertex header */
678 LLVMBuildStore(builder, val, id_ptr);
679
680
681 #if DEBUG_STORE
682 lp_build_printf(builder, " ---- %p storing attribute %d (io = %p)\n", data_ptr, index, io_ptr);
683 #endif
684 #if 0
685 /*lp_build_printf(builder, " ---- %p storing at %d (%p) ", io_ptr, index, data_ptr);
686 print_vectorf(builder, value);*/
687 data_ptr = LLVMBuildBitCast(builder, data_ptr,
688 LLVMPointerType(LLVMArrayType(LLVMVectorType(LLVMFloatTypeInContext(gallivm->context), 4), 0), 0),
689 "datavec");
690 data_ptr = LLVMBuildGEP(builder, data_ptr, indices, 2, "");
691
692 LLVMBuildStore(builder, value, data_ptr);
693 #else
694 {
695 LLVMValueRef x, y, z, w;
696 LLVMValueRef idx0, idx1, idx2, idx3;
697 LLVMValueRef gep0, gep1, gep2, gep3;
698 data_ptr = LLVMBuildGEP(builder, data_ptr, indices, 3, "");
699
700 idx0 = lp_build_const_int32(gallivm, 0);
701 idx1 = lp_build_const_int32(gallivm, 1);
702 idx2 = lp_build_const_int32(gallivm, 2);
703 idx3 = lp_build_const_int32(gallivm, 3);
704
705 x = LLVMBuildExtractElement(builder, value,
706 idx0, "");
707 y = LLVMBuildExtractElement(builder, value,
708 idx1, "");
709 z = LLVMBuildExtractElement(builder, value,
710 idx2, "");
711 w = LLVMBuildExtractElement(builder, value,
712 idx3, "");
713
714 gep0 = LLVMBuildGEP(builder, data_ptr, &idx0, 1, "");
715 gep1 = LLVMBuildGEP(builder, data_ptr, &idx1, 1, "");
716 gep2 = LLVMBuildGEP(builder, data_ptr, &idx2, 1, "");
717 gep3 = LLVMBuildGEP(builder, data_ptr, &idx3, 1, "");
718
719 /*lp_build_printf(builder, "##### x = %f (%p), y = %f (%p), z = %f (%p), w = %f (%p)\n",
720 x, gep0, y, gep1, z, gep2, w, gep3);*/
721 LLVMBuildStore(builder, x, gep0);
722 LLVMBuildStore(builder, y, gep1);
723 LLVMBuildStore(builder, z, gep2);
724 LLVMBuildStore(builder, w, gep3);
725 }
726 #endif
727 }
728
729 static void
730 store_aos_array(struct gallivm_state *gallivm,
731 LLVMValueRef io_ptr,
732 LLVMValueRef aos[NUM_CHANNELS],
733 int attrib,
734 int num_outputs,
735 LLVMValueRef clipmask)
736 {
737 LLVMBuilderRef builder = gallivm->builder;
738 LLVMValueRef attr_index = lp_build_const_int32(gallivm, attrib);
739 LLVMValueRef ind0 = lp_build_const_int32(gallivm, 0);
740 LLVMValueRef ind1 = lp_build_const_int32(gallivm, 1);
741 LLVMValueRef ind2 = lp_build_const_int32(gallivm, 2);
742 LLVMValueRef ind3 = lp_build_const_int32(gallivm, 3);
743 LLVMValueRef io0_ptr, io1_ptr, io2_ptr, io3_ptr;
744 LLVMValueRef clipmask0, clipmask1, clipmask2, clipmask3;
745
746 debug_assert(NUM_CHANNELS == 4);
747
748 io0_ptr = LLVMBuildGEP(builder, io_ptr,
749 &ind0, 1, "");
750 io1_ptr = LLVMBuildGEP(builder, io_ptr,
751 &ind1, 1, "");
752 io2_ptr = LLVMBuildGEP(builder, io_ptr,
753 &ind2, 1, "");
754 io3_ptr = LLVMBuildGEP(builder, io_ptr,
755 &ind3, 1, "");
756
757 clipmask0 = LLVMBuildExtractElement(builder, clipmask,
758 ind0, "");
759 clipmask1 = LLVMBuildExtractElement(builder, clipmask,
760 ind1, "");
761 clipmask2 = LLVMBuildExtractElement(builder, clipmask,
762 ind2, "");
763 clipmask3 = LLVMBuildExtractElement(builder, clipmask,
764 ind3, "");
765
766 #if DEBUG_STORE
767 lp_build_printf(builder, "io = %p, indexes[%d, %d, %d, %d]\n, clipmask0 = %x, clipmask1 = %x, clipmask2 = %x, clipmask3 = %x\n",
768 io_ptr, ind0, ind1, ind2, ind3, clipmask0, clipmask1, clipmask2, clipmask3);
769 #endif
770 /* store for each of the 4 vertices */
771 store_aos(gallivm, io0_ptr, attr_index, aos[0], clipmask0);
772 store_aos(gallivm, io1_ptr, attr_index, aos[1], clipmask1);
773 store_aos(gallivm, io2_ptr, attr_index, aos[2], clipmask2);
774 store_aos(gallivm, io3_ptr, attr_index, aos[3], clipmask3);
775 }
776
777 static void
778 convert_to_aos(struct gallivm_state *gallivm,
779 LLVMValueRef io,
780 LLVMValueRef (*outputs)[NUM_CHANNELS],
781 LLVMValueRef clipmask,
782 int num_outputs,
783 int max_vertices)
784 {
785 LLVMBuilderRef builder = gallivm->builder;
786 unsigned chan, attrib;
787
788 #if DEBUG_STORE
789 lp_build_printf(builder, " # storing begin\n");
790 #endif
791 for (attrib = 0; attrib < num_outputs; ++attrib) {
792 LLVMValueRef soa[4];
793 LLVMValueRef aos[4];
794 for(chan = 0; chan < NUM_CHANNELS; ++chan) {
795 if(outputs[attrib][chan]) {
796 LLVMValueRef out = LLVMBuildLoad(builder, outputs[attrib][chan], "");
797 lp_build_name(out, "output%u.%c", attrib, "xyzw"[chan]);
798 /*lp_build_printf(builder, "output %d : %d ",
799 LLVMConstInt(LLVMInt32Type(), attrib, 0),
800 LLVMConstInt(LLVMInt32Type(), chan, 0));
801 print_vectorf(builder, out);*/
802 soa[chan] = out;
803 } else
804 soa[chan] = 0;
805 }
806 soa_to_aos(gallivm, soa, aos);
807 store_aos_array(gallivm,
808 io,
809 aos,
810 attrib,
811 num_outputs,
812 clipmask);
813 }
814 #if DEBUG_STORE
815 lp_build_printf(builder, " # storing end\n");
816 #endif
817 }
818
819 /*
820 * Stores original vertex positions in clip coordinates
821 * There is probably a more efficient way to do this, 4 floats at once
822 * rather than extracting each element one by one.
823 */
824 static void
825 store_clip(struct gallivm_state *gallivm,
826 LLVMValueRef io_ptr,
827 LLVMValueRef (*outputs)[NUM_CHANNELS])
828 {
829 LLVMBuilderRef builder = gallivm->builder;
830 LLVMValueRef out[4];
831 LLVMValueRef indices[2];
832 LLVMValueRef io0_ptr, io1_ptr, io2_ptr, io3_ptr;
833 LLVMValueRef clip_ptr0, clip_ptr1, clip_ptr2, clip_ptr3;
834 LLVMValueRef clip0_ptr, clip1_ptr, clip2_ptr, clip3_ptr;
835 LLVMValueRef out0elem, out1elem, out2elem, out3elem;
836 int i;
837
838 LLVMValueRef ind0 = lp_build_const_int32(gallivm, 0);
839 LLVMValueRef ind1 = lp_build_const_int32(gallivm, 1);
840 LLVMValueRef ind2 = lp_build_const_int32(gallivm, 2);
841 LLVMValueRef ind3 = lp_build_const_int32(gallivm, 3);
842
843 indices[0] =
844 indices[1] = lp_build_const_int32(gallivm, 0);
845
846 out[0] = LLVMBuildLoad(builder, outputs[0][0], ""); /*x0 x1 x2 x3*/
847 out[1] = LLVMBuildLoad(builder, outputs[0][1], ""); /*y0 y1 y2 y3*/
848 out[2] = LLVMBuildLoad(builder, outputs[0][2], ""); /*z0 z1 z2 z3*/
849 out[3] = LLVMBuildLoad(builder, outputs[0][3], ""); /*w0 w1 w2 w3*/
850
851 io0_ptr = LLVMBuildGEP(builder, io_ptr, &ind0, 1, "");
852 io1_ptr = LLVMBuildGEP(builder, io_ptr, &ind1, 1, "");
853 io2_ptr = LLVMBuildGEP(builder, io_ptr, &ind2, 1, "");
854 io3_ptr = LLVMBuildGEP(builder, io_ptr, &ind3, 1, "");
855
856 clip_ptr0 = draw_jit_header_clip(gallivm, io0_ptr);
857 clip_ptr1 = draw_jit_header_clip(gallivm, io1_ptr);
858 clip_ptr2 = draw_jit_header_clip(gallivm, io2_ptr);
859 clip_ptr3 = draw_jit_header_clip(gallivm, io3_ptr);
860
861 for (i = 0; i<4; i++){
862 clip0_ptr = LLVMBuildGEP(builder, clip_ptr0, indices, 2, ""); /* x0 */
863 clip1_ptr = LLVMBuildGEP(builder, clip_ptr1, indices, 2, ""); /* x1 */
864 clip2_ptr = LLVMBuildGEP(builder, clip_ptr2, indices, 2, ""); /* x2 */
865 clip3_ptr = LLVMBuildGEP(builder, clip_ptr3, indices, 2, ""); /* x3 */
866
867 out0elem = LLVMBuildExtractElement(builder, out[i], ind0, ""); /* x0 */
868 out1elem = LLVMBuildExtractElement(builder, out[i], ind1, ""); /* x1 */
869 out2elem = LLVMBuildExtractElement(builder, out[i], ind2, ""); /* x2 */
870 out3elem = LLVMBuildExtractElement(builder, out[i], ind3, ""); /* x3 */
871
872 LLVMBuildStore(builder, out0elem, clip0_ptr);
873 LLVMBuildStore(builder, out1elem, clip1_ptr);
874 LLVMBuildStore(builder, out2elem, clip2_ptr);
875 LLVMBuildStore(builder, out3elem, clip3_ptr);
876
877 indices[1]= LLVMBuildAdd(builder, indices[1], ind1, "");
878 }
879
880 }
881
882 /* Equivalent of _mm_set1_ps(a)
883 */
884 static LLVMValueRef
885 vec4f_from_scalar(struct gallivm_state *gallivm,
886 LLVMValueRef a,
887 const char *name)
888 {
889 LLVMTypeRef float_type = LLVMFloatTypeInContext(gallivm->context);
890 LLVMValueRef res = LLVMGetUndef(LLVMVectorType(float_type, 4));
891 int i;
892
893 for(i = 0; i < 4; ++i) {
894 LLVMValueRef index = lp_build_const_int32(gallivm, i);
895 res = LLVMBuildInsertElement(gallivm->builder, res, a,
896 index, i == 3 ? name : "");
897 }
898
899 return res;
900 }
901
902 /*
903 * Transforms the outputs for viewport mapping
904 */
905 static void
906 generate_viewport(struct draw_llvm *llvm,
907 LLVMBuilderRef builder,
908 LLVMValueRef (*outputs)[NUM_CHANNELS],
909 LLVMValueRef context_ptr)
910 {
911 int i;
912 struct gallivm_state *gallivm = llvm->gallivm;
913 struct lp_type f32_type = lp_type_float_vec(32);
914 LLVMValueRef out3 = LLVMBuildLoad(builder, outputs[0][3], ""); /*w0 w1 w2 w3*/
915 LLVMValueRef const1 = lp_build_const_vec(gallivm, f32_type, 1.0); /*1.0 1.0 1.0 1.0*/
916 LLVMValueRef vp_ptr = draw_jit_context_viewport(gallivm, context_ptr);
917
918 /* for 1/w convention*/
919 out3 = LLVMBuildFDiv(builder, const1, out3, "");
920 LLVMBuildStore(builder, out3, outputs[0][3]);
921
922 /* Viewport Mapping */
923 for (i=0; i<3; i++){
924 LLVMValueRef out = LLVMBuildLoad(builder, outputs[0][i], ""); /*x0 x1 x2 x3*/
925 LLVMValueRef scale;
926 LLVMValueRef trans;
927 LLVMValueRef scale_i;
928 LLVMValueRef trans_i;
929 LLVMValueRef index;
930
931 index = lp_build_const_int32(gallivm, i);
932 scale_i = LLVMBuildGEP(builder, vp_ptr, &index, 1, "");
933
934 index = lp_build_const_int32(gallivm, i+4);
935 trans_i = LLVMBuildGEP(builder, vp_ptr, &index, 1, "");
936
937 scale = vec4f_from_scalar(gallivm, LLVMBuildLoad(builder, scale_i, ""), "scale");
938 trans = vec4f_from_scalar(gallivm, LLVMBuildLoad(builder, trans_i, ""), "trans");
939
940 /* divide by w */
941 out = LLVMBuildFMul(builder, out, out3, "");
942 /* mult by scale */
943 out = LLVMBuildFMul(builder, out, scale, "");
944 /* add translation */
945 out = LLVMBuildFAdd(builder, out, trans, "");
946
947 /* store transformed outputs */
948 LLVMBuildStore(builder, out, outputs[0][i]);
949 }
950
951 }
952
953
954 /*
955 * Returns clipmask as 4xi32 bitmask for the 4 vertices
956 */
957 static LLVMValueRef
958 generate_clipmask(struct gallivm_state *gallivm,
959 LLVMValueRef (*outputs)[NUM_CHANNELS],
960 boolean clip_xy,
961 boolean clip_z,
962 boolean clip_user,
963 boolean clip_halfz,
964 unsigned nr,
965 LLVMValueRef context_ptr)
966 {
967 LLVMBuilderRef builder = gallivm->builder;
968 LLVMValueRef mask; /* stores the <4xi32> clipmasks */
969 LLVMValueRef test, temp;
970 LLVMValueRef zero, shift;
971 LLVMValueRef pos_x, pos_y, pos_z, pos_w;
972 LLVMValueRef plane1, planes, plane_ptr, sum;
973
974 unsigned i;
975
976 struct lp_type f32_type = lp_type_float_vec(32);
977
978 mask = lp_build_const_int_vec(gallivm, lp_type_int_vec(32), 0);
979 temp = lp_build_const_int_vec(gallivm, lp_type_int_vec(32), 0);
980 zero = lp_build_const_vec(gallivm, f32_type, 0); /* 0.0f 0.0f 0.0f 0.0f */
981 shift = lp_build_const_int_vec(gallivm, lp_type_int_vec(32), 1); /* 1 1 1 1 */
982
983 /* Assuming position stored at output[0] */
984 pos_x = LLVMBuildLoad(builder, outputs[0][0], ""); /*x0 x1 x2 x3*/
985 pos_y = LLVMBuildLoad(builder, outputs[0][1], ""); /*y0 y1 y2 y3*/
986 pos_z = LLVMBuildLoad(builder, outputs[0][2], ""); /*z0 z1 z2 z3*/
987 pos_w = LLVMBuildLoad(builder, outputs[0][3], ""); /*w0 w1 w2 w3*/
988
989 /* Cliptest, for hardwired planes */
990 if (clip_xy){
991 /* plane 1 */
992 test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, pos_x , pos_w);
993 temp = shift;
994 test = LLVMBuildAnd(builder, test, temp, "");
995 mask = test;
996
997 /* plane 2 */
998 test = LLVMBuildFAdd(builder, pos_x, pos_w, "");
999 test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, zero, test);
1000 temp = LLVMBuildShl(builder, temp, shift, "");
1001 test = LLVMBuildAnd(builder, test, temp, "");
1002 mask = LLVMBuildOr(builder, mask, test, "");
1003
1004 /* plane 3 */
1005 test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, pos_y, pos_w);
1006 temp = LLVMBuildShl(builder, temp, shift, "");
1007 test = LLVMBuildAnd(builder, test, temp, "");
1008 mask = LLVMBuildOr(builder, mask, test, "");
1009
1010 /* plane 4 */
1011 test = LLVMBuildFAdd(builder, pos_y, pos_w, "");
1012 test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, zero, test);
1013 temp = LLVMBuildShl(builder, temp, shift, "");
1014 test = LLVMBuildAnd(builder, test, temp, "");
1015 mask = LLVMBuildOr(builder, mask, test, "");
1016 }
1017
1018 if (clip_z){
1019 temp = lp_build_const_int_vec(gallivm, lp_type_int_vec(32), 16);
1020 if (clip_halfz){
1021 /* plane 5 */
1022 test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, zero, pos_z);
1023 test = LLVMBuildAnd(builder, test, temp, "");
1024 mask = LLVMBuildOr(builder, mask, test, "");
1025 }
1026 else{
1027 /* plane 5 */
1028 test = LLVMBuildFAdd(builder, pos_z, pos_w, "");
1029 test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, zero, test);
1030 test = LLVMBuildAnd(builder, test, temp, "");
1031 mask = LLVMBuildOr(builder, mask, test, "");
1032 }
1033 /* plane 6 */
1034 test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, pos_z, pos_w);
1035 temp = LLVMBuildShl(builder, temp, shift, "");
1036 test = LLVMBuildAnd(builder, test, temp, "");
1037 mask = LLVMBuildOr(builder, mask, test, "");
1038 }
1039
1040 if (clip_user){
1041 LLVMValueRef planes_ptr = draw_jit_context_planes(gallivm, context_ptr);
1042 LLVMValueRef indices[3];
1043 temp = lp_build_const_int_vec(gallivm, lp_type_int_vec(32), 32);
1044
1045 /* userclip planes */
1046 for (i = 6; i < nr; i++) {
1047 indices[0] = lp_build_const_int32(gallivm, 0);
1048 indices[1] = lp_build_const_int32(gallivm, i);
1049
1050 indices[2] = lp_build_const_int32(gallivm, 0);
1051 plane_ptr = LLVMBuildGEP(builder, planes_ptr, indices, 3, "");
1052 plane1 = LLVMBuildLoad(builder, plane_ptr, "plane_x");
1053 planes = vec4f_from_scalar(gallivm, plane1, "plane4_x");
1054 sum = LLVMBuildFMul(builder, planes, pos_x, "");
1055
1056 indices[2] = lp_build_const_int32(gallivm, 1);
1057 plane_ptr = LLVMBuildGEP(builder, planes_ptr, indices, 3, "");
1058 plane1 = LLVMBuildLoad(builder, plane_ptr, "plane_y");
1059 planes = vec4f_from_scalar(gallivm, plane1, "plane4_y");
1060 test = LLVMBuildFMul(builder, planes, pos_y, "");
1061 sum = LLVMBuildFAdd(builder, sum, test, "");
1062
1063 indices[2] = lp_build_const_int32(gallivm, 2);
1064 plane_ptr = LLVMBuildGEP(builder, planes_ptr, indices, 3, "");
1065 plane1 = LLVMBuildLoad(builder, plane_ptr, "plane_z");
1066 planes = vec4f_from_scalar(gallivm, plane1, "plane4_z");
1067 test = LLVMBuildFMul(builder, planes, pos_z, "");
1068 sum = LLVMBuildFAdd(builder, sum, test, "");
1069
1070 indices[2] = lp_build_const_int32(gallivm, 3);
1071 plane_ptr = LLVMBuildGEP(builder, planes_ptr, indices, 3, "");
1072 plane1 = LLVMBuildLoad(builder, plane_ptr, "plane_w");
1073 planes = vec4f_from_scalar(gallivm, plane1, "plane4_w");
1074 test = LLVMBuildFMul(builder, planes, pos_w, "");
1075 sum = LLVMBuildFAdd(builder, sum, test, "");
1076
1077 test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, zero, sum);
1078 temp = LLVMBuildShl(builder, temp, shift, "");
1079 test = LLVMBuildAnd(builder, test, temp, "");
1080 mask = LLVMBuildOr(builder, mask, test, "");
1081 }
1082 }
1083 return mask;
1084 }
1085
1086 /*
1087 * Returns boolean if any clipping has occurred
1088 * Used zero/non-zero i32 value to represent boolean
1089 */
1090 static void
1091 clipmask_bool(struct gallivm_state *gallivm,
1092 LLVMValueRef clipmask,
1093 LLVMValueRef ret_ptr)
1094 {
1095 LLVMBuilderRef builder = gallivm->builder;
1096 LLVMValueRef ret = LLVMBuildLoad(builder, ret_ptr, "");
1097 LLVMValueRef temp;
1098 int i;
1099
1100 for (i=0; i<4; i++){
1101 temp = LLVMBuildExtractElement(builder, clipmask,
1102 lp_build_const_int32(gallivm, i) , "");
1103 ret = LLVMBuildOr(builder, ret, temp, "");
1104 }
1105
1106 LLVMBuildStore(builder, ret, ret_ptr);
1107 }
1108
1109 static void
1110 draw_llvm_generate(struct draw_llvm *llvm, struct draw_llvm_variant *variant)
1111 {
1112 struct gallivm_state *gallivm = llvm->gallivm;
1113 LLVMContextRef context = gallivm->context;
1114 LLVMTypeRef int32_type = LLVMInt32TypeInContext(context);
1115 LLVMTypeRef arg_types[8];
1116 LLVMTypeRef func_type;
1117 LLVMValueRef context_ptr;
1118 LLVMBasicBlockRef block;
1119 LLVMBuilderRef builder;
1120 LLVMValueRef start, end, count, stride, step, io_itr;
1121 LLVMValueRef io_ptr, vbuffers_ptr, vb_ptr;
1122 LLVMValueRef instance_id;
1123 LLVMValueRef system_values_array;
1124 struct draw_context *draw = llvm->draw;
1125 const struct tgsi_shader_info *vs_info = &draw->vs.vertex_shader->info;
1126 unsigned i, j;
1127 struct lp_build_context bld;
1128 struct lp_build_loop_state lp_loop;
1129 const int max_vertices = 4;
1130 LLVMValueRef outputs[PIPE_MAX_SHADER_OUTPUTS][NUM_CHANNELS];
1131 void *code;
1132 struct lp_build_sampler_soa *sampler = 0;
1133 LLVMValueRef ret, ret_ptr;
1134 boolean bypass_viewport = variant->key.bypass_viewport;
1135 boolean enable_cliptest = variant->key.clip_xy ||
1136 variant->key.clip_z ||
1137 variant->key.clip_user;
1138
1139 arg_types[0] = get_context_ptr_type(llvm); /* context */
1140 arg_types[1] = get_vertex_header_ptr_type(llvm); /* vertex_header */
1141 arg_types[2] = get_buffer_ptr_type(llvm); /* vbuffers */
1142 arg_types[3] = int32_type; /* start */
1143 arg_types[4] = int32_type; /* count */
1144 arg_types[5] = int32_type; /* stride */
1145 arg_types[6] = get_vb_ptr_type(llvm); /* pipe_vertex_buffer's */
1146 arg_types[7] = int32_type; /* instance_id */
1147
1148 func_type = LLVMFunctionType(int32_type, arg_types, Elements(arg_types), 0);
1149
1150 variant->function = LLVMAddFunction(gallivm->module, "draw_llvm_shader",
1151 func_type);
1152 LLVMSetFunctionCallConv(variant->function, LLVMCCallConv);
1153 for(i = 0; i < Elements(arg_types); ++i)
1154 if(LLVMGetTypeKind(arg_types[i]) == LLVMPointerTypeKind)
1155 LLVMAddAttribute(LLVMGetParam(variant->function, i), LLVMNoAliasAttribute);
1156
1157 context_ptr = LLVMGetParam(variant->function, 0);
1158 io_ptr = LLVMGetParam(variant->function, 1);
1159 vbuffers_ptr = LLVMGetParam(variant->function, 2);
1160 start = LLVMGetParam(variant->function, 3);
1161 count = LLVMGetParam(variant->function, 4);
1162 stride = LLVMGetParam(variant->function, 5);
1163 vb_ptr = LLVMGetParam(variant->function, 6);
1164 instance_id = LLVMGetParam(variant->function, 7);
1165
1166 lp_build_name(context_ptr, "context");
1167 lp_build_name(io_ptr, "io");
1168 lp_build_name(vbuffers_ptr, "vbuffers");
1169 lp_build_name(start, "start");
1170 lp_build_name(count, "count");
1171 lp_build_name(stride, "stride");
1172 lp_build_name(vb_ptr, "vb");
1173 lp_build_name(instance_id, "instance_id");
1174
1175 /*
1176 * Function body
1177 */
1178
1179 block = LLVMAppendBasicBlockInContext(gallivm->context, variant->function, "entry");
1180 builder = gallivm->builder;
1181 assert(builder);
1182 LLVMPositionBuilderAtEnd(builder, block);
1183
1184 lp_build_context_init(&bld, llvm->gallivm, lp_type_int(32));
1185
1186 system_values_array = lp_build_system_values_array(gallivm, vs_info,
1187 instance_id, NULL);
1188
1189 end = lp_build_add(&bld, start, count);
1190
1191 step = lp_build_const_int32(gallivm, max_vertices);
1192
1193 /* function will return non-zero i32 value if any clipped vertices */
1194 ret_ptr = lp_build_alloca(gallivm, int32_type, "");
1195 LLVMBuildStore(builder, lp_build_const_int32(gallivm, 0), ret_ptr);
1196
1197 /* code generated texture sampling */
1198 sampler = draw_llvm_sampler_soa_create(
1199 draw_llvm_variant_key_samplers(&variant->key),
1200 context_ptr);
1201
1202 #if DEBUG_STORE
1203 lp_build_printf(builder, "start = %d, end = %d, step = %d\n",
1204 start, end, step);
1205 #endif
1206 lp_build_loop_begin(&lp_loop, llvm->gallivm, start);
1207 {
1208 LLVMValueRef inputs[PIPE_MAX_SHADER_INPUTS][NUM_CHANNELS];
1209 LLVMValueRef aos_attribs[PIPE_MAX_SHADER_INPUTS][NUM_CHANNELS] = { { 0 } };
1210 LLVMValueRef io;
1211 LLVMValueRef clipmask; /* holds the clipmask value */
1212 const LLVMValueRef (*ptr_aos)[NUM_CHANNELS];
1213
1214 io_itr = LLVMBuildSub(builder, lp_loop.counter, start, "");
1215 io = LLVMBuildGEP(builder, io_ptr, &io_itr, 1, "");
1216 #if DEBUG_STORE
1217 lp_build_printf(builder, " --- io %d = %p, loop counter %d\n",
1218 io_itr, io, lp_loop.counter);
1219 #endif
1220 for (i = 0; i < NUM_CHANNELS; ++i) {
1221 LLVMValueRef true_index = LLVMBuildAdd(
1222 builder,
1223 lp_loop.counter,
1224 lp_build_const_int32(gallivm, i), "");
1225 for (j = 0; j < draw->pt.nr_vertex_elements; ++j) {
1226 struct pipe_vertex_element *velem = &draw->pt.vertex_element[j];
1227 LLVMValueRef vb_index = lp_build_const_int32(gallivm, velem->vertex_buffer_index);
1228 LLVMValueRef vb = LLVMBuildGEP(builder, vb_ptr,
1229 &vb_index, 1, "");
1230 generate_fetch(llvm->gallivm, vbuffers_ptr,
1231 &aos_attribs[j][i], velem, vb, true_index,
1232 instance_id);
1233 }
1234 }
1235 convert_to_soa(gallivm, aos_attribs, inputs,
1236 draw->pt.nr_vertex_elements);
1237
1238 ptr_aos = (const LLVMValueRef (*)[NUM_CHANNELS]) inputs;
1239 generate_vs(llvm,
1240 builder,
1241 outputs,
1242 ptr_aos,
1243 system_values_array,
1244 context_ptr,
1245 sampler);
1246
1247 /* store original positions in clip before further manipulation */
1248 store_clip(gallivm, io, outputs);
1249
1250 /* do cliptest */
1251 if (enable_cliptest){
1252 /* allocate clipmask, assign it integer type */
1253 clipmask = generate_clipmask(gallivm, outputs,
1254 variant->key.clip_xy,
1255 variant->key.clip_z,
1256 variant->key.clip_user,
1257 variant->key.clip_halfz,
1258 variant->key.nr_planes,
1259 context_ptr);
1260 /* return clipping boolean value for function */
1261 clipmask_bool(gallivm, clipmask, ret_ptr);
1262 }
1263 else{
1264 clipmask = lp_build_const_int_vec(gallivm, lp_type_int_vec(32), 0);
1265 }
1266
1267 /* do viewport mapping */
1268 if (!bypass_viewport){
1269 generate_viewport(llvm, builder, outputs, context_ptr);
1270 }
1271
1272 /* store clipmask in vertex header and positions in data */
1273 convert_to_aos(gallivm, io, outputs, clipmask,
1274 vs_info->num_outputs, max_vertices);
1275 }
1276
1277 lp_build_loop_end_cond(&lp_loop, end, step, LLVMIntUGE);
1278
1279 sampler->destroy(sampler);
1280
1281 ret = LLVMBuildLoad(builder, ret_ptr,"");
1282 LLVMBuildRet(builder, ret);
1283
1284 /*
1285 * Translate the LLVM IR into machine code.
1286 */
1287 #ifdef DEBUG
1288 if(LLVMVerifyFunction(variant->function, LLVMPrintMessageAction)) {
1289 lp_debug_dump_value(variant->function);
1290 assert(0);
1291 }
1292 #endif
1293
1294 LLVMRunFunctionPassManager(gallivm->passmgr, variant->function);
1295
1296 if (gallivm_debug & GALLIVM_DEBUG_IR) {
1297 lp_debug_dump_value(variant->function);
1298 debug_printf("\n");
1299 }
1300
1301 code = LLVMGetPointerToGlobal(gallivm->engine, variant->function);
1302 variant->jit_func = (draw_jit_vert_func)pointer_to_func(code);
1303
1304 if (gallivm_debug & GALLIVM_DEBUG_ASM) {
1305 lp_disassemble(code);
1306 }
1307 lp_func_delete_body(variant->function);
1308 }
1309
1310
1311 static void
1312 draw_llvm_generate_elts(struct draw_llvm *llvm, struct draw_llvm_variant *variant)
1313 {
1314 struct gallivm_state *gallivm = llvm->gallivm;
1315 LLVMContextRef context = gallivm->context;
1316 LLVMTypeRef int32_type = LLVMInt32TypeInContext(context);
1317 LLVMTypeRef arg_types[8];
1318 LLVMTypeRef func_type;
1319 LLVMValueRef context_ptr;
1320 LLVMBasicBlockRef block;
1321 LLVMBuilderRef builder;
1322 LLVMValueRef fetch_elts, fetch_count, stride, step, io_itr;
1323 LLVMValueRef io_ptr, vbuffers_ptr, vb_ptr;
1324 LLVMValueRef instance_id;
1325 LLVMValueRef system_values_array;
1326 struct draw_context *draw = llvm->draw;
1327 const struct tgsi_shader_info *vs_info = &draw->vs.vertex_shader->info;
1328 unsigned i, j;
1329 struct lp_build_context bld;
1330 struct lp_build_loop_state lp_loop;
1331 const int max_vertices = 4;
1332 LLVMValueRef outputs[PIPE_MAX_SHADER_OUTPUTS][NUM_CHANNELS];
1333 LLVMValueRef fetch_max;
1334 void *code;
1335 struct lp_build_sampler_soa *sampler = 0;
1336 LLVMValueRef ret, ret_ptr;
1337 boolean bypass_viewport = variant->key.bypass_viewport;
1338 boolean enable_cliptest = variant->key.clip_xy ||
1339 variant->key.clip_z ||
1340 variant->key.clip_user;
1341
1342 arg_types[0] = get_context_ptr_type(llvm); /* context */
1343 arg_types[1] = get_vertex_header_ptr_type(llvm); /* vertex_header */
1344 arg_types[2] = get_buffer_ptr_type(llvm); /* vbuffers */
1345 arg_types[3] = LLVMPointerType(int32_type, 0); /* fetch_elts * */
1346 arg_types[4] = int32_type; /* fetch_count */
1347 arg_types[5] = int32_type; /* stride */
1348 arg_types[6] = get_vb_ptr_type(llvm); /* pipe_vertex_buffer's */
1349 arg_types[7] = int32_type; /* instance_id */
1350
1351 func_type = LLVMFunctionType(int32_type, arg_types, Elements(arg_types), 0);
1352
1353 variant->function_elts = LLVMAddFunction(gallivm->module, "draw_llvm_shader_elts", func_type);
1354 LLVMSetFunctionCallConv(variant->function_elts, LLVMCCallConv);
1355 for(i = 0; i < Elements(arg_types); ++i)
1356 if(LLVMGetTypeKind(arg_types[i]) == LLVMPointerTypeKind)
1357 LLVMAddAttribute(LLVMGetParam(variant->function_elts, i),
1358 LLVMNoAliasAttribute);
1359
1360 context_ptr = LLVMGetParam(variant->function_elts, 0);
1361 io_ptr = LLVMGetParam(variant->function_elts, 1);
1362 vbuffers_ptr = LLVMGetParam(variant->function_elts, 2);
1363 fetch_elts = LLVMGetParam(variant->function_elts, 3);
1364 fetch_count = LLVMGetParam(variant->function_elts, 4);
1365 stride = LLVMGetParam(variant->function_elts, 5);
1366 vb_ptr = LLVMGetParam(variant->function_elts, 6);
1367 instance_id = LLVMGetParam(variant->function_elts, 7);
1368
1369 lp_build_name(context_ptr, "context");
1370 lp_build_name(io_ptr, "io");
1371 lp_build_name(vbuffers_ptr, "vbuffers");
1372 lp_build_name(fetch_elts, "fetch_elts");
1373 lp_build_name(fetch_count, "fetch_count");
1374 lp_build_name(stride, "stride");
1375 lp_build_name(vb_ptr, "vb");
1376 lp_build_name(instance_id, "instance_id");
1377
1378 /*
1379 * Function body
1380 */
1381
1382 block = LLVMAppendBasicBlockInContext(gallivm->context, variant->function_elts, "entry");
1383 builder = gallivm->builder;
1384 LLVMPositionBuilderAtEnd(builder, block);
1385
1386 lp_build_context_init(&bld, gallivm, lp_type_int(32));
1387
1388 system_values_array = lp_build_system_values_array(gallivm, vs_info,
1389 instance_id, NULL);
1390
1391
1392 step = lp_build_const_int32(gallivm, max_vertices);
1393
1394 /* code generated texture sampling */
1395 sampler = draw_llvm_sampler_soa_create(
1396 draw_llvm_variant_key_samplers(&variant->key),
1397 context_ptr);
1398
1399 fetch_max = LLVMBuildSub(builder, fetch_count,
1400 lp_build_const_int32(gallivm, 1),
1401 "fetch_max");
1402
1403 /* function returns non-zero i32 value if any clipped vertices */
1404 ret_ptr = lp_build_alloca(gallivm, int32_type, "");
1405 LLVMBuildStore(builder, lp_build_const_int32(gallivm, 0), ret_ptr);
1406
1407 lp_build_loop_begin(&lp_loop, gallivm, lp_build_const_int32(gallivm, 0));
1408 {
1409 LLVMValueRef inputs[PIPE_MAX_SHADER_INPUTS][NUM_CHANNELS];
1410 LLVMValueRef aos_attribs[PIPE_MAX_SHADER_INPUTS][NUM_CHANNELS] = { { 0 } };
1411 LLVMValueRef io;
1412 LLVMValueRef clipmask; /* holds the clipmask value */
1413 const LLVMValueRef (*ptr_aos)[NUM_CHANNELS];
1414
1415 io_itr = lp_loop.counter;
1416 io = LLVMBuildGEP(builder, io_ptr, &io_itr, 1, "");
1417 #if DEBUG_STORE
1418 lp_build_printf(builder, " --- io %d = %p, loop counter %d\n",
1419 io_itr, io, lp_loop.counter);
1420 #endif
1421 for (i = 0; i < NUM_CHANNELS; ++i) {
1422 LLVMValueRef true_index = LLVMBuildAdd(
1423 builder,
1424 lp_loop.counter,
1425 lp_build_const_int32(gallivm, i), "");
1426 LLVMValueRef fetch_ptr;
1427
1428 /* make sure we're not out of bounds which can happen
1429 * if fetch_count % 4 != 0, because on the last iteration
1430 * a few of the 4 vertex fetches will be out of bounds */
1431 true_index = lp_build_min(&bld, true_index, fetch_max);
1432
1433 fetch_ptr = LLVMBuildGEP(builder, fetch_elts,
1434 &true_index, 1, "");
1435 true_index = LLVMBuildLoad(builder, fetch_ptr, "fetch_elt");
1436 for (j = 0; j < draw->pt.nr_vertex_elements; ++j) {
1437 struct pipe_vertex_element *velem = &draw->pt.vertex_element[j];
1438 LLVMValueRef vb_index = lp_build_const_int32(gallivm, velem->vertex_buffer_index);
1439 LLVMValueRef vb = LLVMBuildGEP(builder, vb_ptr,
1440 &vb_index, 1, "");
1441 generate_fetch(gallivm, vbuffers_ptr,
1442 &aos_attribs[j][i], velem, vb, true_index,
1443 instance_id);
1444 }
1445 }
1446 convert_to_soa(gallivm, aos_attribs, inputs,
1447 draw->pt.nr_vertex_elements);
1448
1449 ptr_aos = (const LLVMValueRef (*)[NUM_CHANNELS]) inputs;
1450 generate_vs(llvm,
1451 builder,
1452 outputs,
1453 ptr_aos,
1454 system_values_array,
1455 context_ptr,
1456 sampler);
1457
1458 /* store original positions in clip before further manipulation */
1459 store_clip(gallivm, io, outputs);
1460
1461 /* do cliptest */
1462 if (enable_cliptest){
1463 /* allocate clipmask, assign it integer type */
1464 clipmask = generate_clipmask(gallivm, outputs,
1465 variant->key.clip_xy,
1466 variant->key.clip_z,
1467 variant->key.clip_user,
1468 variant->key.clip_halfz,
1469 variant->key.nr_planes,
1470 context_ptr);
1471 /* return clipping boolean value for function */
1472 clipmask_bool(gallivm, clipmask, ret_ptr);
1473 }
1474 else{
1475 clipmask = lp_build_const_int_vec(gallivm, lp_type_int_vec(32), 0);
1476 }
1477
1478 /* do viewport mapping */
1479 if (!bypass_viewport){
1480 generate_viewport(llvm, builder, outputs, context_ptr);
1481 }
1482
1483 /* store clipmask in vertex header,
1484 * original positions in clip
1485 * and transformed positions in data
1486 */
1487 convert_to_aos(gallivm, io, outputs, clipmask,
1488 vs_info->num_outputs, max_vertices);
1489 }
1490
1491 lp_build_loop_end_cond(&lp_loop, fetch_count, step, LLVMIntUGE);
1492
1493 sampler->destroy(sampler);
1494
1495 ret = LLVMBuildLoad(builder, ret_ptr,"");
1496 LLVMBuildRet(builder, ret);
1497
1498 /*
1499 * Translate the LLVM IR into machine code.
1500 */
1501 #ifdef DEBUG
1502 if(LLVMVerifyFunction(variant->function_elts, LLVMPrintMessageAction)) {
1503 lp_debug_dump_value(variant->function_elts);
1504 assert(0);
1505 }
1506 #endif
1507
1508 LLVMRunFunctionPassManager(gallivm->passmgr, variant->function_elts);
1509
1510 if (gallivm_debug & GALLIVM_DEBUG_IR) {
1511 lp_debug_dump_value(variant->function_elts);
1512 debug_printf("\n");
1513 }
1514
1515 code = LLVMGetPointerToGlobal(gallivm->engine, variant->function_elts);
1516 variant->jit_func_elts = (draw_jit_vert_func_elts)pointer_to_func(code);
1517
1518 if (gallivm_debug & GALLIVM_DEBUG_ASM) {
1519 lp_disassemble(code);
1520 }
1521 lp_func_delete_body(variant->function_elts);
1522 }
1523
1524
1525 struct draw_llvm_variant_key *
1526 draw_llvm_make_variant_key(struct draw_llvm *llvm, char *store)
1527 {
1528 unsigned i;
1529 struct draw_llvm_variant_key *key;
1530 struct lp_sampler_static_state *sampler;
1531
1532 key = (struct draw_llvm_variant_key *)store;
1533
1534 /* Presumably all variants of the shader should have the same
1535 * number of vertex elements - ie the number of shader inputs.
1536 */
1537 key->nr_vertex_elements = llvm->draw->pt.nr_vertex_elements;
1538
1539 /* will have to rig this up properly later */
1540 key->clip_xy = llvm->draw->clip_xy;
1541 key->clip_z = llvm->draw->clip_z;
1542 key->clip_user = llvm->draw->clip_user;
1543 key->bypass_viewport = llvm->draw->identity_viewport;
1544 key->clip_halfz = !llvm->draw->rasterizer->gl_rasterization_rules;
1545 key->need_edgeflags = (llvm->draw->vs.edgeflag_output ? TRUE : FALSE);
1546 key->nr_planes = llvm->draw->nr_planes;
1547 key->pad = 0;
1548
1549 /* All variants of this shader will have the same value for
1550 * nr_samplers. Not yet trying to compact away holes in the
1551 * sampler array.
1552 */
1553 key->nr_samplers = llvm->draw->vs.vertex_shader->info.file_max[TGSI_FILE_SAMPLER] + 1;
1554
1555 sampler = draw_llvm_variant_key_samplers(key);
1556
1557 memcpy(key->vertex_element,
1558 llvm->draw->pt.vertex_element,
1559 sizeof(struct pipe_vertex_element) * key->nr_vertex_elements);
1560
1561 memset(sampler, 0, key->nr_samplers * sizeof *sampler);
1562
1563 for (i = 0 ; i < key->nr_samplers; i++) {
1564 lp_sampler_static_state(&sampler[i],
1565 llvm->draw->sampler_views[i],
1566 llvm->draw->samplers[i]);
1567 }
1568
1569 return key;
1570 }
1571
1572 void
1573 draw_llvm_set_mapped_texture(struct draw_context *draw,
1574 unsigned sampler_idx,
1575 uint32_t width, uint32_t height, uint32_t depth,
1576 uint32_t last_level,
1577 uint32_t row_stride[PIPE_MAX_TEXTURE_LEVELS],
1578 uint32_t img_stride[PIPE_MAX_TEXTURE_LEVELS],
1579 const void *data[PIPE_MAX_TEXTURE_LEVELS])
1580 {
1581 unsigned j;
1582 struct draw_jit_texture *jit_tex;
1583
1584 assert(sampler_idx < PIPE_MAX_VERTEX_SAMPLERS);
1585
1586
1587 jit_tex = &draw->llvm->jit_context.textures[sampler_idx];
1588
1589 jit_tex->width = width;
1590 jit_tex->height = height;
1591 jit_tex->depth = depth;
1592 jit_tex->last_level = last_level;
1593
1594 for (j = 0; j <= last_level; j++) {
1595 jit_tex->data[j] = data[j];
1596 jit_tex->row_stride[j] = row_stride[j];
1597 jit_tex->img_stride[j] = img_stride[j];
1598 }
1599 }
1600
1601
1602 void
1603 draw_llvm_set_sampler_state(struct draw_context *draw)
1604 {
1605 unsigned i;
1606
1607 for (i = 0; i < draw->num_samplers; i++) {
1608 struct draw_jit_texture *jit_tex = &draw->llvm->jit_context.textures[i];
1609
1610 if (draw->samplers[i]) {
1611 jit_tex->min_lod = draw->samplers[i]->min_lod;
1612 jit_tex->max_lod = draw->samplers[i]->max_lod;
1613 jit_tex->lod_bias = draw->samplers[i]->lod_bias;
1614 COPY_4V(jit_tex->border_color, draw->samplers[i]->border_color);
1615 }
1616 }
1617 }
1618
1619
1620 void
1621 draw_llvm_destroy_variant(struct draw_llvm_variant *variant)
1622 {
1623 struct draw_llvm *llvm = variant->llvm;
1624
1625 if (variant->function_elts) {
1626 LLVMFreeMachineCodeForFunction(llvm->gallivm->engine,
1627 variant->function_elts);
1628 LLVMDeleteFunction(variant->function_elts);
1629 }
1630
1631 if (variant->function) {
1632 LLVMFreeMachineCodeForFunction(llvm->gallivm->engine,
1633 variant->function);
1634 LLVMDeleteFunction(variant->function);
1635 }
1636
1637 remove_from_list(&variant->list_item_local);
1638 variant->shader->variants_cached--;
1639 remove_from_list(&variant->list_item_global);
1640 llvm->nr_variants--;
1641 FREE(variant);
1642 }