projects / mesa.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
history | raw | HEAD
draw: make sure key size is calculated consistently.
[mesa.git] / src / gallium / auxiliary / draw / draw_llvm.c
1 /**************************************************************************
2 *
3 * Copyright 2010 VMware, Inc.
4 * All Rights Reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sub license, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
13 *
14 * The above copyright notice and this permission notice (including the
15 * next paragraph) shall be included in all copies or substantial portions
16 * of the Software.
17 *
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
19 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
20 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
21 * IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR
22 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
23 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
24 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
25 *
26 **************************************************************************/
27
28 #include "draw_llvm.h"
29
30 #include "draw_context.h"
31 #include "draw_vs.h"
32
33 #include "gallivm/lp_bld_arit.h"
34 #include "gallivm/lp_bld_logic.h"
35 #include "gallivm/lp_bld_const.h"
36 #include "gallivm/lp_bld_swizzle.h"
37 #include "gallivm/lp_bld_struct.h"
38 #include "gallivm/lp_bld_type.h"
39 #include "gallivm/lp_bld_flow.h"
40 #include "gallivm/lp_bld_debug.h"
41 #include "gallivm/lp_bld_tgsi.h"
42 #include "gallivm/lp_bld_printf.h"
43 #include "gallivm/lp_bld_intr.h"
44 #include "gallivm/lp_bld_init.h"
45 #include "gallivm/lp_bld_type.h"
46 #include "gallivm/lp_bld_pack.h"
47 #include "gallivm/lp_bld_format.h"
48
49 #include "tgsi/tgsi_exec.h"
50 #include "tgsi/tgsi_dump.h"
51
52 #include "util/u_math.h"
53 #include "util/u_pointer.h"
54 #include "util/u_string.h"
55 #include "util/u_simple_list.h"
56
57
58 #define DEBUG_STORE 0
59
60
61 static void
62 draw_llvm_generate(struct draw_llvm *llvm, struct draw_llvm_variant *var,
63 boolean elts);
64
65
66 /**
67 * Create LLVM type for struct draw_jit_texture
68 */
69 static LLVMTypeRef
70 create_jit_texture_type(struct gallivm_state *gallivm, const char *struct_name)
71 {
72 LLVMTargetDataRef target = gallivm->target;
73 LLVMTypeRef texture_type;
74 LLVMTypeRef elem_types[DRAW_JIT_TEXTURE_NUM_FIELDS];
75 LLVMTypeRef int32_type = LLVMInt32TypeInContext(gallivm->context);
76
77 elem_types[DRAW_JIT_TEXTURE_WIDTH] =
78 elem_types[DRAW_JIT_TEXTURE_HEIGHT] =
79 elem_types[DRAW_JIT_TEXTURE_DEPTH] =
80 elem_types[DRAW_JIT_TEXTURE_FIRST_LEVEL] =
81 elem_types[DRAW_JIT_TEXTURE_LAST_LEVEL] = int32_type;
82 elem_types[DRAW_JIT_TEXTURE_BASE] =
83 LLVMPointerType(LLVMInt8TypeInContext(gallivm->context), 0);
84 elem_types[DRAW_JIT_TEXTURE_ROW_STRIDE] =
85 elem_types[DRAW_JIT_TEXTURE_IMG_STRIDE] =
86 elem_types[DRAW_JIT_TEXTURE_MIP_OFFSETS] =
87 LLVMArrayType(int32_type, PIPE_MAX_TEXTURE_LEVELS);
88
89 texture_type = LLVMStructTypeInContext(gallivm->context, elem_types,
90 Elements(elem_types), 0);
91
92 #if HAVE_LLVM < 0x0300
93 LLVMAddTypeName(gallivm->module, struct_name, texture_type);
94
95 /* Make sure the target's struct layout cache doesn't return
96 * stale/invalid data.
97 */
98 LLVMInvalidateStructLayout(gallivm->target, texture_type);
99 #endif
100
101 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, width,
102 target, texture_type,
103 DRAW_JIT_TEXTURE_WIDTH);
104 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, height,
105 target, texture_type,
106 DRAW_JIT_TEXTURE_HEIGHT);
107 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, depth,
108 target, texture_type,
109 DRAW_JIT_TEXTURE_DEPTH);
110 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, first_level,
111 target, texture_type,
112 DRAW_JIT_TEXTURE_FIRST_LEVEL);
113 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, last_level,
114 target, texture_type,
115 DRAW_JIT_TEXTURE_LAST_LEVEL);
116 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, base,
117 target, texture_type,
118 DRAW_JIT_TEXTURE_BASE);
119 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, row_stride,
120 target, texture_type,
121 DRAW_JIT_TEXTURE_ROW_STRIDE);
122 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, img_stride,
123 target, texture_type,
124 DRAW_JIT_TEXTURE_IMG_STRIDE);
125 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, mip_offsets,
126 target, texture_type,
127 DRAW_JIT_TEXTURE_MIP_OFFSETS);
128
129 LP_CHECK_STRUCT_SIZE(struct draw_jit_texture, target, texture_type);
130
131 return texture_type;
132 }
133
134
135 /**
136 * Create LLVM type for struct draw_jit_sampler
137 */
138 static LLVMTypeRef
139 create_jit_sampler_type(struct gallivm_state *gallivm, const char *struct_name)
140 {
141 LLVMTargetDataRef target = gallivm->target;
142 LLVMTypeRef sampler_type;
143 LLVMTypeRef elem_types[DRAW_JIT_SAMPLER_NUM_FIELDS];
144
145 elem_types[DRAW_JIT_SAMPLER_MIN_LOD] =
146 elem_types[DRAW_JIT_SAMPLER_MAX_LOD] =
147 elem_types[DRAW_JIT_SAMPLER_LOD_BIAS] = LLVMFloatTypeInContext(gallivm->context);
148 elem_types[DRAW_JIT_SAMPLER_BORDER_COLOR] =
149 LLVMArrayType(LLVMFloatTypeInContext(gallivm->context), 4);
150
151 sampler_type = LLVMStructTypeInContext(gallivm->context, elem_types,
152 Elements(elem_types), 0);
153
154 #if HAVE_LLVM < 0x0300
155 LLVMAddTypeName(gallivm->module, struct_name, sampler_type);
156
157 /* Make sure the target's struct layout cache doesn't return
158 * stale/invalid data.
159 */
160 LLVMInvalidateStructLayout(gallivm->target, sampler_type);
161 #endif
162
163 LP_CHECK_MEMBER_OFFSET(struct draw_jit_sampler, min_lod,
164 target, sampler_type,
165 DRAW_JIT_SAMPLER_MIN_LOD);
166 LP_CHECK_MEMBER_OFFSET(struct draw_jit_sampler, max_lod,
167 target, sampler_type,
168 DRAW_JIT_SAMPLER_MAX_LOD);
169 LP_CHECK_MEMBER_OFFSET(struct draw_jit_sampler, lod_bias,
170 target, sampler_type,
171 DRAW_JIT_SAMPLER_LOD_BIAS);
172 LP_CHECK_MEMBER_OFFSET(struct draw_jit_sampler, border_color,
173 target, sampler_type,
174 DRAW_JIT_SAMPLER_BORDER_COLOR);
175
176 LP_CHECK_STRUCT_SIZE(struct draw_jit_sampler, target, sampler_type);
177
178 return sampler_type;
179 }
180
181
182 /**
183 * Create LLVM type for struct draw_jit_context
184 */
185 static LLVMTypeRef
186 create_jit_context_type(struct gallivm_state *gallivm,
187 LLVMTypeRef texture_type, LLVMTypeRef sampler_type,
188 const char *struct_name)
189 {
190 LLVMTargetDataRef target = gallivm->target;
191 LLVMTypeRef float_type = LLVMFloatTypeInContext(gallivm->context);
192 LLVMTypeRef elem_types[6];
193 LLVMTypeRef context_type;
194
195 elem_types[0] = LLVMArrayType(LLVMPointerType(float_type, 0), /* vs_constants */
196 LP_MAX_TGSI_CONST_BUFFERS);
197 elem_types[1] = elem_types[0]; /* gs_constants */
198 elem_types[2] = LLVMPointerType(LLVMArrayType(LLVMArrayType(float_type, 4),
199 DRAW_TOTAL_CLIP_PLANES), 0);
200 elem_types[3] = LLVMPointerType(float_type, 0); /* viewport */
201 elem_types[4] = LLVMArrayType(texture_type,
202 PIPE_MAX_SHADER_SAMPLER_VIEWS); /* textures */
203 elem_types[5] = LLVMArrayType(sampler_type,
204 PIPE_MAX_SAMPLERS); /* samplers */
205 context_type = LLVMStructTypeInContext(gallivm->context, elem_types,
206 Elements(elem_types), 0);
207 #if HAVE_LLVM < 0x0300
208 LLVMAddTypeName(gallivm->module, struct_name, context_type);
209
210 LLVMInvalidateStructLayout(gallivm->target, context_type);
211 #endif
212
213 LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, vs_constants,
214 target, context_type, 0);
215 LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, gs_constants,
216 target, context_type, 1);
217 LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, planes,
218 target, context_type, 2);
219 LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, viewport,
220 target, context_type, 3);
221 LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, textures,
222 target, context_type,
223 DRAW_JIT_CTX_TEXTURES);
224 LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, samplers,
225 target, context_type,
226 DRAW_JIT_CTX_SAMPLERS);
227 LP_CHECK_STRUCT_SIZE(struct draw_jit_context,
228 target, context_type);
229
230 return context_type;
231 }
232
233
234 /**
235 * Create LLVM type for struct pipe_vertex_buffer
236 */
237 static LLVMTypeRef
238 create_jit_vertex_buffer_type(struct gallivm_state *gallivm, const char *struct_name)
239 {
240 LLVMTargetDataRef target = gallivm->target;
241 LLVMTypeRef elem_types[4];
242 LLVMTypeRef vb_type;
243
244 elem_types[0] =
245 elem_types[1] = LLVMInt32TypeInContext(gallivm->context);
246 elem_types[2] =
247 elem_types[3] = LLVMPointerType(LLVMInt8TypeInContext(gallivm->context), 0); /* vs_constants */
248
249 vb_type = LLVMStructTypeInContext(gallivm->context, elem_types,
250 Elements(elem_types), 0);
251 #if HAVE_LLVM < 0x0300
252 LLVMAddTypeName(gallivm->module, struct_name, vb_type);
253
254 LLVMInvalidateStructLayout(gallivm->target, vb_type);
255 #endif
256
257 LP_CHECK_MEMBER_OFFSET(struct pipe_vertex_buffer, stride,
258 target, vb_type, 0);
259 LP_CHECK_MEMBER_OFFSET(struct pipe_vertex_buffer, buffer_offset,
260 target, vb_type, 1);
261
262 LP_CHECK_STRUCT_SIZE(struct pipe_vertex_buffer, target, vb_type);
263
264 return vb_type;
265 }
266
267
268 /**
269 * Create LLVM type for struct vertex_header;
270 */
271 static LLVMTypeRef
272 create_jit_vertex_header(struct gallivm_state *gallivm, int data_elems)
273 {
274 LLVMTargetDataRef target = gallivm->target;
275 LLVMTypeRef elem_types[4];
276 LLVMTypeRef vertex_header;
277 char struct_name[24];
278
279 util_snprintf(struct_name, 23, "vertex_header%d", data_elems);
280
281 elem_types[DRAW_JIT_VERTEX_VERTEX_ID] = LLVMIntTypeInContext(gallivm->context, 32);
282 elem_types[DRAW_JIT_VERTEX_CLIP] = LLVMArrayType(LLVMFloatTypeInContext(gallivm->context), 4);
283 elem_types[DRAW_JIT_VERTEX_PRE_CLIP_POS] = LLVMArrayType(LLVMFloatTypeInContext(gallivm->context), 4);
284 elem_types[DRAW_JIT_VERTEX_DATA] = LLVMArrayType(elem_types[1], data_elems);
285
286 vertex_header = LLVMStructTypeInContext(gallivm->context, elem_types,
287 Elements(elem_types), 0);
288 #if HAVE_LLVM < 0x0300
289 LLVMAddTypeName(gallivm->module, struct_name, vertex_header);
290
291 LLVMInvalidateStructLayout(gallivm->target, vertex_header);
292 #endif
293
294 /* these are bit-fields and we can't take address of them
295 LP_CHECK_MEMBER_OFFSET(struct vertex_header, clipmask,
296 target, vertex_header,
297 DRAW_JIT_VERTEX_CLIPMASK);
298 LP_CHECK_MEMBER_OFFSET(struct vertex_header, edgeflag,
299 target, vertex_header,
300 DRAW_JIT_VERTEX_EDGEFLAG);
301 LP_CHECK_MEMBER_OFFSET(struct vertex_header, pad,
302 target, vertex_header,
303 DRAW_JIT_VERTEX_PAD);
304 LP_CHECK_MEMBER_OFFSET(struct vertex_header, vertex_id,
305 target, vertex_header,
306 DRAW_JIT_VERTEX_VERTEX_ID);
307 */
308 LP_CHECK_MEMBER_OFFSET(struct vertex_header, clip,
309 target, vertex_header,
310 DRAW_JIT_VERTEX_CLIP);
311 LP_CHECK_MEMBER_OFFSET(struct vertex_header, pre_clip_pos,
312 target, vertex_header,
313 DRAW_JIT_VERTEX_PRE_CLIP_POS);
314 LP_CHECK_MEMBER_OFFSET(struct vertex_header, data,
315 target, vertex_header,
316 DRAW_JIT_VERTEX_DATA);
317
318 assert(LLVMABISizeOfType(target, vertex_header) ==
319 offsetof(struct vertex_header, data[data_elems]));
320
321 return vertex_header;
322 }
323
324
325 /**
326 * Create LLVM types for various structures.
327 */
328 static void
329 create_jit_types(struct draw_llvm_variant *variant)
330 {
331 struct gallivm_state *gallivm = variant->gallivm;
332 LLVMTypeRef texture_type, sampler_type, context_type, buffer_type, vb_type;
333
334 texture_type = create_jit_texture_type(gallivm, "texture");
335 sampler_type = create_jit_sampler_type(gallivm, "sampler");
336
337 context_type = create_jit_context_type(gallivm, texture_type, sampler_type,
338 "draw_jit_context");
339 variant->context_ptr_type = LLVMPointerType(context_type, 0);
340
341 buffer_type = LLVMPointerType(LLVMIntTypeInContext(gallivm->context, 8), 0);
342 variant->buffer_ptr_type = LLVMPointerType(buffer_type, 0);
343
344 vb_type = create_jit_vertex_buffer_type(gallivm, "pipe_vertex_buffer");
345 variant->vb_ptr_type = LLVMPointerType(vb_type, 0);
346 }
347
348
349 static LLVMTypeRef
350 get_context_ptr_type(struct draw_llvm_variant *variant)
351 {
352 if (!variant->context_ptr_type)
353 create_jit_types(variant);
354 return variant->context_ptr_type;
355 }
356
357
358 static LLVMTypeRef
359 get_buffer_ptr_type(struct draw_llvm_variant *variant)
360 {
361 if (!variant->buffer_ptr_type)
362 create_jit_types(variant);
363 return variant->buffer_ptr_type;
364 }
365
366
367 static LLVMTypeRef
368 get_vb_ptr_type(struct draw_llvm_variant *variant)
369 {
370 if (!variant->vb_ptr_type)
371 create_jit_types(variant);
372 return variant->vb_ptr_type;
373 }
374
375 static LLVMTypeRef
376 get_vertex_header_ptr_type(struct draw_llvm_variant *variant)
377 {
378 if (!variant->vertex_header_ptr_type)
379 create_jit_types(variant);
380 return variant->vertex_header_ptr_type;
381 }
382
383
384 /**
385 * Create per-context LLVM info.
386 */
387 struct draw_llvm *
388 draw_llvm_create(struct draw_context *draw)
389 {
390 struct draw_llvm *llvm;
391
392 llvm = CALLOC_STRUCT( draw_llvm );
393 if (!llvm)
394 return NULL;
395
396 lp_build_init();
397
398 llvm->draw = draw;
399
400 llvm->nr_variants = 0;
401 make_empty_list(&llvm->vs_variants_list);
402
403 return llvm;
404 }
405
406
407 /**
408 * Free per-context LLVM info.
409 */
410 void
411 draw_llvm_destroy(struct draw_llvm *llvm)
412 {
413 /* XXX free other draw_llvm data? */
414 FREE(llvm);
415 }
416
417
418 /**
419 * Create LLVM-generated code for a vertex shader.
420 */
421 struct draw_llvm_variant *
422 draw_llvm_create_variant(struct draw_llvm *llvm,
423 unsigned num_inputs,
424 const struct draw_llvm_variant_key *key)
425 {
426 struct draw_llvm_variant *variant;
427 struct llvm_vertex_shader *shader =
428 llvm_vertex_shader(llvm->draw->vs.vertex_shader);
429 LLVMTypeRef vertex_header;
430
431 variant = MALLOC(sizeof *variant +
432 shader->variant_key_size -
433 sizeof variant->key);
434 if (variant == NULL)
435 return NULL;
436
437 variant->llvm = llvm;
438
439 variant->gallivm = gallivm_create();
440
441 create_jit_types(variant);
442
443 memcpy(&variant->key, key, shader->variant_key_size);
444
445 vertex_header = create_jit_vertex_header(variant->gallivm, num_inputs);
446
447 variant->vertex_header_ptr_type = LLVMPointerType(vertex_header, 0);
448
449 draw_llvm_generate(llvm, variant, FALSE); /* linear */
450 draw_llvm_generate(llvm, variant, TRUE); /* elts */
451
452 gallivm_compile_module(variant->gallivm);
453
454 variant->jit_func = (draw_jit_vert_func)
455 gallivm_jit_function(variant->gallivm, variant->function);
456
457 variant->jit_func_elts = (draw_jit_vert_func_elts)
458 gallivm_jit_function(variant->gallivm, variant->function_elts);
459
460 variant->shader = shader;
461 variant->list_item_global.base = variant;
462 variant->list_item_local.base = variant;
463 /*variant->no = */shader->variants_created++;
464 variant->list_item_global.base = variant;
465
466 return variant;
467 }
468
469
470 static void
471 generate_vs(struct draw_llvm_variant *variant,
472 LLVMBuilderRef builder,
473 struct lp_type vs_type,
474 LLVMValueRef (*outputs)[TGSI_NUM_CHANNELS],
475 const LLVMValueRef (*inputs)[TGSI_NUM_CHANNELS],
476 const struct lp_bld_tgsi_system_values *system_values,
477 LLVMValueRef context_ptr,
478 struct lp_build_sampler_soa *draw_sampler,
479 boolean clamp_vertex_color)
480 {
481 struct draw_llvm *llvm = variant->llvm;
482 const struct tgsi_token *tokens = llvm->draw->vs.vertex_shader->state.tokens;
483 LLVMValueRef consts_ptr = draw_jit_context_vs_constants(variant->gallivm, context_ptr);
484 struct lp_build_sampler_soa *sampler = 0;
485
486 if (gallivm_debug & (GALLIVM_DEBUG_TGSI | GALLIVM_DEBUG_IR)) {
487 tgsi_dump(tokens, 0);
488 draw_llvm_dump_variant_key(&variant->key);
489 }
490
491 if (llvm->draw->num_sampler_views && llvm->draw->num_samplers)
492 sampler = draw_sampler;
493
494 lp_build_tgsi_soa(variant->gallivm,
495 tokens,
496 vs_type,
497 NULL /*struct lp_build_mask_context *mask*/,
498 consts_ptr,
499 system_values,
500 NULL /*pos*/,
501 inputs,
502 outputs,
503 sampler,
504 &llvm->draw->vs.vertex_shader->info);
505
506 {
507 LLVMValueRef out;
508 unsigned chan, attrib;
509 struct lp_build_context bld;
510 struct tgsi_shader_info* info = &llvm->draw->vs.vertex_shader->info;
511 lp_build_context_init(&bld, variant->gallivm, vs_type);
512
513 for (attrib = 0; attrib < info->num_outputs; ++attrib) {
514 for (chan = 0; chan < TGSI_NUM_CHANNELS; ++chan) {
515 if (outputs[attrib][chan]) {
516 switch (info->output_semantic_name[attrib]) {
517 case TGSI_SEMANTIC_COLOR:
518 case TGSI_SEMANTIC_BCOLOR:
519 if (clamp_vertex_color) {
520 out = LLVMBuildLoad(builder, outputs[attrib][chan], "");
521 out = lp_build_clamp(&bld, out, bld.zero, bld.one);
522 LLVMBuildStore(builder, out, outputs[attrib][chan]);
523 }
524 break;
525 case TGSI_SEMANTIC_FOG:
526 if (chan == 1 || chan == 2)
527 LLVMBuildStore(builder, bld.zero, outputs[attrib][chan]);
528 else if (chan == 3)
529 LLVMBuildStore(builder, bld.one, outputs[attrib][chan]);
530 break;
531 }
532 }
533 }
534 }
535 }
536 }
537
538
539 static void
540 generate_fetch(struct gallivm_state *gallivm,
541 LLVMValueRef vbuffers_ptr,
542 LLVMValueRef *res,
543 struct pipe_vertex_element *velem,
544 LLVMValueRef vbuf,
545 LLVMValueRef index,
546 LLVMValueRef instance_id)
547 {
548 const struct util_format_description *format_desc = util_format_description(velem->src_format);
549 LLVMValueRef zero = LLVMConstNull(LLVMInt32TypeInContext(gallivm->context));
550 LLVMBuilderRef builder = gallivm->builder;
551 LLVMValueRef indices =
552 LLVMConstInt(LLVMInt64TypeInContext(gallivm->context),
553 velem->vertex_buffer_index, 0);
554 LLVMValueRef vbuffer_ptr = LLVMBuildGEP(builder, vbuffers_ptr,
555 &indices, 1, "");
556 LLVMValueRef vb_stride = draw_jit_vbuffer_stride(gallivm, vbuf);
557 LLVMValueRef vb_buffer_offset = draw_jit_vbuffer_offset(gallivm, vbuf);
558 LLVMValueRef stride;
559
560 if (velem->instance_divisor) {
561 /* array index = instance_id / instance_divisor */
562 index = LLVMBuildUDiv(builder, instance_id,
563 lp_build_const_int32(gallivm, velem->instance_divisor),
564 "instance_divisor");
565 }
566
567 stride = LLVMBuildMul(builder, vb_stride, index, "");
568
569 vbuffer_ptr = LLVMBuildLoad(builder, vbuffer_ptr, "vbuffer");
570
571 stride = LLVMBuildAdd(builder, stride,
572 vb_buffer_offset,
573 "");
574 stride = LLVMBuildAdd(builder, stride,
575 lp_build_const_int32(gallivm, velem->src_offset),
576 "");
577
578 /* lp_build_printf(gallivm, "vbuf index = %d, stride is %d\n", indices, stride);*/
579 vbuffer_ptr = LLVMBuildGEP(builder, vbuffer_ptr, &stride, 1, "");
580
581 *res = lp_build_fetch_rgba_aos(gallivm,
582 format_desc,
583 lp_float32_vec4_type(),
584 vbuffer_ptr,
585 zero, zero, zero);
586 }
587
588 static void
589 convert_to_soa(struct gallivm_state *gallivm,
590 LLVMValueRef (*src_aos)[LP_MAX_VECTOR_WIDTH / 32],
591 LLVMValueRef (*dst_soa)[TGSI_NUM_CHANNELS],
592 unsigned num_attribs, const struct lp_type soa_type)
593 {
594 unsigned i, j, k;
595 struct lp_type aos_channel_type = soa_type;
596
597 debug_assert(TGSI_NUM_CHANNELS == 4);
598 debug_assert((soa_type.length % TGSI_NUM_CHANNELS) == 0);
599
600 aos_channel_type.length >>= 1;
601
602 for (i = 0; i < num_attribs; ++i) {
603 LLVMValueRef aos_channels[TGSI_NUM_CHANNELS];
604 unsigned pixels_per_channel = soa_type.length / TGSI_NUM_CHANNELS;
605
606 for (j = 0; j < TGSI_NUM_CHANNELS; ++j) {
607 LLVMValueRef channel[LP_MAX_VECTOR_LENGTH] = { 0 };
608
609 assert(pixels_per_channel <= LP_MAX_VECTOR_LENGTH);
610
611 for (k = 0; k < pixels_per_channel; ++k) {
612 channel[k] = src_aos[i][j + TGSI_NUM_CHANNELS * k];
613 }
614
615 aos_channels[j] = lp_build_concat(gallivm, channel, aos_channel_type, pixels_per_channel);
616 }
617
618 lp_build_transpose_aos(gallivm, soa_type, aos_channels, dst_soa[i]);
619 }
620 }
621
622
623 static void
624 store_aos(struct gallivm_state *gallivm,
625 LLVMValueRef io_ptr,
626 LLVMValueRef index,
627 LLVMValueRef value)
628 {
629 LLVMTypeRef data_ptr_type = LLVMPointerType(lp_build_vec_type(gallivm, lp_float32_vec4_type()), 0);
630 LLVMBuilderRef builder = gallivm->builder;
631 LLVMValueRef data_ptr = draw_jit_header_data(gallivm, io_ptr);
632 LLVMValueRef indices[3];
633
634 indices[0] = lp_build_const_int32(gallivm, 0);
635 indices[1] = index;
636 indices[2] = lp_build_const_int32(gallivm, 0);
637
638 #if DEBUG_STORE
639 lp_build_printf(gallivm, " ---- %p storing attribute %d (io = %p)\n", data_ptr, index, io_ptr);
640 #endif
641
642 data_ptr = LLVMBuildGEP(builder, data_ptr, indices, 3, "");
643 data_ptr = LLVMBuildPointerCast(builder, data_ptr, data_ptr_type, "");
644
645 /* Unaligned store due to the vertex header */
646 lp_set_store_alignment(LLVMBuildStore(builder, value, data_ptr), sizeof(float));
647 }
648
649 /**
650 * Adjust the mask to architecture endianess. The mask will the store in struct:
651 *
652 * struct vertex_header {
653 * unsigned clipmask:DRAW_TOTAL_CLIP_PLANES;
654 * unsigned edgeflag:1;
655 * unsigned have_clipdist:1;
656 * unsigned vertex_id:16;
657 * [...]
658 * }
659 *
660 * On little-endian machine nothing needs to done, however on bit-endian machine
661 * the mask's fields need to be adjusted with the algorithm:
662 *
663 * uint32_t reverse (uint32_t x)
664 * {
665 * return (x >> 16) | // vertex_id
666 * ((x & 0x3fff) << 18) | // clipmask
667 * ((x & 0x4000) << 3) | // have_clipdist
668 * ((x & 0x8000) << 1); // edgeflag
669 * }
670 */
671 static LLVMValueRef
672 adjust_mask(struct gallivm_state *gallivm,
673 LLVMValueRef mask)
674 {
675 #ifdef PIPE_ARCH_BIG_ENDIAN
676 LLVMBuilderRef builder = gallivm->builder;
677 LLVMValueRef vertex_id;
678 LLVMValueRef clipmask;
679 LLVMValueRef have_clipdist;
680 LLVMValueRef edgeflag;
681
682 vertex_id = LLVMBuildLShr(builder, mask, lp_build_const_int32(gallivm, 16), "");
683 clipmask = LLVMBuildAnd(builder, mask, lp_build_const_int32(gallivm, 0x3fff), "");
684 clipmask = LLVMBuildShl(builder, clipmask, lp_build_const_int32(gallivm, 18), "");
685 have_clipdist = LLVMBuildAnd(builder, mask, lp_build_const_int32(gallivm, 0x4000), "");
686 have_clipdist = LLVMBuildShl(builder, have_clipdist, lp_build_const_int32(gallivm, 3), "");
687 edgeflag = LLVMBuildAnd(builder, mask, lp_build_const_int32(gallivm, 0x8000), "");
688 edgeflag = LLVMBuildShl(builder, edgeflag, lp_build_const_int32(gallivm, 1), "");
689
690 mask = LLVMBuildOr(builder, vertex_id, clipmask, "");
691 mask = LLVMBuildOr(builder, mask, have_clipdist, "");
692 mask = LLVMBuildOr(builder, mask, edgeflag, "");
693 #endif
694 return mask;
695 }
696
697 static void
698 store_aos_array(struct gallivm_state *gallivm,
699 struct lp_type soa_type,
700 LLVMValueRef io_ptr,
701 LLVMValueRef* aos,
702 int attrib,
703 int num_outputs,
704 LLVMValueRef clipmask,
705 boolean have_clipdist)
706 {
707 LLVMBuilderRef builder = gallivm->builder;
708 LLVMValueRef attr_index = lp_build_const_int32(gallivm, attrib);
709 LLVMValueRef inds[LP_MAX_VECTOR_WIDTH / 32];
710 LLVMValueRef io_ptrs[LP_MAX_VECTOR_WIDTH / 32];
711 int vector_length = soa_type.length;
712 int i;
713
714 debug_assert(TGSI_NUM_CHANNELS == 4);
715
716 for (i = 0; i < vector_length; i++) {
717 inds[i] = lp_build_const_int32(gallivm, i);
718 io_ptrs[i] = LLVMBuildGEP(builder, io_ptr, &inds[i], 1, "");
719 }
720
721 if (attrib == 0) {
722 /* store vertex header for each of the n vertices */
723 LLVMValueRef val, cliptmp;
724 int vertex_id_pad_edgeflag;
725
726 /* If this assertion fails, it means we need to update the bit twidding
727 * code here. See struct vertex_header in draw_private.h.
728 */
729 assert(DRAW_TOTAL_CLIP_PLANES==14);
730 /* initialize vertex id:16 = 0xffff, have_clipdist:1 = 0, edgeflag:1 = 1 */
731 vertex_id_pad_edgeflag = (0xffff << 16) | (1 << DRAW_TOTAL_CLIP_PLANES);
732 if (have_clipdist)
733 vertex_id_pad_edgeflag |= 1 << (DRAW_TOTAL_CLIP_PLANES+1);
734 val = lp_build_const_int_vec(gallivm, lp_int_type(soa_type), vertex_id_pad_edgeflag);
735 /* OR with the clipmask */
736 cliptmp = LLVMBuildOr(builder, val, clipmask, "");
737 for (i = 0; i < vector_length; i++) {
738 LLVMValueRef id_ptr = draw_jit_header_id(gallivm, io_ptrs[i]);
739 val = LLVMBuildExtractElement(builder, cliptmp, inds[i], "");
740 val = adjust_mask(gallivm, val);
741 LLVMBuildStore(builder, val, id_ptr);
742 #if DEBUG_STORE
743 lp_build_printf(gallivm, "io = %p, index %d\n, clipmask = %x\n",
744 io_ptrs[i], inds[i], val);
745 #endif
746 }
747 }
748
749 /* store for each of the n vertices */
750 for (i = 0; i < vector_length; i++) {
751 store_aos(gallivm, io_ptrs[i], attr_index, aos[i]);
752 }
753 }
754
755
756 static void
757 convert_to_aos(struct gallivm_state *gallivm,
758 LLVMValueRef io,
759 LLVMValueRef (*outputs)[TGSI_NUM_CHANNELS],
760 LLVMValueRef clipmask,
761 int num_outputs,
762 struct lp_type soa_type,
763 boolean have_clipdist)
764 {
765 LLVMBuilderRef builder = gallivm->builder;
766 unsigned chan, attrib, i;
767
768 #if DEBUG_STORE
769 lp_build_printf(gallivm, " # storing begin\n");
770 #endif
771 for (attrib = 0; attrib < num_outputs; ++attrib) {
772 LLVMValueRef soa[TGSI_NUM_CHANNELS];
773 LLVMValueRef aos[LP_MAX_VECTOR_WIDTH / 32];
774 for (chan = 0; chan < TGSI_NUM_CHANNELS; ++chan) {
775 if (outputs[attrib][chan]) {
776 LLVMValueRef out = LLVMBuildLoad(builder, outputs[attrib][chan], "");
777 lp_build_name(out, "output%u.%c", attrib, "xyzw"[chan]);
778 #if DEBUG_STORE
779 lp_build_printf(gallivm, "output %d : %d ",
780 LLVMConstInt(LLVMInt32TypeInContext(gallivm->context),
781 attrib, 0),
782 LLVMConstInt(LLVMInt32TypeInContext(gallivm->context),
783 chan, 0));
784 lp_build_print_value(gallivm, "val = ", out);
785 #endif
786 soa[chan] = out;
787 }
788 else {
789 soa[chan] = 0;
790 }
791 }
792
793
794 if (soa_type.length == TGSI_NUM_CHANNELS) {
795 lp_build_transpose_aos(gallivm, soa_type, soa, aos);
796 } else {
797 lp_build_transpose_aos(gallivm, soa_type, soa, soa);
798
799 for (i = 0; i < soa_type.length; ++i) {
800 aos[i] = lp_build_extract_range(gallivm,
801 soa[i % TGSI_NUM_CHANNELS],
802 (i / TGSI_NUM_CHANNELS) * TGSI_NUM_CHANNELS,
803 TGSI_NUM_CHANNELS);
804 }
805 }
806
807 store_aos_array(gallivm,
808 soa_type,
809 io,
810 aos,
811 attrib,
812 num_outputs,
813 clipmask, have_clipdist);
814 }
815 #if DEBUG_STORE
816 lp_build_printf(gallivm, " # storing end\n");
817 #endif
818 }
819
820
821 /**
822 * Stores original vertex positions in clip coordinates
823 */
824 static void
825 store_clip(struct gallivm_state *gallivm,
826 const struct lp_type vs_type,
827 LLVMValueRef io_ptr,
828 LLVMValueRef (*outputs)[TGSI_NUM_CHANNELS],
829 boolean pre_clip_pos, int idx)
830 {
831 LLVMBuilderRef builder = gallivm->builder;
832 LLVMValueRef soa[4];
833 LLVMValueRef aos[LP_MAX_VECTOR_LENGTH];
834 LLVMValueRef indices[2];
835 LLVMValueRef io_ptrs[LP_MAX_VECTOR_WIDTH / 32];
836 LLVMValueRef inds[LP_MAX_VECTOR_WIDTH / 32];
837 LLVMValueRef clip_ptrs[LP_MAX_VECTOR_WIDTH / 32];
838 int i, j;
839
840 indices[0] =
841 indices[1] = lp_build_const_int32(gallivm, 0);
842
843 for (i = 0; i < vs_type.length; i++) {
844 inds[i] = lp_build_const_int32(gallivm, i);
845 io_ptrs[i] = LLVMBuildGEP(builder, io_ptr, &inds[i], 1, "");
846 }
847
848 soa[0] = LLVMBuildLoad(builder, outputs[idx][0], ""); /*x0 x1 .. xn*/
849 soa[1] = LLVMBuildLoad(builder, outputs[idx][1], ""); /*y0 y1 .. yn*/
850 soa[2] = LLVMBuildLoad(builder, outputs[idx][2], ""); /*z0 z1 .. zn*/
851 soa[3] = LLVMBuildLoad(builder, outputs[idx][3], ""); /*w0 w1 .. wn*/
852
853 if (!pre_clip_pos) {
854 for (i = 0; i < vs_type.length; i++) {
855 clip_ptrs[i] = draw_jit_header_clip(gallivm, io_ptrs[i]);
856 }
857 } else {
858 for (i = 0; i < vs_type.length; i++) {
859 clip_ptrs[i] = draw_jit_header_pre_clip_pos(gallivm, io_ptrs[i]);
860 }
861 }
862
863 lp_build_transpose_aos(gallivm, vs_type, soa, soa);
864 for (i = 0; i < vs_type.length; ++i) {
865 aos[i] = lp_build_extract_range(gallivm,
866 soa[i % TGSI_NUM_CHANNELS],
867 (i / TGSI_NUM_CHANNELS) * TGSI_NUM_CHANNELS,
868 TGSI_NUM_CHANNELS);
869 }
870
871 for (j = 0; j < vs_type.length; j++) {
872 LLVMTypeRef clip_ptr_type = LLVMPointerType(LLVMVectorType(LLVMFloatTypeInContext(gallivm->context), 4), 0);
873 LLVMValueRef clip_ptr;
874
875 clip_ptr = LLVMBuildGEP(builder, clip_ptrs[j], indices, 2, "clipo");
876 clip_ptr = LLVMBuildPointerCast(builder, clip_ptr, clip_ptr_type, "");
877
878 /* Unaligned store */
879 lp_set_store_alignment(LLVMBuildStore(builder, aos[j], clip_ptr), sizeof(float));
880 }
881 }
882
883
884 /**
885 * Transforms the outputs for viewport mapping
886 */
887 static void
888 generate_viewport(struct draw_llvm_variant *variant,
889 LLVMBuilderRef builder,
890 struct lp_type vs_type,
891 LLVMValueRef (*outputs)[TGSI_NUM_CHANNELS],
892 LLVMValueRef context_ptr)
893 {
894 int i;
895 struct gallivm_state *gallivm = variant->gallivm;
896 struct lp_type f32_type = vs_type;
897 LLVMTypeRef vs_type_llvm = lp_build_vec_type(gallivm, vs_type);
898 LLVMValueRef out3 = LLVMBuildLoad(builder, outputs[0][3], ""); /*w0 w1 .. wn*/
899 LLVMValueRef const1 = lp_build_const_vec(gallivm, f32_type, 1.0); /*1.0 1.0 1.0 1.0*/
900 LLVMValueRef vp_ptr = draw_jit_context_viewport(gallivm, context_ptr);
901
902 /* for 1/w convention*/
903 out3 = LLVMBuildFDiv(builder, const1, out3, "");
904 LLVMBuildStore(builder, out3, outputs[0][3]);
905
906 /* Viewport Mapping */
907 for (i=0; i<3; i++) {
908 LLVMValueRef out = LLVMBuildLoad(builder, outputs[0][i], ""); /*x0 x1 .. xn*/
909 LLVMValueRef scale;
910 LLVMValueRef trans;
911 LLVMValueRef scale_i;
912 LLVMValueRef trans_i;
913 LLVMValueRef index;
914
915 index = lp_build_const_int32(gallivm, i);
916 scale_i = LLVMBuildGEP(builder, vp_ptr, &index, 1, "");
917
918 index = lp_build_const_int32(gallivm, i+4);
919 trans_i = LLVMBuildGEP(builder, vp_ptr, &index, 1, "");
920
921 scale = lp_build_broadcast(gallivm, vs_type_llvm,
922 LLVMBuildLoad(builder, scale_i, "scale"));
923 trans = lp_build_broadcast(gallivm, vs_type_llvm,
924 LLVMBuildLoad(builder, trans_i, "trans"));
925
926 /* divide by w */
927 out = LLVMBuildFMul(builder, out, out3, "");
928 /* mult by scale */
929 out = LLVMBuildFMul(builder, out, scale, "");
930 /* add translation */
931 out = LLVMBuildFAdd(builder, out, trans, "");
932
933 /* store transformed outputs */
934 LLVMBuildStore(builder, out, outputs[0][i]);
935 }
936
937 }
938
939
940 /**
941 * Returns clipmask as nxi32 bitmask for the n vertices
942 */
943 static LLVMValueRef
944 generate_clipmask(struct draw_llvm *llvm,
945 struct gallivm_state *gallivm,
946 struct lp_type vs_type,
947 LLVMValueRef (*outputs)[TGSI_NUM_CHANNELS],
948 boolean clip_xy,
949 boolean clip_z,
950 boolean clip_user,
951 boolean clip_halfz,
952 unsigned ucp_enable,
953 LLVMValueRef context_ptr,
954 boolean *have_clipdist)
955 {
956 LLVMBuilderRef builder = gallivm->builder;
957 LLVMValueRef mask; /* stores the <nxi32> clipmasks */
958 LLVMValueRef test, temp;
959 LLVMValueRef zero, shift;
960 LLVMValueRef pos_x, pos_y, pos_z, pos_w;
961 LLVMValueRef cv_x, cv_y, cv_z, cv_w;
962 LLVMValueRef plane1, planes, plane_ptr, sum;
963 struct lp_type f32_type = vs_type;
964 struct lp_type i32_type = lp_int_type(vs_type);
965 const unsigned pos = draw_current_shader_position_output(llvm->draw);
966 const unsigned cv = draw_current_shader_clipvertex_output(llvm->draw);
967 int num_written_clipdistance = llvm->draw->vs.vertex_shader->info.num_written_clipdistance;
968 bool have_cd = false;
969 unsigned cd[2];
970
971 cd[0] = draw_current_shader_clipdistance_output(llvm->draw, 0);
972 cd[1] = draw_current_shader_clipdistance_output(llvm->draw, 1);
973
974 if (cd[0] != pos || cd[1] != pos)
975 have_cd = true;
976
977 mask = lp_build_const_int_vec(gallivm, i32_type, 0);
978 temp = lp_build_const_int_vec(gallivm, i32_type, 0);
979 zero = lp_build_const_vec(gallivm, f32_type, 0); /* 0.0f 0.0f 0.0f 0.0f */
980 shift = lp_build_const_int_vec(gallivm, i32_type, 1); /* 1 1 1 1 */
981
982 /*
983 * load clipvertex and position from correct locations.
984 * if they are the same just load them once.
985 */
986 pos_x = LLVMBuildLoad(builder, outputs[pos][0], ""); /*x0 x1 .. xn */
987 pos_y = LLVMBuildLoad(builder, outputs[pos][1], ""); /*y0 y1 .. yn */
988 pos_z = LLVMBuildLoad(builder, outputs[pos][2], ""); /*z0 z1 .. zn */
989 pos_w = LLVMBuildLoad(builder, outputs[pos][3], ""); /*w0 w1 .. wn */
990
991 if (clip_user && cv != pos) {
992 cv_x = LLVMBuildLoad(builder, outputs[cv][0], ""); /*x0 x1 .. xn */
993 cv_y = LLVMBuildLoad(builder, outputs[cv][1], ""); /*y0 y1 .. yn */
994 cv_z = LLVMBuildLoad(builder, outputs[cv][2], ""); /*z0 z1 .. zn */
995 cv_w = LLVMBuildLoad(builder, outputs[cv][3], ""); /*w0 w1 .. wn */
996 } else {
997 cv_x = pos_x;
998 cv_y = pos_y;
999 cv_z = pos_z;
1000 cv_w = pos_w;
1001 }
1002
1003 /* Cliptest, for hardwired planes */
1004 if (clip_xy) {
1005 /* plane 1 */
1006 test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, pos_x , pos_w);
1007 temp = shift;
1008 test = LLVMBuildAnd(builder, test, temp, "");
1009 mask = test;
1010
1011 /* plane 2 */
1012 test = LLVMBuildFAdd(builder, pos_x, pos_w, "");
1013 test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, zero, test);
1014 temp = LLVMBuildShl(builder, temp, shift, "");
1015 test = LLVMBuildAnd(builder, test, temp, "");
1016 mask = LLVMBuildOr(builder, mask, test, "");
1017
1018 /* plane 3 */
1019 test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, pos_y, pos_w);
1020 temp = LLVMBuildShl(builder, temp, shift, "");
1021 test = LLVMBuildAnd(builder, test, temp, "");
1022 mask = LLVMBuildOr(builder, mask, test, "");
1023
1024 /* plane 4 */
1025 test = LLVMBuildFAdd(builder, pos_y, pos_w, "");
1026 test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, zero, test);
1027 temp = LLVMBuildShl(builder, temp, shift, "");
1028 test = LLVMBuildAnd(builder, test, temp, "");
1029 mask = LLVMBuildOr(builder, mask, test, "");
1030 }
1031
1032 if (clip_z) {
1033 temp = lp_build_const_int_vec(gallivm, i32_type, 16);
1034 if (clip_halfz) {
1035 /* plane 5 */
1036 test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, zero, pos_z);
1037 test = LLVMBuildAnd(builder, test, temp, "");
1038 mask = LLVMBuildOr(builder, mask, test, "");
1039 }
1040 else {
1041 /* plane 5 */
1042 test = LLVMBuildFAdd(builder, pos_z, pos_w, "");
1043 test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, zero, test);
1044 test = LLVMBuildAnd(builder, test, temp, "");
1045 mask = LLVMBuildOr(builder, mask, test, "");
1046 }
1047 /* plane 6 */
1048 test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, pos_z, pos_w);
1049 temp = LLVMBuildShl(builder, temp, shift, "");
1050 test = LLVMBuildAnd(builder, test, temp, "");
1051 mask = LLVMBuildOr(builder, mask, test, "");
1052 }
1053
1054 if (clip_user) {
1055 LLVMValueRef planes_ptr = draw_jit_context_planes(gallivm, context_ptr);
1056 LLVMValueRef indices[3];
1057
1058 /* userclip planes */
1059 while (ucp_enable) {
1060 unsigned plane_idx = ffs(ucp_enable)-1;
1061 ucp_enable &= ~(1 << plane_idx);
1062 plane_idx += 6;
1063
1064 if (have_cd && num_written_clipdistance) {
1065 LLVMValueRef clipdist;
1066 int i;
1067 i = plane_idx - 6;
1068
1069 *have_clipdist = TRUE;
1070 if (i < 4) {
1071 clipdist = LLVMBuildLoad(builder, outputs[cd[0]][i], "");
1072 } else {
1073 clipdist = LLVMBuildLoad(builder, outputs[cd[1]][i-4], "");
1074 }
1075 test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, zero, clipdist);
1076 temp = lp_build_const_int_vec(gallivm, i32_type, 1 << plane_idx);
1077 test = LLVMBuildAnd(builder, test, temp, "");
1078 mask = LLVMBuildOr(builder, mask, test, "");
1079 } else {
1080 LLVMTypeRef vs_type_llvm = lp_build_vec_type(gallivm, vs_type);
1081 indices[0] = lp_build_const_int32(gallivm, 0);
1082 indices[1] = lp_build_const_int32(gallivm, plane_idx);
1083
1084 indices[2] = lp_build_const_int32(gallivm, 0);
1085 plane_ptr = LLVMBuildGEP(builder, planes_ptr, indices, 3, "");
1086 plane1 = LLVMBuildLoad(builder, plane_ptr, "plane_x");
1087 planes = lp_build_broadcast(gallivm, vs_type_llvm, plane1);
1088 sum = LLVMBuildFMul(builder, planes, cv_x, "");
1089
1090 indices[2] = lp_build_const_int32(gallivm, 1);
1091 plane_ptr = LLVMBuildGEP(builder, planes_ptr, indices, 3, "");
1092 plane1 = LLVMBuildLoad(builder, plane_ptr, "plane_y");
1093 planes = lp_build_broadcast(gallivm, vs_type_llvm, plane1);
1094 test = LLVMBuildFMul(builder, planes, cv_y, "");
1095 sum = LLVMBuildFAdd(builder, sum, test, "");
1096
1097 indices[2] = lp_build_const_int32(gallivm, 2);
1098 plane_ptr = LLVMBuildGEP(builder, planes_ptr, indices, 3, "");
1099 plane1 = LLVMBuildLoad(builder, plane_ptr, "plane_z");
1100 planes = lp_build_broadcast(gallivm, vs_type_llvm, plane1);
1101 test = LLVMBuildFMul(builder, planes, cv_z, "");
1102 sum = LLVMBuildFAdd(builder, sum, test, "");
1103
1104 indices[2] = lp_build_const_int32(gallivm, 3);
1105 plane_ptr = LLVMBuildGEP(builder, planes_ptr, indices, 3, "");
1106 plane1 = LLVMBuildLoad(builder, plane_ptr, "plane_w");
1107 planes = lp_build_broadcast(gallivm, vs_type_llvm, plane1);
1108 test = LLVMBuildFMul(builder, planes, cv_w, "");
1109 sum = LLVMBuildFAdd(builder, sum, test, "");
1110
1111 test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, zero, sum);
1112 temp = lp_build_const_int_vec(gallivm, i32_type, 1 << plane_idx);
1113 test = LLVMBuildAnd(builder, test, temp, "");
1114 mask = LLVMBuildOr(builder, mask, test, "");
1115 }
1116 }
1117 }
1118 return mask;
1119 }
1120
1121
1122 /**
1123 * Returns boolean if any clipping has occurred
1124 * Used zero/non-zero i32 value to represent boolean
1125 */
1126 static LLVMValueRef
1127 clipmask_booli32(struct gallivm_state *gallivm,
1128 const struct lp_type vs_type,
1129 LLVMValueRef clipmask_bool_ptr)
1130 {
1131 LLVMBuilderRef builder = gallivm->builder;
1132 LLVMTypeRef int32_type = LLVMInt32TypeInContext(gallivm->context);
1133 LLVMValueRef clipmask_bool = LLVMBuildLoad(builder, clipmask_bool_ptr, "");
1134 LLVMValueRef ret = LLVMConstNull(int32_type);
1135 LLVMValueRef temp;
1136 int i;
1137
1138 /*
1139 * Can do this with log2(vector length) pack instructions and one extract
1140 * (as we don't actually need a or) with sse2 which would be way better.
1141 */
1142 for (i=0; i < vs_type.length; i++) {
1143 temp = LLVMBuildExtractElement(builder, clipmask_bool,
1144 lp_build_const_int32(gallivm, i) , "");
1145 ret = LLVMBuildOr(builder, ret, temp, "");
1146 }
1147 return ret;
1148 }
1149
1150
1151 static void
1152 draw_llvm_generate(struct draw_llvm *llvm, struct draw_llvm_variant *variant,
1153 boolean elts)
1154 {
1155 struct gallivm_state *gallivm = variant->gallivm;
1156 LLVMContextRef context = gallivm->context;
1157 LLVMTypeRef int32_type = LLVMInt32TypeInContext(context);
1158 LLVMTypeRef arg_types[8];
1159 LLVMTypeRef func_type;
1160 LLVMValueRef context_ptr;
1161 LLVMBasicBlockRef block;
1162 LLVMBuilderRef builder;
1163 struct lp_type vs_type;
1164 LLVMValueRef end, start;
1165 LLVMValueRef count, fetch_elts, fetch_count;
1166 LLVMValueRef stride, step, io_itr;
1167 LLVMValueRef io_ptr, vbuffers_ptr, vb_ptr;
1168 LLVMValueRef zero = lp_build_const_int32(gallivm, 0);
1169 LLVMValueRef one = lp_build_const_int32(gallivm, 1);
1170 struct draw_context *draw = llvm->draw;
1171 const struct tgsi_shader_info *vs_info = &draw->vs.vertex_shader->info;
1172 unsigned i, j;
1173 struct lp_build_context bld;
1174 struct lp_build_loop_state lp_loop;
1175 const int vector_length = lp_native_vector_width / 32;
1176 LLVMValueRef outputs[PIPE_MAX_SHADER_OUTPUTS][TGSI_NUM_CHANNELS];
1177 LLVMValueRef fetch_max;
1178 struct lp_build_sampler_soa *sampler = 0;
1179 LLVMValueRef ret, clipmask_bool_ptr;
1180 const boolean bypass_viewport = variant->key.bypass_viewport;
1181 const boolean enable_cliptest = variant->key.clip_xy ||
1182 variant->key.clip_z ||
1183 variant->key.clip_user;
1184 LLVMValueRef variant_func;
1185 const unsigned pos = draw_current_shader_position_output(llvm->draw);
1186 const unsigned cv = draw_current_shader_clipvertex_output(llvm->draw);
1187 boolean have_clipdist = FALSE;
1188 struct lp_bld_tgsi_system_values system_values;
1189
1190 memset(&system_values, 0, sizeof(system_values));
1191
1192 arg_types[0] = get_context_ptr_type(variant); /* context */
1193 arg_types[1] = get_vertex_header_ptr_type(variant); /* vertex_header */
1194 arg_types[2] = get_buffer_ptr_type(variant); /* vbuffers */
1195 if (elts)
1196 arg_types[3] = LLVMPointerType(int32_type, 0);/* fetch_elts * */
1197 else
1198 arg_types[3] = int32_type; /* start */
1199 arg_types[4] = int32_type; /* fetch_count / count */
1200 arg_types[5] = int32_type; /* stride */
1201 arg_types[6] = get_vb_ptr_type(variant); /* pipe_vertex_buffer's */
1202 arg_types[7] = int32_type; /* instance_id */
1203
1204 func_type = LLVMFunctionType(int32_type, arg_types, Elements(arg_types), 0);
1205
1206 variant_func = LLVMAddFunction(gallivm->module,
1207 elts ? "draw_llvm_shader_elts" : "draw_llvm_shader",
1208 func_type);
1209
1210 if (elts)
1211 variant->function_elts = variant_func;
1212 else
1213 variant->function = variant_func;
1214
1215 LLVMSetFunctionCallConv(variant_func, LLVMCCallConv);
1216 for (i = 0; i < Elements(arg_types); ++i)
1217 if (LLVMGetTypeKind(arg_types[i]) == LLVMPointerTypeKind)
1218 LLVMAddAttribute(LLVMGetParam(variant_func, i),
1219 LLVMNoAliasAttribute);
1220
1221 context_ptr = LLVMGetParam(variant_func, 0);
1222 io_ptr = LLVMGetParam(variant_func, 1);
1223 vbuffers_ptr = LLVMGetParam(variant_func, 2);
1224 stride = LLVMGetParam(variant_func, 5);
1225 vb_ptr = LLVMGetParam(variant_func, 6);
1226 system_values.instance_id = LLVMGetParam(variant_func, 7);
1227
1228 lp_build_name(context_ptr, "context");
1229 lp_build_name(io_ptr, "io");
1230 lp_build_name(vbuffers_ptr, "vbuffers");
1231 lp_build_name(stride, "stride");
1232 lp_build_name(vb_ptr, "vb");
1233 lp_build_name(system_values.instance_id, "instance_id");
1234
1235 if (elts) {
1236 fetch_elts = LLVMGetParam(variant_func, 3);
1237 fetch_count = LLVMGetParam(variant_func, 4);
1238 lp_build_name(fetch_elts, "fetch_elts");
1239 lp_build_name(fetch_count, "fetch_count");
1240 start = count = NULL;
1241 }
1242 else {
1243 start = LLVMGetParam(variant_func, 3);
1244 count = LLVMGetParam(variant_func, 4);
1245 lp_build_name(start, "start");
1246 lp_build_name(count, "count");
1247 fetch_elts = fetch_count = NULL;
1248 }
1249
1250 /*
1251 * Function body
1252 */
1253
1254 block = LLVMAppendBasicBlockInContext(gallivm->context, variant_func, "entry");
1255 builder = gallivm->builder;
1256 LLVMPositionBuilderAtEnd(builder, block);
1257
1258 lp_build_context_init(&bld, gallivm, lp_type_int(32));
1259
1260 memset(&vs_type, 0, sizeof vs_type);
1261 vs_type.floating = TRUE; /* floating point values */
1262 vs_type.sign = TRUE; /* values are signed */
1263 vs_type.norm = FALSE; /* values are not limited to [0,1] or [-1,1] */
1264 vs_type.width = 32; /* 32-bit float */
1265 vs_type.length = vector_length;
1266
1267 /* hold temporary "bool" clipmask */
1268 clipmask_bool_ptr = lp_build_alloca(gallivm, lp_build_int_vec_type(gallivm, vs_type), "");
1269 LLVMBuildStore(builder, lp_build_zero(gallivm, lp_int_type(vs_type)), clipmask_bool_ptr);
1270
1271 /* code generated texture sampling */
1272 sampler = draw_llvm_sampler_soa_create(
1273 draw_llvm_variant_key_samplers(&variant->key),
1274 context_ptr);
1275
1276 if (elts) {
1277 start = zero;
1278 end = fetch_count;
1279 }
1280 else {
1281 end = lp_build_add(&bld, start, count);
1282 }
1283
1284 step = lp_build_const_int32(gallivm, vector_length);
1285
1286 fetch_max = LLVMBuildSub(builder, end, one, "fetch_max");
1287
1288 lp_build_loop_begin(&lp_loop, gallivm, start);
1289 {
1290 LLVMValueRef inputs[PIPE_MAX_SHADER_INPUTS][TGSI_NUM_CHANNELS];
1291 LLVMValueRef aos_attribs[PIPE_MAX_SHADER_INPUTS][LP_MAX_VECTOR_WIDTH / 32] = { { 0 } };
1292 LLVMValueRef io;
1293 LLVMValueRef clipmask; /* holds the clipmask value */
1294 const LLVMValueRef (*ptr_aos)[TGSI_NUM_CHANNELS];
1295
1296 if (elts)
1297 io_itr = lp_loop.counter;
1298 else
1299 io_itr = LLVMBuildSub(builder, lp_loop.counter, start, "");
1300
1301 io = LLVMBuildGEP(builder, io_ptr, &io_itr, 1, "");
1302 #if DEBUG_STORE
1303 lp_build_printf(gallivm, " --- io %d = %p, loop counter %d\n",
1304 io_itr, io, lp_loop.counter);
1305 #endif
1306 system_values.vertex_id = lp_build_zero(gallivm, lp_type_uint_vec(32, 32*vector_length));
1307 for (i = 0; i < vector_length; ++i) {
1308 LLVMValueRef true_index =
1309 LLVMBuildAdd(builder,
1310 lp_loop.counter,
1311 lp_build_const_int32(gallivm, i), "");
1312
1313 /* make sure we're not out of bounds which can happen
1314 * if fetch_count % 4 != 0, because on the last iteration
1315 * a few of the 4 vertex fetches will be out of bounds */
1316 true_index = lp_build_min(&bld, true_index, fetch_max);
1317
1318 if (elts) {
1319 LLVMValueRef fetch_ptr;
1320 fetch_ptr = LLVMBuildGEP(builder, fetch_elts,
1321 &true_index, 1, "");
1322 true_index = LLVMBuildLoad(builder, fetch_ptr, "fetch_elt");
1323 }
1324
1325 system_values.vertex_id = LLVMBuildInsertElement(gallivm->builder,
1326 system_values.vertex_id, true_index,
1327 lp_build_const_int32(gallivm, i), "");
1328 for (j = 0; j < draw->pt.nr_vertex_elements; ++j) {
1329 struct pipe_vertex_element *velem = &draw->pt.vertex_element[j];
1330 LLVMValueRef vb_index =
1331 lp_build_const_int32(gallivm, velem->vertex_buffer_index);
1332 LLVMValueRef vb = LLVMBuildGEP(builder, vb_ptr, &vb_index, 1, "");
1333 generate_fetch(gallivm, vbuffers_ptr,
1334 &aos_attribs[j][i], velem, vb, true_index,
1335 system_values.instance_id);
1336 }
1337 }
1338 convert_to_soa(gallivm, aos_attribs, inputs,
1339 draw->pt.nr_vertex_elements, vs_type);
1340
1341 ptr_aos = (const LLVMValueRef (*)[TGSI_NUM_CHANNELS]) inputs;
1342 generate_vs(variant,
1343 builder,
1344 vs_type,
1345 outputs,
1346 ptr_aos,
1347 &system_values,
1348 context_ptr,
1349 sampler,
1350 variant->key.clamp_vertex_color);
1351
1352 /* store original positions in clip before further manipulation */
1353 store_clip(gallivm, vs_type, io, outputs, 0, cv);
1354 store_clip(gallivm, vs_type, io, outputs, 1, pos);
1355
1356 /* do cliptest */
1357 if (enable_cliptest) {
1358 LLVMValueRef temp = LLVMBuildLoad(builder, clipmask_bool_ptr, "");
1359 /* allocate clipmask, assign it integer type */
1360 clipmask = generate_clipmask(llvm,
1361 gallivm,
1362 vs_type,
1363 outputs,
1364 variant->key.clip_xy,
1365 variant->key.clip_z,
1366 variant->key.clip_user,
1367 variant->key.clip_halfz,
1368 variant->key.ucp_enable,
1369 context_ptr, &have_clipdist);
1370 temp = LLVMBuildOr(builder, clipmask, temp, "");
1371 /* store temporary clipping boolean value */
1372 LLVMBuildStore(builder, temp, clipmask_bool_ptr);
1373 }
1374 else {
1375 clipmask = lp_build_const_int_vec(gallivm, lp_int_type(vs_type), 0);
1376 }
1377
1378 /* do viewport mapping */
1379 if (!bypass_viewport) {
1380 generate_viewport(variant, builder, vs_type, outputs, context_ptr);
1381 }
1382
1383 /* store clipmask in vertex header,
1384 * original positions in clip
1385 * and transformed positions in data
1386 */
1387 convert_to_aos(gallivm, io, outputs, clipmask,
1388 vs_info->num_outputs, vs_type,
1389 have_clipdist);
1390 }
1391
1392 lp_build_loop_end_cond(&lp_loop, end, step, LLVMIntUGE);
1393
1394 sampler->destroy(sampler);
1395
1396 /* return clipping boolean value for function */
1397 ret = clipmask_booli32(gallivm, vs_type, clipmask_bool_ptr);
1398
1399 LLVMBuildRet(builder, ret);
1400
1401 gallivm_verify_function(gallivm, variant_func);
1402 }
1403
1404
1405 struct draw_llvm_variant_key *
1406 draw_llvm_make_variant_key(struct draw_llvm *llvm, char *store)
1407 {
1408 unsigned i;
1409 struct draw_llvm_variant_key *key;
1410 struct draw_sampler_static_state *draw_sampler;
1411
1412 key = (struct draw_llvm_variant_key *)store;
1413
1414 key->clamp_vertex_color = llvm->draw->rasterizer->clamp_vertex_color; /**/
1415
1416 /* Presumably all variants of the shader should have the same
1417 * number of vertex elements - ie the number of shader inputs.
1418 * NOTE: we NEED to store the needed number of needed inputs
1419 * here, not the number of provided elements to match keysize
1420 * (and the offset of sampler state in the key).
1421 */
1422 key->nr_vertex_elements = llvm->draw->vs.vertex_shader->info.file_max[TGSI_FILE_INPUT] + 1;
1423 assert(key->nr_vertex_elements <= llvm->draw->pt.nr_vertex_elements);
1424
1425 /* will have to rig this up properly later */
1426 key->clip_xy = llvm->draw->clip_xy;
1427 key->clip_z = llvm->draw->clip_z;
1428 key->clip_user = llvm->draw->clip_user;
1429 key->bypass_viewport = llvm->draw->identity_viewport;
1430 key->clip_halfz = !llvm->draw->rasterizer->gl_rasterization_rules;
1431 key->need_edgeflags = (llvm->draw->vs.edgeflag_output ? TRUE : FALSE);
1432 key->ucp_enable = llvm->draw->rasterizer->clip_plane_enable;
1433 key->pad1 = 0;
1434 key->pad2 = 0;
1435
1436 /* All variants of this shader will have the same value for
1437 * nr_samplers. Not yet trying to compact away holes in the
1438 * sampler array.
1439 */
1440 key->nr_samplers = llvm->draw->vs.vertex_shader->info.file_max[TGSI_FILE_SAMPLER] + 1;
1441 if (llvm->draw->vs.vertex_shader->info.file_max[TGSI_FILE_SAMPLER_VIEW] != -1) {
1442 key->nr_sampler_views =
1443 llvm->draw->vs.vertex_shader->info.file_max[TGSI_FILE_SAMPLER_VIEW] + 1;
1444 }
1445 else {
1446 key->nr_sampler_views = key->nr_samplers;
1447 }
1448
1449 draw_sampler = draw_llvm_variant_key_samplers(key);
1450
1451 memcpy(key->vertex_element,
1452 llvm->draw->pt.vertex_element,
1453 sizeof(struct pipe_vertex_element) * key->nr_vertex_elements);
1454
1455 memset(draw_sampler, 0, MAX2(key->nr_samplers, key->nr_sampler_views) * sizeof *draw_sampler);
1456
1457 for (i = 0 ; i < key->nr_samplers; i++) {
1458 lp_sampler_static_sampler_state(&draw_sampler[i].sampler_state,
1459 llvm->draw->samplers[PIPE_SHADER_VERTEX][i]);
1460 }
1461 for (i = 0 ; i < key->nr_sampler_views; i++) {
1462 lp_sampler_static_texture_state(&draw_sampler[i].texture_state,
1463 llvm->draw->sampler_views[PIPE_SHADER_VERTEX][i]);
1464 }
1465
1466 return key;
1467 }
1468
1469
1470 void
1471 draw_llvm_dump_variant_key(struct draw_llvm_variant_key *key)
1472 {
1473 unsigned i;
1474 struct draw_sampler_static_state *sampler = draw_llvm_variant_key_samplers(key);
1475
1476 debug_printf("clamp_vertex_color = %u\n", key->clamp_vertex_color);
1477 debug_printf("clip_xy = %u\n", key->clip_xy);
1478 debug_printf("clip_z = %u\n", key->clip_z);
1479 debug_printf("clip_user = %u\n", key->clip_user);
1480 debug_printf("bypass_viewport = %u\n", key->bypass_viewport);
1481 debug_printf("clip_halfz = %u\n", key->clip_halfz);
1482 debug_printf("need_edgeflags = %u\n", key->need_edgeflags);
1483 debug_printf("ucp_enable = %u\n", key->ucp_enable);
1484
1485 for (i = 0 ; i < key->nr_vertex_elements; i++) {
1486 debug_printf("vertex_element[%i].src_offset = %u\n", i, key->vertex_element[i].src_offset);
1487 debug_printf("vertex_element[%i].instance_divisor = %u\n", i, key->vertex_element[i].instance_divisor);
1488 debug_printf("vertex_element[%i].vertex_buffer_index = %u\n", i, key->vertex_element[i].vertex_buffer_index);
1489 debug_printf("vertex_element[%i].src_format = %s\n", i, util_format_name(key->vertex_element[i].src_format));
1490 }
1491
1492 for (i = 0 ; i < key->nr_sampler_views; i++) {
1493 debug_printf("sampler[%i].src_format = %s\n", i, util_format_name(sampler[i].texture_state.format));
1494 }
1495 }
1496
1497
1498 void
1499 draw_llvm_set_mapped_texture(struct draw_context *draw,
1500 unsigned sampler_idx,
1501 uint32_t width, uint32_t height, uint32_t depth,
1502 uint32_t first_level, uint32_t last_level,
1503 const void *base_ptr,
1504 uint32_t row_stride[PIPE_MAX_TEXTURE_LEVELS],
1505 uint32_t img_stride[PIPE_MAX_TEXTURE_LEVELS],
1506 uint32_t mip_offsets[PIPE_MAX_TEXTURE_LEVELS])
1507 {
1508 unsigned j;
1509 struct draw_jit_texture *jit_tex;
1510
1511 assert(sampler_idx < Elements(draw->llvm->jit_context.textures));
1512
1513 jit_tex = &draw->llvm->jit_context.textures[sampler_idx];
1514
1515 jit_tex->width = width;
1516 jit_tex->height = height;
1517 jit_tex->depth = depth;
1518 jit_tex->first_level = first_level;
1519 jit_tex->last_level = last_level;
1520 jit_tex->base = base_ptr;
1521
1522 for (j = first_level; j <= last_level; j++) {
1523 jit_tex->mip_offsets[j] = mip_offsets[j];
1524 jit_tex->row_stride[j] = row_stride[j];
1525 jit_tex->img_stride[j] = img_stride[j];
1526 }
1527 }
1528
1529
1530 void
1531 draw_llvm_set_sampler_state(struct draw_context *draw)
1532 {
1533 unsigned i;
1534
1535 for (i = 0; i < draw->num_samplers[PIPE_SHADER_VERTEX]; i++) {
1536 struct draw_jit_sampler *jit_sam = &draw->llvm->jit_context.samplers[i];
1537
1538 if (draw->samplers[i]) {
1539 const struct pipe_sampler_state *s
1540 = draw->samplers[PIPE_SHADER_VERTEX][i];
1541 jit_sam->min_lod = s->min_lod;
1542 jit_sam->max_lod = s->max_lod;
1543 jit_sam->lod_bias = s->lod_bias;
1544 COPY_4V(jit_sam->border_color, s->border_color.f);
1545 }
1546 }
1547 }
1548
1549
1550 void
1551 draw_llvm_destroy_variant(struct draw_llvm_variant *variant)
1552 {
1553 struct draw_llvm *llvm = variant->llvm;
1554
1555 if (variant->function_elts) {
1556 gallivm_free_function(variant->gallivm,
1557 variant->function_elts, variant->jit_func_elts);
1558 }
1559
1560 if (variant->function) {
1561 gallivm_free_function(variant->gallivm,
1562 variant->function, variant->jit_func);
1563 }
1564
1565 gallivm_destroy(variant->gallivm);
1566
1567 remove_from_list(&variant->list_item_local);
1568 variant->shader->variants_cached--;
1569 remove_from_list(&variant->list_item_global);
1570 llvm->nr_variants--;
1571 FREE(variant);
1572 }