b7eb2c3d4e69ce5b5aeb557b464d856ceb92bd22
1 /**************************************************************************
3 * Copyright 2009 VMware, Inc.
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sub license, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
14 * The above copyright notice and this permission notice (including the
15 * next paragraph) shall be included in all copies or substantial portions
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
19 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
20 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
21 * IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR
22 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
23 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
24 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
26 **************************************************************************/
30 * Helper functions for logical operations.
32 * @author Jose Fonseca <jfonseca@vmware.com>
36 #include "util/u_cpu_detect.h"
37 #include "util/u_debug.h"
39 #include "lp_bld_type.h"
40 #include "lp_bld_const.h"
41 #include "lp_bld_intr.h"
42 #include "lp_bld_logic.h"
48 * Selection with vector conditional like
50 * select <4 x i1> %C, %A, %B
52 * is valid IR (e.g. llvm/test/Assembler/vector-select.ll), but it is not
53 * supported on any backend.
55 * Expanding the boolean vector to full SIMD register width, as in
57 * sext <4 x i1> %C to <4 x i32>
59 * is valid and supported (e.g., llvm/test/CodeGen/X86/vec_compare.ll), but
60 * it causes assertion failures in LLVM 2.6. It appears to work correctly on
66 * Build code to compare two values 'a' and 'b' of 'type' using the given func.
67 * \param func one of PIPE_FUNC_x
68 * The result values will be 0 for false or ~0 for true.
71 lp_build_compare(LLVMBuilderRef builder
,
72 const struct lp_type type
,
77 LLVMTypeRef vec_type
= lp_build_vec_type(type
);
78 LLVMTypeRef int_vec_type
= lp_build_int_vec_type(type
);
79 LLVMValueRef zeros
= LLVMConstNull(int_vec_type
);
80 LLVMValueRef ones
= LLVMConstAllOnes(int_vec_type
);
85 assert(func
>= PIPE_FUNC_NEVER
);
86 assert(func
<= PIPE_FUNC_ALWAYS
);
88 if(func
== PIPE_FUNC_NEVER
)
90 if(func
== PIPE_FUNC_ALWAYS
)
93 /* TODO: optimize the constant case */
95 /* XXX: It is not clear if we should use the ordered or unordered operators */
97 #if !defined(HAVE_LLVM) || HAVE_LLVM < 0x0207
98 #if defined(PIPE_ARCH_X86) || defined(PIPE_ARCH_X86_64)
99 if(type
.width
* type
.length
== 128) {
100 if(type
.floating
&& util_cpu_caps
.has_sse
) {
101 /* float[4] comparison */
102 LLVMValueRef args
[3];
108 case PIPE_FUNC_EQUAL
:
111 case PIPE_FUNC_NOTEQUAL
:
117 case PIPE_FUNC_LEQUAL
:
120 case PIPE_FUNC_GREATER
:
124 case PIPE_FUNC_GEQUAL
:
130 return lp_build_undef(type
);
142 args
[2] = LLVMConstInt(LLVMInt8Type(), cc
, 0);
143 res
= lp_build_intrinsic(builder
,
144 "llvm.x86.sse.cmp.ps",
147 res
= LLVMBuildBitCast(builder
, res
, int_vec_type
, "");
150 else if(util_cpu_caps
.has_sse2
) {
151 /* int[4] comparison */
152 static const struct {
158 {0, 0, 0, 1}, /* PIPE_FUNC_NEVER */
159 {1, 0, 1, 0}, /* PIPE_FUNC_LESS */
160 {0, 1, 0, 0}, /* PIPE_FUNC_EQUAL */
161 {0, 0, 1, 1}, /* PIPE_FUNC_LEQUAL */
162 {0, 0, 1, 0}, /* PIPE_FUNC_GREATER */
163 {0, 1, 0, 1}, /* PIPE_FUNC_NOTEQUAL */
164 {1, 0, 1, 1}, /* PIPE_FUNC_GEQUAL */
165 {0, 0, 0, 0} /* PIPE_FUNC_ALWAYS */
169 LLVMValueRef args
[2];
172 switch (type
.width
) {
174 pcmpeq
= "llvm.x86.sse2.pcmpeq.b";
175 pcmpgt
= "llvm.x86.sse2.pcmpgt.b";
178 pcmpeq
= "llvm.x86.sse2.pcmpeq.w";
179 pcmpgt
= "llvm.x86.sse2.pcmpgt.w";
182 pcmpeq
= "llvm.x86.sse2.pcmpeq.d";
183 pcmpgt
= "llvm.x86.sse2.pcmpgt.d";
187 return lp_build_undef(type
);
190 /* There are no signed byte and unsigned word/dword comparison
191 * instructions. So flip the sign bit so that the results match.
194 ((type
.width
== 8 && type
.sign
) ||
195 (type
.width
!= 8 && !type
.sign
))) {
196 LLVMValueRef msb
= lp_build_int_const_scalar(type
, (unsigned long long)1 << (type
.width
- 1));
197 a
= LLVMBuildXor(builder
, a
, msb
, "");
198 b
= LLVMBuildXor(builder
, b
, msb
, "");
201 if(table
[func
].swap
) {
211 res
= lp_build_intrinsic(builder
, pcmpeq
, vec_type
, args
, 2);
212 else if (table
[func
].gt
)
213 res
= lp_build_intrinsic(builder
, pcmpgt
, vec_type
, args
, 2);
215 res
= LLVMConstNull(vec_type
);
218 res
= LLVMBuildNot(builder
, res
, "");
222 } /* if (type.width * type.length == 128) */
224 #endif /* HAVE_LLVM < 0x0207 */
227 LLVMRealPredicate op
;
229 case PIPE_FUNC_NEVER
:
230 op
= LLVMRealPredicateFalse
;
232 case PIPE_FUNC_ALWAYS
:
233 op
= LLVMRealPredicateTrue
;
235 case PIPE_FUNC_EQUAL
:
238 case PIPE_FUNC_NOTEQUAL
:
244 case PIPE_FUNC_LEQUAL
:
247 case PIPE_FUNC_GREATER
:
250 case PIPE_FUNC_GEQUAL
:
255 return lp_build_undef(type
);
258 #if HAVE_LLVM >= 0x0207
259 cond
= LLVMBuildFCmp(builder
, op
, a
, b
, "");
260 res
= LLVMBuildSExt(builder
, cond
, int_vec_type
, "");
262 if (type
.length
== 1) {
263 cond
= LLVMBuildFCmp(builder
, op
, a
, b
, "");
264 res
= LLVMBuildSExt(builder
, cond
, int_vec_type
, "");
267 res
= LLVMGetUndef(int_vec_type
);
269 debug_printf("%s: warning: using slow element-wise float"
270 " vector comparison\n", __FUNCTION__
);
271 for (i
= 0; i
< type
.length
; ++i
) {
272 LLVMValueRef index
= LLVMConstInt(LLVMInt32Type(), i
, 0);
273 cond
= LLVMBuildFCmp(builder
, op
,
274 LLVMBuildExtractElement(builder
, a
, index
, ""),
275 LLVMBuildExtractElement(builder
, b
, index
, ""),
277 cond
= LLVMBuildSelect(builder
, cond
,
278 LLVMConstExtractElement(ones
, index
),
279 LLVMConstExtractElement(zeros
, index
),
281 res
= LLVMBuildInsertElement(builder
, res
, cond
, index
, "");
289 case PIPE_FUNC_EQUAL
:
292 case PIPE_FUNC_NOTEQUAL
:
296 op
= type
.sign
? LLVMIntSLT
: LLVMIntULT
;
298 case PIPE_FUNC_LEQUAL
:
299 op
= type
.sign
? LLVMIntSLE
: LLVMIntULE
;
301 case PIPE_FUNC_GREATER
:
302 op
= type
.sign
? LLVMIntSGT
: LLVMIntUGT
;
304 case PIPE_FUNC_GEQUAL
:
305 op
= type
.sign
? LLVMIntSGE
: LLVMIntUGE
;
309 return lp_build_undef(type
);
312 #if HAVE_LLVM >= 0x0207
313 cond
= LLVMBuildICmp(builder
, op
, a
, b
, "");
314 res
= LLVMBuildSExt(builder
, cond
, int_vec_type
, "");
316 if (type
.length
== 1) {
317 cond
= LLVMBuildICmp(builder
, op
, a
, b
, "");
318 res
= LLVMBuildSExt(builder
, cond
, int_vec_type
, "");
321 res
= LLVMGetUndef(int_vec_type
);
323 debug_printf("%s: warning: using slow element-wise int"
324 " vector comparison\n", __FUNCTION__
);
326 for(i
= 0; i
< type
.length
; ++i
) {
327 LLVMValueRef index
= LLVMConstInt(LLVMInt32Type(), i
, 0);
328 cond
= LLVMBuildICmp(builder
, op
,
329 LLVMBuildExtractElement(builder
, a
, index
, ""),
330 LLVMBuildExtractElement(builder
, b
, index
, ""),
332 cond
= LLVMBuildSelect(builder
, cond
,
333 LLVMConstExtractElement(ones
, index
),
334 LLVMConstExtractElement(zeros
, index
),
336 res
= LLVMBuildInsertElement(builder
, res
, cond
, index
, "");
348 * Build code to compare two values 'a' and 'b' using the given func.
349 * \param func one of PIPE_FUNC_x
350 * The result values will be 0 for false or ~0 for true.
353 lp_build_cmp(struct lp_build_context
*bld
,
358 return lp_build_compare(bld
->builder
, bld
->type
, func
, a
, b
);
363 * Return mask ? a : b;
365 * mask is a bitwise mask, composed of 0 or ~0 for each element.
368 lp_build_select(struct lp_build_context
*bld
,
373 struct lp_type type
= bld
->type
;
379 if (type
.length
== 1) {
380 mask
= LLVMBuildTrunc(bld
->builder
, mask
, LLVMInt1Type(), "");
381 res
= LLVMBuildSelect(bld
->builder
, mask
, a
, b
, "");
385 LLVMTypeRef int_vec_type
= lp_build_int_vec_type(type
);
386 a
= LLVMBuildBitCast(bld
->builder
, a
, int_vec_type
, "");
387 b
= LLVMBuildBitCast(bld
->builder
, b
, int_vec_type
, "");
390 a
= LLVMBuildAnd(bld
->builder
, a
, mask
, "");
392 /* This often gets translated to PANDN, but sometimes the NOT is
393 * pre-computed and stored in another constant. The best strategy depends
394 * on available registers, so it is not a big deal -- hopefully LLVM does
395 * the right decision attending the rest of the program.
397 b
= LLVMBuildAnd(bld
->builder
, b
, LLVMBuildNot(bld
->builder
, mask
, ""), "");
399 res
= LLVMBuildOr(bld
->builder
, a
, b
, "");
402 LLVMTypeRef vec_type
= lp_build_vec_type(type
);
403 res
= LLVMBuildBitCast(bld
->builder
, res
, vec_type
, "");
412 lp_build_select_aos(struct lp_build_context
*bld
,
415 const boolean cond
[4])
417 const struct lp_type type
= bld
->type
;
418 const unsigned n
= type
.length
;
423 if(cond
[0] && cond
[1] && cond
[2] && cond
[3])
425 if(!cond
[0] && !cond
[1] && !cond
[2] && !cond
[3])
427 if(a
== bld
->undef
|| b
== bld
->undef
)
431 * There are three major ways of accomplishing this:
434 * - or with a bit mask.
436 * Select isn't supported for vector types yet.
437 * The flip between these is empirical and might need to be.
443 LLVMTypeRef elem_type
= LLVMInt32Type();
444 LLVMValueRef shuffles
[LP_MAX_VECTOR_LENGTH
];
446 for(j
= 0; j
< n
; j
+= 4)
447 for(i
= 0; i
< 4; ++i
)
448 shuffles
[j
+ i
] = LLVMConstInt(elem_type
, (cond
[i
] ? 0 : n
) + j
+ i
, 0);
450 return LLVMBuildShuffleVector(bld
->builder
, a
, b
, LLVMConstVector(shuffles
, n
), "");
454 /* XXX: Unfortunately select of vectors do not work */
456 LLVMTypeRef elem_type
= LLVMInt1Type();
457 LLVMValueRef cond
[LP_MAX_VECTOR_LENGTH
];
459 for(j
= 0; j
< n
; j
+= 4)
460 for(i
= 0; i
< 4; ++i
)
461 cond
[j
+ i
] = LLVMConstInt(elem_type
, cond
[i
] ? 1 : 0, 0);
463 return LLVMBuildSelect(bld
->builder
, LLVMConstVector(cond
, n
), a
, b
, "");
465 LLVMValueRef mask
= lp_build_const_mask_aos(type
, cond
);
466 return lp_build_select(bld
, mask
, a
, b
);
472 lp_build_alloca(struct lp_build_context
*bld
)
474 const struct lp_type type
= bld
->type
;
476 if (type
.length
> 1) { /*vector*/
477 return LLVMBuildAlloca(bld
->builder
, lp_build_vec_type(type
), "");
479 return LLVMBuildAlloca(bld
->builder
, lp_build_elem_type(type
), "");