*/
-#include "pipe/p_state.h"
-
+#include "util/u_memory.h"
+#include "util/u_debug.h"
+#include "util/u_string.h"
+
+#include "lp_bld_type.h"
+#include "lp_bld_const.h"
+#include "lp_bld_intr.h"
+#include "lp_bld_logic.h"
#include "lp_bld_arit.h"
-LLVMTypeRef
-lp_build_elem_type(union lp_type type)
-{
- if (type.floating) {
- assert(type.sign);
- switch(type.width) {
- case 32:
- return LLVMFloatType();
- break;
- case 64:
- return LLVMDoubleType();
- break;
- default:
- assert(0);
- return LLVMFloatType();
- }
- }
- else {
- return LLVMIntType(type.width);
- }
-}
-
-
-LLVMTypeRef
-lp_build_vec_type(union lp_type type)
-{
- LLVMTypeRef elem_type = lp_build_elem_type(type);
- return LLVMVectorType(elem_type, type.length);
-}
-
-
/**
- * This function is a mirrot of lp_build_elem_type() above.
- *
- * XXX: I'm not sure if it wouldn't be easier/efficient to just recreate the
- * type and check for identity.
+ * Generate min(a, b)
+ * No checks for special case values of a or b = 1 or 0 are done.
*/
-boolean
-lp_check_elem_type(union lp_type type, LLVMTypeRef elem_type)
-{
- LLVMTypeKind elem_kind;
-
- assert(elem_type);
- if(!elem_type)
- return FALSE;
-
- elem_kind = LLVMGetTypeKind(elem_type);
-
- if (type.floating) {
- switch(type.width) {
- case 32:
- if(elem_kind != LLVMFloatTypeKind)
- return FALSE;
- break;
- case 64:
- if(elem_kind != LLVMDoubleTypeKind)
- return FALSE;
- break;
- default:
- assert(0);
- return FALSE;
- }
- }
- else {
- if(elem_kind != LLVMIntegerTypeKind)
- return FALSE;
-
- if(LLVMGetIntTypeWidth(elem_type) != type.width)
- return FALSE;
- }
-
- return TRUE;
-}
-
-
-boolean
-lp_check_vec_type(union lp_type type, LLVMTypeRef vec_type)
-{
- LLVMTypeRef elem_type;
-
- assert(vec_type);
- if(!vec_type)
- return FALSE;
-
- if(LLVMGetTypeKind(vec_type) != LLVMVectorTypeKind)
- return FALSE;
-
- if(LLVMGetVectorSize(vec_type) != type.length)
- return FALSE;
-
- elem_type = LLVMGetElementType(vec_type);
-
- return lp_check_elem_type(type, elem_type);
-}
-
-
-boolean
-lp_check_value(union lp_type type, LLVMValueRef val)
-{
- LLVMTypeRef vec_type;
-
- assert(val);
- if(!val)
- return FALSE;
-
- vec_type = LLVMTypeOf(val);
-
- return lp_check_vec_type(type, vec_type);
-}
-
-
-LLVMValueRef
-lp_build_undef(union lp_type type)
-{
- LLVMTypeRef vec_type = lp_build_vec_type(type);
- return LLVMGetUndef(vec_type);
-}
-
-
-LLVMValueRef
-lp_build_zero(union lp_type type)
-{
- LLVMTypeRef vec_type = lp_build_vec_type(type);
- return LLVMConstNull(vec_type);
-}
-
-
-LLVMValueRef
-lp_build_one(union lp_type type)
-{
- LLVMTypeRef elem_type;
- LLVMValueRef elems[LP_MAX_VECTOR_LENGTH];
- unsigned i;
-
- assert(type.length <= LP_MAX_VECTOR_LENGTH);
-
- elem_type = lp_build_elem_type(type);
-
- if(type.floating)
- elems[0] = LLVMConstReal(elem_type, 1.0);
- else if(type.fixed)
- elems[0] = LLVMConstInt(elem_type, 1LL << (type.width/2), 0);
- else if(!type.norm)
- elems[0] = LLVMConstInt(elem_type, 1, 0);
- else {
- /* special case' -- 1.0 for normalized types is more easily attained if
- * we start with a vector consisting of all bits set */
- LLVMTypeRef vec_type = LLVMVectorType(elem_type, type.length);
- LLVMValueRef vec = LLVMConstAllOnes(vec_type);
-
- if(type.sign)
- vec = LLVMConstLShr(vec, LLVMConstInt(LLVMInt32Type(), 1, 0));
-
- return vec;
- }
-
- for(i = 1; i < type.length; ++i)
- elems[i] = elems[0];
-
- return LLVMConstVector(elems, type.length);
-}
-
-
-LLVMValueRef
-lp_build_const_aos(union lp_type type,
- double r, double g, double b, double a,
- const unsigned char *swizzle)
-{
- const unsigned char default_swizzle[4] = {0, 1, 2, 3};
- LLVMTypeRef elem_type;
- LLVMValueRef elems[LP_MAX_VECTOR_LENGTH];
- unsigned i;
-
- assert(type.length % 4 == 0);
- assert(type.length <= LP_MAX_VECTOR_LENGTH);
-
- elem_type = lp_build_elem_type(type);
-
- if(swizzle == NULL)
- swizzle = default_swizzle;
-
- if(type.floating) {
- elems[swizzle[0]] = LLVMConstReal(elem_type, r);
- elems[swizzle[1]] = LLVMConstReal(elem_type, g);
- elems[swizzle[2]] = LLVMConstReal(elem_type, b);
- elems[swizzle[3]] = LLVMConstReal(elem_type, a);
- }
- else {
- unsigned shift;
- long long llscale;
- double dscale;
-
- if(type.fixed)
- shift = type.width/2;
- else if(type.norm)
- shift = type.sign ? type.width - 1 : type.width;
- else
- shift = 0;
-
- llscale = (long long)1 << shift;
- dscale = (double)llscale;
- assert((long long)dscale == llscale);
-
- elems[swizzle[0]] = LLVMConstInt(elem_type, r*dscale + 0.5, 0);
- elems[swizzle[1]] = LLVMConstInt(elem_type, g*dscale + 0.5, 0);
- elems[swizzle[2]] = LLVMConstInt(elem_type, b*dscale + 0.5, 0);
- elems[swizzle[3]] = LLVMConstInt(elem_type, a*dscale + 0.5, 0);
- }
-
- for(i = 4; i < type.length; ++i)
- elems[i] = elems[i % 4];
-
- return LLVMConstVector(elems, type.length);
-}
-
-
-static LLVMValueRef
-lp_build_intrinsic_binary(LLVMBuilderRef builder,
- const char *name,
- LLVMValueRef a,
- LLVMValueRef b)
-{
- LLVMModuleRef module = LLVMGetGlobalParent(LLVMGetBasicBlockParent(LLVMGetInsertBlock(builder)));
- LLVMValueRef function;
- LLVMValueRef args[2];
-
- function = LLVMGetNamedFunction(module, name);
- if(!function) {
- LLVMTypeRef type = LLVMTypeOf(a);
- LLVMTypeRef arg_types[2];
- arg_types[0] = type;
- arg_types[1] = type;
- function = LLVMAddFunction(module, name, LLVMFunctionType(type, arg_types, 2, 0));
- LLVMSetFunctionCallConv(function, LLVMCCallConv);
- LLVMSetLinkage(function, LLVMExternalLinkage);
- }
- assert(LLVMIsDeclaration(function));
-
- args[0] = a;
- args[1] = b;
-
- return LLVMBuildCall(builder, function, args, 2, "");
-}
-
-
static LLVMValueRef
lp_build_min_simple(struct lp_build_context *bld,
LLVMValueRef a,
LLVMValueRef b)
{
- const union lp_type type = bld->type;
+ const struct lp_type type = bld->type;
const char *intrinsic = NULL;
LLVMValueRef cond;
#if defined(PIPE_ARCH_X86) || defined(PIPE_ARCH_X86_64)
if(type.width * type.length == 128) {
- if(type.floating)
+ if(type.floating) {
if(type.width == 32)
intrinsic = "llvm.x86.sse.min.ps";
if(type.width == 64)
intrinsic = "llvm.x86.sse2.min.pd";
+ }
else {
if(type.width == 8 && !type.sign)
intrinsic = "llvm.x86.sse2.pminu.b";
+ if(type.width == 8 && type.sign)
+ intrinsic = "llvm.x86.sse41.pminsb";
+ if(type.width == 16 && !type.sign)
+ intrinsic = "llvm.x86.sse41.pminuw";
if(type.width == 16 && type.sign)
intrinsic = "llvm.x86.sse2.pmins.w";
+ if(type.width == 32 && !type.sign)
+ intrinsic = "llvm.x86.sse41.pminud";
+ if(type.width == 32 && type.sign)
+ intrinsic = "llvm.x86.sse41.pminsd";
}
}
#endif
-
+
if(intrinsic)
- return lp_build_intrinsic_binary(bld->builder, intrinsic, a, b);
+ return lp_build_intrinsic_binary(bld->builder, intrinsic, lp_build_vec_type(bld->type), a, b);
- if(type.floating)
- cond = LLVMBuildFCmp(bld->builder, LLVMRealULT, a, b, "");
- else
- cond = LLVMBuildICmp(bld->builder, type.sign ? LLVMIntSLT : LLVMIntULT, a, b, "");
- return LLVMBuildSelect(bld->builder, cond, a, b, "");
+ cond = lp_build_cmp(bld, PIPE_FUNC_LESS, a, b);
+ return lp_build_select(bld, cond, a, b);
}
+/**
+ * Generate max(a, b)
+ * No checks for special case values of a or b = 1 or 0 are done.
+ */
static LLVMValueRef
lp_build_max_simple(struct lp_build_context *bld,
LLVMValueRef a,
LLVMValueRef b)
{
- const union lp_type type = bld->type;
+ const struct lp_type type = bld->type;
const char *intrinsic = NULL;
LLVMValueRef cond;
#if defined(PIPE_ARCH_X86) || defined(PIPE_ARCH_X86_64)
if(type.width * type.length == 128) {
- if(type.floating)
+ if(type.floating) {
if(type.width == 32)
intrinsic = "llvm.x86.sse.max.ps";
if(type.width == 64)
intrinsic = "llvm.x86.sse2.max.pd";
+ }
else {
if(type.width == 8 && !type.sign)
intrinsic = "llvm.x86.sse2.pmaxu.b";
+ if(type.width == 8 && type.sign)
+ intrinsic = "llvm.x86.sse41.pmaxsb";
+ if(type.width == 16 && !type.sign)
+ intrinsic = "llvm.x86.sse41.pmaxuw";
if(type.width == 16 && type.sign)
intrinsic = "llvm.x86.sse2.pmaxs.w";
+ if(type.width == 32 && !type.sign)
+ intrinsic = "llvm.x86.sse41.pmaxud";
+ if(type.width == 32 && type.sign)
+ intrinsic = "llvm.x86.sse41.pmaxsd";
}
}
#endif
if(intrinsic)
- return lp_build_intrinsic_binary(bld->builder, intrinsic, a, b);
+ return lp_build_intrinsic_binary(bld->builder, intrinsic, lp_build_vec_type(bld->type), a, b);
- if(type.floating)
- cond = LLVMBuildFCmp(bld->builder, LLVMRealULT, a, b, "");
- else
- cond = LLVMBuildICmp(bld->builder, type.sign ? LLVMIntSLT : LLVMIntULT, a, b, "");
- return LLVMBuildSelect(bld->builder, cond, b, a, "");
+ cond = lp_build_cmp(bld, PIPE_FUNC_GREATER, a, b);
+ return lp_build_select(bld, cond, a, b);
}
+/**
+ * Generate 1 - a, or ~a depending on bld->type.
+ */
LLVMValueRef
lp_build_comp(struct lp_build_context *bld,
LLVMValueRef a)
{
- const union lp_type type = bld->type;
+ const struct lp_type type = bld->type;
if(a == bld->one)
return bld->zero;
}
+/**
+ * Generate a + b
+ */
LLVMValueRef
lp_build_add(struct lp_build_context *bld,
LLVMValueRef a,
LLVMValueRef b)
{
- const union lp_type type = bld->type;
+ const struct lp_type type = bld->type;
LLVMValueRef res;
if(a == bld->zero)
#endif
if(intrinsic)
- return lp_build_intrinsic_binary(bld->builder, intrinsic, a, b);
+ return lp_build_intrinsic_binary(bld->builder, intrinsic, lp_build_vec_type(bld->type), a, b);
}
if(LLVMIsConstant(a) && LLVMIsConstant(b))
else
res = LLVMBuildAdd(bld->builder, a, b, "");
+ /* clamp to ceiling of 1.0 */
if(bld->type.norm && (bld->type.floating || bld->type.fixed))
res = lp_build_min_simple(bld, res, bld->one);
+ /* XXX clamp to floor of -1 or 0??? */
+
return res;
}
+/**
+ * Generate a - b
+ */
LLVMValueRef
lp_build_sub(struct lp_build_context *bld,
LLVMValueRef a,
LLVMValueRef b)
{
- const union lp_type type = bld->type;
+ const struct lp_type type = bld->type;
LLVMValueRef res;
if(b == bld->zero)
#endif
if(intrinsic)
- return lp_build_intrinsic_binary(bld->builder, intrinsic, a, b);
+ return lp_build_intrinsic_binary(bld->builder, intrinsic, lp_build_vec_type(bld->type), a, b);
}
if(LLVMIsConstant(a) && LLVMIsConstant(b))
}
+/**
+ * Build constant int vector of width 'n' and value 'c'.
+ */
static LLVMValueRef
lp_build_const_vec(LLVMTypeRef type, unsigned n, long long c)
{
}
+/**
+ * Generate a * b
+ */
LLVMValueRef
lp_build_mul(struct lp_build_context *bld,
LLVMValueRef a,
LLVMValueRef b)
{
- const union lp_type type = bld->type;
+ const struct lp_type type = bld->type;
if(a == bld->zero)
return bld->zero;
abh = lp_build_mul_u8n(bld->builder, ah, bh);
/* PACKUSWB */
- ab = lp_build_intrinsic_binary(bld->builder, "llvm.x86.sse2.packuswb.128" , abl, abh);
+ ab = lp_build_intrinsic_binary(bld->builder, "llvm.x86.sse2.packuswb.128" , i16x8, abl, abh);
/* NOP */
ab = LLVMBuildBitCast(bld->builder, ab, i8x16, "");
}
+/**
+ * Generate a / b
+ */
+LLVMValueRef
+lp_build_div(struct lp_build_context *bld,
+ LLVMValueRef a,
+ LLVMValueRef b)
+{
+ const struct lp_type type = bld->type;
+
+ if(a == bld->zero)
+ return bld->zero;
+ if(a == bld->one)
+ return lp_build_rcp(bld, b);
+ if(b == bld->zero)
+ return bld->undef;
+ if(b == bld->one)
+ return a;
+ if(a == bld->undef || b == bld->undef)
+ return bld->undef;
+
+ if(LLVMIsConstant(a) && LLVMIsConstant(b))
+ return LLVMConstFDiv(a, b);
+
+#if defined(PIPE_ARCH_X86) || defined(PIPE_ARCH_X86_64)
+ if(type.width == 32 && type.length == 4)
+ return lp_build_mul(bld, a, lp_build_rcp(bld, b));
+#endif
+
+ return LLVMBuildFDiv(bld->builder, a, b, "");
+}
+
+
+LLVMValueRef
+lp_build_lerp(struct lp_build_context *bld,
+ LLVMValueRef x,
+ LLVMValueRef v0,
+ LLVMValueRef v1)
+{
+ return lp_build_add(bld, v0, lp_build_mul(bld, x, lp_build_sub(bld, v1, v0)));
+}
+
+
+LLVMValueRef
+lp_build_lerp_2d(struct lp_build_context *bld,
+ LLVMValueRef x,
+ LLVMValueRef y,
+ LLVMValueRef v00,
+ LLVMValueRef v01,
+ LLVMValueRef v10,
+ LLVMValueRef v11)
+{
+ LLVMValueRef v0 = lp_build_lerp(bld, x, v00, v01);
+ LLVMValueRef v1 = lp_build_lerp(bld, x, v10, v11);
+ return lp_build_lerp(bld, y, v0, v1);
+}
+
+
+/**
+ * Generate min(a, b)
+ * Do checks for special cases.
+ */
LLVMValueRef
lp_build_min(struct lp_build_context *bld,
LLVMValueRef a,
if(a == bld->undef || b == bld->undef)
return bld->undef;
+ if(a == b)
+ return a;
+
if(bld->type.norm) {
if(a == bld->zero || b == bld->zero)
return bld->zero;
}
+/**
+ * Generate max(a, b)
+ * Do checks for special cases.
+ */
LLVMValueRef
lp_build_max(struct lp_build_context *bld,
LLVMValueRef a,
if(a == bld->undef || b == bld->undef)
return bld->undef;
+ if(a == b)
+ return a;
+
if(bld->type.norm) {
if(a == bld->one || b == bld->one)
return bld->one;
return lp_build_max_simple(bld, a, b);
}
+
+
+/**
+ * Generate abs(a)
+ */
+LLVMValueRef
+lp_build_abs(struct lp_build_context *bld,
+ LLVMValueRef a)
+{
+ const struct lp_type type = bld->type;
+ LLVMTypeRef vec_type = lp_build_vec_type(type);
+
+ if(!type.sign)
+ return a;
+
+ if(type.floating) {
+ /* Mask out the sign bit */
+ LLVMTypeRef int_vec_type = lp_build_int_vec_type(type);
+ LLVMValueRef mask = lp_build_int_const_scalar(type, ((unsigned long long)1 << type.width) - 1);
+ a = LLVMBuildBitCast(bld->builder, a, int_vec_type, "");
+ a = LLVMBuildAnd(bld->builder, a, mask, "");
+ a = LLVMBuildBitCast(bld->builder, a, vec_type, "");
+ return a;
+ }
+
+#if defined(PIPE_ARCH_X86) || defined(PIPE_ARCH_X86_64)
+ if(type.width*type.length == 128) {
+ switch(type.width) {
+ case 8:
+ return lp_build_intrinsic_unary(bld->builder, "llvm.x86.ssse3.pabs.b.128", vec_type, a);
+ case 16:
+ return lp_build_intrinsic_unary(bld->builder, "llvm.x86.ssse3.pabs.w.128", vec_type, a);
+ case 32:
+ return lp_build_intrinsic_unary(bld->builder, "llvm.x86.ssse3.pabs.d.128", vec_type, a);
+ }
+ }
+#endif
+
+ return lp_build_max(bld, a, LLVMBuildNeg(bld->builder, a, ""));
+}
+
+
+LLVMValueRef
+lp_build_sgn(struct lp_build_context *bld,
+ LLVMValueRef a)
+{
+ const struct lp_type type = bld->type;
+ LLVMTypeRef vec_type = lp_build_vec_type(type);
+ LLVMValueRef cond;
+ LLVMValueRef res;
+
+ /* Handle non-zero case */
+ if(!type.sign) {
+ /* if not zero then sign must be positive */
+ res = bld->one;
+ }
+ else if(type.floating) {
+ /* Take the sign bit and add it to 1 constant */
+ LLVMTypeRef int_vec_type = lp_build_int_vec_type(type);
+ LLVMValueRef mask = lp_build_int_const_scalar(type, (unsigned long long)1 << (type.width - 1));
+ LLVMValueRef sign;
+ LLVMValueRef one;
+ sign = LLVMBuildBitCast(bld->builder, a, int_vec_type, "");
+ sign = LLVMBuildAnd(bld->builder, sign, mask, "");
+ one = LLVMConstBitCast(bld->one, int_vec_type);
+ res = LLVMBuildOr(bld->builder, sign, one, "");
+ res = LLVMBuildBitCast(bld->builder, res, vec_type, "");
+ }
+ else
+ {
+ LLVMValueRef minus_one = lp_build_const_scalar(type, -1.0);
+ cond = lp_build_cmp(bld, PIPE_FUNC_GREATER, a, bld->zero);
+ res = lp_build_select(bld, cond, bld->one, minus_one);
+ }
+
+ /* Handle zero */
+ cond = lp_build_cmp(bld, PIPE_FUNC_EQUAL, a, bld->zero);
+ res = lp_build_select(bld, cond, bld->zero, bld->one);
+
+ return res;
+}
+
+
+enum lp_build_round_sse41_mode
+{
+ LP_BUILD_ROUND_SSE41_NEAREST = 0,
+ LP_BUILD_ROUND_SSE41_FLOOR = 1,
+ LP_BUILD_ROUND_SSE41_CEIL = 2,
+ LP_BUILD_ROUND_SSE41_TRUNCATE = 3
+};
+
+
+static INLINE LLVMValueRef
+lp_build_round_sse41(struct lp_build_context *bld,
+ LLVMValueRef a,
+ enum lp_build_round_sse41_mode mode)
+{
+ const struct lp_type type = bld->type;
+ LLVMTypeRef vec_type = lp_build_vec_type(type);
+ const char *intrinsic;
+
+ assert(type.floating);
+ assert(type.width*type.length == 128);
+
+ switch(type.width) {
+ case 32:
+ intrinsic = "llvm.x86.sse41.round.ps";
+ break;
+ case 64:
+ intrinsic = "llvm.x86.sse41.round.pd";
+ break;
+ default:
+ assert(0);
+ return bld->undef;
+ }
+
+ return lp_build_intrinsic_binary(bld->builder, intrinsic, vec_type, a,
+ LLVMConstInt(LLVMInt32Type(), mode, 0));
+}
+
+
+LLVMValueRef
+lp_build_round(struct lp_build_context *bld,
+ LLVMValueRef a)
+{
+ const struct lp_type type = bld->type;
+
+ assert(type.floating);
+
+#if defined(PIPE_ARCH_X86) || defined(PIPE_ARCH_X86_64)
+ return lp_build_round_sse41(bld, a, LP_BUILD_ROUND_SSE41_NEAREST);
+#endif
+
+ /* FIXME */
+ assert(0);
+ return bld->undef;
+}
+
+
+LLVMValueRef
+lp_build_floor(struct lp_build_context *bld,
+ LLVMValueRef a)
+{
+ const struct lp_type type = bld->type;
+
+ assert(type.floating);
+
+#if defined(PIPE_ARCH_X86) || defined(PIPE_ARCH_X86_64)
+ return lp_build_round_sse41(bld, a, LP_BUILD_ROUND_SSE41_FLOOR);
+#endif
+
+ /* FIXME */
+ assert(0);
+ return bld->undef;
+}
+
+
+LLVMValueRef
+lp_build_ceil(struct lp_build_context *bld,
+ LLVMValueRef a)
+{
+ const struct lp_type type = bld->type;
+
+ assert(type.floating);
+
+#if defined(PIPE_ARCH_X86) || defined(PIPE_ARCH_X86_64)
+ return lp_build_round_sse41(bld, a, LP_BUILD_ROUND_SSE41_CEIL);
+#endif
+
+ /* FIXME */
+ assert(0);
+ return bld->undef;
+}
+
+
+LLVMValueRef
+lp_build_trunc(struct lp_build_context *bld,
+ LLVMValueRef a)
+{
+ const struct lp_type type = bld->type;
+
+ assert(type.floating);
+
+#if defined(PIPE_ARCH_X86) || defined(PIPE_ARCH_X86_64)
+ return lp_build_round_sse41(bld, a, LP_BUILD_ROUND_SSE41_TRUNCATE);
+#endif
+
+ /* FIXME */
+ assert(0);
+ return bld->undef;
+}
+
+
+/**
+ * Convert to integer, through whichever rounding method that's fastest,
+ * typically truncating to zero.
+ */
+LLVMValueRef
+lp_build_int(struct lp_build_context *bld,
+ LLVMValueRef a)
+{
+ const struct lp_type type = bld->type;
+ LLVMTypeRef int_vec_type = lp_build_int_vec_type(type);
+
+ assert(type.floating);
+
+ return LLVMBuildFPToSI(bld->builder, a, int_vec_type, "");
+}
+
+
+LLVMValueRef
+lp_build_ifloor(struct lp_build_context *bld,
+ LLVMValueRef a)
+{
+ a = lp_build_floor(bld, a);
+ a = lp_build_int(bld, a);
+ return a;
+}
+
+
+LLVMValueRef
+lp_build_sqrt(struct lp_build_context *bld,
+ LLVMValueRef a)
+{
+ const struct lp_type type = bld->type;
+ LLVMTypeRef vec_type = lp_build_vec_type(type);
+ char intrinsic[32];
+
+ /* TODO: optimize the constant case */
+ /* TODO: optimize the constant case */
+
+ assert(type.floating);
+ util_snprintf(intrinsic, sizeof intrinsic, "llvm.sqrt.v%uf%u", type.length, type.width);
+
+ return lp_build_intrinsic_unary(bld->builder, intrinsic, vec_type, a);
+}
+
+
+LLVMValueRef
+lp_build_rcp(struct lp_build_context *bld,
+ LLVMValueRef a)
+{
+ const struct lp_type type = bld->type;
+
+ if(a == bld->zero)
+ return bld->undef;
+ if(a == bld->one)
+ return bld->one;
+ if(a == bld->undef)
+ return bld->undef;
+
+ assert(type.floating);
+
+ if(LLVMIsConstant(a))
+ return LLVMConstFDiv(bld->one, a);
+
+ /* XXX: is this really necessary? */
+#if defined(PIPE_ARCH_X86) || defined(PIPE_ARCH_X86_64)
+ if(type.width == 32 && type.length == 4)
+ return lp_build_intrinsic_unary(bld->builder, "llvm.x86.sse.rcp.ps", lp_build_vec_type(type), a);
+#endif
+
+ return LLVMBuildFDiv(bld->builder, bld->one, a, "");
+}
+
+
+/**
+ * Generate 1/sqrt(a)
+ */
+LLVMValueRef
+lp_build_rsqrt(struct lp_build_context *bld,
+ LLVMValueRef a)
+{
+ const struct lp_type type = bld->type;
+
+ assert(type.floating);
+
+ /* XXX: is this really necessary? */
+#if defined(PIPE_ARCH_X86) || defined(PIPE_ARCH_X86_64)
+ if(type.width == 32 && type.length == 4)
+ return lp_build_intrinsic_unary(bld->builder, "llvm.x86.sse.rsqrt.ps", lp_build_vec_type(type), a);
+#endif
+
+ return lp_build_rcp(bld, lp_build_sqrt(bld, a));
+}
+
+
+/**
+ * Generate cos(a)
+ */
+LLVMValueRef
+lp_build_cos(struct lp_build_context *bld,
+ LLVMValueRef a)
+{
+ const struct lp_type type = bld->type;
+ LLVMTypeRef vec_type = lp_build_vec_type(type);
+ char intrinsic[32];
+
+ /* TODO: optimize the constant case */
+
+ assert(type.floating);
+ util_snprintf(intrinsic, sizeof intrinsic, "llvm.cos.v%uf%u", type.length, type.width);
+
+ return lp_build_intrinsic_unary(bld->builder, intrinsic, vec_type, a);
+}
+
+
+/**
+ * Generate sin(a)
+ */
+LLVMValueRef
+lp_build_sin(struct lp_build_context *bld,
+ LLVMValueRef a)
+{
+ const struct lp_type type = bld->type;
+ LLVMTypeRef vec_type = lp_build_vec_type(type);
+ char intrinsic[32];
+
+ /* TODO: optimize the constant case */
+
+ assert(type.floating);
+ util_snprintf(intrinsic, sizeof intrinsic, "llvm.sin.v%uf%u", type.length, type.width);
+
+ return lp_build_intrinsic_unary(bld->builder, intrinsic, vec_type, a);
+}
+
+
+/**
+ * Generate pow(x, y)
+ */
+LLVMValueRef
+lp_build_pow(struct lp_build_context *bld,
+ LLVMValueRef x,
+ LLVMValueRef y)
+{
+ /* TODO: optimize the constant case */
+ if(LLVMIsConstant(x) && LLVMIsConstant(y))
+ debug_printf("%s: inefficient/imprecise constant arithmetic\n");
+
+ return lp_build_exp2(bld, lp_build_mul(bld, lp_build_log2(bld, x), y));
+}
+
+
+/**
+ * Generate exp(x)
+ */
+LLVMValueRef
+lp_build_exp(struct lp_build_context *bld,
+ LLVMValueRef x)
+{
+ /* log2(e) = 1/log(2) */
+ LLVMValueRef log2e = lp_build_const_scalar(bld->type, 1.4426950408889634);
+
+ return lp_build_mul(bld, log2e, lp_build_exp2(bld, x));
+}
+
+
+/**
+ * Generate log(x)
+ */
+LLVMValueRef
+lp_build_log(struct lp_build_context *bld,
+ LLVMValueRef x)
+{
+ /* log(2) */
+ LLVMValueRef log2 = lp_build_const_scalar(bld->type, 1.4426950408889634);
+
+ return lp_build_mul(bld, log2, lp_build_exp2(bld, x));
+}
+
+
+#define EXP_POLY_DEGREE 3
+#define LOG_POLY_DEGREE 5
+
+
+/**
+ * Generate polynomial.
+ * Ex: x^2 * coeffs[0] + x * coeffs[1] + coeffs[2].
+ */
+static LLVMValueRef
+lp_build_polynomial(struct lp_build_context *bld,
+ LLVMValueRef x,
+ const double *coeffs,
+ unsigned num_coeffs)
+{
+ const struct lp_type type = bld->type;
+ LLVMValueRef res = NULL;
+ unsigned i;
+
+ /* TODO: optimize the constant case */
+ if(LLVMIsConstant(x))
+ debug_printf("%s: inefficient/imprecise constant arithmetic\n");
+
+ for (i = num_coeffs; i--; ) {
+ LLVMValueRef coeff = lp_build_const_scalar(type, coeffs[i]);
+ if(res)
+ res = lp_build_add(bld, coeff, lp_build_mul(bld, x, res));
+ else
+ res = coeff;
+ }
+
+ if(res)
+ return res;
+ else
+ return bld->undef;
+}
+
+
+/**
+ * Minimax polynomial fit of 2**x, in range [-0.5, 0.5[
+ */
+const double lp_build_exp2_polynomial[] = {
+#if EXP_POLY_DEGREE == 5
+ 9.9999994e-1, 6.9315308e-1, 2.4015361e-1, 5.5826318e-2, 8.9893397e-3, 1.8775767e-3
+#elif EXP_POLY_DEGREE == 4
+ 1.0000026, 6.9300383e-1, 2.4144275e-1, 5.2011464e-2, 1.3534167e-2
+#elif EXP_POLY_DEGREE == 3
+ 9.9992520e-1, 6.9583356e-1, 2.2606716e-1, 7.8024521e-2
+#elif EXP_POLY_DEGREE == 2
+ 1.0017247, 6.5763628e-1, 3.3718944e-1
+#else
+#error
+#endif
+};
+
+
+void
+lp_build_exp2_approx(struct lp_build_context *bld,
+ LLVMValueRef x,
+ LLVMValueRef *p_exp2_int_part,
+ LLVMValueRef *p_frac_part,
+ LLVMValueRef *p_exp2)
+{
+ const struct lp_type type = bld->type;
+ LLVMTypeRef vec_type = lp_build_vec_type(type);
+ LLVMTypeRef int_vec_type = lp_build_int_vec_type(type);
+ LLVMValueRef ipart = NULL;
+ LLVMValueRef fpart = NULL;
+ LLVMValueRef expipart = NULL;
+ LLVMValueRef expfpart = NULL;
+ LLVMValueRef res = NULL;
+
+ if(p_exp2_int_part || p_frac_part || p_exp2) {
+ /* TODO: optimize the constant case */
+ if(LLVMIsConstant(x))
+ debug_printf("%s: inefficient/imprecise constant arithmetic\n");
+
+ assert(type.floating && type.width == 32);
+
+ x = lp_build_min(bld, x, lp_build_const_scalar(type, 129.0));
+ x = lp_build_max(bld, x, lp_build_const_scalar(type, -126.99999));
+
+ /* ipart = int(x - 0.5) */
+ ipart = LLVMBuildSub(bld->builder, x, lp_build_const_scalar(type, 0.5f), "");
+ ipart = LLVMBuildFPToSI(bld->builder, ipart, int_vec_type, "");
+
+ /* fpart = x - ipart */
+ fpart = LLVMBuildSIToFP(bld->builder, ipart, vec_type, "");
+ fpart = LLVMBuildSub(bld->builder, x, fpart, "");
+ }
+
+ if(p_exp2_int_part || p_exp2) {
+ /* expipart = (float) (1 << ipart) */
+ expipart = LLVMBuildAdd(bld->builder, ipart, lp_build_int_const_scalar(type, 127), "");
+ expipart = LLVMBuildShl(bld->builder, expipart, lp_build_int_const_scalar(type, 23), "");
+ expipart = LLVMBuildBitCast(bld->builder, expipart, vec_type, "");
+ }
+
+ if(p_exp2) {
+ expfpart = lp_build_polynomial(bld, fpart, lp_build_exp2_polynomial,
+ Elements(lp_build_exp2_polynomial));
+
+ res = LLVMBuildMul(bld->builder, expipart, expfpart, "");
+ }
+
+ if(p_exp2_int_part)
+ *p_exp2_int_part = expipart;
+
+ if(p_frac_part)
+ *p_frac_part = fpart;
+
+ if(p_exp2)
+ *p_exp2 = res;
+}
+
+
+LLVMValueRef
+lp_build_exp2(struct lp_build_context *bld,
+ LLVMValueRef x)
+{
+ LLVMValueRef res;
+ lp_build_exp2_approx(bld, x, NULL, NULL, &res);
+ return res;
+}
+
+
+/**
+ * Minimax polynomial fit of log2(x)/(x - 1), for x in range [1, 2[
+ * These coefficients can be generate with
+ * http://www.boost.org/doc/libs/1_36_0/libs/math/doc/sf_and_dist/html/math_toolkit/toolkit/internals2/minimax.html
+ */
+const double lp_build_log2_polynomial[] = {
+#if LOG_POLY_DEGREE == 6
+ 3.11578814719469302614, -3.32419399085241980044, 2.59883907202499966007, -1.23152682416275988241, 0.318212422185251071475, -0.0344359067839062357313
+#elif LOG_POLY_DEGREE == 5
+ 2.8882704548164776201, -2.52074962577807006663, 1.48116647521213171641, -0.465725644288844778798, 0.0596515482674574969533
+#elif LOG_POLY_DEGREE == 4
+ 2.61761038894603480148, -1.75647175389045657003, 0.688243882994381274313, -0.107254423828329604454
+#elif LOG_POLY_DEGREE == 3
+ 2.28330284476918490682, -1.04913055217340124191, 0.204446009836232697516
+#else
+#error
+#endif
+};
+
+
+/**
+ * See http://www.devmaster.net/forums/showthread.php?p=43580
+ */
+void
+lp_build_log2_approx(struct lp_build_context *bld,
+ LLVMValueRef x,
+ LLVMValueRef *p_exp,
+ LLVMValueRef *p_floor_log2,
+ LLVMValueRef *p_log2)
+{
+ const struct lp_type type = bld->type;
+ LLVMTypeRef vec_type = lp_build_vec_type(type);
+ LLVMTypeRef int_vec_type = lp_build_int_vec_type(type);
+
+ LLVMValueRef expmask = lp_build_int_const_scalar(type, 0x7f800000);
+ LLVMValueRef mantmask = lp_build_int_const_scalar(type, 0x007fffff);
+ LLVMValueRef one = LLVMConstBitCast(bld->one, int_vec_type);
+
+ LLVMValueRef i = NULL;
+ LLVMValueRef exp = NULL;
+ LLVMValueRef mant = NULL;
+ LLVMValueRef logexp = NULL;
+ LLVMValueRef logmant = NULL;
+ LLVMValueRef res = NULL;
+
+ if(p_exp || p_floor_log2 || p_log2) {
+ /* TODO: optimize the constant case */
+ if(LLVMIsConstant(x))
+ debug_printf("%s: inefficient/imprecise constant arithmetic\n");
+
+ assert(type.floating && type.width == 32);
+
+ i = LLVMBuildBitCast(bld->builder, x, int_vec_type, "");
+
+ /* exp = (float) exponent(x) */
+ exp = LLVMBuildAnd(bld->builder, i, expmask, "");
+ }
+
+ if(p_floor_log2 || p_log2) {
+ logexp = LLVMBuildLShr(bld->builder, exp, lp_build_int_const_scalar(type, 23), "");
+ logexp = LLVMBuildSub(bld->builder, logexp, lp_build_int_const_scalar(type, 127), "");
+ logexp = LLVMBuildSIToFP(bld->builder, logexp, vec_type, "");
+ }
+
+ if(p_log2) {
+ /* mant = (float) mantissa(x) */
+ mant = LLVMBuildAnd(bld->builder, i, mantmask, "");
+ mant = LLVMBuildOr(bld->builder, mant, one, "");
+ mant = LLVMBuildSIToFP(bld->builder, mant, vec_type, "");
+
+ logmant = lp_build_polynomial(bld, mant, lp_build_log2_polynomial,
+ Elements(lp_build_log2_polynomial));
+
+ /* This effectively increases the polynomial degree by one, but ensures that log2(1) == 0*/
+ logmant = LLVMBuildMul(bld->builder, logmant, LLVMBuildMul(bld->builder, mant, bld->one, ""), "");
+
+ res = LLVMBuildAdd(bld->builder, logmant, logexp, "");
+ }
+
+ if(p_exp)
+ *p_exp = exp;
+
+ if(p_floor_log2)
+ *p_floor_log2 = logexp;
+
+ if(p_log2)
+ *p_log2 = res;
+}
+
+
+LLVMValueRef
+lp_build_log2(struct lp_build_context *bld,
+ LLVMValueRef x)
+{
+ LLVMValueRef res;
+ lp_build_log2_approx(bld, x, NULL, NULL, &res);
+ return res;
+}
projects / mesa.git / blobdiff