#include "util/u_debug.h"
#include "util/u_math.h"
+#include "util/u_cpu_detect.h"
#include "lp_bld_type.h"
#include "lp_bld_const.h"
#include "lp_bld_intr.h"
#include "lp_bld_arit.h"
+#include "lp_bld_pack.h"
#include "lp_bld_conv.h"
*/
LLVMValueRef
lp_build_clamped_float_to_unsigned_norm(LLVMBuilderRef builder,
- union lp_type src_type,
+ struct lp_type src_type,
unsigned dst_width,
LLVMValueRef src)
{
scale = (double)mask/ubound;
bias = (double)((unsigned long long)1 << (mantissa - n));
- res = LLVMBuildMul(builder, src, lp_build_const_uni(src_type, scale), "");
- res = LLVMBuildAdd(builder, res, lp_build_const_uni(src_type, bias), "");
+ res = LLVMBuildMul(builder, src, lp_build_const_scalar(src_type, scale), "");
+ res = LLVMBuildAdd(builder, res, lp_build_const_scalar(src_type, bias), "");
res = LLVMBuildBitCast(builder, res, int_vec_type, "");
if(dst_width > n) {
int shift = dst_width - n;
- res = LLVMBuildShl(builder, res, lp_build_int_const_uni(src_type, shift), "");
+ res = LLVMBuildShl(builder, res, lp_build_int_const_scalar(src_type, shift), "");
- /* Fill in the empty lower bits for added precision? */
+ /* TODO: Fill in the empty lower bits for additional precision? */
#if 0
{
LLVMValueRef msb;
- msb = LLVMBuildLShr(builder, res, lp_build_int_const_uni(src_type, dst_width - 1), "");
- msb = LLVMBuildShl(builder, msb, lp_build_int_const_uni(src_type, shift), "");
- msb = LLVMBuildSub(builder, msb, lp_build_int_const_uni(src_type, 1), "");
+ msb = LLVMBuildLShr(builder, res, lp_build_int_const_scalar(src_type, dst_width - 1), "");
+ msb = LLVMBuildShl(builder, msb, lp_build_int_const_scalar(src_type, shift), "");
+ msb = LLVMBuildSub(builder, msb, lp_build_int_const_scalar(src_type, 1), "");
res = LLVMBuildOr(builder, res, msb, "");
}
#elif 0
while(shift > 0) {
- res = LLVMBuildOr(builder, res, LLVMBuildLShr(builder, res, lp_build_int_const_uni(src_type, n), ""), "");
+ res = LLVMBuildOr(builder, res, LLVMBuildLShr(builder, res, lp_build_int_const_scalar(src_type, n), ""), "");
shift -= n;
n *= 2;
}
#endif
}
else
- res = LLVMBuildAnd(builder, res, lp_build_int_const_uni(src_type, mask), "");
+ res = LLVMBuildAnd(builder, res, lp_build_int_const_scalar(src_type, mask), "");
return res;
}
LLVMValueRef
lp_build_unsigned_norm_to_float(LLVMBuilderRef builder,
unsigned src_width,
- union lp_type dst_type,
+ struct lp_type dst_type,
LLVMValueRef src)
{
LLVMTypeRef vec_type = lp_build_vec_type(dst_type);
if(src_width > mantissa) {
int shift = src_width - mantissa;
- res = LLVMBuildLShr(builder, res, lp_build_int_const_uni(dst_type, shift), "");
+ res = LLVMBuildLShr(builder, res, lp_build_int_const_scalar(dst_type, shift), "");
}
- bias_ = lp_build_const_uni(dst_type, bias);
+ bias_ = lp_build_const_scalar(dst_type, bias);
res = LLVMBuildOr(builder,
res,
res = LLVMBuildBitCast(builder, res, vec_type, "");
res = LLVMBuildSub(builder, res, bias_, "");
- res = LLVMBuildMul(builder, res, lp_build_const_uni(dst_type, scale), "");
+ res = LLVMBuildMul(builder, res, lp_build_const_scalar(dst_type, scale), "");
return res;
}
-/**
- * Build shuffle vectors that match PUNPCKLxx and PUNPCKHxx instructions.
- */
-static LLVMValueRef
-lp_build_const_unpack_shuffle(unsigned n, unsigned lo_hi)
-{
- LLVMValueRef elems[LP_MAX_VECTOR_LENGTH];
- unsigned i, j;
-
- assert(n <= LP_MAX_VECTOR_LENGTH);
- assert(lo_hi < 2);
-
- /* TODO: cache results in a static table */
-
- for(i = 0, j = lo_hi*n/2; i < n; i += 2, ++j) {
- elems[i + 0] = LLVMConstInt(LLVMInt32Type(), 0 + j, 0);
- elems[i + 1] = LLVMConstInt(LLVMInt32Type(), n + j, 0);
- }
-
- return LLVMConstVector(elems, n);
-}
-
-
-/**
- * Build shuffle vectors that match PACKxx instructions.
- */
-static LLVMValueRef
-lp_build_const_pack_shuffle(unsigned n)
-{
- LLVMValueRef elems[LP_MAX_VECTOR_LENGTH];
- unsigned i;
-
- assert(n <= LP_MAX_VECTOR_LENGTH);
-
- /* TODO: cache results in a static table */
-
- for(i = 0; i < n; ++i)
- elems[i] = LLVMConstInt(LLVMInt32Type(), 2*i, 0);
-
- return LLVMConstVector(elems, n);
-}
-
-
-/**
- * Expand the bit width.
- *
- * This will only change the number of bits the values are represented, not the
- * values themselved.
- */
-static void
-lp_build_expand(LLVMBuilderRef builder,
- union lp_type src_type,
- union lp_type dst_type,
- LLVMValueRef src,
- LLVMValueRef *dst, unsigned num_dsts)
-{
- unsigned num_tmps;
- unsigned i;
-
- /* Register width must remain constant */
- assert(src_type.width * src_type.length == dst_type.width * dst_type.length);
-
- /* We must not loose or gain channels. Only precision */
- assert(src_type.length == dst_type.length * num_dsts);
-
- num_tmps = 1;
- dst[0] = src;
-
- while(src_type.width < dst_type.width) {
- union lp_type new_type = src_type;
- LLVMTypeRef new_vec_type;
-
- new_type.width *= 2;
- new_type.length /= 2;
- new_vec_type = lp_build_vec_type(new_type);
-
- for(i = num_tmps; i--; ) {
- LLVMValueRef zero;
- LLVMValueRef shuffle_lo;
- LLVMValueRef shuffle_hi;
- LLVMValueRef lo;
- LLVMValueRef hi;
-
- zero = lp_build_zero(src_type);
- shuffle_lo = lp_build_const_unpack_shuffle(src_type.length, 0);
- shuffle_hi = lp_build_const_unpack_shuffle(src_type.length, 1);
-
- /* PUNPCKLBW, PUNPCKHBW */
- lo = LLVMBuildShuffleVector(builder, dst[i], zero, shuffle_lo, "");
- hi = LLVMBuildShuffleVector(builder, dst[i], zero, shuffle_hi, "");
-
- dst[2*i + 0] = LLVMBuildBitCast(builder, lo, new_vec_type, "");
- dst[2*i + 1] = LLVMBuildBitCast(builder, hi, new_vec_type, "");
- }
-
- src_type = new_type;
-
- num_tmps *= 2;
- }
-
- assert(num_tmps == num_dsts);
-}
-
-
-/**
- * Non-interleaved pack.
- *
- * This will move values as
- *
- * lo = __ l0 __ l1 __ l2 __.. __ ln
- * hi = __ h0 __ h1 __ h2 __.. __ hn
- * res = l0 l1 l2 .. ln h0 h1 h2 .. hn
- *
- * TODO: handle saturation consistently.
- */
-static LLVMValueRef
-lp_build_pack2(LLVMBuilderRef builder,
- union lp_type src_type,
- union lp_type dst_type,
- boolean clamped,
- LLVMValueRef lo,
- LLVMValueRef hi)
-{
- LLVMTypeRef src_vec_type = lp_build_vec_type(src_type);
- LLVMTypeRef dst_vec_type = lp_build_vec_type(dst_type);
- LLVMValueRef shuffle;
- LLVMValueRef res;
-
- /* Register width must remain constant */
- assert(src_type.width * src_type.length == dst_type.width * dst_type.length);
-
- /* We must not loose or gain channels. Only precision */
- assert(src_type.length * 2 == dst_type.length);
-
- assert(!src_type.floating);
- assert(!dst_type.floating);
-
-#if defined(PIPE_ARCH_X86) || defined(PIPE_ARCH_X86_64)
- if(src_type.width * src_type.length == 128) {
- /* All X86 non-interleaved pack instructions all take signed inputs and
- * saturate them, so saturate beforehand. */
- if(!src_type.sign && !clamped) {
- struct lp_build_context bld;
- unsigned dst_bits = dst_type.sign ? dst_type.width - 1 : dst_type.width;
- LLVMValueRef dst_max = lp_build_int_const_uni(src_type, ((unsigned long long)1 << dst_bits) - 1);
- lp_build_context_init(&bld, builder, src_type);
- lo = lp_build_min(&bld, lo, dst_max);
- hi = lp_build_min(&bld, hi, dst_max);
- }
-
- switch(src_type.width) {
- case 32:
- if(dst_type.sign)
- res = lp_build_intrinsic_binary(builder, "llvm.x86.sse2.packssdw.128", src_vec_type, lo, hi);
- else
- /* PACKUSDW is the only instrinsic with a consistent signature */
- return lp_build_intrinsic_binary(builder, "llvm.x86.sse41.packusdw", dst_vec_type, lo, hi);
- break;
-
- case 16:
- if(dst_type.sign)
- res = lp_build_intrinsic_binary(builder, "llvm.x86.sse2.packsswb.128", src_vec_type, lo, hi);
- else
- res = lp_build_intrinsic_binary(builder, "llvm.x86.sse2.packuswb.128", src_vec_type, lo, hi);
- break;
-
- default:
- assert(0);
- return LLVMGetUndef(dst_vec_type);
- break;
- }
-
- res = LLVMBuildBitCast(builder, res, dst_vec_type, "");
- return res;
- }
-#endif
-
- lo = LLVMBuildBitCast(builder, lo, dst_vec_type, "");
- hi = LLVMBuildBitCast(builder, hi, dst_vec_type, "");
-
- shuffle = lp_build_const_pack_shuffle(dst_type.length);
-
- res = LLVMBuildShuffleVector(builder, lo, hi, shuffle, "");
-
- return res;
-}
-
-
-/**
- * Truncate the bit width.
- *
- * TODO: Handle saturation consistently.
- */
-static LLVMValueRef
-lp_build_trunc(LLVMBuilderRef builder,
- union lp_type src_type,
- union lp_type dst_type,
- boolean clamped,
- const LLVMValueRef *src, unsigned num_srcs)
-{
- LLVMValueRef tmp[LP_MAX_VECTOR_LENGTH];
- unsigned i;
-
- /* Register width must remain constant */
- assert(src_type.width * src_type.length == dst_type.width * dst_type.length);
-
- /* We must not loose or gain channels. Only precision */
- assert(src_type.length * num_srcs == dst_type.length);
-
- for(i = 0; i < num_srcs; ++i)
- tmp[i] = src[i];
-
- while(src_type.width > dst_type.width) {
- union lp_type new_type = src_type;
-
- new_type.width /= 2;
- new_type.length *= 2;
-
- /* Take in consideration the sign changes only in the last step */
- if(new_type.width == dst_type.width)
- new_type.sign = dst_type.sign;
-
- num_srcs /= 2;
-
- for(i = 0; i < num_srcs; ++i)
- tmp[i] = lp_build_pack2(builder, src_type, new_type, clamped,
- tmp[2*i + 0], tmp[2*i + 1]);
-
- src_type = new_type;
- }
-
- assert(num_srcs == 1);
-
- return tmp[0];
-}
-
-
/**
* Generic type conversion.
*
*/
void
lp_build_conv(LLVMBuilderRef builder,
- union lp_type src_type,
- union lp_type dst_type,
+ struct lp_type src_type,
+ struct lp_type dst_type,
const LLVMValueRef *src, unsigned num_srcs,
LLVMValueRef *dst, unsigned num_dsts)
{
- union lp_type tmp_type;
+ struct lp_type tmp_type;
LLVMValueRef tmp[LP_MAX_VECTOR_LENGTH];
unsigned num_tmps;
unsigned i;
* Clamp if necessary
*/
- if(src_type.value != dst_type.value) {
+ if(memcmp(&src_type, &dst_type, sizeof src_type) != 0) {
struct lp_build_context bld;
double src_min = lp_const_min(src_type);
double dst_min = lp_const_min(dst_type);
if(dst_min == 0.0)
thres = bld.zero;
else
- thres = lp_build_const_uni(src_type, dst_min);
+ thres = lp_build_const_scalar(src_type, dst_min);
for(i = 0; i < num_tmps; ++i)
tmp[i] = lp_build_max(&bld, tmp[i], thres);
}
if(dst_max == 1.0)
thres = bld.one;
else
- thres = lp_build_const_uni(src_type, dst_max);
+ thres = lp_build_const_scalar(src_type, dst_max);
for(i = 0; i < num_tmps; ++i)
tmp[i] = lp_build_min(&bld, tmp[i], thres);
}
LLVMTypeRef tmp_vec_type;
if (dst_scale != 1.0) {
- LLVMValueRef scale = lp_build_const_uni(tmp_type, dst_scale);
+ LLVMValueRef scale = lp_build_const_scalar(tmp_type, dst_scale);
for(i = 0; i < num_tmps; ++i)
tmp[i] = LLVMBuildMul(builder, tmp[i], scale, "");
}
/* FIXME: compensate different offsets too */
if(src_shift > dst_shift) {
- LLVMValueRef shift = lp_build_int_const_uni(tmp_type, src_shift - dst_shift);
+ LLVMValueRef shift = lp_build_int_const_scalar(tmp_type, src_shift - dst_shift);
for(i = 0; i < num_tmps; ++i)
if(src_type.sign)
tmp[i] = LLVMBuildAShr(builder, tmp[i], shift, "");
if(tmp_type.width > dst_type.width) {
assert(num_dsts == 1);
- tmp[0] = lp_build_trunc(builder, tmp_type, dst_type, TRUE, tmp, num_tmps);
+ tmp[0] = lp_build_pack(builder, tmp_type, dst_type, TRUE, tmp, num_tmps);
tmp_type.width = dst_type.width;
tmp_type.length = dst_type.length;
num_tmps = 1;
if(tmp_type.width < dst_type.width) {
assert(num_tmps == 1);
- lp_build_expand(builder, tmp_type, dst_type, tmp[0], tmp, num_dsts);
+ lp_build_unpack(builder, tmp_type, dst_type, tmp[0], tmp, num_dsts);
tmp_type.width = dst_type.width;
tmp_type.length = dst_type.length;
num_tmps = num_dsts;
}
if (src_scale != 1.0) {
- LLVMValueRef scale = lp_build_const_uni(tmp_type, 1.0/src_scale);
+ LLVMValueRef scale = lp_build_const_scalar(tmp_type, 1.0/src_scale);
for(i = 0; i < num_tmps; ++i)
tmp[i] = LLVMBuildMul(builder, tmp[i], scale, "");
}
/* FIXME: compensate different offsets too */
if(src_shift < dst_shift) {
- LLVMValueRef shift = lp_build_int_const_uni(tmp_type, dst_shift - src_shift);
+ LLVMValueRef shift = lp_build_int_const_scalar(tmp_type, dst_shift - src_shift);
for(i = 0; i < num_tmps; ++i)
tmp[i] = LLVMBuildShl(builder, tmp[i], shift, "");
}
*/
void
lp_build_conv_mask(LLVMBuilderRef builder,
- union lp_type src_type,
- union lp_type dst_type,
+ struct lp_type src_type,
+ struct lp_type dst_type,
const LLVMValueRef *src, unsigned num_srcs,
LLVMValueRef *dst, unsigned num_dsts)
{
if(src_type.width > dst_type.width) {
assert(num_dsts == 1);
- dst[0] = lp_build_trunc(builder, src_type, dst_type, TRUE, src, num_srcs);
+ dst[0] = lp_build_pack(builder, src_type, dst_type, TRUE, src, num_srcs);
}
else if(src_type.width < dst_type.width) {
assert(num_srcs == 1);
- lp_build_expand(builder, src_type, dst_type, src[0], dst, num_dsts);
+ lp_build_unpack(builder, src_type, dst_type, src[0], dst, num_dsts);
}
else {
assert(num_srcs == num_dsts);
projects / mesa.git / blobdiff