X-Git-Url: https://git.libre-soc.org/?a=blobdiff_plain;f=src%2Fgallium%2Fauxiliary%2Fgallivm%2Flp_bld_conv.c;h=712ce5f92dca1607db2d8db2a7ca88961a72f5f5;hb=5ae31d7e1d3d51c7843571c63aa228f8ca9b879f;hp=053f4132080887658b3c9adf4cc3403f67d503dc;hpb=b101a094b5576337f5bb7aecf525eb1755757b9c;p=mesa.git diff --git a/src/gallium/auxiliary/gallivm/lp_bld_conv.c b/src/gallium/auxiliary/gallivm/lp_bld_conv.c index 053f4132080..712ce5f92dc 100644 --- a/src/gallium/auxiliary/gallivm/lp_bld_conv.c +++ b/src/gallium/auxiliary/gallivm/lp_bld_conv.c @@ -75,400 +75,61 @@ #include "lp_bld_logic.h" #include "lp_bld_intr.h" #include "lp_bld_printf.h" +#include "lp_bld_format.h" -/** - * Byte swap on element. It will construct a call to intrinsic llvm.bswap - * based on the type. - * - * @param res element to byte swap. - * @param type int16_t, int32_t, int64_t, float or double - * @param - */ -LLVMValueRef -lp_build_bswap(struct gallivm_state *gallivm, - LLVMValueRef res, - struct lp_type type) -{ - LLVMTypeRef int_type = LLVMIntTypeInContext(gallivm->context, - type.width); - const char *intrinsic = NULL; - if (type.width == 8) - return res; - if (type.width == 16) - intrinsic = "llvm.bswap.i16"; - else if (type.width == 32) - intrinsic = "llvm.bswap.i32"; - else if (type.width == 64) - intrinsic = "llvm.bswap.i64"; - - assert (intrinsic != NULL); - - /* In case of a floating-point type cast to a int of same size and then - * cast back to fp type. - */ - if (type.floating) - res = LLVMBuildBitCast(gallivm->builder, res, int_type, ""); - res = lp_build_intrinsic_unary(gallivm->builder, intrinsic, int_type, res); - if (type.floating) - res = LLVMBuildBitCast(gallivm->builder, res, - lp_build_elem_type(gallivm, type), ""); - return res; -} - - -/** - * Byte swap every element in the vector. - * - * @param packed to convert - * @param src_type type of int16_t, int32_t, int64_t, float or - * double - * @param dst_type type to return - */ -LLVMValueRef -lp_build_bswap_vec(struct gallivm_state *gallivm, - LLVMValueRef packed, - struct lp_type src_type_vec, - struct lp_type dst_type_vec) -{ - LLVMBuilderRef builder = gallivm->builder; - LLVMTypeRef dst_type = lp_build_elem_type(gallivm, dst_type_vec); - LLVMValueRef res; - - if (src_type_vec.length == 1) { - res = lp_build_bswap(gallivm, packed, src_type_vec); - res = LLVMBuildBitCast(gallivm->builder, res, dst_type, ""); - } else { - unsigned i; - res = LLVMGetUndef(lp_build_vec_type(gallivm, dst_type_vec)); - for (i = 0; i < src_type_vec.length; ++i) { - LLVMValueRef index = lp_build_const_int32(gallivm, i); - LLVMValueRef elem = LLVMBuildExtractElement(builder, packed, index, ""); - elem = lp_build_bswap(gallivm, elem, src_type_vec); - elem = LLVMBuildBitCast(gallivm->builder, elem, dst_type, ""); - res = LLVMBuildInsertElement(gallivm->builder, res, elem, index, ""); - } - } - return res; -} - - -/** - * Convert float32 to a float-like value with less exponent and mantissa - * bits. The mantissa is still biased, and the mantissa still has an implied 1, - * but there's no sign bit. - * - * @param src (vector) float value to convert - * @param mantissa_bits the number of mantissa bits - * @param exponent_bits the number of exponent bits - * - * Unlike float_to_half using accurate method here. - * This implements round-towards-zero (trunc) hence too large numbers get - * converted to largest representable number, not infinity. - * Small numbers may get converted to denorms, depending on normal - * float denorm handling of the cpu. - * Note that compared to the references, below, we skip any rounding bias - * since we do rounding towards zero - OpenGL allows rounding towards zero - * (though not preferred) and DX10 even seems to require it. - * Note that this will not do any packing - the value will - * look like a "rescaled float" (except for Inf/NaN) but be returned - * as int32. - * - * ref http://fgiesen.wordpress.com/2012/03/28/half-to-float-done-quic/ - * ref https://gist.github.com/rygorous/2156668 - */ -static LLVMValueRef -lp_build_float_to_smallfloat_nosign(struct gallivm_state *gallivm, - struct lp_type i32_type, - LLVMValueRef src, - unsigned mantissa_bits, - unsigned exponent_bits) -{ - LLVMBuilderRef builder = gallivm->builder; - LLVMValueRef i32_floatexpmask, i32_smallexpmask, magic, normal; - LLVMValueRef clamped, tmp, i32_roundmask, small_max, src_abs; - LLVMValueRef is_nan, is_posinf, is_nan_or_posinf, i32_qnanbit, nan_or_posinf; - struct lp_type f32_type = lp_type_float_vec(32, 32 * i32_type.length); - struct lp_build_context f32_bld, i32_bld; - LLVMValueRef zero = lp_build_const_vec(gallivm, f32_type, 0.0f); - - lp_build_context_init(&f32_bld, gallivm, f32_type); - lp_build_context_init(&i32_bld, gallivm, i32_type); - - i32_smallexpmask = lp_build_const_int_vec(gallivm, i32_type, - ((1 << exponent_bits) - 1) << 23); - i32_floatexpmask = lp_build_const_int_vec(gallivm, i32_type, 0xff << 23); - - /* "ordinary" number */ - /* clamp to pos range (can still have sign bit if NaN or negative zero) */ - clamped = lp_build_max(&f32_bld, src, zero); - clamped = LLVMBuildBitCast(builder, clamped, i32_bld.vec_type, ""); - /* get rid of excess mantissa bits, and while here also potential sign bit */ - i32_roundmask = lp_build_const_int_vec(gallivm, i32_type, - ~((1 << (23 - mantissa_bits)) - 1) | - 0x7fffffff); - - tmp = lp_build_and(&i32_bld, clamped, i32_roundmask); - tmp = LLVMBuildBitCast(builder, tmp, f32_bld.vec_type, ""); - /* bias exponent (and denormalize if necessary) */ - magic = lp_build_const_int_vec(gallivm, i32_type, - ((1 << (exponent_bits - 1)) - 1) << 23); - magic = LLVMBuildBitCast(builder, magic, f32_bld.vec_type, ""); - normal = lp_build_mul(&f32_bld, tmp, magic); - - /* clamp to max value */ - small_max = lp_build_const_int_vec(gallivm, i32_type, - (((1 << exponent_bits) - 2) << 23) | - (((1 << mantissa_bits) - 1) << (23 - mantissa_bits))); - small_max = LLVMBuildBitCast(builder, small_max, f32_bld.vec_type, ""); - normal = lp_build_min(&f32_bld, normal, small_max); - normal = LLVMBuildBitCast(builder, normal, i32_bld.vec_type, ""); - - /* - * handle nan/inf cases - * a little bit tricky since -Inf -> 0, +Inf -> +Inf, +-Nan -> +Nan - * Note that on a lucky day, we could simplify this a bit, - * by just using the max(src, zero) result - this will have -Inf - * clamped to 0, and MIGHT preserve the NaNs. - */ - src_abs = lp_build_abs(&f32_bld, src); - src_abs = LLVMBuildBitCast(builder, src_abs, i32_bld.vec_type, ""); - src = LLVMBuildBitCast(builder, src, i32_bld.vec_type, ""); - is_nan = lp_build_compare(gallivm, i32_type, PIPE_FUNC_GREATER, - src_abs, i32_floatexpmask); - is_posinf = lp_build_compare(gallivm, i32_type, PIPE_FUNC_EQUAL, - src, i32_floatexpmask); - is_nan_or_posinf = lp_build_and(&i32_bld, is_nan, is_posinf); - /* could also set more mantissa bits but need at least the highest mantissa bit */ - i32_qnanbit = lp_build_const_vec(gallivm, i32_type, 1 << 22); - /* combine maxexp with qnanbit */ - nan_or_posinf = lp_build_or(&i32_bld, i32_smallexpmask, - lp_build_and(&i32_bld, is_nan, i32_qnanbit)); - - return lp_build_select(&i32_bld, is_nan_or_posinf, nan_or_posinf, normal); -} - - -/** - * Convert rgba float SoA values to packed r11g11b10 values. - * - * @param src SoA float (vector) values to convert. - */ -LLVMValueRef -lp_build_float_to_r11g11b10(struct gallivm_state *gallivm, - LLVMValueRef *src) -{ - LLVMValueRef dst, rcomp, bcomp, gcomp, shift, mask; - struct lp_build_context i32_bld; - LLVMTypeRef src_type = LLVMTypeOf(*src); - unsigned src_length = LLVMGetTypeKind(src_type) == LLVMVectorTypeKind ? - LLVMGetVectorSize(src_type) : 1; - struct lp_type i32_type = lp_type_int_vec(32, 32 * src_length); - - lp_build_context_init(&i32_bld, gallivm, i32_type); - - /* "rescale" - this does the actual conversion except the packing */ - rcomp = lp_build_float_to_smallfloat_nosign(gallivm, i32_type, src[0], 6, 5); - gcomp = lp_build_float_to_smallfloat_nosign(gallivm, i32_type, src[1], 6, 5); - bcomp = lp_build_float_to_smallfloat_nosign(gallivm, i32_type, src[2], 5, 5); - - /* pack rescaled SoA floats to r11g11b10 AoS values */ - shift = lp_build_const_int_vec(gallivm, i32_type, 23 - 6); - rcomp = lp_build_shr(&i32_bld, rcomp, shift); - - shift = lp_build_const_int_vec(gallivm, i32_type, 23 - 17); - mask = lp_build_const_int_vec(gallivm, i32_type, 0x7ff << 11); - gcomp = lp_build_shr(&i32_bld, gcomp, shift); - gcomp = lp_build_and(&i32_bld, gcomp, mask); - - shift = lp_build_const_int_vec(gallivm, i32_type, 27 - 23); - mask = lp_build_const_int_vec(gallivm, i32_type, 0x3ff << 22); - bcomp = lp_build_shl(&i32_bld, bcomp, shift); - bcomp = lp_build_and(&i32_bld, bcomp, mask); - - dst = lp_build_or(&i32_bld, rcomp, gcomp); - return lp_build_or(&i32_bld, dst, bcomp); -} - - -/** - * Convert a float-like value with less exponent and mantissa - * bits than a normal float32 to a float32. The mantissa of - * the source value is assumed to have an implied 1, and the exponent - * is biased. There are no negative values. - * The source value to extract must be in a 32bit int. - * While this helper is generic, it is only ever going to be useful for - * r11g11b10 (no other common format exists with the same properties). - * - * @param src (vector) value to convert - * @param mantissa_bits the number of mantissa bits - * @param exponent_bits the number of exponent bits - * @param mantissa_start the bit start position of the packed component - * - * ref http://fgiesen.wordpress.com/2012/03/28/half-to-float-done-quic/ - * ref https://gist.github.com/rygorous/2156668 - */ -static LLVMValueRef -lp_build_smallfloat_nosign_to_float(struct gallivm_state *gallivm, - struct lp_type f32_type, - LLVMValueRef src, - unsigned mantissa_bits, - unsigned exponent_bits, - unsigned mantissa_start) -{ - LLVMBuilderRef builder = gallivm->builder; - LLVMValueRef smallexpmask, i32_floatexpmask, magic; - LLVMValueRef wasinfnan, tmp, res, shift, mask; - unsigned exponent_start = mantissa_start + mantissa_bits; - struct lp_type i32_type = lp_type_int_vec(32, 32 * f32_type.length); - struct lp_build_context f32_bld, i32_bld; - - lp_build_context_init(&f32_bld, gallivm, f32_type); - lp_build_context_init(&i32_bld, gallivm, i32_type); - - /* extract the component to "float position" */ - if (exponent_start < 23) { - shift = lp_build_const_int_vec(gallivm, i32_type, 23 - exponent_start); - src = lp_build_shl(&i32_bld, src, shift); - } - else { - shift = lp_build_const_int_vec(gallivm, i32_type, exponent_start - 23); - src = lp_build_shr(&i32_bld, src, shift); - } - mask = lp_build_const_int_vec(gallivm, i32_type, - ((1 << (mantissa_bits + exponent_bits)) - 1) << - (23 - mantissa_bits)); - src = lp_build_and(&i32_bld, src, mask); - src = LLVMBuildBitCast(builder, src, f32_bld.vec_type, ""); - - /* now do the actual scaling */ - smallexpmask = lp_build_const_int_vec(gallivm, i32_type, - ((1 << exponent_bits) - 1) << 23); - i32_floatexpmask = lp_build_const_int_vec(gallivm, i32_type, 0xff << 23); - /* - * magic number has exponent new exp bias + (new exp bias - old exp bias), - * mantissa is 0. - */ - magic = lp_build_const_int_vec(gallivm, i32_type, - (255 - (1 << (exponent_bits - 1))) << 23); - magic = LLVMBuildBitCast(builder, magic, f32_bld.vec_type, ""); - - /* adjust exponent and fix denorms */ - res = lp_build_mul(&f32_bld, src, magic); - - /* - * if exp was max (== NaN or Inf) set new exp to max (keep mantissa), - * so a simple "or" will do (because exp adjust will leave mantissa intact) - */ - /* use float compare (better for AVX 8-wide / no AVX2 though otherwise should use int) */ - smallexpmask = LLVMBuildBitCast(builder, magic, f32_bld.vec_type, ""); - wasinfnan = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GEQUAL, src, smallexpmask); - res = LLVMBuildBitCast(builder, res, i32_bld.vec_type, ""); - tmp = lp_build_and(&i32_bld, i32_floatexpmask, wasinfnan); - res = lp_build_or(&i32_bld, tmp, res); - - return LLVMBuildBitCast(builder, res, f32_bld.vec_type, ""); -} - - -/** - * Convert packed float format (r11g11b10) value(s) to rgba float SoA values. - * - * @param src packed AoS r11g11b10 values (as (vector) int32) - * @param dst pointer to the SoA result values - */ -void -lp_build_r11g11b10_to_float(struct gallivm_state *gallivm, - LLVMValueRef src, - LLVMValueRef *dst) -{ - LLVMTypeRef src_type = LLVMTypeOf(src); - unsigned src_length = LLVMGetTypeKind(src_type) == LLVMVectorTypeKind ? - LLVMGetVectorSize(src_type) : 1; - struct lp_type f32_type = lp_type_float_vec(32, 32 * src_length); - - dst[0] = lp_build_smallfloat_nosign_to_float(gallivm, f32_type, src, 6, 5, 0); - dst[1] = lp_build_smallfloat_nosign_to_float(gallivm, f32_type, src, 6, 5, 11); - dst[2] = lp_build_smallfloat_nosign_to_float(gallivm, f32_type, src, 5, 5, 22); - - /* Just set alpha to one */ - dst[3] = lp_build_one(gallivm, f32_type); -} - - /** * Converts int16 half-float to float32 - * Note this can be performed in 1 instruction if vcvtph2ps exists (sse5 i think?) + * Note this can be performed in 1 instruction if vcvtph2ps exists (f16c/cvt16) * [llvm.x86.vcvtph2ps / _mm_cvtph_ps] * * @param src value to convert * - * ref http://fgiesen.wordpress.com/2012/03/28/half-to-float-done-quic/ - * ref https://gist.github.com/2144712 */ LLVMValueRef lp_build_half_to_float(struct gallivm_state *gallivm, LLVMValueRef src) { - int src_length = LLVMGetVectorSize(LLVMTypeOf(src)); + LLVMBuilderRef builder = gallivm->builder; + LLVMTypeRef src_type = LLVMTypeOf(src); + unsigned src_length = LLVMGetTypeKind(src_type) == LLVMVectorTypeKind ? + LLVMGetVectorSize(src_type) : 1; struct lp_type f32_type = lp_type_float_vec(32, 32 * src_length); struct lp_type i32_type = lp_type_int_vec(32, 32 * src_length); - - LLVMBuilderRef builder = gallivm->builder; LLVMTypeRef int_vec_type = lp_build_vec_type(gallivm, i32_type); - LLVMTypeRef float_vec_type = lp_build_vec_type(gallivm, f32_type); - - /* Constants */ - LLVMValueRef i32_13 = lp_build_const_int_vec(gallivm, i32_type, 13); - LLVMValueRef i32_16 = lp_build_const_int_vec(gallivm, i32_type, 16); - LLVMValueRef i32_mask_nosign = lp_build_const_int_vec(gallivm, i32_type, 0x7fff); - LLVMValueRef i32_was_infnan = lp_build_const_int_vec(gallivm, i32_type, 0x7bff); - LLVMValueRef i32_exp_infnan = lp_build_const_int_vec(gallivm, i32_type, 0xff << 23); - LLVMValueRef f32_magic = LLVMBuildBitCast(builder, - lp_build_const_int_vec(gallivm, i32_type, (254 - 15) << 23), - float_vec_type, ""); - - /* Convert int16 vector to int32 vector by zero ext */ - LLVMValueRef h = LLVMBuildZExt(builder, src, int_vec_type, ""); - - /* Exponent / mantissa bits */ - LLVMValueRef expmant = LLVMBuildAnd(builder, i32_mask_nosign, h, ""); - LLVMValueRef shifted = LLVMBuildBitCast(builder, LLVMBuildShl(builder, expmant, i32_13, ""), float_vec_type, ""); - - /* Exponent adjust */ - LLVMValueRef scaled = LLVMBuildBitCast(builder, LLVMBuildFMul(builder, shifted, f32_magic, ""), int_vec_type, ""); - - /* Make sure Inf/NaN survive */ - LLVMValueRef b_wasinfnan = lp_build_compare(gallivm, i32_type, PIPE_FUNC_GREATER, expmant, i32_was_infnan); - LLVMValueRef infnanexp = LLVMBuildAnd(builder, b_wasinfnan, i32_exp_infnan, ""); - - /* Sign bit */ - LLVMValueRef justsign = LLVMBuildXor(builder, h, expmant, ""); - LLVMValueRef sign = LLVMBuildShl(builder, justsign, i32_16, ""); - - /* Combine result */ - LLVMValueRef sign_inf = LLVMBuildOr(builder, sign, infnanexp, ""); - LLVMValueRef final = LLVMBuildOr(builder, scaled, sign_inf, ""); - - /* Cast from int32 vector to float32 vector */ - return LLVMBuildBitCast(builder, final, float_vec_type, ""); + LLVMValueRef h; + + if (util_cpu_caps.has_f16c && HAVE_LLVM >= 0x0301 && + (src_length == 4 || src_length == 8)) { + const char *intrinsic = NULL; + if (src_length == 4) { + src = lp_build_pad_vector(gallivm, src, 8); + intrinsic = "llvm.x86.vcvtph2ps.128"; + } + else { + intrinsic = "llvm.x86.vcvtph2ps.256"; + } + return lp_build_intrinsic_unary(builder, intrinsic, + lp_build_vec_type(gallivm, f32_type), src); + } + + /* Convert int16 vector to int32 vector by zero ext (might generate bad code) */ + h = LLVMBuildZExt(builder, src, int_vec_type, ""); + return lp_build_smallfloat_to_float(gallivm, f32_type, h, 10, 5, 0, true); } /** * Converts float32 to int16 half-float - * Note this can be performed in 1 instruction if vcvtps2ph exists (sse5 i think?) + * Note this can be performed in 1 instruction if vcvtps2ph exists (f16c/cvt16) * [llvm.x86.vcvtps2ph / _mm_cvtps_ph] * * @param src value to convert * - * ref http://fgiesen.wordpress.com/2012/03/28/half-to-float-done-quic/ - * ref https://gist.github.com/2156668 - * - * XXX: This is an approximation. It is faster but certain NaNs are converted to - * infinity, and rounding is not correct. + * Convert float32 to half floats, preserving Infs and NaNs, + * with rounding towards zero (trunc). */ LLVMValueRef lp_build_float_to_half(struct gallivm_state *gallivm, @@ -478,60 +139,35 @@ lp_build_float_to_half(struct gallivm_state *gallivm, LLVMTypeRef f32_vec_type = LLVMTypeOf(src); unsigned length = LLVMGetTypeKind(f32_vec_type) == LLVMVectorTypeKind ? LLVMGetVectorSize(f32_vec_type) : 1; - struct lp_type f32_type = lp_type_float_vec(32, 32 * length); - struct lp_type u32_type = lp_type_uint_vec(32, 32 * length); + struct lp_type i32_type = lp_type_int_vec(32, 32 * length); struct lp_type i16_type = lp_type_int_vec(16, 16 * length); - LLVMTypeRef u32_vec_type = lp_build_vec_type(gallivm, u32_type); - LLVMTypeRef i16_vec_type = lp_build_vec_type(gallivm, i16_type); - struct lp_build_context f32_bld; - struct lp_build_context u32_bld; LLVMValueRef result; - lp_build_context_init(&f32_bld, gallivm, f32_type); - lp_build_context_init(&u32_bld, gallivm, u32_type); - - { - /* Constants */ - LLVMValueRef u32_f32inf = lp_build_const_int_vec(gallivm, u32_type, 0xff << 23); - LLVMValueRef u32_expinf = lp_build_const_int_vec(gallivm, u32_type, 0xe0 << 23); - LLVMValueRef f32_f16max = lp_build_const_vec(gallivm, f32_type, 65536.0); // 0x8f << 23 - LLVMValueRef f32_magic = lp_build_const_vec(gallivm, f32_type, 1.92592994e-34); // 0x0f << 23 - - /* Cast from float32 to int32 */ - LLVMValueRef f = LLVMBuildBitCast(builder, src, u32_vec_type, ""); - - /* Remove sign */ - LLVMValueRef srcabs = lp_build_abs(&f32_bld, src); - LLVMValueRef fabs = LLVMBuildBitCast(builder, srcabs, u32_vec_type, ""); - - /* Magic conversion */ - LLVMValueRef clamped = lp_build_min(&f32_bld, f32_f16max, srcabs); - LLVMValueRef scaled = LLVMBuildBitCast(builder, - LLVMBuildFMul(builder, - clamped, - f32_magic, - ""), - u32_vec_type, - ""); - /* Make sure Inf/NaN and unormalised survive */ - LLVMValueRef infnancase = LLVMBuildXor(builder, u32_expinf, fabs, ""); - LLVMValueRef b_notnormal = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GEQUAL, - srcabs, - LLVMBuildBitCast(builder, u32_f32inf, f32_vec_type, "")); - - /* Merge normal / unnormal case */ - LLVMValueRef merged = lp_build_select(&u32_bld, b_notnormal, infnancase, scaled); - LLVMValueRef shifted = lp_build_shr_imm(&u32_bld, merged, 13); - - /* Sign bit */ - LLVMValueRef justsign = LLVMBuildXor(builder, f, fabs, ""); - LLVMValueRef signshifted = lp_build_shr_imm(&u32_bld, justsign, 16); - - /* Combine result */ - result = LLVMBuildOr(builder, shifted, signshifted, ""); + if (util_cpu_caps.has_f16c && HAVE_LLVM >= 0x0301 && + (length == 4 || length == 8)) { + struct lp_type i168_type = lp_type_int_vec(16, 16 * 8); + unsigned mode = 3; /* same as LP_BUILD_ROUND_TRUNCATE */ + LLVMTypeRef i32t = LLVMInt32TypeInContext(gallivm->context); + const char *intrinsic = NULL; + if (length == 4) { + intrinsic = "llvm.x86.vcvtps2ph.128"; + } + else { + intrinsic = "llvm.x86.vcvtps2ph.256"; + } + result = lp_build_intrinsic_binary(builder, intrinsic, + lp_build_vec_type(gallivm, i168_type), + src, LLVMConstInt(i32t, mode, 0)); + if (length == 4) { + result = lp_build_extract_range(gallivm, result, 0, 4); + } } - result = LLVMBuildTrunc(builder, result, i16_vec_type, ""); + else { + result = lp_build_float_to_smallfloat(gallivm, i32_type, src, 10, 5, 0, true); + /* Convert int32 vector to int16 vector by trunc (might generate bad code) */ + result = LLVMBuildTrunc(builder, result, lp_build_vec_type(gallivm, i16_type), ""); + } /* * Debugging code. @@ -621,6 +257,7 @@ lp_build_clamped_float_to_unsigned_norm(struct gallivm_state *gallivm, bias = (double)(1ULL << (mantissa - dst_width)); res = LLVMBuildFMul(builder, src, lp_build_const_vec(gallivm, src_type, scale), ""); + /* instead of fadd/and could (with sse2) just use lp_build_iround */ res = LLVMBuildFAdd(builder, res, lp_build_const_vec(gallivm, src_type, bias), ""); res = LLVMBuildBitCast(builder, res, int_vec_type, ""); res = LLVMBuildAnd(builder, res, @@ -629,17 +266,19 @@ lp_build_clamped_float_to_unsigned_norm(struct gallivm_state *gallivm, else if (dst_width == (mantissa + 1)) { /* * The destination width matches exactly what can be represented in - * floating point (i.e., mantissa + 1 bits). So do a straight - * multiplication followed by casting. No further rounding is necessary. + * floating point (i.e., mantissa + 1 bits). Even so correct rounding + * still needs to be applied (only for numbers in [0.5-1.0] would + * conversion using truncation after scaling be sufficient). */ - double scale; + struct lp_build_context uf32_bld; + lp_build_context_init(&uf32_bld, gallivm, src_type); scale = (double)((1ULL << dst_width) - 1); res = LLVMBuildFMul(builder, src, lp_build_const_vec(gallivm, src_type, scale), ""); - res = LLVMBuildFPToSI(builder, res, int_vec_type, ""); + res = lp_build_iround(&uf32_bld, res); } else { /* @@ -818,24 +457,22 @@ int lp_build_conv_auto(struct gallivm_state *gallivm, dst_type->width == 8) { /* Special case 4x4f --> 1x16ub */ - if (src_type.length == 4 && util_cpu_caps.has_sse2) + if (src_type.length == 4 && + util_cpu_caps.has_sse2) { - assert((num_srcs % 4) == 0); - - num_dsts = num_srcs / 4; - dst_type->length = 16; + num_dsts = (num_srcs + 3) / 4; + dst_type->length = num_srcs * 4 >= 16 ? 16 : num_srcs * 4; lp_build_conv(gallivm, src_type, *dst_type, src, num_srcs, dst, num_dsts); return num_dsts; } /* Special case 2x8f --> 1x16ub */ - if (src_type.length == 8 && util_cpu_caps.has_avx) + if (src_type.length == 8 && + util_cpu_caps.has_avx) { - assert((num_srcs % 2) == 0); - - num_dsts = num_srcs / 2; - dst_type->length = 16; + num_dsts = (num_srcs + 1) / 2; + dst_type->length = num_srcs * 8 >= 16 ? 16 : num_srcs * 8; lp_build_conv(gallivm, src_type, *dst_type, src, num_srcs, dst, num_dsts); return num_dsts; @@ -890,7 +527,7 @@ lp_build_conv(struct gallivm_state *gallivm, num_tmps = num_srcs; - /* Special case 4x4f --> 1x16ub + /* Special case 4x4f --> 1x16ub, 2x4f -> 1x8ub, 1x4f -> 1x4ub */ if (src_type.floating == 1 && src_type.fixed == 0 && @@ -904,20 +541,23 @@ lp_build_conv(struct gallivm_state *gallivm, dst_type.sign == 0 && dst_type.norm == 1 && dst_type.width == 8 && - dst_type.length == 16 && - 4 * num_dsts == num_srcs && + ((dst_type.length == 16 && 4 * num_dsts == num_srcs) || + (num_dsts == 1 && dst_type.length * num_srcs == 16 && num_srcs != 3)) && util_cpu_caps.has_sse2) { struct lp_build_context bld; - struct lp_type int16_type = dst_type; - struct lp_type int32_type = dst_type; + struct lp_type int16_type, int32_type; + struct lp_type dst_type_ext = dst_type; LLVMValueRef const_255f; unsigned i, j; lp_build_context_init(&bld, gallivm, src_type); + dst_type_ext.length = 16; + int16_type = int32_type = dst_type_ext; + int16_type.width *= 2; int16_type.length /= 2; int16_type.sign = 1; @@ -931,21 +571,34 @@ lp_build_conv(struct gallivm_state *gallivm, for (i = 0; i < num_dsts; ++i, src += 4) { LLVMValueRef lo, hi; - for (j = 0; j < 4; ++j) { + for (j = 0; j < dst_type.length / 4; ++j) { tmp[j] = LLVMBuildFMul(builder, src[j], const_255f, ""); tmp[j] = lp_build_iround(&bld, tmp[j]); } + if (num_srcs == 1) { + tmp[1] = tmp[0]; + } + /* relying on clamping behavior of sse2 intrinsics here */ lo = lp_build_pack2(gallivm, int32_type, int16_type, tmp[0], tmp[1]); - hi = lp_build_pack2(gallivm, int32_type, int16_type, tmp[2], tmp[3]); - dst[i] = lp_build_pack2(gallivm, int16_type, dst_type, lo, hi); + + if (num_srcs < 4) { + hi = lo; + } + else { + hi = lp_build_pack2(gallivm, int32_type, int16_type, tmp[2], tmp[3]); + } + dst[i] = lp_build_pack2(gallivm, int16_type, dst_type_ext, lo, hi); + } + if (num_srcs < 4) { + dst[0] = lp_build_extract_range(gallivm, dst[0], 0, dst_type.length); } return; } - /* Special case 2x8f --> 1x16ub + /* Special case 2x8f --> 1x16ub, 1x8f ->1x8ub */ else if (src_type.floating == 1 && src_type.fixed == 0 && @@ -959,20 +612,23 @@ lp_build_conv(struct gallivm_state *gallivm, dst_type.sign == 0 && dst_type.norm == 1 && dst_type.width == 8 && - dst_type.length == 16 && - 2 * num_dsts == num_srcs && + ((dst_type.length == 16 && 2 * num_dsts == num_srcs) || + (num_dsts == 1 && dst_type.length * num_srcs == 8)) && util_cpu_caps.has_avx) { struct lp_build_context bld; - struct lp_type int16_type = dst_type; - struct lp_type int32_type = dst_type; + struct lp_type int16_type, int32_type; + struct lp_type dst_type_ext = dst_type; LLVMValueRef const_255f; unsigned i; lp_build_context_init(&bld, gallivm, src_type); + dst_type_ext.length = 16; + int16_type = int32_type = dst_type_ext; + int16_type.width *= 2; int16_type.length /= 2; int16_type.sign = 1; @@ -987,21 +643,30 @@ lp_build_conv(struct gallivm_state *gallivm, LLVMValueRef lo, hi, a, b; a = LLVMBuildFMul(builder, src[0], const_255f, ""); - b = LLVMBuildFMul(builder, src[1], const_255f, ""); - a = lp_build_iround(&bld, a); - b = lp_build_iround(&bld, b); - tmp[0] = lp_build_extract_range(gallivm, a, 0, 4); tmp[1] = lp_build_extract_range(gallivm, a, 4, 4); - tmp[2] = lp_build_extract_range(gallivm, b, 0, 4); - tmp[3] = lp_build_extract_range(gallivm, b, 4, 4); - /* relying on clamping behavior of sse2 intrinsics here */ lo = lp_build_pack2(gallivm, int32_type, int16_type, tmp[0], tmp[1]); - hi = lp_build_pack2(gallivm, int32_type, int16_type, tmp[2], tmp[3]); - dst[i] = lp_build_pack2(gallivm, int16_type, dst_type, lo, hi); + + if (num_srcs == 1) { + hi = lo; + } + else { + b = LLVMBuildFMul(builder, src[1], const_255f, ""); + b = lp_build_iround(&bld, b); + tmp[2] = lp_build_extract_range(gallivm, b, 0, 4); + tmp[3] = lp_build_extract_range(gallivm, b, 4, 4); + hi = lp_build_pack2(gallivm, int32_type, int16_type, tmp[2], tmp[3]); + + } + dst[i] = lp_build_pack2(gallivm, int16_type, dst_type_ext, lo, hi); } + + if (num_srcs == 1) { + dst[0] = lp_build_extract_range(gallivm, dst[0], 0, dst_type.length); + } + return; } @@ -1080,7 +745,6 @@ lp_build_conv(struct gallivm_state *gallivm, } else { double dst_scale = lp_const_scale(dst_type); - LLVMTypeRef tmp_vec_type; if (dst_scale != 1.0) { LLVMValueRef scale = lp_build_const_vec(gallivm, tmp_type, dst_scale); @@ -1088,19 +752,38 @@ lp_build_conv(struct gallivm_state *gallivm, tmp[i] = LLVMBuildFMul(builder, tmp[i], scale, ""); } - /* Use an equally sized integer for intermediate computations */ - tmp_type.floating = FALSE; - tmp_vec_type = lp_build_vec_type(gallivm, tmp_type); - for(i = 0; i < num_tmps; ++i) { + /* + * these functions will use fptosi in some form which won't work + * with 32bit uint dst. Causes lp_test_conv failures though. + */ + if (0) + assert(dst_type.sign || dst_type.width < 32); + + if (dst_type.sign && dst_type.norm && !dst_type.fixed) { + struct lp_build_context bld; + + lp_build_context_init(&bld, gallivm, tmp_type); + for(i = 0; i < num_tmps; ++i) { + tmp[i] = lp_build_iround(&bld, tmp[i]); + } + tmp_type.floating = FALSE; + } + else { + LLVMTypeRef tmp_vec_type; + + tmp_type.floating = FALSE; + tmp_vec_type = lp_build_vec_type(gallivm, tmp_type); + for(i = 0; i < num_tmps; ++i) { #if 0 - if(dst_type.sign) - tmp[i] = LLVMBuildFPToSI(builder, tmp[i], tmp_vec_type, ""); - else - tmp[i] = LLVMBuildFPToUI(builder, tmp[i], tmp_vec_type, ""); + if(dst_type.sign) + tmp[i] = LLVMBuildFPToSI(builder, tmp[i], tmp_vec_type, ""); + else + tmp[i] = LLVMBuildFPToUI(builder, tmp[i], tmp_vec_type, ""); #else - /* FIXME: there is no SSE counterpart for LLVMBuildFPToUI */ - tmp[i] = LLVMBuildFPToSI(builder, tmp[i], tmp_vec_type, ""); + /* FIXME: there is no SSE counterpart for LLVMBuildFPToUI */ + tmp[i] = LLVMBuildFPToSI(builder, tmp[i], tmp_vec_type, ""); #endif + } } } } @@ -1198,6 +881,18 @@ lp_build_conv(struct gallivm_state *gallivm, for(i = 0; i < num_tmps; ++i) tmp[i] = LLVMBuildFMul(builder, tmp[i], scale, ""); } + + /* the formula above will produce value below -1.0 for most negative + * value but everything seems happy with that hence disable for now */ + if (0 && !src_type.fixed && src_type.norm && src_type.sign) { + struct lp_build_context bld; + + lp_build_context_init(&bld, gallivm, dst_type); + for(i = 0; i < num_tmps; ++i) { + tmp[i] = lp_build_max(&bld, tmp[i], + lp_build_const_vec(gallivm, dst_type, -1.0f)); + } + } } } else {