src/gallium/auxiliary/gallivm/lp_bld_format_float.c

   1 /**************************************************************************
   2  *
   3  * Copyright 2013 VMware, Inc.
   4  * All Rights Reserved.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the
   8  * "Software"), to deal in the Software without restriction, including
   9  * without limitation the rights to use, copy, modify, merge, publish,
  10  * distribute, sub license, and/or sell copies of the Software, and to
  11  * permit persons to whom the Software is furnished to do so, subject to
  12  * the following conditions:
  13  *
  14  * The above copyright notice and this permission notice (including the
  15  * next paragraph) shall be included in all copies or substantial portions
  16  * of the Software.
  17  *
  18  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
  19  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  20  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
  21  * IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR
  22  * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
  23  * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
  24  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  25  *
  26  **************************************************************************/
  27
  28
  29 /**
  30  * @file
  31  * Format conversion code for "special" float formats.
  32  *
  33  * @author Roland Scheidegger <sroland@vmware.com>
  34  */
  35
  36
  37 #include "util/u_debug.h"
  38
  39 #include "lp_bld_type.h"
  40 #include "lp_bld_const.h"
  41 #include "lp_bld_arit.h"
  42 #include "lp_bld_bitarit.h"
  43 #include "lp_bld_logic.h"
  44 #include "lp_bld_format.h"
  45
  46
  47 /**
  48  * Convert float32 to a float-like value with less exponent and mantissa
  49  * bits. The mantissa is still biased, and the mantissa still has an implied 1,
  50  * and there may be a sign bit.
  51  *
  52  * @param src             (vector) float value to convert
  53  * @param mantissa_bits   the number of mantissa bits
  54  * @param exponent_bits   the number of exponent bits
  55  * @param mantissa_start  the start position of the small float in result value
  56  * @param has_sign        if the small float has a sign bit
  57  *
  58  * This implements round-towards-zero (trunc) hence too large numbers get
  59  * converted to largest representable number, not infinity.
  60  * Small numbers may get converted to denorms, depending on normal
  61  * float denorm handling of the cpu.
  62  * Note that compared to the references, below, we skip any rounding bias
  63  * since we do rounding towards zero - OpenGL allows rounding towards zero
  64  * (though not preferred) and DX10 even seems to require it.
  65  * Note that this will pack mantissa, exponent and sign bit (if any) together,
  66  * and shift the result to mantissa_start.
  67  *
  68  * ref http://fgiesen.wordpress.com/2012/03/28/half-to-float-done-quic/
  69  * ref https://gist.github.com/rygorous/2156668
  70  */
  71 LLVMValueRef
  72 lp_build_float_to_smallfloat(struct gallivm_state *gallivm,
  73                              struct lp_type i32_type,
  74                              LLVMValueRef src,
  75                              unsigned mantissa_bits,
  76                              unsigned exponent_bits,
  77                              unsigned mantissa_start,
  78                              boolean has_sign)
  79 {
  80    LLVMBuilderRef builder = gallivm->builder;
  81    LLVMValueRef i32_floatexpmask, i32_smallexpmask, magic, normal;
  82    LLVMValueRef rescale_src, tmp, i32_roundmask, small_max;
  83    LLVMValueRef is_nan, i32_qnanbit, src_abs, shift, infcheck_src, res;
  84    LLVMValueRef is_inf, is_nan_or_inf, nan_or_inf, mask;
  85    struct lp_type f32_type = lp_type_float_vec(32, 32 * i32_type.length);
  86    struct lp_build_context f32_bld, i32_bld;
  87    LLVMValueRef zero = lp_build_const_vec(gallivm, f32_type, 0.0f);
  88    unsigned exponent_start = mantissa_start + mantissa_bits;
  89
  90    lp_build_context_init(&f32_bld, gallivm, f32_type);
  91    lp_build_context_init(&i32_bld, gallivm, i32_type);
  92
  93    i32_smallexpmask = lp_build_const_int_vec(gallivm, i32_type,
  94                                              ((1 << exponent_bits) - 1) << 23);
  95    i32_floatexpmask = lp_build_const_int_vec(gallivm, i32_type, 0xff << 23);
  96
  97    src_abs = lp_build_abs(&f32_bld, src);
  98    src_abs = LLVMBuildBitCast(builder, src_abs, i32_bld.vec_type, "");
  99
 100    if (has_sign) {
 101       rescale_src = src_abs;
 102       infcheck_src = src_abs;
 103       src = LLVMBuildBitCast(builder, src, i32_bld.vec_type, "");
 104    }
 105    else {
 106       /* clamp to pos range (can still have sign bit if NaN or negative zero) */
 107       rescale_src = lp_build_max(&f32_bld, src, zero);
 108       rescale_src = LLVMBuildBitCast(builder, rescale_src, i32_bld.vec_type, "");
 109       src = LLVMBuildBitCast(builder, src, i32_bld.vec_type, "");
 110       infcheck_src = src;
 111    }
 112
 113    /* "ordinary" number */
 114    /* get rid of excess mantissa bits, and while here also potential sign bit */
 115    i32_roundmask = lp_build_const_int_vec(gallivm, i32_type,
 116                                           ~((1 << (23 - mantissa_bits)) - 1) &
 117                                           0x7fffffff);
 118
 119    tmp = lp_build_and(&i32_bld, rescale_src, i32_roundmask);
 120    tmp = LLVMBuildBitCast(builder, tmp, f32_bld.vec_type, "");
 121    /* bias exponent (and denormalize if necessary) */
 122    magic = lp_build_const_int_vec(gallivm, i32_type,
 123                                   ((1 << (exponent_bits - 1)) - 1) << 23);
 124    magic = LLVMBuildBitCast(builder, magic, f32_bld.vec_type, "");
 125    normal = lp_build_mul(&f32_bld, tmp, magic);
 126
 127    /* clamp to max value - largest non-infinity number */
 128    small_max = lp_build_const_int_vec(gallivm, i32_type,
 129                                       (((1 << exponent_bits) - 2) << 23) |
 130                                       (((1 << mantissa_bits) - 1) << (23 - mantissa_bits)));
 131    small_max = LLVMBuildBitCast(builder, small_max, f32_bld.vec_type, "");
 132    normal = lp_build_min(&f32_bld, normal, small_max);
 133    normal = LLVMBuildBitCast(builder, normal, i32_bld.vec_type, "");
 134
 135    /*
 136     * handle nan/inf cases
 137     * a little bit tricky since -Inf -> 0, +Inf -> +Inf, +-Nan -> +Nan
 138     * (for no sign) else ->Inf -> ->Inf too.
 139     * could use explicit "unordered" comparison checking for NaNs
 140     * which might save us from calculating src_abs too.
 141     * (Cannot actually save the comparison since we need to distinguish
 142     * Inf and NaN cases anyway, but it would be better for AVX.)
 143     */
 144    is_nan = lp_build_compare(gallivm, i32_type, PIPE_FUNC_GREATER,
 145                              src_abs, i32_floatexpmask);
 146    is_inf = lp_build_compare(gallivm, i32_type, PIPE_FUNC_EQUAL,
 147                              infcheck_src, i32_floatexpmask);
 148    is_nan_or_inf = lp_build_or(&i32_bld, is_nan, is_inf);
 149    /* could also set more mantissa bits but need at least the highest mantissa bit */
 150    i32_qnanbit = lp_build_const_vec(gallivm, i32_type, 1 << 22);
 151    /* combine maxexp with qnanbit */
 152    nan_or_inf = lp_build_or(&i32_bld, i32_smallexpmask,
 153                             lp_build_and(&i32_bld, is_nan, i32_qnanbit));
 154
 155    res = lp_build_select(&i32_bld, is_nan_or_inf, nan_or_inf, normal);
 156
 157    /* add back sign bit at right position */
 158    if (has_sign) {
 159       LLVMValueRef sign;
 160       struct lp_type u32_type = lp_type_uint_vec(32, 32 * i32_type.length);
 161       struct lp_build_context u32_bld;
 162       lp_build_context_init(&u32_bld, gallivm, u32_type);
 163
 164       mask = lp_build_const_int_vec(gallivm, i32_type, 0x80000000);
 165       shift = lp_build_const_int_vec(gallivm, i32_type, 8 - exponent_bits);
 166       sign = lp_build_and(&i32_bld, mask, src);
 167       sign = lp_build_shr(&u32_bld, sign, shift);
 168       res = lp_build_or(&i32_bld, sign, res);
 169    }
 170
 171    /* shift to final position */
 172    if (exponent_start < 23) {
 173       shift = lp_build_const_int_vec(gallivm, i32_type, 23 - exponent_start);
 174       res = lp_build_shr(&i32_bld, res, shift);
 175    }
 176    else {
 177       shift = lp_build_const_int_vec(gallivm, i32_type, exponent_start - 23);
 178       res = lp_build_shl(&i32_bld, res, shift);
 179    }
 180    if (mantissa_start > 0) {
 181       /* generally shouldn't get bits to mask off but can happen with denormals */
 182       unsigned maskbits = (1 << (mantissa_bits + exponent_bits + has_sign)) - 1;
 183       mask = lp_build_const_int_vec(gallivm, i32_type,
 184                                     maskbits << mantissa_start);
 185       res = lp_build_and(&i32_bld, res, mask);
 186    }
 187    return res;
 188 }
 189
 190
 191 /**
 192  * Convert rgba float SoA values to packed r11g11b10 values.
 193  *
 194  * @param src   SoA float (vector) values to convert.
 195  */
 196 LLVMValueRef
 197 lp_build_float_to_r11g11b10(struct gallivm_state *gallivm,
 198                             LLVMValueRef *src)
 199 {
 200    LLVMValueRef dst, rcomp, bcomp, gcomp;
 201    struct lp_build_context i32_bld;
 202    LLVMTypeRef src_type = LLVMTypeOf(*src);
 203    unsigned src_length = LLVMGetTypeKind(src_type) == LLVMVectorTypeKind ?
 204                             LLVMGetVectorSize(src_type) : 1;
 205    struct lp_type i32_type = lp_type_int_vec(32, 32 * src_length);
 206
 207    lp_build_context_init(&i32_bld, gallivm, i32_type);
 208
 209    /* "rescale" and put in right position */
 210    rcomp = lp_build_float_to_smallfloat(gallivm, i32_type, src[0], 6, 5, 0, false);
 211    gcomp = lp_build_float_to_smallfloat(gallivm, i32_type, src[1], 6, 5, 11, false);
 212    bcomp = lp_build_float_to_smallfloat(gallivm, i32_type, src[2], 5, 5, 22, false);
 213
 214    /* combine the values */
 215    dst = lp_build_or(&i32_bld, rcomp, gcomp);
 216    return lp_build_or(&i32_bld, dst, bcomp);
 217 }
 218
 219
 220 /**
 221  * Convert a float-like value with less exponent and mantissa
 222  * bits than a normal float32 to a float32. The mantissa of
 223  * the source value is assumed to have an implied 1, and the exponent
 224  * is biased. There may be a sign bit.
 225  * The source value to extract must be in a 32bit int (bits not part of
 226  * the value to convert will be masked off).
 227  * This works for things like 11-bit floats or half-floats,
 228  * mantissa, exponent (and sign if present) must be packed
 229  * the same as they are in a ordinary float.
 230  *
 231  * @param src             (vector) value to convert
 232  * @param mantissa_bits   the number of mantissa bits
 233  * @param exponent_bits   the number of exponent bits
 234  * @param mantissa_start  the bit start position of the packed component
 235  * @param has_sign        if the small float has a sign bit
 236  *
 237  * ref http://fgiesen.wordpress.com/2012/03/28/half-to-float-done-quic/
 238  * ref https://gist.github.com/rygorous/2156668
 239  */
 240 LLVMValueRef
 241 lp_build_smallfloat_to_float(struct gallivm_state *gallivm,
 242                              struct lp_type f32_type,
 243                              LLVMValueRef src,
 244                              unsigned mantissa_bits,
 245                              unsigned exponent_bits,
 246                              unsigned mantissa_start,
 247                              boolean has_sign)
 248 {
 249    LLVMBuilderRef builder = gallivm->builder;
 250    LLVMValueRef smallexpmask, i32_floatexpmask, magic;
 251    LLVMValueRef wasinfnan, tmp, res, shift, maskabs, srcabs, sign;
 252    unsigned exponent_start = mantissa_start + mantissa_bits;
 253    struct lp_type i32_type = lp_type_int_vec(32, 32 * f32_type.length);
 254    struct lp_build_context f32_bld, i32_bld;
 255
 256    lp_build_context_init(&f32_bld, gallivm, f32_type);
 257    lp_build_context_init(&i32_bld, gallivm, i32_type);
 258
 259    /* extract the component to "float position" */
 260    if (exponent_start < 23) {
 261       shift = lp_build_const_int_vec(gallivm, i32_type, 23 - exponent_start);
 262       src = lp_build_shl(&i32_bld, src, shift);
 263    }
 264    else {
 265       shift = lp_build_const_int_vec(gallivm, i32_type, exponent_start - 23);
 266       src = lp_build_shr(&i32_bld, src, shift);
 267    }
 268    maskabs = lp_build_const_int_vec(gallivm, i32_type,
 269                                     ((1 << (mantissa_bits + exponent_bits)) - 1)
 270                                     << (23 - mantissa_bits));
 271    srcabs = lp_build_and(&i32_bld, src, maskabs);
 272    srcabs = LLVMBuildBitCast(builder, srcabs, f32_bld.vec_type, "");
 273
 274    /* now do the actual scaling */
 275    smallexpmask = lp_build_const_int_vec(gallivm, i32_type,
 276                                          ((1 << exponent_bits) - 1) << 23);
 277    i32_floatexpmask = lp_build_const_int_vec(gallivm, i32_type, 0xff << 23);
 278    /*
 279     * magic number has exponent new exp bias + (new exp bias - old exp bias),
 280     * mantissa is 0.
 281     */
 282    magic = lp_build_const_int_vec(gallivm, i32_type,
 283                                   (255 - (1 << (exponent_bits - 1))) << 23);
 284    magic = LLVMBuildBitCast(builder, magic, f32_bld.vec_type, "");
 285
 286    /* adjust exponent and fix denorms */
 287    res = lp_build_mul(&f32_bld, srcabs, magic);
 288
 289    /*
 290     * if exp was max (== NaN or Inf) set new exp to max (keep mantissa),
 291     * so a simple "or" will do (because exp adjust will leave mantissa intact)
 292     */
 293    /* use float compare (better for AVX 8-wide / no AVX2 but else should use int) */
 294    smallexpmask = LLVMBuildBitCast(builder, smallexpmask, f32_bld.vec_type, "");
 295    wasinfnan = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GEQUAL, srcabs, smallexpmask);
 296    res = LLVMBuildBitCast(builder, res, i32_bld.vec_type, "");
 297    tmp = lp_build_and(&i32_bld, i32_floatexpmask, wasinfnan);
 298    res = lp_build_or(&i32_bld, tmp, res);
 299
 300    if (has_sign) {
 301       LLVMValueRef signmask = lp_build_const_int_vec(gallivm, i32_type, 0x80000000);
 302       shift = lp_build_const_int_vec(gallivm, i32_type, 8 - exponent_bits);
 303       sign = lp_build_shl(&i32_bld, src, shift);
 304       sign = lp_build_and(&i32_bld, signmask, sign);
 305       res = lp_build_or(&i32_bld, res, sign);
 306    }
 307
 308    return LLVMBuildBitCast(builder, res, f32_bld.vec_type, "");
 309 }
 310
 311
 312 /**
 313  * Convert packed float format (r11g11b10) value(s) to rgba float SoA values.
 314  *
 315  * @param src   packed AoS r11g11b10 values (as (vector) int32)
 316  * @param dst   pointer to the SoA result values
 317  */
 318 void
 319 lp_build_r11g11b10_to_float(struct gallivm_state *gallivm,
 320                             LLVMValueRef src,
 321                             LLVMValueRef *dst)
 322 {
 323    LLVMTypeRef src_type = LLVMTypeOf(src);
 324    unsigned src_length = LLVMGetTypeKind(src_type) == LLVMVectorTypeKind ?
 325                             LLVMGetVectorSize(src_type) : 1;
 326    struct lp_type f32_type = lp_type_float_vec(32, 32 * src_length);
 327
 328    dst[0] = lp_build_smallfloat_to_float(gallivm, f32_type, src, 6, 5, 0, false);
 329    dst[1] = lp_build_smallfloat_to_float(gallivm, f32_type, src, 6, 5, 11, false);
 330    dst[2] = lp_build_smallfloat_to_float(gallivm, f32_type, src, 5, 5, 22, false);
 331
 332    /* Just set alpha to one */
 333    dst[3] = lp_build_one(gallivm, f32_type);
 334 }
 335
 336
 337 static LLVMValueRef
 338 lp_build_rgb9_to_float_helper(struct gallivm_state *gallivm,
 339                               struct lp_type f32_type,
 340                               LLVMValueRef src,
 341                               LLVMValueRef scale,
 342                               unsigned mantissa_start)
 343 {
 344    LLVMValueRef shift, mask;
 345
 346    struct lp_type i32_type = lp_type_int_vec(32, 32 * f32_type.length);
 347    struct lp_build_context i32_bld, f32_bld;
 348
 349    lp_build_context_init(&i32_bld, gallivm, i32_type);
 350    lp_build_context_init(&f32_bld, gallivm, f32_type);
 351
 352    /*
 353     * This is much easier as other weirdo float formats, since
 354     * there's no sign, no Inf/NaN, and there's nothing special
 355     * required for normals/denormals neither (as without the implied one
 356     * for the mantissa for other formats, everything looks like a denormal).
 357     * So just do (float)comp_bits * scale
 358     */
 359    shift = lp_build_const_int_vec(gallivm, i32_type, mantissa_start);
 360    mask = lp_build_const_int_vec(gallivm, i32_type, 0x1ff);
 361    src = lp_build_shr(&i32_bld, src, shift);
 362    src = lp_build_and(&i32_bld, src, mask);
 363    src = lp_build_int_to_float(&f32_bld, src);
 364    return lp_build_mul(&f32_bld, src, scale);
 365 }
 366
 367
 368 /**
 369  * Convert shared exponent format (rgb9e5) value(s) to rgba float SoA values.
 370  *
 371  * @param src   packed AoS rgb9e5 values (as (vector) int32)
 372  * @param dst   pointer to the SoA result values
 373  */
 374 void
 375 lp_build_rgb9e5_to_float(struct gallivm_state *gallivm,
 376                          LLVMValueRef src,
 377                          LLVMValueRef *dst)
 378 {
 379    LLVMBuilderRef builder = gallivm->builder;
 380    LLVMTypeRef src_type = LLVMTypeOf(src);
 381    LLVMValueRef shift, scale, bias, exp;
 382    unsigned src_length = LLVMGetTypeKind(src_type) == LLVMVectorTypeKind ?
 383                             LLVMGetVectorSize(src_type) : 1;
 384    struct lp_type i32_type = lp_type_int_vec(32, 32 * src_length);
 385    struct lp_type u32_type = lp_type_uint_vec(32, 32 * src_length);
 386    struct lp_type f32_type = lp_type_float_vec(32, 32 * src_length);
 387    struct lp_build_context i32_bld, u32_bld, f32_bld;
 388
 389    lp_build_context_init(&i32_bld, gallivm, i32_type);
 390    lp_build_context_init(&u32_bld, gallivm, u32_type);
 391    lp_build_context_init(&f32_bld, gallivm, f32_type);
 392
 393    /* extract exponent */
 394    shift = lp_build_const_int_vec(gallivm, i32_type, 27);
 395    /* this shift needs to be unsigned otherwise need mask */
 396    exp = lp_build_shr(&u32_bld, src, shift);
 397
 398    /*
 399     * scale factor is 2 ^ (exp - bias)
 400     * (and additionally corrected here for the mantissa bits)
 401     * not using shift because
 402     * a) don't have vector shift in a lot of cases
 403     * b) shift direction changes hence need 2 shifts + conditional
 404     *    (or rotate instruction which is even more rare (for instance XOP))
 405     * so use whacky float 2 ^ function instead manipulating exponent
 406     * (saves us the float conversion at the end too)
 407     */
 408    bias = lp_build_const_int_vec(gallivm, i32_type, 127 - (15 + 9));
 409    scale = lp_build_add(&i32_bld, exp, bias);
 410    shift = lp_build_const_int_vec(gallivm, i32_type, 23);
 411    scale = lp_build_shl(&i32_bld, scale, shift);
 412    scale = LLVMBuildBitCast(builder, scale, f32_bld.vec_type, "");
 413
 414    dst[0] = lp_build_rgb9_to_float_helper(gallivm, f32_type, src, scale, 0);
 415    dst[1] = lp_build_rgb9_to_float_helper(gallivm, f32_type, src, scale, 9);
 416    dst[2] = lp_build_rgb9_to_float_helper(gallivm, f32_type, src, scale, 18);
 417
 418    /* Just set alpha to one */
 419    dst[3] = f32_bld.one;
 420 }