src/gallium/auxiliary/gallivm/lp_bld_arit.c

   1 /**************************************************************************
   2  *
   3  * Copyright 2009 VMware, Inc.
   4  * All Rights Reserved.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the
   8  * "Software"), to deal in the Software without restriction, including
   9  * without limitation the rights to use, copy, modify, merge, publish,
  10  * distribute, sub license, and/or sell copies of the Software, and to
  11  * permit persons to whom the Software is furnished to do so, subject to
  12  * the following conditions:
  13  *
  14  * The above copyright notice and this permission notice (including the
  15  * next paragraph) shall be included in all copies or substantial portions
  16  * of the Software.
  17  *
  18  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
  19  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  20  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
  21  * IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR
  22  * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
  23  * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
  24  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  25  *
  26  **************************************************************************/
  27
  28
  29 /**
  30  * @file
  31  * Helper
  32  *
  33  * LLVM IR doesn't support all basic arithmetic operations we care about (most
  34  * notably min/max and saturated operations), and it is often necessary to
  35  * resort machine-specific intrinsics directly. The functions here hide all
  36  * these implementation details from the other modules.
  37  *
  38  * We also do simple expressions simplification here. Reasons are:
  39  * - it is very easy given we have all necessary information readily available
  40  * - LLVM optimization passes fail to simplify several vector expressions
  41  * - We often know value constraints which the optimization passes have no way
  42  *   of knowing, such as when source arguments are known to be in [0, 1] range.
  43  *
  44  * @author Jose Fonseca <jfonseca@vmware.com>
  45  */
  46
  47
  48 #include "util/u_memory.h"
  49 #include "util/u_debug.h"
  50 #include "util/u_math.h"
  51 #include "util/u_string.h"
  52 #include "util/u_cpu_detect.h"
  53
  54 #include "lp_bld_type.h"
  55 #include "lp_bld_const.h"
  56 #include "lp_bld_intr.h"
  57 #include "lp_bld_logic.h"
  58 #include "lp_bld_pack.h"
  59 #include "lp_bld_debug.h"
  60 #include "lp_bld_arit.h"
  61
  62
  63 /**
  64  * Generate min(a, b)
  65  * No checks for special case values of a or b = 1 or 0 are done.
  66  */
  67 static LLVMValueRef
  68 lp_build_min_simple(struct lp_build_context *bld,
  69                     LLVMValueRef a,
  70                     LLVMValueRef b)
  71 {
  72    const struct lp_type type = bld->type;
  73    const char *intrinsic = NULL;
  74    LLVMValueRef cond;
  75
  76    /* TODO: optimize the constant case */
  77
  78    if(type.width * type.length == 128) {
  79       if(type.floating) {
  80          if(type.width == 32 && util_cpu_caps.has_sse)
  81             intrinsic = "llvm.x86.sse.min.ps";
  82          if(type.width == 64 && util_cpu_caps.has_sse2)
  83             intrinsic = "llvm.x86.sse2.min.pd";
  84       }
  85       else {
  86          if(type.width == 8 && !type.sign && util_cpu_caps.has_sse2)
  87             intrinsic = "llvm.x86.sse2.pminu.b";
  88          if(type.width == 8 && type.sign && util_cpu_caps.has_sse4_1)
  89             intrinsic = "llvm.x86.sse41.pminsb";
  90          if(type.width == 16 && !type.sign && util_cpu_caps.has_sse4_1)
  91             intrinsic = "llvm.x86.sse41.pminuw";
  92          if(type.width == 16 && type.sign && util_cpu_caps.has_sse2)
  93             intrinsic = "llvm.x86.sse2.pmins.w";
  94          if(type.width == 32 && !type.sign && util_cpu_caps.has_sse4_1)
  95             intrinsic = "llvm.x86.sse41.pminud";
  96          if(type.width == 32 && type.sign && util_cpu_caps.has_sse4_1)
  97             intrinsic = "llvm.x86.sse41.pminsd";
  98       }
  99    }
 100
 101    if(intrinsic)
 102       return lp_build_intrinsic_binary(bld->builder, intrinsic, lp_build_vec_type(bld->type), a, b);
 103
 104    cond = lp_build_cmp(bld, PIPE_FUNC_LESS, a, b);
 105    return lp_build_select(bld, cond, a, b);
 106 }
 107
 108
 109 /**
 110  * Generate max(a, b)
 111  * No checks for special case values of a or b = 1 or 0 are done.
 112  */
 113 static LLVMValueRef
 114 lp_build_max_simple(struct lp_build_context *bld,
 115                     LLVMValueRef a,
 116                     LLVMValueRef b)
 117 {
 118    const struct lp_type type = bld->type;
 119    const char *intrinsic = NULL;
 120    LLVMValueRef cond;
 121
 122    /* TODO: optimize the constant case */
 123
 124    if(type.width * type.length == 128) {
 125       if(type.floating) {
 126          if(type.width == 32 && util_cpu_caps.has_sse)
 127             intrinsic = "llvm.x86.sse.max.ps";
 128          if(type.width == 64 && util_cpu_caps.has_sse2)
 129             intrinsic = "llvm.x86.sse2.max.pd";
 130       }
 131       else {
 132          if(type.width == 8 && !type.sign && util_cpu_caps.has_sse2)
 133             intrinsic = "llvm.x86.sse2.pmaxu.b";
 134          if(type.width == 8 && type.sign && util_cpu_caps.has_sse4_1)
 135             intrinsic = "llvm.x86.sse41.pmaxsb";
 136          if(type.width == 16 && !type.sign && util_cpu_caps.has_sse4_1)
 137             intrinsic = "llvm.x86.sse41.pmaxuw";
 138          if(type.width == 16 && type.sign && util_cpu_caps.has_sse2)
 139             intrinsic = "llvm.x86.sse2.pmaxs.w";
 140          if(type.width == 32 && !type.sign && util_cpu_caps.has_sse4_1)
 141             intrinsic = "llvm.x86.sse41.pmaxud";
 142          if(type.width == 32 && type.sign && util_cpu_caps.has_sse4_1)
 143             intrinsic = "llvm.x86.sse41.pmaxsd";
 144       }
 145    }
 146
 147    if(intrinsic)
 148       return lp_build_intrinsic_binary(bld->builder, intrinsic, lp_build_vec_type(bld->type), a, b);
 149
 150    cond = lp_build_cmp(bld, PIPE_FUNC_GREATER, a, b);
 151    return lp_build_select(bld, cond, a, b);
 152 }
 153
 154
 155 /**
 156  * Generate 1 - a, or ~a depending on bld->type.
 157  */
 158 LLVMValueRef
 159 lp_build_comp(struct lp_build_context *bld,
 160               LLVMValueRef a)
 161 {
 162    const struct lp_type type = bld->type;
 163
 164    if(a == bld->one)
 165       return bld->zero;
 166    if(a == bld->zero)
 167       return bld->one;
 168
 169    if(type.norm && !type.floating && !type.fixed && !type.sign) {
 170       if(LLVMIsConstant(a))
 171          return LLVMConstNot(a);
 172       else
 173          return LLVMBuildNot(bld->builder, a, "");
 174    }
 175
 176    if(LLVMIsConstant(a))
 177       return LLVMConstSub(bld->one, a);
 178    else
 179       return LLVMBuildSub(bld->builder, bld->one, a, "");
 180 }
 181
 182
 183 /**
 184  * Generate a + b
 185  */
 186 LLVMValueRef
 187 lp_build_add(struct lp_build_context *bld,
 188              LLVMValueRef a,
 189              LLVMValueRef b)
 190 {
 191    const struct lp_type type = bld->type;
 192    LLVMValueRef res;
 193
 194    if(a == bld->zero)
 195       return b;
 196    if(b == bld->zero)
 197       return a;
 198    if(a == bld->undef || b == bld->undef)
 199       return bld->undef;
 200
 201    if(bld->type.norm) {
 202       const char *intrinsic = NULL;
 203
 204       if(a == bld->one || b == bld->one)
 205         return bld->one;
 206
 207       if(util_cpu_caps.has_sse2 &&
 208          type.width * type.length == 128 &&
 209          !type.floating && !type.fixed) {
 210          if(type.width == 8)
 211             intrinsic = type.sign ? "llvm.x86.sse2.padds.b" : "llvm.x86.sse2.paddus.b";
 212          if(type.width == 16)
 213             intrinsic = type.sign ? "llvm.x86.sse2.padds.w" : "llvm.x86.sse2.paddus.w";
 214       }
 215
 216       if(intrinsic)
 217          return lp_build_intrinsic_binary(bld->builder, intrinsic, lp_build_vec_type(bld->type), a, b);
 218    }
 219
 220    if(LLVMIsConstant(a) && LLVMIsConstant(b))
 221       res = LLVMConstAdd(a, b);
 222    else
 223       res = LLVMBuildAdd(bld->builder, a, b, "");
 224
 225    /* clamp to ceiling of 1.0 */
 226    if(bld->type.norm && (bld->type.floating || bld->type.fixed))
 227       res = lp_build_min_simple(bld, res, bld->one);
 228
 229    /* XXX clamp to floor of -1 or 0??? */
 230
 231    return res;
 232 }
 233
 234
 235 /**
 236  * Generate a - b
 237  */
 238 LLVMValueRef
 239 lp_build_sub(struct lp_build_context *bld,
 240              LLVMValueRef a,
 241              LLVMValueRef b)
 242 {
 243    const struct lp_type type = bld->type;
 244    LLVMValueRef res;
 245
 246    if(b == bld->zero)
 247       return a;
 248    if(a == bld->undef || b == bld->undef)
 249       return bld->undef;
 250    if(a == b)
 251       return bld->zero;
 252
 253    if(bld->type.norm) {
 254       const char *intrinsic = NULL;
 255
 256       if(b == bld->one)
 257         return bld->zero;
 258
 259       if(util_cpu_caps.has_sse2 &&
 260          type.width * type.length == 128 &&
 261          !type.floating && !type.fixed) {
 262          if(type.width == 8)
 263             intrinsic = type.sign ? "llvm.x86.sse2.psubs.b" : "llvm.x86.sse2.psubus.b";
 264          if(type.width == 16)
 265             intrinsic = type.sign ? "llvm.x86.sse2.psubs.w" : "llvm.x86.sse2.psubus.w";
 266       }
 267
 268       if(intrinsic)
 269          return lp_build_intrinsic_binary(bld->builder, intrinsic, lp_build_vec_type(bld->type), a, b);
 270    }
 271
 272    if(LLVMIsConstant(a) && LLVMIsConstant(b))
 273       res = LLVMConstSub(a, b);
 274    else
 275       res = LLVMBuildSub(bld->builder, a, b, "");
 276
 277    if(bld->type.norm && (bld->type.floating || bld->type.fixed))
 278       res = lp_build_max_simple(bld, res, bld->zero);
 279
 280    return res;
 281 }
 282
 283
 284 /**
 285  * Normalized 8bit multiplication.
 286  *
 287  * - alpha plus one
 288  *
 289  *     makes the following approximation to the division (Sree)
 290  *
 291  *       a*b/255 ~= (a*(b + 1)) >> 256
 292  *
 293  *     which is the fastest method that satisfies the following OpenGL criteria
 294  *
 295  *       0*0 = 0 and 255*255 = 255
 296  *
 297  * - geometric series
 298  *
 299  *     takes the geometric series approximation to the division
 300  *
 301  *       t/255 = (t >> 8) + (t >> 16) + (t >> 24) ..
 302  *
 303  *     in this case just the first two terms to fit in 16bit arithmetic
 304  *
 305  *       t/255 ~= (t + (t >> 8)) >> 8
 306  *
 307  *     note that just by itself it doesn't satisfies the OpenGL criteria, as
 308  *     255*255 = 254, so the special case b = 255 must be accounted or roundoff
 309  *     must be used
 310  *
 311  * - geometric series plus rounding
 312  *
 313  *     when using a geometric series division instead of truncating the result
 314  *     use roundoff in the approximation (Jim Blinn)
 315  *
 316  *       t/255 ~= (t + (t >> 8) + 0x80) >> 8
 317  *
 318  *     achieving the exact results
 319  *
 320  * @sa Alvy Ray Smith, Image Compositing Fundamentals, Tech Memo 4, Aug 15, 1995,
 321  *     ftp://ftp.alvyray.com/Acrobat/4_Comp.pdf
 322  * @sa Michael Herf, The "double blend trick", May 2000,
 323  *     http://www.stereopsis.com/doubleblend.html
 324  */
 325 static LLVMValueRef
 326 lp_build_mul_u8n(LLVMBuilderRef builder,
 327                  struct lp_type i16_type,
 328                  LLVMValueRef a, LLVMValueRef b)
 329 {
 330    LLVMValueRef c8;
 331    LLVMValueRef ab;
 332
 333    c8 = lp_build_int_const_scalar(i16_type, 8);
 334
 335 #if 0
 336
 337    /* a*b/255 ~= (a*(b + 1)) >> 256 */
 338    b = LLVMBuildAdd(builder, b, lp_build_int_const_scalar(i16_type, 1), "");
 339    ab = LLVMBuildMul(builder, a, b, "");
 340
 341 #else
 342
 343    /* ab/255 ~= (ab + (ab >> 8) + 0x80) >> 8 */
 344    ab = LLVMBuildMul(builder, a, b, "");
 345    ab = LLVMBuildAdd(builder, ab, LLVMBuildLShr(builder, ab, c8, ""), "");
 346    ab = LLVMBuildAdd(builder, ab, lp_build_int_const_scalar(i16_type, 0x80), "");
 347
 348 #endif
 349
 350    ab = LLVMBuildLShr(builder, ab, c8, "");
 351
 352    return ab;
 353 }
 354
 355
 356 /**
 357  * Generate a * b
 358  */
 359 LLVMValueRef
 360 lp_build_mul(struct lp_build_context *bld,
 361              LLVMValueRef a,
 362              LLVMValueRef b)
 363 {
 364    const struct lp_type type = bld->type;
 365    LLVMValueRef shift;
 366    LLVMValueRef res;
 367
 368    if(a == bld->zero)
 369       return bld->zero;
 370    if(a == bld->one)
 371       return b;
 372    if(b == bld->zero)
 373       return bld->zero;
 374    if(b == bld->one)
 375       return a;
 376    if(a == bld->undef || b == bld->undef)
 377       return bld->undef;
 378
 379    if(!type.floating && !type.fixed && type.norm) {
 380       if(type.width == 8) {
 381          struct lp_type i16_type = lp_wider_type(type);
 382          LLVMValueRef al, ah, bl, bh, abl, abh, ab;
 383
 384          lp_build_unpack2(bld->builder, type, i16_type, a, &al, &ah);
 385          lp_build_unpack2(bld->builder, type, i16_type, b, &bl, &bh);
 386
 387          /* PMULLW, PSRLW, PADDW */
 388          abl = lp_build_mul_u8n(bld->builder, i16_type, al, bl);
 389          abh = lp_build_mul_u8n(bld->builder, i16_type, ah, bh);
 390
 391          ab = lp_build_pack2(bld->builder, i16_type, type, abl, abh);
 392
 393          return ab;
 394       }
 395
 396       /* FIXME */
 397       assert(0);
 398    }
 399
 400    if(type.fixed)
 401       shift = lp_build_int_const_scalar(type, type.width/2);
 402    else
 403       shift = NULL;
 404
 405    if(LLVMIsConstant(a) && LLVMIsConstant(b)) {
 406       res =  LLVMConstMul(a, b);
 407       if(shift) {
 408          if(type.sign)
 409             res = LLVMConstAShr(res, shift);
 410          else
 411             res = LLVMConstLShr(res, shift);
 412       }
 413    }
 414    else {
 415       res = LLVMBuildMul(bld->builder, a, b, "");
 416       if(shift) {
 417          if(type.sign)
 418             res = LLVMBuildAShr(bld->builder, res, shift, "");
 419          else
 420             res = LLVMBuildLShr(bld->builder, res, shift, "");
 421       }
 422    }
 423
 424    return res;
 425 }
 426
 427
 428 /**
 429  * Small vector x scale multiplication optimization.
 430  */
 431 LLVMValueRef
 432 lp_build_mul_imm(struct lp_build_context *bld,
 433                  LLVMValueRef a,
 434                  int b)
 435 {
 436    LLVMValueRef factor;
 437
 438    if(b == 0)
 439       return bld->zero;
 440
 441    if(b == 1)
 442       return a;
 443
 444    if(b == -1)
 445       return LLVMBuildNeg(bld->builder, a, "");
 446
 447    if(b == 2 && bld->type.floating)
 448       return lp_build_add(bld, a, a);
 449
 450    if(util_is_pot(b)) {
 451       unsigned shift = ffs(b) - 1;
 452
 453       if(bld->type.floating) {
 454 #if 0
 455          /*
 456           * Power of two multiplication by directly manipulating the mantissa.
 457           *
 458           * XXX: This might not be always faster, it will introduce a small error
 459           * for multiplication by zero, and it will produce wrong results
 460           * for Inf and NaN.
 461           */
 462          unsigned mantissa = lp_mantissa(bld->type);
 463          factor = lp_build_int_const_scalar(bld->type, (unsigned long long)shift << mantissa);
 464          a = LLVMBuildBitCast(bld->builder, a, lp_build_int_vec_type(bld->type), "");
 465          a = LLVMBuildAdd(bld->builder, a, factor, "");
 466          a = LLVMBuildBitCast(bld->builder, a, lp_build_vec_type(bld->type), "");
 467          return a;
 468 #endif
 469       }
 470       else {
 471          factor = lp_build_const_scalar(bld->type, shift);
 472          return LLVMBuildShl(bld->builder, a, factor, "");
 473       }
 474    }
 475
 476    factor = lp_build_const_scalar(bld->type, (double)b);
 477    return lp_build_mul(bld, a, factor);
 478 }
 479
 480
 481 /**
 482  * Generate a / b
 483  */
 484 LLVMValueRef
 485 lp_build_div(struct lp_build_context *bld,
 486              LLVMValueRef a,
 487              LLVMValueRef b)
 488 {
 489    const struct lp_type type = bld->type;
 490
 491    if(a == bld->zero)
 492       return bld->zero;
 493    if(a == bld->one)
 494       return lp_build_rcp(bld, b);
 495    if(b == bld->zero)
 496       return bld->undef;
 497    if(b == bld->one)
 498       return a;
 499    if(a == bld->undef || b == bld->undef)
 500       return bld->undef;
 501
 502    if(LLVMIsConstant(a) && LLVMIsConstant(b))
 503       return LLVMConstFDiv(a, b);
 504
 505    if(util_cpu_caps.has_sse && type.width == 32 && type.length == 4)
 506       return lp_build_mul(bld, a, lp_build_rcp(bld, b));
 507
 508    return LLVMBuildFDiv(bld->builder, a, b, "");
 509 }
 510
 511
 512 /**
 513  * Linear interpolation.
 514  *
 515  * This also works for integer values with a few caveats.
 516  *
 517  * @sa http://www.stereopsis.com/doubleblend.html
 518  */
 519 LLVMValueRef
 520 lp_build_lerp(struct lp_build_context *bld,
 521               LLVMValueRef x,
 522               LLVMValueRef v0,
 523               LLVMValueRef v1)
 524 {
 525    LLVMValueRef delta;
 526    LLVMValueRef res;
 527
 528    delta = lp_build_sub(bld, v1, v0);
 529
 530    res = lp_build_mul(bld, x, delta);
 531
 532    res = lp_build_add(bld, v0, res);
 533
 534    if(bld->type.fixed)
 535       /* XXX: This step is necessary for lerping 8bit colors stored on 16bits,
 536        * but it will be wrong for other uses. Basically we need a more
 537        * powerful lp_type, capable of further distinguishing the values
 538        * interpretation from the value storage. */
 539       res = LLVMBuildAnd(bld->builder, res, lp_build_int_const_scalar(bld->type, (1 << bld->type.width/2) - 1), "");
 540
 541    return res;
 542 }
 543
 544
 545 LLVMValueRef
 546 lp_build_lerp_2d(struct lp_build_context *bld,
 547                  LLVMValueRef x,
 548                  LLVMValueRef y,
 549                  LLVMValueRef v00,
 550                  LLVMValueRef v01,
 551                  LLVMValueRef v10,
 552                  LLVMValueRef v11)
 553 {
 554    LLVMValueRef v0 = lp_build_lerp(bld, x, v00, v01);
 555    LLVMValueRef v1 = lp_build_lerp(bld, x, v10, v11);
 556    return lp_build_lerp(bld, y, v0, v1);
 557 }
 558
 559
 560 /**
 561  * Generate min(a, b)
 562  * Do checks for special cases.
 563  */
 564 LLVMValueRef
 565 lp_build_min(struct lp_build_context *bld,
 566              LLVMValueRef a,
 567              LLVMValueRef b)
 568 {
 569    if(a == bld->undef || b == bld->undef)
 570       return bld->undef;
 571
 572    if(a == b)
 573       return a;
 574
 575    if(bld->type.norm) {
 576       if(a == bld->zero || b == bld->zero)
 577          return bld->zero;
 578       if(a == bld->one)
 579          return b;
 580       if(b == bld->one)
 581          return a;
 582    }
 583
 584    return lp_build_min_simple(bld, a, b);
 585 }
 586
 587
 588 /**
 589  * Generate max(a, b)
 590  * Do checks for special cases.
 591  */
 592 LLVMValueRef
 593 lp_build_max(struct lp_build_context *bld,
 594              LLVMValueRef a,
 595              LLVMValueRef b)
 596 {
 597    if(a == bld->undef || b == bld->undef)
 598       return bld->undef;
 599
 600    if(a == b)
 601       return a;
 602
 603    if(bld->type.norm) {
 604       if(a == bld->one || b == bld->one)
 605          return bld->one;
 606       if(a == bld->zero)
 607          return b;
 608       if(b == bld->zero)
 609          return a;
 610    }
 611
 612    return lp_build_max_simple(bld, a, b);
 613 }
 614
 615
 616 /**
 617  * Generate clamp(a, min, max)
 618  * Do checks for special cases.
 619  */
 620 LLVMValueRef
 621 lp_build_clamp(struct lp_build_context *bld,
 622                LLVMValueRef a,
 623                LLVMValueRef min,
 624                LLVMValueRef max)
 625 {
 626    a = lp_build_min(bld, a, max);
 627    a = lp_build_max(bld, a, min);
 628    return a;
 629 }
 630
 631
 632 /**
 633  * Generate abs(a)
 634  */
 635 LLVMValueRef
 636 lp_build_abs(struct lp_build_context *bld,
 637              LLVMValueRef a)
 638 {
 639    const struct lp_type type = bld->type;
 640    LLVMTypeRef vec_type = lp_build_vec_type(type);
 641
 642    if(!type.sign)
 643       return a;
 644
 645    if(type.floating) {
 646       /* Mask out the sign bit */
 647       if (type.length == 1) {
 648          LLVMTypeRef int_type = LLVMIntType(type.width);
 649          LLVMTypeRef float_type = LLVMFloatType();
 650          unsigned long long absMask = ~(1ULL << (type.width - 1));
 651          LLVMValueRef mask = LLVMConstInt(int_type, absMask, 0);
 652          a = LLVMBuildBitCast(bld->builder, a, int_type, "");
 653          a = LLVMBuildAnd(bld->builder, a, mask, "");
 654          a = LLVMBuildBitCast(bld->builder, a, float_type, "");
 655          return a;
 656       }
 657       else {
 658          /* vector of floats */
 659          LLVMTypeRef int_vec_type = lp_build_int_vec_type(type);
 660          unsigned long long absMask = ~(1ULL << (type.width - 1));
 661          LLVMValueRef mask = lp_build_int_const_scalar(type, ((unsigned long long) absMask));
 662          a = LLVMBuildBitCast(bld->builder, a, int_vec_type, "");
 663          a = LLVMBuildAnd(bld->builder, a, mask, "");
 664          a = LLVMBuildBitCast(bld->builder, a, vec_type, "");
 665          return a;
 666       }
 667    }
 668
 669    if(type.width*type.length == 128 && util_cpu_caps.has_ssse3) {
 670       switch(type.width) {
 671       case 8:
 672          return lp_build_intrinsic_unary(bld->builder, "llvm.x86.ssse3.pabs.b.128", vec_type, a);
 673       case 16:
 674          return lp_build_intrinsic_unary(bld->builder, "llvm.x86.ssse3.pabs.w.128", vec_type, a);
 675       case 32:
 676          return lp_build_intrinsic_unary(bld->builder, "llvm.x86.ssse3.pabs.d.128", vec_type, a);
 677       }
 678    }
 679
 680    return lp_build_max(bld, a, LLVMBuildNeg(bld->builder, a, ""));
 681 }
 682
 683
 684 LLVMValueRef
 685 lp_build_negate(struct lp_build_context *bld,
 686                 LLVMValueRef a)
 687 {
 688    return LLVMBuildNeg(bld->builder, a, "");
 689 }
 690
 691
 692 LLVMValueRef
 693 lp_build_sgn(struct lp_build_context *bld,
 694              LLVMValueRef a)
 695 {
 696    const struct lp_type type = bld->type;
 697    LLVMTypeRef vec_type = lp_build_vec_type(type);
 698    LLVMValueRef cond;
 699    LLVMValueRef res;
 700
 701    /* Handle non-zero case */
 702    if(!type.sign) {
 703       /* if not zero then sign must be positive */
 704       res = bld->one;
 705    }
 706    else if(type.floating) {
 707       /* Take the sign bit and add it to 1 constant */
 708       LLVMTypeRef int_vec_type = lp_build_int_vec_type(type);
 709       LLVMValueRef mask = lp_build_int_const_scalar(type, (unsigned long long)1 << (type.width - 1));
 710       LLVMValueRef sign;
 711       LLVMValueRef one;
 712       sign = LLVMBuildBitCast(bld->builder, a, int_vec_type, "");
 713       sign = LLVMBuildAnd(bld->builder, sign, mask, "");
 714       one = LLVMConstBitCast(bld->one, int_vec_type);
 715       res = LLVMBuildOr(bld->builder, sign, one, "");
 716       res = LLVMBuildBitCast(bld->builder, res, vec_type, "");
 717    }
 718    else
 719    {
 720       LLVMValueRef minus_one = lp_build_const_scalar(type, -1.0);
 721       cond = lp_build_cmp(bld, PIPE_FUNC_GREATER, a, bld->zero);
 722       res = lp_build_select(bld, cond, bld->one, minus_one);
 723    }
 724
 725    /* Handle zero */
 726    cond = lp_build_cmp(bld, PIPE_FUNC_EQUAL, a, bld->zero);
 727    res = lp_build_select(bld, cond, bld->zero, bld->one);
 728
 729    return res;
 730 }
 731
 732
 733 /**
 734  * Set the sign of float vector 'a' according to 'sign'.
 735  * If sign==0, return abs(a).
 736  * If sign==1, return -abs(a);
 737  * Other values for sign produce undefined results.
 738  */
 739 LLVMValueRef
 740 lp_build_set_sign(struct lp_build_context *bld,
 741                   LLVMValueRef a, LLVMValueRef sign)
 742 {
 743    const struct lp_type type = bld->type;
 744    LLVMTypeRef int_vec_type = lp_build_int_vec_type(type);
 745    LLVMTypeRef vec_type = lp_build_vec_type(type);
 746    LLVMValueRef shift = lp_build_int_const_scalar(type, type.width - 1);
 747    LLVMValueRef mask = lp_build_int_const_scalar(type,
 748                              ~((unsigned long long) 1 << (type.width - 1)));
 749    LLVMValueRef val, res;
 750
 751    assert(type.floating);
 752
 753    /* val = reinterpret_cast<int>(a) */
 754    val = LLVMBuildBitCast(bld->builder, a, int_vec_type, "");
 755    /* val = val & mask */
 756    val = LLVMBuildAnd(bld->builder, val, mask, "");
 757    /* sign = sign << shift */
 758    sign = LLVMBuildShl(bld->builder, sign, shift, "");
 759    /* res = val | sign */
 760    res = LLVMBuildOr(bld->builder, val, sign, "");
 761    /* res = reinterpret_cast<float>(res) */
 762    res = LLVMBuildBitCast(bld->builder, res, vec_type, "");
 763
 764    return res;
 765 }
 766
 767
 768 /**
 769  * Convert vector of (or scalar) int to vector of (or scalar) float.
 770  */
 771 LLVMValueRef
 772 lp_build_int_to_float(struct lp_build_context *bld,
 773                       LLVMValueRef a)
 774 {
 775    const struct lp_type type = bld->type;
 776
 777    assert(type.floating);
 778    /*assert(lp_check_value(type, a));*/
 779
 780    if (type.length == 1) {
 781       LLVMTypeRef float_type = LLVMFloatType();
 782       return LLVMBuildSIToFP(bld->builder, a, float_type, "");
 783    }
 784    else {
 785       LLVMTypeRef vec_type = lp_build_vec_type(type);
 786       /*LLVMTypeRef int_vec_type = lp_build_int_vec_type(type);*/
 787       LLVMValueRef res;
 788       res = LLVMBuildSIToFP(bld->builder, a, vec_type, "");
 789       return res;
 790    }
 791 }
 792
 793
 794
 795 enum lp_build_round_sse41_mode
 796 {
 797    LP_BUILD_ROUND_SSE41_NEAREST = 0,
 798    LP_BUILD_ROUND_SSE41_FLOOR = 1,
 799    LP_BUILD_ROUND_SSE41_CEIL = 2,
 800    LP_BUILD_ROUND_SSE41_TRUNCATE = 3
 801 };
 802
 803
 804 static INLINE LLVMValueRef
 805 lp_build_round_sse41(struct lp_build_context *bld,
 806                      LLVMValueRef a,
 807                      enum lp_build_round_sse41_mode mode)
 808 {
 809    const struct lp_type type = bld->type;
 810    LLVMTypeRef vec_type = lp_build_vec_type(type);
 811    const char *intrinsic;
 812
 813    assert(type.floating);
 814    assert(type.width*type.length == 128);
 815    assert(lp_check_value(type, a));
 816    assert(util_cpu_caps.has_sse4_1);
 817
 818    switch(type.width) {
 819    case 32:
 820       intrinsic = "llvm.x86.sse41.round.ps";
 821       break;
 822    case 64:
 823       intrinsic = "llvm.x86.sse41.round.pd";
 824       break;
 825    default:
 826       assert(0);
 827       return bld->undef;
 828    }
 829
 830    return lp_build_intrinsic_binary(bld->builder, intrinsic, vec_type, a,
 831                                     LLVMConstInt(LLVMInt32Type(), mode, 0));
 832 }
 833
 834
 835 LLVMValueRef
 836 lp_build_trunc(struct lp_build_context *bld,
 837                LLVMValueRef a)
 838 {
 839    const struct lp_type type = bld->type;
 840
 841    assert(type.floating);
 842    assert(lp_check_value(type, a));
 843
 844    if(util_cpu_caps.has_sse4_1)
 845       return lp_build_round_sse41(bld, a, LP_BUILD_ROUND_SSE41_TRUNCATE);
 846    else {
 847       LLVMTypeRef vec_type = lp_build_vec_type(type);
 848       LLVMTypeRef int_vec_type = lp_build_int_vec_type(type);
 849       LLVMValueRef res;
 850       res = LLVMBuildFPToSI(bld->builder, a, int_vec_type, "");
 851       res = LLVMBuildSIToFP(bld->builder, res, vec_type, "");
 852       return res;
 853    }
 854 }
 855
 856
 857 LLVMValueRef
 858 lp_build_round(struct lp_build_context *bld,
 859                LLVMValueRef a)
 860 {
 861    const struct lp_type type = bld->type;
 862
 863    assert(type.floating);
 864    assert(lp_check_value(type, a));
 865
 866    if(util_cpu_caps.has_sse4_1)
 867       return lp_build_round_sse41(bld, a, LP_BUILD_ROUND_SSE41_NEAREST);
 868    else {
 869       LLVMTypeRef vec_type = lp_build_vec_type(type);
 870       LLVMValueRef res;
 871       res = lp_build_iround(bld, a);
 872       res = LLVMBuildSIToFP(bld->builder, res, vec_type, "");
 873       return res;
 874    }
 875 }
 876
 877
 878 LLVMValueRef
 879 lp_build_floor(struct lp_build_context *bld,
 880                LLVMValueRef a)
 881 {
 882    const struct lp_type type = bld->type;
 883
 884    assert(type.floating);
 885
 886    if(util_cpu_caps.has_sse4_1)
 887       return lp_build_round_sse41(bld, a, LP_BUILD_ROUND_SSE41_FLOOR);
 888    else {
 889       LLVMTypeRef vec_type = lp_build_vec_type(type);
 890       LLVMValueRef res;
 891       res = lp_build_ifloor(bld, a);
 892       res = LLVMBuildSIToFP(bld->builder, res, vec_type, "");
 893       return res;
 894    }
 895 }
 896
 897
 898 LLVMValueRef
 899 lp_build_ceil(struct lp_build_context *bld,
 900               LLVMValueRef a)
 901 {
 902    const struct lp_type type = bld->type;
 903
 904    assert(type.floating);
 905    assert(lp_check_value(type, a));
 906
 907    if(util_cpu_caps.has_sse4_1)
 908       return lp_build_round_sse41(bld, a, LP_BUILD_ROUND_SSE41_CEIL);
 909    else {
 910       LLVMTypeRef vec_type = lp_build_vec_type(type);
 911       LLVMValueRef res;
 912       res = lp_build_iceil(bld, a);
 913       res = LLVMBuildSIToFP(bld->builder, res, vec_type, "");
 914       return res;
 915    }
 916 }
 917
 918
 919 /**
 920  * Return fractional part of 'a' computed as a - floor(f)
 921  * Typically used in texture coord arithmetic.
 922  */
 923 LLVMValueRef
 924 lp_build_fract(struct lp_build_context *bld,
 925                LLVMValueRef a)
 926 {
 927    assert(bld->type.floating);
 928    return lp_build_sub(bld, a, lp_build_floor(bld, a));
 929 }
 930
 931
 932 /**
 933  * Convert to integer, through whichever rounding method that's fastest,
 934  * typically truncating toward zero.
 935  */
 936 LLVMValueRef
 937 lp_build_itrunc(struct lp_build_context *bld,
 938                 LLVMValueRef a)
 939 {
 940    const struct lp_type type = bld->type;
 941
 942    assert(type.floating);
 943
 944    if (type.length == 1) {
 945       LLVMTypeRef int_type = LLVMIntType(type.width);
 946       return LLVMBuildFPTrunc(bld->builder, a, int_type, "");
 947    }
 948    else {
 949       LLVMTypeRef int_vec_type = lp_build_int_vec_type(type);
 950       assert(lp_check_value(type, a));
 951       return LLVMBuildFPToSI(bld->builder, a, int_vec_type, "");
 952    }
 953 }
 954
 955
 956 LLVMValueRef
 957 lp_build_iround(struct lp_build_context *bld,
 958                 LLVMValueRef a)
 959 {
 960    const struct lp_type type = bld->type;
 961    LLVMTypeRef int_vec_type = lp_build_int_vec_type(type);
 962    LLVMValueRef res;
 963
 964    assert(type.floating);
 965
 966    if (type.length == 1) {
 967       /* scalar float to int */
 968       LLVMTypeRef int_type = LLVMIntType(type.width);
 969       /* XXX we want rounding here! */
 970       res = LLVMBuildFPToSI(bld->builder, a, int_type, "");
 971       return res;
 972    }
 973
 974    assert(lp_check_value(type, a));
 975
 976    if(util_cpu_caps.has_sse4_1) {
 977       res = lp_build_round_sse41(bld, a, LP_BUILD_ROUND_SSE41_NEAREST);
 978    }
 979    else {
 980       LLVMTypeRef vec_type = lp_build_vec_type(type);
 981       LLVMValueRef mask = lp_build_int_const_scalar(type, (unsigned long long)1 << (type.width - 1));
 982       LLVMValueRef sign;
 983       LLVMValueRef half;
 984
 985       /* get sign bit */
 986       sign = LLVMBuildBitCast(bld->builder, a, int_vec_type, "");
 987       sign = LLVMBuildAnd(bld->builder, sign, mask, "");
 988
 989       /* sign * 0.5 */
 990       half = lp_build_const_scalar(type, 0.5);
 991       half = LLVMBuildBitCast(bld->builder, half, int_vec_type, "");
 992       half = LLVMBuildOr(bld->builder, sign, half, "");
 993       half = LLVMBuildBitCast(bld->builder, half, vec_type, "");
 994
 995       res = LLVMBuildAdd(bld->builder, a, half, "");
 996    }
 997
 998    res = LLVMBuildFPToSI(bld->builder, res, int_vec_type, "");
 999
1000    return res;
1001 }
1002
1003
1004 /**
1005  * Convert float[] to int[] with floor().
1006  */
1007 LLVMValueRef
1008 lp_build_ifloor(struct lp_build_context *bld,
1009                 LLVMValueRef a)
1010 {
1011    const struct lp_type type = bld->type;
1012    LLVMTypeRef int_vec_type = lp_build_int_vec_type(type);
1013    LLVMValueRef res;
1014
1015    assert(type.floating);
1016    assert(lp_check_value(type, a));
1017
1018    if(util_cpu_caps.has_sse4_1) {
1019       res = lp_build_round_sse41(bld, a, LP_BUILD_ROUND_SSE41_FLOOR);
1020    }
1021    else {
1022       /* Take the sign bit and add it to 1 constant */
1023       LLVMTypeRef vec_type = lp_build_vec_type(type);
1024       unsigned mantissa = lp_mantissa(type);
1025       LLVMValueRef mask = lp_build_int_const_scalar(type, (unsigned long long)1 << (type.width - 1));
1026       LLVMValueRef sign;
1027       LLVMValueRef offset;
1028
1029       /* sign = a < 0 ? ~0 : 0 */
1030       sign = LLVMBuildBitCast(bld->builder, a, int_vec_type, "");
1031       sign = LLVMBuildAnd(bld->builder, sign, mask, "");
1032       sign = LLVMBuildAShr(bld->builder, sign, lp_build_int_const_scalar(type, type.width - 1), "");
1033       lp_build_name(sign, "floor.sign");
1034
1035       /* offset = -0.99999(9)f */
1036       offset = lp_build_const_scalar(type, -(double)(((unsigned long long)1 << mantissa) - 1)/((unsigned long long)1 << mantissa));
1037       offset = LLVMConstBitCast(offset, int_vec_type);
1038
1039       /* offset = a < 0 ? -0.99999(9)f : 0.0f */
1040       offset = LLVMBuildAnd(bld->builder, offset, sign, "");
1041       offset = LLVMBuildBitCast(bld->builder, offset, vec_type, "");
1042       lp_build_name(offset, "floor.offset");
1043
1044       res = LLVMBuildAdd(bld->builder, a, offset, "");
1045       lp_build_name(res, "floor.res");
1046    }
1047
1048    res = LLVMBuildFPToSI(bld->builder, res, int_vec_type, "");
1049    lp_build_name(res, "floor");
1050
1051    return res;
1052 }
1053
1054
1055 LLVMValueRef
1056 lp_build_iceil(struct lp_build_context *bld,
1057                LLVMValueRef a)
1058 {
1059    const struct lp_type type = bld->type;
1060    LLVMTypeRef int_vec_type = lp_build_int_vec_type(type);
1061    LLVMValueRef res;
1062
1063    assert(type.floating);
1064    assert(lp_check_value(type, a));
1065
1066    if(util_cpu_caps.has_sse4_1) {
1067       res = lp_build_round_sse41(bld, a, LP_BUILD_ROUND_SSE41_CEIL);
1068    }
1069    else {
1070       assert(0);
1071       res = bld->undef;
1072    }
1073
1074    res = LLVMBuildFPToSI(bld->builder, res, int_vec_type, "");
1075
1076    return res;
1077 }
1078
1079
1080 LLVMValueRef
1081 lp_build_sqrt(struct lp_build_context *bld,
1082               LLVMValueRef a)
1083 {
1084    const struct lp_type type = bld->type;
1085    LLVMTypeRef vec_type = lp_build_vec_type(type);
1086    char intrinsic[32];
1087
1088    /* TODO: optimize the constant case */
1089    /* TODO: optimize the constant case */
1090
1091    assert(type.floating);
1092    util_snprintf(intrinsic, sizeof intrinsic, "llvm.sqrt.v%uf%u", type.length, type.width);
1093
1094    return lp_build_intrinsic_unary(bld->builder, intrinsic, vec_type, a);
1095 }
1096
1097
1098 LLVMValueRef
1099 lp_build_rcp(struct lp_build_context *bld,
1100              LLVMValueRef a)
1101 {
1102    const struct lp_type type = bld->type;
1103
1104    if(a == bld->zero)
1105       return bld->undef;
1106    if(a == bld->one)
1107       return bld->one;
1108    if(a == bld->undef)
1109       return bld->undef;
1110
1111    assert(type.floating);
1112
1113    if(LLVMIsConstant(a))
1114       return LLVMConstFDiv(bld->one, a);
1115
1116    if(util_cpu_caps.has_sse && type.width == 32 && type.length == 4)
1117       /* FIXME: improve precision */
1118       return lp_build_intrinsic_unary(bld->builder, "llvm.x86.sse.rcp.ps", lp_build_vec_type(type), a);
1119
1120    return LLVMBuildFDiv(bld->builder, bld->one, a, "");
1121 }
1122
1123
1124 /**
1125  * Generate 1/sqrt(a)
1126  */
1127 LLVMValueRef
1128 lp_build_rsqrt(struct lp_build_context *bld,
1129                LLVMValueRef a)
1130 {
1131    const struct lp_type type = bld->type;
1132
1133    assert(type.floating);
1134
1135    if(util_cpu_caps.has_sse && type.width == 32 && type.length == 4)
1136       return lp_build_intrinsic_unary(bld->builder, "llvm.x86.sse.rsqrt.ps", lp_build_vec_type(type), a);
1137
1138    return lp_build_rcp(bld, lp_build_sqrt(bld, a));
1139 }
1140
1141
1142 /**
1143  * Generate cos(a)
1144  */
1145 LLVMValueRef
1146 lp_build_cos(struct lp_build_context *bld,
1147               LLVMValueRef a)
1148 {
1149    const struct lp_type type = bld->type;
1150    LLVMTypeRef vec_type = lp_build_vec_type(type);
1151    char intrinsic[32];
1152
1153    /* TODO: optimize the constant case */
1154
1155    assert(type.floating);
1156    util_snprintf(intrinsic, sizeof intrinsic, "llvm.cos.v%uf%u", type.length, type.width);
1157
1158    return lp_build_intrinsic_unary(bld->builder, intrinsic, vec_type, a);
1159 }
1160
1161
1162 /**
1163  * Generate sin(a)
1164  */
1165 LLVMValueRef
1166 lp_build_sin(struct lp_build_context *bld,
1167               LLVMValueRef a)
1168 {
1169    const struct lp_type type = bld->type;
1170    LLVMTypeRef vec_type = lp_build_vec_type(type);
1171    char intrinsic[32];
1172
1173    /* TODO: optimize the constant case */
1174
1175    assert(type.floating);
1176    util_snprintf(intrinsic, sizeof intrinsic, "llvm.sin.v%uf%u", type.length, type.width);
1177
1178    return lp_build_intrinsic_unary(bld->builder, intrinsic, vec_type, a);
1179 }
1180
1181
1182 /**
1183  * Generate pow(x, y)
1184  */
1185 LLVMValueRef
1186 lp_build_pow(struct lp_build_context *bld,
1187              LLVMValueRef x,
1188              LLVMValueRef y)
1189 {
1190    /* TODO: optimize the constant case */
1191    if(LLVMIsConstant(x) && LLVMIsConstant(y))
1192       debug_printf("%s: inefficient/imprecise constant arithmetic\n",
1193                    __FUNCTION__);
1194
1195    return lp_build_exp2(bld, lp_build_mul(bld, lp_build_log2(bld, x), y));
1196 }
1197
1198
1199 /**
1200  * Generate exp(x)
1201  */
1202 LLVMValueRef
1203 lp_build_exp(struct lp_build_context *bld,
1204              LLVMValueRef x)
1205 {
1206    /* log2(e) = 1/log(2) */
1207    LLVMValueRef log2e = lp_build_const_scalar(bld->type, 1.4426950408889634);
1208
1209    return lp_build_mul(bld, log2e, lp_build_exp2(bld, x));
1210 }
1211
1212
1213 /**
1214  * Generate log(x)
1215  */
1216 LLVMValueRef
1217 lp_build_log(struct lp_build_context *bld,
1218              LLVMValueRef x)
1219 {
1220    /* log(2) */
1221    LLVMValueRef log2 = lp_build_const_scalar(bld->type, 0.69314718055994529);
1222
1223    return lp_build_mul(bld, log2, lp_build_exp2(bld, x));
1224 }
1225
1226
1227 #define EXP_POLY_DEGREE 3
1228 #define LOG_POLY_DEGREE 5
1229
1230
1231 /**
1232  * Generate polynomial.
1233  * Ex:  coeffs[0] + x * coeffs[1] + x^2 * coeffs[2].
1234  */
1235 static LLVMValueRef
1236 lp_build_polynomial(struct lp_build_context *bld,
1237                     LLVMValueRef x,
1238                     const double *coeffs,
1239                     unsigned num_coeffs)
1240 {
1241    const struct lp_type type = bld->type;
1242    LLVMTypeRef float_type = LLVMFloatType();
1243    LLVMValueRef res = NULL;
1244    unsigned i;
1245
1246    /* TODO: optimize the constant case */
1247    if(LLVMIsConstant(x))
1248       debug_printf("%s: inefficient/imprecise constant arithmetic\n",
1249                    __FUNCTION__);
1250
1251    for (i = num_coeffs; i--; ) {
1252       LLVMValueRef coeff;
1253
1254       if (type.length == 1)
1255          coeff = LLVMConstReal(float_type, coeffs[i]);
1256       else
1257          coeff = lp_build_const_scalar(type, coeffs[i]);
1258
1259       if(res)
1260          res = lp_build_add(bld, coeff, lp_build_mul(bld, x, res));
1261       else
1262          res = coeff;
1263    }
1264
1265    if(res)
1266       return res;
1267    else
1268       return bld->undef;
1269 }
1270
1271
1272 /**
1273  * Minimax polynomial fit of 2**x, in range [-0.5, 0.5[
1274  */
1275 const double lp_build_exp2_polynomial[] = {
1276 #if EXP_POLY_DEGREE == 5
1277    9.9999994e-1, 6.9315308e-1, 2.4015361e-1, 5.5826318e-2, 8.9893397e-3, 1.8775767e-3
1278 #elif EXP_POLY_DEGREE == 4
1279    1.0000026, 6.9300383e-1, 2.4144275e-1, 5.2011464e-2, 1.3534167e-2
1280 #elif EXP_POLY_DEGREE == 3
1281    9.9992520e-1, 6.9583356e-1, 2.2606716e-1, 7.8024521e-2
1282 #elif EXP_POLY_DEGREE == 2
1283    1.0017247, 6.5763628e-1, 3.3718944e-1
1284 #else
1285 #error
1286 #endif
1287 };
1288
1289
1290 void
1291 lp_build_exp2_approx(struct lp_build_context *bld,
1292                      LLVMValueRef x,
1293                      LLVMValueRef *p_exp2_int_part,
1294                      LLVMValueRef *p_frac_part,
1295                      LLVMValueRef *p_exp2)
1296 {
1297    const struct lp_type type = bld->type;
1298    LLVMTypeRef vec_type = lp_build_vec_type(type);
1299    LLVMTypeRef int_vec_type = lp_build_int_vec_type(type);
1300    LLVMValueRef ipart = NULL;
1301    LLVMValueRef fpart = NULL;
1302    LLVMValueRef expipart = NULL;
1303    LLVMValueRef expfpart = NULL;
1304    LLVMValueRef res = NULL;
1305
1306    if(p_exp2_int_part || p_frac_part || p_exp2) {
1307       /* TODO: optimize the constant case */
1308       if(LLVMIsConstant(x))
1309          debug_printf("%s: inefficient/imprecise constant arithmetic\n",
1310                       __FUNCTION__);
1311
1312       assert(type.floating && type.width == 32);
1313
1314       x = lp_build_min(bld, x, lp_build_const_scalar(type,  129.0));
1315       x = lp_build_max(bld, x, lp_build_const_scalar(type, -126.99999));
1316
1317       /* ipart = int(x - 0.5) */
1318       ipart = LLVMBuildSub(bld->builder, x, lp_build_const_scalar(type, 0.5f), "");
1319       ipart = LLVMBuildFPToSI(bld->builder, ipart, int_vec_type, "");
1320
1321       /* fpart = x - ipart */
1322       fpart = LLVMBuildSIToFP(bld->builder, ipart, vec_type, "");
1323       fpart = LLVMBuildSub(bld->builder, x, fpart, "");
1324    }
1325
1326    if(p_exp2_int_part || p_exp2) {
1327       /* expipart = (float) (1 << ipart) */
1328       expipart = LLVMBuildAdd(bld->builder, ipart, lp_build_int_const_scalar(type, 127), "");
1329       expipart = LLVMBuildShl(bld->builder, expipart, lp_build_int_const_scalar(type, 23), "");
1330       expipart = LLVMBuildBitCast(bld->builder, expipart, vec_type, "");
1331    }
1332
1333    if(p_exp2) {
1334       expfpart = lp_build_polynomial(bld, fpart, lp_build_exp2_polynomial,
1335                                      Elements(lp_build_exp2_polynomial));
1336
1337       res = LLVMBuildMul(bld->builder, expipart, expfpart, "");
1338    }
1339
1340    if(p_exp2_int_part)
1341       *p_exp2_int_part = expipart;
1342
1343    if(p_frac_part)
1344       *p_frac_part = fpart;
1345
1346    if(p_exp2)
1347       *p_exp2 = res;
1348 }
1349
1350
1351 LLVMValueRef
1352 lp_build_exp2(struct lp_build_context *bld,
1353               LLVMValueRef x)
1354 {
1355    LLVMValueRef res;
1356    lp_build_exp2_approx(bld, x, NULL, NULL, &res);
1357    return res;
1358 }
1359
1360
1361 /**
1362  * Minimax polynomial fit of log2(x)/(x - 1), for x in range [1, 2[
1363  * These coefficients can be generate with
1364  * http://www.boost.org/doc/libs/1_36_0/libs/math/doc/sf_and_dist/html/math_toolkit/toolkit/internals2/minimax.html
1365  */
1366 const double lp_build_log2_polynomial[] = {
1367 #if LOG_POLY_DEGREE == 6
1368    3.11578814719469302614, -3.32419399085241980044, 2.59883907202499966007, -1.23152682416275988241, 0.318212422185251071475, -0.0344359067839062357313
1369 #elif LOG_POLY_DEGREE == 5
1370    2.8882704548164776201, -2.52074962577807006663, 1.48116647521213171641, -0.465725644288844778798, 0.0596515482674574969533
1371 #elif LOG_POLY_DEGREE == 4
1372    2.61761038894603480148, -1.75647175389045657003, 0.688243882994381274313, -0.107254423828329604454
1373 #elif LOG_POLY_DEGREE == 3
1374    2.28330284476918490682, -1.04913055217340124191, 0.204446009836232697516
1375 #else
1376 #error
1377 #endif
1378 };
1379
1380
1381 /**
1382  * See http://www.devmaster.net/forums/showthread.php?p=43580
1383  */
1384 void
1385 lp_build_log2_approx(struct lp_build_context *bld,
1386                      LLVMValueRef x,
1387                      LLVMValueRef *p_exp,
1388                      LLVMValueRef *p_floor_log2,
1389                      LLVMValueRef *p_log2)
1390 {
1391    const struct lp_type type = bld->type;
1392    LLVMTypeRef vec_type = lp_build_vec_type(type);
1393    LLVMTypeRef int_vec_type = lp_build_int_vec_type(type);
1394
1395    LLVMValueRef expmask = lp_build_int_const_scalar(type, 0x7f800000);
1396    LLVMValueRef mantmask = lp_build_int_const_scalar(type, 0x007fffff);
1397    LLVMValueRef one = LLVMConstBitCast(bld->one, int_vec_type);
1398
1399    LLVMValueRef i = NULL;
1400    LLVMValueRef exp = NULL;
1401    LLVMValueRef mant = NULL;
1402    LLVMValueRef logexp = NULL;
1403    LLVMValueRef logmant = NULL;
1404    LLVMValueRef res = NULL;
1405
1406    if(p_exp || p_floor_log2 || p_log2) {
1407       /* TODO: optimize the constant case */
1408       if(LLVMIsConstant(x))
1409          debug_printf("%s: inefficient/imprecise constant arithmetic\n",
1410                       __FUNCTION__);
1411
1412       assert(type.floating && type.width == 32);
1413
1414       i = LLVMBuildBitCast(bld->builder, x, int_vec_type, "");
1415
1416       /* exp = (float) exponent(x) */
1417       exp = LLVMBuildAnd(bld->builder, i, expmask, "");
1418    }
1419
1420    if(p_floor_log2 || p_log2) {
1421       logexp = LLVMBuildLShr(bld->builder, exp, lp_build_int_const_scalar(type, 23), "");
1422       logexp = LLVMBuildSub(bld->builder, logexp, lp_build_int_const_scalar(type, 127), "");
1423       logexp = LLVMBuildSIToFP(bld->builder, logexp, vec_type, "");
1424    }
1425
1426    if(p_log2) {
1427       /* mant = (float) mantissa(x) */
1428       mant = LLVMBuildAnd(bld->builder, i, mantmask, "");
1429       mant = LLVMBuildOr(bld->builder, mant, one, "");
1430       mant = LLVMBuildBitCast(bld->builder, mant, vec_type, "");
1431
1432       logmant = lp_build_polynomial(bld, mant, lp_build_log2_polynomial,
1433                                     Elements(lp_build_log2_polynomial));
1434
1435       /* This effectively increases the polynomial degree by one, but ensures that log2(1) == 0*/
1436       logmant = LLVMBuildMul(bld->builder, logmant, LLVMBuildSub(bld->builder, mant, bld->one, ""), "");
1437
1438       res = LLVMBuildAdd(bld->builder, logmant, logexp, "");
1439    }
1440
1441    if(p_exp)
1442       *p_exp = exp;
1443
1444    if(p_floor_log2)
1445       *p_floor_log2 = logexp;
1446
1447    if(p_log2)
1448       *p_log2 = res;
1449 }
1450
1451
1452 /** scalar version of above function */
1453 static void
1454 lp_build_float_log2_approx(struct lp_build_context *bld,
1455                            LLVMValueRef x,
1456                            LLVMValueRef *p_exp,
1457                            LLVMValueRef *p_floor_log2,
1458                            LLVMValueRef *p_log2)
1459 {
1460    const struct lp_type type = bld->type;
1461    LLVMTypeRef float_type = LLVMFloatType();
1462    LLVMTypeRef int_type = LLVMIntType(type.width);
1463
1464    LLVMValueRef expmask = LLVMConstInt(int_type, 0x7f800000, 0);
1465    LLVMValueRef mantmask = LLVMConstInt(int_type, 0x007fffff, 0);
1466    LLVMValueRef one = LLVMConstBitCast(bld->one, int_type);
1467
1468    LLVMValueRef i = NULL;
1469    LLVMValueRef exp = NULL;
1470    LLVMValueRef mant = NULL;
1471    LLVMValueRef logexp = NULL;
1472    LLVMValueRef logmant = NULL;
1473    LLVMValueRef res = NULL;
1474
1475    if(p_exp || p_floor_log2 || p_log2) {
1476       /* TODO: optimize the constant case */
1477       if(LLVMIsConstant(x))
1478          debug_printf("%s: inefficient/imprecise constant arithmetic\n",
1479                       __FUNCTION__);
1480
1481       assert(type.floating && type.width == 32);
1482
1483       i = LLVMBuildBitCast(bld->builder, x, int_type, "");
1484
1485       /* exp = (float) exponent(x) */
1486       exp = LLVMBuildAnd(bld->builder, i, expmask, "");
1487    }
1488
1489    if(p_floor_log2 || p_log2) {
1490       LLVMValueRef c23 = LLVMConstInt(int_type, 23, 0);
1491       LLVMValueRef c127 = LLVMConstInt(int_type, 127, 0);
1492       logexp = LLVMBuildLShr(bld->builder, exp, c23, "");
1493       logexp = LLVMBuildSub(bld->builder, logexp, c127, "");
1494       logexp = LLVMBuildSIToFP(bld->builder, logexp, float_type, "");
1495    }
1496
1497    if(p_log2) {
1498       /* mant = (float) mantissa(x) */
1499       mant = LLVMBuildAnd(bld->builder, i, mantmask, "");
1500       mant = LLVMBuildOr(bld->builder, mant, one, "");
1501       mant = LLVMBuildBitCast(bld->builder, mant, float_type, "");
1502
1503       logmant = lp_build_polynomial(bld, mant, lp_build_log2_polynomial,
1504                                     Elements(lp_build_log2_polynomial));
1505
1506       /* This effectively increases the polynomial degree by one, but ensures that log2(1) == 0*/
1507       logmant = LLVMBuildMul(bld->builder, logmant, LLVMBuildSub(bld->builder, mant, bld->one, ""), "");
1508
1509       res = LLVMBuildAdd(bld->builder, logmant, logexp, "");
1510    }
1511
1512    if(p_exp)
1513       *p_exp = exp;
1514
1515    if(p_floor_log2)
1516       *p_floor_log2 = logexp;
1517
1518    if(p_log2)
1519       *p_log2 = res;
1520 }
1521
1522
1523 LLVMValueRef
1524 lp_build_log2(struct lp_build_context *bld,
1525               LLVMValueRef x)
1526 {
1527    LLVMValueRef res;
1528    if (bld->type.length == 1) {
1529       lp_build_float_log2_approx(bld, x, NULL, NULL, &res);
1530    }
1531    else {
1532       lp_build_log2_approx(bld, x, NULL, NULL, &res);
1533    }
1534    return res;
1535 }