src/gallium/auxiliary/gallivm/lp_bld_logic.c

   1 /**************************************************************************
   2  *
   3  * Copyright 2009 VMware, Inc.
   4  * All Rights Reserved.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the
   8  * "Software"), to deal in the Software without restriction, including
   9  * without limitation the rights to use, copy, modify, merge, publish,
  10  * distribute, sub license, and/or sell copies of the Software, and to
  11  * permit persons to whom the Software is furnished to do so, subject to
  12  * the following conditions:
  13  *
  14  * The above copyright notice and this permission notice (including the
  15  * next paragraph) shall be included in all copies or substantial portions
  16  * of the Software.
  17  *
  18  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
  19  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  20  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
  21  * IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR
  22  * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
  23  * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
  24  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  25  *
  26  **************************************************************************/
  27
  28 /**
  29  * @file
  30  * Helper functions for logical operations.
  31  *
  32  * @author Jose Fonseca <jfonseca@vmware.com>
  33  */
  34
  35
  36 #include "util/u_cpu_detect.h"
  37 #include "util/u_memory.h"
  38 #include "util/u_debug.h"
  39
  40 #include "lp_bld_type.h"
  41 #include "lp_bld_const.h"
  42 #include "lp_bld_intr.h"
  43 #include "lp_bld_debug.h"
  44 #include "lp_bld_logic.h"
  45
  46
  47 /*
  48  * XXX
  49  *
  50  * Selection with vector conditional like
  51  *
  52  *    select <4 x i1> %C, %A, %B
  53  *
  54  * is valid IR (e.g. llvm/test/Assembler/vector-select.ll), but it is not
  55  * supported on any backend.
  56  *
  57  * Expanding the boolean vector to full SIMD register width, as in
  58  *
  59  *    sext <4 x i1> %C to <4 x i32>
  60  *
  61  * is valid and supported (e.g., llvm/test/CodeGen/X86/vec_compare.ll), but
  62  * it causes assertion failures in LLVM 2.6. It appears to work correctly on
  63  * LLVM 2.7.
  64  */
  65
  66
  67 /**
  68  * Build code to compare two values 'a' and 'b' of 'type' using the given func.
  69  * \param func  one of PIPE_FUNC_x
  70  * The result values will be 0 for false or ~0 for true.
  71  */
  72 LLVMValueRef
  73 lp_build_compare(LLVMBuilderRef builder,
  74                  const struct lp_type type,
  75                  unsigned func,
  76                  LLVMValueRef a,
  77                  LLVMValueRef b)
  78 {
  79    LLVMTypeRef int_vec_type = lp_build_int_vec_type(type);
  80    LLVMValueRef zeros = LLVMConstNull(int_vec_type);
  81    LLVMValueRef ones = LLVMConstAllOnes(int_vec_type);
  82    LLVMValueRef cond;
  83    LLVMValueRef res;
  84
  85    assert(func >= PIPE_FUNC_NEVER);
  86    assert(func <= PIPE_FUNC_ALWAYS);
  87    assert(lp_check_value(type, a));
  88    assert(lp_check_value(type, b));
  89
  90    if(func == PIPE_FUNC_NEVER)
  91       return zeros;
  92    if(func == PIPE_FUNC_ALWAYS)
  93       return ones;
  94
  95    /* TODO: optimize the constant case */
  96
  97    /* XXX: It is not clear if we should use the ordered or unordered operators */
  98
  99 #if HAVE_LLVM < 0x0207
 100 #if defined(PIPE_ARCH_X86) || defined(PIPE_ARCH_X86_64)
 101    if(type.width * type.length == 128) {
 102       if(type.floating && util_cpu_caps.has_sse) {
 103          /* float[4] comparison */
 104          LLVMTypeRef vec_type = lp_build_vec_type(type);
 105          LLVMValueRef args[3];
 106          unsigned cc;
 107          boolean swap;
 108
 109          swap = FALSE;
 110          switch(func) {
 111          case PIPE_FUNC_EQUAL:
 112             cc = 0;
 113             break;
 114          case PIPE_FUNC_NOTEQUAL:
 115             cc = 4;
 116             break;
 117          case PIPE_FUNC_LESS:
 118             cc = 1;
 119             break;
 120          case PIPE_FUNC_LEQUAL:
 121             cc = 2;
 122             break;
 123          case PIPE_FUNC_GREATER:
 124             cc = 1;
 125             swap = TRUE;
 126             break;
 127          case PIPE_FUNC_GEQUAL:
 128             cc = 2;
 129             swap = TRUE;
 130             break;
 131          default:
 132             assert(0);
 133             return lp_build_undef(type);
 134          }
 135
 136          if(swap) {
 137             args[0] = b;
 138             args[1] = a;
 139          }
 140          else {
 141             args[0] = a;
 142             args[1] = b;
 143          }
 144
 145          args[2] = LLVMConstInt(LLVMInt8Type(), cc, 0);
 146          res = lp_build_intrinsic(builder,
 147                                   "llvm.x86.sse.cmp.ps",
 148                                   vec_type,
 149                                   args, 3);
 150          res = LLVMBuildBitCast(builder, res, int_vec_type, "");
 151          return res;
 152       }
 153       else if(util_cpu_caps.has_sse2) {
 154          /* int[4] comparison */
 155          static const struct {
 156             unsigned swap:1;
 157             unsigned eq:1;
 158             unsigned gt:1;
 159             unsigned not:1;
 160          } table[] = {
 161             {0, 0, 0, 1}, /* PIPE_FUNC_NEVER */
 162             {1, 0, 1, 0}, /* PIPE_FUNC_LESS */
 163             {0, 1, 0, 0}, /* PIPE_FUNC_EQUAL */
 164             {0, 0, 1, 1}, /* PIPE_FUNC_LEQUAL */
 165             {0, 0, 1, 0}, /* PIPE_FUNC_GREATER */
 166             {0, 1, 0, 1}, /* PIPE_FUNC_NOTEQUAL */
 167             {1, 0, 1, 1}, /* PIPE_FUNC_GEQUAL */
 168             {0, 0, 0, 0}  /* PIPE_FUNC_ALWAYS */
 169          };
 170          const char *pcmpeq;
 171          const char *pcmpgt;
 172          LLVMValueRef args[2];
 173          LLVMValueRef res;
 174          LLVMTypeRef vec_type = lp_build_vec_type(type);
 175
 176          switch (type.width) {
 177          case 8:
 178             pcmpeq = "llvm.x86.sse2.pcmpeq.b";
 179             pcmpgt = "llvm.x86.sse2.pcmpgt.b";
 180             break;
 181          case 16:
 182             pcmpeq = "llvm.x86.sse2.pcmpeq.w";
 183             pcmpgt = "llvm.x86.sse2.pcmpgt.w";
 184             break;
 185          case 32:
 186             pcmpeq = "llvm.x86.sse2.pcmpeq.d";
 187             pcmpgt = "llvm.x86.sse2.pcmpgt.d";
 188             break;
 189          default:
 190             assert(0);
 191             return lp_build_undef(type);
 192          }
 193
 194          /* There are no unsigned comparison instructions. So flip the sign bit
 195           * so that the results match.
 196           */
 197          if (table[func].gt && !type.sign) {
 198             LLVMValueRef msb = lp_build_const_int_vec(type, (unsigned long long)1 << (type.width - 1));
 199             a = LLVMBuildXor(builder, a, msb, "");
 200             b = LLVMBuildXor(builder, b, msb, "");
 201          }
 202
 203          if(table[func].swap) {
 204             args[0] = b;
 205             args[1] = a;
 206          }
 207          else {
 208             args[0] = a;
 209             args[1] = b;
 210          }
 211
 212          if(table[func].eq)
 213             res = lp_build_intrinsic(builder, pcmpeq, vec_type, args, 2);
 214          else if (table[func].gt)
 215             res = lp_build_intrinsic(builder, pcmpgt, vec_type, args, 2);
 216          else
 217             res = LLVMConstNull(vec_type);
 218
 219          if(table[func].not)
 220             res = LLVMBuildNot(builder, res, "");
 221
 222          return res;
 223       }
 224    } /* if (type.width * type.length == 128) */
 225 #endif
 226 #endif /* HAVE_LLVM < 0x0207 */
 227
 228    if(type.floating) {
 229       LLVMRealPredicate op;
 230       switch(func) {
 231       case PIPE_FUNC_NEVER:
 232          op = LLVMRealPredicateFalse;
 233          break;
 234       case PIPE_FUNC_ALWAYS:
 235          op = LLVMRealPredicateTrue;
 236          break;
 237       case PIPE_FUNC_EQUAL:
 238          op = LLVMRealUEQ;
 239          break;
 240       case PIPE_FUNC_NOTEQUAL:
 241          op = LLVMRealUNE;
 242          break;
 243       case PIPE_FUNC_LESS:
 244          op = LLVMRealULT;
 245          break;
 246       case PIPE_FUNC_LEQUAL:
 247          op = LLVMRealULE;
 248          break;
 249       case PIPE_FUNC_GREATER:
 250          op = LLVMRealUGT;
 251          break;
 252       case PIPE_FUNC_GEQUAL:
 253          op = LLVMRealUGE;
 254          break;
 255       default:
 256          assert(0);
 257          return lp_build_undef(type);
 258       }
 259
 260 #if HAVE_LLVM >= 0x0207
 261       cond = LLVMBuildFCmp(builder, op, a, b, "");
 262       res = LLVMBuildSExt(builder, cond, int_vec_type, "");
 263 #else
 264       if (type.length == 1) {
 265          cond = LLVMBuildFCmp(builder, op, a, b, "");
 266          res = LLVMBuildSExt(builder, cond, int_vec_type, "");
 267       }
 268       else {
 269          unsigned i;
 270
 271          res = LLVMGetUndef(int_vec_type);
 272
 273          debug_printf("%s: warning: using slow element-wise float"
 274                       " vector comparison\n", __FUNCTION__);
 275          for (i = 0; i < type.length; ++i) {
 276             LLVMValueRef index = LLVMConstInt(LLVMInt32Type(), i, 0);
 277             cond = LLVMBuildFCmp(builder, op,
 278                                  LLVMBuildExtractElement(builder, a, index, ""),
 279                                  LLVMBuildExtractElement(builder, b, index, ""),
 280                                  "");
 281             cond = LLVMBuildSelect(builder, cond,
 282                                    LLVMConstExtractElement(ones, index),
 283                                    LLVMConstExtractElement(zeros, index),
 284                                    "");
 285             res = LLVMBuildInsertElement(builder, res, cond, index, "");
 286          }
 287       }
 288 #endif
 289    }
 290    else {
 291       LLVMIntPredicate op;
 292       switch(func) {
 293       case PIPE_FUNC_EQUAL:
 294          op = LLVMIntEQ;
 295          break;
 296       case PIPE_FUNC_NOTEQUAL:
 297          op = LLVMIntNE;
 298          break;
 299       case PIPE_FUNC_LESS:
 300          op = type.sign ? LLVMIntSLT : LLVMIntULT;
 301          break;
 302       case PIPE_FUNC_LEQUAL:
 303          op = type.sign ? LLVMIntSLE : LLVMIntULE;
 304          break;
 305       case PIPE_FUNC_GREATER:
 306          op = type.sign ? LLVMIntSGT : LLVMIntUGT;
 307          break;
 308       case PIPE_FUNC_GEQUAL:
 309          op = type.sign ? LLVMIntSGE : LLVMIntUGE;
 310          break;
 311       default:
 312          assert(0);
 313          return lp_build_undef(type);
 314       }
 315
 316 #if HAVE_LLVM >= 0x0207
 317       cond = LLVMBuildICmp(builder, op, a, b, "");
 318       res = LLVMBuildSExt(builder, cond, int_vec_type, "");
 319 #else
 320       if (type.length == 1) {
 321          cond = LLVMBuildICmp(builder, op, a, b, "");
 322          res = LLVMBuildSExt(builder, cond, int_vec_type, "");
 323       }
 324       else {
 325          unsigned i;
 326
 327          res = LLVMGetUndef(int_vec_type);
 328
 329          if (gallivm_debug & GALLIVM_DEBUG_PERF) {
 330             debug_printf("%s: using slow element-wise int"
 331                          " vector comparison\n", __FUNCTION__);
 332          }
 333
 334          for(i = 0; i < type.length; ++i) {
 335             LLVMValueRef index = LLVMConstInt(LLVMInt32Type(), i, 0);
 336             cond = LLVMBuildICmp(builder, op,
 337                                  LLVMBuildExtractElement(builder, a, index, ""),
 338                                  LLVMBuildExtractElement(builder, b, index, ""),
 339                                  "");
 340             cond = LLVMBuildSelect(builder, cond,
 341                                    LLVMConstExtractElement(ones, index),
 342                                    LLVMConstExtractElement(zeros, index),
 343                                    "");
 344             res = LLVMBuildInsertElement(builder, res, cond, index, "");
 345          }
 346       }
 347 #endif
 348    }
 349
 350    return res;
 351 }
 352
 353
 354
 355 /**
 356  * Build code to compare two values 'a' and 'b' using the given func.
 357  * \param func  one of PIPE_FUNC_x
 358  * The result values will be 0 for false or ~0 for true.
 359  */
 360 LLVMValueRef
 361 lp_build_cmp(struct lp_build_context *bld,
 362              unsigned func,
 363              LLVMValueRef a,
 364              LLVMValueRef b)
 365 {
 366    return lp_build_compare(bld->builder, bld->type, func, a, b);
 367 }
 368
 369
 370 /**
 371  * Return (mask & a) | (~mask & b);
 372  */
 373 LLVMValueRef
 374 lp_build_select_bitwise(struct lp_build_context *bld,
 375                         LLVMValueRef mask,
 376                         LLVMValueRef a,
 377                         LLVMValueRef b)
 378 {
 379    struct lp_type type = bld->type;
 380    LLVMValueRef res;
 381
 382    assert(lp_check_value(type, a));
 383    assert(lp_check_value(type, b));
 384
 385    if (a == b) {
 386       return a;
 387    }
 388
 389    if(type.floating) {
 390       LLVMTypeRef int_vec_type = lp_build_int_vec_type(type);
 391       a = LLVMBuildBitCast(bld->builder, a, int_vec_type, "");
 392       b = LLVMBuildBitCast(bld->builder, b, int_vec_type, "");
 393    }
 394
 395    a = LLVMBuildAnd(bld->builder, a, mask, "");
 396
 397    /* This often gets translated to PANDN, but sometimes the NOT is
 398     * pre-computed and stored in another constant. The best strategy depends
 399     * on available registers, so it is not a big deal -- hopefully LLVM does
 400     * the right decision attending the rest of the program.
 401     */
 402    b = LLVMBuildAnd(bld->builder, b, LLVMBuildNot(bld->builder, mask, ""), "");
 403
 404    res = LLVMBuildOr(bld->builder, a, b, "");
 405
 406    if(type.floating) {
 407       LLVMTypeRef vec_type = lp_build_vec_type(type);
 408       res = LLVMBuildBitCast(bld->builder, res, vec_type, "");
 409    }
 410
 411    return res;
 412 }
 413
 414
 415 /**
 416  * Return mask ? a : b;
 417  *
 418  * mask is a bitwise mask, composed of 0 or ~0 for each element. Any other value
 419  * will yield unpredictable results.
 420  */
 421 LLVMValueRef
 422 lp_build_select(struct lp_build_context *bld,
 423                 LLVMValueRef mask,
 424                 LLVMValueRef a,
 425                 LLVMValueRef b)
 426 {
 427    struct lp_type type = bld->type;
 428    LLVMValueRef res;
 429
 430    assert(lp_check_value(type, a));
 431    assert(lp_check_value(type, b));
 432
 433    if(a == b)
 434       return a;
 435
 436    if (type.length == 1) {
 437       mask = LLVMBuildTrunc(bld->builder, mask, LLVMInt1Type(), "");
 438       res = LLVMBuildSelect(bld->builder, mask, a, b, "");
 439    }
 440    else if (util_cpu_caps.has_sse4_1 &&
 441             type.width * type.length == 128 &&
 442             !LLVMIsConstant(a) &&
 443             !LLVMIsConstant(b) &&
 444             !LLVMIsConstant(mask)) {
 445       const char *intrinsic;
 446       LLVMTypeRef arg_type;
 447       LLVMValueRef args[3];
 448
 449       if (type.width == 64) {
 450          intrinsic = "llvm.x86.sse41.blendvpd";
 451          arg_type = LLVMVectorType(LLVMDoubleType(), 2);
 452       } else if (type.width == 32) {
 453          intrinsic = "llvm.x86.sse41.blendvps";
 454          arg_type = LLVMVectorType(LLVMFloatType(), 4);
 455       } else {
 456          intrinsic = "llvm.x86.sse41.pblendvb";
 457          arg_type = LLVMVectorType(LLVMInt8Type(), 16);
 458       }
 459
 460       if (arg_type != bld->int_vec_type) {
 461          mask = LLVMBuildBitCast(bld->builder, mask, arg_type, "");
 462       }
 463
 464       if (arg_type != bld->vec_type) {
 465          a = LLVMBuildBitCast(bld->builder, a, arg_type, "");
 466          b = LLVMBuildBitCast(bld->builder, b, arg_type, "");
 467       }
 468
 469       args[0] = b;
 470       args[1] = a;
 471       args[2] = mask;
 472
 473       res = lp_build_intrinsic(bld->builder, intrinsic,
 474                                arg_type, args, Elements(args));
 475
 476       if (arg_type != bld->vec_type) {
 477          res = LLVMBuildBitCast(bld->builder, res, bld->vec_type, "");
 478       }
 479    }
 480    else {
 481       res = lp_build_select_bitwise(bld, mask, a, b);
 482    }
 483
 484    return res;
 485 }
 486
 487
 488 /**
 489  * Return mask ? a : b;
 490  *
 491  * mask is a TGSI_WRITEMASK_xxx.
 492  */
 493 LLVMValueRef
 494 lp_build_select_aos(struct lp_build_context *bld,
 495                     unsigned mask,
 496                     LLVMValueRef a,
 497                     LLVMValueRef b)
 498 {
 499    const struct lp_type type = bld->type;
 500    const unsigned n = type.length;
 501    unsigned i, j;
 502
 503    assert((mask & ~0xf) == 0);
 504    assert(lp_check_value(type, a));
 505    assert(lp_check_value(type, b));
 506
 507    if(a == b)
 508       return a;
 509    if((mask & 0xf) == 0xf)
 510       return a;
 511    if((mask & 0xf) == 0x0)
 512       return b;
 513    if(a == bld->undef || b == bld->undef)
 514       return bld->undef;
 515
 516    /*
 517     * There are three major ways of accomplishing this:
 518     * - with a shuffle,
 519     * - with a select,
 520     * - or with a bit mask.
 521     *
 522     * Select isn't supported for vector types yet.
 523     * The flip between these is empirical and might need to be.
 524     */
 525    if (n <= 4) {
 526       /*
 527        * Shuffle.
 528        */
 529       LLVMTypeRef elem_type = LLVMInt32Type();
 530       LLVMValueRef shuffles[LP_MAX_VECTOR_LENGTH];
 531
 532       for(j = 0; j < n; j += 4)
 533          for(i = 0; i < 4; ++i)
 534             shuffles[j + i] = LLVMConstInt(elem_type,
 535                                            (mask & (1 << i) ? 0 : n) + j + i,
 536                                            0);
 537
 538       return LLVMBuildShuffleVector(bld->builder, a, b, LLVMConstVector(shuffles, n), "");
 539    }
 540    else {
 541 #if 0
 542       /* XXX: Unfortunately select of vectors do not work */
 543       /* Use a select */
 544       LLVMTypeRef elem_type = LLVMInt1Type();
 545       LLVMValueRef cond_vec[LP_MAX_VECTOR_LENGTH];
 546
 547       for(j = 0; j < n; j += 4)
 548          for(i = 0; i < 4; ++i)
 549             cond_vec[j + i] = LLVMConstInt(elem_type,
 550                                            mask & (1 << i) ? 1 : 0, 0);
 551
 552       return LLVMBuildSelect(bld->builder, LLVMConstVector(cond_vec, n), a, b, "");
 553 #else
 554       LLVMValueRef mask_vec = lp_build_const_mask_aos(type, mask);
 555       return lp_build_select(bld, mask_vec, a, b);
 556 #endif
 557    }
 558 }
 559
 560
 561 /** Return (a & ~b) */
 562 LLVMValueRef
 563 lp_build_andc(struct lp_build_context *bld, LLVMValueRef a, LLVMValueRef b)
 564 {
 565    const struct lp_type type = bld->type;
 566
 567    assert(lp_check_value(type, a));
 568    assert(lp_check_value(type, b));
 569
 570    /* can't do bitwise ops on floating-point values */
 571    if(type.floating) {
 572       a = LLVMBuildBitCast(bld->builder, a, bld->int_vec_type, "");
 573       b = LLVMBuildBitCast(bld->builder, b, bld->int_vec_type, "");
 574    }
 575
 576    b = LLVMBuildNot(bld->builder, b, "");
 577    b = LLVMBuildAnd(bld->builder, a, b, "");
 578
 579    if(type.floating) {
 580       b = LLVMBuildBitCast(bld->builder, b, bld->vec_type, "");
 581    }
 582    return b;
 583 }