src/gallium/auxiliary/gallivm/lp_bld_swizzle.c

   1 /**************************************************************************
   2  *
   3  * Copyright 2009 VMware, Inc.
   4  * All Rights Reserved.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the
   8  * "Software"), to deal in the Software without restriction, including
   9  * without limitation the rights to use, copy, modify, merge, publish,
  10  * distribute, sub license, and/or sell copies of the Software, and to
  11  * permit persons to whom the Software is furnished to do so, subject to
  12  * the following conditions:
  13  *
  14  * The above copyright notice and this permission notice (including the
  15  * next paragraph) shall be included in all copies or substantial portions
  16  * of the Software.
  17  *
  18  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
  19  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  20  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
  21  * IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR
  22  * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
  23  * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
  24  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  25  *
  26  **************************************************************************/
  27
  28 /**
  29  * @file
  30  * Helper functions for swizzling/shuffling.
  31  *
  32  * @author Jose Fonseca <jfonseca@vmware.com>
  33  */
  34
  35
  36 #include "util/u_debug.h"
  37
  38 #include "lp_bld_type.h"
  39 #include "lp_bld_const.h"
  40 #include "lp_bld_init.h"
  41 #include "lp_bld_logic.h"
  42 #include "lp_bld_swizzle.h"
  43
  44
  45 LLVMValueRef
  46 lp_build_broadcast(struct gallivm_state *gallivm,
  47                    LLVMTypeRef vec_type,
  48                    LLVMValueRef scalar)
  49 {
  50    LLVMValueRef res;
  51
  52    if (LLVMGetTypeKind(vec_type) != LLVMVectorTypeKind) {
  53       /* scalar */
  54       assert(vec_type == LLVMTypeOf(scalar));
  55       res = scalar;
  56    } else {
  57       LLVMBuilderRef builder = gallivm->builder;
  58       const unsigned length = LLVMGetVectorSize(vec_type);
  59       LLVMValueRef undef = LLVMGetUndef(vec_type);
  60       LLVMTypeRef i32_type = LLVMInt32TypeInContext(gallivm->context);
  61
  62       assert(LLVMGetElementType(vec_type) == LLVMTypeOf(scalar));
  63
  64       if (HAVE_LLVM >= 0x207) {
  65          /* The shuffle vector is always made of int32 elements */
  66          LLVMTypeRef i32_vec_type = LLVMVectorType(i32_type, length);
  67          res = LLVMBuildInsertElement(builder, undef, scalar, LLVMConstNull(i32_type), "");
  68          res = LLVMBuildShuffleVector(builder, res, undef, LLVMConstNull(i32_vec_type), "");
  69       } else {
  70          /* XXX: The above path provokes a bug in LLVM 2.6 */
  71          unsigned i;
  72          res = undef;
  73          for(i = 0; i < length; ++i) {
  74             LLVMValueRef index = lp_build_const_int32(gallivm, i);
  75             res = LLVMBuildInsertElement(builder, res, scalar, index, "");
  76          }
  77       }
  78    }
  79
  80    return res;
  81 }
  82
  83
  84 /**
  85  * Broadcast
  86  */
  87 LLVMValueRef
  88 lp_build_broadcast_scalar(struct lp_build_context *bld,
  89                           LLVMValueRef scalar)
  90 {
  91    assert(lp_check_elem_type(bld->type, LLVMTypeOf(scalar)));
  92
  93    return lp_build_broadcast(bld->gallivm, bld->vec_type, scalar);
  94 }
  95
  96
  97 /**
  98  * Combined extract and broadcast (or a mere shuffle when the two types match)
  99  */
 100 LLVMValueRef
 101 lp_build_extract_broadcast(struct gallivm_state *gallivm,
 102                            struct lp_type src_type,
 103                            struct lp_type dst_type,
 104                            LLVMValueRef vector,
 105                            LLVMValueRef index)
 106 {
 107    LLVMTypeRef i32t = LLVMInt32TypeInContext(gallivm->context);
 108    LLVMValueRef res;
 109
 110    assert(src_type.floating == dst_type.floating);
 111    assert(src_type.width    == dst_type.width);
 112
 113    assert(lp_check_value(src_type, vector));
 114    assert(LLVMTypeOf(index) == i32t);
 115
 116    if (src_type.length == 1) {
 117       if (dst_type.length == 1) {
 118          /*
 119           * Trivial scalar -> scalar.
 120           */
 121
 122          res = vector;
 123       }
 124       else {
 125          /*
 126           * Broadcast scalar -> vector.
 127           */
 128
 129          res = lp_build_broadcast(gallivm,
 130                                   lp_build_vec_type(gallivm, dst_type),
 131                                   vector);
 132       }
 133    }
 134    else {
 135       if (dst_type.length == src_type.length) {
 136          /*
 137           * Special shuffle of the same size.
 138           */
 139
 140          LLVMValueRef shuffle;
 141          shuffle = lp_build_broadcast(gallivm,
 142                                       LLVMVectorType(i32t, dst_type.length),
 143                                       index);
 144          res = LLVMBuildShuffleVector(gallivm->builder, vector,
 145                                       LLVMGetUndef(lp_build_vec_type(gallivm, dst_type)),
 146                                       shuffle, "");
 147       }
 148       else {
 149          LLVMValueRef scalar;
 150          scalar = LLVMBuildExtractElement(gallivm->builder, vector, index, "");
 151          if (dst_type.length == 1) {
 152             /*
 153              * Trivial extract scalar from vector.
 154              */
 155
 156             res = scalar;
 157          }
 158          else {
 159             /*
 160              * General case of different sized vectors.
 161              */
 162
 163             res = lp_build_broadcast(gallivm,
 164                                      lp_build_vec_type(gallivm, dst_type),
 165                                      vector);
 166          }
 167       }
 168    }
 169
 170    return res;
 171 }
 172
 173
 174 /**
 175  * Swizzle one channel into all other three channels.
 176  */
 177 LLVMValueRef
 178 lp_build_swizzle_scalar_aos(struct lp_build_context *bld,
 179                             LLVMValueRef a,
 180                             unsigned channel)
 181 {
 182    LLVMBuilderRef builder = bld->gallivm->builder;
 183    const struct lp_type type = bld->type;
 184    const unsigned n = type.length;
 185    unsigned i, j;
 186
 187    if(a == bld->undef || a == bld->zero || a == bld->one)
 188       return a;
 189
 190    /* XXX: SSE3 has PSHUFB which should be better than bitmasks, but forcing
 191     * using shuffles here actually causes worst results. More investigation is
 192     * needed. */
 193    if (type.width >= 16) {
 194       /*
 195        * Shuffle.
 196        */
 197       LLVMTypeRef elem_type = LLVMInt32TypeInContext(bld->gallivm->context);
 198       LLVMValueRef shuffles[LP_MAX_VECTOR_LENGTH];
 199
 200       for(j = 0; j < n; j += 4)
 201          for(i = 0; i < 4; ++i)
 202             shuffles[j + i] = LLVMConstInt(elem_type, j + channel, 0);
 203
 204       return LLVMBuildShuffleVector(builder, a, bld->undef, LLVMConstVector(shuffles, n), "");
 205    }
 206    else {
 207       /*
 208        * Bit mask and recursive shifts
 209        *
 210        *   XYZW XYZW .... XYZW  <= input
 211        *   0Y00 0Y00 .... 0Y00
 212        *   YY00 YY00 .... YY00
 213        *   YYYY YYYY .... YYYY  <= output
 214        */
 215       struct lp_type type4;
 216       const char shifts[4][2] = {
 217          { 1,  2},
 218          {-1,  2},
 219          { 1, -2},
 220          {-1, -2}
 221       };
 222       unsigned i;
 223
 224       a = LLVMBuildAnd(builder, a,
 225                        lp_build_const_mask_aos(bld->gallivm,
 226                                                type, 1 << channel), "");
 227
 228       /*
 229        * Build a type where each element is an integer that cover the four
 230        * channels.
 231        */
 232
 233       type4 = type;
 234       type4.floating = FALSE;
 235       type4.width *= 4;
 236       type4.length /= 4;
 237
 238       a = LLVMBuildBitCast(builder, a, lp_build_vec_type(bld->gallivm, type4), "");
 239
 240       for(i = 0; i < 2; ++i) {
 241          LLVMValueRef tmp = NULL;
 242          int shift = shifts[channel][i];
 243
 244 #ifdef PIPE_ARCH_LITTLE_ENDIAN
 245          shift = -shift;
 246 #endif
 247
 248          if(shift > 0)
 249             tmp = LLVMBuildLShr(builder, a, lp_build_const_int_vec(bld->gallivm, type4, shift*type.width), "");
 250          if(shift < 0)
 251             tmp = LLVMBuildShl(builder, a, lp_build_const_int_vec(bld->gallivm, type4, -shift*type.width), "");
 252
 253          assert(tmp);
 254          if(tmp)
 255             a = LLVMBuildOr(builder, a, tmp, "");
 256       }
 257
 258       return LLVMBuildBitCast(builder, a, lp_build_vec_type(bld->gallivm, type), "");
 259    }
 260 }
 261
 262
 263 LLVMValueRef
 264 lp_build_swizzle_aos(struct lp_build_context *bld,
 265                      LLVMValueRef a,
 266                      const unsigned char swizzles[4])
 267 {
 268    LLVMBuilderRef builder = bld->gallivm->builder;
 269    const struct lp_type type = bld->type;
 270    const unsigned n = type.length;
 271    unsigned i, j;
 272
 273    if (swizzles[0] == PIPE_SWIZZLE_RED &&
 274        swizzles[1] == PIPE_SWIZZLE_GREEN &&
 275        swizzles[2] == PIPE_SWIZZLE_BLUE &&
 276        swizzles[3] == PIPE_SWIZZLE_ALPHA) {
 277       return a;
 278    }
 279
 280    if (swizzles[0] == swizzles[1] &&
 281        swizzles[1] == swizzles[2] &&
 282        swizzles[2] == swizzles[3]) {
 283       switch (swizzles[0]) {
 284       case PIPE_SWIZZLE_RED:
 285       case PIPE_SWIZZLE_GREEN:
 286       case PIPE_SWIZZLE_BLUE:
 287       case PIPE_SWIZZLE_ALPHA:
 288          return lp_build_swizzle_scalar_aos(bld, a, swizzles[0]);
 289       case PIPE_SWIZZLE_ZERO:
 290          return bld->zero;
 291       case PIPE_SWIZZLE_ONE:
 292          return bld->one;
 293       default:
 294          assert(0);
 295          return bld->undef;
 296       }
 297    }
 298
 299    if (type.width >= 16) {
 300       /*
 301        * Shuffle.
 302        */
 303       LLVMValueRef undef = LLVMGetUndef(lp_build_elem_type(bld->gallivm, type));
 304       LLVMTypeRef i32t = LLVMInt32TypeInContext(bld->gallivm->context);
 305       LLVMValueRef shuffles[LP_MAX_VECTOR_LENGTH];
 306       LLVMValueRef aux[LP_MAX_VECTOR_LENGTH];
 307
 308       memset(aux, 0, sizeof aux);
 309
 310       for(j = 0; j < n; j += 4) {
 311          for(i = 0; i < 4; ++i) {
 312             unsigned shuffle;
 313             switch (swizzles[i]) {
 314             default:
 315                assert(0);
 316                /* fall through */
 317             case PIPE_SWIZZLE_RED:
 318             case PIPE_SWIZZLE_GREEN:
 319             case PIPE_SWIZZLE_BLUE:
 320             case PIPE_SWIZZLE_ALPHA:
 321                shuffle = j + swizzles[i];
 322                break;
 323             case PIPE_SWIZZLE_ZERO:
 324                shuffle = type.length + 0;
 325                if (!aux[0]) {
 326                   aux[0] = lp_build_const_elem(bld->gallivm, type, 0.0);
 327                }
 328                break;
 329             case PIPE_SWIZZLE_ONE:
 330                shuffle = type.length + 1;
 331                if (!aux[1]) {
 332                   aux[1] = lp_build_const_elem(bld->gallivm, type, 1.0);
 333                }
 334                break;
 335             }
 336             shuffles[j + i] = LLVMConstInt(i32t, shuffle, 0);
 337          }
 338       }
 339
 340       for (i = 0; i < n; ++i) {
 341          if (!aux[i]) {
 342             aux[i] = undef;
 343          }
 344       }
 345
 346       return LLVMBuildShuffleVector(builder, a,
 347                                     LLVMConstVector(aux, n),
 348                                     LLVMConstVector(shuffles, n), "");
 349    } else {
 350       /*
 351        * Bit mask and shifts.
 352        *
 353        * For example, this will convert BGRA to RGBA by doing
 354        *
 355        *   rgba = (bgra & 0x00ff0000) >> 16
 356        *        | (bgra & 0xff00ff00)
 357        *        | (bgra & 0x000000ff) << 16
 358        *
 359        * This is necessary not only for faster cause, but because X86 backend
 360        * will refuse shuffles of <4 x i8> vectors
 361        */
 362       LLVMValueRef res;
 363       struct lp_type type4;
 364       unsigned cond = 0;
 365       unsigned chan;
 366       int shift;
 367
 368       /*
 369        * Start with a mixture of 1 and 0.
 370        */
 371       for (chan = 0; chan < 4; ++chan) {
 372          if (swizzles[chan] == PIPE_SWIZZLE_ONE) {
 373             cond |= 1 << chan;
 374          }
 375       }
 376       res = lp_build_select_aos(bld, cond, bld->one, bld->zero);
 377
 378       /*
 379        * Build a type where each element is an integer that cover the four
 380        * channels.
 381        */
 382       type4 = type;
 383       type4.floating = FALSE;
 384       type4.width *= 4;
 385       type4.length /= 4;
 386
 387       a = LLVMBuildBitCast(builder, a, lp_build_vec_type(bld->gallivm, type4), "");
 388       res = LLVMBuildBitCast(builder, res, lp_build_vec_type(bld->gallivm, type4), "");
 389
 390       /*
 391        * Mask and shift the channels, trying to group as many channels in the
 392        * same shift as possible
 393        */
 394       for (shift = -3; shift <= 3; ++shift) {
 395          unsigned long long mask = 0;
 396
 397          assert(type4.width <= sizeof(mask)*8);
 398
 399          for (chan = 0; chan < 4; ++chan) {
 400             /* FIXME: big endian */
 401             if (swizzles[chan] < 4 &&
 402                 chan - swizzles[chan] == shift) {
 403                mask |= ((1ULL << type.width) - 1) << (swizzles[chan] * type.width);
 404             }
 405          }
 406
 407          if (mask) {
 408             LLVMValueRef masked;
 409             LLVMValueRef shifted;
 410
 411             if (0)
 412                debug_printf("shift = %i, mask = 0x%08llx\n", shift, mask);
 413
 414             masked = LLVMBuildAnd(builder, a,
 415                                   lp_build_const_int_vec(bld->gallivm, type4, mask), "");
 416             if (shift > 0) {
 417                shifted = LLVMBuildShl(builder, masked,
 418                                       lp_build_const_int_vec(bld->gallivm, type4, shift*type.width), "");
 419             } else if (shift < 0) {
 420                shifted = LLVMBuildLShr(builder, masked,
 421                                        lp_build_const_int_vec(bld->gallivm, type4, -shift*type.width), "");
 422             } else {
 423                shifted = masked;
 424             }
 425
 426             res = LLVMBuildOr(builder, res, shifted, "");
 427          }
 428       }
 429
 430       return LLVMBuildBitCast(builder, res,
 431                               lp_build_vec_type(bld->gallivm, type), "");
 432    }
 433 }
 434
 435
 436 /**
 437  * Extended swizzle of a single channel of a SoA vector.
 438  *
 439  * @param bld         building context
 440  * @param unswizzled  array with the 4 unswizzled values
 441  * @param swizzle     one of the PIPE_SWIZZLE_*
 442  *
 443  * @return  the swizzled value.
 444  */
 445 LLVMValueRef
 446 lp_build_swizzle_soa_channel(struct lp_build_context *bld,
 447                              const LLVMValueRef *unswizzled,
 448                              unsigned swizzle)
 449 {
 450    switch (swizzle) {
 451    case PIPE_SWIZZLE_RED:
 452    case PIPE_SWIZZLE_GREEN:
 453    case PIPE_SWIZZLE_BLUE:
 454    case PIPE_SWIZZLE_ALPHA:
 455       return unswizzled[swizzle];
 456    case PIPE_SWIZZLE_ZERO:
 457       return bld->zero;
 458    case PIPE_SWIZZLE_ONE:
 459       return bld->one;
 460    default:
 461       assert(0);
 462       return bld->undef;
 463    }
 464 }
 465
 466
 467 /**
 468  * Extended swizzle of a SoA vector.
 469  *
 470  * @param bld         building context
 471  * @param unswizzled  array with the 4 unswizzled values
 472  * @param swizzles    array of PIPE_SWIZZLE_*
 473  * @param swizzled    output swizzled values
 474  */
 475 void
 476 lp_build_swizzle_soa(struct lp_build_context *bld,
 477                      const LLVMValueRef *unswizzled,
 478                      const unsigned char swizzles[4],
 479                      LLVMValueRef *swizzled)
 480 {
 481    unsigned chan;
 482
 483    for (chan = 0; chan < 4; ++chan) {
 484       swizzled[chan] = lp_build_swizzle_soa_channel(bld, unswizzled,
 485                                                     swizzles[chan]);
 486    }
 487 }
 488
 489
 490 /**
 491  * Do an extended swizzle of a SoA vector inplace.
 492  *
 493  * @param bld         building context
 494  * @param values      intput/output array with the 4 values
 495  * @param swizzles    array of PIPE_SWIZZLE_*
 496  */
 497 void
 498 lp_build_swizzle_soa_inplace(struct lp_build_context *bld,
 499                              LLVMValueRef *values,
 500                              const unsigned char swizzles[4])
 501 {
 502    LLVMValueRef unswizzled[4];
 503    unsigned chan;
 504
 505    for (chan = 0; chan < 4; ++chan) {
 506       unswizzled[chan] = values[chan];
 507    }
 508
 509    lp_build_swizzle_soa(bld, unswizzled, swizzles, values);
 510 }