src/gallium/auxiliary/gallivm/lp_bld_swizzle.c

   1 /**************************************************************************
   2  *
   3  * Copyright 2009 VMware, Inc.
   4  * All Rights Reserved.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the
   8  * "Software"), to deal in the Software without restriction, including
   9  * without limitation the rights to use, copy, modify, merge, publish,
  10  * distribute, sub license, and/or sell copies of the Software, and to
  11  * permit persons to whom the Software is furnished to do so, subject to
  12  * the following conditions:
  13  *
  14  * The above copyright notice and this permission notice (including the
  15  * next paragraph) shall be included in all copies or substantial portions
  16  * of the Software.
  17  *
  18  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
  19  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  20  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
  21  * IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR
  22  * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
  23  * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
  24  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  25  *
  26  **************************************************************************/
  27
  28 /**
  29  * @file
  30  * Helper functions for swizzling/shuffling.
  31  *
  32  * @author Jose Fonseca <jfonseca@vmware.com>
  33  */
  34
  35
  36 #include "util/u_debug.h"
  37
  38 #include "lp_bld_type.h"
  39 #include "lp_bld_const.h"
  40 #include "lp_bld_logic.h"
  41 #include "lp_bld_swizzle.h"
  42
  43
  44 LLVMValueRef
  45 lp_build_broadcast(LLVMBuilderRef builder,
  46                    LLVMTypeRef vec_type,
  47                    LLVMValueRef scalar)
  48 {
  49    const unsigned n = LLVMGetVectorSize(vec_type);
  50    LLVMValueRef res;
  51    unsigned i;
  52
  53    res = LLVMGetUndef(vec_type);
  54    for(i = 0; i < n; ++i) {
  55       LLVMValueRef index = LLVMConstInt(LLVMInt32Type(), i, 0);
  56       res = LLVMBuildInsertElement(builder, res, scalar, index, "");
  57    }
  58
  59    return res;
  60 }
  61
  62
  63 /**
  64  * Broadcast
  65  */
  66 LLVMValueRef
  67 lp_build_broadcast_scalar(struct lp_build_context *bld,
  68                           LLVMValueRef scalar)
  69 {
  70    const struct lp_type type = bld->type;
  71
  72    assert(lp_check_elem_type(type, LLVMTypeOf(scalar)));
  73
  74    if (type.length == 1) {
  75       return scalar;
  76    }
  77    else {
  78       LLVMValueRef res;
  79
  80 #if HAVE_LLVM >= 0x207
  81       /* The shuffle vector is always made of int32 elements */
  82       struct lp_type i32_vec_type = lp_type_int_vec(32);
  83       i32_vec_type.length = type.length;
  84
  85       res = LLVMBuildInsertElement(bld->builder, bld->undef, scalar,
  86                                    LLVMConstInt(LLVMInt32Type(), 0, 0), "");
  87       res = LLVMBuildShuffleVector(bld->builder, res, bld->undef,
  88                                    lp_build_const_int_vec(i32_vec_type, 0), "");
  89 #else
  90       /* XXX: The above path provokes a bug in LLVM 2.6 */
  91       unsigned i;
  92       res = bld->undef;
  93       for(i = 0; i < type.length; ++i) {
  94          LLVMValueRef index = LLVMConstInt(LLVMInt32Type(), i, 0);
  95          res = LLVMBuildInsertElement(bld->builder, res, scalar, index, "");
  96       }
  97 #endif
  98       return res;
  99    }
 100 }
 101
 102
 103 /**
 104  * Combined extract and broadcast (or a mere shuffle when the two types match)
 105  */
 106 LLVMValueRef
 107 lp_build_extract_broadcast(LLVMBuilderRef builder,
 108                            struct lp_type src_type,
 109                            struct lp_type dst_type,
 110                            LLVMValueRef vector,
 111                            LLVMValueRef index)
 112 {
 113    LLVMTypeRef i32t = LLVMInt32Type();
 114    LLVMValueRef res;
 115
 116    assert(src_type.floating == dst_type.floating);
 117    assert(src_type.width    == dst_type.width);
 118
 119    assert(lp_check_value(src_type, vector));
 120    assert(LLVMTypeOf(index) == i32t);
 121
 122    if (src_type.length == 1) {
 123       if (dst_type.length == 1) {
 124          /*
 125           * Trivial scalar -> scalar.
 126           */
 127
 128          res = vector;
 129       }
 130       else {
 131          /*
 132           * Broadcast scalar -> vector.
 133           */
 134
 135          res = lp_build_broadcast(builder,
 136                                   lp_build_vec_type(dst_type),
 137                                   vector);
 138       }
 139    }
 140    else {
 141       if (dst_type.length == src_type.length) {
 142          /*
 143           * Special shuffle of the same size.
 144           */
 145
 146          LLVMValueRef shuffle;
 147          shuffle = lp_build_broadcast(builder,
 148                                       LLVMVectorType(i32t, dst_type.length),
 149                                       index);
 150          res = LLVMBuildShuffleVector(builder, vector,
 151                                       LLVMGetUndef(lp_build_vec_type(dst_type)),
 152                                       shuffle, "");
 153       }
 154       else {
 155          LLVMValueRef scalar;
 156          scalar = LLVMBuildExtractElement(builder, vector, index, "");
 157          if (dst_type.length == 1) {
 158             /*
 159              * Trivial extract scalar from vector.
 160              */
 161
 162             res = scalar;
 163          }
 164          else {
 165             /*
 166              * General case of different sized vectors.
 167              */
 168
 169             res = lp_build_broadcast(builder,
 170                                      lp_build_vec_type(dst_type),
 171                                      vector);
 172          }
 173       }
 174    }
 175
 176    return res;
 177 }
 178
 179
 180 /**
 181  * Swizzle one channel into all other three channels.
 182  */
 183 LLVMValueRef
 184 lp_build_swizzle_scalar_aos(struct lp_build_context *bld,
 185                             LLVMValueRef a,
 186                             unsigned channel)
 187 {
 188    const struct lp_type type = bld->type;
 189    const unsigned n = type.length;
 190    unsigned i, j;
 191
 192    if(a == bld->undef || a == bld->zero || a == bld->one)
 193       return a;
 194
 195    /* XXX: SSE3 has PSHUFB which should be better than bitmasks, but forcing
 196     * using shuffles here actually causes worst results. More investigation is
 197     * needed. */
 198    if (type.width >= 16) {
 199       /*
 200        * Shuffle.
 201        */
 202       LLVMTypeRef elem_type = LLVMInt32Type();
 203       LLVMValueRef shuffles[LP_MAX_VECTOR_LENGTH];
 204
 205       for(j = 0; j < n; j += 4)
 206          for(i = 0; i < 4; ++i)
 207             shuffles[j + i] = LLVMConstInt(elem_type, j + channel, 0);
 208
 209       return LLVMBuildShuffleVector(bld->builder, a, bld->undef, LLVMConstVector(shuffles, n), "");
 210    }
 211    else {
 212       /*
 213        * Bit mask and recursive shifts
 214        *
 215        *   XYZW XYZW .... XYZW  <= input
 216        *   0Y00 0Y00 .... 0Y00
 217        *   YY00 YY00 .... YY00
 218        *   YYYY YYYY .... YYYY  <= output
 219        */
 220       struct lp_type type4;
 221       const char shifts[4][2] = {
 222          { 1,  2},
 223          {-1,  2},
 224          { 1, -2},
 225          {-1, -2}
 226       };
 227       unsigned i;
 228
 229       a = LLVMBuildAnd(bld->builder, a,
 230                        lp_build_const_mask_aos(type, 1 << channel), "");
 231
 232       /*
 233        * Build a type where each element is an integer that cover the four
 234        * channels.
 235        */
 236
 237       type4 = type;
 238       type4.floating = FALSE;
 239       type4.width *= 4;
 240       type4.length /= 4;
 241
 242       a = LLVMBuildBitCast(bld->builder, a, lp_build_vec_type(type4), "");
 243
 244       for(i = 0; i < 2; ++i) {
 245          LLVMValueRef tmp = NULL;
 246          int shift = shifts[channel][i];
 247
 248 #ifdef PIPE_ARCH_LITTLE_ENDIAN
 249          shift = -shift;
 250 #endif
 251
 252          if(shift > 0)
 253             tmp = LLVMBuildLShr(bld->builder, a, lp_build_const_int_vec(type4, shift*type.width), "");
 254          if(shift < 0)
 255             tmp = LLVMBuildShl(bld->builder, a, lp_build_const_int_vec(type4, -shift*type.width), "");
 256
 257          assert(tmp);
 258          if(tmp)
 259             a = LLVMBuildOr(bld->builder, a, tmp, "");
 260       }
 261
 262       return LLVMBuildBitCast(bld->builder, a, lp_build_vec_type(type), "");
 263    }
 264 }
 265
 266
 267 LLVMValueRef
 268 lp_build_swizzle_aos(struct lp_build_context *bld,
 269                      LLVMValueRef a,
 270                      const unsigned char swizzles[4])
 271 {
 272    const struct lp_type type = bld->type;
 273    const unsigned n = type.length;
 274    unsigned i, j;
 275
 276    if (swizzles[0] == PIPE_SWIZZLE_RED &&
 277        swizzles[1] == PIPE_SWIZZLE_GREEN &&
 278        swizzles[2] == PIPE_SWIZZLE_BLUE &&
 279        swizzles[3] == PIPE_SWIZZLE_ALPHA) {
 280       return a;
 281    }
 282
 283    if (swizzles[0] == swizzles[1] &&
 284        swizzles[1] == swizzles[2] &&
 285        swizzles[2] == swizzles[3]) {
 286       switch (swizzles[0]) {
 287       case PIPE_SWIZZLE_RED:
 288       case PIPE_SWIZZLE_GREEN:
 289       case PIPE_SWIZZLE_BLUE:
 290       case PIPE_SWIZZLE_ALPHA:
 291          return lp_build_swizzle_scalar_aos(bld, a, swizzles[0]);
 292       case PIPE_SWIZZLE_ZERO:
 293          return bld->zero;
 294       case PIPE_SWIZZLE_ONE:
 295          return bld->one;
 296       default:
 297          assert(0);
 298          return bld->undef;
 299       }
 300    }
 301
 302    if (type.width >= 16) {
 303       /*
 304        * Shuffle.
 305        */
 306       LLVMValueRef undef = LLVMGetUndef(lp_build_elem_type(type));
 307       LLVMTypeRef i32t = LLVMInt32Type();
 308       LLVMValueRef shuffles[LP_MAX_VECTOR_LENGTH];
 309       LLVMValueRef aux[LP_MAX_VECTOR_LENGTH];
 310
 311       memset(aux, 0, sizeof aux);
 312
 313       for(j = 0; j < n; j += 4) {
 314          for(i = 0; i < 4; ++i) {
 315             unsigned shuffle;
 316             switch (swizzles[i]) {
 317             default:
 318                assert(0);
 319                /* fall through */
 320             case PIPE_SWIZZLE_RED:
 321             case PIPE_SWIZZLE_GREEN:
 322             case PIPE_SWIZZLE_BLUE:
 323             case PIPE_SWIZZLE_ALPHA:
 324                shuffle = j + swizzles[i];
 325                break;
 326             case PIPE_SWIZZLE_ZERO:
 327                shuffle = type.length + 0;
 328                if (!aux[0]) {
 329                   aux[0] = lp_build_const_elem(type, 0.0);
 330                }
 331                break;
 332             case PIPE_SWIZZLE_ONE:
 333                shuffle = type.length + 1;
 334                if (!aux[1]) {
 335                   aux[1] = lp_build_const_elem(type, 1.0);
 336                }
 337                break;
 338             }
 339             shuffles[j + i] = LLVMConstInt(i32t, shuffle, 0);
 340          }
 341       }
 342
 343       for (i = 0; i < n; ++i) {
 344          if (!aux[i]) {
 345             aux[i] = undef;
 346          }
 347       }
 348
 349       return LLVMBuildShuffleVector(bld->builder, a,
 350                                     LLVMConstVector(aux, n),
 351                                     LLVMConstVector(shuffles, n), "");
 352    } else {
 353       /*
 354        * Bit mask and shifts.
 355        *
 356        * For example, this will convert BGRA to RGBA by doing
 357        *
 358        *   rgba = (bgra & 0x00ff0000) >> 16
 359        *        | (bgra & 0xff00ff00)
 360        *        | (bgra & 0x000000ff) << 16
 361        *
 362        * This is necessary not only for faster cause, but because X86 backend
 363        * will refuse shuffles of <4 x i8> vectors
 364        */
 365       LLVMValueRef res;
 366       struct lp_type type4;
 367       unsigned cond = 0;
 368       unsigned chan;
 369       int shift;
 370
 371       /*
 372        * Start with a mixture of 1 and 0.
 373        */
 374       for (chan = 0; chan < 4; ++chan) {
 375          if (swizzles[chan] == PIPE_SWIZZLE_ONE) {
 376             cond |= 1 << chan;
 377          }
 378       }
 379       res = lp_build_select_aos(bld, cond, bld->one, bld->zero);
 380
 381       /*
 382        * Build a type where each element is an integer that cover the four
 383        * channels.
 384        */
 385       type4 = type;
 386       type4.floating = FALSE;
 387       type4.width *= 4;
 388       type4.length /= 4;
 389
 390       a = LLVMBuildBitCast(bld->builder, a, lp_build_vec_type(type4), "");
 391       res = LLVMBuildBitCast(bld->builder, res, lp_build_vec_type(type4), "");
 392
 393       /*
 394        * Mask and shift the channels, trying to group as many channels in the
 395        * same shift as possible
 396        */
 397       for (shift = -3; shift <= 3; ++shift) {
 398          unsigned long long mask = 0;
 399
 400          assert(type4.width <= sizeof(mask)*8);
 401
 402          for (chan = 0; chan < 4; ++chan) {
 403             /* FIXME: big endian */
 404             if (swizzles[chan] < 4 &&
 405                 chan - swizzles[chan] == shift) {
 406                mask |= ((1ULL << type.width) - 1) << (swizzles[chan] * type.width);
 407             }
 408          }
 409
 410          if (mask) {
 411             LLVMValueRef masked;
 412             LLVMValueRef shifted;
 413
 414             if (0)
 415                debug_printf("shift = %i, mask = 0x%08llx\n", shift, mask);
 416
 417             masked = LLVMBuildAnd(bld->builder, a,
 418                                   lp_build_const_int_vec(type4, mask), "");
 419             if (shift > 0) {
 420                shifted = LLVMBuildShl(bld->builder, masked,
 421                                       lp_build_const_int_vec(type4, shift*type.width), "");
 422             } else if (shift < 0) {
 423                shifted = LLVMBuildLShr(bld->builder, masked,
 424                                        lp_build_const_int_vec(type4, -shift*type.width), "");
 425             } else {
 426                shifted = masked;
 427             }
 428
 429             res = LLVMBuildOr(bld->builder, res, shifted, "");
 430          }
 431       }
 432
 433       return LLVMBuildBitCast(bld->builder, res, lp_build_vec_type(type), "");
 434    }
 435 }
 436
 437
 438 /**
 439  * Extended swizzle of a single channel of a SoA vector.
 440  *
 441  * @param bld         building context
 442  * @param unswizzled  array with the 4 unswizzled values
 443  * @param swizzle     one of the PIPE_SWIZZLE_*
 444  *
 445  * @return  the swizzled value.
 446  */
 447 LLVMValueRef
 448 lp_build_swizzle_soa_channel(struct lp_build_context *bld,
 449                              const LLVMValueRef *unswizzled,
 450                              unsigned swizzle)
 451 {
 452    switch (swizzle) {
 453    case PIPE_SWIZZLE_RED:
 454    case PIPE_SWIZZLE_GREEN:
 455    case PIPE_SWIZZLE_BLUE:
 456    case PIPE_SWIZZLE_ALPHA:
 457       return unswizzled[swizzle];
 458    case PIPE_SWIZZLE_ZERO:
 459       return bld->zero;
 460    case PIPE_SWIZZLE_ONE:
 461       return bld->one;
 462    default:
 463       assert(0);
 464       return bld->undef;
 465    }
 466 }
 467
 468
 469 /**
 470  * Extended swizzle of a SoA vector.
 471  *
 472  * @param bld         building context
 473  * @param unswizzled  array with the 4 unswizzled values
 474  * @param swizzles    array of PIPE_SWIZZLE_*
 475  * @param swizzled    output swizzled values
 476  */
 477 void
 478 lp_build_swizzle_soa(struct lp_build_context *bld,
 479                      const LLVMValueRef *unswizzled,
 480                      const unsigned char swizzles[4],
 481                      LLVMValueRef *swizzled)
 482 {
 483    unsigned chan;
 484
 485    for (chan = 0; chan < 4; ++chan) {
 486       swizzled[chan] = lp_build_swizzle_soa_channel(bld, unswizzled,
 487                                                     swizzles[chan]);
 488    }
 489 }
 490
 491
 492 /**
 493  * Do an extended swizzle of a SoA vector inplace.
 494  *
 495  * @param bld         building context
 496  * @param values      intput/output array with the 4 values
 497  * @param swizzles    array of PIPE_SWIZZLE_*
 498  */
 499 void
 500 lp_build_swizzle_soa_inplace(struct lp_build_context *bld,
 501                              LLVMValueRef *values,
 502                              const unsigned char swizzles[4])
 503 {
 504    LLVMValueRef unswizzled[4];
 505    unsigned chan;
 506
 507    for (chan = 0; chan < 4; ++chan) {
 508       unswizzled[chan] = values[chan];
 509    }
 510
 511    lp_build_swizzle_soa(bld, unswizzled, swizzles, values);
 512 }