src/gallium/drivers/swr/rasterizer/common/simdlib_256_avx.inl

   1 /****************************************************************************
   2 * Copyright (C) 2017 Intel Corporation.   All Rights Reserved.
   3 *
   4 * Permission is hereby granted, free of charge, to any person obtaining a
   5 * copy of this software and associated documentation files (the "Software"),
   6 * to deal in the Software without restriction, including without limitation
   7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8 * and/or sell copies of the Software, and to permit persons to whom the
   9 * Software is furnished to do so, subject to the following conditions:
  10 *
  11 * The above copyright notice and this permission notice (including the next
  12 * paragraph) shall be included in all copies or substantial portions of the
  13 * Software.
  14 *
  15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21 * IN THE SOFTWARE.
  22 ****************************************************************************/
  23 #if !defined(__SIMD_LIB_AVX_HPP__)
  24 #error Do not include this file directly, use "simdlib.hpp" instead.
  25 #endif
  26
  27 using SIMD128T = SIMD128Impl::AVXImpl;
  28
  29 //============================================================================
  30 // SIMD256 AVX (1) implementation
  31 //============================================================================
  32
  33 #define SIMD_WRAPPER_1(op)  \
  34     static SIMDINLINE Float SIMDCALL op(Float a)   \
  35     {\
  36         return _mm256_##op(a);\
  37     }
  38
  39 #define SIMD_WRAPPER_2(op)  \
  40     static SIMDINLINE Float SIMDCALL op(Float a, Float b)   \
  41     {\
  42         return _mm256_##op(a, b);\
  43     }
  44
  45 #define SIMD_DWRAPPER_2(op)  \
  46     static SIMDINLINE Double SIMDCALL op(Double a, Double b)   \
  47     {\
  48         return _mm256_##op(a, b);\
  49     }
  50
  51 #define SIMD_WRAPPER_2I(op)  \
  52     template<int ImmT>\
  53     static SIMDINLINE Float SIMDCALL op(Float a, Float b)   \
  54     {\
  55         return  _mm256_##op(a, b, ImmT);\
  56     }
  57
  58 #define SIMD_DWRAPPER_2I(op)  \
  59     template<int ImmT>\
  60     static SIMDINLINE Double SIMDCALL op(Double a, Double b)   \
  61     {\
  62         return _mm256_##op(a, b, ImmT);\
  63     }
  64
  65 #define SIMD_WRAPPER_3(op)  \
  66     static SIMDINLINE Float SIMDCALL op(Float a, Float b, Float c)   \
  67     {\
  68         return _mm256_##op(a, b, c);\
  69     }
  70
  71 #define SIMD_IWRAPPER_1(op)  \
  72     static SIMDINLINE Integer SIMDCALL op(Integer a)   \
  73     {\
  74         return _mm256_##op(a);\
  75     }
  76
  77 #define SIMD_IWRAPPER_2(op)  \
  78     static SIMDINLINE Integer SIMDCALL op(Integer a, Integer b)   \
  79     {\
  80         return _mm256_##op(a, b);\
  81     }
  82
  83 #define SIMD_IFWRAPPER_2(op, intrin)  \
  84     static SIMDINLINE Integer SIMDCALL op(Integer a, Integer b)   \
  85     {\
  86         return castps_si( intrin(castsi_ps(a), castsi_ps(b)) );\
  87     }
  88
  89 #define SIMD_IFWRAPPER_2I(op, intrin)  \
  90     template<int ImmT> \
  91     static SIMDINLINE Integer SIMDCALL op(Integer a, Integer b)   \
  92     {\
  93         return castps_si( intrin(castsi_ps(a), castsi_ps(b), ImmT) );\
  94     }
  95
  96 #define SIMD_IWRAPPER_2I_(op, intrin)  \
  97     template<int ImmT>\
  98     static SIMDINLINE Integer SIMDCALL op(Integer a, Integer b)   \
  99     {\
 100         return _mm256_##intrin(a, b, ImmT);\
 101     }
 102 #define SIMD_IWRAPPER_2I(op)  SIMD_IWRAPPER_2I_(op, op)
 103
 104 #define SIMD_IWRAPPER_3(op)  \
 105     static SIMDINLINE Integer SIMDCALL op(Integer a, Integer b, Integer c)   \
 106     {\
 107         return _mm256_##op(a, b, c);\
 108     }
 109
 110 // emulated integer simd
 111 #define SIMD_EMU_IWRAPPER_1(op) \
 112     static SIMDINLINE \
 113     Integer SIMDCALL op(Integer a)\
 114     {\
 115         return Integer\
 116         {\
 117             SIMD128T::op(a.v4[0]),\
 118             SIMD128T::op(a.v4[1]),\
 119         };\
 120     }
 121 #define SIMD_EMU_IWRAPPER_1L(op, shift) \
 122     static SIMDINLINE \
 123     Integer SIMDCALL op(Integer a)\
 124     {\
 125         return Integer \
 126         {\
 127             SIMD128T::op(a.v4[0]), \
 128             SIMD128T::op(SIMD128T::template srli_si<shift>(a.v4[0])), \
 129         };\
 130     }\
 131     static SIMDINLINE \
 132     Integer SIMDCALL op(SIMD128Impl::Integer a)\
 133     {\
 134         return Integer \
 135         {\
 136             SIMD128T::op(a), \
 137             SIMD128T::op(SIMD128T::template srli_si<shift>(a)), \
 138         };\
 139     }
 140
 141 #define SIMD_EMU_IWRAPPER_1I(op) \
 142     template <int ImmT> static SIMDINLINE \
 143     Integer SIMDCALL op(Integer a)\
 144     {\
 145         return Integer\
 146         {\
 147             SIMD128T::template op<ImmT>(a.v4[0]),\
 148             SIMD128T::template op<ImmT>(a.v4[1]),\
 149         };\
 150     }
 151
 152 #define SIMD_EMU_IWRAPPER_2(op) \
 153     static SIMDINLINE \
 154     Integer SIMDCALL op(Integer a, Integer b)\
 155     {\
 156         return Integer\
 157         {\
 158             SIMD128T::op(a.v4[0], b.v4[0]),\
 159             SIMD128T::op(a.v4[1], b.v4[1]),\
 160         };\
 161     }
 162
 163 #define SIMD_EMU_IWRAPPER_2I(op) \
 164     template <int ImmT> static SIMDINLINE \
 165     Integer SIMDCALL op(Integer a, Integer b)\
 166     {\
 167         return Integer\
 168         {\
 169             SIMD128T::template op<ImmT>(a.v4[0], b.v[0]),\
 170             SIMD128T::template op<ImmT>(a.v4[1], b.v[1]),\
 171         };\
 172     }
 173
 174 //-----------------------------------------------------------------------
 175 // Single precision floating point arithmetic operations
 176 //-----------------------------------------------------------------------
 177 SIMD_WRAPPER_2(add_ps);     // return a + b
 178 SIMD_WRAPPER_2(div_ps);     // return a / b
 179
 180 static SIMDINLINE Float SIMDCALL fmadd_ps(Float a, Float b, Float c) // return (a * b) + c
 181 {
 182     return add_ps(mul_ps(a, b), c);
 183 }
 184
 185 static SIMDINLINE Float SIMDCALL fmsub_ps(Float a, Float b, Float c) // return (a * b) - c
 186 {
 187     return sub_ps(mul_ps(a, b), c);
 188 }
 189
 190 SIMD_WRAPPER_2(max_ps);     // return (a > b) ? a : b
 191 SIMD_WRAPPER_2(min_ps);     // return (a < b) ? a : b
 192 SIMD_WRAPPER_2(mul_ps);     // return a * b
 193 SIMD_WRAPPER_1(rcp_ps);     // return 1.0f / a
 194 SIMD_WRAPPER_1(rsqrt_ps);   // return 1.0f / sqrt(a)
 195 SIMD_WRAPPER_2(sub_ps);     // return a - b
 196
 197 template <RoundMode RMT>
 198 static SIMDINLINE Float SIMDCALL round_ps(Float a)
 199 {
 200     return _mm256_round_ps(a, static_cast<int>(RMT));
 201 }
 202
 203 static SIMDINLINE Float SIMDCALL ceil_ps(Float a) { return round_ps<RoundMode::CEIL_NOEXC>(a); }
 204 static SIMDINLINE Float SIMDCALL floor_ps(Float a) { return round_ps<RoundMode::FLOOR_NOEXC>(a); }
 205
 206 //-----------------------------------------------------------------------
 207 // Integer (various width) arithmetic operations
 208 //-----------------------------------------------------------------------
 209 SIMD_EMU_IWRAPPER_1(abs_epi32); // return absolute_value(a) (int32)
 210 SIMD_EMU_IWRAPPER_2(add_epi32); // return a + b (int32)
 211 SIMD_EMU_IWRAPPER_2(add_epi8);  // return a + b (int8)
 212 SIMD_EMU_IWRAPPER_2(adds_epu8); // return ((a + b) > 0xff) ? 0xff : (a + b) (uint8)
 213 SIMD_EMU_IWRAPPER_2(max_epi32); // return (a > b) ? a : b (int32)
 214 SIMD_EMU_IWRAPPER_2(max_epu32); // return (a > b) ? a : b (uint32)
 215 SIMD_EMU_IWRAPPER_2(min_epi32); // return (a < b) ? a : b (int32)
 216 SIMD_EMU_IWRAPPER_2(min_epu32); // return (a < b) ? a : b (uint32)
 217 SIMD_EMU_IWRAPPER_2(mul_epi32); // return a * b (int32)
 218
 219 // return (a * b) & 0xFFFFFFFF
 220 //
 221 // Multiply the packed 32-bit integers in a and b, producing intermediate 64-bit integers,
 222 // and store the low 32 bits of the intermediate integers in dst.
 223 SIMD_EMU_IWRAPPER_2(mullo_epi32);
 224 SIMD_EMU_IWRAPPER_2(sub_epi32); // return a - b (int32)
 225 SIMD_EMU_IWRAPPER_2(sub_epi64); // return a - b (int64)
 226 SIMD_EMU_IWRAPPER_2(subs_epu8); // return (b > a) ? 0 : (a - b) (uint8)
 227
 228 //-----------------------------------------------------------------------
 229 // Logical operations
 230 //-----------------------------------------------------------------------
 231 SIMD_WRAPPER_2(and_ps);         // return a & b       (float treated as int)
 232 SIMD_EMU_IWRAPPER_2(and_si);    // return a & b       (int)
 233 SIMD_WRAPPER_2(andnot_ps);      // return (~a) & b    (float treated as int)
 234 SIMD_EMU_IWRAPPER_2(andnot_si); // return (~a) & b    (int)
 235 SIMD_WRAPPER_2(or_ps);          // return a | b       (float treated as int)
 236 SIMD_EMU_IWRAPPER_2(or_si);     // return a | b       (int)
 237 SIMD_WRAPPER_2(xor_ps);         // return a ^ b       (float treated as int)
 238 SIMD_EMU_IWRAPPER_2(xor_si);    // return a ^ b       (int)
 239
 240
 241 //-----------------------------------------------------------------------
 242 // Shift operations
 243 //-----------------------------------------------------------------------
 244 SIMD_EMU_IWRAPPER_1I(slli_epi32);               // return a << ImmT
 245
 246 static SIMDINLINE Integer SIMDCALL sllv_epi32(Integer vA, Integer vCount) // return a << b      (uint32)
 247 {
 248     int32_t aHi, aLow, countHi, countLow;
 249     __m128i vAHi = _mm_castps_si128(_mm256_extractf128_ps(_mm256_castsi256_ps(vA), 1));
 250     __m128i vALow = _mm_castps_si128(_mm256_extractf128_ps(_mm256_castsi256_ps(vA), 0));
 251     __m128i vCountHi = _mm_castps_si128(_mm256_extractf128_ps(_mm256_castsi256_ps(vCount), 1));
 252     __m128i vCountLow = _mm_castps_si128(_mm256_extractf128_ps(_mm256_castsi256_ps(vCount), 0));
 253
 254     aHi = _mm_extract_epi32(vAHi, 0);
 255     countHi = _mm_extract_epi32(vCountHi, 0);
 256     aHi <<= countHi;
 257     vAHi = _mm_insert_epi32(vAHi, aHi, 0);
 258
 259     aLow = _mm_extract_epi32(vALow, 0);
 260     countLow = _mm_extract_epi32(vCountLow, 0);
 261     aLow <<= countLow;
 262     vALow = _mm_insert_epi32(vALow, aLow, 0);
 263
 264     aHi = _mm_extract_epi32(vAHi, 1);
 265     countHi = _mm_extract_epi32(vCountHi, 1);
 266     aHi <<= countHi;
 267     vAHi = _mm_insert_epi32(vAHi, aHi, 1);
 268
 269     aLow = _mm_extract_epi32(vALow, 1);
 270     countLow = _mm_extract_epi32(vCountLow, 1);
 271     aLow <<= countLow;
 272     vALow = _mm_insert_epi32(vALow, aLow, 1);
 273
 274     aHi = _mm_extract_epi32(vAHi, 2);
 275     countHi = _mm_extract_epi32(vCountHi, 2);
 276     aHi <<= countHi;
 277     vAHi = _mm_insert_epi32(vAHi, aHi, 2);
 278
 279     aLow = _mm_extract_epi32(vALow, 2);
 280     countLow = _mm_extract_epi32(vCountLow, 2);
 281     aLow <<= countLow;
 282     vALow = _mm_insert_epi32(vALow, aLow, 2);
 283
 284     aHi = _mm_extract_epi32(vAHi, 3);
 285     countHi = _mm_extract_epi32(vCountHi, 3);
 286     aHi <<= countHi;
 287     vAHi = _mm_insert_epi32(vAHi, aHi, 3);
 288
 289     aLow = _mm_extract_epi32(vALow, 3);
 290     countLow = _mm_extract_epi32(vCountLow, 3);
 291     aLow <<= countLow;
 292     vALow = _mm_insert_epi32(vALow, aLow, 3);
 293
 294     __m256i ret = _mm256_set1_epi32(0);
 295     ret = _mm256_insertf128_si256(ret, vAHi, 1);
 296     ret = _mm256_insertf128_si256(ret, vALow, 0);
 297     return ret;
 298 }
 299
 300 SIMD_EMU_IWRAPPER_1I(srai_epi32);   // return a >> ImmT   (int32)
 301 SIMD_EMU_IWRAPPER_1I(srli_epi32);   // return a >> ImmT   (uint32)
 302 SIMD_EMU_IWRAPPER_1I(srli_si);      // return a >> (ImmT*8) (uint)
 303
 304 template<int ImmT>                              // same as srli_si, but with Float cast to int
 305 static SIMDINLINE Float SIMDCALL srlisi_ps(Float a)
 306 {
 307     return castsi_ps(srli_si<ImmT>(castps_si(a)));
 308 }
 309
 310 static SIMDINLINE Integer SIMDCALL srlv_epi32(Integer vA, Integer vCount) // return a >> b      (uint32)
 311 {
 312     int32_t aHi, aLow, countHi, countLow;
 313     __m128i vAHi = _mm_castps_si128(_mm256_extractf128_ps(_mm256_castsi256_ps(vA), 1));
 314     __m128i vALow = _mm_castps_si128(_mm256_extractf128_ps(_mm256_castsi256_ps(vA), 0));
 315     __m128i vCountHi = _mm_castps_si128(_mm256_extractf128_ps(_mm256_castsi256_ps(vCount), 1));
 316     __m128i vCountLow = _mm_castps_si128(_mm256_extractf128_ps(_mm256_castsi256_ps(vCount), 0));
 317
 318     aHi = _mm_extract_epi32(vAHi, 0);
 319     countHi = _mm_extract_epi32(vCountHi, 0);
 320     aHi >>= countHi;
 321     vAHi = _mm_insert_epi32(vAHi, aHi, 0);
 322
 323     aLow = _mm_extract_epi32(vALow, 0);
 324     countLow = _mm_extract_epi32(vCountLow, 0);
 325     aLow >>= countLow;
 326     vALow = _mm_insert_epi32(vALow, aLow, 0);
 327
 328     aHi = _mm_extract_epi32(vAHi, 1);
 329     countHi = _mm_extract_epi32(vCountHi, 1);
 330     aHi >>= countHi;
 331     vAHi = _mm_insert_epi32(vAHi, aHi, 1);
 332
 333     aLow = _mm_extract_epi32(vALow, 1);
 334     countLow = _mm_extract_epi32(vCountLow, 1);
 335     aLow >>= countLow;
 336     vALow = _mm_insert_epi32(vALow, aLow, 1);
 337
 338     aHi = _mm_extract_epi32(vAHi, 2);
 339     countHi = _mm_extract_epi32(vCountHi, 2);
 340     aHi >>= countHi;
 341     vAHi = _mm_insert_epi32(vAHi, aHi, 2);
 342
 343     aLow = _mm_extract_epi32(vALow, 2);
 344     countLow = _mm_extract_epi32(vCountLow, 2);
 345     aLow >>= countLow;
 346     vALow = _mm_insert_epi32(vALow, aLow, 2);
 347
 348     aHi = _mm_extract_epi32(vAHi, 3);
 349     countHi = _mm_extract_epi32(vCountHi, 3);
 350     aHi >>= countHi;
 351     vAHi = _mm_insert_epi32(vAHi, aHi, 3);
 352
 353     aLow = _mm_extract_epi32(vALow, 3);
 354     countLow = _mm_extract_epi32(vCountLow, 3);
 355     aLow >>= countLow;
 356     vALow = _mm_insert_epi32(vALow, aLow, 3);
 357
 358     __m256i ret = _mm256_set1_epi32(0);
 359     ret = _mm256_insertf128_si256(ret, vAHi, 1);
 360     ret = _mm256_insertf128_si256(ret, vALow, 0);
 361     return ret;
 362 }
 363
 364
 365
 366 //-----------------------------------------------------------------------
 367 // Conversion operations
 368 //-----------------------------------------------------------------------
 369 static SIMDINLINE Float SIMDCALL castpd_ps(Double a)   // return *(Float*)(&a)
 370 {
 371     return _mm256_castpd_ps(a);
 372 }
 373
 374 static SIMDINLINE Integer SIMDCALL castps_si(Float a)   // return *(Integer*)(&a)
 375 {
 376     return _mm256_castps_si256(a);
 377 }
 378
 379 static SIMDINLINE Double SIMDCALL castsi_pd(Integer a)   // return *(Double*)(&a)
 380 {
 381     return _mm256_castsi256_pd(a);
 382 }
 383
 384 static SIMDINLINE Double SIMDCALL castps_pd(Float a)   // return *(Double*)(&a)
 385 {
 386     return _mm256_castps_pd(a);
 387 }
 388
 389 static SIMDINLINE Integer SIMDCALL castpd_si(Double a)   // return *(Integer*)(&a)
 390 {
 391     return _mm256_castpd_si256(a);
 392 }
 393
 394 static SIMDINLINE Float SIMDCALL castsi_ps(Integer a)   // return *(Float*)(&a)
 395 {
 396     return _mm256_castsi256_ps(a);
 397 }
 398
 399 static SIMDINLINE Float SIMDCALL cvtepi32_ps(Integer a) // return (float)a    (int32 --> float)
 400 {
 401     return _mm256_cvtepi32_ps(a);
 402 }
 403
 404 SIMD_EMU_IWRAPPER_1L(cvtepu8_epi16, 8);                  // return (int16)a    (uint8 --> int16)
 405 SIMD_EMU_IWRAPPER_1L(cvtepu8_epi32, 4);                  // return (int32)a    (uint8 --> int32)
 406 SIMD_EMU_IWRAPPER_1L(cvtepu16_epi32, 8);                 // return (int32)a    (uint16 --> int32)
 407 SIMD_EMU_IWRAPPER_1L(cvtepu16_epi64, 4);                 // return (int64)a    (uint16 --> int64)
 408 SIMD_EMU_IWRAPPER_1L(cvtepu32_epi64, 8);                 // return (int64)a    (uint32 --> int64)
 409
 410 static SIMDINLINE Integer SIMDCALL cvtps_epi32(Float a)            // return (int32)a    (float --> int32)
 411 {
 412     return _mm256_cvtps_epi32(a);
 413 }
 414
 415 static SIMDINLINE Integer SIMDCALL cvttps_epi32(Float a)           // return (int32)a    (rnd_to_zero(float) --> int32)
 416 {
 417     return _mm256_cvttps_epi32(a);
 418 }
 419
 420 //-----------------------------------------------------------------------
 421 // Comparison operations
 422 //-----------------------------------------------------------------------
 423 template<CompareType CmpTypeT>
 424 static SIMDINLINE Float SIMDCALL cmp_ps(Float a, Float b) // return a (CmpTypeT) b
 425 {
 426     return _mm256_cmp_ps(a, b, static_cast<const int>(CmpTypeT));
 427 }
 428 static SIMDINLINE Float SIMDCALL cmplt_ps(Float a, Float b) { return cmp_ps<CompareType::LT_OQ>(a, b); }
 429 static SIMDINLINE Float SIMDCALL cmpgt_ps(Float a, Float b) { return cmp_ps<CompareType::GT_OQ>(a, b); }
 430 static SIMDINLINE Float SIMDCALL cmpneq_ps(Float a, Float b) { return cmp_ps<CompareType::NEQ_OQ>(a, b); }
 431 static SIMDINLINE Float SIMDCALL cmpeq_ps(Float a, Float b) { return cmp_ps<CompareType::EQ_OQ>(a, b); }
 432 static SIMDINLINE Float SIMDCALL cmpge_ps(Float a, Float b) { return cmp_ps<CompareType::GE_OQ>(a, b); }
 433 static SIMDINLINE Float SIMDCALL cmple_ps(Float a, Float b) { return cmp_ps<CompareType::LE_OQ>(a, b); }
 434
 435 SIMD_EMU_IWRAPPER_2(cmpeq_epi8);    // return a == b (int8)
 436 SIMD_EMU_IWRAPPER_2(cmpeq_epi16);   // return a == b (int16)
 437 SIMD_EMU_IWRAPPER_2(cmpeq_epi32);   // return a == b (int32)
 438 SIMD_EMU_IWRAPPER_2(cmpeq_epi64);   // return a == b (int64)
 439 SIMD_EMU_IWRAPPER_2(cmpgt_epi8);    // return a > b (int8)
 440 SIMD_EMU_IWRAPPER_2(cmpgt_epi16);   // return a > b (int16)
 441 SIMD_EMU_IWRAPPER_2(cmpgt_epi32);   // return a > b (int32)
 442 SIMD_EMU_IWRAPPER_2(cmpgt_epi64);   // return a > b (int64)
 443 SIMD_EMU_IWRAPPER_2(cmplt_epi32);   // return a < b (int32)
 444
 445 static SIMDINLINE bool SIMDCALL testz_ps(Float a, Float b)  // return all_lanes_zero(a & b) ? 1 : 0 (float)
 446 {
 447     return  0 != _mm256_testz_ps(a, b);
 448 }
 449
 450 static SIMDINLINE bool SIMDCALL testz_si(Integer a, Integer b)  // return all_lanes_zero(a & b) ? 1 : 0 (int)
 451 {
 452     return  0 != _mm256_testz_si256(a, b);
 453 }
 454
 455 //-----------------------------------------------------------------------
 456 // Blend / shuffle / permute operations
 457 //-----------------------------------------------------------------------
 458 SIMD_WRAPPER_2I(blend_ps);  // return ImmT ? b : a  (float)
 459 SIMD_IFWRAPPER_2I(blend_epi32, _mm256_blend_ps);  // return ImmT ? b : a  (int32)
 460 SIMD_WRAPPER_3(blendv_ps);  // return mask ? b : a  (float)
 461
 462 static SIMDINLINE Integer SIMDCALL blendv_epi32(Integer a, Integer b, Float mask) // return mask ? b : a (int)
 463 {
 464     return castps_si(blendv_ps(castsi_ps(a), castsi_ps(b), mask));
 465 }
 466
 467 static SIMDINLINE Integer SIMDCALL blendv_epi32(Integer a, Integer b, Integer mask) // return mask ? b : a (int)
 468 {
 469     return castps_si(blendv_ps(castsi_ps(a), castsi_ps(b), castsi_ps(mask)));
 470 }
 471
 472 static SIMDINLINE Float SIMDCALL broadcast_ss(float const *p)  // return *p (all elements in vector get same value)
 473 {
 474     return _mm256_broadcast_ss(p);
 475 }
 476
 477 SIMD_EMU_IWRAPPER_2(packs_epi16);   // See documentation for _mm256_packs_epi16 and _mm512_packs_epi16
 478 SIMD_EMU_IWRAPPER_2(packs_epi32);   // See documentation for _mm256_packs_epi32 and _mm512_packs_epi32
 479 SIMD_EMU_IWRAPPER_2(packus_epi16);  // See documentation for _mm256_packus_epi16 and _mm512_packus_epi16
 480 SIMD_EMU_IWRAPPER_2(packus_epi32);  // See documentation for _mm256_packus_epi32 and _mm512_packus_epi32
 481
 482 static SIMDINLINE Integer SIMDCALL permute_epi32(Integer a, Integer swiz) // return a[swiz[i]] for each 32-bit lane i (int32)
 483 {
 484     Integer result;
 485
 486     // Ugly slow implementation
 487     uint32_t const *pA = reinterpret_cast<uint32_t const*>(&a);
 488     uint32_t const *pSwiz = reinterpret_cast<uint32_t const*>(&swiz);
 489     uint32_t *pResult = reinterpret_cast<uint32_t *>(&result);
 490
 491     for (uint32_t i = 0; i < SIMD_WIDTH; ++i)
 492     {
 493         pResult[i] = pA[0xF & pSwiz[i]];
 494     }
 495
 496     return result;
 497 }
 498
 499 static SIMDINLINE Float SIMDCALL permute_ps(Float a, Integer swiz)    // return a[swiz[i]] for each 32-bit lane i (float)
 500 {
 501     Float result;
 502
 503     // Ugly slow implementation
 504     float const *pA = reinterpret_cast<float const*>(&a);
 505     uint32_t const *pSwiz = reinterpret_cast<uint32_t const*>(&swiz);
 506     float *pResult = reinterpret_cast<float *>(&result);
 507
 508     for (uint32_t i = 0; i < SIMD_WIDTH; ++i)
 509     {
 510         pResult[i] = pA[0xF & pSwiz[i]];
 511     }
 512
 513     return result;
 514 }
 515
 516 SIMD_WRAPPER_2I(permute2f128_ps);
 517 SIMD_DWRAPPER_2I(permute2f128_pd);
 518 SIMD_IWRAPPER_2I_(permute2f128_si, permute2f128_si256);
 519
 520
 521 SIMD_EMU_IWRAPPER_1I(shuffle_epi32);
 522
 523 template<int ImmT>
 524 static SIMDINLINE Integer SIMDCALL shuffle_epi64(Integer a, Integer b)
 525 {
 526     return castpd_si(shuffle_pd<ImmT>(castsi_pd(a), castsi_pd(b)));
 527 }
 528 SIMD_EMU_IWRAPPER_2(shuffle_epi8);
 529 SIMD_DWRAPPER_2I(shuffle_pd);
 530 SIMD_WRAPPER_2I(shuffle_ps);
 531 SIMD_EMU_IWRAPPER_2(unpackhi_epi16);
 532 SIMD_IFWRAPPER_2(unpackhi_epi32, _mm256_unpackhi_ps);
 533 SIMD_EMU_IWRAPPER_2(unpackhi_epi64);
 534 SIMD_EMU_IWRAPPER_2(unpackhi_epi8);
 535 SIMD_DWRAPPER_2(unpackhi_pd);
 536 SIMD_WRAPPER_2(unpackhi_ps);
 537 SIMD_EMU_IWRAPPER_2(unpacklo_epi16);
 538 SIMD_IFWRAPPER_2(unpacklo_epi32, _mm256_unpacklo_ps);
 539 SIMD_EMU_IWRAPPER_2(unpacklo_epi64);
 540 SIMD_EMU_IWRAPPER_2(unpacklo_epi8);
 541 SIMD_DWRAPPER_2(unpacklo_pd);
 542 SIMD_WRAPPER_2(unpacklo_ps);
 543
 544 //-----------------------------------------------------------------------
 545 // Load / store operations
 546 //-----------------------------------------------------------------------
 547 template<ScaleFactor ScaleT>
 548 static SIMDINLINE Float SIMDCALL i32gather_ps(float const* p, Integer idx) // return *(float*)(((int8*)p) + (idx * ScaleT))
 549 {
 550     uint32_t *pOffsets = (uint32_t*)&idx;
 551     Float vResult;
 552     float* pResult = (float*)&vResult;
 553     for (uint32_t i = 0; i < SIMD_WIDTH; ++i)
 554     {
 555         uint32_t offset = pOffsets[i];
 556         offset = offset * static_cast<uint32_t>(ScaleT);
 557         pResult[i] = *(float const*)(((uint8_t const*)p + offset));
 558     }
 559
 560     return vResult;
 561 }
 562
 563 static SIMDINLINE Float SIMDCALL load1_ps(float const *p)  // return *p    (broadcast 1 value to all elements)
 564 {
 565     return broadcast_ss(p);
 566 }
 567
 568 static SIMDINLINE Float SIMDCALL load_ps(float const *p)   // return *p    (loads SIMD width elements from memory)
 569 {
 570     return _mm256_load_ps(p);
 571 }
 572
 573 static SIMDINLINE Integer SIMDCALL load_si(Integer const *p)  // return *p
 574 {
 575     return _mm256_load_si256(&p->v);
 576 }
 577
 578 static SIMDINLINE Float SIMDCALL loadu_ps(float const *p)  // return *p    (same as load_ps but allows for unaligned mem)
 579 {
 580     return _mm256_loadu_ps(p);
 581 }
 582
 583 static SIMDINLINE Integer SIMDCALL loadu_si(Integer const *p) // return *p    (same as load_si but allows for unaligned mem)
 584 {
 585     return _mm256_lddqu_si256(&p->v);
 586 }
 587
 588 // for each element: (mask & (1 << 31)) ? (i32gather_ps<ScaleT>(p, idx), mask = 0) : old
 589 template<ScaleFactor ScaleT>
 590 static SIMDINLINE Float SIMDCALL mask_i32gather_ps(Float old, float const* p, Integer idx, Float mask)
 591 {
 592     uint32_t *pOffsets = (uint32_t*)&idx;
 593     Float vResult = old;
 594     float* pResult = (float*)&vResult;
 595     DWORD index;
 596     uint32_t umask = movemask_ps(mask);
 597     while (_BitScanForward(&index, umask))
 598     {
 599         umask &= ~(1 << index);
 600         uint32_t offset = pOffsets[index];
 601         offset = offset * static_cast<uint32_t>(ScaleT);
 602         pResult[index] = *(float const *)(((uint8_t const *)p + offset));
 603     }
 604
 605     return vResult;
 606 }
 607
 608 static SIMDINLINE void SIMDCALL maskstore_ps(float *p, Integer mask, Float src)
 609 {
 610     _mm256_maskstore_ps(p, mask, src);
 611 }
 612
 613 static SIMDINLINE uint32_t SIMDCALL movemask_epi8(Integer a)
 614 {
 615     return SIMD128T::movemask_epi8(a.v4[0]) |
 616            (SIMD128T::movemask_epi8(a.v4[1]) << 16);
 617 }
 618
 619 static SIMDINLINE uint32_t SIMDCALL movemask_pd(Double a)
 620 {
 621     return static_cast<uint32_t>(_mm256_movemask_pd(a));
 622 }
 623 static SIMDINLINE uint32_t SIMDCALL movemask_ps(Float a)
 624 {
 625     return static_cast<uint32_t>(_mm256_movemask_ps(a));
 626 }
 627
 628 static SIMDINLINE Integer SIMDCALL set1_epi32(int i) // return i (all elements are same value)
 629 {
 630     return _mm256_set1_epi32(i);
 631 }
 632
 633 static SIMDINLINE Integer SIMDCALL set1_epi8(char i) // return i (all elements are same value)
 634 {
 635     return _mm256_set1_epi8(i);
 636 }
 637
 638 static SIMDINLINE Float SIMDCALL set1_ps(float f)  // return f (all elements are same value)
 639 {
 640     return _mm256_set1_ps(f);
 641 }
 642
 643 static SIMDINLINE Float SIMDCALL setzero_ps()      // return 0 (float)
 644 {
 645     return _mm256_setzero_ps();
 646 }
 647
 648 static SIMDINLINE Integer SIMDCALL setzero_si()      // return 0 (integer)
 649 {
 650     return _mm256_setzero_si256();
 651 }
 652
 653 static SIMDINLINE void SIMDCALL store_ps(float *p, Float a)    // *p = a   (stores all elements contiguously in memory)
 654 {
 655     _mm256_store_ps(p, a);
 656 }
 657
 658 static SIMDINLINE void SIMDCALL store_si(Integer *p, Integer a)   // *p = a
 659 {
 660     _mm256_store_si256(&p->v, a);
 661 }
 662
 663 static SIMDINLINE void SIMDCALL stream_ps(float *p, Float a)   // *p = a   (same as store_ps, but doesn't keep memory in cache)
 664 {
 665     _mm256_stream_ps(p, a);
 666 }
 667
 668 //=======================================================================
 669 // Legacy interface (available only in SIMD256 width)
 670 //=======================================================================
 671
 672 static SIMDINLINE Float SIMDCALL broadcast_ps(SIMD128Impl::Float const *p)
 673 {
 674     return _mm256_broadcast_ps(&p->v);
 675 }
 676
 677 template<int ImmT>
 678 static SIMDINLINE SIMD128Impl::Double SIMDCALL extractf128_pd(Double a)
 679 {
 680     return _mm256_extractf128_pd(a, ImmT);
 681 }
 682
 683 template<int ImmT>
 684 static SIMDINLINE SIMD128Impl::Float  SIMDCALL extractf128_ps(Float a)
 685 {
 686     return _mm256_extractf128_ps(a, ImmT);
 687 }
 688
 689 template<int ImmT>
 690 static SIMDINLINE SIMD128Impl::Integer SIMDCALL extractf128_si(Integer a)
 691 {
 692     return _mm256_extractf128_si256(a, ImmT);
 693 }
 694
 695 template<int ImmT>
 696 static SIMDINLINE Double SIMDCALL insertf128_pd(Double a, SIMD128Impl::Double b)
 697 {
 698     return _mm256_insertf128_pd(a, b, ImmT);
 699 }
 700
 701 template<int ImmT>
 702 static SIMDINLINE Float SIMDCALL insertf128_ps(Float a, SIMD128Impl::Float b)
 703 {
 704     return _mm256_insertf128_ps(a, b, ImmT);
 705 }
 706
 707 template<int ImmT>
 708 static SIMDINLINE Integer SIMDCALL insertf128_si(Integer a, SIMD128Impl::Integer b)
 709 {
 710     return _mm256_insertf128_si256(a, b, ImmT);
 711 }
 712
 713 #ifndef _mm256_set_m128i
 714 #define _mm256_set_m128i(/* SIMD128Impl::Integer */ hi, /* SIMD128Impl::Integer */ lo) \
 715     _mm256_insertf128_si256(_mm256_castsi128_si256(lo), (hi), 0x1)
 716 #endif
 717
 718 #ifndef _mm256_loadu2_m128i
 719 #define _mm256_loadu2_m128i(/* SIMD128Impl::Integer const* */ hiaddr, \
 720                             /* SIMD128Impl::Integer const* */ loaddr) \
 721     _mm256_set_m128i(_mm_loadu_si128(hiaddr), _mm_loadu_si128(loaddr))
 722 #endif
 723
 724 static SIMDINLINE Integer SIMDCALL loadu2_si(SIMD128Impl::Integer const* phi, SIMD128Impl::Integer const* plo)
 725 {
 726     return _mm256_loadu2_m128i(&phi->v, &plo->v);
 727 }
 728
 729 static SIMDINLINE Integer SIMDCALL set_epi32(int i7, int i6, int i5, int i4, int i3, int i2, int i1, int i0)
 730 {
 731     return _mm256_set_epi32(i7, i6, i5, i4, i3, i2, i1, i0);
 732 }
 733
 734 static SIMDINLINE Float SIMDCALL set_ps(float i7, float i6, float i5, float i4, float i3, float i2, float i1, float i0)
 735 {
 736     return _mm256_set_ps(i7, i6, i5, i4, i3, i2, i1, i0);
 737 }
 738
 739 static SIMDINLINE void SIMDCALL storeu2_si(SIMD128Impl::Integer *phi, SIMD128Impl::Integer *plo, Integer src)
 740 {
 741     _mm256_storeu2_m128i(&phi->v, &plo->v, src);
 742 }
 743
 744 static SIMDINLINE Float SIMDCALL vmask_ps(int32_t mask)
 745 {
 746     Integer vec = set1_epi32(mask);
 747     const Integer bit = set_epi32(
 748         0x80, 0x40, 0x20, 0x10, 0x08, 0x04, 0x02, 0x01);
 749     vec = and_si(vec, bit);
 750     vec = cmplt_epi32(setzero_si(), vec);
 751     return castsi_ps(vec);
 752 }
 753
 754 #undef SIMD_WRAPPER_1
 755 #undef SIMD_WRAPPER_2
 756 #undef SIMD_DWRAPPER_2
 757 #undef SIMD_DWRAPPER_2I
 758 #undef SIMD_WRAPPER_2I
 759 #undef SIMD_WRAPPER_3
 760 #undef SIMD_IWRAPPER_1
 761 #undef SIMD_IWRAPPER_2
 762 #undef SIMD_IFWRAPPER_2
 763 #undef SIMD_IFWRAPPER_2I
 764 #undef SIMD_IWRAPPER_2I
 765 #undef SIMD_IWRAPPER_2I_
 766 #undef SIMD_IWRAPPER_2_
 767 #undef SIMD_IWRAPPER_3
 768 #undef SIMD_EMU_IWRAPPER_1
 769 #undef SIMD_EMU_IWRAPPER_1I
 770 #undef SIMD_EMU_IWRAPPER_2
 771 #undef SIMD_EMU_IWRAPPER_2I