src/gallium/drivers/swr/rasterizer/common/simdlib_128_avx512.inl

   1 /****************************************************************************
   2 * Copyright (C) 2017 Intel Corporation.   All Rights Reserved.
   3 *
   4 * Permission is hereby granted, free of charge, to any person obtaining a
   5 * copy of this software and associated documentation files (the "Software"),
   6 * to deal in the Software without restriction, including without limitation
   7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8 * and/or sell copies of the Software, and to permit persons to whom the
   9 * Software is furnished to do so, subject to the following conditions:
  10 *
  11 * The above copyright notice and this permission notice (including the next
  12 * paragraph) shall be included in all copies or substantial portions of the
  13 * Software.
  14 *
  15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21 * IN THE SOFTWARE.
  22 ****************************************************************************/
  23 #if !defined(__SIMD_LIB_AVX512_HPP__)
  24 #error Do not include this file directly, use "simdlib.hpp" instead.
  25 #endif
  26
  27 //============================================================================
  28 // SIMD128 AVX (512) implementation
  29 //
  30 // Since this implementation inherits from the AVX (2) implementation,
  31 // the only operations below ones that replace AVX (2) operations.
  32 // These use native AVX512 instructions with masking to enable a larger
  33 // register set.
  34 //============================================================================
  35
  36 private:
  37     static SIMDINLINE __m512  __conv(Float r) { return _mm512_castps128_ps512(r.v); }
  38     static SIMDINLINE __m512d __conv(Double r) { return _mm512_castpd128_pd512(r.v); }
  39     static SIMDINLINE __m512i __conv(Integer r) { return _mm512_castsi128_si512(r.v); }
  40     static SIMDINLINE Float   __conv(__m512 r) { return _mm512_castps512_ps128(r); }
  41     static SIMDINLINE Double  __conv(__m512d r) { return _mm512_castpd512_pd128(r); }
  42     static SIMDINLINE Integer __conv(__m512i r) { return _mm512_castsi512_si128(r); }
  43 public:
  44
  45 #define SIMD_WRAPPER_1_(op, intrin, mask)  \
  46     static SIMDINLINE Float SIMDCALL op(Float a)   \
  47     {\
  48         return __conv(_mm512_maskz_##intrin((mask), __conv(a)));\
  49     }
  50 #define SIMD_WRAPPER_1(op)  SIMD_WRAPPER_1_(op, op, __mmask16(0xf))
  51
  52 #define SIMD_WRAPPER_1I_(op, intrin, mask)  \
  53     template<int ImmT> \
  54     static SIMDINLINE Float SIMDCALL op(Float a)   \
  55     {\
  56         return __conv(_mm512_maskz_##intrin((mask), __conv(a), ImmT));\
  57     }
  58 #define SIMD_WRAPPER_1I(op)  SIMD_WRAPPER_1I_(op, op, __mmask16(0xf))
  59
  60 #define SIMD_WRAPPER_2_(op, intrin, mask)  \
  61     static SIMDINLINE Float SIMDCALL op(Float a, Float b)   \
  62     {\
  63         return __conv(_mm512_maskz_##intrin((mask), __conv(a), __conv(b)));\
  64     }
  65 #define SIMD_WRAPPER_2(op)  SIMD_WRAPPER_2_(op, op, __mmask16(0xf))
  66
  67 #define SIMD_WRAPPER_2I(op)  \
  68     template<int ImmT>\
  69     static SIMDINLINE Float SIMDCALL op(Float a, Float b)   \
  70     {\
  71         return __conv(_mm512_maskz_##op(0xf, __conv(a), __conv(b), ImmT));\
  72     }
  73
  74 #define SIMD_WRAPPER_3_(op, intrin, mask)  \
  75     static SIMDINLINE Float SIMDCALL op(Float a, Float b, Float c)   \
  76     {\
  77         return __conv(_mm512_maskz_##intrin((mask), __conv(a), __conv(b), __conv(c)));\
  78     }
  79 #define SIMD_WRAPPER_3(op)  SIMD_WRAPPER_3_(op, op, __mmask16(0xf))
  80
  81 #define SIMD_DWRAPPER_2I(op)  \
  82     template<int ImmT>\
  83     static SIMDINLINE Double SIMDCALL op(Double a, Double b)   \
  84     {\
  85         return __conv(_mm512_maskz_##op(0x3, __conv(a), __conv(b), ImmT));\
  86     }
  87
  88 #define SIMD_IWRAPPER_1_(op, intrin, mask)  \
  89     static SIMDINLINE Integer SIMDCALL op(Integer a)   \
  90     {\
  91         return __conv(_mm512_maskz_##intrin((mask), __conv(a)));\
  92     }
  93 #define SIMD_IWRAPPER_1_32(op)  SIMD_IWRAPPER_1_(op, op, __mmask16(0xf))
  94
  95 #define SIMD_IWRAPPER_1I_(op, intrin, mask)  \
  96     template<int ImmT> \
  97     static SIMDINLINE Integer SIMDCALL op(Integer a)   \
  98     {\
  99         return __conv(_mm512_maskz_##intrin((mask), __conv(a), ImmT));\
 100     }
 101 #define SIMD_IWRAPPER_1I_32(op)  SIMD_IWRAPPER_1I_(op, op, __mmask16(0xf))
 102
 103 #define SIMD_IWRAPPER_2_(op, intrin, mask)  \
 104     static SIMDINLINE Integer SIMDCALL op(Integer a, Integer b)   \
 105     {\
 106         return __conv(_mm512_maskz_##intrin((mask), __conv(a), __conv(b)));\
 107     }
 108 #define SIMD_IWRAPPER_2_32(op)  SIMD_IWRAPPER_2_(op, op, __mmask16(0xf))
 109
 110 #define SIMD_IWRAPPER_2I(op)  \
 111     template<int ImmT>\
 112     static SIMDINLINE Integer SIMDCALL op(Integer a, Integer b)   \
 113     {\
 114         return __conv(_mm512_maskz_##op(0xf, __conv(a), __conv(b), ImmT));\
 115     }
 116
 117 //-----------------------------------------------------------------------
 118 // Single precision floating point arithmetic operations
 119 //-----------------------------------------------------------------------
 120 SIMD_WRAPPER_2(add_ps);     // return a + b
 121 SIMD_WRAPPER_2(div_ps);     // return a / b
 122 SIMD_WRAPPER_3(fmadd_ps);   // return (a * b) + c
 123 SIMD_WRAPPER_3(fmsub_ps);   // return (a * b) - c
 124 SIMD_WRAPPER_2(max_ps);     // return (a > b) ? a : b
 125 SIMD_WRAPPER_2(min_ps);     // return (a < b) ? a : b
 126 SIMD_WRAPPER_2(mul_ps);     // return a * b
 127 SIMD_WRAPPER_1_(rcp_ps, rcp14_ps, __mmask16(0xf));     // return 1.0f / a
 128 SIMD_WRAPPER_1_(rsqrt_ps, rsqrt14_ps, __mmask16(0xf));   // return 1.0f / sqrt(a)
 129 SIMD_WRAPPER_2(sub_ps);     // return a - b
 130
 131 //-----------------------------------------------------------------------
 132 // Integer (various width) arithmetic operations
 133 //-----------------------------------------------------------------------
 134 SIMD_IWRAPPER_1_32(abs_epi32);  // return absolute_value(a) (int32)
 135 SIMD_IWRAPPER_2_32(add_epi32);  // return a + b (int32)
 136 SIMD_IWRAPPER_2_32(max_epi32);  // return (a > b) ? a : b (int32)
 137 SIMD_IWRAPPER_2_32(max_epu32);  // return (a > b) ? a : b (uint32)
 138 SIMD_IWRAPPER_2_32(min_epi32);  // return (a < b) ? a : b (int32)
 139 SIMD_IWRAPPER_2_32(min_epu32);  // return (a < b) ? a : b (uint32)
 140 SIMD_IWRAPPER_2_32(mul_epi32);  // return a * b (int32)
 141
 142 // SIMD_IWRAPPER_2_8(add_epi8);    // return a + b (int8)
 143 // SIMD_IWRAPPER_2_8(adds_epu8);   // return ((a + b) > 0xff) ? 0xff : (a + b) (uint8)
 144
 145 // return (a * b) & 0xFFFFFFFF
 146 //
 147 // Multiply the packed 32-bit integers in a and b, producing intermediate 64-bit integers,
 148 // and store the low 32 bits of the intermediate integers in dst.
 149 SIMD_IWRAPPER_2_32(mullo_epi32);
 150 SIMD_IWRAPPER_2_32(sub_epi32);  // return a - b (int32)
 151
 152 // SIMD_IWRAPPER_2_64(sub_epi64);  // return a - b (int64)
 153 // SIMD_IWRAPPER_2_8(subs_epu8);   // return (b > a) ? 0 : (a - b) (uint8)
 154
 155 //-----------------------------------------------------------------------
 156 // Logical operations
 157 //-----------------------------------------------------------------------
 158 SIMD_IWRAPPER_2_(and_si,    and_epi32, __mmask16(0xf));    // return a & b       (int)
 159 SIMD_IWRAPPER_2_(andnot_si, andnot_epi32, __mmask16(0xf)); // return (~a) & b    (int)
 160 SIMD_IWRAPPER_2_(or_si,     or_epi32, __mmask16(0xf));     // return a | b       (int)
 161 SIMD_IWRAPPER_2_(xor_si,    xor_epi32, __mmask16(0xf));    // return a ^ b       (int)
 162
 163
 164 //-----------------------------------------------------------------------
 165 // Shift operations
 166 //-----------------------------------------------------------------------
 167 SIMD_IWRAPPER_1I_32(slli_epi32);               // return a << ImmT
 168 SIMD_IWRAPPER_2_32(sllv_epi32);                // return a << b      (uint32)
 169 SIMD_IWRAPPER_1I_32(srai_epi32);               // return a >> ImmT   (int32)
 170 SIMD_IWRAPPER_1I_32(srli_epi32);               // return a >> ImmT   (uint32)
 171 SIMD_IWRAPPER_2_32(srlv_epi32);                // return a >> b      (uint32)
 172
 173 // use AVX2 version
 174 //SIMD_IWRAPPER_1I_(srli_si, srli_si256);     // return a >> (ImmT*8) (uint)
 175
 176 //-----------------------------------------------------------------------
 177 // Conversion operations (Use AVX2 versions)
 178 //-----------------------------------------------------------------------
 179 // SIMD_IWRAPPER_1L(cvtepu8_epi16, 0xffff);    // return (int16)a    (uint8 --> int16)
 180 // SIMD_IWRAPPER_1L(cvtepu8_epi32, 0xff);      // return (int32)a    (uint8 --> int32)
 181 // SIMD_IWRAPPER_1L(cvtepu16_epi32, 0xff);     // return (int32)a    (uint16 --> int32)
 182 // SIMD_IWRAPPER_1L(cvtepu16_epi64, 0xf);      // return (int64)a    (uint16 --> int64)
 183 // SIMD_IWRAPPER_1L(cvtepu32_epi64, 0xf);      // return (int64)a    (uint32 --> int64)
 184
 185 //-----------------------------------------------------------------------
 186 // Comparison operations (Use AVX2 versions
 187 //-----------------------------------------------------------------------
 188 //SIMD_IWRAPPER_2_CMP(cmpeq_epi8);    // return a == b (int8)
 189 //SIMD_IWRAPPER_2_CMP(cmpeq_epi16);   // return a == b (int16)
 190 //SIMD_IWRAPPER_2_CMP(cmpeq_epi32);   // return a == b (int32)
 191 //SIMD_IWRAPPER_2_CMP(cmpeq_epi64);   // return a == b (int64)
 192 //SIMD_IWRAPPER_2_CMP(cmpgt_epi8,);   // return a > b (int8)
 193 //SIMD_IWRAPPER_2_CMP(cmpgt_epi16);   // return a > b (int16)
 194 //SIMD_IWRAPPER_2_CMP(cmpgt_epi32);   // return a > b (int32)
 195 //SIMD_IWRAPPER_2_CMP(cmpgt_epi64);   // return a > b (int64)
 196 //
 197 //static SIMDINLINE Integer SIMDCALL cmplt_epi32(Integer a, Integer b)   // return a < b (int32)
 198 //{
 199 //    return cmpgt_epi32(b, a);
 200 //}
 201
 202 //-----------------------------------------------------------------------
 203 // Blend / shuffle / permute operations
 204 //-----------------------------------------------------------------------
 205 // SIMD_IWRAPPER_2_8(packs_epi16);     // int16 --> int8    See documentation for _mm256_packs_epi16 and _mm512_packs_epi16
 206 // SIMD_IWRAPPER_2_16(packs_epi32);    // int32 --> int16   See documentation for _mm256_packs_epi32 and _mm512_packs_epi32
 207 // SIMD_IWRAPPER_2_8(packus_epi16);    // uint16 --> uint8  See documentation for _mm256_packus_epi16 and _mm512_packus_epi16
 208 // SIMD_IWRAPPER_2_16(packus_epi32);   // uint32 --> uint16 See documentation for _mm256_packus_epi32 and _mm512_packus_epi32
 209 // SIMD_IWRAPPER_2_(permute_epi32, permutevar8x32_epi32);
 210
 211 //static SIMDINLINE Float SIMDCALL permute_ps(Float a, Integer swiz)    // return a[swiz[i]] for each 32-bit lane i (float)
 212 //{
 213 //    return _mm256_permutevar8x32_ps(a, swiz);
 214 //}
 215
 216 SIMD_IWRAPPER_1I_32(shuffle_epi32);
 217 //template<int ImmT>
 218 //static SIMDINLINE Integer SIMDCALL shuffle_epi64(Integer a, Integer b)
 219 //{
 220 //    return castpd_si(shuffle_pd<ImmT>(castsi_pd(a), castsi_pd(b)));
 221 //}
 222 //SIMD_IWRAPPER_2(shuffle_epi8);
 223 SIMD_IWRAPPER_2_32(unpackhi_epi32);
 224 SIMD_IWRAPPER_2_32(unpacklo_epi32);
 225
 226 // SIMD_IWRAPPER_2_16(unpackhi_epi16);
 227 // SIMD_IWRAPPER_2_64(unpackhi_epi64);
 228 // SIMD_IWRAPPER_2_8(unpackhi_epi8);
 229 // SIMD_IWRAPPER_2_16(unpacklo_epi16);
 230 // SIMD_IWRAPPER_2_64(unpacklo_epi64);
 231 // SIMD_IWRAPPER_2_8(unpacklo_epi8);
 232
 233 //-----------------------------------------------------------------------
 234 // Load / store operations
 235 //-----------------------------------------------------------------------
 236 static SIMDINLINE Float SIMDCALL load_ps(float const *p)   // return *p    (loads SIMD width elements from memory)
 237 {
 238     return __conv(_mm512_maskz_loadu_ps(__mmask16(0xf), p));
 239 }
 240
 241 static SIMDINLINE Integer SIMDCALL load_si(Integer const *p)  // return *p
 242 {
 243     return __conv(_mm512_maskz_loadu_epi32(__mmask16(0xf), p));
 244 }
 245
 246 static SIMDINLINE Float SIMDCALL loadu_ps(float const *p)  // return *p    (same as load_ps but allows for unaligned mem)
 247 {
 248     return __conv(_mm512_maskz_loadu_ps(__mmask16(0xf), p));
 249 }
 250
 251 static SIMDINLINE Integer SIMDCALL loadu_si(Integer const *p) // return *p    (same as load_si but allows for unaligned mem)
 252 {
 253     return __conv(_mm512_maskz_loadu_epi32(__mmask16(0xf), p));
 254 }
 255
 256 template<ScaleFactor ScaleT>
 257 static SIMDINLINE Float SIMDCALL i32gather_ps(float const* p, Integer idx) // return *(float*)(((int8*)p) + (idx * ScaleT))
 258 {
 259     return __conv(_mm512_mask_i32gather_ps(
 260                     _mm512_setzero_ps(),
 261                     __mmask16(0xf),
 262                     __conv(idx),
 263                     p,
 264                     static_cast<int>(ScaleT)));
 265 }
 266
 267 // for each element: (mask & (1 << 31)) ? (i32gather_ps<ScaleT>(p, idx), mask = 0) : old
 268 template<ScaleFactor ScaleT>
 269 static SIMDINLINE Float SIMDCALL mask_i32gather_ps(Float old, float const* p, Integer idx, Float mask)
 270 {
 271     __mmask16 m = 0xf;
 272     m = _mm512_mask_test_epi32_mask(m, _mm512_castps_si512(__conv(mask)),
 273                                 _mm512_set1_epi32(0x8000000));
 274     return __conv(_mm512_mask_i32gather_ps(
 275                     __conv(old),
 276                     m,
 277                     __conv(idx),
 278                     p,
 279                     static_cast<int>(ScaleT)));
 280 }
 281
 282 // static SIMDINLINE uint32_t SIMDCALL movemask_epi8(Integer a)
 283 // {
 284 //     __mmask64 m = 0xffffull;
 285 //     return static_cast<uint32_t>(
 286 //         _mm512_mask_test_epi8_mask(m, __conv(a), _mm512_set1_epi8(0x80)));
 287 // }
 288
 289 static SIMDINLINE void SIMDCALL maskstore_ps(float *p, Integer mask, Float src)
 290 {
 291     __mmask16 m = 0xf;
 292     m = _mm512_mask_test_epi32_mask(m, __conv(mask), _mm512_set1_epi32(0x80000000));
 293     _mm512_mask_storeu_ps(p, m, __conv(src));
 294 }
 295
 296 static SIMDINLINE void SIMDCALL store_ps(float *p, Float a)    // *p = a   (stores all elements contiguously in memory)
 297 {
 298     _mm512_mask_storeu_ps(p, __mmask16(0xf), __conv(a));
 299 }
 300
 301 static SIMDINLINE void SIMDCALL store_si(Integer *p, Integer a)   // *p = a
 302 {
 303     _mm512_mask_storeu_epi32(p, __mmask16(0xf), __conv(a));
 304 }
 305
 306 static SIMDINLINE Float SIMDCALL vmask_ps(int32_t mask)
 307 {
 308     return castsi_ps(__conv(_mm512_maskz_set1_epi32(__mmask16(mask & 0xf), -1)));
 309 }
 310
 311 //=======================================================================
 312 // Legacy interface (available only in SIMD256 width)
 313 //=======================================================================
 314
 315 #undef SIMD_WRAPPER_1_
 316 #undef SIMD_WRAPPER_1
 317 #undef SIMD_WRAPPER_1I_
 318 #undef SIMD_WRAPPER_1I
 319 #undef SIMD_WRAPPER_2_
 320 #undef SIMD_WRAPPER_2
 321 #undef SIMD_WRAPPER_2I
 322 #undef SIMD_WRAPPER_3_
 323 #undef SIMD_WRAPPER_3
 324 #undef SIMD_DWRAPPER_1_
 325 #undef SIMD_DWRAPPER_1
 326 #undef SIMD_DWRAPPER_1I_
 327 #undef SIMD_DWRAPPER_1I
 328 #undef SIMD_DWRAPPER_2_
 329 #undef SIMD_DWRAPPER_2
 330 #undef SIMD_DWRAPPER_2I
 331 #undef SIMD_IWRAPPER_1_
 332 #undef SIMD_IWRAPPER_1_8
 333 #undef SIMD_IWRAPPER_1_16
 334 #undef SIMD_IWRAPPER_1_32
 335 #undef SIMD_IWRAPPER_1_64
 336 #undef SIMD_IWRAPPER_1I_
 337 #undef SIMD_IWRAPPER_1I_8
 338 #undef SIMD_IWRAPPER_1I_16
 339 #undef SIMD_IWRAPPER_1I_32
 340 #undef SIMD_IWRAPPER_1I_64
 341 #undef SIMD_IWRAPPER_2_
 342 #undef SIMD_IWRAPPER_2_8
 343 #undef SIMD_IWRAPPER_2_16
 344 #undef SIMD_IWRAPPER_2_32
 345 #undef SIMD_IWRAPPER_2_64
 346 #undef SIMD_IWRAPPER_2I
 347 //#undef SIMD_IWRAPPER_2I_8
 348 //#undef SIMD_IWRAPPER_2I_16
 349 //#undef SIMD_IWRAPPER_2I_32
 350 //#undef SIMD_IWRAPPER_2I_64