src/gallium/drivers/swr/rasterizer/common/simdlib_256_avx512.inl

   1 /****************************************************************************
   2 * Copyright (C) 2017 Intel Corporation.   All Rights Reserved.
   3 *
   4 * Permission is hereby granted, free of charge, to any person obtaining a
   5 * copy of this software and associated documentation files (the "Software"),
   6 * to deal in the Software without restriction, including without limitation
   7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8 * and/or sell copies of the Software, and to permit persons to whom the
   9 * Software is furnished to do so, subject to the following conditions:
  10 *
  11 * The above copyright notice and this permission notice (including the next
  12 * paragraph) shall be included in all copies or substantial portions of the
  13 * Software.
  14 *
  15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21 * IN THE SOFTWARE.
  22 ****************************************************************************/
  23 #if !defined(__SIMD_LIB_AVX512_HPP__)
  24 #error Do not include this file directly, use "simdlib.hpp" instead.
  25 #endif
  26
  27 //============================================================================
  28 // SIMD256 AVX (512) implementation
  29 //
  30 // Since this implementation inherits from the AVX (2) implementation,
  31 // the only operations below ones that replace AVX (2) operations.
  32 // These use native AVX512 instructions with masking to enable a larger
  33 // register set.
  34 //============================================================================
  35
  36 private:
  37     static SIMDINLINE __m512  __conv(Float r) { return _mm512_castps256_ps512(r.v); }
  38     static SIMDINLINE __m512d __conv(Double r) { return _mm512_castpd256_pd512(r.v); }
  39     static SIMDINLINE __m512i __conv(Integer r) { return _mm512_castsi256_si512(r.v); }
  40     static SIMDINLINE Float   __conv(__m512 r) { return _mm512_castps512_ps256(r); }
  41     static SIMDINLINE Double  __conv(__m512d r) { return _mm512_castpd512_pd256(r); }
  42     static SIMDINLINE Integer __conv(__m512i r) { return _mm512_castsi512_si256(r); }
  43 public:
  44
  45 #define SIMD_WRAPPER_1_(op, intrin, mask)  \
  46     static SIMDINLINE Float SIMDCALL op(Float a)   \
  47     {\
  48         return __conv(_mm512_maskz_##intrin((mask), __conv(a)));\
  49     }
  50 #define SIMD_WRAPPER_1(op)  SIMD_WRAPPER_1_(op, op, __mmask16(0xff))
  51
  52 #define SIMD_WRAPPER_1I_(op, intrin, mask)  \
  53     template<int ImmT> \
  54     static SIMDINLINE Float SIMDCALL op(Float a)   \
  55     {\
  56         return __conv(_mm512_maskz_##intrin((mask), __conv(a), ImmT));\
  57     }
  58 #define SIMD_WRAPPER_1I(op)  SIMD_WRAPPER_1I_(op, op, __mmask16(0xff))
  59
  60 #define SIMD_WRAPPER_2_(op, intrin, mask)  \
  61     static SIMDINLINE Float SIMDCALL op(Float a, Float b)   \
  62     {\
  63         return __conv(_mm512_maskz_##intrin((mask), __conv(a), __conv(b)));\
  64     }
  65 #define SIMD_WRAPPER_2(op)  SIMD_WRAPPER_2_(op, op, __mmask16(0xff))
  66
  67 #define SIMD_WRAPPER_2I(op)  \
  68     template<int ImmT>\
  69     static SIMDINLINE Float SIMDCALL op(Float a, Float b)   \
  70     {\
  71         return __conv(_mm512_maskz_##op(0xff, __conv(a), __conv(b), ImmT));\
  72     }
  73
  74 #define SIMD_WRAPPER_3_(op, intrin, mask)  \
  75     static SIMDINLINE Float SIMDCALL op(Float a, Float b, Float c)   \
  76     {\
  77         return __conv(_mm512_maskz_##intrin((mask), __conv(a), __conv(b), __conv(c)));\
  78     }
  79 #define SIMD_WRAPPER_3(op)  SIMD_WRAPPER_3_(op, op, __mmask16(0xff))
  80
  81 #define SIMD_DWRAPPER_2I(op)  \
  82     template<int ImmT>\
  83     static SIMDINLINE Double SIMDCALL op(Double a, Double b)   \
  84     {\
  85         return __conv(_mm512_maskz_##op(0xf, __conv(a), __conv(b), ImmT));\
  86     }
  87
  88 #define SIMD_IWRAPPER_1_(op, intrin, mask)  \
  89     static SIMDINLINE Integer SIMDCALL op(Integer a)   \
  90     {\
  91         return __conv(_mm512_maskz_##intrin((mask), __conv(a)));\
  92     }
  93 #define SIMD_IWRAPPER_1_32(op)  SIMD_IWRAPPER_1_(op, op, __mmask16(0xff))
  94
  95 #define SIMD_IWRAPPER_1I_(op, intrin, mask)  \
  96     template<int ImmT> \
  97     static SIMDINLINE Integer SIMDCALL op(Integer a)   \
  98     {\
  99         return __conv(_mm512_maskz_##intrin((mask), __conv(a), ImmT));\
 100     }
 101 #define SIMD_IWRAPPER_1I_32(op)  SIMD_IWRAPPER_1I_(op, op, __mmask16(0xff))
 102
 103 #define SIMD_IWRAPPER_2_(op, intrin, mask)  \
 104     static SIMDINLINE Integer SIMDCALL op(Integer a, Integer b)   \
 105     {\
 106         return __conv(_mm512_maskz_##intrin((mask), __conv(a), __conv(b)));\
 107     }
 108 #define SIMD_IWRAPPER_2_32(op)  SIMD_IWRAPPER_2_(op, op, __mmask16(0xff))
 109
 110 #define SIMD_IWRAPPER_2I(op)  \
 111     template<int ImmT>\
 112     static SIMDINLINE Integer SIMDCALL op(Integer a, Integer b)   \
 113     {\
 114         return __conv(_mm512_maskz_##op(0xff, __conv(a), __conv(b), ImmT));\
 115     }
 116
 117 //-----------------------------------------------------------------------
 118 // Single precision floating point arithmetic operations
 119 //-----------------------------------------------------------------------
 120 SIMD_WRAPPER_2(add_ps);     // return a + b
 121 SIMD_WRAPPER_2(div_ps);     // return a / b
 122 SIMD_WRAPPER_3(fmadd_ps);   // return (a * b) + c
 123 SIMD_WRAPPER_3(fmsub_ps);   // return (a * b) - c
 124 SIMD_WRAPPER_2(max_ps);     // return (a > b) ? a : b
 125 SIMD_WRAPPER_2(min_ps);     // return (a < b) ? a : b
 126 SIMD_WRAPPER_2(mul_ps);     // return a * b
 127 SIMD_WRAPPER_1_(rcp_ps, rcp14_ps, __mmask16(0xff));     // return 1.0f / a
 128 SIMD_WRAPPER_1_(rsqrt_ps, rsqrt14_ps, __mmask16(0xff));   // return 1.0f / sqrt(a)
 129 SIMD_WRAPPER_2(sub_ps);     // return a - b
 130
 131 //-----------------------------------------------------------------------
 132 // Integer (various width) arithmetic operations
 133 //-----------------------------------------------------------------------
 134 SIMD_IWRAPPER_1_32(abs_epi32);  // return absolute_value(a) (int32)
 135 SIMD_IWRAPPER_2_32(add_epi32);  // return a + b (int32)
 136 SIMD_IWRAPPER_2_32(max_epi32);  // return (a > b) ? a : b (int32)
 137 SIMD_IWRAPPER_2_32(max_epu32);  // return (a > b) ? a : b (uint32)
 138 SIMD_IWRAPPER_2_32(min_epi32);  // return (a < b) ? a : b (int32)
 139 SIMD_IWRAPPER_2_32(min_epu32);  // return (a < b) ? a : b (uint32)
 140 SIMD_IWRAPPER_2_32(mul_epi32);  // return a * b (int32)
 141
 142 // SIMD_IWRAPPER_2_8(add_epi8);    // return a + b (int8)
 143 // SIMD_IWRAPPER_2_8(adds_epu8);   // return ((a + b) > 0xff) ? 0xff : (a + b) (uint8)
 144
 145 // return (a * b) & 0xFFFFFFFF
 146 //
 147 // Multiply the packed 32-bit integers in a and b, producing intermediate 64-bit integers,
 148 // and store the low 32 bits of the intermediate integers in dst.
 149 SIMD_IWRAPPER_2_32(mullo_epi32);
 150 SIMD_IWRAPPER_2_32(sub_epi32);  // return a - b (int32)
 151
 152 // SIMD_IWRAPPER_2_64(sub_epi64);  // return a - b (int64)
 153 // SIMD_IWRAPPER_2_8(subs_epu8);   // return (b > a) ? 0 : (a - b) (uint8)
 154
 155 //-----------------------------------------------------------------------
 156 // Logical operations
 157 //-----------------------------------------------------------------------
 158 SIMD_IWRAPPER_2_(and_si,    and_epi32, __mmask16(0xff));    // return a & b       (int)
 159 SIMD_IWRAPPER_2_(andnot_si, andnot_epi32, __mmask16(0xff)); // return (~a) & b    (int)
 160 SIMD_IWRAPPER_2_(or_si,     or_epi32, __mmask16(0xff));     // return a | b       (int)
 161 SIMD_IWRAPPER_2_(xor_si,    xor_epi32, __mmask16(0xff));    // return a ^ b       (int)
 162
 163
 164 //-----------------------------------------------------------------------
 165 // Shift operations
 166 //-----------------------------------------------------------------------
 167 SIMD_IWRAPPER_1I_32(slli_epi32);               // return a << ImmT
 168 SIMD_IWRAPPER_2_32(sllv_epi32);                // return a << b      (uint32)
 169 SIMD_IWRAPPER_1I_32(srai_epi32);               // return a >> ImmT   (int32)
 170 SIMD_IWRAPPER_1I_32(srli_epi32);               // return a >> ImmT   (uint32)
 171 SIMD_IWRAPPER_2_32(srlv_epi32);                // return a >> b      (uint32)
 172
 173 // use AVX2 version
 174 //SIMD_IWRAPPER_1I_(srli_si, srli_si256);     // return a >> (ImmT*8) (uint)
 175
 176 //-----------------------------------------------------------------------
 177 // Conversion operations (Use AVX2 versions)
 178 //-----------------------------------------------------------------------
 179 // SIMD_IWRAPPER_1L(cvtepu8_epi16, 0xffff);    // return (int16)a    (uint8 --> int16)
 180 // SIMD_IWRAPPER_1L(cvtepu8_epi32, 0xff);      // return (int32)a    (uint8 --> int32)
 181 // SIMD_IWRAPPER_1L(cvtepu16_epi32, 0xff);     // return (int32)a    (uint16 --> int32)
 182 // SIMD_IWRAPPER_1L(cvtepu16_epi64, 0xf);      // return (int64)a    (uint16 --> int64)
 183 // SIMD_IWRAPPER_1L(cvtepu32_epi64, 0xf);      // return (int64)a    (uint32 --> int64)
 184
 185 //-----------------------------------------------------------------------
 186 // Comparison operations (Use AVX2 versions
 187 //-----------------------------------------------------------------------
 188 //SIMD_IWRAPPER_2_CMP(cmpeq_epi8);    // return a == b (int8)
 189 //SIMD_IWRAPPER_2_CMP(cmpeq_epi16);   // return a == b (int16)
 190 //SIMD_IWRAPPER_2_CMP(cmpeq_epi32);   // return a == b (int32)
 191 //SIMD_IWRAPPER_2_CMP(cmpeq_epi64);   // return a == b (int64)
 192 //SIMD_IWRAPPER_2_CMP(cmpgt_epi8,);   // return a > b (int8)
 193 //SIMD_IWRAPPER_2_CMP(cmpgt_epi16);   // return a > b (int16)
 194 //SIMD_IWRAPPER_2_CMP(cmpgt_epi32);   // return a > b (int32)
 195 //SIMD_IWRAPPER_2_CMP(cmpgt_epi64);   // return a > b (int64)
 196 //
 197 //static SIMDINLINE Integer SIMDCALL cmplt_epi32(Integer a, Integer b)   // return a < b (int32)
 198 //{
 199 //    return cmpgt_epi32(b, a);
 200 //}
 201
 202 //-----------------------------------------------------------------------
 203 // Blend / shuffle / permute operations
 204 //-----------------------------------------------------------------------
 205 // SIMD_IWRAPPER_2_8(packs_epi16);     // int16 --> int8    See documentation for _mm256_packs_epi16 and _mm512_packs_epi16
 206 // SIMD_IWRAPPER_2_16(packs_epi32);    // int32 --> int16   See documentation for _mm256_packs_epi32 and _mm512_packs_epi32
 207 // SIMD_IWRAPPER_2_8(packus_epi16);    // uint16 --> uint8  See documentation for _mm256_packus_epi16 and _mm512_packus_epi16
 208 // SIMD_IWRAPPER_2_16(packus_epi32);   // uint32 --> uint16 See documentation for _mm256_packus_epi32 and _mm512_packus_epi32
 209
 210 // SIMD_IWRAPPER_2_(permute_epi32, permutevar8x32_epi32);
 211
 212 //static SIMDINLINE Float SIMDCALL permute_ps(Float a, Integer swiz)    // return a[swiz[i]] for each 32-bit lane i (float)
 213 //{
 214 //    return _mm256_permutevar8x32_ps(a, swiz);
 215 //}
 216
 217 SIMD_IWRAPPER_1I_32(shuffle_epi32);
 218 //template<int ImmT>
 219 //static SIMDINLINE Integer SIMDCALL shuffle_epi64(Integer a, Integer b)
 220 //{
 221 //    return castpd_si(shuffle_pd<ImmT>(castsi_pd(a), castsi_pd(b)));
 222 //}
 223 //SIMD_IWRAPPER_2(shuffle_epi8);
 224 SIMD_IWRAPPER_2_32(unpackhi_epi32);
 225 SIMD_IWRAPPER_2_32(unpacklo_epi32);
 226
 227 // SIMD_IWRAPPER_2_16(unpackhi_epi16);
 228 // SIMD_IWRAPPER_2_64(unpackhi_epi64);
 229 // SIMD_IWRAPPER_2_8(unpackhi_epi8);
 230 // SIMD_IWRAPPER_2_16(unpacklo_epi16);
 231 // SIMD_IWRAPPER_2_64(unpacklo_epi64);
 232 // SIMD_IWRAPPER_2_8(unpacklo_epi8);
 233
 234 //-----------------------------------------------------------------------
 235 // Load / store operations
 236 //-----------------------------------------------------------------------
 237 static SIMDINLINE Float SIMDCALL load_ps(float const *p)   // return *p    (loads SIMD width elements from memory)
 238 {
 239     return __conv(_mm512_maskz_loadu_ps(__mmask16(0xff), p));
 240 }
 241
 242 static SIMDINLINE Integer SIMDCALL load_si(Integer const *p)  // return *p
 243 {
 244     return __conv(_mm512_maskz_loadu_epi32(__mmask16(0xff), p));
 245 }
 246
 247 static SIMDINLINE Float SIMDCALL loadu_ps(float const *p)  // return *p    (same as load_ps but allows for unaligned mem)
 248 {
 249     return __conv(_mm512_maskz_loadu_ps(__mmask16(0xff), p));
 250 }
 251
 252 static SIMDINLINE Integer SIMDCALL loadu_si(Integer const *p) // return *p    (same as load_si but allows for unaligned mem)
 253 {
 254     return __conv(_mm512_maskz_loadu_epi32(__mmask16(0xff), p));
 255 }
 256
 257 template<ScaleFactor ScaleT>
 258 static SIMDINLINE Float SIMDCALL i32gather_ps(float const* p, Integer idx) // return *(float*)(((int8*)p) + (idx * ScaleT))
 259 {
 260     return __conv(_mm512_mask_i32gather_ps(
 261                     _mm512_setzero_ps(),
 262                     __mmask16(0xff),
 263                     __conv(idx),
 264                     p,
 265                     static_cast<int>(ScaleT)));
 266 }
 267
 268 // for each element: (mask & (1 << 31)) ? (i32gather_ps<ScaleT>(p, idx), mask = 0) : old
 269 template<ScaleFactor ScaleT>
 270 static SIMDINLINE Float SIMDCALL mask_i32gather_ps(Float old, float const* p, Integer idx, Float mask)
 271 {
 272     __mmask16 m = 0xff;
 273     m = _mm512_mask_test_epi32_mask(m, _mm512_castps_si512(__conv(mask)),
 274                                 _mm512_set1_epi32(0x8000000));
 275     return __conv(_mm512_mask_i32gather_ps(
 276                     __conv(old),
 277                     m,
 278                     __conv(idx),
 279                     p,
 280                     static_cast<int>(ScaleT)));
 281 }
 282
 283 // static SIMDINLINE uint32_t SIMDCALL movemask_epi8(Integer a)
 284 // {
 285 //     __mmask64 m = 0xffffffffull;
 286 //     return static_cast<uint32_t>(
 287 //         _mm512_mask_test_epi8_mask(m, __conv(a), _mm512_set1_epi8(0x80)));
 288 // }
 289
 290 static SIMDINLINE void SIMDCALL maskstore_ps(float *p, Integer mask, Float src)
 291 {
 292     __mmask16 m = 0xff;
 293     m = _mm512_mask_test_epi32_mask(m, __conv(mask), _mm512_set1_epi32(0x80000000));
 294     _mm512_mask_storeu_ps(p, m, __conv(src));
 295 }
 296
 297 static SIMDINLINE void SIMDCALL store_ps(float *p, Float a)    // *p = a   (stores all elements contiguously in memory)
 298 {
 299     _mm512_mask_storeu_ps(p, __mmask16(0xff), __conv(a));
 300 }
 301
 302 static SIMDINLINE void SIMDCALL store_si(Integer *p, Integer a)   // *p = a
 303 {
 304     _mm512_mask_storeu_epi32(p, __mmask16(0xff), __conv(a));
 305 }
 306
 307 static SIMDINLINE Float SIMDCALL vmask_ps(int32_t mask)
 308 {
 309     return castsi_ps(__conv(_mm512_maskz_set1_epi32(__mmask16(mask & 0xff), -1)));
 310 }
 311
 312 //=======================================================================
 313 // Legacy interface (available only in SIMD256 width)
 314 //=======================================================================
 315
 316 #undef SIMD_WRAPPER_1_
 317 #undef SIMD_WRAPPER_1
 318 #undef SIMD_WRAPPER_1I_
 319 #undef SIMD_WRAPPER_1I
 320 #undef SIMD_WRAPPER_2_
 321 #undef SIMD_WRAPPER_2
 322 #undef SIMD_WRAPPER_2I
 323 #undef SIMD_WRAPPER_3_
 324 #undef SIMD_WRAPPER_3
 325 #undef SIMD_IWRAPPER_1_
 326 #undef SIMD_IWRAPPER_1_32
 327 #undef SIMD_IWRAPPER_1I_
 328 #undef SIMD_IWRAPPER_1I_32
 329 #undef SIMD_IWRAPPER_2_
 330 #undef SIMD_IWRAPPER_2_32
 331 #undef SIMD_IWRAPPER_2I