src/gallium/drivers/swr/rasterizer/common/simdlib_512_avx512_core.inl

   1 /****************************************************************************
   2  * Copyright (C) 2017 Intel Corporation.   All Rights Reserved.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  ****************************************************************************/
  23 #if !defined(__SIMD_LIB_AVX512_HPP__)
  24 #error Do not include this file directly, use "simdlib.hpp" instead.
  25 #endif
  26
  27 //============================================================================
  28 // SIMD16 AVX512 (F) implementation for Core processors
  29 //
  30 //============================================================================
  31
  32 #define SIMD_WRAPPER_1_(op, intrin) \
  33     static SIMDINLINE Float SIMDCALL op(Float a) { return intrin(a); }
  34
  35 #define SIMD_WRAPPER_1(op) SIMD_WRAPPER_1_(op, _mm512_##op)
  36
  37 #define SIMD_WRAPPER_2_(op, intrin) \
  38     static SIMDINLINE Float SIMDCALL op(Float a, Float b) { return _mm512_##intrin(a, b); }
  39 #define SIMD_WRAPPER_2(op) SIMD_WRAPPER_2_(op, op)
  40
  41 #define SIMD_WRAPPERI_2_(op, intrin)                                          \
  42     static SIMDINLINE Float SIMDCALL op(Float a, Float b)                     \
  43     {                                                                         \
  44         return _mm512_castsi512_ps(                                           \
  45             _mm512_##intrin(_mm512_castps_si512(a), _mm512_castps_si512(b))); \
  46     }
  47
  48 #define SIMD_DWRAPPER_2(op) \
  49     static SIMDINLINE Double SIMDCALL op(Double a, Double b) { return _mm512_##op(a, b); }
  50
  51 #define SIMD_WRAPPER_2I_(op, intrin)                      \
  52     template <int ImmT>                                   \
  53     static SIMDINLINE Float SIMDCALL op(Float a, Float b) \
  54     {                                                     \
  55         return _mm512_##intrin(a, b, ImmT);               \
  56     }
  57 #define SIMD_WRAPPER_2I(op) SIMD_WRAPPER_2I_(op, op)
  58
  59 #define SIMD_DWRAPPER_2I_(op, intrin)                        \
  60     template <int ImmT>                                      \
  61     static SIMDINLINE Double SIMDCALL op(Double a, Double b) \
  62     {                                                        \
  63         return _mm512_##intrin(a, b, ImmT);                  \
  64     }
  65 #define SIMD_DWRAPPER_2I(op) SIMD_DWRAPPER_2I_(op, op)
  66
  67 #define SIMD_WRAPPER_3(op) \
  68     static SIMDINLINE Float SIMDCALL op(Float a, Float b, Float c) { return _mm512_##op(a, b, c); }
  69
  70 #define SIMD_IWRAPPER_1(op) \
  71     static SIMDINLINE Integer SIMDCALL op(Integer a) { return _mm512_##op(a); }
  72 #define SIMD_IWRAPPER_1_8(op) \
  73     static SIMDINLINE Integer SIMDCALL op(SIMD256Impl::Integer a) { return _mm512_##op(a); }
  74
  75 #define SIMD_IWRAPPER_1_4(op) \
  76     static SIMDINLINE Integer SIMDCALL op(SIMD128Impl::Integer a) { return _mm512_##op(a); }
  77
  78 #define SIMD_IWRAPPER_1I_(op, intrin)                \
  79     template <int ImmT>                              \
  80     static SIMDINLINE Integer SIMDCALL op(Integer a) \
  81     {                                                \
  82         return intrin(a, ImmT);                      \
  83     }
  84 #define SIMD_IWRAPPER_1I(op) SIMD_IWRAPPER_1I_(op, _mm512_##op)
  85
  86 #define SIMD_IWRAPPER_2_(op, intrin) \
  87     static SIMDINLINE Integer SIMDCALL op(Integer a, Integer b) { return _mm512_##intrin(a, b); }
  88 #define SIMD_IWRAPPER_2(op) SIMD_IWRAPPER_2_(op, op)
  89
  90 #define SIMD_IWRAPPER_2_CMP(op, cmp) \
  91     static SIMDINLINE Integer SIMDCALL op(Integer a, Integer b) { return cmp(a, b); }
  92
  93 #define SIMD_IFWRAPPER_2(op, intrin)                                   \
  94     static SIMDINLINE Integer SIMDCALL op(Integer a, Integer b)        \
  95     {                                                                  \
  96         return castps_si(_mm512_##intrin(castsi_ps(a), castsi_ps(b))); \
  97     }
  98
  99 #define SIMD_IWRAPPER_2I_(op, intrin)                           \
 100     template <int ImmT>                                         \
 101     static SIMDINLINE Integer SIMDCALL op(Integer a, Integer b) \
 102     {                                                           \
 103         return _mm512_##intrin(a, b, ImmT);                     \
 104     }
 105 #define SIMD_IWRAPPER_2I(op) SIMD_IWRAPPER_2I_(op, op)
 106
 107 private:
 108 static SIMDINLINE Integer vmask(__mmask32 m)
 109 {
 110     return _mm512_maskz_set1_epi16(m, -1);
 111 }
 112 static SIMDINLINE Integer vmask(__mmask64 m)
 113 {
 114     return _mm512_maskz_set1_epi8(m, -1);
 115 }
 116
 117 public:
 118 SIMD_IWRAPPER_2(add_epi8);  // return a + b (int8)
 119 SIMD_IWRAPPER_2(adds_epu8); // return ((a + b) > 0xff) ? 0xff : (a + b) (uint8)
 120 SIMD_IWRAPPER_2(subs_epu8); // return (b > a) ? 0 : (a - b) (uint8)
 121
 122 SIMD_WRAPPER_2(and_ps);    // return a & b       (float treated as int)
 123 SIMD_WRAPPER_2(andnot_ps); // return (~a) & b    (float treated as int)
 124 SIMD_WRAPPER_2(or_ps);     // return a | b       (float treated as int)
 125 SIMD_WRAPPER_2(xor_ps);    // return a ^ b       (float treated as int)
 126
 127 SIMD_IWRAPPER_1_8(cvtepu8_epi16); // return (int16)a    (uint8 --> int16)
 128
 129 template <CompareTypeInt CmpTypeT>
 130 static SIMDINLINE Integer SIMDCALL cmp_epi8(Integer a, Integer b)
 131 {
 132     // Legacy vector mask generator
 133     __mmask64 result = _mm512_cmp_epi8_mask(a, b, static_cast<const int>(CmpTypeT));
 134     return vmask(result);
 135 }
 136 template <CompareTypeInt CmpTypeT>
 137 static SIMDINLINE Integer SIMDCALL cmp_epi16(Integer a, Integer b)
 138 {
 139     // Legacy vector mask generator
 140     __mmask32 result = _mm512_cmp_epi16_mask(a, b, static_cast<const int>(CmpTypeT));
 141     return vmask(result);
 142 }
 143
 144 SIMD_IWRAPPER_2_CMP(cmpeq_epi8, cmp_epi8<CompareTypeInt::EQ>);   // return a == b (int8)
 145 SIMD_IWRAPPER_2_CMP(cmpeq_epi16, cmp_epi16<CompareTypeInt::EQ>); // return a == b (int16)
 146 SIMD_IWRAPPER_2_CMP(cmpgt_epi8, cmp_epi8<CompareTypeInt::GT>);   // return a > b (int8)
 147 SIMD_IWRAPPER_2_CMP(cmpgt_epi16, cmp_epi16<CompareTypeInt::GT>); // return a > b (int16)
 148
 149 SIMD_IWRAPPER_2(packs_epi16);  // See documentation for _mm512_packs_epi16
 150 SIMD_IWRAPPER_2(packs_epi32);  // See documentation for _mm512_packs_epi32
 151 SIMD_IWRAPPER_2(packus_epi16); // See documentation for _mm512_packus_epi16
 152 SIMD_IWRAPPER_2(packus_epi32); // See documentation for _mm512_packus_epi32
 153
 154 SIMD_IWRAPPER_2(unpackhi_epi8);  // See documentation for _mm512_unpackhi_epi8
 155 SIMD_IWRAPPER_2(unpacklo_epi16); // See documentation for _mm512_unpacklo_epi16
 156 SIMD_IWRAPPER_2(unpacklo_epi8);  // See documentation for _mm512_unpacklo_epi8
 157
 158 SIMD_IWRAPPER_2(shuffle_epi8);
 159
 160 static SIMDINLINE uint64_t SIMDCALL movemask_epi8(Integer a)
 161 {
 162     __mmask64 m = _mm512_cmplt_epi8_mask(a, setzero_si());
 163     return static_cast<uint64_t>(m);
 164 }
 165
 166 #undef SIMD_WRAPPER_1_
 167 #undef SIMD_WRAPPER_1
 168 #undef SIMD_WRAPPER_2
 169 #undef SIMD_WRAPPER_2_
 170 #undef SIMD_WRAPPERI_2_
 171 #undef SIMD_DWRAPPER_2
 172 #undef SIMD_DWRAPPER_2I
 173 #undef SIMD_WRAPPER_2I_
 174 #undef SIMD_WRAPPER_3_
 175 #undef SIMD_WRAPPER_2I
 176 #undef SIMD_WRAPPER_3
 177 #undef SIMD_IWRAPPER_1
 178 #undef SIMD_IWRAPPER_2
 179 #undef SIMD_IFWRAPPER_2
 180 #undef SIMD_IWRAPPER_2I
 181 #undef SIMD_IWRAPPER_1
 182 #undef SIMD_IWRAPPER_1I
 183 #undef SIMD_IWRAPPER_1I_
 184 #undef SIMD_IWRAPPER_2
 185 #undef SIMD_IWRAPPER_2_
 186 #undef SIMD_IWRAPPER_2I