#ifndef __SWR_SIMDINTRIN_H__
#define __SWR_SIMDINTRIN_H__
-#include "common/os.h"
#include "common/intrin.h"
+#include "common/simdlib.hpp"
#if KNOB_SIMD_WIDTH == 8
-#define _simd128_maskstore_ps _mm_maskstore_ps
-#define _simd_load_ps _mm256_load_ps
-#define _simd_load1_ps _mm256_broadcast_ss
-#define _simd_loadu_ps _mm256_loadu_ps
-#define _simd_setzero_ps _mm256_setzero_ps
-#define _simd_set1_ps _mm256_set1_ps
-#define _simd_blend_ps _mm256_blend_ps
-#define _simd_blendv_ps _mm256_blendv_ps
-#define _simd_store_ps _mm256_store_ps
-#define _simd_mul_ps _mm256_mul_ps
-#define _simd_add_ps _mm256_add_ps
-#define _simd_sub_ps _mm256_sub_ps
-#define _simd_rsqrt_ps _mm256_rsqrt_ps
-#define _simd_min_ps _mm256_min_ps
-#define _simd_max_ps _mm256_max_ps
-#define _simd_movemask_ps _mm256_movemask_ps
-#define _simd_cvtps_epi32 _mm256_cvtps_epi32
-#define _simd_cvttps_epi32 _mm256_cvttps_epi32
-#define _simd_cvtepi32_ps _mm256_cvtepi32_ps
-#define _simd_cmplt_ps(a, b) _mm256_cmp_ps(a, b, _CMP_LT_OQ)
-#define _simd_cmpgt_ps(a, b) _mm256_cmp_ps(a, b, _CMP_GT_OQ)
-#define _simd_cmpneq_ps(a, b) _mm256_cmp_ps(a, b, _CMP_NEQ_OQ)
-#define _simd_cmpeq_ps(a, b) _mm256_cmp_ps(a, b, _CMP_EQ_OQ)
-#define _simd_cmpge_ps(a, b) _mm256_cmp_ps(a, b, _CMP_GE_OQ)
-#define _simd_cmple_ps(a, b) _mm256_cmp_ps(a, b, _CMP_LE_OQ)
-#define _simd_cmp_ps(a, b, imm) _mm256_cmp_ps(a, b, imm)
-#define _simd_and_ps _mm256_and_ps
-#define _simd_or_ps _mm256_or_ps
-
-#define _simd_rcp_ps _mm256_rcp_ps
-#define _simd_div_ps _mm256_div_ps
-#define _simd_castsi_ps _mm256_castsi256_ps
-#define _simd_andnot_ps _mm256_andnot_ps
-#define _simd_round_ps _mm256_round_ps
-#define _simd_castpd_ps _mm256_castpd_ps
-#define _simd_broadcast_ps(a) _mm256_broadcast_ps((const __m128*)(a))
-#define _simd_stream_ps _mm256_stream_ps
-
-#define _simd_load_sd _mm256_load_sd
-#define _simd_movemask_pd _mm256_movemask_pd
-#define _simd_castsi_pd _mm256_castsi256_pd
-
-// emulated integer simd
-#define SIMD_EMU_EPI(func, intrin) \
-INLINE \
-__m256i func(__m256i a, __m256i b)\
-{\
- __m128i aHi = _mm256_extractf128_si256(a, 1);\
- __m128i bHi = _mm256_extractf128_si256(b, 1);\
- __m128i aLo = _mm256_castsi256_si128(a);\
- __m128i bLo = _mm256_castsi256_si128(b);\
-\
- __m128i subLo = intrin(aLo, bLo);\
- __m128i subHi = intrin(aHi, bHi);\
-\
- __m256i result = _mm256_castsi128_si256(subLo);\
- result = _mm256_insertf128_si256(result, subHi, 1);\
-\
- return result;\
-}
-
-#if (KNOB_ARCH == KNOB_ARCH_AVX)
-INLINE
-__m256 _simdemu_permute_ps(__m256 a, __m256i b)
-{
- __m128 aHi = _mm256_extractf128_ps(a, 1);
- __m128i bHi = _mm256_extractf128_si256(b, 1);
- __m128 aLo = _mm256_castps256_ps128(a);
- __m128i bLo = _mm256_castsi256_si128(b);
-
- __m128i indexHi = _mm_cmpgt_epi32(bLo, _mm_set1_epi32(3));
- __m128 resLow = _mm_permutevar_ps(aLo, _mm_and_si128(bLo, _mm_set1_epi32(0x3)));
- __m128 resHi = _mm_permutevar_ps(aHi, _mm_and_si128(bLo, _mm_set1_epi32(0x3)));
- __m128 blendLowRes = _mm_blendv_ps(resLow, resHi, _mm_castsi128_ps(indexHi));
-
- indexHi = _mm_cmpgt_epi32(bHi, _mm_set1_epi32(3));
- resLow = _mm_permutevar_ps(aLo, _mm_and_si128(bHi, _mm_set1_epi32(0x3)));
- resHi = _mm_permutevar_ps(aHi, _mm_and_si128(bHi, _mm_set1_epi32(0x3)));
- __m128 blendHiRes = _mm_blendv_ps(resLow, resHi, _mm_castsi128_ps(indexHi));
-
- __m256 result = _mm256_castps128_ps256(blendLowRes);
- result = _mm256_insertf128_ps(result, blendHiRes, 1);
-
- return result;
-}
-
-INLINE
-__m256i _simdemu_permute_epi32(__m256i a, __m256i b)
-{
- return _mm256_castps_si256(_simdemu_permute_ps(_mm256_castsi256_ps(a), b));
-}
-
-INLINE
-__m256i _simdemu_srlv_epi32(__m256i vA, __m256i vCount)
-{
- int32_t aHi, aLow, countHi, countLow;
- __m128i vAHi = _mm_castps_si128(_mm256_extractf128_ps(_mm256_castsi256_ps(vA), 1));
- __m128i vALow = _mm_castps_si128(_mm256_extractf128_ps(_mm256_castsi256_ps(vA), 0));
- __m128i vCountHi = _mm_castps_si128(_mm256_extractf128_ps(_mm256_castsi256_ps(vCount), 1));
- __m128i vCountLow = _mm_castps_si128(_mm256_extractf128_ps(_mm256_castsi256_ps(vCount), 0));
-
- aHi = _mm_extract_epi32(vAHi, 0);
- countHi = _mm_extract_epi32(vCountHi, 0);
- aHi >>= countHi;
- vAHi = _mm_insert_epi32(vAHi, aHi, 0);
-
- aLow = _mm_extract_epi32(vALow, 0);
- countLow = _mm_extract_epi32(vCountLow, 0);
- aLow >>= countLow;
- vALow = _mm_insert_epi32(vALow, aLow, 0);
-
- aHi = _mm_extract_epi32(vAHi, 1);
- countHi = _mm_extract_epi32(vCountHi, 1);
- aHi >>= countHi;
- vAHi = _mm_insert_epi32(vAHi, aHi, 1);
-
- aLow = _mm_extract_epi32(vALow, 1);
- countLow = _mm_extract_epi32(vCountLow, 1);
- aLow >>= countLow;
- vALow = _mm_insert_epi32(vALow, aLow, 1);
-
- aHi = _mm_extract_epi32(vAHi, 2);
- countHi = _mm_extract_epi32(vCountHi, 2);
- aHi >>= countHi;
- vAHi = _mm_insert_epi32(vAHi, aHi, 2);
-
- aLow = _mm_extract_epi32(vALow, 2);
- countLow = _mm_extract_epi32(vCountLow, 2);
- aLow >>= countLow;
- vALow = _mm_insert_epi32(vALow, aLow, 2);
-
- aHi = _mm_extract_epi32(vAHi, 3);
- countHi = _mm_extract_epi32(vCountHi, 3);
- aHi >>= countHi;
- vAHi = _mm_insert_epi32(vAHi, aHi, 3);
-
- aLow = _mm_extract_epi32(vALow, 3);
- countLow = _mm_extract_epi32(vCountLow, 3);
- aLow >>= countLow;
- vALow = _mm_insert_epi32(vALow, aLow, 3);
-
- __m256i ret = _mm256_set1_epi32(0);
- ret = _mm256_insertf128_si256(ret, vAHi, 1);
- ret = _mm256_insertf128_si256(ret, vALow, 0);
- return ret;
-}
-
-
-INLINE
-__m256i _simdemu_sllv_epi32(__m256i vA, __m256i vCount)
-{
- int32_t aHi, aLow, countHi, countLow;
- __m128i vAHi = _mm_castps_si128(_mm256_extractf128_ps(_mm256_castsi256_ps(vA), 1));
- __m128i vALow = _mm_castps_si128(_mm256_extractf128_ps(_mm256_castsi256_ps(vA), 0));
- __m128i vCountHi = _mm_castps_si128(_mm256_extractf128_ps(_mm256_castsi256_ps(vCount), 1));
- __m128i vCountLow = _mm_castps_si128(_mm256_extractf128_ps(_mm256_castsi256_ps(vCount), 0));
-
- aHi = _mm_extract_epi32(vAHi, 0);
- countHi = _mm_extract_epi32(vCountHi, 0);
- aHi <<= countHi;
- vAHi = _mm_insert_epi32(vAHi, aHi, 0);
-
- aLow = _mm_extract_epi32(vALow, 0);
- countLow = _mm_extract_epi32(vCountLow, 0);
- aLow <<= countLow;
- vALow = _mm_insert_epi32(vALow, aLow, 0);
-
- aHi = _mm_extract_epi32(vAHi, 1);
- countHi = _mm_extract_epi32(vCountHi, 1);
- aHi <<= countHi;
- vAHi = _mm_insert_epi32(vAHi, aHi, 1);
-
- aLow = _mm_extract_epi32(vALow, 1);
- countLow = _mm_extract_epi32(vCountLow, 1);
- aLow <<= countLow;
- vALow = _mm_insert_epi32(vALow, aLow, 1);
-
- aHi = _mm_extract_epi32(vAHi, 2);
- countHi = _mm_extract_epi32(vCountHi, 2);
- aHi <<= countHi;
- vAHi = _mm_insert_epi32(vAHi, aHi, 2);
-
- aLow = _mm_extract_epi32(vALow, 2);
- countLow = _mm_extract_epi32(vCountLow, 2);
- aLow <<= countLow;
- vALow = _mm_insert_epi32(vALow, aLow, 2);
-
- aHi = _mm_extract_epi32(vAHi, 3);
- countHi = _mm_extract_epi32(vCountHi, 3);
- aHi <<= countHi;
- vAHi = _mm_insert_epi32(vAHi, aHi, 3);
-
- aLow = _mm_extract_epi32(vALow, 3);
- countLow = _mm_extract_epi32(vCountLow, 3);
- aLow <<= countLow;
- vALow = _mm_insert_epi32(vALow, aLow, 3);
-
- __m256i ret = _mm256_set1_epi32(0);
- ret = _mm256_insertf128_si256(ret, vAHi, 1);
- ret = _mm256_insertf128_si256(ret, vALow, 0);
- return ret;
-}
-
-#define _simd_mul_epi32 _simdemu_mul_epi32
-#define _simd_mullo_epi32 _simdemu_mullo_epi32
-#define _simd_sub_epi32 _simdemu_sub_epi32
-#define _simd_sub_epi64 _simdemu_sub_epi64
-#define _simd_min_epi32 _simdemu_min_epi32
-#define _simd_min_epu32 _simdemu_min_epu32
-#define _simd_max_epi32 _simdemu_max_epi32
-#define _simd_max_epu32 _simdemu_max_epu32
-#define _simd_add_epi32 _simdemu_add_epi32
-#define _simd_and_si _simdemu_and_si
-#define _simd_andnot_si _simdemu_andnot_si
-#define _simd_cmpeq_epi32 _simdemu_cmpeq_epi32
-#define _simd_cmplt_epi32 _simdemu_cmplt_epi32
-#define _simd_cmpgt_epi32 _simdemu_cmpgt_epi32
-#define _simd_or_si _simdemu_or_si
-#define _simd_xor_si _simdemu_xor_si
-#define _simd_castps_si _mm256_castps_si256
-#define _simd_adds_epu8 _simdemu_adds_epu8
-#define _simd_subs_epu8 _simdemu_subs_epu8
-#define _simd_add_epi8 _simdemu_add_epi8
-#define _simd_cmpeq_epi64 _simdemu_cmpeq_epi64
-#define _simd_cmpgt_epi64 _simdemu_cmpgt_epi64
-#define _simd_cmpgt_epi8 _simdemu_cmpgt_epi8
-#define _simd_cmpeq_epi8 _simdemu_cmpeq_epi8
-#define _simd_cmpgt_epi16 _simdemu_cmpgt_epi16
-#define _simd_cmpeq_epi16 _simdemu_cmpeq_epi16
-#define _simd_movemask_epi8 _simdemu_movemask_epi8
-#define _simd_permute_ps _simdemu_permute_ps
-#define _simd_permute_epi32 _simdemu_permute_epi32
-#define _simd_srlv_epi32 _simdemu_srlv_epi32
-#define _simd_sllv_epi32 _simdemu_sllv_epi32
-
-SIMD_EMU_EPI(_simdemu_mul_epi32, _mm_mul_epi32)
-SIMD_EMU_EPI(_simdemu_mullo_epi32, _mm_mullo_epi32)
-SIMD_EMU_EPI(_simdemu_sub_epi32, _mm_sub_epi32)
-SIMD_EMU_EPI(_simdemu_sub_epi64, _mm_sub_epi64)
-SIMD_EMU_EPI(_simdemu_min_epi32, _mm_min_epi32)
-SIMD_EMU_EPI(_simdemu_min_epu32, _mm_min_epu32)
-SIMD_EMU_EPI(_simdemu_max_epi32, _mm_max_epi32)
-SIMD_EMU_EPI(_simdemu_max_epu32, _mm_max_epu32)
-SIMD_EMU_EPI(_simdemu_add_epi32, _mm_add_epi32)
-SIMD_EMU_EPI(_simdemu_and_si, _mm_and_si128)
-SIMD_EMU_EPI(_simdemu_andnot_si, _mm_andnot_si128)
-SIMD_EMU_EPI(_simdemu_cmpeq_epi32, _mm_cmpeq_epi32)
-SIMD_EMU_EPI(_simdemu_cmplt_epi32, _mm_cmplt_epi32)
-SIMD_EMU_EPI(_simdemu_cmpgt_epi32, _mm_cmpgt_epi32)
-SIMD_EMU_EPI(_simdemu_or_si, _mm_or_si128)
-SIMD_EMU_EPI(_simdemu_xor_si, _mm_xor_si128)
-SIMD_EMU_EPI(_simdemu_adds_epu8, _mm_adds_epu8)
-SIMD_EMU_EPI(_simdemu_subs_epu8, _mm_subs_epu8)
-SIMD_EMU_EPI(_simdemu_add_epi8, _mm_add_epi8)
-SIMD_EMU_EPI(_simdemu_cmpeq_epi64, _mm_cmpeq_epi64)
-SIMD_EMU_EPI(_simdemu_cmpgt_epi64, _mm_cmpgt_epi64)
-SIMD_EMU_EPI(_simdemu_cmpgt_epi8, _mm_cmpgt_epi8)
-SIMD_EMU_EPI(_simdemu_cmpeq_epi8, _mm_cmpeq_epi8)
-SIMD_EMU_EPI(_simdemu_cmpgt_epi16, _mm_cmpgt_epi16)
-SIMD_EMU_EPI(_simdemu_cmpeq_epi16, _mm_cmpeq_epi16)
-SIMD_EMU_EPI(_simdemu_unpacklo_epi8, _mm_unpacklo_epi8)
-SIMD_EMU_EPI(_simdemu_unpackhi_epi8, _mm_unpackhi_epi8)
-SIMD_EMU_EPI(_simdemu_unpacklo_epi16, _mm_unpacklo_epi16)
-SIMD_EMU_EPI(_simdemu_unpackhi_epi16, _mm_unpackhi_epi16)
-
-#define _simd_unpacklo_epi8 _simdemu_unpacklo_epi8
-#define _simd_unpackhi_epi8 _simdemu_unpackhi_epi8
-#define _simd_unpacklo_epi16 _simdemu_unpacklo_epi16
-#define _simd_unpackhi_epi16 _simdemu_unpackhi_epi16
-#define _simd_unpacklo_epi32(a, b) _mm256_castps_si256(_mm256_unpacklo_ps(_mm256_castsi256_ps(a), _mm256_castsi256_ps(b)))
-#define _simd_unpackhi_epi32(a, b) _mm256_castps_si256(_mm256_unpackhi_ps(_mm256_castsi256_ps(a), _mm256_castsi256_ps(b)))
-#define _simd_unpacklo_epi64(a, b) _mm256_castpd_si256(_mm256_unpacklo_pd(_mm256_castsi256_pd(a), _mm256_castsi256_pd(b)))
-#define _simd_unpackhi_epi64(a, b) _mm256_castpd_si256(_mm256_unpackhi_pd(_mm256_castsi256_pd(a), _mm256_castsi256_pd(b)))
-
-#define _simd_slli_epi32(a,i) _simdemu_slli_epi32(a,i)
-#define _simd_srai_epi32(a,i) _simdemu_srai_epi32(a,i)
-#define _simd_srli_epi32(a,i) _simdemu_srli_epi32(a,i)
-#define _simd_srlisi_ps(a,i) _mm256_castsi256_ps(_simdemu_srli_si128<i>(_mm256_castps_si256(a)))
-
-#define _simd128_fmadd_ps _mm_fmaddemu_ps
-#define _simd_fmadd_ps _mm_fmaddemu256_ps
-#define _simd_fmsub_ps _mm_fmsubemu256_ps
-#define _simd_shuffle_epi8 _simdemu_shuffle_epi8
-SIMD_EMU_EPI(_simdemu_shuffle_epi8, _mm_shuffle_epi8)
-
-INLINE
-__m128 _mm_fmaddemu_ps(__m128 a, __m128 b, __m128 c)
-{
- __m128 res = _mm_mul_ps(a, b);
- res = _mm_add_ps(res, c);
- return res;
-}
-
-INLINE
-__m256 _mm_fmaddemu256_ps(__m256 a, __m256 b, __m256 c)
-{
- __m256 res = _mm256_mul_ps(a, b);
- res = _mm256_add_ps(res, c);
- return res;
-}
-
-INLINE
-__m256 _mm_fmsubemu256_ps(__m256 a, __m256 b, __m256 c)
-{
- __m256 res = _mm256_mul_ps(a, b);
- res = _mm256_sub_ps(res, c);
- return res;
-}
-
-INLINE
-__m256 _simd_i32gather_ps(const float* pBase, __m256i vOffsets, const int scale)
-{
- uint32_t *pOffsets = (uint32_t*)&vOffsets;
- simdscalar vResult;
- float* pResult = (float*)&vResult;
- for (uint32_t i = 0; i < KNOB_SIMD_WIDTH; ++i)
- {
- uint32_t offset = pOffsets[i];
- offset = offset * scale;
- pResult[i] = *(float*)(((const uint8_t*)pBase + offset));
- }
-
- return vResult;
-}
-
-INLINE
-__m256 _simd_mask_i32gather_ps(__m256 vSrc, const float* pBase, __m256i vOffsets, __m256 vMask, const int scale)
-{
- uint32_t *pOffsets = (uint32_t*)&vOffsets;
- simdscalar vResult = vSrc;
- float* pResult = (float*)&vResult;
- DWORD index;
- uint32_t mask = _simd_movemask_ps(vMask);
- while (_BitScanForward(&index, mask))
- {
- mask &= ~(1 << index);
- uint32_t offset = pOffsets[index];
- offset = offset * scale;
- pResult[index] = *(float*)(((const uint8_t*)pBase + offset));
- }
-
- return vResult;
-}
-
-INLINE
-__m256i _simd_abs_epi32(__m256i a)
-{
- __m128i aHi = _mm256_extractf128_si256(a, 1);
- __m128i aLo = _mm256_castsi256_si128(a);
- __m128i absLo = _mm_abs_epi32(aLo);
- __m128i absHi = _mm_abs_epi32(aHi);
- __m256i result = _mm256_castsi128_si256(absLo);
- result = _mm256_insertf128_si256(result, absHi, 1);
- return result;
-}
-
-INLINE
-int _simdemu_movemask_epi8(__m256i a)
-{
- __m128i aHi = _mm256_extractf128_si256(a, 1);
- __m128i aLo = _mm256_castsi256_si128(a);
-
- int resHi = _mm_movemask_epi8(aHi);
- int resLo = _mm_movemask_epi8(aLo);
-
- return (resHi << 16) | resLo;
-}
-
-INLINE
-__m256i _simd_cvtepu8_epi16(__m128i a)
-{
- __m128i resultlo = _mm_cvtepu8_epi16(a);
- __m128i resulthi = _mm_cvtepu8_epi16(_mm_srli_si128(a, 8));
-
- __m256i result = _mm256_castsi128_si256(resultlo);
-
- return _mm256_insertf128_si256(result, resulthi, 1);
-}
-
-INLINE
-__m256i _simd_cvtepu8_epi32(__m128i a)
-{
- __m128i resultlo = _mm_cvtepu8_epi32(a);
- __m128i resulthi = _mm_cvtepu8_epi32(_mm_srli_si128(a, 4));
-
- __m256i result = _mm256_castsi128_si256(resultlo);
-
- return _mm256_insertf128_si256(result, resulthi, 1);
-}
-
-INLINE
-__m256i _simd_cvtepu16_epi32(__m128i a)
-{
- __m128i resultlo = _mm_cvtepu16_epi32(a);
- __m128i resulthi = _mm_cvtepu16_epi32(_mm_srli_si128(a, 8));
-
- __m256i result = _mm256_castsi128_si256(resultlo);
-
- return _mm256_insertf128_si256(result, resulthi, 1);
-}
-
-INLINE
-__m256i _simd_cvtepu16_epi64(__m128i a)
-{
- __m128i resultlo = _mm_cvtepu16_epi64(a);
- __m128i resulthi = _mm_cvtepu16_epi64(_mm_srli_si128(a, 4));
-
- __m256i result = _mm256_castsi128_si256(resultlo);
-
- return _mm256_insertf128_si256(result, resulthi, 1);
-}
-
-INLINE
-__m256i _simd_cvtepu32_epi64(__m128i a)
-{
- __m128i resultlo = _mm_cvtepu32_epi64(a);
- __m128i resulthi = _mm_cvtepu32_epi64(_mm_srli_si128(a, 8));
-
- __m256i result = _mm256_castsi128_si256(resultlo);
-
- return _mm256_insertf128_si256(result, resulthi, 1);
-}
-
-INLINE
-__m256i _simd_packus_epi16(__m256i a, __m256i b)
-{
- __m128i alo = _mm256_extractf128_si256(a, 0);
- __m128i ahi = _mm256_extractf128_si256(a, 1);
-
- __m128i blo = _mm256_extractf128_si256(b, 0);
- __m128i bhi = _mm256_extractf128_si256(b, 1);
-
- __m128i resultlo = _mm_packus_epi16(alo, blo);
- __m128i resulthi = _mm_packus_epi16(ahi, bhi);
-
- __m256i result = _mm256_castsi128_si256(resultlo);
-
- return _mm256_insertf128_si256(result, resulthi, 1);
-}
-
-INLINE
-__m256i _simd_packs_epi16(__m256i a, __m256i b)
-{
- __m128i alo = _mm256_extractf128_si256(a, 0);
- __m128i ahi = _mm256_extractf128_si256(a, 1);
-
- __m128i blo = _mm256_extractf128_si256(b, 0);
- __m128i bhi = _mm256_extractf128_si256(b, 1);
-
- __m128i resultlo = _mm_packs_epi16(alo, blo);
- __m128i resulthi = _mm_packs_epi16(ahi, bhi);
-
- __m256i result = _mm256_castsi128_si256(resultlo);
-
- return _mm256_insertf128_si256(result, resulthi, 1);
-}
-
-INLINE
-__m256i _simd_packus_epi32(__m256i a, __m256i b)
-{
- __m128i alo = _mm256_extractf128_si256(a, 0);
- __m128i ahi = _mm256_extractf128_si256(a, 1);
-
- __m128i blo = _mm256_extractf128_si256(b, 0);
- __m128i bhi = _mm256_extractf128_si256(b, 1);
-
- __m128i resultlo = _mm_packus_epi32(alo, blo);
- __m128i resulthi = _mm_packus_epi32(ahi, bhi);
-
- __m256i result = _mm256_castsi128_si256(resultlo);
-
- return _mm256_insertf128_si256(result, resulthi, 1);
-}
-
-INLINE
-__m256i _simd_packs_epi32(__m256i a, __m256i b)
-{
- __m128i alo = _mm256_extractf128_si256(a, 0);
- __m128i ahi = _mm256_extractf128_si256(a, 1);
-
- __m128i blo = _mm256_extractf128_si256(b, 0);
- __m128i bhi = _mm256_extractf128_si256(b, 1);
-
- __m128i resultlo = _mm_packs_epi32(alo, blo);
- __m128i resulthi = _mm_packs_epi32(ahi, bhi);
-
- __m256i result = _mm256_castsi128_si256(resultlo);
-
- return _mm256_insertf128_si256(result, resulthi, 1);
-}
-
+typedef SIMD256 SIMD;
#else
-
-#define _simd_mul_epi32 _mm256_mul_epi32
-#define _simd_mullo_epi32 _mm256_mullo_epi32
-#define _simd_sub_epi32 _mm256_sub_epi32
-#define _simd_sub_epi64 _mm256_sub_epi64
-#define _simd_min_epi32 _mm256_min_epi32
-#define _simd_max_epi32 _mm256_max_epi32
-#define _simd_min_epu32 _mm256_min_epu32
-#define _simd_max_epu32 _mm256_max_epu32
-#define _simd_add_epi32 _mm256_add_epi32
-#define _simd_and_si _mm256_and_si256
-#define _simd_andnot_si _mm256_andnot_si256
-#define _simd_cmpeq_epi32 _mm256_cmpeq_epi32
-#define _simd_cmplt_epi32(a,b) _mm256_cmpgt_epi32(b,a)
-#define _simd_cmpgt_epi32(a,b) _mm256_cmpgt_epi32(a,b)
-#define _simd_or_si _mm256_or_si256
-#define _simd_xor_si _mm256_xor_si256
-#define _simd_castps_si _mm256_castps_si256
-
-#define _simd_unpacklo_epi8 _mm256_unpacklo_epi8
-#define _simd_unpackhi_epi8 _mm256_unpackhi_epi8
-#define _simd_unpacklo_epi16 _mm256_unpacklo_epi16
-#define _simd_unpackhi_epi16 _mm256_unpackhi_epi16
-#define _simd_unpacklo_epi32 _mm256_unpacklo_epi32
-#define _simd_unpackhi_epi32 _mm256_unpackhi_epi32
-#define _simd_unpacklo_epi64 _mm256_unpacklo_epi64
-#define _simd_unpackhi_epi64 _mm256_unpackhi_epi64
-
-#define _simd_srli_si(a,i) _simdemu_srli_si128<i>(a)
-#define _simd_slli_epi32 _mm256_slli_epi32
-#define _simd_srai_epi32 _mm256_srai_epi32
-#define _simd_srli_epi32 _mm256_srli_epi32
-#define _simd_srlisi_ps(a,i) _mm256_castsi256_ps(_simdemu_srli_si128<i>(_mm256_castps_si256(a)))
-#define _simd128_fmadd_ps _mm_fmadd_ps
-#define _simd_fmadd_ps _mm256_fmadd_ps
-#define _simd_fmsub_ps _mm256_fmsub_ps
-#define _simd_shuffle_epi8 _mm256_shuffle_epi8
-#define _simd_adds_epu8 _mm256_adds_epu8
-#define _simd_subs_epu8 _mm256_subs_epu8
-#define _simd_add_epi8 _mm256_add_epi8
-#define _simd_i32gather_ps _mm256_i32gather_ps
-#define _simd_mask_i32gather_ps _mm256_mask_i32gather_ps
-#define _simd_abs_epi32 _mm256_abs_epi32
-
-#define _simd_cmpeq_epi64 _mm256_cmpeq_epi64
-#define _simd_cmpgt_epi64 _mm256_cmpgt_epi64
-#define _simd_cmpgt_epi8 _mm256_cmpgt_epi8
-#define _simd_cmpeq_epi8 _mm256_cmpeq_epi8
-#define _simd_cmpgt_epi16 _mm256_cmpgt_epi16
-#define _simd_cmpeq_epi16 _mm256_cmpeq_epi16
-#define _simd_movemask_epi8 _mm256_movemask_epi8
-#define _simd_permute_ps _mm256_permutevar8x32_ps
-#define _simd_permute_epi32 _mm256_permutevar8x32_epi32
-#define _simd_srlv_epi32 _mm256_srlv_epi32
-#define _simd_sllv_epi32 _mm256_sllv_epi32
-#define _simd_cvtepu8_epi16 _mm256_cvtepu8_epi16
-#define _simd_cvtepu8_epi32 _mm256_cvtepu8_epi32
-#define _simd_cvtepu16_epi32 _mm256_cvtepu16_epi32
-#define _simd_cvtepu16_epi64 _mm256_cvtepu16_epi64
-#define _simd_cvtepu32_epi64 _mm256_cvtepu32_epi64
-#define _simd_packus_epi16 _mm256_packus_epi16
-#define _simd_packs_epi16 _mm256_packs_epi16
-#define _simd_packus_epi32 _mm256_packus_epi32
-#define _simd_packs_epi32 _mm256_packs_epi32
-
-#endif
-
-#define _simd_unpacklo_ps _mm256_unpacklo_ps
-#define _simd_unpackhi_ps _mm256_unpackhi_ps
-#define _simd_unpacklo_pd _mm256_unpacklo_pd
-#define _simd_unpackhi_pd _mm256_unpackhi_pd
-#define _simd_insertf128_ps _mm256_insertf128_ps
-#define _simd_insertf128_pd _mm256_insertf128_pd
-#define _simd_insertf128_si _mm256_insertf128_si256
-#define _simd_extractf128_ps _mm256_extractf128_ps
-#define _simd_extractf128_pd _mm256_extractf128_pd
-#define _simd_extractf128_si _mm256_extractf128_si256
-#define _simd_permute2f128_ps _mm256_permute2f128_ps
-#define _simd_permute2f128_pd _mm256_permute2f128_pd
-#define _simd_permute2f128_si _mm256_permute2f128_si256
-#define _simd_shuffle_ps _mm256_shuffle_ps
-#define _simd_shuffle_pd _mm256_shuffle_pd
-#define _simd_shuffle_epi32(a, b, imm8) _mm256_castps_si256(_mm256_shuffle_ps(_mm256_castsi256_ps(a), _mm256_castsi256_ps(b), imm8))
-#define _simd_shuffle_epi64(a, b, imm8) _mm256_castps_si256(_mm256_shuffle_pd(_mm256_castsi256_pd(a), _mm256_castsi256_pd(b), imm8))
-#define _simd_set1_epi32 _mm256_set1_epi32
-#define _simd_set_epi32 _mm256_set_epi32
-#define _simd_set1_epi8 _mm256_set1_epi8
-#define _simd_setzero_si _mm256_setzero_si256
-#define _simd_cvttps_epi32 _mm256_cvttps_epi32
-#define _simd_store_si _mm256_store_si256
-#define _simd_broadcast_ss _mm256_broadcast_ss
-#define _simd_maskstore_ps _mm256_maskstore_ps
-#define _simd_load_si _mm256_load_si256
-#define _simd_loadu_si _mm256_loadu_si256
-#define _simd_sub_ps _mm256_sub_ps
-#define _simd_testz_ps _mm256_testz_ps
-#define _simd_testz_si _mm256_testz_si256
-#define _simd_xor_ps _mm256_xor_ps
-
-INLINE
-simdscalari _simd_loadu2_si(const __m128i *hiaddr, const __m128i *loaddr)
-{
- __m128i lo = _mm_loadu_si128(loaddr);
- __m128i hi = _mm_loadu_si128(hiaddr);
-
- return _mm256_insertf128_si256(_mm256_castsi128_si256(lo), (hi), 1);
-}
-
-INLINE
-void _simd_storeu2_si(__m128i *hiaddr, __m128i *loaddr, simdscalari a)
-{
- _mm_storeu_si128(loaddr, _mm256_castsi256_si128(a));
- _mm_storeu_si128(hiaddr, _mm256_extractf128_si256(a, 1));
-}
-
-INLINE
-simdscalari _simd_blendv_epi32(simdscalari a, simdscalari b, simdscalar mask)
-{
- return _simd_castps_si(_simd_blendv_ps(_simd_castsi_ps(a), _simd_castsi_ps(b), mask));
-}
-
-INLINE
-simdscalari _simd_blendv_epi32(simdscalari a, simdscalari b, simdscalari mask)
-{
- return _simd_castps_si(_simd_blendv_ps(_simd_castsi_ps(a), _simd_castsi_ps(b), _simd_castsi_ps(mask)));
-}
-
-template<int mask>
-INLINE
-__m128i _simd_blend4_epi32(__m128i a, __m128i b)
-{
- return _mm_castps_si128(_mm_blend_ps(_mm_castsi128_ps(a), _mm_castsi128_ps(b), mask));
+#error Unsupported vector width
+#endif//KNOB_SIMD16_WIDTH == 16
+
+
+#define _simd128_maskstore_ps SIMD128::maskstore_ps
+#define _simd128_fmadd_ps SIMD128::fmadd_ps
+
+#define _simd_load_ps SIMD::load_ps
+#define _simd_load1_ps SIMD::broadcast_ss
+#define _simd_loadu_ps SIMD::loadu_ps
+#define _simd_setzero_ps SIMD::setzero_ps
+#define _simd_set1_ps SIMD::set1_ps
+#define _simd_blend_ps(a, b, i) SIMD::blend_ps<i>(a, b)
+#define _simd_blend_epi32(a, b, i) SIMD::blend_epi32<i>(a, b)
+#define _simd_blendv_ps SIMD::blendv_ps
+#define _simd_store_ps SIMD::store_ps
+#define _simd_mul_ps SIMD::mul_ps
+#define _simd_add_ps SIMD::add_ps
+#define _simd_sub_ps SIMD::sub_ps
+#define _simd_rsqrt_ps SIMD::rsqrt_ps
+#define _simd_min_ps SIMD::min_ps
+#define _simd_max_ps SIMD::max_ps
+#define _simd_movemask_ps SIMD::movemask_ps
+#define _simd_cvtps_epi32 SIMD::cvtps_epi32
+#define _simd_cvttps_epi32 SIMD::cvttps_epi32
+#define _simd_cvtepi32_ps SIMD::cvtepi32_ps
+#define _simd_cmplt_ps SIMD::cmplt_ps
+#define _simd_cmpgt_ps SIMD::cmpgt_ps
+#define _simd_cmpneq_ps SIMD::cmpneq_ps
+#define _simd_cmpeq_ps SIMD::cmpeq_ps
+#define _simd_cmpge_ps SIMD::cmpge_ps
+#define _simd_cmple_ps SIMD::cmple_ps
+#define _simd_cmp_ps(a, b, imm) SIMD::cmp_ps<SIMD::CompareType(imm)>(a, b)
+#define _simd_and_ps SIMD::and_ps
+#define _simd_or_ps SIMD::or_ps
+#define _simd_rcp_ps SIMD::rcp_ps
+#define _simd_div_ps SIMD::div_ps
+#define _simd_castsi_ps SIMD::castsi_ps
+#define _simd_castps_pd SIMD::castps_pd
+#define _simd_castpd_ps SIMD::castpd_ps
+#define _simd_andnot_ps SIMD::andnot_ps
+#define _simd_round_ps(a, i) SIMD::round_ps<SIMD::RoundMode(i)>(a)
+#define _simd_castpd_ps SIMD::castpd_ps
+#define _simd_broadcast_ps(a) SIMD::broadcast_ps((SIMD128::Float const *)(a))
+#define _simd_stream_ps SIMD::stream_ps
+
+#define _simd_movemask_pd SIMD::movemask_pd
+#define _simd_castsi_pd SIMD::castsi_pd
+
+#define _simd_mul_epi32 SIMD::mul_epi32
+#define _simd_mullo_epi32 SIMD::mullo_epi32
+#define _simd_sub_epi32 SIMD::sub_epi32
+#define _simd_sub_epi64 SIMD::sub_epi64
+#define _simd_min_epi32 SIMD::min_epi32
+#define _simd_min_epu32 SIMD::min_epu32
+#define _simd_max_epi32 SIMD::max_epi32
+#define _simd_max_epu32 SIMD::max_epu32
+#define _simd_add_epi32 SIMD::add_epi32
+#define _simd_and_si SIMD::and_si
+#define _simd_andnot_si SIMD::andnot_si
+#define _simd_cmpeq_epi32 SIMD::cmpeq_epi32
+#define _simd_cmplt_epi32 SIMD::cmplt_epi32
+#define _simd_cmpgt_epi32 SIMD::cmpgt_epi32
+#define _simd_or_si SIMD::or_si
+#define _simd_xor_si SIMD::xor_si
+#define _simd_castps_si SIMD::castps_si
+#define _simd_adds_epu8 SIMD::adds_epu8
+#define _simd_subs_epu8 SIMD::subs_epu8
+#define _simd_add_epi8 SIMD::add_epi8
+#define _simd_cmpeq_epi64 SIMD::cmpeq_epi64
+#define _simd_cmpgt_epi64 SIMD::cmpgt_epi64
+#define _simd_cmpgt_epi8 SIMD::cmpgt_epi8
+#define _simd_cmpeq_epi8 SIMD::cmpeq_epi8
+#define _simd_cmpgt_epi16 SIMD::cmpgt_epi16
+#define _simd_cmpeq_epi16 SIMD::cmpeq_epi16
+#define _simd_movemask_epi8 SIMD::movemask_epi8
+#define _simd_permute_ps SIMD::permute_ps
+#define _simd_permute_epi32 SIMD::permute_epi32
+#define _simd_srlv_epi32 SIMD::srlv_epi32
+#define _simd_sllv_epi32 SIMD::sllv_epi32
+
+#define _simd_unpacklo_epi8 SIMD::unpacklo_epi8
+#define _simd_unpackhi_epi8 SIMD::unpackhi_epi8
+#define _simd_unpacklo_epi16 SIMD::unpacklo_epi16
+#define _simd_unpackhi_epi16 SIMD::unpackhi_epi16
+#define _simd_unpacklo_epi32 SIMD::unpacklo_epi32
+#define _simd_unpackhi_epi32 SIMD::unpackhi_epi32
+#define _simd_unpacklo_epi64 SIMD::unpacklo_epi64
+#define _simd_unpackhi_epi64 SIMD::unpackhi_epi64
+
+#define _simd_slli_epi32(a,i) SIMD::slli_epi32<i>(a)
+#define _simd_srai_epi32(a,i) SIMD::srai_epi32<i>(a)
+#define _simd_srli_epi32(a,i) SIMD::srli_epi32<i>(a)
+#define _simd_srlisi_ps(a,i) SIMD::srlisi_ps<i>(a)
+
+#define _simd_fmadd_ps SIMD::fmadd_ps
+#define _simd_fmsub_ps SIMD::fmsub_ps
+#define _simd_shuffle_epi8 SIMD::shuffle_epi8
+
+#define _simd_i32gather_ps(p, o, s) SIMD::i32gather_ps<SIMD::ScaleFactor(s)>(p, o)
+#define _simd_mask_i32gather_ps(r, p, o, m, s) SIMD::mask_i32gather_ps<SIMD::ScaleFactor(s)>(r, p, o, m)
+#define _simd_abs_epi32 SIMD::abs_epi32
+
+#define _simd_cvtepu8_epi16 SIMD::cvtepu8_epi16
+#define _simd_cvtepu8_epi32 SIMD::cvtepu8_epi32
+#define _simd_cvtepu16_epi32 SIMD::cvtepu16_epi32
+#define _simd_cvtepu16_epi64 SIMD::cvtepu16_epi64
+#define _simd_cvtepu32_epi64 SIMD::cvtepu32_epi64
+
+#define _simd_packus_epi16 SIMD::packus_epi16
+#define _simd_packs_epi16 SIMD::packs_epi16
+#define _simd_packus_epi32 SIMD::packus_epi32
+#define _simd_packs_epi32 SIMD::packs_epi32
+
+#define _simd_unpacklo_ps SIMD::unpacklo_ps
+#define _simd_unpackhi_ps SIMD::unpackhi_ps
+#define _simd_unpacklo_pd SIMD::unpacklo_pd
+#define _simd_unpackhi_pd SIMD::unpackhi_pd
+#define _simd_insertf128_ps SIMD::insertf128_ps
+#define _simd_insertf128_pd SIMD::insertf128_pd
+#define _simd_insertf128_si(a, b, i) SIMD::insertf128_si<i>(a, b)
+#define _simd_extractf128_ps(a, i) SIMD::extractf128_ps<i>(a)
+#define _simd_extractf128_pd(a, i) SIMD::extractf128_pd<i>(a)
+#define _simd_extractf128_si(a, i) SIMD::extractf128_si<i>(a)
+#define _simd_permute2f128_ps(a, b, i) SIMD::permute2f128_ps<i>(a, b)
+#define _simd_permute2f128_pd(a, b, i) SIMD::permute2f128_pd<i>(a, b)
+#define _simd_permute2f128_si(a, b, i) SIMD::permute2f128_si<i>(a, b)
+#define _simd_shuffle_ps(a, b, i) SIMD::shuffle_ps<i>(a, b)
+#define _simd_shuffle_pd(a, b, i) SIMD::shuffle_pd<i>(a, b)
+#define _simd_shuffle_epi32(a, b, imm8) SIMD::shuffle_epi32<imm8>(a, b)
+#define _simd_shuffle_epi64(a, b, imm8) SIMD::shuffle_epi64<imm8>(a, b)
+#define _simd_set1_epi32 SIMD::set1_epi32
+#define _simd_set_epi32 SIMD::set_epi32
+#define _simd_set_ps SIMD::set_ps
+#define _simd_set1_epi8 SIMD::set1_epi8
+#define _simd_setzero_si SIMD::setzero_si
+#define _simd_cvttps_epi32 SIMD::cvttps_epi32
+#define _simd_store_si SIMD::store_si
+#define _simd_broadcast_ss SIMD::broadcast_ss
+#define _simd_maskstore_ps SIMD::maskstore_ps
+#define _simd_load_si SIMD::load_si
+#define _simd_loadu_si SIMD::loadu_si
+#define _simd_sub_ps SIMD::sub_ps
+#define _simd_testz_ps SIMD::testz_ps
+#define _simd_testz_si SIMD::testz_si
+#define _simd_xor_ps SIMD::xor_ps
+
+#define _simd_loadu2_si SIMD::loadu2_si
+#define _simd_storeu2_si SIMD::storeu2_si
+
+#define _simd_blendv_epi32 SIMD::blendv_epi32
+
+template<int mask> SIMDINLINE
+SIMD128::Integer _simd_blend4_epi32(SIMD128::Integer a, SIMD128::Integer b)
+{
+ return SIMD128::castps_si(SIMD128::blend_ps<mask>(SIMD128::castsi_ps(a), SIMD128::castsi_ps(b)));
}
// convert bitmask to vector mask
-INLINE
-simdscalar vMask(int32_t mask)
+SIMDINLINE
+SIMD256::Float vMask(int32_t mask)
{
- __m256i vec = _mm256_set1_epi32(mask);
- const __m256i bit = _mm256_set_epi32(0x80, 0x40, 0x20, 0x10, 0x08, 0x04, 0x02, 0x01);
- vec = _simd_and_si(vec, bit);
- vec = _simd_cmplt_epi32(_mm256_setzero_si256(), vec);
- return _simd_castsi_ps(vec);
+ SIMD256::Integer vec = SIMD256::set1_epi32(mask);
+ const SIMD256::Integer bit = SIMD256::set_epi32(0x80, 0x40, 0x20, 0x10, 0x08, 0x04, 0x02, 0x01);
+ vec = SIMD256::and_si(vec, bit);
+ vec = SIMD256::cmplt_epi32(SIMD256::setzero_si(), vec);
+ return SIMD256::castsi_ps(vec);
}
-INLINE
-simdscalari vMaski(int32_t mask)
+SIMDINLINE
+SIMD256::Integer vMaski(int32_t mask)
{
- __m256i vec = _mm256_set1_epi32(mask);
- const __m256i bit = _mm256_set_epi32(0x80, 0x40, 0x20, 0x10, 0x08, 0x04, 0x02, 0x01);
- vec = _simd_and_si(vec, bit);
- return _simd_cmplt_epi32(_mm256_setzero_si256(), vec);
+ SIMD256::Integer vec = SIMD256::set1_epi32(mask);
+ const SIMD256::Integer bit = SIMD256::set_epi32(0x80, 0x40, 0x20, 0x10, 0x08, 0x04, 0x02, 0x01);
+ vec = SIMD256::and_si(vec, bit);
+ return SIMD256::cmplt_epi32(SIMD256::setzero_si(), vec);
}
-INLINE
+SIMDINLINE
void _simd_mov(simdscalar &r, unsigned int rlane, simdscalar& s, unsigned int slane)
{
OSALIGNSIMD(float) rArray[KNOB_SIMD_WIDTH], sArray[KNOB_SIMD_WIDTH];
- _mm256_store_ps(rArray, r);
- _mm256_store_ps(sArray, s);
+ SIMD256::store_ps(rArray, r);
+ SIMD256::store_ps(sArray, s);
rArray[rlane] = sArray[slane];
- r = _mm256_load_ps(rArray);
+ r = SIMD256::load_ps(rArray);
}
-INLINE __m256i _simdemu_slli_epi32(__m256i a, uint32_t i)
-{
- __m128i aHi = _mm256_extractf128_si256(a, 1);
- __m128i aLo = _mm256_castsi256_si128(a);
-
- __m128i resHi = _mm_slli_epi32(aHi, i);
- __m128i resLo = _mm_slli_epi32(aLo, i);
-
- __m256i result = _mm256_castsi128_si256(resLo);
- result = _mm256_insertf128_si256(result, resHi, 1);
-
- return result;
-}
-
-INLINE __m256i _simdemu_srai_epi32(__m256i a, uint32_t i)
-{
- __m128i aHi = _mm256_extractf128_si256(a, 1);
- __m128i aLo = _mm256_castsi256_si128(a);
-
- __m128i resHi = _mm_srai_epi32(aHi, i);
- __m128i resLo = _mm_srai_epi32(aLo, i);
-
- __m256i result = _mm256_castsi128_si256(resLo);
- result = _mm256_insertf128_si256(result, resHi, 1);
-
- return result;
-}
-
-INLINE __m256i _simdemu_srli_epi32(__m256i a, uint32_t i)
-{
- __m128i aHi = _mm256_extractf128_si256(a, 1);
- __m128i aLo = _mm256_castsi256_si128(a);
-
- __m128i resHi = _mm_srli_epi32(aHi, i);
- __m128i resLo = _mm_srli_epi32(aLo, i);
-
- __m256i result = _mm256_castsi128_si256(resLo);
- result = _mm256_insertf128_si256(result, resHi, 1);
-
- return result;
-}
-
-INLINE
-void _simdvec_transpose(simdvector &v)
-{
- SWR_INVALID("Need to implement 8 wide version");
-}
-
-#else
-#error Unsupported vector width
-#endif
-
// Populates a simdvector from a vector. So p = xyzw becomes xxxx yyyy zzzz wwww.
-INLINE
-void _simdvec_load_ps(simdvector& r, const float *p)
-{
- r[0] = _simd_set1_ps(p[0]);
- r[1] = _simd_set1_ps(p[1]);
- r[2] = _simd_set1_ps(p[2]);
- r[3] = _simd_set1_ps(p[3]);
-}
+#define _simdvec_load_ps SIMD::vec4_load1_ps
-INLINE
+SIMDINLINE
void _simdvec_mov(simdvector& r, const simdscalar& s)
{
- r[0] = s;
- r[1] = s;
- r[2] = s;
- r[3] = s;
+ SIMD::vec4_set1_vps(r, s);
}
-INLINE
+SIMDINLINE
void _simdvec_mov(simdvector& r, const simdvector& v)
{
- r[0] = v[0];
- r[1] = v[1];
- r[2] = v[2];
- r[3] = v[3];
+ r = v;
}
#if 0
// just move a lane from the source simdvector to dest simdvector
-INLINE
+SIMDINLINE
void _simdvec_mov(simdvector &r, unsigned int rlane, simdvector& s, unsigned int slane)
{
_simd_mov(r[0], rlane, s[0], slane);
}
#endif
-INLINE
-void _simdvec_dp3_ps(simdscalar& r, const simdvector& v0, const simdvector& v1)
-{
- simdscalar tmp;
- r = _simd_mul_ps(v0[0], v1[0]); // (v0.x*v1.x)
-
- tmp = _simd_mul_ps(v0[1], v1[1]); // (v0.y*v1.y)
- r = _simd_add_ps(r, tmp); // (v0.x*v1.x) + (v0.y*v1.y)
- tmp = _simd_mul_ps(v0[2], v1[2]); // (v0.z*v1.z)
- r = _simd_add_ps(r, tmp); // (v0.x*v1.x) + (v0.y*v1.y) + (v0.z*v1.z)
-}
-
-INLINE
-void _simdvec_dp4_ps(simdscalar& r, const simdvector& v0, const simdvector& v1)
-{
- simdscalar tmp;
- r = _simd_mul_ps(v0[0], v1[0]); // (v0.x*v1.x)
-
- tmp = _simd_mul_ps(v0[1], v1[1]); // (v0.y*v1.y)
- r = _simd_add_ps(r, tmp); // (v0.x*v1.x) + (v0.y*v1.y)
-
- tmp = _simd_mul_ps(v0[2], v1[2]); // (v0.z*v1.z)
- r = _simd_add_ps(r, tmp); // (v0.x*v1.x) + (v0.y*v1.y) + (v0.z*v1.z)
-
- tmp = _simd_mul_ps(v0[3], v1[3]); // (v0.w*v1.w)
- r = _simd_add_ps(r, tmp); // (v0.x*v1.x) + (v0.y*v1.y) + (v0.z*v1.z)
-}
-
-INLINE
-simdscalar _simdvec_rcp_length_ps(const simdvector& v)
-{
- simdscalar length;
- _simdvec_dp4_ps(length, v, v);
- return _simd_rsqrt_ps(length);
-}
-
-INLINE
-void _simdvec_normalize_ps(simdvector& r, const simdvector& v)
-{
- simdscalar vecLength;
- vecLength = _simdvec_rcp_length_ps(v);
-
- r[0] = _simd_mul_ps(v[0], vecLength);
- r[1] = _simd_mul_ps(v[1], vecLength);
- r[2] = _simd_mul_ps(v[2], vecLength);
- r[3] = _simd_mul_ps(v[3], vecLength);
-}
-
-INLINE
-void _simdvec_mul_ps(simdvector& r, const simdvector& v, const simdscalar& s)
-{
- r[0] = _simd_mul_ps(v[0], s);
- r[1] = _simd_mul_ps(v[1], s);
- r[2] = _simd_mul_ps(v[2], s);
- r[3] = _simd_mul_ps(v[3], s);
-}
-
-INLINE
-void _simdvec_mul_ps(simdvector& r, const simdvector& v0, const simdvector& v1)
-{
- r[0] = _simd_mul_ps(v0[0], v1[0]);
- r[1] = _simd_mul_ps(v0[1], v1[1]);
- r[2] = _simd_mul_ps(v0[2], v1[2]);
- r[3] = _simd_mul_ps(v0[3], v1[3]);
-}
-
-INLINE
-void _simdvec_add_ps(simdvector& r, const simdvector& v0, const simdvector& v1)
-{
- r[0] = _simd_add_ps(v0[0], v1[0]);
- r[1] = _simd_add_ps(v0[1], v1[1]);
- r[2] = _simd_add_ps(v0[2], v1[2]);
- r[3] = _simd_add_ps(v0[3], v1[3]);
-}
-
-INLINE
-void _simdvec_min_ps(simdvector& r, const simdvector& v0, const simdscalar& s)
-{
- r[0] = _simd_min_ps(v0[0], s);
- r[1] = _simd_min_ps(v0[1], s);
- r[2] = _simd_min_ps(v0[2], s);
- r[3] = _simd_min_ps(v0[3], s);
-}
-
-INLINE
-void _simdvec_max_ps(simdvector& r, const simdvector& v0, const simdscalar& s)
-{
- r[0] = _simd_max_ps(v0[0], s);
- r[1] = _simd_max_ps(v0[1], s);
- r[2] = _simd_max_ps(v0[2], s);
- r[3] = _simd_max_ps(v0[3], s);
-}
-
-// Matrix4x4 * Vector4
-// outVec.x = (m00 * v.x) + (m01 * v.y) + (m02 * v.z) + (m03 * v.w)
-// outVec.y = (m10 * v.x) + (m11 * v.y) + (m12 * v.z) + (m13 * v.w)
-// outVec.z = (m20 * v.x) + (m21 * v.y) + (m22 * v.z) + (m23 * v.w)
-// outVec.w = (m30 * v.x) + (m31 * v.y) + (m32 * v.z) + (m33 * v.w)
-INLINE
-void _simd_mat4x4_vec4_multiply(
- simdvector& result,
- const float *pMatrix,
- const simdvector& v)
-{
- simdscalar m;
- simdscalar r0;
- simdscalar r1;
-
- m = _simd_load1_ps(pMatrix + 0*4 + 0); // m[row][0]
- r0 = _simd_mul_ps(m, v[0]); // (m00 * v.x)
- m = _simd_load1_ps(pMatrix + 0*4 + 1); // m[row][1]
- r1 = _simd_mul_ps(m, v[1]); // (m1 * v.y)
- r0 = _simd_add_ps(r0, r1); // (m0 * v.x) + (m1 * v.y)
- m = _simd_load1_ps(pMatrix + 0*4 + 2); // m[row][2]
- r1 = _simd_mul_ps(m, v[2]); // (m2 * v.z)
- r0 = _simd_add_ps(r0, r1); // (m0 * v.x) + (m1 * v.y) + (m2 * v.z)
- m = _simd_load1_ps(pMatrix + 0*4 + 3); // m[row][3]
- r1 = _simd_mul_ps(m, v[3]); // (m3 * v.z)
- r0 = _simd_add_ps(r0, r1); // (m0 * v.x) + (m1 * v.y) + (m2 * v.z) + (m2 * v.w)
- result[0] = r0;
-
- m = _simd_load1_ps(pMatrix + 1*4 + 0); // m[row][0]
- r0 = _simd_mul_ps(m, v[0]); // (m00 * v.x)
- m = _simd_load1_ps(pMatrix + 1*4 + 1); // m[row][1]
- r1 = _simd_mul_ps(m, v[1]); // (m1 * v.y)
- r0 = _simd_add_ps(r0, r1); // (m0 * v.x) + (m1 * v.y)
- m = _simd_load1_ps(pMatrix + 1*4 + 2); // m[row][2]
- r1 = _simd_mul_ps(m, v[2]); // (m2 * v.z)
- r0 = _simd_add_ps(r0, r1); // (m0 * v.x) + (m1 * v.y) + (m2 * v.z)
- m = _simd_load1_ps(pMatrix + 1*4 + 3); // m[row][3]
- r1 = _simd_mul_ps(m, v[3]); // (m3 * v.z)
- r0 = _simd_add_ps(r0, r1); // (m0 * v.x) + (m1 * v.y) + (m2 * v.z) + (m2 * v.w)
- result[1] = r0;
-
- m = _simd_load1_ps(pMatrix + 2*4 + 0); // m[row][0]
- r0 = _simd_mul_ps(m, v[0]); // (m00 * v.x)
- m = _simd_load1_ps(pMatrix + 2*4 + 1); // m[row][1]
- r1 = _simd_mul_ps(m, v[1]); // (m1 * v.y)
- r0 = _simd_add_ps(r0, r1); // (m0 * v.x) + (m1 * v.y)
- m = _simd_load1_ps(pMatrix + 2*4 + 2); // m[row][2]
- r1 = _simd_mul_ps(m, v[2]); // (m2 * v.z)
- r0 = _simd_add_ps(r0, r1); // (m0 * v.x) + (m1 * v.y) + (m2 * v.z)
- m = _simd_load1_ps(pMatrix + 2*4 + 3); // m[row][3]
- r1 = _simd_mul_ps(m, v[3]); // (m3 * v.z)
- r0 = _simd_add_ps(r0, r1); // (m0 * v.x) + (m1 * v.y) + (m2 * v.z) + (m2 * v.w)
- result[2] = r0;
-
- m = _simd_load1_ps(pMatrix + 3*4 + 0); // m[row][0]
- r0 = _simd_mul_ps(m, v[0]); // (m00 * v.x)
- m = _simd_load1_ps(pMatrix + 3*4 + 1); // m[row][1]
- r1 = _simd_mul_ps(m, v[1]); // (m1 * v.y)
- r0 = _simd_add_ps(r0, r1); // (m0 * v.x) + (m1 * v.y)
- m = _simd_load1_ps(pMatrix + 3*4 + 2); // m[row][2]
- r1 = _simd_mul_ps(m, v[2]); // (m2 * v.z)
- r0 = _simd_add_ps(r0, r1); // (m0 * v.x) + (m1 * v.y) + (m2 * v.z)
- m = _simd_load1_ps(pMatrix + 3*4 + 3); // m[row][3]
- r1 = _simd_mul_ps(m, v[3]); // (m3 * v.z)
- r0 = _simd_add_ps(r0, r1); // (m0 * v.x) + (m1 * v.y) + (m2 * v.z) + (m2 * v.w)
- result[3] = r0;
-}
-
-// Matrix4x4 * Vector3 - Direction Vector where w = 0.
-// outVec.x = (m00 * v.x) + (m01 * v.y) + (m02 * v.z) + (m03 * 0)
-// outVec.y = (m10 * v.x) + (m11 * v.y) + (m12 * v.z) + (m13 * 0)
-// outVec.z = (m20 * v.x) + (m21 * v.y) + (m22 * v.z) + (m23 * 0)
-// outVec.w = (m30 * v.x) + (m31 * v.y) + (m32 * v.z) + (m33 * 0)
-INLINE
-void _simd_mat3x3_vec3_w0_multiply(
- simdvector& result,
- const float *pMatrix,
- const simdvector& v)
-{
- simdscalar m;
- simdscalar r0;
- simdscalar r1;
-
- m = _simd_load1_ps(pMatrix + 0*4 + 0); // m[row][0]
- r0 = _simd_mul_ps(m, v[0]); // (m00 * v.x)
- m = _simd_load1_ps(pMatrix + 0*4 + 1); // m[row][1]
- r1 = _simd_mul_ps(m, v[1]); // (m1 * v.y)
- r0 = _simd_add_ps(r0, r1); // (m0 * v.x) + (m1 * v.y)
- m = _simd_load1_ps(pMatrix + 0*4 + 2); // m[row][2]
- r1 = _simd_mul_ps(m, v[2]); // (m2 * v.z)
- r0 = _simd_add_ps(r0, r1); // (m0 * v.x) + (m1 * v.y) + (m2 * v.z)
- result[0] = r0;
-
- m = _simd_load1_ps(pMatrix + 1*4 + 0); // m[row][0]
- r0 = _simd_mul_ps(m, v[0]); // (m00 * v.x)
- m = _simd_load1_ps(pMatrix + 1*4 + 1); // m[row][1]
- r1 = _simd_mul_ps(m, v[1]); // (m1 * v.y)
- r0 = _simd_add_ps(r0, r1); // (m0 * v.x) + (m1 * v.y)
- m = _simd_load1_ps(pMatrix + 1*4 + 2); // m[row][2]
- r1 = _simd_mul_ps(m, v[2]); // (m2 * v.z)
- r0 = _simd_add_ps(r0, r1); // (m0 * v.x) + (m1 * v.y) + (m2 * v.z)
- result[1] = r0;
-
- m = _simd_load1_ps(pMatrix + 2*4 + 0); // m[row][0]
- r0 = _simd_mul_ps(m, v[0]); // (m00 * v.x)
- m = _simd_load1_ps(pMatrix + 2*4 + 1); // m[row][1]
- r1 = _simd_mul_ps(m, v[1]); // (m1 * v.y)
- r0 = _simd_add_ps(r0, r1); // (m0 * v.x) + (m1 * v.y)
- m = _simd_load1_ps(pMatrix + 2*4 + 2); // m[row][2]
- r1 = _simd_mul_ps(m, v[2]); // (m2 * v.z)
- r0 = _simd_add_ps(r0, r1); // (m0 * v.x) + (m1 * v.y) + (m2 * v.z)
- result[2] = r0;
-
- result[3] = _simd_setzero_ps();
-}
-
-// Matrix4x4 * Vector3 - Position vector where w = 1.
-// outVec.x = (m00 * v.x) + (m01 * v.y) + (m02 * v.z) + (m03 * 1)
-// outVec.y = (m10 * v.x) + (m11 * v.y) + (m12 * v.z) + (m13 * 1)
-// outVec.z = (m20 * v.x) + (m21 * v.y) + (m22 * v.z) + (m23 * 1)
-// outVec.w = (m30 * v.x) + (m31 * v.y) + (m32 * v.z) + (m33 * 1)
-INLINE
-void _simd_mat4x4_vec3_w1_multiply(
- simdvector& result,
- const float *pMatrix,
- const simdvector& v)
-{
- simdscalar m;
- simdscalar r0;
- simdscalar r1;
-
- m = _simd_load1_ps(pMatrix + 0*4 + 0); // m[row][0]
- r0 = _simd_mul_ps(m, v[0]); // (m00 * v.x)
- m = _simd_load1_ps(pMatrix + 0*4 + 1); // m[row][1]
- r1 = _simd_mul_ps(m, v[1]); // (m1 * v.y)
- r0 = _simd_add_ps(r0, r1); // (m0 * v.x) + (m1 * v.y)
- m = _simd_load1_ps(pMatrix + 0*4 + 2); // m[row][2]
- r1 = _simd_mul_ps(m, v[2]); // (m2 * v.z)
- r0 = _simd_add_ps(r0, r1); // (m0 * v.x) + (m1 * v.y) + (m2 * v.z)
- m = _simd_load1_ps(pMatrix + 0*4 + 3); // m[row][3]
- r0 = _simd_add_ps(r0, m); // (m0 * v.x) + (m1 * v.y) + (m2 * v.z) + (m2 * 1)
- result[0] = r0;
-
- m = _simd_load1_ps(pMatrix + 1*4 + 0); // m[row][0]
- r0 = _simd_mul_ps(m, v[0]); // (m00 * v.x)
- m = _simd_load1_ps(pMatrix + 1*4 + 1); // m[row][1]
- r1 = _simd_mul_ps(m, v[1]); // (m1 * v.y)
- r0 = _simd_add_ps(r0, r1); // (m0 * v.x) + (m1 * v.y)
- m = _simd_load1_ps(pMatrix + 1*4 + 2); // m[row][2]
- r1 = _simd_mul_ps(m, v[2]); // (m2 * v.z)
- r0 = _simd_add_ps(r0, r1); // (m0 * v.x) + (m1 * v.y) + (m2 * v.z)
- m = _simd_load1_ps(pMatrix + 1*4 + 3); // m[row][3]
- r0 = _simd_add_ps(r0, m); // (m0 * v.x) + (m1 * v.y) + (m2 * v.z) + (m2 * 1)
- result[1] = r0;
-
- m = _simd_load1_ps(pMatrix + 2*4 + 0); // m[row][0]
- r0 = _simd_mul_ps(m, v[0]); // (m00 * v.x)
- m = _simd_load1_ps(pMatrix + 2*4 + 1); // m[row][1]
- r1 = _simd_mul_ps(m, v[1]); // (m1 * v.y)
- r0 = _simd_add_ps(r0, r1); // (m0 * v.x) + (m1 * v.y)
- m = _simd_load1_ps(pMatrix + 2*4 + 2); // m[row][2]
- r1 = _simd_mul_ps(m, v[2]); // (m2 * v.z)
- r0 = _simd_add_ps(r0, r1); // (m0 * v.x) + (m1 * v.y) + (m2 * v.z)
- m = _simd_load1_ps(pMatrix + 2*4 + 3); // m[row][3]
- r0 = _simd_add_ps(r0, m); // (m0 * v.x) + (m1 * v.y) + (m2 * v.z) + (m2 * 1)
- result[2] = r0;
-
- m = _simd_load1_ps(pMatrix + 3*4 + 0); // m[row][0]
- r0 = _simd_mul_ps(m, v[0]); // (m00 * v.x)
- m = _simd_load1_ps(pMatrix + 3*4 + 1); // m[row][1]
- r1 = _simd_mul_ps(m, v[1]); // (m1 * v.y)
- r0 = _simd_add_ps(r0, r1); // (m0 * v.x) + (m1 * v.y)
- m = _simd_load1_ps(pMatrix + 3*4 + 2); // m[row][2]
- r1 = _simd_mul_ps(m, v[2]); // (m2 * v.z)
- r0 = _simd_add_ps(r0, r1); // (m0 * v.x) + (m1 * v.y) + (m2 * v.z)
- m = _simd_load1_ps(pMatrix + 3*4 + 3); // m[row][3]
- result[3] = _simd_add_ps(r0, m); // (m0 * v.x) + (m1 * v.y) + (m2 * v.z) + (m2 * 1)
-}
-
-INLINE
-void _simd_mat4x3_vec3_w1_multiply(
- simdvector& result,
- const float *pMatrix,
- const simdvector& v)
-{
- simdscalar m;
- simdscalar r0;
- simdscalar r1;
-
- m = _simd_load1_ps(pMatrix + 0*4 + 0); // m[row][0]
- r0 = _simd_mul_ps(m, v[0]); // (m00 * v.x)
- m = _simd_load1_ps(pMatrix + 0*4 + 1); // m[row][1]
- r1 = _simd_mul_ps(m, v[1]); // (m1 * v.y)
- r0 = _simd_add_ps(r0, r1); // (m0 * v.x) + (m1 * v.y)
- m = _simd_load1_ps(pMatrix + 0*4 + 2); // m[row][2]
- r1 = _simd_mul_ps(m, v[2]); // (m2 * v.z)
- r0 = _simd_add_ps(r0, r1); // (m0 * v.x) + (m1 * v.y) + (m2 * v.z)
- m = _simd_load1_ps(pMatrix + 0*4 + 3); // m[row][3]
- r0 = _simd_add_ps(r0, m); // (m0 * v.x) + (m1 * v.y) + (m2 * v.z) + (m2 * 1)
- result[0] = r0;
-
- m = _simd_load1_ps(pMatrix + 1*4 + 0); // m[row][0]
- r0 = _simd_mul_ps(m, v[0]); // (m00 * v.x)
- m = _simd_load1_ps(pMatrix + 1*4 + 1); // m[row][1]
- r1 = _simd_mul_ps(m, v[1]); // (m1 * v.y)
- r0 = _simd_add_ps(r0, r1); // (m0 * v.x) + (m1 * v.y)
- m = _simd_load1_ps(pMatrix + 1*4 + 2); // m[row][2]
- r1 = _simd_mul_ps(m, v[2]); // (m2 * v.z)
- r0 = _simd_add_ps(r0, r1); // (m0 * v.x) + (m1 * v.y) + (m2 * v.z)
- m = _simd_load1_ps(pMatrix + 1*4 + 3); // m[row][3]
- r0 = _simd_add_ps(r0, m); // (m0 * v.x) + (m1 * v.y) + (m2 * v.z) + (m2 * 1)
- result[1] = r0;
-
- m = _simd_load1_ps(pMatrix + 2*4 + 0); // m[row][0]
- r0 = _simd_mul_ps(m, v[0]); // (m00 * v.x)
- m = _simd_load1_ps(pMatrix + 2*4 + 1); // m[row][1]
- r1 = _simd_mul_ps(m, v[1]); // (m1 * v.y)
- r0 = _simd_add_ps(r0, r1); // (m0 * v.x) + (m1 * v.y)
- m = _simd_load1_ps(pMatrix + 2*4 + 2); // m[row][2]
- r1 = _simd_mul_ps(m, v[2]); // (m2 * v.z)
- r0 = _simd_add_ps(r0, r1); // (m0 * v.x) + (m1 * v.y) + (m2 * v.z)
- m = _simd_load1_ps(pMatrix + 2*4 + 3); // m[row][3]
- r0 = _simd_add_ps(r0, m); // (m0 * v.x) + (m1 * v.y) + (m2 * v.z) + (m2 * 1)
- result[2] = r0;
- result[3] = _simd_set1_ps(1.0f);
-}
+#define _simdvec_dp3_ps SIMD::vec4_dp3_ps
+#define _simdvec_dp4_ps SIMD::vec4_dp4_ps
+#define _simdvec_rcp_length_ps SIMD::vec4_rcp_length_ps
+#define _simdvec_normalize_ps SIMD::vec4_normalize_ps
+#define _simdvec_mul_ps SIMD::vec4_mul_ps
+#define _simdvec_add_ps SIMD::vec4_add_ps
+#define _simdvec_min_ps SIMD::vec4_min_ps
+#define _simdvec_max_ps SIMD::vec4_max_ps
+#define _simd_mat4x4_vec4_multiply SIMD::mat4x4_vec4_multiply
+#define _simd_mat3x3_vec3_w0_multiply SIMD::mat3x3_vec3_w0_multiply
+#define _simd_mat4x4_vec3_w1_multiply SIMD::mat4x4_vec3_w1_multiply
+#define _simd_mat4x3_vec3_w1_multiply SIMD::mat4x3_vec3_w1_multiply
//////////////////////////////////////////////////////////////////////////
/// @brief Compute plane equation vA * vX + vB * vY + vC
-INLINE simdscalar vplaneps(simdscalar vA, simdscalar vB, simdscalar vC, simdscalar &vX, simdscalar &vY)
+SIMDINLINE simdscalar vplaneps(simdscalar vA, simdscalar vB, simdscalar vC, simdscalar &vX, simdscalar &vY)
{
simdscalar vOut = _simd_fmadd_ps(vA, vX, vC);
vOut = _simd_fmadd_ps(vB, vY, vOut);
//////////////////////////////////////////////////////////////////////////
/// @brief Compute plane equation vA * vX + vB * vY + vC
-INLINE __m128 vplaneps128(__m128 vA, __m128 vB, __m128 vC, __m128 &vX, __m128 &vY)
+SIMDINLINE simd4scalar vplaneps(simd4scalar vA, simd4scalar vB, simd4scalar vC, simd4scalar &vX, simd4scalar &vY)
{
- __m128 vOut = _simd128_fmadd_ps(vA, vX, vC);
+ simd4scalar vOut = _simd128_fmadd_ps(vA, vX, vC);
vOut = _simd128_fmadd_ps(vB, vY, vOut);
return vOut;
}
/// @param vJ - barycentric J
/// @param pInterpBuffer - pointer to attribute barycentric coeffs
template<UINT Attrib, UINT Comp, UINT numComponents = 4>
-static INLINE simdscalar InterpolateComponent(simdscalar vI, simdscalar vJ, const float *pInterpBuffer)
+static SIMDINLINE simdscalar InterpolateComponent(simdscalar vI, simdscalar vJ, const float *pInterpBuffer)
{
const float *pInterpA = &pInterpBuffer[Attrib * 3 * numComponents + 0 + Comp];
const float *pInterpB = &pInterpBuffer[Attrib * 3 * numComponents + numComponents + Comp];
/// @brief Interpolates a single component (flat shade).
/// @param pInterpBuffer - pointer to attribute barycentric coeffs
template<UINT Attrib, UINT Comp, UINT numComponents = 4>
-static INLINE simdscalar InterpolateComponentFlat(const float *pInterpBuffer)
+static SIMDINLINE simdscalar InterpolateComponentFlat(const float *pInterpBuffer)
{
const float *pInterpA = &pInterpBuffer[Attrib * 3 * numComponents + 0 + Comp];
/// @param vJ - barycentric J
/// @param pInterpBuffer - pointer to attribute barycentric coeffs
template<UINT Attrib, UINT Comp, UINT numComponents = 4>
-static INLINE __m128 InterpolateComponent(__m128 vI, __m128 vJ, const float *pInterpBuffer)
+static SIMDINLINE simd4scalar InterpolateComponent(simd4scalar vI, simd4scalar vJ, const float *pInterpBuffer)
{
const float *pInterpA = &pInterpBuffer[Attrib * 3 * numComponents + 0 + Comp];
const float *pInterpB = &pInterpBuffer[Attrib * 3 * numComponents + numComponents + Comp];
const float *pInterpC = &pInterpBuffer[Attrib * 3 * numComponents + numComponents * 2 + Comp];
- __m128 vA = _mm_broadcast_ss(pInterpA);
- __m128 vB = _mm_broadcast_ss(pInterpB);
- __m128 vC = _mm_broadcast_ss(pInterpC);
+ simd4scalar vA = SIMD128::broadcast_ss(pInterpA);
+ simd4scalar vB = SIMD128::broadcast_ss(pInterpB);
+ simd4scalar vC = SIMD128::broadcast_ss(pInterpC);
- __m128 vk = _mm_sub_ps(_mm_sub_ps(_mm_set1_ps(1.0f), vI), vJ);
- vC = _mm_mul_ps(vk, vC);
+ simd4scalar vk = SIMD128::sub_ps(SIMD128::sub_ps(SIMD128::set1_ps(1.0f), vI), vJ);
+ vC = SIMD128::mul_ps(vk, vC);
- return vplaneps128(vA, vB, vC, vI, vJ);
+ return vplaneps(vA, vB, vC, vI, vJ);
}
-static INLINE __m128 _simd128_abs_ps(__m128 a)
+static SIMDINLINE simd4scalar _simd128_abs_ps(simd4scalar a)
{
- __m128i ai = _mm_castps_si128(a);
- return _mm_castsi128_ps(_mm_and_si128(ai, _mm_set1_epi32(0x7fffffff)));
+ simd4scalari ai = SIMD128::castps_si(a);
+ return SIMD128::castsi_ps(SIMD128::and_si(ai, SIMD128::set1_epi32(0x7fffffff)));
}
-static INLINE simdscalar _simd_abs_ps(simdscalar a)
+static SIMDINLINE simdscalar _simd_abs_ps(simdscalar a)
{
simdscalari ai = _simd_castps_si(a);
return _simd_castsi_ps(_simd_and_si(ai, _simd_set1_epi32(0x7fffffff)));
}
+
#if ENABLE_AVX512_SIMD16
#include "simd16intrin.h"
#endif//ENABLE_AVX512_SIMD16
projects / mesa.git / blobdiff