From 5ea9a30f50dabe60fe95c9077457915dd5ed52c4 Mon Sep 17 00:00:00 2001 From: Tim Rowley Date: Wed, 17 May 2017 17:39:33 -0500 Subject: [PATCH] swr/rast: SIMD16 FE - fix/use SIMD16 calcDeterminantIntVertical() Stop double pumping the SIMD8 version. Reviewed-by: Bruce Cherniak --- .../swr/rasterizer/common/simd16intrin.h | 22 +++++++ .../swr/rasterizer/common/simdintrin.h | 24 +++++++ .../drivers/swr/rasterizer/core/frontend.h | 62 ++++++------------- 3 files changed, 65 insertions(+), 43 deletions(-) diff --git a/src/gallium/drivers/swr/rasterizer/common/simd16intrin.h b/src/gallium/drivers/swr/rasterizer/common/simd16intrin.h index 84585ffc8fc..e303ce59713 100644 --- a/src/gallium/drivers/swr/rasterizer/common/simd16intrin.h +++ b/src/gallium/drivers/swr/rasterizer/common/simd16intrin.h @@ -770,6 +770,26 @@ INLINE simd16scalari SIMDAPI _simd16_cvtepu16_epi32(simdscalari a) return result; } +INLINE simd16scalari SIMDAPI _simd16_cvtepu16_epi64(simdscalari a) +{ + simd16scalari result; + + result.lo = _simd_cvtepu16_epi64(_mm256_extractf128_si256(a, 0)); + result.hi = _simd_cvtepu16_epi64(_mm256_extractf128_si256(a, 1)); + + return result; +} + +INLINE simd16scalari SIMDAPI _simd16_cvtepu32_epi64(simdscalari a) +{ + simd16scalari result; + + result.lo = _simd_cvtepu32_epi64(_mm256_extractf128_si256(a, 0)); + result.hi = _simd_cvtepu32_epi64(_mm256_extractf128_si256(a, 1)); + + return result; +} + SIMD16_EMU_AVX512_2(simd16scalari, _simd16_packus_epi16, _simd_packus_epi16) SIMD16_EMU_AVX512_2(simd16scalari, _simd16_packs_epi16, _simd_packs_epi16) SIMD16_EMU_AVX512_2(simd16scalari, _simd16_packus_epi32, _simd_packus_epi32) @@ -1097,6 +1117,8 @@ INLINE simd16scalari SIMDAPI _simd16_cmpgt_epi8(simd16scalari a, simd16scalari b #define _simd16_cvtepu8_epi16 _mm512_cvtepu8_epi16 #define _simd16_cvtepu8_epi32 _mm512_cvtepu8_epi32 #define _simd16_cvtepu16_epi32 _mm512_cvtepu16_epi32 +#define _simd16_cvtepu16_epi64 _mm512_cvtepu16_epi64 +#define _simd16_cvtepu32_epi64 _mm512_cvtepu32_epi64 #define _simd16_packus_epi16 _mm512_packus_epi16 #define _simd16_packs_epi16 _mm512_packs_epi16 #define _simd16_packus_epi32 _mm512_packus_epi32 diff --git a/src/gallium/drivers/swr/rasterizer/common/simdintrin.h b/src/gallium/drivers/swr/rasterizer/common/simdintrin.h index 61c0c5461a3..ed6e56b5e26 100644 --- a/src/gallium/drivers/swr/rasterizer/common/simdintrin.h +++ b/src/gallium/drivers/swr/rasterizer/common/simdintrin.h @@ -455,6 +455,28 @@ __m256i _simd_cvtepu16_epi32(__m128i a) return _mm256_insertf128_si256(result, resulthi, 1); } +INLINE +__m256i _simd_cvtepu16_epi64(__m128i a) +{ + __m128i resultlo = _mm_cvtepu16_epi64(a); + __m128i resulthi = _mm_cvtepu16_epi64(_mm_srli_si128(a, 4)); + + __m256i result = _mm256_castsi128_si256(resultlo); + + return _mm256_insertf128_si256(result, resulthi, 1); +} + +INLINE +__m256i _simd_cvtepu32_epi64(__m128i a) +{ + __m128i resultlo = _mm_cvtepu32_epi64(a); + __m128i resulthi = _mm_cvtepu32_epi64(_mm_srli_si128(a, 8)); + + __m256i result = _mm256_castsi128_si256(resultlo); + + return _mm256_insertf128_si256(result, resulthi, 1); +} + INLINE __m256i _simd_packus_epi16(__m256i a, __m256i b) { @@ -582,6 +604,8 @@ __m256i _simd_packs_epi32(__m256i a, __m256i b) #define _simd_cvtepu8_epi16 _mm256_cvtepu8_epi16 #define _simd_cvtepu8_epi32 _mm256_cvtepu8_epi32 #define _simd_cvtepu16_epi32 _mm256_cvtepu16_epi32 +#define _simd_cvtepu16_epi64 _mm256_cvtepu16_epi64 +#define _simd_cvtepu32_epi64 _mm256_cvtepu32_epi64 #define _simd_packus_epi16 _mm256_packus_epi16 #define _simd_packs_epi16 _mm256_packs_epi16 #define _simd_packus_epi32 _mm256_packus_epi32 diff --git a/src/gallium/drivers/swr/rasterizer/core/frontend.h b/src/gallium/drivers/swr/rasterizer/core/frontend.h index eedbcfcf7b6..9f347e1e46b 100644 --- a/src/gallium/drivers/swr/rasterizer/core/frontend.h +++ b/src/gallium/drivers/swr/rasterizer/core/frontend.h @@ -162,6 +162,7 @@ INLINE void calcDeterminantIntVertical(const simdscalari vA[3], const simdscalari vB[3], simdscalari *pvDet) { // refer to calcDeterminantInt comment for calculation explanation + // A1*B2 simdscalari vA1Lo = _simd_unpacklo_epi32(vA[1], vA[1]); // 0 0 1 1 4 4 5 5 simdscalari vA1Hi = _simd_unpackhi_epi32(vA[1], vA[1]); // 2 2 3 3 6 6 7 7 @@ -186,8 +187,10 @@ void calcDeterminantIntVertical(const simdscalari vA[3], const simdscalari vB[3] simdscalari detLo = _simd_sub_epi64(vA1B2Lo, vA2B1Lo); simdscalari detHi = _simd_sub_epi64(vA1B2Hi, vA2B1Hi); - // shuffle 0 1 4 5 -> 0 1 2 3 + // shuffle 0 1 4 5 2 3 6 7 -> 0 1 2 3 simdscalari vResultLo = _simd_permute2f128_si(detLo, detHi, 0x20); + + // shuffle 0 1 4 5 2 3 6 7 -> 4 5 6 7 simdscalari vResultHi = _simd_permute2f128_si(detLo, detHi, 0x31); pvDet[0] = vResultLo; @@ -199,57 +202,30 @@ INLINE void calcDeterminantIntVertical(const simd16scalari vA[3], const simd16scalari vB[3], simd16scalari *pvDet) { // refer to calcDeterminantInt comment for calculation explanation - // A1*B2 - -#if 1 - // TODO: get the native SIMD16 version working.. - - simdscalari vA_lo[3]; - simdscalari vA_hi[3]; - simdscalari vB_lo[3]; - simdscalari vB_hi[3]; - - for (uint32_t i = 0; i < 3; i += 1) - { - vA_lo[i] = _simd16_extract_si(vA[i], 0); - vA_hi[i] = _simd16_extract_si(vA[i], 1); - vB_lo[i] = _simd16_extract_si(vB[i], 0); - vB_hi[i] = _simd16_extract_si(vB[i], 1); - } - calcDeterminantIntVertical(vA_lo, vB_lo, reinterpret_cast(&pvDet[0])); - calcDeterminantIntVertical(vA_hi, vB_hi, reinterpret_cast(&pvDet[1])); -#else - simd16scalari vA1Lo = _simd16_unpacklo_epi32(vA[1], vA[1]); // 0 0 1 1 4 4 5 5 8 8 9 9 C C D D - simd16scalari vA1Hi = _simd16_unpackhi_epi32(vA[1], vA[1]); // 2 2 3 3 6 6 7 7 A A B B E E F F + // A1*B2 + simd16scalari vA1_lo = _simd16_cvtepu32_epi64(_simd16_extract_si(vA[1], 0));// 0 1 2 3 4 5 6 7 (64b), upper 32b zero, lower 32b used + simd16scalari vA1_hi = _simd16_cvtepu32_epi64(_simd16_extract_si(vA[1], 1));// 8 9 A B C D E F (64b), upper 32b zero, lower 32b used - simd16scalari vB2Lo = _simd16_unpacklo_epi32(vB[2], vB[2]); - simd16scalari vB2Hi = _simd16_unpackhi_epi32(vB[2], vB[2]); + simd16scalari vB2_lo = _simd16_cvtepu32_epi64(_simd16_extract_si(vB[2], 0));// 0 1 2 3 4 5 6 7 (64b), upper 32b zero, lower 32b used + simd16scalari vB2_hi = _simd16_cvtepu32_epi64(_simd16_extract_si(vB[2], 1));// 8 9 A B C D E F (64b), upper 32b zero, lower 32b used - simd16scalari vA1B2Lo = _simd16_mul_epi32(vA1Lo, vB2Lo); // 0 1 4 5 8 9 C D - simd16scalari vA1B2Hi = _simd16_mul_epi32(vA1Hi, vB2Hi); // 2 3 6 7 A B E F + simd16scalari vA1B2_lo = _simd16_mul_epi32(vA1_lo, vB2_lo); // 0 1 2 3 4 5 6 7 (64b) + simd16scalari vA1B2_hi = _simd16_mul_epi32(vA1_hi, vB2_hi); // 8 9 A B C D E F (64b) // B1*A2 - simd16scalari vA2Lo = _simd16_unpacklo_epi32(vA[2], vA[2]); - simd16scalari vA2Hi = _simd16_unpackhi_epi32(vA[2], vA[2]); + simd16scalari vA2_lo = _simd16_cvtepu32_epi64(_simd16_extract_si(vA[2], 0)); + simd16scalari vA2_hi = _simd16_cvtepu32_epi64(_simd16_extract_si(vA[2], 1)); - simd16scalari vB1Lo = _simd16_unpacklo_epi32(vB[1], vB[1]); - simd16scalari vB1Hi = _simd16_unpackhi_epi32(vB[1], vB[1]); + simd16scalari vB1_lo = _simd16_cvtepu32_epi64(_simd16_extract_si(vB[1], 0)); + simd16scalari vB1_hi = _simd16_cvtepu32_epi64(_simd16_extract_si(vB[1], 1)); - simd16scalari vA2B1Lo = _simd16_mul_epi32(vA2Lo, vB1Lo); - simd16scalari vA2B1Hi = _simd16_mul_epi32(vA2Hi, vB1Hi); + simd16scalari vA2B1_lo = _simd16_mul_epi32(vA2_lo, vB1_lo); + simd16scalari vA2B1_hi = _simd16_mul_epi32(vA2_hi, vB1_hi); // A1*B2 - A2*B1 - simd16scalari detLo = _simd16_sub_epi64(vA1B2Lo, vA2B1Lo); - simd16scalari detHi = _simd16_sub_epi64(vA1B2Hi, vA2B1Hi); - - // shuffle 0 1 4 5 -> 0 1 2 3 - simd16scalari vResultLo = _simd16_permute2f128_si(detLo, detHi, 0x20); - simd16scalari vResultHi = _simd16_permute2f128_si(detLo, detHi, 0x31); - - pvDet[0] = vResultLo; - pvDet[1] = vResultHi; -#endif + pvDet[0] = _simd16_sub_epi64(vA1B2_lo, vA2B1_lo); // 0 1 2 3 4 5 6 7 (64b) + pvDet[1] = _simd16_sub_epi64(vA1B2_hi, vA2B1_hi); // 8 9 A B C D E F (64b) } #endif -- 2.30.2