******************************************************************************/
#pragma once
+#include "utils.h"
+#include "common/simdintrin.h"
+
//////////////////////////////////////////////////////////////////////////
/// PackTraits - Helpers for packing / unpacking same pixel sizes
//////////////////////////////////////////////////////////////////////////
static void storeSOA(uint8_t *pDst, simdscalar src) = delete;
static simdscalar unpack(simdscalar &in) = delete;
static simdscalar pack(simdscalar &in) = delete;
+#if ENABLE_AVX512_SIMD16
+ static simd16scalar loadSOA_16(const uint8_t *pSrc) = delete;
+ static void SIMDCALL storeSOA(uint8_t *pDst, simd16scalar src) = delete;
+ static simd16scalar unpack(simd16scalar &in) = delete;
+ static simd16scalar pack(simd16scalar &in) = delete;
+#endif
};
//////////////////////////////////////////////////////////////////////////
static void storeSOA(uint8_t *pDst, simdscalar src) { return; }
static simdscalar unpack(simdscalar &in) { return _simd_setzero_ps(); }
static simdscalar pack(simdscalar &in) { return _simd_setzero_ps(); }
+#if ENABLE_AVX512_SIMD16
+ static simd16scalar loadSOA_16(const uint8_t *pSrc) { return _simd16_setzero_ps(); }
+ static void SIMDCALL storeSOA(uint8_t *pDst, simd16scalar src) { return; }
+ static simd16scalar unpack(simd16scalar &in) { return _simd16_setzero_ps(); }
+ static simd16scalar pack(simd16scalar &in) { return _simd16_setzero_ps(); }
+#endif
};
-
//////////////////////////////////////////////////////////////////////////
/// PackTraits - Helpers for packing / unpacking 8 bit unsigned channels
//////////////////////////////////////////////////////////////////////////
static simdscalar unpack(simdscalar &in)
{
#if KNOB_SIMD_WIDTH == 8
-#if KNOB_ARCH==KNOB_ARCH_AVX
+#if KNOB_ARCH <= KNOB_ARCH_AVX
__m128i src = _mm_castps_si128(_mm256_castps256_ps128(in));
__m128i resLo = _mm_cvtepu8_epi32(src);
__m128i resHi = _mm_shuffle_epi8(src,
__m256i result = _mm256_castsi128_si256(resLo);
result = _mm256_insertf128_si256(result, resHi, 1);
- return _mm256_castsi256_ps(result);
-#elif KNOB_ARCH==KNOB_ARCH_AVX2
+ return simdscalar{ _mm256_castsi256_ps(result) };
+#else
return _mm256_castsi256_ps(_mm256_cvtepu8_epi32(_mm_castps_si128(_mm256_castps256_ps128(in))));
#endif
#else
#error Unsupported vector width
#endif
}
+#if ENABLE_AVX512_SIMD16
+
+ static simd16scalar loadSOA_16(const uint8_t *pSrc)
+ {
+ simd16scalar result = _simd16_setzero_ps();
+ simdscalar resultlo = _simd_setzero_ps();
+
+ const __m128 src = _mm_load_ps(reinterpret_cast<const float *>(pSrc));
+
+ resultlo = _mm256_insertf128_ps(resultlo, src, 0);
+ result = _simd16_insert_ps(result, resultlo, 0);
+
+ return result;
+ }
+
+ static void SIMDCALL storeSOA(uint8_t *pDst, simd16scalar src)
+ {
+ // store simd16 bytes
+ _mm_store_ps(reinterpret_cast<float *>(pDst), _mm256_castps256_ps128(_simd16_extract_ps(src, 0)));
+ }
+
+ static simd16scalar unpack(simd16scalar &in)
+ {
+ simd4scalari tmp = _mm_castps_si128(_mm256_castps256_ps128(_simd16_extract_ps(in, 0)));
+ simd16scalari result = _simd16_cvtepu8_epi32(tmp);
+
+ return _simd16_castsi_ps(result);
+ }
+
+ static simd16scalar pack(simd16scalar &in)
+ {
+ simd16scalari result = _simd16_setzero_si();
+
+ simdscalari inlo = _simd_castps_si(_simd16_extract_ps(in, 0)); // r0 r1 r2 r3 r4 r5 r6 r7 (32b)
+ simdscalari inhi = _simd_castps_si(_simd16_extract_ps(in, 1)); // r8 r9 rA rB rC rD rE rF
+
+ simdscalari permlo = _simd_permute2f128_si(inlo, inhi, 0x20); // r0 r1 r2 r3 r8 r9 rA rB (32b)
+ simdscalari permhi = _simd_permute2f128_si(inlo, inhi, 0x31); // r4 r5 r6 r7 rC rD rE rF (32b)
+
+ simdscalari pack = _simd_packus_epi32(permlo, permhi); // r0 r1 r2 r3 r4 r5 r6 r7 r8 r9 rA rB rC rD rE rF (16b)
+
+ const simdscalari zero = _simd_setzero_si();
+
+ permlo = _simd_permute2f128_si(pack, zero, 0x20); // (2, 0) // r0 r1 r2 r3 r4 r5 r6 r7 00 00 00 00 00 00 00 00 (16b)
+ permhi = _simd_permute2f128_si(pack, zero, 0x31); // (3, 1) // r8 r9 rA rB rC rD rE rF 00 00 00 00 00 00 00 00 (16b)
+
+ pack = _simd_packus_epi16(permlo, permhi); // r0 r1 r2 r3 r4 r5 r6 r7 r8 r9 rA rB rC rD rE rF 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 (8b)
+
+ result = _simd16_insert_si(result, pack, 0);
+
+ return _simd16_castsi_ps(result);
+ }
+#endif
};
//////////////////////////////////////////////////////////////////////////
static simdscalar unpack(simdscalar &in)
{
#if KNOB_SIMD_WIDTH == 8
-#if KNOB_ARCH==KNOB_ARCH_AVX
- SWR_ASSERT(0); // I think this may be incorrect.
+#if KNOB_ARCH <= KNOB_ARCH_AVX
+ SWR_INVALID("I think this may be incorrect.");
__m128i src = _mm_castps_si128(_mm256_castps256_ps128(in));
__m128i resLo = _mm_cvtepi8_epi32(src);
__m128i resHi = _mm_shuffle_epi8(src,
__m256i result = _mm256_castsi128_si256(resLo);
result = _mm256_insertf128_si256(result, resHi, 1);
return _mm256_castsi256_ps(result);
-#elif KNOB_ARCH==KNOB_ARCH_AVX2
+#else
return _mm256_castsi256_ps(_mm256_cvtepi8_epi32(_mm_castps_si128(_mm256_castps256_ps128(in))));
#endif
#else
#error Unsupported vector width
#endif
}
+#if ENABLE_AVX512_SIMD16
+
+ static simd16scalar loadSOA_16(const uint8_t *pSrc)
+ {
+ simd16scalar result = _simd16_setzero_ps();
+ simdscalar resultlo = _simd_setzero_ps();
+
+ const __m128 src = _mm_load_ps(reinterpret_cast<const float *>(pSrc));
+
+ resultlo = _mm256_insertf128_ps(resultlo, src, 0);
+ result = _simd16_insert_ps(result, resultlo, 0);
+
+ return result;
+ }
+
+ static void SIMDCALL storeSOA(uint8_t *pDst, simd16scalar src)
+ {
+ // store simd16 bytes
+ _mm_store_ps(reinterpret_cast<float *>(pDst), _mm256_castps256_ps128(_simd16_extract_ps(src, 0)));
+ }
+
+ static simd16scalar unpack(simd16scalar &in)
+ {
+ simd4scalari tmp = _mm_castps_si128(_mm256_castps256_ps128(_simd16_extract_ps(in, 0)));
+ simd16scalari result = _simd16_cvtepu8_epi32(tmp);
+
+ return _simd16_castsi_ps(result);
+ }
+
+ static simd16scalar pack(simd16scalar &in)
+ {
+ simd16scalari result = _simd16_setzero_si();
+
+ simdscalari inlo = _simd_castps_si(_simd16_extract_ps(in, 0)); // r0 r1 r2 r3 r4 r5 r6 r7 (32b)
+ simdscalari inhi = _simd_castps_si(_simd16_extract_ps(in, 1)); // r8 r9 rA rB rC rD rE rF
+
+ simdscalari permlo = _simd_permute2f128_si(inlo, inhi, 0x20); // r0 r1 r2 r3 r8 r9 rA rB (32b)
+ simdscalari permhi = _simd_permute2f128_si(inlo, inhi, 0x31); // r4 r5 r6 r7 rC rD rE rF (32b)
+
+ simdscalari pack = _simd_packs_epi32(permlo, permhi); // r0 r1 r2 r3 r4 r5 r6 r7 r8 r9 rA rB rC rD rE rF (16b)
+
+ const simdscalari zero = _simd_setzero_si();
+
+ permlo = _simd_permute2f128_si(pack, zero, 0x20); // (2, 0) // r0 r1 r2 r3 r4 r5 r6 r7 00 00 00 00 00 00 00 00 (16b)
+ permhi = _simd_permute2f128_si(pack, zero, 0x31); // (3, 1) // r8 r9 rA rB rC rD rE rF 00 00 00 00 00 00 00 00 (16b)
+
+ pack = _simd_packs_epi16(permlo, permhi); // r0 r1 r2 r3 r4 r5 r6 r7 r8 r9 rA rB rC rD rE rF 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 (8b)
+
+ result = _simd16_insert_si(result, pack, 0);
+
+ return _simd16_castsi_ps(result);
+ }
+#endif
};
//////////////////////////////////////////////////////////////////////////
static simdscalar unpack(simdscalar &in)
{
#if KNOB_SIMD_WIDTH == 8
-#if KNOB_ARCH==KNOB_ARCH_AVX
+#if KNOB_ARCH <= KNOB_ARCH_AVX
__m128i src = _mm_castps_si128(_mm256_castps256_ps128(in));
__m128i resLo = _mm_cvtepu16_epi32(src);
__m128i resHi = _mm_shuffle_epi8(src,
__m256i result = _mm256_castsi128_si256(resLo);
result = _mm256_insertf128_si256(result, resHi, 1);
return _mm256_castsi256_ps(result);
-#elif KNOB_ARCH==KNOB_ARCH_AVX2
+#else
return _mm256_castsi256_ps(_mm256_cvtepu16_epi32(_mm_castps_si128(_mm256_castps256_ps128(in))));
#endif
#else
#error Unsupported vector width
#endif
}
+#if ENABLE_AVX512_SIMD16
+
+ static simd16scalar loadSOA_16(const uint8_t *pSrc)
+ {
+ simd16scalar result = _simd16_setzero_ps();
+
+ simdscalar resultlo = _simd_load_ps(reinterpret_cast<const float *>(pSrc));
+
+ result = _simd16_insert_ps(result, resultlo, 0);
+
+ return result;
+ }
+
+ static void SIMDCALL storeSOA(uint8_t *pDst, simd16scalar src)
+ {
+ _simd_store_ps(reinterpret_cast<float *>(pDst), _simd16_extract_ps(src, 0));
+ }
+
+ static simd16scalar unpack(simd16scalar &in)
+ {
+ simd16scalari result = _simd16_cvtepu16_epi32(_simd_castps_si(_simd16_extract_ps(in, 0)));
+
+ return _simd16_castsi_ps(result);
+ }
+
+ static simd16scalar pack(simd16scalar &in)
+ {
+ const simd16scalari zero = _simd16_setzero_si();
+
+ simd16scalari permlo = _simd16_permute2f128_si(_simd16_castps_si(in), zero, 0x08); // (0, 0, 2, 0) // r0 r1 r2 r3 r8 r9 rA rB 00 00 00 00 00 00 00 00 (32b)
+ simd16scalari permhi = _simd16_permute2f128_si(_simd16_castps_si(in), zero, 0x0D); // (0, 0, 3, 1) // r4 r5 r6 r7 rC rD rE rF 00 00 00 00 00 00 00 00
+
+ simd16scalari result = _simd16_packus_epi32(permlo, permhi); // r0 r1 r2 r3 r4 r5 r6 r7 r8 r9 rA rB rC rD rE rF 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 (16b)
+
+ return _simd16_castsi_ps(result);
+ }
+#endif
};
//////////////////////////////////////////////////////////////////////////
static simdscalar unpack(simdscalar &in)
{
#if KNOB_SIMD_WIDTH == 8
-#if KNOB_ARCH==KNOB_ARCH_AVX
- SWR_ASSERT(0); // I think this is incorrectly implemented
+#if KNOB_ARCH <= KNOB_ARCH_AVX
+ SWR_INVALID("I think this may be incorrect.");
__m128i src = _mm_castps_si128(_mm256_castps256_ps128(in));
__m128i resLo = _mm_cvtepi16_epi32(src);
__m128i resHi = _mm_shuffle_epi8(src,
__m256i result = _mm256_castsi128_si256(resLo);
result = _mm256_insertf128_si256(result, resHi, 1);
return _mm256_castsi256_ps(result);
-#elif KNOB_ARCH==KNOB_ARCH_AVX2
+#else
return _mm256_castsi256_ps(_mm256_cvtepi16_epi32(_mm_castps_si128(_mm256_castps256_ps128(in))));
#endif
#else
#error Unsupported vector width
#endif
}
+#if ENABLE_AVX512_SIMD16
+
+ static simd16scalar loadSOA_16(const uint8_t *pSrc)
+ {
+ simd16scalar result = _simd16_setzero_ps();
+
+ simdscalar resultlo = _simd_load_ps(reinterpret_cast<const float *>(pSrc));
+
+ result = _simd16_insert_ps(result, resultlo, 0);
+
+ return result;
+ }
+
+ static void SIMDCALL storeSOA(uint8_t *pDst, simd16scalar src)
+ {
+ _simd_store_ps(reinterpret_cast<float *>(pDst), _simd16_extract_ps(src, 0));
+ }
+
+ static simd16scalar unpack(simd16scalar &in)
+ {
+ simd16scalari result = _simd16_cvtepu16_epi32(_simd_castps_si(_simd16_extract_ps(in, 0)));
+
+ return _simd16_castsi_ps(result);
+ }
+
+ static simd16scalar pack(simd16scalar &in)
+ {
+ const simd16scalari zero = _simd16_setzero_si();
+
+ simd16scalari permlo = _simd16_permute2f128_si(_simd16_castps_si(in), zero, 0x08); // (0, 0, 2, 0) // r0 r1 r2 r3 r8 r9 rA rB 00 00 00 00 00 00 00 00 (32b)
+ simd16scalari permhi = _simd16_permute2f128_si(_simd16_castps_si(in), zero, 0x0D); // (0, 0, 3, 1) // r4 r5 r6 r7 rC rD rE rF 00 00 00 00 00 00 00 00
+
+ simd16scalari result = _simd16_packs_epi32(permlo, permhi); // r0 r1 r2 r3 r4 r5 r6 r7 r8 r9 rA rB rC rD rE rF 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 (16b)
+
+ return _simd16_castsi_ps(result);
+ }
+#endif
};
//////////////////////////////////////////////////////////////////////////
static void storeSOA(uint8_t *pDst, simdscalar src) { _simd_store_ps((float*)pDst, src); }
static simdscalar unpack(simdscalar &in) { return in; }
static simdscalar pack(simdscalar &in) { return in; }
+#if ENABLE_AVX512_SIMD16
+
+ static simd16scalar loadSOA_16(const uint8_t *pSrc)
+ {
+ return _simd16_load_ps(reinterpret_cast<const float *>(pSrc));
+ }
+
+ static void SIMDCALL storeSOA(uint8_t *pDst, simd16scalar src)
+ {
+ _simd16_store_ps(reinterpret_cast<float *>(pDst), src);
+ }
+
+ static simd16scalar unpack(simd16scalar &in)
+ {
+ return in;
+ }
+
+ static simd16scalar pack(simd16scalar &in)
+ {
+ return in;
+ }
+#endif
};
//////////////////////////////////////////////////////////////////////////
{
static const SWR_TYPE MyType = type;
static float toFloat() { return 0.0; }
- static float fromFloat() { SWR_ASSERT(0); return 0.0; }
- static simdscalar convertSrgb(simdscalar &in) { SWR_ASSERT(0); return _simd_setzero_ps(); }
+ static float fromFloat() { SWR_NOT_IMPL; return 0.0; }
+ static simdscalar convertSrgb(simdscalar &in) { SWR_NOT_IMPL; return _simd_setzero_ps(); }
};
//////////////////////////////////////////////////////////////////////////
{
static const SWR_TYPE MyType = SWR_TYPE_UINT;
static float toFloat() { return 0.0; }
- static float fromFloat() { SWR_ASSERT(0); return 0.0; }
- static simdscalar convertSrgb(simdscalar &in) { SWR_ASSERT(0); return _simd_setzero_ps(); }
+ static float fromFloat() { SWR_NOT_IMPL; return 0.0; }
+ static simdscalar convertSrgb(simdscalar &in) { SWR_NOT_IMPL; return _simd_setzero_ps(); }
};
//////////////////////////////////////////////////////////////////////////
{
static const SWR_TYPE MyType = SWR_TYPE_SINT;
static float toFloat() { return 0.0; }
- static float fromFloat() { SWR_ASSERT(0); return 0.0; }
- static simdscalar convertSrgb(simdscalar &in) { SWR_ASSERT(0); return _simd_setzero_ps(); }
+ static float fromFloat() { SWR_NOT_IMPL; return 0.0; }
+ static simdscalar convertSrgb(simdscalar &in) { SWR_NOT_IMPL; return _simd_setzero_ps(); }
};
//////////////////////////////////////////////////////////////////////////
{
static const SWR_TYPE MyType = SWR_TYPE_UINT;
static float toFloat() { return 0.0; }
- static float fromFloat() { SWR_ASSERT(0); return 0.0; }
- static simdscalar convertSrgb(simdscalar &in) { SWR_ASSERT(0); return _simd_setzero_ps(); }
+ static float fromFloat() { SWR_NOT_IMPL; return 0.0; }
+ static simdscalar convertSrgb(simdscalar &in) { SWR_NOT_IMPL; return _simd_setzero_ps(); }
};
//////////////////////////////////////////////////////////////////////////
{
static const SWR_TYPE MyType = SWR_TYPE_SINT;
static float toFloat() { return 0.0; }
- static float fromFloat() { SWR_ASSERT(0); return 0.0; }
- static simdscalar convertSrgb(simdscalar &in) { SWR_ASSERT(0); return _simd_setzero_ps(); }
+ static float fromFloat() { SWR_NOT_IMPL; return 0.0; }
+ static simdscalar convertSrgb(simdscalar &in) { SWR_NOT_IMPL; return _simd_setzero_ps(); }
};
//////////////////////////////////////////////////////////////////////////
{
static const SWR_TYPE MyType = SWR_TYPE_UINT;
static float toFloat() { return 0.0; }
- static float fromFloat() { SWR_ASSERT(0); return 0.0; }
- static simdscalar convertSrgb(simdscalar &in) { SWR_ASSERT(0); return _simd_setzero_ps(); }
+ static float fromFloat() { SWR_NOT_IMPL; return 0.0; }
+ static simdscalar convertSrgb(simdscalar &in) { SWR_NOT_IMPL; return _simd_setzero_ps(); }
};
//////////////////////////////////////////////////////////////////////////
{
static const SWR_TYPE MyType = SWR_TYPE_SINT;
static float toFloat() { return 0.0; }
- static float fromFloat() { SWR_ASSERT(0); return 0.0; }
- static simdscalar convertSrgb(simdscalar &in) { SWR_ASSERT(0); return _simd_setzero_ps(); }
+ static float fromFloat() { SWR_NOT_IMPL; return 0.0; }
+ static simdscalar convertSrgb(simdscalar &in) { SWR_NOT_IMPL; return _simd_setzero_ps(); }
};
//////////////////////////////////////////////////////////////////////////
static const SWR_TYPE MyType = SWR_TYPE_UNORM;
static float toFloat() { return 1.0f / 31.0f; }
static float fromFloat() { return 31.0f; }
- static simdscalar convertSrgb(simdscalar &in) { SWR_ASSERT(0); return _simd_setzero_ps(); }
+ static simdscalar convertSrgb(simdscalar &in) { SWR_NOT_IMPL; return _simd_setzero_ps(); }
};
//////////////////////////////////////////////////////////////////////////
static const SWR_TYPE MyType = SWR_TYPE_UNORM;
static float toFloat() { return 1.0f / 63.0f; }
static float fromFloat() { return 63.0f; }
- static simdscalar convertSrgb(simdscalar &in) { SWR_ASSERT(0); return _simd_setzero_ps(); }
+ static simdscalar convertSrgb(simdscalar &in) { SWR_NOT_IMPL; return _simd_setzero_ps(); }
};
//////////////////////////////////////////////////////////////////////////
static const SWR_TYPE MyType = SWR_TYPE_UNORM;
static float toFloat() { return 1.0f / 255.0f; }
static float fromFloat() { return 255.0f; }
- static simdscalar convertSrgb(simdscalar &in) { SWR_ASSERT(0); return _simd_setzero_ps(); }
+ static simdscalar convertSrgb(simdscalar &in) { SWR_NOT_IMPL; return _simd_setzero_ps(); }
};
//////////////////////////////////////////////////////////////////////////
static const SWR_TYPE MyType = SWR_TYPE_SNORM;
static float toFloat() { return 1.0f / 127.0f; }
static float fromFloat() { return 127.0f; }
- static simdscalar convertSrgb(simdscalar &in) { SWR_ASSERT(0); return _simd_setzero_ps(); }
+ static simdscalar convertSrgb(simdscalar &in) { SWR_NOT_IMPL; return _simd_setzero_ps(); }
};
//////////////////////////////////////////////////////////////////////////
static const SWR_TYPE MyType = SWR_TYPE_UNORM;
static float toFloat() { return 1.0f / 65535.0f; }
static float fromFloat() { return 65535.0f; }
- static simdscalar convertSrgb(simdscalar &in) { SWR_ASSERT(0); return _simd_setzero_ps(); }
+ static simdscalar convertSrgb(simdscalar &in) { SWR_NOT_IMPL; return _simd_setzero_ps(); }
};
//////////////////////////////////////////////////////////////////////////
static const SWR_TYPE MyType = SWR_TYPE_UNORM;
static float toFloat() { return 1.0f / 32767.0f; }
static float fromFloat() { return 32767.0f; }
- static simdscalar convertSrgb(simdscalar &in) { SWR_ASSERT(0); return _simd_setzero_ps(); }
+ static simdscalar convertSrgb(simdscalar &in) { SWR_NOT_IMPL; return _simd_setzero_ps(); }
};
//////////////////////////////////////////////////////////////////////////
static const SWR_TYPE MyType = SWR_TYPE_UNORM;
static float toFloat() { return 1.0f / 16777215.0f; }
static float fromFloat() { return 16777215.0f; }
- static simdscalar convertSrgb(simdscalar &in) { SWR_ASSERT(0); return _simd_setzero_ps(); }
+ static simdscalar convertSrgb(simdscalar &in) { SWR_NOT_IMPL; return _simd_setzero_ps(); }
};
//////////////////////////////////////////////////////////////////////////
{
float *f = (float *)(&Base);
- return _mm_set_ps(powf(f[0], Exp),
- powf(f[1], Exp),
+ return _mm_set_ps(powf(f[3], Exp),
powf(f[2], Exp),
- powf(f[3], Exp));
+ powf(f[1], Exp),
+ powf(f[0], Exp));
}
static inline __m128 ConvertFloatToSRGB2(__m128& Src)
return Result;
}
+#if ENABLE_AVX512_SIMD16
+template< unsigned expnum, unsigned expden, unsigned coeffnum, unsigned coeffden >
+inline static simd16scalar SIMDCALL fastpow(simd16scalar value)
+{
+ static const float factor1 = exp2(127.0f * expden / expnum - 127.0f)
+ * powf(1.0f * coeffnum / coeffden, 1.0f * expden / expnum);
+
+ // Apply a constant pre-correction factor.
+ simd16scalar result = _simd16_mul_ps(value, _simd16_set1_ps(factor1));
+
+ // Reinterpret arg as integer to obtain logarithm.
+ //asm("cvtdq2ps %1, %0" : "=x" (result) : "x" (result));
+ result = _simd16_cvtepi32_ps(_simd16_castps_si(result));
+
+ // Multiply logarithm by power.
+ result = _simd16_mul_ps(result, _simd16_set1_ps(1.0f * expnum / expden));
+
+ // Convert back to "integer" to exponentiate.
+ //asm("cvtps2dq %1, %0" : "=x" (result) : "x" (result));
+ result = _simd16_castsi_ps(_simd16_cvtps_epi32(result));
+
+ return result;
+}
+
+inline static simd16scalar SIMDCALL pow512_4(simd16scalar arg)
+{
+ // 5/12 is too small, so compute the 4th root of 20/12 instead.
+ // 20/12 = 5/3 = 1 + 2/3 = 2 - 1/3. 2/3 is a suitable argument for fastpow.
+ // weighting coefficient: a^-1/2 = 2 a; a = 2^-2/3
+ simd16scalar xf = fastpow< 2, 3, int(0.629960524947437 * 1e9), int(1e9) >(arg);
+ simd16scalar xover = _simd16_mul_ps(arg, xf);
+
+ simd16scalar xfm1 = _simd16_rsqrt_ps(xf);
+ simd16scalar x2 = _simd16_mul_ps(arg, arg);
+ simd16scalar xunder = _simd16_mul_ps(x2, xfm1);
+
+ // sqrt2 * over + 2 * sqrt2 * under
+ simd16scalar xavg = _simd16_mul_ps(_simd16_set1_ps(1.0f / (3.0f * 0.629960524947437f) * 0.999852f), _simd16_add_ps(xover, xunder));
+
+ xavg = _simd16_mul_ps(xavg, _simd16_rsqrt_ps(xavg));
+ xavg = _simd16_mul_ps(xavg, _simd16_rsqrt_ps(xavg));
+
+ return xavg;
+}
+
+inline static simd16scalar SIMDCALL powf_wrapper(const simd16scalar base, float exp)
+{
+ const float *f = reinterpret_cast<const float *>(&base);
+
+ return _simd16_set_ps(
+ powf(f[15], exp),
+ powf(f[14], exp),
+ powf(f[13], exp),
+ powf(f[12], exp),
+ powf(f[11], exp),
+ powf(f[10], exp),
+ powf(f[ 9], exp),
+ powf(f[ 8], exp),
+ powf(f[ 7], exp),
+ powf(f[ 6], exp),
+ powf(f[ 5], exp),
+ powf(f[ 4], exp),
+ powf(f[ 3], exp),
+ powf(f[ 2], exp),
+ powf(f[ 1], exp),
+ powf(f[ 0], exp)
+ );
+}
+
+// float to SRGB conversion formula
+//
+// if (value < 0.0031308f)
+// value *= 12.92f;
+// else
+// value = 1.055f * pow(value, 1.0f / 2.4f) - 0.055f;
+//
+static inline simd16scalar ConvertFloatToSRGB2(const simd16scalar &value)
+{
+ // create a mask where the source is < the minimal SRGB float value
+ const simd16mask mask = _simd16_cmplt_ps_mask(value, _simd16_set1_ps(0.0031308f));
+
+ // if all elements are < the threshold, result = value * 12.92
+ simd16scalar result = _simd16_mul_ps(value, _simd16_set1_ps(12.92f));
+
+ if (_simd16_mask2int(mask) != 0xFFFF)
+ {
+ // some elements are >= threshold, result = 1.055 * power(value, 1.0 / 2.4) - 0.055
+#if KNOB_USE_FAST_SRGB == TRUE
+ // 1.0f / 2.4f is 5.0f / 12.0f which is used for approximation.
+ simd16scalar result2 = pow512_4(value);
+#else
+ simd16scalar result2 = powf_wrapper(value, 1.0f / 2.4f);
+#endif
+
+ result2 = _simd16_mul_ps(result2, _simd16_set1_ps(1.055f));
+ result2 = _simd16_sub_ps(result2, _simd16_set1_ps(0.055f));
+
+#if (KNOB_ARCH == KNOB_ARCH_AVX512)
+ // only native AVX512 can directly use the computed mask for the blend operation
+ result = _mm512_mask_blend_ps(mask, result2, result);
+#else
+ result = _simd16_blendv_ps(result2, result, _simd16_cmplt_ps(value, _simd16_set1_ps(0.0031308f)));
+#endif
+ }
+
+ return result;
+}
+
+#endif
//////////////////////////////////////////////////////////////////////////
/// TypeTraits - Format type traits specialization for FLOAT16
//////////////////////////////////////////////////////////////////////////
static const SWR_TYPE MyType = SWR_TYPE_FLOAT;
static float toFloat() { return 1.0f; }
static float fromFloat() { return 1.0f; }
- static simdscalar convertSrgb(simdscalar &in) { SWR_ASSERT(0); return _simd_setzero_ps(); }
+ static simdscalar convertSrgb(simdscalar &in) { SWR_NOT_IMPL; return _simd_setzero_ps(); }
static simdscalar pack(const simdscalar &in)
{
static simdscalar unpack(const simdscalar &in)
{
// input is 8 packed float16, output is 8 packed float32
- SWR_ASSERT(0); // @todo
+ SWR_NOT_IMPL; // @todo
return _simd_setzero_ps();
}
+#if ENABLE_AVX512_SIMD16
+
+ static simd16scalar pack(const simd16scalar &in)
+ {
+ simd16scalari result = _simd16_setzero_si();
+ simdscalari resultlo = _simd_setzero_si();
+
+#if (KNOB_ARCH == KNOB_ARCH_AVX)
+ simdscalar simdlo = pack(_simd16_extract_ps(in, 0));
+ simdscalar simdhi = pack(_simd16_extract_ps(in, 1));
+
+ __m128i templo = _simd_extractf128_si(_simd_castps_si(simdlo), 0);
+ __m128i temphi = _simd_extractf128_si(_simd_castps_si(simdhi), 0);
+
+#else
+ __m128i templo = _mm256_cvtps_ph(_simd16_extract_ps(in, 0), _MM_FROUND_TRUNC);
+ __m128i temphi = _mm256_cvtps_ph(_simd16_extract_ps(in, 1), _MM_FROUND_TRUNC);
+
+#endif
+ resultlo = _simd_insertf128_si(resultlo, templo, 0);
+ resultlo = _simd_insertf128_si(resultlo, temphi, 1);
+
+ result = _simd16_insert_si(result, resultlo, 0);
+
+ return _simd16_castsi_ps(result);
+ }
+
+ static simd16scalar unpack(const simd16scalar &in)
+ {
+ // input is 16 packed float16, output is 16 packed float32
+ SWR_NOT_IMPL; // @todo
+ return _simd16_setzero_ps();
+ }
+#endif
};
//////////////////////////////////////////////////////////////////////////
static float fromFloat() { return 1.0f; }
static inline simdscalar convertSrgb(simdscalar &in)
{
-#if (KNOB_ARCH == KNOB_ARCH_AVX || KNOB_ARCH == KNOB_ARCH_AVX2)
+#if KNOB_SIMD_WIDTH == 8
__m128 srcLo = _mm256_extractf128_ps(in, 0);
__m128 srcHi = _mm256_extractf128_ps(in, 1);
in = _mm256_insertf128_ps(in, srcLo, 0);
in = _mm256_insertf128_ps(in, srcHi, 1);
-
+#else
+#error Unsupported vector width
#endif
return in;
}
-};
+#if ENABLE_AVX512_SIMD16
-//////////////////////////////////////////////////////////////////////////
-/// Format1 - Bitfield for single component formats.
-//////////////////////////////////////////////////////////////////////////
-template<uint32_t x>
-struct Format1
-{
- union
+ static inline simd16scalar convertSrgb(simd16scalar &in)
{
- uint32_t r : x;
-
- ///@ The following are here to provide full template needed in Formats.
- uint32_t g : x;
- uint32_t b : x;
- uint32_t a : x;
- };
+ return ConvertFloatToSRGB2(in);
+ }
+#endif
};
//////////////////////////////////////////////////////////////////////////
-/// Format1 - Bitfield for single component formats - 8 bit specialization
+/// FormatIntType - Calculate base integer type for pixel components based
+/// on total number of bits. Components can be smaller
+/// that this type, but the entire pixel must not be
+/// any smaller than this type.
//////////////////////////////////////////////////////////////////////////
-template<>
-struct Format1<8>
+template <uint32_t bits, bool bits8 = bits <= 8, bool bits16 = bits <= 16>
+struct FormatIntType
{
- union
- {
- uint8_t r;
-
- ///@ The following are here to provide full template needed in Formats.
- uint8_t g;
- uint8_t b;
- uint8_t a;
- };
+ typedef uint32_t TYPE;
};
-//////////////////////////////////////////////////////////////////////////
-/// Format1 - Bitfield for single component formats - 16 bit specialization
-//////////////////////////////////////////////////////////////////////////
-template<>
-struct Format1<16>
+template <uint32_t bits>
+struct FormatIntType<bits, true, true>
{
- union
- {
- uint16_t r;
+ typedef uint8_t TYPE;
+};
- ///@ The following are here to provide full template needed in Formats.
- uint16_t g;
- uint16_t b;
- uint16_t a;
- };
+template <uint32_t bits>
+struct FormatIntType<bits, false, true>
+{
+ typedef uint16_t TYPE;
};
//////////////////////////////////////////////////////////////////////////
-/// Format2 - Bitfield for 2 component formats.
+/// Format1 - Bitfield for single component formats.
//////////////////////////////////////////////////////////////////////////
-template<uint32_t x, uint32_t y>
-union Format2
+template<uint32_t x>
+union Format1
{
+ typedef typename FormatIntType<x>::TYPE TYPE;
struct
{
- uint32_t r : x;
- uint32_t g : y;
+ TYPE r : x;
};
+
+ ///@ The following are here to provide full template needed in Formats.
struct
{
- ///@ The following are here to provide full template needed in Formats.
- uint32_t b : x;
- uint32_t a : y;
+ TYPE g : x;
+ };
+ struct
+ {
+ TYPE b : x;
+ };
+ struct
+ {
+ TYPE a : x;
};
};
//////////////////////////////////////////////////////////////////////////
-/// Format2 - Bitfield for 2 component formats - 16 bit specialization
+/// Format2 - Bitfield for 2 component formats.
//////////////////////////////////////////////////////////////////////////
-template<>
-union Format2<8,8>
+template<uint32_t x, uint32_t y>
+union Format2
{
+ typedef typename FormatIntType<x + y>::TYPE TYPE;
+
struct
{
- uint16_t r : 8;
- uint16_t g : 8;
+ TYPE r : x;
+ TYPE g : y;
};
struct
{
///@ The following are here to provide full template needed in Formats.
- uint16_t b : 8;
- uint16_t a : 8;
+ TYPE b : x;
+ TYPE a : y;
};
};
template<uint32_t x, uint32_t y, uint32_t z>
union Format3
{
- struct
- {
- uint32_t r : x;
- uint32_t g : y;
- uint32_t b : z;
- };
- uint32_t a; ///@note This is here to provide full template needed in Formats.
-};
+ typedef typename FormatIntType<x + y + z>::TYPE TYPE;
-//////////////////////////////////////////////////////////////////////////
-/// Format3 - Bitfield for 3 component formats - 16 bit specialization
-//////////////////////////////////////////////////////////////////////////
-template<>
-union Format3<5,6,5>
-{
struct
{
- uint16_t r : 5;
- uint16_t g : 6;
- uint16_t b : 5;
+ TYPE r : x;
+ TYPE g : y;
+ TYPE b : z;
};
- uint16_t a; ///@note This is here to provide full template needed in Formats.
+ TYPE a; ///@note This is here to provide full template needed in Formats.
};
//////////////////////////////////////////////////////////////////////////
template<uint32_t x, uint32_t y, uint32_t z, uint32_t w>
struct Format4
{
- uint32_t r : x;
- uint32_t g : y;
- uint32_t b : z;
- uint32_t a : w;
-};
+ typedef typename FormatIntType<x + y + z + w>::TYPE TYPE;
-//////////////////////////////////////////////////////////////////////////
-/// Format4 - Bitfield for 4 component formats - 16 bit specialization
-//////////////////////////////////////////////////////////////////////////
-template<>
-struct Format4<5,5,5,1>
-{
- uint16_t r : 5;
- uint16_t g : 5;
- uint16_t b : 5;
- uint16_t a : 1;
-};
-
-//////////////////////////////////////////////////////////////////////////
-/// Format4 - Bitfield for 4 component formats - 16 bit specialization
-//////////////////////////////////////////////////////////////////////////
-template<>
-struct Format4<4,4,4,4>
-{
- uint16_t r : 4;
- uint16_t g : 4;
- uint16_t b : 4;
- uint16_t a : 4;
+ TYPE r : x;
+ TYPE g : y;
+ TYPE b : z;
+ TYPE a : w;
};
//////////////////////////////////////////////////////////////////////////
return CompType[comp];
}
+ INLINE static constexpr uint32_t GetConstBPC(uint32_t comp)
+ {
+ return (comp == 3) ? NumBitsW :
+ ((comp == 2) ? NumBitsZ :
+ ((comp == 1) ? NumBitsY : NumBitsX) );
+ }
+
INLINE static uint32_t GetBPC(uint32_t comp)
{
static const uint32_t MyBpc[4]{ NumBitsX, NumBitsY, NumBitsZ, NumBitsW };
case 3:
return (W == SWR_TYPE_UNORM || W == SWR_TYPE_SNORM) ? true : false;
}
- SWR_ASSERT(0);
+ SWR_INVALID("Invalid component: %d", comp);
return false;
}
case 3:
return TypeTraits<W, NumBitsW>::toFloat();
}
- SWR_ASSERT(0);
+ SWR_INVALID("Invalid component: %d", comp);
return TypeTraits<X, NumBitsX>::toFloat();
}
case 3:
return TypeTraits<W, NumBitsW>::fromFloat();
}
- SWR_ASSERT(0);
+ SWR_INVALID("Invalid component: %d", comp);
return TypeTraits<X, NumBitsX>::fromFloat();
}
case 3:
return TypeTraits<W, NumBitsW>::loadSOA(pSrc);
}
- SWR_ASSERT(0);
+ SWR_INVALID("Invalid component: %d", comp);
return TypeTraits<X, NumBitsX>::loadSOA(pSrc);
}
TypeTraits<W, NumBitsW>::storeSOA(pDst, src);
return;
}
- SWR_ASSERT(0);
+ SWR_INVALID("Invalid component: %d", comp);
TypeTraits<X, NumBitsX>::storeSOA(pDst, src);
}
case 3:
return TypeTraits<W, NumBitsW>::unpack(in);
}
- SWR_ASSERT(0);
+ SWR_INVALID("Invalid component: %d", comp);
return TypeTraits<X, NumBitsX>::unpack(in);
}
case 3:
return TypeTraits<W, NumBitsW>::pack(in);
}
- SWR_ASSERT(0);
+ SWR_INVALID("Invalid component: %d", comp);
return TypeTraits<X, NumBitsX>::pack(in);
}
case 3:
return TypeTraits<W, NumBitsW>::convertSrgb(in);
}
- SWR_ASSERT(0);
+ SWR_INVALID("Invalid component: %d", comp);
return TypeTraits<X, NumBitsX>::convertSrgb(in);
}
+#if ENABLE_AVX512_SIMD16
+
+ INLINE static simd16scalar loadSOA_16(uint32_t comp, const uint8_t* pSrc)
+ {
+ switch (comp)
+ {
+ case 0:
+ return TypeTraits<X, NumBitsX>::loadSOA_16(pSrc);
+ case 1:
+ return TypeTraits<Y, NumBitsY>::loadSOA_16(pSrc);
+ case 2:
+ return TypeTraits<Z, NumBitsZ>::loadSOA_16(pSrc);
+ case 3:
+ return TypeTraits<W, NumBitsW>::loadSOA_16(pSrc);
+ }
+ SWR_INVALID("Invalid component: %d", comp);
+ return TypeTraits<X, NumBitsX>::loadSOA_16(pSrc);
+ }
+
+ INLINE static void SIMDCALL storeSOA(uint32_t comp, uint8_t *pDst, simd16scalar src)
+ {
+ switch (comp)
+ {
+ case 0:
+ TypeTraits<X, NumBitsX>::storeSOA(pDst, src);
+ return;
+ case 1:
+ TypeTraits<Y, NumBitsY>::storeSOA(pDst, src);
+ return;
+ case 2:
+ TypeTraits<Z, NumBitsZ>::storeSOA(pDst, src);
+ return;
+ case 3:
+ TypeTraits<W, NumBitsW>::storeSOA(pDst, src);
+ return;
+ }
+ SWR_INVALID("Invalid component: %d", comp);
+ TypeTraits<X, NumBitsX>::storeSOA(pDst, src);
+ }
+
+ INLINE static simd16scalar unpack(uint32_t comp, simd16scalar &in)
+ {
+ switch (comp)
+ {
+ case 0:
+ return TypeTraits<X, NumBitsX>::unpack(in);
+ case 1:
+ return TypeTraits<Y, NumBitsY>::unpack(in);
+ case 2:
+ return TypeTraits<Z, NumBitsZ>::unpack(in);
+ case 3:
+ return TypeTraits<W, NumBitsW>::unpack(in);
+ }
+ SWR_INVALID("Invalid component: %d", comp);
+ return TypeTraits<X, NumBitsX>::unpack(in);
+ }
+
+ INLINE static simd16scalar pack(uint32_t comp, simd16scalar &in)
+ {
+ switch (comp)
+ {
+ case 0:
+ return TypeTraits<X, NumBitsX>::pack(in);
+ case 1:
+ return TypeTraits<Y, NumBitsY>::pack(in);
+ case 2:
+ return TypeTraits<Z, NumBitsZ>::pack(in);
+ case 3:
+ return TypeTraits<W, NumBitsW>::pack(in);
+ }
+ SWR_INVALID("Invalid component: %d", comp);
+ return TypeTraits<X, NumBitsX>::pack(in);
+ }
+
+ INLINE static simd16scalar convertSrgb(uint32_t comp, simd16scalar &in)
+ {
+ switch (comp)
+ {
+ case 0:
+ return TypeTraits<X, NumBitsX>::convertSrgb(in);
+ case 1:
+ return TypeTraits<Y, NumBitsY>::convertSrgb(in);
+ case 2:
+ return TypeTraits<Z, NumBitsZ>::convertSrgb(in);
+ case 3:
+ return TypeTraits<W, NumBitsW>::convertSrgb(in);
+ }
+ SWR_INVALID("Invalid component: %d", comp);
+ return TypeTraits<X, NumBitsX>::convertSrgb(in);
+ }
+#endif
};
projects / mesa.git / blobdiff