Work in progress (disabled).
USE_8x2_TILE_BACKEND define in knobs.h enables AVX512 code paths
(emulated on non-AVX512 HW).
Signed-off-by: Tim Rowley <timothy.o.rowley@intel.com>
#define OSALIGNLINE(RWORD) OSALIGN(RWORD, 64)
#define OSALIGNSIMD(RWORD) OSALIGN(RWORD, KNOB_SIMD_BYTES)
+#if ENABLE_AVX512_SIMD16
+#define OSALIGNSIMD16(RWORD) OSALIGN(RWORD, KNOB_SIMD16_BYTES)
+#endif
#include "common/swr_assert.h"
return result;
}
-INLINE simd16scalari _simd16_set_epi32(int e7, int e6, int e5, int e4, int e3, int e2, int e1, int e0)
+INLINE simd16scalar _simd16_set_ps(float e15, float e14, float e13, float e12, float e11, float e10, float e9, float e8, float e7, float e6, float e5, float e4, float e3, float e2, float e1, float e0)
{
- simd16scalari result;
+ simd16scalar result;
- result.lo = _mm256_set_epi32(e7, e6, e5, e4, e3, e2, e1, e0);
- result.hi = _mm256_set_epi32(e7, e6, e5, e4, e3, e2, e1, e0);
+ result.lo = _mm256_set_ps(e7, e6, e5, e4, e3, e2, e1, e0);
+ result.hi = _mm256_set_ps(e15, e14, e13, e12, e11, e10, e9, e8);
return result;
}
return result;
}
+INLINE simd16scalar _simd16_set_ps(float e7, float e6, float e5, float e4, float e3, float e2, float e1, float e0)
+{
+ simd16scalar result;
+
+ result.lo = _mm256_set_ps(e7, e6, e5, e4, e3, e2, e1, e0);
+ result.hi = _mm256_set_ps(e7, e6, e5, e4, e3, e2, e1, e0);
+
+ return result;
+}
+
+INLINE simd16scalari _simd16_set_epi32(int e7, int e6, int e5, int e4, int e3, int e2, int e1, int e0)
+{
+ simd16scalari result;
+
+ result.lo = _mm256_set_epi32(e7, e6, e5, e4, e3, e2, e1, e0);
+ result.hi = _mm256_set_epi32(e7, e6, e5, e4, e3, e2, e1, e0);
+
+ return result;
+}
+
INLINE simd16scalar _simd16_load_ps(float const *m)
{
simd16scalar result;
_mm256_store_si256(&m[0].hi, a.hi);
}
+INLINE simdscalar _simd16_extract_ps(simd16scalar a, int imm8)
+{
+ switch (imm8)
+ {
+ case 0:
+ return a.lo;
+ case 1:
+ return a.hi;
+ }
+ return _simd_set1_ps(0.0f);
+}
+
+INLINE simdscalari _simd16_extract_si(simd16scalari a, int imm8)
+{
+ switch (imm8)
+ {
+ case 0:
+ return a.lo;
+ case 1:
+ return a.hi;
+ }
+ return _simd_set1_epi32(0);
+}
+
+INLINE simd16scalar _simd16_insert_ps(simd16scalar a, simdscalar b, int imm8)
+{
+ switch (imm8)
+ {
+ case 0:
+ a.lo = b;
+ break;
+ case 1:
+ a.hi = b;
+ break;
+ }
+ return a;
+}
+
+INLINE simd16scalari _simd16_insert_si(simd16scalari a, simdscalari b, int imm8)
+{
+ switch (imm8)
+ {
+ case 0:
+ a.lo = b;
+ break;
+ case 1:
+ a.hi = b;
+ break;
+ }
+ return a;
+}
+
template <simd16mask mask>
INLINE simd16scalar _simd16_blend_ps_temp(simd16scalar a, simd16scalar b)
{
SIMD16_EMU_AVX512_2(simd16scalari, _simd16_min_epu32, _mm256_min_epu32)
SIMD16_EMU_AVX512_2(simd16scalari, _simd16_max_epu32, _mm256_max_epu32)
SIMD16_EMU_AVX512_2(simd16scalari, _simd16_add_epi32, _mm256_add_epi32)
-SIMD16_EMU_AVX512_2(simd16scalari, _simd16_and_si, _mm256_and_si256)
-SIMD16_EMU_AVX512_2(simd16scalari, _simd16_andnot_si, _mm256_andnot_si256)
-SIMD16_EMU_AVX512_2(simd16scalari, _simd16_or_si, _mm256_or_si256)
-SIMD16_EMU_AVX512_2(simd16scalari, _simd16_xor_si, _mm256_xor_si256)
+SIMD16_EMU_AVX512_2(simd16scalari, _simd16_and_si, _simd_and_si)
+SIMD16_EMU_AVX512_2(simd16scalari, _simd16_andnot_si, _simd_andnot_si)
+SIMD16_EMU_AVX512_2(simd16scalari, _simd16_or_si, _simd_or_si)
+SIMD16_EMU_AVX512_2(simd16scalari, _simd16_xor_si, _simd_xor_si)
SIMD16_EMU_AVX512_2(simd16scalari, _simd16_cmpeq_epi32, _mm256_cmpeq_epi32)
SIMD16_EMU_AVX512_2(simd16scalari, _simd16_cmpgt_epi32, _mm256_cmpgt_epi32)
#define _simd16_cmplt_epi32(a, b) _simd16_cmpgt_epi32(b, a)
-SIMD16_EMU_AVX512_2(simd16scalari, _simd16_unpacklo_epi32, _mm256_unpacklo_epi32)
-SIMD16_EMU_AVX512_2(simd16scalari, _simd16_unpackhi_epi32, _mm256_unpackhi_epi32)
+SIMD16_EMU_AVX512_2(simd16scalari, _simd16_unpacklo_epi32, _simd_unpacklo_epi32)
+SIMD16_EMU_AVX512_2(simd16scalari, _simd16_unpackhi_epi32, _simd_unpackhi_epi32)
+SIMD16_EMU_AVX512_2(simd16scalari, _simd16_unpacklo_epi64, _simd_unpacklo_epi64)
+SIMD16_EMU_AVX512_2(simd16scalari, _simd16_unpackhi_epi64, _simd_unpackhi_epi64)
template <int imm8>
INLINE simd16scalari _simd16_slli_epi32_temp(simd16scalari a)
{
simd16scalari result;
- result.lo = _mm256_slli_epi32(a.lo, imm8);
- result.hi = _mm256_slli_epi32(a.hi, imm8);
+ result.lo = _simd_slli_epi32(a.lo, imm8);
+ result.hi = _simd_slli_epi32(a.hi, imm8);
return result;
}
{
simd16scalari result;
- result.lo = _mm256_srai_epi32(a.lo, imm8);
- result.hi = _mm256_srai_epi32(a.hi, imm8);
+ result.lo = _simd_srai_epi32(a.lo, imm8);
+ result.hi = _simd_srai_epi32(a.hi, imm8);
return result;
}
{
simd16scalari result;
- result.lo = _mm256_srli_epi32(a.lo, imm8);
- result.hi = _mm256_srli_epi32(a.hi, imm8);
+ result.lo = _simd_srli_epi32(a.lo, imm8);
+ result.hi = _simd_srli_epi32(a.hi, imm8);
return result;
}
SIMD16_EMU_AVX512_2(simd16scalari, _simd16_cmpeq_epi8, _mm256_cmpeq_epi8)
SIMD16_EMU_AVX512_2(simd16scalari, _simd16_cmpgt_epi8, _mm256_cmpgt_epi8)
-INLINE simd16scalar _simd16_permute_ps(simd16scalar a, simd16scalari b)
+INLINE simd16scalar _simd16_permute_ps(simd16scalar a, simd16scalari i)
{
simd16scalar result;
- result.lo = _mm256_permutevar8x32_ps(a.lo, b.lo);
- result.hi = _mm256_permutevar8x32_ps(a.hi, b.hi);
+ const simdscalari mask = _simd_set1_epi32(7);
+
+ simdscalar lolo = _simd_permute_ps(a.lo, _simd_and_si(i.lo, mask));
+ simdscalar lohi = _simd_permute_ps(a.hi, _simd_and_si(i.lo, mask));
+
+ simdscalar hilo = _simd_permute_ps(a.lo, _simd_and_si(i.hi, mask));
+ simdscalar hihi = _simd_permute_ps(a.hi, _simd_and_si(i.hi, mask));
+
+ result.lo = _simd_blendv_ps(lolo, lohi, _simd_castsi_ps(_simd_cmpgt_epi32(i.lo, mask)));
+ result.hi = _simd_blendv_ps(hilo, hihi, _simd_castsi_ps(_simd_cmpgt_epi32(i.hi, mask)));
return result;
}
-SIMD16_EMU_AVX512_2(simd16scalari, _simd16_permute_epi32, _mm256_permutevar8x32_epi32)
+INLINE simd16scalari _simd16_permute_epi32(simd16scalari a, simd16scalari i)
+{
+ return _simd16_castps_si(_simd16_permute_ps(_simd16_castsi_ps(a), i));
+}
SIMD16_EMU_AVX512_2(simd16scalari, _simd16_srlv_epi32, _mm256_srlv_epi32)
SIMD16_EMU_AVX512_2(simd16scalari, _simd16_sllv_epi32, _mm256_sllv_epi32)
+template <int imm8>
+INLINE simd16scalar _simd16_permute2f128_ps_temp(simd16scalar a, simd16scalar b)
+{
+ simd16scalar result;
+
+ result.lo = _simd_permute2f128_ps(a.lo, a.hi, ((imm8 & 0x03) << 0) | ((imm8 & 0x0C) << 2));
+ result.hi = _simd_permute2f128_ps(b.lo, b.hi, ((imm8 & 0x30) >> 4) | ((imm8 & 0xC0) >> 2));
+
+ return result;
+}
+
+#define _simd16_permute2f128_ps(a, b, imm8) _simd16_permute2f128_ps_temp<imm8>(a, b)
+
+template <int imm8>
+INLINE simd16scalard _simd16_permute2f128_pd_temp(simd16scalard a, simd16scalard b)
+{
+ simd16scalard result;
+
+ result.lo = _simd_permute2f128_pd(a.lo, a.hi, ((imm8 & 0x03) << 0) | ((imm8 & 0x0C) << 2));
+ result.hi = _simd_permute2f128_pd(b.lo, b.hi, ((imm8 & 0x30) >> 4) | ((imm8 & 0xC0) >> 2));
+
+ return result;
+}
+
+#define _simd16_permute2f128_pd(a, b, imm8) _simd16_permute2f128_pd_temp<imm8>(a, b)
+
+template <int imm8>
+INLINE simd16scalari _simd16_permute2f128_si_temp(simd16scalari a, simd16scalari b)
+{
+ simd16scalari result;
+
+ result.lo = _simd_permute2f128_si(a.lo, a.hi, ((imm8 & 0x03) << 0) | ((imm8 & 0x0C) << 2));
+ result.hi = _simd_permute2f128_si(b.lo, b.hi, ((imm8 & 0x30) >> 4) | ((imm8 & 0xC0) >> 2));
+
+ return result;
+}
+
+#define _simd16_permute2f128_si(a, b, imm8) _simd16_permute2f128_si_temp<imm8>(a, b)
+
template <int imm8>
INLINE simd16scalar _simd16_shuffle_ps_temp(simd16scalar a, simd16scalar b)
{
simd16scalar result;
- result.lo = _mm256_shuffle_ps(a.lo, b.lo, imm8);
- result.hi = _mm256_shuffle_ps(a.hi, b.hi, imm8);
+ result.lo = _simd_shuffle_ps(a.lo, b.lo, imm8);
+ result.hi = _simd_shuffle_ps(a.hi, b.hi, imm8);
return result;
}
#define _simd16_shuffle_ps(a, b, imm8) _simd16_shuffle_ps_temp<imm8>(a, b)
template <int imm8>
-INLINE simd16scalari _simd16_permute_128_temp(simd16scalari a, simd16scalari b)
+INLINE simd16scalard _simd16_shuffle_pd_temp(simd16scalard a, simd16scalard b)
{
- simd16scalari result;
+ simd16scalard result;
- result.lo = _mm256_permute2x128_si256(a.lo, b.lo, imm8);
- result.hi = _mm256_permute2x128_si256(a.hi, b.hi, imm8);
+ result.lo = _simd_shuffle_pd(a.lo, b.lo, (imm8 & 15));
+ result.hi = _simd_shuffle_pd(a.hi, b.hi, (imm8 >> 4));
return result;
}
-#define _simd16_permute_128(a, b, imm8) _simd16_permute_128_temp<imm8>(a, b)
+#define _simd16_shuffle_pd(a, b, imm8) _simd16_shuffle_pd_temp<imm8>(a, b)
+
+template <int imm8>
+INLINE simd16scalari _simd16_shuffle_epi32_temp(simd16scalari a, simd16scalari b)
+{
+ return _simd16_castps_si(_simd16_shuffle_ps(_simd16_castsi_ps(a), _simd16_castsi_ps(b), imm8));
+}
+
+#define _simd16_shuffle_epi32(a, b, imm8) _simd16_shuffle_epi32_temp<imm8>(a, b)
+
+template <int imm8>
+INLINE simd16scalari _simd16_shuffle_epi64_temp(simd16scalari a, simd16scalari b)
+{
+ return _simd16_castpd_si(_simd16_shuffle_pd(_simd16_castsi_pd(a), _simd16_castsi_pd(b), imm8));
+}
+
+#define _simd16_shuffle_epi64(a, b, imm8) _simd16_shuffle_epi64_temp<imm8>(a, b)
+
+INLINE simd16mask _simd16_int2mask(int mask)
+{
+ return mask;
+}
+
+INLINE int _simd16_mask2int(simd16mask mask)
+{
+ return mask;
+}
+
+INLINE simd16mask _simd16_cmplt_ps_mask(simd16scalar a, simd16scalar b)
+{
+ return _simd16_movemask_ps(_simd16_cmplt_ps(a, b));
+}
// convert bitmask to vector mask
INLINE simd16scalar vMask16(int32_t mask)
INLINE simd16mask _simd16_scalari2mask(simd16scalari mask)
{
- __m512i flag = _mm512_set1_epi32(0x80000000);
-
- __m512i temp = _mm512_and_epi32(mask, flag);
-
- return _mm512_cmpeq_epu32_mask(temp, flag);
+ return _mm512_cmpneq_epu32_mask(mask, _mm512_setzero_epi32());
}
#if 0
INLINE simd16mask _simd16_scalard2mask(simd16scalard mask)
{
- __m512i flag = _mm512_set1_epi64(0x8000000000000000);
-
- __m512 tempi = _mm512_and_epi64(_mm512_castpd_si512(mask), flag);
-
- return _mm512_cmpeq_epu64_mask(temp, flag);
+ return _mm512_cmpneq_epu64_mask(mask, _mm512_setzero_epi64());
}
#endif
#define _simd16_set1_epi8 _mm512_set1_epi8
#define _simd16_set1_epi32 _mm512_set1_epi32
-INLINE simd16scalari _simd16_set_epi32(int e7, int e6, int e5, int e4, int e3, int e2, int e1, int e0)
+INLINE simd16scalar _simd16_set_ps(float e15, float e14, float e13, float e12, float e11, float e10, float e9, float e8, float e7, float e6, float e5, float e4, float e3, float e2, float e1, float e0)
{
- simd16scalari result;
-
- result = _mm512_set_epi32(e7, e6, e5, e4, e3, e2, e1, e0, e7, e6, e5, e4, e3, e2, e1, e0);
-
- return result;
+ return _mm512_set_ps(e15, e14, e13, e12, e11, e10, e9, e8, e7, e6, e5, e4, e3, e2, e1, e0);
}
INLINE simd16scalari _simd16_set_epi32(int e15, int e14, int e13, int e12, int e11, int e10, int e9, int e8, int e7, int e6, int e5, int e4, int e3, int e2, int e1, int e0)
{
- simd16scalari result;
+ return _mm512_set_epi32(e15, e14, e13, e12, e11, e10, e9, e8, e7, e6, e5, e4, e3, e2, e1, e0);
+}
- result = _mm512_set_epi32(e15, e14, e13, e12, e11, e10, e9, e8, e7, e6, e5, e4, e3, e2, e1, e0);
+INLINE simd16scalar _simd16_set_ps(float e7, float e6, float e5, float e4, float e3, float e2, float e1, float e0)
+{
+ return _mm512_set_ps(e7, e6, e5, e4, e3, e2, e1, e0, e7, e6, e5, e4, e3, e2, e1, e0);
+}
- return result;
+INLINE simd16scalari _simd16_set_epi32(int e7, int e6, int e5, int e4, int e3, int e2, int e1, int e0)
+{
+ return _mm512_set_epi32(e7, e6, e5, e4, e3, e2, e1, e0, e7, e6, e5, e4, e3, e2, e1, e0);
}
#define _simd16_load_ps _mm512_load_ps
#if 1
#define _simd16_load1_ps _simd16_broadcast_ss
#endif
-#define _simd16_load_si _mm256_load_si256
-#define _simd16_loadu_si _mm256_loadu_si256
+#define _simd16_load_si _mm512_load_si512
+#define _simd16_loadu_si _mm512_loadu_si512
#define _simd16_broadcast_ss(m) _mm512_extload_ps(m, _MM_UPCONV_PS_NONE, _MM_BROADCAST_1X16, 0)
#define _simd16_broadcast_ps(m) _mm512_extload_ps(m, _MM_UPCONV_PS_NONE, _MM_BROADCAST_4X16, 0)
#define _simd16_store_ps _mm512_store_ps
#define _simd16_store_si _mm512_store_si512
+#define _simd16_extract_ps _mm512_extractf32x8_ps
+#define _simd16_extract_si _mm512_extracti32x8_epi32
+#define _simd16_insert_ps _mm512_insertf32x8
+#define _simd16_insert_si _mm512_inserti32x8
INLINE void _simd16_maskstore_ps(float *m, simd16scalari mask, simd16scalar a)
{
#define _simd16_mul_ps _mm512_mul_ps
#define _simd16_add_ps _mm512_add_ps
#define _simd16_sub_ps _mm512_sub_ps
-#define _simd16_rsqrt_ps _mm512_rsqrt23_ps
+#define _simd16_rsqrt_ps _mm512_rsqrt14_ps
#define _simd16_min_ps _mm512_min_ps
#define _simd16_max_ps _mm512_max_ps
{
simd16mask k = _mm512_cmpeq_ps_mask(a, b);
- return _mm512_castsi256_ps(_mm512_mask_blend_epi32(k, _mm512_setzero_epi32(), _mm512_set1_epi32(0xFFFFFFFF)));
+ return _mm512_castsi512_ps(_mm512_mask_blend_epi32(k, _mm512_setzero_epi32(), _mm512_set1_epi32(0xFFFFFFFF)));
}
#define _simd16_cmp_ps(a, b, comp) _simd16_cmp_ps_temp<comp>(a, b)
#define _simd16_unpacklo_epi32 _mm512_unpacklo_epi32
#define _simd16_unpackhi_epi32 _mm512_unpackhi_epi32
+#define _simd16_unpacklo_epi64 _mm512_unpacklo_epi64
+#define _simd16_unpackhi_epi64 _mm512_unpackhi_epi64
#define _simd16_slli_epi32 _mm512_slli_epi32
#define _simd16_srli_epi32 _mm512_srli_epi32
#define _simd16_srai_epi32 _mm512_srai_epi32
return _mm512_mask_blend_epi8(k, _mm512_setzero_si512(), _mm512_set1_epi32(0xFFFFFFFF));
}
-#if 0
-INLINE simd16scalar _simd16_permute_ps(simd16scalar a, simd16scalari b)
-{
- simd16scalar result;
+#define _simd16_permute_ps(a, i) _mm512_permutexvar_ps(i, a)
+#define _simd16_permute_epi32(a, i) _mm512_permutexvar_epi32(i, a)
+#define _simd16_sllv_epi32 _mm512_srlv_epi32
+#define _simd16_srlv_epi32 _mm512_sllv_epi32
+#define _simd16_permute2f128_ps _mm512_shuffle_f32x4
+#define _simd16_permute2f128_pd _mm512_shuffle_f64x2
+#define _simd16_permute2f128_si _mm512_shuffle_i32x4
+#define _simd16_shuffle_ps _mm512_shuffle_ps
+#define _simd16_shuffle_pd _mm512_shuffle_pd
- result.lo = _mm256_permutevar8x32_ps(a.lo, b.lo);
- result.hi = _mm256_permutevar8x32_ps(a.hi, b.hi);
-
- return result;
+template <int imm8>
+INLINE simd16scalari _simd16_shuffle_epi32_temp(simd16scalari a, simd16scalari b)
+{
+ return _simd16_castps_si(_simd16_shuffle_ps(_simd16_castsi_ps(a), _simd16_castsi_ps(b), imm8));
}
-INLINE (simd16scalari _simd16_permute_epi32(simd16scalari a, simd16scalari b)
+#define _simd16_shuffle_epi32(a, b, imm8) _simd16_shuffle_epi32_temp<imm8>(a, b)
+
+template <int imm8>
+INLINE simd16scalari _simd16_shuffle_epi64_temp(simd16scalari a, simd16scalari b)
{
- simd16scalar result;
+ return _simd16_castpd_si(_simd16_shuffle_pd(_simd16_castsi_pd(a), _simd16_castsi_pd(b), imm8));
+}
- result.lo = _mm256_permutevar8x32_epi32(a.lo, b.lo);
- result.hi = _mm256_permutevar8x32_epi32(a.hi, b.hi);
+#define _simd16_shuffle_epi64(a, b, imm8) _simd16_shuffle_epi64_temp<imm8>(a, b)
- return result;
+INLINE simd16mask _simd16_int2mask(int mask)
+{
+ return _mm512_int2mask(mask);
}
-#endif
+INLINE int _simd16_mask2int(simd16mask mask)
+{
+ return _mm512_mask2int(mask);
+}
-#define _simd16_sllv_epi32 _mm512_srlv_epi32
-#define _simd16_srlv_epi32 _mm512_sllv_epi32
-#define _simd16_shuffle_ps _mm512_shuffle_ps
-#define _simd16_permute_128 _mm512_permute4f128_epi32
+INLINE simd16mask _simd16_cmplt_ps_mask(simd16scalar a, simd16scalar b)
+{
+ return _mm512_cmplt_ps_mask(a, b);
+}
// convert bitmask to vector mask
INLINE simd16scalar vMask16(int32_t mask)
#define _simd_cmplt_epi32 _simdemu_cmplt_epi32
#define _simd_cmpgt_epi32 _simdemu_cmpgt_epi32
#define _simd_or_si _simdemu_or_si
+#define _simd_xor_si _simdemu_xor_si
#define _simd_castps_si _mm256_castps_si256
#define _simd_adds_epu8 _simdemu_adds_epu8
#define _simd_subs_epu8 _simdemu_subs_epu8
SIMD_EMU_EPI(_simdemu_cmplt_epi32, _mm_cmplt_epi32)
SIMD_EMU_EPI(_simdemu_cmpgt_epi32, _mm_cmpgt_epi32)
SIMD_EMU_EPI(_simdemu_or_si, _mm_or_si128)
+SIMD_EMU_EPI(_simdemu_xor_si, _mm_xor_si128)
SIMD_EMU_EPI(_simdemu_adds_epu8, _mm_adds_epu8)
SIMD_EMU_EPI(_simdemu_subs_epu8, _mm_subs_epu8)
SIMD_EMU_EPI(_simdemu_add_epi8, _mm_add_epi8)
#define _simd_unpacklo_epi32(a, b) _mm256_castps_si256(_mm256_unpacklo_ps(_mm256_castsi256_ps(a), _mm256_castsi256_ps(b)))
#define _simd_unpackhi_epi32(a, b) _mm256_castps_si256(_mm256_unpackhi_ps(_mm256_castsi256_ps(a), _mm256_castsi256_ps(b)))
+#define _simd_unpacklo_epi64(a, b) _mm256_castpd_si256(_mm256_unpacklo_pd(_mm256_castsi256_pd(a), _mm256_castsi256_pd(b)))
+#define _simd_unpackhi_epi64(a, b) _mm256_castpd_si256(_mm256_unpackhi_pd(_mm256_castsi256_pd(a), _mm256_castsi256_pd(b)))
#define _simd_slli_epi32(a,i) _simdemu_slli_epi32(a,i)
#define _simd_srai_epi32(a,i) _simdemu_srai_epi32(a,i)
return (resHi << 16) | resLo;
}
+
+INLINE
+__m256i _simd_cvtepu8_epi32(__m128i a)
+{
+ __m128i resultlo = _mm_cvtepu8_epi32(a);
+ __m128i resulthi = _mm_shuffle_epi8(a, _mm_set_epi32(0x80808007, 0x80808006, 0x80808005, 0x80808004));
+
+ __m256i result = _mm256_castsi128_si256(resultlo);
+
+ return _mm256_insertf128_si256(result, resulthi, 1);
+}
+
+INLINE
+__m256i _simd_cvtepu16_epi32(__m128i a)
+{
+ __m128i resultlo = _mm_cvtepu16_epi32(a);
+ __m128i resulthi = _mm_shuffle_epi8(a, _mm_set_epi32(0x80800F0E, 0x80800D0C, 0x80800B0A, 0x80800908));
+
+ __m256i result = _mm256_castsi128_si256(resultlo);
+
+ return _mm256_insertf128_si256(result, resulthi, 1);
+}
+
+INLINE
+__m256i _simd_packus_epi32(__m256i a, __m256i b)
+{
+ __m128i alo = _mm256_extractf128_si256(a, 0);
+ __m128i ahi = _mm256_extractf128_si256(a, 1);
+
+ __m128i blo = _mm256_extractf128_si256(b, 0);
+ __m128i bhi = _mm256_extractf128_si256(b, 1);
+
+ __m128i resultlo = _mm_packus_epi32(alo, blo);
+ __m128i resulthi = _mm_packus_epi32(ahi, bhi);
+
+ __m256i result = _mm256_castsi128_si256(resultlo);
+
+ return _mm256_insertf128_si256(result, resulthi, 1);
+}
+
+INLINE
+__m256i _simd_packs_epi32(__m256i a, __m256i b)
+{
+ __m128i alo = _mm256_extractf128_si256(a, 0);
+ __m128i ahi = _mm256_extractf128_si256(a, 1);
+
+ __m128i blo = _mm256_extractf128_si256(b, 0);
+ __m128i bhi = _mm256_extractf128_si256(b, 1);
+
+ __m128i resultlo = _mm_packs_epi32(alo, blo);
+ __m128i resulthi = _mm_packs_epi32(ahi, bhi);
+
+ __m256i result = _mm256_castsi128_si256(resultlo);
+
+ return _mm256_insertf128_si256(result, resulthi, 1);
+}
+
#else
#define _simd_mul_epi32 _mm256_mul_epi32
#define _simd_cmplt_epi32(a,b) _mm256_cmpgt_epi32(b,a)
#define _simd_cmpgt_epi32(a,b) _mm256_cmpgt_epi32(a,b)
#define _simd_or_si _mm256_or_si256
+#define _simd_xor_si _mm256_xor_si256
#define _simd_castps_si _mm256_castps_si256
#define _simd_unpacklo_epi32 _mm256_unpacklo_epi32
#define _simd_unpackhi_epi32 _mm256_unpackhi_epi32
+#define _simd_unpacklo_epi64 _mm256_unpacklo_epi64
+#define _simd_unpackhi_epi64 _mm256_unpackhi_epi64
#define _simd_srli_si(a,i) _simdemu_srli_si128<i>(a)
#define _simd_slli_epi32 _mm256_slli_epi32
#define _simd_cmpeq_epi16 _mm256_cmpeq_epi16
#define _simd_movemask_epi8 _mm256_movemask_epi8
#define _simd_permute_ps _mm256_permutevar8x32_ps
+#define _simd_permute_epi32 _mm256_permutevar8x32_epi32
#define _simd_srlv_epi32 _mm256_srlv_epi32
#define _simd_sllv_epi32 _mm256_sllv_epi32
+#define _simd_cvtepu8_epi32 _mm256_cvtepu8_epi32
+#define _simd_cvtepu16_epi32 _mm256_cvtepu16_epi32
+#define _simd_packus_epi32 _mm256_packus_epi32
+#define _simd_packs_epi32 _mm256_packs_epi32
-INLINE
-simdscalari _simd_permute_epi32(simdscalari a, simdscalari index)
-{
- return _simd_castps_si(_mm256_permutevar8x32_ps(_mm256_castsi256_ps(a), index));
-}
#endif
-#define _simd_permute_128 _mm256_permute2f128_si256
-#define _simd_shuffleps_epi32(vA, vB, imm) _mm256_castps_si256(_mm256_shuffle_ps(_mm256_castsi256_ps(vA), _mm256_castsi256_ps(vB), imm))
+#define _simd_permute2f128_ps _mm256_permute2f128_ps
+#define _simd_permute2f128_pd _mm256_permute2f128_pd
+#define _simd_permute2f128_si _mm256_permute2f128_si256
#define _simd_shuffle_ps _mm256_shuffle_ps
+#define _simd_shuffle_pd _mm256_shuffle_pd
+#define _simd_shuffle_epi32(a, b, imm8) _mm256_castps_si256(_mm256_shuffle_ps(_mm256_castsi256_ps(a), _mm256_castsi256_ps(b), imm8))
+#define _simd_shuffle_epi64(a, b, imm8) _mm256_castps_si256(_mm256_shuffle_pd(_mm256_castsi256_pd(a), _mm256_castsi256_pd(b), imm8))
#define _simd_set1_epi32 _mm256_set1_epi32
#define _simd_set_epi32 _mm256_set_epi32
#define _simd_set1_epi8 _mm256_set1_epi8
#define _simd_testz_ps _mm256_testz_ps
#define _simd_xor_ps _mm256_xor_ps
-
INLINE
simdscalari _simd_blendv_epi32(simdscalari a, simdscalari b, simdscalar mask)
{
return result;
}
-INLINE simdscalari _simd_permute_128(simdscalari a, simdscalari b, const int imm8)
-{
- simdscalari result;
-
- result.lo = _mm256_permute2x128_si256(a.lo, b.lo, imm8);
- result.hi = _mm256_permute2x128_si256(a.hi, b.hi, imm8);
-
- return result;
-}
-
// convert bitmask to vector mask
INLINE simdscalar vMask(int32_t mask)
{
auto lambda = [&](int32_t comp)
{
FormatTraits<format>::storeSOA(comp, pTileBuffer, value.v[comp]);
+
pTileBuffer += (KNOB_SIMD_WIDTH * FormatTraits<format>::GetBPC(comp) / 8);
};
const uint32_t numIter = (KNOB_TILE_Y_DIM / SIMD_TILE_Y_DIM) * (KNOB_TILE_X_DIM / SIMD_TILE_X_DIM);
+
for (uint32_t i = 0; i < numIter; ++i)
{
UnrollerL<0, FormatTraits<format>::numComps, 1>::step(lambda);
}
}
+#if USE_8x2_TILE_BACKEND
+template<SWR_FORMAT format>
+void ClearRasterTile(uint8_t *pTileBuffer, simd16vector &value)
+{
+ auto lambda = [&](int32_t comp)
+ {
+ FormatTraits<format>::storeSOA(comp, pTileBuffer, value.v[comp]);
+
+ pTileBuffer += (KNOB_SIMD16_WIDTH * FormatTraits<format>::GetBPC(comp) / 8);
+ };
+
+ const uint32_t numIter = (KNOB_TILE_Y_DIM / SIMD16_TILE_Y_DIM) * (KNOB_TILE_X_DIM / SIMD16_TILE_X_DIM);
+
+ for (uint32_t i = 0; i < numIter; ++i)
+ {
+ UnrollerL<0, FormatTraits<format>::numComps, 1>::step(lambda);
+ }
+}
+
+#endif
template<SWR_FORMAT format>
INLINE void ClearMacroTile(DRAW_CONTEXT *pDC, SWR_RENDERTARGET_ATTACHMENT rt, uint32_t macroTile, DWORD clear[4], const SWR_RECT& rect)
{
// convert clear color to hottile format
// clear color is in RGBA float/uint32
+#if USE_8x2_TILE_BACKEND
+ simd16vector vClear;
+ for (uint32_t comp = 0; comp < FormatTraits<format>::numComps; ++comp)
+ {
+ simd16scalar vComp;
+ vComp = _simd16_load1_ps((const float*)&clear[comp]);
+ if (FormatTraits<format>::isNormalized(comp))
+ {
+ vComp = _simd16_mul_ps(vComp, _simd16_set1_ps(FormatTraits<format>::fromFloat(comp)));
+ vComp = _simd16_castsi_ps(_simd16_cvtps_epi32(vComp));
+ }
+ vComp = FormatTraits<format>::pack(comp, vComp);
+ vClear.v[FormatTraits<format>::swizzle(comp)] = vComp;
+ }
+
+#else
simdvector vClear;
for (uint32_t comp = 0; comp < FormatTraits<format>::numComps; ++comp)
{
vClear.v[FormatTraits<format>::swizzle(comp)] = vComp;
}
+#endif
uint32_t tileX, tileY;
MacroTileMgr::getTileIndices(macroTile, tileX, tileY);
for(uint32_t xx = x; xx < x + KNOB_TILE_X_DIM; xx += SIMD_TILE_X_DIM)
{
+#if USE_8x2_TILE_BACKEND
+ const bool useAlternateOffset = ((xx & SIMD_TILE_X_DIM) != 0);
+
+#endif
if(coverageMask & MASK)
{
psContext.vX.UL = _simd_add_ps(vULOffsetsX, _simd_set1_ps((float)xx));
// output merger
AR_BEGIN(BEOutputMerger, pDC->drawId);
+#if USE_8x2_TILE_BACKEND
+ OutputMerger(psContext, pColorBase, 0, pBlendState, state.pfnBlendFunc, vCoverageMask, depthPassMask, pPSState->numRenderTargets, useAlternateOffset);
+#else
OutputMerger(psContext, pColorBase, 0, pBlendState, state.pfnBlendFunc, vCoverageMask, depthPassMask, pPSState->numRenderTargets);
+#endif
// do final depth write after all pixel kills
if (!pPSState->forceEarlyZ)
pDepthBase += (KNOB_SIMD_WIDTH * FormatTraits<KNOB_DEPTH_HOT_TILE_FORMAT>::bpp) / 8;
pStencilBase += (KNOB_SIMD_WIDTH * FormatTraits<KNOB_STENCIL_HOT_TILE_FORMAT>::bpp) / 8;
- for(uint32_t rt = 0; rt < NumRT; ++rt)
+#if USE_8x2_TILE_BACKEND
+ if (useAlternateOffset)
+ {
+ for (uint32_t rt = 0; rt < NumRT; ++rt)
+ {
+ pColorBase[rt] += (2 * KNOB_SIMD_WIDTH * FormatTraits<KNOB_COLOR_HOT_TILE_FORMAT>::bpp) / 8;
+ }
+ }
+#else
+ for (uint32_t rt = 0; rt < NumRT; ++rt)
{
pColorBase[rt] += (KNOB_SIMD_WIDTH * FormatTraits<KNOB_COLOR_HOT_TILE_FORMAT>::bpp) / 8;
}
+#endif
AR_END(BEEndTile, 0);
}
}
for (uint32_t xx = x; xx < x + KNOB_TILE_X_DIM; xx += SIMD_TILE_X_DIM)
{
+#if USE_8x2_TILE_BACKEND
+ const bool useAlternateOffset = ((xx & SIMD_TILE_X_DIM) != 0);
+
+#endif
psContext.vX.UL = _simd_add_ps(vULOffsetsX, _simd_set1_ps((float)xx));
// pixel center
psContext.vX.center = _simd_add_ps(vCenterOffsetsX, _simd_set1_ps((float)xx));
// output merger
AR_BEGIN(BEOutputMerger, pDC->drawId);
+#if USE_8x2_TILE_BACKEND
+ OutputMerger(psContext, pColorBase, sample, pBlendState, state.pfnBlendFunc, vCoverageMask, depthPassMask, pPSState->numRenderTargets, useAlternateOffset);
+#else
OutputMerger(psContext, pColorBase, sample, pBlendState, state.pfnBlendFunc, vCoverageMask, depthPassMask, pPSState->numRenderTargets);
+#endif
// do final depth write after all pixel kills
if (!pPSState->forceEarlyZ)
pDepthBase += (KNOB_SIMD_WIDTH * FormatTraits<KNOB_DEPTH_HOT_TILE_FORMAT>::bpp) / 8;
pStencilBase += (KNOB_SIMD_WIDTH * FormatTraits<KNOB_STENCIL_HOT_TILE_FORMAT>::bpp) / 8;
+#if USE_8x2_TILE_BACKEND
+ if (useAlternateOffset)
+ {
+ for (uint32_t rt = 0; rt < NumRT; ++rt)
+ {
+ pColorBase[rt] += (2 * KNOB_SIMD_WIDTH * FormatTraits<KNOB_COLOR_HOT_TILE_FORMAT>::bpp) / 8;
+ }
+ }
+#else
for (uint32_t rt = 0; rt < NumRT; ++rt)
{
pColorBase[rt] += (KNOB_SIMD_WIDTH * FormatTraits<KNOB_COLOR_HOT_TILE_FORMAT>::bpp) / 8;
}
+#endif
AR_END(BEEndTile, 0);
}
}
psContext.vY.center = _simd_add_ps(vCenterOffsetsY, _simd_set1_ps((float)yy));
for(uint32_t xx = x; xx < x + KNOB_TILE_X_DIM; xx += SIMD_TILE_X_DIM)
{
+#if USE_8x2_TILE_BACKEND
+ const bool useAlternateOffset = ((xx & SIMD_TILE_X_DIM) != 0);
+
+#endif
simdscalar activeLanes;
if(!(work.anyCoveredSamples & MASK)) {goto Endtile;};
activeLanes = vMask(work.anyCoveredSamples & MASK);
}
// broadcast the results of the PS to all passing pixels
+#if USE_8x2_TILE_BACKEND
+ OutputMerger(psContext, pColorBase, sample, pBlendState, state.pfnBlendFunc, coverageMask, depthMask, pPSState->numRenderTargets, useAlternateOffset);
+#else
OutputMerger(psContext, pColorBase, sample, pBlendState, state.pfnBlendFunc, coverageMask, depthMask, pPSState->numRenderTargets);
+#endif
if(!pPSState->forceEarlyZ && !T::bForcedSampleCount)
{
pDepthBase += (KNOB_SIMD_WIDTH * FormatTraits<KNOB_DEPTH_HOT_TILE_FORMAT>::bpp) / 8;
pStencilBase += (KNOB_SIMD_WIDTH * FormatTraits<KNOB_STENCIL_HOT_TILE_FORMAT>::bpp) / 8;
+#if USE_8x2_TILE_BACKEND
+ if (useAlternateOffset)
+ {
+ for (uint32_t rt = 0; rt < NumRT; ++rt)
+ {
+ pColorBase[rt] += (2 * KNOB_SIMD_WIDTH * FormatTraits<KNOB_COLOR_HOT_TILE_FORMAT>::bpp) / 8;
+ }
+ }
+#else
for(uint32_t rt = 0; rt < NumRT; ++rt)
{
pColorBase[rt] += (KNOB_SIMD_WIDTH * FormatTraits<KNOB_COLOR_HOT_TILE_FORMAT>::bpp) / 8;
}
+#endif
AR_END(BEEndTile, 0);
}
}
}
}
+#if USE_8x2_TILE_BACKEND
+INLINE void OutputMerger(SWR_PS_CONTEXT &psContext, uint8_t* (&pColorBase)[SWR_NUM_RENDERTARGETS], uint32_t sample, const SWR_BLEND_STATE *pBlendState,
+ const PFN_BLEND_JIT_FUNC(&pfnBlendFunc)[SWR_NUM_RENDERTARGETS], simdscalar &coverageMask, simdscalar depthPassMask, const uint32_t NumRT, bool useAlternateOffset)
+{
+ assert(sample == 0); // will need up upate Raster Tile Color Offsets to support more than single sample here..
+
+ // type safety guaranteed from template instantiation in BEChooser<>::GetFunc
+ uint32_t rasterTileColorOffset = RasterTileColorOffset(sample);
+
+ if (useAlternateOffset)
+ {
+ rasterTileColorOffset += sizeof(simdscalar);
+ }
+
+ simdvector blendSrc;
+ simdvector blendOut;
+
+ for (uint32_t rt = 0; rt < NumRT; ++rt)
+ {
+ simdscalar *pColorSample = reinterpret_cast<simdscalar *>(pColorBase[rt] + rasterTileColorOffset);
+
+ const SWR_RENDER_TARGET_BLEND_STATE *pRTBlend = &pBlendState->renderTarget[rt];
+ // pfnBlendFunc may not update all channels. Initialize with PS output.
+ /// TODO: move this into the blend JIT.
+ blendOut = psContext.shaded[rt];
+
+ blendSrc[0] = pColorSample[0];
+ blendSrc[1] = pColorSample[2];
+ blendSrc[2] = pColorSample[4];
+ blendSrc[3] = pColorSample[6];
+
+ // Blend outputs and update coverage mask for alpha test
+ if (pfnBlendFunc[rt] != nullptr)
+ {
+ pfnBlendFunc[rt](
+ pBlendState,
+ psContext.shaded[rt],
+ psContext.shaded[1],
+ sample,
+ reinterpret_cast<uint8_t *>(&blendSrc),
+ blendOut,
+ &psContext.oMask,
+ reinterpret_cast<simdscalari *>(&coverageMask));
+ }
+
+ // final write mask
+ simdscalari outputMask = _simd_castps_si(_simd_and_ps(coverageMask, depthPassMask));
+
+ ///@todo can only use maskstore fast path if bpc is 32. Assuming hot tile is RGBA32_FLOAT.
+ static_assert(KNOB_COLOR_HOT_TILE_FORMAT == R32G32B32A32_FLOAT, "Unsupported hot tile format");
+
+ // store with color mask
+ if (!pRTBlend->writeDisableRed)
+ {
+ _simd_maskstore_ps(reinterpret_cast<float *>(&pColorSample[0]), outputMask, blendOut.x);
+ }
+ if (!pRTBlend->writeDisableGreen)
+ {
+ _simd_maskstore_ps(reinterpret_cast<float *>(&pColorSample[2]), outputMask, blendOut.y);
+ }
+ if (!pRTBlend->writeDisableBlue)
+ {
+ _simd_maskstore_ps(reinterpret_cast<float *>(&pColorSample[4]), outputMask, blendOut.z);
+ }
+ if (!pRTBlend->writeDisableAlpha)
+ {
+ _simd_maskstore_ps(reinterpret_cast<float *>(&pColorSample[6]), outputMask, blendOut.w);
+ }
+ }
+}
+
+#endif
template<uint32_t sampleCountT = SWR_MULTISAMPLE_1X, uint32_t samplePattern = SWR_MSAA_STANDARD_PATTERN,
uint32_t coverage = 0, uint32_t centroid = 0, uint32_t forced = 0, uint32_t canEarlyZ = 0>
struct SwrBackendTraits
UnrollerL<0, FormatTraits<DstFormat>::numComps, 1>::step(lambda);
}
+
+#if ENABLE_AVX512_SIMD16
+//////////////////////////////////////////////////////////////////////////
+/// @brief Load SIMD packed pixels in SOA format and converts to
+/// SOA RGBA32_FLOAT format.
+/// @param pSrc - source data in SOA form
+/// @param dst - output data in SOA form
+template<SWR_FORMAT SrcFormat>
+INLINE void LoadSOA(const uint8_t *pSrc, simd16vector &dst)
+{
+ // fast path for float32
+ if ((FormatTraits<SrcFormat>::GetType(0) == SWR_TYPE_FLOAT) && (FormatTraits<SrcFormat>::GetBPC(0) == 32))
+ {
+ auto lambda = [&](int comp)
+ {
+ simd16scalar vComp = _simd16_load_ps(reinterpret_cast<const float *>(pSrc + comp * sizeof(simd16scalar)));
+
+ dst.v[FormatTraits<SrcFormat>::swizzle(comp)] = vComp;
+ };
+
+ UnrollerL<0, FormatTraits<SrcFormat>::numComps, 1>::step(lambda);
+ return;
+ }
+
+ auto lambda = [&](int comp)
+ {
+ // load SIMD components
+ simd16scalar vComp = FormatTraits<SrcFormat>::loadSOA_16(comp, pSrc);
+
+ // unpack
+ vComp = FormatTraits<SrcFormat>::unpack(comp, vComp);
+
+ // convert
+ if (FormatTraits<SrcFormat>::isNormalized(comp))
+ {
+ vComp = _simd16_cvtepi32_ps(_simd16_castps_si(vComp));
+ vComp = _simd16_mul_ps(vComp, _simd16_set1_ps(FormatTraits<SrcFormat>::toFloat(comp)));
+ }
+
+ dst.v[FormatTraits<SrcFormat>::swizzle(comp)] = vComp;
+
+ pSrc += (FormatTraits<SrcFormat>::GetBPC(comp) * KNOB_SIMD16_WIDTH) / 8;
+ };
+
+ UnrollerL<0, FormatTraits<SrcFormat>::numComps, 1>::step(lambda);
+}
+
+//////////////////////////////////////////////////////////////////////////
+/// @brief Clamps the given component based on the requirements on the
+/// Format template arg
+/// @param vComp - SIMD vector of floats
+/// @param Component - component
+template<SWR_FORMAT Format>
+INLINE simd16scalar Clamp(simd16scalar vComp, uint32_t Component)
+{
+ if (FormatTraits<Format>::isNormalized(Component))
+ {
+ if (FormatTraits<Format>::GetType(Component) == SWR_TYPE_UNORM)
+ {
+ vComp = _simd16_max_ps(vComp, _simd16_setzero_ps());
+ }
+
+ if (FormatTraits<Format>::GetType(Component) == SWR_TYPE_SNORM)
+ {
+ vComp = _simd16_max_ps(vComp, _simd16_set1_ps(-1.0f));
+ }
+ vComp = _simd16_min_ps(vComp, _simd16_set1_ps(1.0f));
+ }
+ else if (FormatTraits<Format>::GetBPC(Component) < 32)
+ {
+ if (FormatTraits<Format>::GetType(Component) == SWR_TYPE_UINT)
+ {
+ int iMax = (1 << FormatTraits<Format>::GetBPC(Component)) - 1;
+ int iMin = 0;
+ simd16scalari vCompi = _simd16_castps_si(vComp);
+ vCompi = _simd16_max_epu32(vCompi, _simd16_set1_epi32(iMin));
+ vCompi = _simd16_min_epu32(vCompi, _simd16_set1_epi32(iMax));
+ vComp = _simd16_castsi_ps(vCompi);
+ }
+ else if (FormatTraits<Format>::GetType(Component) == SWR_TYPE_SINT)
+ {
+ int iMax = (1 << (FormatTraits<Format>::GetBPC(Component) - 1)) - 1;
+ int iMin = -1 - iMax;
+ simd16scalari vCompi = _simd16_castps_si(vComp);
+ vCompi = _simd16_max_epi32(vCompi, _simd16_set1_epi32(iMin));
+ vCompi = _simd16_min_epi32(vCompi, _simd16_set1_epi32(iMax));
+ vComp = _simd16_castsi_ps(vCompi);
+ }
+ }
+
+ return vComp;
+}
+
+//////////////////////////////////////////////////////////////////////////
+/// @brief Normalize the given component based on the requirements on the
+/// Format template arg
+/// @param vComp - SIMD vector of floats
+/// @param Component - component
+template<SWR_FORMAT Format>
+INLINE simd16scalar Normalize(simd16scalar vComp, uint32_t Component)
+{
+ if (FormatTraits<Format>::isNormalized(Component))
+ {
+ vComp = _simd16_mul_ps(vComp, _simd16_set1_ps(FormatTraits<Format>::fromFloat(Component)));
+ vComp = _simd16_castsi_ps(_simd16_cvtps_epi32(vComp));
+ }
+ return vComp;
+}
+
+//////////////////////////////////////////////////////////////////////////
+/// @brief Convert and store simdvector of pixels in SOA
+/// RGBA32_FLOAT to SOA format
+/// @param src - source data in SOA form
+/// @param dst - output data in SOA form
+template<SWR_FORMAT DstFormat>
+INLINE void StoreSOA(const simd16vector &src, uint8_t *pDst)
+{
+ // fast path for float32
+ if ((FormatTraits<DstFormat>::GetType(0) == SWR_TYPE_FLOAT) && (FormatTraits<DstFormat>::GetBPC(0) == 32))
+ {
+ for (uint32_t comp = 0; comp < FormatTraits<DstFormat>::numComps; ++comp)
+ {
+ simd16scalar vComp = src.v[FormatTraits<DstFormat>::swizzle(comp)];
+
+ // Gamma-correct
+ if (FormatTraits<DstFormat>::isSRGB)
+ {
+ if (comp < 3) // Input format is always RGBA32_FLOAT.
+ {
+ vComp = FormatTraits<R32G32B32A32_FLOAT>::convertSrgb(comp, vComp);
+ }
+ }
+
+ _simd16_store_ps(reinterpret_cast<float *>(pDst + comp * sizeof(simd16scalar)), vComp);
+ }
+ return;
+ }
+
+ auto lambda = [&](int comp)
+ {
+ simd16scalar vComp = src.v[FormatTraits<DstFormat>::swizzle(comp)];
+
+ // Gamma-correct
+ if (FormatTraits<DstFormat>::isSRGB)
+ {
+ if (comp < 3) // Input format is always RGBA32_FLOAT.
+ {
+ vComp = FormatTraits<R32G32B32A32_FLOAT>::convertSrgb(comp, vComp);
+ }
+ }
+
+ // clamp
+ vComp = Clamp<DstFormat>(vComp, comp);
+
+ // normalize
+ vComp = Normalize<DstFormat>(vComp, comp);
+
+ // pack
+ vComp = FormatTraits<DstFormat>::pack(comp, vComp);
+
+ // store
+ FormatTraits<DstFormat>::storeSOA(comp, pDst, vComp);
+
+ pDst += (FormatTraits<DstFormat>::GetBPC(comp) * KNOB_SIMD16_WIDTH) / 8;
+ };
+
+ UnrollerL<0, FormatTraits<DstFormat>::numComps, 1>::step(lambda);
+}
+
+#endif
static void storeSOA(uint8_t *pDst, simdscalar src) = delete;
static simdscalar unpack(simdscalar &in) = delete;
static simdscalar pack(simdscalar &in) = delete;
+#if ENABLE_AVX512_SIMD16
+ static simd16scalar loadSOA_16(const uint8_t *pSrc) = delete;
+ static void storeSOA(uint8_t *pDst, simd16scalar src) = delete;
+ static simd16scalar unpack(simd16scalar &in) = delete;
+ static simd16scalar pack(simd16scalar &in) = delete;
+#endif
};
//////////////////////////////////////////////////////////////////////////
static void storeSOA(uint8_t *pDst, simdscalar src) { return; }
static simdscalar unpack(simdscalar &in) { return _simd_setzero_ps(); }
static simdscalar pack(simdscalar &in) { return _simd_setzero_ps(); }
+#if ENABLE_AVX512_SIMD16
+ static simd16scalar loadSOA_16(const uint8_t *pSrc) { return _simd16_setzero_ps(); }
+ static void storeSOA(uint8_t *pDst, simd16scalar src) { return; }
+ static simd16scalar unpack(simd16scalar &in) { return _simd16_setzero_ps(); }
+ static simd16scalar pack(simd16scalar &in) { return _simd16_setzero_ps(); }
+#endif
};
-
//////////////////////////////////////////////////////////////////////////
/// PackTraits - Helpers for packing / unpacking 8 bit unsigned channels
//////////////////////////////////////////////////////////////////////////
result.lo = _mm256_cvtepu8_epi32(src);
- result.hi = _mm256_cvtepu8_epi32(_mm_bsrli_si128(src, 8));
+ result.hi = _mm256_cvtepu8_epi32(_mm_srli_si128(src, 8));
return _simd_castsi_ps(result);
#endif
#error Unsupported vector width
#endif
}
+#if ENABLE_AVX512_SIMD16
+
+ static simd16scalar loadSOA_16(const uint8_t *pSrc)
+ {
+ simd16scalar result = _simd16_setzero_ps();
+ simdscalar resultlo = _simd_setzero_ps();
+
+ const __m128 src = _mm_load_ps(reinterpret_cast<const float *>(pSrc));
+
+ resultlo = _mm256_insertf128_ps(resultlo, src, 0);
+ result = _simd16_insert_ps(result, resultlo, 0);
+
+ return result;
+ }
+
+ static void storeSOA(uint8_t *pDst, simd16scalar src)
+ {
+ // store simd16 bytes
+ _mm_store_ps(reinterpret_cast<float *>(pDst), _mm256_castps256_ps128(_simd16_extract_ps(src, 0)));
+ }
+
+ static simd16scalar unpack(simd16scalar &in)
+ {
+ simd16scalari result = _simd16_setzero_si();
+
+ __m128i src = _mm_castps_si128(_mm256_castps256_ps128(_simd16_extract_ps(in, 0)));
+
+ result = _simd16_insert_si(result, _simd_cvtepu8_epi32(src), 0);
+ result = _simd16_insert_si(result, _simd_cvtepu8_epi32(_mm_srli_si128(src, 8)), 1);
+
+ return _simd16_castsi_ps(result);
+ }
+
+ static simd16scalar pack(simd16scalar &in)
+ {
+ simd16scalari result = _simd16_setzero_si();
+ simdscalari resultlo = _simd_setzero_si();
+
+ __m128i templo = _mm_packus_epi32(_mm256_castsi256_si128(_mm256_castps_si256(_simd16_extract_ps(in, 0))), _mm256_extractf128_si256(_mm256_castps_si256(_simd16_extract_ps(in, 0)), 1));
+ __m128i temphi = _mm_packus_epi32(_mm256_castsi256_si128(_mm256_castps_si256(_simd16_extract_ps(in, 1))), _mm256_extractf128_si256(_mm256_castps_si256(_simd16_extract_ps(in, 1)), 1));
+
+ __m128i temp = _mm_packus_epi16(templo, temphi);
+
+ resultlo = _mm256_inserti128_si256(resultlo, temp, 0);
+ result = _simd16_insert_si(result, resultlo, 0);
+
+ return _simd16_castsi_ps(result);
+ }
+#endif
};
//////////////////////////////////////////////////////////////////////////
result.lo = _mm256_cvtepu8_epi32(src);
- result.hi = _mm256_cvtepu8_epi32(_mm_bsrli_si128(src, 8));
+ result.hi = _mm256_cvtepu8_epi32(_mm_srli_si128(src, 8));
return _simd_castsi_ps(result);
#endif
#error Unsupported vector width
#endif
}
+#if ENABLE_AVX512_SIMD16
+
+ static simd16scalar loadSOA_16(const uint8_t *pSrc)
+ {
+ simd16scalar result = _simd16_setzero_ps();
+ simdscalar resultlo = _simd_setzero_ps();
+
+ const __m128 src = _mm_load_ps(reinterpret_cast<const float *>(pSrc));
+
+ resultlo = _mm256_insertf128_ps(resultlo, src, 0);
+ result = _simd16_insert_ps(result, resultlo, 0);
+
+ return result;
+ }
+
+ static void storeSOA(uint8_t *pDst, simd16scalar src)
+ {
+ // store simd16 bytes
+ _mm_store_ps(reinterpret_cast<float *>(pDst), _mm256_castps256_ps128(_simd16_extract_ps(src, 0)));
+ }
+
+ static simd16scalar unpack(simd16scalar &in)
+ {
+ simd16scalari result = _simd16_setzero_si();
+
+ __m128i src = _mm_castps_si128(_mm256_castps256_ps128(_simd16_extract_ps(in, 0)));
+
+ result = _simd16_insert_si(result, _simd_cvtepu8_epi32(src), 0);
+ result = _simd16_insert_si(result, _simd_cvtepu8_epi32(_mm_srli_si128(src, 8)), 1);
+
+ return _simd16_castsi_ps(result);
+ }
+
+ static simd16scalar pack(simd16scalar &in)
+ {
+ simd16scalari result = _simd16_setzero_si();
+ simdscalari resultlo = _simd_setzero_si();
+
+ __m128i templo = _mm_packs_epi32(_mm256_castsi256_si128(_mm256_castps_si256(_simd16_extract_ps(in, 0))), _mm256_extractf128_si256(_mm256_castps_si256(_simd16_extract_ps(in, 0)), 1));
+ __m128i temphi = _mm_packs_epi32(_mm256_castsi256_si128(_mm256_castps_si256(_simd16_extract_ps(in, 1))), _mm256_extractf128_si256(_mm256_castps_si256(_simd16_extract_ps(in, 1)), 1));
+
+ __m128i temp = _mm_packs_epi16(templo, temphi);
+
+ resultlo = _mm256_inserti128_si256(resultlo, temp, 0);
+ result = _simd16_insert_si(result, resultlo, 0);
+
+ return _simd16_castsi_ps(result);
+ }
+#endif
};
//////////////////////////////////////////////////////////////////////////
#error Unsupported vector width
#endif
}
+#if ENABLE_AVX512_SIMD16
+
+ static simd16scalar loadSOA_16(const uint8_t *pSrc)
+ {
+ simd16scalar result = _simd16_setzero_ps();
+
+ simdscalar resultlo = _simd_load_ps(reinterpret_cast<const float *>(pSrc));
+
+ result = _simd16_insert_ps(result, resultlo, 0);
+
+ return result;
+ }
+
+ static void storeSOA(uint8_t *pDst, simd16scalar src)
+ {
+ _simd_store_ps(reinterpret_cast<float *>(pDst), _simd16_extract_ps(src, 0));
+ }
+
+ static simd16scalar unpack(simd16scalar &in)
+ {
+ simd16scalari result = _simd16_setzero_si();
+
+ result = _simd16_insert_si(result, _simd_cvtepu16_epi32(_mm256_extracti128_si256(_mm256_castps_si256(_simd16_extract_ps(in, 0)), 0)), 0);
+ result = _simd16_insert_si(result, _simd_cvtepu16_epi32(_mm256_extracti128_si256(_mm256_castps_si256(_simd16_extract_ps(in, 0)), 1)), 1);
+
+ return _simd16_castsi_ps(result);
+ }
+
+ static simd16scalar pack(simd16scalar &in)
+ {
+ simd16scalari result = _simd16_setzero_si();
+
+ simdscalari inlo = _simd_castps_si(_simd16_extract_ps(in, 0));
+ simdscalari inhi = _simd_castps_si(_simd16_extract_ps(in, 1));
+
+ simdscalari templo = _simd_permute2f128_si(inlo, inhi, 0x20);
+ simdscalari temphi = _simd_permute2f128_si(inlo, inhi, 0x31);
+
+ result = _simd16_insert_si(result, _simd_packus_epi32(templo, temphi), 0);
+
+ return _simd16_castsi_ps(result);
+ }
+#endif
};
//////////////////////////////////////////////////////////////////////////
#error Unsupported vector width
#endif
}
+#if ENABLE_AVX512_SIMD16
+
+ static simd16scalar loadSOA_16(const uint8_t *pSrc)
+ {
+ simd16scalar result = _simd16_setzero_ps();
+
+ simdscalar resultlo = _simd_load_ps(reinterpret_cast<const float *>(pSrc));
+
+ result = _simd16_insert_ps(result, resultlo, 0);
+
+ return result;
+ }
+
+ static void storeSOA(uint8_t *pDst, simd16scalar src)
+ {
+ _simd_store_ps(reinterpret_cast<float *>(pDst), _simd16_extract_ps(src, 0));
+ }
+
+ static simd16scalar unpack(simd16scalar &in)
+ {
+ simd16scalari result = _simd16_setzero_si();
+
+ result = _simd16_insert_si(result, _simd_cvtepu16_epi32(_mm256_extracti128_si256(_mm256_castps_si256(_simd16_extract_ps(in, 0)), 0)), 0);
+ result = _simd16_insert_si(result, _simd_cvtepu16_epi32(_mm256_extracti128_si256(_mm256_castps_si256(_simd16_extract_ps(in, 0)), 1)), 1);
+
+ return _simd16_castsi_ps(result);
+ }
+
+ static simd16scalar pack(simd16scalar &in)
+ {
+ simd16scalari result = _simd16_setzero_si();
+
+ simdscalari inlo = _simd_castps_si(_simd16_extract_ps(in, 0));
+ simdscalari inhi = _simd_castps_si(_simd16_extract_ps(in, 1));
+
+ simdscalari templo = _simd_permute2f128_si(inlo, inhi, 0x20);
+ simdscalari temphi = _simd_permute2f128_si(inlo, inhi, 0x31);
+
+ result = _simd16_insert_si(result, _simd_packus_epi32(templo, temphi), 0);
+
+ return _simd16_castsi_ps(result);
+ }
+#endif
};
//////////////////////////////////////////////////////////////////////////
static void storeSOA(uint8_t *pDst, simdscalar src) { _simd_store_ps((float*)pDst, src); }
static simdscalar unpack(simdscalar &in) { return in; }
static simdscalar pack(simdscalar &in) { return in; }
+#if ENABLE_AVX512_SIMD16
+
+ static simd16scalar loadSOA_16(const uint8_t *pSrc)
+ {
+ return _simd16_load_ps(reinterpret_cast<const float *>(pSrc));
+ }
+
+ static void storeSOA(uint8_t *pDst, simd16scalar src)
+ {
+ _simd16_store_ps(reinterpret_cast<float *>(pDst), src);
+ }
+
+ static simd16scalar unpack(simd16scalar &in)
+ {
+ return in;
+ }
+
+ static simd16scalar pack(simd16scalar &in)
+ {
+ return in;
+ }
+#endif
};
//////////////////////////////////////////////////////////////////////////
{
float *f = (float *)(&Base);
- return _mm_set_ps(powf(f[0], Exp),
- powf(f[1], Exp),
+ return _mm_set_ps(powf(f[3], Exp),
powf(f[2], Exp),
- powf(f[3], Exp));
+ powf(f[1], Exp),
+ powf(f[0], Exp));
}
static inline __m128 ConvertFloatToSRGB2(__m128& Src)
return Result;
}
+#if ENABLE_AVX512_SIMD16
+template< unsigned expnum, unsigned expden, unsigned coeffnum, unsigned coeffden >
+inline static simd16scalar fastpow(simd16scalar value)
+{
+ static const float factor1 = exp2(127.0f * expden / expnum - 127.0f)
+ * powf(1.0f * coeffnum / coeffden, 1.0f * expden / expnum);
+
+ // Apply a constant pre-correction factor.
+ simd16scalar result = _simd16_mul_ps(value, _simd16_set1_ps(factor1));
+
+ // Reinterpret arg as integer to obtain logarithm.
+ //asm("cvtdq2ps %1, %0" : "=x" (result) : "x" (result));
+ result = _simd16_cvtepi32_ps(_simd16_castps_si(result));
+
+ // Multiply logarithm by power.
+ result = _simd16_mul_ps(result, _simd16_set1_ps(1.0f * expnum / expden));
+
+ // Convert back to "integer" to exponentiate.
+ //asm("cvtps2dq %1, %0" : "=x" (result) : "x" (result));
+ result = _simd16_castsi_ps(_simd16_cvtps_epi32(result));
+
+ return result;
+}
+
+inline static simd16scalar pow512_4(simd16scalar arg)
+{
+ // 5/12 is too small, so compute the 4th root of 20/12 instead.
+ // 20/12 = 5/3 = 1 + 2/3 = 2 - 1/3. 2/3 is a suitable argument for fastpow.
+ // weighting coefficient: a^-1/2 = 2 a; a = 2^-2/3
+ simd16scalar xf = fastpow< 2, 3, int(0.629960524947437 * 1e9), int(1e9) >(arg);
+ simd16scalar xover = _simd16_mul_ps(arg, xf);
+
+ simd16scalar xfm1 = _simd16_rsqrt_ps(xf);
+ simd16scalar x2 = _simd16_mul_ps(arg, arg);
+ simd16scalar xunder = _simd16_mul_ps(x2, xfm1);
+
+ // sqrt2 * over + 2 * sqrt2 * under
+ simd16scalar xavg = _simd16_mul_ps(_simd16_set1_ps(1.0f / (3.0f * 0.629960524947437f) * 0.999852f), _simd16_add_ps(xover, xunder));
+
+ xavg = _simd16_mul_ps(xavg, _simd16_rsqrt_ps(xavg));
+ xavg = _simd16_mul_ps(xavg, _simd16_rsqrt_ps(xavg));
+
+ return xavg;
+}
+
+inline static simd16scalar powf_wrapper(const simd16scalar base, float exp)
+{
+ const float *f = reinterpret_cast<const float *>(&base);
+
+ return _simd16_set_ps(
+ powf(f[15], exp),
+ powf(f[14], exp),
+ powf(f[13], exp),
+ powf(f[12], exp),
+ powf(f[11], exp),
+ powf(f[10], exp),
+ powf(f[ 9], exp),
+ powf(f[ 8], exp),
+ powf(f[ 7], exp),
+ powf(f[ 6], exp),
+ powf(f[ 5], exp),
+ powf(f[ 4], exp),
+ powf(f[ 3], exp),
+ powf(f[ 2], exp),
+ powf(f[ 1], exp),
+ powf(f[ 0], exp)
+ );
+}
+
+// float to SRGB conversion formula
+//
+// if (value < 0.0031308f)
+// value *= 12.92f;
+// else
+// value = 1.055f * pow(value, 1.0f / 2.4f) - 0.055f;
+//
+static inline simd16scalar ConvertFloatToSRGB2(const simd16scalar &value)
+{
+ // create a mask where the source is < the minimal SRGB float value
+ const simd16mask mask = _simd16_cmplt_ps_mask(value, _simd16_set1_ps(0.0031308f));
+
+ // if all elements are < the threshold, result = value * 12.92
+ simd16scalar result = _simd16_mul_ps(value, _simd16_set1_ps(12.92f));
+
+ if (_simd16_mask2int(mask) != 0xFFFF)
+ {
+ // some elements are >= threshold, result = 1.055 * power(value, 1.0 / 2.4) - 0.055
+#if KNOB_USE_FAST_SRGB == TRUE
+ // 1.0f / 2.4f is 5.0f / 12.0f which is used for approximation.
+ simd16scalar result2 = pow512_4(value);
+#else
+ simd16scalar result2 = powf_wrapper(value, 1.0f / 2.4f);
+#endif
+
+ result2 = _simd16_mul_ps(result2, _simd16_set1_ps(1.055f));
+ result2 = _simd16_sub_ps(result2, _simd16_set1_ps(0.055f));
+
+#if (KNOB_ARCH == KNOB_ARCH_AVX512)
+ // only native AVX512 can directly use the computed mask for the blend operation
+ result = _mm512_mask_blend_ps(mask, result2, result);
+#else
+ result = _simd16_blendv_ps(result2, result, _simd16_cmplt_ps(value, _simd16_set1_ps(0.0031308f)));
+#endif
+ }
+
+ return result;
+}
+
+#endif
//////////////////////////////////////////////////////////////////////////
/// TypeTraits - Format type traits specialization for FLOAT16
//////////////////////////////////////////////////////////////////////////
SWR_ASSERT(0); // @todo
return _simd_setzero_ps();
}
+#if ENABLE_AVX512_SIMD16
+
+ static simd16scalar pack(const simd16scalar &in)
+ {
+ simd16scalari result = _simd16_setzero_si();
+ simdscalari resultlo = _simd_setzero_si();
+
+#if (KNOB_ARCH == KNOB_ARCH_AVX)
+ simdscalar simdlo = pack(_simd16_extract_ps(in, 0));
+ simdscalar simdhi = pack(_simd16_extract_ps(in, 1));
+
+ __m128i templo = _mm256_extractf128_si256(_simd_castps_si(simdlo), 0);
+ __m128i temphi = _mm256_extractf128_si256(_simd_castps_si(simdhi), 0);
+
+#else
+ __m128i templo = _mm256_cvtps_ph(_simd16_extract_ps(in, 0), _MM_FROUND_TRUNC);
+ __m128i temphi = _mm256_cvtps_ph(_simd16_extract_ps(in, 1), _MM_FROUND_TRUNC);
+
+#endif
+ resultlo = _mm256_insertf128_si256(resultlo, templo, 0);
+ resultlo = _mm256_insertf128_si256(resultlo, temphi, 1);
+
+ result = _simd16_insert_si(result, resultlo, 0);
+
+ return _simd16_castsi_ps(result);
+ }
+
+ static simd16scalar unpack(const simd16scalar &in)
+ {
+ // input is 16 packed float16, output is 16 packed float32
+ SWR_ASSERT(0); // @todo
+ return _simd16_setzero_ps();
+ }
+#endif
};
//////////////////////////////////////////////////////////////////////////
#endif
return in;
}
+#if ENABLE_AVX512_SIMD16
+
+ static inline simd16scalar convertSrgb(simd16scalar &in)
+ {
+ return ConvertFloatToSRGB2(in);
+ }
+#endif
};
//////////////////////////////////////////////////////////////////////////
SWR_ASSERT(0);
return TypeTraits<X, NumBitsX>::convertSrgb(in);
}
+#if ENABLE_AVX512_SIMD16
+
+ INLINE static simd16scalar loadSOA_16(uint32_t comp, const uint8_t* pSrc)
+ {
+ switch (comp)
+ {
+ case 0:
+ return TypeTraits<X, NumBitsX>::loadSOA_16(pSrc);
+ case 1:
+ return TypeTraits<Y, NumBitsY>::loadSOA_16(pSrc);
+ case 2:
+ return TypeTraits<Z, NumBitsZ>::loadSOA_16(pSrc);
+ case 3:
+ return TypeTraits<W, NumBitsW>::loadSOA_16(pSrc);
+ }
+ SWR_ASSERT(0);
+ return TypeTraits<X, NumBitsX>::loadSOA_16(pSrc);
+ }
+
+ INLINE static void storeSOA(uint32_t comp, uint8_t *pDst, simd16scalar src)
+ {
+ switch (comp)
+ {
+ case 0:
+ TypeTraits<X, NumBitsX>::storeSOA(pDst, src);
+ return;
+ case 1:
+ TypeTraits<Y, NumBitsY>::storeSOA(pDst, src);
+ return;
+ case 2:
+ TypeTraits<Z, NumBitsZ>::storeSOA(pDst, src);
+ return;
+ case 3:
+ TypeTraits<W, NumBitsW>::storeSOA(pDst, src);
+ return;
+ }
+ SWR_ASSERT(0);
+ TypeTraits<X, NumBitsX>::storeSOA(pDst, src);
+ }
+
+ INLINE static simd16scalar unpack(uint32_t comp, simd16scalar &in)
+ {
+ switch (comp)
+ {
+ case 0:
+ return TypeTraits<X, NumBitsX>::unpack(in);
+ case 1:
+ return TypeTraits<Y, NumBitsY>::unpack(in);
+ case 2:
+ return TypeTraits<Z, NumBitsZ>::unpack(in);
+ case 3:
+ return TypeTraits<W, NumBitsW>::unpack(in);
+ }
+ SWR_ASSERT(0);
+ return TypeTraits<X, NumBitsX>::unpack(in);
+ }
+
+ INLINE static simd16scalar pack(uint32_t comp, simd16scalar &in)
+ {
+ switch (comp)
+ {
+ case 0:
+ return TypeTraits<X, NumBitsX>::pack(in);
+ case 1:
+ return TypeTraits<Y, NumBitsY>::pack(in);
+ case 2:
+ return TypeTraits<Z, NumBitsZ>::pack(in);
+ case 3:
+ return TypeTraits<W, NumBitsW>::pack(in);
+ }
+ SWR_ASSERT(0);
+ return TypeTraits<X, NumBitsX>::pack(in);
+ }
+
+ INLINE static simd16scalar convertSrgb(uint32_t comp, simd16scalar &in)
+ {
+ switch (comp)
+ {
+ case 0:
+ return TypeTraits<X, NumBitsX>::convertSrgb(in);
+ case 1:
+ return TypeTraits<Y, NumBitsY>::convertSrgb(in);
+ case 2:
+ return TypeTraits<Z, NumBitsZ>::convertSrgb(in);
+ case 3:
+ return TypeTraits<W, NumBitsW>::convertSrgb(in);
+ }
+ SWR_ASSERT(0);
+ return TypeTraits<X, NumBitsX>::convertSrgb(in);
+ }
+#endif
};
///////////////////////////////////////////////////////////////////////////////
#define ENABLE_AVX512_SIMD16 0
+#define USE_8x2_TILE_BACKEND 0
///////////////////////////////////////////////////////////////////////////////
// Architecture validation
#endif
#if ENABLE_AVX512_SIMD16
-#if KNOB_SIMD16_WIDTH==16 && KNOB_TILE_X_DIM < 4
+#if KNOB_SIMD16_WIDTH == 16 && KNOB_TILE_X_DIM < 8
#error "incompatible width/tile dimensions"
#endif
#endif
#if ENABLE_AVX512_SIMD16
#if KNOB_SIMD16_WIDTH == 16
-#define SIMD16_TILE_X_DIM 4
-#define SIMD16_TILE_Y_DIM 4
+#define SIMD16_TILE_X_DIM 8
+#define SIMD16_TILE_Y_DIM 2
#else
#error "Invalid simd width"
#endif
return &hotTile;
}
+#if USE_8x2_TILE_BACKEND
+void HotTileMgr::ClearColorHotTile(const HOTTILE* pHotTile) // clear a macro tile from float4 clear data.
+{
+ // Load clear color into SIMD register...
+ float *pClearData = (float *)(pHotTile->clearData);
+ simd16scalar valR = _simd16_broadcast_ss(&pClearData[0]);
+ simd16scalar valG = _simd16_broadcast_ss(&pClearData[1]);
+ simd16scalar valB = _simd16_broadcast_ss(&pClearData[2]);
+ simd16scalar valA = _simd16_broadcast_ss(&pClearData[3]);
+
+ float *pfBuf = (float *)pHotTile->pBuffer;
+ uint32_t numSamples = pHotTile->numSamples;
+
+ for (uint32_t row = 0; row < KNOB_MACROTILE_Y_DIM; row += KNOB_TILE_Y_DIM)
+ {
+ for (uint32_t col = 0; col < KNOB_MACROTILE_X_DIM; col += KNOB_TILE_X_DIM)
+ {
+ for (uint32_t si = 0; si < (KNOB_TILE_X_DIM * KNOB_TILE_Y_DIM * numSamples); si += SIMD16_TILE_X_DIM * SIMD16_TILE_Y_DIM)
+ {
+ _simd16_store_ps(pfBuf, valR);
+ pfBuf += KNOB_SIMD16_WIDTH;
+
+ _simd16_store_ps(pfBuf, valG);
+ pfBuf += KNOB_SIMD16_WIDTH;
+
+ _simd16_store_ps(pfBuf, valB);
+ pfBuf += KNOB_SIMD16_WIDTH;
+
+ _simd16_store_ps(pfBuf, valA);
+ pfBuf += KNOB_SIMD16_WIDTH;
+ }
+ }
+ }
+}
+
+void HotTileMgr::ClearDepthHotTile(const HOTTILE* pHotTile) // clear a macro tile from float4 clear data.
+{
+ // Load clear color into SIMD register...
+ float *pClearData = (float *)(pHotTile->clearData);
+ simd16scalar valZ = _simd16_broadcast_ss(&pClearData[0]);
+
+ float *pfBuf = (float *)pHotTile->pBuffer;
+ uint32_t numSamples = pHotTile->numSamples;
+
+ for (uint32_t row = 0; row < KNOB_MACROTILE_Y_DIM; row += KNOB_TILE_Y_DIM)
+ {
+ for (uint32_t col = 0; col < KNOB_MACROTILE_X_DIM; col += KNOB_TILE_X_DIM)
+ {
+ for (uint32_t si = 0; si < (KNOB_TILE_X_DIM * KNOB_TILE_Y_DIM * numSamples); si += SIMD16_TILE_X_DIM * SIMD16_TILE_Y_DIM)
+ {
+ _simd16_store_ps(pfBuf, valZ);
+ pfBuf += KNOB_SIMD16_WIDTH;
+ }
+ }
+ }
+}
+
+void HotTileMgr::ClearStencilHotTile(const HOTTILE* pHotTile)
+{
+ // convert from F32 to U8.
+ uint8_t clearVal = (uint8_t)(pHotTile->clearData[0]);
+ //broadcast 32x into __m256i...
+ simd16scalari valS = _simd16_set1_epi8(clearVal);
+
+ simd16scalari *pBuf = (simd16scalari *)pHotTile->pBuffer;
+ uint32_t numSamples = pHotTile->numSamples;
+
+ for (uint32_t row = 0; row < KNOB_MACROTILE_Y_DIM; row += KNOB_TILE_Y_DIM)
+ {
+ for (uint32_t col = 0; col < KNOB_MACROTILE_X_DIM; col += KNOB_TILE_X_DIM)
+ {
+ // We're putting 4 pixels in each of the 32-bit slots, so increment 4 times as quickly.
+ for (uint32_t si = 0; si < (KNOB_TILE_X_DIM * KNOB_TILE_Y_DIM * numSamples); si += SIMD16_TILE_X_DIM * SIMD16_TILE_Y_DIM * 4)
+ {
+ _simd16_store_si(pBuf, valS);
+ pBuf += 1;
+ }
+ }
+ }
+}
+
+#else
void HotTileMgr::ClearColorHotTile(const HOTTILE* pHotTile) // clear a macro tile from float4 clear data.
{
// Load clear color into SIMD register...
}
}
+#endif
//////////////////////////////////////////////////////////////////////////
/// @brief InitializeHotTiles
/// for draw calls, we initialize the active hot tiles and perform deferred
{
memcpy(pDst, pSrc, (bpp * KNOB_SIMD_WIDTH) / 8);
}
+#if ENABLE_AVX512_SIMD16
+
+ INLINE static void Transpose_16(const uint8_t* pSrc, uint8_t* pDst)
+ {
+ memcpy(pDst, pSrc, (bpp * KNOB_SIMD16_WIDTH) / 8);
+ }
+#endif
};
//////////////////////////////////////////////////////////////////////////
#error Unsupported vector width
#endif
}
+#if ENABLE_AVX512_SIMD16
+
+ INLINE static void Transpose_16(const uint8_t* pSrc, uint8_t* pDst)
+ {
+ simd16scalari src = _simd16_load_si(reinterpret_cast<const simd16scalari *>(pSrc));
+
+ simd16scalari mask0 = _simd16_set_epi32(0x0f078080, 0x0e068080, 0x0d058080, 0x0c048080, 0x80800b03, 0x80800a02, 0x80800901, 0x80800800);
+
+ simd16scalari dst01 = _simd16_shuffle_epi8(src, mask0);
+
+ simd16scalari perm1 = _simd16_permute2f128_si(src, src, 1);
+
+ simd16scalari mask1 = _simd16_set_epi32(0x80800f07, 0x80800e06, 0x80800d05, 0x80800c04, 0x0b038080, 0x0a028080, 0x09018080, 0x08008080);
+
+ simd16scalari dst23 = _simd16_shuffle_epi8(perm1, mask1);
+
+ simd16scalari dst = _simd16_or_si(dst01, dst23);
+
+ _simd16_store_si(reinterpret_cast<simd16scalari *>(pDst), dst);
+ }
+#endif
};
//////////////////////////////////////////////////////////////////////////
/// @param pSrc - source data in SOA form
/// @param pDst - output data in AOS form
INLINE static void Transpose(const uint8_t* pSrc, uint8_t* pDst) = delete;
+#if ENABLE_AVX512_SIMD16
+
+ INLINE static void Transpose_16(const uint8_t* pSrc, uint8_t* pDst) = delete;
+#endif
};
//////////////////////////////////////////////////////////////////////////
#error Unsupported vector width
#endif
}
+#if ENABLE_AVX512_SIMD16
+
+ INLINE static void Transpose_16(const uint8_t* pSrc, uint8_t* pDst)
+ {
+ __m256i src = _mm256_load_si256(reinterpret_cast<const __m256i *>(pSrc)); // rrrrrrrrrrrrrrrrgggggggggggggggg
+
+ __m256i r = _mm256_permute4x64_epi64(src, 0x50); // 0x50 = 01010000b // rrrrrrrrxxxxxxxxrrrrrrrrxxxxxxxx
+
+ __m256i g = _mm256_permute4x64_epi64(src, 0xFA); // 0xFA = 11111010b // ggggggggxxxxxxxxggggggggxxxxxxxx
+
+ __m256i dst = _mm256_unpacklo_epi8(r, g); // rgrgrgrgrgrgrgrgrgrgrgrgrgrgrgrg
+
+ _mm256_store_si256(reinterpret_cast<__m256i *>(pDst), dst);
+ }
+#endif
};
//////////////////////////////////////////////////////////////////////////
#error Unsupported vector width
#endif
}
+#if ENABLE_AVX512_SIMD16
+
+ INLINE static void Transpose_16(const uint8_t* pSrc, uint8_t* pDst)
+ {
+ simd16scalar src0 = _simd16_load_ps(reinterpret_cast<const float *>(pSrc));
+ simd16scalar src1 = _simd16_load_ps(reinterpret_cast<const float *>(pSrc) + 16);
+ simd16scalar src2 = _simd16_load_ps(reinterpret_cast<const float *>(pSrc) + 32);
+ simd16scalar src3 = _simd16_load_ps(reinterpret_cast<const float *>(pSrc) + 48);
+
+ __m128 vDst[8];
+
+ vTranspose4x8(vDst, _simd16_extract_ps(src0, 0), _simd16_extract_ps(src1, 0), _simd16_extract_ps(src2, 0), _simd16_extract_ps(src3, 0));
+
+#if 1
+ _simd16_store_ps(reinterpret_cast<float *>(pDst) + 0, reinterpret_cast<simd16scalar *>(vDst)[0]);
+ _simd16_store_ps(reinterpret_cast<float *>(pDst) + 16, reinterpret_cast<simd16scalar *>(vDst)[1]);
+#else
+ _mm_store_ps(reinterpret_cast<float *>(pDst), vDst[0]);
+ _mm_store_ps(reinterpret_cast<float *>(pDst) + 4, vDst[1]);
+ _mm_store_ps(reinterpret_cast<float *>(pDst) + 8, vDst[2]);
+ _mm_store_ps(reinterpret_cast<float *>(pDst) + 12, vDst[3]);
+ _mm_store_ps(reinterpret_cast<float *>(pDst) + 16, vDst[4]);
+ _mm_store_ps(reinterpret_cast<float *>(pDst) + 20, vDst[5]);
+ _mm_store_ps(reinterpret_cast<float *>(pDst) + 24, vDst[6]);
+ _mm_store_ps(reinterpret_cast<float *>(pDst) + 28, vDst[7]);
+#endif
+
+ vTranspose4x8(vDst, _simd16_extract_ps(src0, 1), _simd16_extract_ps(src1, 1), _simd16_extract_ps(src2, 1), _simd16_extract_ps(src3, 1));
+
+#if 1
+ _simd16_store_ps(reinterpret_cast<float *>(pDst) + 32, reinterpret_cast<simd16scalar *>(vDst)[2]);
+ _simd16_store_ps(reinterpret_cast<float *>(pDst) + 48, reinterpret_cast<simd16scalar *>(vDst)[3]);
+#else
+ _mm_store_ps(reinterpret_cast<float *>(pDst) + 32, vDst[0]);
+ _mm_store_ps(reinterpret_cast<float *>(pDst) + 36, vDst[1]);
+ _mm_store_ps(reinterpret_cast<float *>(pDst) + 40, vDst[2]);
+ _mm_store_ps(reinterpret_cast<float *>(pDst) + 44, vDst[3]);
+ _mm_store_ps(reinterpret_cast<float *>(pDst) + 48, vDst[4]);
+ _mm_store_ps(reinterpret_cast<float *>(pDst) + 52, vDst[5]);
+ _mm_store_ps(reinterpret_cast<float *>(pDst) + 56, vDst[6]);
+ _mm_store_ps(reinterpret_cast<float *>(pDst) + 60, vDst[7]);
+#endif
+ }
+#endif
};
//////////////////////////////////////////////////////////////////////////
#error Unsupported vector width
#endif
}
+#if ENABLE_AVX512_SIMD16
+
+ INLINE static void Transpose_16(const uint8_t* pSrc, uint8_t* pDst)
+ {
+ simd16scalar src0 = _simd16_load_ps(reinterpret_cast<const float *>(pSrc));
+ simd16scalar src1 = _simd16_load_ps(reinterpret_cast<const float *>(pSrc) + 16);
+ simd16scalar src2 = _simd16_load_ps(reinterpret_cast<const float *>(pSrc) + 32);
+
+ __m128 vDst[8];
+
+ vTranspose3x8(vDst, _simd16_extract_ps(src0, 0), _simd16_extract_ps(src1, 0), _simd16_extract_ps(src2, 0));
+
+#if 1
+ _simd16_store_ps(reinterpret_cast<float *>(pDst) + 0, reinterpret_cast<simd16scalar *>(vDst)[0]);
+ _simd16_store_ps(reinterpret_cast<float *>(pDst) + 16, reinterpret_cast<simd16scalar *>(vDst)[1]);
+#else
+ _mm_store_ps(reinterpret_cast<float *>(pDst), vDst[0]);
+ _mm_store_ps(reinterpret_cast<float *>(pDst) + 4, vDst[1]);
+ _mm_store_ps(reinterpret_cast<float *>(pDst) + 8, vDst[2]);
+ _mm_store_ps(reinterpret_cast<float *>(pDst) + 12, vDst[3]);
+ _mm_store_ps(reinterpret_cast<float *>(pDst) + 16, vDst[4]);
+ _mm_store_ps(reinterpret_cast<float *>(pDst) + 20, vDst[5]);
+ _mm_store_ps(reinterpret_cast<float *>(pDst) + 24, vDst[6]);
+ _mm_store_ps(reinterpret_cast<float *>(pDst) + 28, vDst[7]);
+#endif
+
+ vTranspose3x8(vDst, _simd16_extract_ps(src0, 1), _simd16_extract_ps(src1, 1), _simd16_extract_ps(src2, 1));
+
+#if 1
+ _simd16_store_ps(reinterpret_cast<float *>(pDst) + 32, reinterpret_cast<simd16scalar *>(vDst)[2]);
+ _simd16_store_ps(reinterpret_cast<float *>(pDst) + 48, reinterpret_cast<simd16scalar *>(vDst)[3]);
+#else
+ _mm_store_ps(reinterpret_cast<float *>(pDst) + 32, vDst[0]);
+ _mm_store_ps(reinterpret_cast<float *>(pDst) + 36, vDst[1]);
+ _mm_store_ps(reinterpret_cast<float *>(pDst) + 40, vDst[2]);
+ _mm_store_ps(reinterpret_cast<float *>(pDst) + 44, vDst[3]);
+ _mm_store_ps(reinterpret_cast<float *>(pDst) + 48, vDst[4]);
+ _mm_store_ps(reinterpret_cast<float *>(pDst) + 52, vDst[5]);
+ _mm_store_ps(reinterpret_cast<float *>(pDst) + 56, vDst[6]);
+ _mm_store_ps(reinterpret_cast<float *>(pDst) + 60, vDst[7]);
+#endif
+ }
+#endif
};
//////////////////////////////////////////////////////////////////////////
#error Unsupported vector width
#endif
}
+#if ENABLE_AVX512_SIMD16
+
+ INLINE static void Transpose_16(const uint8_t* pSrc, uint8_t* pDst)
+ {
+ const float *pfSrc = reinterpret_cast<const float *>(pSrc);
+
+ __m256 src_r0 = _mm256_load_ps(pfSrc + 0);
+ __m256 src_r1 = _mm256_load_ps(pfSrc + 8);
+ __m256 src_g0 = _mm256_load_ps(pfSrc + 16);
+ __m256 src_g1 = _mm256_load_ps(pfSrc + 24);
+
+ __m256 dst0 = _mm256_unpacklo_ps(src_r0, src_g0);
+ __m256 dst1 = _mm256_unpackhi_ps(src_r0, src_g0);
+ __m256 dst2 = _mm256_unpacklo_ps(src_r1, src_g1);
+ __m256 dst3 = _mm256_unpackhi_ps(src_r1, src_g1);
+
+ float *pfDst = reinterpret_cast<float *>(pDst);
+
+ _mm256_store_ps(pfDst + 0, dst0);
+ _mm256_store_ps(pfDst + 8, dst1);
+ _mm256_store_ps(pfDst + 16, dst2);
+ _mm256_store_ps(pfDst + 24, dst3);
+ }
+#endif
};
//////////////////////////////////////////////////////////////////////////
#error Unsupported vector width
#endif
}
+#if ENABLE_AVX512_SIMD16
+
+ INLINE static void Transpose_16(const uint8_t* pSrc, uint8_t* pDst)
+ {
+ simd16scalari src_rg = _simd16_load_si(reinterpret_cast<const simd16scalari *>(pSrc));
+ simd16scalari src_ba = _simd16_load_si(reinterpret_cast<const simd16scalari *>(pSrc + sizeof(simd16scalari)));
+
+ __m256i src_r = _simd16_extract_si(src_rg, 0);
+ __m256i src_g = _simd16_extract_si(src_rg, 1);
+ __m256i src_b = _simd16_extract_si(src_ba, 0);
+ __m256i src_a = _simd16_extract_si(src_ba, 1);
+
+ __m256i rg0 = _mm256_unpacklo_epi16(src_r, src_g);
+ __m256i rg1 = _mm256_unpackhi_epi16(src_r, src_g);
+ __m256i ba0 = _mm256_unpacklo_epi16(src_b, src_a);
+ __m256i ba1 = _mm256_unpackhi_epi16(src_b, src_a);
+
+ __m256i dst0 = _mm256_unpacklo_epi32(rg0, ba0);
+ __m256i dst1 = _mm256_unpackhi_epi32(rg0, ba0);
+ __m256i dst2 = _mm256_unpacklo_epi32(rg1, ba1);
+ __m256i dst3 = _mm256_unpackhi_epi32(rg1, ba1);
+
+ _mm256_store_si256(reinterpret_cast<__m256i*>(pDst) + 0, dst0);
+ _mm256_store_si256(reinterpret_cast<__m256i*>(pDst) + 1, dst1);
+ _mm256_store_si256(reinterpret_cast<__m256i*>(pDst) + 2, dst2);
+ _mm256_store_si256(reinterpret_cast<__m256i*>(pDst) + 3, dst3);
+ }
+#endif
};
//////////////////////////////////////////////////////////////////////////
#error Unsupported vector width
#endif
}
+#if ENABLE_AVX512_SIMD16
+
+ INLINE static void Transpose_16(const uint8_t* pSrc, uint8_t* pDst)
+ {
+ simd16scalari src_rg = _simd16_load_si(reinterpret_cast<const simd16scalari *>(pSrc));
+
+ __m256i src_r = _simd16_extract_si(src_rg, 0);
+ __m256i src_g = _simd16_extract_si(src_rg, 1);
+ __m256i src_b = _mm256_load_si256(reinterpret_cast<const __m256i *>(pSrc + sizeof(simd16scalari)));
+ __m256i src_a = _mm256_undefined_si256();
+
+ __m256i rg0 = _mm256_unpacklo_epi16(src_r, src_g);
+ __m256i rg1 = _mm256_unpackhi_epi16(src_r, src_g);
+ __m256i ba0 = _mm256_unpacklo_epi16(src_b, src_a);
+ __m256i ba1 = _mm256_unpackhi_epi16(src_b, src_a);
+
+ __m256i dst0 = _mm256_unpacklo_epi32(rg0, ba0);
+ __m256i dst1 = _mm256_unpackhi_epi32(rg0, ba0);
+ __m256i dst2 = _mm256_unpacklo_epi32(rg1, ba1);
+ __m256i dst3 = _mm256_unpackhi_epi32(rg1, ba1);
+
+ _mm256_store_si256(reinterpret_cast<__m256i*>(pDst) + 0, dst0);
+ _mm256_store_si256(reinterpret_cast<__m256i*>(pDst) + 1, dst1);
+ _mm256_store_si256(reinterpret_cast<__m256i*>(pDst) + 2, dst2);
+ _mm256_store_si256(reinterpret_cast<__m256i*>(pDst) + 3, dst3);
+ }
+#endif
};
//////////////////////////////////////////////////////////////////////////
#error Unsupported vector width
#endif
}
+#if ENABLE_AVX512_SIMD16
+
+ INLINE static void Transpose_16(const uint8_t* pSrc, uint8_t* pDst)
+ {
+ simd16scalari result = _simd16_setzero_si();
+
+ simd16scalari src = _simd16_castps_si(_simd16_load_ps(reinterpret_cast<const float *>(pSrc)));
+
+ simdscalari srclo = _simd16_extract_si(src, 0);
+ simdscalari srchi = _simd16_extract_si(src, 1);
+
+ result = _simd16_insert_si(result, _mm256_unpacklo_epi16(srclo, srchi), 0);
+ result = _simd16_insert_si(result, _mm256_unpackhi_epi16(srclo, srchi), 1);
+
+ _simd16_store_si(reinterpret_cast<simd16scalari *>(pDst), result);
+ }
+#endif
};
//////////////////////////////////////////////////////////////////////////
/// @param pSrc - source data in SOA form
/// @param pDst - output data in AOS form
static void Transpose(const uint8_t* pSrc, uint8_t* pDst) = delete;
+#if ENABLE_AVX512_SIMD16
+
+ static void Transpose_16(const uint8_t* pSrc, uint8_t* pDst) = delete;
+#endif
};
//////////////////////////////////////////////////////////////////////////
/// @param pSrc - source data in SOA form
/// @param pDst - output data in AOS form
static void Transpose(const uint8_t* pSrc, uint8_t* pDst) = delete;
-};
-
+#if ENABLE_AVX512_SIMD16
+ static void Transpose_16(const uint8_t* pSrc, uint8_t* pDst) = delete;
+#endif
+};
//////////////////////////////////////////////////////////////////////////
/// Transpose4_4_4_4
/// @param pSrc - source data in SOA form
/// @param pDst - output data in AOS form
static void Transpose(const uint8_t* pSrc, uint8_t* pDst) = delete;
+#if ENABLE_AVX512_SIMD16
+
+ static void Transpose_16(const uint8_t* pSrc, uint8_t* pDst) = delete;
+#endif
};
//////////////////////////////////////////////////////////////////////////
/// @param pSrc - source data in SOA form
/// @param pDst - output data in AOS form
static void Transpose(const uint8_t* pSrc, uint8_t* pDst) = delete;
+#if ENABLE_AVX512_SIMD16
+
+ static void Transpose_16(const uint8_t* pSrc, uint8_t* pDst) = delete;
+#endif
};
//////////////////////////////////////////////////////////////////////////
/// @param pSrc - source data in SOA form
/// @param pDst - output data in AOS form
static void Transpose(const uint8_t* pSrc, uint8_t* pDst) = delete;
+#if ENABLE_AVX512_SIMD16
+
+ static void Transpose_16(const uint8_t* pSrc, uint8_t* pDst) = delete;
+#endif
};
//////////////////////////////////////////////////////////////////////////
/// @param pSrc - source data in SOA form
/// @param pDst - output data in AOS form
static void Transpose(const uint8_t* pSrc, uint8_t* pDst) = delete;
+#if ENABLE_AVX512_SIMD16
+
+ static void Transpose_16(const uint8_t* pSrc, uint8_t* pDst) = delete;
+#endif
};
//////////////////////////////////////////////////////////////////////////
/// @param pSrc - source data in SOA form
/// @param pDst - output data in AOS form
static void Transpose(const uint8_t* pSrc, uint8_t* pDst) = delete;
+#if ENABLE_AVX512_SIMD16
+
+ static void Transpose_16(const uint8_t* pSrc, uint8_t* pDst) = delete;
+#endif
};
//////////////////////////////////////////////////////////////////////////
/// @param pSrc - source data in SOA form
/// @param pDst - output data in AOS form
static void Transpose(const uint8_t* pSrc, uint8_t* pDst) = delete;
+#if ENABLE_AVX512_SIMD16
+
+ static void Transpose_16(const uint8_t* pSrc, uint8_t* pDst) = delete;
+#endif
};
// helper function to unroll loops
uint32_t x, uint32_t y,
uint8_t* pDst)
{
+#if USE_8x2_TILE_BACKEND
+ typedef SimdTile_16<DstFormat, SrcFormat> SimdT;
+
+ SimdT* pDstSimdTiles = (SimdT*)pDst;
+
+ // Compute which simd tile we're accessing within 8x8 tile.
+ // i.e. Compute linear simd tile coordinate given (x, y) in pixel coordinates.
+ uint32_t simdIndex = (y / SIMD16_TILE_Y_DIM) * (KNOB_TILE_X_DIM / SIMD16_TILE_X_DIM) + (x / SIMD16_TILE_X_DIM);
+
+ SimdT* pSimdTile = &pDstSimdTiles[simdIndex];
+
+ uint32_t simdOffset = (y % SIMD16_TILE_Y_DIM) * SIMD16_TILE_X_DIM + (x % SIMD16_TILE_X_DIM);
+
+ pSimdTile->SetSwizzledColor(simdOffset, srcColor);
+#else
typedef SimdTile<DstFormat, SrcFormat> SimdT;
SimdT* pDstSimdTiles = (SimdT*)pDst;
uint32_t simdOffset = (y % SIMD_TILE_Y_DIM) * SIMD_TILE_X_DIM + (x % SIMD_TILE_X_DIM);
pSimdTile->SetSwizzledColor(simdOffset, srcColor);
+#endif
}
//////////////////////////////////////////////////////////////////////////
}
};
+#if USE_8x2_TILE_BACKEND
+template <>
+struct StorePixels<32, 4>
+{
+ static void Store(const uint8_t* pSrc, uint8_t* (&ppDsts)[4])
+ {
+ __m128i quad0 = _mm_load_si128(&reinterpret_cast<const __m128i *>(pSrc)[0]);
+ __m128i quad1 = _mm_load_si128(&reinterpret_cast<const __m128i *>(pSrc)[1]);
+ __m128i quad2 = _mm_load_si128(&reinterpret_cast<const __m128i *>(pSrc)[2]);
+ __m128i quad3 = _mm_load_si128(&reinterpret_cast<const __m128i *>(pSrc)[3]);
+
+ _mm_storeu_si128(reinterpret_cast<__m128i *>(ppDsts[0]), _mm_unpacklo_epi64(quad0, quad1));
+ _mm_storeu_si128(reinterpret_cast<__m128i *>(ppDsts[1]), _mm_unpackhi_epi64(quad0, quad1));
+ _mm_storeu_si128(reinterpret_cast<__m128i *>(ppDsts[2]), _mm_unpacklo_epi64(quad2, quad3));
+ _mm_storeu_si128(reinterpret_cast<__m128i *>(ppDsts[3]), _mm_unpackhi_epi64(quad2, quad3));
+ }
+};
+
+#endif
//////////////////////////////////////////////////////////////////////////
/// StorePixels (32-bit pixel specialization)
/// @brief Stores a 4x2 (AVX) raster-tile to two rows.
template <size_t NumDests>
INLINE static void Convert(const uint8_t* pSrc, uint8_t* (&ppDsts)[NumDests])
{
+#if USE_8x2_TILE_BACKEND
+ static const uint32_t MAX_RASTER_TILE_BYTES = 16 * 16; // 16 pixels * 16 bytes per pixel
+
+ OSALIGNSIMD16(uint8_t) soaTile[MAX_RASTER_TILE_BYTES];
+ OSALIGNSIMD16(uint8_t) aosTile[MAX_RASTER_TILE_BYTES];
+
+ // Convert from SrcFormat --> DstFormat
+ simd16vector src;
+ LoadSOA<SrcFormat>(pSrc, src);
+ StoreSOA<DstFormat>(src, soaTile);
+
+ // Convert from SOA --> AOS
+ FormatTraits<DstFormat>::TransposeT::Transpose_16(soaTile, aosTile);
+
+#else
static const uint32_t MAX_RASTER_TILE_BYTES = 128; // 8 pixels * 16 bytes per pixel
OSALIGNSIMD(uint8_t) soaTile[MAX_RASTER_TILE_BYTES];
// Convert from SOA --> AOS
FormatTraits<DstFormat>::TransposeT::Transpose(soaTile, aosTile);
+#endif
// Store data into destination
StorePixels<FormatTraits<DstFormat>::bpp, NumDests>::Store(aosTile, ppDsts);
}
template <size_t NumDests>
INLINE static void Convert(const uint8_t* pSrc, uint8_t* (&ppDsts)[NumDests])
{
+#if USE_8x2_TILE_BACKEND
+ static const uint32_t MAX_RASTER_TILE_BYTES = 16 * 16; // 16 pixels * 16 bytes per pixel
+
+ OSALIGNSIMD16(uint8_t) aosTile[MAX_RASTER_TILE_BYTES];
+
+ // Convert from SOA --> AOS
+ FormatTraits<Format>::TransposeT::Transpose_16(pSrc, aosTile);
+
+#else
static const uint32_t MAX_RASTER_TILE_BYTES = 128; // 8 pixels * 16 bytes per pixel
OSALIGNSIMD(uint8_t) aosTile[MAX_RASTER_TILE_BYTES];
// Convert from SOA --> AOS
FormatTraits<Format>::TransposeT::Transpose(pSrc, aosTile);
+#endif
// Store data into destination
StorePixels<FormatTraits<Format>::bpp, NumDests>::Store(aosTile, ppDsts);
}
template <size_t NumDests>
INLINE static void Convert(const uint8_t* pSrc, uint8_t* (&ppDsts)[NumDests])
{
+#if USE_8x2_TILE_BACKEND
+ static const uint32_t MAX_RASTER_TILE_BYTES = 16 * 4; // 16 pixels * 4 bytes per pixel
+
+ OSALIGNSIMD16(uint8_t) soaTile[MAX_RASTER_TILE_BYTES];
+ OSALIGNSIMD16(uint8_t) aosTile[MAX_RASTER_TILE_BYTES];
+
+ // Convert from SrcFormat --> DstFormat
+ simd16vector src;
+ LoadSOA<SrcFormat>(pSrc, src);
+ StoreSOA<DstFormat>(src, soaTile);
+
+ // Convert from SOA --> AOS
+ FormatTraits<DstFormat>::TransposeT::Transpose_16(soaTile, aosTile);
+
+ // Store data into destination but don't overwrite the X8 bits
+ // Each 4-pixel row is 16-bytes
+#if 1
+ simdscalari loadlo = _simd_load_si(reinterpret_cast<simdscalari *>(aosTile));
+ simdscalari loadhi = _simd_load_si(reinterpret_cast<simdscalari *>(aosTile + sizeof(simdscalari)));
+
+ simdscalari templo = _simd_unpacklo_epi64(loadlo, loadhi);
+ simdscalari temphi = _simd_unpackhi_epi64(loadlo, loadhi);
+
+ simdscalari destlo = _mm256_loadu2_m128i(reinterpret_cast<__m128i *>(ppDsts[1]), reinterpret_cast<__m128i *>(ppDsts[0]));
+ simdscalari desthi = _mm256_loadu2_m128i(reinterpret_cast<__m128i *>(ppDsts[3]), reinterpret_cast<__m128i *>(ppDsts[2]));
+
+ simdscalari mask = _simd_set1_epi32(0xFFFFFF);
+
+ destlo = _simd_or_si(_simd_andnot_si(mask, destlo), _simd_and_si(mask, templo));
+ desthi = _simd_or_si(_simd_andnot_si(mask, desthi), _simd_and_si(mask, templo));
+
+ _mm256_storeu2_m128i(reinterpret_cast<__m128i *>(ppDsts[1]), reinterpret_cast<__m128i *>(ppDsts[0]), destlo);
+ _mm256_storeu2_m128i(reinterpret_cast<__m128i *>(ppDsts[3]), reinterpret_cast<__m128i *>(ppDsts[2]), desthi);
+#else
+ __m128i *pZRow01 = (__m128i*)aosTile;
+ __m128i vQuad00 = _mm_load_si128(pZRow01);
+ __m128i vQuad01 = _mm_load_si128(pZRow01 + 1);
+ __m128i vQuad02 = _mm_load_si128(pZRow01 + 2);
+ __m128i vQuad03 = _mm_load_si128(pZRow01 + 3);
+
+ __m128i vRow00 = _mm_unpacklo_epi64(vQuad00, vQuad01);
+ __m128i vRow10 = _mm_unpackhi_epi64(vQuad00, vQuad01);
+ __m128i vRow20 = _mm_unpacklo_epi64(vQuad02, vQuad03);
+ __m128i vRow30 = _mm_unpackhi_epi64(vQuad02, vQuad03);
+
+ __m128i vDst0 = _mm_loadu_si128((const __m128i*)ppDsts[0]);
+ __m128i vDst1 = _mm_loadu_si128((const __m128i*)ppDsts[1]);
+ __m128i vDst2 = _mm_loadu_si128((const __m128i*)ppDsts[2]);
+ __m128i vDst3 = _mm_loadu_si128((const __m128i*)ppDsts[3]);
+
+ __m128i vMask = _mm_set1_epi32(0xFFFFFF);
+
+ vDst0 = _mm_andnot_si128(vMask, vDst0);
+ vDst0 = _mm_or_si128(vDst0, _mm_and_si128(vRow00, vMask));
+ vDst1 = _mm_andnot_si128(vMask, vDst1);
+ vDst1 = _mm_or_si128(vDst1, _mm_and_si128(vRow10, vMask));
+ vDst2 = _mm_andnot_si128(vMask, vDst2);
+ vDst2 = _mm_or_si128(vDst2, _mm_and_si128(vRow20, vMask));
+ vDst3 = _mm_andnot_si128(vMask, vDst3);
+ vDst3 = _mm_or_si128(vDst3, _mm_and_si128(vRow10, vMask));
+
+ _mm_storeu_si128((__m128i*)ppDsts[0], vDst0);
+ _mm_storeu_si128((__m128i*)ppDsts[1], vDst1);
+ _mm_storeu_si128((__m128i*)ppDsts[2], vDst2);
+ _mm_storeu_si128((__m128i*)ppDsts[3], vDst3);
+#endif
+#else
static const uint32_t MAX_RASTER_TILE_BYTES = 128; // 8 pixels * 16 bytes per pixel
OSALIGNSIMD(uint8_t) soaTile[MAX_RASTER_TILE_BYTES];
_mm_storeu_si128((__m128i*)ppDsts[0], vDst0);
_mm_storeu_si128((__m128i*)ppDsts[1], vDst1);
+#endif
}
};
+#if USE_8x2_TILE_BACKEND
+template<SWR_FORMAT DstFormat>
+INLINE static void FlatConvert(const uint8_t* pSrc, uint8_t* pDst0, uint8_t* pDst1, uint8_t* pDst2, uint8_t* pDst3)
+{
+ // swizzle rgba -> bgra while we load
+ simd16scalar comp0 = _simd16_load_ps(reinterpret_cast<const float*>(pSrc + FormatTraits<DstFormat>::swizzle(0) * sizeof(simd16scalar))); // float32 rrrrrrrrrrrrrrrr
+ simd16scalar comp1 = _simd16_load_ps(reinterpret_cast<const float*>(pSrc + FormatTraits<DstFormat>::swizzle(1) * sizeof(simd16scalar))); // float32 gggggggggggggggg
+ simd16scalar comp2 = _simd16_load_ps(reinterpret_cast<const float*>(pSrc + FormatTraits<DstFormat>::swizzle(2) * sizeof(simd16scalar))); // float32 bbbbbbbbbbbbbbbb
+ simd16scalar comp3 = _simd16_load_ps(reinterpret_cast<const float*>(pSrc + FormatTraits<DstFormat>::swizzle(3) * sizeof(simd16scalar))); // float32 aaaaaaaaaaaaaaaa
+
+ // clamp
+ const simd16scalar zero = _simd16_setzero_ps();
+ const simd16scalar ones = _simd16_set1_ps(1.0f);
+
+ comp0 = _simd16_max_ps(comp0, zero);
+ comp0 = _simd16_min_ps(comp0, ones);
+
+ comp1 = _simd16_max_ps(comp1, zero);
+ comp1 = _simd16_min_ps(comp1, ones);
+
+ comp2 = _simd16_max_ps(comp2, zero);
+ comp2 = _simd16_min_ps(comp2, ones);
+
+ comp3 = _simd16_max_ps(comp3, zero);
+ comp3 = _simd16_min_ps(comp3, ones);
+
+ if (FormatTraits<DstFormat>::isSRGB)
+ {
+ // Gamma-correct only rgb
+ comp0 = FormatTraits<R32G32B32A32_FLOAT>::convertSrgb(0, comp0);
+ comp1 = FormatTraits<R32G32B32A32_FLOAT>::convertSrgb(1, comp1);
+ comp2 = FormatTraits<R32G32B32A32_FLOAT>::convertSrgb(2, comp2);
+ }
+
+ // convert float components from 0.0f .. 1.0f to correct scale for 0 .. 255 dest format
+ comp0 = _simd16_mul_ps(comp0, _simd16_set1_ps(FormatTraits<DstFormat>::fromFloat(0)));
+ comp1 = _simd16_mul_ps(comp1, _simd16_set1_ps(FormatTraits<DstFormat>::fromFloat(1)));
+ comp2 = _simd16_mul_ps(comp2, _simd16_set1_ps(FormatTraits<DstFormat>::fromFloat(2)));
+ comp3 = _simd16_mul_ps(comp3, _simd16_set1_ps(FormatTraits<DstFormat>::fromFloat(3)));
+
+ // moving to 16 wide integer vector types
+ simd16scalari src0 = _simd16_cvtps_epi32(comp0); // padded byte rrrrrrrrrrrrrrrr
+ simd16scalari src1 = _simd16_cvtps_epi32(comp1); // padded byte gggggggggggggggg
+ simd16scalari src2 = _simd16_cvtps_epi32(comp2); // padded byte bbbbbbbbbbbbbbbb
+ simd16scalari src3 = _simd16_cvtps_epi32(comp3); // padded byte aaaaaaaaaaaaaaaa
+
+#if 1
+ // SOA to AOS conversion
+ src1 = _simd16_slli_epi32(src1, 8);
+ src2 = _simd16_slli_epi32(src2, 16);
+ src3 = _simd16_slli_epi32(src3, 24);
+
+ simd16scalari final = _simd16_or_si(_simd16_or_si(src0, src1), _simd16_or_si(src2, src3)); // 0 1 2 3 4 5 6 7 8 9 A B C D E F
+
+ // de-swizzle conversion
+#if 1
+ simd16scalari final0 = _simd16_permute2f128_si(final, final, 0xA0); // (2, 2, 0, 0) // 0 1 2 3 0 1 2 3 8 9 A B 8 9 A B
+ simd16scalari final1 = _simd16_permute2f128_si(final, final, 0xF5); // (3, 3, 1, 1) // 4 5 6 7 4 5 6 7 C D E F C D E F
+
+ final = _simd16_shuffle_epi64(final0, final1, 0xCC); // (1 1 0 0 1 1 0 0) // 0 1 4 5 2 3 6 7 8 9 C D A B E F
+
+#else
+ final = _simd16_permute_epi32(final, _simd16_set_epi32(15, 14, 11, 10, 13, 12, 9, 8, 7, 6, 3, 2, 5, 4, 1, 0));
+
+#endif
+#endif
+#if KNOB_ARCH == KNOB_ARCH_AVX
+
+ // splitting into two sets of 4 wide integer vector types
+ // because AVX doesn't have instructions to support this operation at 8 wide
+#if 0
+ __m128i srcLo0 = _mm256_castsi256_si128(src0); // 000r000r000r000r
+ __m128i srcLo1 = _mm256_castsi256_si128(src1); // 000g000g000g000g
+ __m128i srcLo2 = _mm256_castsi256_si128(src2); // 000b000b000b000b
+ __m128i srcLo3 = _mm256_castsi256_si128(src3); // 000a000a000a000a
+
+ __m128i srcHi0 = _mm256_extractf128_si256(src0, 1); // 000r000r000r000r
+ __m128i srcHi1 = _mm256_extractf128_si256(src1, 1); // 000g000g000g000g
+ __m128i srcHi2 = _mm256_extractf128_si256(src2, 1); // 000b000b000b000b
+ __m128i srcHi3 = _mm256_extractf128_si256(src3, 1); // 000a000a000a000a
+
+ srcLo1 = _mm_slli_si128(srcLo1, 1); // 00g000g000g000g0
+ srcHi1 = _mm_slli_si128(srcHi1, 1); // 00g000g000g000g0
+ srcLo2 = _mm_slli_si128(srcLo2, 2); // 0b000b000b000b00
+ srcHi2 = _mm_slli_si128(srcHi2, 2); // 0b000b000b000b00
+ srcLo3 = _mm_slli_si128(srcLo3, 3); // a000a000a000a000
+ srcHi3 = _mm_slli_si128(srcHi3, 3); // a000a000a000a000
+
+ srcLo0 = _mm_or_si128(srcLo0, srcLo1); // 00gr00gr00gr00gr
+ srcLo2 = _mm_or_si128(srcLo2, srcLo3); // ab00ab00ab00ab00
+
+ srcHi0 = _mm_or_si128(srcHi0, srcHi1); // 00gr00gr00gr00gr
+ srcHi2 = _mm_or_si128(srcHi2, srcHi3); // ab00ab00ab00ab00
+
+ srcLo0 = _mm_or_si128(srcLo0, srcLo2); // abgrabgrabgrabgr
+ srcHi0 = _mm_or_si128(srcHi0, srcHi2); // abgrabgrabgrabgr
+
+ // unpack into rows that get the tiling order correct
+ __m128i vRow00 = _mm_unpacklo_epi64(srcLo0, srcHi0); // abgrabgrabgrabgrabgrabgrabgrabgr
+ __m128i vRow10 = _mm_unpackhi_epi64(srcLo0, srcHi0);
+
+ __m256i final = _mm256_castsi128_si256(vRow00);
+ final = _mm256_insertf128_si256(final, vRow10, 1);
+
+#else
+#if 0
+ simd16scalari final = _simd16_setzero_si();
+
+#endif
+#endif
+#elif KNOB_ARCH >= KNOB_ARCH_AVX2
+ // logic is as above, only wider
+#if 0
+ src1 = _simd16_slli_epi32(src1, 8);
+ src2 = _simd16_slli_epi32(src2, 16);
+ src3 = _simd16_slli_epi32(src3, 24);
+
+ simd16scalari final = _simd16_or_si(_simd16_or_si(src0, src1), _simd16_or_si(src2, src3));
+
+ final = _simd16_permute_epi32(final, _simd16_set_epi32(15, 14, 11, 10, 13, 12, 9, 8, 7, 6, 3, 2, 5, 4, 1, 0));
+
+#endif
+#endif
+ _mm256_storeu2_m128i(reinterpret_cast<__m128i *>(pDst1), reinterpret_cast<__m128i *>(pDst0), _simd16_extract_si(final, 0));
+ _mm256_storeu2_m128i(reinterpret_cast<__m128i *>(pDst3), reinterpret_cast<__m128i *>(pDst2), _simd16_extract_si(final, 1));
+}
+
+#endif
template<SWR_FORMAT DstFormat>
INLINE static void FlatConvert(const uint8_t* pSrc, uint8_t* pDst, uint8_t* pDst1)
{
src2 = _mm256_or_si256(src2, src3);
__m256i final = _mm256_or_si256(src0, src2);
-
+#if 0
+
+ __m256i perm = _mm256_set_epi32(7, 6, 3, 2, 5, 4, 1, 0);
+
+ final = _mm256_permutevar8x32_epi32(final, perm);
+#else
+
// adjust the data to get the tiling order correct 0 1 2 3 -> 0 2 1 3
final = _mm256_permute4x64_epi64(final, 0xD8);
-
+#endif
#endif
_mm256_storeu2_m128i((__m128i*)pDst1, (__m128i*)pDst, final);
template <size_t NumDests>
INLINE static void Convert(const uint8_t* pSrc, uint8_t* (&ppDsts)[NumDests])
{
+#if USE_8x2_TILE_BACKEND
+ FlatConvert<R8G8B8A8_UNORM>(pSrc, ppDsts[0], ppDsts[1], ppDsts[2], ppDsts[3]);
+#else
FlatConvert<R8G8B8A8_UNORM>(pSrc, ppDsts[0], ppDsts[1]);
+#endif
}
};
template <size_t NumDests>
INLINE static void Convert(const uint8_t* pSrc, uint8_t* (&ppDsts)[NumDests])
{
+#if USE_8x2_TILE_BACKEND
+ FlatConvert<R8G8B8A8_UNORM_SRGB>(pSrc, ppDsts[0], ppDsts[1], ppDsts[2], ppDsts[3]);
+#else
FlatConvert<R8G8B8A8_UNORM_SRGB>(pSrc, ppDsts[0], ppDsts[1]);
+#endif
}
};
uint32_t x, uint32_t y,
float outputColor[4])
{
+#if USE_8x2_TILE_BACKEND
+ typedef SimdTile_16<SrcFormat, DstFormat> SimdT;
+
+ SimdT* pSrcSimdTiles = (SimdT*)pSrc;
+
+ // Compute which simd tile we're accessing within 8x8 tile.
+ // i.e. Compute linear simd tile coordinate given (x, y) in pixel coordinates.
+ uint32_t simdIndex = (y / SIMD16_TILE_Y_DIM) * (KNOB_TILE_X_DIM / SIMD16_TILE_X_DIM) + (x / SIMD16_TILE_X_DIM);
+
+ SimdT* pSimdTile = &pSrcSimdTiles[simdIndex];
+
+ uint32_t simdOffset = (y % SIMD16_TILE_Y_DIM) * SIMD16_TILE_X_DIM + (x % SIMD16_TILE_X_DIM);
+
+ pSimdTile->GetSwizzledColor(simdOffset, outputColor);
+#else
typedef SimdTile<SrcFormat, DstFormat> SimdT;
SimdT* pSrcSimdTiles = (SimdT*)pSrc;
uint32_t simdOffset = (y % SIMD_TILE_Y_DIM) * SIMD_TILE_X_DIM + (x % SIMD_TILE_X_DIM);
pSimdTile->GetSwizzledColor(simdOffset, outputColor);
+#endif
}
//////////////////////////////////////////////////////////////////////////
uint8_t* pDst = (uint8_t*)ComputeSurfaceAddress<false, false>(x, y, pDstSurface->arrayIndex + renderTargetArrayIndex,
pDstSurface->arrayIndex + renderTargetArrayIndex, sampleNum, pDstSurface->lod, pDstSurface);
+#if USE_8x2_TILE_BACKEND
+ uint8_t* ppRows[] = { pDst, pDst + pDstSurface->pitch, pDst + (SIMD16_TILE_X_DIM * DST_BYTES_PER_PIXEL) / 2, pDst + pDstSurface->pitch + (SIMD16_TILE_X_DIM * DST_BYTES_PER_PIXEL) / 2 };
+
+ for (uint32_t row = 0; row < KNOB_TILE_Y_DIM / SIMD16_TILE_Y_DIM; ++row)
+ {
+ uint8_t* ppStartRows[] = { ppRows[0], ppRows[1], ppRows[2], ppRows[3] };
+
+ for (uint32_t col = 0; col < KNOB_TILE_X_DIM / SIMD16_TILE_X_DIM; ++col)
+ {
+ // Format conversion and convert from SOA to AOS, and store the rows.
+ ConvertPixelsSOAtoAOS<SrcFormat, DstFormat>::Convert(pSrc, ppRows);
+
+ ppRows[0] += SIMD16_TILE_X_DIM * DST_BYTES_PER_PIXEL;
+ ppRows[1] += SIMD16_TILE_X_DIM * DST_BYTES_PER_PIXEL;
+ ppRows[2] += SIMD16_TILE_X_DIM * DST_BYTES_PER_PIXEL;
+ ppRows[3] += SIMD16_TILE_X_DIM * DST_BYTES_PER_PIXEL;
+
+ pSrc += KNOB_SIMD16_WIDTH * SRC_BYTES_PER_PIXEL;
+ }
+
+ ppRows[0] = ppStartRows[0] + SIMD16_TILE_Y_DIM * pDstSurface->pitch;
+ ppRows[1] = ppStartRows[1] + SIMD16_TILE_Y_DIM * pDstSurface->pitch;
+ ppRows[2] = ppStartRows[2] + SIMD16_TILE_Y_DIM * pDstSurface->pitch;
+ ppRows[3] = ppStartRows[3] + SIMD16_TILE_Y_DIM * pDstSurface->pitch;
+ }
+#else
uint8_t* ppRows[] = { pDst, pDst + pDstSurface->pitch };
for (uint32_t row = 0; row < KNOB_TILE_Y_DIM / SIMD_TILE_Y_DIM; ++row)
ppRows[0] = ppStartRows[0] + 2 * pDstSurface->pitch;
ppRows[1] = ppStartRows[1] + 2 * pDstSurface->pitch;
}
+#endif
}
};
struct OptStoreRasterTile< TilingTraits<SWR_TILE_MODE_YMAJOR, 32>, SrcFormat, DstFormat>
{
typedef StoreRasterTile<TilingTraits<SWR_TILE_MODE_YMAJOR, 32>, SrcFormat, DstFormat> GenericStoreTile;
+ static const size_t SRC_BYTES_PER_PIXEL = FormatTraits<SrcFormat>::bpp / 8;
//////////////////////////////////////////////////////////////////////////
/// @brief Stores an 8x8 raster tile to the destination surface.
uint8_t* pCol0 = (uint8_t*)ComputeSurfaceAddress<false, false>(x, y, pDstSurface->arrayIndex + renderTargetArrayIndex,
pDstSurface->arrayIndex + renderTargetArrayIndex, sampleNum, pDstSurface->lod, pDstSurface);
+#if USE_8x2_TILE_BACKEND
+ // The Hot Tile uses a row-major tiling mode and has a larger memory footprint. So we iterate in a row-major pattern.
+ for (uint32_t row = 0; row < KNOB_TILE_Y_DIM; row += SIMD16_TILE_Y_DIM)
+ {
+ uint8_t *pRow = pCol0 + row * DestRowWidthBytes;
+
+ uint8_t *ppDsts[] = { pRow, pRow + DestRowWidthBytes, pRow + DestColumnBytes, pRow + DestColumnBytes + DestRowWidthBytes };
+
+ ConvertPixelsSOAtoAOS<SrcFormat, DstFormat>::Convert(pSrc, ppDsts);
+
+ pSrc += KNOB_SIMD16_WIDTH * SRC_BYTES_PER_PIXEL;
+ }
+#else
// Increment by a whole SIMD. 4x2 for AVX. 2x2 for SSE.
uint32_t pSrcInc = (FormatTraits<SrcFormat>::bpp * KNOB_SIMD_WIDTH) / 8;
ConvertPixelsSOAtoAOS<SrcFormat, DstFormat>::Convert(pSrc, ppDsts);
pSrc += pSrcInc;
}
+#endif
}
};
}
};
+#if ENABLE_AVX512_SIMD16
+//////////////////////////////////////////////////////////////////////////
+/// SimdTile 8x2 for AVX-512
+//////////////////////////////////////////////////////////////////////////
+
+template<SWR_FORMAT HotTileFormat, SWR_FORMAT SrcOrDstFormat>
+struct SimdTile_16
+{
+ // SimdTile is SOA (e.g. rrrrrrrrrrrrrrrr gggggggggggggggg bbbbbbbbbbbbbbbb aaaaaaaaaaaaaaaa )
+ float color[FormatTraits<HotTileFormat>::numComps][KNOB_SIMD16_WIDTH];
+
+ //////////////////////////////////////////////////////////////////////////
+ /// @brief Retrieve color from simd.
+ /// @param index - linear index to color within simd.
+ /// @param outputColor - output color
+ INLINE void GetSwizzledColor(
+ uint32_t index,
+ float outputColor[4])
+ {
+ // SOA pattern for 8x2..
+ // 0 1 4 5 8 9 C D
+ // 2 3 6 7 A B E F
+ // The offset converts pattern to linear
+ static const uint32_t offset[KNOB_SIMD16_WIDTH] = { 0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15 };
+
+ for (uint32_t i = 0; i < FormatTraits<SrcOrDstFormat>::numComps; ++i)
+ {
+ outputColor[i] = this->color[FormatTraits<SrcOrDstFormat>::swizzle(i)][offset[index]];
+ }
+ }
+
+ //////////////////////////////////////////////////////////////////////////
+ /// @brief Retrieve color from simd.
+ /// @param index - linear index to color within simd.
+ /// @param outputColor - output color
+ INLINE void SetSwizzledColor(
+ uint32_t index,
+ const float src[4])
+ {
+ // SOA pattern for 8x2..
+ // 0 1 4 5 8 9 C D
+ // 2 3 6 7 A B E F
+ // The offset converts pattern to linear
+ static const uint32_t offset[KNOB_SIMD16_WIDTH] = { 0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15 };
+
+ for (uint32_t i = 0; i < FormatTraits<SrcOrDstFormat>::numComps; ++i)
+ {
+ this->color[i][offset[index]] = src[i];
+ }
+ }
+};
+
+template<>
+struct SimdTile_16 <R8_UINT, R8_UINT>
+{
+ // SimdTile is SOA (e.g. rrrrrrrrrrrrrrrr gggggggggggggggg bbbbbbbbbbbbbbbb aaaaaaaaaaaaaaaa )
+ uint8_t color[FormatTraits<R8_UINT>::numComps][KNOB_SIMD16_WIDTH];
+
+ //////////////////////////////////////////////////////////////////////////
+ /// @brief Retrieve color from simd.
+ /// @param index - linear index to color within simd.
+ /// @param outputColor - output color
+ INLINE void GetSwizzledColor(
+ uint32_t index,
+ float outputColor[4])
+ {
+ // SOA pattern for 8x2..
+ // 0 1 4 5 8 9 C D
+ // 2 3 6 7 A B E F
+ // The offset converts pattern to linear
+ static const uint32_t offset[KNOB_SIMD16_WIDTH] = { 0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15 };
+
+ for (uint32_t i = 0; i < FormatTraits<R8_UINT>::numComps; ++i)
+ {
+ uint32_t src = this->color[FormatTraits<R8_UINT>::swizzle(i)][offset[index]];
+ outputColor[i] = *(float*)&src;
+ }
+ }
+
+ //////////////////////////////////////////////////////////////////////////
+ /// @brief Retrieve color from simd.
+ /// @param index - linear index to color within simd.
+ /// @param outputColor - output color
+ INLINE void SetSwizzledColor(
+ uint32_t index,
+ const float src[4])
+ {
+ // SOA pattern for 8x2..
+ // 0 1 4 5 8 9 C D
+ // 2 3 6 7 A B E F
+ // The offset converts pattern to linear
+ static const uint32_t offset[KNOB_SIMD16_WIDTH] = { 0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15 };
+
+ for (uint32_t i = 0; i < FormatTraits<R8_UINT>::numComps; ++i)
+ {
+ this->color[i][offset[index]] = *(uint8_t*)&src[i];
+ }
+ }
+};
+
+#endif
//////////////////////////////////////////////////////////////////////////
/// @brief Computes lod offset for 1D surface at specified lod.
/// @param baseWidth - width of basemip (mip 0).
projects / mesa.git / commitdiff