projects / mesa.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
history | raw | HEAD
swr/rast: Separate RDTSC code from archrast
[mesa.git] / src / gallium / drivers / swr / rasterizer / core / format_types.h
1 /****************************************************************************
2 * Copyright (C) 2014-2015 Intel Corporation. All Rights Reserved.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 * @file formats.h
24 *
25 * @brief Definitions for SWR_FORMAT functions.
26 *
27 ******************************************************************************/
28 #pragma once
29
30 #include "utils.h"
31 #include "common/simdintrin.h"
32
33 //////////////////////////////////////////////////////////////////////////
34 /// PackTraits - Helpers for packing / unpacking same pixel sizes
35 //////////////////////////////////////////////////////////////////////////
36 template <uint32_t NumBits, bool Signed = false>
37 struct PackTraits
38 {
39 static const uint32_t MyNumBits = NumBits;
40 static simdscalar loadSOA(const uint8_t *pSrc) = delete;
41 static void storeSOA(uint8_t *pDst, simdscalar const &src) = delete;
42 static simdscalar unpack(simdscalar &in) = delete;
43 static simdscalar pack(simdscalar &in) = delete;
44 #if ENABLE_AVX512_SIMD16
45 static simd16scalar loadSOA_16(const uint8_t *pSrc) = delete;
46 static void SIMDCALL storeSOA(uint8_t *pDst, simd16scalar const &src) = delete;
47 static simd16scalar unpack(simd16scalar &in) = delete;
48 static simd16scalar pack(simd16scalar &in) = delete;
49 #endif
50 };
51
52 //////////////////////////////////////////////////////////////////////////
53 /// PackTraits - Helpers for packing / unpacking unused channels
54 //////////////////////////////////////////////////////////////////////////
55 template <>
56 struct PackTraits<0, false>
57 {
58 static const uint32_t MyNumBits = 0;
59
60 static simdscalar loadSOA(const uint8_t *pSrc) { return _simd_setzero_ps(); }
61 static void storeSOA(uint8_t *pDst, simdscalar const &src) { return; }
62 static simdscalar unpack(simdscalar &in) { return _simd_setzero_ps(); }
63 static simdscalar pack(simdscalar &in) { return _simd_setzero_ps(); }
64 #if ENABLE_AVX512_SIMD16
65 static simd16scalar loadSOA_16(const uint8_t *pSrc) { return _simd16_setzero_ps(); }
66 static void SIMDCALL storeSOA(uint8_t *pDst, simd16scalar const &src) { return; }
67 static simd16scalar unpack(simd16scalar &in) { return _simd16_setzero_ps(); }
68 static simd16scalar pack(simd16scalar &in) { return _simd16_setzero_ps(); }
69 #endif
70 };
71
72 //////////////////////////////////////////////////////////////////////////
73 /// PackTraits - Helpers for packing / unpacking 8 bit unsigned channels
74 //////////////////////////////////////////////////////////////////////////
75 template <>
76 struct PackTraits<8, false>
77 {
78 static const uint32_t MyNumBits = 8;
79
80 static simdscalar loadSOA(const uint8_t *pSrc)
81 {
82 #if KNOB_SIMD_WIDTH == 8
83 __m256 result = _mm256_setzero_ps();
84 __m128 vLo = _mm_castpd_ps(_mm_load_sd((double*)pSrc));
85 return _mm256_insertf128_ps(result, vLo, 0);
86 #else
87 #error Unsupported vector width
88 #endif
89 }
90
91 static void storeSOA(uint8_t *pDst, simdscalar const &src)
92 {
93 // store simd bytes
94 #if KNOB_SIMD_WIDTH == 8
95 _mm_storel_pd((double*)pDst, _mm_castps_pd(_mm256_castps256_ps128(src)));
96 #else
97 #error Unsupported vector width
98 #endif
99 }
100
101 static simdscalar unpack(simdscalar &in)
102 {
103 #if KNOB_SIMD_WIDTH == 8
104 #if KNOB_ARCH <= KNOB_ARCH_AVX
105 __m128i src = _mm_castps_si128(_mm256_castps256_ps128(in));
106 __m128i resLo = _mm_cvtepu8_epi32(src);
107 __m128i resHi = _mm_shuffle_epi8(src,
108 _mm_set_epi32(0x80808007, 0x80808006, 0x80808005, 0x80808004));
109
110 __m256i result = _mm256_castsi128_si256(resLo);
111 result = _mm256_insertf128_si256(result, resHi, 1);
112 return simdscalar{ _mm256_castsi256_ps(result) };
113 #else
114 return _mm256_castsi256_ps(_mm256_cvtepu8_epi32(_mm_castps_si128(_mm256_castps256_ps128(in))));
115 #endif
116 #else
117 #error Unsupported vector width
118 #endif
119 }
120
121 static simdscalar pack(simdscalar &in)
122 {
123 #if KNOB_SIMD_WIDTH == 8
124 simdscalari src = _simd_castps_si(in);
125 __m128i res16 = _mm_packus_epi32(_mm256_castsi256_si128(src), _mm256_extractf128_si256(src, 1));
126 __m128i res8 = _mm_packus_epi16(res16, _mm_undefined_si128());
127 return _mm256_castsi256_ps(_mm256_castsi128_si256(res8));
128 #else
129 #error Unsupported vector width
130 #endif
131 }
132 #if ENABLE_AVX512_SIMD16
133
134 static simd16scalar loadSOA_16(const uint8_t *pSrc)
135 {
136 simd16scalar result = _simd16_setzero_ps();
137 simdscalar resultlo = _simd_setzero_ps();
138
139 const __m128 src = _mm_load_ps(reinterpret_cast<const float *>(pSrc));
140
141 resultlo = _mm256_insertf128_ps(resultlo, src, 0);
142 result = _simd16_insert_ps(result, resultlo, 0);
143
144 return result;
145 }
146
147 static void SIMDCALL storeSOA(uint8_t *pDst, simd16scalar const &src)
148 {
149 // store simd16 bytes
150 _mm_store_ps(reinterpret_cast<float *>(pDst), _mm256_castps256_ps128(_simd16_extract_ps(src, 0)));
151 }
152
153 static simd16scalar unpack(simd16scalar &in)
154 {
155 simd4scalari tmp = _mm_castps_si128(_mm256_castps256_ps128(_simd16_extract_ps(in, 0)));
156 simd16scalari result = _simd16_cvtepu8_epi32(tmp);
157
158 return _simd16_castsi_ps(result);
159 }
160
161 static simd16scalar pack(simd16scalar &in)
162 {
163 simd16scalari result = _simd16_setzero_si();
164
165 simdscalari inlo = _simd_castps_si(_simd16_extract_ps(in, 0)); // r0 r1 r2 r3 r4 r5 r6 r7 (32b)
166 simdscalari inhi = _simd_castps_si(_simd16_extract_ps(in, 1)); // r8 r9 rA rB rC rD rE rF
167
168 simdscalari permlo = _simd_permute2f128_si(inlo, inhi, 0x20); // r0 r1 r2 r3 r8 r9 rA rB (32b)
169 simdscalari permhi = _simd_permute2f128_si(inlo, inhi, 0x31); // r4 r5 r6 r7 rC rD rE rF (32b)
170
171 simdscalari pack = _simd_packus_epi32(permlo, permhi); // r0 r1 r2 r3 r4 r5 r6 r7 r8 r9 rA rB rC rD rE rF (16b)
172
173 const simdscalari zero = _simd_setzero_si();
174
175 permlo = _simd_permute2f128_si(pack, zero, 0x20); // (2, 0) // r0 r1 r2 r3 r4 r5 r6 r7 00 00 00 00 00 00 00 00 (16b)
176 permhi = _simd_permute2f128_si(pack, zero, 0x31); // (3, 1) // r8 r9 rA rB rC rD rE rF 00 00 00 00 00 00 00 00 (16b)
177
178 pack = _simd_packus_epi16(permlo, permhi); // r0 r1 r2 r3 r4 r5 r6 r7 r8 r9 rA rB rC rD rE rF 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 (8b)
179
180 result = _simd16_insert_si(result, pack, 0);
181
182 return _simd16_castsi_ps(result);
183 }
184 #endif
185 };
186
187 //////////////////////////////////////////////////////////////////////////
188 /// PackTraits - Helpers for packing / unpacking 8 bit signed channels
189 //////////////////////////////////////////////////////////////////////////
190 template <>
191 struct PackTraits<8, true>
192 {
193 static const uint32_t MyNumBits = 8;
194
195 static simdscalar loadSOA(const uint8_t *pSrc)
196 {
197 #if KNOB_SIMD_WIDTH == 8
198 __m256 result = _mm256_setzero_ps();
199 __m128 vLo = _mm_castpd_ps(_mm_load_sd((double*)pSrc));
200 return _mm256_insertf128_ps(result, vLo, 0);
201 #else
202 #error Unsupported vector width
203 #endif
204 }
205
206 static void storeSOA(uint8_t *pDst, simdscalar const &src)
207 {
208 // store simd bytes
209 #if KNOB_SIMD_WIDTH == 8
210 _mm_storel_pd((double*)pDst, _mm_castps_pd(_mm256_castps256_ps128(src)));
211 #else
212 #error Unsupported vector width
213 #endif
214 }
215
216 static simdscalar unpack(simdscalar &in)
217 {
218 #if KNOB_SIMD_WIDTH == 8
219 #if KNOB_ARCH <= KNOB_ARCH_AVX
220 SWR_INVALID("I think this may be incorrect.");
221 __m128i src = _mm_castps_si128(_mm256_castps256_ps128(in));
222 __m128i resLo = _mm_cvtepi8_epi32(src);
223 __m128i resHi = _mm_shuffle_epi8(src,
224 _mm_set_epi32(0x80808007, 0x80808006, 0x80808005, 0x80808004));
225
226 __m256i result = _mm256_castsi128_si256(resLo);
227 result = _mm256_insertf128_si256(result, resHi, 1);
228 return _mm256_castsi256_ps(result);
229 #else
230 return _mm256_castsi256_ps(_mm256_cvtepi8_epi32(_mm_castps_si128(_mm256_castps256_ps128(in))));
231 #endif
232 #else
233 #error Unsupported vector width
234 #endif
235 }
236
237 static simdscalar pack(simdscalar &in)
238 {
239 #if KNOB_SIMD_WIDTH == 8
240 simdscalari src = _simd_castps_si(in);
241 __m128i res16 = _mm_packs_epi32(_mm256_castsi256_si128(src), _mm256_extractf128_si256(src, 1));
242 __m128i res8 = _mm_packs_epi16(res16, _mm_undefined_si128());
243 return _mm256_castsi256_ps(_mm256_castsi128_si256(res8));
244 #else
245 #error Unsupported vector width
246 #endif
247 }
248 #if ENABLE_AVX512_SIMD16
249
250 static simd16scalar loadSOA_16(const uint8_t *pSrc)
251 {
252 simd16scalar result = _simd16_setzero_ps();
253 simdscalar resultlo = _simd_setzero_ps();
254
255 const __m128 src = _mm_load_ps(reinterpret_cast<const float *>(pSrc));
256
257 resultlo = _mm256_insertf128_ps(resultlo, src, 0);
258 result = _simd16_insert_ps(result, resultlo, 0);
259
260 return result;
261 }
262
263 static void SIMDCALL storeSOA(uint8_t *pDst, simd16scalar const &src)
264 {
265 // store simd16 bytes
266 _mm_store_ps(reinterpret_cast<float *>(pDst), _mm256_castps256_ps128(_simd16_extract_ps(src, 0)));
267 }
268
269 static simd16scalar unpack(simd16scalar &in)
270 {
271 simd4scalari tmp = _mm_castps_si128(_mm256_castps256_ps128(_simd16_extract_ps(in, 0)));
272 simd16scalari result = _simd16_cvtepu8_epi32(tmp);
273
274 return _simd16_castsi_ps(result);
275 }
276
277 static simd16scalar pack(simd16scalar &in)
278 {
279 simd16scalari result = _simd16_setzero_si();
280
281 simdscalari inlo = _simd_castps_si(_simd16_extract_ps(in, 0)); // r0 r1 r2 r3 r4 r5 r6 r7 (32b)
282 simdscalari inhi = _simd_castps_si(_simd16_extract_ps(in, 1)); // r8 r9 rA rB rC rD rE rF
283
284 simdscalari permlo = _simd_permute2f128_si(inlo, inhi, 0x20); // r0 r1 r2 r3 r8 r9 rA rB (32b)
285 simdscalari permhi = _simd_permute2f128_si(inlo, inhi, 0x31); // r4 r5 r6 r7 rC rD rE rF (32b)
286
287 simdscalari pack = _simd_packs_epi32(permlo, permhi); // r0 r1 r2 r3 r4 r5 r6 r7 r8 r9 rA rB rC rD rE rF (16b)
288
289 const simdscalari zero = _simd_setzero_si();
290
291 permlo = _simd_permute2f128_si(pack, zero, 0x20); // (2, 0) // r0 r1 r2 r3 r4 r5 r6 r7 00 00 00 00 00 00 00 00 (16b)
292 permhi = _simd_permute2f128_si(pack, zero, 0x31); // (3, 1) // r8 r9 rA rB rC rD rE rF 00 00 00 00 00 00 00 00 (16b)
293
294 pack = _simd_packs_epi16(permlo, permhi); // r0 r1 r2 r3 r4 r5 r6 r7 r8 r9 rA rB rC rD rE rF 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 (8b)
295
296 result = _simd16_insert_si(result, pack, 0);
297
298 return _simd16_castsi_ps(result);
299 }
300 #endif
301 };
302
303 //////////////////////////////////////////////////////////////////////////
304 /// PackTraits - Helpers for packing / unpacking 16 bit unsigned channels
305 //////////////////////////////////////////////////////////////////////////
306 template <>
307 struct PackTraits<16, false>
308 {
309 static const uint32_t MyNumBits = 16;
310
311 static simdscalar loadSOA(const uint8_t *pSrc)
312 {
313 #if KNOB_SIMD_WIDTH == 8
314 __m256 result = _mm256_setzero_ps();
315 __m128 vLo = _mm_load_ps((const float*)pSrc);
316 return _mm256_insertf128_ps(result, vLo, 0);
317 #else
318 #error Unsupported vector width
319 #endif
320 }
321
322 static void storeSOA(uint8_t *pDst, simdscalar const &src)
323 {
324 #if KNOB_SIMD_WIDTH == 8
325 // store 16B (2B * 8)
326 _mm_store_ps((float*)pDst, _mm256_castps256_ps128(src));
327 #else
328 #error Unsupported vector width
329 #endif
330 }
331
332 static simdscalar unpack(simdscalar &in)
333 {
334 #if KNOB_SIMD_WIDTH == 8
335 #if KNOB_ARCH <= KNOB_ARCH_AVX
336 __m128i src = _mm_castps_si128(_mm256_castps256_ps128(in));
337 __m128i resLo = _mm_cvtepu16_epi32(src);
338 __m128i resHi = _mm_shuffle_epi8(src,
339 _mm_set_epi32(0x80800F0E, 0x80800D0C, 0x80800B0A, 0x80800908));
340
341 __m256i result = _mm256_castsi128_si256(resLo);
342 result = _mm256_insertf128_si256(result, resHi, 1);
343 return _mm256_castsi256_ps(result);
344 #else
345 return _mm256_castsi256_ps(_mm256_cvtepu16_epi32(_mm_castps_si128(_mm256_castps256_ps128(in))));
346 #endif
347 #else
348 #error Unsupported vector width
349 #endif
350 }
351
352 static simdscalar pack(simdscalar &in)
353 {
354 #if KNOB_SIMD_WIDTH == 8
355 simdscalari src = _simd_castps_si(in);
356 __m256i res = _mm256_castsi128_si256(_mm_packus_epi32(_mm256_castsi256_si128(src), _mm256_extractf128_si256(src, 1)));
357 return _mm256_castsi256_ps(res);
358 #else
359 #error Unsupported vector width
360 #endif
361 }
362 #if ENABLE_AVX512_SIMD16
363
364 static simd16scalar loadSOA_16(const uint8_t *pSrc)
365 {
366 simd16scalar result = _simd16_setzero_ps();
367
368 simdscalar resultlo = _simd_load_ps(reinterpret_cast<const float *>(pSrc));
369
370 result = _simd16_insert_ps(result, resultlo, 0);
371
372 return result;
373 }
374
375 static void SIMDCALL storeSOA(uint8_t *pDst, simd16scalar const &src)
376 {
377 _simd_store_ps(reinterpret_cast<float *>(pDst), _simd16_extract_ps(src, 0));
378 }
379
380 static simd16scalar unpack(simd16scalar &in)
381 {
382 simd16scalari result = _simd16_cvtepu16_epi32(_simd_castps_si(_simd16_extract_ps(in, 0)));
383
384 return _simd16_castsi_ps(result);
385 }
386
387 static simd16scalar pack(simd16scalar &in)
388 {
389 const simd16scalari zero = _simd16_setzero_si();
390
391 simd16scalari permlo = _simd16_permute2f128_si(_simd16_castps_si(in), zero, 0x08); // (0, 0, 2, 0) // r0 r1 r2 r3 r8 r9 rA rB 00 00 00 00 00 00 00 00 (32b)
392 simd16scalari permhi = _simd16_permute2f128_si(_simd16_castps_si(in), zero, 0x0D); // (0, 0, 3, 1) // r4 r5 r6 r7 rC rD rE rF 00 00 00 00 00 00 00 00
393
394 simd16scalari result = _simd16_packus_epi32(permlo, permhi); // r0 r1 r2 r3 r4 r5 r6 r7 r8 r9 rA rB rC rD rE rF 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 (16b)
395
396 return _simd16_castsi_ps(result);
397 }
398 #endif
399 };
400
401 //////////////////////////////////////////////////////////////////////////
402 /// PackTraits - Helpers for packing / unpacking 16 bit signed channels
403 //////////////////////////////////////////////////////////////////////////
404 template <>
405 struct PackTraits<16, true>
406 {
407 static const uint32_t MyNumBits = 16;
408
409 static simdscalar loadSOA(const uint8_t *pSrc)
410 {
411 #if KNOB_SIMD_WIDTH == 8
412 __m256 result = _mm256_setzero_ps();
413 __m128 vLo = _mm_load_ps((const float*)pSrc);
414 return _mm256_insertf128_ps(result, vLo, 0);
415 #else
416 #error Unsupported vector width
417 #endif
418 }
419
420 static void storeSOA(uint8_t *pDst, simdscalar const &src)
421 {
422 #if KNOB_SIMD_WIDTH == 8
423 // store 16B (2B * 8)
424 _mm_store_ps((float*)pDst, _mm256_castps256_ps128(src));
425 #else
426 #error Unsupported vector width
427 #endif
428 }
429
430 static simdscalar unpack(simdscalar &in)
431 {
432 #if KNOB_SIMD_WIDTH == 8
433 #if KNOB_ARCH <= KNOB_ARCH_AVX
434 SWR_INVALID("I think this may be incorrect.");
435 __m128i src = _mm_castps_si128(_mm256_castps256_ps128(in));
436 __m128i resLo = _mm_cvtepi16_epi32(src);
437 __m128i resHi = _mm_shuffle_epi8(src,
438 _mm_set_epi32(0x80800F0E, 0x80800D0C, 0x80800B0A, 0x80800908));
439
440 __m256i result = _mm256_castsi128_si256(resLo);
441 result = _mm256_insertf128_si256(result, resHi, 1);
442 return _mm256_castsi256_ps(result);
443 #else
444 return _mm256_castsi256_ps(_mm256_cvtepi16_epi32(_mm_castps_si128(_mm256_castps256_ps128(in))));
445 #endif
446 #else
447 #error Unsupported vector width
448 #endif
449 }
450
451 static simdscalar pack(simdscalar &in)
452 {
453 #if KNOB_SIMD_WIDTH == 8
454 simdscalari src = _simd_castps_si(in);
455 __m256i res = _mm256_castsi128_si256(_mm_packs_epi32(_mm256_castsi256_si128(src), _mm256_extractf128_si256(src, 1)));
456 return _mm256_castsi256_ps(res);
457 #else
458 #error Unsupported vector width
459 #endif
460 }
461 #if ENABLE_AVX512_SIMD16
462
463 static simd16scalar loadSOA_16(const uint8_t *pSrc)
464 {
465 simd16scalar result = _simd16_setzero_ps();
466
467 simdscalar resultlo = _simd_load_ps(reinterpret_cast<const float *>(pSrc));
468
469 result = _simd16_insert_ps(result, resultlo, 0);
470
471 return result;
472 }
473
474 static void SIMDCALL storeSOA(uint8_t *pDst, simd16scalar const &src)
475 {
476 _simd_store_ps(reinterpret_cast<float *>(pDst), _simd16_extract_ps(src, 0));
477 }
478
479 static simd16scalar unpack(simd16scalar &in)
480 {
481 simd16scalari result = _simd16_cvtepu16_epi32(_simd_castps_si(_simd16_extract_ps(in, 0)));
482
483 return _simd16_castsi_ps(result);
484 }
485
486 static simd16scalar pack(simd16scalar &in)
487 {
488 const simd16scalari zero = _simd16_setzero_si();
489
490 simd16scalari permlo = _simd16_permute2f128_si(_simd16_castps_si(in), zero, 0x08); // (0, 0, 2, 0) // r0 r1 r2 r3 r8 r9 rA rB 00 00 00 00 00 00 00 00 (32b)
491 simd16scalari permhi = _simd16_permute2f128_si(_simd16_castps_si(in), zero, 0x0D); // (0, 0, 3, 1) // r4 r5 r6 r7 rC rD rE rF 00 00 00 00 00 00 00 00
492
493 simd16scalari result = _simd16_packs_epi32(permlo, permhi); // r0 r1 r2 r3 r4 r5 r6 r7 r8 r9 rA rB rC rD rE rF 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 (16b)
494
495 return _simd16_castsi_ps(result);
496 }
497 #endif
498 };
499
500 //////////////////////////////////////////////////////////////////////////
501 /// PackTraits - Helpers for packing / unpacking 32 bit channels
502 //////////////////////////////////////////////////////////////////////////
503 template <>
504 struct PackTraits<32, false>
505 {
506 static const uint32_t MyNumBits = 32;
507
508 static simdscalar loadSOA(const uint8_t *pSrc) { return _simd_load_ps((const float*)pSrc); }
509 static void storeSOA(uint8_t *pDst, simdscalar const &src) { _simd_store_ps((float*)pDst, src); }
510 static simdscalar unpack(simdscalar &in) { return in; }
511 static simdscalar pack(simdscalar &in) { return in; }
512 #if ENABLE_AVX512_SIMD16
513
514 static simd16scalar loadSOA_16(const uint8_t *pSrc)
515 {
516 return _simd16_load_ps(reinterpret_cast<const float *>(pSrc));
517 }
518
519 static void SIMDCALL storeSOA(uint8_t *pDst, simd16scalar const &src)
520 {
521 _simd16_store_ps(reinterpret_cast<float *>(pDst), src);
522 }
523
524 static simd16scalar unpack(simd16scalar &in)
525 {
526 return in;
527 }
528
529 static simd16scalar pack(simd16scalar &in)
530 {
531 return in;
532 }
533 #endif
534 };
535
536 //////////////////////////////////////////////////////////////////////////
537 /// TypeTraits - Format type traits.
538 //////////////////////////////////////////////////////////////////////////
539 template<SWR_TYPE type, uint32_t NumBits>
540 struct TypeTraits : PackTraits<NumBits>
541 {
542 static const SWR_TYPE MyType = type;
543 static float toFloat() { return 0.0; }
544 static float fromFloat() { SWR_NOT_IMPL; return 0.0; }
545 static simdscalar convertSrgb(simdscalar &in) { SWR_NOT_IMPL; return _simd_setzero_ps(); }
546 };
547
548 //////////////////////////////////////////////////////////////////////////
549 /// TypeTraits - Format type traits specialization for UINT8
550 //////////////////////////////////////////////////////////////////////////
551 template<> struct TypeTraits<SWR_TYPE_UINT, 8> : PackTraits<8>
552 {
553 static const SWR_TYPE MyType = SWR_TYPE_UINT;
554 static float toFloat() { return 0.0; }
555 static float fromFloat() { SWR_NOT_IMPL; return 0.0; }
556 static simdscalar convertSrgb(simdscalar &in) { SWR_NOT_IMPL; return _simd_setzero_ps(); }
557 };
558
559 //////////////////////////////////////////////////////////////////////////
560 /// TypeTraits - Format type traits specialization for UINT8
561 //////////////////////////////////////////////////////////////////////////
562 template<> struct TypeTraits<SWR_TYPE_SINT, 8> : PackTraits<8, true>
563 {
564 static const SWR_TYPE MyType = SWR_TYPE_SINT;
565 static float toFloat() { return 0.0; }
566 static float fromFloat() { SWR_NOT_IMPL; return 0.0; }
567 static simdscalar convertSrgb(simdscalar &in) { SWR_NOT_IMPL; return _simd_setzero_ps(); }
568 };
569
570 //////////////////////////////////////////////////////////////////////////
571 /// TypeTraits - Format type traits specialization for UINT16
572 //////////////////////////////////////////////////////////////////////////
573 template<> struct TypeTraits<SWR_TYPE_UINT, 16> : PackTraits<16>
574 {
575 static const SWR_TYPE MyType = SWR_TYPE_UINT;
576 static float toFloat() { return 0.0; }
577 static float fromFloat() { SWR_NOT_IMPL; return 0.0; }
578 static simdscalar convertSrgb(simdscalar &in) { SWR_NOT_IMPL; return _simd_setzero_ps(); }
579 };
580
581 //////////////////////////////////////////////////////////////////////////
582 /// TypeTraits - Format type traits specialization for SINT16
583 //////////////////////////////////////////////////////////////////////////
584 template<> struct TypeTraits<SWR_TYPE_SINT, 16> : PackTraits<16, true>
585 {
586 static const SWR_TYPE MyType = SWR_TYPE_SINT;
587 static float toFloat() { return 0.0; }
588 static float fromFloat() { SWR_NOT_IMPL; return 0.0; }
589 static simdscalar convertSrgb(simdscalar &in) { SWR_NOT_IMPL; return _simd_setzero_ps(); }
590 };
591
592 //////////////////////////////////////////////////////////////////////////
593 /// TypeTraits - Format type traits specialization for UINT32
594 //////////////////////////////////////////////////////////////////////////
595 template<> struct TypeTraits<SWR_TYPE_UINT, 32> : PackTraits<32>
596 {
597 static const SWR_TYPE MyType = SWR_TYPE_UINT;
598 static float toFloat() { return 0.0; }
599 static float fromFloat() { SWR_NOT_IMPL; return 0.0; }
600 static simdscalar convertSrgb(simdscalar &in) { SWR_NOT_IMPL; return _simd_setzero_ps(); }
601 };
602
603 //////////////////////////////////////////////////////////////////////////
604 /// TypeTraits - Format type traits specialization for UINT32
605 //////////////////////////////////////////////////////////////////////////
606 template<> struct TypeTraits<SWR_TYPE_SINT, 32> : PackTraits<32>
607 {
608 static const SWR_TYPE MyType = SWR_TYPE_SINT;
609 static float toFloat() { return 0.0; }
610 static float fromFloat() { SWR_NOT_IMPL; return 0.0; }
611 static simdscalar convertSrgb(simdscalar &in) { SWR_NOT_IMPL; return _simd_setzero_ps(); }
612 };
613
614 //////////////////////////////////////////////////////////////////////////
615 /// TypeTraits - Format type traits specialization for UNORM5
616 //////////////////////////////////////////////////////////////////////////
617 template<> struct TypeTraits<SWR_TYPE_UNORM, 5> : PackTraits<5>
618 {
619 static const SWR_TYPE MyType = SWR_TYPE_UNORM;
620 static float toFloat() { return 1.0f / 31.0f; }
621 static float fromFloat() { return 31.0f; }
622 static simdscalar convertSrgb(simdscalar &in) { SWR_NOT_IMPL; return _simd_setzero_ps(); }
623 };
624
625 //////////////////////////////////////////////////////////////////////////
626 /// TypeTraits - Format type traits specialization for UNORM6
627 //////////////////////////////////////////////////////////////////////////
628 template<> struct TypeTraits<SWR_TYPE_UNORM, 6> : PackTraits<6>
629 {
630 static const SWR_TYPE MyType = SWR_TYPE_UNORM;
631 static float toFloat() { return 1.0f / 63.0f; }
632 static float fromFloat() { return 63.0f; }
633 static simdscalar convertSrgb(simdscalar &in) { SWR_NOT_IMPL; return _simd_setzero_ps(); }
634 };
635
636 //////////////////////////////////////////////////////////////////////////
637 /// TypeTraits - Format type traits specialization for UNORM8
638 //////////////////////////////////////////////////////////////////////////
639 template<> struct TypeTraits<SWR_TYPE_UNORM, 8> : PackTraits<8>
640 {
641 static const SWR_TYPE MyType = SWR_TYPE_UNORM;
642 static float toFloat() { return 1.0f / 255.0f; }
643 static float fromFloat() { return 255.0f; }
644 static simdscalar convertSrgb(simdscalar &in) { SWR_NOT_IMPL; return _simd_setzero_ps(); }
645 };
646
647 //////////////////////////////////////////////////////////////////////////
648 /// TypeTraits - Format type traits specialization for UNORM8
649 //////////////////////////////////////////////////////////////////////////
650 template<> struct TypeTraits<SWR_TYPE_SNORM, 8> : PackTraits<8, true>
651 {
652 static const SWR_TYPE MyType = SWR_TYPE_SNORM;
653 static float toFloat() { return 1.0f / 127.0f; }
654 static float fromFloat() { return 127.0f; }
655 static simdscalar convertSrgb(simdscalar &in) { SWR_NOT_IMPL; return _simd_setzero_ps(); }
656 };
657
658 //////////////////////////////////////////////////////////////////////////
659 /// TypeTraits - Format type traits specialization for UNORM16
660 //////////////////////////////////////////////////////////////////////////
661 template<> struct TypeTraits<SWR_TYPE_UNORM, 16> : PackTraits<16>
662 {
663 static const SWR_TYPE MyType = SWR_TYPE_UNORM;
664 static float toFloat() { return 1.0f / 65535.0f; }
665 static float fromFloat() { return 65535.0f; }
666 static simdscalar convertSrgb(simdscalar &in) { SWR_NOT_IMPL; return _simd_setzero_ps(); }
667 };
668
669 //////////////////////////////////////////////////////////////////////////
670 /// TypeTraits - Format type traits specialization for SNORM16
671 //////////////////////////////////////////////////////////////////////////
672 template<> struct TypeTraits<SWR_TYPE_SNORM, 16> : PackTraits<16, true>
673 {
674 static const SWR_TYPE MyType = SWR_TYPE_UNORM;
675 static float toFloat() { return 1.0f / 32767.0f; }
676 static float fromFloat() { return 32767.0f; }
677 static simdscalar convertSrgb(simdscalar &in) { SWR_NOT_IMPL; return _simd_setzero_ps(); }
678 };
679
680 //////////////////////////////////////////////////////////////////////////
681 /// TypeTraits - Format type traits specialization for UNORM24
682 //////////////////////////////////////////////////////////////////////////
683 template<>
684 struct TypeTraits < SWR_TYPE_UNORM, 24 > : PackTraits<32>
685 {
686 static const SWR_TYPE MyType = SWR_TYPE_UNORM;
687 static float toFloat() { return 1.0f / 16777215.0f; }
688 static float fromFloat() { return 16777215.0f; }
689 static simdscalar convertSrgb(simdscalar &in) { SWR_NOT_IMPL; return _simd_setzero_ps(); }
690 };
691
692 //////////////////////////////////////////////////////////////////////////
693 // FLOAT Specializations from here on...
694 //////////////////////////////////////////////////////////////////////////
695 #define TO_M128i(a) _mm_castps_si128(a)
696 #define TO_M128(a) _mm_castsi128_ps(a)
697
698 #include "math.h"
699
700 template< unsigned expnum, unsigned expden, unsigned coeffnum, unsigned coeffden >
701 inline static __m128 fastpow(__m128 arg) {
702 __m128 ret = arg;
703
704 static const __m128 factor = _mm_set1_ps(exp2(127.0f * expden / expnum - 127.0f)
705 * powf(1.0f * coeffnum / coeffden, 1.0f * expden / expnum));
706
707 // Apply a constant pre-correction factor.
708 ret = _mm_mul_ps(ret, factor);
709
710 // Reinterpret arg as integer to obtain logarithm.
711 //asm("cvtdq2ps %1, %0" : "=x" (ret) : "x" (ret));
712 ret = _mm_cvtepi32_ps(_mm_castps_si128(ret));
713
714 // Multiply logarithm by power.
715 ret = _mm_mul_ps(ret, _mm_set1_ps(1.0f * expnum / expden));
716
717 // Convert back to "integer" to exponentiate.
718 //asm("cvtps2dq %1, %0" : "=x" (ret) : "x" (ret));
719 ret = _mm_castsi128_ps(_mm_cvtps_epi32(ret));
720
721 return ret;
722 }
723
724 inline static __m128 pow512_4(__m128 arg) {
725 // 5/12 is too small, so compute the 4th root of 20/12 instead.
726 // 20/12 = 5/3 = 1 + 2/3 = 2 - 1/3. 2/3 is a suitable argument for fastpow.
727 // weighting coefficient: a^-1/2 = 2 a; a = 2^-2/3
728 __m128 xf = fastpow< 2, 3, int(0.629960524947437 * 1e9), int(1e9) >(arg);
729 __m128 xover = _mm_mul_ps(arg, xf);
730
731 __m128 xfm1 = _mm_rsqrt_ps(xf);
732 __m128 x2 = _mm_mul_ps(arg, arg);
733 __m128 xunder = _mm_mul_ps(x2, xfm1);
734
735 // sqrt2 * over + 2 * sqrt2 * under
736 __m128 xavg = _mm_mul_ps(_mm_set1_ps(1.0f / (3.0f * 0.629960524947437f) * 0.999852f),
737 _mm_add_ps(xover, xunder));
738
739 xavg = _mm_mul_ps(xavg, _mm_rsqrt_ps(xavg));
740 xavg = _mm_mul_ps(xavg, _mm_rsqrt_ps(xavg));
741 return xavg;
742 }
743
744 inline static __m128 powf_wrapper(__m128 Base, float Exp)
745 {
746 float *f = (float *)(&Base);
747
748 return _mm_set_ps(powf(f[3], Exp),
749 powf(f[2], Exp),
750 powf(f[1], Exp),
751 powf(f[0], Exp));
752 }
753
754 static inline __m128 ConvertFloatToSRGB2(__m128& Src)
755 {
756 // create a mask with 0xFFFFFFFF in the DWORDs where the source is <= the minimal SRGB float value
757 __m128i CmpToSRGBThresholdMask = TO_M128i(_mm_cmpnlt_ps(_mm_set1_ps(0.0031308f), Src));
758
759 // squeeze the mask down to 16 bits (4 bits per DWORD)
760 int CompareResult = _mm_movemask_epi8(CmpToSRGBThresholdMask);
761
762 __m128 Result;
763
764 //
765 if (CompareResult == 0xFFFF)
766 {
767 // all DWORDs are <= the threshold
768 Result = _mm_mul_ps(Src, _mm_set1_ps(12.92f));
769 }
770 else if (CompareResult == 0x0)
771 {
772 // all DWORDs are > the threshold
773 __m128 fSrc_0RGB = Src;
774
775 // --> 1.055f * c(1.0f/2.4f) - 0.055f
776 #if KNOB_USE_FAST_SRGB == TRUE
777 // 1.0f / 2.4f is 5.0f / 12.0f which is used for approximation.
778 __m128 f = pow512_4(fSrc_0RGB);
779 #else
780 __m128 f = powf_wrapper(fSrc_0RGB, 1.0f / 2.4f);
781 #endif
782 f = _mm_mul_ps(f, _mm_set1_ps(1.055f));
783 Result = _mm_sub_ps(f, _mm_set1_ps(0.055f));
784 }
785 else
786 {
787 // some DWORDs are <= the threshold and some are > threshold
788 __m128 Src_0RGB_mul_denorm = _mm_mul_ps(Src, _mm_set1_ps(12.92f));
789
790 __m128 fSrc_0RGB = Src;
791
792 // --> 1.055f * c(1.0f/2.4f) - 0.055f
793 #if KNOB_USE_FAST_SRGB == TRUE
794 // 1.0f / 2.4f is 5.0f / 12.0f which is used for approximation.
795 __m128 f = pow512_4(fSrc_0RGB);
796 #else
797 __m128 f = powf_wrapper(fSrc_0RGB, 1.0f / 2.4f);
798 #endif
799 f = _mm_mul_ps(f, _mm_set1_ps(1.055f));
800 f = _mm_sub_ps(f, _mm_set1_ps(0.055f));
801
802 // Clear the alpha (is garbage after the sub)
803 __m128i i = _mm_and_si128(TO_M128i(f), _mm_set_epi32(0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF));
804
805 __m128i LessThanPart = _mm_and_si128(CmpToSRGBThresholdMask, TO_M128i(Src_0RGB_mul_denorm));
806 __m128i GreaterEqualPart = _mm_andnot_si128(CmpToSRGBThresholdMask, i);
807 __m128i CombinedParts = _mm_or_si128(LessThanPart, GreaterEqualPart);
808
809 Result = TO_M128(CombinedParts);
810 }
811
812 return Result;
813 }
814
815 #if ENABLE_AVX512_SIMD16
816 template< unsigned expnum, unsigned expden, unsigned coeffnum, unsigned coeffden >
817 inline static simd16scalar SIMDCALL fastpow(simd16scalar const &value)
818 {
819 static const float factor1 = exp2(127.0f * expden / expnum - 127.0f)
820 * powf(1.0f * coeffnum / coeffden, 1.0f * expden / expnum);
821
822 // Apply a constant pre-correction factor.
823 simd16scalar result = _simd16_mul_ps(value, _simd16_set1_ps(factor1));
824
825 // Reinterpret arg as integer to obtain logarithm.
826 //asm("cvtdq2ps %1, %0" : "=x" (result) : "x" (result));
827 result = _simd16_cvtepi32_ps(_simd16_castps_si(result));
828
829 // Multiply logarithm by power.
830 result = _simd16_mul_ps(result, _simd16_set1_ps(1.0f * expnum / expden));
831
832 // Convert back to "integer" to exponentiate.
833 //asm("cvtps2dq %1, %0" : "=x" (result) : "x" (result));
834 result = _simd16_castsi_ps(_simd16_cvtps_epi32(result));
835
836 return result;
837 }
838
839 inline static simd16scalar SIMDCALL pow512_4(simd16scalar const &arg)
840 {
841 // 5/12 is too small, so compute the 4th root of 20/12 instead.
842 // 20/12 = 5/3 = 1 + 2/3 = 2 - 1/3. 2/3 is a suitable argument for fastpow.
843 // weighting coefficient: a^-1/2 = 2 a; a = 2^-2/3
844 simd16scalar xf = fastpow< 2, 3, int(0.629960524947437 * 1e9), int(1e9) >(arg);
845 simd16scalar xover = _simd16_mul_ps(arg, xf);
846
847 simd16scalar xfm1 = _simd16_rsqrt_ps(xf);
848 simd16scalar x2 = _simd16_mul_ps(arg, arg);
849 simd16scalar xunder = _simd16_mul_ps(x2, xfm1);
850
851 // sqrt2 * over + 2 * sqrt2 * under
852 simd16scalar xavg = _simd16_mul_ps(_simd16_set1_ps(1.0f / (3.0f * 0.629960524947437f) * 0.999852f), _simd16_add_ps(xover, xunder));
853
854 xavg = _simd16_mul_ps(xavg, _simd16_rsqrt_ps(xavg));
855 xavg = _simd16_mul_ps(xavg, _simd16_rsqrt_ps(xavg));
856
857 return xavg;
858 }
859
860 inline static simd16scalar SIMDCALL powf_wrapper(const simd16scalar &base, float exp)
861 {
862 const float *f = reinterpret_cast<const float *>(&base);
863
864 return _simd16_set_ps(
865 powf(f[15], exp),
866 powf(f[14], exp),
867 powf(f[13], exp),
868 powf(f[12], exp),
869 powf(f[11], exp),
870 powf(f[10], exp),
871 powf(f[ 9], exp),
872 powf(f[ 8], exp),
873 powf(f[ 7], exp),
874 powf(f[ 6], exp),
875 powf(f[ 5], exp),
876 powf(f[ 4], exp),
877 powf(f[ 3], exp),
878 powf(f[ 2], exp),
879 powf(f[ 1], exp),
880 powf(f[ 0], exp)
881 );
882 }
883
884 // float to SRGB conversion formula
885 //
886 // if (value < 0.0031308f)
887 // value *= 12.92f;
888 // else
889 // value = 1.055f * pow(value, 1.0f / 2.4f) - 0.055f;
890 //
891 static inline simd16scalar ConvertFloatToSRGB2(const simd16scalar &value)
892 {
893 // create a mask where the source is < the minimal SRGB float value
894 const simd16mask mask = _simd16_cmplt_ps_mask(value, _simd16_set1_ps(0.0031308f));
895
896 // if all elements are < the threshold, result = value * 12.92
897 simd16scalar result = _simd16_mul_ps(value, _simd16_set1_ps(12.92f));
898
899 if (_simd16_mask2int(mask) != 0xFFFF)
900 {
901 // some elements are >= threshold, result = 1.055 * power(value, 1.0 / 2.4) - 0.055
902 #if KNOB_USE_FAST_SRGB == TRUE
903 // 1.0f / 2.4f is 5.0f / 12.0f which is used for approximation.
904 simd16scalar result2 = pow512_4(value);
905 #else
906 simd16scalar result2 = powf_wrapper(value, 1.0f / 2.4f);
907 #endif
908
909 result2 = _simd16_mul_ps(result2, _simd16_set1_ps(1.055f));
910 result2 = _simd16_sub_ps(result2, _simd16_set1_ps(0.055f));
911
912 #if (KNOB_ARCH == KNOB_ARCH_AVX512)
913 // only native AVX512 can directly use the computed mask for the blend operation
914 result = _mm512_mask_blend_ps(mask, result2, result);
915 #else
916 result = _simd16_blendv_ps(result2, result, _simd16_cmplt_ps(value, _simd16_set1_ps(0.0031308f)));
917 #endif
918 }
919
920 return result;
921 }
922
923 #endif
924 //////////////////////////////////////////////////////////////////////////
925 /// TypeTraits - Format type traits specialization for FLOAT16
926 //////////////////////////////////////////////////////////////////////////
927 template<> struct TypeTraits<SWR_TYPE_FLOAT, 16> : PackTraits<16>
928 {
929 static const SWR_TYPE MyType = SWR_TYPE_FLOAT;
930 static float toFloat() { return 1.0f; }
931 static float fromFloat() { return 1.0f; }
932 static simdscalar convertSrgb(simdscalar &in) { SWR_NOT_IMPL; return _simd_setzero_ps(); }
933
934 static simdscalar pack(const simdscalar &in)
935 {
936 #if KNOB_SIMD_WIDTH == 8
937 #if (KNOB_ARCH == KNOB_ARCH_AVX)
938 // input is 8 packed float32, output is 8 packed float16
939 simdscalari src = _simd_castps_si(in);
940
941 static const uint32_t FLOAT_EXP_BITS = 8;
942 static const uint32_t FLOAT_MANTISSA_BITS = 23;
943 static const uint32_t FLOAT_MANTISSA_MASK = (1U << FLOAT_MANTISSA_BITS) - 1;
944 static const uint32_t FLOAT_EXP_MASK = ((1U << FLOAT_EXP_BITS) - 1) << FLOAT_MANTISSA_BITS;
945
946 static const uint32_t HALF_EXP_BITS = 5;
947 static const uint32_t HALF_MANTISSA_BITS = 10;
948 static const uint32_t HALF_EXP_MASK = ((1U << HALF_EXP_BITS) - 1) << HALF_MANTISSA_BITS;
949
950 // minimum exponent required, exponents below this are flushed to 0.
951 static const int32_t HALF_EXP_MIN = -14;
952 static const int32_t FLOAT_EXP_BIAS = 127;
953 static const int32_t FLOAT_EXP_MIN = HALF_EXP_MIN + FLOAT_EXP_BIAS;
954 static const int32_t FLOAT_EXP_MIN_FTZ = FLOAT_EXP_MIN - (HALF_MANTISSA_BITS + 1); // +1 for the lack of implicit significand
955
956 // maximum exponent required, exponents above this are set to infinity
957 static const int32_t HALF_EXP_MAX = 15;
958 static const int32_t FLOAT_EXP_MAX = HALF_EXP_MAX + FLOAT_EXP_BIAS;
959
960 const simdscalari vSignMask = _simd_set1_epi32(0x80000000);
961 const simdscalari vExpMask = _simd_set1_epi32(FLOAT_EXP_MASK);
962 const simdscalari vManMask = _simd_set1_epi32(FLOAT_MANTISSA_MASK);
963 const simdscalari vExpMin = _simd_set1_epi32(FLOAT_EXP_MASK & uint32_t(FLOAT_EXP_MIN << FLOAT_MANTISSA_BITS));
964 const simdscalari vExpMinFtz = _simd_set1_epi32(FLOAT_EXP_MASK & uint32_t(FLOAT_EXP_MIN_FTZ << FLOAT_MANTISSA_BITS));
965 const simdscalari vExpMax = _simd_set1_epi32(FLOAT_EXP_MASK & uint32_t(FLOAT_EXP_MAX << FLOAT_MANTISSA_BITS));
966
967 simdscalari vSign = _simd_and_si(src, vSignMask);
968 simdscalari vExp = _simd_and_si(src, vExpMask);
969 simdscalari vMan = _simd_and_si(src, vManMask);
970
971 simdscalari vFTZMask = _simd_cmplt_epi32(vExp, vExpMinFtz);
972 simdscalari vDenormMask = _simd_andnot_si(vFTZMask, _simd_cmplt_epi32(vExp, vExpMin));
973 simdscalari vInfMask = _simd_cmpeq_epi32(vExpMask, vExp);
974 simdscalari vClampMask = _simd_andnot_si(vInfMask, _simd_cmplt_epi32(vExpMax, vExp));
975
976 simdscalari vHalfExp = _simd_add_epi32(_simd_sub_epi32(vExp, vExpMin), _simd_set1_epi32(1U << FLOAT_MANTISSA_BITS));
977
978 // pack output 16-bits into the lower 16-bits of each 32-bit channel
979 simdscalari vDst = _simd_and_si(_simd_srli_epi32(vHalfExp, 13), _simd_set1_epi32(HALF_EXP_MASK));
980 vDst = _simd_or_si(vDst, _simd_srli_epi32(vMan, FLOAT_MANTISSA_BITS - HALF_MANTISSA_BITS));
981
982 // Flush To Zero
983 vDst = _simd_andnot_si(vFTZMask, vDst);
984 // Apply Infinites / NaN
985 vDst = _simd_or_si(vDst, _simd_and_si(vInfMask, _simd_set1_epi32(HALF_EXP_MASK)));
986
987 // Apply clamps
988 vDst = _simd_andnot_si(vClampMask, vDst);
989 vDst = _simd_or_si(vDst,
990 _simd_and_si(vClampMask, _simd_set1_epi32(0x7BFF)));
991
992 // Compute Denormals (subnormals)
993 if (!_mm256_testz_si256(vDenormMask, vDenormMask))
994 {
995 uint32_t *pDenormMask = (uint32_t*)&vDenormMask;
996 uint32_t *pExp = (uint32_t*)&vExp;
997 uint32_t *pMan = (uint32_t*)&vMan;
998 uint32_t *pDst = (uint32_t*)&vDst;
999 for (uint32_t i = 0; i < KNOB_SIMD_WIDTH; ++i)
1000 {
1001 if (pDenormMask[i])
1002 {
1003 // Need to compute subnormal value
1004 uint32_t exponent = pExp[i] >> FLOAT_MANTISSA_BITS;
1005 uint32_t mantissa = pMan[i] |
1006 (1U << FLOAT_MANTISSA_BITS); // Denorms include no "implicit" 1s. Make it explicit
1007
1008 pDst[i] = mantissa >> ((FLOAT_EXP_MIN - exponent) + (FLOAT_MANTISSA_BITS - HALF_MANTISSA_BITS));
1009 }
1010 }
1011 }
1012
1013 // Add in sign bits
1014 vDst = _simd_or_si(vDst, _simd_srli_epi32(vSign, 16));
1015
1016 // Pack to lower 128-bits
1017 vDst = _mm256_castsi128_si256(_mm_packus_epi32(_mm256_castsi256_si128(vDst), _mm256_extractf128_si256(vDst, 1)));
1018
1019 #if 0
1020 #if !defined(NDEBUG)
1021 simdscalari vCheck = _mm256_castsi128_si256(_mm256_cvtps_ph(in, _MM_FROUND_TRUNC));
1022
1023 for (uint32_t i = 0; i < 4; ++i)
1024 {
1025 SWR_ASSERT(vCheck.m256i_i32[i] == vDst.m256i_i32[i]);
1026 }
1027 #endif
1028 #endif
1029
1030 return _simd_castsi_ps(vDst);
1031
1032 #else
1033 return _mm256_castsi256_ps(_mm256_castsi128_si256(_mm256_cvtps_ph(in, _MM_FROUND_TRUNC)));
1034 #endif
1035 #else
1036 #error Unsupported vector width
1037 #endif
1038 }
1039
1040 static simdscalar unpack(const simdscalar &in)
1041 {
1042 // input is 8 packed float16, output is 8 packed float32
1043 SWR_NOT_IMPL; // @todo
1044 return _simd_setzero_ps();
1045 }
1046 #if ENABLE_AVX512_SIMD16
1047
1048 static simd16scalar pack(const simd16scalar &in)
1049 {
1050 simd16scalari result = _simd16_setzero_si();
1051 simdscalari resultlo = _simd_setzero_si();
1052
1053 #if (KNOB_ARCH == KNOB_ARCH_AVX)
1054 simdscalar simdlo = pack(_simd16_extract_ps(in, 0));
1055 simdscalar simdhi = pack(_simd16_extract_ps(in, 1));
1056
1057 __m128i templo = _simd_extractf128_si(_simd_castps_si(simdlo), 0);
1058 __m128i temphi = _simd_extractf128_si(_simd_castps_si(simdhi), 0);
1059
1060 #else
1061 __m128i templo = _mm256_cvtps_ph(_simd16_extract_ps(in, 0), _MM_FROUND_TRUNC);
1062 __m128i temphi = _mm256_cvtps_ph(_simd16_extract_ps(in, 1), _MM_FROUND_TRUNC);
1063
1064 #endif
1065 resultlo = _simd_insertf128_si(resultlo, templo, 0);
1066 resultlo = _simd_insertf128_si(resultlo, temphi, 1);
1067
1068 result = _simd16_insert_si(result, resultlo, 0);
1069
1070 return _simd16_castsi_ps(result);
1071 }
1072
1073 static simd16scalar unpack(const simd16scalar &in)
1074 {
1075 // input is 16 packed float16, output is 16 packed float32
1076 SWR_NOT_IMPL; // @todo
1077 return _simd16_setzero_ps();
1078 }
1079 #endif
1080 };
1081
1082 //////////////////////////////////////////////////////////////////////////
1083 /// TypeTraits - Format type traits specialization for FLOAT32
1084 //////////////////////////////////////////////////////////////////////////
1085 template<> struct TypeTraits<SWR_TYPE_FLOAT, 32> : PackTraits<32>
1086 {
1087 static const SWR_TYPE MyType = SWR_TYPE_FLOAT;
1088 static float toFloat() { return 1.0f; }
1089 static float fromFloat() { return 1.0f; }
1090 static inline simdscalar convertSrgb(simdscalar &in)
1091 {
1092 #if KNOB_SIMD_WIDTH == 8
1093 __m128 srcLo = _mm256_extractf128_ps(in, 0);
1094 __m128 srcHi = _mm256_extractf128_ps(in, 1);
1095
1096 srcLo = ConvertFloatToSRGB2(srcLo);
1097 srcHi = ConvertFloatToSRGB2(srcHi);
1098
1099 in = _mm256_insertf128_ps(in, srcLo, 0);
1100 in = _mm256_insertf128_ps(in, srcHi, 1);
1101 #else
1102 #error Unsupported vector width
1103 #endif
1104 return in;
1105 }
1106 #if ENABLE_AVX512_SIMD16
1107
1108 static inline simd16scalar convertSrgb(simd16scalar &in)
1109 {
1110 return ConvertFloatToSRGB2(in);
1111 }
1112 #endif
1113 };
1114
1115 //////////////////////////////////////////////////////////////////////////
1116 /// FormatIntType - Calculate base integer type for pixel components based
1117 /// on total number of bits. Components can be smaller
1118 /// that this type, but the entire pixel must not be
1119 /// any smaller than this type.
1120 //////////////////////////////////////////////////////////////////////////
1121 template <uint32_t bits, bool bits8 = bits <= 8, bool bits16 = bits <= 16>
1122 struct FormatIntType
1123 {
1124 typedef uint32_t TYPE;
1125 };
1126
1127 template <uint32_t bits>
1128 struct FormatIntType<bits, true, true>
1129 {
1130 typedef uint8_t TYPE;
1131 };
1132
1133 template <uint32_t bits>
1134 struct FormatIntType<bits, false, true>
1135 {
1136 typedef uint16_t TYPE;
1137 };
1138
1139 //////////////////////////////////////////////////////////////////////////
1140 /// Format1 - Bitfield for single component formats.
1141 //////////////////////////////////////////////////////////////////////////
1142 template<uint32_t x>
1143 union Format1
1144 {
1145 typedef typename FormatIntType<x>::TYPE TYPE;
1146 struct
1147 {
1148 TYPE r : x;
1149 };
1150
1151 ///@ The following are here to provide full template needed in Formats.
1152 struct
1153 {
1154 TYPE g : x;
1155 };
1156 struct
1157 {
1158 TYPE b : x;
1159 };
1160 struct
1161 {
1162 TYPE a : x;
1163 };
1164 };
1165
1166 //////////////////////////////////////////////////////////////////////////
1167 /// Format2 - Bitfield for 2 component formats.
1168 //////////////////////////////////////////////////////////////////////////
1169 template<uint32_t x, uint32_t y>
1170 union Format2
1171 {
1172 typedef typename FormatIntType<x + y>::TYPE TYPE;
1173
1174 struct
1175 {
1176 TYPE r : x;
1177 TYPE g : y;
1178 };
1179 struct
1180 {
1181 ///@ The following are here to provide full template needed in Formats.
1182 TYPE b : x;
1183 TYPE a : y;
1184 };
1185 };
1186
1187 //////////////////////////////////////////////////////////////////////////
1188 /// Format3 - Bitfield for 3 component formats.
1189 //////////////////////////////////////////////////////////////////////////
1190 template<uint32_t x, uint32_t y, uint32_t z>
1191 union Format3
1192 {
1193 typedef typename FormatIntType<x + y + z>::TYPE TYPE;
1194
1195 struct
1196 {
1197 TYPE r : x;
1198 TYPE g : y;
1199 TYPE b : z;
1200 };
1201 TYPE a; ///@note This is here to provide full template needed in Formats.
1202 };
1203
1204 //////////////////////////////////////////////////////////////////////////
1205 /// Format4 - Bitfield for 4 component formats.
1206 //////////////////////////////////////////////////////////////////////////
1207 template<uint32_t x, uint32_t y, uint32_t z, uint32_t w>
1208 struct Format4
1209 {
1210 typedef typename FormatIntType<x + y + z + w>::TYPE TYPE;
1211
1212 TYPE r : x;
1213 TYPE g : y;
1214 TYPE b : z;
1215 TYPE a : w;
1216 };
1217
1218 //////////////////////////////////////////////////////////////////////////
1219 /// ComponentTraits - Default components
1220 //////////////////////////////////////////////////////////////////////////
1221 template<uint32_t x, uint32_t y, uint32_t z, uint32_t w>
1222 struct Defaults
1223 {
1224 INLINE static uint32_t GetDefault(uint32_t comp)
1225 {
1226 static const uint32_t defaults[4]{ x, y, z, w };
1227 return defaults[comp];
1228 }
1229 };
1230
1231 //////////////////////////////////////////////////////////////////////////
1232 /// ComponentTraits - Component type traits.
1233 //////////////////////////////////////////////////////////////////////////
1234 template<SWR_TYPE X, uint32_t NumBitsX, SWR_TYPE Y = SWR_TYPE_UNKNOWN, uint32_t NumBitsY = 0, SWR_TYPE Z = SWR_TYPE_UNKNOWN, uint32_t NumBitsZ = 0, SWR_TYPE W = SWR_TYPE_UNKNOWN, uint32_t NumBitsW = 0>
1235 struct ComponentTraits
1236 {
1237 INLINE static SWR_TYPE GetType(uint32_t comp)
1238 {
1239 static const SWR_TYPE CompType[4]{ X, Y, Z, W };
1240 return CompType[comp];
1241 }
1242
1243 INLINE static constexpr uint32_t GetConstBPC(uint32_t comp)
1244 {
1245 return (comp == 3) ? NumBitsW :
1246 ((comp == 2) ? NumBitsZ :
1247 ((comp == 1) ? NumBitsY : NumBitsX) );
1248 }
1249
1250 INLINE static uint32_t GetBPC(uint32_t comp)
1251 {
1252 static const uint32_t MyBpc[4]{ NumBitsX, NumBitsY, NumBitsZ, NumBitsW };
1253 return MyBpc[comp];
1254 }
1255
1256 INLINE static bool isNormalized(uint32_t comp)
1257 {
1258 switch (comp)
1259 {
1260 case 0:
1261 return (X == SWR_TYPE_UNORM || X == SWR_TYPE_SNORM) ? true : false;
1262 case 1:
1263 return (Y == SWR_TYPE_UNORM || Y == SWR_TYPE_SNORM) ? true : false;
1264 case 2:
1265 return (Z == SWR_TYPE_UNORM || Z == SWR_TYPE_SNORM) ? true : false;
1266 case 3:
1267 return (W == SWR_TYPE_UNORM || W == SWR_TYPE_SNORM) ? true : false;
1268 }
1269 SWR_INVALID("Invalid component: %d", comp);
1270 return false;
1271 }
1272
1273 INLINE static float toFloat(uint32_t comp)
1274 {
1275 switch (comp)
1276 {
1277 case 0:
1278 return TypeTraits<X, NumBitsX>::toFloat();
1279 case 1:
1280 return TypeTraits<Y, NumBitsY>::toFloat();
1281 case 2:
1282 return TypeTraits<Z, NumBitsZ>::toFloat();
1283 case 3:
1284 return TypeTraits<W, NumBitsW>::toFloat();
1285 }
1286 SWR_INVALID("Invalid component: %d", comp);
1287 return TypeTraits<X, NumBitsX>::toFloat();
1288
1289 }
1290
1291 INLINE static float fromFloat(uint32_t comp)
1292 {
1293 switch (comp)
1294 {
1295 case 0:
1296 return TypeTraits<X, NumBitsX>::fromFloat();
1297 case 1:
1298 return TypeTraits<Y, NumBitsY>::fromFloat();
1299 case 2:
1300 return TypeTraits<Z, NumBitsZ>::fromFloat();
1301 case 3:
1302 return TypeTraits<W, NumBitsW>::fromFloat();
1303 }
1304 SWR_INVALID("Invalid component: %d", comp);
1305 return TypeTraits<X, NumBitsX>::fromFloat();
1306 }
1307
1308 INLINE static simdscalar loadSOA(uint32_t comp, const uint8_t* pSrc)
1309 {
1310 switch (comp)
1311 {
1312 case 0:
1313 return TypeTraits<X, NumBitsX>::loadSOA(pSrc);
1314 case 1:
1315 return TypeTraits<Y, NumBitsY>::loadSOA(pSrc);
1316 case 2:
1317 return TypeTraits<Z, NumBitsZ>::loadSOA(pSrc);
1318 case 3:
1319 return TypeTraits<W, NumBitsW>::loadSOA(pSrc);
1320 }
1321 SWR_INVALID("Invalid component: %d", comp);
1322 return TypeTraits<X, NumBitsX>::loadSOA(pSrc);
1323 }
1324
1325 INLINE static void storeSOA(uint32_t comp, uint8_t *pDst, simdscalar const &src)
1326 {
1327 switch (comp)
1328 {
1329 case 0:
1330 TypeTraits<X, NumBitsX>::storeSOA(pDst, src);
1331 return;
1332 case 1:
1333 TypeTraits<Y, NumBitsY>::storeSOA(pDst, src);
1334 return;
1335 case 2:
1336 TypeTraits<Z, NumBitsZ>::storeSOA(pDst, src);
1337 return;
1338 case 3:
1339 TypeTraits<W, NumBitsW>::storeSOA(pDst, src);
1340 return;
1341 }
1342 SWR_INVALID("Invalid component: %d", comp);
1343 }
1344
1345 INLINE static simdscalar unpack(uint32_t comp, simdscalar &in)
1346 {
1347 simdscalar out;
1348 switch (comp)
1349 {
1350 case 0:
1351 out = TypeTraits<X, NumBitsX>::unpack(in); break;
1352 case 1:
1353 out = TypeTraits<Y, NumBitsY>::unpack(in); break;
1354 case 2:
1355 out = TypeTraits<Z, NumBitsZ>::unpack(in); break;
1356 case 3:
1357 out = TypeTraits<W, NumBitsW>::unpack(in); break;
1358 default:
1359 SWR_INVALID("Invalid component: %d", comp);
1360 out = in;
1361 break;
1362 }
1363 return out;
1364 }
1365
1366 INLINE static simdscalar pack(uint32_t comp, simdscalar &in)
1367 {
1368 simdscalar out;
1369 switch (comp)
1370 {
1371 case 0:
1372 out = TypeTraits<X, NumBitsX>::pack(in); break;
1373 case 1:
1374 out = TypeTraits<Y, NumBitsY>::pack(in); break;
1375 case 2:
1376 out = TypeTraits<Z, NumBitsZ>::pack(in); break;
1377 case 3:
1378 out = TypeTraits<W, NumBitsW>::pack(in); break;
1379 default:
1380 SWR_INVALID("Invalid component: %d", comp);
1381 out = in;
1382 break;
1383 }
1384 return out;
1385 }
1386
1387 INLINE static simdscalar convertSrgb(uint32_t comp, simdscalar &in)
1388 {
1389 switch (comp)
1390 {
1391 case 0:
1392 return TypeTraits<X, NumBitsX>::convertSrgb(in);
1393 case 1:
1394 return TypeTraits<Y, NumBitsY>::convertSrgb(in);
1395 case 2:
1396 return TypeTraits<Z, NumBitsZ>::convertSrgb(in);
1397 case 3:
1398 return TypeTraits<W, NumBitsW>::convertSrgb(in);
1399 }
1400 SWR_INVALID("Invalid component: %d", comp);
1401 return TypeTraits<X, NumBitsX>::convertSrgb(in);
1402 }
1403 #if ENABLE_AVX512_SIMD16
1404
1405 INLINE static simd16scalar loadSOA_16(uint32_t comp, const uint8_t* pSrc)
1406 {
1407 switch (comp)
1408 {
1409 case 0:
1410 return TypeTraits<X, NumBitsX>::loadSOA_16(pSrc);
1411 case 1:
1412 return TypeTraits<Y, NumBitsY>::loadSOA_16(pSrc);
1413 case 2:
1414 return TypeTraits<Z, NumBitsZ>::loadSOA_16(pSrc);
1415 case 3:
1416 return TypeTraits<W, NumBitsW>::loadSOA_16(pSrc);
1417 }
1418 SWR_INVALID("Invalid component: %d", comp);
1419 return TypeTraits<X, NumBitsX>::loadSOA_16(pSrc);
1420 }
1421
1422 INLINE static void SIMDCALL storeSOA(uint32_t comp, uint8_t *pDst, simd16scalar const &src)
1423 {
1424 switch (comp)
1425 {
1426 case 0:
1427 TypeTraits<X, NumBitsX>::storeSOA(pDst, src);
1428 return;
1429 case 1:
1430 TypeTraits<Y, NumBitsY>::storeSOA(pDst, src);
1431 return;
1432 case 2:
1433 TypeTraits<Z, NumBitsZ>::storeSOA(pDst, src);
1434 return;
1435 case 3:
1436 TypeTraits<W, NumBitsW>::storeSOA(pDst, src);
1437 return;
1438 }
1439 SWR_INVALID("Invalid component: %d", comp);
1440 TypeTraits<X, NumBitsX>::storeSOA(pDst, src);
1441 }
1442
1443 INLINE static simd16scalar unpack(uint32_t comp, simd16scalar &in)
1444 {
1445 switch (comp)
1446 {
1447 case 0:
1448 return TypeTraits<X, NumBitsX>::unpack(in);
1449 case 1:
1450 return TypeTraits<Y, NumBitsY>::unpack(in);
1451 case 2:
1452 return TypeTraits<Z, NumBitsZ>::unpack(in);
1453 case 3:
1454 return TypeTraits<W, NumBitsW>::unpack(in);
1455 }
1456 SWR_INVALID("Invalid component: %d", comp);
1457 return TypeTraits<X, NumBitsX>::unpack(in);
1458 }
1459
1460 INLINE static simd16scalar pack(uint32_t comp, simd16scalar &in)
1461 {
1462 switch (comp)
1463 {
1464 case 0:
1465 return TypeTraits<X, NumBitsX>::pack(in);
1466 case 1:
1467 return TypeTraits<Y, NumBitsY>::pack(in);
1468 case 2:
1469 return TypeTraits<Z, NumBitsZ>::pack(in);
1470 case 3:
1471 return TypeTraits<W, NumBitsW>::pack(in);
1472 }
1473 SWR_INVALID("Invalid component: %d", comp);
1474 return TypeTraits<X, NumBitsX>::pack(in);
1475 }
1476
1477 INLINE static simd16scalar convertSrgb(uint32_t comp, simd16scalar &in)
1478 {
1479 switch (comp)
1480 {
1481 case 0:
1482 return TypeTraits<X, NumBitsX>::convertSrgb(in);
1483 case 1:
1484 return TypeTraits<Y, NumBitsY>::convertSrgb(in);
1485 case 2:
1486 return TypeTraits<Z, NumBitsZ>::convertSrgb(in);
1487 case 3:
1488 return TypeTraits<W, NumBitsW>::convertSrgb(in);
1489 }
1490 SWR_INVALID("Invalid component: %d", comp);
1491 return TypeTraits<X, NumBitsX>::convertSrgb(in);
1492 }
1493 #endif
1494 };