projects / mesa.git / blob
? search:


142007eb3412afcdcf57c91da49fec3d28540b55
[mesa.git] / src / gallium / drivers / swr / rasterizer / jitter / builder_misc.cpp
1 /****************************************************************************
2 * Copyright (C) 2014-2015 Intel Corporation. All Rights Reserved.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 * @file builder_misc.cpp
24 *
25 * @brief Implementation for miscellaneous builder functions
26 *
27 * Notes:
28 *
29 ******************************************************************************/
30 #include "jit_pch.hpp"
31 #include "builder.h"
32 #include "common/rdtsc_buckets.h"
33
34 #include <cstdarg>
35
36 namespace SwrJit
37 {
38 void __cdecl CallPrint(const char* fmt, ...);
39
40 //////////////////////////////////////////////////////////////////////////
41 /// @brief Convert an IEEE 754 32-bit single precision float to an
42 /// 16 bit float with 5 exponent bits and a variable
43 /// number of mantissa bits.
44 /// @param val - 32-bit float
45 /// @todo Maybe move this outside of this file into a header?
46 static uint16_t ConvertFloat32ToFloat16(float val)
47 {
48 uint32_t sign, exp, mant;
49 uint32_t roundBits;
50
51 // Extract the sign, exponent, and mantissa
52 uint32_t uf = *(uint32_t*)&val;
53 sign = (uf & 0x80000000) >> 31;
54 exp = (uf & 0x7F800000) >> 23;
55 mant = uf & 0x007FFFFF;
56
57 // Check for out of range
58 if (std::isnan(val))
59 {
60 exp = 0x1F;
61 mant = 0x200;
62 sign = 1; // set the sign bit for NANs
63 }
64 else if (std::isinf(val))
65 {
66 exp = 0x1f;
67 mant = 0x0;
68 }
69 else if (exp > (0x70 + 0x1E)) // Too big to represent -> max representable value
70 {
71 exp = 0x1E;
72 mant = 0x3FF;
73 }
74 else if ((exp <= 0x70) && (exp >= 0x66)) // It's a denorm
75 {
76 mant |= 0x00800000;
77 for (; exp <= 0x70; mant >>= 1, exp++)
78 ;
79 exp = 0;
80 mant = mant >> 13;
81 }
82 else if (exp < 0x66) // Too small to represent -> Zero
83 {
84 exp = 0;
85 mant = 0;
86 }
87 else
88 {
89 // Saves bits that will be shifted off for rounding
90 roundBits = mant & 0x1FFFu;
91 // convert exponent and mantissa to 16 bit format
92 exp = exp - 0x70;
93 mant = mant >> 13;
94
95 // Essentially RTZ, but round up if off by only 1 lsb
96 if (roundBits == 0x1FFFu)
97 {
98 mant++;
99 // check for overflow
100 if ((mant & 0xC00u) != 0)
101 exp++;
102 // make sure only the needed bits are used
103 mant &= 0x3FF;
104 }
105 }
106
107 uint32_t tmpVal = (sign << 15) | (exp << 10) | mant;
108 return (uint16_t)tmpVal;
109 }
110
111 //////////////////////////////////////////////////////////////////////////
112 /// @brief Convert an IEEE 754 16-bit float to an 32-bit single precision
113 /// float
114 /// @param val - 16-bit float
115 /// @todo Maybe move this outside of this file into a header?
116 static float ConvertFloat16ToFloat32(uint32_t val)
117 {
118 uint32_t result;
119 if ((val & 0x7fff) == 0)
120 {
121 result = ((uint32_t)(val & 0x8000)) << 16;
122 }
123 else if ((val & 0x7c00) == 0x7c00)
124 {
125 result = ((val & 0x3ff) == 0) ? 0x7f800000 : 0x7fc00000;
126 result |= ((uint32_t)val & 0x8000) << 16;
127 }
128 else
129 {
130 uint32_t sign = (val & 0x8000) << 16;
131 uint32_t mant = (val & 0x3ff) << 13;
132 uint32_t exp = (val >> 10) & 0x1f;
133 if ((exp == 0) && (mant != 0)) // Adjust exponent and mantissa for denormals
134 {
135 mant <<= 1;
136 while (mant < (0x400 << 13))
137 {
138 exp--;
139 mant <<= 1;
140 }
141 mant &= (0x3ff << 13);
142 }
143 exp = ((exp - 15 + 127) & 0xff) << 23;
144 result = sign | exp | mant;
145 }
146
147 return *(float*)&result;
148 }
149
150 Constant *Builder::C(bool i)
151 {
152 return ConstantInt::get(IRB()->getInt1Ty(), (i ? 1 : 0));
153 }
154
155 Constant *Builder::C(char i)
156 {
157 return ConstantInt::get(IRB()->getInt8Ty(), i);
158 }
159
160 Constant *Builder::C(uint8_t i)
161 {
162 return ConstantInt::get(IRB()->getInt8Ty(), i);
163 }
164
165 Constant *Builder::C(int i)
166 {
167 return ConstantInt::get(IRB()->getInt32Ty(), i);
168 }
169
170 Constant *Builder::C(int64_t i)
171 {
172 return ConstantInt::get(IRB()->getInt64Ty(), i);
173 }
174
175 Constant *Builder::C(uint16_t i)
176 {
177 return ConstantInt::get(mInt16Ty,i);
178 }
179
180 Constant *Builder::C(uint32_t i)
181 {
182 return ConstantInt::get(IRB()->getInt32Ty(), i);
183 }
184
185 Constant *Builder::C(float i)
186 {
187 return ConstantFP::get(IRB()->getFloatTy(), i);
188 }
189
190 Constant *Builder::PRED(bool pred)
191 {
192 return ConstantInt::get(IRB()->getInt1Ty(), (pred ? 1 : 0));
193 }
194
195 Value *Builder::VIMMED1(int i)
196 {
197 return ConstantVector::getSplat(mVWidth, cast<ConstantInt>(C(i)));
198 }
199
200 Value *Builder::VIMMED1_16(int i)
201 {
202 return ConstantVector::getSplat(mVWidth16, cast<ConstantInt>(C(i)));
203 }
204
205 Value *Builder::VIMMED1(uint32_t i)
206 {
207 return ConstantVector::getSplat(mVWidth, cast<ConstantInt>(C(i)));
208 }
209
210 Value *Builder::VIMMED1_16(uint32_t i)
211 {
212 return ConstantVector::getSplat(mVWidth16, cast<ConstantInt>(C(i)));
213 }
214
215 Value *Builder::VIMMED1(float i)
216 {
217 return ConstantVector::getSplat(mVWidth, cast<ConstantFP>(C(i)));
218 }
219
220 Value *Builder::VIMMED1_16(float i)
221 {
222 return ConstantVector::getSplat(mVWidth16, cast<ConstantFP>(C(i)));
223 }
224
225 Value *Builder::VIMMED1(bool i)
226 {
227 return ConstantVector::getSplat(mVWidth, cast<ConstantInt>(C(i)));
228 }
229
230 Value *Builder::VIMMED1_16(bool i)
231 {
232 return ConstantVector::getSplat(mVWidth16, cast<ConstantInt>(C(i)));
233 }
234
235 Value *Builder::VUNDEF_IPTR()
236 {
237 return UndefValue::get(VectorType::get(mInt32PtrTy,mVWidth));
238 }
239
240 Value *Builder::VUNDEF(Type* t)
241 {
242 return UndefValue::get(VectorType::get(t, mVWidth));
243 }
244
245 Value *Builder::VUNDEF_I()
246 {
247 return UndefValue::get(VectorType::get(mInt32Ty, mVWidth));
248 }
249
250 Value *Builder::VUNDEF_I_16()
251 {
252 return UndefValue::get(VectorType::get(mInt32Ty, mVWidth16));
253 }
254
255 Value *Builder::VUNDEF_F()
256 {
257 return UndefValue::get(VectorType::get(mFP32Ty, mVWidth));
258 }
259
260 Value *Builder::VUNDEF_F_16()
261 {
262 return UndefValue::get(VectorType::get(mFP32Ty, mVWidth16));
263 }
264
265 Value *Builder::VUNDEF(Type *ty, uint32_t size)
266 {
267 return UndefValue::get(VectorType::get(ty, size));
268 }
269
270 Value *Builder::VBROADCAST(Value *src, const llvm::Twine& name)
271 {
272 // check if src is already a vector
273 if (src->getType()->isVectorTy())
274 {
275 return src;
276 }
277
278 return VECTOR_SPLAT(mVWidth, src, name);
279 }
280
281 Value *Builder::VBROADCAST_16(Value *src)
282 {
283 // check if src is already a vector
284 if (src->getType()->isVectorTy())
285 {
286 return src;
287 }
288
289 return VECTOR_SPLAT(mVWidth16, src);
290 }
291
292 uint32_t Builder::IMMED(Value* v)
293 {
294 SWR_ASSERT(isa<ConstantInt>(v));
295 ConstantInt *pValConst = cast<ConstantInt>(v);
296 return pValConst->getZExtValue();
297 }
298
299 int32_t Builder::S_IMMED(Value* v)
300 {
301 SWR_ASSERT(isa<ConstantInt>(v));
302 ConstantInt *pValConst = cast<ConstantInt>(v);
303 return pValConst->getSExtValue();
304 }
305
306 Value *Builder::GEP(Value* ptr, const std::initializer_list<Value*> &indexList)
307 {
308 std::vector<Value*> indices;
309 for (auto i : indexList)
310 indices.push_back(i);
311 return GEPA(ptr, indices);
312 }
313
314 Value *Builder::GEP(Value* ptr, const std::initializer_list<uint32_t> &indexList)
315 {
316 std::vector<Value*> indices;
317 for (auto i : indexList)
318 indices.push_back(C(i));
319 return GEPA(ptr, indices);
320 }
321
322 Value *Builder::IN_BOUNDS_GEP(Value* ptr, const std::initializer_list<Value*> &indexList)
323 {
324 std::vector<Value*> indices;
325 for (auto i : indexList)
326 indices.push_back(i);
327 return IN_BOUNDS_GEP(ptr, indices);
328 }
329
330 Value *Builder::IN_BOUNDS_GEP(Value* ptr, const std::initializer_list<uint32_t> &indexList)
331 {
332 std::vector<Value*> indices;
333 for (auto i : indexList)
334 indices.push_back(C(i));
335 return IN_BOUNDS_GEP(ptr, indices);
336 }
337
338 LoadInst *Builder::LOAD(Value *basePtr, const std::initializer_list<uint32_t> &indices, const llvm::Twine& name)
339 {
340 std::vector<Value*> valIndices;
341 for (auto i : indices)
342 valIndices.push_back(C(i));
343 return LOAD(GEPA(basePtr, valIndices), name);
344 }
345
346 LoadInst *Builder::LOADV(Value *basePtr, const std::initializer_list<Value*> &indices, const llvm::Twine& name)
347 {
348 std::vector<Value*> valIndices;
349 for (auto i : indices)
350 valIndices.push_back(i);
351 return LOAD(GEPA(basePtr, valIndices), name);
352 }
353
354 StoreInst *Builder::STORE(Value *val, Value *basePtr, const std::initializer_list<uint32_t> &indices)
355 {
356 std::vector<Value*> valIndices;
357 for (auto i : indices)
358 valIndices.push_back(C(i));
359 return STORE(val, GEPA(basePtr, valIndices));
360 }
361
362 StoreInst *Builder::STOREV(Value *val, Value *basePtr, const std::initializer_list<Value*> &indices)
363 {
364 std::vector<Value*> valIndices;
365 for (auto i : indices)
366 valIndices.push_back(i);
367 return STORE(val, GEPA(basePtr, valIndices));
368 }
369
370 CallInst *Builder::CALL(Value *Callee, const std::initializer_list<Value*> &argsList, const llvm::Twine& name)
371 {
372 std::vector<Value*> args;
373 for (auto arg : argsList)
374 args.push_back(arg);
375 return CALLA(Callee, args, name);
376 }
377
378 CallInst *Builder::CALL(Value *Callee, Value* arg)
379 {
380 std::vector<Value*> args;
381 args.push_back(arg);
382 return CALLA(Callee, args);
383 }
384
385 CallInst *Builder::CALL2(Value *Callee, Value* arg1, Value* arg2)
386 {
387 std::vector<Value*> args;
388 args.push_back(arg1);
389 args.push_back(arg2);
390 return CALLA(Callee, args);
391 }
392
393 CallInst *Builder::CALL3(Value *Callee, Value* arg1, Value* arg2, Value* arg3)
394 {
395 std::vector<Value*> args;
396 args.push_back(arg1);
397 args.push_back(arg2);
398 args.push_back(arg3);
399 return CALLA(Callee, args);
400 }
401
402 //////////////////////////////////////////////////////////////////////////
403 Value *Builder::DEBUGTRAP()
404 {
405 Function *func = Intrinsic::getDeclaration(JM()->mpCurrentModule, Intrinsic::debugtrap);
406 return CALL(func);
407 }
408
409 Value *Builder::VRCP(Value *va, const llvm::Twine& name)
410 {
411 return FDIV(VIMMED1(1.0f), va, name); // 1 / a
412 }
413
414 Value *Builder::VPLANEPS(Value* vA, Value* vB, Value* vC, Value* &vX, Value* &vY)
415 {
416 Value* vOut = FMADDPS(vA, vX, vC);
417 vOut = FMADDPS(vB, vY, vOut);
418 return vOut;
419 }
420
421 //////////////////////////////////////////////////////////////////////////
422 /// @brief Generate an i32 masked load operation in LLVM IR. If not
423 /// supported on the underlying platform, emulate it with float masked load
424 /// @param src - base address pointer for the load
425 /// @param vMask - SIMD wide mask that controls whether to access memory load 0
426 Value *Builder::MASKLOADD(Value* src,Value* mask)
427 {
428 Value* vResult;
429 // use avx2 gather instruction is available
430 if(JM()->mArch.AVX2())
431 {
432 Function *func = Intrinsic::getDeclaration(JM()->mpCurrentModule, Intrinsic::x86_avx2_maskload_d_256);
433 vResult = CALL(func,{src,mask});
434 }
435 else
436 {
437 // maskload intrinsic expects integer mask operand in llvm >= 3.8
438 #if (LLVM_VERSION_MAJOR > 3) || (LLVM_VERSION_MAJOR == 3 && LLVM_VERSION_MINOR >= 8)
439 mask = BITCAST(mask,VectorType::get(mInt32Ty,mVWidth));
440 #else
441 mask = BITCAST(mask,VectorType::get(mFP32Ty,mVWidth));
442 #endif
443 Function *func = Intrinsic::getDeclaration(JM()->mpCurrentModule,Intrinsic::x86_avx_maskload_ps_256);
444 vResult = BITCAST(CALL(func,{src,mask}), VectorType::get(mInt32Ty,mVWidth));
445 }
446 return vResult;
447 }
448
449 //////////////////////////////////////////////////////////////////////////
450 /// @brief insert a JIT call to CallPrint
451 /// - outputs formatted string to both stdout and VS output window
452 /// - DEBUG builds only
453 /// Usage example:
454 /// PRINT("index %d = 0x%p\n",{C(lane), pIndex});
455 /// where C(lane) creates a constant value to print, and pIndex is the Value*
456 /// result from a GEP, printing out the pointer to memory
457 /// @param printStr - constant string to print, which includes format specifiers
458 /// @param printArgs - initializer list of Value*'s to print to std out
459 CallInst *Builder::PRINT(const std::string &printStr,const std::initializer_list<Value*> &printArgs)
460 {
461 // push the arguments to CallPrint into a vector
462 std::vector<Value*> printCallArgs;
463 // save room for the format string. we still need to modify it for vectors
464 printCallArgs.resize(1);
465
466 // search through the format string for special processing
467 size_t pos = 0;
468 std::string tempStr(printStr);
469 pos = tempStr.find('%', pos);
470 auto v = printArgs.begin();
471
472 while ((pos != std::string::npos) && (v != printArgs.end()))
473 {
474 Value* pArg = *v;
475 Type* pType = pArg->getType();
476
477 if (pType->isVectorTy())
478 {
479 Type* pContainedType = pType->getContainedType(0);
480
481 if (toupper(tempStr[pos + 1]) == 'X')
482 {
483 tempStr[pos] = '0';
484 tempStr[pos + 1] = 'x';
485 tempStr.insert(pos + 2, "%08X ");
486 pos += 7;
487
488 printCallArgs.push_back(VEXTRACT(pArg, C(0)));
489
490 std::string vectorFormatStr;
491 for (uint32_t i = 1; i < pType->getVectorNumElements(); ++i)
492 {
493 vectorFormatStr += "0x%08X ";
494 printCallArgs.push_back(VEXTRACT(pArg, C(i)));
495 }
496
497 tempStr.insert(pos, vectorFormatStr);
498 pos += vectorFormatStr.size();
499 }
500 else if ((tempStr[pos + 1] == 'f') && (pContainedType->isFloatTy()))
501 {
502 uint32_t i = 0;
503 for (; i < (pArg->getType()->getVectorNumElements()) - 1; i++)
504 {
505 tempStr.insert(pos, std::string("%f "));
506 pos += 3;
507 printCallArgs.push_back(FP_EXT(VEXTRACT(pArg, C(i)), Type::getDoubleTy(JM()->mContext)));
508 }
509 printCallArgs.push_back(FP_EXT(VEXTRACT(pArg, C(i)), Type::getDoubleTy(JM()->mContext)));
510 }
511 else if ((tempStr[pos + 1] == 'd') && (pContainedType->isIntegerTy()))
512 {
513 uint32_t i = 0;
514 for (; i < (pArg->getType()->getVectorNumElements()) - 1; i++)
515 {
516 tempStr.insert(pos, std::string("%d "));
517 pos += 3;
518 printCallArgs.push_back(VEXTRACT(pArg, C(i)));
519 }
520 printCallArgs.push_back(VEXTRACT(pArg, C(i)));
521 }
522 }
523 else
524 {
525 if (toupper(tempStr[pos + 1]) == 'X')
526 {
527 tempStr[pos] = '0';
528 tempStr.insert(pos + 1, "x%08");
529 printCallArgs.push_back(pArg);
530 pos += 3;
531 }
532 // for %f we need to cast float Values to doubles so that they print out correctly
533 else if ((tempStr[pos + 1] == 'f') && (pType->isFloatTy()))
534 {
535 printCallArgs.push_back(FP_EXT(pArg, Type::getDoubleTy(JM()->mContext)));
536 pos++;
537 }
538 else
539 {
540 printCallArgs.push_back(pArg);
541 }
542 }
543
544 // advance to the next arguement
545 v++;
546 pos = tempStr.find('%', ++pos);
547 }
548
549 // create global variable constant string
550 Constant *constString = ConstantDataArray::getString(JM()->mContext,tempStr,true);
551 GlobalVariable *gvPtr = new GlobalVariable(constString->getType(),true,GlobalValue::InternalLinkage,constString,"printStr");
552 JM()->mpCurrentModule->getGlobalList().push_back(gvPtr);
553
554 // get a pointer to the first character in the constant string array
555 std::vector<Constant*> geplist{C(0),C(0)};
556 Constant *strGEP = ConstantExpr::getGetElementPtr(nullptr, gvPtr,geplist,false);
557
558 // insert the pointer to the format string in the argument vector
559 printCallArgs[0] = strGEP;
560
561 // get pointer to CallPrint function and insert decl into the module if needed
562 std::vector<Type*> args;
563 args.push_back(PointerType::get(mInt8Ty,0));
564 FunctionType* callPrintTy = FunctionType::get(Type::getVoidTy(JM()->mContext),args,true);
565 Function *callPrintFn = cast<Function>(JM()->mpCurrentModule->getOrInsertFunction("CallPrint", callPrintTy));
566
567 // if we haven't yet added the symbol to the symbol table
568 if((sys::DynamicLibrary::SearchForAddressOfSymbol("CallPrint")) == nullptr)
569 {
570 sys::DynamicLibrary::AddSymbol("CallPrint", (void *)&CallPrint);
571 }
572
573 // insert a call to CallPrint
574 return CALLA(callPrintFn,printCallArgs);
575 }
576
577 //////////////////////////////////////////////////////////////////////////
578 /// @brief Wrapper around PRINT with initializer list.
579 CallInst* Builder::PRINT(const std::string &printStr)
580 {
581 return PRINT(printStr, {});
582 }
583
584 //////////////////////////////////////////////////////////////////////////
585 /// @brief Generate a masked gather operation in LLVM IR. If not
586 /// supported on the underlying platform, emulate it with loads
587 /// @param vSrc - SIMD wide value that will be loaded if mask is invalid
588 /// @param pBase - Int8* base VB address pointer value
589 /// @param vIndices - SIMD wide value of VB byte offsets
590 /// @param vMask - SIMD wide mask that controls whether to access memory or the src values
591 /// @param scale - value to scale indices by
592 Value *Builder::GATHERPS(Value *vSrc, Value *pBase, Value *vIndices, Value *vMask, uint8_t scale)
593 {
594 Value *vGather;
595
596 // use avx2 gather instruction if available
597 if(JM()->mArch.AVX2())
598 {
599 // force mask to <N x float>, required by vgather
600 Value *mask = BITCAST(VMASK(vMask), mSimdFP32Ty);
601
602 vGather = VGATHERPS(vSrc, pBase, vIndices, mask, C(scale));
603 }
604 else
605 {
606 Value* pStack = STACKSAVE();
607
608 // store vSrc on the stack. this way we can select between a valid load address and the vSrc address
609 Value* vSrcPtr = ALLOCA(vSrc->getType());
610 STORE(vSrc, vSrcPtr);
611
612 vGather = VUNDEF_F();
613 Value *vScaleVec = VIMMED1((uint32_t)scale);
614 Value *vOffsets = MUL(vIndices,vScaleVec);
615 for(uint32_t i = 0; i < mVWidth; ++i)
616 {
617 // single component byte index
618 Value *offset = VEXTRACT(vOffsets,C(i));
619 // byte pointer to component
620 Value *loadAddress = GEP(pBase,offset);
621 loadAddress = BITCAST(loadAddress,PointerType::get(mFP32Ty,0));
622 // pointer to the value to load if we're masking off a component
623 Value *maskLoadAddress = GEP(vSrcPtr,{C(0), C(i)});
624 Value *selMask = VEXTRACT(vMask,C(i));
625 // switch in a safe address to load if we're trying to access a vertex
626 Value *validAddress = SELECT(selMask, loadAddress, maskLoadAddress);
627 Value *val = LOAD(validAddress);
628 vGather = VINSERT(vGather,val,C(i));
629 }
630
631 STACKRESTORE(pStack);
632 }
633
634 return vGather;
635 }
636
637 Value *Builder::GATHERPS_16(Value *vSrc, Value *pBase, Value *vIndices, Value *vMask, uint8_t scale)
638 {
639 Value *vGather = VUNDEF_F_16();
640
641 // use AVX512F gather instruction if available
642 if (JM()->mArch.AVX512F())
643 {
644 // force mask to <N-bit Integer>, required by vgather2
645 Value *mask = BITCAST(vMask, mInt16Ty);
646
647 vGather = VGATHERPS_16(vSrc, pBase, vIndices, mask, C((uint32_t)scale));
648 }
649 else
650 {
651 Value *src0 = EXTRACT_16(vSrc, 0);
652 Value *src1 = EXTRACT_16(vSrc, 1);
653
654 Value *indices0 = EXTRACT_16(vIndices, 0);
655 Value *indices1 = EXTRACT_16(vIndices, 1);
656
657 Value *mask0 = EXTRACT_16(vMask, 0);
658 Value *mask1 = EXTRACT_16(vMask, 1);
659
660 Value *gather0 = GATHERPS(src0, pBase, indices0, mask0, scale);
661 Value *gather1 = GATHERPS(src1, pBase, indices1, mask1, scale);
662
663 vGather = JOIN_16(gather0, gather1);
664 }
665
666 return vGather;
667 }
668
669 //////////////////////////////////////////////////////////////////////////
670 /// @brief Generate a masked gather operation in LLVM IR. If not
671 /// supported on the underlying platform, emulate it with loads
672 /// @param vSrc - SIMD wide value that will be loaded if mask is invalid
673 /// @param pBase - Int8* base VB address pointer value
674 /// @param vIndices - SIMD wide value of VB byte offsets
675 /// @param vMask - SIMD wide mask that controls whether to access memory or the src values
676 /// @param scale - value to scale indices by
677 Value *Builder::GATHERDD(Value* vSrc, Value* pBase, Value* vIndices, Value* vMask, uint8_t scale)
678 {
679 Value* vGather;
680
681 // use avx2 gather instruction if available
682 if(JM()->mArch.AVX2())
683 {
684 vGather = VGATHERDD(vSrc, pBase, vIndices, VMASK(vMask), C(scale));
685 }
686 else
687 {
688 Value* pStack = STACKSAVE();
689
690 // store vSrc on the stack. this way we can select between a valid load address and the vSrc address
691 Value* vSrcPtr = ALLOCA(vSrc->getType());
692 STORE(vSrc, vSrcPtr);
693
694 vGather = VUNDEF_I();
695 Value *vScaleVec = VIMMED1((uint32_t)scale);
696 Value *vOffsets = MUL(vIndices, vScaleVec);
697 for(uint32_t i = 0; i < mVWidth; ++i)
698 {
699 // single component byte index
700 Value *offset = VEXTRACT(vOffsets, C(i));
701 // byte pointer to component
702 Value *loadAddress = GEP(pBase, offset);
703 loadAddress = BITCAST(loadAddress, PointerType::get(mInt32Ty, 0));
704 // pointer to the value to load if we're masking off a component
705 Value *maskLoadAddress = GEP(vSrcPtr, {C(0), C(i)});
706 Value *selMask = VEXTRACT(vMask, C(i));
707 // switch in a safe address to load if we're trying to access a vertex
708 Value *validAddress = SELECT(selMask, loadAddress, maskLoadAddress);
709 Value *val = LOAD(validAddress, C(0));
710 vGather = VINSERT(vGather, val, C(i));
711 }
712
713 STACKRESTORE(pStack);
714 }
715
716 return vGather;
717 }
718
719 Value *Builder::GATHERDD_16(Value *vSrc, Value *pBase, Value *vIndices, Value *vMask, uint8_t scale)
720 {
721 Value *vGather = VUNDEF_I_16();
722
723 // use AVX512F gather instruction if available
724 if (JM()->mArch.AVX512F())
725 {
726 // force mask to <N-bit Integer>, required by vgather2
727 Value *mask = BITCAST(vMask, mInt16Ty);
728
729 vGather = VGATHERDD_16(vSrc, pBase, vIndices, mask, C((uint32_t)scale));
730 }
731 else
732 {
733 Value *src0 = EXTRACT_16(vSrc, 0);
734 Value *src1 = EXTRACT_16(vSrc, 1);
735
736 Value *indices0 = EXTRACT_16(vIndices, 0);
737 Value *indices1 = EXTRACT_16(vIndices, 1);
738
739 Value *mask0 = EXTRACT_16(vMask, 0);
740 Value *mask1 = EXTRACT_16(vMask, 1);
741
742 Value *gather0 = GATHERDD(src0, pBase, indices0, mask0, scale);
743 Value *gather1 = GATHERDD(src1, pBase, indices1, mask1, scale);
744
745 vGather = JOIN_16(gather0, gather1);
746 }
747
748 return vGather;
749 }
750
751 //////////////////////////////////////////////////////////////////////////
752 /// @brief Generate a masked gather operation in LLVM IR. If not
753 /// supported on the underlying platform, emulate it with loads
754 /// @param vSrc - SIMD wide value that will be loaded if mask is invalid
755 /// @param pBase - Int8* base VB address pointer value
756 /// @param vIndices - SIMD wide value of VB byte offsets
757 /// @param vMask - SIMD wide mask that controls whether to access memory or the src values
758 /// @param scale - value to scale indices by
759 Value *Builder::GATHERPD(Value* vSrc, Value* pBase, Value* vIndices, Value* vMask, uint8_t scale)
760 {
761 Value* vGather;
762
763 // use avx2 gather instruction if available
764 if(JM()->mArch.AVX2())
765 {
766 vMask = BITCAST(S_EXT(vMask, VectorType::get(mInt64Ty, mVWidth/2)), VectorType::get(mDoubleTy, mVWidth/2));
767 vGather = VGATHERPD(vSrc, pBase, vIndices, vMask, C(scale));
768 }
769 else
770 {
771 Value* pStack = STACKSAVE();
772
773 // store vSrc on the stack. this way we can select between a valid load address and the vSrc address
774 Value* vSrcPtr = ALLOCA(vSrc->getType());
775 STORE(vSrc, vSrcPtr);
776
777 vGather = UndefValue::get(VectorType::get(mDoubleTy, 4));
778 Value *vScaleVec = VECTOR_SPLAT(4, C((uint32_t)scale));
779 Value *vOffsets = MUL(vIndices,vScaleVec);
780 for(uint32_t i = 0; i < mVWidth/2; ++i)
781 {
782 // single component byte index
783 Value *offset = VEXTRACT(vOffsets,C(i));
784 // byte pointer to component
785 Value *loadAddress = GEP(pBase,offset);
786 loadAddress = BITCAST(loadAddress,PointerType::get(mDoubleTy,0));
787 // pointer to the value to load if we're masking off a component
788 Value *maskLoadAddress = GEP(vSrcPtr,{C(0), C(i)});
789 Value *selMask = VEXTRACT(vMask,C(i));
790 // switch in a safe address to load if we're trying to access a vertex
791 Value *validAddress = SELECT(selMask, loadAddress, maskLoadAddress);
792 Value *val = LOAD(validAddress);
793 vGather = VINSERT(vGather,val,C(i));
794 }
795 STACKRESTORE(pStack);
796 }
797 return vGather;
798 }
799
800 Value *Builder::EXTRACT_16(Value *x, uint32_t imm)
801 {
802 if (imm == 0)
803 {
804 return VSHUFFLE(x, UndefValue::get(x->getType()), { 0, 1, 2, 3, 4, 5, 6, 7 });
805 }
806 else
807 {
808 return VSHUFFLE(x, UndefValue::get(x->getType()), { 8, 9, 10, 11, 12, 13, 14, 15 });
809 }
810 }
811
812 Value *Builder::JOIN_16(Value *a, Value *b)
813 {
814 return VSHUFFLE(a, b, { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 });
815 }
816
817 //////////////////////////////////////////////////////////////////////////
818 /// @brief convert x86 <N x float> mask to llvm <N x i1> mask
819 Value *Builder::MASK(Value *vmask)
820 {
821 Value *src = BITCAST(vmask, mSimdInt32Ty);
822 return ICMP_SLT(src, VIMMED1(0));
823 }
824
825 Value *Builder::MASK_16(Value *vmask)
826 {
827 Value *src = BITCAST(vmask, mSimd16Int32Ty);
828 return ICMP_SLT(src, VIMMED1_16(0));
829 }
830
831 //////////////////////////////////////////////////////////////////////////
832 /// @brief convert llvm <N x i1> mask to x86 <N x i32> mask
833 Value *Builder::VMASK(Value *mask)
834 {
835 return S_EXT(mask, mSimdInt32Ty);
836 }
837
838 Value *Builder::VMASK_16(Value *mask)
839 {
840 return S_EXT(mask, mSimd16Int32Ty);
841 }
842
843 //////////////////////////////////////////////////////////////////////////
844 /// @brief Generate a VPSHUFB operation in LLVM IR. If not
845 /// supported on the underlying platform, emulate it
846 /// @param a - 256bit SIMD(32x8bit) of 8bit integer values
847 /// @param b - 256bit SIMD(32x8bit) of 8bit integer mask values
848 /// Byte masks in lower 128 lane of b selects 8 bit values from lower
849 /// 128bits of a, and vice versa for the upper lanes. If the mask
850 /// value is negative, '0' is inserted.
851 Value *Builder::PSHUFB(Value* a, Value* b)
852 {
853 Value* res;
854 // use avx2 pshufb instruction if available
855 if(JM()->mArch.AVX2())
856 {
857 res = VPSHUFB(a, b);
858 }
859 else
860 {
861 Constant* cB = dyn_cast<Constant>(b);
862 // number of 8 bit elements in b
863 uint32_t numElms = cast<VectorType>(cB->getType())->getNumElements();
864 // output vector
865 Value* vShuf = UndefValue::get(VectorType::get(mInt8Ty, numElms));
866
867 // insert an 8 bit value from the high and low lanes of a per loop iteration
868 numElms /= 2;
869 for(uint32_t i = 0; i < numElms; i++)
870 {
871 ConstantInt* cLow128b = cast<ConstantInt>(cB->getAggregateElement(i));
872 ConstantInt* cHigh128b = cast<ConstantInt>(cB->getAggregateElement(i + numElms));
873
874 // extract values from constant mask
875 char valLow128bLane = (char)(cLow128b->getSExtValue());
876 char valHigh128bLane = (char)(cHigh128b->getSExtValue());
877
878 Value* insertValLow128b;
879 Value* insertValHigh128b;
880
881 // if the mask value is negative, insert a '0' in the respective output position
882 // otherwise, lookup the value at mask position (bits 3..0 of the respective mask byte) in a and insert in output vector
883 insertValLow128b = (valLow128bLane < 0) ? C((char)0) : VEXTRACT(a, C((valLow128bLane & 0xF)));
884 insertValHigh128b = (valHigh128bLane < 0) ? C((char)0) : VEXTRACT(a, C((valHigh128bLane & 0xF) + numElms));
885
886 vShuf = VINSERT(vShuf, insertValLow128b, i);
887 vShuf = VINSERT(vShuf, insertValHigh128b, (i + numElms));
888 }
889 res = vShuf;
890 }
891 return res;
892 }
893
894 //////////////////////////////////////////////////////////////////////////
895 /// @brief Generate a VPSHUFB operation (sign extend 8 8bit values to 32
896 /// bits)in LLVM IR. If not supported on the underlying platform, emulate it
897 /// @param a - 128bit SIMD lane(16x8bit) of 8bit integer values. Only
898 /// lower 8 values are used.
899 Value *Builder::PMOVSXBD(Value* a)
900 {
901 // VPMOVSXBD output type
902 Type* v8x32Ty = VectorType::get(mInt32Ty, 8);
903 // Extract 8 values from 128bit lane and sign extend
904 return S_EXT(VSHUFFLE(a, a, C<int>({0, 1, 2, 3, 4, 5, 6, 7})), v8x32Ty);
905 }
906
907 //////////////////////////////////////////////////////////////////////////
908 /// @brief Generate a VPSHUFB operation (sign extend 8 16bit values to 32
909 /// bits)in LLVM IR. If not supported on the underlying platform, emulate it
910 /// @param a - 128bit SIMD lane(8x16bit) of 16bit integer values.
911 Value *Builder::PMOVSXWD(Value* a)
912 {
913 // VPMOVSXWD output type
914 Type* v8x32Ty = VectorType::get(mInt32Ty, 8);
915 // Extract 8 values from 128bit lane and sign extend
916 return S_EXT(VSHUFFLE(a, a, C<int>({0, 1, 2, 3, 4, 5, 6, 7})), v8x32Ty);
917 }
918
919 //////////////////////////////////////////////////////////////////////////
920 /// @brief Generate a VPERMD operation (shuffle 32 bit integer values
921 /// across 128 bit lanes) in LLVM IR. If not supported on the underlying
922 /// platform, emulate it
923 /// @param a - 256bit SIMD lane(8x32bit) of integer values.
924 /// @param idx - 256bit SIMD lane(8x32bit) of 3 bit lane index values
925 Value *Builder::PERMD(Value* a, Value* idx)
926 {
927 Value* res;
928 // use avx2 permute instruction if available
929 if(JM()->mArch.AVX2())
930 {
931 res = VPERMD(a, idx);
932 }
933 else
934 {
935 if (isa<Constant>(idx))
936 {
937 res = VSHUFFLE(a, a, idx);
938 }
939 else
940 {
941 res = VUNDEF_I();
942 for (uint32_t l = 0; l < JM()->mVWidth; ++l)
943 {
944 Value* pIndex = VEXTRACT(idx, C(l));
945 Value* pVal = VEXTRACT(a, pIndex);
946 res = VINSERT(res, pVal, C(l));
947 }
948 }
949 }
950 return res;
951 }
952
953 //////////////////////////////////////////////////////////////////////////
954 /// @brief Generate a VPERMPS operation (shuffle 32 bit float values
955 /// across 128 bit lanes) in LLVM IR. If not supported on the underlying
956 /// platform, emulate it
957 /// @param a - 256bit SIMD lane(8x32bit) of float values.
958 /// @param idx - 256bit SIMD lane(8x32bit) of 3 bit lane index values
959 Value *Builder::PERMPS(Value* a, Value* idx)
960 {
961 Value* res;
962 // use avx2 permute instruction if available
963 if (JM()->mArch.AVX2())
964 {
965 // llvm 3.6.0 swapped the order of the args to vpermd
966 res = VPERMPS(idx, a);
967 }
968 else
969 {
970 if (isa<Constant>(idx))
971 {
972 res = VSHUFFLE(a, a, idx);
973 }
974 else
975 {
976 res = VUNDEF_F();
977 for (uint32_t l = 0; l < JM()->mVWidth; ++l)
978 {
979 Value* pIndex = VEXTRACT(idx, C(l));
980 Value* pVal = VEXTRACT(a, pIndex);
981 res = VINSERT(res, pVal, C(l));
982 }
983 }
984 }
985
986 return res;
987 }
988
989 //////////////////////////////////////////////////////////////////////////
990 /// @brief Generate a VCVTPH2PS operation (float16->float32 conversion)
991 /// in LLVM IR. If not supported on the underlying platform, emulate it
992 /// @param a - 128bit SIMD lane(8x16bit) of float16 in int16 format.
993 Value *Builder::CVTPH2PS(Value* a, const llvm::Twine& name)
994 {
995 if (JM()->mArch.F16C())
996 {
997 return VCVTPH2PS(a, name);
998 }
999 else
1000 {
1001 FunctionType* pFuncTy = FunctionType::get(mFP32Ty, mInt16Ty);
1002 Function* pCvtPh2Ps = cast<Function>(JM()->mpCurrentModule->getOrInsertFunction("ConvertFloat16ToFloat32", pFuncTy));
1003
1004 if (sys::DynamicLibrary::SearchForAddressOfSymbol("ConvertFloat16ToFloat32") == nullptr)
1005 {
1006 sys::DynamicLibrary::AddSymbol("ConvertFloat16ToFloat32", (void *)&ConvertFloat16ToFloat32);
1007 }
1008
1009 Value* pResult = UndefValue::get(mSimdFP32Ty);
1010 for (uint32_t i = 0; i < mVWidth; ++i)
1011 {
1012 Value* pSrc = VEXTRACT(a, C(i));
1013 Value* pConv = CALL(pCvtPh2Ps, std::initializer_list<Value*>{pSrc});
1014 pResult = VINSERT(pResult, pConv, C(i));
1015 }
1016
1017 pResult->setName(name);
1018 return pResult;
1019 }
1020 }
1021
1022 //////////////////////////////////////////////////////////////////////////
1023 /// @brief Generate a VCVTPS2PH operation (float32->float16 conversion)
1024 /// in LLVM IR. If not supported on the underlying platform, emulate it
1025 /// @param a - 128bit SIMD lane(8x16bit) of float16 in int16 format.
1026 Value *Builder::CVTPS2PH(Value* a, Value* rounding)
1027 {
1028 if (JM()->mArch.F16C())
1029 {
1030 return VCVTPS2PH(a, rounding);
1031 }
1032 else
1033 {
1034 // call scalar C function for now
1035 FunctionType* pFuncTy = FunctionType::get(mInt16Ty, mFP32Ty);
1036 Function* pCvtPs2Ph = cast<Function>(JM()->mpCurrentModule->getOrInsertFunction("ConvertFloat32ToFloat16", pFuncTy));
1037
1038 if (sys::DynamicLibrary::SearchForAddressOfSymbol("ConvertFloat32ToFloat16") == nullptr)
1039 {
1040 sys::DynamicLibrary::AddSymbol("ConvertFloat32ToFloat16", (void *)&ConvertFloat32ToFloat16);
1041 }
1042
1043 Value* pResult = UndefValue::get(mSimdInt16Ty);
1044 for (uint32_t i = 0; i < mVWidth; ++i)
1045 {
1046 Value* pSrc = VEXTRACT(a, C(i));
1047 Value* pConv = CALL(pCvtPs2Ph, std::initializer_list<Value*>{pSrc});
1048 pResult = VINSERT(pResult, pConv, C(i));
1049 }
1050
1051 return pResult;
1052 }
1053 }
1054
1055 Value *Builder::PMAXSD(Value* a, Value* b)
1056 {
1057 Value* cmp = ICMP_SGT(a, b);
1058 return SELECT(cmp, a, b);
1059 }
1060
1061 Value *Builder::PMINSD(Value* a, Value* b)
1062 {
1063 Value* cmp = ICMP_SLT(a, b);
1064 return SELECT(cmp, a, b);
1065 }
1066
1067 void Builder::Gather4(const SWR_FORMAT format, Value* pSrcBase, Value* byteOffsets,
1068 Value* mask, Value* vGatherComponents[], bool bPackedOutput)
1069 {
1070 const SWR_FORMAT_INFO &info = GetFormatInfo(format);
1071 if(info.type[0] == SWR_TYPE_FLOAT && info.bpc[0] == 32)
1072 {
1073 GATHER4PS(info, pSrcBase, byteOffsets, mask, vGatherComponents, bPackedOutput);
1074 }
1075 else
1076 {
1077 GATHER4DD(info, pSrcBase, byteOffsets, mask, vGatherComponents, bPackedOutput);
1078 }
1079 }
1080
1081 void Builder::GATHER4PS(const SWR_FORMAT_INFO &info, Value* pSrcBase, Value* byteOffsets,
1082 Value* vMask, Value* vGatherComponents[], bool bPackedOutput)
1083 {
1084 switch(info.bpp / info.numComps)
1085 {
1086 case 16:
1087 {
1088 Value* vGatherResult[2];
1089
1090 // TODO: vGatherMaskedVal
1091 Value* vGatherMaskedVal = VIMMED1((float)0);
1092
1093 // always have at least one component out of x or y to fetch
1094
1095 vGatherResult[0] = GATHERPS(vGatherMaskedVal, pSrcBase, byteOffsets, vMask);
1096 // e.g. result of first 8x32bit integer gather for 16bit components
1097 // 256i - 0 1 2 3 4 5 6 7
1098 // xyxy xyxy xyxy xyxy xyxy xyxy xyxy xyxy
1099 //
1100
1101 // if we have at least one component out of x or y to fetch
1102 if(info.numComps > 2)
1103 {
1104 // offset base to the next components(zw) in the vertex to gather
1105 pSrcBase = GEP(pSrcBase, C((char)4));
1106
1107 vGatherResult[1] = GATHERPS(vGatherMaskedVal, pSrcBase, byteOffsets, vMask);
1108 // e.g. result of second 8x32bit integer gather for 16bit components
1109 // 256i - 0 1 2 3 4 5 6 7
1110 // zwzw zwzw zwzw zwzw zwzw zwzw zwzw zwzw
1111 //
1112 }
1113 else
1114 {
1115 vGatherResult[1] = vGatherMaskedVal;
1116 }
1117
1118 // Shuffle gathered components into place, each row is a component
1119 Shuffle16bpcGather4(info, vGatherResult, vGatherComponents, bPackedOutput);
1120 }
1121 break;
1122 case 32:
1123 {
1124 // apply defaults
1125 for (uint32_t i = 0; i < 4; ++i)
1126 {
1127 vGatherComponents[i] = VIMMED1(*(float*)&info.defaults[i]);
1128 }
1129
1130 for(uint32_t i = 0; i < info.numComps; i++)
1131 {
1132 uint32_t swizzleIndex = info.swizzle[i];
1133
1134 // Gather a SIMD of components
1135 vGatherComponents[swizzleIndex] = GATHERPS(vGatherComponents[swizzleIndex], pSrcBase, byteOffsets, vMask);
1136
1137 // offset base to the next component to gather
1138 pSrcBase = GEP(pSrcBase, C((char)4));
1139 }
1140 }
1141 break;
1142 default:
1143 SWR_INVALID("Invalid float format");
1144 break;
1145 }
1146 }
1147
1148 void Builder::GATHER4DD(const SWR_FORMAT_INFO &info, Value* pSrcBase, Value* byteOffsets,
1149 Value* vMask, Value* vGatherComponents[], bool bPackedOutput)
1150 {
1151 switch (info.bpp / info.numComps)
1152 {
1153 case 8:
1154 {
1155 Value* vGatherMaskedVal = VIMMED1((int32_t)0);
1156 Value* vGatherResult = GATHERDD(vGatherMaskedVal, pSrcBase, byteOffsets, vMask);
1157 // e.g. result of an 8x32bit integer gather for 8bit components
1158 // 256i - 0 1 2 3 4 5 6 7
1159 // xyzw xyzw xyzw xyzw xyzw xyzw xyzw xyzw
1160
1161 Shuffle8bpcGather4(info, vGatherResult, vGatherComponents, bPackedOutput);
1162 }
1163 break;
1164 case 16:
1165 {
1166 Value* vGatherResult[2];
1167
1168 // TODO: vGatherMaskedVal
1169 Value* vGatherMaskedVal = VIMMED1((int32_t)0);
1170
1171 // always have at least one component out of x or y to fetch
1172
1173 vGatherResult[0] = GATHERDD(vGatherMaskedVal, pSrcBase, byteOffsets, vMask);
1174 // e.g. result of first 8x32bit integer gather for 16bit components
1175 // 256i - 0 1 2 3 4 5 6 7
1176 // xyxy xyxy xyxy xyxy xyxy xyxy xyxy xyxy
1177 //
1178
1179 // if we have at least one component out of x or y to fetch
1180 if(info.numComps > 2)
1181 {
1182 // offset base to the next components(zw) in the vertex to gather
1183 pSrcBase = GEP(pSrcBase, C((char)4));
1184
1185 vGatherResult[1] = GATHERDD(vGatherMaskedVal, pSrcBase, byteOffsets, vMask);
1186 // e.g. result of second 8x32bit integer gather for 16bit components
1187 // 256i - 0 1 2 3 4 5 6 7
1188 // zwzw zwzw zwzw zwzw zwzw zwzw zwzw zwzw
1189 //
1190 }
1191 else
1192 {
1193 vGatherResult[1] = vGatherMaskedVal;
1194 }
1195
1196 // Shuffle gathered components into place, each row is a component
1197 Shuffle16bpcGather4(info, vGatherResult, vGatherComponents, bPackedOutput);
1198
1199 }
1200 break;
1201 case 32:
1202 {
1203 // apply defaults
1204 for (uint32_t i = 0; i < 4; ++i)
1205 {
1206 vGatherComponents[i] = VIMMED1((int)info.defaults[i]);
1207 }
1208
1209 for(uint32_t i = 0; i < info.numComps; i++)
1210 {
1211 uint32_t swizzleIndex = info.swizzle[i];
1212
1213 // Gather a SIMD of components
1214 vGatherComponents[swizzleIndex] = GATHERDD(vGatherComponents[swizzleIndex], pSrcBase, byteOffsets, vMask);
1215
1216 // offset base to the next component to gather
1217 pSrcBase = GEP(pSrcBase, C((char)4));
1218 }
1219 }
1220 break;
1221 default:
1222 SWR_INVALID("unsupported format");
1223 break;
1224 }
1225 }
1226
1227 void Builder::Shuffle16bpcGather4(const SWR_FORMAT_INFO &info, Value* vGatherInput[2], Value* vGatherOutput[4], bool bPackedOutput)
1228 {
1229 // cast types
1230 Type* vGatherTy = VectorType::get(IntegerType::getInt32Ty(JM()->mContext), mVWidth);
1231 Type* v32x8Ty = VectorType::get(mInt8Ty, mVWidth * 4); // vwidth is units of 32 bits
1232
1233 // input could either be float or int vector; do shuffle work in int
1234 vGatherInput[0] = BITCAST(vGatherInput[0], mSimdInt32Ty);
1235 vGatherInput[1] = BITCAST(vGatherInput[1], mSimdInt32Ty);
1236
1237 if(bPackedOutput)
1238 {
1239 Type* v128bitTy = VectorType::get(IntegerType::getIntNTy(JM()->mContext, 128), mVWidth / 4); // vwidth is units of 32 bits
1240
1241 // shuffle mask
1242 Value* vConstMask = C<char>({0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15,
1243 0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15});
1244 Value* vShufResult = BITCAST(PSHUFB(BITCAST(vGatherInput[0], v32x8Ty), vConstMask), vGatherTy);
1245 // after pshufb: group components together in each 128bit lane
1246 // 256i - 0 1 2 3 4 5 6 7
1247 // xxxx xxxx yyyy yyyy xxxx xxxx yyyy yyyy
1248
1249 Value* vi128XY = BITCAST(PERMD(vShufResult, C<int32_t>({0, 1, 4, 5, 2, 3, 6, 7})), v128bitTy);
1250 // after PERMD: move and pack xy components into each 128bit lane
1251 // 256i - 0 1 2 3 4 5 6 7
1252 // xxxx xxxx xxxx xxxx yyyy yyyy yyyy yyyy
1253
1254 // do the same for zw components
1255 Value* vi128ZW = nullptr;
1256 if(info.numComps > 2)
1257 {
1258 Value* vShufResult = BITCAST(PSHUFB(BITCAST(vGatherInput[1], v32x8Ty), vConstMask), vGatherTy);
1259 vi128ZW = BITCAST(PERMD(vShufResult, C<int32_t>({0, 1, 4, 5, 2, 3, 6, 7})), v128bitTy);
1260 }
1261
1262 for(uint32_t i = 0; i < 4; i++)
1263 {
1264 uint32_t swizzleIndex = info.swizzle[i];
1265 // todo: fixed for packed
1266 Value* vGatherMaskedVal = VIMMED1((int32_t)(info.defaults[i]));
1267 if(i >= info.numComps)
1268 {
1269 // set the default component val
1270 vGatherOutput[swizzleIndex] = vGatherMaskedVal;
1271 continue;
1272 }
1273
1274 // if x or z, extract 128bits from lane 0, else for y or w, extract from lane 1
1275 uint32_t lane = ((i == 0) || (i == 2)) ? 0 : 1;
1276 // if x or y, use vi128XY permute result, else use vi128ZW
1277 Value* selectedPermute = (i < 2) ? vi128XY : vi128ZW;
1278
1279 // extract packed component 128 bit lanes
1280 vGatherOutput[swizzleIndex] = VEXTRACT(selectedPermute, C(lane));
1281 }
1282
1283 }
1284 else
1285 {
1286 // pshufb masks for each component
1287 Value* vConstMask[2];
1288 // x/z shuffle mask
1289 vConstMask[0] = C<char>({0, 1, -1, -1, 4, 5, -1, -1, 8, 9, -1, -1, 12, 13, -1, -1,
1290 0, 1, -1, -1, 4, 5, -1, -1, 8, 9, -1, -1, 12, 13, -1, -1, });
1291
1292 // y/w shuffle mask
1293 vConstMask[1] = C<char>({2, 3, -1, -1, 6, 7, -1, -1, 10, 11, -1, -1, 14, 15, -1, -1,
1294 2, 3, -1, -1, 6, 7, -1, -1, 10, 11, -1, -1, 14, 15, -1, -1});
1295
1296
1297 // shuffle enabled components into lower word of each 32bit lane, 0 extending to 32 bits
1298 // apply defaults
1299 for (uint32_t i = 0; i < 4; ++i)
1300 {
1301 vGatherOutput[i] = VIMMED1((int32_t)info.defaults[i]);
1302 }
1303
1304 for(uint32_t i = 0; i < info.numComps; i++)
1305 {
1306 uint32_t swizzleIndex = info.swizzle[i];
1307
1308 // select correct constMask for x/z or y/w pshufb
1309 uint32_t selectedMask = ((i == 0) || (i == 2)) ? 0 : 1;
1310 // if x or y, use vi128XY permute result, else use vi128ZW
1311 uint32_t selectedGather = (i < 2) ? 0 : 1;
1312
1313 vGatherOutput[swizzleIndex] = BITCAST(PSHUFB(BITCAST(vGatherInput[selectedGather], v32x8Ty), vConstMask[selectedMask]), vGatherTy);
1314 // after pshufb mask for x channel; z uses the same shuffle from the second gather
1315 // 256i - 0 1 2 3 4 5 6 7
1316 // xx00 xx00 xx00 xx00 xx00 xx00 xx00 xx00
1317 }
1318 }
1319 }
1320
1321 void Builder::Shuffle8bpcGather4(const SWR_FORMAT_INFO &info, Value* vGatherInput, Value* vGatherOutput[], bool bPackedOutput)
1322 {
1323 // cast types
1324 Type* vGatherTy = VectorType::get(IntegerType::getInt32Ty(JM()->mContext), mVWidth);
1325 Type* v32x8Ty = VectorType::get(mInt8Ty, mVWidth * 4 ); // vwidth is units of 32 bits
1326
1327 if(bPackedOutput)
1328 {
1329 Type* v128Ty = VectorType::get(IntegerType::getIntNTy(JM()->mContext, 128), mVWidth / 4); // vwidth is units of 32 bits
1330 // shuffle mask
1331 Value* vConstMask = C<char>({0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15,
1332 0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15});
1333 Value* vShufResult = BITCAST(PSHUFB(BITCAST(vGatherInput, v32x8Ty), vConstMask), vGatherTy);
1334 // after pshufb: group components together in each 128bit lane
1335 // 256i - 0 1 2 3 4 5 6 7
1336 // xxxx yyyy zzzz wwww xxxx yyyy zzzz wwww
1337
1338 Value* vi128XY = BITCAST(PERMD(vShufResult, C<int32_t>({0, 4, 0, 0, 1, 5, 0, 0})), v128Ty);
1339 // after PERMD: move and pack xy and zw components in low 64 bits of each 128bit lane
1340 // 256i - 0 1 2 3 4 5 6 7
1341 // xxxx xxxx dcdc dcdc yyyy yyyy dcdc dcdc (dc - don't care)
1342
1343 // do the same for zw components
1344 Value* vi128ZW = nullptr;
1345 if(info.numComps > 2)
1346 {
1347 vi128ZW = BITCAST(PERMD(vShufResult, C<int32_t>({2, 6, 0, 0, 3, 7, 0, 0})), v128Ty);
1348 }
1349
1350 // sign extend all enabled components. If we have a fill vVertexElements, output to current simdvertex
1351 for(uint32_t i = 0; i < 4; i++)
1352 {
1353 uint32_t swizzleIndex = info.swizzle[i];
1354 // todo: fix for packed
1355 Value* vGatherMaskedVal = VIMMED1((int32_t)(info.defaults[i]));
1356 if(i >= info.numComps)
1357 {
1358 // set the default component val
1359 vGatherOutput[swizzleIndex] = vGatherMaskedVal;
1360 continue;
1361 }
1362
1363 // if x or z, extract 128bits from lane 0, else for y or w, extract from lane 1
1364 uint32_t lane = ((i == 0) || (i == 2)) ? 0 : 1;
1365 // if x or y, use vi128XY permute result, else use vi128ZW
1366 Value* selectedPermute = (i < 2) ? vi128XY : vi128ZW;
1367
1368 // sign extend
1369 vGatherOutput[swizzleIndex] = VEXTRACT(selectedPermute, C(lane));
1370 }
1371 }
1372 // else zero extend
1373 else{
1374 // shuffle enabled components into lower byte of each 32bit lane, 0 extending to 32 bits
1375 // apply defaults
1376 for (uint32_t i = 0; i < 4; ++i)
1377 {
1378 vGatherOutput[i] = VIMMED1((int32_t)info.defaults[i]);
1379 }
1380
1381 for(uint32_t i = 0; i < info.numComps; i++){
1382 uint32_t swizzleIndex = info.swizzle[i];
1383
1384 // pshufb masks for each component
1385 Value* vConstMask;
1386 switch(i)
1387 {
1388 case 0:
1389 // x shuffle mask
1390 vConstMask = C<char>({0, -1, -1, -1, 4, -1, -1, -1, 8, -1, -1, -1, 12, -1, -1, -1,
1391 0, -1, -1, -1, 4, -1, -1, -1, 8, -1, -1, -1, 12, -1, -1, -1});
1392 break;
1393 case 1:
1394 // y shuffle mask
1395 vConstMask = C<char>({1, -1, -1, -1, 5, -1, -1, -1, 9, -1, -1, -1, 13, -1, -1, -1,
1396 1, -1, -1, -1, 5, -1, -1, -1, 9, -1, -1, -1, 13, -1, -1, -1});
1397 break;
1398 case 2:
1399 // z shuffle mask
1400 vConstMask = C<char>({2, -1, -1, -1, 6, -1, -1, -1, 10, -1, -1, -1, 14, -1, -1, -1,
1401 2, -1, -1, -1, 6, -1, -1, -1, 10, -1, -1, -1, 14, -1, -1, -1});
1402 break;
1403 case 3:
1404 // w shuffle mask
1405 vConstMask = C<char>({3, -1, -1, -1, 7, -1, -1, -1, 11, -1, -1, -1, 15, -1, -1, -1,
1406 3, -1, -1, -1, 7, -1, -1, -1, 11, -1, -1, -1, 15, -1, -1, -1});
1407 break;
1408 default:
1409 vConstMask = nullptr;
1410 break;
1411 }
1412
1413 vGatherOutput[swizzleIndex] = BITCAST(PSHUFB(BITCAST(vGatherInput, v32x8Ty), vConstMask), vGatherTy);
1414 // after pshufb for x channel
1415 // 256i - 0 1 2 3 4 5 6 7
1416 // x000 x000 x000 x000 x000 x000 x000 x000
1417 }
1418 }
1419 }
1420
1421 // Helper function to create alloca in entry block of function
1422 Value* Builder::CreateEntryAlloca(Function* pFunc, Type* pType)
1423 {
1424 auto saveIP = IRB()->saveIP();
1425 IRB()->SetInsertPoint(&pFunc->getEntryBlock(),
1426 pFunc->getEntryBlock().begin());
1427 Value* pAlloca = ALLOCA(pType);
1428 if (saveIP.isSet()) IRB()->restoreIP(saveIP);
1429 return pAlloca;
1430 }
1431
1432 Value* Builder::CreateEntryAlloca(Function* pFunc, Type* pType, Value* pArraySize)
1433 {
1434 auto saveIP = IRB()->saveIP();
1435 IRB()->SetInsertPoint(&pFunc->getEntryBlock(),
1436 pFunc->getEntryBlock().begin());
1437 Value* pAlloca = ALLOCA(pType, pArraySize);
1438 if (saveIP.isSet()) IRB()->restoreIP(saveIP);
1439 return pAlloca;
1440 }
1441
1442 //////////////////////////////////////////////////////////////////////////
1443 /// @brief emulates a scatter operation.
1444 /// @param pDst - pointer to destination
1445 /// @param vSrc - vector of src data to scatter
1446 /// @param vOffsets - vector of byte offsets from pDst
1447 /// @param vMask - mask of valid lanes
1448 void Builder::SCATTERPS(Value* pDst, Value* vSrc, Value* vOffsets, Value* vMask)
1449 {
1450 /* Scatter algorithm
1451
1452 while(Index = BitScanForward(mask))
1453 srcElem = srcVector[Index]
1454 offsetElem = offsetVector[Index]
1455 *(pDst + offsetElem) = srcElem
1456 Update mask (&= ~(1<<Index)
1457
1458 */
1459
1460 BasicBlock* pCurBB = IRB()->GetInsertBlock();
1461 Function* pFunc = pCurBB->getParent();
1462 Type* pSrcTy = vSrc->getType()->getVectorElementType();
1463
1464 // Store vectors on stack
1465 if (pScatterStackSrc == nullptr)
1466 {
1467 // Save off stack allocations and reuse per scatter. Significantly reduces stack
1468 // requirements for shaders with a lot of scatters.
1469 pScatterStackSrc = CreateEntryAlloca(pFunc, mSimdInt64Ty);
1470 pScatterStackOffsets = CreateEntryAlloca(pFunc, mSimdInt32Ty);
1471 }
1472
1473 Value* pSrcArrayPtr = BITCAST(pScatterStackSrc, PointerType::get(vSrc->getType(), 0));
1474 Value* pOffsetsArrayPtr = pScatterStackOffsets;
1475 STORE(vSrc, pSrcArrayPtr);
1476 STORE(vOffsets, pOffsetsArrayPtr);
1477
1478 // Cast to pointers for random access
1479 pSrcArrayPtr = POINTER_CAST(pSrcArrayPtr, PointerType::get(pSrcTy, 0));
1480 pOffsetsArrayPtr = POINTER_CAST(pOffsetsArrayPtr, PointerType::get(mInt32Ty, 0));
1481
1482 Value* pMask = VMOVMSKPS(BITCAST(vMask, mSimdFP32Ty));
1483
1484 // Get cttz function
1485 Function* pfnCttz = Intrinsic::getDeclaration(mpJitMgr->mpCurrentModule, Intrinsic::cttz, { mInt32Ty });
1486
1487 // Setup loop basic block
1488 BasicBlock* pLoop = BasicBlock::Create(mpJitMgr->mContext, "Scatter_Loop", pFunc);
1489
1490 // compute first set bit
1491 Value* pIndex = CALL(pfnCttz, { pMask, C(false) });
1492
1493 Value* pIsUndef = ICMP_EQ(pIndex, C(32));
1494
1495 // Split current block
1496 BasicBlock* pPostLoop = pCurBB->splitBasicBlock(cast<Instruction>(pIsUndef)->getNextNode());
1497
1498 // Remove unconditional jump created by splitBasicBlock
1499 pCurBB->getTerminator()->eraseFromParent();
1500
1501 // Add terminator to end of original block
1502 IRB()->SetInsertPoint(pCurBB);
1503
1504 // Add conditional branch
1505 COND_BR(pIsUndef, pPostLoop, pLoop);
1506
1507 // Add loop basic block contents
1508 IRB()->SetInsertPoint(pLoop);
1509 PHINode* pIndexPhi = PHI(mInt32Ty, 2);
1510 PHINode* pMaskPhi = PHI(mInt32Ty, 2);
1511
1512 pIndexPhi->addIncoming(pIndex, pCurBB);
1513 pMaskPhi->addIncoming(pMask, pCurBB);
1514
1515 // Extract elements for this index
1516 Value* pSrcElem = LOADV(pSrcArrayPtr, { pIndexPhi });
1517 Value* pOffsetElem = LOADV(pOffsetsArrayPtr, { pIndexPhi });
1518
1519 // GEP to this offset in dst
1520 Value* pCurDst = GEP(pDst, pOffsetElem);
1521 pCurDst = POINTER_CAST(pCurDst, PointerType::get(pSrcTy, 0));
1522 STORE(pSrcElem, pCurDst);
1523
1524 // Update the mask
1525 Value* pNewMask = AND(pMaskPhi, NOT(SHL(C(1), pIndexPhi)));
1526
1527 // Terminator
1528 Value* pNewIndex = CALL(pfnCttz, { pNewMask, C(false) });
1529
1530 pIsUndef = ICMP_EQ(pNewIndex, C(32));
1531 COND_BR(pIsUndef, pPostLoop, pLoop);
1532
1533 // Update phi edges
1534 pIndexPhi->addIncoming(pNewIndex, pLoop);
1535 pMaskPhi->addIncoming(pNewMask, pLoop);
1536
1537 // Move builder to beginning of post loop
1538 IRB()->SetInsertPoint(pPostLoop, pPostLoop->begin());
1539 }
1540
1541 Value* Builder::VABSPS(Value* a)
1542 {
1543 Value* asInt = BITCAST(a, mSimdInt32Ty);
1544 Value* result = BITCAST(AND(asInt, VIMMED1(0x7fffffff)), mSimdFP32Ty);
1545 return result;
1546 }
1547
1548 Value *Builder::ICLAMP(Value* src, Value* low, Value* high, const llvm::Twine& name)
1549 {
1550 Value *lowCmp = ICMP_SLT(src, low);
1551 Value *ret = SELECT(lowCmp, low, src);
1552
1553 Value *highCmp = ICMP_SGT(ret, high);
1554 ret = SELECT(highCmp, high, ret, name);
1555
1556 return ret;
1557 }
1558
1559 Value *Builder::FCLAMP(Value* src, Value* low, Value* high)
1560 {
1561 Value *lowCmp = FCMP_OLT(src, low);
1562 Value *ret = SELECT(lowCmp, low, src);
1563
1564 Value *highCmp = FCMP_OGT(ret, high);
1565 ret = SELECT(highCmp, high, ret);
1566
1567 return ret;
1568 }
1569
1570 Value *Builder::FCLAMP(Value* src, float low, float high)
1571 {
1572 Value* result = VMAXPS(src, VIMMED1(low));
1573 result = VMINPS(result, VIMMED1(high));
1574
1575 return result;
1576 }
1577
1578 //////////////////////////////////////////////////////////////////////////
1579 /// @brief save/restore stack, providing ability to push/pop the stack and
1580 /// reduce overall stack requirements for temporary stack use
1581 Value* Builder::STACKSAVE()
1582 {
1583 Function* pfnStackSave = Intrinsic::getDeclaration(JM()->mpCurrentModule, Intrinsic::stacksave);
1584 return CALLA(pfnStackSave);
1585 }
1586
1587 void Builder::STACKRESTORE(Value* pSaved)
1588 {
1589 Function* pfnStackRestore = Intrinsic::getDeclaration(JM()->mpCurrentModule, Intrinsic::stackrestore);
1590 CALL(pfnStackRestore, std::initializer_list<Value*>{pSaved});
1591 }
1592
1593 Value *Builder::FMADDPS(Value* a, Value* b, Value* c)
1594 {
1595 Value* vOut;
1596 // use FMADs if available
1597 if(JM()->mArch.AVX2())
1598 {
1599 vOut = VFMADDPS(a, b, c);
1600 }
1601 else
1602 {
1603 vOut = FADD(FMUL(a, b), c);
1604 }
1605 return vOut;
1606 }
1607
1608 Value* Builder::POPCNT(Value* a)
1609 {
1610 Function* pCtPop = Intrinsic::getDeclaration(JM()->mpCurrentModule, Intrinsic::ctpop, { a->getType() });
1611 return CALL(pCtPop, std::initializer_list<Value*>{a});
1612 }
1613
1614 //////////////////////////////////////////////////////////////////////////
1615 /// @brief C functions called by LLVM IR
1616 //////////////////////////////////////////////////////////////////////////
1617
1618 //////////////////////////////////////////////////////////////////////////
1619 /// @brief called in JIT code, inserted by PRINT
1620 /// output to both stdout and visual studio debug console
1621 void __cdecl CallPrint(const char* fmt, ...)
1622 {
1623 va_list args;
1624 va_start(args, fmt);
1625 vprintf(fmt, args);
1626
1627 #if defined( _WIN32 )
1628 char strBuf[1024];
1629 vsnprintf_s(strBuf, _TRUNCATE, fmt, args);
1630 OutputDebugStringA(strBuf);
1631 #endif
1632
1633 va_end(args);
1634 }
1635
1636 Value *Builder::VEXTRACTI128(Value* a, Constant* imm8)
1637 {
1638 bool flag = !imm8->isZeroValue();
1639 SmallVector<Constant*,8> idx;
1640 for (unsigned i = 0; i < mVWidth / 2; i++) {
1641 idx.push_back(C(flag ? i + mVWidth / 2 : i));
1642 }
1643 return VSHUFFLE(a, VUNDEF_I(), ConstantVector::get(idx));
1644 }
1645
1646 Value *Builder::VINSERTI128(Value* a, Value* b, Constant* imm8)
1647 {
1648 bool flag = !imm8->isZeroValue();
1649 SmallVector<Constant*,8> idx;
1650 for (unsigned i = 0; i < mVWidth; i++) {
1651 idx.push_back(C(i));
1652 }
1653 Value *inter = VSHUFFLE(b, VUNDEF_I(), ConstantVector::get(idx));
1654
1655 SmallVector<Constant*,8> idx2;
1656 for (unsigned i = 0; i < mVWidth / 2; i++) {
1657 idx2.push_back(C(flag ? i : i + mVWidth));
1658 }
1659 for (unsigned i = mVWidth / 2; i < mVWidth; i++) {
1660 idx2.push_back(C(flag ? i + mVWidth / 2 : i));
1661 }
1662 return VSHUFFLE(a, inter, ConstantVector::get(idx2));
1663 }
1664
1665 // rdtsc buckets macros
1666 void Builder::RDTSC_START(Value* pBucketMgr, Value* pId)
1667 {
1668 // @todo due to an issue with thread local storage propagation in llvm, we can only safely call into
1669 // buckets framework when single threaded
1670 if (KNOB_SINGLE_THREADED)
1671 {
1672 std::vector<Type*> args{
1673 PointerType::get(mInt32Ty, 0), // pBucketMgr
1674 mInt32Ty // id
1675 };
1676
1677 FunctionType* pFuncTy = FunctionType::get(Type::getVoidTy(JM()->mContext), args, false);
1678 Function* pFunc = cast<Function>(JM()->mpCurrentModule->getOrInsertFunction("BucketManager_StartBucket", pFuncTy));
1679 if (sys::DynamicLibrary::SearchForAddressOfSymbol("BucketManager_StartBucket") == nullptr)
1680 {
1681 sys::DynamicLibrary::AddSymbol("BucketManager_StartBucket", (void*)&BucketManager_StartBucket);
1682 }
1683
1684 CALL(pFunc, { pBucketMgr, pId });
1685 }
1686 }
1687
1688 void Builder::RDTSC_STOP(Value* pBucketMgr, Value* pId)
1689 {
1690 // @todo due to an issue with thread local storage propagation in llvm, we can only safely call into
1691 // buckets framework when single threaded
1692 if (KNOB_SINGLE_THREADED)
1693 {
1694 std::vector<Type*> args{
1695 PointerType::get(mInt32Ty, 0), // pBucketMgr
1696 mInt32Ty // id
1697 };
1698
1699 FunctionType* pFuncTy = FunctionType::get(Type::getVoidTy(JM()->mContext), args, false);
1700 Function* pFunc = cast<Function>(JM()->mpCurrentModule->getOrInsertFunction("BucketManager_StopBucket", pFuncTy));
1701 if (sys::DynamicLibrary::SearchForAddressOfSymbol("BucketManager_StopBucket") == nullptr)
1702 {
1703 sys::DynamicLibrary::AddSymbol("BucketManager_StopBucket", (void*)&BucketManager_StopBucket);
1704 }
1705
1706 CALL(pFunc, { pBucketMgr, pId });
1707 }
1708 }
1709
1710
1711 uint32_t Builder::GetTypeSize(Type* pType)
1712 {
1713 if (pType->isStructTy())
1714 {
1715 uint32_t numElems = pType->getStructNumElements();
1716 Type* pElemTy = pType->getStructElementType(0);
1717 return numElems * GetTypeSize(pElemTy);
1718 }
1719
1720 if (pType->isArrayTy())
1721 {
1722 uint32_t numElems = pType->getArrayNumElements();
1723 Type* pElemTy = pType->getArrayElementType();
1724 return numElems * GetTypeSize(pElemTy);
1725 }
1726
1727 if (pType->isIntegerTy())
1728 {
1729 uint32_t bitSize = pType->getIntegerBitWidth();
1730 return bitSize / 8;
1731 }
1732
1733 if (pType->isFloatTy())
1734 {
1735 return 4;
1736 }
1737
1738 if (pType->isHalfTy())
1739 {
1740 return 2;
1741 }
1742
1743 if (pType->isDoubleTy())
1744 {
1745 return 8;
1746 }
1747
1748 SWR_ASSERT(false, "Unimplemented type.");
1749 return 0;
1750 }
1751 }