projects / mesa.git / blob
? search:


04092541e5dce75f3f637ea9568f9102d8261066
[mesa.git] / src / gallium / drivers / swr / rasterizer / jitter / builder_misc.cpp
1 /****************************************************************************
2 * Copyright (C) 2014-2015 Intel Corporation. All Rights Reserved.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 * @file builder_misc.cpp
24 *
25 * @brief Implementation for miscellaneous builder functions
26 *
27 * Notes:
28 *
29 ******************************************************************************/
30 #include "builder.h"
31 #include "common/rdtsc_buckets.h"
32
33 #include <cstdarg>
34
35 namespace SwrJit
36 {
37 void __cdecl CallPrint(const char* fmt, ...);
38
39 //////////////////////////////////////////////////////////////////////////
40 /// @brief Convert an IEEE 754 32-bit single precision float to an
41 /// 16 bit float with 5 exponent bits and a variable
42 /// number of mantissa bits.
43 /// @param val - 32-bit float
44 /// @todo Maybe move this outside of this file into a header?
45 static uint16_t ConvertFloat32ToFloat16(float val)
46 {
47 uint32_t sign, exp, mant;
48 uint32_t roundBits;
49
50 // Extract the sign, exponent, and mantissa
51 uint32_t uf = *(uint32_t*)&val;
52 sign = (uf & 0x80000000) >> 31;
53 exp = (uf & 0x7F800000) >> 23;
54 mant = uf & 0x007FFFFF;
55
56 // Check for out of range
57 if (std::isnan(val))
58 {
59 exp = 0x1F;
60 mant = 0x200;
61 sign = 1; // set the sign bit for NANs
62 }
63 else if (std::isinf(val))
64 {
65 exp = 0x1f;
66 mant = 0x0;
67 }
68 else if (exp > (0x70 + 0x1E)) // Too big to represent -> max representable value
69 {
70 exp = 0x1E;
71 mant = 0x3FF;
72 }
73 else if ((exp <= 0x70) && (exp >= 0x66)) // It's a denorm
74 {
75 mant |= 0x00800000;
76 for (; exp <= 0x70; mant >>= 1, exp++)
77 ;
78 exp = 0;
79 mant = mant >> 13;
80 }
81 else if (exp < 0x66) // Too small to represent -> Zero
82 {
83 exp = 0;
84 mant = 0;
85 }
86 else
87 {
88 // Saves bits that will be shifted off for rounding
89 roundBits = mant & 0x1FFFu;
90 // convert exponent and mantissa to 16 bit format
91 exp = exp - 0x70;
92 mant = mant >> 13;
93
94 // Essentially RTZ, but round up if off by only 1 lsb
95 if (roundBits == 0x1FFFu)
96 {
97 mant++;
98 // check for overflow
99 if ((mant & 0xC00u) != 0)
100 exp++;
101 // make sure only the needed bits are used
102 mant &= 0x3FF;
103 }
104 }
105
106 uint32_t tmpVal = (sign << 15) | (exp << 10) | mant;
107 return (uint16_t)tmpVal;
108 }
109
110 //////////////////////////////////////////////////////////////////////////
111 /// @brief Convert an IEEE 754 16-bit float to an 32-bit single precision
112 /// float
113 /// @param val - 16-bit float
114 /// @todo Maybe move this outside of this file into a header?
115 static float ConvertFloat16ToFloat32(uint32_t val)
116 {
117 uint32_t result;
118 if ((val & 0x7fff) == 0)
119 {
120 result = ((uint32_t)(val & 0x8000)) << 16;
121 }
122 else if ((val & 0x7c00) == 0x7c00)
123 {
124 result = ((val & 0x3ff) == 0) ? 0x7f800000 : 0x7fc00000;
125 result |= ((uint32_t)val & 0x8000) << 16;
126 }
127 else
128 {
129 uint32_t sign = (val & 0x8000) << 16;
130 uint32_t mant = (val & 0x3ff) << 13;
131 uint32_t exp = (val >> 10) & 0x1f;
132 if ((exp == 0) && (mant != 0)) // Adjust exponent and mantissa for denormals
133 {
134 mant <<= 1;
135 while (mant < (0x400 << 13))
136 {
137 exp--;
138 mant <<= 1;
139 }
140 mant &= (0x3ff << 13);
141 }
142 exp = ((exp - 15 + 127) & 0xff) << 23;
143 result = sign | exp | mant;
144 }
145
146 return *(float*)&result;
147 }
148
149 Constant *Builder::C(bool i)
150 {
151 return ConstantInt::get(IRB()->getInt1Ty(), (i ? 1 : 0));
152 }
153
154 Constant *Builder::C(char i)
155 {
156 return ConstantInt::get(IRB()->getInt8Ty(), i);
157 }
158
159 Constant *Builder::C(uint8_t i)
160 {
161 return ConstantInt::get(IRB()->getInt8Ty(), i);
162 }
163
164 Constant *Builder::C(int i)
165 {
166 return ConstantInt::get(IRB()->getInt32Ty(), i);
167 }
168
169 Constant *Builder::C(int64_t i)
170 {
171 return ConstantInt::get(IRB()->getInt64Ty(), i);
172 }
173
174 Constant *Builder::C(uint16_t i)
175 {
176 return ConstantInt::get(mInt16Ty,i);
177 }
178
179 Constant *Builder::C(uint32_t i)
180 {
181 return ConstantInt::get(IRB()->getInt32Ty(), i);
182 }
183
184 Constant *Builder::C(float i)
185 {
186 return ConstantFP::get(IRB()->getFloatTy(), i);
187 }
188
189 Constant *Builder::PRED(bool pred)
190 {
191 return ConstantInt::get(IRB()->getInt1Ty(), (pred ? 1 : 0));
192 }
193
194 Value *Builder::VIMMED1(int i)
195 {
196 return ConstantVector::getSplat(mVWidth, cast<ConstantInt>(C(i)));
197 }
198
199 Value *Builder::VIMMED1(uint32_t i)
200 {
201 return ConstantVector::getSplat(mVWidth, cast<ConstantInt>(C(i)));
202 }
203
204 Value *Builder::VIMMED1(float i)
205 {
206 return ConstantVector::getSplat(mVWidth, cast<ConstantFP>(C(i)));
207 }
208
209 Value *Builder::VIMMED1(bool i)
210 {
211 return ConstantVector::getSplat(mVWidth, cast<ConstantInt>(C(i)));
212 }
213
214 #if USE_SIMD16_BUILDER
215 Value *Builder::VIMMED2_1(int i)
216 {
217 return ConstantVector::getSplat(mVWidth2, cast<ConstantInt>(C(i)));
218 }
219
220 Value *Builder::VIMMED2_1(uint32_t i)
221 {
222 return ConstantVector::getSplat(mVWidth2, cast<ConstantInt>(C(i)));
223 }
224
225 Value *Builder::VIMMED2_1(float i)
226 {
227 return ConstantVector::getSplat(mVWidth2, cast<ConstantFP>(C(i)));
228 }
229
230 Value *Builder::VIMMED2_1(bool i)
231 {
232 return ConstantVector::getSplat(mVWidth2, cast<ConstantInt>(C(i)));
233 }
234
235 #endif
236 Value *Builder::VUNDEF_IPTR()
237 {
238 return UndefValue::get(VectorType::get(mInt32PtrTy,mVWidth));
239 }
240
241 Value *Builder::VUNDEF_I()
242 {
243 return UndefValue::get(VectorType::get(mInt32Ty, mVWidth));
244 }
245
246 Value *Builder::VUNDEF(Type *ty, uint32_t size)
247 {
248 return UndefValue::get(VectorType::get(ty, size));
249 }
250
251 Value *Builder::VUNDEF_F()
252 {
253 return UndefValue::get(VectorType::get(mFP32Ty, mVWidth));
254 }
255
256 #if USE_SIMD16_BUILDER
257 Value *Builder::VUNDEF2_F()
258 {
259 return UndefValue::get(VectorType::get(mFP32Ty, mVWidth2));
260 }
261
262 Value *Builder::VUNDEF2_I()
263 {
264 return UndefValue::get(VectorType::get(mInt32Ty, mVWidth2));
265 }
266
267 #endif
268 Value *Builder::VUNDEF(Type* t)
269 {
270 return UndefValue::get(VectorType::get(t, mVWidth));
271 }
272
273 Value *Builder::VBROADCAST(Value *src)
274 {
275 // check if src is already a vector
276 if (src->getType()->isVectorTy())
277 {
278 return src;
279 }
280
281 return VECTOR_SPLAT(mVWidth, src);
282 }
283
284 #if USE_SIMD16_BUILDER
285 Value *Builder::VBROADCAST2(Value *src)
286 {
287 // check if src is already a vector
288 if (src->getType()->isVectorTy())
289 {
290 return src;
291 }
292
293 return VECTOR_SPLAT(mVWidth2, src);
294 }
295
296 #endif
297 uint32_t Builder::IMMED(Value* v)
298 {
299 SWR_ASSERT(isa<ConstantInt>(v));
300 ConstantInt *pValConst = cast<ConstantInt>(v);
301 return pValConst->getZExtValue();
302 }
303
304 int32_t Builder::S_IMMED(Value* v)
305 {
306 SWR_ASSERT(isa<ConstantInt>(v));
307 ConstantInt *pValConst = cast<ConstantInt>(v);
308 return pValConst->getSExtValue();
309 }
310
311 Value *Builder::GEP(Value* ptr, const std::initializer_list<Value*> &indexList)
312 {
313 std::vector<Value*> indices;
314 for (auto i : indexList)
315 indices.push_back(i);
316 return GEPA(ptr, indices);
317 }
318
319 Value *Builder::GEP(Value* ptr, const std::initializer_list<uint32_t> &indexList)
320 {
321 std::vector<Value*> indices;
322 for (auto i : indexList)
323 indices.push_back(C(i));
324 return GEPA(ptr, indices);
325 }
326
327 Value *Builder::IN_BOUNDS_GEP(Value* ptr, const std::initializer_list<Value*> &indexList)
328 {
329 std::vector<Value*> indices;
330 for (auto i : indexList)
331 indices.push_back(i);
332 return IN_BOUNDS_GEP(ptr, indices);
333 }
334
335 Value *Builder::IN_BOUNDS_GEP(Value* ptr, const std::initializer_list<uint32_t> &indexList)
336 {
337 std::vector<Value*> indices;
338 for (auto i : indexList)
339 indices.push_back(C(i));
340 return IN_BOUNDS_GEP(ptr, indices);
341 }
342
343 LoadInst *Builder::LOAD(Value *basePtr, const std::initializer_list<uint32_t> &indices, const llvm::Twine& name)
344 {
345 std::vector<Value*> valIndices;
346 for (auto i : indices)
347 valIndices.push_back(C(i));
348 return LOAD(GEPA(basePtr, valIndices), name);
349 }
350
351 LoadInst *Builder::LOADV(Value *basePtr, const std::initializer_list<Value*> &indices, const llvm::Twine& name)
352 {
353 std::vector<Value*> valIndices;
354 for (auto i : indices)
355 valIndices.push_back(i);
356 return LOAD(GEPA(basePtr, valIndices), name);
357 }
358
359 StoreInst *Builder::STORE(Value *val, Value *basePtr, const std::initializer_list<uint32_t> &indices)
360 {
361 std::vector<Value*> valIndices;
362 for (auto i : indices)
363 valIndices.push_back(C(i));
364 return STORE(val, GEPA(basePtr, valIndices));
365 }
366
367 StoreInst *Builder::STOREV(Value *val, Value *basePtr, const std::initializer_list<Value*> &indices)
368 {
369 std::vector<Value*> valIndices;
370 for (auto i : indices)
371 valIndices.push_back(i);
372 return STORE(val, GEPA(basePtr, valIndices));
373 }
374
375 CallInst *Builder::CALL(Value *Callee, const std::initializer_list<Value*> &argsList)
376 {
377 std::vector<Value*> args;
378 for (auto arg : argsList)
379 args.push_back(arg);
380 return CALLA(Callee, args);
381 }
382
383 CallInst *Builder::CALL(Value *Callee, Value* arg)
384 {
385 std::vector<Value*> args;
386 args.push_back(arg);
387 return CALLA(Callee, args);
388 }
389
390 CallInst *Builder::CALL2(Value *Callee, Value* arg1, Value* arg2)
391 {
392 std::vector<Value*> args;
393 args.push_back(arg1);
394 args.push_back(arg2);
395 return CALLA(Callee, args);
396 }
397
398 CallInst *Builder::CALL3(Value *Callee, Value* arg1, Value* arg2, Value* arg3)
399 {
400 std::vector<Value*> args;
401 args.push_back(arg1);
402 args.push_back(arg2);
403 args.push_back(arg3);
404 return CALLA(Callee, args);
405 }
406
407 //////////////////////////////////////////////////////////////////////////
408 Value *Builder::DEBUGTRAP()
409 {
410 Function *func = Intrinsic::getDeclaration(JM()->mpCurrentModule, Intrinsic::debugtrap);
411 return CALL(func);
412 }
413
414 Value *Builder::VRCP(Value *va)
415 {
416 return FDIV(VIMMED1(1.0f), va); // 1 / a
417 }
418
419 Value *Builder::VPLANEPS(Value* vA, Value* vB, Value* vC, Value* &vX, Value* &vY)
420 {
421 Value* vOut = FMADDPS(vA, vX, vC);
422 vOut = FMADDPS(vB, vY, vOut);
423 return vOut;
424 }
425
426 //////////////////////////////////////////////////////////////////////////
427 /// @brief Generate an i32 masked load operation in LLVM IR. If not
428 /// supported on the underlying platform, emulate it with float masked load
429 /// @param src - base address pointer for the load
430 /// @param vMask - SIMD wide mask that controls whether to access memory load 0
431 Value *Builder::MASKLOADD(Value* src,Value* mask)
432 {
433 Value* vResult;
434 // use avx2 gather instruction is available
435 if(JM()->mArch.AVX2())
436 {
437 Function *func = Intrinsic::getDeclaration(JM()->mpCurrentModule, Intrinsic::x86_avx2_maskload_d_256);
438 vResult = CALL(func,{src,mask});
439 }
440 else
441 {
442 // maskload intrinsic expects integer mask operand in llvm >= 3.8
443 #if (LLVM_VERSION_MAJOR > 3) || (LLVM_VERSION_MAJOR == 3 && LLVM_VERSION_MINOR >= 8)
444 mask = BITCAST(mask,VectorType::get(mInt32Ty,mVWidth));
445 #else
446 mask = BITCAST(mask,VectorType::get(mFP32Ty,mVWidth));
447 #endif
448 Function *func = Intrinsic::getDeclaration(JM()->mpCurrentModule,Intrinsic::x86_avx_maskload_ps_256);
449 vResult = BITCAST(CALL(func,{src,mask}), VectorType::get(mInt32Ty,mVWidth));
450 }
451 return vResult;
452 }
453
454 //////////////////////////////////////////////////////////////////////////
455 /// @brief insert a JIT call to CallPrint
456 /// - outputs formatted string to both stdout and VS output window
457 /// - DEBUG builds only
458 /// Usage example:
459 /// PRINT("index %d = 0x%p\n",{C(lane), pIndex});
460 /// where C(lane) creates a constant value to print, and pIndex is the Value*
461 /// result from a GEP, printing out the pointer to memory
462 /// @param printStr - constant string to print, which includes format specifiers
463 /// @param printArgs - initializer list of Value*'s to print to std out
464 CallInst *Builder::PRINT(const std::string &printStr,const std::initializer_list<Value*> &printArgs)
465 {
466 // push the arguments to CallPrint into a vector
467 std::vector<Value*> printCallArgs;
468 // save room for the format string. we still need to modify it for vectors
469 printCallArgs.resize(1);
470
471 // search through the format string for special processing
472 size_t pos = 0;
473 std::string tempStr(printStr);
474 pos = tempStr.find('%', pos);
475 auto v = printArgs.begin();
476
477 while ((pos != std::string::npos) && (v != printArgs.end()))
478 {
479 Value* pArg = *v;
480 Type* pType = pArg->getType();
481
482 if (pType->isVectorTy())
483 {
484 Type* pContainedType = pType->getContainedType(0);
485
486 if (toupper(tempStr[pos + 1]) == 'X')
487 {
488 tempStr[pos] = '0';
489 tempStr[pos + 1] = 'x';
490 tempStr.insert(pos + 2, "%08X ");
491 pos += 7;
492
493 printCallArgs.push_back(VEXTRACT(pArg, C(0)));
494
495 std::string vectorFormatStr;
496 for (uint32_t i = 1; i < pType->getVectorNumElements(); ++i)
497 {
498 vectorFormatStr += "0x%08X ";
499 printCallArgs.push_back(VEXTRACT(pArg, C(i)));
500 }
501
502 tempStr.insert(pos, vectorFormatStr);
503 pos += vectorFormatStr.size();
504 }
505 else if ((tempStr[pos + 1] == 'f') && (pContainedType->isFloatTy()))
506 {
507 uint32_t i = 0;
508 for (; i < (pArg->getType()->getVectorNumElements()) - 1; i++)
509 {
510 tempStr.insert(pos, std::string("%f "));
511 pos += 3;
512 printCallArgs.push_back(FP_EXT(VEXTRACT(pArg, C(i)), Type::getDoubleTy(JM()->mContext)));
513 }
514 printCallArgs.push_back(FP_EXT(VEXTRACT(pArg, C(i)), Type::getDoubleTy(JM()->mContext)));
515 }
516 else if ((tempStr[pos + 1] == 'd') && (pContainedType->isIntegerTy()))
517 {
518 uint32_t i = 0;
519 for (; i < (pArg->getType()->getVectorNumElements()) - 1; i++)
520 {
521 tempStr.insert(pos, std::string("%d "));
522 pos += 3;
523 printCallArgs.push_back(VEXTRACT(pArg, C(i)));
524 }
525 printCallArgs.push_back(VEXTRACT(pArg, C(i)));
526 }
527 }
528 else
529 {
530 if (toupper(tempStr[pos + 1]) == 'X')
531 {
532 tempStr[pos] = '0';
533 tempStr.insert(pos + 1, "x%08");
534 printCallArgs.push_back(pArg);
535 pos += 3;
536 }
537 // for %f we need to cast float Values to doubles so that they print out correctly
538 else if ((tempStr[pos + 1] == 'f') && (pType->isFloatTy()))
539 {
540 printCallArgs.push_back(FP_EXT(pArg, Type::getDoubleTy(JM()->mContext)));
541 pos++;
542 }
543 else
544 {
545 printCallArgs.push_back(pArg);
546 }
547 }
548
549 // advance to the next arguement
550 v++;
551 pos = tempStr.find('%', ++pos);
552 }
553
554 // create global variable constant string
555 Constant *constString = ConstantDataArray::getString(JM()->mContext,tempStr,true);
556 GlobalVariable *gvPtr = new GlobalVariable(constString->getType(),true,GlobalValue::InternalLinkage,constString,"printStr");
557 JM()->mpCurrentModule->getGlobalList().push_back(gvPtr);
558
559 // get a pointer to the first character in the constant string array
560 std::vector<Constant*> geplist{C(0),C(0)};
561 Constant *strGEP = ConstantExpr::getGetElementPtr(nullptr, gvPtr,geplist,false);
562
563 // insert the pointer to the format string in the argument vector
564 printCallArgs[0] = strGEP;
565
566 // get pointer to CallPrint function and insert decl into the module if needed
567 std::vector<Type*> args;
568 args.push_back(PointerType::get(mInt8Ty,0));
569 FunctionType* callPrintTy = FunctionType::get(Type::getVoidTy(JM()->mContext),args,true);
570 Function *callPrintFn = cast<Function>(JM()->mpCurrentModule->getOrInsertFunction("CallPrint", callPrintTy));
571
572 // if we haven't yet added the symbol to the symbol table
573 if((sys::DynamicLibrary::SearchForAddressOfSymbol("CallPrint")) == nullptr)
574 {
575 sys::DynamicLibrary::AddSymbol("CallPrint", (void *)&CallPrint);
576 }
577
578 // insert a call to CallPrint
579 return CALLA(callPrintFn,printCallArgs);
580 }
581
582 //////////////////////////////////////////////////////////////////////////
583 /// @brief Wrapper around PRINT with initializer list.
584 CallInst* Builder::PRINT(const std::string &printStr)
585 {
586 return PRINT(printStr, {});
587 }
588
589 //////////////////////////////////////////////////////////////////////////
590 /// @brief Generate a masked gather operation in LLVM IR. If not
591 /// supported on the underlying platform, emulate it with loads
592 /// @param vSrc - SIMD wide value that will be loaded if mask is invalid
593 /// @param pBase - Int8* base VB address pointer value
594 /// @param vIndices - SIMD wide value of VB byte offsets
595 /// @param vMask - SIMD wide mask that controls whether to access memory or the src values
596 /// @param scale - value to scale indices by
597 Value *Builder::GATHERPS(Value *vSrc, Value *pBase, Value *vIndices, Value *vMask, uint8_t scale)
598 {
599 Value *vGather;
600
601 // use avx2 gather instruction if available
602 if(JM()->mArch.AVX2())
603 {
604 // force mask to <N x float>, required by vgather
605 Value *mask = BITCAST(VMASK(vMask), mSimdFP32Ty);
606
607 vGather = VGATHERPS(vSrc, pBase, vIndices, mask, C(scale));
608 }
609 else
610 {
611 Value* pStack = STACKSAVE();
612
613 // store vSrc on the stack. this way we can select between a valid load address and the vSrc address
614 Value* vSrcPtr = ALLOCA(vSrc->getType());
615 STORE(vSrc, vSrcPtr);
616
617 vGather = VUNDEF_F();
618 Value *vScaleVec = VIMMED1((uint32_t)scale);
619 Value *vOffsets = MUL(vIndices,vScaleVec);
620 for(uint32_t i = 0; i < mVWidth; ++i)
621 {
622 // single component byte index
623 Value *offset = VEXTRACT(vOffsets,C(i));
624 // byte pointer to component
625 Value *loadAddress = GEP(pBase,offset);
626 loadAddress = BITCAST(loadAddress,PointerType::get(mFP32Ty,0));
627 // pointer to the value to load if we're masking off a component
628 Value *maskLoadAddress = GEP(vSrcPtr,{C(0), C(i)});
629 Value *selMask = VEXTRACT(vMask,C(i));
630 // switch in a safe address to load if we're trying to access a vertex
631 Value *validAddress = SELECT(selMask, loadAddress, maskLoadAddress);
632 Value *val = LOAD(validAddress);
633 vGather = VINSERT(vGather,val,C(i));
634 }
635 STACKRESTORE(pStack);
636 }
637
638 return vGather;
639 }
640
641 #if USE_SIMD16_BUILDER
642 Value *Builder::GATHERPS2(Value *vSrc, Value *pBase, Value *vIndices, Value *vMask, uint8_t scale)
643 {
644 Value *vGather = VUNDEF2_F();
645
646 // use avx512 gather instruction if available
647 if (JM()->mArch.AVX512F())
648 {
649 // force mask to <N-bit Integer>, required by vgather2
650 Value *mask = BITCAST(vMask, mInt16Ty);
651
652 vGather = VGATHERPS2(vSrc, pBase, vIndices, mask, C((uint32_t)scale));
653 }
654 else
655 {
656 Value *src0 = EXTRACT2_F(vSrc, 0);
657 Value *src1 = EXTRACT2_F(vSrc, 1);
658
659 Value *indices0 = EXTRACT2_I(vIndices, 0);
660 Value *indices1 = EXTRACT2_I(vIndices, 1);
661
662 Value *mask0 = EXTRACT2_I(vMask, 0);
663 Value *mask1 = EXTRACT2_I(vMask, 1);
664
665 Value *gather0 = GATHERPS(src0, pBase, indices0, mask0, scale);
666 Value *gather1 = GATHERPS(src1, pBase, indices1, mask1, scale);
667
668 vGather = INSERT2_F(vGather, gather0, 0);
669 vGather = INSERT2_F(vGather, gather1, 1);
670 }
671
672 return vGather;
673 }
674
675 #endif
676 //////////////////////////////////////////////////////////////////////////
677 /// @brief Generate a masked gather operation in LLVM IR. If not
678 /// supported on the underlying platform, emulate it with loads
679 /// @param vSrc - SIMD wide value that will be loaded if mask is invalid
680 /// @param pBase - Int8* base VB address pointer value
681 /// @param vIndices - SIMD wide value of VB byte offsets
682 /// @param vMask - SIMD wide mask that controls whether to access memory or the src values
683 /// @param scale - value to scale indices by
684 Value *Builder::GATHERDD(Value* vSrc, Value* pBase, Value* vIndices, Value* vMask, uint8_t scale)
685 {
686 Value* vGather;
687
688 // use avx2 gather instruction if available
689 if(JM()->mArch.AVX2())
690 {
691 vGather = VGATHERDD(vSrc, pBase, vIndices, VMASK(vMask), C(scale));
692 }
693 else
694 {
695 Value* pStack = STACKSAVE();
696
697 // store vSrc on the stack. this way we can select between a valid load address and the vSrc address
698 Value* vSrcPtr = ALLOCA(vSrc->getType());
699 STORE(vSrc, vSrcPtr);
700
701 vGather = VUNDEF_I();
702 Value *vScaleVec = VIMMED1((uint32_t)scale);
703 Value *vOffsets = MUL(vIndices, vScaleVec);
704 for(uint32_t i = 0; i < mVWidth; ++i)
705 {
706 // single component byte index
707 Value *offset = VEXTRACT(vOffsets, C(i));
708 // byte pointer to component
709 Value *loadAddress = GEP(pBase, offset);
710 loadAddress = BITCAST(loadAddress, PointerType::get(mInt32Ty, 0));
711 // pointer to the value to load if we're masking off a component
712 Value *maskLoadAddress = GEP(vSrcPtr, {C(0), C(i)});
713 Value *selMask = VEXTRACT(vMask, C(i));
714 // switch in a safe address to load if we're trying to access a vertex
715 Value *validAddress = SELECT(selMask, loadAddress, maskLoadAddress);
716 Value *val = LOAD(validAddress, C(0));
717 vGather = VINSERT(vGather, val, C(i));
718 }
719
720 STACKRESTORE(pStack);
721 }
722 return vGather;
723 }
724
725 //////////////////////////////////////////////////////////////////////////
726 /// @brief Generate a masked gather operation in LLVM IR. If not
727 /// supported on the underlying platform, emulate it with loads
728 /// @param vSrc - SIMD wide value that will be loaded if mask is invalid
729 /// @param pBase - Int8* base VB address pointer value
730 /// @param vIndices - SIMD wide value of VB byte offsets
731 /// @param vMask - SIMD wide mask that controls whether to access memory or the src values
732 /// @param scale - value to scale indices by
733 Value *Builder::GATHERPD(Value* vSrc, Value* pBase, Value* vIndices, Value* vMask, uint8_t scale)
734 {
735 Value* vGather;
736
737 // use avx2 gather instruction if available
738 if(JM()->mArch.AVX2())
739 {
740 vMask = BITCAST(S_EXT(vMask, VectorType::get(mInt64Ty, mVWidth/2)), VectorType::get(mDoubleTy, mVWidth/2));
741 vGather = VGATHERPD(vSrc, pBase, vIndices, vMask, C(scale));
742 }
743 else
744 {
745 Value* pStack = STACKSAVE();
746
747 // store vSrc on the stack. this way we can select between a valid load address and the vSrc address
748 Value* vSrcPtr = ALLOCA(vSrc->getType());
749 STORE(vSrc, vSrcPtr);
750
751 vGather = UndefValue::get(VectorType::get(mDoubleTy, 4));
752 Value *vScaleVec = VECTOR_SPLAT(4, C((uint32_t)scale));
753 Value *vOffsets = MUL(vIndices,vScaleVec);
754 for(uint32_t i = 0; i < mVWidth/2; ++i)
755 {
756 // single component byte index
757 Value *offset = VEXTRACT(vOffsets,C(i));
758 // byte pointer to component
759 Value *loadAddress = GEP(pBase,offset);
760 loadAddress = BITCAST(loadAddress,PointerType::get(mDoubleTy,0));
761 // pointer to the value to load if we're masking off a component
762 Value *maskLoadAddress = GEP(vSrcPtr,{C(0), C(i)});
763 Value *selMask = VEXTRACT(vMask,C(i));
764 // switch in a safe address to load if we're trying to access a vertex
765 Value *validAddress = SELECT(selMask, loadAddress, maskLoadAddress);
766 Value *val = LOAD(validAddress);
767 vGather = VINSERT(vGather,val,C(i));
768 }
769 STACKRESTORE(pStack);
770 }
771 return vGather;
772 }
773
774 #if USE_SIMD16_BUILDER
775 //////////////////////////////////////////////////////////////////////////
776 /// @brief
777 Value *Builder::EXTRACT2_F(Value *a2, uint32_t imm)
778 {
779 const uint32_t i0 = (imm > 0) ? mVWidth : 0;
780
781 Value *result = VUNDEF_F();
782
783 for (uint32_t i = 0; i < mVWidth; i += 1)
784 {
785 #if 1
786 if (!a2->getType()->getScalarType()->isFloatTy())
787 {
788 a2 = BITCAST(a2, mSimd2FP32Ty);
789 }
790
791 #endif
792 Value *temp = VEXTRACT(a2, C(i0 + i));
793
794 result = VINSERT(result, temp, C(i));
795 }
796
797 return result;
798 }
799
800 Value *Builder::EXTRACT2_I(Value *a2, uint32_t imm)
801 {
802 return BITCAST(EXTRACT2_F(a2, imm), mSimdInt32Ty);
803 }
804
805 //////////////////////////////////////////////////////////////////////////
806 /// @brief
807 Value *Builder::INSERT2_F(Value *a2, Value *b, uint32_t imm)
808 {
809 const uint32_t i0 = (imm > 0) ? mVWidth : 0;
810
811 Value *result = BITCAST(a2, mSimd2FP32Ty);
812
813 for (uint32_t i = 0; i < mVWidth; i += 1)
814 {
815 #if 1
816 if (!b->getType()->getScalarType()->isFloatTy())
817 {
818 b = BITCAST(b, mSimdFP32Ty);
819 }
820
821 #endif
822 Value *temp = VEXTRACT(b, C(i));
823
824 result = VINSERT(result, temp, C(i0 + i));
825 }
826
827 return result;
828 }
829
830 Value *Builder::INSERT2_I(Value *a2, Value *b, uint32_t imm)
831 {
832 return BITCAST(INSERT2_F(a2, b, imm), mSimd2Int32Ty);
833 }
834
835 #endif
836 //////////////////////////////////////////////////////////////////////////
837 /// @brief convert x86 <N x float> mask to llvm <N x i1> mask
838 Value *Builder::MASK(Value *vmask)
839 {
840 Value *src = BITCAST(vmask, mSimdInt32Ty);
841 return ICMP_SLT(src, VIMMED1(0));
842 }
843
844 #if USE_SIMD16_BUILDER
845 Value *Builder::MASK2(Value *vmask)
846 {
847 Value *src = BITCAST(vmask, mSimd2Int32Ty);
848 return ICMP_SLT(src, VIMMED2_1(0));
849 }
850
851 #endif
852 //////////////////////////////////////////////////////////////////////////
853 /// @brief convert llvm <N x i1> mask to x86 <N x i32> mask
854 Value *Builder::VMASK(Value *mask)
855 {
856 return S_EXT(mask, mSimdInt32Ty);
857 }
858
859 #if USE_SIMD16_BUILDER
860 Value *Builder::VMASK2(Value *mask)
861 {
862 return S_EXT(mask, mSimd2Int32Ty);
863 }
864
865 #endif
866 //////////////////////////////////////////////////////////////////////////
867 /// @brief Generate a VPSHUFB operation in LLVM IR. If not
868 /// supported on the underlying platform, emulate it
869 /// @param a - 256bit SIMD(32x8bit) of 8bit integer values
870 /// @param b - 256bit SIMD(32x8bit) of 8bit integer mask values
871 /// Byte masks in lower 128 lane of b selects 8 bit values from lower
872 /// 128bits of a, and vice versa for the upper lanes. If the mask
873 /// value is negative, '0' is inserted.
874 Value *Builder::PSHUFB(Value* a, Value* b)
875 {
876 Value* res;
877 // use avx2 pshufb instruction if available
878 if(JM()->mArch.AVX2())
879 {
880 res = VPSHUFB(a, b);
881 }
882 else
883 {
884 Constant* cB = dyn_cast<Constant>(b);
885 // number of 8 bit elements in b
886 uint32_t numElms = cast<VectorType>(cB->getType())->getNumElements();
887 // output vector
888 Value* vShuf = UndefValue::get(VectorType::get(mInt8Ty, numElms));
889
890 // insert an 8 bit value from the high and low lanes of a per loop iteration
891 numElms /= 2;
892 for(uint32_t i = 0; i < numElms; i++)
893 {
894 ConstantInt* cLow128b = cast<ConstantInt>(cB->getAggregateElement(i));
895 ConstantInt* cHigh128b = cast<ConstantInt>(cB->getAggregateElement(i + numElms));
896
897 // extract values from constant mask
898 char valLow128bLane = (char)(cLow128b->getSExtValue());
899 char valHigh128bLane = (char)(cHigh128b->getSExtValue());
900
901 Value* insertValLow128b;
902 Value* insertValHigh128b;
903
904 // if the mask value is negative, insert a '0' in the respective output position
905 // otherwise, lookup the value at mask position (bits 3..0 of the respective mask byte) in a and insert in output vector
906 insertValLow128b = (valLow128bLane < 0) ? C((char)0) : VEXTRACT(a, C((valLow128bLane & 0xF)));
907 insertValHigh128b = (valHigh128bLane < 0) ? C((char)0) : VEXTRACT(a, C((valHigh128bLane & 0xF) + numElms));
908
909 vShuf = VINSERT(vShuf, insertValLow128b, i);
910 vShuf = VINSERT(vShuf, insertValHigh128b, (i + numElms));
911 }
912 res = vShuf;
913 }
914 return res;
915 }
916
917 //////////////////////////////////////////////////////////////////////////
918 /// @brief Generate a VPSHUFB operation (sign extend 8 8bit values to 32
919 /// bits)in LLVM IR. If not supported on the underlying platform, emulate it
920 /// @param a - 128bit SIMD lane(16x8bit) of 8bit integer values. Only
921 /// lower 8 values are used.
922 Value *Builder::PMOVSXBD(Value* a)
923 {
924 // VPMOVSXBD output type
925 Type* v8x32Ty = VectorType::get(mInt32Ty, 8);
926 // Extract 8 values from 128bit lane and sign extend
927 return S_EXT(VSHUFFLE(a, a, C<int>({0, 1, 2, 3, 4, 5, 6, 7})), v8x32Ty);
928 }
929
930 //////////////////////////////////////////////////////////////////////////
931 /// @brief Generate a VPSHUFB operation (sign extend 8 16bit values to 32
932 /// bits)in LLVM IR. If not supported on the underlying platform, emulate it
933 /// @param a - 128bit SIMD lane(8x16bit) of 16bit integer values.
934 Value *Builder::PMOVSXWD(Value* a)
935 {
936 // VPMOVSXWD output type
937 Type* v8x32Ty = VectorType::get(mInt32Ty, 8);
938 // Extract 8 values from 128bit lane and sign extend
939 return S_EXT(VSHUFFLE(a, a, C<int>({0, 1, 2, 3, 4, 5, 6, 7})), v8x32Ty);
940 }
941
942 //////////////////////////////////////////////////////////////////////////
943 /// @brief Generate a VPERMD operation (shuffle 32 bit integer values
944 /// across 128 bit lanes) in LLVM IR. If not supported on the underlying
945 /// platform, emulate it
946 /// @param a - 256bit SIMD lane(8x32bit) of integer values.
947 /// @param idx - 256bit SIMD lane(8x32bit) of 3 bit lane index values
948 Value *Builder::PERMD(Value* a, Value* idx)
949 {
950 Value* res;
951 // use avx2 permute instruction if available
952 if(JM()->mArch.AVX2())
953 {
954 res = VPERMD(a, idx);
955 }
956 else
957 {
958 if (isa<Constant>(idx))
959 {
960 res = VSHUFFLE(a, a, idx);
961 }
962 else
963 {
964 res = VUNDEF_I();
965 for (uint32_t l = 0; l < JM()->mVWidth; ++l)
966 {
967 Value* pIndex = VEXTRACT(idx, C(l));
968 Value* pVal = VEXTRACT(a, pIndex);
969 res = VINSERT(res, pVal, C(l));
970 }
971 }
972 }
973 return res;
974 }
975
976 //////////////////////////////////////////////////////////////////////////
977 /// @brief Generate a VPERMPS operation (shuffle 32 bit float values
978 /// across 128 bit lanes) in LLVM IR. If not supported on the underlying
979 /// platform, emulate it
980 /// @param a - 256bit SIMD lane(8x32bit) of float values.
981 /// @param idx - 256bit SIMD lane(8x32bit) of 3 bit lane index values
982 Value *Builder::PERMPS(Value* a, Value* idx)
983 {
984 Value* res;
985 // use avx2 permute instruction if available
986 if (JM()->mArch.AVX2())
987 {
988 // llvm 3.6.0 swapped the order of the args to vpermd
989 res = VPERMPS(idx, a);
990 }
991 else
992 {
993 if (isa<Constant>(idx))
994 {
995 res = VSHUFFLE(a, a, idx);
996 }
997 else
998 {
999 res = VUNDEF_F();
1000 for (uint32_t l = 0; l < JM()->mVWidth; ++l)
1001 {
1002 Value* pIndex = VEXTRACT(idx, C(l));
1003 Value* pVal = VEXTRACT(a, pIndex);
1004 res = VINSERT(res, pVal, C(l));
1005 }
1006 }
1007 }
1008
1009 return res;
1010 }
1011
1012 //////////////////////////////////////////////////////////////////////////
1013 /// @brief Generate a VCVTPH2PS operation (float16->float32 conversion)
1014 /// in LLVM IR. If not supported on the underlying platform, emulate it
1015 /// @param a - 128bit SIMD lane(8x16bit) of float16 in int16 format.
1016 Value *Builder::CVTPH2PS(Value* a)
1017 {
1018 if (JM()->mArch.F16C())
1019 {
1020 return VCVTPH2PS(a);
1021 }
1022 else
1023 {
1024 FunctionType* pFuncTy = FunctionType::get(mFP32Ty, mInt16Ty);
1025 Function* pCvtPh2Ps = cast<Function>(JM()->mpCurrentModule->getOrInsertFunction("ConvertFloat16ToFloat32", pFuncTy));
1026
1027 if (sys::DynamicLibrary::SearchForAddressOfSymbol("ConvertFloat16ToFloat32") == nullptr)
1028 {
1029 sys::DynamicLibrary::AddSymbol("ConvertFloat16ToFloat32", (void *)&ConvertFloat16ToFloat32);
1030 }
1031
1032 Value* pResult = UndefValue::get(mSimdFP32Ty);
1033 for (uint32_t i = 0; i < mVWidth; ++i)
1034 {
1035 Value* pSrc = VEXTRACT(a, C(i));
1036 Value* pConv = CALL(pCvtPh2Ps, std::initializer_list<Value*>{pSrc});
1037 pResult = VINSERT(pResult, pConv, C(i));
1038 }
1039
1040 return pResult;
1041 }
1042 }
1043
1044 //////////////////////////////////////////////////////////////////////////
1045 /// @brief Generate a VCVTPS2PH operation (float32->float16 conversion)
1046 /// in LLVM IR. If not supported on the underlying platform, emulate it
1047 /// @param a - 128bit SIMD lane(8x16bit) of float16 in int16 format.
1048 Value *Builder::CVTPS2PH(Value* a, Value* rounding)
1049 {
1050 if (JM()->mArch.F16C())
1051 {
1052 return VCVTPS2PH(a, rounding);
1053 }
1054 else
1055 {
1056 // call scalar C function for now
1057 FunctionType* pFuncTy = FunctionType::get(mInt16Ty, mFP32Ty);
1058 Function* pCvtPs2Ph = cast<Function>(JM()->mpCurrentModule->getOrInsertFunction("ConvertFloat32ToFloat16", pFuncTy));
1059
1060 if (sys::DynamicLibrary::SearchForAddressOfSymbol("ConvertFloat32ToFloat16") == nullptr)
1061 {
1062 sys::DynamicLibrary::AddSymbol("ConvertFloat32ToFloat16", (void *)&ConvertFloat32ToFloat16);
1063 }
1064
1065 Value* pResult = UndefValue::get(mSimdInt16Ty);
1066 for (uint32_t i = 0; i < mVWidth; ++i)
1067 {
1068 Value* pSrc = VEXTRACT(a, C(i));
1069 Value* pConv = CALL(pCvtPs2Ph, std::initializer_list<Value*>{pSrc});
1070 pResult = VINSERT(pResult, pConv, C(i));
1071 }
1072
1073 return pResult;
1074 }
1075 }
1076
1077 Value *Builder::PMAXSD(Value* a, Value* b)
1078 {
1079 Value* cmp = ICMP_SGT(a, b);
1080 return SELECT(cmp, a, b);
1081 }
1082
1083 Value *Builder::PMINSD(Value* a, Value* b)
1084 {
1085 Value* cmp = ICMP_SLT(a, b);
1086 return SELECT(cmp, a, b);
1087 }
1088
1089 void Builder::Gather4(const SWR_FORMAT format, Value* pSrcBase, Value* byteOffsets,
1090 Value* mask, Value* vGatherComponents[], bool bPackedOutput)
1091 {
1092 const SWR_FORMAT_INFO &info = GetFormatInfo(format);
1093 if(info.type[0] == SWR_TYPE_FLOAT && info.bpc[0] == 32)
1094 {
1095 GATHER4PS(info, pSrcBase, byteOffsets, mask, vGatherComponents, bPackedOutput);
1096 }
1097 else
1098 {
1099 GATHER4DD(info, pSrcBase, byteOffsets, mask, vGatherComponents, bPackedOutput);
1100 }
1101 }
1102
1103 void Builder::GATHER4PS(const SWR_FORMAT_INFO &info, Value* pSrcBase, Value* byteOffsets,
1104 Value* vMask, Value* vGatherComponents[], bool bPackedOutput)
1105 {
1106 switch(info.bpp / info.numComps)
1107 {
1108 case 16:
1109 {
1110 Value* vGatherResult[2];
1111
1112 // TODO: vGatherMaskedVal
1113 Value* vGatherMaskedVal = VIMMED1((float)0);
1114
1115 // always have at least one component out of x or y to fetch
1116
1117 vGatherResult[0] = GATHERPS(vGatherMaskedVal, pSrcBase, byteOffsets, vMask);
1118 // e.g. result of first 8x32bit integer gather for 16bit components
1119 // 256i - 0 1 2 3 4 5 6 7
1120 // xyxy xyxy xyxy xyxy xyxy xyxy xyxy xyxy
1121 //
1122
1123 // if we have at least one component out of x or y to fetch
1124 if(info.numComps > 2)
1125 {
1126 // offset base to the next components(zw) in the vertex to gather
1127 pSrcBase = GEP(pSrcBase, C((char)4));
1128
1129 vGatherResult[1] = GATHERPS(vGatherMaskedVal, pSrcBase, byteOffsets, vMask);
1130 // e.g. result of second 8x32bit integer gather for 16bit components
1131 // 256i - 0 1 2 3 4 5 6 7
1132 // zwzw zwzw zwzw zwzw zwzw zwzw zwzw zwzw
1133 //
1134 }
1135 else
1136 {
1137 vGatherResult[1] = vGatherMaskedVal;
1138 }
1139
1140 // Shuffle gathered components into place, each row is a component
1141 Shuffle16bpcGather4(info, vGatherResult, vGatherComponents, bPackedOutput);
1142 }
1143 break;
1144 case 32:
1145 {
1146 // apply defaults
1147 for (uint32_t i = 0; i < 4; ++i)
1148 {
1149 vGatherComponents[i] = VIMMED1(*(float*)&info.defaults[i]);
1150 }
1151
1152 for(uint32_t i = 0; i < info.numComps; i++)
1153 {
1154 uint32_t swizzleIndex = info.swizzle[i];
1155
1156 // Gather a SIMD of components
1157 vGatherComponents[swizzleIndex] = GATHERPS(vGatherComponents[swizzleIndex], pSrcBase, byteOffsets, vMask);
1158
1159 // offset base to the next component to gather
1160 pSrcBase = GEP(pSrcBase, C((char)4));
1161 }
1162 }
1163 break;
1164 default:
1165 SWR_INVALID("Invalid float format");
1166 break;
1167 }
1168 }
1169
1170 void Builder::GATHER4DD(const SWR_FORMAT_INFO &info, Value* pSrcBase, Value* byteOffsets,
1171 Value* vMask, Value* vGatherComponents[], bool bPackedOutput)
1172 {
1173 switch (info.bpp / info.numComps)
1174 {
1175 case 8:
1176 {
1177 Value* vGatherMaskedVal = VIMMED1((int32_t)0);
1178 Value* vGatherResult = GATHERDD(vGatherMaskedVal, pSrcBase, byteOffsets, vMask);
1179 // e.g. result of an 8x32bit integer gather for 8bit components
1180 // 256i - 0 1 2 3 4 5 6 7
1181 // xyzw xyzw xyzw xyzw xyzw xyzw xyzw xyzw
1182
1183 Shuffle8bpcGather4(info, vGatherResult, vGatherComponents, bPackedOutput);
1184 }
1185 break;
1186 case 16:
1187 {
1188 Value* vGatherResult[2];
1189
1190 // TODO: vGatherMaskedVal
1191 Value* vGatherMaskedVal = VIMMED1((int32_t)0);
1192
1193 // always have at least one component out of x or y to fetch
1194
1195 vGatherResult[0] = GATHERDD(vGatherMaskedVal, pSrcBase, byteOffsets, vMask);
1196 // e.g. result of first 8x32bit integer gather for 16bit components
1197 // 256i - 0 1 2 3 4 5 6 7
1198 // xyxy xyxy xyxy xyxy xyxy xyxy xyxy xyxy
1199 //
1200
1201 // if we have at least one component out of x or y to fetch
1202 if(info.numComps > 2)
1203 {
1204 // offset base to the next components(zw) in the vertex to gather
1205 pSrcBase = GEP(pSrcBase, C((char)4));
1206
1207 vGatherResult[1] = GATHERDD(vGatherMaskedVal, pSrcBase, byteOffsets, vMask);
1208 // e.g. result of second 8x32bit integer gather for 16bit components
1209 // 256i - 0 1 2 3 4 5 6 7
1210 // zwzw zwzw zwzw zwzw zwzw zwzw zwzw zwzw
1211 //
1212 }
1213 else
1214 {
1215 vGatherResult[1] = vGatherMaskedVal;
1216 }
1217
1218 // Shuffle gathered components into place, each row is a component
1219 Shuffle16bpcGather4(info, vGatherResult, vGatherComponents, bPackedOutput);
1220
1221 }
1222 break;
1223 case 32:
1224 {
1225 // apply defaults
1226 for (uint32_t i = 0; i < 4; ++i)
1227 {
1228 vGatherComponents[i] = VIMMED1((int)info.defaults[i]);
1229 }
1230
1231 for(uint32_t i = 0; i < info.numComps; i++)
1232 {
1233 uint32_t swizzleIndex = info.swizzle[i];
1234
1235 // Gather a SIMD of components
1236 vGatherComponents[swizzleIndex] = GATHERDD(vGatherComponents[swizzleIndex], pSrcBase, byteOffsets, vMask);
1237
1238 // offset base to the next component to gather
1239 pSrcBase = GEP(pSrcBase, C((char)4));
1240 }
1241 }
1242 break;
1243 default:
1244 SWR_INVALID("unsupported format");
1245 break;
1246 }
1247 }
1248
1249 void Builder::Shuffle16bpcGather4(const SWR_FORMAT_INFO &info, Value* vGatherInput[2], Value* vGatherOutput[4], bool bPackedOutput)
1250 {
1251 // cast types
1252 Type* vGatherTy = VectorType::get(IntegerType::getInt32Ty(JM()->mContext), mVWidth);
1253 Type* v32x8Ty = VectorType::get(mInt8Ty, mVWidth * 4); // vwidth is units of 32 bits
1254
1255 // input could either be float or int vector; do shuffle work in int
1256 vGatherInput[0] = BITCAST(vGatherInput[0], mSimdInt32Ty);
1257 vGatherInput[1] = BITCAST(vGatherInput[1], mSimdInt32Ty);
1258
1259 if(bPackedOutput)
1260 {
1261 Type* v128bitTy = VectorType::get(IntegerType::getIntNTy(JM()->mContext, 128), mVWidth / 4); // vwidth is units of 32 bits
1262
1263 // shuffle mask
1264 Value* vConstMask = C<char>({0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15,
1265 0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15});
1266 Value* vShufResult = BITCAST(PSHUFB(BITCAST(vGatherInput[0], v32x8Ty), vConstMask), vGatherTy);
1267 // after pshufb: group components together in each 128bit lane
1268 // 256i - 0 1 2 3 4 5 6 7
1269 // xxxx xxxx yyyy yyyy xxxx xxxx yyyy yyyy
1270
1271 Value* vi128XY = BITCAST(PERMD(vShufResult, C<int32_t>({0, 1, 4, 5, 2, 3, 6, 7})), v128bitTy);
1272 // after PERMD: move and pack xy components into each 128bit lane
1273 // 256i - 0 1 2 3 4 5 6 7
1274 // xxxx xxxx xxxx xxxx yyyy yyyy yyyy yyyy
1275
1276 // do the same for zw components
1277 Value* vi128ZW = nullptr;
1278 if(info.numComps > 2)
1279 {
1280 Value* vShufResult = BITCAST(PSHUFB(BITCAST(vGatherInput[1], v32x8Ty), vConstMask), vGatherTy);
1281 vi128ZW = BITCAST(PERMD(vShufResult, C<int32_t>({0, 1, 4, 5, 2, 3, 6, 7})), v128bitTy);
1282 }
1283
1284 for(uint32_t i = 0; i < 4; i++)
1285 {
1286 uint32_t swizzleIndex = info.swizzle[i];
1287 // todo: fixed for packed
1288 Value* vGatherMaskedVal = VIMMED1((int32_t)(info.defaults[i]));
1289 if(i >= info.numComps)
1290 {
1291 // set the default component val
1292 vGatherOutput[swizzleIndex] = vGatherMaskedVal;
1293 continue;
1294 }
1295
1296 // if x or z, extract 128bits from lane 0, else for y or w, extract from lane 1
1297 uint32_t lane = ((i == 0) || (i == 2)) ? 0 : 1;
1298 // if x or y, use vi128XY permute result, else use vi128ZW
1299 Value* selectedPermute = (i < 2) ? vi128XY : vi128ZW;
1300
1301 // extract packed component 128 bit lanes
1302 vGatherOutput[swizzleIndex] = VEXTRACT(selectedPermute, C(lane));
1303 }
1304
1305 }
1306 else
1307 {
1308 // pshufb masks for each component
1309 Value* vConstMask[2];
1310 // x/z shuffle mask
1311 vConstMask[0] = C<char>({0, 1, -1, -1, 4, 5, -1, -1, 8, 9, -1, -1, 12, 13, -1, -1,
1312 0, 1, -1, -1, 4, 5, -1, -1, 8, 9, -1, -1, 12, 13, -1, -1, });
1313
1314 // y/w shuffle mask
1315 vConstMask[1] = C<char>({2, 3, -1, -1, 6, 7, -1, -1, 10, 11, -1, -1, 14, 15, -1, -1,
1316 2, 3, -1, -1, 6, 7, -1, -1, 10, 11, -1, -1, 14, 15, -1, -1});
1317
1318
1319 // shuffle enabled components into lower word of each 32bit lane, 0 extending to 32 bits
1320 // apply defaults
1321 for (uint32_t i = 0; i < 4; ++i)
1322 {
1323 vGatherOutput[i] = VIMMED1((int32_t)info.defaults[i]);
1324 }
1325
1326 for(uint32_t i = 0; i < info.numComps; i++)
1327 {
1328 uint32_t swizzleIndex = info.swizzle[i];
1329
1330 // select correct constMask for x/z or y/w pshufb
1331 uint32_t selectedMask = ((i == 0) || (i == 2)) ? 0 : 1;
1332 // if x or y, use vi128XY permute result, else use vi128ZW
1333 uint32_t selectedGather = (i < 2) ? 0 : 1;
1334
1335 vGatherOutput[swizzleIndex] = BITCAST(PSHUFB(BITCAST(vGatherInput[selectedGather], v32x8Ty), vConstMask[selectedMask]), vGatherTy);
1336 // after pshufb mask for x channel; z uses the same shuffle from the second gather
1337 // 256i - 0 1 2 3 4 5 6 7
1338 // xx00 xx00 xx00 xx00 xx00 xx00 xx00 xx00
1339 }
1340 }
1341 }
1342
1343 void Builder::Shuffle8bpcGather4(const SWR_FORMAT_INFO &info, Value* vGatherInput, Value* vGatherOutput[], bool bPackedOutput)
1344 {
1345 // cast types
1346 Type* vGatherTy = VectorType::get(IntegerType::getInt32Ty(JM()->mContext), mVWidth);
1347 Type* v32x8Ty = VectorType::get(mInt8Ty, mVWidth * 4 ); // vwidth is units of 32 bits
1348
1349 if(bPackedOutput)
1350 {
1351 Type* v128Ty = VectorType::get(IntegerType::getIntNTy(JM()->mContext, 128), mVWidth / 4); // vwidth is units of 32 bits
1352 // shuffle mask
1353 Value* vConstMask = C<char>({0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15,
1354 0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15});
1355 Value* vShufResult = BITCAST(PSHUFB(BITCAST(vGatherInput, v32x8Ty), vConstMask), vGatherTy);
1356 // after pshufb: group components together in each 128bit lane
1357 // 256i - 0 1 2 3 4 5 6 7
1358 // xxxx yyyy zzzz wwww xxxx yyyy zzzz wwww
1359
1360 Value* vi128XY = BITCAST(PERMD(vShufResult, C<int32_t>({0, 4, 0, 0, 1, 5, 0, 0})), v128Ty);
1361 // after PERMD: move and pack xy and zw components in low 64 bits of each 128bit lane
1362 // 256i - 0 1 2 3 4 5 6 7
1363 // xxxx xxxx dcdc dcdc yyyy yyyy dcdc dcdc (dc - don't care)
1364
1365 // do the same for zw components
1366 Value* vi128ZW = nullptr;
1367 if(info.numComps > 2)
1368 {
1369 vi128ZW = BITCAST(PERMD(vShufResult, C<int32_t>({2, 6, 0, 0, 3, 7, 0, 0})), v128Ty);
1370 }
1371
1372 // sign extend all enabled components. If we have a fill vVertexElements, output to current simdvertex
1373 for(uint32_t i = 0; i < 4; i++)
1374 {
1375 uint32_t swizzleIndex = info.swizzle[i];
1376 // todo: fix for packed
1377 Value* vGatherMaskedVal = VIMMED1((int32_t)(info.defaults[i]));
1378 if(i >= info.numComps)
1379 {
1380 // set the default component val
1381 vGatherOutput[swizzleIndex] = vGatherMaskedVal;
1382 continue;
1383 }
1384
1385 // if x or z, extract 128bits from lane 0, else for y or w, extract from lane 1
1386 uint32_t lane = ((i == 0) || (i == 2)) ? 0 : 1;
1387 // if x or y, use vi128XY permute result, else use vi128ZW
1388 Value* selectedPermute = (i < 2) ? vi128XY : vi128ZW;
1389
1390 // sign extend
1391 vGatherOutput[swizzleIndex] = VEXTRACT(selectedPermute, C(lane));
1392 }
1393 }
1394 // else zero extend
1395 else{
1396 // shuffle enabled components into lower byte of each 32bit lane, 0 extending to 32 bits
1397 // apply defaults
1398 for (uint32_t i = 0; i < 4; ++i)
1399 {
1400 vGatherOutput[i] = VIMMED1((int32_t)info.defaults[i]);
1401 }
1402
1403 for(uint32_t i = 0; i < info.numComps; i++){
1404 uint32_t swizzleIndex = info.swizzle[i];
1405
1406 // pshufb masks for each component
1407 Value* vConstMask;
1408 switch(i)
1409 {
1410 case 0:
1411 // x shuffle mask
1412 vConstMask = C<char>({0, -1, -1, -1, 4, -1, -1, -1, 8, -1, -1, -1, 12, -1, -1, -1,
1413 0, -1, -1, -1, 4, -1, -1, -1, 8, -1, -1, -1, 12, -1, -1, -1});
1414 break;
1415 case 1:
1416 // y shuffle mask
1417 vConstMask = C<char>({1, -1, -1, -1, 5, -1, -1, -1, 9, -1, -1, -1, 13, -1, -1, -1,
1418 1, -1, -1, -1, 5, -1, -1, -1, 9, -1, -1, -1, 13, -1, -1, -1});
1419 break;
1420 case 2:
1421 // z shuffle mask
1422 vConstMask = C<char>({2, -1, -1, -1, 6, -1, -1, -1, 10, -1, -1, -1, 14, -1, -1, -1,
1423 2, -1, -1, -1, 6, -1, -1, -1, 10, -1, -1, -1, 14, -1, -1, -1});
1424 break;
1425 case 3:
1426 // w shuffle mask
1427 vConstMask = C<char>({3, -1, -1, -1, 7, -1, -1, -1, 11, -1, -1, -1, 15, -1, -1, -1,
1428 3, -1, -1, -1, 7, -1, -1, -1, 11, -1, -1, -1, 15, -1, -1, -1});
1429 break;
1430 default:
1431 vConstMask = nullptr;
1432 break;
1433 }
1434
1435 vGatherOutput[swizzleIndex] = BITCAST(PSHUFB(BITCAST(vGatherInput, v32x8Ty), vConstMask), vGatherTy);
1436 // after pshufb for x channel
1437 // 256i - 0 1 2 3 4 5 6 7
1438 // x000 x000 x000 x000 x000 x000 x000 x000
1439 }
1440 }
1441 }
1442
1443 // Helper function to create alloca in entry block of function
1444 Value* Builder::CreateEntryAlloca(Function* pFunc, Type* pType)
1445 {
1446 auto saveIP = IRB()->saveIP();
1447 IRB()->SetInsertPoint(&pFunc->getEntryBlock(),
1448 pFunc->getEntryBlock().begin());
1449 Value* pAlloca = ALLOCA(pType);
1450 if (saveIP.isSet()) IRB()->restoreIP(saveIP);
1451 return pAlloca;
1452 }
1453
1454 Value* Builder::CreateEntryAlloca(Function* pFunc, Type* pType, Value* pArraySize)
1455 {
1456 auto saveIP = IRB()->saveIP();
1457 IRB()->SetInsertPoint(&pFunc->getEntryBlock(),
1458 pFunc->getEntryBlock().begin());
1459 Value* pAlloca = ALLOCA(pType, pArraySize);
1460 if (saveIP.isSet()) IRB()->restoreIP(saveIP);
1461 return pAlloca;
1462 }
1463
1464 //////////////////////////////////////////////////////////////////////////
1465 /// @brief emulates a scatter operation.
1466 /// @param pDst - pointer to destination
1467 /// @param vSrc - vector of src data to scatter
1468 /// @param vOffsets - vector of byte offsets from pDst
1469 /// @param vMask - mask of valid lanes
1470 void Builder::SCATTERPS(Value* pDst, Value* vSrc, Value* vOffsets, Value* vMask)
1471 {
1472 /* Scatter algorithm
1473
1474 while(Index = BitScanForward(mask))
1475 srcElem = srcVector[Index]
1476 offsetElem = offsetVector[Index]
1477 *(pDst + offsetElem) = srcElem
1478 Update mask (&= ~(1<<Index)
1479
1480 */
1481
1482 BasicBlock* pCurBB = IRB()->GetInsertBlock();
1483 Function* pFunc = pCurBB->getParent();
1484 Type* pSrcTy = vSrc->getType()->getVectorElementType();
1485
1486 // Store vectors on stack
1487 if (pScatterStackSrc == nullptr)
1488 {
1489 // Save off stack allocations and reuse per scatter. Significantly reduces stack
1490 // requirements for shaders with a lot of scatters.
1491 pScatterStackSrc = CreateEntryAlloca(pFunc, mSimdInt64Ty);
1492 pScatterStackOffsets = CreateEntryAlloca(pFunc, mSimdInt32Ty);
1493 }
1494
1495 Value* pSrcArrayPtr = BITCAST(pScatterStackSrc, PointerType::get(vSrc->getType(), 0));
1496 Value* pOffsetsArrayPtr = pScatterStackOffsets;
1497 STORE(vSrc, pSrcArrayPtr);
1498 STORE(vOffsets, pOffsetsArrayPtr);
1499
1500 // Cast to pointers for random access
1501 pSrcArrayPtr = POINTER_CAST(pSrcArrayPtr, PointerType::get(pSrcTy, 0));
1502 pOffsetsArrayPtr = POINTER_CAST(pOffsetsArrayPtr, PointerType::get(mInt32Ty, 0));
1503
1504 Value* pMask = VMOVMSKPS(BITCAST(vMask, mSimdFP32Ty));
1505
1506 // Get cttz function
1507 Function* pfnCttz = Intrinsic::getDeclaration(mpJitMgr->mpCurrentModule, Intrinsic::cttz, { mInt32Ty });
1508
1509 // Setup loop basic block
1510 BasicBlock* pLoop = BasicBlock::Create(mpJitMgr->mContext, "Scatter Loop", pFunc);
1511
1512 // compute first set bit
1513 Value* pIndex = CALL(pfnCttz, { pMask, C(false) });
1514
1515 Value* pIsUndef = ICMP_EQ(pIndex, C(32));
1516
1517 // Split current block
1518 BasicBlock* pPostLoop = pCurBB->splitBasicBlock(cast<Instruction>(pIsUndef)->getNextNode());
1519
1520 // Remove unconditional jump created by splitBasicBlock
1521 pCurBB->getTerminator()->eraseFromParent();
1522
1523 // Add terminator to end of original block
1524 IRB()->SetInsertPoint(pCurBB);
1525
1526 // Add conditional branch
1527 COND_BR(pIsUndef, pPostLoop, pLoop);
1528
1529 // Add loop basic block contents
1530 IRB()->SetInsertPoint(pLoop);
1531 PHINode* pIndexPhi = PHI(mInt32Ty, 2);
1532 PHINode* pMaskPhi = PHI(mInt32Ty, 2);
1533
1534 pIndexPhi->addIncoming(pIndex, pCurBB);
1535 pMaskPhi->addIncoming(pMask, pCurBB);
1536
1537 // Extract elements for this index
1538 Value* pSrcElem = LOADV(pSrcArrayPtr, { pIndexPhi });
1539 Value* pOffsetElem = LOADV(pOffsetsArrayPtr, { pIndexPhi });
1540
1541 // GEP to this offset in dst
1542 Value* pCurDst = GEP(pDst, pOffsetElem);
1543 pCurDst = POINTER_CAST(pCurDst, PointerType::get(pSrcTy, 0));
1544 STORE(pSrcElem, pCurDst);
1545
1546 // Update the mask
1547 Value* pNewMask = AND(pMaskPhi, NOT(SHL(C(1), pIndexPhi)));
1548
1549 // Terminator
1550 Value* pNewIndex = CALL(pfnCttz, { pNewMask, C(false) });
1551
1552 pIsUndef = ICMP_EQ(pNewIndex, C(32));
1553 COND_BR(pIsUndef, pPostLoop, pLoop);
1554
1555 // Update phi edges
1556 pIndexPhi->addIncoming(pNewIndex, pLoop);
1557 pMaskPhi->addIncoming(pNewMask, pLoop);
1558
1559 // Move builder to beginning of post loop
1560 IRB()->SetInsertPoint(pPostLoop, pPostLoop->begin());
1561 }
1562
1563 Value* Builder::VABSPS(Value* a)
1564 {
1565 Value* asInt = BITCAST(a, mSimdInt32Ty);
1566 Value* result = BITCAST(AND(asInt, VIMMED1(0x7fffffff)), mSimdFP32Ty);
1567 return result;
1568 }
1569
1570 Value *Builder::ICLAMP(Value* src, Value* low, Value* high)
1571 {
1572 Value *lowCmp = ICMP_SLT(src, low);
1573 Value *ret = SELECT(lowCmp, low, src);
1574
1575 Value *highCmp = ICMP_SGT(ret, high);
1576 ret = SELECT(highCmp, high, ret);
1577
1578 return ret;
1579 }
1580
1581 Value *Builder::FCLAMP(Value* src, Value* low, Value* high)
1582 {
1583 Value *lowCmp = FCMP_OLT(src, low);
1584 Value *ret = SELECT(lowCmp, low, src);
1585
1586 Value *highCmp = FCMP_OGT(ret, high);
1587 ret = SELECT(highCmp, high, ret);
1588
1589 return ret;
1590 }
1591
1592 Value *Builder::FCLAMP(Value* src, float low, float high)
1593 {
1594 Value* result = VMAXPS(src, VIMMED1(low));
1595 result = VMINPS(result, VIMMED1(high));
1596
1597 return result;
1598 }
1599
1600 //////////////////////////////////////////////////////////////////////////
1601 /// @brief save/restore stack, providing ability to push/pop the stack and
1602 /// reduce overall stack requirements for temporary stack use
1603 Value* Builder::STACKSAVE()
1604 {
1605 Function* pfnStackSave = Intrinsic::getDeclaration(JM()->mpCurrentModule, Intrinsic::stacksave);
1606 return CALLA(pfnStackSave);
1607 }
1608
1609 void Builder::STACKRESTORE(Value* pSaved)
1610 {
1611 Function* pfnStackRestore = Intrinsic::getDeclaration(JM()->mpCurrentModule, Intrinsic::stackrestore);
1612 CALL(pfnStackRestore, std::initializer_list<Value*>{pSaved});
1613 }
1614
1615 Value *Builder::FMADDPS(Value* a, Value* b, Value* c)
1616 {
1617 Value* vOut;
1618 // use FMADs if available
1619 if(JM()->mArch.AVX2())
1620 {
1621 vOut = VFMADDPS(a, b, c);
1622 }
1623 else
1624 {
1625 vOut = FADD(FMUL(a, b), c);
1626 }
1627 return vOut;
1628 }
1629
1630 Value* Builder::POPCNT(Value* a)
1631 {
1632 Function* pCtPop = Intrinsic::getDeclaration(JM()->mpCurrentModule, Intrinsic::ctpop, { a->getType() });
1633 return CALL(pCtPop, std::initializer_list<Value*>{a});
1634 }
1635
1636 //////////////////////////////////////////////////////////////////////////
1637 /// @brief C functions called by LLVM IR
1638 //////////////////////////////////////////////////////////////////////////
1639
1640 //////////////////////////////////////////////////////////////////////////
1641 /// @brief called in JIT code, inserted by PRINT
1642 /// output to both stdout and visual studio debug console
1643 void __cdecl CallPrint(const char* fmt, ...)
1644 {
1645 va_list args;
1646 va_start(args, fmt);
1647 vprintf(fmt, args);
1648
1649 #if defined( _WIN32 )
1650 char strBuf[1024];
1651 vsnprintf_s(strBuf, _TRUNCATE, fmt, args);
1652 OutputDebugStringA(strBuf);
1653 #endif
1654
1655 va_end(args);
1656 }
1657
1658 Value *Builder::VEXTRACTI128(Value* a, Constant* imm8)
1659 {
1660 bool flag = !imm8->isZeroValue();
1661 SmallVector<Constant*,8> idx;
1662 for (unsigned i = 0; i < mVWidth / 2; i++) {
1663 idx.push_back(C(flag ? i + mVWidth / 2 : i));
1664 }
1665 return VSHUFFLE(a, VUNDEF_I(), ConstantVector::get(idx));
1666 }
1667
1668 Value *Builder::VINSERTI128(Value* a, Value* b, Constant* imm8)
1669 {
1670 bool flag = !imm8->isZeroValue();
1671 SmallVector<Constant*,8> idx;
1672 for (unsigned i = 0; i < mVWidth; i++) {
1673 idx.push_back(C(i));
1674 }
1675 Value *inter = VSHUFFLE(b, VUNDEF_I(), ConstantVector::get(idx));
1676
1677 SmallVector<Constant*,8> idx2;
1678 for (unsigned i = 0; i < mVWidth / 2; i++) {
1679 idx2.push_back(C(flag ? i : i + mVWidth));
1680 }
1681 for (unsigned i = mVWidth / 2; i < mVWidth; i++) {
1682 idx2.push_back(C(flag ? i + mVWidth / 2 : i));
1683 }
1684 return VSHUFFLE(a, inter, ConstantVector::get(idx2));
1685 }
1686
1687 // rdtsc buckets macros
1688 void Builder::RDTSC_START(Value* pBucketMgr, Value* pId)
1689 {
1690 // @todo due to an issue with thread local storage propagation in llvm, we can only safely call into
1691 // buckets framework when single threaded
1692 if (KNOB_SINGLE_THREADED)
1693 {
1694 std::vector<Type*> args{
1695 PointerType::get(mInt32Ty, 0), // pBucketMgr
1696 mInt32Ty // id
1697 };
1698
1699 FunctionType* pFuncTy = FunctionType::get(Type::getVoidTy(JM()->mContext), args, false);
1700 Function* pFunc = cast<Function>(JM()->mpCurrentModule->getOrInsertFunction("BucketManager_StartBucket", pFuncTy));
1701 if (sys::DynamicLibrary::SearchForAddressOfSymbol("BucketManager_StartBucket") == nullptr)
1702 {
1703 sys::DynamicLibrary::AddSymbol("BucketManager_StartBucket", (void*)&BucketManager_StartBucket);
1704 }
1705
1706 CALL(pFunc, { pBucketMgr, pId });
1707 }
1708 }
1709
1710 void Builder::RDTSC_STOP(Value* pBucketMgr, Value* pId)
1711 {
1712 // @todo due to an issue with thread local storage propagation in llvm, we can only safely call into
1713 // buckets framework when single threaded
1714 if (KNOB_SINGLE_THREADED)
1715 {
1716 std::vector<Type*> args{
1717 PointerType::get(mInt32Ty, 0), // pBucketMgr
1718 mInt32Ty // id
1719 };
1720
1721 FunctionType* pFuncTy = FunctionType::get(Type::getVoidTy(JM()->mContext), args, false);
1722 Function* pFunc = cast<Function>(JM()->mpCurrentModule->getOrInsertFunction("BucketManager_StopBucket", pFuncTy));
1723 if (sys::DynamicLibrary::SearchForAddressOfSymbol("BucketManager_StopBucket") == nullptr)
1724 {
1725 sys::DynamicLibrary::AddSymbol("BucketManager_StopBucket", (void*)&BucketManager_StopBucket);
1726 }
1727
1728 CALL(pFunc, { pBucketMgr, pId });
1729 }
1730 }
1731
1732
1733 uint32_t Builder::GetTypeSize(Type* pType)
1734 {
1735 if (pType->isStructTy())
1736 {
1737 uint32_t numElems = pType->getStructNumElements();
1738 Type* pElemTy = pType->getStructElementType(0);
1739 return numElems * GetTypeSize(pElemTy);
1740 }
1741
1742 if (pType->isArrayTy())
1743 {
1744 uint32_t numElems = pType->getArrayNumElements();
1745 Type* pElemTy = pType->getArrayElementType();
1746 return numElems * GetTypeSize(pElemTy);
1747 }
1748
1749 if (pType->isIntegerTy())
1750 {
1751 uint32_t bitSize = pType->getIntegerBitWidth();
1752 return bitSize / 8;
1753 }
1754
1755 if (pType->isFloatTy())
1756 {
1757 return 4;
1758 }
1759
1760 if (pType->isHalfTy())
1761 {
1762 return 2;
1763 }
1764
1765 if (pType->isDoubleTy())
1766 {
1767 return 8;
1768 }
1769
1770 SWR_ASSERT(false, "Unimplemented type.");
1771 return 0;
1772 }
1773 }