src/gallium/drivers/swr/rasterizer/jitter/builder_misc.cpp

   1 /****************************************************************************
   2 * Copyright (C) 2014-2015 Intel Corporation.   All Rights Reserved.
   3 *
   4 * Permission is hereby granted, free of charge, to any person obtaining a
   5 * copy of this software and associated documentation files (the "Software"),
   6 * to deal in the Software without restriction, including without limitation
   7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8 * and/or sell copies of the Software, and to permit persons to whom the
   9 * Software is furnished to do so, subject to the following conditions:
  10 *
  11 * The above copyright notice and this permission notice (including the next
  12 * paragraph) shall be included in all copies or substantial portions of the
  13 * Software.
  14 *
  15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21 * IN THE SOFTWARE.
  22 *
  23 * @file builder_misc.cpp
  24 *
  25 * @brief Implementation for miscellaneous builder functions
  26 *
  27 * Notes:
  28 *
  29 ******************************************************************************/
  30 #include "builder.h"
  31 #include "common/rdtsc_buckets.h"
  32
  33 #include "llvm/Support/DynamicLibrary.h"
  34
  35 void __cdecl CallPrint(const char* fmt, ...);
  36
  37 //////////////////////////////////////////////////////////////////////////
  38 /// @brief Convert an IEEE 754 32-bit single precision float to an
  39 ///        16 bit float with 5 exponent bits and a variable
  40 ///        number of mantissa bits.
  41 /// @param val - 32-bit float
  42 /// @todo Maybe move this outside of this file into a header?
  43 static uint16_t Convert32To16Float(float val)
  44 {
  45     uint32_t sign, exp, mant;
  46     uint32_t roundBits;
  47
  48     // Extract the sign, exponent, and mantissa
  49     uint32_t uf = *(uint32_t*)&val;
  50     sign = (uf & 0x80000000) >> 31;
  51     exp = (uf & 0x7F800000) >> 23;
  52     mant = uf & 0x007FFFFF;
  53
  54     // Check for out of range
  55     if (std::isnan(val))
  56     {
  57         exp = 0x1F;
  58         mant = 0x200;
  59         sign = 1;                     // set the sign bit for NANs
  60     }
  61     else if (std::isinf(val))
  62     {
  63         exp = 0x1f;
  64         mant = 0x0;
  65     }
  66     else if (exp > (0x70 + 0x1E)) // Too big to represent -> max representable value
  67     {
  68         exp = 0x1E;
  69         mant = 0x3FF;
  70     }
  71     else if ((exp <= 0x70) && (exp >= 0x66)) // It's a denorm
  72     {
  73         mant |= 0x00800000;
  74         for (; exp <= 0x70; mant >>= 1, exp++)
  75             ;
  76         exp = 0;
  77         mant = mant >> 13;
  78     }
  79     else if (exp < 0x66) // Too small to represent -> Zero
  80     {
  81         exp = 0;
  82         mant = 0;
  83     }
  84     else
  85     {
  86         // Saves bits that will be shifted off for rounding
  87         roundBits = mant & 0x1FFFu;
  88         // convert exponent and mantissa to 16 bit format
  89         exp = exp - 0x70;
  90         mant = mant >> 13;
  91
  92         // Essentially RTZ, but round up if off by only 1 lsb
  93         if (roundBits == 0x1FFFu)
  94         {
  95             mant++;
  96             // check for overflow
  97             if ((mant & 0xC00u) != 0)
  98                 exp++;
  99             // make sure only the needed bits are used
 100             mant &= 0x3FF;
 101         }
 102     }
 103
 104     uint32_t tmpVal = (sign << 15) | (exp << 10) | mant;
 105     return (uint16_t)tmpVal;
 106 }
 107
 108 //////////////////////////////////////////////////////////////////////////
 109 /// @brief Convert an IEEE 754 16-bit float to an 32-bit single precision
 110 ///        float
 111 /// @param val - 16-bit float
 112 /// @todo Maybe move this outside of this file into a header?
 113 static float ConvertSmallFloatTo32(UINT val)
 114 {
 115     UINT result;
 116     if ((val & 0x7fff) == 0)
 117     {
 118         result = ((uint32_t)(val & 0x8000)) << 16;
 119     }
 120     else if ((val & 0x7c00) == 0x7c00)
 121     {
 122         result = ((val & 0x3ff) == 0) ? 0x7f800000 : 0x7fc00000;
 123         result |= ((uint32_t)val & 0x8000) << 16;
 124     }
 125     else
 126     {
 127         uint32_t sign = (val & 0x8000) << 16;
 128         uint32_t mant = (val & 0x3ff) << 13;
 129         uint32_t exp = (val >> 10) & 0x1f;
 130         if ((exp == 0) && (mant != 0)) // Adjust exponent and mantissa for denormals
 131         {
 132             mant <<= 1;
 133             while (mant < (0x400 << 13))
 134             {
 135                 exp--;
 136                 mant <<= 1;
 137             }
 138             mant &= (0x3ff << 13);
 139         }
 140         exp = ((exp - 15 + 127) & 0xff) << 23;
 141         result = sign | exp | mant;
 142     }
 143
 144     return *(float*)&result;
 145 }
 146
 147 Constant *Builder::C(bool i)
 148 {
 149     return ConstantInt::get(IRB()->getInt1Ty(), (i ? 1 : 0));
 150 }
 151
 152 Constant *Builder::C(char i)
 153 {
 154     return ConstantInt::get(IRB()->getInt8Ty(), i);
 155 }
 156
 157 Constant *Builder::C(uint8_t i)
 158 {
 159     return ConstantInt::get(IRB()->getInt8Ty(), i);
 160 }
 161
 162 Constant *Builder::C(int i)
 163 {
 164     return ConstantInt::get(IRB()->getInt32Ty(), i);
 165 }
 166
 167 Constant *Builder::C(int64_t i)
 168 {
 169     return ConstantInt::get(IRB()->getInt64Ty(), i);
 170 }
 171
 172 Constant *Builder::C(uint16_t i)
 173 {
 174     return ConstantInt::get(mInt16Ty,i);
 175 }
 176
 177 Constant *Builder::C(uint32_t i)
 178 {
 179     return ConstantInt::get(IRB()->getInt32Ty(), i);
 180 }
 181
 182 Constant *Builder::C(float i)
 183 {
 184     return ConstantFP::get(IRB()->getFloatTy(), i);
 185 }
 186
 187 Constant *Builder::PRED(bool pred)
 188 {
 189     return ConstantInt::get(IRB()->getInt1Ty(), (pred ? 1 : 0));
 190 }
 191
 192 Value *Builder::VIMMED1(int i)
 193 {
 194     return ConstantVector::getSplat(mVWidth, cast<ConstantInt>(C(i)));
 195 }
 196
 197 Value *Builder::VIMMED1(uint32_t i)
 198 {
 199     return ConstantVector::getSplat(mVWidth, cast<ConstantInt>(C(i)));
 200 }
 201
 202 Value *Builder::VIMMED1(float i)
 203 {
 204     return ConstantVector::getSplat(mVWidth, cast<ConstantFP>(C(i)));
 205 }
 206
 207 Value *Builder::VIMMED1(bool i)
 208 {
 209     return ConstantVector::getSplat(mVWidth, cast<ConstantInt>(C(i)));
 210 }
 211
 212 Value *Builder::VUNDEF_IPTR()
 213 {
 214     return UndefValue::get(VectorType::get(mInt32PtrTy,mVWidth));
 215 }
 216
 217 Value *Builder::VUNDEF_I()
 218 {
 219     return UndefValue::get(VectorType::get(mInt32Ty, mVWidth));
 220 }
 221
 222 Value *Builder::VUNDEF(Type *ty, uint32_t size)
 223 {
 224     return UndefValue::get(VectorType::get(ty, size));
 225 }
 226
 227 Value *Builder::VUNDEF_F()
 228 {
 229     return UndefValue::get(VectorType::get(mFP32Ty, mVWidth));
 230 }
 231
 232 Value *Builder::VUNDEF(Type* t)
 233 {
 234     return UndefValue::get(VectorType::get(t, mVWidth));
 235 }
 236
 237 #if HAVE_LLVM == 0x306
 238 Value *Builder::VINSERT(Value *vec, Value *val, uint64_t index)
 239 {
 240     return VINSERT(vec, val, C((int64_t)index));
 241 }
 242 #endif
 243
 244 Value *Builder::VBROADCAST(Value *src)
 245 {
 246     // check if src is already a vector
 247     if (src->getType()->isVectorTy())
 248     {
 249         return src;
 250     }
 251
 252     return VECTOR_SPLAT(mVWidth, src);
 253 }
 254
 255 uint32_t Builder::IMMED(Value* v)
 256 {
 257     SWR_ASSERT(isa<ConstantInt>(v));
 258     ConstantInt *pValConst = cast<ConstantInt>(v);
 259     return pValConst->getZExtValue();
 260 }
 261
 262 int32_t Builder::S_IMMED(Value* v)
 263 {
 264     SWR_ASSERT(isa<ConstantInt>(v));
 265     ConstantInt *pValConst = cast<ConstantInt>(v);
 266     return pValConst->getSExtValue();
 267 }
 268
 269 Value *Builder::GEP(Value* ptr, const std::initializer_list<Value*> &indexList)
 270 {
 271     std::vector<Value*> indices;
 272     for (auto i : indexList)
 273         indices.push_back(i);
 274     return GEPA(ptr, indices);
 275 }
 276
 277 Value *Builder::GEP(Value* ptr, const std::initializer_list<uint32_t> &indexList)
 278 {
 279     std::vector<Value*> indices;
 280     for (auto i : indexList)
 281         indices.push_back(C(i));
 282     return GEPA(ptr, indices);
 283 }
 284
 285 LoadInst *Builder::LOAD(Value *basePtr, const std::initializer_list<uint32_t> &indices, const llvm::Twine& name)
 286 {
 287     std::vector<Value*> valIndices;
 288     for (auto i : indices)
 289         valIndices.push_back(C(i));
 290     return LOAD(GEPA(basePtr, valIndices), name);
 291 }
 292
 293 LoadInst *Builder::LOADV(Value *basePtr, const std::initializer_list<Value*> &indices, const llvm::Twine& name)
 294 {
 295     std::vector<Value*> valIndices;
 296     for (auto i : indices)
 297         valIndices.push_back(i);
 298     return LOAD(GEPA(basePtr, valIndices), name);
 299 }
 300
 301 StoreInst *Builder::STORE(Value *val, Value *basePtr, const std::initializer_list<uint32_t> &indices)
 302 {
 303     std::vector<Value*> valIndices;
 304     for (auto i : indices)
 305         valIndices.push_back(C(i));
 306     return STORE(val, GEPA(basePtr, valIndices));
 307 }
 308
 309 StoreInst *Builder::STOREV(Value *val, Value *basePtr, const std::initializer_list<Value*> &indices)
 310 {
 311     std::vector<Value*> valIndices;
 312     for (auto i : indices)
 313         valIndices.push_back(i);
 314     return STORE(val, GEPA(basePtr, valIndices));
 315 }
 316
 317 CallInst *Builder::CALL(Value *Callee, const std::initializer_list<Value*> &argsList)
 318 {
 319     std::vector<Value*> args;
 320     for (auto arg : argsList)
 321         args.push_back(arg);
 322     return CALLA(Callee, args);
 323 }
 324
 325 Value *Builder::VRCP(Value *va)
 326 {
 327     return FDIV(VIMMED1(1.0f), va);  // 1 / a
 328 }
 329
 330 Value *Builder::VPLANEPS(Value* vA, Value* vB, Value* vC, Value* &vX, Value* &vY)
 331 {
 332     Value* vOut = FMADDPS(vA, vX, vC);
 333     vOut = FMADDPS(vB, vY, vOut);
 334     return vOut;
 335 }
 336
 337 //////////////////////////////////////////////////////////////////////////
 338 /// @brief Generate an i32 masked load operation in LLVM IR.  If not
 339 /// supported on the underlying platform, emulate it with float masked load
 340 /// @param src - base address pointer for the load
 341 /// @param vMask - SIMD wide mask that controls whether to access memory load 0
 342 Value *Builder::MASKLOADD(Value* src,Value* mask)
 343 {
 344     Value* vResult;
 345     // use avx2 gather instruction is available
 346     if(JM()->mArch.AVX2())
 347     {
 348         Function *func = Intrinsic::getDeclaration(JM()->mpCurrentModule, Intrinsic::x86_avx2_maskload_d_256);
 349         vResult = CALL(func,{src,mask});
 350     }
 351     else
 352     {
 353         Function *func = Intrinsic::getDeclaration(JM()->mpCurrentModule,Intrinsic::x86_avx_maskload_ps_256);
 354         Value* fMask = BITCAST(mask,VectorType::get(mInt32Ty,mVWidth));
 355         vResult = BITCAST(CALL(func,{src,fMask}), VectorType::get(mInt32Ty,mVWidth));
 356     }
 357     return vResult;
 358 }
 359
 360 //////////////////////////////////////////////////////////////////////////
 361 /// @brief insert a JIT call to CallPrint
 362 /// - outputs formatted string to both stdout and VS output window
 363 /// - DEBUG builds only
 364 /// Usage example:
 365 ///   PRINT("index %d = 0x%p\n",{C(lane), pIndex});
 366 ///   where C(lane) creates a constant value to print, and pIndex is the Value*
 367 ///   result from a GEP, printing out the pointer to memory
 368 /// @param printStr - constant string to print, which includes format specifiers
 369 /// @param printArgs - initializer list of Value*'s to print to std out
 370 CallInst *Builder::PRINT(const std::string &printStr,const std::initializer_list<Value*> &printArgs)
 371 {
 372     // push the arguments to CallPrint into a vector
 373     std::vector<Value*> printCallArgs;
 374     // save room for the format string.  we still need to modify it for vectors
 375     printCallArgs.resize(1);
 376
 377     // search through the format string for special processing
 378     size_t pos = 0;
 379     std::string tempStr(printStr);
 380     pos = tempStr.find('%', pos);
 381     auto v = printArgs.begin();
 382
 383     while ((pos != std::string::npos) && (v != printArgs.end()))
 384     {
 385         Value* pArg = *v;
 386         Type* pType = pArg->getType();
 387
 388         if (pType->isVectorTy())
 389         {
 390             Type* pContainedType = pType->getContainedType(0);
 391
 392             if (toupper(tempStr[pos + 1]) == 'X')
 393             {
 394                 tempStr[pos] = '0';
 395                 tempStr[pos + 1] = 'x';
 396                 tempStr.insert(pos + 2, "%08X ");
 397                 pos += 7;
 398
 399                 printCallArgs.push_back(VEXTRACT(pArg, C(0)));
 400
 401                 std::string vectorFormatStr;
 402                 for (uint32_t i = 1; i < pType->getVectorNumElements(); ++i)
 403                 {
 404                     vectorFormatStr += "0x%08X ";
 405                     printCallArgs.push_back(VEXTRACT(pArg, C(i)));
 406                 }
 407
 408                 tempStr.insert(pos, vectorFormatStr);
 409                 pos += vectorFormatStr.size();
 410             }
 411             else if ((tempStr[pos + 1] == 'f') && (pContainedType->isFloatTy()))
 412             {
 413                 uint32_t i = 0;
 414                 for (; i < (pArg->getType()->getVectorNumElements()) - 1; i++)
 415                 {
 416                     tempStr.insert(pos, std::string("%f "));
 417                     pos += 3;
 418                     printCallArgs.push_back(FP_EXT(VEXTRACT(pArg, C(i)), Type::getDoubleTy(JM()->mContext)));
 419                 }
 420                 printCallArgs.push_back(FP_EXT(VEXTRACT(pArg, C(i)), Type::getDoubleTy(JM()->mContext)));
 421             }
 422             else if ((tempStr[pos + 1] == 'd') && (pContainedType->isIntegerTy()))
 423             {
 424                 uint32_t i = 0;
 425                 for (; i < (pArg->getType()->getVectorNumElements()) - 1; i++)
 426                 {
 427                     tempStr.insert(pos, std::string("%d "));
 428                     pos += 3;
 429                     printCallArgs.push_back(VEXTRACT(pArg, C(i)));
 430                 }
 431                 printCallArgs.push_back(VEXTRACT(pArg, C(i)));
 432             }
 433         }
 434         else
 435         {
 436             if (toupper(tempStr[pos + 1]) == 'X')
 437             {
 438                 tempStr[pos] = '0';
 439                 tempStr.insert(pos + 1, "x%08");
 440                 printCallArgs.push_back(pArg);
 441                 pos += 3;
 442             }
 443             // for %f we need to cast float Values to doubles so that they print out correctly
 444             else if ((tempStr[pos + 1] == 'f') && (pType->isFloatTy()))
 445             {
 446                 printCallArgs.push_back(FP_EXT(pArg, Type::getDoubleTy(JM()->mContext)));
 447                 pos++;
 448             }
 449             else
 450             {
 451                 printCallArgs.push_back(pArg);
 452             }
 453         }
 454
 455         // advance to the next arguement
 456         v++;
 457         pos = tempStr.find('%', ++pos);
 458     }
 459
 460     // create global variable constant string
 461     Constant *constString = ConstantDataArray::getString(JM()->mContext,tempStr,true);
 462     GlobalVariable *gvPtr = new GlobalVariable(constString->getType(),true,GlobalValue::InternalLinkage,constString,"printStr");
 463     JM()->mpCurrentModule->getGlobalList().push_back(gvPtr);
 464
 465     // get a pointer to the first character in the constant string array
 466     std::vector<Constant*> geplist{C(0),C(0)};
 467 #if HAVE_LLVM == 0x306
 468     Constant *strGEP = ConstantExpr::getGetElementPtr(gvPtr,geplist,false);
 469 #else
 470     Constant *strGEP = ConstantExpr::getGetElementPtr(nullptr, gvPtr,geplist,false);
 471 #endif
 472
 473     // insert the pointer to the format string in the argument vector
 474     printCallArgs[0] = strGEP;
 475
 476     // get pointer to CallPrint function and insert decl into the module if needed
 477     std::vector<Type*> args;
 478     args.push_back(PointerType::get(mInt8Ty,0));
 479     FunctionType* callPrintTy = FunctionType::get(Type::getVoidTy(JM()->mContext),args,true);
 480     Function *callPrintFn = cast<Function>(JM()->mpCurrentModule->getOrInsertFunction("CallPrint", callPrintTy));
 481
 482     // if we haven't yet added the symbol to the symbol table
 483     if((sys::DynamicLibrary::SearchForAddressOfSymbol("CallPrint")) == nullptr)
 484     {
 485         sys::DynamicLibrary::AddSymbol("CallPrint", (void *)&CallPrint);
 486     }
 487
 488     // insert a call to CallPrint
 489     return CALLA(callPrintFn,printCallArgs);
 490 }
 491
 492 //////////////////////////////////////////////////////////////////////////
 493 /// @brief Wrapper around PRINT with initializer list.
 494 CallInst* Builder::PRINT(const std::string &printStr)
 495 {
 496     return PRINT(printStr, {});
 497 }
 498
 499 //////////////////////////////////////////////////////////////////////////
 500 /// @brief Generate a masked gather operation in LLVM IR.  If not
 501 /// supported on the underlying platform, emulate it with loads
 502 /// @param vSrc - SIMD wide value that will be loaded if mask is invalid
 503 /// @param pBase - Int8* base VB address pointer value
 504 /// @param vIndices - SIMD wide value of VB byte offsets
 505 /// @param vMask - SIMD wide mask that controls whether to access memory or the src values
 506 /// @param scale - value to scale indices by
 507 Value *Builder::GATHERPS(Value* vSrc, Value* pBase, Value* vIndices, Value* vMask, Value* scale)
 508 {
 509     Value* vGather;
 510
 511     // use avx2 gather instruction if available
 512     if(JM()->mArch.AVX2())
 513     {
 514         // force mask to <N x float>, required by vgather
 515         vMask = BITCAST(vMask, mSimdFP32Ty);
 516         vGather = VGATHERPS(vSrc,pBase,vIndices,vMask,scale);
 517     }
 518     else
 519     {
 520         Value* pStack = STACKSAVE();
 521
 522         // store vSrc on the stack.  this way we can select between a valid load address and the vSrc address
 523         Value* vSrcPtr = ALLOCA(vSrc->getType());
 524         STORE(vSrc, vSrcPtr);
 525
 526         vGather = VUNDEF_F();
 527         Value *vScaleVec = VBROADCAST(Z_EXT(scale,mInt32Ty));
 528         Value *vOffsets = MUL(vIndices,vScaleVec);
 529         Value *mask = MASK(vMask);
 530         for(uint32_t i = 0; i < mVWidth; ++i)
 531         {
 532             // single component byte index
 533             Value *offset = VEXTRACT(vOffsets,C(i));
 534             // byte pointer to component
 535             Value *loadAddress = GEP(pBase,offset);
 536             loadAddress = BITCAST(loadAddress,PointerType::get(mFP32Ty,0));
 537             // pointer to the value to load if we're masking off a component
 538             Value *maskLoadAddress = GEP(vSrcPtr,{C(0), C(i)});
 539             Value *selMask = VEXTRACT(mask,C(i));
 540             // switch in a safe address to load if we're trying to access a vertex
 541             Value *validAddress = SELECT(selMask, loadAddress, maskLoadAddress);
 542             Value *val = LOAD(validAddress);
 543             vGather = VINSERT(vGather,val,C(i));
 544         }
 545         STACKRESTORE(pStack);
 546     }
 547
 548     return vGather;
 549 }
 550
 551 //////////////////////////////////////////////////////////////////////////
 552 /// @brief Generate a masked gather operation in LLVM IR.  If not
 553 /// supported on the underlying platform, emulate it with loads
 554 /// @param vSrc - SIMD wide value that will be loaded if mask is invalid
 555 /// @param pBase - Int8* base VB address pointer value
 556 /// @param vIndices - SIMD wide value of VB byte offsets
 557 /// @param vMask - SIMD wide mask that controls whether to access memory or the src values
 558 /// @param scale - value to scale indices by
 559 Value *Builder::GATHERDD(Value* vSrc, Value* pBase, Value* vIndices, Value* vMask, Value* scale)
 560 {
 561     Value* vGather;
 562
 563     // use avx2 gather instruction if available
 564     if(JM()->mArch.AVX2())
 565     {
 566         vGather = VGATHERDD(vSrc, pBase, vIndices, vMask, scale);
 567     }
 568     else
 569     {
 570         Value* pStack = STACKSAVE();
 571
 572         // store vSrc on the stack.  this way we can select between a valid load address and the vSrc address
 573         Value* vSrcPtr = ALLOCA(vSrc->getType());
 574         STORE(vSrc, vSrcPtr);
 575
 576         vGather = VUNDEF_I();
 577         Value *vScaleVec = VBROADCAST(Z_EXT(scale, mInt32Ty));
 578         Value *vOffsets = MUL(vIndices, vScaleVec);
 579         Value *mask = MASK(vMask);
 580         for(uint32_t i = 0; i < mVWidth; ++i)
 581         {
 582             // single component byte index
 583             Value *offset = VEXTRACT(vOffsets, C(i));
 584             // byte pointer to component
 585             Value *loadAddress = GEP(pBase, offset);
 586             loadAddress = BITCAST(loadAddress, PointerType::get(mInt32Ty, 0));
 587             // pointer to the value to load if we're masking off a component
 588             Value *maskLoadAddress = GEP(vSrcPtr, {C(0), C(i)});
 589             Value *selMask = VEXTRACT(mask, C(i));
 590             // switch in a safe address to load if we're trying to access a vertex
 591             Value *validAddress = SELECT(selMask, loadAddress, maskLoadAddress);
 592             Value *val = LOAD(validAddress, C(0));
 593             vGather = VINSERT(vGather, val, C(i));
 594         }
 595
 596         STACKRESTORE(pStack);
 597     }
 598     return vGather;
 599 }
 600
 601 //////////////////////////////////////////////////////////////////////////
 602 /// @brief convert x86 <N x float> mask to llvm <N x i1> mask
 603 Value* Builder::MASK(Value* vmask)
 604 {
 605     Value* src = BITCAST(vmask, mSimdInt32Ty);
 606     return ICMP_SLT(src, VIMMED1(0));
 607 }
 608
 609 //////////////////////////////////////////////////////////////////////////
 610 /// @brief convert llvm <N x i1> mask to x86 <N x i32> mask
 611 Value* Builder::VMASK(Value* mask)
 612 {
 613     return S_EXT(mask, mSimdInt32Ty);
 614 }
 615
 616 //////////////////////////////////////////////////////////////////////////
 617 /// @brief Generate a VPSHUFB operation in LLVM IR.  If not
 618 /// supported on the underlying platform, emulate it
 619 /// @param a - 256bit SIMD(32x8bit) of 8bit integer values
 620 /// @param b - 256bit SIMD(32x8bit) of 8bit integer mask values
 621 /// Byte masks in lower 128 lane of b selects 8 bit values from lower
 622 /// 128bits of a, and vice versa for the upper lanes.  If the mask
 623 /// value is negative, '0' is inserted.
 624 Value *Builder::PSHUFB(Value* a, Value* b)
 625 {
 626     Value* res;
 627     // use avx2 pshufb instruction if available
 628     if(JM()->mArch.AVX2())
 629     {
 630         res = VPSHUFB(a, b);
 631     }
 632     else
 633     {
 634         Constant* cB = dyn_cast<Constant>(b);
 635         // number of 8 bit elements in b
 636         uint32_t numElms = cast<VectorType>(cB->getType())->getNumElements();
 637         // output vector
 638         Value* vShuf = UndefValue::get(VectorType::get(mInt8Ty, numElms));
 639
 640         // insert an 8 bit value from the high and low lanes of a per loop iteration
 641         numElms /= 2;
 642         for(uint32_t i = 0; i < numElms; i++)
 643         {
 644             ConstantInt* cLow128b = cast<ConstantInt>(cB->getAggregateElement(i));
 645             ConstantInt* cHigh128b = cast<ConstantInt>(cB->getAggregateElement(i + numElms));
 646
 647             // extract values from constant mask
 648             char valLow128bLane =  (char)(cLow128b->getSExtValue());
 649             char valHigh128bLane = (char)(cHigh128b->getSExtValue());
 650
 651             Value* insertValLow128b;
 652             Value* insertValHigh128b;
 653
 654             // if the mask value is negative, insert a '0' in the respective output position
 655             // otherwise, lookup the value at mask position (bits 3..0 of the respective mask byte) in a and insert in output vector
 656             insertValLow128b = (valLow128bLane < 0) ? C((char)0) : VEXTRACT(a, C((valLow128bLane & 0xF)));
 657             insertValHigh128b = (valHigh128bLane < 0) ? C((char)0) : VEXTRACT(a, C((valHigh128bLane & 0xF) + numElms));
 658
 659             vShuf = VINSERT(vShuf, insertValLow128b, i);
 660             vShuf = VINSERT(vShuf, insertValHigh128b, (i + numElms));
 661         }
 662         res = vShuf;
 663     }
 664     return res;
 665 }
 666
 667 //////////////////////////////////////////////////////////////////////////
 668 /// @brief Generate a VPSHUFB operation (sign extend 8 8bit values to 32
 669 /// bits)in LLVM IR.  If not supported on the underlying platform, emulate it
 670 /// @param a - 128bit SIMD lane(16x8bit) of 8bit integer values.  Only
 671 /// lower 8 values are used.
 672 Value *Builder::PMOVSXBD(Value* a)
 673 {
 674     Value* res;
 675     // use avx2 byte sign extend instruction if available
 676     if(JM()->mArch.AVX2())
 677     {
 678         res = VPMOVSXBD(a);
 679     }
 680     else
 681     {
 682         // VPMOVSXBD output type
 683         Type* v8x32Ty = VectorType::get(mInt32Ty, 8);
 684         // Extract 8 values from 128bit lane and sign extend
 685         res = S_EXT(VSHUFFLE(a, a, C<int>({0, 1, 2, 3, 4, 5, 6, 7})), v8x32Ty);
 686     }
 687     return res;
 688 }
 689
 690 //////////////////////////////////////////////////////////////////////////
 691 /// @brief Generate a VPSHUFB operation (sign extend 8 16bit values to 32
 692 /// bits)in LLVM IR.  If not supported on the underlying platform, emulate it
 693 /// @param a - 128bit SIMD lane(8x16bit) of 16bit integer values.
 694 Value *Builder::PMOVSXWD(Value* a)
 695 {
 696     Value* res;
 697     // use avx2 word sign extend if available
 698     if(JM()->mArch.AVX2())
 699     {
 700         res = VPMOVSXWD(a);
 701     }
 702     else
 703     {
 704         // VPMOVSXWD output type
 705         Type* v8x32Ty = VectorType::get(mInt32Ty, 8);
 706         // Extract 8 values from 128bit lane and sign extend
 707         res = S_EXT(VSHUFFLE(a, a, C<int>({0, 1, 2, 3, 4, 5, 6, 7})), v8x32Ty);
 708     }
 709     return res;
 710 }
 711
 712 //////////////////////////////////////////////////////////////////////////
 713 /// @brief Generate a VPERMD operation (shuffle 32 bit integer values
 714 /// across 128 bit lanes) in LLVM IR.  If not supported on the underlying
 715 /// platform, emulate it
 716 /// @param a - 256bit SIMD lane(8x32bit) of integer values.
 717 /// @param idx - 256bit SIMD lane(8x32bit) of 3 bit lane index values
 718 Value *Builder::PERMD(Value* a, Value* idx)
 719 {
 720     Value* res;
 721     // use avx2 permute instruction if available
 722     if(JM()->mArch.AVX2())
 723     {
 724         // llvm 3.6.0 swapped the order of the args to vpermd
 725         res = VPERMD(idx, a);
 726     }
 727     else
 728     {
 729         if (isa<Constant>(idx))
 730         {
 731             res = VSHUFFLE(a, a, idx);
 732         }
 733         else
 734         {
 735             res = VUNDEF_I();
 736             for (uint32_t l = 0; l < JM()->mVWidth; ++l)
 737             {
 738                 Value* pIndex = VEXTRACT(idx, C(l));
 739                 Value* pVal = VEXTRACT(a, pIndex);
 740                 res = VINSERT(res, pVal, C(l));
 741             }
 742         }
 743     }
 744     return res;
 745 }
 746
 747 //////////////////////////////////////////////////////////////////////////
 748 /// @brief Generate a VPERMPS operation (shuffle 32 bit float values
 749 /// across 128 bit lanes) in LLVM IR.  If not supported on the underlying
 750 /// platform, emulate it
 751 /// @param a - 256bit SIMD lane(8x32bit) of float values.
 752 /// @param idx - 256bit SIMD lane(8x32bit) of 3 bit lane index values
 753 Value *Builder::PERMPS(Value* a, Value* idx)
 754 {
 755     Value* res;
 756     // use avx2 permute instruction if available
 757     if (JM()->mArch.AVX2())
 758     {
 759         // llvm 3.6.0 swapped the order of the args to vpermd
 760         res = VPERMPS(idx, a);
 761     }
 762     else
 763     {
 764         if (isa<Constant>(idx))
 765         {
 766             res = VSHUFFLE(a, a, idx);
 767         }
 768         else
 769         {
 770             res = VUNDEF_F();
 771             for (uint32_t l = 0; l < JM()->mVWidth; ++l)
 772             {
 773                 Value* pIndex = VEXTRACT(idx, C(l));
 774                 Value* pVal = VEXTRACT(a, pIndex);
 775                 res = VINSERT(res, pVal, C(l));
 776             }
 777         }
 778     }
 779
 780     return res;
 781 }
 782
 783 //////////////////////////////////////////////////////////////////////////
 784 /// @brief Generate a VCVTPH2PS operation (float16->float32 conversion)
 785 /// in LLVM IR.  If not supported on the underlying platform, emulate it
 786 /// @param a - 128bit SIMD lane(8x16bit) of float16 in int16 format.
 787 Value *Builder::CVTPH2PS(Value* a)
 788 {
 789     if (JM()->mArch.F16C())
 790     {
 791         return VCVTPH2PS(a);
 792     }
 793     else
 794     {
 795         FunctionType* pFuncTy = FunctionType::get(mFP32Ty, mInt16Ty);
 796         Function* pCvtPh2Ps = cast<Function>(JM()->mpCurrentModule->getOrInsertFunction("ConvertSmallFloatTo32", pFuncTy));
 797
 798         if (sys::DynamicLibrary::SearchForAddressOfSymbol("ConvertSmallFloatTo32") == nullptr)
 799         {
 800             sys::DynamicLibrary::AddSymbol("ConvertSmallFloatTo32", (void *)&ConvertSmallFloatTo32);
 801         }
 802
 803         Value* pResult = UndefValue::get(mSimdFP32Ty);
 804         for (uint32_t i = 0; i < mVWidth; ++i)
 805         {
 806             Value* pSrc = VEXTRACT(a, C(i));
 807             Value* pConv = CALL(pCvtPh2Ps, std::initializer_list<Value*>{pSrc});
 808             pResult = VINSERT(pResult, pConv, C(i));
 809         }
 810
 811         return pResult;
 812     }
 813 }
 814
 815 //////////////////////////////////////////////////////////////////////////
 816 /// @brief Generate a VCVTPS2PH operation (float32->float16 conversion)
 817 /// in LLVM IR.  If not supported on the underlying platform, emulate it
 818 /// @param a - 128bit SIMD lane(8x16bit) of float16 in int16 format.
 819 Value *Builder::CVTPS2PH(Value* a, Value* rounding)
 820 {
 821     if (JM()->mArch.F16C())
 822     {
 823         return VCVTPS2PH(a, rounding);
 824     }
 825     else
 826     {
 827         // call scalar C function for now
 828         FunctionType* pFuncTy = FunctionType::get(mInt16Ty, mFP32Ty);
 829         Function* pCvtPs2Ph = cast<Function>(JM()->mpCurrentModule->getOrInsertFunction("Convert32To16Float", pFuncTy));
 830
 831         if (sys::DynamicLibrary::SearchForAddressOfSymbol("Convert32To16Float") == nullptr)
 832         {
 833             sys::DynamicLibrary::AddSymbol("Convert32To16Float", (void *)&Convert32To16Float);
 834         }
 835
 836         Value* pResult = UndefValue::get(mSimdInt16Ty);
 837         for (uint32_t i = 0; i < mVWidth; ++i)
 838         {
 839             Value* pSrc = VEXTRACT(a, C(i));
 840             Value* pConv = CALL(pCvtPs2Ph, std::initializer_list<Value*>{pSrc});
 841             pResult = VINSERT(pResult, pConv, C(i));
 842         }
 843
 844         return pResult;
 845     }
 846 }
 847
 848 Value *Builder::PMAXSD(Value* a, Value* b)
 849 {
 850     if (JM()->mArch.AVX2())
 851     {
 852         return VPMAXSD(a, b);
 853     }
 854     else
 855     {
 856         // use 4-wide sse max intrinsic on lower/upper halves of 8-wide sources
 857         Function* pmaxsd = Intrinsic::getDeclaration(JM()->mpCurrentModule, Intrinsic::x86_sse41_pmaxsd);
 858
 859         // low 128
 860         Value* aLo = VEXTRACTI128(a, C((uint8_t)0));
 861         Value* bLo = VEXTRACTI128(b, C((uint8_t)0));
 862         Value* resLo = CALL(pmaxsd, {aLo, bLo});
 863
 864         // high 128
 865         Value* aHi = VEXTRACTI128(a, C((uint8_t)1));
 866         Value* bHi = VEXTRACTI128(b, C((uint8_t)1));
 867         Value* resHi = CALL(pmaxsd, {aHi, bHi});
 868
 869         // combine
 870         Value* result = VINSERTI128(VUNDEF_I(), resLo, C((uint8_t)0));
 871         result = VINSERTI128(result, resHi, C((uint8_t)1));
 872
 873         return result;
 874     }
 875 }
 876
 877 Value *Builder::PMINSD(Value* a, Value* b)
 878 {
 879     if (JM()->mArch.AVX2())
 880     {
 881         return VPMINSD(a, b);
 882     }
 883     else
 884     {
 885         // use 4-wide sse max intrinsic on lower/upper halves of 8-wide sources
 886         Function* pminsd = Intrinsic::getDeclaration(JM()->mpCurrentModule, Intrinsic::x86_sse41_pminsd);
 887
 888         // low 128
 889         Value* aLo = VEXTRACTI128(a, C((uint8_t)0));
 890         Value* bLo = VEXTRACTI128(b, C((uint8_t)0));
 891         Value* resLo = CALL(pminsd, {aLo, bLo});
 892
 893         // high 128
 894         Value* aHi = VEXTRACTI128(a, C((uint8_t)1));
 895         Value* bHi = VEXTRACTI128(b, C((uint8_t)1));
 896         Value* resHi = CALL(pminsd, {aHi, bHi});
 897
 898         // combine
 899         Value* result = VINSERTI128(VUNDEF_I(), resLo, C((uint8_t)0));
 900         result = VINSERTI128(result, resHi, C((uint8_t)1));
 901
 902         return result;
 903     }
 904 }
 905
 906 void Builder::Gather4(const SWR_FORMAT format, Value* pSrcBase, Value* byteOffsets,
 907                       Value* mask, Value* vGatherComponents[], bool bPackedOutput)
 908 {
 909     const SWR_FORMAT_INFO &info = GetFormatInfo(format);
 910     if(info.type[0] == SWR_TYPE_FLOAT && info.bpc[0] == 32)
 911     {
 912         // ensure our mask is the correct type
 913         mask = BITCAST(mask, mSimdFP32Ty);
 914         GATHER4PS(info, pSrcBase, byteOffsets, mask, vGatherComponents, bPackedOutput);
 915     }
 916     else
 917     {
 918         // ensure our mask is the correct type
 919         mask = BITCAST(mask, mSimdInt32Ty);
 920         GATHER4DD(info, pSrcBase, byteOffsets, mask, vGatherComponents, bPackedOutput);
 921     }
 922 }
 923
 924 void Builder::GATHER4PS(const SWR_FORMAT_INFO &info, Value* pSrcBase, Value* byteOffsets,
 925                         Value* mask, Value* vGatherComponents[], bool bPackedOutput)
 926 {
 927     switch(info.bpp / info.numComps)
 928     {
 929         case 16:
 930         {
 931                 Value* vGatherResult[2];
 932                 Value *vMask;
 933
 934                 // TODO: vGatherMaskedVal
 935                 Value* vGatherMaskedVal = VIMMED1((float)0);
 936
 937                 // always have at least one component out of x or y to fetch
 938
 939                 // save mask as it is zero'd out after each gather
 940                 vMask = mask;
 941
 942                 vGatherResult[0] = GATHERPS(vGatherMaskedVal, pSrcBase, byteOffsets, vMask, C((char)1));
 943                 // e.g. result of first 8x32bit integer gather for 16bit components
 944                 // 256i - 0    1    2    3    4    5    6    7
 945                 //        xyxy xyxy xyxy xyxy xyxy xyxy xyxy xyxy
 946                 //
 947
 948                 // if we have at least one component out of x or y to fetch
 949                 if(info.numComps > 2)
 950                 {
 951                     // offset base to the next components(zw) in the vertex to gather
 952                     pSrcBase = GEP(pSrcBase, C((char)4));
 953                     vMask = mask;
 954
 955                     vGatherResult[1] =  GATHERPS(vGatherMaskedVal, pSrcBase, byteOffsets, vMask, C((char)1));
 956                     // e.g. result of second 8x32bit integer gather for 16bit components
 957                     // 256i - 0    1    2    3    4    5    6    7
 958                     //        zwzw zwzw zwzw zwzw zwzw zwzw zwzw zwzw
 959                     //
 960                 }
 961                 else
 962                 {
 963                     vGatherResult[1] =  vGatherMaskedVal;
 964                 }
 965
 966                 // Shuffle gathered components into place, each row is a component
 967                 Shuffle16bpcGather4(info, vGatherResult, vGatherComponents, bPackedOutput);
 968         }
 969             break;
 970         case 32:
 971         {
 972             // apply defaults
 973             for (uint32_t i = 0; i < 4; ++i)
 974             {
 975                 vGatherComponents[i] = VIMMED1(*(float*)&info.defaults[i]);
 976             }
 977
 978             for(uint32_t i = 0; i < info.numComps; i++)
 979             {
 980                 uint32_t swizzleIndex = info.swizzle[i];
 981
 982                 // save mask as it is zero'd out after each gather
 983                 Value *vMask = mask;
 984
 985                 // Gather a SIMD of components
 986                 vGatherComponents[swizzleIndex] = GATHERPS(vGatherComponents[swizzleIndex], pSrcBase, byteOffsets, vMask, C((char)1));
 987
 988                 // offset base to the next component to gather
 989                 pSrcBase = GEP(pSrcBase, C((char)4));
 990             }
 991         }
 992             break;
 993         default:
 994             SWR_ASSERT(0, "Invalid float format");
 995             break;
 996     }
 997 }
 998
 999 void Builder::GATHER4DD(const SWR_FORMAT_INFO &info, Value* pSrcBase, Value* byteOffsets,
1000                         Value* mask, Value* vGatherComponents[], bool bPackedOutput)
1001 {
1002     switch (info.bpp / info.numComps)
1003     {
1004         case 8:
1005         {
1006             Value* vGatherMaskedVal = VIMMED1((int32_t)0);
1007             Value* vGatherResult = GATHERDD(vGatherMaskedVal, pSrcBase, byteOffsets, mask, C((char)1));
1008             // e.g. result of an 8x32bit integer gather for 8bit components
1009             // 256i - 0    1    2    3    4    5    6    7
1010             //        xyzw xyzw xyzw xyzw xyzw xyzw xyzw xyzw
1011
1012             Shuffle8bpcGather4(info, vGatherResult, vGatherComponents, bPackedOutput);
1013         }
1014             break;
1015         case 16:
1016         {
1017             Value* vGatherResult[2];
1018             Value *vMask;
1019
1020             // TODO: vGatherMaskedVal
1021             Value* vGatherMaskedVal = VIMMED1((int32_t)0);
1022
1023             // always have at least one component out of x or y to fetch
1024
1025             // save mask as it is zero'd out after each gather
1026             vMask = mask;
1027
1028             vGatherResult[0] = GATHERDD(vGatherMaskedVal, pSrcBase, byteOffsets, vMask, C((char)1));
1029             // e.g. result of first 8x32bit integer gather for 16bit components
1030             // 256i - 0    1    2    3    4    5    6    7
1031             //        xyxy xyxy xyxy xyxy xyxy xyxy xyxy xyxy
1032             //
1033
1034             // if we have at least one component out of x or y to fetch
1035             if(info.numComps > 2)
1036             {
1037                 // offset base to the next components(zw) in the vertex to gather
1038                 pSrcBase = GEP(pSrcBase, C((char)4));
1039                 vMask = mask;
1040
1041                 vGatherResult[1] = GATHERDD(vGatherMaskedVal, pSrcBase, byteOffsets, vMask, C((char)1));
1042                 // e.g. result of second 8x32bit integer gather for 16bit components
1043                 // 256i - 0    1    2    3    4    5    6    7
1044                 //        zwzw zwzw zwzw zwzw zwzw zwzw zwzw zwzw
1045                 //
1046             }
1047             else
1048             {
1049                 vGatherResult[1] = vGatherMaskedVal;
1050             }
1051
1052             // Shuffle gathered components into place, each row is a component
1053             Shuffle16bpcGather4(info, vGatherResult, vGatherComponents, bPackedOutput);
1054
1055         }
1056             break;
1057         case 32:
1058         {
1059             // apply defaults
1060             for (uint32_t i = 0; i < 4; ++i)
1061             {
1062                 vGatherComponents[i] = VIMMED1((int)info.defaults[i]);
1063             }
1064
1065             for(uint32_t i = 0; i < info.numComps; i++)
1066             {
1067                 uint32_t swizzleIndex = info.swizzle[i];
1068
1069                 // save mask as it is zero'd out after each gather
1070                 Value *vMask = mask;
1071
1072                 // Gather a SIMD of components
1073                 vGatherComponents[swizzleIndex] = GATHERDD(vGatherComponents[swizzleIndex], pSrcBase, byteOffsets, vMask, C((char)1));
1074
1075                 // offset base to the next component to gather
1076                 pSrcBase = GEP(pSrcBase, C((char)4));
1077             }
1078         }
1079             break;
1080         default:
1081             SWR_ASSERT(0, "unsupported format");
1082         break;
1083     }
1084 }
1085
1086 void Builder::Shuffle16bpcGather4(const SWR_FORMAT_INFO &info, Value* vGatherInput[2], Value* vGatherOutput[4], bool bPackedOutput)
1087 {
1088     // cast types
1089     Type* vGatherTy = VectorType::get(IntegerType::getInt32Ty(JM()->mContext), mVWidth);
1090     Type* v32x8Ty = VectorType::get(mInt8Ty, mVWidth * 4); // vwidth is units of 32 bits
1091
1092     // input could either be float or int vector; do shuffle work in int
1093     vGatherInput[0] = BITCAST(vGatherInput[0], mSimdInt32Ty);
1094     vGatherInput[1] = BITCAST(vGatherInput[1], mSimdInt32Ty);
1095
1096     if(bPackedOutput)
1097     {
1098         Type* v128bitTy = VectorType::get(IntegerType::getIntNTy(JM()->mContext, 128), mVWidth / 4); // vwidth is units of 32 bits
1099
1100         // shuffle mask
1101         Value* vConstMask = C<char>({0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15,
1102                                      0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15});
1103         Value* vShufResult = BITCAST(PSHUFB(BITCAST(vGatherInput[0], v32x8Ty), vConstMask), vGatherTy);
1104         // after pshufb: group components together in each 128bit lane
1105         // 256i - 0    1    2    3    4    5    6    7
1106         //        xxxx xxxx yyyy yyyy xxxx xxxx yyyy yyyy
1107
1108         Value* vi128XY = BITCAST(PERMD(vShufResult, C<int32_t>({0, 1, 4, 5, 2, 3, 6, 7})), v128bitTy);
1109         // after PERMD: move and pack xy components into each 128bit lane
1110         // 256i - 0    1    2    3    4    5    6    7
1111         //        xxxx xxxx xxxx xxxx yyyy yyyy yyyy yyyy
1112
1113         // do the same for zw components
1114         Value* vi128ZW = nullptr;
1115         if(info.numComps > 2)
1116         {
1117             Value* vShufResult = BITCAST(PSHUFB(BITCAST(vGatherInput[1], v32x8Ty), vConstMask), vGatherTy);
1118             vi128ZW = BITCAST(PERMD(vShufResult, C<int32_t>({0, 1, 4, 5, 2, 3, 6, 7})), v128bitTy);
1119         }
1120
1121         for(uint32_t i = 0; i < 4; i++)
1122         {
1123             uint32_t swizzleIndex = info.swizzle[i];
1124             // todo: fixed for packed
1125             Value* vGatherMaskedVal = VIMMED1((int32_t)(info.defaults[i]));
1126             if(i >= info.numComps)
1127             {
1128                 // set the default component val
1129                 vGatherOutput[swizzleIndex] = vGatherMaskedVal;
1130                 continue;
1131             }
1132
1133             // if x or z, extract 128bits from lane 0, else for y or w, extract from lane 1
1134             uint32_t lane = ((i == 0) || (i == 2)) ? 0 : 1;
1135             // if x or y, use vi128XY permute result, else use vi128ZW
1136             Value* selectedPermute = (i < 2) ? vi128XY : vi128ZW;
1137
1138             // extract packed component 128 bit lanes
1139             vGatherOutput[swizzleIndex] = VEXTRACT(selectedPermute, C(lane));
1140         }
1141
1142     }
1143     else
1144     {
1145         // pshufb masks for each component
1146         Value* vConstMask[2];
1147         // x/z shuffle mask
1148         vConstMask[0] = C<char>({0, 1, -1, -1, 4, 5, -1, -1, 8, 9, -1, -1, 12, 13, -1, -1,
1149                                  0, 1, -1, -1, 4, 5, -1, -1, 8, 9, -1, -1, 12, 13, -1, -1, });
1150
1151         // y/w shuffle mask
1152         vConstMask[1] = C<char>({2, 3, -1, -1, 6, 7, -1, -1, 10, 11, -1, -1, 14, 15, -1, -1,
1153                                  2, 3, -1, -1, 6, 7, -1, -1, 10, 11, -1, -1, 14, 15, -1, -1});
1154
1155
1156         // shuffle enabled components into lower word of each 32bit lane, 0 extending to 32 bits
1157         // apply defaults
1158         for (uint32_t i = 0; i < 4; ++i)
1159         {
1160             vGatherOutput[i] = VIMMED1((int32_t)info.defaults[i]);
1161         }
1162
1163         for(uint32_t i = 0; i < info.numComps; i++)
1164         {
1165             uint32_t swizzleIndex = info.swizzle[i];
1166
1167             // select correct constMask for x/z or y/w pshufb
1168             uint32_t selectedMask = ((i == 0) || (i == 2)) ? 0 : 1;
1169             // if x or y, use vi128XY permute result, else use vi128ZW
1170             uint32_t selectedGather = (i < 2) ? 0 : 1;
1171
1172             vGatherOutput[swizzleIndex] = BITCAST(PSHUFB(BITCAST(vGatherInput[selectedGather], v32x8Ty), vConstMask[selectedMask]), vGatherTy);
1173             // after pshufb mask for x channel; z uses the same shuffle from the second gather
1174             // 256i - 0    1    2    3    4    5    6    7
1175             //        xx00 xx00 xx00 xx00 xx00 xx00 xx00 xx00
1176         }
1177     }
1178 }
1179
1180 void Builder::Shuffle8bpcGather4(const SWR_FORMAT_INFO &info, Value* vGatherInput, Value* vGatherOutput[], bool bPackedOutput)
1181 {
1182     // cast types
1183     Type* vGatherTy = VectorType::get(IntegerType::getInt32Ty(JM()->mContext), mVWidth);
1184     Type* v32x8Ty =  VectorType::get(mInt8Ty, mVWidth * 4 ); // vwidth is units of 32 bits
1185
1186     if(bPackedOutput)
1187     {
1188         Type* v128Ty = VectorType::get(IntegerType::getIntNTy(JM()->mContext, 128), mVWidth / 4); // vwidth is units of 32 bits
1189         // shuffle mask
1190         Value* vConstMask = C<char>({0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15,
1191                                      0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15});
1192         Value* vShufResult = BITCAST(PSHUFB(BITCAST(vGatherInput, v32x8Ty), vConstMask), vGatherTy);
1193         // after pshufb: group components together in each 128bit lane
1194         // 256i - 0    1    2    3    4    5    6    7
1195         //        xxxx yyyy zzzz wwww xxxx yyyy zzzz wwww
1196
1197         Value* vi128XY = BITCAST(PERMD(vShufResult, C<int32_t>({0, 4, 0, 0, 1, 5, 0, 0})), v128Ty);
1198         // after PERMD: move and pack xy and zw components in low 64 bits of each 128bit lane
1199         // 256i - 0    1    2    3    4    5    6    7
1200         //        xxxx xxxx dcdc dcdc yyyy yyyy dcdc dcdc (dc - don't care)
1201
1202         // do the same for zw components
1203         Value* vi128ZW = nullptr;
1204         if(info.numComps > 2)
1205         {
1206             vi128ZW = BITCAST(PERMD(vShufResult, C<int32_t>({2, 6, 0, 0, 3, 7, 0, 0})), v128Ty);
1207         }
1208
1209         // sign extend all enabled components. If we have a fill vVertexElements, output to current simdvertex
1210         for(uint32_t i = 0; i < 4; i++)
1211         {
1212             uint32_t swizzleIndex = info.swizzle[i];
1213             // todo: fix for packed
1214             Value* vGatherMaskedVal = VIMMED1((int32_t)(info.defaults[i]));
1215             if(i >= info.numComps)
1216             {
1217                 // set the default component val
1218                 vGatherOutput[swizzleIndex] = vGatherMaskedVal;
1219                 continue;
1220             }
1221
1222             // if x or z, extract 128bits from lane 0, else for y or w, extract from lane 1
1223             uint32_t lane = ((i == 0) || (i == 2)) ? 0 : 1;
1224             // if x or y, use vi128XY permute result, else use vi128ZW
1225             Value* selectedPermute = (i < 2) ? vi128XY : vi128ZW;
1226
1227             // sign extend
1228             vGatherOutput[swizzleIndex] = VEXTRACT(selectedPermute, C(lane));
1229         }
1230     }
1231     // else zero extend
1232     else{
1233         // shuffle enabled components into lower byte of each 32bit lane, 0 extending to 32 bits
1234         // apply defaults
1235         for (uint32_t i = 0; i < 4; ++i)
1236         {
1237             vGatherOutput[i] = VIMMED1((int32_t)info.defaults[i]);
1238         }
1239
1240         for(uint32_t i = 0; i < info.numComps; i++){
1241             uint32_t swizzleIndex = info.swizzle[i];
1242
1243             // pshufb masks for each component
1244             Value* vConstMask;
1245             switch(i)
1246             {
1247                 case 0:
1248                     // x shuffle mask
1249                     vConstMask = C<char>({0, -1, -1, -1, 4, -1, -1, -1, 8, -1, -1, -1, 12, -1, -1, -1,
1250                                           0, -1, -1, -1, 4, -1, -1, -1, 8, -1, -1, -1, 12, -1, -1, -1});
1251                     break;
1252                 case 1:
1253                     // y shuffle mask
1254                     vConstMask = C<char>({1, -1, -1, -1, 5, -1, -1, -1, 9, -1, -1, -1, 13, -1, -1, -1,
1255                                           1, -1, -1, -1, 5, -1, -1, -1, 9, -1, -1, -1, 13, -1, -1, -1});
1256                     break;
1257                 case 2:
1258                     // z shuffle mask
1259                     vConstMask = C<char>({2, -1, -1, -1, 6, -1, -1, -1, 10, -1, -1, -1, 14, -1, -1, -1,
1260                                           2, -1, -1, -1, 6, -1, -1, -1, 10, -1, -1, -1, 14, -1, -1, -1});
1261                     break;
1262                 case 3:
1263                     // w shuffle mask
1264                     vConstMask = C<char>({3, -1, -1, -1, 7, -1, -1, -1, 11, -1, -1, -1, 15, -1, -1, -1,
1265                                           3, -1, -1, -1, 7, -1, -1, -1, 11, -1, -1, -1, 15, -1, -1, -1});
1266                     break;
1267                 default:
1268                     vConstMask = nullptr;
1269                     break;
1270             }
1271
1272                 vGatherOutput[swizzleIndex] = BITCAST(PSHUFB(BITCAST(vGatherInput, v32x8Ty), vConstMask), vGatherTy);
1273                 // after pshufb for x channel
1274                 // 256i - 0    1    2    3    4    5    6    7
1275                 //        x000 x000 x000 x000 x000 x000 x000 x000
1276         }
1277     }
1278 }
1279
1280 //////////////////////////////////////////////////////////////////////////
1281 /// @brief emulates a scatter operation.
1282 /// @param pDst - pointer to destination
1283 /// @param vSrc - vector of src data to scatter
1284 /// @param vOffsets - vector of byte offsets from pDst
1285 /// @param vMask - mask of valid lanes
1286 void Builder::SCATTERPS(Value* pDst, Value* vSrc, Value* vOffsets, Value* vMask)
1287 {
1288     Value* pStack = STACKSAVE();
1289
1290     Type* pSrcTy = vSrc->getType()->getVectorElementType();
1291
1292     // allocate tmp stack for masked off lanes
1293     Value* vTmpPtr = ALLOCA(pSrcTy);
1294
1295     Value *mask = MASK(vMask);
1296     for (uint32_t i = 0; i < mVWidth; ++i)
1297     {
1298         Value *offset = VEXTRACT(vOffsets, C(i));
1299         // byte pointer to component
1300         Value *storeAddress = GEP(pDst, offset);
1301         storeAddress = BITCAST(storeAddress, PointerType::get(pSrcTy, 0));
1302         Value *selMask = VEXTRACT(mask, C(i));
1303         Value *srcElem = VEXTRACT(vSrc, C(i));
1304         // switch in a safe address to load if we're trying to access a vertex
1305         Value *validAddress = SELECT(selMask, storeAddress, vTmpPtr);
1306         STORE(srcElem, validAddress);
1307     }
1308
1309     STACKRESTORE(pStack);
1310 }
1311
1312 Value* Builder::VABSPS(Value* a)
1313 {
1314     Value* asInt = BITCAST(a, mSimdInt32Ty);
1315     Value* result = BITCAST(AND(asInt, VIMMED1(0x7fffffff)), mSimdFP32Ty);
1316     return result;
1317 }
1318
1319 Value *Builder::ICLAMP(Value* src, Value* low, Value* high)
1320 {
1321     Value *lowCmp = ICMP_SLT(src, low);
1322     Value *ret = SELECT(lowCmp, low, src);
1323
1324     Value *highCmp = ICMP_SGT(ret, high);
1325     ret = SELECT(highCmp, high, ret);
1326
1327     return ret;
1328 }
1329
1330 Value *Builder::FCLAMP(Value* src, Value* low, Value* high)
1331 {
1332     Value *lowCmp = FCMP_OLT(src, low);
1333     Value *ret = SELECT(lowCmp, low, src);
1334
1335     Value *highCmp = FCMP_OGT(ret, high);
1336     ret = SELECT(highCmp, high, ret);
1337
1338     return ret;
1339 }
1340
1341 Value *Builder::FCLAMP(Value* src, float low, float high)
1342 {
1343     Value* result = VMAXPS(src, VIMMED1(low));
1344     result = VMINPS(result, VIMMED1(high));
1345
1346     return result;
1347 }
1348
1349 //////////////////////////////////////////////////////////////////////////
1350 /// @brief save/restore stack, providing ability to push/pop the stack and
1351 ///        reduce overall stack requirements for temporary stack use
1352 Value* Builder::STACKSAVE()
1353 {
1354     Function* pfnStackSave = Intrinsic::getDeclaration(JM()->mpCurrentModule, Intrinsic::stacksave);
1355 #if HAVE_LLVM == 0x306
1356     return CALL(pfnStackSave);
1357 #else
1358     return CALLA(pfnStackSave);
1359 #endif
1360 }
1361
1362 void Builder::STACKRESTORE(Value* pSaved)
1363 {
1364     Function* pfnStackRestore = Intrinsic::getDeclaration(JM()->mpCurrentModule, Intrinsic::stackrestore);
1365     CALL(pfnStackRestore, std::initializer_list<Value*>{pSaved});
1366 }
1367
1368 Value *Builder::FMADDPS(Value* a, Value* b, Value* c)
1369 {
1370     Value* vOut;
1371     // use FMADs if available
1372     if(JM()->mArch.AVX2())
1373     {
1374         vOut = VFMADDPS(a, b, c);
1375     }
1376     else
1377     {
1378         vOut = FADD(FMUL(a, b), c);
1379     }
1380     return vOut;
1381 }
1382
1383 Value* Builder::POPCNT(Value* a)
1384 {
1385     Function* pCtPop = Intrinsic::getDeclaration(JM()->mpCurrentModule, Intrinsic::ctpop, { a->getType() });
1386     return CALL(pCtPop, std::initializer_list<Value*>{a});
1387 }
1388
1389 //////////////////////////////////////////////////////////////////////////
1390 /// @brief C functions called by LLVM IR
1391 //////////////////////////////////////////////////////////////////////////
1392
1393 //////////////////////////////////////////////////////////////////////////
1394 /// @brief called in JIT code, inserted by PRINT
1395 /// output to both stdout and visual studio debug console
1396 void __cdecl CallPrint(const char* fmt, ...)
1397 {
1398     va_list args;
1399     va_start(args, fmt);
1400     vprintf(fmt, args);
1401
1402 #if defined( _WIN32 )
1403     char strBuf[1024];
1404     vsnprintf_s(strBuf, _TRUNCATE, fmt, args);
1405     OutputDebugString(strBuf);
1406 #endif
1407
1408     va_end(args);
1409 }
1410
1411 Value *Builder::VEXTRACTI128(Value* a, Constant* imm8)
1412 {
1413 #if HAVE_LLVM == 0x306
1414     Function *func =
1415         Intrinsic::getDeclaration(JM()->mpCurrentModule,
1416                                   Intrinsic::x86_avx_vextractf128_si_256);
1417     return CALL(func, {a, imm8});
1418 #else
1419     bool flag = !imm8->isZeroValue();
1420     SmallVector<Constant*,8> idx;
1421     for (unsigned i = 0; i < mVWidth / 2; i++) {
1422         idx.push_back(C(flag ? i + mVWidth / 2 : i));
1423     }
1424     return VSHUFFLE(a, VUNDEF_I(), ConstantVector::get(idx));
1425 #endif
1426 }
1427
1428 Value *Builder::VINSERTI128(Value* a, Value* b, Constant* imm8)
1429 {
1430 #if HAVE_LLVM == 0x306
1431     Function *func =
1432         Intrinsic::getDeclaration(JM()->mpCurrentModule,
1433                                   Intrinsic::x86_avx_vinsertf128_si_256);
1434     return CALL(func, {a, b, imm8});
1435 #else
1436     bool flag = !imm8->isZeroValue();
1437     SmallVector<Constant*,8> idx;
1438     for (unsigned i = 0; i < mVWidth; i++) {
1439         idx.push_back(C(i));
1440     }
1441     Value *inter = VSHUFFLE(b, VUNDEF_I(), ConstantVector::get(idx));
1442
1443     SmallVector<Constant*,8> idx2;
1444     for (unsigned i = 0; i < mVWidth / 2; i++) {
1445         idx2.push_back(C(flag ? i : i + mVWidth));
1446     }
1447     for (unsigned i = mVWidth / 2; i < mVWidth; i++) {
1448         idx2.push_back(C(flag ? i + mVWidth / 2 : i));
1449     }
1450     return VSHUFFLE(a, inter, ConstantVector::get(idx2));
1451 #endif
1452 }
1453
1454 // rdtsc buckets macros
1455 void Builder::RDTSC_START(Value* pBucketMgr, Value* pId)
1456 {
1457     std::vector<Type*> args{
1458         PointerType::get(mInt32Ty, 0),   // pBucketMgr
1459         mInt32Ty                        // id
1460     };
1461
1462     FunctionType* pFuncTy = FunctionType::get(Type::getVoidTy(JM()->mContext), args, false);
1463     Function* pFunc = cast<Function>(JM()->mpCurrentModule->getOrInsertFunction("BucketManager_StartBucket", pFuncTy));
1464     if (sys::DynamicLibrary::SearchForAddressOfSymbol("BucketManager_StartBucket") == nullptr)
1465     {
1466         sys::DynamicLibrary::AddSymbol("BucketManager_StartBucket", (void*)&BucketManager_StartBucket);
1467     }
1468
1469     CALL(pFunc, { pBucketMgr, pId });
1470 }
1471
1472 void Builder::RDTSC_STOP(Value* pBucketMgr, Value* pId)
1473 {
1474     std::vector<Type*> args{
1475         PointerType::get(mInt32Ty, 0),   // pBucketMgr
1476         mInt32Ty                        // id
1477     };
1478
1479     FunctionType* pFuncTy = FunctionType::get(Type::getVoidTy(JM()->mContext), args, false);
1480     Function* pFunc = cast<Function>(JM()->mpCurrentModule->getOrInsertFunction("BucketManager_StopBucket", pFuncTy));
1481     if (sys::DynamicLibrary::SearchForAddressOfSymbol("BucketManager_StopBucket") == nullptr)
1482     {
1483         sys::DynamicLibrary::AddSymbol("BucketManager_StopBucket", (void*)&BucketManager_StopBucket);
1484     }
1485
1486     CALL(pFunc, { pBucketMgr, pId });
1487 }
1488