src/gallium/drivers/swr/rasterizer/jitter/functionpasses/lower_x86.cpp

   1 /****************************************************************************
   2  * Copyright (C) 2014-2018 Intel Corporation.   All Rights Reserved.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * @file lower_x86.cpp
  24  *
  25  * @brief llvm pass to lower meta code to x86
  26  *
  27  * Notes:
  28  *
  29  ******************************************************************************/
  30
  31 #include "jit_pch.hpp"
  32 #include "passes.h"
  33 #include "JitManager.h"
  34
  35 #include "common/simdlib.hpp"
  36
  37 #include <unordered_map>
  38
  39 extern "C" void ScatterPS_256(uint8_t*, SIMD256::Integer, SIMD256::Float, uint8_t, uint32_t);
  40
  41 namespace llvm
  42 {
  43     // foward declare the initializer
  44     void initializeLowerX86Pass(PassRegistry&);
  45 } // namespace llvm
  46
  47 namespace SwrJit
  48 {
  49     using namespace llvm;
  50
  51 #if LLVM_VERSION_MAJOR > 10
  52     typedef unsigned IntrinsicID;
  53 #else
  54     typedef Intrinsic::ID IntrinsicID;
  55 #endif
  56
  57     enum TargetArch
  58     {
  59         AVX    = 0,
  60         AVX2   = 1,
  61         AVX512 = 2
  62     };
  63
  64     enum TargetWidth
  65     {
  66         W256       = 0,
  67         W512       = 1,
  68         NUM_WIDTHS = 2
  69     };
  70
  71     struct LowerX86;
  72
  73     typedef std::function<Instruction*(LowerX86*, TargetArch, TargetWidth, CallInst*)> EmuFunc;
  74
  75     struct X86Intrinsic
  76     {
  77         IntrinsicID intrin[NUM_WIDTHS];
  78         EmuFunc       emuFunc;
  79     };
  80
  81     // Map of intrinsics that haven't been moved to the new mechanism yet. If used, these get the
  82     // previous behavior of mapping directly to avx/avx2 intrinsics.
  83     static std::map<std::string, IntrinsicID> intrinsicMap = {
  84         {"meta.intrinsic.BEXTR_32", Intrinsic::x86_bmi_bextr_32},
  85         {"meta.intrinsic.VPSHUFB", Intrinsic::x86_avx2_pshuf_b},
  86         {"meta.intrinsic.VCVTPS2PH", Intrinsic::x86_vcvtps2ph_256},
  87         {"meta.intrinsic.VPTESTC", Intrinsic::x86_avx_ptestc_256},
  88         {"meta.intrinsic.VPTESTZ", Intrinsic::x86_avx_ptestz_256},
  89         {"meta.intrinsic.VPHADDD", Intrinsic::x86_avx2_phadd_d},
  90         {"meta.intrinsic.PDEP32", Intrinsic::x86_bmi_pdep_32},
  91         {"meta.intrinsic.RDTSC", Intrinsic::x86_rdtsc},
  92     };
  93
  94     // Forward decls
  95     Instruction* NO_EMU(LowerX86* pThis, TargetArch arch, TargetWidth width, CallInst* pCallInst);
  96     Instruction*
  97     VPERM_EMU(LowerX86* pThis, TargetArch arch, TargetWidth width, CallInst* pCallInst);
  98     Instruction*
  99     VGATHER_EMU(LowerX86* pThis, TargetArch arch, TargetWidth width, CallInst* pCallInst);
 100     Instruction*
 101     VSCATTER_EMU(LowerX86* pThis, TargetArch arch, TargetWidth width, CallInst* pCallInst);
 102     Instruction*
 103     VROUND_EMU(LowerX86* pThis, TargetArch arch, TargetWidth width, CallInst* pCallInst);
 104     Instruction*
 105     VHSUB_EMU(LowerX86* pThis, TargetArch arch, TargetWidth width, CallInst* pCallInst);
 106     Instruction*
 107     VCONVERT_EMU(LowerX86* pThis, TargetArch arch, TargetWidth width, CallInst* pCallInst);
 108
 109     Instruction* DOUBLE_EMU(LowerX86*     pThis,
 110                             TargetArch    arch,
 111                             TargetWidth   width,
 112                             CallInst*     pCallInst,
 113                             Intrinsic::ID intrin);
 114
 115     static Intrinsic::ID DOUBLE = (Intrinsic::ID)-1;
 116
 117     // clang-format off
 118     static std::map<std::string, X86Intrinsic> intrinsicMap2[] = {
 119         //                               256 wide                               512 wide
 120         {
 121             // AVX
 122             {"meta.intrinsic.VRCPPS",    {{Intrinsic::x86_avx_rcp_ps_256,       DOUBLE},                    NO_EMU}},
 123             {"meta.intrinsic.VPERMPS",   {{Intrinsic::not_intrinsic,            Intrinsic::not_intrinsic},  VPERM_EMU}},
 124             {"meta.intrinsic.VPERMD",    {{Intrinsic::not_intrinsic,            Intrinsic::not_intrinsic},  VPERM_EMU}},
 125             {"meta.intrinsic.VGATHERPD", {{Intrinsic::not_intrinsic,            Intrinsic::not_intrinsic},  VGATHER_EMU}},
 126             {"meta.intrinsic.VGATHERPS", {{Intrinsic::not_intrinsic,            Intrinsic::not_intrinsic},  VGATHER_EMU}},
 127             {"meta.intrinsic.VGATHERDD", {{Intrinsic::not_intrinsic,            Intrinsic::not_intrinsic},  VGATHER_EMU}},
 128             {"meta.intrinsic.VSCATTERPS", {{Intrinsic::not_intrinsic,           Intrinsic::not_intrinsic}, VSCATTER_EMU}},
 129             {"meta.intrinsic.VCVTPD2PS", {{Intrinsic::x86_avx_cvt_pd2_ps_256,   Intrinsic::not_intrinsic},  NO_EMU}},
 130             {"meta.intrinsic.VCVTPH2PS", {{Intrinsic::x86_vcvtph2ps_256,        Intrinsic::not_intrinsic},  NO_EMU}},
 131             {"meta.intrinsic.VROUND",    {{Intrinsic::x86_avx_round_ps_256,     DOUBLE},                    NO_EMU}},
 132             {"meta.intrinsic.VHSUBPS",   {{Intrinsic::x86_avx_hsub_ps_256,      DOUBLE},                    NO_EMU}},
 133         },
 134         {
 135             // AVX2
 136             {"meta.intrinsic.VRCPPS",       {{Intrinsic::x86_avx_rcp_ps_256,    DOUBLE},                    NO_EMU}},
 137             {"meta.intrinsic.VPERMPS",      {{Intrinsic::x86_avx2_permps,       Intrinsic::not_intrinsic},  VPERM_EMU}},
 138             {"meta.intrinsic.VPERMD",       {{Intrinsic::x86_avx2_permd,        Intrinsic::not_intrinsic},  VPERM_EMU}},
 139             {"meta.intrinsic.VGATHERPD",    {{Intrinsic::not_intrinsic,         Intrinsic::not_intrinsic},  VGATHER_EMU}},
 140             {"meta.intrinsic.VGATHERPS",    {{Intrinsic::not_intrinsic,         Intrinsic::not_intrinsic},  VGATHER_EMU}},
 141             {"meta.intrinsic.VGATHERDD",    {{Intrinsic::not_intrinsic,         Intrinsic::not_intrinsic},  VGATHER_EMU}},
 142             {"meta.intrinsic.VSCATTERPS", {{Intrinsic::not_intrinsic,           Intrinsic::not_intrinsic}, VSCATTER_EMU}},
 143             {"meta.intrinsic.VCVTPD2PS",    {{Intrinsic::x86_avx_cvt_pd2_ps_256, DOUBLE},                   NO_EMU}},
 144             {"meta.intrinsic.VCVTPH2PS",    {{Intrinsic::x86_vcvtph2ps_256,     Intrinsic::not_intrinsic},  NO_EMU}},
 145             {"meta.intrinsic.VROUND",       {{Intrinsic::x86_avx_round_ps_256,  DOUBLE},                    NO_EMU}},
 146             {"meta.intrinsic.VHSUBPS",      {{Intrinsic::x86_avx_hsub_ps_256,   DOUBLE},                    NO_EMU}},
 147         },
 148         {
 149             // AVX512
 150             {"meta.intrinsic.VRCPPS", {{Intrinsic::x86_avx512_rcp14_ps_256,     Intrinsic::x86_avx512_rcp14_ps_512}, NO_EMU}},
 151 #if LLVM_VERSION_MAJOR < 7
 152             {"meta.intrinsic.VPERMPS", {{Intrinsic::x86_avx512_mask_permvar_sf_256, Intrinsic::x86_avx512_mask_permvar_sf_512}, NO_EMU}},
 153             {"meta.intrinsic.VPERMD", {{Intrinsic::x86_avx512_mask_permvar_si_256, Intrinsic::x86_avx512_mask_permvar_si_512}, NO_EMU}},
 154 #else
 155             {"meta.intrinsic.VPERMPS", {{Intrinsic::not_intrinsic,              Intrinsic::not_intrinsic}, VPERM_EMU}},
 156             {"meta.intrinsic.VPERMD", {{Intrinsic::not_intrinsic,               Intrinsic::not_intrinsic}, VPERM_EMU}},
 157 #endif
 158             {"meta.intrinsic.VGATHERPD", {{Intrinsic::not_intrinsic,            Intrinsic::not_intrinsic}, VGATHER_EMU}},
 159             {"meta.intrinsic.VGATHERPS", {{Intrinsic::not_intrinsic,            Intrinsic::not_intrinsic}, VGATHER_EMU}},
 160             {"meta.intrinsic.VGATHERDD", {{Intrinsic::not_intrinsic,            Intrinsic::not_intrinsic}, VGATHER_EMU}},
 161             {"meta.intrinsic.VSCATTERPS", {{Intrinsic::not_intrinsic,           Intrinsic::not_intrinsic}, VSCATTER_EMU}},
 162 #if LLVM_VERSION_MAJOR < 7
 163             {"meta.intrinsic.VCVTPD2PS", {{Intrinsic::x86_avx512_mask_cvtpd2ps_256, Intrinsic::x86_avx512_mask_cvtpd2ps_512}, NO_EMU}},
 164 #else
 165             {"meta.intrinsic.VCVTPD2PS", {{Intrinsic::not_intrinsic,            Intrinsic::not_intrinsic}, VCONVERT_EMU}},
 166 #endif
 167             {"meta.intrinsic.VCVTPH2PS", {{Intrinsic::x86_avx512_mask_vcvtph2ps_256, Intrinsic::x86_avx512_mask_vcvtph2ps_512}, NO_EMU}},
 168             {"meta.intrinsic.VROUND", {{Intrinsic::not_intrinsic,               Intrinsic::not_intrinsic}, VROUND_EMU}},
 169             {"meta.intrinsic.VHSUBPS", {{Intrinsic::not_intrinsic,              Intrinsic::not_intrinsic}, VHSUB_EMU}},
 170         }};
 171     // clang-format on
 172
 173     struct LowerX86 : public FunctionPass
 174     {
 175         LowerX86(Builder* b = nullptr) : FunctionPass(ID), B(b)
 176         {
 177             initializeLowerX86Pass(*PassRegistry::getPassRegistry());
 178
 179             // Determine target arch
 180             if (JM()->mArch.AVX512F())
 181             {
 182                 mTarget = AVX512;
 183             }
 184             else if (JM()->mArch.AVX2())
 185             {
 186                 mTarget = AVX2;
 187             }
 188             else if (JM()->mArch.AVX())
 189             {
 190                 mTarget = AVX;
 191             }
 192             else
 193             {
 194                 SWR_ASSERT(false, "Unsupported AVX architecture.");
 195                 mTarget = AVX;
 196             }
 197
 198             // Setup scatter function for 256 wide
 199             uint32_t curWidth = B->mVWidth;
 200             B->SetTargetWidth(8);
 201             std::vector<Type*> args = {
 202                 B->mInt8PtrTy,   // pBase
 203                 B->mSimdInt32Ty, // vIndices
 204                 B->mSimdFP32Ty,  // vSrc
 205                 B->mInt8Ty,      // mask
 206                 B->mInt32Ty      // scale
 207             };
 208
 209             FunctionType* pfnScatterTy = FunctionType::get(B->mVoidTy, args, false);
 210             mPfnScatter256             = cast<Function>(
 211 #if LLVM_VERSION_MAJOR >= 9
 212                 B->JM()->mpCurrentModule->getOrInsertFunction("ScatterPS_256", pfnScatterTy).getCallee());
 213 #else
 214                 B->JM()->mpCurrentModule->getOrInsertFunction("ScatterPS_256", pfnScatterTy));
 215 #endif
 216             if (sys::DynamicLibrary::SearchForAddressOfSymbol("ScatterPS_256") == nullptr)
 217             {
 218                 sys::DynamicLibrary::AddSymbol("ScatterPS_256", (void*)&ScatterPS_256);
 219             }
 220
 221             B->SetTargetWidth(curWidth);
 222         }
 223
 224         // Try to decipher the vector type of the instruction. This does not work properly
 225         // across all intrinsics, and will have to be rethought. Probably need something
 226         // similar to llvm's getDeclaration() utility to map a set of inputs to a specific typed
 227         // intrinsic.
 228         void GetRequestedWidthAndType(CallInst*       pCallInst,
 229                                       const StringRef intrinName,
 230                                       TargetWidth*    pWidth,
 231                                       Type**          pTy)
 232         {
 233             assert(pCallInst);
 234             Type* pVecTy = pCallInst->getType();
 235
 236             // Check for intrinsic specific types
 237             // VCVTPD2PS type comes from src, not dst
 238             if (intrinName.equals("meta.intrinsic.VCVTPD2PS"))
 239             {
 240                 Value* pOp = pCallInst->getOperand(0);
 241                 assert(pOp);
 242                 pVecTy = pOp->getType();
 243             }
 244
 245             if (!pVecTy->isVectorTy())
 246             {
 247                 for (auto& op : pCallInst->arg_operands())
 248                 {
 249                     if (op.get()->getType()->isVectorTy())
 250                     {
 251                         pVecTy = op.get()->getType();
 252                         break;
 253                     }
 254                 }
 255             }
 256             SWR_ASSERT(pVecTy->isVectorTy(), "Couldn't determine vector size");
 257
 258             uint32_t width = cast<VectorType>(pVecTy)->getBitWidth();
 259             switch (width)
 260             {
 261             case 256:
 262                 *pWidth = W256;
 263                 break;
 264             case 512:
 265                 *pWidth = W512;
 266                 break;
 267             default:
 268                 SWR_ASSERT(false, "Unhandled vector width %d", width);
 269                 *pWidth = W256;
 270             }
 271
 272             *pTy = pVecTy->getScalarType();
 273         }
 274
 275         Value* GetZeroVec(TargetWidth width, Type* pTy)
 276         {
 277             uint32_t numElem = 0;
 278             switch (width)
 279             {
 280             case W256:
 281                 numElem = 8;
 282                 break;
 283             case W512:
 284                 numElem = 16;
 285                 break;
 286             default:
 287                 SWR_ASSERT(false, "Unhandled vector width type %d\n", width);
 288             }
 289
 290             return ConstantVector::getNullValue(VectorType::get(pTy, numElem));
 291         }
 292
 293         Value* GetMask(TargetWidth width)
 294         {
 295             Value* mask;
 296             switch (width)
 297             {
 298             case W256:
 299                 mask = B->C((uint8_t)-1);
 300                 break;
 301             case W512:
 302                 mask = B->C((uint16_t)-1);
 303                 break;
 304             default:
 305                 SWR_ASSERT(false, "Unhandled vector width type %d\n", width);
 306             }
 307             return mask;
 308         }
 309
 310         // Convert <N x i1> mask to <N x i32> x86 mask
 311         Value* VectorMask(Value* vi1Mask)
 312         {
 313             uint32_t numElem = vi1Mask->getType()->getVectorNumElements();
 314             return B->S_EXT(vi1Mask, VectorType::get(B->mInt32Ty, numElem));
 315         }
 316
 317         Instruction* ProcessIntrinsicAdvanced(CallInst* pCallInst)
 318         {
 319             Function*   pFunc = pCallInst->getCalledFunction();
 320             assert(pFunc);
 321
 322             auto&       intrinsic = intrinsicMap2[mTarget][pFunc->getName().str()];
 323             TargetWidth vecWidth;
 324             Type*       pElemTy;
 325             GetRequestedWidthAndType(pCallInst, pFunc->getName(), &vecWidth, &pElemTy);
 326
 327             // Check if there is a native intrinsic for this instruction
 328             IntrinsicID id = intrinsic.intrin[vecWidth];
 329             if (id == DOUBLE)
 330             {
 331                 // Double pump the next smaller SIMD intrinsic
 332                 SWR_ASSERT(vecWidth != 0, "Cannot double pump smallest SIMD width.");
 333                 Intrinsic::ID id2 = intrinsic.intrin[vecWidth - 1];
 334                 SWR_ASSERT(id2 != Intrinsic::not_intrinsic,
 335                            "Cannot find intrinsic to double pump.");
 336                 return DOUBLE_EMU(this, mTarget, vecWidth, pCallInst, id2);
 337             }
 338             else if (id != Intrinsic::not_intrinsic)
 339             {
 340                 Function* pIntrin = Intrinsic::getDeclaration(B->JM()->mpCurrentModule, id);
 341                 SmallVector<Value*, 8> args;
 342                 for (auto& arg : pCallInst->arg_operands())
 343                 {
 344                     args.push_back(arg.get());
 345                 }
 346
 347                 // If AVX512, all instructions add a src operand and mask. We'll pass in 0 src and
 348                 // full mask for now Assuming the intrinsics are consistent and place the src
 349                 // operand and mask last in the argument list.
 350                 if (mTarget == AVX512)
 351                 {
 352                     if (pFunc->getName().equals("meta.intrinsic.VCVTPD2PS"))
 353                     {
 354                         args.push_back(GetZeroVec(W256, pCallInst->getType()->getScalarType()));
 355                         args.push_back(GetMask(W256));
 356                         // for AVX512 VCVTPD2PS, we also have to add rounding mode
 357                         args.push_back(B->C(_MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC));
 358                     }
 359                     else
 360                     {
 361                         args.push_back(GetZeroVec(vecWidth, pElemTy));
 362                         args.push_back(GetMask(vecWidth));
 363                     }
 364                 }
 365
 366                 return B->CALLA(pIntrin, args);
 367             }
 368             else
 369             {
 370                 // No native intrinsic, call emulation function
 371                 return intrinsic.emuFunc(this, mTarget, vecWidth, pCallInst);
 372             }
 373
 374             SWR_ASSERT(false);
 375             return nullptr;
 376         }
 377
 378         Instruction* ProcessIntrinsic(CallInst* pCallInst)
 379         {
 380             Function* pFunc = pCallInst->getCalledFunction();
 381             assert(pFunc);
 382
 383             // Forward to the advanced support if found
 384             if (intrinsicMap2[mTarget].find(pFunc->getName().str()) != intrinsicMap2[mTarget].end())
 385             {
 386                 return ProcessIntrinsicAdvanced(pCallInst);
 387             }
 388
 389             SWR_ASSERT(intrinsicMap.find(pFunc->getName().str()) != intrinsicMap.end(),
 390                        "Unimplemented intrinsic %s.",
 391                        pFunc->getName().str().c_str());
 392
 393             Intrinsic::ID x86Intrinsic = intrinsicMap[pFunc->getName().str()];
 394             Function*     pX86IntrinFunc =
 395                 Intrinsic::getDeclaration(B->JM()->mpCurrentModule, x86Intrinsic);
 396
 397             SmallVector<Value*, 8> args;
 398             for (auto& arg : pCallInst->arg_operands())
 399             {
 400                 args.push_back(arg.get());
 401             }
 402             return B->CALLA(pX86IntrinFunc, args);
 403         }
 404
 405         //////////////////////////////////////////////////////////////////////////
 406         /// @brief LLVM funtion pass run method.
 407         /// @param f- The function we're working on with this pass.
 408         virtual bool runOnFunction(Function& F)
 409         {
 410             std::vector<Instruction*> toRemove;
 411             std::vector<BasicBlock*>  bbs;
 412
 413             // Make temp copy of the basic blocks and instructions, as the intrinsic
 414             // replacement code might invalidate the iterators
 415             for (auto& b : F.getBasicBlockList())
 416             {
 417                 bbs.push_back(&b);
 418             }
 419
 420             for (auto* BB : bbs)
 421             {
 422                 std::vector<Instruction*> insts;
 423                 for (auto& i : BB->getInstList())
 424                 {
 425                     insts.push_back(&i);
 426                 }
 427
 428                 for (auto* I : insts)
 429                 {
 430                     if (CallInst* pCallInst = dyn_cast<CallInst>(I))
 431                     {
 432                         Function* pFunc = pCallInst->getCalledFunction();
 433                         if (pFunc)
 434                         {
 435                             if (pFunc->getName().startswith("meta.intrinsic"))
 436                             {
 437                                 B->IRB()->SetInsertPoint(I);
 438                                 Instruction* pReplace = ProcessIntrinsic(pCallInst);
 439                                 toRemove.push_back(pCallInst);
 440                                 if (pReplace)
 441                                 {
 442                                     pCallInst->replaceAllUsesWith(pReplace);
 443                                 }
 444                             }
 445                         }
 446                     }
 447                 }
 448             }
 449
 450             for (auto* pInst : toRemove)
 451             {
 452                 pInst->eraseFromParent();
 453             }
 454
 455             JitManager::DumpToFile(&F, "lowerx86");
 456
 457             return true;
 458         }
 459
 460         virtual void getAnalysisUsage(AnalysisUsage& AU) const {}
 461
 462         JitManager* JM() { return B->JM(); }
 463         Builder*    B;
 464         TargetArch  mTarget;
 465         Function*   mPfnScatter256;
 466
 467         static char ID; ///< Needed by LLVM to generate ID for FunctionPass.
 468     };
 469
 470     char LowerX86::ID = 0; // LLVM uses address of ID as the actual ID.
 471
 472     FunctionPass* createLowerX86Pass(Builder* b) { return new LowerX86(b); }
 473
 474     Instruction* NO_EMU(LowerX86* pThis, TargetArch arch, TargetWidth width, CallInst* pCallInst)
 475     {
 476         SWR_ASSERT(false, "Unimplemented intrinsic emulation.");
 477         return nullptr;
 478     }
 479
 480     Instruction* VPERM_EMU(LowerX86* pThis, TargetArch arch, TargetWidth width, CallInst* pCallInst)
 481     {
 482         // Only need vperm emulation for AVX
 483         SWR_ASSERT(arch == AVX);
 484
 485         Builder* B         = pThis->B;
 486         auto     v32A      = pCallInst->getArgOperand(0);
 487         auto     vi32Index = pCallInst->getArgOperand(1);
 488
 489         Value* v32Result;
 490         if (isa<Constant>(vi32Index))
 491         {
 492             // Can use llvm shuffle vector directly with constant shuffle indices
 493             v32Result = B->VSHUFFLE(v32A, v32A, vi32Index);
 494         }
 495         else
 496         {
 497             v32Result = UndefValue::get(v32A->getType());
 498             for (uint32_t l = 0; l < v32A->getType()->getVectorNumElements(); ++l)
 499             {
 500                 auto i32Index = B->VEXTRACT(vi32Index, B->C(l));
 501                 auto val      = B->VEXTRACT(v32A, i32Index);
 502                 v32Result     = B->VINSERT(v32Result, val, B->C(l));
 503             }
 504         }
 505         return cast<Instruction>(v32Result);
 506     }
 507
 508     Instruction*
 509     VGATHER_EMU(LowerX86* pThis, TargetArch arch, TargetWidth width, CallInst* pCallInst)
 510     {
 511         Builder* B           = pThis->B;
 512         auto     vSrc        = pCallInst->getArgOperand(0);
 513         auto     pBase       = pCallInst->getArgOperand(1);
 514         auto     vi32Indices = pCallInst->getArgOperand(2);
 515         auto     vi1Mask     = pCallInst->getArgOperand(3);
 516         auto     i8Scale     = pCallInst->getArgOperand(4);
 517
 518         pBase             = B->POINTER_CAST(pBase, PointerType::get(B->mInt8Ty, 0));
 519         uint32_t numElem  = vSrc->getType()->getVectorNumElements();
 520         auto     i32Scale = B->Z_EXT(i8Scale, B->mInt32Ty);
 521         auto     srcTy    = vSrc->getType()->getVectorElementType();
 522         Value*   v32Gather = nullptr;
 523         if (arch == AVX)
 524         {
 525             // Full emulation for AVX
 526             // Store source on stack to provide a valid address to load from inactive lanes
 527             auto pStack = B->STACKSAVE();
 528             auto pTmp   = B->ALLOCA(vSrc->getType());
 529             B->STORE(vSrc, pTmp);
 530
 531             v32Gather        = UndefValue::get(vSrc->getType());
 532             auto vi32Scale   = ConstantVector::getSplat(numElem, cast<ConstantInt>(i32Scale));
 533             auto vi32Offsets = B->MUL(vi32Indices, vi32Scale);
 534
 535             for (uint32_t i = 0; i < numElem; ++i)
 536             {
 537                 auto i32Offset          = B->VEXTRACT(vi32Offsets, B->C(i));
 538                 auto pLoadAddress       = B->GEP(pBase, i32Offset);
 539                 pLoadAddress            = B->BITCAST(pLoadAddress, PointerType::get(srcTy, 0));
 540                 auto pMaskedLoadAddress = B->GEP(pTmp, {0, i});
 541                 auto i1Mask             = B->VEXTRACT(vi1Mask, B->C(i));
 542                 auto pValidAddress      = B->SELECT(i1Mask, pLoadAddress, pMaskedLoadAddress);
 543                 auto val                = B->LOAD(pValidAddress);
 544                 v32Gather               = B->VINSERT(v32Gather, val, B->C(i));
 545             }
 546
 547             B->STACKRESTORE(pStack);
 548         }
 549         else if (arch == AVX2 || (arch == AVX512 && width == W256))
 550         {
 551             Function* pX86IntrinFunc = nullptr;
 552             if (srcTy == B->mFP32Ty)
 553             {
 554                 pX86IntrinFunc = Intrinsic::getDeclaration(B->JM()->mpCurrentModule,
 555                                                            Intrinsic::x86_avx2_gather_d_ps_256);
 556             }
 557             else if (srcTy == B->mInt32Ty)
 558             {
 559                 pX86IntrinFunc = Intrinsic::getDeclaration(B->JM()->mpCurrentModule,
 560                                                            Intrinsic::x86_avx2_gather_d_d_256);
 561             }
 562             else if (srcTy == B->mDoubleTy)
 563             {
 564                 pX86IntrinFunc = Intrinsic::getDeclaration(B->JM()->mpCurrentModule,
 565                                                            Intrinsic::x86_avx2_gather_d_q_256);
 566             }
 567             else
 568             {
 569                 SWR_ASSERT(false, "Unsupported vector element type for gather.");
 570             }
 571
 572             if (width == W256)
 573             {
 574                 auto v32Mask = B->BITCAST(pThis->VectorMask(vi1Mask), vSrc->getType());
 575                 v32Gather = B->CALL(pX86IntrinFunc, {vSrc, pBase, vi32Indices, v32Mask, i8Scale});
 576             }
 577             else if (width == W512)
 578             {
 579                 // Double pump 4-wide for 64bit elements
 580                 if (vSrc->getType()->getVectorElementType() == B->mDoubleTy)
 581                 {
 582                     auto v64Mask = pThis->VectorMask(vi1Mask);
 583                     v64Mask      = B->S_EXT(
 584                         v64Mask,
 585                         VectorType::get(B->mInt64Ty, v64Mask->getType()->getVectorNumElements()));
 586                     v64Mask = B->BITCAST(v64Mask, vSrc->getType());
 587
 588                     Value* src0 = B->VSHUFFLE(vSrc, vSrc, B->C({0, 1, 2, 3}));
 589                     Value* src1 = B->VSHUFFLE(vSrc, vSrc, B->C({4, 5, 6, 7}));
 590
 591                     Value* indices0 = B->VSHUFFLE(vi32Indices, vi32Indices, B->C({0, 1, 2, 3}));
 592                     Value* indices1 = B->VSHUFFLE(vi32Indices, vi32Indices, B->C({4, 5, 6, 7}));
 593
 594                     Value* mask0 = B->VSHUFFLE(v64Mask, v64Mask, B->C({0, 1, 2, 3}));
 595                     Value* mask1 = B->VSHUFFLE(v64Mask, v64Mask, B->C({4, 5, 6, 7}));
 596
 597                     src0 = B->BITCAST(
 598                         src0,
 599                         VectorType::get(B->mInt64Ty, src0->getType()->getVectorNumElements()));
 600                     mask0 = B->BITCAST(
 601                         mask0,
 602                         VectorType::get(B->mInt64Ty, mask0->getType()->getVectorNumElements()));
 603                     Value* gather0 =
 604                         B->CALL(pX86IntrinFunc, {src0, pBase, indices0, mask0, i8Scale});
 605                     src1 = B->BITCAST(
 606                         src1,
 607                         VectorType::get(B->mInt64Ty, src1->getType()->getVectorNumElements()));
 608                     mask1 = B->BITCAST(
 609                         mask1,
 610                         VectorType::get(B->mInt64Ty, mask1->getType()->getVectorNumElements()));
 611                     Value* gather1 =
 612                         B->CALL(pX86IntrinFunc, {src1, pBase, indices1, mask1, i8Scale});
 613
 614                     v32Gather = B->VSHUFFLE(gather0, gather1, B->C({0, 1, 2, 3, 4, 5, 6, 7}));
 615                     v32Gather = B->BITCAST(v32Gather, vSrc->getType());
 616                 }
 617                 else
 618                 {
 619                     // Double pump 8-wide for 32bit elements
 620                     auto v32Mask = pThis->VectorMask(vi1Mask);
 621                     v32Mask      = B->BITCAST(v32Mask, vSrc->getType());
 622                     Value* src0  = B->EXTRACT_16(vSrc, 0);
 623                     Value* src1  = B->EXTRACT_16(vSrc, 1);
 624
 625                     Value* indices0 = B->EXTRACT_16(vi32Indices, 0);
 626                     Value* indices1 = B->EXTRACT_16(vi32Indices, 1);
 627
 628                     Value* mask0 = B->EXTRACT_16(v32Mask, 0);
 629                     Value* mask1 = B->EXTRACT_16(v32Mask, 1);
 630
 631                     Value* gather0 =
 632                         B->CALL(pX86IntrinFunc, {src0, pBase, indices0, mask0, i8Scale});
 633                     Value* gather1 =
 634                         B->CALL(pX86IntrinFunc, {src1, pBase, indices1, mask1, i8Scale});
 635
 636                     v32Gather = B->JOIN_16(gather0, gather1);
 637                 }
 638             }
 639         }
 640         else if (arch == AVX512)
 641         {
 642             Value*    iMask = nullptr;
 643             Function* pX86IntrinFunc = nullptr;
 644             if (srcTy == B->mFP32Ty)
 645             {
 646                 pX86IntrinFunc = Intrinsic::getDeclaration(B->JM()->mpCurrentModule,
 647                                                            Intrinsic::x86_avx512_gather_dps_512);
 648                 iMask          = B->BITCAST(vi1Mask, B->mInt16Ty);
 649             }
 650             else if (srcTy == B->mInt32Ty)
 651             {
 652                 pX86IntrinFunc = Intrinsic::getDeclaration(B->JM()->mpCurrentModule,
 653                                                            Intrinsic::x86_avx512_gather_dpi_512);
 654                 iMask          = B->BITCAST(vi1Mask, B->mInt16Ty);
 655             }
 656             else if (srcTy == B->mDoubleTy)
 657             {
 658                 pX86IntrinFunc = Intrinsic::getDeclaration(B->JM()->mpCurrentModule,
 659                                                            Intrinsic::x86_avx512_gather_dpd_512);
 660                 iMask          = B->BITCAST(vi1Mask, B->mInt8Ty);
 661             }
 662             else
 663             {
 664                 SWR_ASSERT(false, "Unsupported vector element type for gather.");
 665             }
 666
 667             auto i32Scale = B->Z_EXT(i8Scale, B->mInt32Ty);
 668             v32Gather     = B->CALL(pX86IntrinFunc, {vSrc, pBase, vi32Indices, iMask, i32Scale});
 669         }
 670
 671         return cast<Instruction>(v32Gather);
 672     }
 673     Instruction*
 674     VSCATTER_EMU(LowerX86* pThis, TargetArch arch, TargetWidth width, CallInst* pCallInst)
 675     {
 676         Builder* B           = pThis->B;
 677         auto     pBase       = pCallInst->getArgOperand(0);
 678         auto     vi1Mask     = pCallInst->getArgOperand(1);
 679         auto     vi32Indices = pCallInst->getArgOperand(2);
 680         auto     v32Src      = pCallInst->getArgOperand(3);
 681         auto     i32Scale    = pCallInst->getArgOperand(4);
 682
 683         if (arch != AVX512)
 684         {
 685             // Call into C function to do the scatter. This has significantly better compile perf
 686             // compared to jitting scatter loops for every scatter
 687             if (width == W256)
 688             {
 689                 auto mask = B->BITCAST(vi1Mask, B->mInt8Ty);
 690                 B->CALL(pThis->mPfnScatter256, {pBase, vi32Indices, v32Src, mask, i32Scale});
 691             }
 692             else
 693             {
 694                 // Need to break up 512 wide scatter to two 256 wide
 695                 auto maskLo = B->VSHUFFLE(vi1Mask, vi1Mask, B->C({0, 1, 2, 3, 4, 5, 6, 7}));
 696                 auto indicesLo =
 697                     B->VSHUFFLE(vi32Indices, vi32Indices, B->C({0, 1, 2, 3, 4, 5, 6, 7}));
 698                 auto srcLo = B->VSHUFFLE(v32Src, v32Src, B->C({0, 1, 2, 3, 4, 5, 6, 7}));
 699
 700                 auto mask = B->BITCAST(maskLo, B->mInt8Ty);
 701                 B->CALL(pThis->mPfnScatter256, {pBase, indicesLo, srcLo, mask, i32Scale});
 702
 703                 auto maskHi = B->VSHUFFLE(vi1Mask, vi1Mask, B->C({8, 9, 10, 11, 12, 13, 14, 15}));
 704                 auto indicesHi =
 705                     B->VSHUFFLE(vi32Indices, vi32Indices, B->C({8, 9, 10, 11, 12, 13, 14, 15}));
 706                 auto srcHi = B->VSHUFFLE(v32Src, v32Src, B->C({8, 9, 10, 11, 12, 13, 14, 15}));
 707
 708                 mask = B->BITCAST(maskHi, B->mInt8Ty);
 709                 B->CALL(pThis->mPfnScatter256, {pBase, indicesHi, srcHi, mask, i32Scale});
 710             }
 711             return nullptr;
 712         }
 713
 714         Value*    iMask;
 715         Function* pX86IntrinFunc;
 716         if (width == W256)
 717         {
 718             // No direct intrinsic supported in llvm to scatter 8 elem with 32bit indices, but we
 719             // can use the scatter of 8 elements with 64bit indices
 720             pX86IntrinFunc = Intrinsic::getDeclaration(B->JM()->mpCurrentModule,
 721                                                        Intrinsic::x86_avx512_scatter_qps_512);
 722
 723             auto vi32IndicesExt = B->Z_EXT(vi32Indices, B->mSimdInt64Ty);
 724             iMask               = B->BITCAST(vi1Mask, B->mInt8Ty);
 725             B->CALL(pX86IntrinFunc, {pBase, iMask, vi32IndicesExt, v32Src, i32Scale});
 726         }
 727         else if (width == W512)
 728         {
 729             pX86IntrinFunc = Intrinsic::getDeclaration(B->JM()->mpCurrentModule,
 730                                                        Intrinsic::x86_avx512_scatter_dps_512);
 731             iMask          = B->BITCAST(vi1Mask, B->mInt16Ty);
 732             B->CALL(pX86IntrinFunc, {pBase, iMask, vi32Indices, v32Src, i32Scale});
 733         }
 734         return nullptr;
 735     }
 736
 737     // No support for vroundps in avx512 (it is available in kncni), so emulate with avx
 738     // instructions
 739     Instruction*
 740     VROUND_EMU(LowerX86* pThis, TargetArch arch, TargetWidth width, CallInst* pCallInst)
 741     {
 742         SWR_ASSERT(arch == AVX512);
 743
 744         auto B       = pThis->B;
 745         auto vf32Src = pCallInst->getOperand(0);
 746         assert(vf32Src);
 747         auto i8Round = pCallInst->getOperand(1);
 748         assert(i8Round);
 749         auto pfnFunc =
 750             Intrinsic::getDeclaration(B->JM()->mpCurrentModule, Intrinsic::x86_avx_round_ps_256);
 751
 752         if (width == W256)
 753         {
 754             return cast<Instruction>(B->CALL2(pfnFunc, vf32Src, i8Round));
 755         }
 756         else if (width == W512)
 757         {
 758             auto v8f32SrcLo = B->EXTRACT_16(vf32Src, 0);
 759             auto v8f32SrcHi = B->EXTRACT_16(vf32Src, 1);
 760
 761             auto v8f32ResLo = B->CALL2(pfnFunc, v8f32SrcLo, i8Round);
 762             auto v8f32ResHi = B->CALL2(pfnFunc, v8f32SrcHi, i8Round);
 763
 764             return cast<Instruction>(B->JOIN_16(v8f32ResLo, v8f32ResHi));
 765         }
 766         else
 767         {
 768             SWR_ASSERT(false, "Unimplemented vector width.");
 769         }
 770
 771         return nullptr;
 772     }
 773
 774     Instruction*
 775     VCONVERT_EMU(LowerX86* pThis, TargetArch arch, TargetWidth width, CallInst* pCallInst)
 776     {
 777         SWR_ASSERT(arch == AVX512);
 778
 779         auto B       = pThis->B;
 780         auto vf32Src = pCallInst->getOperand(0);
 781
 782         if (width == W256)
 783         {
 784             auto vf32SrcRound = Intrinsic::getDeclaration(B->JM()->mpCurrentModule,
 785                                                           Intrinsic::x86_avx_round_ps_256);
 786             return cast<Instruction>(B->FP_TRUNC(vf32SrcRound, B->mFP32Ty));
 787         }
 788         else if (width == W512)
 789         {
 790             // 512 can use intrinsic
 791             auto pfnFunc = Intrinsic::getDeclaration(B->JM()->mpCurrentModule,
 792                                                      Intrinsic::x86_avx512_mask_cvtpd2ps_512);
 793             return cast<Instruction>(B->CALL(pfnFunc, vf32Src));
 794         }
 795         else
 796         {
 797             SWR_ASSERT(false, "Unimplemented vector width.");
 798         }
 799
 800         return nullptr;
 801     }
 802
 803     // No support for hsub in AVX512
 804     Instruction* VHSUB_EMU(LowerX86* pThis, TargetArch arch, TargetWidth width, CallInst* pCallInst)
 805     {
 806         SWR_ASSERT(arch == AVX512);
 807
 808         auto B    = pThis->B;
 809         auto src0 = pCallInst->getOperand(0);
 810         auto src1 = pCallInst->getOperand(1);
 811
 812         // 256b hsub can just use avx intrinsic
 813         if (width == W256)
 814         {
 815             auto pX86IntrinFunc =
 816                 Intrinsic::getDeclaration(B->JM()->mpCurrentModule, Intrinsic::x86_avx_hsub_ps_256);
 817             return cast<Instruction>(B->CALL2(pX86IntrinFunc, src0, src1));
 818         }
 819         else if (width == W512)
 820         {
 821             // 512b hsub can be accomplished with shuf/sub combo
 822             auto minuend    = B->VSHUFFLE(src0, src1, B->C({0, 2, 8, 10, 4, 6, 12, 14}));
 823             auto subtrahend = B->VSHUFFLE(src0, src1, B->C({1, 3, 9, 11, 5, 7, 13, 15}));
 824             return cast<Instruction>(B->SUB(minuend, subtrahend));
 825         }
 826         else
 827         {
 828             SWR_ASSERT(false, "Unimplemented vector width.");
 829             return nullptr;
 830         }
 831     }
 832
 833     // Double pump input using Intrin template arg. This blindly extracts lower and upper 256 from
 834     // each vector argument and calls the 256 wide intrinsic, then merges the results to 512 wide
 835     Instruction* DOUBLE_EMU(LowerX86*     pThis,
 836                             TargetArch    arch,
 837                             TargetWidth   width,
 838                             CallInst*     pCallInst,
 839                             Intrinsic::ID intrin)
 840     {
 841         auto B = pThis->B;
 842         SWR_ASSERT(width == W512);
 843         Value*    result[2];
 844         Function* pX86IntrinFunc = Intrinsic::getDeclaration(B->JM()->mpCurrentModule, intrin);
 845         for (uint32_t i = 0; i < 2; ++i)
 846         {
 847             SmallVector<Value*, 8> args;
 848             for (auto& arg : pCallInst->arg_operands())
 849             {
 850                 auto argType = arg.get()->getType();
 851                 if (argType->isVectorTy())
 852                 {
 853                     uint32_t vecWidth  = argType->getVectorNumElements();
 854                     Value*   lanes     = B->CInc<int>(i * vecWidth / 2, vecWidth / 2);
 855                     Value*   argToPush = B->VSHUFFLE(
 856                         arg.get(), B->VUNDEF(argType->getVectorElementType(), vecWidth), lanes);
 857                     args.push_back(argToPush);
 858                 }
 859                 else
 860                 {
 861                     args.push_back(arg.get());
 862                 }
 863             }
 864             result[i] = B->CALLA(pX86IntrinFunc, args);
 865         }
 866         uint32_t vecWidth;
 867         if (result[0]->getType()->isVectorTy())
 868         {
 869             assert(result[1]->getType()->isVectorTy());
 870             vecWidth = result[0]->getType()->getVectorNumElements() +
 871                        result[1]->getType()->getVectorNumElements();
 872         }
 873         else
 874         {
 875             vecWidth = 2;
 876         }
 877         Value* lanes = B->CInc<int>(0, vecWidth);
 878         return cast<Instruction>(B->VSHUFFLE(result[0], result[1], lanes));
 879     }
 880
 881 } // namespace SwrJit
 882
 883 using namespace SwrJit;
 884
 885 INITIALIZE_PASS_BEGIN(LowerX86, "LowerX86", "LowerX86", false, false)
 886 INITIALIZE_PASS_END(LowerX86, "LowerX86", "LowerX86", false, false)