src/gallium/drivers/swr/rasterizer/jitter/builder_gfx_mem.cpp

   1 /****************************************************************************
   2  * Copyright (C) 2014-2018 Intel Corporation.   All Rights Reserved.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * @file builder_gfx_mem.cpp
  24  *
  25  * @brief Definition of the gfx mem builder
  26  *
  27  * Notes:
  28  *
  29  ******************************************************************************/
  30 #include "jit_pch.hpp"
  31 #include "builder.h"
  32 #include "common/rdtsc_buckets.h"
  33 #include "builder_gfx_mem.h"
  34
  35 namespace SwrJit
  36 {
  37     using namespace llvm;
  38
  39     BuilderGfxMem::BuilderGfxMem(JitManager* pJitMgr) : Builder(pJitMgr)
  40     {
  41         mpTranslationFuncTy             = nullptr;
  42         mpfnTranslateGfxAddressForRead  = nullptr;
  43         mpfnTranslateGfxAddressForWrite = nullptr;
  44         mpfnTrackMemAccess              = nullptr;
  45         mpParamSimDC                    = nullptr;
  46         mpWorkerData                    = nullptr;
  47
  48     }
  49
  50     void BuilderGfxMem::NotifyPrivateContextSet()
  51     {
  52     }
  53
  54     void BuilderGfxMem::AssertGFXMemoryParams(Value* ptr, MEM_CLIENT usage)
  55     {
  56         SWR_ASSERT(!(ptr->getType() == mInt64Ty && usage == MEM_CLIENT::MEM_CLIENT_INTERNAL),
  57                    "Internal memory should not be gfxptr_t.");
  58     }
  59
  60     //////////////////////////////////////////////////////////////////////////
  61     /// @brief Generate a masked gather operation in LLVM IR.  If not
  62     /// supported on the underlying platform, emulate it with loads
  63     /// @param vSrc - SIMD wide value that will be loaded if mask is invalid
  64     /// @param pBase - Int8* base VB address pointer value
  65     /// @param vIndices - SIMD wide value of VB byte offsets
  66     /// @param vMask - SIMD wide mask that controls whether to access memory or the src values
  67     /// @param scale - value to scale indices by
  68     Value* BuilderGfxMem::GATHERPS(Value*         vSrc,
  69                                    Value*         pBase,
  70                                    Value*         vIndices,
  71                                    Value*         vMask,
  72                                    uint8_t        scale,
  73                                    MEM_CLIENT     usage)
  74     {
  75        // address may be coming in as 64bit int now so get the pointer
  76         if (pBase->getType() == mInt64Ty)
  77         {
  78             pBase = INT_TO_PTR(pBase, PointerType::get(mInt8Ty, 0));
  79         }
  80
  81         Value* vGather = Builder::GATHERPS(vSrc, pBase, vIndices, vMask, scale);
  82         return vGather;
  83     }
  84
  85     //////////////////////////////////////////////////////////////////////////
  86     /// @brief Generate a masked gather operation in LLVM IR.  If not
  87     /// supported on the underlying platform, emulate it with loads
  88     /// @param vSrc - SIMD wide value that will be loaded if mask is invalid
  89     /// @param pBase - Int8* base VB address pointer value
  90     /// @param vIndices - SIMD wide value of VB byte offsets
  91     /// @param vMask - SIMD wide mask that controls whether to access memory or the src values
  92     /// @param scale - value to scale indices by
  93     Value* BuilderGfxMem::GATHERDD(Value*         vSrc,
  94                                    Value*         pBase,
  95                                    Value*         vIndices,
  96                                    Value*         vMask,
  97                                    uint8_t        scale,
  98                                    MEM_CLIENT     usage)
  99     {
 100
 101         // address may be coming in as 64bit int now so get the pointer
 102         if (pBase->getType() == mInt64Ty)
 103         {
 104             pBase = INT_TO_PTR(pBase, PointerType::get(mInt8Ty, 0));
 105         }
 106
 107         Value* vGather = Builder::GATHERDD(vSrc, pBase, vIndices, vMask, scale);
 108         return vGather;
 109     }
 110
 111     void BuilderGfxMem::SCATTERPS(
 112         Value* pDst, Value* vSrc, Value* vOffsets, Value* vMask, MEM_CLIENT usage)
 113     {
 114
 115         // address may be coming in as 64bit int now so get the pointer
 116         if (pDst->getType() == mInt64Ty)
 117         {
 118             pDst = INT_TO_PTR(pDst, PointerType::get(mInt8Ty, 0));
 119         }
 120
 121         Builder::SCATTERPS(pDst, BITCAST(vSrc, mSimdFP32Ty), vOffsets, vMask, usage);
 122     }
 123
 124     Value* BuilderGfxMem::OFFSET_TO_NEXT_COMPONENT(Value* base, Constant* offset)
 125     {
 126         return ADD(base, offset);
 127     }
 128
 129     Value* BuilderGfxMem::GEP(Value* Ptr, Value* Idx, Type* Ty, bool isReadOnly, const Twine& Name)
 130     {
 131         bool xlate = (Ptr->getType() == mInt64Ty);
 132         if (xlate)
 133         {
 134             Ptr = INT_TO_PTR(Ptr, Ty);
 135             Ptr = Builder::GEP(Ptr, Idx, nullptr, isReadOnly, Name);
 136             Ptr = PTR_TO_INT(Ptr, mInt64Ty);
 137             if (isReadOnly)
 138             {
 139                 Ptr = TranslationHelper(Ptr, Ty, mpfnTranslateGfxAddressForRead);
 140             }
 141             else
 142             {
 143                 Ptr = TranslationHelper(Ptr, Ty, mpfnTranslateGfxAddressForWrite);
 144             }
 145         }
 146         else
 147         {
 148             Ptr = Builder::GEP(Ptr, Idx, nullptr, isReadOnly, Name);
 149         }
 150         return Ptr;
 151     }
 152
 153     Value* BuilderGfxMem::GEP(Type* Ty, Value* Ptr, Value* Idx, const Twine& Name)
 154     {
 155         bool xlate = (Ptr->getType() == mInt64Ty);
 156         if (xlate)
 157         {
 158             Ptr = INT_TO_PTR(Ptr, Ty);
 159             Ptr = Builder::GEP(Ty, Ptr, Idx, Name);
 160             Ptr = PTR_TO_INT(Ptr, mInt64Ty);
 161             Ptr = TranslationHelper(Ptr, Ty, mpfnTranslateGfxAddressForRead);
 162         }
 163         else
 164         {
 165             Ptr = Builder::GEP(Ty, Ptr, Idx, Name);
 166         }
 167         return Ptr;
 168     }
 169
 170     Value* BuilderGfxMem::GEP(Value* Ptr, const std::initializer_list<Value*>& indexList, Type* Ty)
 171     {
 172         bool xlate = (Ptr->getType() == mInt64Ty);
 173         if (xlate)
 174         {
 175             Ptr = INT_TO_PTR(Ptr, Ty);
 176             Ptr = Builder::GEP(Ptr, indexList);
 177             Ptr = PTR_TO_INT(Ptr, mInt64Ty);
 178             Ptr = TranslationHelper(Ptr, Ty, mpfnTranslateGfxAddressForRead);
 179         }
 180         else
 181         {
 182             Ptr = Builder::GEP(Ptr, indexList);
 183         }
 184         return Ptr;
 185     }
 186
 187     Value*
 188     BuilderGfxMem::GEP(Value* Ptr, const std::initializer_list<uint32_t>& indexList, Type* Ty)
 189     {
 190         bool xlate = (Ptr->getType() == mInt64Ty);
 191         if (xlate)
 192         {
 193             Ptr = INT_TO_PTR(Ptr, Ty);
 194             Ptr = Builder::GEP(Ptr, indexList);
 195             Ptr = PTR_TO_INT(Ptr, mInt64Ty);
 196             Ptr = TranslationHelper(Ptr, Ty, mpfnTranslateGfxAddressForRead);
 197         }
 198         else
 199         {
 200             Ptr = Builder::GEP(Ptr, indexList);
 201         }
 202         return Ptr;
 203     }
 204
 205     Value* BuilderGfxMem::TranslationHelper(Value* Ptr, Type* Ty, Value* pfnTranslateGfxAddress)
 206     {
 207         SWR_ASSERT(!(Ptr->getType() == mInt64Ty && Ty == nullptr),
 208                    "Access of GFX pointers must have non-null type specified.");
 209
 210         // address may be coming in as 64bit int now so get the pointer
 211         if (Ptr->getType() == mInt64Ty)
 212         {
 213             Ptr = INT_TO_PTR(Ptr, Ty);
 214         }
 215
 216         return Ptr;
 217     }
 218
 219     void BuilderGfxMem::TrackerHelper(Value* Ptr, Type* Ty, MEM_CLIENT usage, bool isRead)
 220     {
 221 #if defined(KNOB_ENABLE_AR)
 222         if (!KNOB_AR_ENABLE_MEMORY_EVENTS)
 223         {
 224             return;
 225         }
 226
 227         Value* tmpPtr;
 228         // convert actual pointers to int64.
 229         uint32_t size = 0;
 230
 231         if (Ptr->getType() == mInt64Ty)
 232         {
 233             DataLayout dataLayout(JM()->mpCurrentModule);
 234             size = (uint32_t)dataLayout.getTypeAllocSize(Ty);
 235
 236             tmpPtr = Ptr;
 237         }
 238         else
 239         {
 240             DataLayout dataLayout(JM()->mpCurrentModule);
 241             size = (uint32_t)dataLayout.getTypeAllocSize(Ptr->getType());
 242
 243             tmpPtr = PTR_TO_INT(Ptr, mInt64Ty);
 244         }
 245
 246         // There are some shader compile setups where there's no translation functions set up.
 247         // This would be a situation where the accesses are to internal rasterizer memory and won't
 248         // be logged.
 249         // TODO:  we may wish to revisit this for URB reads/writes, though.
 250         if (mpfnTrackMemAccess)
 251         {
 252             SWR_ASSERT(mpWorkerData != nullptr);
 253             CALL(mpfnTrackMemAccess,
 254                  {mpParamSimDC,
 255                   mpWorkerData,
 256                   tmpPtr,
 257                   C((uint32_t)size),
 258                   C((uint8_t)isRead),
 259                   C((uint32_t)usage)});
 260         }
 261 #endif
 262
 263         return;
 264     }
 265
 266     LoadInst* BuilderGfxMem::LOAD(Value* Ptr, const char* Name, Type* Ty, MEM_CLIENT usage)
 267     {
 268         AssertGFXMemoryParams(Ptr, usage);
 269         TrackerHelper(Ptr, Ty, usage, true);
 270
 271         Ptr = TranslationHelper(Ptr, Ty, mpfnTranslateGfxAddressForRead);
 272         return Builder::LOAD(Ptr, Name);
 273     }
 274
 275     LoadInst* BuilderGfxMem::LOAD(Value* Ptr, const Twine& Name, Type* Ty, MEM_CLIENT usage)
 276     {
 277         AssertGFXMemoryParams(Ptr, usage);
 278         TrackerHelper(Ptr, Ty, usage, true);
 279
 280         Ptr = TranslationHelper(Ptr, Ty, mpfnTranslateGfxAddressForRead);
 281         return Builder::LOAD(Ptr, Name);
 282     }
 283
 284     LoadInst* BuilderGfxMem::LOAD(
 285         Value* Ptr, bool isVolatile, const Twine& Name, Type* Ty, MEM_CLIENT usage)
 286     {
 287         AssertGFXMemoryParams(Ptr, usage);
 288         TrackerHelper(Ptr, Ty, usage, true);
 289
 290         Ptr = TranslationHelper(Ptr, Ty, mpfnTranslateGfxAddressForRead);
 291         return Builder::LOAD(Ptr, isVolatile, Name);
 292     }
 293
 294     LoadInst* BuilderGfxMem::LOAD(Value*                                 BasePtr,
 295                                   const std::initializer_list<uint32_t>& offset,
 296                                   const llvm::Twine&                     name,
 297                                   Type*                                  Ty,
 298                                   MEM_CLIENT                             usage)
 299     {
 300         AssertGFXMemoryParams(BasePtr, usage);
 301
 302         bool bNeedTranslation = false;
 303         if (BasePtr->getType() == mInt64Ty)
 304         {
 305             SWR_ASSERT(Ty);
 306             BasePtr          = INT_TO_PTR(BasePtr, Ty, name);
 307             bNeedTranslation = true;
 308         }
 309         std::vector<Value*> valIndices;
 310         for (auto i : offset)
 311         {
 312             valIndices.push_back(C(i));
 313         }
 314         BasePtr = Builder::GEPA(BasePtr, valIndices, name);
 315         if (bNeedTranslation)
 316         {
 317             BasePtr = PTR_TO_INT(BasePtr, mInt64Ty, name);
 318         }
 319
 320         return LOAD(BasePtr, name, Ty, usage);
 321     }
 322
 323     CallInst* BuilderGfxMem::MASKED_LOAD(Value*         Ptr,
 324                                          unsigned       Align,
 325                                          Value*         Mask,
 326                                          Value*         PassThru,
 327                                          const Twine&   Name,
 328                                          Type*          Ty,
 329                                          MEM_CLIENT     usage)
 330     {
 331         AssertGFXMemoryParams(Ptr, usage);
 332         TrackerHelper(Ptr, Ty, usage, true);
 333
 334         Ptr = TranslationHelper(Ptr, Ty, mpfnTranslateGfxAddressForRead);
 335         return Builder::MASKED_LOAD(Ptr, Align, Mask, PassThru, Name, Ty, usage);
 336     }
 337
 338     StoreInst*
 339     BuilderGfxMem::STORE(Value* Val, Value* Ptr, bool isVolatile, Type* Ty, MEM_CLIENT usage)
 340     {
 341         AssertGFXMemoryParams(Ptr, usage);
 342         TrackerHelper(Ptr, Ty, usage, false);
 343
 344         Ptr = TranslationHelper(Ptr, Ty, mpfnTranslateGfxAddressForRead);
 345         return Builder::STORE(Val, Ptr, isVolatile, Ty, usage);
 346     }
 347
 348     StoreInst* BuilderGfxMem::STORE(Value*                                 Val,
 349                                     Value*                                 BasePtr,
 350                                     const std::initializer_list<uint32_t>& offset,
 351                                     Type*                                  Ty,
 352                                     MEM_CLIENT                             usage)
 353     {
 354         AssertGFXMemoryParams(BasePtr, usage);
 355         TrackerHelper(BasePtr, Ty, usage, false);
 356
 357         BasePtr = TranslationHelper(BasePtr, Ty, mpfnTranslateGfxAddressForRead);
 358         return Builder::STORE(Val, BasePtr, offset, Ty, usage);
 359     }
 360
 361     CallInst* BuilderGfxMem::MASKED_STORE(
 362         Value* Val, Value* Ptr, unsigned Align, Value* Mask, Type* Ty, MEM_CLIENT usage)
 363     {
 364         AssertGFXMemoryParams(Ptr, usage);
 365
 366         TrackerHelper(Ptr, Ty, usage, false);
 367
 368         Ptr = TranslationHelper(Ptr, Ty, mpfnTranslateGfxAddressForRead);
 369         return Builder::MASKED_STORE(Val, Ptr, Align, Mask, Ty, usage);
 370     }
 371
 372     Value* BuilderGfxMem::TranslateGfxAddressForRead(Value*       xpGfxAddress,
 373                                                      Type*        PtrTy,
 374                                                      const Twine& Name,
 375                                                      MEM_CLIENT /* usage */)
 376     {
 377         if (PtrTy == nullptr)
 378         {
 379             PtrTy = mInt8PtrTy;
 380         }
 381         return INT_TO_PTR(xpGfxAddress, PtrTy, Name);
 382     }
 383
 384     Value* BuilderGfxMem::TranslateGfxAddressForWrite(Value*       xpGfxAddress,
 385                                                       Type*        PtrTy,
 386                                                       const Twine& Name,
 387                                                       MEM_CLIENT /* usage */)
 388     {
 389         if (PtrTy == nullptr)
 390         {
 391             PtrTy = mInt8PtrTy;
 392         }
 393         return INT_TO_PTR(xpGfxAddress, PtrTy, Name);
 394     }
 395
 396 } // namespace SwrJit