From f695e433542847cc396b6b55c6f6e912a196a107 Mon Sep 17 00:00:00 2001 From: Alok Hota Date: Fri, 14 Sep 2018 09:45:26 -0500 Subject: [PATCH] swr/rast: Add translation support to streamout Reviewed-by: Bruce Cherniak --- .../rasterizer/codegen/gen_llvm_ir_macros.py | 2 ++ .../drivers/swr/rasterizer/core/api.cpp | 16 ++++++------ src/gallium/drivers/swr/rasterizer/core/api.h | 25 +++++++++++++------ .../drivers/swr/rasterizer/core/context.h | 16 ++++++------ .../drivers/swr/rasterizer/core/frontend.cpp | 6 +++-- .../drivers/swr/rasterizer/core/state.h | 6 ++--- .../swr/rasterizer/jitter/builder_gfx_mem.cpp | 24 ++++++++++++++++++ .../swr/rasterizer/jitter/builder_gfx_mem.h | 6 +++++ .../swr/rasterizer/jitter/builder_mem.cpp | 2 +- .../swr/rasterizer/jitter/builder_mem.h | 17 ++++++++++--- .../swr/rasterizer/jitter/streamout_jit.cpp | 21 +++++++++++----- src/gallium/drivers/swr/swr_state.cpp | 2 +- 12 files changed, 106 insertions(+), 37 deletions(-) diff --git a/src/gallium/drivers/swr/rasterizer/codegen/gen_llvm_ir_macros.py b/src/gallium/drivers/swr/rasterizer/codegen/gen_llvm_ir_macros.py index 485403ae1ec..97a08d21f35 100644 --- a/src/gallium/drivers/swr/rasterizer/codegen/gen_llvm_ir_macros.py +++ b/src/gallium/drivers/swr/rasterizer/codegen/gen_llvm_ir_macros.py @@ -162,6 +162,8 @@ def parse_ir_builder(input_file): func_name == 'CreateGEP' or func_name == 'CreateLoad' or func_name == 'CreateMaskedLoad' or + func_name == 'CreateStore' or + func_name == 'CreateMaskedStore' or func_name == 'CreateElementUnorderedAtomicMemCpy'): ignore = True diff --git a/src/gallium/drivers/swr/rasterizer/core/api.cpp b/src/gallium/drivers/swr/rasterizer/core/api.cpp index 0423a4fd600..e260c951078 100644 --- a/src/gallium/drivers/swr/rasterizer/core/api.cpp +++ b/src/gallium/drivers/swr/rasterizer/core/api.cpp @@ -193,13 +193,15 @@ HANDLE SwrCreateContext(SWR_CREATECONTEXT_INFO* pCreateInfo) pContext->pHotTileMgr = new HotTileMgr(); // initialize callback functions - pContext->pfnLoadTile = pCreateInfo->pfnLoadTile; - pContext->pfnStoreTile = pCreateInfo->pfnStoreTile; - pContext->pfnClearTile = pCreateInfo->pfnClearTile; - pContext->pfnMakeGfxPtr = pCreateInfo->pfnMakeGfxPtr; - pContext->pfnUpdateSoWriteOffset = pCreateInfo->pfnUpdateSoWriteOffset; - pContext->pfnUpdateStats = pCreateInfo->pfnUpdateStats; - pContext->pfnUpdateStatsFE = pCreateInfo->pfnUpdateStatsFE; + pContext->pfnLoadTile = pCreateInfo->pfnLoadTile; + pContext->pfnStoreTile = pCreateInfo->pfnStoreTile; + pContext->pfnClearTile = pCreateInfo->pfnClearTile; + pContext->pfnTranslateGfxptrForRead = pCreateInfo->pfnTranslateGfxptrForRead; + pContext->pfnTranslateGfxptrForWrite = pCreateInfo->pfnTranslateGfxptrForWrite; + pContext->pfnMakeGfxPtr = pCreateInfo->pfnMakeGfxPtr; + pContext->pfnUpdateSoWriteOffset = pCreateInfo->pfnUpdateSoWriteOffset; + pContext->pfnUpdateStats = pCreateInfo->pfnUpdateStats; + pContext->pfnUpdateStatsFE = pCreateInfo->pfnUpdateStatsFE; // pass pointer to bucket manager back to caller diff --git a/src/gallium/drivers/swr/rasterizer/core/api.h b/src/gallium/drivers/swr/rasterizer/core/api.h index b33dc96e339..e31e0446ba6 100644 --- a/src/gallium/drivers/swr/rasterizer/core/api.h +++ b/src/gallium/drivers/swr/rasterizer/core/api.h @@ -145,6 +145,15 @@ typedef void(SWR_API* PFN_CLEAR_TILE)(HANDLE hPrivateContex uint32_t renderTargetArrayIndex, const float* pClearColor); + +typedef void* (SWR_API* PFN_TRANSLATE_GFXPTR_FOR_READ)(HANDLE hPrivateContext, + gfxptr_t xpAddr, + bool* pbNullTileAccessed); + +typedef void* (SWR_API* PFN_TRANSLATE_GFXPTR_FOR_WRITE)(HANDLE hPrivateContext, + gfxptr_t xpAddr, + bool* pbNullTileAccessed); + typedef gfxptr_t(SWR_API* PFN_MAKE_GFXPTR)(HANDLE hPrivateContext, void* sysAddr); @@ -241,13 +250,15 @@ struct SWR_CREATECONTEXT_INFO SWR_WORKER_PRIVATE_STATE* pWorkerPrivateState; // Callback functions - PFN_LOAD_TILE pfnLoadTile; - PFN_STORE_TILE pfnStoreTile; - PFN_CLEAR_TILE pfnClearTile; - PFN_MAKE_GFXPTR pfnMakeGfxPtr; - PFN_UPDATE_SO_WRITE_OFFSET pfnUpdateSoWriteOffset; - PFN_UPDATE_STATS pfnUpdateStats; - PFN_UPDATE_STATS_FE pfnUpdateStatsFE; + PFN_LOAD_TILE pfnLoadTile; + PFN_STORE_TILE pfnStoreTile; + PFN_CLEAR_TILE pfnClearTile; + PFN_TRANSLATE_GFXPTR_FOR_READ pfnTranslateGfxptrForRead; + PFN_TRANSLATE_GFXPTR_FOR_WRITE pfnTranslateGfxptrForWrite; + PFN_MAKE_GFXPTR pfnMakeGfxPtr; + PFN_UPDATE_SO_WRITE_OFFSET pfnUpdateSoWriteOffset; + PFN_UPDATE_STATS pfnUpdateStats; + PFN_UPDATE_STATS_FE pfnUpdateStatsFE; // Pointer to rdtsc buckets mgr returned to the caller. diff --git a/src/gallium/drivers/swr/rasterizer/core/context.h b/src/gallium/drivers/swr/rasterizer/core/context.h index 393d42b477e..efbddb01e0e 100644 --- a/src/gallium/drivers/swr/rasterizer/core/context.h +++ b/src/gallium/drivers/swr/rasterizer/core/context.h @@ -525,13 +525,15 @@ struct SWR_CONTEXT HotTileMgr* pHotTileMgr; // Callback functions, passed in at create context time - PFN_LOAD_TILE pfnLoadTile; - PFN_STORE_TILE pfnStoreTile; - PFN_CLEAR_TILE pfnClearTile; - PFN_MAKE_GFXPTR pfnMakeGfxPtr; - PFN_UPDATE_SO_WRITE_OFFSET pfnUpdateSoWriteOffset; - PFN_UPDATE_STATS pfnUpdateStats; - PFN_UPDATE_STATS_FE pfnUpdateStatsFE; + PFN_LOAD_TILE pfnLoadTile; + PFN_STORE_TILE pfnStoreTile; + PFN_CLEAR_TILE pfnClearTile; + PFN_TRANSLATE_GFXPTR_FOR_READ pfnTranslateGfxptrForRead; + PFN_TRANSLATE_GFXPTR_FOR_WRITE pfnTranslateGfxptrForWrite; + PFN_MAKE_GFXPTR pfnMakeGfxPtr; + PFN_UPDATE_SO_WRITE_OFFSET pfnUpdateSoWriteOffset; + PFN_UPDATE_STATS pfnUpdateStats; + PFN_UPDATE_STATS_FE pfnUpdateStatsFE; // Global Stats diff --git a/src/gallium/drivers/swr/rasterizer/core/frontend.cpp b/src/gallium/drivers/swr/rasterizer/core/frontend.cpp index 6ba6784f518..85b2e8d54b0 100644 --- a/src/gallium/drivers/swr/rasterizer/core/frontend.cpp +++ b/src/gallium/drivers/swr/rasterizer/core/frontend.cpp @@ -575,7 +575,7 @@ static void StreamOut( // Call SOS SWR_ASSERT(state.pfnSoFunc[streamIndex] != nullptr, "Trying to execute uninitialized streamout jit function."); - state.pfnSoFunc[streamIndex](soContext); + state.pfnSoFunc[streamIndex](GetPrivateState(pDC), soContext); } // Update SO write offset. The driver provides memory for the update. @@ -583,7 +583,9 @@ static void StreamOut( { if (state.soBuffer[i].pWriteOffset) { - *state.soBuffer[i].pWriteOffset = soContext.pBuffer[i]->streamOffset * sizeof(uint32_t); + bool nullTileAccessed = false; + void* pWriteOffset = pDC->pContext->pfnTranslateGfxptrForWrite(GetPrivateState(pDC), soContext.pBuffer[i]->pWriteOffset, &nullTileAccessed); + *((uint32_t*)pWriteOffset) = soContext.pBuffer[i]->streamOffset * sizeof(uint32_t); } if (state.soBuffer[i].soWriteEnable) diff --git a/src/gallium/drivers/swr/rasterizer/core/state.h b/src/gallium/drivers/swr/rasterizer/core/state.h index c2594b04cc4..04fad69feae 100644 --- a/src/gallium/drivers/swr/rasterizer/core/state.h +++ b/src/gallium/drivers/swr/rasterizer/core/state.h @@ -678,10 +678,10 @@ OSALIGNLINE(struct) SWR_STATS_FE struct SWR_STREAMOUT_BUFFER { // Pointers to streamout buffers. - uint32_t* pBuffer; + gfxptr_t pBuffer; // Offset to the SO write offset. If not null then we update offset here. - uint32_t* pWriteOffset; + gfxptr_t pWriteOffset; bool enable; bool soWriteEnable; @@ -936,7 +936,7 @@ typedef void(__cdecl *PFN_HS_FUNC)(HANDLE hPrivateData, HANDLE hWorkerPrivateDat typedef void(__cdecl *PFN_DS_FUNC)(HANDLE hPrivateData, HANDLE hWorkerPrivateData, SWR_DS_CONTEXT* pDsContext); typedef void(__cdecl *PFN_GS_FUNC)(HANDLE hPrivateData, HANDLE hWorkerPrivateData, SWR_GS_CONTEXT* pGsContext); typedef void(__cdecl *PFN_CS_FUNC)(HANDLE hPrivateData, HANDLE hWorkerPrivateData, SWR_CS_CONTEXT* pCsContext); -typedef void(__cdecl *PFN_SO_FUNC)(SWR_STREAMOUT_CONTEXT& soContext); +typedef void(__cdecl *PFN_SO_FUNC)(HANDLE hPrivateData, SWR_STREAMOUT_CONTEXT& soContext); typedef void(__cdecl *PFN_PIXEL_KERNEL)(HANDLE hPrivateData, HANDLE hWorkerPrivateData, SWR_PS_CONTEXT* pContext); typedef void(__cdecl *PFN_CPIXEL_KERNEL)(HANDLE hPrivateData, HANDLE hWorkerPrivateData, SWR_PS_CONTEXT* pContext); typedef void(__cdecl *PFN_BLEND_JIT_FUNC)(SWR_BLEND_CONTEXT*); diff --git a/src/gallium/drivers/swr/rasterizer/jitter/builder_gfx_mem.cpp b/src/gallium/drivers/swr/rasterizer/jitter/builder_gfx_mem.cpp index 19eec7e99e0..b3d0b706ac0 100644 --- a/src/gallium/drivers/swr/rasterizer/jitter/builder_gfx_mem.cpp +++ b/src/gallium/drivers/swr/rasterizer/jitter/builder_gfx_mem.cpp @@ -237,6 +237,30 @@ namespace SwrJit return Builder::MASKED_LOAD(Ptr, Align, Mask, PassThru, Name, Ty, usage); } + StoreInst* BuilderGfxMem::STORE(Value *Val, Value *Ptr, bool isVolatile, Type* Ty, JIT_MEM_CLIENT usage) + { + AssertGFXMemoryParams(Ptr, usage); + + Ptr = TranslationHelper(Ptr, Ty); + return Builder::STORE(Val, Ptr, isVolatile, Ty, usage); + } + + StoreInst* BuilderGfxMem::STORE(Value* Val, Value* BasePtr, const std::initializer_list& offset, Type* Ty, JIT_MEM_CLIENT usage) + { + AssertGFXMemoryParams(BasePtr, usage); + + BasePtr = TranslationHelper(BasePtr, Ty); + return Builder::STORE(Val, BasePtr, offset, Ty, usage); + } + + CallInst* BuilderGfxMem::MASKED_STORE(Value *Val, Value *Ptr, unsigned Align, Value *Mask, Type* Ty, JIT_MEM_CLIENT usage) + { + AssertGFXMemoryParams(Ptr, usage); + + Ptr = TranslationHelper(Ptr, Ty); + return Builder::MASKED_STORE(Val, Ptr, Align, Mask, Ty, usage); + } + Value* BuilderGfxMem::TranslateGfxAddressForRead(Value* xpGfxAddress, Type* PtrTy, const Twine& Name, diff --git a/src/gallium/drivers/swr/rasterizer/jitter/builder_gfx_mem.h b/src/gallium/drivers/swr/rasterizer/jitter/builder_gfx_mem.h index 4cf06253695..1bbe86d16cd 100644 --- a/src/gallium/drivers/swr/rasterizer/jitter/builder_gfx_mem.h +++ b/src/gallium/drivers/swr/rasterizer/jitter/builder_gfx_mem.h @@ -76,6 +76,12 @@ namespace SwrJit Type* Ty = nullptr, JIT_MEM_CLIENT usage = JIT_MEM_CLIENT::MEM_CLIENT_INTERNAL); + virtual StoreInst* STORE(Value *Val, Value *Ptr, bool isVolatile = false, Type* Ty = nullptr, JIT_MEM_CLIENT usage = JIT_MEM_CLIENT::MEM_CLIENT_INTERNAL); + + virtual StoreInst* STORE(Value* Val, Value* BasePtr, const std::initializer_list& offset, Type* Ty = nullptr, JIT_MEM_CLIENT usage = JIT_MEM_CLIENT::MEM_CLIENT_INTERNAL); + + virtual CallInst* MASKED_STORE(Value *Val, Value *Ptr, unsigned Align, Value *Mask, Type* Ty = nullptr, JIT_MEM_CLIENT usage = JIT_MEM_CLIENT::MEM_CLIENT_INTERNAL); + virtual Value* GATHERPS(Value* src, Value* pBase, Value* indices, diff --git a/src/gallium/drivers/swr/rasterizer/jitter/builder_mem.cpp b/src/gallium/drivers/swr/rasterizer/jitter/builder_mem.cpp index a9d649092d4..90a0e03e197 100644 --- a/src/gallium/drivers/swr/rasterizer/jitter/builder_mem.cpp +++ b/src/gallium/drivers/swr/rasterizer/jitter/builder_mem.cpp @@ -141,7 +141,7 @@ namespace SwrJit } StoreInst* - Builder::STORE(Value* val, Value* basePtr, const std::initializer_list& indices) + Builder::STORE(Value* val, Value* basePtr, const std::initializer_list& indices, Type* Ty, JIT_MEM_CLIENT usage) { std::vector valIndices; for (auto i : indices) diff --git a/src/gallium/drivers/swr/rasterizer/jitter/builder_mem.h b/src/gallium/drivers/swr/rasterizer/jitter/builder_mem.h index c533984da30..ccf42c8dab0 100644 --- a/src/gallium/drivers/swr/rasterizer/jitter/builder_mem.h +++ b/src/gallium/drivers/swr/rasterizer/jitter/builder_mem.h @@ -36,6 +36,7 @@ enum class JIT_MEM_CLIENT GFX_MEM_CLIENT_FETCH, GFX_MEM_CLIENT_SAMPLER, GFX_MEM_CLIENT_SHADER, + GFX_MEM_CLIENT_STREAMOUT }; protected: @@ -85,9 +86,19 @@ virtual CallInst* MASKED_LOAD(Value* Ptr, return IRB()->CreateMaskedLoad(Ptr, Align, Mask, PassThru, Name); } -LoadInst* - LOADV(Value* BasePtr, const std::initializer_list& offset, const llvm::Twine& name = ""); -StoreInst* STORE(Value* Val, Value* BasePtr, const std::initializer_list& offset); +virtual StoreInst* STORE(Value *Val, Value *Ptr, bool isVolatile = false, Type* Ty = nullptr, JIT_MEM_CLIENT usage = JIT_MEM_CLIENT::MEM_CLIENT_INTERNAL) +{ + return IRB()->CreateStore(Val, Ptr, isVolatile); +} + +virtual StoreInst* STORE(Value* Val, Value* BasePtr, const std::initializer_list& offset, Type* Ty = nullptr, JIT_MEM_CLIENT usage = JIT_MEM_CLIENT::MEM_CLIENT_INTERNAL); + +virtual CallInst* MASKED_STORE(Value *Val, Value *Ptr, unsigned Align, Value *Mask, Type* Ty = nullptr, JIT_MEM_CLIENT usage = JIT_MEM_CLIENT::MEM_CLIENT_INTERNAL) +{ + return IRB()->CreateMaskedStore(Val, Ptr, Align, Mask); +} + +LoadInst* LOADV(Value* BasePtr, const std::initializer_list& offset, const llvm::Twine& name = ""); StoreInst* STOREV(Value* Val, Value* BasePtr, const std::initializer_list& offset); Value* MEM_ADD(Value* i32Incr, diff --git a/src/gallium/drivers/swr/rasterizer/jitter/streamout_jit.cpp b/src/gallium/drivers/swr/rasterizer/jitter/streamout_jit.cpp index 11ad36521b3..2e99bb7a5d5 100644 --- a/src/gallium/drivers/swr/rasterizer/jitter/streamout_jit.cpp +++ b/src/gallium/drivers/swr/rasterizer/jitter/streamout_jit.cpp @@ -28,7 +28,7 @@ * ******************************************************************************/ #include "jit_pch.hpp" -#include "builder.h" +#include "builder_gfx_mem.h" #include "jit_api.h" #include "streamout_jit.h" #include "gen_state_llvm.h" @@ -40,9 +40,9 @@ using namespace SwrJit; ////////////////////////////////////////////////////////////////////////// /// Interface to Jitting a fetch shader ////////////////////////////////////////////////////////////////////////// -struct StreamOutJit : public Builder +struct StreamOutJit : public BuilderGfxMem { - StreamOutJit(JitManager* pJitMgr) : Builder(pJitMgr){}; + StreamOutJit(JitManager* pJitMgr) : BuilderGfxMem(pJitMgr){}; // returns pointer to SWR_STREAMOUT_BUFFER Value* getSOBuffer(Value* pSoCtx, uint32_t buffer) @@ -155,7 +155,7 @@ struct StreamOutJit : public Builder // cast mask to <4xi1> Value* mask = ToMask(packedMask); - MASKED_STORE(src, pOut, 4, mask); + MASKED_STORE(src, pOut, 4, mask, PointerType::get(simd4Ty, 0), JIT_MEM_CLIENT::GFX_MEM_CLIENT_STREAMOUT); } // increment SO buffer @@ -223,7 +223,7 @@ struct StreamOutJit : public Builder Value* pBuf = getSOBuffer(pSoCtx, b); Value* pData = LOAD(pBuf, {0, SWR_STREAMOUT_BUFFER_pBuffer}); Value* streamOffset = LOAD(pBuf, {0, SWR_STREAMOUT_BUFFER_streamOffset}); - pOutBuffer[b] = GEP(pData, streamOffset); + pOutBuffer[b] = GEP(pData, streamOffset, PointerType::get(IRB()->getInt32Ty(), 0)); pOutBufferStartVertex[b] = pOutBuffer[b]; outBufferPitch[b] = LOAD(pBuf, {0, SWR_STREAMOUT_BUFFER_pitch}); @@ -264,9 +264,13 @@ struct StreamOutJit : public Builder fnName << ComputeCRC(0, &state, sizeof(state)); // SO function signature - // typedef void(__cdecl *PFN_SO_FUNC)(SWR_STREAMOUT_CONTEXT*) + // typedef void(__cdecl *PFN_SO_FUNC)(SimDrawContext, SWR_STREAMOUT_CONTEXT*) + + Type* typeParam0; + typeParam0 = mInt8PtrTy; std::vector args{ + typeParam0, PointerType::get(Gen_SWR_STREAMOUT_CONTEXT(JM()), 0), // SWR_STREAMOUT_CONTEXT* }; @@ -284,6 +288,11 @@ struct StreamOutJit : public Builder // arguments auto argitr = soFunc->arg_begin(); + + Value* privateContext = &*argitr++; + privateContext->setName("privateContext"); + SetPrivateContext(privateContext); + Value* pSoCtx = &*argitr++; pSoCtx->setName("pSoCtx"); diff --git a/src/gallium/drivers/swr/swr_state.cpp b/src/gallium/drivers/swr/swr_state.cpp index e540ddb06c8..d7baa7127b4 100644 --- a/src/gallium/drivers/swr/swr_state.cpp +++ b/src/gallium/drivers/swr/swr_state.cpp @@ -1746,7 +1746,7 @@ swr_update_derived(struct pipe_context *pipe, continue; buffer.enable = true; buffer.pBuffer = - (uint32_t *)(swr_resource_data(ctx->so_targets[i]->buffer) + + (gfxptr_t)(swr_resource_data(ctx->so_targets[i]->buffer) + ctx->so_targets[i]->buffer_offset); buffer.bufferSize = ctx->so_targets[i]->buffer_size >> 2; buffer.pitch = stream_output->stride[i]; -- 2.30.2