/****************************************************************************
-* Copyright (C) 2014-2015 Intel Corporation. All Rights Reserved.
+* Copyright (C) 2014-2016 Intel Corporation. All Rights Reserved.
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
#include <cfloat>
#include <cmath>
#include <cstdio>
+#include <new>
#include "core/api.h"
#include "core/backend.h"
#include "core/context.h"
+#include "core/depthstencil.h"
#include "core/frontend.h"
#include "core/rasterizer.h"
#include "core/rdtsc_core.h"
#include "core/threads.h"
#include "core/tilemgr.h"
#include "core/clip.h"
+#include "core/utils.h"
#include "common/simdintrin.h"
#include "common/os.h"
void SetupDefaultState(SWR_CONTEXT *pContext);
+static INLINE SWR_CONTEXT* GetContext(HANDLE hContext)
+{
+ return (SWR_CONTEXT*)hContext;
+}
+
//////////////////////////////////////////////////////////////////////////
/// @brief Create SWR Context.
/// @param pCreateInfo - pointer to creation info.
HANDLE SwrCreateContext(
- const SWR_CREATECONTEXT_INFO* pCreateInfo)
+ SWR_CREATECONTEXT_INFO* pCreateInfo)
{
RDTSC_RESET();
RDTSC_INIT(0);
- void* pContextMem = _aligned_malloc(sizeof(SWR_CONTEXT), KNOB_SIMD_WIDTH * 4);
+ void* pContextMem = AlignedMalloc(sizeof(SWR_CONTEXT), KNOB_SIMD_WIDTH * 4);
memset(pContextMem, 0, sizeof(SWR_CONTEXT));
SWR_CONTEXT *pContext = new (pContextMem) SWR_CONTEXT();
pContext->driverType = pCreateInfo->driver;
pContext->privateStateSize = pCreateInfo->privateStateSize;
- pContext->dcRing = (DRAW_CONTEXT*)_aligned_malloc(sizeof(DRAW_CONTEXT)*KNOB_MAX_DRAWS_IN_FLIGHT, 64);
- memset(pContext->dcRing, 0, sizeof(DRAW_CONTEXT)*KNOB_MAX_DRAWS_IN_FLIGHT);
+ pContext->dcRing.Init(KNOB_MAX_DRAWS_IN_FLIGHT);
+ pContext->dsRing.Init(KNOB_MAX_DRAWS_IN_FLIGHT);
+
+ pContext->pMacroTileManagerArray = (MacroTileMgr*)AlignedMalloc(sizeof(MacroTileMgr) * KNOB_MAX_DRAWS_IN_FLIGHT, 64);
+ pContext->pDispatchQueueArray = (DispatchQueue*)AlignedMalloc(sizeof(DispatchQueue) * KNOB_MAX_DRAWS_IN_FLIGHT, 64);
- pContext->dsRing = (DRAW_STATE*)_aligned_malloc(sizeof(DRAW_STATE)*KNOB_MAX_DRAWS_IN_FLIGHT, 64);
- memset(pContext->dsRing, 0, sizeof(DRAW_STATE)*KNOB_MAX_DRAWS_IN_FLIGHT);
+ pContext->threadInfo.MAX_WORKER_THREADS = KNOB_MAX_WORKER_THREADS;
+ pContext->threadInfo.MAX_NUMA_NODES = KNOB_MAX_NUMA_NODES;
+ pContext->threadInfo.MAX_CORES_PER_NUMA_NODE = KNOB_MAX_CORES_PER_NUMA_NODE;
+ pContext->threadInfo.MAX_THREADS_PER_CORE = KNOB_MAX_THREADS_PER_CORE;
+ pContext->threadInfo.SINGLE_THREADED = KNOB_SINGLE_THREADED;
- pContext->numSubContexts = pCreateInfo->maxSubContexts;
- if (pContext->numSubContexts > 1)
+ if (pCreateInfo->pThreadInfo)
{
- pContext->subCtxSave = (DRAW_STATE*)_aligned_malloc(sizeof(DRAW_STATE) * pContext->numSubContexts, 64);
- memset(pContext->subCtxSave, 0, sizeof(DRAW_STATE) * pContext->numSubContexts);
+ pContext->threadInfo = *pCreateInfo->pThreadInfo;
}
for (uint32_t dc = 0; dc < KNOB_MAX_DRAWS_IN_FLIGHT; ++dc)
{
- pContext->dcRing[dc].pArena = new Arena();
- pContext->dcRing[dc].inUse = false;
- pContext->dcRing[dc].pTileMgr = new MacroTileMgr(*(pContext->dcRing[dc].pArena));
- pContext->dcRing[dc].pDispatch = new DispatchQueue(); /// @todo Could lazily allocate this if Dispatch seen.
+ pContext->dcRing[dc].pArena = new CachingArena(pContext->cachingArenaAllocator);
+ new (&pContext->pMacroTileManagerArray[dc]) MacroTileMgr(*pContext->dcRing[dc].pArena);
+ new (&pContext->pDispatchQueueArray[dc]) DispatchQueue();
- pContext->dsRing[dc].pArena = new Arena();
+ pContext->dsRing[dc].pArena = new CachingArena(pContext->cachingArenaAllocator);
}
- if (!KNOB_SINGLE_THREADED)
+ if (!pContext->threadInfo.SINGLE_THREADED)
{
memset(&pContext->WaitLock, 0, sizeof(pContext->WaitLock));
memset(&pContext->FifosNotEmpty, 0, sizeof(pContext->FifosNotEmpty));
}
// Calling createThreadPool() above can set SINGLE_THREADED
- if (KNOB_SINGLE_THREADED)
+ if (pContext->threadInfo.SINGLE_THREADED)
{
pContext->NumWorkerThreads = 1;
+ pContext->NumFEThreads = 1;
+ pContext->NumBEThreads = 1;
}
// Allocate scratch space for workers.
///@note We could lazily allocate this but its rather small amount of memory.
for (uint32_t i = 0; i < pContext->NumWorkerThreads; ++i)
{
- ///@todo Use numa API for allocations using numa information from thread data (if exists).
- pContext->pScratch[i] = (uint8_t*)_aligned_malloc((32 * 1024), KNOB_SIMD_WIDTH * 4);
+#if defined(_WIN32)
+ uint32_t numaNode = pContext->threadPool.pThreadData ?
+ pContext->threadPool.pThreadData[i].numaId : 0;
+ pContext->pScratch[i] = (uint8_t*)VirtualAllocExNuma(
+ GetCurrentProcess(), nullptr, 32 * sizeof(KILOBYTE),
+ MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE,
+ numaNode);
+#else
+ pContext->pScratch[i] = (uint8_t*)AlignedMalloc(32 * sizeof(KILOBYTE), KNOB_SIMD_WIDTH * 4);
+#endif
}
- pContext->nextDrawId = 1;
- pContext->DrawEnqueued = 1;
-
// State setup AFTER context is fully initialized
SetupDefaultState(pContext);
// initialize function pointer tables
InitClearTilesTable();
- // initialize store tiles function
+ // initialize callback functions
pContext->pfnLoadTile = pCreateInfo->pfnLoadTile;
pContext->pfnStoreTile = pCreateInfo->pfnStoreTile;
pContext->pfnClearTile = pCreateInfo->pfnClearTile;
+ pContext->pfnUpdateSoWriteOffset = pCreateInfo->pfnUpdateSoWriteOffset;
+
+ // pass pointer to bucket manager back to caller
+#ifdef KNOB_ENABLE_RDTSC
+ pCreateInfo->pBucketMgr = &gBucketMgr;
+#endif
+
+ pCreateInfo->contextSaveSize = sizeof(API_STATE);
return (HANDLE)pContext;
}
void SwrDestroyContext(HANDLE hContext)
{
- SWR_CONTEXT *pContext = (SWR_CONTEXT*)hContext;
+ SWR_CONTEXT *pContext = GetContext(hContext);
DestroyThreadPool(pContext, &pContext->threadPool);
// free the fifos
{
delete pContext->dcRing[i].pArena;
delete pContext->dsRing[i].pArena;
- delete(pContext->dcRing[i].pTileMgr);
- delete(pContext->dcRing[i].pDispatch);
+ pContext->pMacroTileManagerArray[i].~MacroTileMgr();
+ pContext->pDispatchQueueArray[i].~DispatchQueue();
}
+ AlignedFree(pContext->pDispatchQueueArray);
+ AlignedFree(pContext->pMacroTileManagerArray);
+
// Free scratch space.
for (uint32_t i = 0; i < pContext->NumWorkerThreads; ++i)
{
- _aligned_free(pContext->pScratch[i]);
+#if defined(_WIN32)
+ VirtualFree(pContext->pScratch[i], 0, MEM_RELEASE);
+#else
+ AlignedFree(pContext->pScratch[i]);
+#endif
}
- _aligned_free(pContext->dcRing);
- _aligned_free(pContext->dsRing);
- _aligned_free(pContext->subCtxSave);
-
delete(pContext->pHotTileMgr);
pContext->~SWR_CONTEXT();
- _aligned_free((SWR_CONTEXT*)hContext);
+ AlignedFree(GetContext(hContext));
}
void CopyState(DRAW_STATE& dst, const DRAW_STATE& src)
pContext->FifosNotEmpty.notify_all();
}
-bool StillDrawing(SWR_CONTEXT *pContext, DRAW_CONTEXT *pDC)
+static TileSet gSingleThreadLockedTiles;
+
+template<bool IsDraw>
+void QueueWork(SWR_CONTEXT *pContext)
{
- // For single thread nothing should still be drawing.
- if (KNOB_SINGLE_THREADED) { return false; }
+ DRAW_CONTEXT* pDC = pContext->pCurDrawContext;
+ uint32_t dcIndex = pDC->drawId % KNOB_MAX_DRAWS_IN_FLIGHT;
- if (pDC->isCompute)
+ if (IsDraw)
{
- if (pDC->doneCompute)
- {
- pDC->inUse = false;
- return false;
- }
+ pDC->pTileMgr = &pContext->pMacroTileManagerArray[dcIndex];
+ pDC->pTileMgr->initialize();
}
- // Check if backend work is done. First make sure all triangles have been binned.
- if (pDC->doneFE == true)
- {
- // ensure workers have all moved passed this draw
- if (pDC->threadsDoneFE != pContext->NumWorkerThreads)
- {
- return true;
- }
-
- if (pDC->threadsDoneBE != pContext->NumWorkerThreads)
- {
- return true;
- }
+ // Each worker thread looks at a DC for both FE and BE work at different times and so we
+ // multiply threadDone by 2. When the threadDone counter has reached 0 then all workers
+ // have moved past this DC. (i.e. Each worker has checked this DC for both FE and BE work and
+ // then moved on if all work is done.)
+ pContext->pCurDrawContext->threadsDone = pContext->NumFEThreads + pContext->NumBEThreads;
- pDC->inUse = false; // all work is done.
+ if (IsDraw)
+ {
+ InterlockedIncrement((volatile LONG*)&pContext->drawsOutstandingFE);
}
- return pDC->inUse;
-}
-
-void QueueDraw(SWR_CONTEXT *pContext)
-{
- SWR_ASSERT(pContext->pCurDrawContext->inUse == false);
- pContext->pCurDrawContext->inUse = true;
-
_ReadWriteBarrier();
{
std::unique_lock<std::mutex> lock(pContext->WaitLock);
- pContext->DrawEnqueued++;
+ pContext->dcRing.Enqueue();
}
- if (KNOB_SINGLE_THREADED)
+ if (pContext->threadInfo.SINGLE_THREADED)
{
// flush denormals to 0
uint32_t mxcsr = _mm_getcsr();
_mm_setcsr(mxcsr | _MM_FLUSH_ZERO_ON | _MM_DENORMALS_ZERO_ON);
- std::unordered_set<uint32_t> lockedTiles;
- uint64_t curDraw[2] = { pContext->pCurDrawContext->drawId, pContext->pCurDrawContext->drawId };
- WorkOnFifoFE(pContext, 0, curDraw[0], 0);
- WorkOnFifoBE(pContext, 0, curDraw[1], lockedTiles);
+ if (IsDraw)
+ {
+ uint32_t curDraw[2] = { pContext->pCurDrawContext->drawId, pContext->pCurDrawContext->drawId };
+ WorkOnFifoFE(pContext, 0, curDraw[0]);
+ WorkOnFifoBE(pContext, 0, curDraw[1], gSingleThreadLockedTiles, 0, 0);
+ }
+ else
+ {
+ uint32_t curDispatch = pContext->pCurDrawContext->drawId;
+ WorkOnCompute(pContext, 0, curDispatch);
+ }
+
+ // Dequeue the work here, if not already done, since we're single threaded (i.e. no workers).
+ while (CompleteDrawContext(pContext, pContext->pCurDrawContext) > 0) {}
// restore csr
_mm_setcsr(mxcsr);
pContext->pCurDrawContext = nullptr;
}
-///@todo Combine this with QueueDraw
-void QueueDispatch(SWR_CONTEXT *pContext)
+INLINE void QueueDraw(SWR_CONTEXT* pContext)
{
- SWR_ASSERT(pContext->pCurDrawContext->inUse == false);
- pContext->pCurDrawContext->inUse = true;
-
- _ReadWriteBarrier();
- {
- std::unique_lock<std::mutex> lock(pContext->WaitLock);
- pContext->DrawEnqueued++;
- }
-
- if (KNOB_SINGLE_THREADED)
- {
- // flush denormals to 0
- uint32_t mxcsr = _mm_getcsr();
- _mm_setcsr(mxcsr | _MM_FLUSH_ZERO_ON | _MM_DENORMALS_ZERO_ON);
-
- uint64_t curDispatch = pContext->pCurDrawContext->drawId;
- WorkOnCompute(pContext, 0, curDispatch);
-
- // restore csr
- _mm_setcsr(mxcsr);
- }
- else
- {
- RDTSC_START(APIDrawWakeAllThreads);
- WakeAllThreads(pContext);
- RDTSC_STOP(APIDrawWakeAllThreads, 1, 0);
- }
+ QueueWork<true>(pContext);
+}
- // Set current draw context to NULL so that next state call forces a new draw context to be created and populated.
- pContext->pPrevDrawContext = pContext->pCurDrawContext;
- pContext->pCurDrawContext = nullptr;
+INLINE void QueueDispatch(SWR_CONTEXT* pContext)
+{
+ QueueWork<false>(pContext);
}
DRAW_CONTEXT* GetDrawContext(SWR_CONTEXT *pContext, bool isSplitDraw = false)
// If current draw context is null then need to obtain a new draw context to use from ring.
if (pContext->pCurDrawContext == nullptr)
{
- uint32_t dcIndex = pContext->nextDrawId % KNOB_MAX_DRAWS_IN_FLIGHT;
+ // Need to wait for a free entry.
+ while (pContext->dcRing.IsFull())
+ {
+ _mm_pause();
+ }
- DRAW_CONTEXT* pCurDrawContext = &pContext->dcRing[dcIndex];
- pContext->pCurDrawContext = pCurDrawContext;
+ uint64_t curDraw = pContext->dcRing.GetHead();
+ uint32_t dcIndex = curDraw % KNOB_MAX_DRAWS_IN_FLIGHT;
- // Need to wait until this draw context is available to use.
- while (StillDrawing(pContext, pCurDrawContext))
+ static uint64_t lastDrawChecked;
+ static uint32_t lastFrameChecked;
+ if ((pContext->frameCount - lastFrameChecked) > 2 ||
+ (curDraw - lastDrawChecked) > 0x10000)
{
- _mm_pause();
+ // Take this opportunity to clean-up old arena allocations
+ pContext->cachingArenaAllocator.FreeOldBlocks();
+
+ lastFrameChecked = pContext->frameCount;
+ lastDrawChecked = curDraw;
}
+ DRAW_CONTEXT* pCurDrawContext = &pContext->dcRing[dcIndex];
+ pContext->pCurDrawContext = pCurDrawContext;
+
// Assign next available entry in DS ring to this DC.
uint32_t dsIndex = pContext->curStateId % KNOB_MAX_DRAWS_IN_FLIGHT;
pCurDrawContext->pState = &pContext->dsRing[dsIndex];
- Arena& stateArena = *(pCurDrawContext->pState->pArena);
-
// Copy previous state to current state.
if (pContext->pPrevDrawContext)
{
{
CopyState(*pCurDrawContext->pState, *pPrevDrawContext->pState);
- stateArena.Reset(true); // Reset memory.
+ // Should have been cleaned up previously
+ SWR_ASSERT(pCurDrawContext->pState->pArena->IsEmpty() == true);
+
pCurDrawContext->pState->pPrivateState = nullptr;
pContext->curStateId++; // Progress state ring index forward.
// If its a split draw then just copy the state pointer over
// since its the same draw.
pCurDrawContext->pState = pPrevDrawContext->pState;
+ SWR_ASSERT(pPrevDrawContext->cleanupState == false);
}
}
else
{
- stateArena.Reset(); // Reset memory.
+ SWR_ASSERT(pCurDrawContext->pState->pArena->IsEmpty() == true);
pContext->curStateId++; // Progress state ring index forward.
}
- pCurDrawContext->dependency = 0;
- pCurDrawContext->pArena->Reset();
+ SWR_ASSERT(pCurDrawContext->pArena->IsEmpty() == true);
+
+ pCurDrawContext->dependent = false;
pCurDrawContext->pContext = pContext;
pCurDrawContext->isCompute = false; // Dispatch has to set this to true.
- pCurDrawContext->inUse = false;
- pCurDrawContext->doneCompute = false;
pCurDrawContext->doneFE = false;
pCurDrawContext->FeLock = 0;
- pCurDrawContext->threadsDoneFE = 0;
- pCurDrawContext->threadsDoneBE = 0;
+ pCurDrawContext->threadsDone = 0;
+ pCurDrawContext->retireCallback.pfnCallbackFunc = nullptr;
- pCurDrawContext->pTileMgr->initialize();
+ memset(&pCurDrawContext->dynState, 0, sizeof(pCurDrawContext->dynState));
// Assign unique drawId for this DC
- pCurDrawContext->drawId = pContext->nextDrawId++;
+ pCurDrawContext->drawId = pContext->dcRing.GetHead();
+
+ pCurDrawContext->cleanupState = true;
}
else
{
return pContext->pCurDrawContext;
}
-void SWR_API SwrSetActiveSubContext(
- HANDLE hContext,
- uint32_t subContextIndex)
+API_STATE* GetDrawState(SWR_CONTEXT *pContext)
{
- SWR_CONTEXT *pContext = (SWR_CONTEXT*)hContext;
- if (subContextIndex >= pContext->numSubContexts)
- {
- return;
- }
+ DRAW_CONTEXT* pDC = GetDrawContext(pContext);
+ SWR_ASSERT(pDC->pState != nullptr);
- if (subContextIndex != pContext->curSubCtxId)
- {
- // Save and restore draw state
- DRAW_CONTEXT* pDC = GetDrawContext(pContext);
- CopyState(
- pContext->subCtxSave[pContext->curSubCtxId],
- *(pDC->pState));
+ return &pDC->pState->state;
+}
- CopyState(
- *(pDC->pState),
- pContext->subCtxSave[subContextIndex]);
+void SWR_API SwrSaveState(
+ HANDLE hContext,
+ void* pOutputStateBlock,
+ size_t memSize)
+{
+ SWR_CONTEXT *pContext = GetContext(hContext);
+ auto pSrc = GetDrawState(pContext);
+ SWR_ASSERT(pOutputStateBlock && memSize >= sizeof(*pSrc));
- pContext->curSubCtxId = subContextIndex;
- }
+ memcpy(pOutputStateBlock, pSrc, sizeof(*pSrc));
}
-API_STATE* GetDrawState(SWR_CONTEXT *pContext)
+void SWR_API SwrRestoreState(
+ HANDLE hContext,
+ const void* pStateBlock,
+ size_t memSize)
{
- DRAW_CONTEXT* pDC = GetDrawContext(pContext);
- SWR_ASSERT(pDC->pState != nullptr);
+ SWR_CONTEXT *pContext = GetContext(hContext);
+ auto pDst = GetDrawState(pContext);
+ SWR_ASSERT(pStateBlock && memSize >= sizeof(*pDst));
- return &pDC->pState->state;
+ memcpy(pDst, pStateBlock, sizeof(*pDst));
}
void SetupDefaultState(SWR_CONTEXT *pContext)
pState->rastState.frontWinding = SWR_FRONTWINDING_CCW;
}
-static INLINE SWR_CONTEXT* GetContext(HANDLE hContext)
-{
- return (SWR_CONTEXT*)hContext;
-}
-
void SwrSync(HANDLE hContext, PFN_CALLBACK_FUNC pfnFunc, uint64_t userData, uint64_t userData2, uint64_t userData3)
{
RDTSC_START(APISync);
pDC->FeWork.type = SYNC;
pDC->FeWork.pfnWork = ProcessSync;
- pDC->FeWork.desc.sync.pfnCallbackFunc = pfnFunc;
- pDC->FeWork.desc.sync.userData = userData;
- pDC->FeWork.desc.sync.userData2 = userData2;
- pDC->FeWork.desc.sync.userData3 = userData3;
- // cannot execute until all previous draws have completed
- pDC->dependency = pDC->drawId - 1;
+ // Setup callback function
+ pDC->retireCallback.pfnCallbackFunc = pfnFunc;
+ pDC->retireCallback.userData = userData;
+ pDC->retireCallback.userData2 = userData2;
+ pDC->retireCallback.userData3 = userData3;
//enqueue
QueueDraw(pContext);
SWR_CONTEXT *pContext = GetContext(hContext);
RDTSC_START(APIWaitForIdle);
- // Wait for all work to complete.
- for (uint32_t dc = 0; dc < KNOB_MAX_DRAWS_IN_FLIGHT; ++dc)
+
+ while (!pContext->dcRing.IsEmpty())
{
- DRAW_CONTEXT *pDC = &pContext->dcRing[dc];
+ _mm_pause();
+ }
- while (StillDrawing(pContext, pDC))
- {
- _mm_pause();
- }
+ RDTSC_STOP(APIWaitForIdle, 1, 0);
+}
+
+void SwrWaitForIdleFE(HANDLE hContext)
+{
+ SWR_CONTEXT *pContext = GetContext(hContext);
+
+ RDTSC_START(APIWaitForIdle);
+
+ while (pContext->drawsOutstandingFE > 0)
+ {
+ _mm_pause();
}
+
RDTSC_STOP(APIWaitForIdle, 1, 0);
}
void SwrSetCsFunc(
HANDLE hContext,
PFN_CS_FUNC pfnCsFunc,
- uint32_t totalThreadsInGroup)
+ uint32_t totalThreadsInGroup,
+ uint32_t totalSpillFillSize)
{
API_STATE* pState = GetDrawState(GetContext(hContext));
pState->pfnCsFunc = pfnCsFunc;
pState->totalThreadsInGroup = totalThreadsInGroup;
+ pState->totalSpillFillSize = totalSpillFillSize;
}
void SwrSetTsState(
pState->pfnBlendFunc[renderTarget] = pfnBlendFunc;
}
-void SwrSetLinkage(
- HANDLE hContext,
- uint32_t mask,
- const uint8_t* pMap)
-{
- API_STATE* pState = GetDrawState(GetContext(hContext));
-
- static const uint8_t IDENTITY_MAP[] =
- {
- 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
- 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31,
- };
- static_assert(sizeof(IDENTITY_MAP) == sizeof(pState->linkageMap),
- "Update for new value of MAX_ATTRIBUTES");
-
- pState->linkageMask = mask;
- pState->linkageCount = _mm_popcnt_u32(mask);
-
- if (!pMap)
- {
- pMap = IDENTITY_MAP;
- }
- memcpy(pState->linkageMap, pMap, pState->linkageCount);
-}
-
// update guardband multipliers for the viewport
void updateGuardband(API_STATE *pState)
{
}
else
{
+ // the vp width and height must be added to origin un-rounded then the result round to -inf.
+ // The cast to int works for rounding assuming all [left, right, top, bottom] are positive.
left = (int32_t)pState->vp[0].x;
- right = (int32_t)pState->vp[0].x + (int32_t)pState->vp[0].width;
+ right = (int32_t)(pState->vp[0].x + pState->vp[0].width);
top = (int32_t)pState->vp[0].y;
- bottom = (int32_t)pState->vp[0].y + (int32_t)pState->vp[0].height;
+ bottom = (int32_t)(pState->vp[0].y + pState->vp[0].height);
}
right = std::min<uint32_t>(right, KNOB_MAX_SCISSOR_X);
}
}
+// templated backend function tables
+extern PFN_BACKEND_FUNC gBackendNullPs[SWR_MULTISAMPLE_TYPE_COUNT];
+extern PFN_BACKEND_FUNC gBackendSingleSample[SWR_INPUT_COVERAGE_COUNT][2][2];
+extern PFN_BACKEND_FUNC gBackendPixelRateTable[SWR_MULTISAMPLE_TYPE_COUNT][SWR_MSAA_SAMPLE_PATTERN_COUNT][SWR_INPUT_COVERAGE_COUNT][2][2][2];
+extern PFN_BACKEND_FUNC gBackendSampleRateTable[SWR_MULTISAMPLE_TYPE_COUNT][SWR_INPUT_COVERAGE_COUNT][2][2];
void SetupPipeline(DRAW_CONTEXT *pDC)
{
DRAW_STATE* pState = pDC->pState;
const SWR_RASTSTATE &rastState = pState->state.rastState;
+ const SWR_PS_STATE &psState = pState->state.psState;
BACKEND_FUNCS& backendFuncs = pState->backendFuncs;
- const uint32_t forcedSampleCount = (rastState.bForcedSampleCount) ? 1 : 0;
+ const uint32_t forcedSampleCount = (rastState.forcedSampleCount) ? 1 : 0;
// setup backend
- if (pState->state.psState.pfnPixelShader == nullptr)
+ if (psState.pfnPixelShader == nullptr)
{
backendFuncs.pfnBackend = gBackendNullPs[pState->state.rastState.sampleCount];
- // always need to generate I & J per sample for Z interpolation
- backendFuncs.pfnCalcSampleBarycentrics = gSampleBarycentricTable[1];
}
else
{
- const bool bMultisampleEnable = ((rastState.sampleCount > SWR_MULTISAMPLE_1X) || rastState.bForcedSampleCount) ? 1 : 0;
- const uint32_t centroid = ((pState->state.psState.barycentricsMask & SWR_BARYCENTRIC_CENTROID_MASK) > 0) ? 1 : 0;
-
- // currently only support 'normal' input coverage
- SWR_ASSERT(pState->state.psState.inputCoverage == SWR_INPUT_COVERAGE_NORMAL ||
- pState->state.psState.inputCoverage == SWR_INPUT_COVERAGE_NONE);
+ const bool bMultisampleEnable = ((rastState.sampleCount > SWR_MULTISAMPLE_1X) || rastState.forcedSampleCount) ? 1 : 0;
+ const uint32_t centroid = ((psState.barycentricsMask & SWR_BARYCENTRIC_CENTROID_MASK) > 0) ? 1 : 0;
+ const uint32_t canEarlyZ = (psState.forceEarlyZ || (!psState.writesODepth && !psState.usesSourceDepth && !psState.usesUAV)) ? 1 : 0;
- SWR_BARYCENTRICS_MASK barycentricsMask = (SWR_BARYCENTRICS_MASK)pState->state.psState.barycentricsMask;
+ SWR_BARYCENTRICS_MASK barycentricsMask = (SWR_BARYCENTRICS_MASK)psState.barycentricsMask;
// select backend function
- switch(pState->state.psState.shadingRate)
+ switch(psState.shadingRate)
{
case SWR_SHADING_RATE_PIXEL:
if(bMultisampleEnable)
{
// always need to generate I & J per sample for Z interpolation
barycentricsMask = (SWR_BARYCENTRICS_MASK)(barycentricsMask | SWR_BARYCENTRIC_PER_SAMPLE_MASK);
- backendFuncs.pfnBackend = gBackendPixelRateTable[rastState.sampleCount][rastState.samplePattern][pState->state.psState.inputCoverage][centroid][forcedSampleCount];
- backendFuncs.pfnOutputMerger = gBackendOutputMergerTable[pState->state.psState.numRenderTargets][pState->state.blendState.sampleCount];
+ backendFuncs.pfnBackend = gBackendPixelRateTable[rastState.sampleCount][rastState.samplePattern][psState.inputCoverage][centroid][forcedSampleCount][canEarlyZ];
}
else
{
// always need to generate I & J per pixel for Z interpolation
barycentricsMask = (SWR_BARYCENTRICS_MASK)(barycentricsMask | SWR_BARYCENTRIC_PER_PIXEL_MASK);
- backendFuncs.pfnBackend = gBackendSingleSample[pState->state.psState.inputCoverage][centroid];
- backendFuncs.pfnOutputMerger = gBackendOutputMergerTable[pState->state.psState.numRenderTargets][SWR_MULTISAMPLE_1X];
+ backendFuncs.pfnBackend = gBackendSingleSample[psState.inputCoverage][centroid][canEarlyZ];
}
break;
case SWR_SHADING_RATE_SAMPLE:
SWR_ASSERT(rastState.samplePattern == SWR_MSAA_STANDARD_PATTERN);
// always need to generate I & J per sample for Z interpolation
barycentricsMask = (SWR_BARYCENTRICS_MASK)(barycentricsMask | SWR_BARYCENTRIC_PER_SAMPLE_MASK);
- backendFuncs.pfnBackend = gBackendSampleRateTable[rastState.sampleCount][pState->state.psState.inputCoverage][centroid];
- backendFuncs.pfnOutputMerger = gBackendOutputMergerTable[pState->state.psState.numRenderTargets][pState->state.blendState.sampleCount];
+ backendFuncs.pfnBackend = gBackendSampleRateTable[rastState.sampleCount][psState.inputCoverage][centroid][canEarlyZ];
break;
- case SWR_SHADING_RATE_COARSE:
default:
SWR_ASSERT(0 && "Invalid shading rate");
break;
}
-
- // setup pointer to function that generates necessary barycentrics required by the PS
- bool bBarycentrics = (barycentricsMask & SWR_BARYCENTRIC_PER_PIXEL_MASK) > 0 ? 1 : 0;
- backendFuncs.pfnCalcPixelBarycentrics = gPixelBarycentricTable[bBarycentrics];
-
- bBarycentrics = (barycentricsMask & SWR_BARYCENTRIC_PER_SAMPLE_MASK) > 0 ? 1 : 0;
- backendFuncs.pfnCalcSampleBarycentrics = gSampleBarycentricTable[bBarycentrics];
-
- bBarycentrics = (barycentricsMask & SWR_BARYCENTRIC_CENTROID_MASK) > 0 ? 1 : 0;
- backendFuncs.pfnCalcCentroidBarycentrics = gCentroidBarycentricTable[rastState.sampleCount][bBarycentrics][rastState.samplePattern][forcedSampleCount];
}
PFN_PROCESS_PRIMS pfnBinner;
break;
default:
pState->pfnProcessPrims = ClipTriangles;
- pfnBinner = BinTriangles;
+ pfnBinner = GetBinTrianglesFunc((rastState.conservativeRast > 0));
break;
};
(pState->state.depthStencilState.depthWriteEnable == FALSE) &&
(pState->state.depthStencilState.stencilTestEnable == FALSE) &&
(pState->state.depthStencilState.stencilWriteEnable == FALSE) &&
- (pState->state.linkageCount == 0))
+ (pState->state.backendState.numAttributes == 0))
{
pState->pfnProcessPrims = nullptr;
- pState->state.linkageMask = 0;
}
if (pState->state.soState.rasterizerDisable == true)
{
pState->pfnProcessPrims = nullptr;
- pState->state.linkageMask = 0;
}
- // set up the frontend attrib mask
- pState->state.feAttribMask = pState->state.linkageMask;
+ // set up the frontend attribute count
+ pState->state.feNumAttributes = 0;
+ const SWR_BACKEND_STATE& backendState = pState->state.backendState;
+ if (backendState.swizzleEnable)
+ {
+ // attribute swizzling is enabled, iterate over the map and record the max attribute used
+ for (uint32_t i = 0; i < backendState.numAttributes; ++i)
+ {
+ pState->state.feNumAttributes = std::max(pState->state.feNumAttributes, (uint32_t)backendState.swizzleMap[i].sourceAttrib + 1);
+ }
+ }
+ else
+ {
+ pState->state.feNumAttributes = pState->state.backendState.numAttributes;
+ }
+
if (pState->state.soState.soEnable)
{
+ uint32_t streamMasks = 0;
for (uint32_t i = 0; i < 4; ++i)
{
- pState->state.feAttribMask |= pState->state.soState.streamMasks[i];
+ streamMasks |= pState->state.soState.streamMasks[i];
+ }
+
+ DWORD maxAttrib;
+ if (_BitScanReverse(&maxAttrib, streamMasks))
+ {
+ pState->state.feNumAttributes = std::max(pState->state.feNumAttributes, (uint32_t)(maxAttrib + 1));
}
}
uint32_t numRTs = pState->state.psState.numRenderTargets;
pState->state.colorHottileEnable = 0;
- if(pState->state.psState.pfnPixelShader != nullptr)
+ if (psState.pfnPixelShader != nullptr)
{
for (uint32_t rt = 0; rt < numRTs; ++rt)
{
!pState->state.blendState.renderTarget[rt].writeDisableBlue) ? (1 << rt) : 0;
}
}
+
+ // Setup depth quantization function
+ if (pState->state.depthHottileEnable)
+ {
+ switch (pState->state.rastState.depthFormat)
+ {
+ case R32_FLOAT_X8X24_TYPELESS: pState->state.pfnQuantizeDepth = QuantizeDepth < R32_FLOAT_X8X24_TYPELESS > ; break;
+ case R32_FLOAT: pState->state.pfnQuantizeDepth = QuantizeDepth < R32_FLOAT > ; break;
+ case R24_UNORM_X8_TYPELESS: pState->state.pfnQuantizeDepth = QuantizeDepth < R24_UNORM_X8_TYPELESS > ; break;
+ case R16_UNORM: pState->state.pfnQuantizeDepth = QuantizeDepth < R16_UNORM > ; break;
+ default: SWR_ASSERT(false, "Unsupported depth format for depth quantiztion.");
+ pState->state.pfnQuantizeDepth = QuantizeDepth < R32_FLOAT > ;
+ }
+ }
+ else
+ {
+ // set up pass-through quantize if depth isn't enabled
+ pState->state.pfnQuantizeDepth = QuantizeDepth < R32_FLOAT > ;
+ }
}
//////////////////////////////////////////////////////////////////////////
}
break;
+ // The Primitive Assembly code can only handle 1 RECT at a time.
+ case TOP_RECT_LIST:
+ vertsPerDraw = 3;
+ break;
+
default:
// We are not splitting up draws for other topologies.
break;
return vertsPerDraw;
}
-// Recursive template used to auto-nest conditionals. Converts dynamic boolean function
-// arguments to static template arguments.
-template <bool... ArgsB>
-struct FEDrawChooser
-{
- // Last Arg Terminator
- static PFN_FE_WORK_FUNC GetFunc(bool bArg)
- {
- if (bArg)
- {
- return ProcessDraw<ArgsB..., true>;
- }
-
- return ProcessDraw<ArgsB..., false>;
- }
-
- // Recursively parse args
- template <typename... TArgsT>
- static PFN_FE_WORK_FUNC GetFunc(bool bArg, TArgsT... remainingArgs)
- {
- if (bArg)
- {
- return FEDrawChooser<ArgsB..., true>::GetFunc(remainingArgs...);
- }
-
- return FEDrawChooser<ArgsB..., false>::GetFunc(remainingArgs...);
- }
-};
-
-// Selector for correct templated Draw front-end function
-INLINE
-static PFN_FE_WORK_FUNC GetFEDrawFunc(bool IsIndexed, bool HasTessellation, bool HasGeometryShader, bool HasStreamOut, bool RasterizerEnabled)
-{
- return FEDrawChooser<>::GetFunc(IsIndexed, HasTessellation, HasGeometryShader, HasStreamOut, RasterizerEnabled);
-}
-
//////////////////////////////////////////////////////////////////////////
/// @brief DrawInstanced
SWR_CONTEXT *pContext = GetContext(hContext);
DRAW_CONTEXT* pDC = GetDrawContext(pContext);
- int32_t maxVertsPerDraw = MaxVertsPerDraw(pDC, numVertices, topology);
+ uint32_t maxVertsPerDraw = MaxVertsPerDraw(pDC, numVertices, topology);
uint32_t primsPerDraw = GetNumPrims(topology, maxVertsPerDraw);
- int32_t remainingVerts = numVertices;
+ uint32_t remainingVerts = numVertices;
API_STATE *pState = &pDC->pState->state;
pState->topology = topology;
InitDraw(pDC, isSplitDraw);
pDC->FeWork.type = DRAW;
- pDC->FeWork.pfnWork = GetFEDrawFunc(
+ pDC->FeWork.pfnWork = GetProcessDrawFunc(
false, // IsIndexed
+ false, // bEnableCutIndex
pState->tsState.tsEnable,
pState->gsState.gsEnable,
pState->soState.soEnable,
pDC->FeWork.desc.draw.startPrimID = draw * primsPerDraw;
pDC->FeWork.desc.draw.startVertexID = draw * maxVertsPerDraw;
+ pDC->cleanupState = (remainingVerts == numVertsForDraw);
+
//enqueue DC
QueueDraw(pContext);
DRAW_CONTEXT* pDC = GetDrawContext(pContext);
API_STATE* pState = &pDC->pState->state;
- int32_t maxIndicesPerDraw = MaxVertsPerDraw(pDC, numIndices, topology);
+ uint32_t maxIndicesPerDraw = MaxVertsPerDraw(pDC, numIndices, topology);
uint32_t primsPerDraw = GetNumPrims(topology, maxIndicesPerDraw);
- int32_t remainingIndices = numIndices;
+ uint32_t remainingIndices = numIndices;
uint32_t indexSize = 0;
switch (pState->indexBuffer.format)
InitDraw(pDC, isSplitDraw);
pDC->FeWork.type = DRAW;
- pDC->FeWork.pfnWork = GetFEDrawFunc(
+ pDC->FeWork.pfnWork = GetProcessDrawFunc(
true, // IsIndexed
+ pState->frontendState.bEnableCutIndex,
pState->tsState.tsEnable,
pState->gsState.gsEnable,
pState->soState.soEnable,
pDC->FeWork.desc.draw.baseVertex = baseVertex;
pDC->FeWork.desc.draw.startPrimID = draw * primsPerDraw;
+ pDC->cleanupState = (remainingIndices == numIndicesForDraw);
+
//enqueue DC
QueueDraw(pContext);
DrawIndexedInstance(hContext, topology, numIndices, indexOffset, baseVertex, numInstances, startInstance);
}
-// Attach surfaces to pipeline
+//////////////////////////////////////////////////////////////////////////
+/// @brief SwrInvalidateTiles
+/// @param hContext - Handle passed back from SwrCreateContext
+/// @param attachmentMask - The mask specifies which surfaces attached to the hottiles to invalidate.
void SwrInvalidateTiles(
HANDLE hContext,
uint32_t attachmentMask)
{
- SWR_CONTEXT *pContext = (SWR_CONTEXT*)hContext;
+ if (KNOB_TOSS_DRAW)
+ {
+ return;
+ }
+
+ SWR_CONTEXT *pContext = GetContext(hContext);
+ DRAW_CONTEXT* pDC = GetDrawContext(pContext);
+
+ pDC->FeWork.type = DISCARDINVALIDATETILES;
+ pDC->FeWork.pfnWork = ProcessDiscardInvalidateTiles;
+ pDC->FeWork.desc.discardInvalidateTiles.attachmentMask = attachmentMask;
+ memset(&pDC->FeWork.desc.discardInvalidateTiles.rect, 0, sizeof(SWR_RECT));
+ pDC->FeWork.desc.discardInvalidateTiles.newTileState = SWR_TILE_INVALID;
+ pDC->FeWork.desc.discardInvalidateTiles.createNewTiles = false;
+ pDC->FeWork.desc.discardInvalidateTiles.fullTilesOnly = false;
+
+ //enqueue
+ QueueDraw(pContext);
+}
+
+//////////////////////////////////////////////////////////////////////////
+/// @brief SwrDiscardRect
+/// @param hContext - Handle passed back from SwrCreateContext
+/// @param attachmentMask - The mask specifies which surfaces attached to the hottiles to discard.
+/// @param rect - if rect is all zeros, the entire attachment surface will be discarded
+void SwrDiscardRect(
+ HANDLE hContext,
+ uint32_t attachmentMask,
+ SWR_RECT rect)
+{
+ if (KNOB_TOSS_DRAW)
+ {
+ return;
+ }
+
+ SWR_CONTEXT *pContext = GetContext(hContext);
DRAW_CONTEXT* pDC = GetDrawContext(pContext);
// Queue a load to the hottile
- pDC->FeWork.type = INVALIDATETILES;
- pDC->FeWork.pfnWork = ProcessInvalidateTiles;
- pDC->FeWork.desc.invalidateTiles.attachmentMask = attachmentMask;
+ pDC->FeWork.type = DISCARDINVALIDATETILES;
+ pDC->FeWork.pfnWork = ProcessDiscardInvalidateTiles;
+ pDC->FeWork.desc.discardInvalidateTiles.attachmentMask = attachmentMask;
+ pDC->FeWork.desc.discardInvalidateTiles.rect = rect;
+ pDC->FeWork.desc.discardInvalidateTiles.newTileState = SWR_TILE_RESOLVED;
+ pDC->FeWork.desc.discardInvalidateTiles.createNewTiles = true;
+ pDC->FeWork.desc.discardInvalidateTiles.fullTilesOnly = true;
//enqueue
QueueDraw(pContext);
}
RDTSC_START(APIDispatch);
- SWR_CONTEXT *pContext = (SWR_CONTEXT*)hContext;
+ SWR_CONTEXT *pContext = GetContext(hContext);
DRAW_CONTEXT* pDC = GetDrawContext(pContext);
pDC->isCompute = true; // This is a compute context.
- // Ensure spill fill pointers are initialized to nullptr.
- memset(pDC->pSpillFill, 0, sizeof(pDC->pSpillFill));
-
COMPUTE_DESC* pTaskData = (COMPUTE_DESC*)pDC->pArena->AllocAligned(sizeof(COMPUTE_DESC), 64);
pTaskData->threadGroupCountX = threadGroupCountX;
pTaskData->threadGroupCountZ = threadGroupCountZ;
uint32_t totalThreadGroups = threadGroupCountX * threadGroupCountY * threadGroupCountZ;
+ uint32_t dcIndex = pDC->drawId % KNOB_MAX_DRAWS_IN_FLIGHT;
+ pDC->pDispatch = &pContext->pDispatchQueueArray[dcIndex];
pDC->pDispatch->initialize(totalThreadGroups, pTaskData);
QueueDispatch(pContext);
SWR_RENDERTARGET_ATTACHMENT attachment,
SWR_TILE_STATE postStoreTileState)
{
+ if (KNOB_TOSS_DRAW)
+ {
+ return;
+ }
+
RDTSC_START(APIStoreTiles);
- SWR_CONTEXT *pContext = (SWR_CONTEXT*)hContext;
+ SWR_CONTEXT *pContext = GetContext(hContext);
DRAW_CONTEXT* pDC = GetDrawContext(pContext);
SetupMacroTileScissors(pDC);
uint32_t clearMask,
const float clearColor[4],
float z,
- BYTE stencil)
+ uint8_t stencil)
{
+ if (KNOB_TOSS_DRAW)
+ {
+ return;
+ }
+
RDTSC_START(APIClearRenderTarget);
- SWR_CONTEXT *pContext = (SWR_CONTEXT*)hContext;
+ SWR_CONTEXT *pContext = GetContext(hContext);
DRAW_CONTEXT* pDC = GetDrawContext(pContext);
pDC->FeWork.desc.queryStats.pStats = pStats;
// cannot execute until all previous draws have completed
- pDC->dependency = pDC->drawId - 1;
+ pDC->dependent = true;
//enqueue
QueueDraw(pContext);
HANDLE hContext)
{
RDTSC_ENDFRAME();
+ SWR_CONTEXT *pContext = GetContext(hContext);
+ pContext->frameCount++;
}
projects / mesa.git / blobdiff