src/gallium/drivers/swr/rasterizer/core/api.cpp

   1 /****************************************************************************
   2 * Copyright (C) 2014-2015 Intel Corporation.   All Rights Reserved.
   3 *
   4 * Permission is hereby granted, free of charge, to any person obtaining a
   5 * copy of this software and associated documentation files (the "Software"),
   6 * to deal in the Software without restriction, including without limitation
   7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8 * and/or sell copies of the Software, and to permit persons to whom the
   9 * Software is furnished to do so, subject to the following conditions:
  10 *
  11 * The above copyright notice and this permission notice (including the next
  12 * paragraph) shall be included in all copies or substantial portions of the
  13 * Software.
  14 *
  15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21 * IN THE SOFTWARE.
  22 *
  23 * @file api.cpp
  24 *
  25 * @brief API implementation
  26 *
  27 ******************************************************************************/
  28
  29 #include <cfloat>
  30 #include <cmath>
  31 #include <cstdio>
  32 #include <new>
  33
  34 #include "core/api.h"
  35 #include "core/backend.h"
  36 #include "core/context.h"
  37 #include "core/depthstencil.h"
  38 #include "core/frontend.h"
  39 #include "core/rasterizer.h"
  40 #include "core/rdtsc_core.h"
  41 #include "core/threads.h"
  42 #include "core/tilemgr.h"
  43 #include "core/clip.h"
  44 #include "core/utils.h"
  45
  46 #include "common/simdintrin.h"
  47 #include "common/os.h"
  48
  49 void SetupDefaultState(SWR_CONTEXT *pContext);
  50
  51 //////////////////////////////////////////////////////////////////////////
  52 /// @brief Create SWR Context.
  53 /// @param pCreateInfo - pointer to creation info.
  54 HANDLE SwrCreateContext(
  55     SWR_CREATECONTEXT_INFO* pCreateInfo)
  56 {
  57     RDTSC_RESET();
  58     RDTSC_INIT(0);
  59
  60     void* pContextMem = _aligned_malloc(sizeof(SWR_CONTEXT), KNOB_SIMD_WIDTH * 4);
  61     memset(pContextMem, 0, sizeof(SWR_CONTEXT));
  62     SWR_CONTEXT *pContext = new (pContextMem) SWR_CONTEXT();
  63
  64     pContext->driverType = pCreateInfo->driver;
  65     pContext->privateStateSize = pCreateInfo->privateStateSize;
  66
  67     pContext->dcRing.Init(KNOB_MAX_DRAWS_IN_FLIGHT);
  68     pContext->dsRing.Init(KNOB_MAX_DRAWS_IN_FLIGHT);
  69
  70     pContext->pMacroTileManagerArray = (MacroTileMgr*)_aligned_malloc(sizeof(MacroTileMgr) * KNOB_MAX_DRAWS_IN_FLIGHT, 64);
  71     pContext->pDispatchQueueArray = (DispatchQueue*)_aligned_malloc(sizeof(DispatchQueue) * KNOB_MAX_DRAWS_IN_FLIGHT, 64);
  72
  73     for (uint32_t dc = 0; dc < KNOB_MAX_DRAWS_IN_FLIGHT; ++dc)
  74     {
  75         pContext->dcRing[dc].pArena = new CachingArena(pContext->cachingArenaAllocator);
  76         new (&pContext->pMacroTileManagerArray[dc]) MacroTileMgr(*pContext->dcRing[dc].pArena);
  77         new (&pContext->pDispatchQueueArray[dc]) DispatchQueue();
  78
  79         pContext->dsRing[dc].pArena = new CachingArena(pContext->cachingArenaAllocator);
  80     }
  81
  82     if (!KNOB_SINGLE_THREADED)
  83     {
  84         memset(&pContext->WaitLock, 0, sizeof(pContext->WaitLock));
  85         memset(&pContext->FifosNotEmpty, 0, sizeof(pContext->FifosNotEmpty));
  86         new (&pContext->WaitLock) std::mutex();
  87         new (&pContext->FifosNotEmpty) std::condition_variable();
  88
  89         CreateThreadPool(pContext, &pContext->threadPool);
  90     }
  91
  92     // Calling createThreadPool() above can set SINGLE_THREADED
  93     if (KNOB_SINGLE_THREADED)
  94     {
  95         SET_KNOB(HYPERTHREADED_FE, false);
  96         pContext->NumWorkerThreads = 1;
  97         pContext->NumFEThreads = 1;
  98         pContext->NumBEThreads = 1;
  99     }
 100
 101     // Allocate scratch space for workers.
 102     ///@note We could lazily allocate this but its rather small amount of memory.
 103     for (uint32_t i = 0; i < pContext->NumWorkerThreads; ++i)
 104     {
 105 #if defined(_WIN32)
 106         uint32_t numaNode = pContext->threadPool.pThreadData ?
 107             pContext->threadPool.pThreadData[i].numaId : 0;
 108         pContext->pScratch[i] = (uint8_t*)VirtualAllocExNuma(
 109             GetCurrentProcess(), nullptr, 32 * sizeof(KILOBYTE),
 110             MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE,
 111             numaNode);
 112 #else
 113         pContext->pScratch[i] = (uint8_t*)_aligned_malloc(32 * sizeof(KILOBYTE), KNOB_SIMD_WIDTH * 4);
 114 #endif
 115     }
 116
 117     // State setup AFTER context is fully initialized
 118     SetupDefaultState(pContext);
 119
 120     // initialize hot tile manager
 121     pContext->pHotTileMgr = new HotTileMgr();
 122
 123     // initialize function pointer tables
 124     InitClearTilesTable();
 125
 126     // initialize store tiles function
 127     pContext->pfnLoadTile = pCreateInfo->pfnLoadTile;
 128     pContext->pfnStoreTile = pCreateInfo->pfnStoreTile;
 129     pContext->pfnClearTile = pCreateInfo->pfnClearTile;
 130
 131     // pass pointer to bucket manager back to caller
 132 #ifdef KNOB_ENABLE_RDTSC
 133     pCreateInfo->pBucketMgr = &gBucketMgr;
 134 #endif
 135
 136     pCreateInfo->contextSaveSize = sizeof(API_STATE);
 137
 138     return (HANDLE)pContext;
 139 }
 140
 141 void SwrDestroyContext(HANDLE hContext)
 142 {
 143     SWR_CONTEXT *pContext = (SWR_CONTEXT*)hContext;
 144     DestroyThreadPool(pContext, &pContext->threadPool);
 145
 146     // free the fifos
 147     for (uint32_t i = 0; i < KNOB_MAX_DRAWS_IN_FLIGHT; ++i)
 148     {
 149         delete pContext->dcRing[i].pArena;
 150         delete pContext->dsRing[i].pArena;
 151         pContext->pMacroTileManagerArray[i].~MacroTileMgr();
 152         pContext->pDispatchQueueArray[i].~DispatchQueue();
 153     }
 154
 155     _aligned_free(pContext->pDispatchQueueArray);
 156     _aligned_free(pContext->pMacroTileManagerArray);
 157
 158     // Free scratch space.
 159     for (uint32_t i = 0; i < pContext->NumWorkerThreads; ++i)
 160     {
 161 #if defined(_WIN32)
 162         VirtualFree(pContext->pScratch[i], 0, MEM_RELEASE);
 163 #else
 164         _aligned_free(pContext->pScratch[i]);
 165 #endif
 166     }
 167
 168     delete(pContext->pHotTileMgr);
 169
 170     pContext->~SWR_CONTEXT();
 171     _aligned_free((SWR_CONTEXT*)hContext);
 172 }
 173
 174 void CopyState(DRAW_STATE& dst, const DRAW_STATE& src)
 175 {
 176     memcpy(&dst.state, &src.state, sizeof(API_STATE));
 177 }
 178
 179 void WakeAllThreads(SWR_CONTEXT *pContext)
 180 {
 181     pContext->FifosNotEmpty.notify_all();
 182 }
 183
 184 template<bool IsDraw>
 185 void QueueWork(SWR_CONTEXT *pContext)
 186 {
 187     DRAW_CONTEXT* pDC = pContext->pCurDrawContext;
 188     uint32_t dcIndex = pDC->drawId % KNOB_MAX_DRAWS_IN_FLIGHT;
 189
 190     if (IsDraw)
 191     {
 192         pDC->pTileMgr = &pContext->pMacroTileManagerArray[dcIndex];
 193         pDC->pTileMgr->initialize();
 194     }
 195
 196     // Each worker thread looks at a DC for both FE and BE work at different times and so we
 197     // multiply threadDone by 2.  When the threadDone counter has reached 0 then all workers
 198     // have moved past this DC. (i.e. Each worker has checked this DC for both FE and BE work and
 199     // then moved on if all work is done.)
 200     pContext->pCurDrawContext->threadsDone = pContext->NumFEThreads + pContext->NumBEThreads;
 201
 202     _ReadWriteBarrier();
 203     {
 204         std::unique_lock<std::mutex> lock(pContext->WaitLock);
 205         pContext->dcRing.Enqueue();
 206     }
 207
 208     if (KNOB_SINGLE_THREADED)
 209     {
 210         // flush denormals to 0
 211         uint32_t mxcsr = _mm_getcsr();
 212         _mm_setcsr(mxcsr | _MM_FLUSH_ZERO_ON | _MM_DENORMALS_ZERO_ON);
 213
 214         if (IsDraw)
 215         {
 216             static TileSet lockedTiles;
 217             uint64_t curDraw[2] = { pContext->pCurDrawContext->drawId, pContext->pCurDrawContext->drawId };
 218             WorkOnFifoFE(pContext, 0, curDraw[0]);
 219             WorkOnFifoBE(pContext, 0, curDraw[1], lockedTiles, 0, 0);
 220         }
 221         else
 222         {
 223             uint64_t curDispatch = pContext->pCurDrawContext->drawId;
 224             WorkOnCompute(pContext, 0, curDispatch);
 225         }
 226
 227         // Dequeue the work here, if not already done, since we're single threaded (i.e. no workers).
 228         while (CompleteDrawContext(pContext, pContext->pCurDrawContext) > 0) {}
 229
 230         // restore csr
 231         _mm_setcsr(mxcsr);
 232     }
 233     else
 234     {
 235         RDTSC_START(APIDrawWakeAllThreads);
 236         WakeAllThreads(pContext);
 237         RDTSC_STOP(APIDrawWakeAllThreads, 1, 0);
 238     }
 239
 240     // Set current draw context to NULL so that next state call forces a new draw context to be created and populated.
 241     pContext->pPrevDrawContext = pContext->pCurDrawContext;
 242     pContext->pCurDrawContext = nullptr;
 243 }
 244
 245 INLINE void QueueDraw(SWR_CONTEXT* pContext)
 246 {
 247     QueueWork<true>(pContext);
 248 }
 249
 250 INLINE void QueueDispatch(SWR_CONTEXT* pContext)
 251 {
 252     QueueWork<false>(pContext);
 253 }
 254
 255 DRAW_CONTEXT* GetDrawContext(SWR_CONTEXT *pContext, bool isSplitDraw = false)
 256 {
 257     RDTSC_START(APIGetDrawContext);
 258     // If current draw context is null then need to obtain a new draw context to use from ring.
 259     if (pContext->pCurDrawContext == nullptr)
 260     {
 261         // Need to wait for a free entry.
 262         while (pContext->dcRing.IsFull())
 263         {
 264             _mm_pause();
 265         }
 266
 267         uint64_t curDraw = pContext->dcRing.GetHead();
 268         uint32_t dcIndex = curDraw % KNOB_MAX_DRAWS_IN_FLIGHT;
 269
 270         static uint64_t lastDrawChecked;
 271         static uint32_t lastFrameChecked;
 272         if ((pContext->frameCount - lastFrameChecked) > 2 ||
 273             (curDraw - lastDrawChecked) > 0x10000)
 274         {
 275             // Take this opportunity to clean-up old arena allocations
 276             pContext->cachingArenaAllocator.FreeOldBlocks();
 277
 278             lastFrameChecked = pContext->frameCount;
 279             lastDrawChecked = curDraw;
 280         }
 281
 282         DRAW_CONTEXT* pCurDrawContext = &pContext->dcRing[dcIndex];
 283         pContext->pCurDrawContext = pCurDrawContext;
 284
 285         // Assign next available entry in DS ring to this DC.
 286         uint32_t dsIndex = pContext->curStateId % KNOB_MAX_DRAWS_IN_FLIGHT;
 287         pCurDrawContext->pState = &pContext->dsRing[dsIndex];
 288
 289         // Copy previous state to current state.
 290         if (pContext->pPrevDrawContext)
 291         {
 292             DRAW_CONTEXT* pPrevDrawContext = pContext->pPrevDrawContext;
 293
 294             // If we're splitting our draw then we can just use the same state from the previous
 295             // draw. In this case, we won't increment the DS ring index so the next non-split
 296             // draw can receive the state.
 297             if (isSplitDraw == false)
 298             {
 299                 CopyState(*pCurDrawContext->pState, *pPrevDrawContext->pState);
 300
 301                 // Should have been cleaned up previously
 302                 SWR_ASSERT(pCurDrawContext->pState->pArena->IsEmpty() == true);
 303
 304                 pCurDrawContext->pState->pPrivateState = nullptr;
 305
 306                 pContext->curStateId++;  // Progress state ring index forward.
 307             }
 308             else
 309             {
 310                 // If its a split draw then just copy the state pointer over
 311                 // since its the same draw.
 312                 pCurDrawContext->pState = pPrevDrawContext->pState;
 313                 SWR_ASSERT(pPrevDrawContext->cleanupState == false);
 314             }
 315         }
 316         else
 317         {
 318             SWR_ASSERT(pCurDrawContext->pState->pArena->IsEmpty() == true);
 319             pContext->curStateId++;  // Progress state ring index forward.
 320         }
 321
 322         SWR_ASSERT(pCurDrawContext->pArena->IsEmpty() == true);
 323
 324         pCurDrawContext->dependency = 0;
 325         pCurDrawContext->pContext = pContext;
 326         pCurDrawContext->isCompute = false; // Dispatch has to set this to true.
 327
 328         pCurDrawContext->doneFE = false;
 329         pCurDrawContext->FeLock = 0;
 330         pCurDrawContext->threadsDone = 0;
 331
 332         // Assign unique drawId for this DC
 333         pCurDrawContext->drawId = pContext->dcRing.GetHead();
 334
 335         pCurDrawContext->cleanupState = true;
 336     }
 337     else
 338     {
 339         SWR_ASSERT(isSplitDraw == false, "Split draw should only be used when obtaining a new DC");
 340     }
 341
 342     RDTSC_STOP(APIGetDrawContext, 0, 0);
 343     return pContext->pCurDrawContext;
 344 }
 345
 346 API_STATE* GetDrawState(SWR_CONTEXT *pContext)
 347 {
 348     DRAW_CONTEXT* pDC = GetDrawContext(pContext);
 349     SWR_ASSERT(pDC->pState != nullptr);
 350
 351     return &pDC->pState->state;
 352 }
 353
 354 void SWR_API SwrSaveState(
 355     HANDLE hContext,
 356     void* pOutputStateBlock,
 357     size_t memSize)
 358 {
 359     SWR_CONTEXT *pContext = (SWR_CONTEXT*)hContext;
 360     auto pSrc = GetDrawState(pContext);
 361     SWR_ASSERT(pOutputStateBlock && memSize >= sizeof(*pSrc));
 362
 363     memcpy(pOutputStateBlock, pSrc, sizeof(*pSrc));
 364 }
 365
 366 void SWR_API SwrRestoreState(
 367     HANDLE hContext,
 368     const void* pStateBlock,
 369     size_t memSize)
 370 {
 371     SWR_CONTEXT *pContext = (SWR_CONTEXT*)hContext;
 372     auto pDst = GetDrawState(pContext);
 373     SWR_ASSERT(pStateBlock && memSize >= sizeof(*pDst));
 374
 375     memcpy(pDst, pStateBlock, sizeof(*pDst));
 376 }
 377
 378 void SetupDefaultState(SWR_CONTEXT *pContext)
 379 {
 380     API_STATE* pState = GetDrawState(pContext);
 381
 382     pState->rastState.cullMode = SWR_CULLMODE_NONE;
 383     pState->rastState.frontWinding = SWR_FRONTWINDING_CCW;
 384 }
 385
 386 static INLINE SWR_CONTEXT* GetContext(HANDLE hContext)
 387 {
 388     return (SWR_CONTEXT*)hContext;
 389 }
 390
 391 void SwrSync(HANDLE hContext, PFN_CALLBACK_FUNC pfnFunc, uint64_t userData, uint64_t userData2, uint64_t userData3)
 392 {
 393     RDTSC_START(APISync);
 394
 395     SWR_ASSERT(pfnFunc != nullptr);
 396
 397     SWR_CONTEXT *pContext = GetContext(hContext);
 398     DRAW_CONTEXT* pDC = GetDrawContext(pContext);
 399
 400     pDC->FeWork.type = SYNC;
 401     pDC->FeWork.pfnWork = ProcessSync;
 402     pDC->FeWork.desc.sync.pfnCallbackFunc = pfnFunc;
 403     pDC->FeWork.desc.sync.userData = userData;
 404     pDC->FeWork.desc.sync.userData2 = userData2;
 405     pDC->FeWork.desc.sync.userData3 = userData3;
 406
 407     // cannot execute until all previous draws have completed
 408     pDC->dependency = pDC->drawId - 1;
 409
 410     //enqueue
 411     QueueDraw(pContext);
 412
 413     RDTSC_STOP(APISync, 1, 0);
 414 }
 415
 416 void SwrWaitForIdle(HANDLE hContext)
 417 {
 418     SWR_CONTEXT *pContext = GetContext(hContext);
 419
 420     RDTSC_START(APIWaitForIdle);
 421
 422     while (!pContext->dcRing.IsEmpty())
 423     {
 424         _mm_pause();
 425     }
 426
 427     RDTSC_STOP(APIWaitForIdle, 1, 0);
 428 }
 429
 430 void SwrSetVertexBuffers(
 431     HANDLE hContext,
 432     uint32_t numBuffers,
 433     const SWR_VERTEX_BUFFER_STATE* pVertexBuffers)
 434 {
 435     API_STATE* pState = GetDrawState(GetContext(hContext));
 436
 437     for (uint32_t i = 0; i < numBuffers; ++i)
 438     {
 439         const SWR_VERTEX_BUFFER_STATE *pVB = &pVertexBuffers[i];
 440         pState->vertexBuffers[pVB->index] = *pVB;
 441     }
 442 }
 443
 444 void SwrSetIndexBuffer(
 445     HANDLE hContext,
 446     const SWR_INDEX_BUFFER_STATE* pIndexBuffer)
 447 {
 448     API_STATE* pState = GetDrawState(GetContext(hContext));
 449
 450     pState->indexBuffer = *pIndexBuffer;
 451 }
 452
 453 void SwrSetFetchFunc(
 454     HANDLE hContext,
 455     PFN_FETCH_FUNC    pfnFetchFunc)
 456 {
 457     API_STATE* pState = GetDrawState(GetContext(hContext));
 458
 459     pState->pfnFetchFunc = pfnFetchFunc;
 460 }
 461
 462 void SwrSetSoFunc(
 463     HANDLE hContext,
 464     PFN_SO_FUNC    pfnSoFunc,
 465     uint32_t streamIndex)
 466 {
 467     API_STATE* pState = GetDrawState(GetContext(hContext));
 468
 469     SWR_ASSERT(streamIndex < MAX_SO_STREAMS);
 470
 471     pState->pfnSoFunc[streamIndex] = pfnSoFunc;
 472 }
 473
 474 void SwrSetSoState(
 475     HANDLE hContext,
 476     SWR_STREAMOUT_STATE* pSoState)
 477 {
 478     API_STATE* pState = GetDrawState(GetContext(hContext));
 479
 480     pState->soState = *pSoState;
 481 }
 482
 483 void SwrSetSoBuffers(
 484     HANDLE hContext,
 485     SWR_STREAMOUT_BUFFER* pSoBuffer,
 486     uint32_t slot)
 487 {
 488     API_STATE* pState = GetDrawState(GetContext(hContext));
 489
 490     SWR_ASSERT((slot < 4), "There are only 4 SO buffer slots [0, 3]\nSlot requested: %d", slot);
 491
 492     pState->soBuffer[slot] = *pSoBuffer;
 493 }
 494
 495 void SwrSetVertexFunc(
 496     HANDLE hContext,
 497     PFN_VERTEX_FUNC pfnVertexFunc)
 498 {
 499     API_STATE* pState = GetDrawState(GetContext(hContext));
 500
 501     pState->pfnVertexFunc = pfnVertexFunc;
 502 }
 503
 504 void SwrSetFrontendState(
 505     HANDLE hContext,
 506     SWR_FRONTEND_STATE *pFEState)
 507 {
 508     API_STATE* pState = GetDrawState(GetContext(hContext));
 509     pState->frontendState = *pFEState;
 510 }
 511
 512 void SwrSetGsState(
 513     HANDLE hContext,
 514     SWR_GS_STATE *pGSState)
 515 {
 516     API_STATE* pState = GetDrawState(GetContext(hContext));
 517     pState->gsState = *pGSState;
 518 }
 519
 520 void SwrSetGsFunc(
 521     HANDLE hContext,
 522     PFN_GS_FUNC pfnGsFunc)
 523 {
 524     API_STATE* pState = GetDrawState(GetContext(hContext));
 525     pState->pfnGsFunc = pfnGsFunc;
 526 }
 527
 528 void SwrSetCsFunc(
 529     HANDLE hContext,
 530     PFN_CS_FUNC pfnCsFunc,
 531     uint32_t totalThreadsInGroup,
 532     uint32_t totalSpillFillSize)
 533 {
 534     API_STATE* pState = GetDrawState(GetContext(hContext));
 535     pState->pfnCsFunc = pfnCsFunc;
 536     pState->totalThreadsInGroup = totalThreadsInGroup;
 537     pState->totalSpillFillSize = totalSpillFillSize;
 538 }
 539
 540 void SwrSetTsState(
 541     HANDLE hContext,
 542     SWR_TS_STATE *pState)
 543 {
 544     API_STATE* pApiState = GetDrawState(GetContext(hContext));
 545     pApiState->tsState = *pState;
 546 }
 547
 548 void SwrSetHsFunc(
 549     HANDLE hContext,
 550     PFN_HS_FUNC pfnFunc)
 551 {
 552     API_STATE* pApiState = GetDrawState(GetContext(hContext));
 553     pApiState->pfnHsFunc = pfnFunc;
 554 }
 555
 556 void SwrSetDsFunc(
 557     HANDLE hContext,
 558     PFN_DS_FUNC pfnFunc)
 559 {
 560     API_STATE* pApiState = GetDrawState(GetContext(hContext));
 561     pApiState->pfnDsFunc = pfnFunc;
 562 }
 563
 564 void SwrSetDepthStencilState(
 565     HANDLE hContext,
 566     SWR_DEPTH_STENCIL_STATE *pDSState)
 567 {
 568     API_STATE* pState = GetDrawState(GetContext(hContext));
 569
 570     pState->depthStencilState = *pDSState;
 571 }
 572
 573 void SwrSetBackendState(
 574     HANDLE hContext,
 575     SWR_BACKEND_STATE *pBEState)
 576 {
 577     API_STATE* pState = GetDrawState(GetContext(hContext));
 578
 579     pState->backendState = *pBEState;
 580 }
 581
 582 void SwrSetPixelShaderState(
 583     HANDLE hContext,
 584     SWR_PS_STATE *pPSState)
 585 {
 586     API_STATE *pState = GetDrawState(GetContext(hContext));
 587     pState->psState = *pPSState;
 588 }
 589
 590 void SwrSetBlendState(
 591     HANDLE hContext,
 592     SWR_BLEND_STATE *pBlendState)
 593 {
 594     API_STATE *pState = GetDrawState(GetContext(hContext));
 595     memcpy(&pState->blendState, pBlendState, sizeof(SWR_BLEND_STATE));
 596 }
 597
 598 void SwrSetBlendFunc(
 599     HANDLE hContext,
 600     uint32_t renderTarget,
 601     PFN_BLEND_JIT_FUNC pfnBlendFunc)
 602 {
 603     SWR_ASSERT(renderTarget < SWR_NUM_RENDERTARGETS);
 604     API_STATE *pState = GetDrawState(GetContext(hContext));
 605     pState->pfnBlendFunc[renderTarget] = pfnBlendFunc;
 606 }
 607
 608 void SwrSetLinkage(
 609     HANDLE hContext,
 610     uint32_t mask,
 611     const uint8_t* pMap)
 612 {
 613     API_STATE* pState = GetDrawState(GetContext(hContext));
 614
 615     static const uint8_t IDENTITY_MAP[] =
 616     {
 617          0,  1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11, 12, 13, 14, 15,
 618         16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31,
 619     };
 620     static_assert(sizeof(IDENTITY_MAP) == sizeof(pState->linkageMap),
 621         "Update for new value of MAX_ATTRIBUTES");
 622
 623     pState->linkageMask = mask;
 624     pState->linkageCount = _mm_popcnt_u32(mask);
 625
 626     if (!pMap)
 627     {
 628         pMap = IDENTITY_MAP;
 629     }
 630     memcpy(pState->linkageMap, pMap, pState->linkageCount);
 631 }
 632
 633 // update guardband multipliers for the viewport
 634 void updateGuardband(API_STATE *pState)
 635 {
 636     // guardband center is viewport center
 637     pState->gbState.left    = KNOB_GUARDBAND_WIDTH  / pState->vp[0].width;
 638     pState->gbState.right   = KNOB_GUARDBAND_WIDTH  / pState->vp[0].width;
 639     pState->gbState.top     = KNOB_GUARDBAND_HEIGHT / pState->vp[0].height;
 640     pState->gbState.bottom  = KNOB_GUARDBAND_HEIGHT / pState->vp[0].height;
 641 }
 642
 643 void SwrSetRastState(
 644     HANDLE hContext,
 645     const SWR_RASTSTATE *pRastState)
 646 {
 647     SWR_CONTEXT *pContext = GetContext(hContext);
 648     API_STATE* pState = GetDrawState(pContext);
 649
 650     memcpy(&pState->rastState, pRastState, sizeof(SWR_RASTSTATE));
 651 }
 652
 653 void SwrSetViewports(
 654     HANDLE hContext,
 655     uint32_t numViewports,
 656     const SWR_VIEWPORT* pViewports,
 657     const SWR_VIEWPORT_MATRIX* pMatrices)
 658 {
 659     SWR_ASSERT(numViewports <= KNOB_NUM_VIEWPORTS_SCISSORS,
 660         "Invalid number of viewports.");
 661
 662     SWR_CONTEXT *pContext = GetContext(hContext);
 663     API_STATE* pState = GetDrawState(pContext);
 664
 665     memcpy(&pState->vp[0], pViewports, sizeof(SWR_VIEWPORT) * numViewports);
 666
 667     if (pMatrices != nullptr)
 668     {
 669         memcpy(&pState->vpMatrix[0], pMatrices, sizeof(SWR_VIEWPORT_MATRIX) * numViewports);
 670     }
 671     else
 672     {
 673         // Compute default viewport transform.
 674         for (uint32_t i = 0; i < numViewports; ++i)
 675         {
 676             if (pContext->driverType == DX)
 677             {
 678                 pState->vpMatrix[i].m00 = pState->vp[i].width / 2.0f;
 679                 pState->vpMatrix[i].m11 = -pState->vp[i].height / 2.0f;
 680                 pState->vpMatrix[i].m22 = pState->vp[i].maxZ - pState->vp[i].minZ;
 681                 pState->vpMatrix[i].m30 = pState->vp[i].x + pState->vpMatrix[i].m00;
 682                 pState->vpMatrix[i].m31 = pState->vp[i].y - pState->vpMatrix[i].m11;
 683                 pState->vpMatrix[i].m32 = pState->vp[i].minZ;
 684             }
 685             else
 686             {
 687                 // Standard, with the exception that Y is inverted.
 688                 pState->vpMatrix[i].m00 = (pState->vp[i].width - pState->vp[i].x) / 2.0f;
 689                 pState->vpMatrix[i].m11 = (pState->vp[i].y - pState->vp[i].height) / 2.0f;
 690                 pState->vpMatrix[i].m22 = (pState->vp[i].maxZ - pState->vp[i].minZ) / 2.0f;
 691                 pState->vpMatrix[i].m30 = pState->vp[i].x + pState->vpMatrix[i].m00;
 692                 pState->vpMatrix[i].m31 = pState->vp[i].height + pState->vpMatrix[i].m11;
 693                 pState->vpMatrix[i].m32 = pState->vp[i].minZ + pState->vpMatrix[i].m22;
 694
 695                 // Now that the matrix is calculated, clip the view coords to screen size.
 696                 // OpenGL allows for -ve x,y in the viewport.
 697                 pState->vp[i].x = std::max(pState->vp[i].x, 0.0f);
 698                 pState->vp[i].y = std::max(pState->vp[i].y, 0.0f);
 699             }
 700         }
 701     }
 702
 703     updateGuardband(pState);
 704 }
 705
 706 void SwrSetScissorRects(
 707     HANDLE hContext,
 708     uint32_t numScissors,
 709     const BBOX* pScissors)
 710 {
 711     SWR_ASSERT(numScissors <= KNOB_NUM_VIEWPORTS_SCISSORS,
 712         "Invalid number of scissor rects.");
 713
 714     API_STATE* pState = GetDrawState(GetContext(hContext));
 715     memcpy(&pState->scissorRects[0], pScissors, numScissors * sizeof(BBOX));
 716 };
 717
 718 void SetupMacroTileScissors(DRAW_CONTEXT *pDC)
 719 {
 720     API_STATE *pState = &pDC->pState->state;
 721     uint32_t left, right, top, bottom;
 722
 723     // Set up scissor dimensions based on scissor or viewport
 724     if (pState->rastState.scissorEnable)
 725     {
 726         // scissor rect right/bottom edge are exclusive, core expects scissor dimensions to be inclusive, so subtract one pixel from right/bottom edges
 727         left = pState->scissorRects[0].left;
 728         right = pState->scissorRects[0].right;
 729         top = pState->scissorRects[0].top;
 730         bottom = pState->scissorRects[0].bottom;
 731     }
 732     else
 733     {
 734         left = (int32_t)pState->vp[0].x;
 735         right = (int32_t)pState->vp[0].x + (int32_t)pState->vp[0].width;
 736         top = (int32_t)pState->vp[0].y;
 737         bottom = (int32_t)pState->vp[0].y + (int32_t)pState->vp[0].height;
 738     }
 739
 740     right = std::min<uint32_t>(right, KNOB_MAX_SCISSOR_X);
 741     bottom = std::min<uint32_t>(bottom, KNOB_MAX_SCISSOR_Y);
 742
 743     if (left > KNOB_MAX_SCISSOR_X || top > KNOB_MAX_SCISSOR_Y)
 744     {
 745         pState->scissorInFixedPoint.left = 0;
 746         pState->scissorInFixedPoint.right = 0;
 747         pState->scissorInFixedPoint.top = 0;
 748         pState->scissorInFixedPoint.bottom = 0;
 749     }
 750     else
 751     {
 752         pState->scissorInFixedPoint.left = left * FIXED_POINT_SCALE;
 753         pState->scissorInFixedPoint.right = right * FIXED_POINT_SCALE - 1;
 754         pState->scissorInFixedPoint.top = top * FIXED_POINT_SCALE;
 755         pState->scissorInFixedPoint.bottom = bottom * FIXED_POINT_SCALE - 1;
 756     }
 757 }
 758 // templated backend function tables
 759 extern PFN_BACKEND_FUNC gBackendNullPs[SWR_MULTISAMPLE_TYPE_MAX];
 760 extern PFN_BACKEND_FUNC gBackendSingleSample[2][2];
 761 extern PFN_BACKEND_FUNC gBackendPixelRateTable[SWR_MULTISAMPLE_TYPE_MAX][SWR_MSAA_SAMPLE_PATTERN_MAX][SWR_INPUT_COVERAGE_MAX][2][2];
 762 extern PFN_BACKEND_FUNC gBackendSampleRateTable[SWR_MULTISAMPLE_TYPE_MAX][SWR_INPUT_COVERAGE_MAX][2];
 763 extern PFN_OUTPUT_MERGER gBackendOutputMergerTable[SWR_NUM_RENDERTARGETS + 1][SWR_MULTISAMPLE_TYPE_MAX];
 764 extern PFN_CALC_PIXEL_BARYCENTRICS gPixelBarycentricTable[2];
 765 extern PFN_CALC_SAMPLE_BARYCENTRICS gSampleBarycentricTable[2];
 766 void SetupPipeline(DRAW_CONTEXT *pDC)
 767 {
 768     DRAW_STATE* pState = pDC->pState;
 769     const SWR_RASTSTATE &rastState = pState->state.rastState;
 770     const SWR_PS_STATE &psState = pState->state.psState;
 771     BACKEND_FUNCS& backendFuncs = pState->backendFuncs;
 772     const uint32_t forcedSampleCount = (rastState.bForcedSampleCount) ? 1 : 0;
 773
 774     // setup backend
 775     if (psState.pfnPixelShader == nullptr)
 776     {
 777         backendFuncs.pfnBackend = gBackendNullPs[pState->state.rastState.sampleCount];
 778         // always need to generate I & J per sample for Z interpolation
 779         backendFuncs.pfnCalcSampleBarycentrics = gSampleBarycentricTable[1];
 780     }
 781     else
 782     {
 783         const bool bMultisampleEnable = ((rastState.sampleCount > SWR_MULTISAMPLE_1X) || rastState.bForcedSampleCount) ? 1 : 0;
 784         const uint32_t centroid = ((psState.barycentricsMask & SWR_BARYCENTRIC_CENTROID_MASK) > 0) ? 1 : 0;
 785
 786         // currently only support 'normal' input coverage
 787         SWR_ASSERT(psState.inputCoverage == SWR_INPUT_COVERAGE_NORMAL ||
 788                    psState.inputCoverage == SWR_INPUT_COVERAGE_NONE);
 789
 790         SWR_BARYCENTRICS_MASK barycentricsMask = (SWR_BARYCENTRICS_MASK)psState.barycentricsMask;
 791
 792         // select backend function
 793         switch(psState.shadingRate)
 794         {
 795         case SWR_SHADING_RATE_PIXEL:
 796             if(bMultisampleEnable)
 797             {
 798                 // always need to generate I & J per sample for Z interpolation
 799                 barycentricsMask = (SWR_BARYCENTRICS_MASK)(barycentricsMask | SWR_BARYCENTRIC_PER_SAMPLE_MASK);
 800                 backendFuncs.pfnBackend = gBackendPixelRateTable[rastState.sampleCount][rastState.samplePattern][psState.inputCoverage][centroid][forcedSampleCount];
 801                 backendFuncs.pfnOutputMerger = gBackendOutputMergerTable[psState.numRenderTargets][pState->state.blendState.sampleCount];
 802             }
 803             else
 804             {
 805                 // always need to generate I & J per pixel for Z interpolation
 806                 barycentricsMask = (SWR_BARYCENTRICS_MASK)(barycentricsMask | SWR_BARYCENTRIC_PER_PIXEL_MASK);
 807                 backendFuncs.pfnBackend = gBackendSingleSample[psState.inputCoverage][centroid];
 808                 backendFuncs.pfnOutputMerger = gBackendOutputMergerTable[psState.numRenderTargets][SWR_MULTISAMPLE_1X];
 809             }
 810             break;
 811         case SWR_SHADING_RATE_SAMPLE:
 812             SWR_ASSERT(rastState.samplePattern == SWR_MSAA_STANDARD_PATTERN);
 813             // always need to generate I & J per sample for Z interpolation
 814             barycentricsMask = (SWR_BARYCENTRICS_MASK)(barycentricsMask | SWR_BARYCENTRIC_PER_SAMPLE_MASK);
 815             backendFuncs.pfnBackend = gBackendSampleRateTable[rastState.sampleCount][psState.inputCoverage][centroid];
 816             backendFuncs.pfnOutputMerger = gBackendOutputMergerTable[psState.numRenderTargets][pState->state.blendState.sampleCount];
 817             break;
 818         default:
 819             SWR_ASSERT(0 && "Invalid shading rate");
 820             break;
 821         }
 822
 823         // setup pointer to function that generates necessary barycentrics required by the PS
 824         bool bBarycentrics = (barycentricsMask & SWR_BARYCENTRIC_PER_PIXEL_MASK) > 0 ? 1 : 0;
 825         backendFuncs.pfnCalcPixelBarycentrics = gPixelBarycentricTable[bBarycentrics];
 826
 827         bBarycentrics = (barycentricsMask & SWR_BARYCENTRIC_PER_SAMPLE_MASK) > 0 ? 1 : 0;
 828         backendFuncs.pfnCalcSampleBarycentrics = gSampleBarycentricTable[bBarycentrics];
 829     }
 830
 831     PFN_PROCESS_PRIMS pfnBinner;
 832     switch (pState->state.topology)
 833     {
 834     case TOP_POINT_LIST:
 835         pState->pfnProcessPrims = ClipPoints;
 836         pfnBinner = BinPoints;
 837         break;
 838     case TOP_LINE_LIST:
 839     case TOP_LINE_STRIP:
 840     case TOP_LINE_LOOP:
 841     case TOP_LINE_LIST_ADJ:
 842     case TOP_LISTSTRIP_ADJ:
 843         pState->pfnProcessPrims = ClipLines;
 844         pfnBinner = BinLines;
 845         break;
 846     default:
 847         pState->pfnProcessPrims = ClipTriangles;
 848         pfnBinner = BinTriangles;
 849         break;
 850     };
 851
 852     // disable clipper if viewport transform is disabled
 853     if (pState->state.frontendState.vpTransformDisable)
 854     {
 855         pState->pfnProcessPrims = pfnBinner;
 856     }
 857
 858     if ((pState->state.psState.pfnPixelShader == nullptr) &&
 859         (pState->state.depthStencilState.depthTestEnable == FALSE) &&
 860         (pState->state.depthStencilState.depthWriteEnable == FALSE) &&
 861         (pState->state.depthStencilState.stencilTestEnable == FALSE) &&
 862         (pState->state.depthStencilState.stencilWriteEnable == FALSE) &&
 863         (pState->state.linkageCount == 0))
 864     {
 865         pState->pfnProcessPrims = nullptr;
 866         pState->state.linkageMask = 0;
 867     }
 868
 869     if (pState->state.soState.rasterizerDisable == true)
 870     {
 871         pState->pfnProcessPrims = nullptr;
 872         pState->state.linkageMask = 0;
 873     }
 874
 875     // set up the frontend attrib mask
 876     pState->state.feAttribMask = pState->state.linkageMask;
 877     if (pState->state.soState.soEnable)
 878     {
 879         for (uint32_t i = 0; i < 4; ++i)
 880         {
 881             pState->state.feAttribMask |= pState->state.soState.streamMasks[i];
 882         }
 883     }
 884
 885     // complicated logic to test for cases where we don't need backing hottile memory for a draw
 886     // have to check for the special case where depth/stencil test is enabled but depthwrite is disabled.
 887     pState->state.depthHottileEnable = ((!(pState->state.depthStencilState.depthTestEnable &&
 888                                            !pState->state.depthStencilState.depthWriteEnable &&
 889                                            pState->state.depthStencilState.depthTestFunc == ZFUNC_ALWAYS)) &&
 890                                         (pState->state.depthStencilState.depthTestEnable ||
 891                                          pState->state.depthStencilState.depthWriteEnable)) ? true : false;
 892
 893     pState->state.stencilHottileEnable = (((!(pState->state.depthStencilState.stencilTestEnable &&
 894                                              !pState->state.depthStencilState.stencilWriteEnable &&
 895                                               pState->state.depthStencilState.stencilTestFunc == ZFUNC_ALWAYS)) ||
 896                                           // for stencil we have to check the double sided state as well
 897                                           (!(pState->state.depthStencilState.doubleSidedStencilTestEnable &&
 898                                              !pState->state.depthStencilState.stencilWriteEnable &&
 899                                               pState->state.depthStencilState.backfaceStencilTestFunc == ZFUNC_ALWAYS))) &&
 900                                           (pState->state.depthStencilState.stencilTestEnable  ||
 901                                            pState->state.depthStencilState.stencilWriteEnable)) ? true : false;
 902
 903     uint32_t numRTs = pState->state.psState.numRenderTargets;
 904     pState->state.colorHottileEnable = 0;
 905     if (psState.pfnPixelShader != nullptr)
 906     {
 907         for (uint32_t rt = 0; rt < numRTs; ++rt)
 908         {
 909             pState->state.colorHottileEnable |=
 910                 (!pState->state.blendState.renderTarget[rt].writeDisableAlpha ||
 911                  !pState->state.blendState.renderTarget[rt].writeDisableRed ||
 912                  !pState->state.blendState.renderTarget[rt].writeDisableGreen ||
 913                  !pState->state.blendState.renderTarget[rt].writeDisableBlue) ? (1 << rt) : 0;
 914         }
 915     }
 916
 917     // Setup depth quantization function
 918     if (pState->state.depthHottileEnable)
 919     {
 920         switch (pState->state.rastState.depthFormat)
 921         {
 922         case R32_FLOAT_X8X24_TYPELESS: pState->state.pfnQuantizeDepth = QuantizeDepth < R32_FLOAT_X8X24_TYPELESS > ; break;
 923         case R32_FLOAT: pState->state.pfnQuantizeDepth = QuantizeDepth < R32_FLOAT > ; break;
 924         case R24_UNORM_X8_TYPELESS: pState->state.pfnQuantizeDepth = QuantizeDepth < R24_UNORM_X8_TYPELESS > ; break;
 925         case R16_UNORM: pState->state.pfnQuantizeDepth = QuantizeDepth < R16_UNORM > ; break;
 926         default: SWR_ASSERT(false, "Unsupported depth format for depth quantiztion.");
 927             pState->state.pfnQuantizeDepth = QuantizeDepth < R32_FLOAT > ;
 928         }
 929     }
 930     else
 931     {
 932         // set up pass-through quantize if depth isn't enabled
 933         pState->state.pfnQuantizeDepth = QuantizeDepth < R32_FLOAT > ;
 934     }
 935 }
 936
 937 //////////////////////////////////////////////////////////////////////////
 938 /// @brief InitDraw
 939 /// @param pDC - Draw context to initialize for this draw.
 940 void InitDraw(
 941     DRAW_CONTEXT *pDC,
 942     bool isSplitDraw)
 943 {
 944     // We don't need to re-setup the scissors/pipeline state again for split draw.
 945     if (isSplitDraw == false)
 946     {
 947         SetupMacroTileScissors(pDC);
 948         SetupPipeline(pDC);
 949     }
 950 }
 951
 952 //////////////////////////////////////////////////////////////////////////
 953 /// @brief We can split the draw for certain topologies for better performance.
 954 /// @param totalVerts - Total vertices for draw
 955 /// @param topology - Topology used for draw
 956 uint32_t MaxVertsPerDraw(
 957     DRAW_CONTEXT* pDC,
 958     uint32_t totalVerts,
 959     PRIMITIVE_TOPOLOGY topology)
 960 {
 961     API_STATE& state = pDC->pState->state;
 962
 963     uint32_t vertsPerDraw = totalVerts;
 964
 965     if (state.soState.soEnable)
 966     {
 967         return totalVerts;
 968     }
 969
 970     switch (topology)
 971     {
 972     case TOP_POINT_LIST:
 973     case TOP_TRIANGLE_LIST:
 974         vertsPerDraw = KNOB_MAX_PRIMS_PER_DRAW;
 975         break;
 976
 977     case TOP_PATCHLIST_1:
 978     case TOP_PATCHLIST_2:
 979     case TOP_PATCHLIST_3:
 980     case TOP_PATCHLIST_4:
 981     case TOP_PATCHLIST_5:
 982     case TOP_PATCHLIST_6:
 983     case TOP_PATCHLIST_7:
 984     case TOP_PATCHLIST_8:
 985     case TOP_PATCHLIST_9:
 986     case TOP_PATCHLIST_10:
 987     case TOP_PATCHLIST_11:
 988     case TOP_PATCHLIST_12:
 989     case TOP_PATCHLIST_13:
 990     case TOP_PATCHLIST_14:
 991     case TOP_PATCHLIST_15:
 992     case TOP_PATCHLIST_16:
 993     case TOP_PATCHLIST_17:
 994     case TOP_PATCHLIST_18:
 995     case TOP_PATCHLIST_19:
 996     case TOP_PATCHLIST_20:
 997     case TOP_PATCHLIST_21:
 998     case TOP_PATCHLIST_22:
 999     case TOP_PATCHLIST_23:
1000     case TOP_PATCHLIST_24:
1001     case TOP_PATCHLIST_25:
1002     case TOP_PATCHLIST_26:
1003     case TOP_PATCHLIST_27:
1004     case TOP_PATCHLIST_28:
1005     case TOP_PATCHLIST_29:
1006     case TOP_PATCHLIST_30:
1007     case TOP_PATCHLIST_31:
1008     case TOP_PATCHLIST_32:
1009         if (pDC->pState->state.tsState.tsEnable)
1010         {
1011             uint32_t vertsPerPrim = topology - TOP_PATCHLIST_BASE;
1012             vertsPerDraw = vertsPerPrim * KNOB_MAX_TESS_PRIMS_PER_DRAW;
1013         }
1014         break;
1015
1016     // The Primitive Assembly code can only handle 1 RECT at a time.
1017     case TOP_RECT_LIST:
1018         vertsPerDraw = 3;
1019         break;
1020
1021     default:
1022         // We are not splitting up draws for other topologies.
1023         break;
1024     }
1025
1026     return vertsPerDraw;
1027 }
1028
1029
1030 //////////////////////////////////////////////////////////////////////////
1031 /// @brief DrawInstanced
1032 /// @param hContext - Handle passed back from SwrCreateContext
1033 /// @param topology - Specifies topology for draw.
1034 /// @param numVerts - How many vertices to read sequentially from vertex data (per instance).
1035 /// @param startVertex - Specifies start vertex for draw. (vertex data)
1036 /// @param numInstances - How many instances to render.
1037 /// @param startInstance - Which instance to start sequentially fetching from in each buffer (instanced data)
1038 void DrawInstanced(
1039     HANDLE hContext,
1040     PRIMITIVE_TOPOLOGY topology,
1041     uint32_t numVertices,
1042     uint32_t startVertex,
1043     uint32_t numInstances = 1,
1044     uint32_t startInstance = 0)
1045 {
1046     if (KNOB_TOSS_DRAW)
1047     {
1048         return;
1049     }
1050
1051     RDTSC_START(APIDraw);
1052
1053     SWR_CONTEXT *pContext = GetContext(hContext);
1054     DRAW_CONTEXT* pDC = GetDrawContext(pContext);
1055
1056     uint32_t maxVertsPerDraw = MaxVertsPerDraw(pDC, numVertices, topology);
1057     uint32_t primsPerDraw = GetNumPrims(topology, maxVertsPerDraw);
1058     uint32_t remainingVerts = numVertices;
1059
1060     API_STATE    *pState = &pDC->pState->state;
1061     pState->topology = topology;
1062     pState->forceFront = false;
1063
1064     // disable culling for points/lines
1065     uint32_t oldCullMode = pState->rastState.cullMode;
1066     if (topology == TOP_POINT_LIST)
1067     {
1068         pState->rastState.cullMode = SWR_CULLMODE_NONE;
1069         pState->forceFront = true;
1070     }
1071
1072     int draw = 0;
1073     while (remainingVerts)
1074     {
1075         uint32_t numVertsForDraw = (remainingVerts < maxVertsPerDraw) ?
1076         remainingVerts : maxVertsPerDraw;
1077
1078         bool isSplitDraw = (draw > 0) ? true : false;
1079         DRAW_CONTEXT* pDC = GetDrawContext(pContext, isSplitDraw);
1080         InitDraw(pDC, isSplitDraw);
1081
1082         pDC->FeWork.type = DRAW;
1083         pDC->FeWork.pfnWork = GetProcessDrawFunc(
1084             false,  // IsIndexed
1085             pState->tsState.tsEnable,
1086             pState->gsState.gsEnable,
1087             pState->soState.soEnable,
1088             pDC->pState->pfnProcessPrims != nullptr);
1089         pDC->FeWork.desc.draw.numVerts = numVertsForDraw;
1090         pDC->FeWork.desc.draw.startVertex = startVertex;
1091         pDC->FeWork.desc.draw.numInstances = numInstances;
1092         pDC->FeWork.desc.draw.startInstance = startInstance;
1093         pDC->FeWork.desc.draw.startPrimID = draw * primsPerDraw;
1094         pDC->FeWork.desc.draw.startVertexID = draw * maxVertsPerDraw;
1095
1096         pDC->cleanupState = (remainingVerts == numVertsForDraw);
1097
1098         //enqueue DC
1099         QueueDraw(pContext);
1100
1101         remainingVerts -= numVertsForDraw;
1102         draw++;
1103     }
1104
1105     // restore culling state
1106     pDC = GetDrawContext(pContext);
1107     pDC->pState->state.rastState.cullMode = oldCullMode;
1108
1109     RDTSC_STOP(APIDraw, numVertices * numInstances, 0);
1110 }
1111
1112 //////////////////////////////////////////////////////////////////////////
1113 /// @brief SwrDraw
1114 /// @param hContext - Handle passed back from SwrCreateContext
1115 /// @param topology - Specifies topology for draw.
1116 /// @param startVertex - Specifies start vertex in vertex buffer for draw.
1117 /// @param primCount - Number of vertices.
1118 void SwrDraw(
1119     HANDLE hContext,
1120     PRIMITIVE_TOPOLOGY topology,
1121     uint32_t startVertex,
1122     uint32_t numVertices)
1123 {
1124     DrawInstanced(hContext, topology, numVertices, startVertex);
1125 }
1126
1127 //////////////////////////////////////////////////////////////////////////
1128 /// @brief SwrDrawInstanced
1129 /// @param hContext - Handle passed back from SwrCreateContext
1130 /// @param topology - Specifies topology for draw.
1131 /// @param numVertsPerInstance - How many vertices to read sequentially from vertex data.
1132 /// @param numInstances - How many instances to render.
1133 /// @param startVertex - Specifies start vertex for draw. (vertex data)
1134 /// @param startInstance - Which instance to start sequentially fetching from in each buffer (instanced data)
1135 void SwrDrawInstanced(
1136     HANDLE hContext,
1137     PRIMITIVE_TOPOLOGY topology,
1138     uint32_t numVertsPerInstance,
1139     uint32_t numInstances,
1140     uint32_t startVertex,
1141     uint32_t startInstance
1142     )
1143 {
1144     DrawInstanced(hContext, topology, numVertsPerInstance, startVertex, numInstances, startInstance);
1145 }
1146
1147 //////////////////////////////////////////////////////////////////////////
1148 /// @brief DrawIndexedInstanced
1149 /// @param hContext - Handle passed back from SwrCreateContext
1150 /// @param topology - Specifies topology for draw.
1151 /// @param numIndices - Number of indices to read sequentially from index buffer.
1152 /// @param indexOffset - Starting index into index buffer.
1153 /// @param baseVertex - Vertex in vertex buffer to consider as index "0". Note value is signed.
1154 /// @param numInstances - Number of instances to render.
1155 /// @param startInstance - Which instance to start sequentially fetching from in each buffer (instanced data)
1156 void DrawIndexedInstance(
1157     HANDLE hContext,
1158     PRIMITIVE_TOPOLOGY topology,
1159     uint32_t numIndices,
1160     uint32_t indexOffset,
1161     int32_t baseVertex,
1162     uint32_t numInstances = 1,
1163     uint32_t startInstance = 0)
1164 {
1165     if (KNOB_TOSS_DRAW)
1166     {
1167         return;
1168     }
1169
1170     RDTSC_START(APIDrawIndexed);
1171
1172     SWR_CONTEXT *pContext = GetContext(hContext);
1173     DRAW_CONTEXT* pDC = GetDrawContext(pContext);
1174     API_STATE* pState = &pDC->pState->state;
1175
1176     uint32_t maxIndicesPerDraw = MaxVertsPerDraw(pDC, numIndices, topology);
1177     uint32_t primsPerDraw = GetNumPrims(topology, maxIndicesPerDraw);
1178     uint32_t remainingIndices = numIndices;
1179
1180     uint32_t indexSize = 0;
1181     switch (pState->indexBuffer.format)
1182     {
1183     case R32_UINT: indexSize = sizeof(uint32_t); break;
1184     case R16_UINT: indexSize = sizeof(uint16_t); break;
1185     case R8_UINT: indexSize = sizeof(uint8_t); break;
1186     default:
1187         SWR_ASSERT(0);
1188     }
1189
1190     int draw = 0;
1191     uint8_t *pIB = (uint8_t*)pState->indexBuffer.pIndices;
1192     pIB += (uint64_t)indexOffset * (uint64_t)indexSize;
1193
1194     pState->topology = topology;
1195     pState->forceFront = false;
1196
1197     // disable culling for points/lines
1198     uint32_t oldCullMode = pState->rastState.cullMode;
1199     if (topology == TOP_POINT_LIST)
1200     {
1201         pState->rastState.cullMode = SWR_CULLMODE_NONE;
1202         pState->forceFront = true;
1203     }
1204
1205     while (remainingIndices)
1206     {
1207         uint32_t numIndicesForDraw = (remainingIndices < maxIndicesPerDraw) ?
1208         remainingIndices : maxIndicesPerDraw;
1209
1210         // When breaking up draw, we need to obtain new draw context for each iteration.
1211         bool isSplitDraw = (draw > 0) ? true : false;
1212         pDC = GetDrawContext(pContext, isSplitDraw);
1213         InitDraw(pDC, isSplitDraw);
1214
1215         pDC->FeWork.type = DRAW;
1216         pDC->FeWork.pfnWork = GetProcessDrawFunc(
1217             true,   // IsIndexed
1218             pState->tsState.tsEnable,
1219             pState->gsState.gsEnable,
1220             pState->soState.soEnable,
1221             pDC->pState->pfnProcessPrims != nullptr);
1222         pDC->FeWork.desc.draw.pDC = pDC;
1223         pDC->FeWork.desc.draw.numIndices = numIndicesForDraw;
1224         pDC->FeWork.desc.draw.pIB = (int*)pIB;
1225         pDC->FeWork.desc.draw.type = pDC->pState->state.indexBuffer.format;
1226
1227         pDC->FeWork.desc.draw.numInstances = numInstances;
1228         pDC->FeWork.desc.draw.startInstance = startInstance;
1229         pDC->FeWork.desc.draw.baseVertex = baseVertex;
1230         pDC->FeWork.desc.draw.startPrimID = draw * primsPerDraw;
1231
1232         pDC->cleanupState = (remainingIndices == numIndicesForDraw);
1233
1234         //enqueue DC
1235         QueueDraw(pContext);
1236
1237         pIB += maxIndicesPerDraw * indexSize;
1238         remainingIndices -= numIndicesForDraw;
1239         draw++;
1240     }
1241
1242     // restore culling state
1243     pDC = GetDrawContext(pContext);
1244     pDC->pState->state.rastState.cullMode = oldCullMode;
1245
1246     RDTSC_STOP(APIDrawIndexed, numIndices * numInstances, 0);
1247 }
1248
1249
1250 //////////////////////////////////////////////////////////////////////////
1251 /// @brief DrawIndexed
1252 /// @param hContext - Handle passed back from SwrCreateContext
1253 /// @param topology - Specifies topology for draw.
1254 /// @param numIndices - Number of indices to read sequentially from index buffer.
1255 /// @param indexOffset - Starting index into index buffer.
1256 /// @param baseVertex - Vertex in vertex buffer to consider as index "0". Note value is signed.
1257 void SwrDrawIndexed(
1258     HANDLE hContext,
1259     PRIMITIVE_TOPOLOGY topology,
1260     uint32_t numIndices,
1261     uint32_t indexOffset,
1262     int32_t baseVertex
1263     )
1264 {
1265     DrawIndexedInstance(hContext, topology, numIndices, indexOffset, baseVertex);
1266 }
1267
1268 //////////////////////////////////////////////////////////////////////////
1269 /// @brief SwrDrawIndexedInstanced
1270 /// @param hContext - Handle passed back from SwrCreateContext
1271 /// @param topology - Specifies topology for draw.
1272 /// @param numIndices - Number of indices to read sequentially from index buffer.
1273 /// @param numInstances - Number of instances to render.
1274 /// @param indexOffset - Starting index into index buffer.
1275 /// @param baseVertex - Vertex in vertex buffer to consider as index "0". Note value is signed.
1276 /// @param startInstance - Which instance to start sequentially fetching from in each buffer (instanced data)
1277 void SwrDrawIndexedInstanced(
1278     HANDLE hContext,
1279     PRIMITIVE_TOPOLOGY topology,
1280     uint32_t numIndices,
1281     uint32_t numInstances,
1282     uint32_t indexOffset,
1283     int32_t baseVertex,
1284     uint32_t startInstance)
1285 {
1286     DrawIndexedInstance(hContext, topology, numIndices, indexOffset, baseVertex, numInstances, startInstance);
1287 }
1288
1289 //////////////////////////////////////////////////////////////////////////
1290 /// @brief SwrInvalidateTiles
1291 /// @param hContext - Handle passed back from SwrCreateContext
1292 /// @param attachmentMask - The mask specifies which surfaces attached to the hottiles to invalidate.
1293 void SwrInvalidateTiles(
1294     HANDLE hContext,
1295     uint32_t attachmentMask)
1296 {
1297     SWR_CONTEXT *pContext = (SWR_CONTEXT*)hContext;
1298     DRAW_CONTEXT* pDC = GetDrawContext(pContext);
1299
1300     pDC->FeWork.type = DISCARDINVALIDATETILES;
1301     pDC->FeWork.pfnWork = ProcessDiscardInvalidateTiles;
1302     pDC->FeWork.desc.discardInvalidateTiles.attachmentMask = attachmentMask;
1303     memset(&pDC->FeWork.desc.discardInvalidateTiles.rect, 0, sizeof(SWR_RECT));
1304     pDC->FeWork.desc.discardInvalidateTiles.newTileState = SWR_TILE_INVALID;
1305     pDC->FeWork.desc.discardInvalidateTiles.createNewTiles = false;
1306     pDC->FeWork.desc.discardInvalidateTiles.fullTilesOnly = false;
1307
1308     //enqueue
1309     QueueDraw(pContext);
1310 }
1311
1312 //////////////////////////////////////////////////////////////////////////
1313 /// @brief SwrDiscardRect
1314 /// @param hContext - Handle passed back from SwrCreateContext
1315 /// @param attachmentMask - The mask specifies which surfaces attached to the hottiles to discard.
1316 /// @param rect - if rect is all zeros, the entire attachment surface will be discarded
1317 void SwrDiscardRect(
1318     HANDLE hContext,
1319     uint32_t attachmentMask,
1320     SWR_RECT rect)
1321 {
1322     SWR_CONTEXT *pContext = (SWR_CONTEXT*)hContext;
1323     DRAW_CONTEXT* pDC = GetDrawContext(pContext);
1324
1325     // Queue a load to the hottile
1326     pDC->FeWork.type = DISCARDINVALIDATETILES;
1327     pDC->FeWork.pfnWork = ProcessDiscardInvalidateTiles;
1328     pDC->FeWork.desc.discardInvalidateTiles.attachmentMask = attachmentMask;
1329     pDC->FeWork.desc.discardInvalidateTiles.rect = rect;
1330     pDC->FeWork.desc.discardInvalidateTiles.newTileState = SWR_TILE_RESOLVED;
1331     pDC->FeWork.desc.discardInvalidateTiles.createNewTiles = true;
1332     pDC->FeWork.desc.discardInvalidateTiles.fullTilesOnly = true;
1333
1334     //enqueue
1335     QueueDraw(pContext);
1336 }
1337
1338 //////////////////////////////////////////////////////////////////////////
1339 /// @brief SwrDispatch
1340 /// @param hContext - Handle passed back from SwrCreateContext
1341 /// @param threadGroupCountX - Number of thread groups dispatched in X direction
1342 /// @param threadGroupCountY - Number of thread groups dispatched in Y direction
1343 /// @param threadGroupCountZ - Number of thread groups dispatched in Z direction
1344 void SwrDispatch(
1345     HANDLE hContext,
1346     uint32_t threadGroupCountX,
1347     uint32_t threadGroupCountY,
1348     uint32_t threadGroupCountZ)
1349 {
1350     if (KNOB_TOSS_DRAW)
1351     {
1352         return;
1353     }
1354
1355     RDTSC_START(APIDispatch);
1356     SWR_CONTEXT *pContext = (SWR_CONTEXT*)hContext;
1357     DRAW_CONTEXT* pDC = GetDrawContext(pContext);
1358
1359     pDC->isCompute = true;      // This is a compute context.
1360
1361     COMPUTE_DESC* pTaskData = (COMPUTE_DESC*)pDC->pArena->AllocAligned(sizeof(COMPUTE_DESC), 64);
1362
1363     pTaskData->threadGroupCountX = threadGroupCountX;
1364     pTaskData->threadGroupCountY = threadGroupCountY;
1365     pTaskData->threadGroupCountZ = threadGroupCountZ;
1366
1367     uint32_t totalThreadGroups = threadGroupCountX * threadGroupCountY * threadGroupCountZ;
1368     uint32_t dcIndex = pDC->drawId % KNOB_MAX_DRAWS_IN_FLIGHT;
1369     pDC->pDispatch = &pContext->pDispatchQueueArray[dcIndex];
1370     pDC->pDispatch->initialize(totalThreadGroups, pTaskData);
1371
1372     QueueDispatch(pContext);
1373     RDTSC_STOP(APIDispatch, threadGroupCountX * threadGroupCountY * threadGroupCountZ, 0);
1374 }
1375
1376 // Deswizzles, converts and stores current contents of the hot tiles to surface
1377 // described by pState
1378 void SwrStoreTiles(
1379     HANDLE hContext,
1380     SWR_RENDERTARGET_ATTACHMENT attachment,
1381     SWR_TILE_STATE postStoreTileState)
1382 {
1383     RDTSC_START(APIStoreTiles);
1384
1385     SWR_CONTEXT *pContext = (SWR_CONTEXT*)hContext;
1386     DRAW_CONTEXT* pDC = GetDrawContext(pContext);
1387
1388     SetupMacroTileScissors(pDC);
1389
1390     pDC->FeWork.type = STORETILES;
1391     pDC->FeWork.pfnWork = ProcessStoreTiles;
1392     pDC->FeWork.desc.storeTiles.attachment = attachment;
1393     pDC->FeWork.desc.storeTiles.postStoreTileState = postStoreTileState;
1394
1395     //enqueue
1396     QueueDraw(pContext);
1397
1398     RDTSC_STOP(APIStoreTiles, 0, 0);
1399 }
1400
1401 void SwrClearRenderTarget(
1402     HANDLE hContext,
1403     uint32_t clearMask,
1404     const float clearColor[4],
1405     float z,
1406     uint8_t stencil)
1407 {
1408     RDTSC_START(APIClearRenderTarget);
1409
1410     SWR_CONTEXT *pContext = (SWR_CONTEXT*)hContext;
1411
1412     DRAW_CONTEXT* pDC = GetDrawContext(pContext);
1413
1414     SetupMacroTileScissors(pDC);
1415
1416     CLEAR_FLAGS flags;
1417     flags.mask = clearMask;
1418
1419     pDC->FeWork.type = CLEAR;
1420     pDC->FeWork.pfnWork = ProcessClear;
1421     pDC->FeWork.desc.clear.flags = flags;
1422     pDC->FeWork.desc.clear.clearDepth = z;
1423     pDC->FeWork.desc.clear.clearRTColor[0] = clearColor[0];
1424     pDC->FeWork.desc.clear.clearRTColor[1] = clearColor[1];
1425     pDC->FeWork.desc.clear.clearRTColor[2] = clearColor[2];
1426     pDC->FeWork.desc.clear.clearRTColor[3] = clearColor[3];
1427     pDC->FeWork.desc.clear.clearStencil = stencil;
1428
1429     // enqueue draw
1430     QueueDraw(pContext);
1431
1432     RDTSC_STOP(APIClearRenderTarget, 0, pDC->drawId);
1433 }
1434
1435 //////////////////////////////////////////////////////////////////////////
1436 /// @brief Returns a pointer to the private context state for the current
1437 ///        draw operation. This is used for external componets such as the
1438 ///        sampler.
1439 ///        SWR is responsible for the allocation of the private context state.
1440 /// @param hContext - Handle passed back from SwrCreateContext
1441 VOID* SwrGetPrivateContextState(
1442     HANDLE hContext)
1443 {
1444     SWR_CONTEXT* pContext = GetContext(hContext);
1445     DRAW_CONTEXT* pDC = GetDrawContext(pContext);
1446     DRAW_STATE* pState = pDC->pState;
1447
1448     if (pState->pPrivateState == nullptr)
1449     {
1450         pState->pPrivateState = pState->pArena->AllocAligned(pContext->privateStateSize, KNOB_SIMD_WIDTH*sizeof(float));
1451     }
1452
1453     return pState->pPrivateState;
1454 }
1455
1456 //////////////////////////////////////////////////////////////////////////
1457 /// @brief Clients can use this to allocate memory for draw/dispatch
1458 ///        operations. The memory will automatically be freed once operation
1459 ///        has completed. Client can use this to allocate binding tables,
1460 ///        etc. needed for shader execution.
1461 /// @param hContext - Handle passed back from SwrCreateContext
1462 /// @param size - Size of allocation
1463 /// @param align - Alignment needed for allocation.
1464 VOID* SwrAllocDrawContextMemory(
1465     HANDLE hContext,
1466     uint32_t size,
1467     uint32_t align)
1468 {
1469     SWR_CONTEXT* pContext = GetContext(hContext);
1470     DRAW_CONTEXT* pDC = GetDrawContext(pContext);
1471
1472     return pDC->pState->pArena->AllocAligned(size, align);
1473 }
1474
1475 //////////////////////////////////////////////////////////////////////////
1476 /// @brief Returns pointer to SWR stats.
1477 /// @note The counters are atomically incremented by multiple threads.
1478 ///       When calling this, you need to ensure all previous operations
1479 ///       have completed.
1480 /// @todo If necessary, add a callback to avoid stalling the pipe to
1481 ///       sample the counters.
1482 /// @param hContext - Handle passed back from SwrCreateContext
1483 /// @param pStats - SWR will fill this out for caller.
1484 void SwrGetStats(
1485     HANDLE hContext,
1486     SWR_STATS* pStats)
1487 {
1488     SWR_CONTEXT *pContext = GetContext(hContext);
1489     DRAW_CONTEXT* pDC = GetDrawContext(pContext);
1490
1491     pDC->FeWork.type = QUERYSTATS;
1492     pDC->FeWork.pfnWork = ProcessQueryStats;
1493     pDC->FeWork.desc.queryStats.pStats = pStats;
1494
1495     // cannot execute until all previous draws have completed
1496     pDC->dependency = pDC->drawId - 1;
1497
1498     //enqueue
1499     QueueDraw(pContext);
1500 }
1501
1502 //////////////////////////////////////////////////////////////////////////
1503 /// @brief Enables stats counting
1504 /// @param hContext - Handle passed back from SwrCreateContext
1505 /// @param enable - If true then counts are incremented.
1506 void SwrEnableStats(
1507     HANDLE hContext,
1508     bool enable)
1509 {
1510     SWR_CONTEXT *pContext = GetContext(hContext);
1511     DRAW_CONTEXT* pDC = GetDrawContext(pContext);
1512
1513     pDC->pState->state.enableStats = enable;
1514 }
1515
1516 //////////////////////////////////////////////////////////////////////////
1517 /// @brief Mark end of frame - used for performance profiling
1518 /// @param hContext - Handle passed back from SwrCreateContext
1519 void SWR_API SwrEndFrame(
1520     HANDLE hContext)
1521 {
1522     RDTSC_ENDFRAME();
1523     SWR_CONTEXT *pContext = GetContext(hContext);
1524     pContext->frameCount++;
1525 }