src/gallium/drivers/swr/rasterizer/core/api.cpp

   1 /****************************************************************************
   2 * Copyright (C) 2014-2015 Intel Corporation.   All Rights Reserved.
   3 *
   4 * Permission is hereby granted, free of charge, to any person obtaining a
   5 * copy of this software and associated documentation files (the "Software"),
   6 * to deal in the Software without restriction, including without limitation
   7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8 * and/or sell copies of the Software, and to permit persons to whom the
   9 * Software is furnished to do so, subject to the following conditions:
  10 *
  11 * The above copyright notice and this permission notice (including the next
  12 * paragraph) shall be included in all copies or substantial portions of the
  13 * Software.
  14 *
  15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21 * IN THE SOFTWARE.
  22 *
  23 * @file api.cpp
  24 *
  25 * @brief API implementation
  26 *
  27 ******************************************************************************/
  28
  29 #include <cfloat>
  30 #include <cmath>
  31 #include <cstdio>
  32 #include <new>
  33
  34 #include "core/api.h"
  35 #include "core/backend.h"
  36 #include "core/context.h"
  37 #include "core/depthstencil.h"
  38 #include "core/frontend.h"
  39 #include "core/rasterizer.h"
  40 #include "core/rdtsc_core.h"
  41 #include "core/threads.h"
  42 #include "core/tilemgr.h"
  43 #include "core/clip.h"
  44 #include "core/utils.h"
  45
  46 #include "common/simdintrin.h"
  47 #include "common/os.h"
  48
  49 void SetupDefaultState(SWR_CONTEXT *pContext);
  50
  51 //////////////////////////////////////////////////////////////////////////
  52 /// @brief Create SWR Context.
  53 /// @param pCreateInfo - pointer to creation info.
  54 HANDLE SwrCreateContext(
  55     SWR_CREATECONTEXT_INFO* pCreateInfo)
  56 {
  57     RDTSC_RESET();
  58     RDTSC_INIT(0);
  59
  60     void* pContextMem = AlignedMalloc(sizeof(SWR_CONTEXT), KNOB_SIMD_WIDTH * 4);
  61     memset(pContextMem, 0, sizeof(SWR_CONTEXT));
  62     SWR_CONTEXT *pContext = new (pContextMem) SWR_CONTEXT();
  63
  64     pContext->driverType = pCreateInfo->driver;
  65     pContext->privateStateSize = pCreateInfo->privateStateSize;
  66
  67     pContext->dcRing.Init(KNOB_MAX_DRAWS_IN_FLIGHT);
  68     pContext->dsRing.Init(KNOB_MAX_DRAWS_IN_FLIGHT);
  69
  70     pContext->pMacroTileManagerArray = (MacroTileMgr*)AlignedMalloc(sizeof(MacroTileMgr) * KNOB_MAX_DRAWS_IN_FLIGHT, 64);
  71     pContext->pDispatchQueueArray = (DispatchQueue*)AlignedMalloc(sizeof(DispatchQueue) * KNOB_MAX_DRAWS_IN_FLIGHT, 64);
  72
  73     for (uint32_t dc = 0; dc < KNOB_MAX_DRAWS_IN_FLIGHT; ++dc)
  74     {
  75         pContext->dcRing[dc].pArena = new CachingArena(pContext->cachingArenaAllocator);
  76         new (&pContext->pMacroTileManagerArray[dc]) MacroTileMgr(*pContext->dcRing[dc].pArena);
  77         new (&pContext->pDispatchQueueArray[dc]) DispatchQueue();
  78
  79         pContext->dsRing[dc].pArena = new CachingArena(pContext->cachingArenaAllocator);
  80     }
  81
  82     if (!KNOB_SINGLE_THREADED)
  83     {
  84         memset(&pContext->WaitLock, 0, sizeof(pContext->WaitLock));
  85         memset(&pContext->FifosNotEmpty, 0, sizeof(pContext->FifosNotEmpty));
  86         new (&pContext->WaitLock) std::mutex();
  87         new (&pContext->FifosNotEmpty) std::condition_variable();
  88
  89         CreateThreadPool(pContext, &pContext->threadPool);
  90     }
  91
  92     // Calling createThreadPool() above can set SINGLE_THREADED
  93     if (KNOB_SINGLE_THREADED)
  94     {
  95         SET_KNOB(HYPERTHREADED_FE, false);
  96         pContext->NumWorkerThreads = 1;
  97         pContext->NumFEThreads = 1;
  98         pContext->NumBEThreads = 1;
  99     }
 100
 101     // Allocate scratch space for workers.
 102     ///@note We could lazily allocate this but its rather small amount of memory.
 103     for (uint32_t i = 0; i < pContext->NumWorkerThreads; ++i)
 104     {
 105 #if defined(_WIN32)
 106         uint32_t numaNode = pContext->threadPool.pThreadData ?
 107             pContext->threadPool.pThreadData[i].numaId : 0;
 108         pContext->pScratch[i] = (uint8_t*)VirtualAllocExNuma(
 109             GetCurrentProcess(), nullptr, 32 * sizeof(KILOBYTE),
 110             MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE,
 111             numaNode);
 112 #else
 113         pContext->pScratch[i] = (uint8_t*)AlignedMalloc(32 * sizeof(KILOBYTE), KNOB_SIMD_WIDTH * 4);
 114 #endif
 115     }
 116
 117     // State setup AFTER context is fully initialized
 118     SetupDefaultState(pContext);
 119
 120     // initialize hot tile manager
 121     pContext->pHotTileMgr = new HotTileMgr();
 122
 123     // initialize function pointer tables
 124     InitClearTilesTable();
 125
 126     // initialize store tiles function
 127     pContext->pfnLoadTile = pCreateInfo->pfnLoadTile;
 128     pContext->pfnStoreTile = pCreateInfo->pfnStoreTile;
 129     pContext->pfnClearTile = pCreateInfo->pfnClearTile;
 130
 131     // pass pointer to bucket manager back to caller
 132 #ifdef KNOB_ENABLE_RDTSC
 133     pCreateInfo->pBucketMgr = &gBucketMgr;
 134 #endif
 135
 136     pCreateInfo->contextSaveSize = sizeof(API_STATE);
 137
 138     return (HANDLE)pContext;
 139 }
 140
 141 void SwrDestroyContext(HANDLE hContext)
 142 {
 143     SWR_CONTEXT *pContext = (SWR_CONTEXT*)hContext;
 144     DestroyThreadPool(pContext, &pContext->threadPool);
 145
 146     // free the fifos
 147     for (uint32_t i = 0; i < KNOB_MAX_DRAWS_IN_FLIGHT; ++i)
 148     {
 149         delete pContext->dcRing[i].pArena;
 150         delete pContext->dsRing[i].pArena;
 151         pContext->pMacroTileManagerArray[i].~MacroTileMgr();
 152         pContext->pDispatchQueueArray[i].~DispatchQueue();
 153     }
 154
 155     AlignedFree(pContext->pDispatchQueueArray);
 156     AlignedFree(pContext->pMacroTileManagerArray);
 157
 158     // Free scratch space.
 159     for (uint32_t i = 0; i < pContext->NumWorkerThreads; ++i)
 160     {
 161 #if defined(_WIN32)
 162         VirtualFree(pContext->pScratch[i], 0, MEM_RELEASE);
 163 #else
 164         AlignedFree(pContext->pScratch[i]);
 165 #endif
 166     }
 167
 168     delete(pContext->pHotTileMgr);
 169
 170     pContext->~SWR_CONTEXT();
 171     AlignedFree((SWR_CONTEXT*)hContext);
 172 }
 173
 174 void CopyState(DRAW_STATE& dst, const DRAW_STATE& src)
 175 {
 176     memcpy(&dst.state, &src.state, sizeof(API_STATE));
 177 }
 178
 179 void WakeAllThreads(SWR_CONTEXT *pContext)
 180 {
 181     pContext->FifosNotEmpty.notify_all();
 182 }
 183
 184 template<bool IsDraw>
 185 void QueueWork(SWR_CONTEXT *pContext)
 186 {
 187     DRAW_CONTEXT* pDC = pContext->pCurDrawContext;
 188     uint32_t dcIndex = pDC->drawId % KNOB_MAX_DRAWS_IN_FLIGHT;
 189
 190     if (IsDraw)
 191     {
 192         pDC->pTileMgr = &pContext->pMacroTileManagerArray[dcIndex];
 193         pDC->pTileMgr->initialize();
 194     }
 195
 196     // Each worker thread looks at a DC for both FE and BE work at different times and so we
 197     // multiply threadDone by 2.  When the threadDone counter has reached 0 then all workers
 198     // have moved past this DC. (i.e. Each worker has checked this DC for both FE and BE work and
 199     // then moved on if all work is done.)
 200     pContext->pCurDrawContext->threadsDone = pContext->NumFEThreads + pContext->NumBEThreads;
 201
 202     _ReadWriteBarrier();
 203     {
 204         std::unique_lock<std::mutex> lock(pContext->WaitLock);
 205         pContext->dcRing.Enqueue();
 206     }
 207
 208     if (KNOB_SINGLE_THREADED)
 209     {
 210         // flush denormals to 0
 211         uint32_t mxcsr = _mm_getcsr();
 212         _mm_setcsr(mxcsr | _MM_FLUSH_ZERO_ON | _MM_DENORMALS_ZERO_ON);
 213
 214         if (IsDraw)
 215         {
 216             static TileSet lockedTiles;
 217             uint64_t curDraw[2] = { pContext->pCurDrawContext->drawId, pContext->pCurDrawContext->drawId };
 218             WorkOnFifoFE(pContext, 0, curDraw[0]);
 219             WorkOnFifoBE(pContext, 0, curDraw[1], lockedTiles, 0, 0);
 220         }
 221         else
 222         {
 223             uint64_t curDispatch = pContext->pCurDrawContext->drawId;
 224             WorkOnCompute(pContext, 0, curDispatch);
 225         }
 226
 227         // Dequeue the work here, if not already done, since we're single threaded (i.e. no workers).
 228         while (CompleteDrawContext(pContext, pContext->pCurDrawContext) > 0) {}
 229
 230         // restore csr
 231         _mm_setcsr(mxcsr);
 232     }
 233     else
 234     {
 235         RDTSC_START(APIDrawWakeAllThreads);
 236         WakeAllThreads(pContext);
 237         RDTSC_STOP(APIDrawWakeAllThreads, 1, 0);
 238     }
 239
 240     // Set current draw context to NULL so that next state call forces a new draw context to be created and populated.
 241     pContext->pPrevDrawContext = pContext->pCurDrawContext;
 242     pContext->pCurDrawContext = nullptr;
 243 }
 244
 245 INLINE void QueueDraw(SWR_CONTEXT* pContext)
 246 {
 247     QueueWork<true>(pContext);
 248 }
 249
 250 INLINE void QueueDispatch(SWR_CONTEXT* pContext)
 251 {
 252     QueueWork<false>(pContext);
 253 }
 254
 255 DRAW_CONTEXT* GetDrawContext(SWR_CONTEXT *pContext, bool isSplitDraw = false)
 256 {
 257     RDTSC_START(APIGetDrawContext);
 258     // If current draw context is null then need to obtain a new draw context to use from ring.
 259     if (pContext->pCurDrawContext == nullptr)
 260     {
 261         // Need to wait for a free entry.
 262         while (pContext->dcRing.IsFull())
 263         {
 264             _mm_pause();
 265         }
 266
 267         uint64_t curDraw = pContext->dcRing.GetHead();
 268         uint32_t dcIndex = curDraw % KNOB_MAX_DRAWS_IN_FLIGHT;
 269
 270         static uint64_t lastDrawChecked;
 271         static uint32_t lastFrameChecked;
 272         if ((pContext->frameCount - lastFrameChecked) > 2 ||
 273             (curDraw - lastDrawChecked) > 0x10000)
 274         {
 275             // Take this opportunity to clean-up old arena allocations
 276             pContext->cachingArenaAllocator.FreeOldBlocks();
 277
 278             lastFrameChecked = pContext->frameCount;
 279             lastDrawChecked = curDraw;
 280         }
 281
 282         DRAW_CONTEXT* pCurDrawContext = &pContext->dcRing[dcIndex];
 283         pContext->pCurDrawContext = pCurDrawContext;
 284
 285         // Assign next available entry in DS ring to this DC.
 286         uint32_t dsIndex = pContext->curStateId % KNOB_MAX_DRAWS_IN_FLIGHT;
 287         pCurDrawContext->pState = &pContext->dsRing[dsIndex];
 288
 289         // Copy previous state to current state.
 290         if (pContext->pPrevDrawContext)
 291         {
 292             DRAW_CONTEXT* pPrevDrawContext = pContext->pPrevDrawContext;
 293
 294             // If we're splitting our draw then we can just use the same state from the previous
 295             // draw. In this case, we won't increment the DS ring index so the next non-split
 296             // draw can receive the state.
 297             if (isSplitDraw == false)
 298             {
 299                 CopyState(*pCurDrawContext->pState, *pPrevDrawContext->pState);
 300
 301                 // Should have been cleaned up previously
 302                 SWR_ASSERT(pCurDrawContext->pState->pArena->IsEmpty() == true);
 303
 304                 pCurDrawContext->pState->pPrivateState = nullptr;
 305
 306                 pContext->curStateId++;  // Progress state ring index forward.
 307             }
 308             else
 309             {
 310                 // If its a split draw then just copy the state pointer over
 311                 // since its the same draw.
 312                 pCurDrawContext->pState = pPrevDrawContext->pState;
 313                 SWR_ASSERT(pPrevDrawContext->cleanupState == false);
 314             }
 315         }
 316         else
 317         {
 318             SWR_ASSERT(pCurDrawContext->pState->pArena->IsEmpty() == true);
 319             pContext->curStateId++;  // Progress state ring index forward.
 320         }
 321
 322         SWR_ASSERT(pCurDrawContext->pArena->IsEmpty() == true);
 323
 324         pCurDrawContext->dependency = 0;
 325         pCurDrawContext->pContext = pContext;
 326         pCurDrawContext->isCompute = false; // Dispatch has to set this to true.
 327
 328         pCurDrawContext->doneFE = false;
 329         pCurDrawContext->FeLock = 0;
 330         pCurDrawContext->threadsDone = 0;
 331
 332         // Assign unique drawId for this DC
 333         pCurDrawContext->drawId = pContext->dcRing.GetHead();
 334
 335         pCurDrawContext->cleanupState = true;
 336     }
 337     else
 338     {
 339         SWR_ASSERT(isSplitDraw == false, "Split draw should only be used when obtaining a new DC");
 340     }
 341
 342     RDTSC_STOP(APIGetDrawContext, 0, 0);
 343     return pContext->pCurDrawContext;
 344 }
 345
 346 API_STATE* GetDrawState(SWR_CONTEXT *pContext)
 347 {
 348     DRAW_CONTEXT* pDC = GetDrawContext(pContext);
 349     SWR_ASSERT(pDC->pState != nullptr);
 350
 351     return &pDC->pState->state;
 352 }
 353
 354 void SWR_API SwrSaveState(
 355     HANDLE hContext,
 356     void* pOutputStateBlock,
 357     size_t memSize)
 358 {
 359     SWR_CONTEXT *pContext = (SWR_CONTEXT*)hContext;
 360     auto pSrc = GetDrawState(pContext);
 361     SWR_ASSERT(pOutputStateBlock && memSize >= sizeof(*pSrc));
 362
 363     memcpy(pOutputStateBlock, pSrc, sizeof(*pSrc));
 364 }
 365
 366 void SWR_API SwrRestoreState(
 367     HANDLE hContext,
 368     const void* pStateBlock,
 369     size_t memSize)
 370 {
 371     SWR_CONTEXT *pContext = (SWR_CONTEXT*)hContext;
 372     auto pDst = GetDrawState(pContext);
 373     SWR_ASSERT(pStateBlock && memSize >= sizeof(*pDst));
 374
 375     memcpy(pDst, pStateBlock, sizeof(*pDst));
 376 }
 377
 378 void SetupDefaultState(SWR_CONTEXT *pContext)
 379 {
 380     API_STATE* pState = GetDrawState(pContext);
 381
 382     pState->rastState.cullMode = SWR_CULLMODE_NONE;
 383     pState->rastState.frontWinding = SWR_FRONTWINDING_CCW;
 384 }
 385
 386 static INLINE SWR_CONTEXT* GetContext(HANDLE hContext)
 387 {
 388     return (SWR_CONTEXT*)hContext;
 389 }
 390
 391 void SwrSync(HANDLE hContext, PFN_CALLBACK_FUNC pfnFunc, uint64_t userData, uint64_t userData2, uint64_t userData3)
 392 {
 393     RDTSC_START(APISync);
 394
 395     SWR_ASSERT(pfnFunc != nullptr);
 396
 397     SWR_CONTEXT *pContext = GetContext(hContext);
 398     DRAW_CONTEXT* pDC = GetDrawContext(pContext);
 399
 400     pDC->FeWork.type = SYNC;
 401     pDC->FeWork.pfnWork = ProcessSync;
 402     pDC->FeWork.desc.sync.pfnCallbackFunc = pfnFunc;
 403     pDC->FeWork.desc.sync.userData = userData;
 404     pDC->FeWork.desc.sync.userData2 = userData2;
 405     pDC->FeWork.desc.sync.userData3 = userData3;
 406
 407     // cannot execute until all previous draws have completed
 408     pDC->dependency = pDC->drawId - 1;
 409
 410     //enqueue
 411     QueueDraw(pContext);
 412
 413     RDTSC_STOP(APISync, 1, 0);
 414 }
 415
 416 void SwrWaitForIdle(HANDLE hContext)
 417 {
 418     SWR_CONTEXT *pContext = GetContext(hContext);
 419
 420     RDTSC_START(APIWaitForIdle);
 421
 422     while (!pContext->dcRing.IsEmpty())
 423     {
 424         _mm_pause();
 425     }
 426
 427     RDTSC_STOP(APIWaitForIdle, 1, 0);
 428 }
 429
 430 void SwrSetVertexBuffers(
 431     HANDLE hContext,
 432     uint32_t numBuffers,
 433     const SWR_VERTEX_BUFFER_STATE* pVertexBuffers)
 434 {
 435     API_STATE* pState = GetDrawState(GetContext(hContext));
 436
 437     for (uint32_t i = 0; i < numBuffers; ++i)
 438     {
 439         const SWR_VERTEX_BUFFER_STATE *pVB = &pVertexBuffers[i];
 440         pState->vertexBuffers[pVB->index] = *pVB;
 441     }
 442 }
 443
 444 void SwrSetIndexBuffer(
 445     HANDLE hContext,
 446     const SWR_INDEX_BUFFER_STATE* pIndexBuffer)
 447 {
 448     API_STATE* pState = GetDrawState(GetContext(hContext));
 449
 450     pState->indexBuffer = *pIndexBuffer;
 451 }
 452
 453 void SwrSetFetchFunc(
 454     HANDLE hContext,
 455     PFN_FETCH_FUNC    pfnFetchFunc)
 456 {
 457     API_STATE* pState = GetDrawState(GetContext(hContext));
 458
 459     pState->pfnFetchFunc = pfnFetchFunc;
 460 }
 461
 462 void SwrSetSoFunc(
 463     HANDLE hContext,
 464     PFN_SO_FUNC    pfnSoFunc,
 465     uint32_t streamIndex)
 466 {
 467     API_STATE* pState = GetDrawState(GetContext(hContext));
 468
 469     SWR_ASSERT(streamIndex < MAX_SO_STREAMS);
 470
 471     pState->pfnSoFunc[streamIndex] = pfnSoFunc;
 472 }
 473
 474 void SwrSetSoState(
 475     HANDLE hContext,
 476     SWR_STREAMOUT_STATE* pSoState)
 477 {
 478     API_STATE* pState = GetDrawState(GetContext(hContext));
 479
 480     pState->soState = *pSoState;
 481 }
 482
 483 void SwrSetSoBuffers(
 484     HANDLE hContext,
 485     SWR_STREAMOUT_BUFFER* pSoBuffer,
 486     uint32_t slot)
 487 {
 488     API_STATE* pState = GetDrawState(GetContext(hContext));
 489
 490     SWR_ASSERT((slot < 4), "There are only 4 SO buffer slots [0, 3]\nSlot requested: %d", slot);
 491
 492     pState->soBuffer[slot] = *pSoBuffer;
 493 }
 494
 495 void SwrSetVertexFunc(
 496     HANDLE hContext,
 497     PFN_VERTEX_FUNC pfnVertexFunc)
 498 {
 499     API_STATE* pState = GetDrawState(GetContext(hContext));
 500
 501     pState->pfnVertexFunc = pfnVertexFunc;
 502 }
 503
 504 void SwrSetFrontendState(
 505     HANDLE hContext,
 506     SWR_FRONTEND_STATE *pFEState)
 507 {
 508     API_STATE* pState = GetDrawState(GetContext(hContext));
 509     pState->frontendState = *pFEState;
 510 }
 511
 512 void SwrSetGsState(
 513     HANDLE hContext,
 514     SWR_GS_STATE *pGSState)
 515 {
 516     API_STATE* pState = GetDrawState(GetContext(hContext));
 517     pState->gsState = *pGSState;
 518 }
 519
 520 void SwrSetGsFunc(
 521     HANDLE hContext,
 522     PFN_GS_FUNC pfnGsFunc)
 523 {
 524     API_STATE* pState = GetDrawState(GetContext(hContext));
 525     pState->pfnGsFunc = pfnGsFunc;
 526 }
 527
 528 void SwrSetCsFunc(
 529     HANDLE hContext,
 530     PFN_CS_FUNC pfnCsFunc,
 531     uint32_t totalThreadsInGroup,
 532     uint32_t totalSpillFillSize)
 533 {
 534     API_STATE* pState = GetDrawState(GetContext(hContext));
 535     pState->pfnCsFunc = pfnCsFunc;
 536     pState->totalThreadsInGroup = totalThreadsInGroup;
 537     pState->totalSpillFillSize = totalSpillFillSize;
 538 }
 539
 540 void SwrSetTsState(
 541     HANDLE hContext,
 542     SWR_TS_STATE *pState)
 543 {
 544     API_STATE* pApiState = GetDrawState(GetContext(hContext));
 545     pApiState->tsState = *pState;
 546 }
 547
 548 void SwrSetHsFunc(
 549     HANDLE hContext,
 550     PFN_HS_FUNC pfnFunc)
 551 {
 552     API_STATE* pApiState = GetDrawState(GetContext(hContext));
 553     pApiState->pfnHsFunc = pfnFunc;
 554 }
 555
 556 void SwrSetDsFunc(
 557     HANDLE hContext,
 558     PFN_DS_FUNC pfnFunc)
 559 {
 560     API_STATE* pApiState = GetDrawState(GetContext(hContext));
 561     pApiState->pfnDsFunc = pfnFunc;
 562 }
 563
 564 void SwrSetDepthStencilState(
 565     HANDLE hContext,
 566     SWR_DEPTH_STENCIL_STATE *pDSState)
 567 {
 568     API_STATE* pState = GetDrawState(GetContext(hContext));
 569
 570     pState->depthStencilState = *pDSState;
 571 }
 572
 573 void SwrSetBackendState(
 574     HANDLE hContext,
 575     SWR_BACKEND_STATE *pBEState)
 576 {
 577     API_STATE* pState = GetDrawState(GetContext(hContext));
 578
 579     pState->backendState = *pBEState;
 580 }
 581
 582 void SwrSetPixelShaderState(
 583     HANDLE hContext,
 584     SWR_PS_STATE *pPSState)
 585 {
 586     API_STATE *pState = GetDrawState(GetContext(hContext));
 587     pState->psState = *pPSState;
 588 }
 589
 590 void SwrSetBlendState(
 591     HANDLE hContext,
 592     SWR_BLEND_STATE *pBlendState)
 593 {
 594     API_STATE *pState = GetDrawState(GetContext(hContext));
 595     memcpy(&pState->blendState, pBlendState, sizeof(SWR_BLEND_STATE));
 596 }
 597
 598 void SwrSetBlendFunc(
 599     HANDLE hContext,
 600     uint32_t renderTarget,
 601     PFN_BLEND_JIT_FUNC pfnBlendFunc)
 602 {
 603     SWR_ASSERT(renderTarget < SWR_NUM_RENDERTARGETS);
 604     API_STATE *pState = GetDrawState(GetContext(hContext));
 605     pState->pfnBlendFunc[renderTarget] = pfnBlendFunc;
 606 }
 607
 608 void SwrSetLinkage(
 609     HANDLE hContext,
 610     uint32_t mask,
 611     const uint8_t* pMap)
 612 {
 613     API_STATE* pState = GetDrawState(GetContext(hContext));
 614
 615     static const uint8_t IDENTITY_MAP[] =
 616     {
 617          0,  1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11, 12, 13, 14, 15,
 618         16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31,
 619     };
 620     static_assert(sizeof(IDENTITY_MAP) == sizeof(pState->linkageMap),
 621         "Update for new value of MAX_ATTRIBUTES");
 622
 623     pState->linkageMask = mask;
 624     pState->linkageCount = _mm_popcnt_u32(mask);
 625
 626     if (!pMap)
 627     {
 628         pMap = IDENTITY_MAP;
 629     }
 630     memcpy(pState->linkageMap, pMap, pState->linkageCount);
 631 }
 632
 633 // update guardband multipliers for the viewport
 634 void updateGuardband(API_STATE *pState)
 635 {
 636     // guardband center is viewport center
 637     pState->gbState.left    = KNOB_GUARDBAND_WIDTH  / pState->vp[0].width;
 638     pState->gbState.right   = KNOB_GUARDBAND_WIDTH  / pState->vp[0].width;
 639     pState->gbState.top     = KNOB_GUARDBAND_HEIGHT / pState->vp[0].height;
 640     pState->gbState.bottom  = KNOB_GUARDBAND_HEIGHT / pState->vp[0].height;
 641 }
 642
 643 void SwrSetRastState(
 644     HANDLE hContext,
 645     const SWR_RASTSTATE *pRastState)
 646 {
 647     SWR_CONTEXT *pContext = GetContext(hContext);
 648     API_STATE* pState = GetDrawState(pContext);
 649
 650     memcpy(&pState->rastState, pRastState, sizeof(SWR_RASTSTATE));
 651 }
 652
 653 void SwrSetViewports(
 654     HANDLE hContext,
 655     uint32_t numViewports,
 656     const SWR_VIEWPORT* pViewports,
 657     const SWR_VIEWPORT_MATRIX* pMatrices)
 658 {
 659     SWR_ASSERT(numViewports <= KNOB_NUM_VIEWPORTS_SCISSORS,
 660         "Invalid number of viewports.");
 661
 662     SWR_CONTEXT *pContext = GetContext(hContext);
 663     API_STATE* pState = GetDrawState(pContext);
 664
 665     memcpy(&pState->vp[0], pViewports, sizeof(SWR_VIEWPORT) * numViewports);
 666
 667     if (pMatrices != nullptr)
 668     {
 669         memcpy(&pState->vpMatrix[0], pMatrices, sizeof(SWR_VIEWPORT_MATRIX) * numViewports);
 670     }
 671     else
 672     {
 673         // Compute default viewport transform.
 674         for (uint32_t i = 0; i < numViewports; ++i)
 675         {
 676             if (pContext->driverType == DX)
 677             {
 678                 pState->vpMatrix[i].m00 = pState->vp[i].width / 2.0f;
 679                 pState->vpMatrix[i].m11 = -pState->vp[i].height / 2.0f;
 680                 pState->vpMatrix[i].m22 = pState->vp[i].maxZ - pState->vp[i].minZ;
 681                 pState->vpMatrix[i].m30 = pState->vp[i].x + pState->vpMatrix[i].m00;
 682                 pState->vpMatrix[i].m31 = pState->vp[i].y - pState->vpMatrix[i].m11;
 683                 pState->vpMatrix[i].m32 = pState->vp[i].minZ;
 684             }
 685             else
 686             {
 687                 // Standard, with the exception that Y is inverted.
 688                 pState->vpMatrix[i].m00 = (pState->vp[i].width - pState->vp[i].x) / 2.0f;
 689                 pState->vpMatrix[i].m11 = (pState->vp[i].y - pState->vp[i].height) / 2.0f;
 690                 pState->vpMatrix[i].m22 = (pState->vp[i].maxZ - pState->vp[i].minZ) / 2.0f;
 691                 pState->vpMatrix[i].m30 = pState->vp[i].x + pState->vpMatrix[i].m00;
 692                 pState->vpMatrix[i].m31 = pState->vp[i].height + pState->vpMatrix[i].m11;
 693                 pState->vpMatrix[i].m32 = pState->vp[i].minZ + pState->vpMatrix[i].m22;
 694
 695                 // Now that the matrix is calculated, clip the view coords to screen size.
 696                 // OpenGL allows for -ve x,y in the viewport.
 697                 pState->vp[i].x = std::max(pState->vp[i].x, 0.0f);
 698                 pState->vp[i].y = std::max(pState->vp[i].y, 0.0f);
 699             }
 700         }
 701     }
 702
 703     updateGuardband(pState);
 704 }
 705
 706 void SwrSetScissorRects(
 707     HANDLE hContext,
 708     uint32_t numScissors,
 709     const BBOX* pScissors)
 710 {
 711     SWR_ASSERT(numScissors <= KNOB_NUM_VIEWPORTS_SCISSORS,
 712         "Invalid number of scissor rects.");
 713
 714     API_STATE* pState = GetDrawState(GetContext(hContext));
 715     memcpy(&pState->scissorRects[0], pScissors, numScissors * sizeof(BBOX));
 716 };
 717
 718 void SetupMacroTileScissors(DRAW_CONTEXT *pDC)
 719 {
 720     API_STATE *pState = &pDC->pState->state;
 721     uint32_t left, right, top, bottom;
 722
 723     // Set up scissor dimensions based on scissor or viewport
 724     if (pState->rastState.scissorEnable)
 725     {
 726         // scissor rect right/bottom edge are exclusive, core expects scissor dimensions to be inclusive, so subtract one pixel from right/bottom edges
 727         left = pState->scissorRects[0].left;
 728         right = pState->scissorRects[0].right;
 729         top = pState->scissorRects[0].top;
 730         bottom = pState->scissorRects[0].bottom;
 731     }
 732     else
 733     {
 734         left = (int32_t)pState->vp[0].x;
 735         right = (int32_t)pState->vp[0].x + (int32_t)pState->vp[0].width;
 736         top = (int32_t)pState->vp[0].y;
 737         bottom = (int32_t)pState->vp[0].y + (int32_t)pState->vp[0].height;
 738     }
 739
 740     right = std::min<uint32_t>(right, KNOB_MAX_SCISSOR_X);
 741     bottom = std::min<uint32_t>(bottom, KNOB_MAX_SCISSOR_Y);
 742
 743     if (left > KNOB_MAX_SCISSOR_X || top > KNOB_MAX_SCISSOR_Y)
 744     {
 745         pState->scissorInFixedPoint.left = 0;
 746         pState->scissorInFixedPoint.right = 0;
 747         pState->scissorInFixedPoint.top = 0;
 748         pState->scissorInFixedPoint.bottom = 0;
 749     }
 750     else
 751     {
 752         pState->scissorInFixedPoint.left = left * FIXED_POINT_SCALE;
 753         pState->scissorInFixedPoint.right = right * FIXED_POINT_SCALE - 1;
 754         pState->scissorInFixedPoint.top = top * FIXED_POINT_SCALE;
 755         pState->scissorInFixedPoint.bottom = bottom * FIXED_POINT_SCALE - 1;
 756     }
 757 }
 758
 759 // templated backend function tables
 760 extern PFN_BACKEND_FUNC gBackendNullPs[SWR_MULTISAMPLE_TYPE_MAX];
 761 extern PFN_BACKEND_FUNC gBackendSingleSample[2][2][2];
 762 extern PFN_BACKEND_FUNC gBackendPixelRateTable[SWR_MULTISAMPLE_TYPE_MAX][SWR_MSAA_SAMPLE_PATTERN_MAX][SWR_INPUT_COVERAGE_MAX][2][2][2];
 763 extern PFN_BACKEND_FUNC gBackendSampleRateTable[SWR_MULTISAMPLE_TYPE_MAX][SWR_INPUT_COVERAGE_MAX][2][2];
 764 void SetupPipeline(DRAW_CONTEXT *pDC)
 765 {
 766     DRAW_STATE* pState = pDC->pState;
 767     const SWR_RASTSTATE &rastState = pState->state.rastState;
 768     const SWR_PS_STATE &psState = pState->state.psState;
 769     BACKEND_FUNCS& backendFuncs = pState->backendFuncs;
 770     const uint32_t forcedSampleCount = (rastState.forcedSampleCount) ? 1 : 0;
 771
 772     // setup backend
 773     if (psState.pfnPixelShader == nullptr)
 774     {
 775         backendFuncs.pfnBackend = gBackendNullPs[pState->state.rastState.sampleCount];
 776     }
 777     else
 778     {
 779         const bool bMultisampleEnable = ((rastState.sampleCount > SWR_MULTISAMPLE_1X) || rastState.forcedSampleCount) ? 1 : 0;
 780         const uint32_t centroid = ((psState.barycentricsMask & SWR_BARYCENTRIC_CENTROID_MASK) > 0) ? 1 : 0;
 781         const uint32_t canEarlyZ = (psState.forceEarlyZ || (!psState.writesODepth && !psState.usesSourceDepth && !psState.usesUAV)) ? 1 : 0;
 782
 783         // currently only support 'normal' input coverage
 784         SWR_ASSERT(psState.inputCoverage == SWR_INPUT_COVERAGE_NORMAL ||
 785                    psState.inputCoverage == SWR_INPUT_COVERAGE_NONE);
 786
 787         SWR_BARYCENTRICS_MASK barycentricsMask = (SWR_BARYCENTRICS_MASK)psState.barycentricsMask;
 788
 789         // select backend function
 790         switch(psState.shadingRate)
 791         {
 792         case SWR_SHADING_RATE_PIXEL:
 793             if(bMultisampleEnable)
 794             {
 795                 // always need to generate I & J per sample for Z interpolation
 796                 barycentricsMask = (SWR_BARYCENTRICS_MASK)(barycentricsMask | SWR_BARYCENTRIC_PER_SAMPLE_MASK);
 797                 backendFuncs.pfnBackend = gBackendPixelRateTable[rastState.sampleCount][rastState.samplePattern][psState.inputCoverage][centroid][forcedSampleCount][canEarlyZ];
 798             }
 799             else
 800             {
 801                 // always need to generate I & J per pixel for Z interpolation
 802                 barycentricsMask = (SWR_BARYCENTRICS_MASK)(barycentricsMask | SWR_BARYCENTRIC_PER_PIXEL_MASK);
 803                 backendFuncs.pfnBackend = gBackendSingleSample[psState.inputCoverage][centroid][canEarlyZ];
 804             }
 805             break;
 806         case SWR_SHADING_RATE_SAMPLE:
 807             SWR_ASSERT(rastState.samplePattern == SWR_MSAA_STANDARD_PATTERN);
 808             // always need to generate I & J per sample for Z interpolation
 809             barycentricsMask = (SWR_BARYCENTRICS_MASK)(barycentricsMask | SWR_BARYCENTRIC_PER_SAMPLE_MASK);
 810             backendFuncs.pfnBackend = gBackendSampleRateTable[rastState.sampleCount][psState.inputCoverage][centroid][canEarlyZ];
 811             break;
 812         default:
 813             SWR_ASSERT(0 && "Invalid shading rate");
 814             break;
 815         }
 816     }
 817
 818     PFN_PROCESS_PRIMS pfnBinner;
 819     switch (pState->state.topology)
 820     {
 821     case TOP_POINT_LIST:
 822         pState->pfnProcessPrims = ClipPoints;
 823         pfnBinner = BinPoints;
 824         break;
 825     case TOP_LINE_LIST:
 826     case TOP_LINE_STRIP:
 827     case TOP_LINE_LOOP:
 828     case TOP_LINE_LIST_ADJ:
 829     case TOP_LISTSTRIP_ADJ:
 830         pState->pfnProcessPrims = ClipLines;
 831         pfnBinner = BinLines;
 832         break;
 833     default:
 834         pState->pfnProcessPrims = ClipTriangles;
 835         pfnBinner = BinTriangles;
 836         break;
 837     };
 838
 839     // disable clipper if viewport transform is disabled
 840     if (pState->state.frontendState.vpTransformDisable)
 841     {
 842         pState->pfnProcessPrims = pfnBinner;
 843     }
 844
 845     if ((pState->state.psState.pfnPixelShader == nullptr) &&
 846         (pState->state.depthStencilState.depthTestEnable == FALSE) &&
 847         (pState->state.depthStencilState.depthWriteEnable == FALSE) &&
 848         (pState->state.depthStencilState.stencilTestEnable == FALSE) &&
 849         (pState->state.depthStencilState.stencilWriteEnable == FALSE) &&
 850         (pState->state.linkageCount == 0))
 851     {
 852         pState->pfnProcessPrims = nullptr;
 853         pState->state.linkageMask = 0;
 854     }
 855
 856     if (pState->state.soState.rasterizerDisable == true)
 857     {
 858         pState->pfnProcessPrims = nullptr;
 859         pState->state.linkageMask = 0;
 860     }
 861
 862     // set up the frontend attrib mask
 863     pState->state.feAttribMask = pState->state.linkageMask;
 864     if (pState->state.soState.soEnable)
 865     {
 866         for (uint32_t i = 0; i < 4; ++i)
 867         {
 868             pState->state.feAttribMask |= pState->state.soState.streamMasks[i];
 869         }
 870     }
 871
 872     // complicated logic to test for cases where we don't need backing hottile memory for a draw
 873     // have to check for the special case where depth/stencil test is enabled but depthwrite is disabled.
 874     pState->state.depthHottileEnable = ((!(pState->state.depthStencilState.depthTestEnable &&
 875                                            !pState->state.depthStencilState.depthWriteEnable &&
 876                                            pState->state.depthStencilState.depthTestFunc == ZFUNC_ALWAYS)) &&
 877                                         (pState->state.depthStencilState.depthTestEnable ||
 878                                          pState->state.depthStencilState.depthWriteEnable)) ? true : false;
 879
 880     pState->state.stencilHottileEnable = (((!(pState->state.depthStencilState.stencilTestEnable &&
 881                                              !pState->state.depthStencilState.stencilWriteEnable &&
 882                                               pState->state.depthStencilState.stencilTestFunc == ZFUNC_ALWAYS)) ||
 883                                           // for stencil we have to check the double sided state as well
 884                                           (!(pState->state.depthStencilState.doubleSidedStencilTestEnable &&
 885                                              !pState->state.depthStencilState.stencilWriteEnable &&
 886                                               pState->state.depthStencilState.backfaceStencilTestFunc == ZFUNC_ALWAYS))) &&
 887                                           (pState->state.depthStencilState.stencilTestEnable  ||
 888                                            pState->state.depthStencilState.stencilWriteEnable)) ? true : false;
 889
 890     uint32_t numRTs = pState->state.psState.numRenderTargets;
 891     pState->state.colorHottileEnable = 0;
 892     if (psState.pfnPixelShader != nullptr)
 893     {
 894         for (uint32_t rt = 0; rt < numRTs; ++rt)
 895         {
 896             pState->state.colorHottileEnable |=
 897                 (!pState->state.blendState.renderTarget[rt].writeDisableAlpha ||
 898                  !pState->state.blendState.renderTarget[rt].writeDisableRed ||
 899                  !pState->state.blendState.renderTarget[rt].writeDisableGreen ||
 900                  !pState->state.blendState.renderTarget[rt].writeDisableBlue) ? (1 << rt) : 0;
 901         }
 902     }
 903
 904     // Setup depth quantization function
 905     if (pState->state.depthHottileEnable)
 906     {
 907         switch (pState->state.rastState.depthFormat)
 908         {
 909         case R32_FLOAT_X8X24_TYPELESS: pState->state.pfnQuantizeDepth = QuantizeDepth < R32_FLOAT_X8X24_TYPELESS > ; break;
 910         case R32_FLOAT: pState->state.pfnQuantizeDepth = QuantizeDepth < R32_FLOAT > ; break;
 911         case R24_UNORM_X8_TYPELESS: pState->state.pfnQuantizeDepth = QuantizeDepth < R24_UNORM_X8_TYPELESS > ; break;
 912         case R16_UNORM: pState->state.pfnQuantizeDepth = QuantizeDepth < R16_UNORM > ; break;
 913         default: SWR_ASSERT(false, "Unsupported depth format for depth quantiztion.");
 914             pState->state.pfnQuantizeDepth = QuantizeDepth < R32_FLOAT > ;
 915         }
 916     }
 917     else
 918     {
 919         // set up pass-through quantize if depth isn't enabled
 920         pState->state.pfnQuantizeDepth = QuantizeDepth < R32_FLOAT > ;
 921     }
 922 }
 923
 924 //////////////////////////////////////////////////////////////////////////
 925 /// @brief InitDraw
 926 /// @param pDC - Draw context to initialize for this draw.
 927 void InitDraw(
 928     DRAW_CONTEXT *pDC,
 929     bool isSplitDraw)
 930 {
 931     // We don't need to re-setup the scissors/pipeline state again for split draw.
 932     if (isSplitDraw == false)
 933     {
 934         SetupMacroTileScissors(pDC);
 935         SetupPipeline(pDC);
 936     }
 937 }
 938
 939 //////////////////////////////////////////////////////////////////////////
 940 /// @brief We can split the draw for certain topologies for better performance.
 941 /// @param totalVerts - Total vertices for draw
 942 /// @param topology - Topology used for draw
 943 uint32_t MaxVertsPerDraw(
 944     DRAW_CONTEXT* pDC,
 945     uint32_t totalVerts,
 946     PRIMITIVE_TOPOLOGY topology)
 947 {
 948     API_STATE& state = pDC->pState->state;
 949
 950     uint32_t vertsPerDraw = totalVerts;
 951
 952     if (state.soState.soEnable)
 953     {
 954         return totalVerts;
 955     }
 956
 957     switch (topology)
 958     {
 959     case TOP_POINT_LIST:
 960     case TOP_TRIANGLE_LIST:
 961         vertsPerDraw = KNOB_MAX_PRIMS_PER_DRAW;
 962         break;
 963
 964     case TOP_PATCHLIST_1:
 965     case TOP_PATCHLIST_2:
 966     case TOP_PATCHLIST_3:
 967     case TOP_PATCHLIST_4:
 968     case TOP_PATCHLIST_5:
 969     case TOP_PATCHLIST_6:
 970     case TOP_PATCHLIST_7:
 971     case TOP_PATCHLIST_8:
 972     case TOP_PATCHLIST_9:
 973     case TOP_PATCHLIST_10:
 974     case TOP_PATCHLIST_11:
 975     case TOP_PATCHLIST_12:
 976     case TOP_PATCHLIST_13:
 977     case TOP_PATCHLIST_14:
 978     case TOP_PATCHLIST_15:
 979     case TOP_PATCHLIST_16:
 980     case TOP_PATCHLIST_17:
 981     case TOP_PATCHLIST_18:
 982     case TOP_PATCHLIST_19:
 983     case TOP_PATCHLIST_20:
 984     case TOP_PATCHLIST_21:
 985     case TOP_PATCHLIST_22:
 986     case TOP_PATCHLIST_23:
 987     case TOP_PATCHLIST_24:
 988     case TOP_PATCHLIST_25:
 989     case TOP_PATCHLIST_26:
 990     case TOP_PATCHLIST_27:
 991     case TOP_PATCHLIST_28:
 992     case TOP_PATCHLIST_29:
 993     case TOP_PATCHLIST_30:
 994     case TOP_PATCHLIST_31:
 995     case TOP_PATCHLIST_32:
 996         if (pDC->pState->state.tsState.tsEnable)
 997         {
 998             uint32_t vertsPerPrim = topology - TOP_PATCHLIST_BASE;
 999             vertsPerDraw = vertsPerPrim * KNOB_MAX_TESS_PRIMS_PER_DRAW;
1000         }
1001         break;
1002
1003     // The Primitive Assembly code can only handle 1 RECT at a time.
1004     case TOP_RECT_LIST:
1005         vertsPerDraw = 3;
1006         break;
1007
1008     default:
1009         // We are not splitting up draws for other topologies.
1010         break;
1011     }
1012
1013     return vertsPerDraw;
1014 }
1015
1016
1017 //////////////////////////////////////////////////////////////////////////
1018 /// @brief DrawInstanced
1019 /// @param hContext - Handle passed back from SwrCreateContext
1020 /// @param topology - Specifies topology for draw.
1021 /// @param numVerts - How many vertices to read sequentially from vertex data (per instance).
1022 /// @param startVertex - Specifies start vertex for draw. (vertex data)
1023 /// @param numInstances - How many instances to render.
1024 /// @param startInstance - Which instance to start sequentially fetching from in each buffer (instanced data)
1025 void DrawInstanced(
1026     HANDLE hContext,
1027     PRIMITIVE_TOPOLOGY topology,
1028     uint32_t numVertices,
1029     uint32_t startVertex,
1030     uint32_t numInstances = 1,
1031     uint32_t startInstance = 0)
1032 {
1033     if (KNOB_TOSS_DRAW)
1034     {
1035         return;
1036     }
1037
1038     RDTSC_START(APIDraw);
1039
1040     SWR_CONTEXT *pContext = GetContext(hContext);
1041     DRAW_CONTEXT* pDC = GetDrawContext(pContext);
1042
1043     uint32_t maxVertsPerDraw = MaxVertsPerDraw(pDC, numVertices, topology);
1044     uint32_t primsPerDraw = GetNumPrims(topology, maxVertsPerDraw);
1045     uint32_t remainingVerts = numVertices;
1046
1047     API_STATE    *pState = &pDC->pState->state;
1048     pState->topology = topology;
1049     pState->forceFront = false;
1050
1051     // disable culling for points/lines
1052     uint32_t oldCullMode = pState->rastState.cullMode;
1053     if (topology == TOP_POINT_LIST)
1054     {
1055         pState->rastState.cullMode = SWR_CULLMODE_NONE;
1056         pState->forceFront = true;
1057     }
1058
1059     int draw = 0;
1060     while (remainingVerts)
1061     {
1062         uint32_t numVertsForDraw = (remainingVerts < maxVertsPerDraw) ?
1063         remainingVerts : maxVertsPerDraw;
1064
1065         bool isSplitDraw = (draw > 0) ? true : false;
1066         DRAW_CONTEXT* pDC = GetDrawContext(pContext, isSplitDraw);
1067         InitDraw(pDC, isSplitDraw);
1068
1069         pDC->FeWork.type = DRAW;
1070         pDC->FeWork.pfnWork = GetProcessDrawFunc(
1071             false,  // IsIndexed
1072             pState->tsState.tsEnable,
1073             pState->gsState.gsEnable,
1074             pState->soState.soEnable,
1075             pDC->pState->pfnProcessPrims != nullptr);
1076         pDC->FeWork.desc.draw.numVerts = numVertsForDraw;
1077         pDC->FeWork.desc.draw.startVertex = startVertex;
1078         pDC->FeWork.desc.draw.numInstances = numInstances;
1079         pDC->FeWork.desc.draw.startInstance = startInstance;
1080         pDC->FeWork.desc.draw.startPrimID = draw * primsPerDraw;
1081         pDC->FeWork.desc.draw.startVertexID = draw * maxVertsPerDraw;
1082
1083         pDC->cleanupState = (remainingVerts == numVertsForDraw);
1084
1085         //enqueue DC
1086         QueueDraw(pContext);
1087
1088         remainingVerts -= numVertsForDraw;
1089         draw++;
1090     }
1091
1092     // restore culling state
1093     pDC = GetDrawContext(pContext);
1094     pDC->pState->state.rastState.cullMode = oldCullMode;
1095
1096     RDTSC_STOP(APIDraw, numVertices * numInstances, 0);
1097 }
1098
1099 //////////////////////////////////////////////////////////////////////////
1100 /// @brief SwrDraw
1101 /// @param hContext - Handle passed back from SwrCreateContext
1102 /// @param topology - Specifies topology for draw.
1103 /// @param startVertex - Specifies start vertex in vertex buffer for draw.
1104 /// @param primCount - Number of vertices.
1105 void SwrDraw(
1106     HANDLE hContext,
1107     PRIMITIVE_TOPOLOGY topology,
1108     uint32_t startVertex,
1109     uint32_t numVertices)
1110 {
1111     DrawInstanced(hContext, topology, numVertices, startVertex);
1112 }
1113
1114 //////////////////////////////////////////////////////////////////////////
1115 /// @brief SwrDrawInstanced
1116 /// @param hContext - Handle passed back from SwrCreateContext
1117 /// @param topology - Specifies topology for draw.
1118 /// @param numVertsPerInstance - How many vertices to read sequentially from vertex data.
1119 /// @param numInstances - How many instances to render.
1120 /// @param startVertex - Specifies start vertex for draw. (vertex data)
1121 /// @param startInstance - Which instance to start sequentially fetching from in each buffer (instanced data)
1122 void SwrDrawInstanced(
1123     HANDLE hContext,
1124     PRIMITIVE_TOPOLOGY topology,
1125     uint32_t numVertsPerInstance,
1126     uint32_t numInstances,
1127     uint32_t startVertex,
1128     uint32_t startInstance
1129     )
1130 {
1131     DrawInstanced(hContext, topology, numVertsPerInstance, startVertex, numInstances, startInstance);
1132 }
1133
1134 //////////////////////////////////////////////////////////////////////////
1135 /// @brief DrawIndexedInstanced
1136 /// @param hContext - Handle passed back from SwrCreateContext
1137 /// @param topology - Specifies topology for draw.
1138 /// @param numIndices - Number of indices to read sequentially from index buffer.
1139 /// @param indexOffset - Starting index into index buffer.
1140 /// @param baseVertex - Vertex in vertex buffer to consider as index "0". Note value is signed.
1141 /// @param numInstances - Number of instances to render.
1142 /// @param startInstance - Which instance to start sequentially fetching from in each buffer (instanced data)
1143 void DrawIndexedInstance(
1144     HANDLE hContext,
1145     PRIMITIVE_TOPOLOGY topology,
1146     uint32_t numIndices,
1147     uint32_t indexOffset,
1148     int32_t baseVertex,
1149     uint32_t numInstances = 1,
1150     uint32_t startInstance = 0)
1151 {
1152     if (KNOB_TOSS_DRAW)
1153     {
1154         return;
1155     }
1156
1157     RDTSC_START(APIDrawIndexed);
1158
1159     SWR_CONTEXT *pContext = GetContext(hContext);
1160     DRAW_CONTEXT* pDC = GetDrawContext(pContext);
1161     API_STATE* pState = &pDC->pState->state;
1162
1163     uint32_t maxIndicesPerDraw = MaxVertsPerDraw(pDC, numIndices, topology);
1164     uint32_t primsPerDraw = GetNumPrims(topology, maxIndicesPerDraw);
1165     uint32_t remainingIndices = numIndices;
1166
1167     uint32_t indexSize = 0;
1168     switch (pState->indexBuffer.format)
1169     {
1170     case R32_UINT: indexSize = sizeof(uint32_t); break;
1171     case R16_UINT: indexSize = sizeof(uint16_t); break;
1172     case R8_UINT: indexSize = sizeof(uint8_t); break;
1173     default:
1174         SWR_ASSERT(0);
1175     }
1176
1177     int draw = 0;
1178     uint8_t *pIB = (uint8_t*)pState->indexBuffer.pIndices;
1179     pIB += (uint64_t)indexOffset * (uint64_t)indexSize;
1180
1181     pState->topology = topology;
1182     pState->forceFront = false;
1183
1184     // disable culling for points/lines
1185     uint32_t oldCullMode = pState->rastState.cullMode;
1186     if (topology == TOP_POINT_LIST)
1187     {
1188         pState->rastState.cullMode = SWR_CULLMODE_NONE;
1189         pState->forceFront = true;
1190     }
1191
1192     while (remainingIndices)
1193     {
1194         uint32_t numIndicesForDraw = (remainingIndices < maxIndicesPerDraw) ?
1195         remainingIndices : maxIndicesPerDraw;
1196
1197         // When breaking up draw, we need to obtain new draw context for each iteration.
1198         bool isSplitDraw = (draw > 0) ? true : false;
1199         pDC = GetDrawContext(pContext, isSplitDraw);
1200         InitDraw(pDC, isSplitDraw);
1201
1202         pDC->FeWork.type = DRAW;
1203         pDC->FeWork.pfnWork = GetProcessDrawFunc(
1204             true,   // IsIndexed
1205             pState->tsState.tsEnable,
1206             pState->gsState.gsEnable,
1207             pState->soState.soEnable,
1208             pDC->pState->pfnProcessPrims != nullptr);
1209         pDC->FeWork.desc.draw.pDC = pDC;
1210         pDC->FeWork.desc.draw.numIndices = numIndicesForDraw;
1211         pDC->FeWork.desc.draw.pIB = (int*)pIB;
1212         pDC->FeWork.desc.draw.type = pDC->pState->state.indexBuffer.format;
1213
1214         pDC->FeWork.desc.draw.numInstances = numInstances;
1215         pDC->FeWork.desc.draw.startInstance = startInstance;
1216         pDC->FeWork.desc.draw.baseVertex = baseVertex;
1217         pDC->FeWork.desc.draw.startPrimID = draw * primsPerDraw;
1218
1219         pDC->cleanupState = (remainingIndices == numIndicesForDraw);
1220
1221         //enqueue DC
1222         QueueDraw(pContext);
1223
1224         pIB += maxIndicesPerDraw * indexSize;
1225         remainingIndices -= numIndicesForDraw;
1226         draw++;
1227     }
1228
1229     // restore culling state
1230     pDC = GetDrawContext(pContext);
1231     pDC->pState->state.rastState.cullMode = oldCullMode;
1232
1233     RDTSC_STOP(APIDrawIndexed, numIndices * numInstances, 0);
1234 }
1235
1236
1237 //////////////////////////////////////////////////////////////////////////
1238 /// @brief DrawIndexed
1239 /// @param hContext - Handle passed back from SwrCreateContext
1240 /// @param topology - Specifies topology for draw.
1241 /// @param numIndices - Number of indices to read sequentially from index buffer.
1242 /// @param indexOffset - Starting index into index buffer.
1243 /// @param baseVertex - Vertex in vertex buffer to consider as index "0". Note value is signed.
1244 void SwrDrawIndexed(
1245     HANDLE hContext,
1246     PRIMITIVE_TOPOLOGY topology,
1247     uint32_t numIndices,
1248     uint32_t indexOffset,
1249     int32_t baseVertex
1250     )
1251 {
1252     DrawIndexedInstance(hContext, topology, numIndices, indexOffset, baseVertex);
1253 }
1254
1255 //////////////////////////////////////////////////////////////////////////
1256 /// @brief SwrDrawIndexedInstanced
1257 /// @param hContext - Handle passed back from SwrCreateContext
1258 /// @param topology - Specifies topology for draw.
1259 /// @param numIndices - Number of indices to read sequentially from index buffer.
1260 /// @param numInstances - Number of instances to render.
1261 /// @param indexOffset - Starting index into index buffer.
1262 /// @param baseVertex - Vertex in vertex buffer to consider as index "0". Note value is signed.
1263 /// @param startInstance - Which instance to start sequentially fetching from in each buffer (instanced data)
1264 void SwrDrawIndexedInstanced(
1265     HANDLE hContext,
1266     PRIMITIVE_TOPOLOGY topology,
1267     uint32_t numIndices,
1268     uint32_t numInstances,
1269     uint32_t indexOffset,
1270     int32_t baseVertex,
1271     uint32_t startInstance)
1272 {
1273     DrawIndexedInstance(hContext, topology, numIndices, indexOffset, baseVertex, numInstances, startInstance);
1274 }
1275
1276 //////////////////////////////////////////////////////////////////////////
1277 /// @brief SwrInvalidateTiles
1278 /// @param hContext - Handle passed back from SwrCreateContext
1279 /// @param attachmentMask - The mask specifies which surfaces attached to the hottiles to invalidate.
1280 void SwrInvalidateTiles(
1281     HANDLE hContext,
1282     uint32_t attachmentMask)
1283 {
1284     if (KNOB_TOSS_DRAW)
1285     {
1286         return;
1287     }
1288
1289     SWR_CONTEXT *pContext = (SWR_CONTEXT*)hContext;
1290     DRAW_CONTEXT* pDC = GetDrawContext(pContext);
1291
1292     pDC->FeWork.type = DISCARDINVALIDATETILES;
1293     pDC->FeWork.pfnWork = ProcessDiscardInvalidateTiles;
1294     pDC->FeWork.desc.discardInvalidateTiles.attachmentMask = attachmentMask;
1295     memset(&pDC->FeWork.desc.discardInvalidateTiles.rect, 0, sizeof(SWR_RECT));
1296     pDC->FeWork.desc.discardInvalidateTiles.newTileState = SWR_TILE_INVALID;
1297     pDC->FeWork.desc.discardInvalidateTiles.createNewTiles = false;
1298     pDC->FeWork.desc.discardInvalidateTiles.fullTilesOnly = false;
1299
1300     //enqueue
1301     QueueDraw(pContext);
1302 }
1303
1304 //////////////////////////////////////////////////////////////////////////
1305 /// @brief SwrDiscardRect
1306 /// @param hContext - Handle passed back from SwrCreateContext
1307 /// @param attachmentMask - The mask specifies which surfaces attached to the hottiles to discard.
1308 /// @param rect - if rect is all zeros, the entire attachment surface will be discarded
1309 void SwrDiscardRect(
1310     HANDLE hContext,
1311     uint32_t attachmentMask,
1312     SWR_RECT rect)
1313 {
1314     if (KNOB_TOSS_DRAW)
1315     {
1316         return;
1317     }
1318
1319     SWR_CONTEXT *pContext = (SWR_CONTEXT*)hContext;
1320     DRAW_CONTEXT* pDC = GetDrawContext(pContext);
1321
1322     // Queue a load to the hottile
1323     pDC->FeWork.type = DISCARDINVALIDATETILES;
1324     pDC->FeWork.pfnWork = ProcessDiscardInvalidateTiles;
1325     pDC->FeWork.desc.discardInvalidateTiles.attachmentMask = attachmentMask;
1326     pDC->FeWork.desc.discardInvalidateTiles.rect = rect;
1327     pDC->FeWork.desc.discardInvalidateTiles.newTileState = SWR_TILE_RESOLVED;
1328     pDC->FeWork.desc.discardInvalidateTiles.createNewTiles = true;
1329     pDC->FeWork.desc.discardInvalidateTiles.fullTilesOnly = true;
1330
1331     //enqueue
1332     QueueDraw(pContext);
1333 }
1334
1335 //////////////////////////////////////////////////////////////////////////
1336 /// @brief SwrDispatch
1337 /// @param hContext - Handle passed back from SwrCreateContext
1338 /// @param threadGroupCountX - Number of thread groups dispatched in X direction
1339 /// @param threadGroupCountY - Number of thread groups dispatched in Y direction
1340 /// @param threadGroupCountZ - Number of thread groups dispatched in Z direction
1341 void SwrDispatch(
1342     HANDLE hContext,
1343     uint32_t threadGroupCountX,
1344     uint32_t threadGroupCountY,
1345     uint32_t threadGroupCountZ)
1346 {
1347     if (KNOB_TOSS_DRAW)
1348     {
1349         return;
1350     }
1351
1352     RDTSC_START(APIDispatch);
1353     SWR_CONTEXT *pContext = (SWR_CONTEXT*)hContext;
1354     DRAW_CONTEXT* pDC = GetDrawContext(pContext);
1355
1356     pDC->isCompute = true;      // This is a compute context.
1357
1358     COMPUTE_DESC* pTaskData = (COMPUTE_DESC*)pDC->pArena->AllocAligned(sizeof(COMPUTE_DESC), 64);
1359
1360     pTaskData->threadGroupCountX = threadGroupCountX;
1361     pTaskData->threadGroupCountY = threadGroupCountY;
1362     pTaskData->threadGroupCountZ = threadGroupCountZ;
1363
1364     uint32_t totalThreadGroups = threadGroupCountX * threadGroupCountY * threadGroupCountZ;
1365     uint32_t dcIndex = pDC->drawId % KNOB_MAX_DRAWS_IN_FLIGHT;
1366     pDC->pDispatch = &pContext->pDispatchQueueArray[dcIndex];
1367     pDC->pDispatch->initialize(totalThreadGroups, pTaskData);
1368
1369     QueueDispatch(pContext);
1370     RDTSC_STOP(APIDispatch, threadGroupCountX * threadGroupCountY * threadGroupCountZ, 0);
1371 }
1372
1373 // Deswizzles, converts and stores current contents of the hot tiles to surface
1374 // described by pState
1375 void SwrStoreTiles(
1376     HANDLE hContext,
1377     SWR_RENDERTARGET_ATTACHMENT attachment,
1378     SWR_TILE_STATE postStoreTileState)
1379 {
1380     if (KNOB_TOSS_DRAW)
1381     {
1382         return;
1383     }
1384
1385     RDTSC_START(APIStoreTiles);
1386
1387     SWR_CONTEXT *pContext = (SWR_CONTEXT*)hContext;
1388     DRAW_CONTEXT* pDC = GetDrawContext(pContext);
1389
1390     SetupMacroTileScissors(pDC);
1391
1392     pDC->FeWork.type = STORETILES;
1393     pDC->FeWork.pfnWork = ProcessStoreTiles;
1394     pDC->FeWork.desc.storeTiles.attachment = attachment;
1395     pDC->FeWork.desc.storeTiles.postStoreTileState = postStoreTileState;
1396
1397     //enqueue
1398     QueueDraw(pContext);
1399
1400     RDTSC_STOP(APIStoreTiles, 0, 0);
1401 }
1402
1403 void SwrClearRenderTarget(
1404     HANDLE hContext,
1405     uint32_t clearMask,
1406     const float clearColor[4],
1407     float z,
1408     uint8_t stencil)
1409 {
1410     if (KNOB_TOSS_DRAW)
1411     {
1412         return;
1413     }
1414
1415     RDTSC_START(APIClearRenderTarget);
1416
1417     SWR_CONTEXT *pContext = (SWR_CONTEXT*)hContext;
1418
1419     DRAW_CONTEXT* pDC = GetDrawContext(pContext);
1420
1421     SetupMacroTileScissors(pDC);
1422
1423     CLEAR_FLAGS flags;
1424     flags.mask = clearMask;
1425
1426     pDC->FeWork.type = CLEAR;
1427     pDC->FeWork.pfnWork = ProcessClear;
1428     pDC->FeWork.desc.clear.flags = flags;
1429     pDC->FeWork.desc.clear.clearDepth = z;
1430     pDC->FeWork.desc.clear.clearRTColor[0] = clearColor[0];
1431     pDC->FeWork.desc.clear.clearRTColor[1] = clearColor[1];
1432     pDC->FeWork.desc.clear.clearRTColor[2] = clearColor[2];
1433     pDC->FeWork.desc.clear.clearRTColor[3] = clearColor[3];
1434     pDC->FeWork.desc.clear.clearStencil = stencil;
1435
1436     // enqueue draw
1437     QueueDraw(pContext);
1438
1439     RDTSC_STOP(APIClearRenderTarget, 0, pDC->drawId);
1440 }
1441
1442 //////////////////////////////////////////////////////////////////////////
1443 /// @brief Returns a pointer to the private context state for the current
1444 ///        draw operation. This is used for external componets such as the
1445 ///        sampler.
1446 ///        SWR is responsible for the allocation of the private context state.
1447 /// @param hContext - Handle passed back from SwrCreateContext
1448 VOID* SwrGetPrivateContextState(
1449     HANDLE hContext)
1450 {
1451     SWR_CONTEXT* pContext = GetContext(hContext);
1452     DRAW_CONTEXT* pDC = GetDrawContext(pContext);
1453     DRAW_STATE* pState = pDC->pState;
1454
1455     if (pState->pPrivateState == nullptr)
1456     {
1457         pState->pPrivateState = pState->pArena->AllocAligned(pContext->privateStateSize, KNOB_SIMD_WIDTH*sizeof(float));
1458     }
1459
1460     return pState->pPrivateState;
1461 }
1462
1463 //////////////////////////////////////////////////////////////////////////
1464 /// @brief Clients can use this to allocate memory for draw/dispatch
1465 ///        operations. The memory will automatically be freed once operation
1466 ///        has completed. Client can use this to allocate binding tables,
1467 ///        etc. needed for shader execution.
1468 /// @param hContext - Handle passed back from SwrCreateContext
1469 /// @param size - Size of allocation
1470 /// @param align - Alignment needed for allocation.
1471 VOID* SwrAllocDrawContextMemory(
1472     HANDLE hContext,
1473     uint32_t size,
1474     uint32_t align)
1475 {
1476     SWR_CONTEXT* pContext = GetContext(hContext);
1477     DRAW_CONTEXT* pDC = GetDrawContext(pContext);
1478
1479     return pDC->pState->pArena->AllocAligned(size, align);
1480 }
1481
1482 //////////////////////////////////////////////////////////////////////////
1483 /// @brief Returns pointer to SWR stats.
1484 /// @note The counters are atomically incremented by multiple threads.
1485 ///       When calling this, you need to ensure all previous operations
1486 ///       have completed.
1487 /// @todo If necessary, add a callback to avoid stalling the pipe to
1488 ///       sample the counters.
1489 /// @param hContext - Handle passed back from SwrCreateContext
1490 /// @param pStats - SWR will fill this out for caller.
1491 void SwrGetStats(
1492     HANDLE hContext,
1493     SWR_STATS* pStats)
1494 {
1495     SWR_CONTEXT *pContext = GetContext(hContext);
1496     DRAW_CONTEXT* pDC = GetDrawContext(pContext);
1497
1498     pDC->FeWork.type = QUERYSTATS;
1499     pDC->FeWork.pfnWork = ProcessQueryStats;
1500     pDC->FeWork.desc.queryStats.pStats = pStats;
1501
1502     // cannot execute until all previous draws have completed
1503     pDC->dependency = pDC->drawId - 1;
1504
1505     //enqueue
1506     QueueDraw(pContext);
1507 }
1508
1509 //////////////////////////////////////////////////////////////////////////
1510 /// @brief Enables stats counting
1511 /// @param hContext - Handle passed back from SwrCreateContext
1512 /// @param enable - If true then counts are incremented.
1513 void SwrEnableStats(
1514     HANDLE hContext,
1515     bool enable)
1516 {
1517     SWR_CONTEXT *pContext = GetContext(hContext);
1518     DRAW_CONTEXT* pDC = GetDrawContext(pContext);
1519
1520     pDC->pState->state.enableStats = enable;
1521 }
1522
1523 //////////////////////////////////////////////////////////////////////////
1524 /// @brief Mark end of frame - used for performance profiling
1525 /// @param hContext - Handle passed back from SwrCreateContext
1526 void SWR_API SwrEndFrame(
1527     HANDLE hContext)
1528 {
1529     RDTSC_ENDFRAME();
1530     SWR_CONTEXT *pContext = GetContext(hContext);
1531     pContext->frameCount++;
1532 }