src/gallium/drivers/swr/rasterizer/core/api.cpp

   1 /****************************************************************************
   2 * Copyright (C) 2014-2016 Intel Corporation.   All Rights Reserved.
   3 *
   4 * Permission is hereby granted, free of charge, to any person obtaining a
   5 * copy of this software and associated documentation files (the "Software"),
   6 * to deal in the Software without restriction, including without limitation
   7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8 * and/or sell copies of the Software, and to permit persons to whom the
   9 * Software is furnished to do so, subject to the following conditions:
  10 *
  11 * The above copyright notice and this permission notice (including the next
  12 * paragraph) shall be included in all copies or substantial portions of the
  13 * Software.
  14 *
  15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21 * IN THE SOFTWARE.
  22 *
  23 * @file api.cpp
  24 *
  25 * @brief API implementation
  26 *
  27 ******************************************************************************/
  28
  29 #include <cfloat>
  30 #include <cmath>
  31 #include <cstdio>
  32 #include <new>
  33
  34 #include "core/api.h"
  35 #include "core/backend.h"
  36 #include "core/context.h"
  37 #include "core/depthstencil.h"
  38 #include "core/frontend.h"
  39 #include "core/rasterizer.h"
  40 #include "core/rdtsc_core.h"
  41 #include "core/threads.h"
  42 #include "core/tilemgr.h"
  43 #include "core/clip.h"
  44 #include "core/utils.h"
  45
  46 #include "common/simdintrin.h"
  47 #include "common/os.h"
  48
  49 static const SWR_RECT g_MaxScissorRect = { 0, 0, KNOB_MAX_SCISSOR_X, KNOB_MAX_SCISSOR_Y };
  50
  51 void SetupDefaultState(SWR_CONTEXT *pContext);
  52
  53 static INLINE SWR_CONTEXT* GetContext(HANDLE hContext)
  54 {
  55     return (SWR_CONTEXT*)hContext;
  56 }
  57
  58 //////////////////////////////////////////////////////////////////////////
  59 /// @brief Create SWR Context.
  60 /// @param pCreateInfo - pointer to creation info.
  61 HANDLE SwrCreateContext(
  62     SWR_CREATECONTEXT_INFO* pCreateInfo)
  63 {
  64     RDTSC_RESET();
  65     RDTSC_INIT(0);
  66
  67     void* pContextMem = AlignedMalloc(sizeof(SWR_CONTEXT), KNOB_SIMD_WIDTH * 4);
  68     memset(pContextMem, 0, sizeof(SWR_CONTEXT));
  69     SWR_CONTEXT *pContext = new (pContextMem) SWR_CONTEXT();
  70
  71     pContext->driverType = pCreateInfo->driver;
  72     pContext->privateStateSize = pCreateInfo->privateStateSize;
  73
  74     pContext->dcRing.Init(KNOB_MAX_DRAWS_IN_FLIGHT);
  75     pContext->dsRing.Init(KNOB_MAX_DRAWS_IN_FLIGHT);
  76
  77     pContext->pMacroTileManagerArray = (MacroTileMgr*)AlignedMalloc(sizeof(MacroTileMgr) * KNOB_MAX_DRAWS_IN_FLIGHT, 64);
  78     pContext->pDispatchQueueArray = (DispatchQueue*)AlignedMalloc(sizeof(DispatchQueue) * KNOB_MAX_DRAWS_IN_FLIGHT, 64);
  79
  80     for (uint32_t dc = 0; dc < KNOB_MAX_DRAWS_IN_FLIGHT; ++dc)
  81     {
  82         pContext->dcRing[dc].pArena = new CachingArena(pContext->cachingArenaAllocator);
  83         new (&pContext->pMacroTileManagerArray[dc]) MacroTileMgr(*pContext->dcRing[dc].pArena);
  84         new (&pContext->pDispatchQueueArray[dc]) DispatchQueue();
  85
  86         pContext->dsRing[dc].pArena = new CachingArena(pContext->cachingArenaAllocator);
  87     }
  88
  89     pContext->threadInfo.MAX_WORKER_THREADS        = KNOB_MAX_WORKER_THREADS;
  90     pContext->threadInfo.MAX_NUMA_NODES            = KNOB_MAX_NUMA_NODES;
  91     pContext->threadInfo.MAX_CORES_PER_NUMA_NODE   = KNOB_MAX_CORES_PER_NUMA_NODE;
  92     pContext->threadInfo.MAX_THREADS_PER_CORE      = KNOB_MAX_THREADS_PER_CORE;
  93     pContext->threadInfo.SINGLE_THREADED           = KNOB_SINGLE_THREADED;
  94
  95     if (pCreateInfo->pThreadInfo)
  96     {
  97         pContext->threadInfo = *pCreateInfo->pThreadInfo;
  98     }
  99
 100     memset(&pContext->WaitLock, 0, sizeof(pContext->WaitLock));
 101     memset(&pContext->FifosNotEmpty, 0, sizeof(pContext->FifosNotEmpty));
 102     new (&pContext->WaitLock) std::mutex();
 103     new (&pContext->FifosNotEmpty) std::condition_variable();
 104
 105     CreateThreadPool(pContext, &pContext->threadPool);
 106
 107     // Calling createThreadPool() above can set SINGLE_THREADED
 108     if (pContext->threadInfo.SINGLE_THREADED)
 109     {
 110         pContext->NumWorkerThreads = 1;
 111         pContext->NumFEThreads = 1;
 112         pContext->NumBEThreads = 1;
 113     }
 114
 115     pContext->ppScratch = new uint8_t*[pContext->NumWorkerThreads];
 116     pContext->pStats = new SWR_STATS[pContext->NumWorkerThreads];
 117
 118     // Allocate scratch space for workers.
 119     ///@note We could lazily allocate this but its rather small amount of memory.
 120     for (uint32_t i = 0; i < pContext->NumWorkerThreads; ++i)
 121     {
 122 #if defined(_WIN32)
 123         uint32_t numaNode = pContext->threadPool.pThreadData ?
 124             pContext->threadPool.pThreadData[i].numaId : 0;
 125         pContext->ppScratch[i] = (uint8_t*)VirtualAllocExNuma(
 126             GetCurrentProcess(), nullptr, 32 * sizeof(KILOBYTE),
 127             MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE,
 128             numaNode);
 129 #else
 130         pContext->ppScratch[i] = (uint8_t*)AlignedMalloc(32 * sizeof(KILOBYTE), KNOB_SIMD_WIDTH * 4);
 131 #endif
 132     }
 133
 134     // State setup AFTER context is fully initialized
 135     SetupDefaultState(pContext);
 136
 137     // initialize hot tile manager
 138     pContext->pHotTileMgr = new HotTileMgr();
 139
 140     // initialize function pointer tables
 141     InitClearTilesTable();
 142
 143     // initialize callback functions
 144     pContext->pfnLoadTile = pCreateInfo->pfnLoadTile;
 145     pContext->pfnStoreTile = pCreateInfo->pfnStoreTile;
 146     pContext->pfnClearTile = pCreateInfo->pfnClearTile;
 147     pContext->pfnUpdateSoWriteOffset = pCreateInfo->pfnUpdateSoWriteOffset;
 148     pContext->pfnUpdateStats = pCreateInfo->pfnUpdateStats;
 149     pContext->pfnUpdateStatsFE = pCreateInfo->pfnUpdateStatsFE;
 150
 151     // pass pointer to bucket manager back to caller
 152 #ifdef KNOB_ENABLE_RDTSC
 153     pCreateInfo->pBucketMgr = &gBucketMgr;
 154 #endif
 155
 156     pCreateInfo->contextSaveSize = sizeof(API_STATE);
 157
 158     return (HANDLE)pContext;
 159 }
 160
 161 void SwrDestroyContext(HANDLE hContext)
 162 {
 163     SWR_CONTEXT *pContext = GetContext(hContext);
 164     DestroyThreadPool(pContext, &pContext->threadPool);
 165
 166     // free the fifos
 167     for (uint32_t i = 0; i < KNOB_MAX_DRAWS_IN_FLIGHT; ++i)
 168     {
 169         delete [] pContext->dcRing[i].dynState.pStats;
 170         delete pContext->dcRing[i].pArena;
 171         delete pContext->dsRing[i].pArena;
 172         pContext->pMacroTileManagerArray[i].~MacroTileMgr();
 173         pContext->pDispatchQueueArray[i].~DispatchQueue();
 174     }
 175
 176     AlignedFree(pContext->pDispatchQueueArray);
 177     AlignedFree(pContext->pMacroTileManagerArray);
 178
 179     // Free scratch space.
 180     for (uint32_t i = 0; i < pContext->NumWorkerThreads; ++i)
 181     {
 182 #if defined(_WIN32)
 183         VirtualFree(pContext->ppScratch[i], 0, MEM_RELEASE);
 184 #else
 185         AlignedFree(pContext->ppScratch[i]);
 186 #endif
 187     }
 188
 189     delete [] pContext->ppScratch;
 190     delete [] pContext->pStats;
 191
 192     delete(pContext->pHotTileMgr);
 193
 194     pContext->~SWR_CONTEXT();
 195     AlignedFree(GetContext(hContext));
 196 }
 197
 198 void CopyState(DRAW_STATE& dst, const DRAW_STATE& src)
 199 {
 200     memcpy(&dst.state, &src.state, sizeof(API_STATE));
 201 }
 202
 203 void WakeAllThreads(SWR_CONTEXT *pContext)
 204 {
 205     pContext->FifosNotEmpty.notify_all();
 206 }
 207
 208 template<bool IsDraw>
 209 void QueueWork(SWR_CONTEXT *pContext)
 210 {
 211     DRAW_CONTEXT* pDC = pContext->pCurDrawContext;
 212     uint32_t dcIndex = pDC->drawId % KNOB_MAX_DRAWS_IN_FLIGHT;
 213
 214     if (IsDraw)
 215     {
 216         pDC->pTileMgr = &pContext->pMacroTileManagerArray[dcIndex];
 217         pDC->pTileMgr->initialize();
 218     }
 219
 220     // Each worker thread looks at a DC for both FE and BE work at different times and so we
 221     // multiply threadDone by 2.  When the threadDone counter has reached 0 then all workers
 222     // have moved past this DC. (i.e. Each worker has checked this DC for both FE and BE work and
 223     // then moved on if all work is done.)
 224     pContext->pCurDrawContext->threadsDone = pContext->NumFEThreads + pContext->NumBEThreads;
 225
 226     if (IsDraw)
 227     {
 228         InterlockedIncrement((volatile LONG*)&pContext->drawsOutstandingFE);
 229     }
 230
 231     _ReadWriteBarrier();
 232     {
 233         std::unique_lock<std::mutex> lock(pContext->WaitLock);
 234         pContext->dcRing.Enqueue();
 235     }
 236
 237     if (pContext->threadInfo.SINGLE_THREADED)
 238     {
 239         // flush denormals to 0
 240         uint32_t mxcsr = _mm_getcsr();
 241         _mm_setcsr(mxcsr | _MM_FLUSH_ZERO_ON | _MM_DENORMALS_ZERO_ON);
 242
 243         if (IsDraw)
 244         {
 245             uint32_t curDraw[2] = { pContext->pCurDrawContext->drawId, pContext->pCurDrawContext->drawId };
 246             WorkOnFifoFE(pContext, 0, curDraw[0]);
 247             WorkOnFifoBE(pContext, 0, curDraw[1], pContext->singleThreadLockedTiles, 0, 0);
 248         }
 249         else
 250         {
 251             uint32_t curDispatch = pContext->pCurDrawContext->drawId;
 252             WorkOnCompute(pContext, 0, curDispatch);
 253         }
 254
 255         // Dequeue the work here, if not already done, since we're single threaded (i.e. no workers).
 256         while (CompleteDrawContext(pContext, pContext->pCurDrawContext) > 0) {}
 257
 258         // restore csr
 259         _mm_setcsr(mxcsr);
 260     }
 261     else
 262     {
 263         RDTSC_START(APIDrawWakeAllThreads);
 264         WakeAllThreads(pContext);
 265         RDTSC_STOP(APIDrawWakeAllThreads, 1, 0);
 266     }
 267
 268     // Set current draw context to NULL so that next state call forces a new draw context to be created and populated.
 269     pContext->pPrevDrawContext = pContext->pCurDrawContext;
 270     pContext->pCurDrawContext = nullptr;
 271 }
 272
 273 INLINE void QueueDraw(SWR_CONTEXT* pContext)
 274 {
 275     QueueWork<true>(pContext);
 276 }
 277
 278 INLINE void QueueDispatch(SWR_CONTEXT* pContext)
 279 {
 280     QueueWork<false>(pContext);
 281 }
 282
 283 DRAW_CONTEXT* GetDrawContext(SWR_CONTEXT *pContext, bool isSplitDraw = false)
 284 {
 285     RDTSC_START(APIGetDrawContext);
 286     // If current draw context is null then need to obtain a new draw context to use from ring.
 287     if (pContext->pCurDrawContext == nullptr)
 288     {
 289         // Need to wait for a free entry.
 290         while (pContext->dcRing.IsFull())
 291         {
 292             _mm_pause();
 293         }
 294
 295         uint64_t curDraw = pContext->dcRing.GetHead();
 296         uint32_t dcIndex = curDraw % KNOB_MAX_DRAWS_IN_FLIGHT;
 297
 298         if ((pContext->frameCount - pContext->lastFrameChecked) > 2 ||
 299             (curDraw - pContext->lastDrawChecked) > 0x10000)
 300         {
 301             // Take this opportunity to clean-up old arena allocations
 302             pContext->cachingArenaAllocator.FreeOldBlocks();
 303
 304             pContext->lastFrameChecked = pContext->frameCount;
 305             pContext->lastDrawChecked = curDraw;
 306         }
 307
 308         DRAW_CONTEXT* pCurDrawContext = &pContext->dcRing[dcIndex];
 309         pContext->pCurDrawContext = pCurDrawContext;
 310
 311         // Assign next available entry in DS ring to this DC.
 312         uint32_t dsIndex = pContext->curStateId % KNOB_MAX_DRAWS_IN_FLIGHT;
 313         pCurDrawContext->pState = &pContext->dsRing[dsIndex];
 314
 315         // Copy previous state to current state.
 316         if (pContext->pPrevDrawContext)
 317         {
 318             DRAW_CONTEXT* pPrevDrawContext = pContext->pPrevDrawContext;
 319
 320             // If we're splitting our draw then we can just use the same state from the previous
 321             // draw. In this case, we won't increment the DS ring index so the next non-split
 322             // draw can receive the state.
 323             if (isSplitDraw == false)
 324             {
 325                 CopyState(*pCurDrawContext->pState, *pPrevDrawContext->pState);
 326
 327                 // Should have been cleaned up previously
 328                 SWR_ASSERT(pCurDrawContext->pState->pArena->IsEmpty() == true);
 329
 330                 pCurDrawContext->pState->pPrivateState = nullptr;
 331
 332                 pContext->curStateId++;  // Progress state ring index forward.
 333             }
 334             else
 335             {
 336                 // If its a split draw then just copy the state pointer over
 337                 // since its the same draw.
 338                 pCurDrawContext->pState = pPrevDrawContext->pState;
 339                 SWR_ASSERT(pPrevDrawContext->cleanupState == false);
 340             }
 341         }
 342         else
 343         {
 344             SWR_ASSERT(pCurDrawContext->pState->pArena->IsEmpty() == true);
 345             pContext->curStateId++;  // Progress state ring index forward.
 346         }
 347
 348         SWR_ASSERT(pCurDrawContext->pArena->IsEmpty() == true);
 349
 350         pCurDrawContext->dependent = false;
 351         pCurDrawContext->pContext = pContext;
 352         pCurDrawContext->isCompute = false; // Dispatch has to set this to true.
 353
 354         pCurDrawContext->doneFE = false;
 355         pCurDrawContext->FeLock = 0;
 356         pCurDrawContext->threadsDone = 0;
 357         pCurDrawContext->retireCallback.pfnCallbackFunc = nullptr;
 358
 359         pCurDrawContext->dynState.Reset(pContext->threadPool.numThreads);
 360
 361         // Assign unique drawId for this DC
 362         pCurDrawContext->drawId = pContext->dcRing.GetHead();
 363
 364         pCurDrawContext->cleanupState = true;
 365     }
 366     else
 367     {
 368         SWR_ASSERT(isSplitDraw == false, "Split draw should only be used when obtaining a new DC");
 369     }
 370
 371     RDTSC_STOP(APIGetDrawContext, 0, 0);
 372     return pContext->pCurDrawContext;
 373 }
 374
 375 API_STATE* GetDrawState(SWR_CONTEXT *pContext)
 376 {
 377     DRAW_CONTEXT* pDC = GetDrawContext(pContext);
 378     SWR_ASSERT(pDC->pState != nullptr);
 379
 380     return &pDC->pState->state;
 381 }
 382
 383 void SWR_API SwrSaveState(
 384     HANDLE hContext,
 385     void* pOutputStateBlock,
 386     size_t memSize)
 387 {
 388     SWR_CONTEXT *pContext = GetContext(hContext);
 389     auto pSrc = GetDrawState(pContext);
 390     SWR_ASSERT(pOutputStateBlock && memSize >= sizeof(*pSrc));
 391
 392     memcpy(pOutputStateBlock, pSrc, sizeof(*pSrc));
 393 }
 394
 395 void SWR_API SwrRestoreState(
 396     HANDLE hContext,
 397     const void* pStateBlock,
 398     size_t memSize)
 399 {
 400     SWR_CONTEXT *pContext = GetContext(hContext);
 401     auto pDst = GetDrawState(pContext);
 402     SWR_ASSERT(pStateBlock && memSize >= sizeof(*pDst));
 403
 404     memcpy(pDst, pStateBlock, sizeof(*pDst));
 405 }
 406
 407 void SetupDefaultState(SWR_CONTEXT *pContext)
 408 {
 409     API_STATE* pState = GetDrawState(pContext);
 410
 411     pState->rastState.cullMode = SWR_CULLMODE_NONE;
 412     pState->rastState.frontWinding = SWR_FRONTWINDING_CCW;
 413 }
 414
 415 void SwrSync(HANDLE hContext, PFN_CALLBACK_FUNC pfnFunc, uint64_t userData, uint64_t userData2, uint64_t userData3)
 416 {
 417     RDTSC_START(APISync);
 418
 419     SWR_ASSERT(pfnFunc != nullptr);
 420
 421     SWR_CONTEXT *pContext = GetContext(hContext);
 422     DRAW_CONTEXT* pDC = GetDrawContext(pContext);
 423
 424     pDC->FeWork.type = SYNC;
 425     pDC->FeWork.pfnWork = ProcessSync;
 426
 427     // Setup callback function
 428     pDC->retireCallback.pfnCallbackFunc = pfnFunc;
 429     pDC->retireCallback.userData = userData;
 430     pDC->retireCallback.userData2 = userData2;
 431     pDC->retireCallback.userData3 = userData3;
 432
 433     //enqueue
 434     QueueDraw(pContext);
 435
 436     RDTSC_STOP(APISync, 1, 0);
 437 }
 438
 439 void SwrWaitForIdle(HANDLE hContext)
 440 {
 441     SWR_CONTEXT *pContext = GetContext(hContext);
 442
 443     RDTSC_START(APIWaitForIdle);
 444
 445     while (!pContext->dcRing.IsEmpty())
 446     {
 447         _mm_pause();
 448     }
 449
 450     RDTSC_STOP(APIWaitForIdle, 1, 0);
 451 }
 452
 453 void SwrWaitForIdleFE(HANDLE hContext)
 454 {
 455     SWR_CONTEXT *pContext = GetContext(hContext);
 456
 457     RDTSC_START(APIWaitForIdle);
 458
 459     while (pContext->drawsOutstandingFE > 0)
 460     {
 461         _mm_pause();
 462     }
 463
 464     RDTSC_STOP(APIWaitForIdle, 1, 0);
 465 }
 466
 467 void SwrSetVertexBuffers(
 468     HANDLE hContext,
 469     uint32_t numBuffers,
 470     const SWR_VERTEX_BUFFER_STATE* pVertexBuffers)
 471 {
 472     API_STATE* pState = GetDrawState(GetContext(hContext));
 473
 474     for (uint32_t i = 0; i < numBuffers; ++i)
 475     {
 476         const SWR_VERTEX_BUFFER_STATE *pVB = &pVertexBuffers[i];
 477         pState->vertexBuffers[pVB->index] = *pVB;
 478     }
 479 }
 480
 481 void SwrSetIndexBuffer(
 482     HANDLE hContext,
 483     const SWR_INDEX_BUFFER_STATE* pIndexBuffer)
 484 {
 485     API_STATE* pState = GetDrawState(GetContext(hContext));
 486
 487     pState->indexBuffer = *pIndexBuffer;
 488 }
 489
 490 void SwrSetFetchFunc(
 491     HANDLE hContext,
 492     PFN_FETCH_FUNC    pfnFetchFunc)
 493 {
 494     API_STATE* pState = GetDrawState(GetContext(hContext));
 495
 496     pState->pfnFetchFunc = pfnFetchFunc;
 497 }
 498
 499 void SwrSetSoFunc(
 500     HANDLE hContext,
 501     PFN_SO_FUNC    pfnSoFunc,
 502     uint32_t streamIndex)
 503 {
 504     API_STATE* pState = GetDrawState(GetContext(hContext));
 505
 506     SWR_ASSERT(streamIndex < MAX_SO_STREAMS);
 507
 508     pState->pfnSoFunc[streamIndex] = pfnSoFunc;
 509 }
 510
 511 void SwrSetSoState(
 512     HANDLE hContext,
 513     SWR_STREAMOUT_STATE* pSoState)
 514 {
 515     API_STATE* pState = GetDrawState(GetContext(hContext));
 516
 517     pState->soState = *pSoState;
 518 }
 519
 520 void SwrSetSoBuffers(
 521     HANDLE hContext,
 522     SWR_STREAMOUT_BUFFER* pSoBuffer,
 523     uint32_t slot)
 524 {
 525     API_STATE* pState = GetDrawState(GetContext(hContext));
 526
 527     SWR_ASSERT((slot < 4), "There are only 4 SO buffer slots [0, 3]\nSlot requested: %d", slot);
 528
 529     pState->soBuffer[slot] = *pSoBuffer;
 530 }
 531
 532 void SwrSetVertexFunc(
 533     HANDLE hContext,
 534     PFN_VERTEX_FUNC pfnVertexFunc)
 535 {
 536     API_STATE* pState = GetDrawState(GetContext(hContext));
 537
 538     pState->pfnVertexFunc = pfnVertexFunc;
 539 }
 540
 541 void SwrSetFrontendState(
 542     HANDLE hContext,
 543     SWR_FRONTEND_STATE *pFEState)
 544 {
 545     API_STATE* pState = GetDrawState(GetContext(hContext));
 546     pState->frontendState = *pFEState;
 547 }
 548
 549 void SwrSetGsState(
 550     HANDLE hContext,
 551     SWR_GS_STATE *pGSState)
 552 {
 553     API_STATE* pState = GetDrawState(GetContext(hContext));
 554     pState->gsState = *pGSState;
 555 }
 556
 557 void SwrSetGsFunc(
 558     HANDLE hContext,
 559     PFN_GS_FUNC pfnGsFunc)
 560 {
 561     API_STATE* pState = GetDrawState(GetContext(hContext));
 562     pState->pfnGsFunc = pfnGsFunc;
 563 }
 564
 565 void SwrSetCsFunc(
 566     HANDLE hContext,
 567     PFN_CS_FUNC pfnCsFunc,
 568     uint32_t totalThreadsInGroup,
 569     uint32_t totalSpillFillSize)
 570 {
 571     API_STATE* pState = GetDrawState(GetContext(hContext));
 572     pState->pfnCsFunc = pfnCsFunc;
 573     pState->totalThreadsInGroup = totalThreadsInGroup;
 574     pState->totalSpillFillSize = totalSpillFillSize;
 575 }
 576
 577 void SwrSetTsState(
 578     HANDLE hContext,
 579     SWR_TS_STATE *pState)
 580 {
 581     API_STATE* pApiState = GetDrawState(GetContext(hContext));
 582     pApiState->tsState = *pState;
 583 }
 584
 585 void SwrSetHsFunc(
 586     HANDLE hContext,
 587     PFN_HS_FUNC pfnFunc)
 588 {
 589     API_STATE* pApiState = GetDrawState(GetContext(hContext));
 590     pApiState->pfnHsFunc = pfnFunc;
 591 }
 592
 593 void SwrSetDsFunc(
 594     HANDLE hContext,
 595     PFN_DS_FUNC pfnFunc)
 596 {
 597     API_STATE* pApiState = GetDrawState(GetContext(hContext));
 598     pApiState->pfnDsFunc = pfnFunc;
 599 }
 600
 601 void SwrSetDepthStencilState(
 602     HANDLE hContext,
 603     SWR_DEPTH_STENCIL_STATE *pDSState)
 604 {
 605     API_STATE* pState = GetDrawState(GetContext(hContext));
 606
 607     pState->depthStencilState = *pDSState;
 608 }
 609
 610 void SwrSetBackendState(
 611     HANDLE hContext,
 612     SWR_BACKEND_STATE *pBEState)
 613 {
 614     API_STATE* pState = GetDrawState(GetContext(hContext));
 615
 616     pState->backendState = *pBEState;
 617 }
 618
 619 void SwrSetPixelShaderState(
 620     HANDLE hContext,
 621     SWR_PS_STATE *pPSState)
 622 {
 623     API_STATE *pState = GetDrawState(GetContext(hContext));
 624     pState->psState = *pPSState;
 625 }
 626
 627 void SwrSetBlendState(
 628     HANDLE hContext,
 629     SWR_BLEND_STATE *pBlendState)
 630 {
 631     API_STATE *pState = GetDrawState(GetContext(hContext));
 632     memcpy(&pState->blendState, pBlendState, sizeof(SWR_BLEND_STATE));
 633 }
 634
 635 void SwrSetBlendFunc(
 636     HANDLE hContext,
 637     uint32_t renderTarget,
 638     PFN_BLEND_JIT_FUNC pfnBlendFunc)
 639 {
 640     SWR_ASSERT(renderTarget < SWR_NUM_RENDERTARGETS);
 641     API_STATE *pState = GetDrawState(GetContext(hContext));
 642     pState->pfnBlendFunc[renderTarget] = pfnBlendFunc;
 643 }
 644
 645 // update guardband multipliers for the viewport
 646 void updateGuardbands(API_STATE *pState)
 647 {
 648     uint32_t numGbs = pState->gsState.emitsRenderTargetArrayIndex ? KNOB_NUM_VIEWPORTS_SCISSORS : 1;
 649
 650     for(uint32_t i = 0; i < numGbs; ++i)
 651     {
 652         // guardband center is viewport center
 653         pState->gbState.left[i] = KNOB_GUARDBAND_WIDTH / pState->vp[i].width;
 654         pState->gbState.right[i] = KNOB_GUARDBAND_WIDTH / pState->vp[i].width;
 655         pState->gbState.top[i] = KNOB_GUARDBAND_HEIGHT / pState->vp[i].height;
 656         pState->gbState.bottom[i] = KNOB_GUARDBAND_HEIGHT / pState->vp[i].height;
 657     }
 658 }
 659
 660 void SwrSetRastState(
 661     HANDLE hContext,
 662     const SWR_RASTSTATE *pRastState)
 663 {
 664     SWR_CONTEXT *pContext = GetContext(hContext);
 665     API_STATE* pState = GetDrawState(pContext);
 666
 667     memcpy(&pState->rastState, pRastState, sizeof(SWR_RASTSTATE));
 668 }
 669
 670 void SwrSetViewports(
 671     HANDLE hContext,
 672     uint32_t numViewports,
 673     const SWR_VIEWPORT* pViewports,
 674     const SWR_VIEWPORT_MATRICES* pMatrices)
 675 {
 676     SWR_ASSERT(numViewports <= KNOB_NUM_VIEWPORTS_SCISSORS,
 677         "Invalid number of viewports.");
 678
 679     SWR_CONTEXT *pContext = GetContext(hContext);
 680     API_STATE* pState = GetDrawState(pContext);
 681
 682     memcpy(&pState->vp[0], pViewports, sizeof(SWR_VIEWPORT) * numViewports);
 683
 684     if (pMatrices != nullptr)
 685     {
 686         // @todo Faster to copy portions of the SOA or just copy all of it?
 687         memcpy(&pState->vpMatrices, pMatrices, sizeof(SWR_VIEWPORT_MATRICES));
 688     }
 689     else
 690     {
 691         // Compute default viewport transform.
 692         for (uint32_t i = 0; i < numViewports; ++i)
 693         {
 694             if (pContext->driverType == DX)
 695             {
 696                 pState->vpMatrices.m00[i] = pState->vp[i].width / 2.0f;
 697                 pState->vpMatrices.m11[i] = -pState->vp[i].height / 2.0f;
 698                 pState->vpMatrices.m22[i] = pState->vp[i].maxZ - pState->vp[i].minZ;
 699                 pState->vpMatrices.m30[i] = pState->vp[i].x + pState->vpMatrices.m00[i];
 700                 pState->vpMatrices.m31[i] = pState->vp[i].y - pState->vpMatrices.m11[i];
 701                 pState->vpMatrices.m32[i] = pState->vp[i].minZ;
 702             }
 703             else
 704             {
 705                 // Standard, with the exception that Y is inverted.
 706                 pState->vpMatrices.m00[i] = (pState->vp[i].width - pState->vp[i].x) / 2.0f;
 707                 pState->vpMatrices.m11[i] = (pState->vp[i].y - pState->vp[i].height) / 2.0f;
 708                 pState->vpMatrices.m22[i] = (pState->vp[i].maxZ - pState->vp[i].minZ) / 2.0f;
 709                 pState->vpMatrices.m30[i] = pState->vp[i].x + pState->vpMatrices.m00[i];
 710                 pState->vpMatrices.m31[i] = pState->vp[i].height + pState->vpMatrices.m11[i];
 711                 pState->vpMatrices.m32[i] = pState->vp[i].minZ + pState->vpMatrices.m22[i];
 712
 713                 // Now that the matrix is calculated, clip the view coords to screen size.
 714                 // OpenGL allows for -ve x,y in the viewport.
 715                 pState->vp[i].x = std::max(pState->vp[i].x, 0.0f);
 716                 pState->vp[i].y = std::max(pState->vp[i].y, 0.0f);
 717             }
 718         }
 719     }
 720
 721     updateGuardbands(pState);
 722 }
 723
 724 void SwrSetScissorRects(
 725     HANDLE hContext,
 726     uint32_t numScissors,
 727     const SWR_RECT* pScissors)
 728 {
 729     SWR_ASSERT(numScissors <= KNOB_NUM_VIEWPORTS_SCISSORS,
 730         "Invalid number of scissor rects.");
 731
 732     API_STATE* pState = GetDrawState(GetContext(hContext));
 733     memcpy(&pState->scissorRects[0], pScissors, numScissors * sizeof(pScissors[0]));
 734 };
 735
 736 void SetupMacroTileScissors(DRAW_CONTEXT *pDC)
 737 {
 738     API_STATE *pState = &pDC->pState->state;
 739     uint32_t numScissors = pState->gsState.emitsViewportArrayIndex ? KNOB_NUM_VIEWPORTS_SCISSORS : 1;
 740     pState->scissorsTileAligned = true;
 741
 742     for (uint32_t index = 0; index < numScissors; ++index)
 743     {
 744         SWR_RECT &scissorInFixedPoint = pState->scissorsInFixedPoint[index];
 745
 746         // Set up scissor dimensions based on scissor or viewport
 747         if (pState->rastState.scissorEnable)
 748         {
 749             scissorInFixedPoint = pState->scissorRects[index];
 750         }
 751         else
 752         {
 753             // the vp width and height must be added to origin un-rounded then the result round to -inf.
 754             // The cast to int works for rounding assuming all [left, right, top, bottom] are positive.
 755             scissorInFixedPoint.xmin = (int32_t)pState->vp[index].x;
 756             scissorInFixedPoint.xmax = (int32_t)(pState->vp[index].x + pState->vp[index].width);
 757             scissorInFixedPoint.ymin = (int32_t)pState->vp[index].y;
 758             scissorInFixedPoint.ymax = (int32_t)(pState->vp[index].y + pState->vp[index].height);
 759         }
 760
 761         // Clamp to max rect
 762         scissorInFixedPoint &= g_MaxScissorRect;
 763
 764         // Test for tile alignment
 765         bool tileAligned;
 766         tileAligned  = (scissorInFixedPoint.xmin % KNOB_TILE_X_DIM) == 0;
 767         tileAligned &= (scissorInFixedPoint.ymin % KNOB_TILE_Y_DIM) == 0;
 768         tileAligned &= (scissorInFixedPoint.xmax % KNOB_TILE_X_DIM) == 0;
 769         tileAligned &= (scissorInFixedPoint.xmax % KNOB_TILE_Y_DIM) == 0;
 770
 771         pState->scissorsTileAligned &= tileAligned;
 772
 773         // Scale to fixed point
 774         scissorInFixedPoint.xmin *= FIXED_POINT_SCALE;
 775         scissorInFixedPoint.xmax *= FIXED_POINT_SCALE;
 776         scissorInFixedPoint.ymin *= FIXED_POINT_SCALE;
 777         scissorInFixedPoint.ymax *= FIXED_POINT_SCALE;
 778
 779         // Make scissor inclusive
 780         scissorInFixedPoint.xmax -= 1;
 781         scissorInFixedPoint.ymax -= 1;
 782     }
 783
 784
 785 }
 786
 787 // templated backend function tables
 788 extern PFN_BACKEND_FUNC gBackendNullPs[SWR_MULTISAMPLE_TYPE_COUNT];
 789 extern PFN_BACKEND_FUNC gBackendSingleSample[SWR_INPUT_COVERAGE_COUNT][2][2];
 790 extern PFN_BACKEND_FUNC gBackendPixelRateTable[SWR_MULTISAMPLE_TYPE_COUNT][SWR_MSAA_SAMPLE_PATTERN_COUNT][SWR_INPUT_COVERAGE_COUNT][2][2][2];
 791 extern PFN_BACKEND_FUNC gBackendSampleRateTable[SWR_MULTISAMPLE_TYPE_COUNT][SWR_INPUT_COVERAGE_COUNT][2][2];
 792 void SetupPipeline(DRAW_CONTEXT *pDC)
 793 {
 794     DRAW_STATE* pState = pDC->pState;
 795     const SWR_RASTSTATE &rastState = pState->state.rastState;
 796     const SWR_PS_STATE &psState = pState->state.psState;
 797     BACKEND_FUNCS& backendFuncs = pState->backendFuncs;
 798     const uint32_t forcedSampleCount = (rastState.forcedSampleCount) ? 1 : 0;
 799
 800     // setup backend
 801     if (psState.pfnPixelShader == nullptr)
 802     {
 803         backendFuncs.pfnBackend = gBackendNullPs[pState->state.rastState.sampleCount];
 804     }
 805     else
 806     {
 807         const bool bMultisampleEnable = ((rastState.sampleCount > SWR_MULTISAMPLE_1X) || rastState.forcedSampleCount) ? 1 : 0;
 808         const uint32_t centroid = ((psState.barycentricsMask & SWR_BARYCENTRIC_CENTROID_MASK) > 0) ? 1 : 0;
 809         const uint32_t canEarlyZ = (psState.forceEarlyZ || (!psState.writesODepth && !psState.usesSourceDepth && !psState.usesUAV)) ? 1 : 0;
 810
 811         SWR_BARYCENTRICS_MASK barycentricsMask = (SWR_BARYCENTRICS_MASK)psState.barycentricsMask;
 812
 813         // select backend function
 814         switch(psState.shadingRate)
 815         {
 816         case SWR_SHADING_RATE_PIXEL:
 817             if(bMultisampleEnable)
 818             {
 819                 // always need to generate I & J per sample for Z interpolation
 820                 barycentricsMask = (SWR_BARYCENTRICS_MASK)(barycentricsMask | SWR_BARYCENTRIC_PER_SAMPLE_MASK);
 821                 backendFuncs.pfnBackend = gBackendPixelRateTable[rastState.sampleCount][rastState.samplePattern][psState.inputCoverage][centroid][forcedSampleCount][canEarlyZ];
 822             }
 823             else
 824             {
 825                 // always need to generate I & J per pixel for Z interpolation
 826                 barycentricsMask = (SWR_BARYCENTRICS_MASK)(barycentricsMask | SWR_BARYCENTRIC_PER_PIXEL_MASK);
 827                 backendFuncs.pfnBackend = gBackendSingleSample[psState.inputCoverage][centroid][canEarlyZ];
 828             }
 829             break;
 830         case SWR_SHADING_RATE_SAMPLE:
 831             SWR_ASSERT(rastState.samplePattern == SWR_MSAA_STANDARD_PATTERN);
 832             // always need to generate I & J per sample for Z interpolation
 833             barycentricsMask = (SWR_BARYCENTRICS_MASK)(barycentricsMask | SWR_BARYCENTRIC_PER_SAMPLE_MASK);
 834             backendFuncs.pfnBackend = gBackendSampleRateTable[rastState.sampleCount][psState.inputCoverage][centroid][canEarlyZ];
 835             break;
 836         default:
 837             SWR_ASSERT(0 && "Invalid shading rate");
 838             break;
 839         }
 840     }
 841
 842     PFN_PROCESS_PRIMS pfnBinner;
 843     switch (pState->state.topology)
 844     {
 845     case TOP_POINT_LIST:
 846         pState->pfnProcessPrims = ClipPoints;
 847         pfnBinner = BinPoints;
 848         break;
 849     case TOP_LINE_LIST:
 850     case TOP_LINE_STRIP:
 851     case TOP_LINE_LOOP:
 852     case TOP_LINE_LIST_ADJ:
 853     case TOP_LISTSTRIP_ADJ:
 854         pState->pfnProcessPrims = ClipLines;
 855         pfnBinner = BinLines;
 856         break;
 857     default:
 858         pState->pfnProcessPrims = ClipTriangles;
 859         pfnBinner = GetBinTrianglesFunc((rastState.conservativeRast > 0));
 860         break;
 861     };
 862
 863     // disable clipper if viewport transform is disabled
 864     if (pState->state.frontendState.vpTransformDisable)
 865     {
 866         pState->pfnProcessPrims = pfnBinner;
 867     }
 868
 869     if ((pState->state.psState.pfnPixelShader == nullptr) &&
 870         (pState->state.depthStencilState.depthTestEnable == FALSE) &&
 871         (pState->state.depthStencilState.depthWriteEnable == FALSE) &&
 872         (pState->state.depthStencilState.stencilTestEnable == FALSE) &&
 873         (pState->state.depthStencilState.stencilWriteEnable == FALSE) &&
 874         (pState->state.backendState.numAttributes == 0))
 875     {
 876         pState->pfnProcessPrims = nullptr;
 877     }
 878
 879     if (pState->state.soState.rasterizerDisable == true)
 880     {
 881         pState->pfnProcessPrims = nullptr;
 882     }
 883
 884     // set up the frontend attribute count
 885     pState->state.feNumAttributes = 0;
 886     const SWR_BACKEND_STATE& backendState = pState->state.backendState;
 887     if (backendState.swizzleEnable)
 888     {
 889         // attribute swizzling is enabled, iterate over the map and record the max attribute used
 890         for (uint32_t i = 0; i < backendState.numAttributes; ++i)
 891         {
 892             pState->state.feNumAttributes = std::max(pState->state.feNumAttributes, (uint32_t)backendState.swizzleMap[i].sourceAttrib + 1);
 893         }
 894     }
 895     else
 896     {
 897         pState->state.feNumAttributes = pState->state.backendState.numAttributes;
 898     }
 899
 900     if (pState->state.soState.soEnable)
 901     {
 902         uint32_t streamMasks = 0;
 903         for (uint32_t i = 0; i < 4; ++i)
 904         {
 905             streamMasks |= pState->state.soState.streamMasks[i];
 906         }
 907
 908         DWORD maxAttrib;
 909         if (_BitScanReverse(&maxAttrib, streamMasks))
 910         {
 911             pState->state.feNumAttributes = std::max(pState->state.feNumAttributes, (uint32_t)(maxAttrib + 1));
 912         }
 913     }
 914
 915     // complicated logic to test for cases where we don't need backing hottile memory for a draw
 916     // have to check for the special case where depth/stencil test is enabled but depthwrite is disabled.
 917     pState->state.depthHottileEnable = ((!(pState->state.depthStencilState.depthTestEnable &&
 918                                            !pState->state.depthStencilState.depthWriteEnable &&
 919                                            pState->state.depthStencilState.depthTestFunc == ZFUNC_ALWAYS)) &&
 920                                         (pState->state.depthStencilState.depthTestEnable ||
 921                                          pState->state.depthStencilState.depthWriteEnable)) ? true : false;
 922
 923     pState->state.stencilHottileEnable = (((!(pState->state.depthStencilState.stencilTestEnable &&
 924                                              !pState->state.depthStencilState.stencilWriteEnable &&
 925                                               pState->state.depthStencilState.stencilTestFunc == ZFUNC_ALWAYS)) ||
 926                                           // for stencil we have to check the double sided state as well
 927                                           (!(pState->state.depthStencilState.doubleSidedStencilTestEnable &&
 928                                              !pState->state.depthStencilState.stencilWriteEnable &&
 929                                               pState->state.depthStencilState.backfaceStencilTestFunc == ZFUNC_ALWAYS))) &&
 930                                           (pState->state.depthStencilState.stencilTestEnable  ||
 931                                            pState->state.depthStencilState.stencilWriteEnable)) ? true : false;
 932
 933     uint32_t numRTs = pState->state.psState.numRenderTargets;
 934     pState->state.colorHottileEnable = 0;
 935     if (psState.pfnPixelShader != nullptr)
 936     {
 937         for (uint32_t rt = 0; rt < numRTs; ++rt)
 938         {
 939             pState->state.colorHottileEnable |=
 940                 (!pState->state.blendState.renderTarget[rt].writeDisableAlpha ||
 941                  !pState->state.blendState.renderTarget[rt].writeDisableRed ||
 942                  !pState->state.blendState.renderTarget[rt].writeDisableGreen ||
 943                  !pState->state.blendState.renderTarget[rt].writeDisableBlue) ? (1 << rt) : 0;
 944         }
 945     }
 946
 947     // Setup depth quantization function
 948     if (pState->state.depthHottileEnable)
 949     {
 950         switch (pState->state.rastState.depthFormat)
 951         {
 952         case R32_FLOAT_X8X24_TYPELESS: pState->state.pfnQuantizeDepth = QuantizeDepth < R32_FLOAT_X8X24_TYPELESS > ; break;
 953         case R32_FLOAT: pState->state.pfnQuantizeDepth = QuantizeDepth < R32_FLOAT > ; break;
 954         case R24_UNORM_X8_TYPELESS: pState->state.pfnQuantizeDepth = QuantizeDepth < R24_UNORM_X8_TYPELESS > ; break;
 955         case R16_UNORM: pState->state.pfnQuantizeDepth = QuantizeDepth < R16_UNORM > ; break;
 956         default: SWR_ASSERT(false, "Unsupported depth format for depth quantiztion.");
 957             pState->state.pfnQuantizeDepth = QuantizeDepth < R32_FLOAT > ;
 958         }
 959     }
 960     else
 961     {
 962         // set up pass-through quantize if depth isn't enabled
 963         pState->state.pfnQuantizeDepth = QuantizeDepth < R32_FLOAT > ;
 964     }
 965 }
 966
 967 //////////////////////////////////////////////////////////////////////////
 968 /// @brief InitDraw
 969 /// @param pDC - Draw context to initialize for this draw.
 970 void InitDraw(
 971     DRAW_CONTEXT *pDC,
 972     bool isSplitDraw)
 973 {
 974     // We don't need to re-setup the scissors/pipeline state again for split draw.
 975     if (isSplitDraw == false)
 976     {
 977         SetupMacroTileScissors(pDC);
 978         SetupPipeline(pDC);
 979     }
 980 }
 981
 982 //////////////////////////////////////////////////////////////////////////
 983 /// @brief We can split the draw for certain topologies for better performance.
 984 /// @param totalVerts - Total vertices for draw
 985 /// @param topology - Topology used for draw
 986 uint32_t MaxVertsPerDraw(
 987     DRAW_CONTEXT* pDC,
 988     uint32_t totalVerts,
 989     PRIMITIVE_TOPOLOGY topology)
 990 {
 991     API_STATE& state = pDC->pState->state;
 992
 993     uint32_t vertsPerDraw = totalVerts;
 994
 995     if (state.soState.soEnable)
 996     {
 997         return totalVerts;
 998     }
 999
1000     switch (topology)
1001     {
1002     case TOP_POINT_LIST:
1003     case TOP_TRIANGLE_LIST:
1004         vertsPerDraw = KNOB_MAX_PRIMS_PER_DRAW;
1005         break;
1006
1007     case TOP_PATCHLIST_1:
1008     case TOP_PATCHLIST_2:
1009     case TOP_PATCHLIST_3:
1010     case TOP_PATCHLIST_4:
1011     case TOP_PATCHLIST_5:
1012     case TOP_PATCHLIST_6:
1013     case TOP_PATCHLIST_7:
1014     case TOP_PATCHLIST_8:
1015     case TOP_PATCHLIST_9:
1016     case TOP_PATCHLIST_10:
1017     case TOP_PATCHLIST_11:
1018     case TOP_PATCHLIST_12:
1019     case TOP_PATCHLIST_13:
1020     case TOP_PATCHLIST_14:
1021     case TOP_PATCHLIST_15:
1022     case TOP_PATCHLIST_16:
1023     case TOP_PATCHLIST_17:
1024     case TOP_PATCHLIST_18:
1025     case TOP_PATCHLIST_19:
1026     case TOP_PATCHLIST_20:
1027     case TOP_PATCHLIST_21:
1028     case TOP_PATCHLIST_22:
1029     case TOP_PATCHLIST_23:
1030     case TOP_PATCHLIST_24:
1031     case TOP_PATCHLIST_25:
1032     case TOP_PATCHLIST_26:
1033     case TOP_PATCHLIST_27:
1034     case TOP_PATCHLIST_28:
1035     case TOP_PATCHLIST_29:
1036     case TOP_PATCHLIST_30:
1037     case TOP_PATCHLIST_31:
1038     case TOP_PATCHLIST_32:
1039         if (pDC->pState->state.tsState.tsEnable)
1040         {
1041             uint32_t vertsPerPrim = topology - TOP_PATCHLIST_BASE;
1042             vertsPerDraw = vertsPerPrim * KNOB_MAX_TESS_PRIMS_PER_DRAW;
1043         }
1044         break;
1045
1046     // The Primitive Assembly code can only handle 1 RECT at a time.
1047     case TOP_RECT_LIST:
1048         vertsPerDraw = 3;
1049         break;
1050
1051     default:
1052         // We are not splitting up draws for other topologies.
1053         break;
1054     }
1055
1056     return vertsPerDraw;
1057 }
1058
1059
1060 //////////////////////////////////////////////////////////////////////////
1061 /// @brief DrawInstanced
1062 /// @param hContext - Handle passed back from SwrCreateContext
1063 /// @param topology - Specifies topology for draw.
1064 /// @param numVerts - How many vertices to read sequentially from vertex data (per instance).
1065 /// @param startVertex - Specifies start vertex for draw. (vertex data)
1066 /// @param numInstances - How many instances to render.
1067 /// @param startInstance - Which instance to start sequentially fetching from in each buffer (instanced data)
1068 void DrawInstanced(
1069     HANDLE hContext,
1070     PRIMITIVE_TOPOLOGY topology,
1071     uint32_t numVertices,
1072     uint32_t startVertex,
1073     uint32_t numInstances = 1,
1074     uint32_t startInstance = 0)
1075 {
1076     if (KNOB_TOSS_DRAW)
1077     {
1078         return;
1079     }
1080
1081     RDTSC_START(APIDraw);
1082
1083     SWR_CONTEXT *pContext = GetContext(hContext);
1084     DRAW_CONTEXT* pDC = GetDrawContext(pContext);
1085
1086     uint32_t maxVertsPerDraw = MaxVertsPerDraw(pDC, numVertices, topology);
1087     uint32_t primsPerDraw = GetNumPrims(topology, maxVertsPerDraw);
1088     uint32_t remainingVerts = numVertices;
1089
1090     API_STATE    *pState = &pDC->pState->state;
1091     pState->topology = topology;
1092     pState->forceFront = false;
1093
1094     // disable culling for points/lines
1095     uint32_t oldCullMode = pState->rastState.cullMode;
1096     if (topology == TOP_POINT_LIST)
1097     {
1098         pState->rastState.cullMode = SWR_CULLMODE_NONE;
1099         pState->forceFront = true;
1100     }
1101
1102     int draw = 0;
1103     while (remainingVerts)
1104     {
1105         uint32_t numVertsForDraw = (remainingVerts < maxVertsPerDraw) ?
1106         remainingVerts : maxVertsPerDraw;
1107
1108         bool isSplitDraw = (draw > 0) ? true : false;
1109         DRAW_CONTEXT* pDC = GetDrawContext(pContext, isSplitDraw);
1110         InitDraw(pDC, isSplitDraw);
1111
1112         pDC->FeWork.type = DRAW;
1113         pDC->FeWork.pfnWork = GetProcessDrawFunc(
1114             false,  // IsIndexed
1115             false, // bEnableCutIndex
1116             pState->tsState.tsEnable,
1117             pState->gsState.gsEnable,
1118             pState->soState.soEnable,
1119             pDC->pState->pfnProcessPrims != nullptr);
1120         pDC->FeWork.desc.draw.numVerts = numVertsForDraw;
1121         pDC->FeWork.desc.draw.startVertex = startVertex;
1122         pDC->FeWork.desc.draw.numInstances = numInstances;
1123         pDC->FeWork.desc.draw.startInstance = startInstance;
1124         pDC->FeWork.desc.draw.startPrimID = draw * primsPerDraw;
1125         pDC->FeWork.desc.draw.startVertexID = draw * maxVertsPerDraw;
1126
1127         pDC->cleanupState = (remainingVerts == numVertsForDraw);
1128
1129         //enqueue DC
1130         QueueDraw(pContext);
1131
1132         remainingVerts -= numVertsForDraw;
1133         draw++;
1134     }
1135
1136     // restore culling state
1137     pDC = GetDrawContext(pContext);
1138     pDC->pState->state.rastState.cullMode = oldCullMode;
1139
1140     RDTSC_STOP(APIDraw, numVertices * numInstances, 0);
1141 }
1142
1143 //////////////////////////////////////////////////////////////////////////
1144 /// @brief SwrDraw
1145 /// @param hContext - Handle passed back from SwrCreateContext
1146 /// @param topology - Specifies topology for draw.
1147 /// @param startVertex - Specifies start vertex in vertex buffer for draw.
1148 /// @param primCount - Number of vertices.
1149 void SwrDraw(
1150     HANDLE hContext,
1151     PRIMITIVE_TOPOLOGY topology,
1152     uint32_t startVertex,
1153     uint32_t numVertices)
1154 {
1155     DrawInstanced(hContext, topology, numVertices, startVertex);
1156 }
1157
1158 //////////////////////////////////////////////////////////////////////////
1159 /// @brief SwrDrawInstanced
1160 /// @param hContext - Handle passed back from SwrCreateContext
1161 /// @param topology - Specifies topology for draw.
1162 /// @param numVertsPerInstance - How many vertices to read sequentially from vertex data.
1163 /// @param numInstances - How many instances to render.
1164 /// @param startVertex - Specifies start vertex for draw. (vertex data)
1165 /// @param startInstance - Which instance to start sequentially fetching from in each buffer (instanced data)
1166 void SwrDrawInstanced(
1167     HANDLE hContext,
1168     PRIMITIVE_TOPOLOGY topology,
1169     uint32_t numVertsPerInstance,
1170     uint32_t numInstances,
1171     uint32_t startVertex,
1172     uint32_t startInstance
1173     )
1174 {
1175     DrawInstanced(hContext, topology, numVertsPerInstance, startVertex, numInstances, startInstance);
1176 }
1177
1178 //////////////////////////////////////////////////////////////////////////
1179 /// @brief DrawIndexedInstanced
1180 /// @param hContext - Handle passed back from SwrCreateContext
1181 /// @param topology - Specifies topology for draw.
1182 /// @param numIndices - Number of indices to read sequentially from index buffer.
1183 /// @param indexOffset - Starting index into index buffer.
1184 /// @param baseVertex - Vertex in vertex buffer to consider as index "0". Note value is signed.
1185 /// @param numInstances - Number of instances to render.
1186 /// @param startInstance - Which instance to start sequentially fetching from in each buffer (instanced data)
1187 void DrawIndexedInstance(
1188     HANDLE hContext,
1189     PRIMITIVE_TOPOLOGY topology,
1190     uint32_t numIndices,
1191     uint32_t indexOffset,
1192     int32_t baseVertex,
1193     uint32_t numInstances = 1,
1194     uint32_t startInstance = 0)
1195 {
1196     if (KNOB_TOSS_DRAW)
1197     {
1198         return;
1199     }
1200
1201     RDTSC_START(APIDrawIndexed);
1202
1203     SWR_CONTEXT *pContext = GetContext(hContext);
1204     DRAW_CONTEXT* pDC = GetDrawContext(pContext);
1205     API_STATE* pState = &pDC->pState->state;
1206
1207     uint32_t maxIndicesPerDraw = MaxVertsPerDraw(pDC, numIndices, topology);
1208     uint32_t primsPerDraw = GetNumPrims(topology, maxIndicesPerDraw);
1209     uint32_t remainingIndices = numIndices;
1210
1211     uint32_t indexSize = 0;
1212     switch (pState->indexBuffer.format)
1213     {
1214     case R32_UINT: indexSize = sizeof(uint32_t); break;
1215     case R16_UINT: indexSize = sizeof(uint16_t); break;
1216     case R8_UINT: indexSize = sizeof(uint8_t); break;
1217     default:
1218         SWR_ASSERT(0);
1219     }
1220
1221     int draw = 0;
1222     uint8_t *pIB = (uint8_t*)pState->indexBuffer.pIndices;
1223     pIB += (uint64_t)indexOffset * (uint64_t)indexSize;
1224
1225     pState->topology = topology;
1226     pState->forceFront = false;
1227
1228     // disable culling for points/lines
1229     uint32_t oldCullMode = pState->rastState.cullMode;
1230     if (topology == TOP_POINT_LIST)
1231     {
1232         pState->rastState.cullMode = SWR_CULLMODE_NONE;
1233         pState->forceFront = true;
1234     }
1235
1236     while (remainingIndices)
1237     {
1238         uint32_t numIndicesForDraw = (remainingIndices < maxIndicesPerDraw) ?
1239         remainingIndices : maxIndicesPerDraw;
1240
1241         // When breaking up draw, we need to obtain new draw context for each iteration.
1242         bool isSplitDraw = (draw > 0) ? true : false;
1243         pDC = GetDrawContext(pContext, isSplitDraw);
1244         InitDraw(pDC, isSplitDraw);
1245
1246         pDC->FeWork.type = DRAW;
1247         pDC->FeWork.pfnWork = GetProcessDrawFunc(
1248             true,   // IsIndexed
1249             pState->frontendState.bEnableCutIndex,
1250             pState->tsState.tsEnable,
1251             pState->gsState.gsEnable,
1252             pState->soState.soEnable,
1253             pDC->pState->pfnProcessPrims != nullptr);
1254         pDC->FeWork.desc.draw.pDC = pDC;
1255         pDC->FeWork.desc.draw.numIndices = numIndicesForDraw;
1256         pDC->FeWork.desc.draw.pIB = (int*)pIB;
1257         pDC->FeWork.desc.draw.type = pDC->pState->state.indexBuffer.format;
1258
1259         pDC->FeWork.desc.draw.numInstances = numInstances;
1260         pDC->FeWork.desc.draw.startInstance = startInstance;
1261         pDC->FeWork.desc.draw.baseVertex = baseVertex;
1262         pDC->FeWork.desc.draw.startPrimID = draw * primsPerDraw;
1263
1264         pDC->cleanupState = (remainingIndices == numIndicesForDraw);
1265
1266         //enqueue DC
1267         QueueDraw(pContext);
1268
1269         pIB += maxIndicesPerDraw * indexSize;
1270         remainingIndices -= numIndicesForDraw;
1271         draw++;
1272     }
1273
1274     // restore culling state
1275     pDC = GetDrawContext(pContext);
1276     pDC->pState->state.rastState.cullMode = oldCullMode;
1277
1278     RDTSC_STOP(APIDrawIndexed, numIndices * numInstances, 0);
1279 }
1280
1281
1282 //////////////////////////////////////////////////////////////////////////
1283 /// @brief DrawIndexed
1284 /// @param hContext - Handle passed back from SwrCreateContext
1285 /// @param topology - Specifies topology for draw.
1286 /// @param numIndices - Number of indices to read sequentially from index buffer.
1287 /// @param indexOffset - Starting index into index buffer.
1288 /// @param baseVertex - Vertex in vertex buffer to consider as index "0". Note value is signed.
1289 void SwrDrawIndexed(
1290     HANDLE hContext,
1291     PRIMITIVE_TOPOLOGY topology,
1292     uint32_t numIndices,
1293     uint32_t indexOffset,
1294     int32_t baseVertex
1295     )
1296 {
1297     DrawIndexedInstance(hContext, topology, numIndices, indexOffset, baseVertex);
1298 }
1299
1300 //////////////////////////////////////////////////////////////////////////
1301 /// @brief SwrDrawIndexedInstanced
1302 /// @param hContext - Handle passed back from SwrCreateContext
1303 /// @param topology - Specifies topology for draw.
1304 /// @param numIndices - Number of indices to read sequentially from index buffer.
1305 /// @param numInstances - Number of instances to render.
1306 /// @param indexOffset - Starting index into index buffer.
1307 /// @param baseVertex - Vertex in vertex buffer to consider as index "0". Note value is signed.
1308 /// @param startInstance - Which instance to start sequentially fetching from in each buffer (instanced data)
1309 void SwrDrawIndexedInstanced(
1310     HANDLE hContext,
1311     PRIMITIVE_TOPOLOGY topology,
1312     uint32_t numIndices,
1313     uint32_t numInstances,
1314     uint32_t indexOffset,
1315     int32_t baseVertex,
1316     uint32_t startInstance)
1317 {
1318     DrawIndexedInstance(hContext, topology, numIndices, indexOffset, baseVertex, numInstances, startInstance);
1319 }
1320
1321 //////////////////////////////////////////////////////////////////////////
1322 /// @brief SwrInvalidateTiles
1323 /// @param hContext - Handle passed back from SwrCreateContext
1324 /// @param attachmentMask - The mask specifies which surfaces attached to the hottiles to invalidate.
1325 /// @param invalidateRect - The pixel-coordinate rectangle to invalidate.  This will be expanded to
1326 ///                         be hottile size-aligned.
1327 void SWR_API SwrInvalidateTiles(
1328     HANDLE hContext,
1329     uint32_t attachmentMask,
1330     const SWR_RECT& invalidateRect)
1331 {
1332     if (KNOB_TOSS_DRAW)
1333     {
1334         return;
1335     }
1336
1337     SWR_CONTEXT *pContext = GetContext(hContext);
1338     DRAW_CONTEXT* pDC = GetDrawContext(pContext);
1339
1340     pDC->FeWork.type = DISCARDINVALIDATETILES;
1341     pDC->FeWork.pfnWork = ProcessDiscardInvalidateTiles;
1342     pDC->FeWork.desc.discardInvalidateTiles.attachmentMask = attachmentMask;
1343     pDC->FeWork.desc.discardInvalidateTiles.rect = invalidateRect;
1344     pDC->FeWork.desc.discardInvalidateTiles.rect &= g_MaxScissorRect;
1345     pDC->FeWork.desc.discardInvalidateTiles.newTileState = SWR_TILE_INVALID;
1346     pDC->FeWork.desc.discardInvalidateTiles.createNewTiles = false;
1347     pDC->FeWork.desc.discardInvalidateTiles.fullTilesOnly = false;
1348
1349     //enqueue
1350     QueueDraw(pContext);
1351 }
1352
1353 //////////////////////////////////////////////////////////////////////////
1354 /// @brief SwrDiscardRect
1355 /// @param hContext - Handle passed back from SwrCreateContext
1356 /// @param attachmentMask - The mask specifies which surfaces attached to the hottiles to discard.
1357 /// @param rect - The pixel-coordinate rectangle to discard.  Only fully-covered hottiles will be
1358 ///               discarded.
1359 void SWR_API SwrDiscardRect(
1360     HANDLE hContext,
1361     uint32_t attachmentMask,
1362     const SWR_RECT& rect)
1363 {
1364     if (KNOB_TOSS_DRAW)
1365     {
1366         return;
1367     }
1368
1369     SWR_CONTEXT *pContext = GetContext(hContext);
1370     DRAW_CONTEXT* pDC = GetDrawContext(pContext);
1371
1372     // Queue a load to the hottile
1373     pDC->FeWork.type = DISCARDINVALIDATETILES;
1374     pDC->FeWork.pfnWork = ProcessDiscardInvalidateTiles;
1375     pDC->FeWork.desc.discardInvalidateTiles.attachmentMask = attachmentMask;
1376     pDC->FeWork.desc.discardInvalidateTiles.rect = rect;
1377     pDC->FeWork.desc.discardInvalidateTiles.rect &= g_MaxScissorRect;
1378     pDC->FeWork.desc.discardInvalidateTiles.newTileState = SWR_TILE_RESOLVED;
1379     pDC->FeWork.desc.discardInvalidateTiles.createNewTiles = true;
1380     pDC->FeWork.desc.discardInvalidateTiles.fullTilesOnly = true;
1381
1382     //enqueue
1383     QueueDraw(pContext);
1384 }
1385
1386 //////////////////////////////////////////////////////////////////////////
1387 /// @brief SwrDispatch
1388 /// @param hContext - Handle passed back from SwrCreateContext
1389 /// @param threadGroupCountX - Number of thread groups dispatched in X direction
1390 /// @param threadGroupCountY - Number of thread groups dispatched in Y direction
1391 /// @param threadGroupCountZ - Number of thread groups dispatched in Z direction
1392 void SwrDispatch(
1393     HANDLE hContext,
1394     uint32_t threadGroupCountX,
1395     uint32_t threadGroupCountY,
1396     uint32_t threadGroupCountZ)
1397 {
1398     if (KNOB_TOSS_DRAW)
1399     {
1400         return;
1401     }
1402
1403     RDTSC_START(APIDispatch);
1404     SWR_CONTEXT *pContext = GetContext(hContext);
1405     DRAW_CONTEXT* pDC = GetDrawContext(pContext);
1406
1407     pDC->isCompute = true;      // This is a compute context.
1408
1409     COMPUTE_DESC* pTaskData = (COMPUTE_DESC*)pDC->pArena->AllocAligned(sizeof(COMPUTE_DESC), 64);
1410
1411     pTaskData->threadGroupCountX = threadGroupCountX;
1412     pTaskData->threadGroupCountY = threadGroupCountY;
1413     pTaskData->threadGroupCountZ = threadGroupCountZ;
1414
1415     uint32_t totalThreadGroups = threadGroupCountX * threadGroupCountY * threadGroupCountZ;
1416     uint32_t dcIndex = pDC->drawId % KNOB_MAX_DRAWS_IN_FLIGHT;
1417     pDC->pDispatch = &pContext->pDispatchQueueArray[dcIndex];
1418     pDC->pDispatch->initialize(totalThreadGroups, pTaskData);
1419
1420     QueueDispatch(pContext);
1421     RDTSC_STOP(APIDispatch, threadGroupCountX * threadGroupCountY * threadGroupCountZ, 0);
1422 }
1423
1424 // Deswizzles, converts and stores current contents of the hot tiles to surface
1425 // described by pState
1426 void SWR_API SwrStoreTiles(
1427     HANDLE hContext,
1428     SWR_RENDERTARGET_ATTACHMENT attachment,
1429     SWR_TILE_STATE postStoreTileState,
1430     const SWR_RECT& storeRect)
1431 {
1432     if (KNOB_TOSS_DRAW)
1433     {
1434         return;
1435     }
1436
1437     RDTSC_START(APIStoreTiles);
1438
1439     SWR_CONTEXT *pContext = GetContext(hContext);
1440     DRAW_CONTEXT* pDC = GetDrawContext(pContext);
1441
1442     pDC->FeWork.type = STORETILES;
1443     pDC->FeWork.pfnWork = ProcessStoreTiles;
1444     pDC->FeWork.desc.storeTiles.attachment = attachment;
1445     pDC->FeWork.desc.storeTiles.postStoreTileState = postStoreTileState;
1446     pDC->FeWork.desc.storeTiles.rect = storeRect;
1447     pDC->FeWork.desc.storeTiles.rect &= g_MaxScissorRect;
1448
1449     //enqueue
1450     QueueDraw(pContext);
1451
1452     RDTSC_STOP(APIStoreTiles, 0, 0);
1453 }
1454
1455 //////////////////////////////////////////////////////////////////////////
1456 /// @brief SwrClearRenderTarget - Clear attached render targets / depth / stencil
1457 /// @param hContext - Handle passed back from SwrCreateContext
1458 /// @param clearMask - combination of SWR_CLEAR_COLOR / SWR_CLEAR_DEPTH / SWR_CLEAR_STENCIL flags (or SWR_CLEAR_NONE)
1459 /// @param clearColor - color use for clearing render targets
1460 /// @param z - depth value use for clearing depth buffer
1461 /// @param stencil - stencil value used for clearing stencil buffer
1462 /// @param clearRect - The pixel-coordinate rectangle to clear in all cleared buffers
1463 void SWR_API SwrClearRenderTarget(
1464     HANDLE hContext,
1465     uint32_t clearMask,
1466     const float clearColor[4],
1467     float z,
1468     uint8_t stencil,
1469     const SWR_RECT& clearRect)
1470 {
1471     if (KNOB_TOSS_DRAW)
1472     {
1473         return;
1474     }
1475
1476     RDTSC_START(APIClearRenderTarget);
1477
1478     SWR_CONTEXT *pContext = GetContext(hContext);
1479     DRAW_CONTEXT* pDC = GetDrawContext(pContext);
1480
1481     CLEAR_FLAGS flags;
1482     flags.bits = 0;
1483     flags.mask = clearMask;
1484
1485     pDC->FeWork.type = CLEAR;
1486     pDC->FeWork.pfnWork = ProcessClear;
1487     pDC->FeWork.desc.clear.rect = clearRect;
1488     pDC->FeWork.desc.clear.rect &= g_MaxScissorRect;
1489     pDC->FeWork.desc.clear.flags = flags;
1490     pDC->FeWork.desc.clear.clearDepth = z;
1491     pDC->FeWork.desc.clear.clearRTColor[0] = clearColor[0];
1492     pDC->FeWork.desc.clear.clearRTColor[1] = clearColor[1];
1493     pDC->FeWork.desc.clear.clearRTColor[2] = clearColor[2];
1494     pDC->FeWork.desc.clear.clearRTColor[3] = clearColor[3];
1495     pDC->FeWork.desc.clear.clearStencil = stencil;
1496
1497     // enqueue draw
1498     QueueDraw(pContext);
1499
1500     RDTSC_STOP(APIClearRenderTarget, 0, pDC->drawId);
1501 }
1502
1503 //////////////////////////////////////////////////////////////////////////
1504 /// @brief Returns a pointer to the private context state for the current
1505 ///        draw operation. This is used for external componets such as the
1506 ///        sampler.
1507 ///        SWR is responsible for the allocation of the private context state.
1508 /// @param hContext - Handle passed back from SwrCreateContext
1509 VOID* SwrGetPrivateContextState(
1510     HANDLE hContext)
1511 {
1512     SWR_CONTEXT* pContext = GetContext(hContext);
1513     DRAW_CONTEXT* pDC = GetDrawContext(pContext);
1514     DRAW_STATE* pState = pDC->pState;
1515
1516     if (pState->pPrivateState == nullptr)
1517     {
1518         pState->pPrivateState = pState->pArena->AllocAligned(pContext->privateStateSize, KNOB_SIMD_WIDTH*sizeof(float));
1519     }
1520
1521     return pState->pPrivateState;
1522 }
1523
1524 //////////////////////////////////////////////////////////////////////////
1525 /// @brief Clients can use this to allocate memory for draw/dispatch
1526 ///        operations. The memory will automatically be freed once operation
1527 ///        has completed. Client can use this to allocate binding tables,
1528 ///        etc. needed for shader execution.
1529 /// @param hContext - Handle passed back from SwrCreateContext
1530 /// @param size - Size of allocation
1531 /// @param align - Alignment needed for allocation.
1532 VOID* SwrAllocDrawContextMemory(
1533     HANDLE hContext,
1534     uint32_t size,
1535     uint32_t align)
1536 {
1537     SWR_CONTEXT* pContext = GetContext(hContext);
1538     DRAW_CONTEXT* pDC = GetDrawContext(pContext);
1539
1540     return pDC->pState->pArena->AllocAligned(size, align);
1541 }
1542
1543 //////////////////////////////////////////////////////////////////////////
1544 /// @brief Enables stats counting
1545 /// @param hContext - Handle passed back from SwrCreateContext
1546 /// @param enable - If true then counts are incremented.
1547 void SwrEnableStats(
1548     HANDLE hContext,
1549     bool enable)
1550 {
1551     SWR_CONTEXT *pContext = GetContext(hContext);
1552     DRAW_CONTEXT* pDC = GetDrawContext(pContext);
1553
1554     pDC->pState->state.enableStats = enable;
1555 }
1556
1557 //////////////////////////////////////////////////////////////////////////
1558 /// @brief Mark end of frame - used for performance profiling
1559 /// @param hContext - Handle passed back from SwrCreateContext
1560 void SWR_API SwrEndFrame(
1561     HANDLE hContext)
1562 {
1563     RDTSC_ENDFRAME();
1564     SWR_CONTEXT *pContext = GetContext(hContext);
1565     pContext->frameCount++;
1566 }