src/gallium/drivers/swr/rasterizer/core/backend.cpp

   1 /****************************************************************************
   2 * Copyright (C) 2014-2015 Intel Corporation.   All Rights Reserved.
   3 *
   4 * Permission is hereby granted, free of charge, to any person obtaining a
   5 * copy of this software and associated documentation files (the "Software"),
   6 * to deal in the Software without restriction, including without limitation
   7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8 * and/or sell copies of the Software, and to permit persons to whom the
   9 * Software is furnished to do so, subject to the following conditions:
  10 *
  11 * The above copyright notice and this permission notice (including the next
  12 * paragraph) shall be included in all copies or substantial portions of the
  13 * Software.
  14 *
  15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21 * IN THE SOFTWARE.
  22 *
  23 * @file backend.cpp
  24 *
  25 * @brief Backend handles rasterization, pixel shading and output merger
  26 *        operations.
  27 *
  28 ******************************************************************************/
  29
  30 #include <smmintrin.h>
  31
  32 #include "rdtsc_core.h"
  33 #include "backend.h"
  34 #include "depthstencil.h"
  35 #include "tilemgr.h"
  36 #include "memory/tilingtraits.h"
  37 #include "core/multisample.h"
  38
  39 #include <algorithm>
  40
  41 const __m128 vTileOffsetsX = {0.5, KNOB_TILE_X_DIM - 0.5, 0.5, KNOB_TILE_X_DIM - 0.5};
  42 const __m128 vTileOffsetsY = {0.5, 0.5, KNOB_TILE_Y_DIM - 0.5, KNOB_TILE_Y_DIM - 0.5};
  43
  44 /// @todo move to common lib
  45 #define MASKTOVEC(i3,i2,i1,i0) {-i0,-i1,-i2,-i3}
  46 static const __m128 gMaskToVec[] = {
  47     MASKTOVEC(0,0,0,0),
  48     MASKTOVEC(0,0,0,1),
  49     MASKTOVEC(0,0,1,0),
  50     MASKTOVEC(0,0,1,1),
  51     MASKTOVEC(0,1,0,0),
  52     MASKTOVEC(0,1,0,1),
  53     MASKTOVEC(0,1,1,0),
  54     MASKTOVEC(0,1,1,1),
  55     MASKTOVEC(1,0,0,0),
  56     MASKTOVEC(1,0,0,1),
  57     MASKTOVEC(1,0,1,0),
  58     MASKTOVEC(1,0,1,1),
  59     MASKTOVEC(1,1,0,0),
  60     MASKTOVEC(1,1,0,1),
  61     MASKTOVEC(1,1,1,0),
  62     MASKTOVEC(1,1,1,1),
  63 };
  64
  65 typedef void(*PFN_CLEAR_TILES)(DRAW_CONTEXT*, SWR_RENDERTARGET_ATTACHMENT rt, uint32_t, DWORD[4]);
  66 static PFN_CLEAR_TILES sClearTilesTable[NUM_SWR_FORMATS];
  67
  68 //////////////////////////////////////////////////////////////////////////
  69 /// @brief Process compute work.
  70 /// @param pDC - pointer to draw context (dispatch).
  71 /// @param workerId - The unique worker ID that is assigned to this thread.
  72 /// @param threadGroupId - the linear index for the thread group within the dispatch.
  73 void ProcessComputeBE(DRAW_CONTEXT* pDC, uint32_t workerId, uint32_t threadGroupId)
  74 {
  75     RDTSC_START(BEDispatch);
  76
  77     SWR_CONTEXT *pContext = pDC->pContext;
  78
  79     const COMPUTE_DESC* pTaskData = (COMPUTE_DESC*)pDC->pDispatch->GetTasksData();
  80     SWR_ASSERT(pTaskData != nullptr);
  81
  82     // Ensure spill fill memory has been allocated.
  83     if (pDC->pSpillFill[workerId] == nullptr)
  84     {
  85         ///@todo Add state which indicates the spill fill size.
  86         pDC->pSpillFill[workerId] = (uint8_t*)pDC->pArena->AllocAlignedSync(4096 * 1024, sizeof(float) * 8);
  87     }
  88
  89     const API_STATE& state = GetApiState(pDC);
  90
  91     SWR_CS_CONTEXT csContext{ 0 };
  92     csContext.tileCounter = threadGroupId;
  93     csContext.dispatchDims[0] = pTaskData->threadGroupCountX;
  94     csContext.dispatchDims[1] = pTaskData->threadGroupCountY;
  95     csContext.dispatchDims[2] = pTaskData->threadGroupCountZ;
  96     csContext.pTGSM = pContext->pScratch[workerId];
  97     csContext.pSpillFillBuffer = pDC->pSpillFill[workerId];
  98
  99     state.pfnCsFunc(GetPrivateState(pDC), &csContext);
 100
 101     UPDATE_STAT(CsInvocations, state.totalThreadsInGroup);
 102
 103     RDTSC_STOP(BEDispatch, 1, 0);
 104 }
 105
 106 void ProcessSyncBE(DRAW_CONTEXT *pDC, uint32_t workerId, uint32_t macroTile, void *pUserData)
 107 {
 108     SYNC_DESC *pSync = (SYNC_DESC*)pUserData;
 109
 110     uint32_t x, y;
 111     MacroTileMgr::getTileIndices(macroTile, x, y);
 112     SWR_ASSERT(x == 0 && y == 0);
 113
 114     if (pSync->pfnCallbackFunc != nullptr)
 115     {
 116         pSync->pfnCallbackFunc(pSync->userData, pSync->userData2, pSync->userData3);
 117     }
 118 }
 119
 120 void ProcessQueryStatsBE(DRAW_CONTEXT *pDC, uint32_t workerId, uint32_t macroTile, void *pUserData)
 121 {
 122     QUERY_DESC* pQueryDesc = (QUERY_DESC*)pUserData;
 123     SWR_STATS* pStats = pQueryDesc->pStats;
 124     SWR_CONTEXT *pContext = pDC->pContext;
 125
 126     SWR_ASSERT(pStats != nullptr);
 127
 128     for (uint32_t i = 0; i < pContext->NumWorkerThreads; ++i)
 129     {
 130         pStats->DepthPassCount += pContext->stats[i].DepthPassCount;
 131
 132         pStats->IaVertices    += pContext->stats[i].IaVertices;
 133         pStats->IaPrimitives  += pContext->stats[i].IaPrimitives;
 134         pStats->VsInvocations += pContext->stats[i].VsInvocations;
 135         pStats->HsInvocations += pContext->stats[i].HsInvocations;
 136         pStats->DsInvocations += pContext->stats[i].DsInvocations;
 137         pStats->GsInvocations += pContext->stats[i].GsInvocations;
 138         pStats->PsInvocations += pContext->stats[i].PsInvocations;
 139         pStats->CInvocations  += pContext->stats[i].CInvocations;
 140         pStats->CsInvocations += pContext->stats[i].CsInvocations;
 141         pStats->CPrimitives   += pContext->stats[i].CPrimitives;
 142         pStats->GsPrimitives  += pContext->stats[i].GsPrimitives;
 143
 144         for (uint32_t stream = 0; stream < MAX_SO_STREAMS; ++stream)
 145         {
 146             pStats->SoWriteOffset[stream] += pContext->stats[i].SoWriteOffset[stream];
 147
 148             /// @note client is required to provide valid write offset before every draw, so we clear
 149             /// out the contents of the write offset when storing stats
 150             pContext->stats[i].SoWriteOffset[stream] = 0;
 151
 152             pStats->SoPrimStorageNeeded[stream] += pContext->stats[i].SoPrimStorageNeeded[stream];
 153             pStats->SoNumPrimsWritten[stream] += pContext->stats[i].SoNumPrimsWritten[stream];
 154         }
 155     }
 156 }
 157
 158 template<SWR_FORMAT format>
 159 void ClearRasterTile(uint8_t *pTileBuffer, simdvector &value)
 160 {
 161     auto lambda = [&](int comp)
 162     {
 163         FormatTraits<format>::storeSOA(comp, pTileBuffer, value.v[comp]);
 164         pTileBuffer += (KNOB_SIMD_WIDTH * FormatTraits<format>::GetBPC(comp) / 8);
 165     };
 166
 167     const uint32_t numIter = (KNOB_TILE_Y_DIM / SIMD_TILE_Y_DIM) * (KNOB_TILE_X_DIM / SIMD_TILE_X_DIM);
 168     for (uint32_t i = 0; i < numIter; ++i)
 169     {
 170         UnrollerL<0, FormatTraits<format>::numComps, 1>::step(lambda);
 171     }
 172 }
 173
 174 template<SWR_FORMAT format>
 175 INLINE void ClearMacroTile(DRAW_CONTEXT *pDC, SWR_RENDERTARGET_ATTACHMENT rt, uint32_t macroTile, DWORD clear[4])
 176 {
 177     // convert clear color to hottile format
 178     // clear color is in RGBA float/uint32
 179     simdvector vClear;
 180     for (uint32_t comp = 0; comp < FormatTraits<format>::numComps; ++comp)
 181     {
 182         simdscalar vComp;
 183         vComp = _simd_load1_ps((const float*)&clear[comp]);
 184         if (FormatTraits<format>::isNormalized(comp))
 185         {
 186             vComp = _simd_mul_ps(vComp, _simd_set1_ps(FormatTraits<format>::fromFloat(comp)));
 187             vComp = _simd_castsi_ps(_simd_cvtps_epi32(vComp));
 188         }
 189         vComp = FormatTraits<format>::pack(comp, vComp);
 190         vClear.v[FormatTraits<format>::swizzle(comp)] = vComp;
 191     }
 192
 193     uint32_t tileX, tileY;
 194     MacroTileMgr::getTileIndices(macroTile, tileX, tileY);
 195     const API_STATE& state = GetApiState(pDC);
 196
 197     int top = KNOB_MACROTILE_Y_DIM_FIXED * tileY;
 198     int bottom = top + KNOB_MACROTILE_Y_DIM_FIXED - 1;
 199     int left = KNOB_MACROTILE_X_DIM_FIXED * tileX;
 200     int right = left + KNOB_MACROTILE_X_DIM_FIXED - 1;
 201
 202     // intersect with scissor
 203     top = std::max(top, state.scissorInFixedPoint.top);
 204     left = std::max(left, state.scissorInFixedPoint.left);
 205     bottom = std::min(bottom, state.scissorInFixedPoint.bottom);
 206     right = std::min(right, state.scissorInFixedPoint.right);
 207
 208     // translate to local hottile origin
 209     top -= KNOB_MACROTILE_Y_DIM_FIXED * tileY;
 210     bottom -= KNOB_MACROTILE_Y_DIM_FIXED * tileY;
 211     left -= KNOB_MACROTILE_X_DIM_FIXED * tileX;
 212     right -= KNOB_MACROTILE_X_DIM_FIXED * tileX;
 213
 214     // convert to raster tiles
 215     top >>= (KNOB_TILE_Y_DIM_SHIFT + FIXED_POINT_SHIFT);
 216     bottom >>= (KNOB_TILE_Y_DIM_SHIFT + FIXED_POINT_SHIFT);
 217     left >>= (KNOB_TILE_X_DIM_SHIFT + FIXED_POINT_SHIFT);
 218     right >>= (KNOB_TILE_X_DIM_SHIFT + FIXED_POINT_SHIFT);
 219
 220     const int numSamples = GetNumSamples(pDC->pState->state.rastState.sampleCount);
 221     // compute steps between raster tile samples / raster tiles / macro tile rows
 222     const uint32_t rasterTileSampleStep = KNOB_TILE_X_DIM * KNOB_TILE_Y_DIM * FormatTraits<format>::bpp / 8;
 223     const uint32_t rasterTileStep = (KNOB_TILE_X_DIM * KNOB_TILE_Y_DIM * (FormatTraits<format>::bpp / 8)) * numSamples;
 224     const uint32_t macroTileRowStep = (KNOB_MACROTILE_X_DIM / KNOB_TILE_X_DIM) * rasterTileStep;
 225     const uint32_t pitch = (FormatTraits<format>::bpp * KNOB_MACROTILE_X_DIM / 8);
 226
 227     HOTTILE *pHotTile = pDC->pContext->pHotTileMgr->GetHotTile(pDC->pContext, pDC, macroTile, rt, true, numSamples);
 228     uint32_t rasterTileStartOffset = (ComputeTileOffset2D< TilingTraits<SWR_TILE_SWRZ, FormatTraits<format>::bpp > >(pitch, left, top)) * numSamples;
 229     uint8_t* pRasterTileRow = pHotTile->pBuffer + rasterTileStartOffset; //(ComputeTileOffset2D< TilingTraits<SWR_TILE_SWRZ, FormatTraits<format>::bpp > >(pitch, x, y)) * numSamples;
 230
 231     // loop over all raster tiles in the current hot tile
 232     for (int y = top; y <= bottom; ++y)
 233     {
 234         uint8_t* pRasterTile = pRasterTileRow;
 235         for (int x = left; x <= right; ++x)
 236         {
 237             for( int sampleNum = 0; sampleNum < numSamples; sampleNum++)
 238             {
 239                 ClearRasterTile<format>(pRasterTile, vClear);
 240                 pRasterTile += rasterTileSampleStep;
 241             }
 242         }
 243         pRasterTileRow += macroTileRowStep;
 244     }
 245
 246     pHotTile->state = HOTTILE_DIRTY;
 247 }
 248
 249
 250 void ProcessClearBE(DRAW_CONTEXT *pDC, uint32_t workerId, uint32_t macroTile, void *pUserData)
 251 {
 252     if (KNOB_FAST_CLEAR)
 253     {
 254         CLEAR_DESC *pClear = (CLEAR_DESC*)pUserData;
 255         SWR_CONTEXT *pContext = pDC->pContext;
 256         SWR_MULTISAMPLE_COUNT sampleCount = pDC->pState->state.rastState.sampleCount;
 257         uint32_t numSamples = GetNumSamples(sampleCount);
 258
 259         SWR_ASSERT(pClear->flags.bits != 0); // shouldn't be here without a reason.
 260
 261         RDTSC_START(BEClear);
 262
 263         if (pClear->flags.mask & SWR_CLEAR_COLOR)
 264         {
 265             HOTTILE *pHotTile = pContext->pHotTileMgr->GetHotTile(pContext, pDC, macroTile, SWR_ATTACHMENT_COLOR0, true, numSamples);
 266             // All we want to do here is to mark the hot tile as being in a "needs clear" state.
 267             pHotTile->clearData[0] = *(DWORD*)&(pClear->clearRTColor[0]);
 268             pHotTile->clearData[1] = *(DWORD*)&(pClear->clearRTColor[1]);
 269             pHotTile->clearData[2] = *(DWORD*)&(pClear->clearRTColor[2]);
 270             pHotTile->clearData[3] = *(DWORD*)&(pClear->clearRTColor[3]);
 271             pHotTile->state = HOTTILE_CLEAR;
 272         }
 273
 274         if (pClear->flags.mask & SWR_CLEAR_DEPTH)
 275         {
 276             HOTTILE *pHotTile = pContext->pHotTileMgr->GetHotTile(pContext, pDC, macroTile, SWR_ATTACHMENT_DEPTH, true, numSamples);
 277             pHotTile->clearData[0] = *(DWORD*)&pClear->clearDepth;
 278             pHotTile->state = HOTTILE_CLEAR;
 279         }
 280
 281         if (pClear->flags.mask & SWR_CLEAR_STENCIL)
 282         {
 283             HOTTILE *pHotTile = pContext->pHotTileMgr->GetHotTile(pContext, pDC, macroTile, SWR_ATTACHMENT_STENCIL, true, numSamples);
 284
 285             pHotTile->clearData[0] = *(DWORD*)&pClear->clearStencil;
 286             pHotTile->state = HOTTILE_CLEAR;
 287         }
 288
 289         RDTSC_STOP(BEClear, 0, 0);
 290     }
 291     else
 292     {
 293         // Legacy clear
 294         CLEAR_DESC *pClear = (CLEAR_DESC*)pUserData;
 295         RDTSC_START(BEClear);
 296
 297         if (pClear->flags.mask & SWR_CLEAR_COLOR)
 298         {
 299             /// @todo clear data should come in as RGBA32_FLOAT
 300             DWORD clearData[4];
 301             float clearFloat[4];
 302             clearFloat[0] = ((uint8_t*)(&pClear->clearRTColor))[0] / 255.0f;
 303             clearFloat[1] = ((uint8_t*)(&pClear->clearRTColor))[1] / 255.0f;
 304             clearFloat[2] = ((uint8_t*)(&pClear->clearRTColor))[2] / 255.0f;
 305             clearFloat[3] = ((uint8_t*)(&pClear->clearRTColor))[3] / 255.0f;
 306             clearData[0] = *(DWORD*)&clearFloat[0];
 307             clearData[1] = *(DWORD*)&clearFloat[1];
 308             clearData[2] = *(DWORD*)&clearFloat[2];
 309             clearData[3] = *(DWORD*)&clearFloat[3];
 310
 311             PFN_CLEAR_TILES pfnClearTiles = sClearTilesTable[KNOB_COLOR_HOT_TILE_FORMAT];
 312             SWR_ASSERT(pfnClearTiles != nullptr);
 313
 314             pfnClearTiles(pDC, SWR_ATTACHMENT_COLOR0, macroTile, clearData);
 315         }
 316
 317         if (pClear->flags.mask & SWR_CLEAR_DEPTH)
 318         {
 319             DWORD clearData[4];
 320             clearData[0] = *(DWORD*)&pClear->clearDepth;
 321             PFN_CLEAR_TILES pfnClearTiles = sClearTilesTable[KNOB_DEPTH_HOT_TILE_FORMAT];
 322             SWR_ASSERT(pfnClearTiles != nullptr);
 323
 324             pfnClearTiles(pDC, SWR_ATTACHMENT_DEPTH, macroTile, clearData);
 325         }
 326
 327         if (pClear->flags.mask & SWR_CLEAR_STENCIL)
 328         {
 329             uint32_t value = pClear->clearStencil;
 330             DWORD clearData[4];
 331             clearData[0] = *(DWORD*)&value;
 332             PFN_CLEAR_TILES pfnClearTiles = sClearTilesTable[KNOB_STENCIL_HOT_TILE_FORMAT];
 333
 334             pfnClearTiles(pDC, SWR_ATTACHMENT_STENCIL, macroTile, clearData);
 335         }
 336
 337         RDTSC_STOP(BEClear, 0, 0);
 338     }
 339 }
 340
 341
 342 void ProcessStoreTileBE(DRAW_CONTEXT *pDC, uint32_t workerId, uint32_t macroTile, void *pData)
 343 {
 344     RDTSC_START(BEStoreTiles);
 345     STORE_TILES_DESC *pDesc = (STORE_TILES_DESC*)pData;
 346     SWR_CONTEXT *pContext = pDC->pContext;
 347
 348 #ifdef KNOB_ENABLE_RDTSC
 349     uint32_t numTiles = 0;
 350 #endif
 351     SWR_FORMAT srcFormat;
 352     switch (pDesc->attachment)
 353     {
 354     case SWR_ATTACHMENT_COLOR0:
 355     case SWR_ATTACHMENT_COLOR1:
 356     case SWR_ATTACHMENT_COLOR2:
 357     case SWR_ATTACHMENT_COLOR3:
 358     case SWR_ATTACHMENT_COLOR4:
 359     case SWR_ATTACHMENT_COLOR5:
 360     case SWR_ATTACHMENT_COLOR6:
 361     case SWR_ATTACHMENT_COLOR7: srcFormat = KNOB_COLOR_HOT_TILE_FORMAT; break;
 362     case SWR_ATTACHMENT_DEPTH: srcFormat = KNOB_DEPTH_HOT_TILE_FORMAT; break;
 363     case SWR_ATTACHMENT_STENCIL: srcFormat = KNOB_STENCIL_HOT_TILE_FORMAT; break;
 364     default: SWR_ASSERT(false, "Unknown attachment: %d", pDesc->attachment); srcFormat = KNOB_COLOR_HOT_TILE_FORMAT; break;
 365     }
 366
 367     uint32_t x, y;
 368     MacroTileMgr::getTileIndices(macroTile, x, y);
 369
 370     // Only need to store the hottile if it's been rendered to...
 371     HOTTILE *pHotTile = pContext->pHotTileMgr->GetHotTile(pContext, pDC, macroTile, pDesc->attachment, false);
 372     if (pHotTile)
 373     {
 374         // clear if clear is pending (i.e., not rendered to), then mark as dirty for store.
 375         if (pHotTile->state == HOTTILE_CLEAR)
 376         {
 377             PFN_CLEAR_TILES pfnClearTiles = sClearTilesTable[srcFormat];
 378             SWR_ASSERT(pfnClearTiles != nullptr);
 379
 380             pfnClearTiles(pDC, pDesc->attachment, macroTile, pHotTile->clearData);
 381         }
 382
 383         if (pHotTile->state == HOTTILE_DIRTY || pDesc->postStoreTileState == (SWR_TILE_STATE)HOTTILE_DIRTY)
 384         {
 385             int destX = KNOB_MACROTILE_X_DIM * x;
 386             int destY = KNOB_MACROTILE_Y_DIM * y;
 387
 388             pContext->pfnStoreTile(GetPrivateState(pDC), srcFormat,
 389                 pDesc->attachment, destX, destY, pHotTile->renderTargetArrayIndex, pHotTile->pBuffer);
 390         }
 391
 392
 393         if (pHotTile->state == HOTTILE_DIRTY || pHotTile->state == HOTTILE_RESOLVED)
 394         {
 395             pHotTile->state = (HOTTILE_STATE)pDesc->postStoreTileState;
 396         }
 397     }
 398     RDTSC_STOP(BEStoreTiles, numTiles, pDC->drawId);
 399 }
 400
 401
 402 void ProcessDiscardInvalidateTilesBE(DRAW_CONTEXT *pDC, uint32_t workerId, uint32_t macroTile, void *pData)
 403 {
 404     DISCARD_INVALIDATE_TILES_DESC *pDesc = (DISCARD_INVALIDATE_TILES_DESC *)pData;
 405     SWR_CONTEXT *pContext = pDC->pContext;
 406
 407     const int numSamples = GetNumSamples(pDC->pState->state.rastState.sampleCount);
 408
 409     for (uint32_t i = 0; i < SWR_NUM_ATTACHMENTS; ++i)
 410     {
 411         if (pDesc->attachmentMask & (1 << i))
 412         {
 413             HOTTILE *pHotTile = pContext->pHotTileMgr->GetHotTileNoLoad(
 414                 pContext, pDC, macroTile, (SWR_RENDERTARGET_ATTACHMENT)i, pDesc->createNewTiles, numSamples);
 415             if (pHotTile)
 416             {
 417                 pHotTile->state = (HOTTILE_STATE)pDesc->newTileState;
 418             }
 419         }
 420     }
 421 }
 422
 423 #if KNOB_SIMD_WIDTH == 8
 424 const __m256 vCenterOffsetsX = {0.5, 1.5, 0.5, 1.5, 2.5, 3.5, 2.5, 3.5};
 425 const __m256 vCenterOffsetsY = {0.5, 0.5, 1.5, 1.5, 0.5, 0.5, 1.5, 1.5};
 426 const __m256 vULOffsetsX = {0.0, 1.0, 0.0, 1.0, 2.0, 3.0, 2.0, 3.0};
 427 const __m256 vULOffsetsY = {0.0, 0.0, 1.0, 1.0, 0.0, 0.0, 1.0, 1.0};
 428 #else
 429 #error Unsupported vector width
 430 #endif
 431
 432 INLINE
 433 bool CanEarlyZ(const SWR_PS_STATE *pPSState)
 434 {
 435     return (pPSState->forceEarlyZ || (!pPSState->writesODepth && !pPSState->usesSourceDepth && !pPSState->usesUAV));
 436 }
 437
 438 simdmask ComputeUserClipMask(uint8_t clipMask, float* pUserClipBuffer, simdscalar vI, simdscalar vJ)
 439 {
 440     simdscalar vClipMask = _simd_setzero_ps();
 441     uint32_t numClipDistance = _mm_popcnt_u32(clipMask);
 442
 443     for (uint32_t i = 0; i < numClipDistance; ++i)
 444     {
 445         // pull triangle clip distance values from clip buffer
 446         simdscalar vA = _simd_broadcast_ss(pUserClipBuffer++);
 447         simdscalar vB = _simd_broadcast_ss(pUserClipBuffer++);
 448         simdscalar vC = _simd_broadcast_ss(pUserClipBuffer++);
 449
 450         // interpolate
 451         simdscalar vInterp = vplaneps(vA, vB, vC, vI, vJ);
 452
 453         // clip if interpolated clip distance is < 0 || NAN
 454         simdscalar vCull = _simd_cmp_ps(_simd_setzero_ps(), vInterp, _CMP_NLE_UQ);
 455
 456         vClipMask = _simd_or_ps(vClipMask, vCull);
 457     }
 458
 459     return _simd_movemask_ps(vClipMask);
 460 }
 461
 462 template<bool perspMask>
 463 INLINE void CalcPixelBarycentrics(const BarycentricCoeffs& coeffs, SWR_PS_CONTEXT &psContext)
 464 {
 465     if(perspMask)
 466     {
 467         // evaluate I,J
 468         psContext.vI.center = vplaneps(coeffs.vIa, coeffs.vIb, coeffs.vIc, psContext.vX.center, psContext.vY.center);
 469         psContext.vJ.center = vplaneps(coeffs.vJa, coeffs.vJb, coeffs.vJc, psContext.vX.center, psContext.vY.center);
 470         psContext.vI.center = _simd_mul_ps(psContext.vI.center, coeffs.vRecipDet);
 471         psContext.vJ.center = _simd_mul_ps(psContext.vJ.center, coeffs.vRecipDet);
 472
 473         // interpolate 1/w
 474         psContext.vOneOverW.center = vplaneps(coeffs.vAOneOverW, coeffs.vBOneOverW, coeffs.vCOneOverW, psContext.vI.center, psContext.vJ.center);
 475     }
 476 }
 477
 478 template<bool perspMask>
 479 INLINE void CalcSampleBarycentrics(const BarycentricCoeffs& coeffs, SWR_PS_CONTEXT &psContext)
 480 {
 481     if(perspMask)
 482     {
 483         // evaluate I,J
 484         psContext.vI.sample = vplaneps(coeffs.vIa, coeffs.vIb, coeffs.vIc, psContext.vX.sample, psContext.vY.sample);
 485         psContext.vJ.sample = vplaneps(coeffs.vJa, coeffs.vJb, coeffs.vJc, psContext.vX.sample, psContext.vY.sample);
 486         psContext.vI.sample = _simd_mul_ps(psContext.vI.sample, coeffs.vRecipDet);
 487         psContext.vJ.sample = _simd_mul_ps(psContext.vJ.sample, coeffs.vRecipDet);
 488
 489         // interpolate 1/w
 490         psContext.vOneOverW.sample = vplaneps(coeffs.vAOneOverW, coeffs.vBOneOverW, coeffs.vCOneOverW, psContext.vI.sample, psContext.vJ.sample);
 491     }
 492 }
 493
 494
 495 ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
 496 // Centroid behaves exactly as follows :
 497 // (1) If all samples in the primitive are covered, the attribute is evaluated at the pixel center (even if the sample pattern does not happen to
 498 //     have a sample location there).
 499 // (2) Else the attribute is evaluated at the first covered sample, in increasing order of sample index, where sample coverage is after ANDing the
 500 //     coverage with the SampleMask Rasterizer State.
 501 // (3) If no samples are covered, such as on helper pixels executed off the bounds of a primitive to fill out 2x2 pixel stamps, the attribute is
 502 //     evaluated as follows : If the SampleMask Rasterizer state is a subset of the samples in the pixel, then the first sample covered by the
 503 //     SampleMask Rasterizer State is the evaluation point.Otherwise (full SampleMask), the pixel center is the evaluation point.
 504 ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
 505 template<SWR_MULTISAMPLE_COUNT sampleCount, bool bForcedSampleCount>
 506 INLINE void CalcCentroidPos(SWR_PS_CONTEXT &psContext, const uint64_t *const coverageMask, const uint32_t sampleMask,
 507                             const simdscalar vXSamplePosUL, const simdscalar vYSamplePosUL)
 508 {
 509     uint32_t inputMask[KNOB_SIMD_WIDTH];
 510
 511     generateInputCoverage<sampleCount, 1, bForcedSampleCount>(coverageMask, inputMask, sampleMask);
 512
 513     // Case (2) - partially covered pixel
 514
 515     // scan for first covered sample per pixel in the 4x2 span
 516     unsigned long sampleNum[KNOB_SIMD_WIDTH];
 517     (inputMask[0] > 0) ? (_BitScanForward(&sampleNum[0], inputMask[0])) : (sampleNum[0] = 0);
 518     (inputMask[1] > 0) ? (_BitScanForward(&sampleNum[1], inputMask[1])) : (sampleNum[1] = 0);
 519     (inputMask[2] > 0) ? (_BitScanForward(&sampleNum[2], inputMask[2])) : (sampleNum[2] = 0);
 520     (inputMask[3] > 0) ? (_BitScanForward(&sampleNum[3], inputMask[3])) : (sampleNum[3] = 0);
 521     (inputMask[4] > 0) ? (_BitScanForward(&sampleNum[4], inputMask[4])) : (sampleNum[4] = 0);
 522     (inputMask[5] > 0) ? (_BitScanForward(&sampleNum[5], inputMask[5])) : (sampleNum[5] = 0);
 523     (inputMask[6] > 0) ? (_BitScanForward(&sampleNum[6], inputMask[6])) : (sampleNum[6] = 0);
 524     (inputMask[7] > 0) ? (_BitScanForward(&sampleNum[7], inputMask[7])) : (sampleNum[7] = 0);
 525
 526     // look up and set the sample offsets from UL pixel corner for first covered sample
 527     __m256 vXSample = _mm256_set_ps(MultisampleTraits<sampleCount>::X(sampleNum[7]),
 528                                     MultisampleTraits<sampleCount>::X(sampleNum[6]),
 529                                     MultisampleTraits<sampleCount>::X(sampleNum[5]),
 530                                     MultisampleTraits<sampleCount>::X(sampleNum[4]),
 531                                     MultisampleTraits<sampleCount>::X(sampleNum[3]),
 532                                     MultisampleTraits<sampleCount>::X(sampleNum[2]),
 533                                     MultisampleTraits<sampleCount>::X(sampleNum[1]),
 534                                     MultisampleTraits<sampleCount>::X(sampleNum[0]));
 535
 536     __m256 vYSample = _mm256_set_ps(MultisampleTraits<sampleCount>::Y(sampleNum[7]),
 537                                     MultisampleTraits<sampleCount>::Y(sampleNum[6]),
 538                                     MultisampleTraits<sampleCount>::Y(sampleNum[5]),
 539                                     MultisampleTraits<sampleCount>::Y(sampleNum[4]),
 540                                     MultisampleTraits<sampleCount>::Y(sampleNum[3]),
 541                                     MultisampleTraits<sampleCount>::Y(sampleNum[2]),
 542                                     MultisampleTraits<sampleCount>::Y(sampleNum[1]),
 543                                     MultisampleTraits<sampleCount>::Y(sampleNum[0]));
 544     // add sample offset to UL pixel corner
 545     vXSample = _simd_add_ps(vXSamplePosUL, vXSample);
 546     vYSample = _simd_add_ps(vYSamplePosUL, vYSample);
 547
 548     // Case (1) and case (3b) - All samples covered or not covered with full SampleMask
 549     static const __m256i vFullyCoveredMask = MultisampleTraits<sampleCount>::FullSampleMask();
 550     __m256i vInputCoveragei =  _mm256_set_epi32(inputMask[7], inputMask[6], inputMask[5], inputMask[4], inputMask[3], inputMask[2], inputMask[1], inputMask[0]);
 551     __m256i vAllSamplesCovered = _simd_cmpeq_epi32(vInputCoveragei, vFullyCoveredMask);
 552
 553     static const __m256i vZero = _simd_setzero_si();
 554     const __m256i vSampleMask = _simd_and_si(_simd_set1_epi32(sampleMask), vFullyCoveredMask);
 555     __m256i vNoSamplesCovered = _simd_cmpeq_epi32(vInputCoveragei, vZero);
 556     __m256i vIsFullSampleMask = _simd_cmpeq_epi32(vSampleMask, vFullyCoveredMask);
 557     __m256i vCase3b = _simd_and_si(vNoSamplesCovered, vIsFullSampleMask);
 558
 559     __m256i vEvalAtCenter = _simd_or_si(vAllSamplesCovered, vCase3b);
 560
 561     // set the centroid position based on results from above
 562     psContext.vX.centroid = _simd_blendv_ps(vXSample, psContext.vX.center, _simd_castsi_ps(vEvalAtCenter));
 563     psContext.vY.centroid = _simd_blendv_ps(vYSample, psContext.vY.center, _simd_castsi_ps(vEvalAtCenter));
 564
 565     // Case (3a) No samples covered and partial sample mask
 566     __m256i vSomeSampleMaskSamples = _simd_cmplt_epi32(vSampleMask, vFullyCoveredMask);
 567     // sample mask should never be all 0's for this case, but handle it anyways
 568     unsigned long firstCoveredSampleMaskSample = 0;
 569     (sampleMask > 0) ? (_BitScanForward(&firstCoveredSampleMaskSample, sampleMask)) : (firstCoveredSampleMaskSample = 0);
 570
 571     __m256i vCase3a = _simd_and_si(vNoSamplesCovered, vSomeSampleMaskSamples);
 572
 573     vXSample = _simd_set1_ps(MultisampleTraits<sampleCount>::X(firstCoveredSampleMaskSample));
 574     vYSample = _simd_set1_ps(MultisampleTraits<sampleCount>::Y(firstCoveredSampleMaskSample));
 575
 576     // blend in case 3a pixel locations
 577     psContext.vX.centroid = _simd_blendv_ps(psContext.vX.centroid, vXSample, _simd_castsi_ps(vCase3a));
 578     psContext.vY.centroid = _simd_blendv_ps(psContext.vY.centroid, vYSample, _simd_castsi_ps(vCase3a));
 579 }
 580
 581 template<uint32_t sampleCount, uint32_t persp, uint32_t standardPattern, uint32_t forcedMultisampleCount>
 582 INLINE void CalcCentroidBarycentrics(const BarycentricCoeffs& coeffs, SWR_PS_CONTEXT &psContext,
 583                                      const uint64_t *const coverageMask, const uint32_t sampleMask,
 584                                      const simdscalar vXSamplePosUL, const simdscalar vYSamplePosUL)
 585 {
 586     static const bool bPersp = (bool)persp;
 587     static const bool bIsStandardPattern = (bool)standardPattern;
 588     static const bool bForcedMultisampleCount = (bool)forcedMultisampleCount;
 589
 590     // calculate centroid positions
 591     if(bPersp)
 592     {
 593         if(bIsStandardPattern)
 594         {
 595             ///@ todo: don't need to generate input coverage 2x if input coverage and centroid
 596             CalcCentroidPos<(SWR_MULTISAMPLE_COUNT)sampleCount, bForcedMultisampleCount>(psContext, coverageMask, sampleMask, vXSamplePosUL, vYSamplePosUL);
 597         }
 598         else
 599         {
 600             static const __m256 pixelCenter = _simd_set1_ps(0.5f);
 601             psContext.vX.centroid = _simd_add_ps(vXSamplePosUL, pixelCenter);
 602             psContext.vY.centroid = _simd_add_ps(vYSamplePosUL, pixelCenter);
 603         }
 604         // evaluate I,J
 605         psContext.vI.centroid = vplaneps(coeffs.vIa, coeffs.vIb, coeffs.vIc, psContext.vX.centroid, psContext.vY.centroid);
 606         psContext.vJ.centroid = vplaneps(coeffs.vJa, coeffs.vJb, coeffs.vJc, psContext.vX.centroid, psContext.vY.centroid);
 607         psContext.vI.centroid = _simd_mul_ps(psContext.vI.centroid, coeffs.vRecipDet);
 608         psContext.vJ.centroid = _simd_mul_ps(psContext.vJ.centroid, coeffs.vRecipDet);
 609
 610         // interpolate 1/w
 611         psContext.vOneOverW.centroid = vplaneps(coeffs.vAOneOverW, coeffs.vBOneOverW, coeffs.vCOneOverW, psContext.vI.centroid, psContext.vJ.centroid);
 612     }
 613 }
 614
 615 template<uint32_t NumRT, uint32_t sampleCountT>
 616 void OutputMerger(SWR_PS_CONTEXT &psContext, uint8_t* (&pColorBase)[SWR_NUM_RENDERTARGETS], uint32_t sample, const SWR_BLEND_STATE *pBlendState,
 617                   const PFN_BLEND_JIT_FUNC (&pfnBlendFunc)[SWR_NUM_RENDERTARGETS], simdscalar &coverageMask, simdscalar depthPassMask)
 618 {
 619     // type safety guaranteed from template instantiation in BEChooser<>::GetFunc
 620     static const SWR_MULTISAMPLE_COUNT sampleCount = (SWR_MULTISAMPLE_COUNT)sampleCountT;
 621     uint32_t rasterTileColorOffset = MultisampleTraits<sampleCount>::RasterTileColorOffset(sample);
 622     simdvector blendOut;
 623
 624     for(uint32_t rt = 0; rt < NumRT; ++rt)
 625     {
 626         uint8_t *pColorSample;
 627         if(sampleCount == SWR_MULTISAMPLE_1X)
 628         {
 629             pColorSample = pColorBase[rt];
 630         }
 631         else
 632         {
 633             pColorSample = pColorBase[rt] + rasterTileColorOffset;
 634         }
 635
 636         const SWR_RENDER_TARGET_BLEND_STATE *pRTBlend = &pBlendState->renderTarget[rt];
 637         // pfnBlendFunc may not update all channels.  Initialize with PS output.
 638         /// TODO: move this into the blend JIT.
 639         blendOut = psContext.shaded[rt];
 640
 641         // Blend outputs and update coverage mask for alpha test
 642         if(pfnBlendFunc[rt] != nullptr)
 643         {
 644             pfnBlendFunc[rt](
 645                 pBlendState,
 646                 psContext.shaded[rt],
 647                 psContext.shaded[1],
 648                 sample,
 649                 pColorSample,
 650                 blendOut,
 651                 &psContext.oMask,
 652                 (simdscalari*)&coverageMask);
 653         }
 654
 655         // final write mask
 656         simdscalari outputMask = _simd_castps_si(_simd_and_ps(coverageMask, depthPassMask));
 657
 658         ///@todo can only use maskstore fast path if bpc is 32. Assuming hot tile is RGBA32_FLOAT.
 659         static_assert(KNOB_COLOR_HOT_TILE_FORMAT == R32G32B32A32_FLOAT, "Unsupported hot tile format");
 660
 661         const uint32_t simd = KNOB_SIMD_WIDTH * sizeof(float);
 662
 663         // store with color mask
 664         if(!pRTBlend->writeDisableRed)
 665         {
 666             _simd_maskstore_ps((float*)pColorSample, outputMask, blendOut.x);
 667         }
 668         if(!pRTBlend->writeDisableGreen)
 669         {
 670             _simd_maskstore_ps((float*)(pColorSample + simd), outputMask, blendOut.y);
 671         }
 672         if(!pRTBlend->writeDisableBlue)
 673         {
 674             _simd_maskstore_ps((float*)(pColorSample + simd * 2), outputMask, blendOut.z);
 675         }
 676         if(!pRTBlend->writeDisableAlpha)
 677         {
 678             _simd_maskstore_ps((float*)(pColorSample + simd * 3), outputMask, blendOut.w);
 679         }
 680     }
 681 }
 682
 683 template<uint32_t sampleCountT, uint32_t samplePattern, uint32_t inputCoverage, uint32_t centroidPos, uint32_t forcedSampleCount>
 684 void BackendSingleSample(DRAW_CONTEXT *pDC, uint32_t workerId, uint32_t x, uint32_t y, SWR_TRIANGLE_DESC &work, RenderOutputBuffers &renderBuffers)
 685 {
 686     RDTSC_START(BESetup);
 687     // type safety guaranteed from template instantiation in BEChooser<>::GetFunc
 688     static const bool bInputCoverage = (bool)inputCoverage;
 689     static const bool bCentroidPos = (bool)centroidPos;
 690
 691     SWR_CONTEXT *pContext = pDC->pContext;
 692     const API_STATE& state = GetApiState(pDC);
 693     const SWR_RASTSTATE& rastState = state.rastState;
 694     const SWR_PS_STATE *pPSState = &state.psState;
 695     const SWR_BLEND_STATE *pBlendState = &state.blendState;
 696     const BACKEND_FUNCS& backendFuncs = pDC->pState->backendFuncs;
 697     uint64_t coverageMask = work.coverageMask[0];
 698
 699     // broadcast scalars
 700     BarycentricCoeffs coeffs;
 701     coeffs.vIa = _simd_broadcast_ss(&work.I[0]);
 702     coeffs.vIb = _simd_broadcast_ss(&work.I[1]);
 703     coeffs.vIc = _simd_broadcast_ss(&work.I[2]);
 704
 705     coeffs.vJa = _simd_broadcast_ss(&work.J[0]);
 706     coeffs.vJb = _simd_broadcast_ss(&work.J[1]);
 707     coeffs.vJc = _simd_broadcast_ss(&work.J[2]);
 708
 709     coeffs.vZa = _simd_broadcast_ss(&work.Z[0]);
 710     coeffs.vZb = _simd_broadcast_ss(&work.Z[1]);
 711     coeffs.vZc = _simd_broadcast_ss(&work.Z[2]);
 712
 713     coeffs.vRecipDet = _simd_broadcast_ss(&work.recipDet);
 714
 715     coeffs.vAOneOverW = _simd_broadcast_ss(&work.OneOverW[0]);
 716     coeffs.vBOneOverW = _simd_broadcast_ss(&work.OneOverW[1]);
 717     coeffs.vCOneOverW = _simd_broadcast_ss(&work.OneOverW[2]);
 718
 719     uint8_t *pColorBase[SWR_NUM_RENDERTARGETS];
 720     uint32_t NumRT = state.psState.numRenderTargets;
 721     for(uint32_t rt = 0; rt < NumRT; ++rt)
 722     {
 723         pColorBase[rt] = renderBuffers.pColor[rt];
 724     }
 725     uint8_t *pDepthBase = renderBuffers.pDepth, *pStencilBase = renderBuffers.pStencil;
 726     RDTSC_STOP(BESetup, 0, 0);
 727
 728     SWR_PS_CONTEXT psContext;
 729     psContext.pAttribs = work.pAttribs;
 730     psContext.pPerspAttribs = work.pPerspAttribs;
 731     psContext.frontFace = work.triFlags.frontFacing;
 732     psContext.primID = work.triFlags.primID;
 733
 734     // save Ia/Ib/Ic and Ja/Jb/Jc if we need to reevaluate i/j/k in the shader because of pull attribs
 735     psContext.I = work.I;
 736     psContext.J = work.J;
 737     psContext.recipDet = work.recipDet;
 738     psContext.pRecipW = work.pRecipW;
 739     psContext.pSamplePosX = (const float*)&MultisampleTraits<SWR_MULTISAMPLE_1X>::samplePosX;
 740     psContext.pSamplePosY = (const float*)&MultisampleTraits<SWR_MULTISAMPLE_1X>::samplePosY;
 741
 742     for(uint32_t yy = y; yy < y + KNOB_TILE_Y_DIM; yy += SIMD_TILE_Y_DIM)
 743     {
 744         // UL pixel corner
 745         psContext.vY.UL = _simd_add_ps(vULOffsetsY, _simd_set1_ps((float)yy));
 746         // pixel center
 747         psContext.vY.center = _simd_add_ps(vCenterOffsetsY, _simd_set1_ps((float)yy));
 748
 749         for(uint32_t xx = x; xx < x + KNOB_TILE_X_DIM; xx += SIMD_TILE_X_DIM)
 750         {
 751             if(bInputCoverage)
 752             {
 753                 generateInputCoverage<SWR_MULTISAMPLE_1X, SWR_MSAA_STANDARD_PATTERN, false>(&work.coverageMask[0], psContext.inputMask, pBlendState->sampleMask);
 754             }
 755
 756             if(coverageMask & MASK)
 757             {
 758                 RDTSC_START(BEBarycentric);
 759                 psContext.vX.UL = _simd_add_ps(vULOffsetsX, _simd_set1_ps((float)xx));
 760                 // pixel center
 761                 psContext.vX.center = _simd_add_ps(vCenterOffsetsX, _simd_set1_ps((float)xx));
 762
 763                 backendFuncs.pfnCalcPixelBarycentrics(coeffs, psContext);
 764
 765                 if(bCentroidPos)
 766                 {
 767                     // for 1x case, centroid is pixel center
 768                     psContext.vX.centroid = psContext.vX.center;
 769                     psContext.vY.centroid = psContext.vY.center;
 770                     psContext.vI.centroid = psContext.vI.center;
 771                     psContext.vJ.centroid = psContext.vJ.center;
 772                     psContext.vOneOverW.centroid = psContext.vOneOverW.center;
 773                 }
 774
 775                 // interpolate z
 776                 psContext.vZ = vplaneps(coeffs.vZa, coeffs.vZb, coeffs.vZc, psContext.vI.center, psContext.vJ.center);
 777                 RDTSC_STOP(BEBarycentric, 0, 0);
 778
 779                 simdmask clipCoverageMask = coverageMask & MASK;
 780
 781                 // interpolate user clip distance if available
 782                 if(rastState.clipDistanceMask)
 783                 {
 784                     clipCoverageMask &= ~ComputeUserClipMask(rastState.clipDistanceMask, work.pUserClipBuffer,
 785                                                              psContext.vI.center, psContext.vJ.center);
 786                 }
 787
 788                 simdscalar vCoverageMask = vMask(clipCoverageMask);
 789                 simdscalar depthPassMask = vCoverageMask;
 790                 simdscalar stencilPassMask = vCoverageMask;
 791
 792                 // Early-Z?
 793                 if(CanEarlyZ(pPSState))
 794                 {
 795                     RDTSC_START(BEEarlyDepthTest);
 796                     depthPassMask = DepthStencilTest(&state.vp[0], &state.depthStencilState, work.triFlags.frontFacing,
 797                                                         psContext.vZ, pDepthBase, vCoverageMask, pStencilBase, &stencilPassMask);
 798                     RDTSC_STOP(BEEarlyDepthTest, 0, 0);
 799
 800                     // early-exit if no pixels passed depth or earlyZ is forced on
 801                     if(pPSState->forceEarlyZ || !_simd_movemask_ps(depthPassMask))
 802                     {
 803                         DepthStencilWrite(&state.vp[0], &state.depthStencilState, work.triFlags.frontFacing, psContext.vZ,
 804                                             pDepthBase, depthPassMask, vCoverageMask, pStencilBase, stencilPassMask);
 805
 806                         if (!_simd_movemask_ps(depthPassMask))
 807                         {
 808                             goto Endtile;
 809                         }
 810                     }
 811                 }
 812
 813                 psContext.sampleIndex = 0;
 814                 psContext.activeMask = _simd_castps_si(vCoverageMask);
 815
 816                 // execute pixel shader
 817                 RDTSC_START(BEPixelShader);
 818                 UPDATE_STAT(PsInvocations, _mm_popcnt_u32(_simd_movemask_ps(vCoverageMask)));
 819                 state.psState.pfnPixelShader(GetPrivateState(pDC), &psContext);
 820                 RDTSC_STOP(BEPixelShader, 0, 0);
 821
 822                 vCoverageMask = _simd_castsi_ps(psContext.activeMask);
 823
 824                 // late-Z
 825                 if(!CanEarlyZ(pPSState))
 826                 {
 827                     RDTSC_START(BELateDepthTest);
 828                     depthPassMask = DepthStencilTest(&state.vp[0], &state.depthStencilState, work.triFlags.frontFacing,
 829                                                         psContext.vZ, pDepthBase, vCoverageMask, pStencilBase, &stencilPassMask);
 830                     RDTSC_STOP(BELateDepthTest, 0, 0);
 831
 832                     if(!_simd_movemask_ps(depthPassMask))
 833                     {
 834                         // need to call depth/stencil write for stencil write
 835                         DepthStencilWrite(&state.vp[0], &state.depthStencilState, work.triFlags.frontFacing, psContext.vZ,
 836                                             pDepthBase, depthPassMask, vCoverageMask, pStencilBase, stencilPassMask);
 837                         goto Endtile;
 838                     }
 839                 }
 840
 841                 uint32_t statMask = _simd_movemask_ps(depthPassMask);
 842                 uint32_t statCount = _mm_popcnt_u32(statMask);
 843                 UPDATE_STAT(DepthPassCount, statCount);
 844
 845                 // output merger
 846                 RDTSC_START(BEOutputMerger);
 847                 backendFuncs.pfnOutputMerger(psContext, pColorBase, 0, pBlendState, state.pfnBlendFunc,
 848                                              vCoverageMask, depthPassMask);
 849
 850                 // do final depth write after all pixel kills
 851                 if (!pPSState->forceEarlyZ)
 852                 {
 853                     DepthStencilWrite(&state.vp[0], &state.depthStencilState, work.triFlags.frontFacing, psContext.vZ,
 854                         pDepthBase, depthPassMask, vCoverageMask, pStencilBase, stencilPassMask);
 855                 }
 856                 RDTSC_STOP(BEOutputMerger, 0, 0);
 857             }
 858
 859 Endtile:
 860             RDTSC_START(BEEndTile);
 861             coverageMask >>= (SIMD_TILE_Y_DIM * SIMD_TILE_X_DIM);
 862             pDepthBase += (KNOB_SIMD_WIDTH * FormatTraits<KNOB_DEPTH_HOT_TILE_FORMAT>::bpp) / 8;
 863             pStencilBase += (KNOB_SIMD_WIDTH * FormatTraits<KNOB_STENCIL_HOT_TILE_FORMAT>::bpp) / 8;
 864
 865             for(uint32_t rt = 0; rt < NumRT; ++rt)
 866             {
 867                 pColorBase[rt] += (KNOB_SIMD_WIDTH * FormatTraits<KNOB_COLOR_HOT_TILE_FORMAT>::bpp) / 8;
 868             }
 869             RDTSC_STOP(BEEndTile, 0, 0);
 870         }
 871     }
 872 }
 873
 874 template<uint32_t sampleCountT, uint32_t samplePattern, uint32_t inputCoverage, uint32_t centroidPos, uint32_t forcedSampleCount>
 875 void BackendSampleRate(DRAW_CONTEXT *pDC, uint32_t workerId, uint32_t x, uint32_t y, SWR_TRIANGLE_DESC &work, RenderOutputBuffers &renderBuffers)
 876 {
 877     // type safety guaranteed from template instantiation in BEChooser<>::GetFunc
 878     static const SWR_MULTISAMPLE_COUNT sampleCount = (SWR_MULTISAMPLE_COUNT)sampleCountT;
 879     static const bool bInputCoverage = (bool)inputCoverage;
 880     static const bool bCentroidPos = (bool)centroidPos;
 881
 882     RDTSC_START(BESetup);
 883
 884     SWR_CONTEXT *pContext = pDC->pContext;
 885     const API_STATE& state = GetApiState(pDC);
 886     const SWR_RASTSTATE& rastState = state.rastState;
 887     const SWR_PS_STATE *pPSState = &state.psState;
 888     const SWR_BLEND_STATE *pBlendState = &state.blendState;
 889     const BACKEND_FUNCS& backendFuncs = pDC->pState->backendFuncs;
 890
 891     // broadcast scalars
 892     BarycentricCoeffs coeffs;
 893     coeffs.vIa = _simd_broadcast_ss(&work.I[0]);
 894     coeffs.vIb = _simd_broadcast_ss(&work.I[1]);
 895     coeffs.vIc = _simd_broadcast_ss(&work.I[2]);
 896
 897     coeffs.vJa = _simd_broadcast_ss(&work.J[0]);
 898     coeffs.vJb = _simd_broadcast_ss(&work.J[1]);
 899     coeffs.vJc = _simd_broadcast_ss(&work.J[2]);
 900
 901     coeffs.vZa = _simd_broadcast_ss(&work.Z[0]);
 902     coeffs.vZb = _simd_broadcast_ss(&work.Z[1]);
 903     coeffs.vZc = _simd_broadcast_ss(&work.Z[2]);
 904
 905     coeffs.vRecipDet = _simd_broadcast_ss(&work.recipDet);
 906
 907     coeffs.vAOneOverW = _simd_broadcast_ss(&work.OneOverW[0]);
 908     coeffs.vBOneOverW = _simd_broadcast_ss(&work.OneOverW[1]);
 909     coeffs.vCOneOverW = _simd_broadcast_ss(&work.OneOverW[2]);
 910
 911     uint8_t *pColorBase[SWR_NUM_RENDERTARGETS];
 912     uint32_t NumRT = state.psState.numRenderTargets;
 913     for(uint32_t rt = 0; rt < NumRT; ++rt)
 914     {
 915         pColorBase[rt] = renderBuffers.pColor[rt];
 916     }
 917     uint8_t *pDepthBase = renderBuffers.pDepth, *pStencilBase = renderBuffers.pStencil;
 918     RDTSC_STOP(BESetup, 0, 0);
 919
 920     SWR_PS_CONTEXT psContext;
 921     psContext.pAttribs = work.pAttribs;
 922     psContext.pPerspAttribs = work.pPerspAttribs;
 923     psContext.pRecipW = work.pRecipW;
 924     psContext.frontFace = work.triFlags.frontFacing;
 925     psContext.primID = work.triFlags.primID;
 926
 927     // save Ia/Ib/Ic and Ja/Jb/Jc if we need to reevaluate i/j/k in the shader because of pull attribs
 928     psContext.I = work.I;
 929     psContext.J = work.J;
 930     psContext.recipDet = work.recipDet;
 931     psContext.pSamplePosX = (const float*)&MultisampleTraits<sampleCount>::samplePosX;
 932     psContext.pSamplePosY = (const float*)&MultisampleTraits<sampleCount>::samplePosY;
 933     const uint32_t numSamples = MultisampleTraits<sampleCount>::numSamples;
 934
 935     for (uint32_t yy = y; yy < y + KNOB_TILE_Y_DIM; yy += SIMD_TILE_Y_DIM)
 936     {
 937         // UL pixel corner
 938         psContext.vY.UL = _simd_add_ps(vULOffsetsY, _simd_set1_ps((float)yy));
 939         // pixel center
 940         psContext.vY.center = _simd_add_ps(vCenterOffsetsY, _simd_set1_ps((float)yy));
 941
 942         for (uint32_t xx = x; xx < x + KNOB_TILE_X_DIM; xx += SIMD_TILE_X_DIM)
 943         {
 944             psContext.vX.UL = _simd_add_ps(vULOffsetsX, _simd_set1_ps((float)xx));
 945             // pixel center
 946             psContext.vX.center = _simd_add_ps(vCenterOffsetsX, _simd_set1_ps((float)xx));
 947
 948             RDTSC_START(BEBarycentric);
 949             backendFuncs.pfnCalcPixelBarycentrics(coeffs, psContext);
 950             RDTSC_STOP(BEBarycentric, 0, 0);
 951
 952             if(bInputCoverage)
 953             {
 954                 generateInputCoverage<sampleCount, SWR_MSAA_STANDARD_PATTERN, false>(&work.coverageMask[0], psContext.inputMask, pBlendState->sampleMask);
 955             }
 956
 957             if(bCentroidPos)
 958             {
 959                 ///@ todo: don't need to genererate input coverage 2x if input coverage and centroid
 960                 RDTSC_START(BEBarycentric);
 961                 backendFuncs.pfnCalcCentroidBarycentrics(coeffs, psContext, &work.coverageMask[0], pBlendState->sampleMask, psContext.vX.UL, psContext.vY.UL);
 962                 RDTSC_STOP(BEBarycentric, 0, 0);
 963             }
 964
 965             for(uint32_t sample = 0; sample < numSamples; sample++)
 966             {
 967                 if (work.coverageMask[sample] & MASK)
 968                 {
 969                     RDTSC_START(BEBarycentric);
 970
 971                     // calculate per sample positions
 972                     psContext.vX.sample = _simd_add_ps(psContext.vX.UL, MultisampleTraits<sampleCount>::vX(sample));
 973                     psContext.vY.sample = _simd_add_ps(psContext.vY.UL, MultisampleTraits<sampleCount>::vY(sample));
 974
 975                     simdmask coverageMask = work.coverageMask[sample] & MASK;
 976                     simdscalar vCoverageMask = vMask(coverageMask);
 977
 978                     backendFuncs.pfnCalcSampleBarycentrics(coeffs, psContext);
 979
 980                     // interpolate z
 981                     psContext.vZ = vplaneps(coeffs.vZa, coeffs.vZb, coeffs.vZc, psContext.vI.sample, psContext.vJ.sample);
 982
 983                     RDTSC_STOP(BEBarycentric, 0, 0);
 984
 985                     // interpolate user clip distance if available
 986                     if (rastState.clipDistanceMask)
 987                     {
 988                         coverageMask &= ~ComputeUserClipMask(rastState.clipDistanceMask, work.pUserClipBuffer,
 989                             psContext.vI.sample, psContext.vJ.sample);
 990                     }
 991
 992                     simdscalar depthPassMask = vCoverageMask;
 993                     simdscalar stencilPassMask = vCoverageMask;
 994
 995                     // offset depth/stencil buffers current sample
 996                     uint8_t *pDepthSample = pDepthBase + MultisampleTraits<sampleCount>::RasterTileDepthOffset(sample);
 997                     uint8_t *pStencilSample = pStencilBase + MultisampleTraits<sampleCount>::RasterTileStencilOffset(sample);
 998
 999                     // Early-Z?
1000                     if (CanEarlyZ(pPSState))
1001                     {
1002                         RDTSC_START(BEEarlyDepthTest);
1003                         depthPassMask = DepthStencilTest(&state.vp[0], &state.depthStencilState, work.triFlags.frontFacing,
1004                                               psContext.vZ, pDepthSample, vCoverageMask, pStencilSample, &stencilPassMask);
1005                         RDTSC_STOP(BEEarlyDepthTest, 0, 0);
1006
1007                         // early-exit if no samples passed depth or earlyZ is forced on.
1008                         if (pPSState->forceEarlyZ || !_simd_movemask_ps(depthPassMask))
1009                         {
1010                             DepthStencilWrite(&state.vp[0], &state.depthStencilState, work.triFlags.frontFacing, psContext.vZ,
1011                                 pDepthSample, depthPassMask, vCoverageMask, pStencilSample, stencilPassMask);
1012
1013                             if (!_simd_movemask_ps(depthPassMask))
1014                             {
1015                                 work.coverageMask[sample] >>= (SIMD_TILE_Y_DIM * SIMD_TILE_X_DIM);
1016                                 continue;
1017                             }
1018                         }
1019                     }
1020
1021                     psContext.sampleIndex = sample;
1022                     psContext.activeMask = _simd_castps_si(vCoverageMask);
1023
1024                     // execute pixel shader
1025                     RDTSC_START(BEPixelShader);
1026                     UPDATE_STAT(PsInvocations, _mm_popcnt_u32(_simd_movemask_ps(vCoverageMask)));
1027                     state.psState.pfnPixelShader(GetPrivateState(pDC), &psContext);
1028                     RDTSC_STOP(BEPixelShader, 0, 0);
1029
1030                     vCoverageMask = _simd_castsi_ps(psContext.activeMask);
1031
1032                     //// late-Z
1033                     if (!CanEarlyZ(pPSState))
1034                     {
1035                         RDTSC_START(BELateDepthTest);
1036                         depthPassMask = DepthStencilTest(&state.vp[0], &state.depthStencilState, work.triFlags.frontFacing,
1037                                               psContext.vZ, pDepthSample, vCoverageMask, pStencilSample, &stencilPassMask);
1038                         RDTSC_STOP(BELateDepthTest, 0, 0);
1039
1040                         if (!_simd_movemask_ps(depthPassMask))
1041                         {
1042                             // need to call depth/stencil write for stencil write
1043                             DepthStencilWrite(&state.vp[0], &state.depthStencilState, work.triFlags.frontFacing, psContext.vZ,
1044                                 pDepthSample, depthPassMask, vCoverageMask, pStencilSample, stencilPassMask);
1045
1046                             work.coverageMask[sample] >>= (SIMD_TILE_Y_DIM * SIMD_TILE_X_DIM);
1047                             continue;
1048                         }
1049                     }
1050
1051                     uint32_t statMask = _simd_movemask_ps(depthPassMask);
1052                     uint32_t statCount = _mm_popcnt_u32(statMask);
1053                     UPDATE_STAT(DepthPassCount, statCount);
1054
1055                     // output merger
1056                     RDTSC_START(BEOutputMerger);
1057                     backendFuncs.pfnOutputMerger(psContext, pColorBase, sample, pBlendState, state.pfnBlendFunc,
1058                                                  vCoverageMask, depthPassMask);
1059
1060                     // do final depth write after all pixel kills
1061                     if (!pPSState->forceEarlyZ)
1062                     {
1063                         DepthStencilWrite(&state.vp[0], &state.depthStencilState, work.triFlags.frontFacing, psContext.vZ,
1064                             pDepthSample, depthPassMask, vCoverageMask, pStencilSample, stencilPassMask);
1065                     }
1066                     RDTSC_STOP(BEOutputMerger, 0, 0);
1067                 }
1068                 work.coverageMask[sample] >>= (SIMD_TILE_Y_DIM * SIMD_TILE_X_DIM);
1069             }
1070             RDTSC_START(BEEndTile);
1071             pDepthBase += (KNOB_SIMD_WIDTH * FormatTraits<KNOB_DEPTH_HOT_TILE_FORMAT>::bpp) / 8;
1072             pStencilBase += (KNOB_SIMD_WIDTH * FormatTraits<KNOB_STENCIL_HOT_TILE_FORMAT>::bpp) / 8;
1073
1074             for (uint32_t rt = 0; rt < NumRT; ++rt)
1075             {
1076                 pColorBase[rt] += (KNOB_SIMD_WIDTH * FormatTraits<KNOB_COLOR_HOT_TILE_FORMAT>::bpp) / 8;
1077             }
1078             RDTSC_STOP(BEEndTile, 0, 0);
1079         }
1080     }
1081 }
1082
1083 template<uint32_t sampleCountT, uint32_t samplePattern, uint32_t inputCoverage, uint32_t centroidPos, uint32_t forcedSampleCount>
1084 void BackendPixelRate(DRAW_CONTEXT *pDC, uint32_t workerId, uint32_t x, uint32_t y, SWR_TRIANGLE_DESC &work, RenderOutputBuffers &renderBuffers)
1085 {
1086     // type safety guaranteed from template instantiation in BEChooser<>::GetFunc
1087     static const SWR_MULTISAMPLE_COUNT sampleCount = (SWR_MULTISAMPLE_COUNT)sampleCountT;
1088     static const bool bIsStandardPattern = (bool)samplePattern;
1089     static const bool bInputCoverage = (bool)inputCoverage;
1090     static const bool bCentroidPos = (bool)centroidPos;
1091     static const bool bForcedSampleCount = (bool)forcedSampleCount;
1092
1093     RDTSC_START(BESetup);
1094
1095     SWR_CONTEXT *pContext = pDC->pContext;
1096     const API_STATE& state = GetApiState(pDC);
1097     const SWR_RASTSTATE& rastState = state.rastState;
1098     const SWR_PS_STATE *pPSState = &state.psState;
1099     const SWR_BLEND_STATE *pBlendState = &state.blendState;
1100     const BACKEND_FUNCS& backendFuncs = pDC->pState->backendFuncs;
1101
1102     // broadcast scalars
1103     BarycentricCoeffs coeffs;
1104     coeffs.vIa = _simd_broadcast_ss(&work.I[0]);
1105     coeffs.vIb = _simd_broadcast_ss(&work.I[1]);
1106     coeffs.vIc = _simd_broadcast_ss(&work.I[2]);
1107
1108     coeffs.vJa = _simd_broadcast_ss(&work.J[0]);
1109     coeffs.vJb = _simd_broadcast_ss(&work.J[1]);
1110     coeffs.vJc = _simd_broadcast_ss(&work.J[2]);
1111
1112     coeffs.vZa = _simd_broadcast_ss(&work.Z[0]);
1113     coeffs.vZb = _simd_broadcast_ss(&work.Z[1]);
1114     coeffs.vZc = _simd_broadcast_ss(&work.Z[2]);
1115
1116     coeffs.vRecipDet = _simd_broadcast_ss(&work.recipDet);
1117
1118     coeffs.vAOneOverW = _simd_broadcast_ss(&work.OneOverW[0]);
1119     coeffs.vBOneOverW = _simd_broadcast_ss(&work.OneOverW[1]);
1120     coeffs.vCOneOverW = _simd_broadcast_ss(&work.OneOverW[2]);
1121
1122     uint8_t *pColorBase[SWR_NUM_RENDERTARGETS];
1123     uint32_t NumRT = state.psState.numRenderTargets;
1124     for(uint32_t rt = 0; rt < NumRT; ++rt)
1125     {
1126         pColorBase[rt] = renderBuffers.pColor[rt];
1127     }
1128     uint8_t *pDepthBase = renderBuffers.pDepth, *pStencilBase = renderBuffers.pStencil;
1129     RDTSC_STOP(BESetup, 0, 0);
1130
1131     SWR_PS_CONTEXT psContext;
1132     psContext.pAttribs = work.pAttribs;
1133     psContext.pPerspAttribs = work.pPerspAttribs;
1134     psContext.frontFace = work.triFlags.frontFacing;
1135     psContext.primID = work.triFlags.primID;
1136     psContext.pRecipW = work.pRecipW;
1137     // save Ia/Ib/Ic and Ja/Jb/Jc if we need to reevaluate i/j/k in the shader because of pull attribs
1138     psContext.I = work.I;
1139     psContext.J = work.J;
1140     psContext.recipDet = work.recipDet;
1141     psContext.pSamplePosX = (const float*)&MultisampleTraits<sampleCount>::samplePosX;
1142     psContext.pSamplePosY = (const float*)&MultisampleTraits<sampleCount>::samplePosY;
1143     psContext.sampleIndex = 0;
1144
1145     uint32_t numCoverageSamples;
1146     if(bIsStandardPattern)
1147     {
1148         numCoverageSamples = MultisampleTraits<sampleCount>::numSamples;
1149     }
1150     else
1151     {
1152         numCoverageSamples = 1;
1153     }
1154
1155     uint32_t numOMSamples;
1156     // RT has to be single sample if we're in forcedMSAA mode
1157     if(bForcedSampleCount && (sampleCount > SWR_MULTISAMPLE_1X))
1158     {
1159         numOMSamples = 1;
1160     }
1161     // unless we're forced to single sample, in which case we run the OM at the sample count of the RT
1162     else if(bForcedSampleCount && (sampleCount == SWR_MULTISAMPLE_1X))
1163     {
1164         numOMSamples = GetNumSamples(pBlendState->sampleCount);
1165     }
1166     // else we're in normal MSAA mode and rasterizer and OM are running at the same sample count
1167     else
1168     {
1169         numOMSamples = MultisampleTraits<sampleCount>::numSamples;
1170     }
1171
1172     for(uint32_t yy = y; yy < y + KNOB_TILE_Y_DIM; yy += SIMD_TILE_Y_DIM)
1173     {
1174         psContext.vY.UL = _simd_add_ps(vULOffsetsY, _simd_set1_ps((float)yy));
1175         psContext.vY.center = _simd_add_ps(vCenterOffsetsY, _simd_set1_ps((float)yy));
1176         for(uint32_t xx = x; xx < x + KNOB_TILE_X_DIM; xx += SIMD_TILE_X_DIM)
1177         {
1178             simdscalar vZ[MultisampleTraits<sampleCount>::numSamples]{ 0 };
1179             psContext.vX.UL = _simd_add_ps(vULOffsetsX, _simd_set1_ps((float)xx));
1180             // set pixel center positions
1181             psContext.vX.center = _simd_add_ps(vCenterOffsetsX, _simd_set1_ps((float)xx));
1182
1183             if (bInputCoverage)
1184             {
1185                 generateInputCoverage<sampleCount, bIsStandardPattern, bForcedSampleCount>(&work.coverageMask[0], psContext.inputMask, pBlendState->sampleMask);
1186             }
1187
1188             if(bCentroidPos)
1189             {
1190                 ///@ todo: don't need to genererate input coverage 2x if input coverage and centroid
1191                 RDTSC_START(BEBarycentric);
1192                 backendFuncs.pfnCalcCentroidBarycentrics(coeffs, psContext, &work.coverageMask[0], pBlendState->sampleMask, psContext.vX.UL, psContext.vY.UL);
1193                 RDTSC_STOP(BEBarycentric, 0, 0);
1194             }
1195
1196             // if oDepth written to, or there is a potential to discard any samples, we need to
1197             // run the PS early, then interp or broadcast Z and test
1198             if(pPSState->writesODepth || pPSState->killsPixel)
1199             {
1200                 RDTSC_START(BEBarycentric);
1201                 backendFuncs.pfnCalcPixelBarycentrics(coeffs, psContext);
1202
1203                 // interpolate z
1204                 psContext.vZ = vplaneps(coeffs.vZa, coeffs.vZb, coeffs.vZc, psContext.vI.center, psContext.vJ.center);
1205                 RDTSC_STOP(BEBarycentric, 0, 0);
1206
1207                 // execute pixel shader
1208                 RDTSC_START(BEPixelShader);
1209                 state.psState.pfnPixelShader(GetPrivateState(pDC), &psContext);
1210                 RDTSC_STOP(BEPixelShader, 0, 0);
1211             }
1212             else
1213             {
1214                                 psContext.activeMask = _simd_set1_epi32(-1);
1215             }
1216
1217             // need to declare enough space for all samples
1218             simdscalar vCoverageMask[MultisampleTraits<sampleCount>::numSamples];
1219             simdscalar depthPassMask[MultisampleTraits<sampleCount>::numSamples];
1220             simdscalar stencilPassMask[MultisampleTraits<sampleCount>::numSamples];
1221             simdscalar anyDepthSamplePassed = _simd_setzero_ps();
1222             simdscalar anyStencilSamplePassed = _simd_setzero_ps();
1223             for(uint32_t sample = 0; sample < numCoverageSamples; sample++)
1224             {
1225                 vCoverageMask[sample] = vMask(work.coverageMask[sample] & MASK);
1226
1227                 // pull mask back out for any discards and and with coverage
1228                 vCoverageMask[sample] = _simd_and_ps(vCoverageMask[sample], _simd_castsi_ps(psContext.activeMask));
1229
1230                 if (!_simd_movemask_ps(vCoverageMask[sample]))
1231                 {
1232                     vCoverageMask[sample] = depthPassMask[sample] = stencilPassMask[sample] =  _simd_setzero_ps();
1233                     continue;
1234                 }
1235
1236                 if(bForcedSampleCount)
1237                 {
1238                     // candidate pixels (that passed coverage) will cause shader invocation if any bits in the samplemask are set
1239                     const simdscalar vSampleMask = _simd_castsi_ps(_simd_cmpgt_epi32(_simd_set1_epi32(pBlendState->sampleMask), _simd_setzero_si()));
1240                     anyDepthSamplePassed = _simd_or_ps(anyDepthSamplePassed, _simd_and_ps(vCoverageMask[sample], vSampleMask));
1241                     continue;
1242                 }
1243
1244                 depthPassMask[sample] = vCoverageMask[sample];
1245
1246                 // if oDepth isn't written to, we need to interpolate Z for each sample
1247                 // if clip distances are enabled, we need to interpolate for each sample
1248                 if(!pPSState->writesODepth || rastState.clipDistanceMask)
1249                 {
1250                     RDTSC_START(BEBarycentric);
1251                     if(bIsStandardPattern)
1252                     {
1253                         // calculate per sample positions
1254                         psContext.vX.sample = _simd_add_ps(psContext.vX.UL, MultisampleTraits<sampleCount>::vX(sample));
1255                         psContext.vY.sample = _simd_add_ps(psContext.vY.UL, MultisampleTraits<sampleCount>::vY(sample));
1256                     }
1257                     else
1258                     {
1259                         psContext.vX.sample = psContext.vX.center;
1260                         psContext.vY.sample = psContext.vY.center;
1261                     }
1262
1263                     // calc I & J per sample
1264                     backendFuncs.pfnCalcSampleBarycentrics(coeffs, psContext);
1265
1266                     // interpolate z
1267                     if (!pPSState->writesODepth)
1268                     {
1269                         vZ[sample] = vplaneps(coeffs.vZa, coeffs.vZb, coeffs.vZc, psContext.vI.sample, psContext.vJ.sample);
1270                     }
1271
1272                     ///@todo: perspective correct vs non-perspective correct clipping?
1273                     // interpolate clip distances
1274                     if (rastState.clipDistanceMask)
1275                     {
1276                         uint8_t clipMask = ComputeUserClipMask(rastState.clipDistanceMask, work.pUserClipBuffer,
1277                             psContext.vI.sample, psContext.vJ.sample);
1278                         vCoverageMask[sample] = _simd_and_ps(vCoverageMask[sample], vMask(~clipMask));
1279                     }
1280                     RDTSC_STOP(BEBarycentric, 0, 0);
1281                 }
1282                 // else 'broadcast' and test psContext.vZ written from the PS each sample
1283                 else
1284                 {
1285                     vZ[sample] = psContext.vZ;
1286                 }
1287
1288                 // offset depth/stencil buffers current sample
1289                 uint8_t *pDepthSample = pDepthBase + MultisampleTraits<sampleCount>::RasterTileDepthOffset(sample);
1290                 uint8_t * pStencilSample = pStencilBase + MultisampleTraits<sampleCount>::RasterTileStencilOffset(sample);
1291
1292                 // ZTest for this sample
1293                 RDTSC_START(BEEarlyDepthTest);
1294                 stencilPassMask[sample] = vCoverageMask[sample];
1295                 depthPassMask[sample] = DepthStencilTest(&state.vp[0], &state.depthStencilState, work.triFlags.frontFacing,
1296                                         vZ[sample], pDepthSample, vCoverageMask[sample], pStencilSample, &stencilPassMask[sample]);
1297                 RDTSC_STOP(BEEarlyDepthTest, 0, 0);
1298
1299                 anyDepthSamplePassed = _simd_or_ps(anyDepthSamplePassed, depthPassMask[sample]);
1300                 anyStencilSamplePassed = _simd_or_ps(anyStencilSamplePassed, stencilPassMask[sample]);
1301                 uint32_t statMask = _simd_movemask_ps(depthPassMask[sample]);
1302                 uint32_t statCount = _mm_popcnt_u32(statMask);
1303                 UPDATE_STAT(DepthPassCount, statCount);
1304             }
1305
1306             // if we didn't have to execute the PS early, and at least 1 sample passed the depth test, run the PS
1307             if(!pPSState->writesODepth && !pPSState->killsPixel && _simd_movemask_ps(anyDepthSamplePassed))
1308             {
1309                 RDTSC_START(BEBarycentric);
1310                 backendFuncs.pfnCalcPixelBarycentrics(coeffs, psContext);
1311                 // interpolate z
1312                 psContext.vZ = vplaneps(coeffs.vZa, coeffs.vZb, coeffs.vZc, psContext.vI.center, psContext.vJ.center);
1313                 RDTSC_STOP(BEBarycentric, 0, 0);
1314
1315                 // execute pixel shader
1316                 RDTSC_START(BEPixelShader);
1317                 state.psState.pfnPixelShader(GetPrivateState(pDC), &psContext);
1318                 RDTSC_STOP(BEPixelShader, 0, 0);
1319             }
1320             ///@todo: make sure this works for kill pixel
1321             else if(!_simd_movemask_ps(anyStencilSamplePassed))
1322             {
1323                 goto Endtile;
1324             }
1325
1326             // loop over all samples, broadcasting the results of the PS to all passing pixels
1327             for(uint32_t sample = 0; sample < numOMSamples; sample++)
1328             {
1329                 uint8_t *pDepthSample = pDepthBase + MultisampleTraits<sampleCount>::RasterTileDepthOffset(sample);
1330                 uint8_t * pStencilSample = pStencilBase + MultisampleTraits<sampleCount>::RasterTileStencilOffset(sample);
1331
1332                 // output merger
1333                 RDTSC_START(BEOutputMerger);
1334
1335                 // skip if none of the pixels for this sample passed
1336                 simdscalar coverageMaskSample;
1337                 simdscalar depthMaskSample;
1338                 simdscalar stencilMaskSample;
1339                 simdscalar vInterpolatedZ;
1340
1341                 // forcedSampleCount outputs to any pixels with covered samples not masked off by SampleMask
1342                 // depth test is disabled, so just set the z val to 0.
1343                 if(bForcedSampleCount)
1344                 {
1345                     coverageMaskSample = depthMaskSample = anyDepthSamplePassed;
1346                     vInterpolatedZ = _simd_setzero_ps();
1347                 }
1348                 else if(bIsStandardPattern)
1349                 {
1350                     if(!_simd_movemask_ps(depthPassMask[sample]))
1351                     {
1352                         depthPassMask[sample] = _simd_setzero_ps();
1353                         DepthStencilWrite(&state.vp[0], &state.depthStencilState, work.triFlags.frontFacing, vZ[sample], pDepthSample, depthPassMask[sample],
1354                                           vCoverageMask[sample], pStencilSample, stencilPassMask[sample]);
1355                         continue;
1356                     }
1357                     coverageMaskSample = vCoverageMask[sample];
1358                     depthMaskSample = depthPassMask[sample];
1359                     stencilMaskSample = stencilPassMask[sample];
1360                     vInterpolatedZ = vZ[sample];
1361                 }
1362                 else
1363                 {
1364                     // center pattern only needs to use a single depth test as all samples are at the same position
1365                     if(!_simd_movemask_ps(depthPassMask[0]))
1366                     {
1367                         depthPassMask[0] = _simd_setzero_ps();
1368                         DepthStencilWrite(&state.vp[0], &state.depthStencilState, work.triFlags.frontFacing, vZ[0], pDepthSample, depthPassMask[0],
1369                                           vCoverageMask[0], pStencilSample, stencilPassMask[0]);
1370                         continue;
1371                     }
1372                     coverageMaskSample = (vCoverageMask[0]);
1373                     depthMaskSample = depthPassMask[0];
1374                     stencilMaskSample = stencilPassMask[0];
1375                     vInterpolatedZ = vZ[0];
1376                 }
1377
1378                 // output merger
1379                 RDTSC_START(BEOutputMerger);
1380                 backendFuncs.pfnOutputMerger(psContext, pColorBase, sample, pBlendState, state.pfnBlendFunc,
1381                                              coverageMaskSample, depthMaskSample);
1382
1383                 DepthStencilWrite(&state.vp[0], &state.depthStencilState, work.triFlags.frontFacing, vInterpolatedZ, pDepthSample, depthMaskSample,
1384                                   coverageMaskSample, pStencilSample, stencilMaskSample);
1385                 RDTSC_STOP(BEOutputMerger, 0, 0);
1386             }
1387
1388 Endtile:
1389             RDTSC_START(BEEndTile);
1390             for(uint32_t sample = 0; sample < numCoverageSamples; sample++)
1391             {
1392                 work.coverageMask[sample] >>= (SIMD_TILE_Y_DIM * SIMD_TILE_X_DIM);
1393             }
1394
1395             pDepthBase += (KNOB_SIMD_WIDTH * FormatTraits<KNOB_DEPTH_HOT_TILE_FORMAT>::bpp) / 8;
1396             pStencilBase += (KNOB_SIMD_WIDTH * FormatTraits<KNOB_STENCIL_HOT_TILE_FORMAT>::bpp) / 8;
1397
1398             for(uint32_t rt = 0; rt < NumRT; ++rt)
1399             {
1400                 pColorBase[rt] += (KNOB_SIMD_WIDTH * FormatTraits<KNOB_COLOR_HOT_TILE_FORMAT>::bpp) / 8;
1401             }
1402             RDTSC_STOP(BEEndTile, 0, 0);
1403         }
1404     }
1405 }
1406 // optimized backend flow with NULL PS
1407 template<uint32_t sampleCountT>
1408 void BackendNullPS(DRAW_CONTEXT *pDC, uint32_t workerId, uint32_t x, uint32_t y, SWR_TRIANGLE_DESC &work, RenderOutputBuffers &renderBuffers)
1409 {
1410     RDTSC_START(BESetup);
1411
1412     static const SWR_MULTISAMPLE_COUNT sampleCount = (SWR_MULTISAMPLE_COUNT)sampleCountT;
1413     SWR_CONTEXT *pContext = pDC->pContext;
1414     const API_STATE& state = GetApiState(pDC);
1415     const BACKEND_FUNCS& backendFuncs = pDC->pState->backendFuncs;
1416
1417     // broadcast scalars
1418     BarycentricCoeffs coeffs;
1419     coeffs.vIa = _simd_broadcast_ss(&work.I[0]);
1420     coeffs.vIb = _simd_broadcast_ss(&work.I[1]);
1421     coeffs.vIc = _simd_broadcast_ss(&work.I[2]);
1422
1423     coeffs.vJa = _simd_broadcast_ss(&work.J[0]);
1424     coeffs.vJb = _simd_broadcast_ss(&work.J[1]);
1425     coeffs.vJc = _simd_broadcast_ss(&work.J[2]);
1426
1427     coeffs.vZa = _simd_broadcast_ss(&work.Z[0]);
1428     coeffs.vZb = _simd_broadcast_ss(&work.Z[1]);
1429     coeffs.vZc = _simd_broadcast_ss(&work.Z[2]);
1430
1431     coeffs.vRecipDet = _simd_broadcast_ss(&work.recipDet);
1432
1433     uint8_t *pDepthBase = renderBuffers.pDepth, *pStencilBase = renderBuffers.pStencil;
1434
1435     RDTSC_STOP(BESetup, 0, 0);
1436
1437     SWR_PS_CONTEXT psContext;
1438     for (uint32_t yy = y; yy < y + KNOB_TILE_Y_DIM; yy += SIMD_TILE_Y_DIM)
1439     {
1440         // UL pixel corner
1441         simdscalar vYSamplePosUL = _simd_add_ps(vULOffsetsY, _simd_set1_ps((float)yy));
1442
1443         for (uint32_t xx = x; xx < x + KNOB_TILE_X_DIM; xx += SIMD_TILE_X_DIM)
1444         {
1445             // UL pixel corners
1446             simdscalar vXSamplePosUL = _simd_add_ps(vULOffsetsX, _simd_set1_ps((float)xx));
1447
1448             // iterate over active samples
1449             unsigned long sample = 0;
1450             uint32_t sampleMask = state.blendState.sampleMask;
1451             while (_BitScanForward(&sample, sampleMask))
1452             {
1453                 sampleMask &= ~(1 << sample);
1454                 if (work.coverageMask[sample] & MASK)
1455                 {
1456                     RDTSC_START(BEBarycentric);
1457                     // calculate per sample positions
1458                     psContext.vX.sample = _simd_add_ps(vXSamplePosUL, MultisampleTraits<sampleCount>::vX(sample));
1459                     psContext.vY.sample = _simd_add_ps(vYSamplePosUL, MultisampleTraits<sampleCount>::vY(sample));
1460
1461                     backendFuncs.pfnCalcSampleBarycentrics(coeffs, psContext);
1462
1463                     // interpolate z
1464                     psContext.vZ = vplaneps(coeffs.vZa, coeffs.vZb, coeffs.vZc, psContext.vI.sample, psContext.vJ.sample);
1465
1466                     RDTSC_STOP(BEBarycentric, 0, 0);
1467
1468                     simdscalar vCoverageMask = vMask(work.coverageMask[sample] & MASK);
1469                     simdscalar stencilPassMask = vCoverageMask;
1470
1471                     // offset depth/stencil buffers current sample
1472                     uint8_t *pDepthSample = pDepthBase + MultisampleTraits<sampleCount>::RasterTileDepthOffset(sample);
1473                     uint8_t *pStencilSample = pStencilBase + MultisampleTraits<sampleCount>::RasterTileStencilOffset(sample);
1474
1475                     RDTSC_START(BEEarlyDepthTest);
1476                     simdscalar depthPassMask = DepthStencilTest(&state.vp[0], &state.depthStencilState, work.triFlags.frontFacing,
1477                         psContext.vZ, pDepthSample, vCoverageMask, pStencilSample, &stencilPassMask);
1478                     DepthStencilWrite(&state.vp[0], &state.depthStencilState, work.triFlags.frontFacing, psContext.vZ,
1479                         pDepthSample, depthPassMask, vCoverageMask, pStencilSample, stencilPassMask);
1480                     RDTSC_STOP(BEEarlyDepthTest, 0, 0);
1481
1482                     uint32_t statMask = _simd_movemask_ps(depthPassMask);
1483                     uint32_t statCount = _mm_popcnt_u32(statMask);
1484                     UPDATE_STAT(DepthPassCount, statCount);
1485                 }
1486                 work.coverageMask[sample] >>= (SIMD_TILE_Y_DIM * SIMD_TILE_X_DIM);
1487             }
1488             pDepthBase += (KNOB_SIMD_WIDTH * FormatTraits<KNOB_DEPTH_HOT_TILE_FORMAT>::bpp) / 8;
1489             pStencilBase += (KNOB_SIMD_WIDTH * FormatTraits<KNOB_STENCIL_HOT_TILE_FORMAT>::bpp) / 8;
1490         }
1491     }
1492 }
1493
1494 void InitClearTilesTable()
1495 {
1496     memset(sClearTilesTable, 0, sizeof(sClearTilesTable));
1497
1498     sClearTilesTable[R8G8B8A8_UNORM] = ClearMacroTile<R8G8B8A8_UNORM>;
1499     sClearTilesTable[B8G8R8A8_UNORM] = ClearMacroTile<B8G8R8A8_UNORM>;
1500     sClearTilesTable[R32_FLOAT] = ClearMacroTile<R32_FLOAT>;
1501     sClearTilesTable[R32G32B32A32_FLOAT] = ClearMacroTile<R32G32B32A32_FLOAT>;
1502     sClearTilesTable[R8_UINT] = ClearMacroTile<R8_UINT>;
1503 }
1504
1505 PFN_BACKEND_FUNC gBackendNullPs[SWR_MULTISAMPLE_TYPE_MAX];
1506 PFN_BACKEND_FUNC gBackendSingleSample[2][2] = {};
1507 PFN_BACKEND_FUNC gBackendPixelRateTable[SWR_MULTISAMPLE_TYPE_MAX][SWR_MSAA_SAMPLE_PATTERN_MAX][SWR_INPUT_COVERAGE_MAX][2][2] = {};
1508 PFN_BACKEND_FUNC gBackendSampleRateTable[SWR_MULTISAMPLE_TYPE_MAX][SWR_INPUT_COVERAGE_MAX][2] = {};
1509 PFN_OUTPUT_MERGER gBackendOutputMergerTable[SWR_NUM_RENDERTARGETS+1][SWR_MULTISAMPLE_TYPE_MAX] = {};
1510 PFN_CALC_PIXEL_BARYCENTRICS gPixelBarycentricTable[2] = {};
1511 PFN_CALC_SAMPLE_BARYCENTRICS gSampleBarycentricTable[2] = {};
1512 PFN_CALC_CENTROID_BARYCENTRICS gCentroidBarycentricTable[SWR_MULTISAMPLE_TYPE_MAX][2][2][2] = {};
1513
1514 // Recursive template used to auto-nest conditionals.  Converts dynamic enum function
1515 // arguments to static template arguments.
1516 template <uint32_t... ArgsT>
1517 struct OMChooser
1518 {
1519     // Last Arg Terminator
1520     static PFN_OUTPUT_MERGER GetFunc(SWR_MULTISAMPLE_COUNT tArg)
1521     {
1522         switch(tArg)
1523         {
1524         case SWR_MULTISAMPLE_1X: return OutputMerger<ArgsT..., SWR_MULTISAMPLE_1X>; break;
1525         case SWR_MULTISAMPLE_2X: return OutputMerger<ArgsT..., SWR_MULTISAMPLE_2X>; break;
1526         case SWR_MULTISAMPLE_4X: return OutputMerger<ArgsT..., SWR_MULTISAMPLE_4X>; break;
1527         case SWR_MULTISAMPLE_8X: return OutputMerger<ArgsT..., SWR_MULTISAMPLE_8X>; break;
1528         case SWR_MULTISAMPLE_16X: return OutputMerger<ArgsT..., SWR_MULTISAMPLE_16X>; break;
1529         default:
1530             SWR_ASSERT(0 && "Invalid sample count\n");
1531             return nullptr;
1532             break;
1533         }
1534     }
1535
1536     // Recursively parse args
1537     template <typename... TArgsT>
1538     static PFN_OUTPUT_MERGER GetFunc(uint32_t tArg, TArgsT... remainingArgs)
1539     {
1540         switch(tArg)
1541         {
1542         case 0: return OMChooser<ArgsT..., 0>::GetFunc(remainingArgs...); break;
1543         case 1: return OMChooser<ArgsT..., 1>::GetFunc(remainingArgs...); break;
1544         case 2: return OMChooser<ArgsT..., 2>::GetFunc(remainingArgs...); break;
1545         case 3: return OMChooser<ArgsT..., 3>::GetFunc(remainingArgs...); break;
1546         case 4: return OMChooser<ArgsT..., 4>::GetFunc(remainingArgs...); break;
1547         case 5: return OMChooser<ArgsT..., 5>::GetFunc(remainingArgs...); break;
1548         case 6: return OMChooser<ArgsT..., 6>::GetFunc(remainingArgs...); break;
1549         case 7: return OMChooser<ArgsT..., 7>::GetFunc(remainingArgs...); break;
1550         case 8: return OMChooser<ArgsT..., 8>::GetFunc(remainingArgs...); break;
1551         default:
1552             SWR_ASSERT(0 && "Invalid RT index\n");
1553             return nullptr;
1554             break;
1555         }
1556     }
1557 };
1558
1559 // Recursive template used to auto-nest conditionals.  Converts dynamic enum function
1560 // arguments to static template arguments.
1561 template <uint32_t... ArgsT>
1562 struct BECentroidBarycentricChooser
1563 {
1564
1565     // Last Arg Terminator
1566     template <typename... TArgsT>
1567     static PFN_CALC_CENTROID_BARYCENTRICS GetFunc(uint32_t tArg)
1568     {
1569         if(tArg > 0)
1570         {
1571             return CalcCentroidBarycentrics<ArgsT..., 1>;
1572         }
1573
1574         return CalcCentroidBarycentrics<ArgsT..., 0>;
1575     }
1576
1577     // Recursively parse args
1578     template <typename... TArgsT>
1579     static PFN_CALC_CENTROID_BARYCENTRICS GetFunc(SWR_MULTISAMPLE_COUNT tArg, TArgsT... remainingArgs)
1580     {
1581         switch(tArg)
1582         {
1583         case SWR_MULTISAMPLE_1X: return BECentroidBarycentricChooser<ArgsT..., SWR_MULTISAMPLE_1X>::GetFunc(remainingArgs...); break;
1584         case SWR_MULTISAMPLE_2X: return BECentroidBarycentricChooser<ArgsT..., SWR_MULTISAMPLE_2X>::GetFunc(remainingArgs...); break;
1585         case SWR_MULTISAMPLE_4X: return BECentroidBarycentricChooser<ArgsT..., SWR_MULTISAMPLE_4X>::GetFunc(remainingArgs...); break;
1586         case SWR_MULTISAMPLE_8X: return BECentroidBarycentricChooser<ArgsT..., SWR_MULTISAMPLE_8X>::GetFunc(remainingArgs...); break;
1587         case SWR_MULTISAMPLE_16X: return BECentroidBarycentricChooser<ArgsT..., SWR_MULTISAMPLE_16X>::GetFunc(remainingArgs...); break;
1588         default:
1589             SWR_ASSERT(0 && "Invalid sample count\n");
1590             return nullptr;
1591             break;
1592         }
1593     }
1594
1595     // Recursively parse args
1596     template <typename... TArgsT>
1597     static PFN_CALC_CENTROID_BARYCENTRICS GetFunc(uint32_t tArg, TArgsT... remainingArgs)
1598     {
1599         if(tArg > 0)
1600         {
1601             return BECentroidBarycentricChooser<ArgsT..., 1>::GetFunc(remainingArgs...);
1602         }
1603
1604         return BECentroidBarycentricChooser<ArgsT..., 0>::GetFunc(remainingArgs...);
1605     }
1606 };
1607
1608 // Recursive template used to auto-nest conditionals.  Converts dynamic enum function
1609 // arguments to static template arguments.
1610 template <uint32_t... ArgsT>
1611 struct BEChooser
1612 {
1613     // Last Arg Terminator
1614     static PFN_BACKEND_FUNC GetFunc(SWR_BACKEND_FUNCS tArg)
1615     {
1616         switch(tArg)
1617         {
1618         case SWR_BACKEND_SINGLE_SAMPLE: return BackendSingleSample<ArgsT...>; break;
1619         case SWR_BACKEND_MSAA_PIXEL_RATE: return BackendPixelRate<ArgsT...>; break;
1620         case SWR_BACKEND_MSAA_SAMPLE_RATE: return BackendSampleRate<ArgsT...>; break;
1621         default:
1622             SWR_ASSERT(0 && "Invalid backend func\n");
1623             return nullptr;
1624             break;
1625         }
1626     }
1627
1628
1629     // Recursively parse args
1630     template <typename... TArgsT>
1631     static PFN_BACKEND_FUNC GetFunc(SWR_MULTISAMPLE_COUNT tArg, TArgsT... remainingArgs)
1632     {
1633         switch(tArg)
1634         {
1635         case SWR_MULTISAMPLE_1X: return BEChooser<ArgsT..., SWR_MULTISAMPLE_1X>::GetFunc(remainingArgs...); break;
1636         case SWR_MULTISAMPLE_2X: return BEChooser<ArgsT..., SWR_MULTISAMPLE_2X>::GetFunc(remainingArgs...); break;
1637         case SWR_MULTISAMPLE_4X: return BEChooser<ArgsT..., SWR_MULTISAMPLE_4X>::GetFunc(remainingArgs...); break;
1638         case SWR_MULTISAMPLE_8X: return BEChooser<ArgsT..., SWR_MULTISAMPLE_8X>::GetFunc(remainingArgs...); break;
1639         case SWR_MULTISAMPLE_16X: return BEChooser<ArgsT..., SWR_MULTISAMPLE_16X>::GetFunc(remainingArgs...); break;
1640         default:
1641             SWR_ASSERT(0 && "Invalid sample count\n");
1642             return nullptr;
1643             break;
1644         }
1645     }
1646
1647     // Recursively parse args
1648     template <typename... TArgsT>
1649     static PFN_BACKEND_FUNC GetFunc(uint32_t tArg, TArgsT... remainingArgs)
1650     {
1651         if(tArg > 0)
1652         {
1653             return BEChooser<ArgsT..., 1>::GetFunc(remainingArgs...);
1654         }
1655
1656          return BEChooser<ArgsT..., 0>::GetFunc(remainingArgs...);
1657     }
1658 };
1659
1660 template <uint32_t numRenderTargets, SWR_MULTISAMPLE_COUNT numSampleRates>
1661 void InitBackendOMFuncTable(PFN_OUTPUT_MERGER (&table)[numRenderTargets][numSampleRates])
1662 {
1663     for(uint32_t rtNum = SWR_ATTACHMENT_COLOR0; rtNum < numRenderTargets; rtNum++)
1664     {
1665         for(uint32_t sampleCount = SWR_MULTISAMPLE_1X; sampleCount < numSampleRates; sampleCount++)
1666         {
1667             table[rtNum][sampleCount] =
1668                 OMChooser<>::GetFunc((SWR_RENDERTARGET_ATTACHMENT)rtNum, (SWR_MULTISAMPLE_COUNT)sampleCount);
1669         }
1670     }
1671 }
1672
1673 template <SWR_MULTISAMPLE_COUNT numSampleRates>
1674 void InitBackendBarycentricsTables(PFN_CALC_PIXEL_BARYCENTRICS (&pixelTable)[2],
1675                                    PFN_CALC_SAMPLE_BARYCENTRICS (&sampleTable)[2],
1676                                    PFN_CALC_CENTROID_BARYCENTRICS (&centroidTable)[numSampleRates][2][2][2])
1677 {
1678     pixelTable[0] = CalcPixelBarycentrics<0>;
1679     pixelTable[1] = CalcPixelBarycentrics<1>;
1680
1681     sampleTable[0] = CalcSampleBarycentrics<0>;
1682     sampleTable[1] = CalcSampleBarycentrics<1>;
1683
1684     for(uint32_t sampleCount = SWR_MULTISAMPLE_1X; sampleCount < numSampleRates; sampleCount++)
1685     {
1686         for(uint32_t baryMask = 0; baryMask < 2; baryMask++)
1687         {
1688             for(uint32_t patternNum = 0; patternNum < 2; patternNum++)
1689             {
1690                 for(uint32_t forcedSampleEnable = 0; forcedSampleEnable < 2; forcedSampleEnable++)
1691                 {
1692                     centroidTable[sampleCount][baryMask][patternNum][forcedSampleEnable]=
1693                         BECentroidBarycentricChooser<>::GetFunc((SWR_MULTISAMPLE_COUNT)sampleCount, baryMask, patternNum, forcedSampleEnable);
1694                 }
1695             }
1696         }
1697     }
1698 }
1699
1700 void InitBackendSampleFuncTable(PFN_BACKEND_FUNC (&table)[2][2])
1701 {
1702     gBackendSingleSample[0][0] = BEChooser<>::GetFunc(SWR_MULTISAMPLE_1X, SWR_MSAA_STANDARD_PATTERN, SWR_INPUT_COVERAGE_NONE, 0, 0, (SWR_BACKEND_FUNCS)SWR_BACKEND_SINGLE_SAMPLE);
1703     gBackendSingleSample[0][1] = BEChooser<>::GetFunc(SWR_MULTISAMPLE_1X, SWR_MSAA_STANDARD_PATTERN, SWR_INPUT_COVERAGE_NONE, 1, 0, (SWR_BACKEND_FUNCS)SWR_BACKEND_SINGLE_SAMPLE);
1704     gBackendSingleSample[1][0] = BEChooser<>::GetFunc(SWR_MULTISAMPLE_1X, SWR_MSAA_STANDARD_PATTERN, SWR_INPUT_COVERAGE_NORMAL, 0, 0, (SWR_BACKEND_FUNCS)SWR_BACKEND_SINGLE_SAMPLE);
1705     gBackendSingleSample[1][1] = BEChooser<>::GetFunc(SWR_MULTISAMPLE_1X, SWR_MSAA_STANDARD_PATTERN, SWR_INPUT_COVERAGE_NORMAL, 1, 0, (SWR_BACKEND_FUNCS)SWR_BACKEND_SINGLE_SAMPLE);
1706 }
1707
1708 template <SWR_MULTISAMPLE_COUNT numSampleRates, SWR_MSAA_SAMPLE_PATTERN numSamplePatterns, SWR_INPUT_COVERAGE numCoverageModes>
1709 void InitBackendPixelFuncTable(PFN_BACKEND_FUNC (&table)[numSampleRates][numSamplePatterns][numCoverageModes][2][2])
1710 {
1711     for(uint32_t sampleCount = SWR_MULTISAMPLE_1X; sampleCount < numSampleRates; sampleCount++)
1712     {
1713         for(uint32_t samplePattern = SWR_MSAA_CENTER_PATTERN; samplePattern < numSamplePatterns; samplePattern++)
1714         {
1715             for(uint32_t inputCoverage = SWR_INPUT_COVERAGE_NONE; inputCoverage < numCoverageModes; inputCoverage++)
1716             {
1717                 for(uint32_t isCentroid = 0; isCentroid < 2; isCentroid++)
1718                 {
1719                     table[sampleCount][samplePattern][inputCoverage][isCentroid][0] =
1720                         BEChooser<>::GetFunc((SWR_MULTISAMPLE_COUNT)sampleCount, samplePattern, inputCoverage, isCentroid, 0, (SWR_BACKEND_FUNCS)SWR_BACKEND_MSAA_PIXEL_RATE);
1721                     table[sampleCount][samplePattern][inputCoverage][isCentroid][1] =
1722                         BEChooser<>::GetFunc((SWR_MULTISAMPLE_COUNT)sampleCount, samplePattern, inputCoverage, isCentroid, 1, (SWR_BACKEND_FUNCS)SWR_BACKEND_MSAA_PIXEL_RATE);
1723                 }
1724             }
1725         }
1726     }
1727 }
1728
1729 template <uint32_t numSampleRates, uint32_t numCoverageModes>
1730 void InitBackendSampleFuncTable(PFN_BACKEND_FUNC (&table)[numSampleRates][numCoverageModes][2])
1731 {
1732     for(uint32_t sampleCount = SWR_MULTISAMPLE_1X; sampleCount < numSampleRates; sampleCount++)
1733     {
1734         for(uint32_t inputCoverage = SWR_INPUT_COVERAGE_NONE; inputCoverage < numCoverageModes; inputCoverage++)
1735         {
1736             table[sampleCount][inputCoverage][0] =
1737                 BEChooser<>::GetFunc((SWR_MULTISAMPLE_COUNT)sampleCount, SWR_MSAA_STANDARD_PATTERN, inputCoverage, 0, 0, (SWR_BACKEND_FUNCS)SWR_BACKEND_MSAA_SAMPLE_RATE);
1738             table[sampleCount][inputCoverage][1] =
1739                 BEChooser<>::GetFunc((SWR_MULTISAMPLE_COUNT)sampleCount, SWR_MSAA_STANDARD_PATTERN, inputCoverage, 1, 0, (SWR_BACKEND_FUNCS)SWR_BACKEND_MSAA_SAMPLE_RATE);
1740         }
1741     }
1742 }
1743
1744 void InitBackendFuncTables()
1745 {
1746     InitBackendSampleFuncTable(gBackendSingleSample);
1747     InitBackendPixelFuncTable<(SWR_MULTISAMPLE_COUNT)SWR_MULTISAMPLE_TYPE_MAX, SWR_MSAA_SAMPLE_PATTERN_MAX, SWR_INPUT_COVERAGE_MAX>(gBackendPixelRateTable);
1748     InitBackendSampleFuncTable<SWR_MULTISAMPLE_TYPE_MAX, SWR_INPUT_COVERAGE_MAX>(gBackendSampleRateTable);
1749     InitBackendOMFuncTable<SWR_NUM_RENDERTARGETS+1, SWR_MULTISAMPLE_TYPE_MAX>(gBackendOutputMergerTable);
1750     InitBackendBarycentricsTables<(SWR_MULTISAMPLE_COUNT)(SWR_MULTISAMPLE_TYPE_MAX)>(gPixelBarycentricTable, gSampleBarycentricTable, gCentroidBarycentricTable);
1751
1752     gBackendNullPs[SWR_MULTISAMPLE_1X] = &BackendNullPS < SWR_MULTISAMPLE_1X > ;
1753     gBackendNullPs[SWR_MULTISAMPLE_2X] = &BackendNullPS < SWR_MULTISAMPLE_2X > ;
1754     gBackendNullPs[SWR_MULTISAMPLE_4X] = &BackendNullPS < SWR_MULTISAMPLE_4X > ;
1755     gBackendNullPs[SWR_MULTISAMPLE_8X] = &BackendNullPS < SWR_MULTISAMPLE_8X > ;
1756     gBackendNullPs[SWR_MULTISAMPLE_16X] = &BackendNullPS < SWR_MULTISAMPLE_16X > ;
1757 }