src/gallium/drivers/swr/rasterizer/core/backend.cpp

   1 /****************************************************************************
   2 * Copyright (C) 2014-2015 Intel Corporation.   All Rights Reserved.
   3 *
   4 * Permission is hereby granted, free of charge, to any person obtaining a
   5 * copy of this software and associated documentation files (the "Software"),
   6 * to deal in the Software without restriction, including without limitation
   7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8 * and/or sell copies of the Software, and to permit persons to whom the
   9 * Software is furnished to do so, subject to the following conditions:
  10 *
  11 * The above copyright notice and this permission notice (including the next
  12 * paragraph) shall be included in all copies or substantial portions of the
  13 * Software.
  14 *
  15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21 * IN THE SOFTWARE.
  22 *
  23 * @file backend.cpp
  24 *
  25 * @brief Backend handles rasterization, pixel shading and output merger
  26 *        operations.
  27 *
  28 ******************************************************************************/
  29
  30 #include <smmintrin.h>
  31
  32 #include "rdtsc_core.h"
  33 #include "backend.h"
  34 #include "depthstencil.h"
  35 #include "tilemgr.h"
  36 #include "memory/tilingtraits.h"
  37 #include "core/multisample.h"
  38
  39 #include <algorithm>
  40
  41 const __m128 vTileOffsetsX = {0.5, KNOB_TILE_X_DIM - 0.5, 0.5, KNOB_TILE_X_DIM - 0.5};
  42 const __m128 vTileOffsetsY = {0.5, 0.5, KNOB_TILE_Y_DIM - 0.5, KNOB_TILE_Y_DIM - 0.5};
  43
  44 /// @todo move to common lib
  45 #define MASKTOVEC(i3,i2,i1,i0) {-i0,-i1,-i2,-i3}
  46 static const __m128 gMaskToVec[] = {
  47     MASKTOVEC(0,0,0,0),
  48     MASKTOVEC(0,0,0,1),
  49     MASKTOVEC(0,0,1,0),
  50     MASKTOVEC(0,0,1,1),
  51     MASKTOVEC(0,1,0,0),
  52     MASKTOVEC(0,1,0,1),
  53     MASKTOVEC(0,1,1,0),
  54     MASKTOVEC(0,1,1,1),
  55     MASKTOVEC(1,0,0,0),
  56     MASKTOVEC(1,0,0,1),
  57     MASKTOVEC(1,0,1,0),
  58     MASKTOVEC(1,0,1,1),
  59     MASKTOVEC(1,1,0,0),
  60     MASKTOVEC(1,1,0,1),
  61     MASKTOVEC(1,1,1,0),
  62     MASKTOVEC(1,1,1,1),
  63 };
  64
  65 typedef void(*PFN_CLEAR_TILES)(DRAW_CONTEXT*, SWR_RENDERTARGET_ATTACHMENT rt, uint32_t, DWORD[4]);
  66 static PFN_CLEAR_TILES sClearTilesTable[NUM_SWR_FORMATS];
  67
  68 //////////////////////////////////////////////////////////////////////////
  69 /// @brief Process compute work.
  70 /// @param pDC - pointer to draw context (dispatch).
  71 /// @param workerId - The unique worker ID that is assigned to this thread.
  72 /// @param threadGroupId - the linear index for the thread group within the dispatch.
  73 void ProcessComputeBE(DRAW_CONTEXT* pDC, uint32_t workerId, uint32_t threadGroupId)
  74 {
  75     RDTSC_START(BEDispatch);
  76
  77     SWR_CONTEXT *pContext = pDC->pContext;
  78
  79     const COMPUTE_DESC* pTaskData = (COMPUTE_DESC*)pDC->pDispatch->GetTasksData();
  80     SWR_ASSERT(pTaskData != nullptr);
  81
  82     // Ensure spill fill memory has been allocated.
  83     if (pDC->pSpillFill[workerId] == nullptr)
  84     {
  85         ///@todo Add state which indicates the spill fill size.
  86         pDC->pSpillFill[workerId] = (uint8_t*)pDC->pArena->AllocAlignedSync(4096 * 1024, sizeof(float) * 8);
  87     }
  88
  89     const API_STATE& state = GetApiState(pDC);
  90
  91     SWR_CS_CONTEXT csContext{ 0 };
  92     csContext.tileCounter = threadGroupId;
  93     csContext.dispatchDims[0] = pTaskData->threadGroupCountX;
  94     csContext.dispatchDims[1] = pTaskData->threadGroupCountY;
  95     csContext.dispatchDims[2] = pTaskData->threadGroupCountZ;
  96     csContext.pTGSM = pContext->pScratch[workerId];
  97     csContext.pSpillFillBuffer = pDC->pSpillFill[workerId];
  98
  99     state.pfnCsFunc(GetPrivateState(pDC), &csContext);
 100
 101     UPDATE_STAT(CsInvocations, state.totalThreadsInGroup);
 102
 103     RDTSC_STOP(BEDispatch, 1, 0);
 104 }
 105
 106 void ProcessSyncBE(DRAW_CONTEXT *pDC, uint32_t workerId, uint32_t macroTile, void *pUserData)
 107 {
 108     SYNC_DESC *pSync = (SYNC_DESC*)pUserData;
 109
 110     uint32_t x, y;
 111     MacroTileMgr::getTileIndices(macroTile, x, y);
 112     SWR_ASSERT(x == 0 && y == 0);
 113
 114     if (pSync->pfnCallbackFunc != nullptr)
 115     {
 116         pSync->pfnCallbackFunc(pSync->userData, pSync->userData2, pSync->userData3);
 117     }
 118 }
 119
 120 void ProcessQueryStatsBE(DRAW_CONTEXT *pDC, uint32_t workerId, uint32_t macroTile, void *pUserData)
 121 {
 122     QUERY_DESC* pQueryDesc = (QUERY_DESC*)pUserData;
 123     SWR_STATS* pStats = pQueryDesc->pStats;
 124     SWR_CONTEXT *pContext = pDC->pContext;
 125
 126     SWR_ASSERT(pStats != nullptr);
 127
 128     for (uint32_t i = 0; i < pContext->NumWorkerThreads; ++i)
 129     {
 130         pStats->DepthPassCount += pContext->stats[i].DepthPassCount;
 131
 132         pStats->IaVertices    += pContext->stats[i].IaVertices;
 133         pStats->IaPrimitives  += pContext->stats[i].IaPrimitives;
 134         pStats->VsInvocations += pContext->stats[i].VsInvocations;
 135         pStats->HsInvocations += pContext->stats[i].HsInvocations;
 136         pStats->DsInvocations += pContext->stats[i].DsInvocations;
 137         pStats->GsInvocations += pContext->stats[i].GsInvocations;
 138         pStats->PsInvocations += pContext->stats[i].PsInvocations;
 139         pStats->CInvocations  += pContext->stats[i].CInvocations;
 140         pStats->CsInvocations += pContext->stats[i].CsInvocations;
 141         pStats->CPrimitives   += pContext->stats[i].CPrimitives;
 142         pStats->GsPrimitives  += pContext->stats[i].GsPrimitives;
 143
 144         for (uint32_t stream = 0; stream < MAX_SO_STREAMS; ++stream)
 145         {
 146             pStats->SoWriteOffset[stream] += pContext->stats[i].SoWriteOffset[stream];
 147
 148             /// @note client is required to provide valid write offset before every draw, so we clear
 149             /// out the contents of the write offset when storing stats
 150             pContext->stats[i].SoWriteOffset[stream] = 0;
 151
 152             pStats->SoPrimStorageNeeded[stream] += pContext->stats[i].SoPrimStorageNeeded[stream];
 153             pStats->SoNumPrimsWritten[stream] += pContext->stats[i].SoNumPrimsWritten[stream];
 154         }
 155     }
 156 }
 157
 158 template<SWR_FORMAT format>
 159 void ClearRasterTile(BYTE *pTileBuffer, simdvector &value)
 160 {
 161     auto lambda = [&](int comp)
 162     {
 163         FormatTraits<format>::storeSOA(comp, pTileBuffer, value.v[comp]);
 164         pTileBuffer += (KNOB_SIMD_WIDTH * FormatTraits<format>::GetBPC(comp) / 8);
 165     };
 166
 167     const uint32_t numIter = (KNOB_TILE_Y_DIM / SIMD_TILE_Y_DIM) * (KNOB_TILE_X_DIM / SIMD_TILE_X_DIM);
 168     for (uint32_t i = 0; i < numIter; ++i)
 169     {
 170         UnrollerL<0, FormatTraits<format>::numComps, 1>::step(lambda);
 171     }
 172 }
 173
 174 template<SWR_FORMAT format>
 175 INLINE void ClearMacroTile(DRAW_CONTEXT *pDC, SWR_RENDERTARGET_ATTACHMENT rt, uint32_t macroTile, DWORD clear[4])
 176 {
 177     // convert clear color to hottile format
 178     // clear color is in RGBA float/uint32
 179     simdvector vClear;
 180     for (uint32_t comp = 0; comp < FormatTraits<format>::numComps; ++comp)
 181     {
 182         simdscalar vComp;
 183         vComp = _simd_load1_ps((const float*)&clear[comp]);
 184         if (FormatTraits<format>::isNormalized(comp))
 185         {
 186             vComp = _simd_mul_ps(vComp, _simd_set1_ps(FormatTraits<format>::fromFloat(comp)));
 187             vComp = _simd_castsi_ps(_simd_cvtps_epi32(vComp));
 188         }
 189         vComp = FormatTraits<format>::pack(comp, vComp);
 190         vClear.v[FormatTraits<format>::swizzle(comp)] = vComp;
 191     }
 192
 193     uint32_t tileX, tileY;
 194     MacroTileMgr::getTileIndices(macroTile, tileX, tileY);
 195     const API_STATE& state = GetApiState(pDC);
 196
 197     int top = KNOB_MACROTILE_Y_DIM_FIXED * tileY;
 198     int bottom = top + KNOB_MACROTILE_Y_DIM_FIXED - 1;
 199     int left = KNOB_MACROTILE_X_DIM_FIXED * tileX;
 200     int right = left + KNOB_MACROTILE_X_DIM_FIXED - 1;
 201
 202     // intersect with scissor
 203     top = std::max(top, state.scissorInFixedPoint.top);
 204     left = std::max(left, state.scissorInFixedPoint.left);
 205     bottom = std::min(bottom, state.scissorInFixedPoint.bottom);
 206     right = std::min(right, state.scissorInFixedPoint.right);
 207
 208     // translate to local hottile origin
 209     top -= KNOB_MACROTILE_Y_DIM_FIXED * tileY;
 210     bottom -= KNOB_MACROTILE_Y_DIM_FIXED * tileY;
 211     left -= KNOB_MACROTILE_X_DIM_FIXED * tileX;
 212     right -= KNOB_MACROTILE_X_DIM_FIXED * tileX;
 213
 214     // convert to raster tiles
 215     top >>= (KNOB_TILE_Y_DIM_SHIFT + FIXED_POINT_SHIFT);
 216     bottom >>= (KNOB_TILE_Y_DIM_SHIFT + FIXED_POINT_SHIFT);
 217     left >>= (KNOB_TILE_X_DIM_SHIFT + FIXED_POINT_SHIFT);
 218     right >>= (KNOB_TILE_X_DIM_SHIFT + FIXED_POINT_SHIFT);
 219
 220     const int numSamples = GetNumSamples(pDC->pState->state.rastState.sampleCount);
 221     // compute steps between raster tile samples / raster tiles / macro tile rows
 222     const uint32_t rasterTileSampleStep = KNOB_TILE_X_DIM * KNOB_TILE_Y_DIM * FormatTraits<format>::bpp / 8;
 223     const uint32_t rasterTileStep = (KNOB_TILE_X_DIM * KNOB_TILE_Y_DIM * (FormatTraits<format>::bpp / 8)) * numSamples;
 224     const uint32_t macroTileRowStep = (KNOB_MACROTILE_X_DIM / KNOB_TILE_X_DIM) * rasterTileStep;
 225     const uint32_t pitch = (FormatTraits<format>::bpp * KNOB_MACROTILE_X_DIM / 8);
 226
 227     HOTTILE *pHotTile = pDC->pContext->pHotTileMgr->GetHotTile(pDC->pContext, pDC, macroTile, rt, true, numSamples);
 228     uint32_t rasterTileStartOffset = (ComputeTileOffset2D< TilingTraits<SWR_TILE_SWRZ, FormatTraits<format>::bpp > >(pitch, left, top)) * numSamples;
 229     uint8_t* pRasterTileRow = pHotTile->pBuffer + rasterTileStartOffset; //(ComputeTileOffset2D< TilingTraits<SWR_TILE_SWRZ, FormatTraits<format>::bpp > >(pitch, x, y)) * numSamples;
 230
 231     // loop over all raster tiles in the current hot tile
 232     for (int y = top; y <= bottom; ++y)
 233     {
 234         uint8_t* pRasterTile = pRasterTileRow;
 235         for (int x = left; x <= right; ++x)
 236         {
 237             for( int sampleNum = 0; sampleNum < numSamples; sampleNum++)
 238             {
 239                 ClearRasterTile<format>(pRasterTile, vClear);
 240                 pRasterTile += rasterTileSampleStep;
 241             }
 242         }
 243         pRasterTileRow += macroTileRowStep;
 244     }
 245
 246     pHotTile->state = HOTTILE_DIRTY;
 247 }
 248
 249
 250 void ProcessClearBE(DRAW_CONTEXT *pDC, uint32_t workerId, uint32_t macroTile, void *pUserData)
 251 {
 252     if (KNOB_FAST_CLEAR)
 253     {
 254         CLEAR_DESC *pClear = (CLEAR_DESC*)pUserData;
 255         SWR_CONTEXT *pContext = pDC->pContext;
 256         SWR_MULTISAMPLE_COUNT sampleCount = pDC->pState->state.rastState.sampleCount;
 257         uint32_t numSamples = GetNumSamples(sampleCount);
 258
 259         SWR_ASSERT(pClear->flags.bits != 0); // shouldn't be here without a reason.
 260
 261         RDTSC_START(BEClear);
 262
 263         if (pClear->flags.mask & SWR_CLEAR_COLOR)
 264         {
 265             HOTTILE *pHotTile = pContext->pHotTileMgr->GetHotTile(pContext, pDC, macroTile, SWR_ATTACHMENT_COLOR0, true, numSamples);
 266             // All we want to do here is to mark the hot tile as being in a "needs clear" state.
 267             pHotTile->clearData[0] = *(DWORD*)&(pClear->clearRTColor[0]);
 268             pHotTile->clearData[1] = *(DWORD*)&(pClear->clearRTColor[1]);
 269             pHotTile->clearData[2] = *(DWORD*)&(pClear->clearRTColor[2]);
 270             pHotTile->clearData[3] = *(DWORD*)&(pClear->clearRTColor[3]);
 271             pHotTile->state = HOTTILE_CLEAR;
 272         }
 273
 274         if (pClear->flags.mask & SWR_CLEAR_DEPTH)
 275         {
 276             HOTTILE *pHotTile = pContext->pHotTileMgr->GetHotTile(pContext, pDC, macroTile, SWR_ATTACHMENT_DEPTH, true, numSamples);
 277             pHotTile->clearData[0] = *(DWORD*)&pClear->clearDepth;
 278             pHotTile->state = HOTTILE_CLEAR;
 279         }
 280
 281         if (pClear->flags.mask & SWR_CLEAR_STENCIL)
 282         {
 283             HOTTILE *pHotTile = pContext->pHotTileMgr->GetHotTile(pContext, pDC, macroTile, SWR_ATTACHMENT_STENCIL, true, numSamples);
 284
 285             pHotTile->clearData[0] = *(DWORD*)&pClear->clearStencil;
 286             pHotTile->state = HOTTILE_CLEAR;
 287         }
 288
 289         RDTSC_STOP(BEClear, 0, 0);
 290     }
 291     else
 292     {
 293         // Legacy clear
 294         CLEAR_DESC *pClear = (CLEAR_DESC*)pUserData;
 295         RDTSC_START(BEClear);
 296
 297         if (pClear->flags.mask & SWR_CLEAR_COLOR)
 298         {
 299             /// @todo clear data should come in as RGBA32_FLOAT
 300             DWORD clearData[4];
 301             float clearFloat[4];
 302             clearFloat[0] = ((BYTE*)(&pClear->clearRTColor))[0] / 255.0f;
 303             clearFloat[1] = ((BYTE*)(&pClear->clearRTColor))[1] / 255.0f;
 304             clearFloat[2] = ((BYTE*)(&pClear->clearRTColor))[2] / 255.0f;
 305             clearFloat[3] = ((BYTE*)(&pClear->clearRTColor))[3] / 255.0f;
 306             clearData[0] = *(DWORD*)&clearFloat[0];
 307             clearData[1] = *(DWORD*)&clearFloat[1];
 308             clearData[2] = *(DWORD*)&clearFloat[2];
 309             clearData[3] = *(DWORD*)&clearFloat[3];
 310
 311             PFN_CLEAR_TILES pfnClearTiles = sClearTilesTable[KNOB_COLOR_HOT_TILE_FORMAT];
 312             SWR_ASSERT(pfnClearTiles != nullptr);
 313
 314             pfnClearTiles(pDC, SWR_ATTACHMENT_COLOR0, macroTile, clearData);
 315         }
 316
 317         if (pClear->flags.mask & SWR_CLEAR_DEPTH)
 318         {
 319             DWORD clearData[4];
 320             clearData[0] = *(DWORD*)&pClear->clearDepth;
 321             PFN_CLEAR_TILES pfnClearTiles = sClearTilesTable[KNOB_DEPTH_HOT_TILE_FORMAT];
 322             SWR_ASSERT(pfnClearTiles != nullptr);
 323
 324             pfnClearTiles(pDC, SWR_ATTACHMENT_DEPTH, macroTile, clearData);
 325         }
 326
 327         if (pClear->flags.mask & SWR_CLEAR_STENCIL)
 328         {
 329             uint32_t value = pClear->clearStencil;
 330             DWORD clearData[4];
 331             clearData[0] = *(DWORD*)&value;
 332             PFN_CLEAR_TILES pfnClearTiles = sClearTilesTable[KNOB_STENCIL_HOT_TILE_FORMAT];
 333
 334             pfnClearTiles(pDC, SWR_ATTACHMENT_STENCIL, macroTile, clearData);
 335         }
 336
 337         RDTSC_STOP(BEClear, 0, 0);
 338     }
 339 }
 340
 341
 342 void ProcessStoreTileBE(DRAW_CONTEXT *pDC, uint32_t workerId, uint32_t macroTile, void *pData)
 343 {
 344     RDTSC_START(BEStoreTiles);
 345     STORE_TILES_DESC *pDesc = (STORE_TILES_DESC*)pData;
 346     SWR_CONTEXT *pContext = pDC->pContext;
 347
 348 #ifdef KNOB_ENABLE_RDTSC
 349     uint32_t numTiles = 0;
 350 #endif
 351     SWR_FORMAT srcFormat;
 352     switch (pDesc->attachment)
 353     {
 354     case SWR_ATTACHMENT_COLOR0:
 355     case SWR_ATTACHMENT_COLOR1:
 356     case SWR_ATTACHMENT_COLOR2:
 357     case SWR_ATTACHMENT_COLOR3:
 358     case SWR_ATTACHMENT_COLOR4:
 359     case SWR_ATTACHMENT_COLOR5:
 360     case SWR_ATTACHMENT_COLOR6:
 361     case SWR_ATTACHMENT_COLOR7: srcFormat = KNOB_COLOR_HOT_TILE_FORMAT; break;
 362     case SWR_ATTACHMENT_DEPTH: srcFormat = KNOB_DEPTH_HOT_TILE_FORMAT; break;
 363     case SWR_ATTACHMENT_STENCIL: srcFormat = KNOB_STENCIL_HOT_TILE_FORMAT; break;
 364     default: SWR_ASSERT(false, "Unknown attachment: %d", pDesc->attachment); srcFormat = KNOB_COLOR_HOT_TILE_FORMAT; break;
 365     }
 366
 367     uint32_t x, y;
 368     MacroTileMgr::getTileIndices(macroTile, x, y);
 369
 370     // Only need to store the hottile if it's been rendered to...
 371     HOTTILE *pHotTile = pContext->pHotTileMgr->GetHotTile(pContext, pDC, macroTile, pDesc->attachment, false);
 372     if (pHotTile)
 373     {
 374         // clear if clear is pending (i.e., not rendered to), then mark as dirty for store.
 375         if (pHotTile->state == HOTTILE_CLEAR)
 376         {
 377             PFN_CLEAR_TILES pfnClearTiles = sClearTilesTable[srcFormat];
 378             SWR_ASSERT(pfnClearTiles != nullptr);
 379
 380             pfnClearTiles(pDC, pDesc->attachment, macroTile, pHotTile->clearData);
 381         }
 382
 383         if (pHotTile->state == HOTTILE_DIRTY || pDesc->postStoreTileState == (SWR_TILE_STATE)HOTTILE_DIRTY)
 384         {
 385             int destX = KNOB_MACROTILE_X_DIM * x;
 386             int destY = KNOB_MACROTILE_Y_DIM * y;
 387
 388             pContext->pfnStoreTile(GetPrivateState(pDC), srcFormat,
 389                 pDesc->attachment, destX, destY, pHotTile->renderTargetArrayIndex, pHotTile->pBuffer);
 390         }
 391
 392
 393         if (pHotTile->state == HOTTILE_DIRTY || pHotTile->state == HOTTILE_RESOLVED)
 394         {
 395             pHotTile->state = (HOTTILE_STATE)pDesc->postStoreTileState;
 396         }
 397     }
 398     RDTSC_STOP(BEStoreTiles, numTiles, pDC->drawId);
 399 }
 400
 401
 402 void ProcessInvalidateTilesBE(DRAW_CONTEXT *pDC, uint32_t workerId, uint32_t macroTile, void *pData)
 403 {
 404     INVALIDATE_TILES_DESC *pDesc = (INVALIDATE_TILES_DESC*)pData;
 405     SWR_CONTEXT *pContext = pDC->pContext;
 406
 407     for (uint32_t i = 0; i < SWR_NUM_ATTACHMENTS; ++i)
 408     {
 409         if (pDesc->attachmentMask & (1 << i))
 410         {
 411             HOTTILE *pHotTile = pContext->pHotTileMgr->GetHotTile(pContext, pDC, macroTile, (SWR_RENDERTARGET_ATTACHMENT)i, false);
 412             if (pHotTile)
 413             {
 414                 pHotTile->state = HOTTILE_INVALID;
 415             }
 416         }
 417     }
 418 }
 419
 420 #if KNOB_SIMD_WIDTH == 8
 421 const __m256 vCenterOffsetsX = {0.5, 1.5, 0.5, 1.5, 2.5, 3.5, 2.5, 3.5};
 422 const __m256 vCenterOffsetsY = {0.5, 0.5, 1.5, 1.5, 0.5, 0.5, 1.5, 1.5};
 423 const __m256 vULOffsetsX = {0.0, 1.0, 0.0, 1.0, 2.0, 3.0, 2.0, 3.0};
 424 const __m256 vULOffsetsY = {0.0, 0.0, 1.0, 1.0, 0.0, 0.0, 1.0, 1.0};
 425 #else
 426 #error Unsupported vector width
 427 #endif
 428
 429 INLINE
 430 bool CanEarlyZ(const SWR_PS_STATE *pPSState)
 431 {
 432     return (pPSState->forceEarlyZ || (!pPSState->writesODepth && !pPSState->usesSourceDepth && !pPSState->usesUAV));
 433 }
 434
 435 simdmask ComputeUserClipMask(uint8_t clipMask, float* pUserClipBuffer, simdscalar vI, simdscalar vJ)
 436 {
 437     simdscalar vClipMask = _simd_setzero_ps();
 438     uint32_t numClipDistance = _mm_popcnt_u32(clipMask);
 439
 440     for (uint32_t i = 0; i < numClipDistance; ++i)
 441     {
 442         // pull triangle clip distance values from clip buffer
 443         simdscalar vA = _simd_broadcast_ss(pUserClipBuffer++);
 444         simdscalar vB = _simd_broadcast_ss(pUserClipBuffer++);
 445         simdscalar vC = _simd_broadcast_ss(pUserClipBuffer++);
 446
 447         // interpolate
 448         simdscalar vInterp = vplaneps(vA, vB, vC, vI, vJ);
 449
 450         // clip if interpolated clip distance is < 0 || NAN
 451         simdscalar vCull = _simd_cmp_ps(_simd_setzero_ps(), vInterp, _CMP_NLE_UQ);
 452
 453         vClipMask = _simd_or_ps(vClipMask, vCull);
 454     }
 455
 456     return _simd_movemask_ps(vClipMask);
 457 }
 458
 459 template<bool perspMask>
 460 INLINE void CalcPixelBarycentrics(const BarycentricCoeffs& coeffs, SWR_PS_CONTEXT &psContext)
 461 {
 462     if(perspMask)
 463     {
 464         // evaluate I,J
 465         psContext.vI.center = vplaneps(coeffs.vIa, coeffs.vIb, coeffs.vIc, psContext.vX.center, psContext.vY.center);
 466         psContext.vJ.center = vplaneps(coeffs.vJa, coeffs.vJb, coeffs.vJc, psContext.vX.center, psContext.vY.center);
 467         psContext.vI.center = _simd_mul_ps(psContext.vI.center, coeffs.vRecipDet);
 468         psContext.vJ.center = _simd_mul_ps(psContext.vJ.center, coeffs.vRecipDet);
 469
 470         // interpolate 1/w
 471         psContext.vOneOverW.center = vplaneps(coeffs.vAOneOverW, coeffs.vBOneOverW, coeffs.vCOneOverW, psContext.vI.center, psContext.vJ.center);
 472     }
 473 }
 474
 475 template<bool perspMask>
 476 INLINE void CalcSampleBarycentrics(const BarycentricCoeffs& coeffs, SWR_PS_CONTEXT &psContext)
 477 {
 478     if(perspMask)
 479     {
 480         // evaluate I,J
 481         psContext.vI.sample = vplaneps(coeffs.vIa, coeffs.vIb, coeffs.vIc, psContext.vX.sample, psContext.vY.sample);
 482         psContext.vJ.sample = vplaneps(coeffs.vJa, coeffs.vJb, coeffs.vJc, psContext.vX.sample, psContext.vY.sample);
 483         psContext.vI.sample = _simd_mul_ps(psContext.vI.sample, coeffs.vRecipDet);
 484         psContext.vJ.sample = _simd_mul_ps(psContext.vJ.sample, coeffs.vRecipDet);
 485
 486         // interpolate 1/w
 487         psContext.vOneOverW.sample = vplaneps(coeffs.vAOneOverW, coeffs.vBOneOverW, coeffs.vCOneOverW, psContext.vI.sample, psContext.vJ.sample);
 488     }
 489 }
 490
 491
 492 ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
 493 // Centroid behaves exactly as follows :
 494 // (1) If all samples in the primitive are covered, the attribute is evaluated at the pixel center (even if the sample pattern does not happen to
 495 //     have a sample location there).
 496 // (2) Else the attribute is evaluated at the first covered sample, in increasing order of sample index, where sample coverage is after ANDing the
 497 //     coverage with the SampleMask Rasterizer State.
 498 // (3) If no samples are covered, such as on helper pixels executed off the bounds of a primitive to fill out 2x2 pixel stamps, the attribute is
 499 //     evaluated as follows : If the SampleMask Rasterizer state is a subset of the samples in the pixel, then the first sample covered by the
 500 //     SampleMask Rasterizer State is the evaluation point.Otherwise (full SampleMask), the pixel center is the evaluation point.
 501 ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
 502 template<SWR_MULTISAMPLE_COUNT sampleCount, bool bForcedSampleCount>
 503 INLINE void CalcCentroidPos(SWR_PS_CONTEXT &psContext, const uint64_t *const coverageMask, const uint32_t sampleMask,
 504                             const simdscalar vXSamplePosUL, const simdscalar vYSamplePosUL)
 505 {
 506     uint32_t inputMask[KNOB_SIMD_WIDTH];
 507
 508     generateInputCoverage<sampleCount, 1, bForcedSampleCount>(coverageMask, inputMask, sampleMask);
 509
 510     // Case (2) - partially covered pixel
 511
 512     // scan for first covered sample per pixel in the 4x2 span
 513     unsigned long sampleNum[KNOB_SIMD_WIDTH];
 514     (inputMask[0] > 0) ? (_BitScanForward(&sampleNum[0], inputMask[0])) : (sampleNum[0] = 0);
 515     (inputMask[1] > 0) ? (_BitScanForward(&sampleNum[1], inputMask[1])) : (sampleNum[1] = 0);
 516     (inputMask[2] > 0) ? (_BitScanForward(&sampleNum[2], inputMask[2])) : (sampleNum[2] = 0);
 517     (inputMask[3] > 0) ? (_BitScanForward(&sampleNum[3], inputMask[3])) : (sampleNum[3] = 0);
 518     (inputMask[4] > 0) ? (_BitScanForward(&sampleNum[4], inputMask[4])) : (sampleNum[4] = 0);
 519     (inputMask[5] > 0) ? (_BitScanForward(&sampleNum[5], inputMask[5])) : (sampleNum[5] = 0);
 520     (inputMask[6] > 0) ? (_BitScanForward(&sampleNum[6], inputMask[6])) : (sampleNum[6] = 0);
 521     (inputMask[7] > 0) ? (_BitScanForward(&sampleNum[7], inputMask[7])) : (sampleNum[7] = 0);
 522
 523     // look up and set the sample offsets from UL pixel corner for first covered sample
 524     __m256 vXSample = _mm256_set_ps(MultisampleTraits<sampleCount>::X(sampleNum[7]),
 525                                     MultisampleTraits<sampleCount>::X(sampleNum[6]),
 526                                     MultisampleTraits<sampleCount>::X(sampleNum[5]),
 527                                     MultisampleTraits<sampleCount>::X(sampleNum[4]),
 528                                     MultisampleTraits<sampleCount>::X(sampleNum[3]),
 529                                     MultisampleTraits<sampleCount>::X(sampleNum[2]),
 530                                     MultisampleTraits<sampleCount>::X(sampleNum[1]),
 531                                     MultisampleTraits<sampleCount>::X(sampleNum[0]));
 532
 533     __m256 vYSample = _mm256_set_ps(MultisampleTraits<sampleCount>::Y(sampleNum[7]),
 534                                     MultisampleTraits<sampleCount>::Y(sampleNum[6]),
 535                                     MultisampleTraits<sampleCount>::Y(sampleNum[5]),
 536                                     MultisampleTraits<sampleCount>::Y(sampleNum[4]),
 537                                     MultisampleTraits<sampleCount>::Y(sampleNum[3]),
 538                                     MultisampleTraits<sampleCount>::Y(sampleNum[2]),
 539                                     MultisampleTraits<sampleCount>::Y(sampleNum[1]),
 540                                     MultisampleTraits<sampleCount>::Y(sampleNum[0]));
 541     // add sample offset to UL pixel corner
 542     vXSample = _simd_add_ps(vXSamplePosUL, vXSample);
 543     vYSample = _simd_add_ps(vYSamplePosUL, vYSample);
 544
 545     // Case (1) and case (3b) - All samples covered or not covered with full SampleMask
 546     static const __m256i vFullyCoveredMask = MultisampleTraits<sampleCount>::FullSampleMask();
 547     __m256i vInputCoveragei =  _mm256_set_epi32(inputMask[7], inputMask[6], inputMask[5], inputMask[4], inputMask[3], inputMask[2], inputMask[1], inputMask[0]);
 548     __m256i vAllSamplesCovered = _simd_cmpeq_epi32(vInputCoveragei, vFullyCoveredMask);
 549
 550     static const __m256i vZero = _simd_setzero_si();
 551     const __m256i vSampleMask = _simd_and_si(_simd_set1_epi32(sampleMask), vFullyCoveredMask);
 552     __m256i vNoSamplesCovered = _simd_cmpeq_epi32(vInputCoveragei, vZero);
 553     __m256i vIsFullSampleMask = _simd_cmpeq_epi32(vSampleMask, vFullyCoveredMask);
 554     __m256i vCase3b = _simd_and_si(vNoSamplesCovered, vIsFullSampleMask);
 555
 556     __m256i vEvalAtCenter = _simd_or_si(vAllSamplesCovered, vCase3b);
 557
 558     // set the centroid position based on results from above
 559     psContext.vX.centroid = _simd_blendv_ps(vXSample, psContext.vX.center, _simd_castsi_ps(vEvalAtCenter));
 560     psContext.vY.centroid = _simd_blendv_ps(vYSample, psContext.vY.center, _simd_castsi_ps(vEvalAtCenter));
 561
 562     // Case (3a) No samples covered and partial sample mask
 563     __m256i vSomeSampleMaskSamples = _simd_cmplt_epi32(vSampleMask, vFullyCoveredMask);
 564     // sample mask should never be all 0's for this case, but handle it anyways
 565     unsigned long firstCoveredSampleMaskSample = 0;
 566     (sampleMask > 0) ? (_BitScanForward(&firstCoveredSampleMaskSample, sampleMask)) : (firstCoveredSampleMaskSample = 0);
 567
 568     __m256i vCase3a = _simd_and_si(vNoSamplesCovered, vSomeSampleMaskSamples);
 569
 570     vXSample = _simd_set1_ps(MultisampleTraits<sampleCount>::X(firstCoveredSampleMaskSample));
 571     vYSample = _simd_set1_ps(MultisampleTraits<sampleCount>::Y(firstCoveredSampleMaskSample));
 572
 573     // blend in case 3a pixel locations
 574     psContext.vX.centroid = _simd_blendv_ps(psContext.vX.centroid, vXSample, _simd_castsi_ps(vCase3a));
 575     psContext.vY.centroid = _simd_blendv_ps(psContext.vY.centroid, vYSample, _simd_castsi_ps(vCase3a));
 576 }
 577
 578 template<uint32_t sampleCount, uint32_t persp, uint32_t standardPattern, uint32_t forcedMultisampleCount>
 579 INLINE void CalcCentroidBarycentrics(const BarycentricCoeffs& coeffs, SWR_PS_CONTEXT &psContext,
 580                                      const uint64_t *const coverageMask, const uint32_t sampleMask,
 581                                      const simdscalar vXSamplePosUL, const simdscalar vYSamplePosUL)
 582 {
 583     static const bool bPersp = (bool)persp;
 584     static const bool bIsStandardPattern = (bool)standardPattern;
 585     static const bool bForcedMultisampleCount = (bool)forcedMultisampleCount;
 586
 587     // calculate centroid positions
 588     if(bPersp)
 589     {
 590         if(bIsStandardPattern)
 591         {
 592             ///@ todo: don't need to generate input coverage 2x if input coverage and centroid
 593             CalcCentroidPos<(SWR_MULTISAMPLE_COUNT)sampleCount, bForcedMultisampleCount>(psContext, coverageMask, sampleMask, vXSamplePosUL, vYSamplePosUL);
 594         }
 595         else
 596         {
 597             static const __m256 pixelCenter = _simd_set1_ps(0.5f);
 598             psContext.vX.centroid = _simd_add_ps(vXSamplePosUL, pixelCenter);
 599             psContext.vY.centroid = _simd_add_ps(vYSamplePosUL, pixelCenter);
 600         }
 601         // evaluate I,J
 602         psContext.vI.centroid = vplaneps(coeffs.vIa, coeffs.vIb, coeffs.vIc, psContext.vX.centroid, psContext.vY.centroid);
 603         psContext.vJ.centroid = vplaneps(coeffs.vJa, coeffs.vJb, coeffs.vJc, psContext.vX.centroid, psContext.vY.centroid);
 604         psContext.vI.centroid = _simd_mul_ps(psContext.vI.centroid, coeffs.vRecipDet);
 605         psContext.vJ.centroid = _simd_mul_ps(psContext.vJ.centroid, coeffs.vRecipDet);
 606
 607         // interpolate 1/w
 608         psContext.vOneOverW.centroid = vplaneps(coeffs.vAOneOverW, coeffs.vBOneOverW, coeffs.vCOneOverW, psContext.vI.centroid, psContext.vJ.centroid);
 609     }
 610 }
 611
 612 template<uint32_t NumRT, uint32_t sampleCountT>
 613 void OutputMerger(SWR_PS_CONTEXT &psContext, uint8_t* (&pColorBase)[SWR_NUM_RENDERTARGETS], uint32_t sample, const SWR_BLEND_STATE *pBlendState,
 614                   const PFN_BLEND_JIT_FUNC (&pfnBlendFunc)[SWR_NUM_RENDERTARGETS], simdscalar &coverageMask, simdscalar depthPassMask)
 615 {
 616     // type safety guaranteed from template instantiation in BEChooser<>::GetFunc
 617     static const SWR_MULTISAMPLE_COUNT sampleCount = (SWR_MULTISAMPLE_COUNT)sampleCountT;
 618     uint32_t rasterTileColorOffset = MultisampleTraits<sampleCount>::RasterTileColorOffset(sample);
 619     simdvector blendOut;
 620
 621     for(uint32_t rt = 0; rt < NumRT; ++rt)
 622     {
 623         uint8_t *pColorSample;
 624         if(sampleCount == SWR_MULTISAMPLE_1X)
 625         {
 626             pColorSample = pColorBase[rt];
 627         }
 628         else
 629         {
 630             pColorSample = pColorBase[rt] + rasterTileColorOffset;
 631         }
 632
 633         const SWR_RENDER_TARGET_BLEND_STATE *pRTBlend = &pBlendState->renderTarget[rt];
 634         // pfnBlendFunc may not update all channels.  Initialize with PS output.
 635         /// TODO: move this into the blend JIT.
 636         blendOut = psContext.shaded[rt];
 637
 638         // Blend outputs and update coverage mask for alpha test
 639         if(pfnBlendFunc[rt] != nullptr)
 640         {
 641             pfnBlendFunc[rt](
 642                 pBlendState,
 643                 psContext.shaded[rt],
 644                 psContext.shaded[1],
 645                 sample,
 646                 pColorSample,
 647                 blendOut,
 648                 &psContext.oMask,
 649                 (simdscalari*)&coverageMask);
 650         }
 651
 652         // final write mask
 653         simdscalari outputMask = _simd_castps_si(_simd_and_ps(coverageMask, depthPassMask));
 654
 655         ///@todo can only use maskstore fast path if bpc is 32. Assuming hot tile is RGBA32_FLOAT.
 656         static_assert(KNOB_COLOR_HOT_TILE_FORMAT == R32G32B32A32_FLOAT, "Unsupported hot tile format");
 657
 658         const uint32_t simd = KNOB_SIMD_WIDTH * sizeof(float);
 659
 660         // store with color mask
 661         if(!pRTBlend->writeDisableRed)
 662         {
 663             _simd_maskstore_ps((float*)pColorSample, outputMask, blendOut.x);
 664         }
 665         if(!pRTBlend->writeDisableGreen)
 666         {
 667             _simd_maskstore_ps((float*)(pColorSample + simd), outputMask, blendOut.y);
 668         }
 669         if(!pRTBlend->writeDisableBlue)
 670         {
 671             _simd_maskstore_ps((float*)(pColorSample + simd * 2), outputMask, blendOut.z);
 672         }
 673         if(!pRTBlend->writeDisableAlpha)
 674         {
 675             _simd_maskstore_ps((float*)(pColorSample + simd * 3), outputMask, blendOut.w);
 676         }
 677     }
 678 }
 679
 680 template<uint32_t sampleCountT, uint32_t samplePattern, uint32_t inputCoverage, uint32_t centroidPos, uint32_t forcedSampleCount>
 681 void BackendSingleSample(DRAW_CONTEXT *pDC, uint32_t workerId, uint32_t x, uint32_t y, SWR_TRIANGLE_DESC &work, RenderOutputBuffers &renderBuffers)
 682 {
 683     RDTSC_START(BESetup);
 684     // type safety guaranteed from template instantiation in BEChooser<>::GetFunc
 685     static const bool bInputCoverage = (bool)inputCoverage;
 686     static const bool bCentroidPos = (bool)centroidPos;
 687
 688     SWR_CONTEXT *pContext = pDC->pContext;
 689     const API_STATE& state = GetApiState(pDC);
 690     const SWR_RASTSTATE& rastState = state.rastState;
 691     const SWR_PS_STATE *pPSState = &state.psState;
 692     const SWR_BLEND_STATE *pBlendState = &state.blendState;
 693     const BACKEND_FUNCS& backendFuncs = pDC->pState->backendFuncs;
 694     uint64_t coverageMask = work.coverageMask[0];
 695
 696     // broadcast scalars
 697     BarycentricCoeffs coeffs;
 698     coeffs.vIa = _simd_broadcast_ss(&work.I[0]);
 699     coeffs.vIb = _simd_broadcast_ss(&work.I[1]);
 700     coeffs.vIc = _simd_broadcast_ss(&work.I[2]);
 701
 702     coeffs.vJa = _simd_broadcast_ss(&work.J[0]);
 703     coeffs.vJb = _simd_broadcast_ss(&work.J[1]);
 704     coeffs.vJc = _simd_broadcast_ss(&work.J[2]);
 705
 706     coeffs.vZa = _simd_broadcast_ss(&work.Z[0]);
 707     coeffs.vZb = _simd_broadcast_ss(&work.Z[1]);
 708     coeffs.vZc = _simd_broadcast_ss(&work.Z[2]);
 709
 710     coeffs.vRecipDet = _simd_broadcast_ss(&work.recipDet);
 711
 712     coeffs.vAOneOverW = _simd_broadcast_ss(&work.OneOverW[0]);
 713     coeffs.vBOneOverW = _simd_broadcast_ss(&work.OneOverW[1]);
 714     coeffs.vCOneOverW = _simd_broadcast_ss(&work.OneOverW[2]);
 715
 716     uint8_t *pColorBase[SWR_NUM_RENDERTARGETS];
 717     uint32_t NumRT = state.psState.numRenderTargets;
 718     for(uint32_t rt = 0; rt < NumRT; ++rt)
 719     {
 720         pColorBase[rt] = renderBuffers.pColor[rt];
 721     }
 722     uint8_t *pDepthBase = renderBuffers.pDepth, *pStencilBase = renderBuffers.pStencil;
 723     RDTSC_STOP(BESetup, 0, 0);
 724
 725     SWR_PS_CONTEXT psContext;
 726     psContext.pAttribs = work.pAttribs;
 727     psContext.pPerspAttribs = work.pPerspAttribs;
 728     psContext.frontFace = work.triFlags.frontFacing;
 729     psContext.primID = work.triFlags.primID;
 730
 731     // save Ia/Ib/Ic and Ja/Jb/Jc if we need to reevaluate i/j/k in the shader because of pull attribs
 732     psContext.I = work.I;
 733     psContext.J = work.J;
 734     psContext.recipDet = work.recipDet;
 735     psContext.pRecipW = work.pRecipW;
 736     psContext.pSamplePosX = (const float*)&MultisampleTraits<SWR_MULTISAMPLE_1X>::samplePosX;
 737     psContext.pSamplePosY = (const float*)&MultisampleTraits<SWR_MULTISAMPLE_1X>::samplePosY;
 738
 739     for(uint32_t yy = y; yy < y + KNOB_TILE_Y_DIM; yy += SIMD_TILE_Y_DIM)
 740     {
 741         // UL pixel corner
 742         psContext.vY.UL = _simd_add_ps(vULOffsetsY, _simd_set1_ps((float)yy));
 743         // pixel center
 744         psContext.vY.center = _simd_add_ps(vCenterOffsetsY, _simd_set1_ps((float)yy));
 745
 746         for(uint32_t xx = x; xx < x + KNOB_TILE_X_DIM; xx += SIMD_TILE_X_DIM)
 747         {
 748             if(bInputCoverage)
 749             {
 750                 generateInputCoverage<SWR_MULTISAMPLE_1X, SWR_MSAA_STANDARD_PATTERN, false>(&work.coverageMask[0], psContext.inputMask, pBlendState->sampleMask);
 751             }
 752
 753             if(coverageMask & MASK)
 754             {
 755                 RDTSC_START(BEBarycentric);
 756                 psContext.vX.UL = _simd_add_ps(vULOffsetsX, _simd_set1_ps((float)xx));
 757                 // pixel center
 758                 psContext.vX.center = _simd_add_ps(vCenterOffsetsX, _simd_set1_ps((float)xx));
 759
 760                 backendFuncs.pfnCalcPixelBarycentrics(coeffs, psContext);
 761
 762                 if(bCentroidPos)
 763                 {
 764                     // for 1x case, centroid is pixel center
 765                     psContext.vX.centroid = psContext.vX.center;
 766                     psContext.vY.centroid = psContext.vY.center;
 767                     psContext.vI.centroid = psContext.vI.center;
 768                     psContext.vJ.centroid = psContext.vJ.center;
 769                     psContext.vOneOverW.centroid = psContext.vOneOverW.center;
 770                 }
 771
 772                 // interpolate z
 773                 psContext.vZ = vplaneps(coeffs.vZa, coeffs.vZb, coeffs.vZc, psContext.vI.center, psContext.vJ.center);
 774                 RDTSC_STOP(BEBarycentric, 0, 0);
 775
 776                 simdmask clipCoverageMask = coverageMask & MASK;
 777
 778                 // interpolate user clip distance if available
 779                 if(rastState.clipDistanceMask)
 780                 {
 781                     clipCoverageMask &= ~ComputeUserClipMask(rastState.clipDistanceMask, work.pUserClipBuffer,
 782                                                              psContext.vI.center, psContext.vJ.center);
 783                 }
 784
 785                 simdscalar vCoverageMask = vMask(clipCoverageMask);
 786                 simdscalar depthPassMask = vCoverageMask;
 787                 simdscalar stencilPassMask = vCoverageMask;
 788
 789                 // Early-Z?
 790                 if(CanEarlyZ(pPSState))
 791                 {
 792                     RDTSC_START(BEEarlyDepthTest);
 793                     depthPassMask = DepthStencilTest(&state.vp[0], &state.depthStencilState, work.triFlags.frontFacing,
 794                                                         psContext.vZ, pDepthBase, vCoverageMask, pStencilBase, &stencilPassMask);
 795                     RDTSC_STOP(BEEarlyDepthTest, 0, 0);
 796
 797                     // early-exit if no pixels passed depth or earlyZ is forced on
 798                     if(pPSState->forceEarlyZ || !_simd_movemask_ps(depthPassMask))
 799                     {
 800                         DepthStencilWrite(&state.vp[0], &state.depthStencilState, work.triFlags.frontFacing, psContext.vZ,
 801                                             pDepthBase, depthPassMask, vCoverageMask, pStencilBase, stencilPassMask);
 802
 803                         if (!_simd_movemask_ps(depthPassMask))
 804                         {
 805                             goto Endtile;
 806                         }
 807                     }
 808                 }
 809
 810                 psContext.sampleIndex = 0;
 811                 psContext.activeMask = _simd_castps_si(vCoverageMask);
 812
 813                 // execute pixel shader
 814                 RDTSC_START(BEPixelShader);
 815                 UPDATE_STAT(PsInvocations, _mm_popcnt_u32(_simd_movemask_ps(vCoverageMask)));
 816                 state.psState.pfnPixelShader(GetPrivateState(pDC), &psContext);
 817                 RDTSC_STOP(BEPixelShader, 0, 0);
 818
 819                 vCoverageMask = _simd_castsi_ps(psContext.activeMask);
 820
 821                 // late-Z
 822                 if(!CanEarlyZ(pPSState))
 823                 {
 824                     RDTSC_START(BELateDepthTest);
 825                     depthPassMask = DepthStencilTest(&state.vp[0], &state.depthStencilState, work.triFlags.frontFacing,
 826                                                         psContext.vZ, pDepthBase, vCoverageMask, pStencilBase, &stencilPassMask);
 827                     RDTSC_STOP(BELateDepthTest, 0, 0);
 828
 829                     if(!_simd_movemask_ps(depthPassMask))
 830                     {
 831                         // need to call depth/stencil write for stencil write
 832                         DepthStencilWrite(&state.vp[0], &state.depthStencilState, work.triFlags.frontFacing, psContext.vZ,
 833                                             pDepthBase, depthPassMask, vCoverageMask, pStencilBase, stencilPassMask);
 834                         goto Endtile;
 835                     }
 836                 }
 837
 838                 uint32_t statMask = _simd_movemask_ps(depthPassMask);
 839                 uint32_t statCount = _mm_popcnt_u32(statMask);
 840                 UPDATE_STAT(DepthPassCount, statCount);
 841
 842                 // output merger
 843                 RDTSC_START(BEOutputMerger);
 844                 backendFuncs.pfnOutputMerger(psContext, pColorBase, 0, pBlendState, state.pfnBlendFunc,
 845                                              vCoverageMask, depthPassMask);
 846
 847                 // do final depth write after all pixel kills
 848                 if (!pPSState->forceEarlyZ)
 849                 {
 850                     DepthStencilWrite(&state.vp[0], &state.depthStencilState, work.triFlags.frontFacing, psContext.vZ,
 851                         pDepthBase, depthPassMask, vCoverageMask, pStencilBase, stencilPassMask);
 852                 }
 853                 RDTSC_STOP(BEOutputMerger, 0, 0);
 854             }
 855
 856 Endtile:
 857             RDTSC_START(BEEndTile);
 858             coverageMask >>= (SIMD_TILE_Y_DIM * SIMD_TILE_X_DIM);
 859             pDepthBase += (KNOB_SIMD_WIDTH * FormatTraits<KNOB_DEPTH_HOT_TILE_FORMAT>::bpp) / 8;
 860             pStencilBase += (KNOB_SIMD_WIDTH * FormatTraits<KNOB_STENCIL_HOT_TILE_FORMAT>::bpp) / 8;
 861
 862             for(uint32_t rt = 0; rt < NumRT; ++rt)
 863             {
 864                 pColorBase[rt] += (KNOB_SIMD_WIDTH * FormatTraits<KNOB_COLOR_HOT_TILE_FORMAT>::bpp) / 8;
 865             }
 866             RDTSC_STOP(BEEndTile, 0, 0);
 867         }
 868     }
 869 }
 870
 871 template<uint32_t sampleCountT, uint32_t samplePattern, uint32_t inputCoverage, uint32_t centroidPos, uint32_t forcedSampleCount>
 872 void BackendSampleRate(DRAW_CONTEXT *pDC, uint32_t workerId, uint32_t x, uint32_t y, SWR_TRIANGLE_DESC &work, RenderOutputBuffers &renderBuffers)
 873 {
 874     // type safety guaranteed from template instantiation in BEChooser<>::GetFunc
 875     static const SWR_MULTISAMPLE_COUNT sampleCount = (SWR_MULTISAMPLE_COUNT)sampleCountT;
 876     static const bool bInputCoverage = (bool)inputCoverage;
 877     static const bool bCentroidPos = (bool)centroidPos;
 878
 879     RDTSC_START(BESetup);
 880
 881     SWR_CONTEXT *pContext = pDC->pContext;
 882     const API_STATE& state = GetApiState(pDC);
 883     const SWR_RASTSTATE& rastState = state.rastState;
 884     const SWR_PS_STATE *pPSState = &state.psState;
 885     const SWR_BLEND_STATE *pBlendState = &state.blendState;
 886     const BACKEND_FUNCS& backendFuncs = pDC->pState->backendFuncs;
 887
 888     // broadcast scalars
 889     BarycentricCoeffs coeffs;
 890     coeffs.vIa = _simd_broadcast_ss(&work.I[0]);
 891     coeffs.vIb = _simd_broadcast_ss(&work.I[1]);
 892     coeffs.vIc = _simd_broadcast_ss(&work.I[2]);
 893
 894     coeffs.vJa = _simd_broadcast_ss(&work.J[0]);
 895     coeffs.vJb = _simd_broadcast_ss(&work.J[1]);
 896     coeffs.vJc = _simd_broadcast_ss(&work.J[2]);
 897
 898     coeffs.vZa = _simd_broadcast_ss(&work.Z[0]);
 899     coeffs.vZb = _simd_broadcast_ss(&work.Z[1]);
 900     coeffs.vZc = _simd_broadcast_ss(&work.Z[2]);
 901
 902     coeffs.vRecipDet = _simd_broadcast_ss(&work.recipDet);
 903
 904     coeffs.vAOneOverW = _simd_broadcast_ss(&work.OneOverW[0]);
 905     coeffs.vBOneOverW = _simd_broadcast_ss(&work.OneOverW[1]);
 906     coeffs.vCOneOverW = _simd_broadcast_ss(&work.OneOverW[2]);
 907
 908     uint8_t *pColorBase[SWR_NUM_RENDERTARGETS];
 909     uint32_t NumRT = state.psState.numRenderTargets;
 910     for(uint32_t rt = 0; rt < NumRT; ++rt)
 911     {
 912         pColorBase[rt] = renderBuffers.pColor[rt];
 913     }
 914     uint8_t *pDepthBase = renderBuffers.pDepth, *pStencilBase = renderBuffers.pStencil;
 915     RDTSC_STOP(BESetup, 0, 0);
 916
 917     SWR_PS_CONTEXT psContext;
 918     psContext.pAttribs = work.pAttribs;
 919     psContext.pPerspAttribs = work.pPerspAttribs;
 920     psContext.pRecipW = work.pRecipW;
 921     psContext.frontFace = work.triFlags.frontFacing;
 922     psContext.primID = work.triFlags.primID;
 923
 924     // save Ia/Ib/Ic and Ja/Jb/Jc if we need to reevaluate i/j/k in the shader because of pull attribs
 925     psContext.I = work.I;
 926     psContext.J = work.J;
 927     psContext.recipDet = work.recipDet;
 928     psContext.pSamplePosX = (const float*)&MultisampleTraits<sampleCount>::samplePosX;
 929     psContext.pSamplePosY = (const float*)&MultisampleTraits<sampleCount>::samplePosY;
 930     const uint32_t numSamples = MultisampleTraits<sampleCount>::numSamples;
 931
 932     for (uint32_t yy = y; yy < y + KNOB_TILE_Y_DIM; yy += SIMD_TILE_Y_DIM)
 933     {
 934         // UL pixel corner
 935         psContext.vY.UL = _simd_add_ps(vULOffsetsY, _simd_set1_ps((float)yy));
 936         // pixel center
 937         psContext.vY.center = _simd_add_ps(vCenterOffsetsY, _simd_set1_ps((float)yy));
 938
 939         for (uint32_t xx = x; xx < x + KNOB_TILE_X_DIM; xx += SIMD_TILE_X_DIM)
 940         {
 941             psContext.vX.UL = _simd_add_ps(vULOffsetsX, _simd_set1_ps((float)xx));
 942             // pixel center
 943             psContext.vX.center = _simd_add_ps(vCenterOffsetsX, _simd_set1_ps((float)xx));
 944
 945             RDTSC_START(BEBarycentric);
 946             backendFuncs.pfnCalcPixelBarycentrics(coeffs, psContext);
 947             RDTSC_STOP(BEBarycentric, 0, 0);
 948
 949             if(bInputCoverage)
 950             {
 951                 generateInputCoverage<sampleCount, SWR_MSAA_STANDARD_PATTERN, false>(&work.coverageMask[0], psContext.inputMask, pBlendState->sampleMask);
 952             }
 953
 954             if(bCentroidPos)
 955             {
 956                 ///@ todo: don't need to genererate input coverage 2x if input coverage and centroid
 957                 RDTSC_START(BEBarycentric);
 958                 backendFuncs.pfnCalcCentroidBarycentrics(coeffs, psContext, &work.coverageMask[0], pBlendState->sampleMask, psContext.vX.UL, psContext.vY.UL);
 959                 RDTSC_STOP(BEBarycentric, 0, 0);
 960             }
 961
 962             for(uint32_t sample = 0; sample < numSamples; sample++)
 963             {
 964                 if (work.coverageMask[sample] & MASK)
 965                 {
 966                     RDTSC_START(BEBarycentric);
 967
 968                     // calculate per sample positions
 969                     psContext.vX.sample = _simd_add_ps(psContext.vX.UL, MultisampleTraits<sampleCount>::vX(sample));
 970                     psContext.vY.sample = _simd_add_ps(psContext.vY.UL, MultisampleTraits<sampleCount>::vY(sample));
 971
 972                     simdmask coverageMask = work.coverageMask[sample] & MASK;
 973                     simdscalar vCoverageMask = vMask(coverageMask);
 974
 975                     backendFuncs.pfnCalcSampleBarycentrics(coeffs, psContext);
 976
 977                     // interpolate z
 978                     psContext.vZ = vplaneps(coeffs.vZa, coeffs.vZb, coeffs.vZc, psContext.vI.sample, psContext.vJ.sample);
 979
 980                     RDTSC_STOP(BEBarycentric, 0, 0);
 981
 982                     // interpolate user clip distance if available
 983                     if (rastState.clipDistanceMask)
 984                     {
 985                         coverageMask &= ~ComputeUserClipMask(rastState.clipDistanceMask, work.pUserClipBuffer,
 986                             psContext.vI.sample, psContext.vJ.sample);
 987                     }
 988
 989                     simdscalar depthPassMask = vCoverageMask;
 990                     simdscalar stencilPassMask = vCoverageMask;
 991
 992                     // offset depth/stencil buffers current sample
 993                     uint8_t *pDepthSample = pDepthBase + MultisampleTraits<sampleCount>::RasterTileDepthOffset(sample);
 994                     uint8_t *pStencilSample = pStencilBase + MultisampleTraits<sampleCount>::RasterTileStencilOffset(sample);
 995
 996                     // Early-Z?
 997                     if (CanEarlyZ(pPSState))
 998                     {
 999                         RDTSC_START(BEEarlyDepthTest);
1000                         depthPassMask = DepthStencilTest(&state.vp[0], &state.depthStencilState, work.triFlags.frontFacing,
1001                                               psContext.vZ, pDepthSample, vCoverageMask, pStencilSample, &stencilPassMask);
1002                         RDTSC_STOP(BEEarlyDepthTest, 0, 0);
1003
1004                         // early-exit if no samples passed depth or earlyZ is forced on.
1005                         if (pPSState->forceEarlyZ || !_simd_movemask_ps(depthPassMask))
1006                         {
1007                             DepthStencilWrite(&state.vp[0], &state.depthStencilState, work.triFlags.frontFacing, psContext.vZ,
1008                                 pDepthSample, depthPassMask, vCoverageMask, pStencilSample, stencilPassMask);
1009
1010                             if (!_simd_movemask_ps(depthPassMask))
1011                             {
1012                                 work.coverageMask[sample] >>= (SIMD_TILE_Y_DIM * SIMD_TILE_X_DIM);
1013                                 continue;
1014                             }
1015                         }
1016                     }
1017
1018                     psContext.sampleIndex = sample;
1019                     psContext.activeMask = _simd_castps_si(vCoverageMask);
1020
1021                     // execute pixel shader
1022                     RDTSC_START(BEPixelShader);
1023                     UPDATE_STAT(PsInvocations, _mm_popcnt_u32(_simd_movemask_ps(vCoverageMask)));
1024                     state.psState.pfnPixelShader(GetPrivateState(pDC), &psContext);
1025                     RDTSC_STOP(BEPixelShader, 0, 0);
1026
1027                     vCoverageMask = _simd_castsi_ps(psContext.activeMask);
1028
1029                     //// late-Z
1030                     if (!CanEarlyZ(pPSState))
1031                     {
1032                         RDTSC_START(BELateDepthTest);
1033                         depthPassMask = DepthStencilTest(&state.vp[0], &state.depthStencilState, work.triFlags.frontFacing,
1034                                               psContext.vZ, pDepthSample, vCoverageMask, pStencilSample, &stencilPassMask);
1035                         RDTSC_STOP(BELateDepthTest, 0, 0);
1036
1037                         if (!_simd_movemask_ps(depthPassMask))
1038                         {
1039                             // need to call depth/stencil write for stencil write
1040                             DepthStencilWrite(&state.vp[0], &state.depthStencilState, work.triFlags.frontFacing, psContext.vZ,
1041                                 pDepthSample, depthPassMask, vCoverageMask, pStencilSample, stencilPassMask);
1042
1043                             work.coverageMask[sample] >>= (SIMD_TILE_Y_DIM * SIMD_TILE_X_DIM);
1044                             continue;
1045                         }
1046                     }
1047
1048                     uint32_t statMask = _simd_movemask_ps(depthPassMask);
1049                     uint32_t statCount = _mm_popcnt_u32(statMask);
1050                     UPDATE_STAT(DepthPassCount, statCount);
1051
1052                     // output merger
1053                     RDTSC_START(BEOutputMerger);
1054                     backendFuncs.pfnOutputMerger(psContext, pColorBase, sample, pBlendState, state.pfnBlendFunc,
1055                                                  vCoverageMask, depthPassMask);
1056
1057                     // do final depth write after all pixel kills
1058                     if (!pPSState->forceEarlyZ)
1059                     {
1060                         DepthStencilWrite(&state.vp[0], &state.depthStencilState, work.triFlags.frontFacing, psContext.vZ,
1061                             pDepthSample, depthPassMask, vCoverageMask, pStencilSample, stencilPassMask);
1062                     }
1063                     RDTSC_STOP(BEOutputMerger, 0, 0);
1064                 }
1065                 work.coverageMask[sample] >>= (SIMD_TILE_Y_DIM * SIMD_TILE_X_DIM);
1066             }
1067             RDTSC_START(BEEndTile);
1068             pDepthBase += (KNOB_SIMD_WIDTH * FormatTraits<KNOB_DEPTH_HOT_TILE_FORMAT>::bpp) / 8;
1069             pStencilBase += (KNOB_SIMD_WIDTH * FormatTraits<KNOB_STENCIL_HOT_TILE_FORMAT>::bpp) / 8;
1070
1071             for (uint32_t rt = 0; rt < NumRT; ++rt)
1072             {
1073                 pColorBase[rt] += (KNOB_SIMD_WIDTH * FormatTraits<KNOB_COLOR_HOT_TILE_FORMAT>::bpp) / 8;
1074             }
1075             RDTSC_STOP(BEEndTile, 0, 0);
1076         }
1077     }
1078 }
1079
1080 template<uint32_t sampleCountT, uint32_t samplePattern, uint32_t inputCoverage, uint32_t centroidPos, uint32_t forcedSampleCount>
1081 void BackendPixelRate(DRAW_CONTEXT *pDC, uint32_t workerId, uint32_t x, uint32_t y, SWR_TRIANGLE_DESC &work, RenderOutputBuffers &renderBuffers)
1082 {
1083     // type safety guaranteed from template instantiation in BEChooser<>::GetFunc
1084     static const SWR_MULTISAMPLE_COUNT sampleCount = (SWR_MULTISAMPLE_COUNT)sampleCountT;
1085     static const bool bIsStandardPattern = (bool)samplePattern;
1086     static const bool bInputCoverage = (bool)inputCoverage;
1087     static const bool bCentroidPos = (bool)centroidPos;
1088     static const bool bForcedSampleCount = (bool)forcedSampleCount;
1089
1090     RDTSC_START(BESetup);
1091
1092     SWR_CONTEXT *pContext = pDC->pContext;
1093     const API_STATE& state = GetApiState(pDC);
1094     const SWR_RASTSTATE& rastState = state.rastState;
1095     const SWR_PS_STATE *pPSState = &state.psState;
1096     const SWR_BLEND_STATE *pBlendState = &state.blendState;
1097     const BACKEND_FUNCS& backendFuncs = pDC->pState->backendFuncs;
1098
1099     // broadcast scalars
1100     BarycentricCoeffs coeffs;
1101     coeffs.vIa = _simd_broadcast_ss(&work.I[0]);
1102     coeffs.vIb = _simd_broadcast_ss(&work.I[1]);
1103     coeffs.vIc = _simd_broadcast_ss(&work.I[2]);
1104
1105     coeffs.vJa = _simd_broadcast_ss(&work.J[0]);
1106     coeffs.vJb = _simd_broadcast_ss(&work.J[1]);
1107     coeffs.vJc = _simd_broadcast_ss(&work.J[2]);
1108
1109     coeffs.vZa = _simd_broadcast_ss(&work.Z[0]);
1110     coeffs.vZb = _simd_broadcast_ss(&work.Z[1]);
1111     coeffs.vZc = _simd_broadcast_ss(&work.Z[2]);
1112
1113     coeffs.vRecipDet = _simd_broadcast_ss(&work.recipDet);
1114
1115     coeffs.vAOneOverW = _simd_broadcast_ss(&work.OneOverW[0]);
1116     coeffs.vBOneOverW = _simd_broadcast_ss(&work.OneOverW[1]);
1117     coeffs.vCOneOverW = _simd_broadcast_ss(&work.OneOverW[2]);
1118
1119     uint8_t *pColorBase[SWR_NUM_RENDERTARGETS];
1120     uint32_t NumRT = state.psState.numRenderTargets;
1121     for(uint32_t rt = 0; rt < NumRT; ++rt)
1122     {
1123         pColorBase[rt] = renderBuffers.pColor[rt];
1124     }
1125     uint8_t *pDepthBase = renderBuffers.pDepth, *pStencilBase = renderBuffers.pStencil;
1126     RDTSC_STOP(BESetup, 0, 0);
1127
1128     SWR_PS_CONTEXT psContext;
1129     psContext.pAttribs = work.pAttribs;
1130     psContext.pPerspAttribs = work.pPerspAttribs;
1131     psContext.frontFace = work.triFlags.frontFacing;
1132     psContext.primID = work.triFlags.primID;
1133     psContext.pRecipW = work.pRecipW;
1134     // save Ia/Ib/Ic and Ja/Jb/Jc if we need to reevaluate i/j/k in the shader because of pull attribs
1135     psContext.I = work.I;
1136     psContext.J = work.J;
1137     psContext.recipDet = work.recipDet;
1138     psContext.pSamplePosX = (const float*)&MultisampleTraits<sampleCount>::samplePosX;
1139     psContext.pSamplePosY = (const float*)&MultisampleTraits<sampleCount>::samplePosY;
1140     psContext.sampleIndex = 0;
1141
1142     uint32_t numCoverageSamples;
1143     if(bIsStandardPattern)
1144     {
1145         numCoverageSamples = MultisampleTraits<sampleCount>::numSamples;
1146     }
1147     else
1148     {
1149         numCoverageSamples = 1;
1150     }
1151
1152     uint32_t numOMSamples;
1153     // RT has to be single sample if we're in forcedMSAA mode
1154     if(bForcedSampleCount && (sampleCount > SWR_MULTISAMPLE_1X))
1155     {
1156         numOMSamples = 1;
1157     }
1158     // unless we're forced to single sample, in which case we run the OM at the sample count of the RT
1159     else if(bForcedSampleCount && (sampleCount == SWR_MULTISAMPLE_1X))
1160     {
1161         numOMSamples = GetNumSamples(pBlendState->sampleCount);
1162     }
1163     // else we're in normal MSAA mode and rasterizer and OM are running at the same sample count
1164     else
1165     {
1166         numOMSamples = MultisampleTraits<sampleCount>::numSamples;
1167     }
1168
1169     for(uint32_t yy = y; yy < y + KNOB_TILE_Y_DIM; yy += SIMD_TILE_Y_DIM)
1170     {
1171         psContext.vY.UL = _simd_add_ps(vULOffsetsY, _simd_set1_ps((float)yy));
1172         psContext.vY.center = _simd_add_ps(vCenterOffsetsY, _simd_set1_ps((float)yy));
1173         for(uint32_t xx = x; xx < x + KNOB_TILE_X_DIM; xx += SIMD_TILE_X_DIM)
1174         {
1175             simdscalar vZ[MultisampleTraits<sampleCount>::numSamples]{ 0 };
1176             psContext.vX.UL = _simd_add_ps(vULOffsetsX, _simd_set1_ps((float)xx));
1177             // set pixel center positions
1178             psContext.vX.center = _simd_add_ps(vCenterOffsetsX, _simd_set1_ps((float)xx));
1179
1180             if (bInputCoverage)
1181             {
1182                 generateInputCoverage<sampleCount, bIsStandardPattern, bForcedSampleCount>(&work.coverageMask[0], psContext.inputMask, pBlendState->sampleMask);
1183             }
1184
1185             if(bCentroidPos)
1186             {
1187                 ///@ todo: don't need to genererate input coverage 2x if input coverage and centroid
1188                 RDTSC_START(BEBarycentric);
1189                 backendFuncs.pfnCalcCentroidBarycentrics(coeffs, psContext, &work.coverageMask[0], pBlendState->sampleMask, psContext.vX.UL, psContext.vY.UL);
1190                 RDTSC_STOP(BEBarycentric, 0, 0);
1191             }
1192
1193             // if oDepth written to, or there is a potential to discard any samples, we need to
1194             // run the PS early, then interp or broadcast Z and test
1195             if(pPSState->writesODepth || pPSState->killsPixel)
1196             {
1197                 RDTSC_START(BEBarycentric);
1198                 backendFuncs.pfnCalcPixelBarycentrics(coeffs, psContext);
1199
1200                 // interpolate z
1201                 psContext.vZ = vplaneps(coeffs.vZa, coeffs.vZb, coeffs.vZc, psContext.vI.center, psContext.vJ.center);
1202                 RDTSC_STOP(BEBarycentric, 0, 0);
1203
1204                 // execute pixel shader
1205                 RDTSC_START(BEPixelShader);
1206                 state.psState.pfnPixelShader(GetPrivateState(pDC), &psContext);
1207                 RDTSC_STOP(BEPixelShader, 0, 0);
1208             }
1209             else
1210             {
1211                                 psContext.activeMask = _simd_set1_epi32(-1);
1212             }
1213
1214             // need to declare enough space for all samples
1215             simdscalar vCoverageMask[MultisampleTraits<sampleCount>::numSamples];
1216             simdscalar depthPassMask[MultisampleTraits<sampleCount>::numSamples];
1217             simdscalar stencilPassMask[MultisampleTraits<sampleCount>::numSamples];
1218             simdscalar anyDepthSamplePassed = _simd_setzero_ps();
1219             simdscalar anyStencilSamplePassed = _simd_setzero_ps();
1220             for(uint32_t sample = 0; sample < numCoverageSamples; sample++)
1221             {
1222                 vCoverageMask[sample] = vMask(work.coverageMask[sample] & MASK);
1223
1224                 // pull mask back out for any discards and and with coverage
1225                 vCoverageMask[sample] = _simd_and_ps(vCoverageMask[sample], _simd_castsi_ps(psContext.activeMask));
1226
1227                 if (!_simd_movemask_ps(vCoverageMask[sample]))
1228                 {
1229                     vCoverageMask[sample] = depthPassMask[sample] = stencilPassMask[sample] =  _simd_setzero_ps();
1230                     continue;
1231                 }
1232
1233                 if(bForcedSampleCount)
1234                 {
1235                     // candidate pixels (that passed coverage) will cause shader invocation if any bits in the samplemask are set
1236                     const simdscalar vSampleMask = _simd_castsi_ps(_simd_cmpgt_epi32(_simd_set1_epi32(pBlendState->sampleMask), _simd_setzero_si()));
1237                     anyDepthSamplePassed = _simd_or_ps(anyDepthSamplePassed, _simd_and_ps(vCoverageMask[sample], vSampleMask));
1238                     continue;
1239                 }
1240
1241                 depthPassMask[sample] = vCoverageMask[sample];
1242
1243                 // if oDepth isn't written to, we need to interpolate Z for each sample
1244                 // if clip distances are enabled, we need to interpolate for each sample
1245                 if(!pPSState->writesODepth || rastState.clipDistanceMask)
1246                 {
1247                     RDTSC_START(BEBarycentric);
1248                     if(bIsStandardPattern)
1249                     {
1250                         // calculate per sample positions
1251                         psContext.vX.sample = _simd_add_ps(psContext.vX.UL, MultisampleTraits<sampleCount>::vX(sample));
1252                         psContext.vY.sample = _simd_add_ps(psContext.vY.UL, MultisampleTraits<sampleCount>::vY(sample));
1253                     }
1254                     else
1255                     {
1256                         psContext.vX.sample = psContext.vX.center;
1257                         psContext.vY.sample = psContext.vY.center;
1258                     }
1259
1260                     // calc I & J per sample
1261                     backendFuncs.pfnCalcSampleBarycentrics(coeffs, psContext);
1262
1263                     // interpolate z
1264                     if (!pPSState->writesODepth)
1265                     {
1266                         vZ[sample] = vplaneps(coeffs.vZa, coeffs.vZb, coeffs.vZc, psContext.vI.sample, psContext.vJ.sample);
1267                     }
1268
1269                     ///@todo: perspective correct vs non-perspective correct clipping?
1270                     // interpolate clip distances
1271                     if (rastState.clipDistanceMask)
1272                     {
1273                         uint8_t clipMask = ComputeUserClipMask(rastState.clipDistanceMask, work.pUserClipBuffer,
1274                             psContext.vI.sample, psContext.vJ.sample);
1275                         vCoverageMask[sample] = _simd_and_ps(vCoverageMask[sample], vMask(~clipMask));
1276                     }
1277                     RDTSC_STOP(BEBarycentric, 0, 0);
1278                 }
1279                 // else 'broadcast' and test psContext.vZ written from the PS each sample
1280                 else
1281                 {
1282                     vZ[sample] = psContext.vZ;
1283                 }
1284
1285                 // offset depth/stencil buffers current sample
1286                 uint8_t *pDepthSample = pDepthBase + MultisampleTraits<sampleCount>::RasterTileDepthOffset(sample);
1287                 uint8_t * pStencilSample = pStencilBase + MultisampleTraits<sampleCount>::RasterTileStencilOffset(sample);
1288
1289                 // ZTest for this sample
1290                 RDTSC_START(BEEarlyDepthTest);
1291                 stencilPassMask[sample] = vCoverageMask[sample];
1292                 depthPassMask[sample] = DepthStencilTest(&state.vp[0], &state.depthStencilState, work.triFlags.frontFacing,
1293                                         vZ[sample], pDepthSample, vCoverageMask[sample], pStencilSample, &stencilPassMask[sample]);
1294                 RDTSC_STOP(BEEarlyDepthTest, 0, 0);
1295
1296                 anyDepthSamplePassed = _simd_or_ps(anyDepthSamplePassed, depthPassMask[sample]);
1297                 anyStencilSamplePassed = _simd_or_ps(anyStencilSamplePassed, stencilPassMask[sample]);
1298                 uint32_t statMask = _simd_movemask_ps(depthPassMask[sample]);
1299                 uint32_t statCount = _mm_popcnt_u32(statMask);
1300                 UPDATE_STAT(DepthPassCount, statCount);
1301             }
1302
1303             // if we didn't have to execute the PS early, and at least 1 sample passed the depth test, run the PS
1304             if(!pPSState->writesODepth && !pPSState->killsPixel && _simd_movemask_ps(anyDepthSamplePassed))
1305             {
1306                 RDTSC_START(BEBarycentric);
1307                 backendFuncs.pfnCalcPixelBarycentrics(coeffs, psContext);
1308                 // interpolate z
1309                 psContext.vZ = vplaneps(coeffs.vZa, coeffs.vZb, coeffs.vZc, psContext.vI.center, psContext.vJ.center);
1310                 RDTSC_STOP(BEBarycentric, 0, 0);
1311
1312                 // execute pixel shader
1313                 RDTSC_START(BEPixelShader);
1314                 state.psState.pfnPixelShader(GetPrivateState(pDC), &psContext);
1315                 RDTSC_STOP(BEPixelShader, 0, 0);
1316             }
1317             ///@todo: make sure this works for kill pixel
1318             else if(!_simd_movemask_ps(anyStencilSamplePassed))
1319             {
1320                 goto Endtile;
1321             }
1322
1323             // loop over all samples, broadcasting the results of the PS to all passing pixels
1324             for(uint32_t sample = 0; sample < numOMSamples; sample++)
1325             {
1326                 uint8_t *pDepthSample = pDepthBase + MultisampleTraits<sampleCount>::RasterTileDepthOffset(sample);
1327                 uint8_t * pStencilSample = pStencilBase + MultisampleTraits<sampleCount>::RasterTileStencilOffset(sample);
1328
1329                 // output merger
1330                 RDTSC_START(BEOutputMerger);
1331
1332                 // skip if none of the pixels for this sample passed
1333                 simdscalar coverageMaskSample;
1334                 simdscalar depthMaskSample;
1335                 simdscalar stencilMaskSample;
1336                 simdscalar vInterpolatedZ;
1337
1338                 // forcedSampleCount outputs to any pixels with covered samples not masked off by SampleMask
1339                 // depth test is disabled, so just set the z val to 0.
1340                 if(bForcedSampleCount)
1341                 {
1342                     coverageMaskSample = depthMaskSample = anyDepthSamplePassed;
1343                     vInterpolatedZ = _simd_setzero_ps();
1344                 }
1345                 else if(bIsStandardPattern)
1346                 {
1347                     if(!_simd_movemask_ps(depthPassMask[sample]))
1348                     {
1349                         depthPassMask[sample] = _simd_setzero_ps();
1350                         DepthStencilWrite(&state.vp[0], &state.depthStencilState, work.triFlags.frontFacing, vZ[sample], pDepthSample, depthPassMask[sample],
1351                                           vCoverageMask[sample], pStencilSample, stencilPassMask[sample]);
1352                         continue;
1353                     }
1354                     coverageMaskSample = vCoverageMask[sample];
1355                     depthMaskSample = depthPassMask[sample];
1356                     stencilMaskSample = stencilPassMask[sample];
1357                     vInterpolatedZ = vZ[sample];
1358                 }
1359                 else
1360                 {
1361                     // center pattern only needs to use a single depth test as all samples are at the same position
1362                     if(!_simd_movemask_ps(depthPassMask[0]))
1363                     {
1364                         depthPassMask[0] = _simd_setzero_ps();
1365                         DepthStencilWrite(&state.vp[0], &state.depthStencilState, work.triFlags.frontFacing, vZ[0], pDepthSample, depthPassMask[0],
1366                                           vCoverageMask[0], pStencilSample, stencilPassMask[0]);
1367                         continue;
1368                     }
1369                     coverageMaskSample = (vCoverageMask[0]);
1370                     depthMaskSample = depthPassMask[0];
1371                     stencilMaskSample = stencilPassMask[0];
1372                     vInterpolatedZ = vZ[0];
1373                 }
1374
1375                 // output merger
1376                 RDTSC_START(BEOutputMerger);
1377                 backendFuncs.pfnOutputMerger(psContext, pColorBase, sample, pBlendState, state.pfnBlendFunc,
1378                                              coverageMaskSample, depthMaskSample);
1379
1380                 DepthStencilWrite(&state.vp[0], &state.depthStencilState, work.triFlags.frontFacing, vInterpolatedZ, pDepthSample, depthMaskSample,
1381                                   coverageMaskSample, pStencilSample, stencilMaskSample);
1382                 RDTSC_STOP(BEOutputMerger, 0, 0);
1383             }
1384
1385 Endtile:
1386             RDTSC_START(BEEndTile);
1387             for(uint32_t sample = 0; sample < numCoverageSamples; sample++)
1388             {
1389                 work.coverageMask[sample] >>= (SIMD_TILE_Y_DIM * SIMD_TILE_X_DIM);
1390             }
1391
1392             pDepthBase += (KNOB_SIMD_WIDTH * FormatTraits<KNOB_DEPTH_HOT_TILE_FORMAT>::bpp) / 8;
1393             pStencilBase += (KNOB_SIMD_WIDTH * FormatTraits<KNOB_STENCIL_HOT_TILE_FORMAT>::bpp) / 8;
1394
1395             for(uint32_t rt = 0; rt < NumRT; ++rt)
1396             {
1397                 pColorBase[rt] += (KNOB_SIMD_WIDTH * FormatTraits<KNOB_COLOR_HOT_TILE_FORMAT>::bpp) / 8;
1398             }
1399             RDTSC_STOP(BEEndTile, 0, 0);
1400         }
1401     }
1402 }
1403 // optimized backend flow with NULL PS
1404 template<uint32_t sampleCountT>
1405 void BackendNullPS(DRAW_CONTEXT *pDC, uint32_t workerId, uint32_t x, uint32_t y, SWR_TRIANGLE_DESC &work, RenderOutputBuffers &renderBuffers)
1406 {
1407     RDTSC_START(BESetup);
1408
1409     static const SWR_MULTISAMPLE_COUNT sampleCount = (SWR_MULTISAMPLE_COUNT)sampleCountT;
1410     SWR_CONTEXT *pContext = pDC->pContext;
1411     const API_STATE& state = GetApiState(pDC);
1412     const BACKEND_FUNCS& backendFuncs = pDC->pState->backendFuncs;
1413
1414     // broadcast scalars
1415     BarycentricCoeffs coeffs;
1416     coeffs.vIa = _simd_broadcast_ss(&work.I[0]);
1417     coeffs.vIb = _simd_broadcast_ss(&work.I[1]);
1418     coeffs.vIc = _simd_broadcast_ss(&work.I[2]);
1419
1420     coeffs.vJa = _simd_broadcast_ss(&work.J[0]);
1421     coeffs.vJb = _simd_broadcast_ss(&work.J[1]);
1422     coeffs.vJc = _simd_broadcast_ss(&work.J[2]);
1423
1424     coeffs.vZa = _simd_broadcast_ss(&work.Z[0]);
1425     coeffs.vZb = _simd_broadcast_ss(&work.Z[1]);
1426     coeffs.vZc = _simd_broadcast_ss(&work.Z[2]);
1427
1428     coeffs.vRecipDet = _simd_broadcast_ss(&work.recipDet);
1429
1430     BYTE *pDepthBase = renderBuffers.pDepth, *pStencilBase = renderBuffers.pStencil;
1431
1432     RDTSC_STOP(BESetup, 0, 0);
1433
1434     SWR_PS_CONTEXT psContext;
1435     for (uint32_t yy = y; yy < y + KNOB_TILE_Y_DIM; yy += SIMD_TILE_Y_DIM)
1436     {
1437         // UL pixel corner
1438         simdscalar vYSamplePosUL = _simd_add_ps(vULOffsetsY, _simd_set1_ps((float)yy));
1439
1440         for (uint32_t xx = x; xx < x + KNOB_TILE_X_DIM; xx += SIMD_TILE_X_DIM)
1441         {
1442             // UL pixel corners
1443             simdscalar vXSamplePosUL = _simd_add_ps(vULOffsetsX, _simd_set1_ps((float)xx));
1444
1445             // iterate over active samples
1446             unsigned long sample = 0;
1447             uint32_t sampleMask = state.blendState.sampleMask;
1448             while (_BitScanForward(&sample, sampleMask))
1449             {
1450                 sampleMask &= ~(1 << sample);
1451                 if (work.coverageMask[sample] & MASK)
1452                 {
1453                     RDTSC_START(BEBarycentric);
1454                     // calculate per sample positions
1455                     psContext.vX.sample = _simd_add_ps(vXSamplePosUL, MultisampleTraits<sampleCount>::vX(sample));
1456                     psContext.vY.sample = _simd_add_ps(vYSamplePosUL, MultisampleTraits<sampleCount>::vY(sample));
1457
1458                     backendFuncs.pfnCalcSampleBarycentrics(coeffs, psContext);
1459
1460                     // interpolate z
1461                     psContext.vZ = vplaneps(coeffs.vZa, coeffs.vZb, coeffs.vZc, psContext.vI.sample, psContext.vJ.sample);
1462
1463                     RDTSC_STOP(BEBarycentric, 0, 0);
1464
1465                     simdscalar vCoverageMask = vMask(work.coverageMask[sample] & MASK);
1466                     simdscalar stencilPassMask = vCoverageMask;
1467
1468                     // offset depth/stencil buffers current sample
1469                     uint8_t *pDepthSample = pDepthBase + MultisampleTraits<sampleCount>::RasterTileDepthOffset(sample);
1470                     uint8_t *pStencilSample = pStencilBase + MultisampleTraits<sampleCount>::RasterTileStencilOffset(sample);
1471
1472                     RDTSC_START(BEEarlyDepthTest);
1473                     simdscalar depthPassMask = DepthStencilTest(&state.vp[0], &state.depthStencilState, work.triFlags.frontFacing,
1474                         psContext.vZ, pDepthSample, vCoverageMask, pStencilSample, &stencilPassMask);
1475                     DepthStencilWrite(&state.vp[0], &state.depthStencilState, work.triFlags.frontFacing, psContext.vZ,
1476                         pDepthSample, depthPassMask, vCoverageMask, pStencilSample, stencilPassMask);
1477                     RDTSC_STOP(BEEarlyDepthTest, 0, 0);
1478
1479                     uint32_t statMask = _simd_movemask_ps(depthPassMask);
1480                     uint32_t statCount = _mm_popcnt_u32(statMask);
1481                     UPDATE_STAT(DepthPassCount, statCount);
1482                 }
1483                 work.coverageMask[sample] >>= (SIMD_TILE_Y_DIM * SIMD_TILE_X_DIM);
1484             }
1485             pDepthBase += (KNOB_SIMD_WIDTH * FormatTraits<KNOB_DEPTH_HOT_TILE_FORMAT>::bpp) / 8;
1486             pStencilBase += (KNOB_SIMD_WIDTH * FormatTraits<KNOB_STENCIL_HOT_TILE_FORMAT>::bpp) / 8;
1487         }
1488     }
1489 }
1490
1491 void InitClearTilesTable()
1492 {
1493     memset(sClearTilesTable, 0, sizeof(sClearTilesTable));
1494
1495     sClearTilesTable[R8G8B8A8_UNORM] = ClearMacroTile<R8G8B8A8_UNORM>;
1496     sClearTilesTable[B8G8R8A8_UNORM] = ClearMacroTile<B8G8R8A8_UNORM>;
1497     sClearTilesTable[R32_FLOAT] = ClearMacroTile<R32_FLOAT>;
1498     sClearTilesTable[R32G32B32A32_FLOAT] = ClearMacroTile<R32G32B32A32_FLOAT>;
1499     sClearTilesTable[R8_UINT] = ClearMacroTile<R8_UINT>;
1500 }
1501
1502 PFN_BACKEND_FUNC gBackendNullPs[SWR_MULTISAMPLE_TYPE_MAX];
1503 PFN_BACKEND_FUNC gBackendSingleSample[2][2] = {};
1504 PFN_BACKEND_FUNC gBackendPixelRateTable[SWR_MULTISAMPLE_TYPE_MAX][SWR_MSAA_SAMPLE_PATTERN_MAX][SWR_INPUT_COVERAGE_MAX][2][2] = {};
1505 PFN_BACKEND_FUNC gBackendSampleRateTable[SWR_MULTISAMPLE_TYPE_MAX][SWR_INPUT_COVERAGE_MAX][2] = {};
1506 PFN_OUTPUT_MERGER gBackendOutputMergerTable[SWR_NUM_RENDERTARGETS+1][SWR_MULTISAMPLE_TYPE_MAX] = {};
1507 PFN_CALC_PIXEL_BARYCENTRICS gPixelBarycentricTable[2] = {};
1508 PFN_CALC_SAMPLE_BARYCENTRICS gSampleBarycentricTable[2] = {};
1509 PFN_CALC_CENTROID_BARYCENTRICS gCentroidBarycentricTable[SWR_MULTISAMPLE_TYPE_MAX][2][2][2] = {};
1510
1511 // Recursive template used to auto-nest conditionals.  Converts dynamic enum function
1512 // arguments to static template arguments.
1513 template <uint32_t... ArgsT>
1514 struct OMChooser
1515 {
1516     // Last Arg Terminator
1517     static PFN_OUTPUT_MERGER GetFunc(SWR_MULTISAMPLE_COUNT tArg)
1518     {
1519         switch(tArg)
1520         {
1521         case SWR_MULTISAMPLE_1X: return OutputMerger<ArgsT..., SWR_MULTISAMPLE_1X>; break;
1522         case SWR_MULTISAMPLE_2X: return OutputMerger<ArgsT..., SWR_MULTISAMPLE_2X>; break;
1523         case SWR_MULTISAMPLE_4X: return OutputMerger<ArgsT..., SWR_MULTISAMPLE_4X>; break;
1524         case SWR_MULTISAMPLE_8X: return OutputMerger<ArgsT..., SWR_MULTISAMPLE_8X>; break;
1525         case SWR_MULTISAMPLE_16X: return OutputMerger<ArgsT..., SWR_MULTISAMPLE_16X>; break;
1526         default:
1527             SWR_ASSERT(0 && "Invalid sample count\n");
1528             return nullptr;
1529             break;
1530         }
1531     }
1532
1533     // Recursively parse args
1534     template <typename... TArgsT>
1535     static PFN_OUTPUT_MERGER GetFunc(uint32_t tArg, TArgsT... remainingArgs)
1536     {
1537         switch(tArg)
1538         {
1539         case 0: return OMChooser<ArgsT..., 0>::GetFunc(remainingArgs...); break;
1540         case 1: return OMChooser<ArgsT..., 1>::GetFunc(remainingArgs...); break;
1541         case 2: return OMChooser<ArgsT..., 2>::GetFunc(remainingArgs...); break;
1542         case 3: return OMChooser<ArgsT..., 3>::GetFunc(remainingArgs...); break;
1543         case 4: return OMChooser<ArgsT..., 4>::GetFunc(remainingArgs...); break;
1544         case 5: return OMChooser<ArgsT..., 5>::GetFunc(remainingArgs...); break;
1545         case 6: return OMChooser<ArgsT..., 6>::GetFunc(remainingArgs...); break;
1546         case 7: return OMChooser<ArgsT..., 7>::GetFunc(remainingArgs...); break;
1547         case 8: return OMChooser<ArgsT..., 8>::GetFunc(remainingArgs...); break;
1548         default:
1549             SWR_ASSERT(0 && "Invalid RT index\n");
1550             return nullptr;
1551             break;
1552         }
1553     }
1554 };
1555
1556 // Recursive template used to auto-nest conditionals.  Converts dynamic enum function
1557 // arguments to static template arguments.
1558 template <uint32_t... ArgsT>
1559 struct BECentroidBarycentricChooser
1560 {
1561
1562     // Last Arg Terminator
1563     template <typename... TArgsT>
1564     static PFN_CALC_CENTROID_BARYCENTRICS GetFunc(uint32_t tArg)
1565     {
1566         if(tArg > 0)
1567         {
1568             return CalcCentroidBarycentrics<ArgsT..., 1>;
1569         }
1570
1571         return CalcCentroidBarycentrics<ArgsT..., 0>;
1572     }
1573
1574     // Recursively parse args
1575     template <typename... TArgsT>
1576     static PFN_CALC_CENTROID_BARYCENTRICS GetFunc(SWR_MULTISAMPLE_COUNT tArg, TArgsT... remainingArgs)
1577     {
1578         switch(tArg)
1579         {
1580         case SWR_MULTISAMPLE_1X: return BECentroidBarycentricChooser<ArgsT..., SWR_MULTISAMPLE_1X>::GetFunc(remainingArgs...); break;
1581         case SWR_MULTISAMPLE_2X: return BECentroidBarycentricChooser<ArgsT..., SWR_MULTISAMPLE_2X>::GetFunc(remainingArgs...); break;
1582         case SWR_MULTISAMPLE_4X: return BECentroidBarycentricChooser<ArgsT..., SWR_MULTISAMPLE_4X>::GetFunc(remainingArgs...); break;
1583         case SWR_MULTISAMPLE_8X: return BECentroidBarycentricChooser<ArgsT..., SWR_MULTISAMPLE_8X>::GetFunc(remainingArgs...); break;
1584         case SWR_MULTISAMPLE_16X: return BECentroidBarycentricChooser<ArgsT..., SWR_MULTISAMPLE_16X>::GetFunc(remainingArgs...); break;
1585         default:
1586             SWR_ASSERT(0 && "Invalid sample count\n");
1587             return nullptr;
1588             break;
1589         }
1590     }
1591
1592     // Recursively parse args
1593     template <typename... TArgsT>
1594     static PFN_CALC_CENTROID_BARYCENTRICS GetFunc(uint32_t tArg, TArgsT... remainingArgs)
1595     {
1596         if(tArg > 0)
1597         {
1598             return BECentroidBarycentricChooser<ArgsT..., 1>::GetFunc(remainingArgs...);
1599         }
1600
1601         return BECentroidBarycentricChooser<ArgsT..., 0>::GetFunc(remainingArgs...);
1602     }
1603 };
1604
1605 // Recursive template used to auto-nest conditionals.  Converts dynamic enum function
1606 // arguments to static template arguments.
1607 template <uint32_t... ArgsT>
1608 struct BEChooser
1609 {
1610     // Last Arg Terminator
1611     static PFN_BACKEND_FUNC GetFunc(SWR_BACKEND_FUNCS tArg)
1612     {
1613         switch(tArg)
1614         {
1615         case SWR_BACKEND_SINGLE_SAMPLE: return BackendSingleSample<ArgsT...>; break;
1616         case SWR_BACKEND_MSAA_PIXEL_RATE: return BackendPixelRate<ArgsT...>; break;
1617         case SWR_BACKEND_MSAA_SAMPLE_RATE: return BackendSampleRate<ArgsT...>; break;
1618         default:
1619             SWR_ASSERT(0 && "Invalid backend func\n");
1620             return nullptr;
1621             break;
1622         }
1623     }
1624
1625
1626     // Recursively parse args
1627     template <typename... TArgsT>
1628     static PFN_BACKEND_FUNC GetFunc(SWR_MULTISAMPLE_COUNT tArg, TArgsT... remainingArgs)
1629     {
1630         switch(tArg)
1631         {
1632         case SWR_MULTISAMPLE_1X: return BEChooser<ArgsT..., SWR_MULTISAMPLE_1X>::GetFunc(remainingArgs...); break;
1633         case SWR_MULTISAMPLE_2X: return BEChooser<ArgsT..., SWR_MULTISAMPLE_2X>::GetFunc(remainingArgs...); break;
1634         case SWR_MULTISAMPLE_4X: return BEChooser<ArgsT..., SWR_MULTISAMPLE_4X>::GetFunc(remainingArgs...); break;
1635         case SWR_MULTISAMPLE_8X: return BEChooser<ArgsT..., SWR_MULTISAMPLE_8X>::GetFunc(remainingArgs...); break;
1636         case SWR_MULTISAMPLE_16X: return BEChooser<ArgsT..., SWR_MULTISAMPLE_16X>::GetFunc(remainingArgs...); break;
1637         default:
1638             SWR_ASSERT(0 && "Invalid sample count\n");
1639             return nullptr;
1640             break;
1641         }
1642     }
1643
1644     // Recursively parse args
1645     template <typename... TArgsT>
1646     static PFN_BACKEND_FUNC GetFunc(uint32_t tArg, TArgsT... remainingArgs)
1647     {
1648         if(tArg > 0)
1649         {
1650             return BEChooser<ArgsT..., 1>::GetFunc(remainingArgs...);
1651         }
1652
1653          return BEChooser<ArgsT..., 0>::GetFunc(remainingArgs...);
1654     }
1655 };
1656
1657 template <uint32_t numRenderTargets, SWR_MULTISAMPLE_COUNT numSampleRates>
1658 void InitBackendOMFuncTable(PFN_OUTPUT_MERGER (&table)[numRenderTargets][numSampleRates])
1659 {
1660     for(uint32_t rtNum = SWR_ATTACHMENT_COLOR0; rtNum < numRenderTargets; rtNum++)
1661     {
1662         for(uint32_t sampleCount = SWR_MULTISAMPLE_1X; sampleCount < numSampleRates; sampleCount++)
1663         {
1664             table[rtNum][sampleCount] =
1665                 OMChooser<>::GetFunc((SWR_RENDERTARGET_ATTACHMENT)rtNum, (SWR_MULTISAMPLE_COUNT)sampleCount);
1666         }
1667     }
1668 }
1669
1670 template <SWR_MULTISAMPLE_COUNT numSampleRates>
1671 void InitBackendBarycentricsTables(PFN_CALC_PIXEL_BARYCENTRICS (&pixelTable)[2],
1672                                    PFN_CALC_SAMPLE_BARYCENTRICS (&sampleTable)[2],
1673                                    PFN_CALC_CENTROID_BARYCENTRICS (&centroidTable)[numSampleRates][2][2][2])
1674 {
1675     pixelTable[0] = CalcPixelBarycentrics<0>;
1676     pixelTable[1] = CalcPixelBarycentrics<1>;
1677
1678     sampleTable[0] = CalcSampleBarycentrics<0>;
1679     sampleTable[1] = CalcSampleBarycentrics<1>;
1680
1681     for(uint32_t sampleCount = SWR_MULTISAMPLE_1X; sampleCount < numSampleRates; sampleCount++)
1682     {
1683         for(uint32_t baryMask = 0; baryMask < 2; baryMask++)
1684         {
1685             for(uint32_t patternNum = 0; patternNum < 2; patternNum++)
1686             {
1687                 for(uint32_t forcedSampleEnable = 0; forcedSampleEnable < 2; forcedSampleEnable++)
1688                 {
1689                     centroidTable[sampleCount][baryMask][patternNum][forcedSampleEnable]=
1690                         BECentroidBarycentricChooser<>::GetFunc((SWR_MULTISAMPLE_COUNT)sampleCount, baryMask, patternNum, forcedSampleEnable);
1691                 }
1692             }
1693         }
1694     }
1695 }
1696
1697 void InitBackendSampleFuncTable(PFN_BACKEND_FUNC (&table)[2][2])
1698 {
1699     gBackendSingleSample[0][0] = BEChooser<>::GetFunc(SWR_MULTISAMPLE_1X, SWR_MSAA_STANDARD_PATTERN, SWR_INPUT_COVERAGE_NONE, 0, 0, (SWR_BACKEND_FUNCS)SWR_BACKEND_SINGLE_SAMPLE);
1700     gBackendSingleSample[0][1] = BEChooser<>::GetFunc(SWR_MULTISAMPLE_1X, SWR_MSAA_STANDARD_PATTERN, SWR_INPUT_COVERAGE_NONE, 1, 0, (SWR_BACKEND_FUNCS)SWR_BACKEND_SINGLE_SAMPLE);
1701     gBackendSingleSample[1][0] = BEChooser<>::GetFunc(SWR_MULTISAMPLE_1X, SWR_MSAA_STANDARD_PATTERN, SWR_INPUT_COVERAGE_NORMAL, 0, 0, (SWR_BACKEND_FUNCS)SWR_BACKEND_SINGLE_SAMPLE);
1702     gBackendSingleSample[1][1] = BEChooser<>::GetFunc(SWR_MULTISAMPLE_1X, SWR_MSAA_STANDARD_PATTERN, SWR_INPUT_COVERAGE_NORMAL, 1, 0, (SWR_BACKEND_FUNCS)SWR_BACKEND_SINGLE_SAMPLE);
1703 }
1704
1705 template <SWR_MULTISAMPLE_COUNT numSampleRates, SWR_MSAA_SAMPLE_PATTERN numSamplePatterns, SWR_INPUT_COVERAGE numCoverageModes>
1706 void InitBackendPixelFuncTable(PFN_BACKEND_FUNC (&table)[numSampleRates][numSamplePatterns][numCoverageModes][2][2])
1707 {
1708     for(uint32_t sampleCount = SWR_MULTISAMPLE_1X; sampleCount < numSampleRates; sampleCount++)
1709     {
1710         for(uint32_t samplePattern = SWR_MSAA_CENTER_PATTERN; samplePattern < numSamplePatterns; samplePattern++)
1711         {
1712             for(uint32_t inputCoverage = SWR_INPUT_COVERAGE_NONE; inputCoverage < numCoverageModes; inputCoverage++)
1713             {
1714                 for(uint32_t isCentroid = 0; isCentroid < 2; isCentroid++)
1715                 {
1716                     table[sampleCount][samplePattern][inputCoverage][isCentroid][0] =
1717                         BEChooser<>::GetFunc((SWR_MULTISAMPLE_COUNT)sampleCount, samplePattern, inputCoverage, isCentroid, 0, (SWR_BACKEND_FUNCS)SWR_BACKEND_MSAA_PIXEL_RATE);
1718                     table[sampleCount][samplePattern][inputCoverage][isCentroid][1] =
1719                         BEChooser<>::GetFunc((SWR_MULTISAMPLE_COUNT)sampleCount, samplePattern, inputCoverage, isCentroid, 1, (SWR_BACKEND_FUNCS)SWR_BACKEND_MSAA_PIXEL_RATE);
1720                 }
1721             }
1722         }
1723     }
1724 }
1725
1726 template <uint32_t numSampleRates, uint32_t numCoverageModes>
1727 void InitBackendSampleFuncTable(PFN_BACKEND_FUNC (&table)[numSampleRates][numCoverageModes][2])
1728 {
1729     for(uint32_t sampleCount = SWR_MULTISAMPLE_1X; sampleCount < numSampleRates; sampleCount++)
1730     {
1731         for(uint32_t inputCoverage = SWR_INPUT_COVERAGE_NONE; inputCoverage < numCoverageModes; inputCoverage++)
1732         {
1733             table[sampleCount][inputCoverage][0] =
1734                 BEChooser<>::GetFunc((SWR_MULTISAMPLE_COUNT)sampleCount, SWR_MSAA_STANDARD_PATTERN, inputCoverage, 0, 0, (SWR_BACKEND_FUNCS)SWR_BACKEND_MSAA_SAMPLE_RATE);
1735             table[sampleCount][inputCoverage][1] =
1736                 BEChooser<>::GetFunc((SWR_MULTISAMPLE_COUNT)sampleCount, SWR_MSAA_STANDARD_PATTERN, inputCoverage, 1, 0, (SWR_BACKEND_FUNCS)SWR_BACKEND_MSAA_SAMPLE_RATE);
1737         }
1738     }
1739 }
1740
1741 void InitBackendFuncTables()
1742 {
1743     InitBackendSampleFuncTable(gBackendSingleSample);
1744     InitBackendPixelFuncTable<(SWR_MULTISAMPLE_COUNT)SWR_MULTISAMPLE_TYPE_MAX, SWR_MSAA_SAMPLE_PATTERN_MAX, SWR_INPUT_COVERAGE_MAX>(gBackendPixelRateTable);
1745     InitBackendSampleFuncTable<SWR_MULTISAMPLE_TYPE_MAX, SWR_INPUT_COVERAGE_MAX>(gBackendSampleRateTable);
1746     InitBackendOMFuncTable<SWR_NUM_RENDERTARGETS+1, SWR_MULTISAMPLE_TYPE_MAX>(gBackendOutputMergerTable);
1747     InitBackendBarycentricsTables<(SWR_MULTISAMPLE_COUNT)(SWR_MULTISAMPLE_TYPE_MAX)>(gPixelBarycentricTable, gSampleBarycentricTable, gCentroidBarycentricTable);
1748
1749     gBackendNullPs[SWR_MULTISAMPLE_1X] = &BackendNullPS < SWR_MULTISAMPLE_1X > ;
1750     gBackendNullPs[SWR_MULTISAMPLE_2X] = &BackendNullPS < SWR_MULTISAMPLE_2X > ;
1751     gBackendNullPs[SWR_MULTISAMPLE_4X] = &BackendNullPS < SWR_MULTISAMPLE_4X > ;
1752     gBackendNullPs[SWR_MULTISAMPLE_8X] = &BackendNullPS < SWR_MULTISAMPLE_8X > ;
1753     gBackendNullPs[SWR_MULTISAMPLE_16X] = &BackendNullPS < SWR_MULTISAMPLE_16X > ;
1754 }