src/gallium/drivers/swr/rasterizer/core/backend.cpp

   1 /****************************************************************************
   2 * Copyright (C) 2014-2015 Intel Corporation.   All Rights Reserved.
   3 *
   4 * Permission is hereby granted, free of charge, to any person obtaining a
   5 * copy of this software and associated documentation files (the "Software"),
   6 * to deal in the Software without restriction, including without limitation
   7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8 * and/or sell copies of the Software, and to permit persons to whom the
   9 * Software is furnished to do so, subject to the following conditions:
  10 *
  11 * The above copyright notice and this permission notice (including the next
  12 * paragraph) shall be included in all copies or substantial portions of the
  13 * Software.
  14 *
  15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21 * IN THE SOFTWARE.
  22 *
  23 * @file backend.cpp
  24 *
  25 * @brief Backend handles rasterization, pixel shading and output merger
  26 *        operations.
  27 *
  28 ******************************************************************************/
  29
  30 #include <smmintrin.h>
  31
  32 #include "rdtsc_core.h"
  33 #include "backend.h"
  34 #include "depthstencil.h"
  35 #include "tilemgr.h"
  36 #include "memory/tilingtraits.h"
  37 #include "core/multisample.h"
  38
  39 #include <algorithm>
  40
  41 const __m128 vTileOffsetsX = {0.5, KNOB_TILE_X_DIM - 0.5, 0.5, KNOB_TILE_X_DIM - 0.5};
  42 const __m128 vTileOffsetsY = {0.5, 0.5, KNOB_TILE_Y_DIM - 0.5, KNOB_TILE_Y_DIM - 0.5};
  43
  44 /// @todo move to common lib
  45 #define MASKTOVEC(i3,i2,i1,i0) {-i0,-i1,-i2,-i3}
  46 static const __m128 gMaskToVec[] = {
  47     MASKTOVEC(0,0,0,0),
  48     MASKTOVEC(0,0,0,1),
  49     MASKTOVEC(0,0,1,0),
  50     MASKTOVEC(0,0,1,1),
  51     MASKTOVEC(0,1,0,0),
  52     MASKTOVEC(0,1,0,1),
  53     MASKTOVEC(0,1,1,0),
  54     MASKTOVEC(0,1,1,1),
  55     MASKTOVEC(1,0,0,0),
  56     MASKTOVEC(1,0,0,1),
  57     MASKTOVEC(1,0,1,0),
  58     MASKTOVEC(1,0,1,1),
  59     MASKTOVEC(1,1,0,0),
  60     MASKTOVEC(1,1,0,1),
  61     MASKTOVEC(1,1,1,0),
  62     MASKTOVEC(1,1,1,1),
  63 };
  64
  65 typedef void(*PFN_CLEAR_TILES)(DRAW_CONTEXT*, SWR_RENDERTARGET_ATTACHMENT rt, uint32_t, DWORD[4]);
  66 static PFN_CLEAR_TILES sClearTilesTable[NUM_SWR_FORMATS];
  67
  68 //////////////////////////////////////////////////////////////////////////
  69 /// @brief Process compute work.
  70 /// @param pDC - pointer to draw context (dispatch).
  71 /// @param workerId - The unique worker ID that is assigned to this thread.
  72 /// @param threadGroupId - the linear index for the thread group within the dispatch.
  73 void ProcessComputeBE(DRAW_CONTEXT* pDC, uint32_t workerId, uint32_t threadGroupId)
  74 {
  75     RDTSC_START(BEDispatch);
  76
  77     SWR_CONTEXT *pContext = pDC->pContext;
  78
  79     const COMPUTE_DESC* pTaskData = (COMPUTE_DESC*)pDC->pDispatch->GetTasksData();
  80     SWR_ASSERT(pTaskData != nullptr);
  81
  82     // Ensure spill fill memory has been allocated.
  83     if (pDC->pSpillFill[workerId] == nullptr)
  84     {
  85         ///@todo Add state which indicates the spill fill size.
  86         pDC->pSpillFill[workerId] = (uint8_t*)pDC->pArena->AllocAlignedSync(4096 * 1024, sizeof(float) * 8);
  87     }
  88
  89     const API_STATE& state = GetApiState(pDC);
  90
  91     SWR_CS_CONTEXT csContext{ 0 };
  92     csContext.tileCounter = threadGroupId;
  93     csContext.dispatchDims[0] = pTaskData->threadGroupCountX;
  94     csContext.dispatchDims[1] = pTaskData->threadGroupCountY;
  95     csContext.dispatchDims[2] = pTaskData->threadGroupCountZ;
  96     csContext.pTGSM = pContext->pScratch[workerId];
  97     csContext.pSpillFillBuffer = pDC->pSpillFill[workerId];
  98
  99     state.pfnCsFunc(GetPrivateState(pDC), &csContext);
 100
 101     UPDATE_STAT(CsInvocations, state.totalThreadsInGroup);
 102
 103     RDTSC_STOP(BEDispatch, 1, 0);
 104 }
 105
 106 void ProcessSyncBE(DRAW_CONTEXT *pDC, uint32_t workerId, uint32_t macroTile, void *pUserData)
 107 {
 108     SYNC_DESC *pSync = (SYNC_DESC*)pUserData;
 109
 110     uint32_t x, y;
 111     MacroTileMgr::getTileIndices(macroTile, x, y);
 112     SWR_ASSERT(x == 0 && y == 0);
 113
 114     if (pSync->pfnCallbackFunc != nullptr)
 115     {
 116         pSync->pfnCallbackFunc(pSync->userData, pSync->userData2, pSync->userData3);
 117     }
 118 }
 119
 120 void ProcessQueryStatsBE(DRAW_CONTEXT *pDC, uint32_t workerId, uint32_t macroTile, void *pUserData)
 121 {
 122     QUERY_DESC* pQueryDesc = (QUERY_DESC*)pUserData;
 123     SWR_STATS* pStats = pQueryDesc->pStats;
 124     SWR_CONTEXT *pContext = pDC->pContext;
 125
 126     SWR_ASSERT(pStats != nullptr);
 127
 128     for (uint32_t i = 0; i < pContext->NumWorkerThreads; ++i)
 129     {
 130         pStats->DepthPassCount += pContext->stats[i].DepthPassCount;
 131
 132         pStats->IaVertices    += pContext->stats[i].IaVertices;
 133         pStats->IaPrimitives  += pContext->stats[i].IaPrimitives;
 134         pStats->VsInvocations += pContext->stats[i].VsInvocations;
 135         pStats->HsInvocations += pContext->stats[i].HsInvocations;
 136         pStats->DsInvocations += pContext->stats[i].DsInvocations;
 137         pStats->GsInvocations += pContext->stats[i].GsInvocations;
 138         pStats->PsInvocations += pContext->stats[i].PsInvocations;
 139         pStats->CInvocations  += pContext->stats[i].CInvocations;
 140         pStats->CsInvocations += pContext->stats[i].CsInvocations;
 141         pStats->CPrimitives   += pContext->stats[i].CPrimitives;
 142         pStats->GsPrimitives  += pContext->stats[i].GsPrimitives;
 143
 144         for (uint32_t stream = 0; stream < MAX_SO_STREAMS; ++stream)
 145         {
 146             pStats->SoWriteOffset[stream] += pContext->stats[i].SoWriteOffset[stream];
 147
 148             /// @note client is required to provide valid write offset before every draw, so we clear
 149             /// out the contents of the write offset when storing stats
 150             pContext->stats[i].SoWriteOffset[stream] = 0;
 151
 152             pStats->SoPrimStorageNeeded[stream] += pContext->stats[i].SoPrimStorageNeeded[stream];
 153             pStats->SoNumPrimsWritten[stream] += pContext->stats[i].SoNumPrimsWritten[stream];
 154         }
 155     }
 156 }
 157
 158 template<SWR_FORMAT format>
 159 void ClearRasterTile(BYTE *pTileBuffer, simdvector &value)
 160 {
 161     auto lambda = [&](int comp)
 162     {
 163         FormatTraits<format>::storeSOA(comp, pTileBuffer, value.v[comp]);
 164         pTileBuffer += (KNOB_SIMD_WIDTH * FormatTraits<format>::GetBPC(comp) / 8);
 165     };
 166
 167     const uint32_t numIter = (KNOB_TILE_Y_DIM / SIMD_TILE_Y_DIM) * (KNOB_TILE_X_DIM / SIMD_TILE_X_DIM);
 168     for (uint32_t i = 0; i < numIter; ++i)
 169     {
 170         UnrollerL<0, FormatTraits<format>::numComps, 1>::step(lambda);
 171     }
 172 }
 173
 174 template<SWR_FORMAT format>
 175 INLINE void ClearMacroTile(DRAW_CONTEXT *pDC, SWR_RENDERTARGET_ATTACHMENT rt, uint32_t macroTile, DWORD clear[4])
 176 {
 177     // convert clear color to hottile format
 178     // clear color is in RGBA float/uint32
 179     simdvector vClear;
 180     for (uint32_t comp = 0; comp < FormatTraits<format>::numComps; ++comp)
 181     {
 182         simdscalar vComp;
 183         vComp = _simd_load1_ps((const float*)&clear[comp]);
 184         if (FormatTraits<format>::isNormalized(comp))
 185         {
 186             vComp = _simd_mul_ps(vComp, _simd_set1_ps(FormatTraits<format>::fromFloat(comp)));
 187             vComp = _simd_castsi_ps(_simd_cvtps_epi32(vComp));
 188         }
 189         vComp = FormatTraits<format>::pack(comp, vComp);
 190         vClear.v[FormatTraits<format>::swizzle(comp)] = vComp;
 191     }
 192
 193     uint32_t tileX, tileY;
 194     MacroTileMgr::getTileIndices(macroTile, tileX, tileY);
 195     const API_STATE& state = GetApiState(pDC);
 196
 197     int top = KNOB_MACROTILE_Y_DIM_FIXED * tileY;
 198     int bottom = top + KNOB_MACROTILE_Y_DIM_FIXED - 1;
 199     int left = KNOB_MACROTILE_X_DIM_FIXED * tileX;
 200     int right = left + KNOB_MACROTILE_X_DIM_FIXED - 1;
 201
 202     // intersect with scissor
 203     top = std::max(top, state.scissorInFixedPoint.top);
 204     left = std::max(left, state.scissorInFixedPoint.left);
 205     bottom = std::min(bottom, state.scissorInFixedPoint.bottom);
 206     right = std::min(right, state.scissorInFixedPoint.right);
 207
 208     // translate to local hottile origin
 209     top -= KNOB_MACROTILE_Y_DIM_FIXED * tileY;
 210     bottom -= KNOB_MACROTILE_Y_DIM_FIXED * tileY;
 211     left -= KNOB_MACROTILE_X_DIM_FIXED * tileX;
 212     right -= KNOB_MACROTILE_X_DIM_FIXED * tileX;
 213
 214     // convert to raster tiles
 215     top >>= (KNOB_TILE_Y_DIM_SHIFT + FIXED_POINT_SHIFT);
 216     bottom >>= (KNOB_TILE_Y_DIM_SHIFT + FIXED_POINT_SHIFT);
 217     left >>= (KNOB_TILE_X_DIM_SHIFT + FIXED_POINT_SHIFT);
 218     right >>= (KNOB_TILE_X_DIM_SHIFT + FIXED_POINT_SHIFT);
 219
 220     const int numSamples = GetNumSamples(pDC->pState->state.rastState.sampleCount);
 221     // compute steps between raster tile samples / raster tiles / macro tile rows
 222     const uint32_t rasterTileSampleStep = KNOB_TILE_X_DIM * KNOB_TILE_Y_DIM * FormatTraits<format>::bpp / 8;
 223     const uint32_t rasterTileStep = (KNOB_TILE_X_DIM * KNOB_TILE_Y_DIM * (FormatTraits<format>::bpp / 8)) * numSamples;
 224     const uint32_t macroTileRowStep = (KNOB_MACROTILE_X_DIM / KNOB_TILE_X_DIM) * rasterTileStep;
 225     const uint32_t pitch = (FormatTraits<format>::bpp * KNOB_MACROTILE_X_DIM / 8);
 226
 227     HOTTILE *pHotTile = pDC->pContext->pHotTileMgr->GetHotTile(pDC->pContext, pDC, macroTile, rt, true, numSamples);
 228     uint32_t rasterTileStartOffset = (ComputeTileOffset2D< TilingTraits<SWR_TILE_SWRZ, FormatTraits<format>::bpp > >(pitch, left, top)) * numSamples;
 229     uint8_t* pRasterTileRow = pHotTile->pBuffer + rasterTileStartOffset; //(ComputeTileOffset2D< TilingTraits<SWR_TILE_SWRZ, FormatTraits<format>::bpp > >(pitch, x, y)) * numSamples;
 230
 231     // loop over all raster tiles in the current hot tile
 232     for (int y = top; y <= bottom; ++y)
 233     {
 234         uint8_t* pRasterTile = pRasterTileRow;
 235         for (int x = left; x <= right; ++x)
 236         {
 237             for( int sampleNum = 0; sampleNum < numSamples; sampleNum++)
 238             {
 239                 ClearRasterTile<format>(pRasterTile, vClear);
 240                 pRasterTile += rasterTileSampleStep;
 241             }
 242         }
 243         pRasterTileRow += macroTileRowStep;
 244     }
 245
 246     pHotTile->state = HOTTILE_DIRTY;
 247 }
 248
 249
 250 void ProcessClearBE(DRAW_CONTEXT *pDC, uint32_t workerId, uint32_t macroTile, void *pUserData)
 251 {
 252     if (KNOB_FAST_CLEAR)
 253     {
 254         CLEAR_DESC *pClear = (CLEAR_DESC*)pUserData;
 255         SWR_CONTEXT *pContext = pDC->pContext;
 256         SWR_MULTISAMPLE_COUNT sampleCount = pDC->pState->state.rastState.sampleCount;
 257         uint32_t numSamples = GetNumSamples(sampleCount);
 258
 259         SWR_ASSERT(pClear->flags.bits != 0); // shouldn't be here without a reason.
 260
 261         RDTSC_START(BEClear);
 262
 263         if (pClear->flags.mask & SWR_CLEAR_COLOR)
 264         {
 265             HOTTILE *pHotTile = pContext->pHotTileMgr->GetHotTile(pContext, pDC, macroTile, SWR_ATTACHMENT_COLOR0, true, numSamples);
 266             // All we want to do here is to mark the hot tile as being in a "needs clear" state.
 267             pHotTile->clearData[0] = *(DWORD*)&(pClear->clearRTColor[0]);
 268             pHotTile->clearData[1] = *(DWORD*)&(pClear->clearRTColor[1]);
 269             pHotTile->clearData[2] = *(DWORD*)&(pClear->clearRTColor[2]);
 270             pHotTile->clearData[3] = *(DWORD*)&(pClear->clearRTColor[3]);
 271             pHotTile->state = HOTTILE_CLEAR;
 272         }
 273
 274         if (pClear->flags.mask & SWR_CLEAR_DEPTH)
 275         {
 276             HOTTILE *pHotTile = pContext->pHotTileMgr->GetHotTile(pContext, pDC, macroTile, SWR_ATTACHMENT_DEPTH, true, numSamples);
 277             pHotTile->clearData[0] = *(DWORD*)&pClear->clearDepth;
 278             pHotTile->state = HOTTILE_CLEAR;
 279         }
 280
 281         if (pClear->flags.mask & SWR_CLEAR_STENCIL)
 282         {
 283             HOTTILE *pHotTile = pContext->pHotTileMgr->GetHotTile(pContext, pDC, macroTile, SWR_ATTACHMENT_STENCIL, true, numSamples);
 284
 285             pHotTile->clearData[0] = *(DWORD*)&pClear->clearStencil;
 286             pHotTile->state = HOTTILE_CLEAR;
 287         }
 288
 289         RDTSC_STOP(BEClear, 0, 0);
 290     }
 291     else
 292     {
 293         // Legacy clear
 294         CLEAR_DESC *pClear = (CLEAR_DESC*)pUserData;
 295         RDTSC_START(BEClear);
 296
 297         if (pClear->flags.mask & SWR_CLEAR_COLOR)
 298         {
 299             /// @todo clear data should come in as RGBA32_FLOAT
 300             DWORD clearData[4];
 301             float clearFloat[4];
 302             clearFloat[0] = ((BYTE*)(&pClear->clearRTColor))[0] / 255.0f;
 303             clearFloat[1] = ((BYTE*)(&pClear->clearRTColor))[1] / 255.0f;
 304             clearFloat[2] = ((BYTE*)(&pClear->clearRTColor))[2] / 255.0f;
 305             clearFloat[3] = ((BYTE*)(&pClear->clearRTColor))[3] / 255.0f;
 306             clearData[0] = *(DWORD*)&clearFloat[0];
 307             clearData[1] = *(DWORD*)&clearFloat[1];
 308             clearData[2] = *(DWORD*)&clearFloat[2];
 309             clearData[3] = *(DWORD*)&clearFloat[3];
 310
 311             PFN_CLEAR_TILES pfnClearTiles = sClearTilesTable[KNOB_COLOR_HOT_TILE_FORMAT];
 312             SWR_ASSERT(pfnClearTiles != nullptr);
 313
 314             pfnClearTiles(pDC, SWR_ATTACHMENT_COLOR0, macroTile, clearData);
 315         }
 316
 317         if (pClear->flags.mask & SWR_CLEAR_DEPTH)
 318         {
 319             DWORD clearData[4];
 320             clearData[0] = *(DWORD*)&pClear->clearDepth;
 321             PFN_CLEAR_TILES pfnClearTiles = sClearTilesTable[KNOB_DEPTH_HOT_TILE_FORMAT];
 322             SWR_ASSERT(pfnClearTiles != nullptr);
 323
 324             pfnClearTiles(pDC, SWR_ATTACHMENT_DEPTH, macroTile, clearData);
 325         }
 326
 327         if (pClear->flags.mask & SWR_CLEAR_STENCIL)
 328         {
 329             uint32_t value = pClear->clearStencil;
 330             DWORD clearData[4];
 331             clearData[0] = *(DWORD*)&value;
 332             PFN_CLEAR_TILES pfnClearTiles = sClearTilesTable[KNOB_STENCIL_HOT_TILE_FORMAT];
 333
 334             pfnClearTiles(pDC, SWR_ATTACHMENT_STENCIL, macroTile, clearData);
 335         }
 336
 337         RDTSC_STOP(BEClear, 0, 0);
 338     }
 339 }
 340
 341
 342 void ProcessStoreTileBE(DRAW_CONTEXT *pDC, uint32_t workerId, uint32_t macroTile, void *pData)
 343 {
 344     RDTSC_START(BEStoreTiles);
 345     STORE_TILES_DESC *pDesc = (STORE_TILES_DESC*)pData;
 346     SWR_CONTEXT *pContext = pDC->pContext;
 347
 348 #ifdef KNOB_ENABLE_RDTSC
 349     uint32_t numTiles = 0;
 350 #endif
 351     SWR_FORMAT srcFormat;
 352     switch (pDesc->attachment)
 353     {
 354     case SWR_ATTACHMENT_COLOR0:
 355     case SWR_ATTACHMENT_COLOR1:
 356     case SWR_ATTACHMENT_COLOR2:
 357     case SWR_ATTACHMENT_COLOR3:
 358     case SWR_ATTACHMENT_COLOR4:
 359     case SWR_ATTACHMENT_COLOR5:
 360     case SWR_ATTACHMENT_COLOR6:
 361     case SWR_ATTACHMENT_COLOR7: srcFormat = KNOB_COLOR_HOT_TILE_FORMAT; break;
 362     case SWR_ATTACHMENT_DEPTH: srcFormat = KNOB_DEPTH_HOT_TILE_FORMAT; break;
 363     case SWR_ATTACHMENT_STENCIL: srcFormat = KNOB_STENCIL_HOT_TILE_FORMAT; break;
 364     default: SWR_ASSERT(false, "Unknown attachment: %d", pDesc->attachment); srcFormat = KNOB_COLOR_HOT_TILE_FORMAT; break;
 365     }
 366
 367     uint32_t x, y;
 368     MacroTileMgr::getTileIndices(macroTile, x, y);
 369
 370     // Only need to store the hottile if it's been rendered to...
 371     HOTTILE *pHotTile = pContext->pHotTileMgr->GetHotTile(pContext, pDC, macroTile, pDesc->attachment, false);
 372     if (pHotTile)
 373     {
 374         // clear if clear is pending (i.e., not rendered to), then mark as dirty for store.
 375         if (pHotTile->state == HOTTILE_CLEAR)
 376         {
 377             PFN_CLEAR_TILES pfnClearTiles = sClearTilesTable[srcFormat];
 378             SWR_ASSERT(pfnClearTiles != nullptr);
 379
 380             pfnClearTiles(pDC, pDesc->attachment, macroTile, pHotTile->clearData);
 381         }
 382
 383         if (pHotTile->state == HOTTILE_DIRTY || pDesc->postStoreTileState == (SWR_TILE_STATE)HOTTILE_DIRTY)
 384         {
 385             int destX = KNOB_MACROTILE_X_DIM * x;
 386             int destY = KNOB_MACROTILE_Y_DIM * y;
 387
 388             pContext->pfnStoreTile(GetPrivateState(pDC), srcFormat,
 389                 pDesc->attachment, destX, destY, pHotTile->renderTargetArrayIndex, pHotTile->pBuffer);
 390         }
 391
 392
 393         if (pHotTile->state == HOTTILE_DIRTY || pHotTile->state == HOTTILE_RESOLVED)
 394         {
 395             pHotTile->state = (HOTTILE_STATE)pDesc->postStoreTileState;
 396         }
 397     }
 398     RDTSC_STOP(BEStoreTiles, numTiles, pDC->drawId);
 399 }
 400
 401
 402 void ProcessInvalidateTilesBE(DRAW_CONTEXT *pDC, uint32_t workerId, uint32_t macroTile, void *pData)
 403 {
 404     INVALIDATE_TILES_DESC *pDesc = (INVALIDATE_TILES_DESC*)pData;
 405     SWR_CONTEXT *pContext = pDC->pContext;
 406
 407     for (uint32_t i = 0; i < SWR_NUM_ATTACHMENTS; ++i)
 408     {
 409         if (pDesc->attachmentMask & (1 << i))
 410         {
 411             HOTTILE *pHotTile = pContext->pHotTileMgr->GetHotTileNoLoad(pContext, pDC, macroTile, (SWR_RENDERTARGET_ATTACHMENT)i);
 412             if (pHotTile)
 413             {
 414                 SWR_ASSERT(pHotTile->state == HOTTILE_INVALID || pHotTile->state == HOTTILE_RESOLVED);
 415                 pHotTile->state = HOTTILE_INVALID;
 416             }
 417         }
 418     }
 419 }
 420
 421 #if KNOB_SIMD_WIDTH == 8
 422 const __m256 vCenterOffsetsX = {0.5, 1.5, 0.5, 1.5, 2.5, 3.5, 2.5, 3.5};
 423 const __m256 vCenterOffsetsY = {0.5, 0.5, 1.5, 1.5, 0.5, 0.5, 1.5, 1.5};
 424 const __m256 vULOffsetsX = {0.0, 1.0, 0.0, 1.0, 2.0, 3.0, 2.0, 3.0};
 425 const __m256 vULOffsetsY = {0.0, 0.0, 1.0, 1.0, 0.0, 0.0, 1.0, 1.0};
 426 #else
 427 #error Unsupported vector width
 428 #endif
 429
 430 INLINE
 431 bool CanEarlyZ(const SWR_PS_STATE *pPSState)
 432 {
 433     return (pPSState->forceEarlyZ || (!pPSState->writesODepth && !pPSState->usesSourceDepth && !pPSState->usesUAV));
 434 }
 435
 436 simdmask ComputeUserClipMask(uint8_t clipMask, float* pUserClipBuffer, simdscalar vI, simdscalar vJ)
 437 {
 438     simdscalar vClipMask = _simd_setzero_ps();
 439     uint32_t numClipDistance = _mm_popcnt_u32(clipMask);
 440
 441     for (uint32_t i = 0; i < numClipDistance; ++i)
 442     {
 443         // pull triangle clip distance values from clip buffer
 444         simdscalar vA = _simd_broadcast_ss(pUserClipBuffer++);
 445         simdscalar vB = _simd_broadcast_ss(pUserClipBuffer++);
 446         simdscalar vC = _simd_broadcast_ss(pUserClipBuffer++);
 447
 448         // interpolate
 449         simdscalar vInterp = vplaneps(vA, vB, vC, vI, vJ);
 450
 451         // clip if interpolated clip distance is < 0 || NAN
 452         simdscalar vCull = _simd_cmp_ps(_simd_setzero_ps(), vInterp, _CMP_NLE_UQ);
 453
 454         vClipMask = _simd_or_ps(vClipMask, vCull);
 455     }
 456
 457     return _simd_movemask_ps(vClipMask);
 458 }
 459
 460 template<bool perspMask>
 461 INLINE void CalcPixelBarycentrics(const BarycentricCoeffs& coeffs, SWR_PS_CONTEXT &psContext)
 462 {
 463     if(perspMask)
 464     {
 465         // evaluate I,J
 466         psContext.vI.center = vplaneps(coeffs.vIa, coeffs.vIb, coeffs.vIc, psContext.vX.center, psContext.vY.center);
 467         psContext.vJ.center = vplaneps(coeffs.vJa, coeffs.vJb, coeffs.vJc, psContext.vX.center, psContext.vY.center);
 468         psContext.vI.center = _simd_mul_ps(psContext.vI.center, coeffs.vRecipDet);
 469         psContext.vJ.center = _simd_mul_ps(psContext.vJ.center, coeffs.vRecipDet);
 470
 471         // interpolate 1/w
 472         psContext.vOneOverW.center = vplaneps(coeffs.vAOneOverW, coeffs.vBOneOverW, coeffs.vCOneOverW, psContext.vI.center, psContext.vJ.center);
 473     }
 474 }
 475
 476 template<bool perspMask>
 477 INLINE void CalcSampleBarycentrics(const BarycentricCoeffs& coeffs, SWR_PS_CONTEXT &psContext)
 478 {
 479     if(perspMask)
 480     {
 481         // evaluate I,J
 482         psContext.vI.sample = vplaneps(coeffs.vIa, coeffs.vIb, coeffs.vIc, psContext.vX.sample, psContext.vY.sample);
 483         psContext.vJ.sample = vplaneps(coeffs.vJa, coeffs.vJb, coeffs.vJc, psContext.vX.sample, psContext.vY.sample);
 484         psContext.vI.sample = _simd_mul_ps(psContext.vI.sample, coeffs.vRecipDet);
 485         psContext.vJ.sample = _simd_mul_ps(psContext.vJ.sample, coeffs.vRecipDet);
 486
 487         // interpolate 1/w
 488         psContext.vOneOverW.sample = vplaneps(coeffs.vAOneOverW, coeffs.vBOneOverW, coeffs.vCOneOverW, psContext.vI.sample, psContext.vJ.sample);
 489     }
 490 }
 491
 492
 493 ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
 494 // Centroid behaves exactly as follows :
 495 // (1) If all samples in the primitive are covered, the attribute is evaluated at the pixel center (even if the sample pattern does not happen to
 496 //     have a sample location there).
 497 // (2) Else the attribute is evaluated at the first covered sample, in increasing order of sample index, where sample coverage is after ANDing the
 498 //     coverage with the SampleMask Rasterizer State.
 499 // (3) If no samples are covered, such as on helper pixels executed off the bounds of a primitive to fill out 2x2 pixel stamps, the attribute is
 500 //     evaluated as follows : If the SampleMask Rasterizer state is a subset of the samples in the pixel, then the first sample covered by the
 501 //     SampleMask Rasterizer State is the evaluation point.Otherwise (full SampleMask), the pixel center is the evaluation point.
 502 ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
 503 template<SWR_MULTISAMPLE_COUNT sampleCount, bool bForcedSampleCount>
 504 INLINE void CalcCentroidPos(SWR_PS_CONTEXT &psContext, const uint64_t *const coverageMask, const uint32_t sampleMask,
 505                             const simdscalar vXSamplePosUL, const simdscalar vYSamplePosUL)
 506 {
 507     uint32_t inputMask[KNOB_SIMD_WIDTH];
 508
 509     generateInputCoverage<sampleCount, 1, bForcedSampleCount>(coverageMask, inputMask, sampleMask);
 510
 511     // Case (2) - partially covered pixel
 512
 513     // scan for first covered sample per pixel in the 4x2 span
 514     unsigned long sampleNum[KNOB_SIMD_WIDTH];
 515     (inputMask[0] > 0) ? (_BitScanForward(&sampleNum[0], inputMask[0])) : (sampleNum[0] = 0);
 516     (inputMask[1] > 0) ? (_BitScanForward(&sampleNum[1], inputMask[1])) : (sampleNum[1] = 0);
 517     (inputMask[2] > 0) ? (_BitScanForward(&sampleNum[2], inputMask[2])) : (sampleNum[2] = 0);
 518     (inputMask[3] > 0) ? (_BitScanForward(&sampleNum[3], inputMask[3])) : (sampleNum[3] = 0);
 519     (inputMask[4] > 0) ? (_BitScanForward(&sampleNum[4], inputMask[4])) : (sampleNum[4] = 0);
 520     (inputMask[5] > 0) ? (_BitScanForward(&sampleNum[5], inputMask[5])) : (sampleNum[5] = 0);
 521     (inputMask[6] > 0) ? (_BitScanForward(&sampleNum[6], inputMask[6])) : (sampleNum[6] = 0);
 522     (inputMask[7] > 0) ? (_BitScanForward(&sampleNum[7], inputMask[7])) : (sampleNum[7] = 0);
 523
 524     // look up and set the sample offsets from UL pixel corner for first covered sample
 525     __m256 vXSample = _mm256_set_ps(MultisampleTraits<sampleCount>::X(sampleNum[7]),
 526                                     MultisampleTraits<sampleCount>::X(sampleNum[6]),
 527                                     MultisampleTraits<sampleCount>::X(sampleNum[5]),
 528                                     MultisampleTraits<sampleCount>::X(sampleNum[4]),
 529                                     MultisampleTraits<sampleCount>::X(sampleNum[3]),
 530                                     MultisampleTraits<sampleCount>::X(sampleNum[2]),
 531                                     MultisampleTraits<sampleCount>::X(sampleNum[1]),
 532                                     MultisampleTraits<sampleCount>::X(sampleNum[0]));
 533
 534     __m256 vYSample = _mm256_set_ps(MultisampleTraits<sampleCount>::Y(sampleNum[7]),
 535                                     MultisampleTraits<sampleCount>::Y(sampleNum[6]),
 536                                     MultisampleTraits<sampleCount>::Y(sampleNum[5]),
 537                                     MultisampleTraits<sampleCount>::Y(sampleNum[4]),
 538                                     MultisampleTraits<sampleCount>::Y(sampleNum[3]),
 539                                     MultisampleTraits<sampleCount>::Y(sampleNum[2]),
 540                                     MultisampleTraits<sampleCount>::Y(sampleNum[1]),
 541                                     MultisampleTraits<sampleCount>::Y(sampleNum[0]));
 542     // add sample offset to UL pixel corner
 543     vXSample = _simd_add_ps(vXSamplePosUL, vXSample);
 544     vYSample = _simd_add_ps(vYSamplePosUL, vYSample);
 545
 546     // Case (1) and case (3b) - All samples covered or not covered with full SampleMask
 547     static const __m256i vFullyCoveredMask = MultisampleTraits<sampleCount>::FullSampleMask();
 548     __m256i vInputCoveragei =  _mm256_set_epi32(inputMask[7], inputMask[6], inputMask[5], inputMask[4], inputMask[3], inputMask[2], inputMask[1], inputMask[0]);
 549     __m256i vAllSamplesCovered = _simd_cmpeq_epi32(vInputCoveragei, vFullyCoveredMask);
 550
 551     static const __m256i vZero = _simd_setzero_si();
 552     const __m256i vSampleMask = _simd_and_si(_simd_set1_epi32(sampleMask), vFullyCoveredMask);
 553     __m256i vNoSamplesCovered = _simd_cmpeq_epi32(vInputCoveragei, vZero);
 554     __m256i vIsFullSampleMask = _simd_cmpeq_epi32(vSampleMask, vFullyCoveredMask);
 555     __m256i vCase3b = _simd_and_si(vNoSamplesCovered, vIsFullSampleMask);
 556
 557     __m256i vEvalAtCenter = _simd_or_si(vAllSamplesCovered, vCase3b);
 558
 559     // set the centroid position based on results from above
 560     psContext.vX.centroid = _simd_blendv_ps(vXSample, psContext.vX.center, _simd_castsi_ps(vEvalAtCenter));
 561     psContext.vY.centroid = _simd_blendv_ps(vYSample, psContext.vY.center, _simd_castsi_ps(vEvalAtCenter));
 562
 563     // Case (3a) No samples covered and partial sample mask
 564     __m256i vSomeSampleMaskSamples = _simd_cmplt_epi32(vSampleMask, vFullyCoveredMask);
 565     // sample mask should never be all 0's for this case, but handle it anyways
 566     unsigned long firstCoveredSampleMaskSample = 0;
 567     (sampleMask > 0) ? (_BitScanForward(&firstCoveredSampleMaskSample, sampleMask)) : (firstCoveredSampleMaskSample = 0);
 568
 569     __m256i vCase3a = _simd_and_si(vNoSamplesCovered, vSomeSampleMaskSamples);
 570
 571     vXSample = _simd_set1_ps(MultisampleTraits<sampleCount>::X(firstCoveredSampleMaskSample));
 572     vYSample = _simd_set1_ps(MultisampleTraits<sampleCount>::Y(firstCoveredSampleMaskSample));
 573
 574     // blend in case 3a pixel locations
 575     psContext.vX.centroid = _simd_blendv_ps(psContext.vX.centroid, vXSample, _simd_castsi_ps(vCase3a));
 576     psContext.vY.centroid = _simd_blendv_ps(psContext.vY.centroid, vYSample, _simd_castsi_ps(vCase3a));
 577 }
 578
 579 template<uint32_t sampleCount, uint32_t persp, uint32_t standardPattern, uint32_t forcedMultisampleCount>
 580 INLINE void CalcCentroidBarycentrics(const BarycentricCoeffs& coeffs, SWR_PS_CONTEXT &psContext,
 581                                      const uint64_t *const coverageMask, const uint32_t sampleMask,
 582                                      const simdscalar vXSamplePosUL, const simdscalar vYSamplePosUL)
 583 {
 584     static const bool bPersp = (bool)persp;
 585     static const bool bIsStandardPattern = (bool)standardPattern;
 586     static const bool bForcedMultisampleCount = (bool)forcedMultisampleCount;
 587
 588     // calculate centroid positions
 589     if(bPersp)
 590     {
 591         if(bIsStandardPattern)
 592         {
 593             ///@ todo: don't need to generate input coverage 2x if input coverage and centroid
 594             CalcCentroidPos<(SWR_MULTISAMPLE_COUNT)sampleCount, bForcedMultisampleCount>(psContext, coverageMask, sampleMask, vXSamplePosUL, vYSamplePosUL);
 595         }
 596         else
 597         {
 598             static const __m256 pixelCenter = _simd_set1_ps(0.5f);
 599             psContext.vX.centroid = _simd_add_ps(vXSamplePosUL, pixelCenter);
 600             psContext.vY.centroid = _simd_add_ps(vYSamplePosUL, pixelCenter);
 601         }
 602         // evaluate I,J
 603         psContext.vI.centroid = vplaneps(coeffs.vIa, coeffs.vIb, coeffs.vIc, psContext.vX.centroid, psContext.vY.centroid);
 604         psContext.vJ.centroid = vplaneps(coeffs.vJa, coeffs.vJb, coeffs.vJc, psContext.vX.centroid, psContext.vY.centroid);
 605         psContext.vI.centroid = _simd_mul_ps(psContext.vI.centroid, coeffs.vRecipDet);
 606         psContext.vJ.centroid = _simd_mul_ps(psContext.vJ.centroid, coeffs.vRecipDet);
 607
 608         // interpolate 1/w
 609         psContext.vOneOverW.centroid = vplaneps(coeffs.vAOneOverW, coeffs.vBOneOverW, coeffs.vCOneOverW, psContext.vI.centroid, psContext.vJ.centroid);
 610     }
 611 }
 612
 613 template<uint32_t NumRT, uint32_t sampleCountT>
 614 void OutputMerger(SWR_PS_CONTEXT &psContext, uint8_t* (&pColorBase)[SWR_NUM_RENDERTARGETS], uint32_t sample, const SWR_BLEND_STATE *pBlendState,
 615                   const PFN_BLEND_JIT_FUNC (&pfnBlendFunc)[SWR_NUM_RENDERTARGETS], simdscalar &coverageMask, simdscalar depthPassMask)
 616 {
 617     // type safety guaranteed from template instantiation in BEChooser<>::GetFunc
 618     static const SWR_MULTISAMPLE_COUNT sampleCount = (SWR_MULTISAMPLE_COUNT)sampleCountT;
 619     uint32_t rasterTileColorOffset = MultisampleTraits<sampleCount>::RasterTileColorOffset(sample);
 620     simdvector blendOut;
 621
 622     for(uint32_t rt = 0; rt < NumRT; ++rt)
 623     {
 624         uint8_t *pColorSample;
 625         if(sampleCount == SWR_MULTISAMPLE_1X)
 626         {
 627             pColorSample = pColorBase[rt];
 628         }
 629         else
 630         {
 631             pColorSample = pColorBase[rt] + rasterTileColorOffset;
 632         }
 633
 634         const SWR_RENDER_TARGET_BLEND_STATE *pRTBlend = &pBlendState->renderTarget[rt];
 635         // pfnBlendFunc may not update all channels.  Initialize with PS output.
 636         /// TODO: move this into the blend JIT.
 637         blendOut = psContext.shaded[rt];
 638
 639         // Blend outputs and update coverage mask for alpha test
 640         if(pfnBlendFunc[rt] != nullptr)
 641         {
 642             pfnBlendFunc[rt](
 643                 pBlendState,
 644                 psContext.shaded[rt],
 645                 psContext.shaded[1],
 646                 sample,
 647                 pColorSample,
 648                 blendOut,
 649                 &psContext.oMask,
 650                 (simdscalari*)&coverageMask);
 651         }
 652
 653         // final write mask
 654         simdscalari outputMask = _simd_castps_si(_simd_and_ps(coverageMask, depthPassMask));
 655
 656         ///@todo can only use maskstore fast path if bpc is 32. Assuming hot tile is RGBA32_FLOAT.
 657         static_assert(KNOB_COLOR_HOT_TILE_FORMAT == R32G32B32A32_FLOAT, "Unsupported hot tile format");
 658
 659         const uint32_t simd = KNOB_SIMD_WIDTH * sizeof(float);
 660
 661         // store with color mask
 662         if(!pRTBlend->writeDisableRed)
 663         {
 664             _simd_maskstore_ps((float*)pColorSample, outputMask, blendOut.x);
 665         }
 666         if(!pRTBlend->writeDisableGreen)
 667         {
 668             _simd_maskstore_ps((float*)(pColorSample + simd), outputMask, blendOut.y);
 669         }
 670         if(!pRTBlend->writeDisableBlue)
 671         {
 672             _simd_maskstore_ps((float*)(pColorSample + simd * 2), outputMask, blendOut.z);
 673         }
 674         if(!pRTBlend->writeDisableAlpha)
 675         {
 676             _simd_maskstore_ps((float*)(pColorSample + simd * 3), outputMask, blendOut.w);
 677         }
 678     }
 679 }
 680
 681 template<uint32_t sampleCountT, uint32_t samplePattern, uint32_t inputCoverage, uint32_t centroidPos, uint32_t forcedSampleCount>
 682 void BackendSingleSample(DRAW_CONTEXT *pDC, uint32_t workerId, uint32_t x, uint32_t y, SWR_TRIANGLE_DESC &work, RenderOutputBuffers &renderBuffers)
 683 {
 684     RDTSC_START(BESetup);
 685     // type safety guaranteed from template instantiation in BEChooser<>::GetFunc
 686     static const bool bInputCoverage = (bool)inputCoverage;
 687     static const bool bCentroidPos = (bool)centroidPos;
 688
 689     SWR_CONTEXT *pContext = pDC->pContext;
 690     const API_STATE& state = GetApiState(pDC);
 691     const SWR_RASTSTATE& rastState = state.rastState;
 692     const SWR_PS_STATE *pPSState = &state.psState;
 693     const SWR_BLEND_STATE *pBlendState = &state.blendState;
 694     const BACKEND_FUNCS& backendFuncs = pDC->pState->backendFuncs;
 695     uint64_t coverageMask = work.coverageMask[0];
 696
 697     // broadcast scalars
 698     BarycentricCoeffs coeffs;
 699     coeffs.vIa = _simd_broadcast_ss(&work.I[0]);
 700     coeffs.vIb = _simd_broadcast_ss(&work.I[1]);
 701     coeffs.vIc = _simd_broadcast_ss(&work.I[2]);
 702
 703     coeffs.vJa = _simd_broadcast_ss(&work.J[0]);
 704     coeffs.vJb = _simd_broadcast_ss(&work.J[1]);
 705     coeffs.vJc = _simd_broadcast_ss(&work.J[2]);
 706
 707     coeffs.vZa = _simd_broadcast_ss(&work.Z[0]);
 708     coeffs.vZb = _simd_broadcast_ss(&work.Z[1]);
 709     coeffs.vZc = _simd_broadcast_ss(&work.Z[2]);
 710
 711     coeffs.vRecipDet = _simd_broadcast_ss(&work.recipDet);
 712
 713     coeffs.vAOneOverW = _simd_broadcast_ss(&work.OneOverW[0]);
 714     coeffs.vBOneOverW = _simd_broadcast_ss(&work.OneOverW[1]);
 715     coeffs.vCOneOverW = _simd_broadcast_ss(&work.OneOverW[2]);
 716
 717     uint8_t *pColorBase[SWR_NUM_RENDERTARGETS];
 718     uint32_t NumRT = state.psState.numRenderTargets;
 719     for(uint32_t rt = 0; rt < NumRT; ++rt)
 720     {
 721         pColorBase[rt] = renderBuffers.pColor[rt];
 722     }
 723     uint8_t *pDepthBase = renderBuffers.pDepth, *pStencilBase = renderBuffers.pStencil;
 724     RDTSC_STOP(BESetup, 0, 0);
 725
 726     SWR_PS_CONTEXT psContext;
 727     psContext.pAttribs = work.pAttribs;
 728     psContext.pPerspAttribs = work.pPerspAttribs;
 729     psContext.frontFace = work.triFlags.frontFacing;
 730     psContext.primID = work.triFlags.primID;
 731
 732     // save Ia/Ib/Ic and Ja/Jb/Jc if we need to reevaluate i/j/k in the shader because of pull attribs
 733     psContext.I = work.I;
 734     psContext.J = work.J;
 735     psContext.recipDet = work.recipDet;
 736     psContext.pRecipW = work.pRecipW;
 737     psContext.pSamplePosX = (const float*)&MultisampleTraits<SWR_MULTISAMPLE_1X>::samplePosX;
 738     psContext.pSamplePosY = (const float*)&MultisampleTraits<SWR_MULTISAMPLE_1X>::samplePosY;
 739
 740     for(uint32_t yy = y; yy < y + KNOB_TILE_Y_DIM; yy += SIMD_TILE_Y_DIM)
 741     {
 742         // UL pixel corner
 743         psContext.vY.UL = _simd_add_ps(vULOffsetsY, _simd_set1_ps((float)yy));
 744         // pixel center
 745         psContext.vY.center = _simd_add_ps(vCenterOffsetsY, _simd_set1_ps((float)yy));
 746
 747         for(uint32_t xx = x; xx < x + KNOB_TILE_X_DIM; xx += SIMD_TILE_X_DIM)
 748         {
 749             if(bInputCoverage)
 750             {
 751                 generateInputCoverage<SWR_MULTISAMPLE_1X, SWR_MSAA_STANDARD_PATTERN, false>(&work.coverageMask[0], psContext.inputMask, pBlendState->sampleMask);
 752             }
 753
 754             if(coverageMask & MASK)
 755             {
 756                 RDTSC_START(BEBarycentric);
 757                 psContext.vX.UL = _simd_add_ps(vULOffsetsX, _simd_set1_ps((float)xx));
 758                 // pixel center
 759                 psContext.vX.center = _simd_add_ps(vCenterOffsetsX, _simd_set1_ps((float)xx));
 760
 761                 backendFuncs.pfnCalcPixelBarycentrics(coeffs, psContext);
 762
 763                 if(bCentroidPos)
 764                 {
 765                     // for 1x case, centroid is pixel center
 766                     psContext.vX.centroid = psContext.vX.center;
 767                     psContext.vY.centroid = psContext.vY.center;
 768                     psContext.vI.centroid = psContext.vI.center;
 769                     psContext.vJ.centroid = psContext.vJ.center;
 770                     psContext.vOneOverW.centroid = psContext.vOneOverW.center;
 771                 }
 772
 773                 // interpolate z
 774                 psContext.vZ = vplaneps(coeffs.vZa, coeffs.vZb, coeffs.vZc, psContext.vI.center, psContext.vJ.center);
 775                 RDTSC_STOP(BEBarycentric, 0, 0);
 776
 777                 simdmask clipCoverageMask = coverageMask & MASK;
 778
 779                 // interpolate user clip distance if available
 780                 if(rastState.clipDistanceMask)
 781                 {
 782                     clipCoverageMask &= ~ComputeUserClipMask(rastState.clipDistanceMask, work.pUserClipBuffer,
 783                                                              psContext.vI.center, psContext.vJ.center);
 784                 }
 785
 786                 simdscalar vCoverageMask = vMask(clipCoverageMask);
 787                 simdscalar depthPassMask = vCoverageMask;
 788                 simdscalar stencilPassMask = vCoverageMask;
 789
 790                 // Early-Z?
 791                 if(CanEarlyZ(pPSState))
 792                 {
 793                     RDTSC_START(BEEarlyDepthTest);
 794                     depthPassMask = DepthStencilTest(&state.vp[0], &state.depthStencilState, work.triFlags.frontFacing,
 795                                                         psContext.vZ, pDepthBase, vCoverageMask, pStencilBase, &stencilPassMask);
 796                     RDTSC_STOP(BEEarlyDepthTest, 0, 0);
 797
 798                     // early-exit if no pixels passed depth or earlyZ is forced on
 799                     if(pPSState->forceEarlyZ || !_simd_movemask_ps(depthPassMask))
 800                     {
 801                         DepthStencilWrite(&state.vp[0], &state.depthStencilState, work.triFlags.frontFacing, psContext.vZ,
 802                                             pDepthBase, depthPassMask, vCoverageMask, pStencilBase, stencilPassMask);
 803
 804                         if (!_simd_movemask_ps(depthPassMask))
 805                         {
 806                             goto Endtile;
 807                         }
 808                     }
 809                 }
 810
 811                 psContext.sampleIndex = 0;
 812                 psContext.activeMask = _simd_castps_si(vCoverageMask);
 813
 814                 // execute pixel shader
 815                 RDTSC_START(BEPixelShader);
 816                 UPDATE_STAT(PsInvocations, _mm_popcnt_u32(_simd_movemask_ps(vCoverageMask)));
 817                 state.psState.pfnPixelShader(GetPrivateState(pDC), &psContext);
 818                 RDTSC_STOP(BEPixelShader, 0, 0);
 819
 820                 vCoverageMask = _simd_castsi_ps(psContext.activeMask);
 821
 822                 // late-Z
 823                 if(!CanEarlyZ(pPSState))
 824                 {
 825                     RDTSC_START(BELateDepthTest);
 826                     depthPassMask = DepthStencilTest(&state.vp[0], &state.depthStencilState, work.triFlags.frontFacing,
 827                                                         psContext.vZ, pDepthBase, vCoverageMask, pStencilBase, &stencilPassMask);
 828                     RDTSC_STOP(BELateDepthTest, 0, 0);
 829
 830                     if(!_simd_movemask_ps(depthPassMask))
 831                     {
 832                         // need to call depth/stencil write for stencil write
 833                         DepthStencilWrite(&state.vp[0], &state.depthStencilState, work.triFlags.frontFacing, psContext.vZ,
 834                                             pDepthBase, depthPassMask, vCoverageMask, pStencilBase, stencilPassMask);
 835                         goto Endtile;
 836                     }
 837                 }
 838
 839                 uint32_t statMask = _simd_movemask_ps(depthPassMask);
 840                 uint32_t statCount = _mm_popcnt_u32(statMask);
 841                 UPDATE_STAT(DepthPassCount, statCount);
 842
 843                 // output merger
 844                 RDTSC_START(BEOutputMerger);
 845                 backendFuncs.pfnOutputMerger(psContext, pColorBase, 0, pBlendState, state.pfnBlendFunc,
 846                                              vCoverageMask, depthPassMask);
 847
 848                 // do final depth write after all pixel kills
 849                 if (!pPSState->forceEarlyZ)
 850                 {
 851                     DepthStencilWrite(&state.vp[0], &state.depthStencilState, work.triFlags.frontFacing, psContext.vZ,
 852                         pDepthBase, depthPassMask, vCoverageMask, pStencilBase, stencilPassMask);
 853                 }
 854                 RDTSC_STOP(BEOutputMerger, 0, 0);
 855             }
 856
 857 Endtile:
 858             RDTSC_START(BEEndTile);
 859             coverageMask >>= (SIMD_TILE_Y_DIM * SIMD_TILE_X_DIM);
 860             pDepthBase += (KNOB_SIMD_WIDTH * FormatTraits<KNOB_DEPTH_HOT_TILE_FORMAT>::bpp) / 8;
 861             pStencilBase += (KNOB_SIMD_WIDTH * FormatTraits<KNOB_STENCIL_HOT_TILE_FORMAT>::bpp) / 8;
 862
 863             for(uint32_t rt = 0; rt < NumRT; ++rt)
 864             {
 865                 pColorBase[rt] += (KNOB_SIMD_WIDTH * FormatTraits<KNOB_COLOR_HOT_TILE_FORMAT>::bpp) / 8;
 866             }
 867             RDTSC_STOP(BEEndTile, 0, 0);
 868         }
 869     }
 870 }
 871
 872 template<uint32_t sampleCountT, uint32_t samplePattern, uint32_t inputCoverage, uint32_t centroidPos, uint32_t forcedSampleCount>
 873 void BackendSampleRate(DRAW_CONTEXT *pDC, uint32_t workerId, uint32_t x, uint32_t y, SWR_TRIANGLE_DESC &work, RenderOutputBuffers &renderBuffers)
 874 {
 875     // type safety guaranteed from template instantiation in BEChooser<>::GetFunc
 876     static const SWR_MULTISAMPLE_COUNT sampleCount = (SWR_MULTISAMPLE_COUNT)sampleCountT;
 877     static const bool bInputCoverage = (bool)inputCoverage;
 878     static const bool bCentroidPos = (bool)centroidPos;
 879
 880     RDTSC_START(BESetup);
 881
 882     SWR_CONTEXT *pContext = pDC->pContext;
 883     const API_STATE& state = GetApiState(pDC);
 884     const SWR_RASTSTATE& rastState = state.rastState;
 885     const SWR_PS_STATE *pPSState = &state.psState;
 886     const SWR_BLEND_STATE *pBlendState = &state.blendState;
 887     const BACKEND_FUNCS& backendFuncs = pDC->pState->backendFuncs;
 888
 889     // broadcast scalars
 890     BarycentricCoeffs coeffs;
 891     coeffs.vIa = _simd_broadcast_ss(&work.I[0]);
 892     coeffs.vIb = _simd_broadcast_ss(&work.I[1]);
 893     coeffs.vIc = _simd_broadcast_ss(&work.I[2]);
 894
 895     coeffs.vJa = _simd_broadcast_ss(&work.J[0]);
 896     coeffs.vJb = _simd_broadcast_ss(&work.J[1]);
 897     coeffs.vJc = _simd_broadcast_ss(&work.J[2]);
 898
 899     coeffs.vZa = _simd_broadcast_ss(&work.Z[0]);
 900     coeffs.vZb = _simd_broadcast_ss(&work.Z[1]);
 901     coeffs.vZc = _simd_broadcast_ss(&work.Z[2]);
 902
 903     coeffs.vRecipDet = _simd_broadcast_ss(&work.recipDet);
 904
 905     coeffs.vAOneOverW = _simd_broadcast_ss(&work.OneOverW[0]);
 906     coeffs.vBOneOverW = _simd_broadcast_ss(&work.OneOverW[1]);
 907     coeffs.vCOneOverW = _simd_broadcast_ss(&work.OneOverW[2]);
 908
 909     uint8_t *pColorBase[SWR_NUM_RENDERTARGETS];
 910     uint32_t NumRT = state.psState.numRenderTargets;
 911     for(uint32_t rt = 0; rt < NumRT; ++rt)
 912     {
 913         pColorBase[rt] = renderBuffers.pColor[rt];
 914     }
 915     uint8_t *pDepthBase = renderBuffers.pDepth, *pStencilBase = renderBuffers.pStencil;
 916     RDTSC_STOP(BESetup, 0, 0);
 917
 918     SWR_PS_CONTEXT psContext;
 919     psContext.pAttribs = work.pAttribs;
 920     psContext.pPerspAttribs = work.pPerspAttribs;
 921     psContext.pRecipW = work.pRecipW;
 922     psContext.frontFace = work.triFlags.frontFacing;
 923     psContext.primID = work.triFlags.primID;
 924
 925     // save Ia/Ib/Ic and Ja/Jb/Jc if we need to reevaluate i/j/k in the shader because of pull attribs
 926     psContext.I = work.I;
 927     psContext.J = work.J;
 928     psContext.recipDet = work.recipDet;
 929     psContext.pSamplePosX = (const float*)&MultisampleTraits<sampleCount>::samplePosX;
 930     psContext.pSamplePosY = (const float*)&MultisampleTraits<sampleCount>::samplePosY;
 931     const uint32_t numSamples = MultisampleTraits<sampleCount>::numSamples;
 932
 933     for (uint32_t yy = y; yy < y + KNOB_TILE_Y_DIM; yy += SIMD_TILE_Y_DIM)
 934     {
 935         // UL pixel corner
 936         psContext.vY.UL = _simd_add_ps(vULOffsetsY, _simd_set1_ps((float)yy));
 937         // pixel center
 938         psContext.vY.center = _simd_add_ps(vCenterOffsetsY, _simd_set1_ps((float)yy));
 939
 940         for (uint32_t xx = x; xx < x + KNOB_TILE_X_DIM; xx += SIMD_TILE_X_DIM)
 941         {
 942             psContext.vX.UL = _simd_add_ps(vULOffsetsX, _simd_set1_ps((float)xx));
 943             // pixel center
 944             psContext.vX.center = _simd_add_ps(vCenterOffsetsX, _simd_set1_ps((float)xx));
 945
 946             RDTSC_START(BEBarycentric);
 947             backendFuncs.pfnCalcPixelBarycentrics(coeffs, psContext);
 948             RDTSC_STOP(BEBarycentric, 0, 0);
 949
 950             if(bInputCoverage)
 951             {
 952                 generateInputCoverage<sampleCount, SWR_MSAA_STANDARD_PATTERN, false>(&work.coverageMask[0], psContext.inputMask, pBlendState->sampleMask);
 953             }
 954
 955             if(bCentroidPos)
 956             {
 957                 ///@ todo: don't need to genererate input coverage 2x if input coverage and centroid
 958                 RDTSC_START(BEBarycentric);
 959                 backendFuncs.pfnCalcCentroidBarycentrics(coeffs, psContext, &work.coverageMask[0], pBlendState->sampleMask, psContext.vX.UL, psContext.vY.UL);
 960                 RDTSC_STOP(BEBarycentric, 0, 0);
 961             }
 962
 963             for(uint32_t sample = 0; sample < numSamples; sample++)
 964             {
 965                 if (work.coverageMask[sample] & MASK)
 966                 {
 967                     RDTSC_START(BEBarycentric);
 968
 969                     // calculate per sample positions
 970                     psContext.vX.sample = _simd_add_ps(psContext.vX.UL, MultisampleTraits<sampleCount>::vX(sample));
 971                     psContext.vY.sample = _simd_add_ps(psContext.vY.UL, MultisampleTraits<sampleCount>::vY(sample));
 972
 973                     simdmask coverageMask = work.coverageMask[sample] & MASK;
 974                     simdscalar vCoverageMask = vMask(coverageMask);
 975
 976                     backendFuncs.pfnCalcSampleBarycentrics(coeffs, psContext);
 977
 978                     // interpolate z
 979                     psContext.vZ = vplaneps(coeffs.vZa, coeffs.vZb, coeffs.vZc, psContext.vI.sample, psContext.vJ.sample);
 980
 981                     RDTSC_STOP(BEBarycentric, 0, 0);
 982
 983                     // interpolate user clip distance if available
 984                     if (rastState.clipDistanceMask)
 985                     {
 986                         coverageMask &= ~ComputeUserClipMask(rastState.clipDistanceMask, work.pUserClipBuffer,
 987                             psContext.vI.sample, psContext.vJ.sample);
 988                     }
 989
 990                     simdscalar depthPassMask = vCoverageMask;
 991                     simdscalar stencilPassMask = vCoverageMask;
 992
 993                     // offset depth/stencil buffers current sample
 994                     uint8_t *pDepthSample = pDepthBase + MultisampleTraits<sampleCount>::RasterTileDepthOffset(sample);
 995                     uint8_t *pStencilSample = pStencilBase + MultisampleTraits<sampleCount>::RasterTileStencilOffset(sample);
 996
 997                     // Early-Z?
 998                     if (CanEarlyZ(pPSState))
 999                     {
1000                         RDTSC_START(BEEarlyDepthTest);
1001                         depthPassMask = DepthStencilTest(&state.vp[0], &state.depthStencilState, work.triFlags.frontFacing,
1002                                               psContext.vZ, pDepthSample, vCoverageMask, pStencilSample, &stencilPassMask);
1003                         RDTSC_STOP(BEEarlyDepthTest, 0, 0);
1004
1005                         // early-exit if no samples passed depth or earlyZ is forced on.
1006                         if (pPSState->forceEarlyZ || !_simd_movemask_ps(depthPassMask))
1007                         {
1008                             DepthStencilWrite(&state.vp[0], &state.depthStencilState, work.triFlags.frontFacing, psContext.vZ,
1009                                 pDepthSample, depthPassMask, vCoverageMask, pStencilSample, stencilPassMask);
1010
1011                             if (!_simd_movemask_ps(depthPassMask))
1012                             {
1013                                 work.coverageMask[sample] >>= (SIMD_TILE_Y_DIM * SIMD_TILE_X_DIM);
1014                                 continue;
1015                             }
1016                         }
1017                     }
1018
1019                     psContext.sampleIndex = sample;
1020                     psContext.activeMask = _simd_castps_si(vCoverageMask);
1021
1022                     // execute pixel shader
1023                     RDTSC_START(BEPixelShader);
1024                     UPDATE_STAT(PsInvocations, _mm_popcnt_u32(_simd_movemask_ps(vCoverageMask)));
1025                     state.psState.pfnPixelShader(GetPrivateState(pDC), &psContext);
1026                     RDTSC_STOP(BEPixelShader, 0, 0);
1027
1028                     vCoverageMask = _simd_castsi_ps(psContext.activeMask);
1029
1030                     //// late-Z
1031                     if (!CanEarlyZ(pPSState))
1032                     {
1033                         RDTSC_START(BELateDepthTest);
1034                         depthPassMask = DepthStencilTest(&state.vp[0], &state.depthStencilState, work.triFlags.frontFacing,
1035                                               psContext.vZ, pDepthSample, vCoverageMask, pStencilSample, &stencilPassMask);
1036                         RDTSC_STOP(BELateDepthTest, 0, 0);
1037
1038                         if (!_simd_movemask_ps(depthPassMask))
1039                         {
1040                             // need to call depth/stencil write for stencil write
1041                             DepthStencilWrite(&state.vp[0], &state.depthStencilState, work.triFlags.frontFacing, psContext.vZ,
1042                                 pDepthSample, depthPassMask, vCoverageMask, pStencilSample, stencilPassMask);
1043
1044                             work.coverageMask[sample] >>= (SIMD_TILE_Y_DIM * SIMD_TILE_X_DIM);
1045                             continue;
1046                         }
1047                     }
1048
1049                     uint32_t statMask = _simd_movemask_ps(depthPassMask);
1050                     uint32_t statCount = _mm_popcnt_u32(statMask);
1051                     UPDATE_STAT(DepthPassCount, statCount);
1052
1053                     // output merger
1054                     RDTSC_START(BEOutputMerger);
1055                     backendFuncs.pfnOutputMerger(psContext, pColorBase, sample, pBlendState, state.pfnBlendFunc,
1056                                                  vCoverageMask, depthPassMask);
1057
1058                     // do final depth write after all pixel kills
1059                     if (!pPSState->forceEarlyZ)
1060                     {
1061                         DepthStencilWrite(&state.vp[0], &state.depthStencilState, work.triFlags.frontFacing, psContext.vZ,
1062                             pDepthSample, depthPassMask, vCoverageMask, pStencilSample, stencilPassMask);
1063                     }
1064                     RDTSC_STOP(BEOutputMerger, 0, 0);
1065                 }
1066                 work.coverageMask[sample] >>= (SIMD_TILE_Y_DIM * SIMD_TILE_X_DIM);
1067             }
1068             RDTSC_START(BEEndTile);
1069             pDepthBase += (KNOB_SIMD_WIDTH * FormatTraits<KNOB_DEPTH_HOT_TILE_FORMAT>::bpp) / 8;
1070             pStencilBase += (KNOB_SIMD_WIDTH * FormatTraits<KNOB_STENCIL_HOT_TILE_FORMAT>::bpp) / 8;
1071
1072             for (uint32_t rt = 0; rt < NumRT; ++rt)
1073             {
1074                 pColorBase[rt] += (KNOB_SIMD_WIDTH * FormatTraits<KNOB_COLOR_HOT_TILE_FORMAT>::bpp) / 8;
1075             }
1076             RDTSC_STOP(BEEndTile, 0, 0);
1077         }
1078     }
1079 }
1080
1081 template<uint32_t sampleCountT, uint32_t samplePattern, uint32_t inputCoverage, uint32_t centroidPos, uint32_t forcedSampleCount>
1082 void BackendPixelRate(DRAW_CONTEXT *pDC, uint32_t workerId, uint32_t x, uint32_t y, SWR_TRIANGLE_DESC &work, RenderOutputBuffers &renderBuffers)
1083 {
1084     // type safety guaranteed from template instantiation in BEChooser<>::GetFunc
1085     static const SWR_MULTISAMPLE_COUNT sampleCount = (SWR_MULTISAMPLE_COUNT)sampleCountT;
1086     static const bool bIsStandardPattern = (bool)samplePattern;
1087     static const bool bInputCoverage = (bool)inputCoverage;
1088     static const bool bCentroidPos = (bool)centroidPos;
1089     static const bool bForcedSampleCount = (bool)forcedSampleCount;
1090
1091     RDTSC_START(BESetup);
1092
1093     SWR_CONTEXT *pContext = pDC->pContext;
1094     const API_STATE& state = GetApiState(pDC);
1095     const SWR_RASTSTATE& rastState = state.rastState;
1096     const SWR_PS_STATE *pPSState = &state.psState;
1097     const SWR_BLEND_STATE *pBlendState = &state.blendState;
1098     const BACKEND_FUNCS& backendFuncs = pDC->pState->backendFuncs;
1099
1100     // broadcast scalars
1101     BarycentricCoeffs coeffs;
1102     coeffs.vIa = _simd_broadcast_ss(&work.I[0]);
1103     coeffs.vIb = _simd_broadcast_ss(&work.I[1]);
1104     coeffs.vIc = _simd_broadcast_ss(&work.I[2]);
1105
1106     coeffs.vJa = _simd_broadcast_ss(&work.J[0]);
1107     coeffs.vJb = _simd_broadcast_ss(&work.J[1]);
1108     coeffs.vJc = _simd_broadcast_ss(&work.J[2]);
1109
1110     coeffs.vZa = _simd_broadcast_ss(&work.Z[0]);
1111     coeffs.vZb = _simd_broadcast_ss(&work.Z[1]);
1112     coeffs.vZc = _simd_broadcast_ss(&work.Z[2]);
1113
1114     coeffs.vRecipDet = _simd_broadcast_ss(&work.recipDet);
1115
1116     coeffs.vAOneOverW = _simd_broadcast_ss(&work.OneOverW[0]);
1117     coeffs.vBOneOverW = _simd_broadcast_ss(&work.OneOverW[1]);
1118     coeffs.vCOneOverW = _simd_broadcast_ss(&work.OneOverW[2]);
1119
1120     uint8_t *pColorBase[SWR_NUM_RENDERTARGETS];
1121     uint32_t NumRT = state.psState.numRenderTargets;
1122     for(uint32_t rt = 0; rt < NumRT; ++rt)
1123     {
1124         pColorBase[rt] = renderBuffers.pColor[rt];
1125     }
1126     uint8_t *pDepthBase = renderBuffers.pDepth, *pStencilBase = renderBuffers.pStencil;
1127     RDTSC_STOP(BESetup, 0, 0);
1128
1129     SWR_PS_CONTEXT psContext;
1130     psContext.pAttribs = work.pAttribs;
1131     psContext.pPerspAttribs = work.pPerspAttribs;
1132     psContext.frontFace = work.triFlags.frontFacing;
1133     psContext.primID = work.triFlags.primID;
1134     psContext.pRecipW = work.pRecipW;
1135     // save Ia/Ib/Ic and Ja/Jb/Jc if we need to reevaluate i/j/k in the shader because of pull attribs
1136     psContext.I = work.I;
1137     psContext.J = work.J;
1138     psContext.recipDet = work.recipDet;
1139     psContext.pSamplePosX = (const float*)&MultisampleTraits<sampleCount>::samplePosX;
1140     psContext.pSamplePosY = (const float*)&MultisampleTraits<sampleCount>::samplePosY;
1141     psContext.sampleIndex = 0;
1142
1143     uint32_t numCoverageSamples;
1144     if(bIsStandardPattern)
1145     {
1146         numCoverageSamples = MultisampleTraits<sampleCount>::numSamples;
1147     }
1148     else
1149     {
1150         numCoverageSamples = 1;
1151     }
1152
1153     uint32_t numOMSamples;
1154     // RT has to be single sample if we're in forcedMSAA mode
1155     if(bForcedSampleCount && (sampleCount > SWR_MULTISAMPLE_1X))
1156     {
1157         numOMSamples = 1;
1158     }
1159     // unless we're forced to single sample, in which case we run the OM at the sample count of the RT
1160     else if(bForcedSampleCount && (sampleCount == SWR_MULTISAMPLE_1X))
1161     {
1162         numOMSamples = GetNumSamples(pBlendState->sampleCount);
1163     }
1164     // else we're in normal MSAA mode and rasterizer and OM are running at the same sample count
1165     else
1166     {
1167         numOMSamples = MultisampleTraits<sampleCount>::numSamples;
1168     }
1169
1170     for(uint32_t yy = y; yy < y + KNOB_TILE_Y_DIM; yy += SIMD_TILE_Y_DIM)
1171     {
1172         psContext.vY.UL = _simd_add_ps(vULOffsetsY, _simd_set1_ps((float)yy));
1173         psContext.vY.center = _simd_add_ps(vCenterOffsetsY, _simd_set1_ps((float)yy));
1174         for(uint32_t xx = x; xx < x + KNOB_TILE_X_DIM; xx += SIMD_TILE_X_DIM)
1175         {
1176             simdscalar vZ[MultisampleTraits<sampleCount>::numSamples]{ 0 };
1177             psContext.vX.UL = _simd_add_ps(vULOffsetsX, _simd_set1_ps((float)xx));
1178             // set pixel center positions
1179             psContext.vX.center = _simd_add_ps(vCenterOffsetsX, _simd_set1_ps((float)xx));
1180
1181             if (bInputCoverage)
1182             {
1183                 generateInputCoverage<sampleCount, bIsStandardPattern, bForcedSampleCount>(&work.coverageMask[0], psContext.inputMask, pBlendState->sampleMask);
1184             }
1185
1186             if(bCentroidPos)
1187             {
1188                 ///@ todo: don't need to genererate input coverage 2x if input coverage and centroid
1189                 RDTSC_START(BEBarycentric);
1190                 backendFuncs.pfnCalcCentroidBarycentrics(coeffs, psContext, &work.coverageMask[0], pBlendState->sampleMask, psContext.vX.UL, psContext.vY.UL);
1191                 RDTSC_STOP(BEBarycentric, 0, 0);
1192             }
1193
1194             // if oDepth written to, or there is a potential to discard any samples, we need to
1195             // run the PS early, then interp or broadcast Z and test
1196             if(pPSState->writesODepth || pPSState->killsPixel)
1197             {
1198                 RDTSC_START(BEBarycentric);
1199                 backendFuncs.pfnCalcPixelBarycentrics(coeffs, psContext);
1200
1201                 // interpolate z
1202                 psContext.vZ = vplaneps(coeffs.vZa, coeffs.vZb, coeffs.vZc, psContext.vI.center, psContext.vJ.center);
1203                 RDTSC_STOP(BEBarycentric, 0, 0);
1204
1205                 // execute pixel shader
1206                 RDTSC_START(BEPixelShader);
1207                 state.psState.pfnPixelShader(GetPrivateState(pDC), &psContext);
1208                 RDTSC_STOP(BEPixelShader, 0, 0);
1209             }
1210             else
1211             {
1212                                 psContext.activeMask = _simd_set1_epi32(-1);
1213             }
1214
1215             // need to declare enough space for all samples
1216             simdscalar vCoverageMask[MultisampleTraits<sampleCount>::numSamples];
1217             simdscalar depthPassMask[MultisampleTraits<sampleCount>::numSamples];
1218             simdscalar stencilPassMask[MultisampleTraits<sampleCount>::numSamples];
1219             simdscalar anyDepthSamplePassed = _simd_setzero_ps();
1220             simdscalar anyStencilSamplePassed = _simd_setzero_ps();
1221             for(uint32_t sample = 0; sample < numCoverageSamples; sample++)
1222             {
1223                 vCoverageMask[sample] = vMask(work.coverageMask[sample] & MASK);
1224
1225                 // pull mask back out for any discards and and with coverage
1226                 vCoverageMask[sample] = _simd_and_ps(vCoverageMask[sample], _simd_castsi_ps(psContext.activeMask));
1227
1228                 if (!_simd_movemask_ps(vCoverageMask[sample]))
1229                 {
1230                     vCoverageMask[sample] = depthPassMask[sample] = stencilPassMask[sample] =  _simd_setzero_ps();
1231                     continue;
1232                 }
1233
1234                 if(bForcedSampleCount)
1235                 {
1236                     // candidate pixels (that passed coverage) will cause shader invocation if any bits in the samplemask are set
1237                     const simdscalar vSampleMask = _simd_castsi_ps(_simd_cmpgt_epi32(_simd_set1_epi32(pBlendState->sampleMask), _simd_setzero_si()));
1238                     anyDepthSamplePassed = _simd_or_ps(anyDepthSamplePassed, _simd_and_ps(vCoverageMask[sample], vSampleMask));
1239                     continue;
1240                 }
1241
1242                 depthPassMask[sample] = vCoverageMask[sample];
1243
1244                 // if oDepth isn't written to, we need to interpolate Z for each sample
1245                 // if clip distances are enabled, we need to interpolate for each sample
1246                 if(!pPSState->writesODepth || rastState.clipDistanceMask)
1247                 {
1248                     RDTSC_START(BEBarycentric);
1249                     if(bIsStandardPattern)
1250                     {
1251                         // calculate per sample positions
1252                         psContext.vX.sample = _simd_add_ps(psContext.vX.UL, MultisampleTraits<sampleCount>::vX(sample));
1253                         psContext.vY.sample = _simd_add_ps(psContext.vY.UL, MultisampleTraits<sampleCount>::vY(sample));
1254                     }
1255                     else
1256                     {
1257                         psContext.vX.sample = psContext.vX.center;
1258                         psContext.vY.sample = psContext.vY.center;
1259                     }
1260
1261                     // calc I & J per sample
1262                     backendFuncs.pfnCalcSampleBarycentrics(coeffs, psContext);
1263
1264                     // interpolate z
1265                     if (!pPSState->writesODepth)
1266                     {
1267                         vZ[sample] = vplaneps(coeffs.vZa, coeffs.vZb, coeffs.vZc, psContext.vI.sample, psContext.vJ.sample);
1268                     }
1269
1270                     ///@todo: perspective correct vs non-perspective correct clipping?
1271                     // interpolate clip distances
1272                     if (rastState.clipDistanceMask)
1273                     {
1274                         uint8_t clipMask = ComputeUserClipMask(rastState.clipDistanceMask, work.pUserClipBuffer,
1275                             psContext.vI.sample, psContext.vJ.sample);
1276                         vCoverageMask[sample] = _simd_and_ps(vCoverageMask[sample], vMask(~clipMask));
1277                     }
1278                     RDTSC_STOP(BEBarycentric, 0, 0);
1279                 }
1280                 // else 'broadcast' and test psContext.vZ written from the PS each sample
1281                 else
1282                 {
1283                     vZ[sample] = psContext.vZ;
1284                 }
1285
1286                 // offset depth/stencil buffers current sample
1287                 uint8_t *pDepthSample = pDepthBase + MultisampleTraits<sampleCount>::RasterTileDepthOffset(sample);
1288                 uint8_t * pStencilSample = pStencilBase + MultisampleTraits<sampleCount>::RasterTileStencilOffset(sample);
1289
1290                 // ZTest for this sample
1291                 RDTSC_START(BEEarlyDepthTest);
1292                 stencilPassMask[sample] = vCoverageMask[sample];
1293                 depthPassMask[sample] = DepthStencilTest(&state.vp[0], &state.depthStencilState, work.triFlags.frontFacing,
1294                                         vZ[sample], pDepthSample, vCoverageMask[sample], pStencilSample, &stencilPassMask[sample]);
1295                 RDTSC_STOP(BEEarlyDepthTest, 0, 0);
1296
1297                 anyDepthSamplePassed = _simd_or_ps(anyDepthSamplePassed, depthPassMask[sample]);
1298                 anyStencilSamplePassed = _simd_or_ps(anyStencilSamplePassed, stencilPassMask[sample]);
1299                 uint32_t statMask = _simd_movemask_ps(depthPassMask[sample]);
1300                 uint32_t statCount = _mm_popcnt_u32(statMask);
1301                 UPDATE_STAT(DepthPassCount, statCount);
1302             }
1303
1304             // if we didn't have to execute the PS early, and at least 1 sample passed the depth test, run the PS
1305             if(!pPSState->writesODepth && !pPSState->killsPixel && _simd_movemask_ps(anyDepthSamplePassed))
1306             {
1307                 RDTSC_START(BEBarycentric);
1308                 backendFuncs.pfnCalcPixelBarycentrics(coeffs, psContext);
1309                 // interpolate z
1310                 psContext.vZ = vplaneps(coeffs.vZa, coeffs.vZb, coeffs.vZc, psContext.vI.center, psContext.vJ.center);
1311                 RDTSC_STOP(BEBarycentric, 0, 0);
1312
1313                 // execute pixel shader
1314                 RDTSC_START(BEPixelShader);
1315                 state.psState.pfnPixelShader(GetPrivateState(pDC), &psContext);
1316                 RDTSC_STOP(BEPixelShader, 0, 0);
1317             }
1318             ///@todo: make sure this works for kill pixel
1319             else if(!_simd_movemask_ps(anyStencilSamplePassed))
1320             {
1321                 goto Endtile;
1322             }
1323
1324             // loop over all samples, broadcasting the results of the PS to all passing pixels
1325             for(uint32_t sample = 0; sample < numOMSamples; sample++)
1326             {
1327                 uint8_t *pDepthSample = pDepthBase + MultisampleTraits<sampleCount>::RasterTileDepthOffset(sample);
1328                 uint8_t * pStencilSample = pStencilBase + MultisampleTraits<sampleCount>::RasterTileStencilOffset(sample);
1329
1330                 // output merger
1331                 RDTSC_START(BEOutputMerger);
1332
1333                 // skip if none of the pixels for this sample passed
1334                 simdscalar coverageMaskSample;
1335                 simdscalar depthMaskSample;
1336                 simdscalar stencilMaskSample;
1337                 simdscalar vInterpolatedZ;
1338
1339                 // forcedSampleCount outputs to any pixels with covered samples not masked off by SampleMask
1340                 // depth test is disabled, so just set the z val to 0.
1341                 if(bForcedSampleCount)
1342                 {
1343                     coverageMaskSample = depthMaskSample = anyDepthSamplePassed;
1344                     vInterpolatedZ = _simd_setzero_ps();
1345                 }
1346                 else if(bIsStandardPattern)
1347                 {
1348                     if(!_simd_movemask_ps(depthPassMask[sample]))
1349                     {
1350                         depthPassMask[sample] = _simd_setzero_ps();
1351                         DepthStencilWrite(&state.vp[0], &state.depthStencilState, work.triFlags.frontFacing, vZ[sample], pDepthSample, depthPassMask[sample],
1352                                           vCoverageMask[sample], pStencilSample, stencilPassMask[sample]);
1353                         continue;
1354                     }
1355                     coverageMaskSample = vCoverageMask[sample];
1356                     depthMaskSample = depthPassMask[sample];
1357                     stencilMaskSample = stencilPassMask[sample];
1358                     vInterpolatedZ = vZ[sample];
1359                 }
1360                 else
1361                 {
1362                     // center pattern only needs to use a single depth test as all samples are at the same position
1363                     if(!_simd_movemask_ps(depthPassMask[0]))
1364                     {
1365                         depthPassMask[0] = _simd_setzero_ps();
1366                         DepthStencilWrite(&state.vp[0], &state.depthStencilState, work.triFlags.frontFacing, vZ[0], pDepthSample, depthPassMask[0],
1367                                           vCoverageMask[0], pStencilSample, stencilPassMask[0]);
1368                         continue;
1369                     }
1370                     coverageMaskSample = (vCoverageMask[0]);
1371                     depthMaskSample = depthPassMask[0];
1372                     stencilMaskSample = stencilPassMask[0];
1373                     vInterpolatedZ = vZ[0];
1374                 }
1375
1376                 // output merger
1377                 RDTSC_START(BEOutputMerger);
1378                 backendFuncs.pfnOutputMerger(psContext, pColorBase, sample, pBlendState, state.pfnBlendFunc,
1379                                              coverageMaskSample, depthMaskSample);
1380
1381                 DepthStencilWrite(&state.vp[0], &state.depthStencilState, work.triFlags.frontFacing, vInterpolatedZ, pDepthSample, depthMaskSample,
1382                                   coverageMaskSample, pStencilSample, stencilMaskSample);
1383                 RDTSC_STOP(BEOutputMerger, 0, 0);
1384             }
1385
1386 Endtile:
1387             RDTSC_START(BEEndTile);
1388             for(uint32_t sample = 0; sample < numCoverageSamples; sample++)
1389             {
1390                 work.coverageMask[sample] >>= (SIMD_TILE_Y_DIM * SIMD_TILE_X_DIM);
1391             }
1392
1393             pDepthBase += (KNOB_SIMD_WIDTH * FormatTraits<KNOB_DEPTH_HOT_TILE_FORMAT>::bpp) / 8;
1394             pStencilBase += (KNOB_SIMD_WIDTH * FormatTraits<KNOB_STENCIL_HOT_TILE_FORMAT>::bpp) / 8;
1395
1396             for(uint32_t rt = 0; rt < NumRT; ++rt)
1397             {
1398                 pColorBase[rt] += (KNOB_SIMD_WIDTH * FormatTraits<KNOB_COLOR_HOT_TILE_FORMAT>::bpp) / 8;
1399             }
1400             RDTSC_STOP(BEEndTile, 0, 0);
1401         }
1402     }
1403 }
1404 // optimized backend flow with NULL PS
1405 template<uint32_t sampleCountT>
1406 void BackendNullPS(DRAW_CONTEXT *pDC, uint32_t workerId, uint32_t x, uint32_t y, SWR_TRIANGLE_DESC &work, RenderOutputBuffers &renderBuffers)
1407 {
1408     RDTSC_START(BESetup);
1409
1410     static const SWR_MULTISAMPLE_COUNT sampleCount = (SWR_MULTISAMPLE_COUNT)sampleCountT;
1411     SWR_CONTEXT *pContext = pDC->pContext;
1412     const API_STATE& state = GetApiState(pDC);
1413     const BACKEND_FUNCS& backendFuncs = pDC->pState->backendFuncs;
1414
1415     // broadcast scalars
1416     BarycentricCoeffs coeffs;
1417     coeffs.vIa = _simd_broadcast_ss(&work.I[0]);
1418     coeffs.vIb = _simd_broadcast_ss(&work.I[1]);
1419     coeffs.vIc = _simd_broadcast_ss(&work.I[2]);
1420
1421     coeffs.vJa = _simd_broadcast_ss(&work.J[0]);
1422     coeffs.vJb = _simd_broadcast_ss(&work.J[1]);
1423     coeffs.vJc = _simd_broadcast_ss(&work.J[2]);
1424
1425     coeffs.vZa = _simd_broadcast_ss(&work.Z[0]);
1426     coeffs.vZb = _simd_broadcast_ss(&work.Z[1]);
1427     coeffs.vZc = _simd_broadcast_ss(&work.Z[2]);
1428
1429     coeffs.vRecipDet = _simd_broadcast_ss(&work.recipDet);
1430
1431     BYTE *pDepthBase = renderBuffers.pDepth, *pStencilBase = renderBuffers.pStencil;
1432
1433     RDTSC_STOP(BESetup, 0, 0);
1434
1435     SWR_PS_CONTEXT psContext;
1436     for (uint32_t yy = y; yy < y + KNOB_TILE_Y_DIM; yy += SIMD_TILE_Y_DIM)
1437     {
1438         // UL pixel corner
1439         simdscalar vYSamplePosUL = _simd_add_ps(vULOffsetsY, _simd_set1_ps((float)yy));
1440
1441         for (uint32_t xx = x; xx < x + KNOB_TILE_X_DIM; xx += SIMD_TILE_X_DIM)
1442         {
1443             // UL pixel corners
1444             simdscalar vXSamplePosUL = _simd_add_ps(vULOffsetsX, _simd_set1_ps((float)xx));
1445
1446             // iterate over active samples
1447             unsigned long sample = 0;
1448             uint32_t sampleMask = state.blendState.sampleMask;
1449             while (_BitScanForward(&sample, sampleMask))
1450             {
1451                 sampleMask &= ~(1 << sample);
1452                 if (work.coverageMask[sample] & MASK)
1453                 {
1454                     RDTSC_START(BEBarycentric);
1455                     // calculate per sample positions
1456                     psContext.vX.sample = _simd_add_ps(vXSamplePosUL, MultisampleTraits<sampleCount>::vX(sample));
1457                     psContext.vY.sample = _simd_add_ps(vYSamplePosUL, MultisampleTraits<sampleCount>::vY(sample));
1458
1459                     backendFuncs.pfnCalcSampleBarycentrics(coeffs, psContext);
1460
1461                     // interpolate z
1462                     psContext.vZ = vplaneps(coeffs.vZa, coeffs.vZb, coeffs.vZc, psContext.vI.sample, psContext.vJ.sample);
1463
1464                     RDTSC_STOP(BEBarycentric, 0, 0);
1465
1466                     simdscalar vCoverageMask = vMask(work.coverageMask[sample] & MASK);
1467                     simdscalar stencilPassMask = vCoverageMask;
1468
1469                     // offset depth/stencil buffers current sample
1470                     uint8_t *pDepthSample = pDepthBase + MultisampleTraits<sampleCount>::RasterTileDepthOffset(sample);
1471                     uint8_t *pStencilSample = pStencilBase + MultisampleTraits<sampleCount>::RasterTileStencilOffset(sample);
1472
1473                     RDTSC_START(BEEarlyDepthTest);
1474                     simdscalar depthPassMask = DepthStencilTest(&state.vp[0], &state.depthStencilState, work.triFlags.frontFacing,
1475                         psContext.vZ, pDepthSample, vCoverageMask, pStencilSample, &stencilPassMask);
1476                     DepthStencilWrite(&state.vp[0], &state.depthStencilState, work.triFlags.frontFacing, psContext.vZ,
1477                         pDepthSample, depthPassMask, vCoverageMask, pStencilSample, stencilPassMask);
1478                     RDTSC_STOP(BEEarlyDepthTest, 0, 0);
1479
1480                     uint32_t statMask = _simd_movemask_ps(depthPassMask);
1481                     uint32_t statCount = _mm_popcnt_u32(statMask);
1482                     UPDATE_STAT(DepthPassCount, statCount);
1483                 }
1484                 work.coverageMask[sample] >>= (SIMD_TILE_Y_DIM * SIMD_TILE_X_DIM);
1485             }
1486             pDepthBase += (KNOB_SIMD_WIDTH * FormatTraits<KNOB_DEPTH_HOT_TILE_FORMAT>::bpp) / 8;
1487             pStencilBase += (KNOB_SIMD_WIDTH * FormatTraits<KNOB_STENCIL_HOT_TILE_FORMAT>::bpp) / 8;
1488         }
1489     }
1490 }
1491
1492 void InitClearTilesTable()
1493 {
1494     memset(sClearTilesTable, 0, sizeof(sClearTilesTable));
1495
1496     sClearTilesTable[R8G8B8A8_UNORM] = ClearMacroTile<R8G8B8A8_UNORM>;
1497     sClearTilesTable[B8G8R8A8_UNORM] = ClearMacroTile<B8G8R8A8_UNORM>;
1498     sClearTilesTable[R32_FLOAT] = ClearMacroTile<R32_FLOAT>;
1499     sClearTilesTable[R32G32B32A32_FLOAT] = ClearMacroTile<R32G32B32A32_FLOAT>;
1500     sClearTilesTable[R8_UINT] = ClearMacroTile<R8_UINT>;
1501 }
1502
1503 PFN_BACKEND_FUNC gBackendNullPs[SWR_MULTISAMPLE_TYPE_MAX];
1504 PFN_BACKEND_FUNC gBackendSingleSample[2][2] = {};
1505 PFN_BACKEND_FUNC gBackendPixelRateTable[SWR_MULTISAMPLE_TYPE_MAX][SWR_MSAA_SAMPLE_PATTERN_MAX][SWR_INPUT_COVERAGE_MAX][2][2] = {};
1506 PFN_BACKEND_FUNC gBackendSampleRateTable[SWR_MULTISAMPLE_TYPE_MAX][SWR_INPUT_COVERAGE_MAX][2] = {};
1507 PFN_OUTPUT_MERGER gBackendOutputMergerTable[SWR_NUM_RENDERTARGETS+1][SWR_MULTISAMPLE_TYPE_MAX] = {};
1508 PFN_CALC_PIXEL_BARYCENTRICS gPixelBarycentricTable[2] = {};
1509 PFN_CALC_SAMPLE_BARYCENTRICS gSampleBarycentricTable[2] = {};
1510 PFN_CALC_CENTROID_BARYCENTRICS gCentroidBarycentricTable[SWR_MULTISAMPLE_TYPE_MAX][2][2][2] = {};
1511
1512 // Recursive template used to auto-nest conditionals.  Converts dynamic enum function
1513 // arguments to static template arguments.
1514 template <uint32_t... ArgsT>
1515 struct OMChooser
1516 {
1517     // Last Arg Terminator
1518     static PFN_OUTPUT_MERGER GetFunc(SWR_MULTISAMPLE_COUNT tArg)
1519     {
1520         switch(tArg)
1521         {
1522         case SWR_MULTISAMPLE_1X: return OutputMerger<ArgsT..., SWR_MULTISAMPLE_1X>; break;
1523         case SWR_MULTISAMPLE_2X: return OutputMerger<ArgsT..., SWR_MULTISAMPLE_2X>; break;
1524         case SWR_MULTISAMPLE_4X: return OutputMerger<ArgsT..., SWR_MULTISAMPLE_4X>; break;
1525         case SWR_MULTISAMPLE_8X: return OutputMerger<ArgsT..., SWR_MULTISAMPLE_8X>; break;
1526         case SWR_MULTISAMPLE_16X: return OutputMerger<ArgsT..., SWR_MULTISAMPLE_16X>; break;
1527         default:
1528             SWR_ASSERT(0 && "Invalid sample count\n");
1529             return nullptr;
1530             break;
1531         }
1532     }
1533
1534     // Recursively parse args
1535     template <typename... TArgsT>
1536     static PFN_OUTPUT_MERGER GetFunc(uint32_t tArg, TArgsT... remainingArgs)
1537     {
1538         switch(tArg)
1539         {
1540         case 0: return OMChooser<ArgsT..., 0>::GetFunc(remainingArgs...); break;
1541         case 1: return OMChooser<ArgsT..., 1>::GetFunc(remainingArgs...); break;
1542         case 2: return OMChooser<ArgsT..., 2>::GetFunc(remainingArgs...); break;
1543         case 3: return OMChooser<ArgsT..., 3>::GetFunc(remainingArgs...); break;
1544         case 4: return OMChooser<ArgsT..., 4>::GetFunc(remainingArgs...); break;
1545         case 5: return OMChooser<ArgsT..., 5>::GetFunc(remainingArgs...); break;
1546         case 6: return OMChooser<ArgsT..., 6>::GetFunc(remainingArgs...); break;
1547         case 7: return OMChooser<ArgsT..., 7>::GetFunc(remainingArgs...); break;
1548         case 8: return OMChooser<ArgsT..., 8>::GetFunc(remainingArgs...); break;
1549         default:
1550             SWR_ASSERT(0 && "Invalid RT index\n");
1551             return nullptr;
1552             break;
1553         }
1554     }
1555 };
1556
1557 // Recursive template used to auto-nest conditionals.  Converts dynamic enum function
1558 // arguments to static template arguments.
1559 template <uint32_t... ArgsT>
1560 struct BECentroidBarycentricChooser
1561 {
1562
1563     // Last Arg Terminator
1564     template <typename... TArgsT>
1565     static PFN_CALC_CENTROID_BARYCENTRICS GetFunc(uint32_t tArg)
1566     {
1567         if(tArg > 0)
1568         {
1569             return CalcCentroidBarycentrics<ArgsT..., 1>;
1570         }
1571
1572         return CalcCentroidBarycentrics<ArgsT..., 0>;
1573     }
1574
1575     // Recursively parse args
1576     template <typename... TArgsT>
1577     static PFN_CALC_CENTROID_BARYCENTRICS GetFunc(SWR_MULTISAMPLE_COUNT tArg, TArgsT... remainingArgs)
1578     {
1579         switch(tArg)
1580         {
1581         case SWR_MULTISAMPLE_1X: return BECentroidBarycentricChooser<ArgsT..., SWR_MULTISAMPLE_1X>::GetFunc(remainingArgs...); break;
1582         case SWR_MULTISAMPLE_2X: return BECentroidBarycentricChooser<ArgsT..., SWR_MULTISAMPLE_2X>::GetFunc(remainingArgs...); break;
1583         case SWR_MULTISAMPLE_4X: return BECentroidBarycentricChooser<ArgsT..., SWR_MULTISAMPLE_4X>::GetFunc(remainingArgs...); break;
1584         case SWR_MULTISAMPLE_8X: return BECentroidBarycentricChooser<ArgsT..., SWR_MULTISAMPLE_8X>::GetFunc(remainingArgs...); break;
1585         case SWR_MULTISAMPLE_16X: return BECentroidBarycentricChooser<ArgsT..., SWR_MULTISAMPLE_16X>::GetFunc(remainingArgs...); break;
1586         default:
1587             SWR_ASSERT(0 && "Invalid sample count\n");
1588             return nullptr;
1589             break;
1590         }
1591     }
1592
1593     // Recursively parse args
1594     template <typename... TArgsT>
1595     static PFN_CALC_CENTROID_BARYCENTRICS GetFunc(uint32_t tArg, TArgsT... remainingArgs)
1596     {
1597         if(tArg > 0)
1598         {
1599             return BECentroidBarycentricChooser<ArgsT..., 1>::GetFunc(remainingArgs...);
1600         }
1601
1602         return BECentroidBarycentricChooser<ArgsT..., 0>::GetFunc(remainingArgs...);
1603     }
1604 };
1605
1606 // Recursive template used to auto-nest conditionals.  Converts dynamic enum function
1607 // arguments to static template arguments.
1608 template <uint32_t... ArgsT>
1609 struct BEChooser
1610 {
1611     // Last Arg Terminator
1612     static PFN_BACKEND_FUNC GetFunc(SWR_BACKEND_FUNCS tArg)
1613     {
1614         switch(tArg)
1615         {
1616         case SWR_BACKEND_SINGLE_SAMPLE: return BackendSingleSample<ArgsT...>; break;
1617         case SWR_BACKEND_MSAA_PIXEL_RATE: return BackendPixelRate<ArgsT...>; break;
1618         case SWR_BACKEND_MSAA_SAMPLE_RATE: return BackendSampleRate<ArgsT...>; break;
1619         default:
1620             SWR_ASSERT(0 && "Invalid backend func\n");
1621             return nullptr;
1622             break;
1623         }
1624     }
1625
1626
1627     // Recursively parse args
1628     template <typename... TArgsT>
1629     static PFN_BACKEND_FUNC GetFunc(SWR_MULTISAMPLE_COUNT tArg, TArgsT... remainingArgs)
1630     {
1631         switch(tArg)
1632         {
1633         case SWR_MULTISAMPLE_1X: return BEChooser<ArgsT..., SWR_MULTISAMPLE_1X>::GetFunc(remainingArgs...); break;
1634         case SWR_MULTISAMPLE_2X: return BEChooser<ArgsT..., SWR_MULTISAMPLE_2X>::GetFunc(remainingArgs...); break;
1635         case SWR_MULTISAMPLE_4X: return BEChooser<ArgsT..., SWR_MULTISAMPLE_4X>::GetFunc(remainingArgs...); break;
1636         case SWR_MULTISAMPLE_8X: return BEChooser<ArgsT..., SWR_MULTISAMPLE_8X>::GetFunc(remainingArgs...); break;
1637         case SWR_MULTISAMPLE_16X: return BEChooser<ArgsT..., SWR_MULTISAMPLE_16X>::GetFunc(remainingArgs...); break;
1638         default:
1639             SWR_ASSERT(0 && "Invalid sample count\n");
1640             return nullptr;
1641             break;
1642         }
1643     }
1644
1645     // Recursively parse args
1646     template <typename... TArgsT>
1647     static PFN_BACKEND_FUNC GetFunc(uint32_t tArg, TArgsT... remainingArgs)
1648     {
1649         if(tArg > 0)
1650         {
1651             return BEChooser<ArgsT..., 1>::GetFunc(remainingArgs...);
1652         }
1653
1654          return BEChooser<ArgsT..., 0>::GetFunc(remainingArgs...);
1655     }
1656 };
1657
1658 template <uint32_t numRenderTargets, SWR_MULTISAMPLE_COUNT numSampleRates>
1659 void InitBackendOMFuncTable(PFN_OUTPUT_MERGER (&table)[numRenderTargets][numSampleRates])
1660 {
1661     for(uint32_t rtNum = SWR_ATTACHMENT_COLOR0; rtNum < numRenderTargets; rtNum++)
1662     {
1663         for(uint32_t sampleCount = SWR_MULTISAMPLE_1X; sampleCount < numSampleRates; sampleCount++)
1664         {
1665             table[rtNum][sampleCount] =
1666                 OMChooser<>::GetFunc((SWR_RENDERTARGET_ATTACHMENT)rtNum, (SWR_MULTISAMPLE_COUNT)sampleCount);
1667         }
1668     }
1669 }
1670
1671 template <SWR_MULTISAMPLE_COUNT numSampleRates>
1672 void InitBackendBarycentricsTables(PFN_CALC_PIXEL_BARYCENTRICS (&pixelTable)[2],
1673                                    PFN_CALC_SAMPLE_BARYCENTRICS (&sampleTable)[2],
1674                                    PFN_CALC_CENTROID_BARYCENTRICS (&centroidTable)[numSampleRates][2][2][2])
1675 {
1676     pixelTable[0] = CalcPixelBarycentrics<0>;
1677     pixelTable[1] = CalcPixelBarycentrics<1>;
1678
1679     sampleTable[0] = CalcSampleBarycentrics<0>;
1680     sampleTable[1] = CalcSampleBarycentrics<1>;
1681
1682     for(uint32_t sampleCount = SWR_MULTISAMPLE_1X; sampleCount < numSampleRates; sampleCount++)
1683     {
1684         for(uint32_t baryMask = 0; baryMask < 2; baryMask++)
1685         {
1686             for(uint32_t patternNum = 0; patternNum < 2; patternNum++)
1687             {
1688                 for(uint32_t forcedSampleEnable = 0; forcedSampleEnable < 2; forcedSampleEnable++)
1689                 {
1690                     centroidTable[sampleCount][baryMask][patternNum][forcedSampleEnable]=
1691                         BECentroidBarycentricChooser<>::GetFunc((SWR_MULTISAMPLE_COUNT)sampleCount, baryMask, patternNum, forcedSampleEnable);
1692                 }
1693             }
1694         }
1695     }
1696 }
1697
1698 void InitBackendSampleFuncTable(PFN_BACKEND_FUNC (&table)[2][2])
1699 {
1700     gBackendSingleSample[0][0] = BEChooser<>::GetFunc(SWR_MULTISAMPLE_1X, SWR_MSAA_STANDARD_PATTERN, SWR_INPUT_COVERAGE_NONE, 0, 0, (SWR_BACKEND_FUNCS)SWR_BACKEND_SINGLE_SAMPLE);
1701     gBackendSingleSample[0][1] = BEChooser<>::GetFunc(SWR_MULTISAMPLE_1X, SWR_MSAA_STANDARD_PATTERN, SWR_INPUT_COVERAGE_NONE, 1, 0, (SWR_BACKEND_FUNCS)SWR_BACKEND_SINGLE_SAMPLE);
1702     gBackendSingleSample[1][0] = BEChooser<>::GetFunc(SWR_MULTISAMPLE_1X, SWR_MSAA_STANDARD_PATTERN, SWR_INPUT_COVERAGE_NORMAL, 0, 0, (SWR_BACKEND_FUNCS)SWR_BACKEND_SINGLE_SAMPLE);
1703     gBackendSingleSample[1][1] = BEChooser<>::GetFunc(SWR_MULTISAMPLE_1X, SWR_MSAA_STANDARD_PATTERN, SWR_INPUT_COVERAGE_NORMAL, 1, 0, (SWR_BACKEND_FUNCS)SWR_BACKEND_SINGLE_SAMPLE);
1704 }
1705
1706 template <SWR_MULTISAMPLE_COUNT numSampleRates, SWR_MSAA_SAMPLE_PATTERN numSamplePatterns, SWR_INPUT_COVERAGE numCoverageModes>
1707 void InitBackendPixelFuncTable(PFN_BACKEND_FUNC (&table)[numSampleRates][numSamplePatterns][numCoverageModes][2][2])
1708 {
1709     for(uint32_t sampleCount = SWR_MULTISAMPLE_1X; sampleCount < numSampleRates; sampleCount++)
1710     {
1711         for(uint32_t samplePattern = SWR_MSAA_CENTER_PATTERN; samplePattern < numSamplePatterns; samplePattern++)
1712         {
1713             for(uint32_t inputCoverage = SWR_INPUT_COVERAGE_NONE; inputCoverage < numCoverageModes; inputCoverage++)
1714             {
1715                 for(uint32_t isCentroid = 0; isCentroid < 2; isCentroid++)
1716                 {
1717                     table[sampleCount][samplePattern][inputCoverage][isCentroid][0] =
1718                         BEChooser<>::GetFunc((SWR_MULTISAMPLE_COUNT)sampleCount, samplePattern, inputCoverage, isCentroid, 0, (SWR_BACKEND_FUNCS)SWR_BACKEND_MSAA_PIXEL_RATE);
1719                     table[sampleCount][samplePattern][inputCoverage][isCentroid][1] =
1720                         BEChooser<>::GetFunc((SWR_MULTISAMPLE_COUNT)sampleCount, samplePattern, inputCoverage, isCentroid, 1, (SWR_BACKEND_FUNCS)SWR_BACKEND_MSAA_PIXEL_RATE);
1721                 }
1722             }
1723         }
1724     }
1725 }
1726
1727 template <uint32_t numSampleRates, uint32_t numCoverageModes>
1728 void InitBackendSampleFuncTable(PFN_BACKEND_FUNC (&table)[numSampleRates][numCoverageModes][2])
1729 {
1730     for(uint32_t sampleCount = SWR_MULTISAMPLE_1X; sampleCount < numSampleRates; sampleCount++)
1731     {
1732         for(uint32_t inputCoverage = SWR_INPUT_COVERAGE_NONE; inputCoverage < numCoverageModes; inputCoverage++)
1733         {
1734             table[sampleCount][inputCoverage][0] =
1735                 BEChooser<>::GetFunc((SWR_MULTISAMPLE_COUNT)sampleCount, SWR_MSAA_STANDARD_PATTERN, inputCoverage, 0, 0, (SWR_BACKEND_FUNCS)SWR_BACKEND_MSAA_SAMPLE_RATE);
1736             table[sampleCount][inputCoverage][1] =
1737                 BEChooser<>::GetFunc((SWR_MULTISAMPLE_COUNT)sampleCount, SWR_MSAA_STANDARD_PATTERN, inputCoverage, 1, 0, (SWR_BACKEND_FUNCS)SWR_BACKEND_MSAA_SAMPLE_RATE);
1738         }
1739     }
1740 }
1741
1742 void InitBackendFuncTables()
1743 {
1744     InitBackendSampleFuncTable(gBackendSingleSample);
1745     InitBackendPixelFuncTable<(SWR_MULTISAMPLE_COUNT)SWR_MULTISAMPLE_TYPE_MAX, SWR_MSAA_SAMPLE_PATTERN_MAX, SWR_INPUT_COVERAGE_MAX>(gBackendPixelRateTable);
1746     InitBackendSampleFuncTable<SWR_MULTISAMPLE_TYPE_MAX, SWR_INPUT_COVERAGE_MAX>(gBackendSampleRateTable);
1747     InitBackendOMFuncTable<SWR_NUM_RENDERTARGETS+1, SWR_MULTISAMPLE_TYPE_MAX>(gBackendOutputMergerTable);
1748     InitBackendBarycentricsTables<(SWR_MULTISAMPLE_COUNT)(SWR_MULTISAMPLE_TYPE_MAX)>(gPixelBarycentricTable, gSampleBarycentricTable, gCentroidBarycentricTable);
1749
1750     gBackendNullPs[SWR_MULTISAMPLE_1X] = &BackendNullPS < SWR_MULTISAMPLE_1X > ;
1751     gBackendNullPs[SWR_MULTISAMPLE_2X] = &BackendNullPS < SWR_MULTISAMPLE_2X > ;
1752     gBackendNullPs[SWR_MULTISAMPLE_4X] = &BackendNullPS < SWR_MULTISAMPLE_4X > ;
1753     gBackendNullPs[SWR_MULTISAMPLE_8X] = &BackendNullPS < SWR_MULTISAMPLE_8X > ;
1754     gBackendNullPs[SWR_MULTISAMPLE_16X] = &BackendNullPS < SWR_MULTISAMPLE_16X > ;
1755 }