src/gallium/drivers/swr/rasterizer/core/context.h

   1 /****************************************************************************
   2 * Copyright (C) 2014-2016 Intel Corporation.   All Rights Reserved.
   3 *
   4 * Permission is hereby granted, free of charge, to any person obtaining a
   5 * copy of this software and associated documentation files (the "Software"),
   6 * to deal in the Software without restriction, including without limitation
   7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8 * and/or sell copies of the Software, and to permit persons to whom the
   9 * Software is furnished to do so, subject to the following conditions:
  10 *
  11 * The above copyright notice and this permission notice (including the next
  12 * paragraph) shall be included in all copies or substantial portions of the
  13 * Software.
  14 *
  15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21 * IN THE SOFTWARE.
  22 *
  23 * @file context.h
  24 *
  25 * @brief Definitions for SWR_CONTEXT and DRAW_CONTEXT
  26 *        The SWR_CONTEXT is our global context and contains the DC ring,
  27 *        thread state, etc.
  28 *
  29 *        The DRAW_CONTEXT contains all state associated with a draw operation.
  30 *
  31 ******************************************************************************/
  32 #pragma once
  33
  34 #include <condition_variable>
  35 #include <algorithm>
  36
  37 #include "core/api.h"
  38 #include "core/utils.h"
  39 #include "core/arena.h"
  40 #include "core/fifo.hpp"
  41 #include "core/knobs.h"
  42 #include "common/intrin.h"
  43 #include "core/threads.h"
  44 #include "ringbuffer.h"
  45 #include "archrast/archrast.h"
  46
  47 // x.8 fixed point precision values
  48 #define FIXED_POINT_SHIFT 8
  49 #define FIXED_POINT_SCALE 256
  50
  51 // x.16 fixed point precision values
  52 #define FIXED_POINT16_SHIFT 16
  53 #define FIXED_POINT16_SCALE 65536
  54
  55 struct SWR_CONTEXT;
  56 struct DRAW_CONTEXT;
  57
  58 struct TRI_FLAGS
  59 {
  60     uint32_t frontFacing : 1;
  61     uint32_t yMajor : 1;
  62     uint32_t coverageMask : (SIMD_TILE_X_DIM * SIMD_TILE_Y_DIM);
  63     uint32_t reserved : 32 - 1 - 1 - (SIMD_TILE_X_DIM * SIMD_TILE_Y_DIM);
  64     float pointSize;
  65     uint32_t renderTargetArrayIndex;
  66     uint32_t viewportIndex;
  67 };
  68
  69 //////////////////////////////////////////////////////////////////////////
  70 /// SWR_TRIANGLE_DESC
  71 /////////////////////////////////////////////////////////////////////////
  72 struct SWR_TRIANGLE_DESC
  73 {
  74     float I[3];
  75     float J[3];
  76     float Z[3];
  77     float OneOverW[3];
  78     float recipDet;
  79
  80     float *pRecipW;
  81     float *pAttribs;
  82     float *pPerspAttribs;
  83     float *pSamplePos;
  84     float *pUserClipBuffer;
  85
  86     uint64_t coverageMask[SWR_MAX_NUM_MULTISAMPLES];
  87     uint64_t innerCoverageMask; // Conservative rasterization inner coverage: marked covered if entire pixel is covered
  88     uint64_t anyCoveredSamples;
  89
  90     TRI_FLAGS triFlags;
  91 };
  92
  93 struct TRIANGLE_WORK_DESC
  94 {
  95     float *pTriBuffer;
  96     float *pAttribs;
  97     float *pUserClipBuffer;
  98     uint32_t numAttribs;
  99     TRI_FLAGS triFlags;
 100 };
 101
 102 struct CLEAR_DESC
 103 {
 104     SWR_RECT rect;
 105     uint32_t attachmentMask;
 106     uint32_t renderTargetArrayIndex;
 107     float clearRTColor[4];  // RGBA_32F
 108     float clearDepth;   // [0..1]
 109     uint8_t clearStencil;
 110 };
 111
 112 struct DISCARD_INVALIDATE_TILES_DESC
 113 {
 114     uint32_t attachmentMask;
 115     SWR_RECT rect;
 116     SWR_TILE_STATE newTileState;
 117     bool createNewTiles;
 118     bool fullTilesOnly;
 119 };
 120
 121 struct SYNC_DESC
 122 {
 123     PFN_CALLBACK_FUNC pfnCallbackFunc;
 124     uint64_t userData;
 125     uint64_t userData2;
 126     uint64_t userData3;
 127 };
 128
 129 struct STORE_TILES_DESC
 130 {
 131     uint32_t attachmentMask;
 132     SWR_TILE_STATE postStoreTileState;
 133     SWR_RECT rect;
 134 };
 135
 136 struct COMPUTE_DESC
 137 {
 138     uint32_t threadGroupCountX;
 139     uint32_t threadGroupCountY;
 140     uint32_t threadGroupCountZ;
 141 };
 142
 143 typedef void(*PFN_WORK_FUNC)(DRAW_CONTEXT* pDC, uint32_t workerId, uint32_t macroTile, void* pDesc);
 144
 145 enum WORK_TYPE
 146 {
 147     SYNC,
 148     DRAW,
 149     CLEAR,
 150     DISCARDINVALIDATETILES,
 151     STORETILES,
 152     SHUTDOWN,
 153 };
 154
 155 OSALIGNSIMD(struct) BE_WORK
 156 {
 157     WORK_TYPE type;
 158     PFN_WORK_FUNC pfnWork;
 159     union
 160     {
 161         SYNC_DESC sync;
 162         TRIANGLE_WORK_DESC tri;
 163         CLEAR_DESC clear;
 164         DISCARD_INVALIDATE_TILES_DESC discardInvalidateTiles;
 165         STORE_TILES_DESC storeTiles;
 166     } desc;
 167 };
 168
 169 struct DRAW_WORK
 170 {
 171     DRAW_CONTEXT*   pDC;
 172     union
 173     {
 174         uint32_t   numIndices;      // DrawIndexed: Number of indices for draw.
 175         uint32_t   numVerts;        // Draw: Number of verts (triangles, lines, etc)
 176     };
 177     union
 178     {
 179         const int32_t* pIB;        // DrawIndexed: App supplied indices
 180         uint32_t   startVertex;    // Draw: Starting vertex in VB to render from.
 181     };
 182     int32_t    baseVertex;
 183     uint32_t   numInstances;        // Number of instances
 184     uint32_t   startInstance;       // Instance offset
 185     uint32_t   startPrimID;         // starting primitiveID for this draw batch
 186     uint32_t   startVertexID;       // starting VertexID for this draw batch (only needed for non-indexed draws)
 187     SWR_FORMAT type;                // index buffer type
 188 };
 189
 190 typedef void(*PFN_FE_WORK_FUNC)(SWR_CONTEXT* pContext, DRAW_CONTEXT* pDC, uint32_t workerId, void* pDesc);
 191 struct FE_WORK
 192 {
 193     WORK_TYPE type;
 194     PFN_FE_WORK_FUNC pfnWork;
 195     union
 196     {
 197         SYNC_DESC sync;
 198         DRAW_WORK draw;
 199         CLEAR_DESC clear;
 200         DISCARD_INVALIDATE_TILES_DESC discardInvalidateTiles;
 201         STORE_TILES_DESC storeTiles;
 202     } desc;
 203 };
 204
 205 struct GUARDBANDS
 206 {
 207     float left[KNOB_NUM_VIEWPORTS_SCISSORS];
 208     float right[KNOB_NUM_VIEWPORTS_SCISSORS];
 209     float top[KNOB_NUM_VIEWPORTS_SCISSORS];
 210     float bottom[KNOB_NUM_VIEWPORTS_SCISSORS];
 211 };
 212
 213 struct PA_STATE;
 214
 215 // function signature for pipeline stages that execute after primitive assembly
 216 typedef void(*PFN_PROCESS_PRIMS)(DRAW_CONTEXT *pDC, PA_STATE& pa, uint32_t workerId, simdvector prims[],
 217     uint32_t primMask, simdscalari const &primID, simdscalari const &viewportIdx);
 218
 219 #if ENABLE_AVX512_SIMD16
 220 // function signature for pipeline stages that execute after primitive assembly
 221 typedef void(SIMDCALL *PFN_PROCESS_PRIMS_SIMD16)(DRAW_CONTEXT *pDC, PA_STATE& pa, uint32_t workerId, simd16vector prims[],
 222     uint32_t primMask, simd16scalari const &primID, simd16scalari const &viewportIdx);
 223
 224 #endif
 225 OSALIGNLINE(struct) API_STATE
 226 {
 227     // Vertex Buffers
 228     SWR_VERTEX_BUFFER_STATE vertexBuffers[KNOB_NUM_STREAMS];
 229
 230     // Index Buffer
 231     SWR_INDEX_BUFFER_STATE  indexBuffer;
 232
 233     // FS - Fetch Shader State
 234     PFN_FETCH_FUNC          pfnFetchFunc;
 235
 236     // VS - Vertex Shader State
 237     PFN_VERTEX_FUNC         pfnVertexFunc;
 238
 239     // GS - Geometry Shader State
 240     PFN_GS_FUNC             pfnGsFunc;
 241     SWR_GS_STATE            gsState;
 242
 243     // CS - Compute Shader
 244     PFN_CS_FUNC             pfnCsFunc;
 245     uint32_t                totalThreadsInGroup;
 246     uint32_t                totalSpillFillSize;
 247     uint32_t                scratchSpaceSize;
 248     uint32_t                scratchSpaceNumInstances;
 249
 250     // FE - Frontend State
 251     SWR_FRONTEND_STATE      frontendState;
 252
 253     // SOS - Streamout Shader State
 254     PFN_SO_FUNC             pfnSoFunc[MAX_SO_STREAMS];
 255
 256     // Streamout state
 257     SWR_STREAMOUT_STATE     soState;
 258     mutable SWR_STREAMOUT_BUFFER soBuffer[MAX_SO_STREAMS];
 259
 260     // Tessellation State
 261     PFN_HS_FUNC             pfnHsFunc;
 262     PFN_DS_FUNC             pfnDsFunc;
 263     SWR_TS_STATE            tsState;
 264
 265     // Number of attributes used by the frontend (vs, so, gs)
 266     uint32_t                feNumAttributes;
 267
 268     PRIMITIVE_TOPOLOGY      topology;
 269     bool                    forceFront;
 270
 271     // RS - Rasterizer State
 272     SWR_RASTSTATE           rastState;
 273     // floating point multisample offsets
 274     float samplePos[SWR_MAX_NUM_MULTISAMPLES * 2];
 275
 276     GUARDBANDS               gbState;
 277
 278     SWR_VIEWPORT            vp[KNOB_NUM_VIEWPORTS_SCISSORS];
 279     SWR_VIEWPORT_MATRICES   vpMatrices;
 280
 281     SWR_RECT                scissorRects[KNOB_NUM_VIEWPORTS_SCISSORS];
 282     SWR_RECT                scissorsInFixedPoint[KNOB_NUM_VIEWPORTS_SCISSORS];
 283     bool                    scissorsTileAligned;
 284
 285     // Backend state
 286     SWR_BACKEND_STATE       backendState;
 287
 288     SWR_DEPTH_BOUNDS_STATE  depthBoundsState;
 289
 290     // PS - Pixel shader state
 291     SWR_PS_STATE            psState;
 292
 293     SWR_DEPTH_STENCIL_STATE depthStencilState;
 294
 295     // OM - Output Merger State
 296     SWR_BLEND_STATE         blendState;
 297     PFN_BLEND_JIT_FUNC      pfnBlendFunc[SWR_NUM_RENDERTARGETS];
 298
 299     struct
 300     {
 301         uint32_t enableStatsFE : 1;             // Enable frontend pipeline stats
 302         uint32_t enableStatsBE : 1;             // Enable backend pipeline stats
 303         uint32_t colorHottileEnable : 8;        // Bitmask of enabled color hottiles
 304         uint32_t depthHottileEnable: 1;         // Enable depth buffer hottile
 305         uint32_t stencilHottileEnable : 1;      // Enable stencil buffer hottile
 306     };
 307
 308     PFN_QUANTIZE_DEPTH      pfnQuantizeDepth;
 309 };
 310
 311 class MacroTileMgr;
 312 class DispatchQueue;
 313
 314 struct RenderOutputBuffers
 315 {
 316     uint8_t* pColor[SWR_NUM_RENDERTARGETS];
 317     uint8_t* pDepth;
 318     uint8_t* pStencil;
 319 };
 320
 321 // Plane equation A/B/C coeffs used to evaluate I/J barycentric coords
 322 struct BarycentricCoeffs
 323 {
 324     simdscalar vIa;
 325     simdscalar vIb;
 326     simdscalar vIc;
 327
 328     simdscalar vJa;
 329     simdscalar vJb;
 330     simdscalar vJc;
 331
 332     simdscalar vZa;
 333     simdscalar vZb;
 334     simdscalar vZc;
 335
 336     simdscalar vRecipDet;
 337
 338     simdscalar vAOneOverW;
 339     simdscalar vBOneOverW;
 340     simdscalar vCOneOverW;
 341 };
 342
 343 // pipeline function pointer types
 344 typedef void(*PFN_BACKEND_FUNC)(DRAW_CONTEXT*, uint32_t, uint32_t, uint32_t, SWR_TRIANGLE_DESC&, RenderOutputBuffers&);
 345 typedef void(*PFN_OUTPUT_MERGER)(SWR_PS_CONTEXT &, uint8_t* (&)[SWR_NUM_RENDERTARGETS], uint32_t, const SWR_BLEND_STATE*,
 346                                  const PFN_BLEND_JIT_FUNC (&)[SWR_NUM_RENDERTARGETS], simdscalar&, simdscalar const &);
 347 typedef void(*PFN_CALC_PIXEL_BARYCENTRICS)(const BarycentricCoeffs&, SWR_PS_CONTEXT &);
 348 typedef void(*PFN_CALC_SAMPLE_BARYCENTRICS)(const BarycentricCoeffs&, SWR_PS_CONTEXT&);
 349 typedef void(*PFN_CALC_CENTROID_BARYCENTRICS)(const BarycentricCoeffs&, SWR_PS_CONTEXT &, const uint64_t *const, const uint32_t,
 350                                               simdscalar const &, simdscalar const &);
 351
 352 struct BACKEND_FUNCS
 353 {
 354     PFN_BACKEND_FUNC pfnBackend;
 355 };
 356
 357 // Draw State
 358 struct DRAW_STATE
 359 {
 360     API_STATE state;
 361
 362     void* pPrivateState;  // Its required the driver sets this up for each draw.
 363
 364     // pipeline function pointers, filled in by API thread when setting up the draw
 365     BACKEND_FUNCS backendFuncs;
 366     PFN_PROCESS_PRIMS pfnProcessPrims;
 367 #if USE_SIMD16_FRONTEND
 368     PFN_PROCESS_PRIMS_SIMD16 pfnProcessPrims_simd16;
 369 #endif
 370
 371     CachingArena* pArena;     // This should only be used by API thread.
 372 };
 373
 374 struct DRAW_DYNAMIC_STATE
 375 {
 376     void Reset(uint32_t numThreads)
 377     {
 378         SWR_STATS* pSavePtr = pStats;
 379         memset(this, 0, sizeof(*this));
 380         pStats = pSavePtr;
 381         memset(pStats, 0, sizeof(SWR_STATS) * numThreads);
 382     }
 383     ///@todo Currently assumes only a single FE can do stream output for a draw.
 384     uint32_t SoWriteOffset[4];
 385     bool     SoWriteOffsetDirty[4];
 386
 387     SWR_STATS_FE statsFE;   // Only one FE thread per DC.
 388     SWR_STATS*   pStats;
 389 };
 390
 391 // Draw Context
 392 //    The api thread sets up a draw context that exists for the life of the draw.
 393 //    This draw context maintains all of the state needed for the draw operation.
 394 struct DRAW_CONTEXT
 395 {
 396     SWR_CONTEXT*    pContext;
 397     union
 398     {
 399         MacroTileMgr*   pTileMgr;
 400         DispatchQueue*  pDispatch;      // Queue for thread groups. (isCompute)
 401     };
 402     DRAW_STATE*     pState;             // Read-only state. Core should not update this outside of API thread.
 403     DRAW_DYNAMIC_STATE dynState;
 404
 405     CachingArena*   pArena;
 406
 407     uint32_t        drawId;
 408     bool            dependentFE;    // Frontend work is dependent on all previous FE
 409     bool            dependent;      // Backend work is dependent on all previous BE
 410     bool            isCompute;      // Is this DC a compute context?
 411     bool            cleanupState;   // True if this is the last draw using an entry in the state ring.
 412
 413     FE_WORK         FeWork;
 414
 415     volatile OSALIGNLINE(bool)       doneFE;         // Is FE work done for this draw?
 416     volatile OSALIGNLINE(uint32_t)   FeLock;
 417     volatile OSALIGNLINE(uint32_t)   threadsDone;
 418
 419     SYNC_DESC       retireCallback; // Call this func when this DC is retired.
 420 };
 421
 422 static_assert((sizeof(DRAW_CONTEXT) & 63) == 0, "Invalid size for DRAW_CONTEXT");
 423
 424 INLINE const API_STATE& GetApiState(const DRAW_CONTEXT* pDC)
 425 {
 426     SWR_ASSERT(pDC != nullptr);
 427     SWR_ASSERT(pDC->pState != nullptr);
 428
 429     return pDC->pState->state;
 430 }
 431
 432 INLINE void* GetPrivateState(const DRAW_CONTEXT* pDC)
 433 {
 434     SWR_ASSERT(pDC != nullptr);
 435     SWR_ASSERT(pDC->pState != nullptr);
 436
 437     return pDC->pState->pPrivateState;
 438 }
 439
 440 class HotTileMgr;
 441
 442 struct SWR_CONTEXT
 443 {
 444     // Draw Context Ring
 445     //  Each draw needs its own state in order to support mulitple draws in flight across multiple threads.
 446     //  We maintain N draw contexts configured as a ring. The size of the ring limits the maximum number
 447     //  of draws that can be in flight at any given time.
 448     //
 449     //  Description:
 450     //  1. State - When an application first sets state we'll request a new draw context to use.
 451     //     a. If there are no available draw contexts then we'll have to wait until one becomes free.
 452     //     b. If one is available then set pCurDrawContext to point to it and mark it in use.
 453     //     c. All state calls set state on pCurDrawContext.
 454     //  2. Draw - Creates submits a work item that is associated with current draw context.
 455     //     a. Set pPrevDrawContext = pCurDrawContext
 456     //     b. Set pCurDrawContext to NULL.
 457     //  3. State - When an applications sets state after draw
 458     //     a. Same as step 1.
 459     //     b. State is copied from prev draw context to current.
 460     RingBuffer<DRAW_CONTEXT> dcRing;
 461
 462     DRAW_CONTEXT *pCurDrawContext;    // This points to DC entry in ring for an unsubmitted draw.
 463     DRAW_CONTEXT *pPrevDrawContext;   // This points to DC entry for the previous context submitted that we can copy state from.
 464
 465     MacroTileMgr* pMacroTileManagerArray;
 466     DispatchQueue* pDispatchQueueArray;
 467
 468     // Draw State Ring
 469     //  When draw are very large (lots of primitives) then the API thread will break these up.
 470     //  These split draws all have identical state. So instead of storing the state directly
 471     //  in the Draw Context (DC) we instead store it in a Draw State (DS). This allows multiple DCs
 472     //  to reference a single entry in the DS ring.
 473     RingBuffer<DRAW_STATE> dsRing;
 474
 475     uint32_t curStateId;               // Current index to the next available entry in the DS ring.
 476
 477     uint32_t NumWorkerThreads;
 478     uint32_t NumFEThreads;
 479     uint32_t NumBEThreads;
 480
 481     THREAD_POOL threadPool; // Thread pool associated with this context
 482     SWR_THREADING_INFO threadInfo;
 483
 484     uint32_t MAX_DRAWS_IN_FLIGHT;
 485
 486     std::condition_variable FifosNotEmpty;
 487     std::mutex WaitLock;
 488
 489     uint32_t privateStateSize;
 490
 491     HotTileMgr *pHotTileMgr;
 492
 493     // Callback functions, passed in at create context time
 494     PFN_LOAD_TILE               pfnLoadTile;
 495     PFN_STORE_TILE              pfnStoreTile;
 496     PFN_CLEAR_TILE              pfnClearTile;
 497     PFN_UPDATE_SO_WRITE_OFFSET  pfnUpdateSoWriteOffset;
 498     PFN_UPDATE_STATS            pfnUpdateStats;
 499     PFN_UPDATE_STATS_FE         pfnUpdateStatsFE;
 500
 501
 502     // Global Stats
 503     SWR_STATS* pStats;
 504
 505     // Scratch space for workers.
 506     uint8_t** ppScratch;
 507
 508     volatile OSALIGNLINE(uint32_t)  drawsOutstandingFE;
 509
 510     OSALIGNLINE(CachingAllocator) cachingArenaAllocator;
 511     uint32_t frameCount;
 512
 513     uint32_t lastFrameChecked;
 514     uint64_t lastDrawChecked;
 515     TileSet singleThreadLockedTiles;
 516
 517     // ArchRast thread contexts.
 518     HANDLE* pArContext;
 519 };
 520
 521 #define UPDATE_STAT_BE(name, count) if (GetApiState(pDC).enableStatsBE) { pDC->dynState.pStats[workerId].name += count; }
 522 #define UPDATE_STAT_FE(name, count) if (GetApiState(pDC).enableStatsFE) { pDC->dynState.statsFE.name += count; }
 523
 524 // ArchRast instrumentation framework
 525 #define AR_WORKER_CTX  pContext->pArContext[workerId]
 526 #define AR_API_CTX     pContext->pArContext[pContext->NumWorkerThreads]
 527
 528 #ifdef KNOB_ENABLE_AR
 529     #define _AR_BEGIN(ctx, type, id)    ArchRast::Dispatch(ctx, ArchRast::Start(ArchRast::type, id))
 530     #define _AR_END(ctx, type, count)   ArchRast::Dispatch(ctx, ArchRast::End(ArchRast::type, count))
 531     #define _AR_EVENT(ctx, event)       ArchRast::Dispatch(ctx, ArchRast::event)
 532     #define _AR_FLUSH(ctx, id)          ArchRast::FlushDraw(ctx, id)
 533 #else
 534     #ifdef KNOB_ENABLE_RDTSC
 535         #define _AR_BEGIN(ctx, type, id) (void)ctx; RDTSC_START(type)
 536         #define _AR_END(ctx, type, id)   RDTSC_STOP(type, id, 0)
 537     #else
 538         #define _AR_BEGIN(ctx, type, id) (void)ctx
 539         #define _AR_END(ctx, type, id)
 540     #endif
 541     #define _AR_EVENT(ctx, event)
 542     #define _AR_FLUSH(ctx, id)
 543 #endif
 544
 545 // Use these macros for api thread.
 546 #define AR_API_BEGIN(type, id) _AR_BEGIN(AR_API_CTX, type, id)
 547 #define AR_API_END(type, count) _AR_END(AR_API_CTX, type, count)
 548 #define AR_API_EVENT(event) _AR_EVENT(AR_API_CTX, event)
 549
 550 // Use these macros for worker threads.
 551 #define AR_BEGIN(type, id) _AR_BEGIN(AR_WORKER_CTX, type, id)
 552 #define AR_END(type, count) _AR_END(AR_WORKER_CTX, type, count)
 553 #define AR_EVENT(event) _AR_EVENT(AR_WORKER_CTX, event)
 554 #define AR_FLUSH(id) _AR_FLUSH(AR_WORKER_CTX, id)