1 /****************************************************************************
2 * Copyright (C) 2014-2016 Intel Corporation. All Rights Reserved.
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
22 ****************************************************************************/
33 #if defined(__linux__) || defined(__gnu_linux__)
39 #include "common/os.h"
43 #include "rasterizer.h"
44 #include "rdtsc_core.h"
53 uint32_t procGroup
= 0;
54 std::vector
<uint32_t> threadIds
;
60 std::vector
<Core
> cores
;
63 typedef std::vector
<NumaNode
> CPUNumaNodes
;
65 void CalculateProcessorTopology(CPUNumaNodes
& out_nodes
, uint32_t& out_numThreadsPerProcGroup
)
68 out_numThreadsPerProcGroup
= 0;
72 std::vector
<KAFFINITY
> threadMaskPerProcGroup
;
75 std::lock_guard
<std::mutex
> l(m
);
79 BOOL ret
= GetLogicalProcessorInformationEx(RelationProcessorCore
, nullptr, &bufSize
);
80 SWR_ASSERT(ret
== FALSE
&& GetLastError() == ERROR_INSUFFICIENT_BUFFER
);
82 PSYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX pBufferMem
= (PSYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX
)malloc(bufSize
);
83 SWR_ASSERT(pBufferMem
);
85 ret
= GetLogicalProcessorInformationEx(RelationProcessorCore
, pBufferMem
, &bufSize
);
86 SWR_ASSERT(ret
!= FALSE
, "Failed to get Processor Topology Information");
88 uint32_t count
= bufSize
/ pBufferMem
->Size
;
89 PSYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX pBuffer
= pBufferMem
;
91 for (uint32_t i
= 0; i
< count
; ++i
)
93 SWR_ASSERT(pBuffer
->Relationship
== RelationProcessorCore
);
94 for (uint32_t g
= 0; g
< pBuffer
->Processor
.GroupCount
; ++g
)
96 auto& gmask
= pBuffer
->Processor
.GroupMask
[g
];
97 uint32_t threadId
= 0;
98 uint32_t procGroup
= gmask
.Group
;
100 Core
* pCore
= nullptr;
102 uint32_t numThreads
= (uint32_t)_mm_popcount_sizeT(gmask
.Mask
);
104 while (BitScanForwardSizeT((unsigned long*)&threadId
, gmask
.Mask
))
107 KAFFINITY threadMask
= KAFFINITY(1) << threadId
;
108 gmask
.Mask
&= ~threadMask
;
110 if (procGroup
>= threadMaskPerProcGroup
.size())
112 threadMaskPerProcGroup
.resize(procGroup
+ 1);
115 if (threadMaskPerProcGroup
[procGroup
] & threadMask
)
117 // Already seen this mask. This means that we are in 32-bit mode and
118 // have seen more than 32 HW threads for this procGroup
121 SWR_INVALID("Shouldn't get here in 64-bit mode");
126 threadMaskPerProcGroup
[procGroup
] |= (KAFFINITY(1) << threadId
);
130 PROCESSOR_NUMBER procNum
= {};
131 procNum
.Group
= WORD(procGroup
);
132 procNum
.Number
= UCHAR(threadId
);
134 ret
= GetNumaProcessorNodeEx(&procNum
, (PUSHORT
)&numaId
);
138 if (out_nodes
.size() <= numaId
)
140 out_nodes
.resize(numaId
+ 1);
142 auto& numaNode
= out_nodes
[numaId
];
143 numaNode
.numaId
= numaId
;
147 if (nullptr == pCore
)
149 numaNode
.cores
.push_back(Core());
150 pCore
= &numaNode
.cores
.back();
151 pCore
->procGroup
= procGroup
;
153 pCore
->threadIds
.push_back(threadId
);
156 out_numThreadsPerProcGroup
++;
160 pBuffer
= PtrAdd(pBuffer
, pBuffer
->Size
);
166 #elif defined(__linux__) || defined (__gnu_linux__)
168 // Parse /proc/cpuinfo to get full topology
169 std::ifstream
input("/proc/cpuinfo");
172 uint32_t threadId
= uint32_t(-1);
173 uint32_t coreId
= uint32_t(-1);
174 uint32_t numaId
= uint32_t(-1);
176 while (std::getline(input
, line
))
178 if (line
.find("processor") != std::string::npos
)
180 if (threadId
!= uint32_t(-1))
183 if (out_nodes
.size() <= numaId
)
185 out_nodes
.resize(numaId
+ 1);
188 auto& numaNode
= out_nodes
[numaId
];
189 if (numaNode
.cores
.size() <= coreId
)
191 numaNode
.cores
.resize(coreId
+ 1);
194 auto& core
= numaNode
.cores
[coreId
];
195 core
.procGroup
= coreId
;
196 core
.threadIds
.push_back(threadId
);
198 out_numThreadsPerProcGroup
++;
201 auto data_start
= line
.find(": ") + 2;
202 threadId
= std::strtoul(&line
.c_str()[data_start
], &c
, 10);
205 if (line
.find("core id") != std::string::npos
)
207 auto data_start
= line
.find(": ") + 2;
208 coreId
= std::strtoul(&line
.c_str()[data_start
], &c
, 10);
211 if (line
.find("physical id") != std::string::npos
)
213 auto data_start
= line
.find(": ") + 2;
214 numaId
= std::strtoul(&line
.c_str()[data_start
], &c
, 10);
219 if (threadId
!= uint32_t(-1))
222 if (out_nodes
.size() <= numaId
)
224 out_nodes
.resize(numaId
+ 1);
226 auto& numaNode
= out_nodes
[numaId
];
227 numaNode
.numaId
= numaId
;
228 if (numaNode
.cores
.size() <= coreId
)
230 numaNode
.cores
.resize(coreId
+ 1);
232 auto& core
= numaNode
.cores
[coreId
];
234 core
.procGroup
= coreId
;
235 core
.threadIds
.push_back(threadId
);
236 out_numThreadsPerProcGroup
++;
241 #error Unsupported platform
245 // Prune empty cores and numa nodes
246 for (auto node_it
= out_nodes
.begin(); node_it
!= out_nodes
.end(); )
248 // Erase empty cores (first)
249 for (auto core_it
= node_it
->cores
.begin(); core_it
!= node_it
->cores
.end(); )
251 if (core_it
->threadIds
.size() == 0)
253 core_it
= node_it
->cores
.erase(core_it
);
261 // Erase empty numa nodes (second)
262 if (node_it
->cores
.size() == 0)
264 node_it
= out_nodes
.erase(node_it
);
274 void bindThread(SWR_CONTEXT
* pContext
, uint32_t threadId
, uint32_t procGroupId
= 0, bool bindProcGroup
=false)
276 // Only bind threads when MAX_WORKER_THREADS isn't set.
277 if (pContext
->threadInfo
.SINGLE_THREADED
|| (pContext
->threadInfo
.MAX_WORKER_THREADS
&& bindProcGroup
== false))
284 GROUP_AFFINITY affinity
= {};
285 affinity
.Group
= procGroupId
;
290 // Hopefully we don't get here. Logic in CreateThreadPool should prevent this.
291 SWR_INVALID("Shouldn't get here");
293 // In a 32-bit process on Windows it is impossible to bind
294 // to logical processors 32-63 within a processor group.
295 // In this case set the mask to 0 and let the system assign
296 // the processor. Hopefully it will make smart choices.
302 // If MAX_WORKER_THREADS is set, only bind to the proc group,
303 // Not the individual HW thread.
304 if (!pContext
->threadInfo
.MAX_WORKER_THREADS
)
306 affinity
.Mask
= KAFFINITY(1) << threadId
;
310 SetThreadGroupAffinity(GetCurrentThread(), &affinity
, nullptr);
315 pthread_t thread
= pthread_self();
317 CPU_SET(threadId
, &cpuset
);
319 pthread_setaffinity_np(thread
, sizeof(cpu_set_t
), &cpuset
);
325 uint32_t GetEnqueuedDraw(SWR_CONTEXT
*pContext
)
327 return pContext
->dcRing
.GetHead();
331 DRAW_CONTEXT
*GetDC(SWR_CONTEXT
*pContext
, uint32_t drawId
)
333 return &pContext
->dcRing
[(drawId
-1) % KNOB_MAX_DRAWS_IN_FLIGHT
];
337 bool IDComparesLess(uint32_t a
, uint32_t b
)
339 // Use signed delta to ensure that wrap-around to 0 is correctly handled.
340 int32_t delta
= int32_t(a
- b
);
344 // returns true if dependency not met
346 bool CheckDependency(SWR_CONTEXT
*pContext
, DRAW_CONTEXT
*pDC
, uint32_t lastRetiredDraw
)
348 return pDC
->dependent
&& IDComparesLess(lastRetiredDraw
, pDC
->drawId
- 1);
351 bool CheckDependencyFE(SWR_CONTEXT
*pContext
, DRAW_CONTEXT
*pDC
, uint32_t lastRetiredDraw
)
353 return pDC
->dependentFE
&& IDComparesLess(lastRetiredDraw
, pDC
->drawId
- 1);
356 //////////////////////////////////////////////////////////////////////////
357 /// @brief Update client stats.
358 INLINE
void UpdateClientStats(SWR_CONTEXT
* pContext
, uint32_t workerId
, DRAW_CONTEXT
* pDC
)
360 if ((pContext
->pfnUpdateStats
== nullptr) || (GetApiState(pDC
).enableStatsBE
== false))
365 DRAW_DYNAMIC_STATE
& dynState
= pDC
->dynState
;
366 OSALIGNLINE(SWR_STATS
) stats
{ 0 };
368 // Sum up stats across all workers before sending to client.
369 for (uint32_t i
= 0; i
< pContext
->NumWorkerThreads
; ++i
)
371 stats
.DepthPassCount
+= dynState
.pStats
[i
].DepthPassCount
;
373 stats
.PsInvocations
+= dynState
.pStats
[i
].PsInvocations
;
374 stats
.CsInvocations
+= dynState
.pStats
[i
].CsInvocations
;
378 pContext
->pfnUpdateStats(GetPrivateState(pDC
), &stats
);
381 INLINE
void ExecuteCallbacks(SWR_CONTEXT
* pContext
, uint32_t workerId
, DRAW_CONTEXT
* pDC
)
383 UpdateClientStats(pContext
, workerId
, pDC
);
385 if (pDC
->retireCallback
.pfnCallbackFunc
)
387 pDC
->retireCallback
.pfnCallbackFunc(pDC
->retireCallback
.userData
,
388 pDC
->retireCallback
.userData2
,
389 pDC
->retireCallback
.userData3
);
393 // inlined-only version
394 INLINE
int32_t CompleteDrawContextInl(SWR_CONTEXT
* pContext
, uint32_t workerId
, DRAW_CONTEXT
* pDC
)
396 int32_t result
= InterlockedDecrement((volatile long*)&pDC
->threadsDone
);
397 SWR_ASSERT(result
>= 0);
399 AR_FLUSH(pDC
->drawId
);
403 ExecuteCallbacks(pContext
, workerId
, pDC
);
405 // Cleanup memory allocations
406 pDC
->pArena
->Reset(true);
409 pDC
->pTileMgr
->initialize();
411 if (pDC
->cleanupState
)
413 pDC
->pState
->pArena
->Reset(true);
418 pContext
->dcRing
.Dequeue(); // Remove from tail
424 // available to other translation modules
425 int32_t CompleteDrawContext(SWR_CONTEXT
* pContext
, DRAW_CONTEXT
* pDC
)
427 return CompleteDrawContextInl(pContext
, 0, pDC
);
430 INLINE
bool FindFirstIncompleteDraw(SWR_CONTEXT
* pContext
, uint32_t workerId
, uint32_t& curDrawBE
, uint32_t& drawEnqueued
)
432 // increment our current draw id to the first incomplete draw
433 drawEnqueued
= GetEnqueuedDraw(pContext
);
434 while (IDComparesLess(curDrawBE
, drawEnqueued
))
436 DRAW_CONTEXT
*pDC
= &pContext
->dcRing
[curDrawBE
% KNOB_MAX_DRAWS_IN_FLIGHT
];
438 // If its not compute and FE is not done then break out of loop.
439 if (!pDC
->doneFE
&& !pDC
->isCompute
) break;
441 bool isWorkComplete
= pDC
->isCompute
?
442 pDC
->pDispatch
->isWorkComplete() :
443 pDC
->pTileMgr
->isWorkComplete();
448 CompleteDrawContextInl(pContext
, workerId
, pDC
);
456 // If there are no more incomplete draws then return false.
457 return IDComparesLess(curDrawBE
, drawEnqueued
);
460 //////////////////////////////////////////////////////////////////////////
461 /// @brief If there is any BE work then go work on it.
462 /// @param pContext - pointer to SWR context.
463 /// @param workerId - The unique worker ID that is assigned to this thread.
464 /// @param curDrawBE - This tracks the draw contexts that this thread has processed. Each worker thread
465 /// has its own curDrawBE counter and this ensures that each worker processes all the
467 /// @param lockedTiles - This is the set of tiles locked by other threads. Each thread maintains its
468 /// own set and each time it fails to lock a macrotile, because its already locked,
469 /// then it will add that tile to the lockedTiles set. As a worker begins to work
470 /// on future draws the lockedTiles ensure that it doesn't work on tiles that may
471 /// still have work pending in a previous draw. Additionally, the lockedTiles is
472 /// hueristic that can steer a worker back to the same macrotile that it had been
473 /// working on in a previous draw.
474 /// @returns true if worker thread should shutdown
476 SWR_CONTEXT
*pContext
,
479 TileSet
& lockedTiles
,
483 bool bShutdown
= false;
485 // Find the first incomplete draw that has pending work. If no such draw is found then
486 // return. FindFirstIncompleteDraw is responsible for incrementing the curDrawBE.
487 uint32_t drawEnqueued
= 0;
488 if (FindFirstIncompleteDraw(pContext
, workerId
, curDrawBE
, drawEnqueued
) == false)
493 uint32_t lastRetiredDraw
= pContext
->dcRing
[curDrawBE
% KNOB_MAX_DRAWS_IN_FLIGHT
].drawId
- 1;
495 // Reset our history for locked tiles. We'll have to re-learn which tiles are locked.
498 // Try to work on each draw in order of the available draws in flight.
499 // 1. If we're on curDrawBE, we can work on any macrotile that is available.
500 // 2. If we're trying to work on draws after curDrawBE, we are restricted to
501 // working on those macrotiles that are known to be complete in the prior draw to
502 // maintain order. The locked tiles provides the history to ensures this.
503 for (uint32_t i
= curDrawBE
; IDComparesLess(i
, drawEnqueued
); ++i
)
505 DRAW_CONTEXT
*pDC
= &pContext
->dcRing
[i
% KNOB_MAX_DRAWS_IN_FLIGHT
];
507 if (pDC
->isCompute
) return false; // We don't look at compute work.
509 // First wait for FE to be finished with this draw. This keeps threading model simple
510 // but if there are lots of bubbles between draws then serializing FE and BE may
511 // need to be revisited.
512 if (!pDC
->doneFE
) return false;
514 // If this draw is dependent on a previous draw then we need to bail.
515 if (CheckDependency(pContext
, pDC
, lastRetiredDraw
))
520 // Grab the list of all dirty macrotiles. A tile is dirty if it has work queued to it.
521 auto ¯oTiles
= pDC
->pTileMgr
->getDirtyTiles();
523 for (auto tile
: macroTiles
)
525 uint32_t tileID
= tile
->mId
;
527 // Only work on tiles for this numa node
529 pDC
->pTileMgr
->getTileIndices(tileID
, x
, y
);
530 if (((x
^ y
) & numaMask
) != numaNode
)
535 if (!tile
->getNumQueued())
540 // can only work on this draw if it's not in use by other threads
541 if (lockedTiles
.find(tileID
) != lockedTiles
.end())
550 AR_BEGIN(WorkerFoundWork
, pDC
->drawId
);
552 uint32_t numWorkItems
= tile
->getNumQueued();
553 SWR_ASSERT(numWorkItems
);
555 pWork
= tile
->peek();
557 if (pWork
->type
== DRAW
)
559 pContext
->pHotTileMgr
->InitializeHotTiles(pContext
, pDC
, workerId
, tileID
);
561 else if (pWork
->type
== SHUTDOWN
)
566 while ((pWork
= tile
->peek()) != nullptr)
568 pWork
->pfnWork(pDC
, workerId
, tileID
, &pWork
->desc
);
571 AR_END(WorkerFoundWork
, numWorkItems
);
575 pDC
->pTileMgr
->markTileComplete(tileID
);
577 // Optimization: If the draw is complete and we're the last one to have worked on it then
578 // we can reset the locked list as we know that all previous draws before the next are guaranteed to be complete.
579 if ((curDrawBE
== i
) && (bShutdown
|| pDC
->pTileMgr
->isWorkComplete()))
581 // We can increment the current BE and safely move to next draw since we know this draw is complete.
583 CompleteDrawContextInl(pContext
, workerId
, pDC
);
598 // This tile is already locked. So let's add it to our locked tiles set. This way we don't try locking this one again.
599 lockedTiles
.insert(tileID
);
607 //////////////////////////////////////////////////////////////////////////
608 /// @brief Called when FE work is complete for this DC.
609 INLINE
void CompleteDrawFE(SWR_CONTEXT
* pContext
, uint32_t workerId
, DRAW_CONTEXT
* pDC
)
611 if (pContext
->pfnUpdateStatsFE
&& GetApiState(pDC
).enableStatsFE
)
613 SWR_STATS_FE
& stats
= pDC
->dynState
.statsFE
;
615 AR_EVENT(FrontendStatsEvent(pDC
->drawId
,
616 stats
.IaVertices
, stats
.IaPrimitives
, stats
.VsInvocations
, stats
.HsInvocations
,
617 stats
.DsInvocations
, stats
.GsInvocations
, stats
.GsPrimitives
, stats
.CInvocations
, stats
.CPrimitives
,
618 stats
.SoPrimStorageNeeded
[0], stats
.SoPrimStorageNeeded
[1], stats
.SoPrimStorageNeeded
[2], stats
.SoPrimStorageNeeded
[3],
619 stats
.SoNumPrimsWritten
[0], stats
.SoNumPrimsWritten
[1], stats
.SoNumPrimsWritten
[2], stats
.SoNumPrimsWritten
[3]
621 AR_EVENT(FrontendDrawEndEvent(pDC
->drawId
));
623 pContext
->pfnUpdateStatsFE(GetPrivateState(pDC
), &stats
);
626 if (pContext
->pfnUpdateSoWriteOffset
)
628 for (uint32_t i
= 0; i
< MAX_SO_BUFFERS
; ++i
)
630 if ((pDC
->dynState
.SoWriteOffsetDirty
[i
]) &&
631 (pDC
->pState
->state
.soBuffer
[i
].soWriteEnable
))
633 pContext
->pfnUpdateSoWriteOffset(GetPrivateState(pDC
), i
, pDC
->dynState
.SoWriteOffset
[i
]);
638 // Ensure all streaming writes are globally visible before marking this FE done
642 InterlockedDecrement((volatile long*)&pContext
->drawsOutstandingFE
);
645 void WorkOnFifoFE(SWR_CONTEXT
*pContext
, uint32_t workerId
, uint32_t &curDrawFE
)
647 // Try to grab the next DC from the ring
648 uint32_t drawEnqueued
= GetEnqueuedDraw(pContext
);
649 while (IDComparesLess(curDrawFE
, drawEnqueued
))
651 uint32_t dcSlot
= curDrawFE
% KNOB_MAX_DRAWS_IN_FLIGHT
;
652 DRAW_CONTEXT
*pDC
= &pContext
->dcRing
[dcSlot
];
653 if (pDC
->isCompute
|| pDC
->doneFE
)
655 CompleteDrawContextInl(pContext
, workerId
, pDC
);
664 uint32_t lastRetiredFE
= curDrawFE
- 1;
665 uint32_t curDraw
= curDrawFE
;
666 while (IDComparesLess(curDraw
, drawEnqueued
))
668 uint32_t dcSlot
= curDraw
% KNOB_MAX_DRAWS_IN_FLIGHT
;
669 DRAW_CONTEXT
*pDC
= &pContext
->dcRing
[dcSlot
];
671 if (!pDC
->isCompute
&& !pDC
->FeLock
)
673 if (CheckDependencyFE(pContext
, pDC
, lastRetiredFE
))
678 uint32_t initial
= InterlockedCompareExchange((volatile uint32_t*)&pDC
->FeLock
, 1, 0);
681 // successfully grabbed the DC, now run the FE
682 pDC
->FeWork
.pfnWork(pContext
, pDC
, workerId
, &pDC
->FeWork
.desc
);
684 CompleteDrawFE(pContext
, workerId
, pDC
);
691 //////////////////////////////////////////////////////////////////////////
692 /// @brief If there is any compute work then go work on it.
693 /// @param pContext - pointer to SWR context.
694 /// @param workerId - The unique worker ID that is assigned to this thread.
695 /// @param curDrawBE - This tracks the draw contexts that this thread has processed. Each worker thread
696 /// has its own curDrawBE counter and this ensures that each worker processes all the
699 SWR_CONTEXT
*pContext
,
703 uint32_t drawEnqueued
= 0;
704 if (FindFirstIncompleteDraw(pContext
, workerId
, curDrawBE
, drawEnqueued
) == false)
709 uint32_t lastRetiredDraw
= pContext
->dcRing
[curDrawBE
% KNOB_MAX_DRAWS_IN_FLIGHT
].drawId
- 1;
711 for (uint64_t i
= curDrawBE
; IDComparesLess(i
, drawEnqueued
); ++i
)
713 DRAW_CONTEXT
*pDC
= &pContext
->dcRing
[i
% KNOB_MAX_DRAWS_IN_FLIGHT
];
714 if (pDC
->isCompute
== false) return;
716 // check dependencies
717 if (CheckDependency(pContext
, pDC
, lastRetiredDraw
))
722 SWR_ASSERT(pDC
->pDispatch
!= nullptr);
723 DispatchQueue
& queue
= *pDC
->pDispatch
;
725 // Is there any work remaining?
726 if (queue
.getNumQueued() > 0)
728 void* pSpillFillBuffer
= nullptr;
729 void* pScratchSpace
= nullptr;
730 uint32_t threadGroupId
= 0;
731 while (queue
.getWork(threadGroupId
))
733 queue
.dispatch(pDC
, workerId
, threadGroupId
, pSpillFillBuffer
, pScratchSpace
);
734 queue
.finishedWork();
737 // Ensure all streaming writes are globally visible before moving onto the next draw
743 template<bool IsFEThread
, bool IsBEThread
>
744 DWORD
workerThreadMain(LPVOID pData
)
746 THREAD_DATA
*pThreadData
= (THREAD_DATA
*)pData
;
747 SWR_CONTEXT
*pContext
= pThreadData
->pContext
;
748 uint32_t threadId
= pThreadData
->threadId
;
749 uint32_t workerId
= pThreadData
->workerId
;
751 bindThread(pContext
, threadId
, pThreadData
->procGroupId
, pThreadData
->forceBindProcGroup
);
755 sprintf_s(threadName
,
757 "SWRWorker_%02d_NUMA%d_Core%02d_T%d",
759 // linux pthread name limited to 16 chars (including \0)
760 "w%03d-n%d-c%03d-t%d",
762 workerId
, pThreadData
->numaId
, pThreadData
->coreId
, pThreadData
->htId
);
763 SetCurrentThreadName(threadName
);
766 RDTSC_INIT(threadId
);
768 uint32_t numaNode
= pThreadData
->numaId
;
769 uint32_t numaMask
= pContext
->threadPool
.numaMask
;
771 // flush denormals to 0
772 _mm_setcsr(_mm_getcsr() | _MM_FLUSH_ZERO_ON
| _MM_DENORMALS_ZERO_ON
);
774 // Track tiles locked by other threads. If we try to lock a macrotile and find its already
775 // locked then we'll add it to this list so that we don't try and lock it again.
778 // each worker has the ability to work on any of the queued draws as long as certain
779 // conditions are met. the data associated
780 // with a draw is guaranteed to be active as long as a worker hasn't signaled that he
781 // has moved on to the next draw when he determines there is no more work to do. The api
782 // thread will not increment the head of the dc ring until all workers have moved past the
784 // the logic to determine what to work on is:
785 // 1- try to work on the FE any draw that is queued. For now there are no dependencies
786 // on the FE work, so any worker can grab any FE and process in parallel. Eventually
787 // we'll need dependency tracking to force serialization on FEs. The worker will try
788 // to pick an FE by atomically incrementing a counter in the swr context. he'll keep
789 // trying until he reaches the tail.
790 // 2- BE work must be done in strict order. we accomplish this today by pulling work off
791 // the oldest draw (ie the head) of the dcRing. the worker can determine if there is
792 // any work left by comparing the total # of binned work items and the total # of completed
793 // work items. If they are equal, then there is no more work to do for this draw, and
794 // the worker can safely increment its oldestDraw counter and move on to the next draw.
795 std::unique_lock
<std::mutex
> lock(pContext
->WaitLock
, std::defer_lock
);
797 auto threadHasWork
= [&](uint32_t curDraw
) { return curDraw
!= pContext
->dcRing
.GetHead(); };
799 uint32_t curDrawBE
= 0;
800 uint32_t curDrawFE
= 0;
802 bool bShutdown
= false;
806 if (bShutdown
&& !threadHasWork(curDrawBE
))
812 while (loop
++ < KNOB_WORKER_SPIN_LOOP_COUNT
&& !threadHasWork(curDrawBE
))
817 if (!threadHasWork(curDrawBE
))
821 // check for thread idle condition again under lock
822 if (threadHasWork(curDrawBE
))
828 pContext
->FifosNotEmpty
.wait(lock
);
834 AR_BEGIN(WorkerWorkOnFifoBE
, 0);
835 bShutdown
|= WorkOnFifoBE(pContext
, workerId
, curDrawBE
, lockedTiles
, numaNode
, numaMask
);
836 AR_END(WorkerWorkOnFifoBE
, 0);
838 WorkOnCompute(pContext
, workerId
, curDrawBE
);
843 WorkOnFifoFE(pContext
, workerId
, curDrawFE
);
847 curDrawBE
= curDrawFE
;
854 template<> DWORD workerThreadMain
<false, false>(LPVOID
) = delete;
856 template <bool IsFEThread
, bool IsBEThread
>
857 DWORD
workerThreadInit(LPVOID pData
)
863 return workerThreadMain
<IsFEThread
, IsBEThread
>(pData
);
867 __except(EXCEPTION_CONTINUE_SEARCH
)
875 template<> DWORD workerThreadInit
<false, false>(LPVOID pData
) = delete;
877 //////////////////////////////////////////////////////////////////////////
878 /// @brief Creates thread pool info but doesn't launch threads.
879 /// @param pContext - pointer to context
880 /// @param pPool - pointer to thread pool object.
881 void CreateThreadPool(SWR_CONTEXT
* pContext
, THREAD_POOL
* pPool
)
883 bindThread(pContext
, 0);
886 uint32_t numThreadsPerProcGroup
= 0;
887 CalculateProcessorTopology(nodes
, numThreadsPerProcGroup
);
889 uint32_t numHWNodes
= (uint32_t)nodes
.size();
890 uint32_t numHWCoresPerNode
= (uint32_t)nodes
[0].cores
.size();
891 uint32_t numHWHyperThreads
= (uint32_t)nodes
[0].cores
[0].threadIds
.size();
893 // Calculate num HW threads. Due to asymmetric topologies, this is not
894 // a trivial multiplication.
895 uint32_t numHWThreads
= 0;
896 for (auto& node
: nodes
)
898 for (auto& core
: node
.cores
)
900 numHWThreads
+= (uint32_t)core
.threadIds
.size();
904 uint32_t numNodes
= numHWNodes
;
905 uint32_t numCoresPerNode
= numHWCoresPerNode
;
906 uint32_t numHyperThreads
= numHWHyperThreads
;
908 if (pContext
->threadInfo
.MAX_NUMA_NODES
)
910 numNodes
= std::min(numNodes
, pContext
->threadInfo
.MAX_NUMA_NODES
);
913 if (pContext
->threadInfo
.MAX_CORES_PER_NUMA_NODE
)
915 numCoresPerNode
= std::min(numCoresPerNode
, pContext
->threadInfo
.MAX_CORES_PER_NUMA_NODE
);
918 if (pContext
->threadInfo
.MAX_THREADS_PER_CORE
)
920 numHyperThreads
= std::min(numHyperThreads
, pContext
->threadInfo
.MAX_THREADS_PER_CORE
);
923 #if defined(_WIN32) && !defined(_WIN64)
924 if (!pContext
->threadInfo
.MAX_WORKER_THREADS
)
926 // Limit 32-bit windows to bindable HW threads only
927 if ((numCoresPerNode
* numHWHyperThreads
) > 32)
929 numCoresPerNode
= 32 / numHWHyperThreads
;
934 // Calculate numThreads
935 uint32_t numThreads
= numNodes
* numCoresPerNode
* numHyperThreads
;
936 numThreads
= std::min(numThreads
, numHWThreads
);
938 if (pContext
->threadInfo
.MAX_WORKER_THREADS
)
940 uint32_t maxHWThreads
= numHWNodes
* numHWCoresPerNode
* numHWHyperThreads
;
941 numThreads
= std::min(pContext
->threadInfo
.MAX_WORKER_THREADS
, maxHWThreads
);
944 uint32_t numAPIReservedThreads
= 1;
949 // If only 1 worker threads, try to move it to an available
950 // HW thread. If that fails, use the API thread.
951 if (numCoresPerNode
< numHWCoresPerNode
)
955 else if (numHyperThreads
< numHWHyperThreads
)
959 else if (numNodes
< numHWNodes
)
965 pContext
->threadInfo
.SINGLE_THREADED
= true;
970 // Save HW threads for the API if we can
971 if (numThreads
> numAPIReservedThreads
)
973 numThreads
-= numAPIReservedThreads
;
977 numAPIReservedThreads
= 0;
981 if (pContext
->threadInfo
.SINGLE_THREADED
)
986 // Initialize DRAW_CONTEXT's per-thread stats
987 for (uint32_t dc
= 0; dc
< KNOB_MAX_DRAWS_IN_FLIGHT
; ++dc
)
989 pContext
->dcRing
[dc
].dynState
.pStats
= (SWR_STATS
*)AlignedMalloc(sizeof(SWR_STATS
) * numThreads
, 64);
990 memset(pContext
->dcRing
[dc
].dynState
.pStats
, 0, sizeof(SWR_STATS
) * numThreads
);
993 if (pContext
->threadInfo
.SINGLE_THREADED
)
995 pContext
->NumWorkerThreads
= 1;
996 pContext
->NumFEThreads
= 1;
997 pContext
->NumBEThreads
= 1;
998 pPool
->numThreads
= 0;
1003 pPool
->numThreads
= numThreads
;
1004 pContext
->NumWorkerThreads
= pPool
->numThreads
;
1006 pPool
->pThreadData
= (THREAD_DATA
*)malloc(pPool
->numThreads
* sizeof(THREAD_DATA
));
1007 pPool
->numaMask
= 0;
1009 pPool
->pThreads
= new THREAD_PTR
[pPool
->numThreads
];
1011 if (pContext
->threadInfo
.MAX_WORKER_THREADS
)
1013 bool bForceBindProcGroup
= (numThreads
> numThreadsPerProcGroup
);
1014 uint32_t numProcGroups
= (numThreads
+ numThreadsPerProcGroup
- 1) / numThreadsPerProcGroup
;
1015 // When MAX_WORKER_THREADS is set we don't bother to bind to specific HW threads
1016 // But Windows will still require binding to specific process groups
1017 for (uint32_t workerId
= 0; workerId
< numThreads
; ++workerId
)
1019 pPool
->pThreadData
[workerId
].workerId
= workerId
;
1020 pPool
->pThreadData
[workerId
].procGroupId
= workerId
% numProcGroups
;
1021 pPool
->pThreadData
[workerId
].threadId
= 0;
1022 pPool
->pThreadData
[workerId
].numaId
= 0;
1023 pPool
->pThreadData
[workerId
].coreId
= 0;
1024 pPool
->pThreadData
[workerId
].htId
= 0;
1025 pPool
->pThreadData
[workerId
].pContext
= pContext
;
1026 pPool
->pThreadData
[workerId
].forceBindProcGroup
= bForceBindProcGroup
;
1028 pContext
->NumBEThreads
++;
1029 pContext
->NumFEThreads
++;
1034 pPool
->numaMask
= numNodes
- 1; // Only works for 2**n numa nodes (1, 2, 4, etc.)
1036 uint32_t workerId
= 0;
1037 for (uint32_t n
= 0; n
< numNodes
; ++n
)
1039 auto& node
= nodes
[n
];
1040 uint32_t numCores
= numCoresPerNode
;
1041 for (uint32_t c
= 0; c
< numCores
; ++c
)
1043 if (c
>= node
.cores
.size())
1048 auto& core
= node
.cores
[c
];
1049 for (uint32_t t
= 0; t
< numHyperThreads
; ++t
)
1051 if (t
>= core
.threadIds
.size())
1056 if (numAPIReservedThreads
)
1058 --numAPIReservedThreads
;
1062 SWR_ASSERT(workerId
< numThreads
);
1064 pPool
->pThreadData
[workerId
].workerId
= workerId
;
1065 pPool
->pThreadData
[workerId
].procGroupId
= core
.procGroup
;
1066 pPool
->pThreadData
[workerId
].threadId
= core
.threadIds
[t
];
1067 pPool
->pThreadData
[workerId
].numaId
= node
.numaId
;
1068 pPool
->pThreadData
[workerId
].coreId
= c
;
1069 pPool
->pThreadData
[workerId
].htId
= t
;
1070 pPool
->pThreadData
[workerId
].pContext
= pContext
;
1072 pContext
->NumBEThreads
++;
1073 pContext
->NumFEThreads
++;
1079 SWR_ASSERT(workerId
== pContext
->NumWorkerThreads
);
1083 //////////////////////////////////////////////////////////////////////////
1084 /// @brief Launches worker threads in thread pool.
1085 /// @param pContext - pointer to context
1086 /// @param pPool - pointer to thread pool object.
1087 void StartThreadPool(SWR_CONTEXT
* pContext
, THREAD_POOL
* pPool
)
1089 if (pContext
->threadInfo
.SINGLE_THREADED
)
1094 for (uint32_t workerId
= 0; workerId
< pContext
->NumWorkerThreads
; ++workerId
)
1096 pPool
->pThreads
[workerId
] = new std::thread(workerThreadInit
<true, true>, &pPool
->pThreadData
[workerId
]);
1100 //////////////////////////////////////////////////////////////////////////
1101 /// @brief Destroys thread pool.
1102 /// @param pContext - pointer to context
1103 /// @param pPool - pointer to thread pool object.
1104 void DestroyThreadPool(SWR_CONTEXT
*pContext
, THREAD_POOL
*pPool
)
1106 if (!pContext
->threadInfo
.SINGLE_THREADED
)
1108 // Wait for all threads to finish
1109 SwrWaitForIdle(pContext
);
1111 // Wait for threads to finish and destroy them
1112 for (uint32_t t
= 0; t
< pPool
->numThreads
; ++t
)
1114 // Detach from thread. Cannot join() due to possibility (in Windows) of code
1115 // in some DLLMain(THREAD_DETATCH case) blocking the thread until after this returns.
1116 pPool
->pThreads
[t
]->detach();
1117 delete(pPool
->pThreads
[t
]);
1120 delete [] pPool
->pThreads
;
1122 // Clean up data used by threads
1123 free(pPool
->pThreadData
);