//
// Design doc at http://golang.org/s/go11sched.
-typedef struct Sched Sched;
-struct Sched {
- Lock;
-
- uint64 goidgen;
- M* midle; // idle m's waiting for work
- int32 nmidle; // number of idle m's waiting for work
- int32 nmidlelocked; // number of locked m's waiting for work
- int32 mcount; // number of m's that have been created
- int32 maxmcount; // maximum number of m's allowed (or die)
-
- P* pidle; // idle P's
- uint32 npidle;
- uint32 nmspinning;
-
- // Global runnable queue.
- G* runqhead;
- G* runqtail;
- int32 runqsize;
-
- // Global cache of dead G's.
- Lock gflock;
- G* gfree;
-
- uint32 gcwaiting; // gc is waiting to run
- int32 stopwait;
- Note stopnote;
- uint32 sysmonwait;
- Note sysmonnote;
- uint64 lastpoll;
-
- int32 profilehz; // cpu profiling rate
-};
+typedef struct schedt Sched;
enum
{
- // Number of goroutine ids to grab from runtime_sched.goidgen to local per-P cache at once.
+ // Number of goroutine ids to grab from runtime_sched->goidgen to local per-P cache at once.
// 16 seems to provide enough amortization, but other than that it's mostly arbitrary number.
GoidCacheBatch = 16,
};
-Sched runtime_sched;
+extern Sched* runtime_getsched() __asm__ (GOSYM_PREFIX "runtime.getsched");
+
+static Sched* runtime_sched;
int32 runtime_gomaxprocs;
uint32 runtime_needextram = 1;
M runtime_m0;
const byte *p;
Eface i;
+ runtime_sched = runtime_getsched();
+
m = &runtime_m0;
g = &runtime_g0;
m->g0 = g;
initcontext();
- runtime_sched.maxmcount = 10000;
+ runtime_sched->maxmcount = 10000;
runtime_precisestack = 0;
// runtime_symtabinit();
runtime_goenvs();
runtime_parsedebugvars();
- runtime_sched.lastpoll = runtime_nanotime();
+ runtime_sched->lastpoll = runtime_nanotime();
procs = 1;
s = runtime_getenv("GOMAXPROCS");
p = s.str;
checkmcount(void)
{
// sched lock is held
- if(runtime_sched.mcount > runtime_sched.maxmcount) {
- runtime_printf("runtime: program exceeds %d-thread limit\n", runtime_sched.maxmcount);
+ if(runtime_sched->mcount > runtime_sched->maxmcount) {
+ runtime_printf("runtime: program exceeds %d-thread limit\n", runtime_sched->maxmcount);
runtime_throw("thread exhaustion");
}
}
mp->fastrand = 0x49f6428aUL + mp->id + runtime_cputicks();
- runtime_lock(&runtime_sched);
- mp->id = runtime_sched.mcount++;
+ runtime_lock(&runtime_sched->lock);
+ mp->id = runtime_sched->mcount++;
checkmcount();
runtime_mpreinit(mp);
// runtime_NumCgoCall() iterates over allm w/o schedlock,
// so we need to publish it safely.
runtime_atomicstorep(&runtime_allm, mp);
- runtime_unlock(&runtime_sched);
+ runtime_unlock(&runtime_sched->lock);
}
// Mark gp ready to run.
}
gp->atomicstatus = _Grunnable;
runqput((P*)g->m->p, gp);
- if(runtime_atomicload(&runtime_sched.npidle) != 0 && runtime_atomicload(&runtime_sched.nmspinning) == 0) // TODO: fast atomic
+ if(runtime_atomicload(&runtime_sched->npidle) != 0 && runtime_atomicload(&runtime_sched->nmspinning) == 0) // TODO: fast atomic
wakep();
g->m->locks--;
}
// Figure out how many CPUs to use during GC.
// Limited by gomaxprocs, number of actual CPUs, and MaxGcproc.
- runtime_lock(&runtime_sched);
+ runtime_lock(&runtime_sched->lock);
n = runtime_gomaxprocs;
if(n > runtime_ncpu)
n = runtime_ncpu > 0 ? runtime_ncpu : 1;
if(n > MaxGcproc)
n = MaxGcproc;
- if(n > runtime_sched.nmidle+1) // one M is currently running
- n = runtime_sched.nmidle+1;
- runtime_unlock(&runtime_sched);
+ if(n > runtime_sched->nmidle+1) // one M is currently running
+ n = runtime_sched->nmidle+1;
+ runtime_unlock(&runtime_sched->lock);
return n;
}
{
int32 n;
- runtime_lock(&runtime_sched);
+ runtime_lock(&runtime_sched->lock);
n = runtime_gomaxprocs;
if(n > runtime_ncpu)
n = runtime_ncpu;
if(n > MaxGcproc)
n = MaxGcproc;
- n -= runtime_sched.nmidle+1; // one M is currently running
- runtime_unlock(&runtime_sched);
+ n -= runtime_sched->nmidle+1; // one M is currently running
+ runtime_unlock(&runtime_sched->lock);
return n > 0;
}
M *mp;
int32 n, pos;
- runtime_lock(&runtime_sched);
+ runtime_lock(&runtime_sched->lock);
pos = 0;
for(n = 1; n < nproc; n++) { // one M is currently running
if(runtime_allp[pos]->mcache == g->m->mcache)
pos++;
runtime_notewakeup(&mp->park);
}
- runtime_unlock(&runtime_sched);
+ runtime_unlock(&runtime_sched->lock);
}
// Similar to stoptheworld but best-effort and can be called several times.
// so try several times
for(i = 0; i < 5; i++) {
// this should tell the scheduler to not start any new goroutines
- runtime_sched.stopwait = 0x7fffffff;
- runtime_atomicstore((uint32*)&runtime_sched.gcwaiting, 1);
+ runtime_sched->stopwait = 0x7fffffff;
+ runtime_atomicstore((uint32*)&runtime_sched->gcwaiting, 1);
// this should stop running goroutines
if(!preemptall())
break; // no running goroutines
P *p;
bool wait;
- runtime_lock(&runtime_sched);
- runtime_sched.stopwait = runtime_gomaxprocs;
- runtime_atomicstore((uint32*)&runtime_sched.gcwaiting, 1);
+ runtime_lock(&runtime_sched->lock);
+ runtime_sched->stopwait = runtime_gomaxprocs;
+ runtime_atomicstore((uint32*)&runtime_sched->gcwaiting, 1);
preemptall();
// stop current P
((P*)g->m->p)->status = _Pgcstop;
- runtime_sched.stopwait--;
+ runtime_sched->stopwait--;
// try to retake all P's in _Psyscall status
for(i = 0; i < runtime_gomaxprocs; i++) {
p = runtime_allp[i];
s = p->status;
if(s == _Psyscall && runtime_cas(&p->status, s, _Pgcstop))
- runtime_sched.stopwait--;
+ runtime_sched->stopwait--;
}
// stop idle P's
while((p = pidleget()) != nil) {
p->status = _Pgcstop;
- runtime_sched.stopwait--;
+ runtime_sched->stopwait--;
}
- wait = runtime_sched.stopwait > 0;
- runtime_unlock(&runtime_sched);
+ wait = runtime_sched->stopwait > 0;
+ runtime_unlock(&runtime_sched->lock);
// wait for remaining P's to stop voluntarily
if(wait) {
- runtime_notesleep(&runtime_sched.stopnote);
- runtime_noteclear(&runtime_sched.stopnote);
+ runtime_notesleep(&runtime_sched->stopnote);
+ runtime_noteclear(&runtime_sched->stopnote);
}
- if(runtime_sched.stopwait)
+ if(runtime_sched->stopwait)
runtime_throw("stoptheworld: not stopped");
for(i = 0; i < runtime_gomaxprocs; i++) {
p = runtime_allp[i];
gp = runtime_netpoll(false); // non-blocking
injectglist(gp);
add = needaddgcproc();
- runtime_lock(&runtime_sched);
+ runtime_lock(&runtime_sched->lock);
if(newprocs) {
procresize(newprocs);
newprocs = 0;
} else
procresize(runtime_gomaxprocs);
- runtime_sched.gcwaiting = 0;
+ runtime_sched->gcwaiting = 0;
p1 = nil;
while((p = pidleget()) != nil) {
p->link = (uintptr)p1;
p1 = p;
}
- if(runtime_sched.sysmonwait) {
- runtime_sched.sysmonwait = false;
- runtime_notewakeup(&runtime_sched.sysmonnote);
+ if(runtime_sched->sysmonwait) {
+ runtime_sched->sysmonwait = false;
+ runtime_notewakeup(&runtime_sched->sysmonnote);
}
- runtime_unlock(&runtime_sched);
+ runtime_unlock(&runtime_sched->lock);
while(p1) {
p = p1;
mp->locked = _LockInternal;
mp->lockedg = gp;
gp->lockedm = mp;
- gp->goid = runtime_xadd64(&runtime_sched.goidgen, 1);
+ gp->goid = runtime_xadd64(&runtime_sched->goidgen, 1);
// put on allg for garbage collector
allgadd(gp);
runtime_throw("stopm holding p");
if(m->spinning) {
m->spinning = false;
- runtime_xadd(&runtime_sched.nmspinning, -1);
+ runtime_xadd(&runtime_sched->nmspinning, -1);
}
retry:
- runtime_lock(&runtime_sched);
+ runtime_lock(&runtime_sched->lock);
mput(m);
- runtime_unlock(&runtime_sched);
+ runtime_unlock(&runtime_sched->lock);
runtime_notesleep(&m->park);
m = g->m;
runtime_noteclear(&m->park);
M *mp;
void (*fn)(void);
- runtime_lock(&runtime_sched);
+ runtime_lock(&runtime_sched->lock);
if(p == nil) {
p = pidleget();
if(p == nil) {
- runtime_unlock(&runtime_sched);
+ runtime_unlock(&runtime_sched->lock);
if(spinning)
- runtime_xadd(&runtime_sched.nmspinning, -1);
+ runtime_xadd(&runtime_sched->nmspinning, -1);
return;
}
}
mp = mget();
- runtime_unlock(&runtime_sched);
+ runtime_unlock(&runtime_sched->lock);
if(mp == nil) {
fn = nil;
if(spinning)
handoffp(P *p)
{
// if it has local work, start it straight away
- if(p->runqhead != p->runqtail || runtime_sched.runqsize) {
+ if(p->runqhead != p->runqtail || runtime_sched->runqsize) {
startm(p, false);
return;
}
// no local work, check that there are no spinning/idle M's,
// otherwise our help is not required
- if(runtime_atomicload(&runtime_sched.nmspinning) + runtime_atomicload(&runtime_sched.npidle) == 0 && // TODO: fast atomic
- runtime_cas(&runtime_sched.nmspinning, 0, 1)) {
+ if(runtime_atomicload(&runtime_sched->nmspinning) + runtime_atomicload(&runtime_sched->npidle) == 0 && // TODO: fast atomic
+ runtime_cas(&runtime_sched->nmspinning, 0, 1)) {
startm(p, true);
return;
}
- runtime_lock(&runtime_sched);
- if(runtime_sched.gcwaiting) {
+ runtime_lock(&runtime_sched->lock);
+ if(runtime_sched->gcwaiting) {
p->status = _Pgcstop;
- if(--runtime_sched.stopwait == 0)
- runtime_notewakeup(&runtime_sched.stopnote);
- runtime_unlock(&runtime_sched);
+ if(--runtime_sched->stopwait == 0)
+ runtime_notewakeup(&runtime_sched->stopnote);
+ runtime_unlock(&runtime_sched->lock);
return;
}
- if(runtime_sched.runqsize) {
- runtime_unlock(&runtime_sched);
+ if(runtime_sched->runqsize) {
+ runtime_unlock(&runtime_sched->lock);
startm(p, false);
return;
}
// If this is the last running P and nobody is polling network,
// need to wakeup another M to poll network.
- if(runtime_sched.npidle == (uint32)runtime_gomaxprocs-1 && runtime_atomicload64(&runtime_sched.lastpoll) != 0) {
- runtime_unlock(&runtime_sched);
+ if(runtime_sched->npidle == (uint32)runtime_gomaxprocs-1 && runtime_atomicload64(&runtime_sched->lastpoll) != 0) {
+ runtime_unlock(&runtime_sched->lock);
startm(p, false);
return;
}
pidleput(p);
- runtime_unlock(&runtime_sched);
+ runtime_unlock(&runtime_sched->lock);
}
// Tries to add one more P to execute G's.
wakep(void)
{
// be conservative about spinning threads
- if(!runtime_cas(&runtime_sched.nmspinning, 0, 1))
+ if(!runtime_cas(&runtime_sched->nmspinning, 0, 1))
return;
startm(nil, true);
}
{
P *p;
- if(!runtime_sched.gcwaiting)
+ if(!runtime_sched->gcwaiting)
runtime_throw("gcstopm: not waiting for gc");
if(g->m->spinning) {
g->m->spinning = false;
- runtime_xadd(&runtime_sched.nmspinning, -1);
+ runtime_xadd(&runtime_sched->nmspinning, -1);
}
p = releasep();
- runtime_lock(&runtime_sched);
+ runtime_lock(&runtime_sched->lock);
p->status = _Pgcstop;
- if(--runtime_sched.stopwait == 0)
- runtime_notewakeup(&runtime_sched.stopnote);
- runtime_unlock(&runtime_sched);
+ if(--runtime_sched->stopwait == 0)
+ runtime_notewakeup(&runtime_sched->stopnote);
+ runtime_unlock(&runtime_sched->lock);
stopm();
}
gp->m = g->m;
// Check whether the profiler needs to be turned on or off.
- hz = runtime_sched.profilehz;
+ hz = runtime_sched->profilehz;
if(g->m->profilehz != hz)
runtime_resetcpuprofiler(hz);
int32 i;
top:
- if(runtime_sched.gcwaiting) {
+ if(runtime_sched->gcwaiting) {
gcstopm();
goto top;
}
if(gp)
return gp;
// global runq
- if(runtime_sched.runqsize) {
- runtime_lock(&runtime_sched);
+ if(runtime_sched->runqsize) {
+ runtime_lock(&runtime_sched->lock);
gp = globrunqget((P*)g->m->p, 0);
- runtime_unlock(&runtime_sched);
+ runtime_unlock(&runtime_sched->lock);
if(gp)
return gp;
}
// If number of spinning M's >= number of busy P's, block.
// This is necessary to prevent excessive CPU consumption
// when GOMAXPROCS>>1 but the program parallelism is low.
- if(!g->m->spinning && 2 * runtime_atomicload(&runtime_sched.nmspinning) >= runtime_gomaxprocs - runtime_atomicload(&runtime_sched.npidle)) // TODO: fast atomic
+ if(!g->m->spinning && 2 * runtime_atomicload(&runtime_sched->nmspinning) >= runtime_gomaxprocs - runtime_atomicload(&runtime_sched->npidle)) // TODO: fast atomic
goto stop;
if(!g->m->spinning) {
g->m->spinning = true;
- runtime_xadd(&runtime_sched.nmspinning, 1);
+ runtime_xadd(&runtime_sched->nmspinning, 1);
}
// random steal from other P's
for(i = 0; i < 2*runtime_gomaxprocs; i++) {
- if(runtime_sched.gcwaiting)
+ if(runtime_sched->gcwaiting)
goto top;
p = runtime_allp[runtime_fastrand1()%runtime_gomaxprocs];
if(p == (P*)g->m->p)
}
stop:
// return P and block
- runtime_lock(&runtime_sched);
- if(runtime_sched.gcwaiting) {
- runtime_unlock(&runtime_sched);
+ runtime_lock(&runtime_sched->lock);
+ if(runtime_sched->gcwaiting) {
+ runtime_unlock(&runtime_sched->lock);
goto top;
}
- if(runtime_sched.runqsize) {
+ if(runtime_sched->runqsize) {
gp = globrunqget((P*)g->m->p, 0);
- runtime_unlock(&runtime_sched);
+ runtime_unlock(&runtime_sched->lock);
return gp;
}
p = releasep();
pidleput(p);
- runtime_unlock(&runtime_sched);
+ runtime_unlock(&runtime_sched->lock);
if(g->m->spinning) {
g->m->spinning = false;
- runtime_xadd(&runtime_sched.nmspinning, -1);
+ runtime_xadd(&runtime_sched->nmspinning, -1);
}
// check all runqueues once again
for(i = 0; i < runtime_gomaxprocs; i++) {
p = runtime_allp[i];
if(p && p->runqhead != p->runqtail) {
- runtime_lock(&runtime_sched);
+ runtime_lock(&runtime_sched->lock);
p = pidleget();
- runtime_unlock(&runtime_sched);
+ runtime_unlock(&runtime_sched->lock);
if(p) {
acquirep(p);
goto top;
}
}
// poll network
- if(runtime_xchg64(&runtime_sched.lastpoll, 0) != 0) {
+ if(runtime_xchg64(&runtime_sched->lastpoll, 0) != 0) {
if(g->m->p)
runtime_throw("findrunnable: netpoll with p");
if(g->m->spinning)
runtime_throw("findrunnable: netpoll with spinning");
gp = runtime_netpoll(true); // block until new work is available
- runtime_atomicstore64(&runtime_sched.lastpoll, runtime_nanotime());
+ runtime_atomicstore64(&runtime_sched->lastpoll, runtime_nanotime());
if(gp) {
- runtime_lock(&runtime_sched);
+ runtime_lock(&runtime_sched->lock);
p = pidleget();
- runtime_unlock(&runtime_sched);
+ runtime_unlock(&runtime_sched->lock);
if(p) {
acquirep(p);
injectglist((G*)gp->schedlink);
if(g->m->spinning) {
g->m->spinning = false;
- nmspinning = runtime_xadd(&runtime_sched.nmspinning, -1);
+ nmspinning = runtime_xadd(&runtime_sched->nmspinning, -1);
if(nmspinning < 0)
runtime_throw("findrunnable: negative nmspinning");
} else
- nmspinning = runtime_atomicload(&runtime_sched.nmspinning);
+ nmspinning = runtime_atomicload(&runtime_sched->nmspinning);
// M wakeup policy is deliberately somewhat conservative (see nmspinning handling),
// so see if we need to wakeup another P here.
- if (nmspinning == 0 && runtime_atomicload(&runtime_sched.npidle) > 0)
+ if (nmspinning == 0 && runtime_atomicload(&runtime_sched->npidle) > 0)
wakep();
}
if(glist == nil)
return;
- runtime_lock(&runtime_sched);
+ runtime_lock(&runtime_sched->lock);
for(n = 0; glist; n++) {
gp = glist;
glist = (G*)gp->schedlink;
gp->atomicstatus = _Grunnable;
globrunqput(gp);
}
- runtime_unlock(&runtime_sched);
+ runtime_unlock(&runtime_sched->lock);
- for(; n && runtime_sched.npidle; n--)
+ for(; n && runtime_sched->npidle; n--)
startm(nil, false);
}
runtime_throw("schedule: holding locks");
top:
- if(runtime_sched.gcwaiting) {
+ if(runtime_sched->gcwaiting) {
gcstopm();
goto top;
}
tick = ((P*)g->m->p)->schedtick;
// This is a fancy way to say tick%61==0,
// it uses 2 MUL instructions instead of a single DIV and so is faster on modern processors.
- if(tick - (((uint64)tick*0x4325c53fu)>>36)*61 == 0 && runtime_sched.runqsize > 0) {
- runtime_lock(&runtime_sched);
+ if(tick - (((uint64)tick*0x4325c53fu)>>36)*61 == 0 && runtime_sched->runqsize > 0) {
+ runtime_lock(&runtime_sched->lock);
gp = globrunqget((P*)g->m->p, 1);
- runtime_unlock(&runtime_sched);
+ runtime_unlock(&runtime_sched->lock);
if(gp)
resetspinning();
}
gp->atomicstatus = _Grunnable;
gp->m = nil;
m->curg = nil;
- runtime_lock(&runtime_sched);
+ runtime_lock(&runtime_sched->lock);
globrunqput(gp);
- runtime_unlock(&runtime_sched);
+ runtime_unlock(&runtime_sched->lock);
if(m->lockedg) {
stoplockedm();
execute(gp); // Never returns.
g->atomicstatus = _Gsyscall;
- if(runtime_atomicload(&runtime_sched.sysmonwait)) { // TODO: fast atomic
- runtime_lock(&runtime_sched);
- if(runtime_atomicload(&runtime_sched.sysmonwait)) {
- runtime_atomicstore(&runtime_sched.sysmonwait, 0);
- runtime_notewakeup(&runtime_sched.sysmonnote);
+ if(runtime_atomicload(&runtime_sched->sysmonwait)) { // TODO: fast atomic
+ runtime_lock(&runtime_sched->lock);
+ if(runtime_atomicload(&runtime_sched->sysmonwait)) {
+ runtime_atomicstore(&runtime_sched->sysmonwait, 0);
+ runtime_notewakeup(&runtime_sched->sysmonnote);
}
- runtime_unlock(&runtime_sched);
+ runtime_unlock(&runtime_sched->lock);
}
g->m->mcache = nil;
((P*)(g->m->p))->m = 0;
runtime_atomicstore(&((P*)g->m->p)->status, _Psyscall);
- if(runtime_atomicload(&runtime_sched.gcwaiting)) {
- runtime_lock(&runtime_sched);
- if (runtime_sched.stopwait > 0 && runtime_cas(&((P*)g->m->p)->status, _Psyscall, _Pgcstop)) {
- if(--runtime_sched.stopwait == 0)
- runtime_notewakeup(&runtime_sched.stopnote);
+ if(runtime_atomicload(&runtime_sched->gcwaiting)) {
+ runtime_lock(&runtime_sched->lock);
+ if (runtime_sched->stopwait > 0 && runtime_cas(&((P*)g->m->p)->status, _Psyscall, _Pgcstop)) {
+ if(--runtime_sched->stopwait == 0)
+ runtime_notewakeup(&runtime_sched->stopnote);
}
- runtime_unlock(&runtime_sched);
+ runtime_unlock(&runtime_sched->lock);
}
g->m->locks--;
gp = g;
// Freezetheworld sets stopwait but does not retake P's.
- if(runtime_sched.stopwait) {
+ if(runtime_sched->stopwait) {
gp->m->p = 0;
return false;
}
}
// Try to get any other idle P.
gp->m->p = 0;
- if(runtime_sched.pidle) {
- runtime_lock(&runtime_sched);
+ if(runtime_sched->pidle) {
+ runtime_lock(&runtime_sched->lock);
p = pidleget();
- if(p && runtime_atomicload(&runtime_sched.sysmonwait)) {
- runtime_atomicstore(&runtime_sched.sysmonwait, 0);
- runtime_notewakeup(&runtime_sched.sysmonnote);
+ if(p && runtime_atomicload(&runtime_sched->sysmonwait)) {
+ runtime_atomicstore(&runtime_sched->sysmonwait, 0);
+ runtime_notewakeup(&runtime_sched->sysmonnote);
}
- runtime_unlock(&runtime_sched);
+ runtime_unlock(&runtime_sched->lock);
if(p) {
acquirep(p);
return true;
gp->atomicstatus = _Grunnable;
gp->m = nil;
m->curg = nil;
- runtime_lock(&runtime_sched);
+ runtime_lock(&runtime_sched->lock);
p = pidleget();
if(p == nil)
globrunqput(gp);
- else if(runtime_atomicload(&runtime_sched.sysmonwait)) {
- runtime_atomicstore(&runtime_sched.sysmonwait, 0);
- runtime_notewakeup(&runtime_sched.sysmonnote);
+ else if(runtime_atomicload(&runtime_sched->sysmonwait)) {
+ runtime_atomicstore(&runtime_sched->sysmonwait, 0);
+ runtime_notewakeup(&runtime_sched->sysmonnote);
}
- runtime_unlock(&runtime_sched);
+ runtime_unlock(&runtime_sched->lock);
if(p) {
acquirep(p);
execute(gp); // Never returns.
{
int32 hz;
- hz = runtime_sched.profilehz;
+ hz = runtime_sched->profilehz;
if(hz != 0)
runtime_resetcpuprofiler(hz);
runtime_m()->locks--;
newg->gopc = (uintptr)__builtin_return_address(0);
newg->atomicstatus = _Grunnable;
if(p->goidcache == p->goidcacheend) {
- p->goidcache = runtime_xadd64(&runtime_sched.goidgen, GoidCacheBatch);
+ p->goidcache = runtime_xadd64(&runtime_sched->goidgen, GoidCacheBatch);
p->goidcacheend = p->goidcache + GoidCacheBatch;
}
newg->goid = p->goidcache++;
runqput(p, vnewg);
- if(runtime_atomicload(&runtime_sched.npidle) != 0 && runtime_atomicload(&runtime_sched.nmspinning) == 0 && fn != runtime_main) // TODO: fast atomic
+ if(runtime_atomicload(&runtime_sched->npidle) != 0 && runtime_atomicload(&runtime_sched->nmspinning) == 0 && fn != runtime_main) // TODO: fast atomic
wakep();
g->m->locks--;
return vnewg;
p->gfree = gp;
p->gfreecnt++;
if(p->gfreecnt >= 64) {
- runtime_lock(&runtime_sched.gflock);
+ runtime_lock(&runtime_sched->gflock);
while(p->gfreecnt >= 32) {
p->gfreecnt--;
gp = p->gfree;
p->gfree = (G*)gp->schedlink;
- gp->schedlink = (uintptr)runtime_sched.gfree;
- runtime_sched.gfree = gp;
+ gp->schedlink = (uintptr)runtime_sched->gfree;
+ runtime_sched->gfree = gp;
}
- runtime_unlock(&runtime_sched.gflock);
+ runtime_unlock(&runtime_sched->gflock);
}
}
retry:
gp = p->gfree;
- if(gp == nil && runtime_sched.gfree) {
- runtime_lock(&runtime_sched.gflock);
- while(p->gfreecnt < 32 && runtime_sched.gfree) {
+ if(gp == nil && runtime_sched->gfree) {
+ runtime_lock(&runtime_sched->gflock);
+ while(p->gfreecnt < 32 && runtime_sched->gfree) {
p->gfreecnt++;
- gp = runtime_sched.gfree;
- runtime_sched.gfree = (G*)gp->schedlink;
+ gp = runtime_sched->gfree;
+ runtime_sched->gfree = (G*)gp->schedlink;
gp->schedlink = (uintptr)p->gfree;
p->gfree = gp;
}
- runtime_unlock(&runtime_sched.gflock);
+ runtime_unlock(&runtime_sched->gflock);
goto retry;
}
if(gp) {
{
G *gp;
- runtime_lock(&runtime_sched.gflock);
+ runtime_lock(&runtime_sched->gflock);
while(p->gfreecnt) {
p->gfreecnt--;
gp = p->gfree;
p->gfree = (G*)gp->schedlink;
- gp->schedlink = (uintptr)runtime_sched.gfree;
- runtime_sched.gfree = gp;
+ gp->schedlink = (uintptr)runtime_sched->gfree;
+ runtime_sched->gfree = gp;
}
- runtime_unlock(&runtime_sched.gflock);
+ runtime_unlock(&runtime_sched->gflock);
}
void
if(n > _MaxGomaxprocs)
n = _MaxGomaxprocs;
- runtime_lock(&runtime_sched);
+ runtime_lock(&runtime_sched->lock);
ret = (intgo)runtime_gomaxprocs;
if(n <= 0 || n == ret) {
- runtime_unlock(&runtime_sched);
+ runtime_unlock(&runtime_sched->lock);
return ret;
}
- runtime_unlock(&runtime_sched);
+ runtime_unlock(&runtime_sched->lock);
runtime_acquireWorldsema();
g->m->gcing = 1;
int32
runtime_mcount(void)
{
- return runtime_sched.mcount;
+ return runtime_sched->mcount;
}
static struct {
prof.hz = hz;
runtime_atomicstore(&prof.lock, 0);
- runtime_lock(&runtime_sched);
- runtime_sched.profilehz = hz;
- runtime_unlock(&runtime_sched);
+ runtime_lock(&runtime_sched->lock);
+ runtime_sched->profilehz = hz;
+ runtime_unlock(&runtime_sched->lock);
if(hz != 0)
runtime_resetcpuprofiler(hz);
p->runqtail--;
gp = (G*)p->runq[p->runqtail%nelem(p->runq)];
// push onto head of global queue
- gp->schedlink = (uintptr)runtime_sched.runqhead;
- runtime_sched.runqhead = gp;
- if(runtime_sched.runqtail == nil)
- runtime_sched.runqtail = gp;
- runtime_sched.runqsize++;
+ gp->schedlink = runtime_sched->runqhead;
+ runtime_sched->runqhead = (uintptr)gp;
+ if(runtime_sched->runqtail == 0)
+ runtime_sched->runqtail = (uintptr)gp;
+ runtime_sched->runqsize++;
}
}
// fill local queues with at most nelem(p->runq)/2 goroutines
// start at 1 because current M already executes some G and will acquire allp[0] below,
// so if we have a spare G we want to put it into allp[1].
- for(i = 1; (uint32)i < (uint32)new * nelem(p->runq)/2 && runtime_sched.runqsize > 0; i++) {
- gp = runtime_sched.runqhead;
- runtime_sched.runqhead = (G*)gp->schedlink;
- if(runtime_sched.runqhead == nil)
- runtime_sched.runqtail = nil;
- runtime_sched.runqsize--;
+ for(i = 1; (uint32)i < (uint32)new * nelem(p->runq)/2 && runtime_sched->runqsize > 0; i++) {
+ gp = (G*)runtime_sched->runqhead;
+ runtime_sched->runqhead = gp->schedlink;
+ if(runtime_sched->runqhead == 0)
+ runtime_sched->runqtail = 0;
+ runtime_sched->runqsize--;
runqput(runtime_allp[i%new], gp);
}
static void
incidlelocked(int32 v)
{
- runtime_lock(&runtime_sched);
- runtime_sched.nmidlelocked += v;
+ runtime_lock(&runtime_sched->lock);
+ runtime_sched->nmidlelocked += v;
if(v > 0)
checkdead();
- runtime_unlock(&runtime_sched);
+ runtime_unlock(&runtime_sched->lock);
}
// Check for deadlock situation.
}
// -1 for sysmon
- run = runtime_sched.mcount - runtime_sched.nmidle - runtime_sched.nmidlelocked - 1 - countextra();
+ run = runtime_sched->mcount - runtime_sched->nmidle - runtime_sched->nmidlelocked - 1 - countextra();
if(run > 0)
return;
// If we are dying because of a signal caught on an already idle thread,
return;
if(run < 0) {
runtime_printf("runtime: checkdead: nmidle=%d nmidlelocked=%d mcount=%d\n",
- runtime_sched.nmidle, runtime_sched.nmidlelocked, runtime_sched.mcount);
+ runtime_sched->nmidle, runtime_sched->nmidlelocked, runtime_sched->mcount);
runtime_throw("checkdead: inconsistent counts");
}
grunning = 0;
delay = 10*1000;
runtime_usleep(delay);
if(runtime_debug.schedtrace <= 0 &&
- (runtime_sched.gcwaiting || runtime_atomicload(&runtime_sched.npidle) == (uint32)runtime_gomaxprocs)) { // TODO: fast atomic
- runtime_lock(&runtime_sched);
- if(runtime_atomicload(&runtime_sched.gcwaiting) || runtime_atomicload(&runtime_sched.npidle) == (uint32)runtime_gomaxprocs) {
- runtime_atomicstore(&runtime_sched.sysmonwait, 1);
- runtime_unlock(&runtime_sched);
- runtime_notesleep(&runtime_sched.sysmonnote);
- runtime_noteclear(&runtime_sched.sysmonnote);
+ (runtime_sched->gcwaiting || runtime_atomicload(&runtime_sched->npidle) == (uint32)runtime_gomaxprocs)) { // TODO: fast atomic
+ runtime_lock(&runtime_sched->lock);
+ if(runtime_atomicload(&runtime_sched->gcwaiting) || runtime_atomicload(&runtime_sched->npidle) == (uint32)runtime_gomaxprocs) {
+ runtime_atomicstore(&runtime_sched->sysmonwait, 1);
+ runtime_unlock(&runtime_sched->lock);
+ runtime_notesleep(&runtime_sched->sysmonnote);
+ runtime_noteclear(&runtime_sched->sysmonnote);
idle = 0;
delay = 20;
} else
- runtime_unlock(&runtime_sched);
+ runtime_unlock(&runtime_sched->lock);
}
// poll network if not polled for more than 10ms
- lastpoll = runtime_atomicload64(&runtime_sched.lastpoll);
+ lastpoll = runtime_atomicload64(&runtime_sched->lastpoll);
now = runtime_nanotime();
if(lastpoll != 0 && lastpoll + 10*1000*1000 < now) {
- runtime_cas64(&runtime_sched.lastpoll, lastpoll, now);
+ runtime_cas64(&runtime_sched->lastpoll, lastpoll, now);
gp = runtime_netpoll(false); // non-blocking
if(gp) {
// Need to decrement number of idle locked M's
// but on the other hand we want to retake them eventually
// because they can prevent the sysmon thread from deep sleep.
if(p->runqhead == p->runqtail &&
- runtime_atomicload(&runtime_sched.nmspinning) + runtime_atomicload(&runtime_sched.npidle) > 0 &&
+ runtime_atomicload(&runtime_sched->nmspinning) + runtime_atomicload(&runtime_sched->npidle) > 0 &&
pd->syscallwhen + 10*1000*1000 > now)
continue;
// Need to decrement number of idle locked M's
if(starttime == 0)
starttime = now;
- runtime_lock(&runtime_sched);
+ runtime_lock(&runtime_sched->lock);
runtime_printf("SCHED %Dms: gomaxprocs=%d idleprocs=%d threads=%d idlethreads=%d runqueue=%d",
- (now-starttime)/1000000, runtime_gomaxprocs, runtime_sched.npidle, runtime_sched.mcount,
- runtime_sched.nmidle, runtime_sched.runqsize);
+ (now-starttime)/1000000, runtime_gomaxprocs, runtime_sched->npidle, runtime_sched->mcount,
+ runtime_sched->nmidle, runtime_sched->runqsize);
if(detailed) {
runtime_printf(" gcwaiting=%d nmidlelocked=%d nmspinning=%d stopwait=%d sysmonwait=%d\n",
- runtime_sched.gcwaiting, runtime_sched.nmidlelocked, runtime_sched.nmspinning,
- runtime_sched.stopwait, runtime_sched.sysmonwait);
+ runtime_sched->gcwaiting, runtime_sched->nmidlelocked, runtime_sched->nmspinning,
+ runtime_sched->stopwait, runtime_sched->sysmonwait);
}
// We must be careful while reading data from P's, M's and G's.
// Even if we hold schedlock, most data can be changed concurrently.
}
}
if(!detailed) {
- runtime_unlock(&runtime_sched);
+ runtime_unlock(&runtime_sched->lock);
return;
}
for(mp = runtime_allm; mp; mp = mp->alllink) {
lockedm ? lockedm->id : -1);
}
runtime_unlock(&allglock);
- runtime_unlock(&runtime_sched);
+ runtime_unlock(&runtime_sched->lock);
}
// Put mp on midle list.
static void
mput(M *mp)
{
- mp->schedlink = (uintptr)runtime_sched.midle;
- runtime_sched.midle = mp;
- runtime_sched.nmidle++;
+ mp->schedlink = runtime_sched->midle;
+ runtime_sched->midle = (uintptr)mp;
+ runtime_sched->nmidle++;
checkdead();
}
{
M *mp;
- if((mp = runtime_sched.midle) != nil){
- runtime_sched.midle = (M*)mp->schedlink;
- runtime_sched.nmidle--;
+ if((mp = (M*)runtime_sched->midle) != nil){
+ runtime_sched->midle = mp->schedlink;
+ runtime_sched->nmidle--;
}
return mp;
}
globrunqput(G *gp)
{
gp->schedlink = 0;
- if(runtime_sched.runqtail)
- runtime_sched.runqtail->schedlink = (uintptr)gp;
+ if(runtime_sched->runqtail)
+ ((G*)runtime_sched->runqtail)->schedlink = (uintptr)gp;
else
- runtime_sched.runqhead = gp;
- runtime_sched.runqtail = gp;
- runtime_sched.runqsize++;
+ runtime_sched->runqhead = (uintptr)gp;
+ runtime_sched->runqtail = (uintptr)gp;
+ runtime_sched->runqsize++;
}
// Put a batch of runnable goroutines on the global runnable queue.
globrunqputbatch(G *ghead, G *gtail, int32 n)
{
gtail->schedlink = 0;
- if(runtime_sched.runqtail)
- runtime_sched.runqtail->schedlink = (uintptr)ghead;
+ if(runtime_sched->runqtail)
+ ((G*)runtime_sched->runqtail)->schedlink = (uintptr)ghead;
else
- runtime_sched.runqhead = ghead;
- runtime_sched.runqtail = gtail;
- runtime_sched.runqsize += n;
+ runtime_sched->runqhead = (uintptr)ghead;
+ runtime_sched->runqtail = (uintptr)gtail;
+ runtime_sched->runqsize += n;
}
// Try get a batch of G's from the global runnable queue.
G *gp, *gp1;
int32 n;
- if(runtime_sched.runqsize == 0)
+ if(runtime_sched->runqsize == 0)
return nil;
- n = runtime_sched.runqsize/runtime_gomaxprocs+1;
- if(n > runtime_sched.runqsize)
- n = runtime_sched.runqsize;
+ n = runtime_sched->runqsize/runtime_gomaxprocs+1;
+ if(n > runtime_sched->runqsize)
+ n = runtime_sched->runqsize;
if(max > 0 && n > max)
n = max;
if((uint32)n > nelem(p->runq)/2)
n = nelem(p->runq)/2;
- runtime_sched.runqsize -= n;
- if(runtime_sched.runqsize == 0)
- runtime_sched.runqtail = nil;
- gp = runtime_sched.runqhead;
- runtime_sched.runqhead = (G*)gp->schedlink;
+ runtime_sched->runqsize -= n;
+ if(runtime_sched->runqsize == 0)
+ runtime_sched->runqtail = 0;
+ gp = (G*)runtime_sched->runqhead;
+ runtime_sched->runqhead = gp->schedlink;
n--;
while(n--) {
- gp1 = runtime_sched.runqhead;
- runtime_sched.runqhead = (G*)gp1->schedlink;
+ gp1 = (G*)runtime_sched->runqhead;
+ runtime_sched->runqhead = gp1->schedlink;
runqput(p, gp1);
}
return gp;
static void
pidleput(P *p)
{
- p->link = (uintptr)runtime_sched.pidle;
- runtime_sched.pidle = p;
- runtime_xadd(&runtime_sched.npidle, 1); // TODO: fast atomic
+ p->link = runtime_sched->pidle;
+ runtime_sched->pidle = (uintptr)p;
+ runtime_xadd(&runtime_sched->npidle, 1); // TODO: fast atomic
}
// Try get a p from pidle list.
{
P *p;
- p = runtime_sched.pidle;
+ p = (P*)runtime_sched->pidle;
if(p) {
- runtime_sched.pidle = (P*)p->link;
- runtime_xadd(&runtime_sched.npidle, -1); // TODO: fast atomic
+ runtime_sched->pidle = p->link;
+ runtime_xadd(&runtime_sched->npidle, -1); // TODO: fast atomic
}
return p;
}
for(i=0; i<n; i++)
batch[i]->schedlink = (uintptr)batch[i+1];
// Now put the batch on global queue.
- runtime_lock(&runtime_sched);
+ runtime_lock(&runtime_sched->lock);
globrunqputbatch(batch[0], batch[n], n+1);
- runtime_unlock(&runtime_sched);
+ runtime_unlock(&runtime_sched->lock);
return true;
}
{
intgo out;
- runtime_lock(&runtime_sched);
- out = (intgo)runtime_sched.maxmcount;
- runtime_sched.maxmcount = (int32)in;
+ runtime_lock(&runtime_sched->lock);
+ out = (intgo)runtime_sched->maxmcount;
+ runtime_sched->maxmcount = (int32)in;
checkmcount();
- runtime_unlock(&runtime_sched);
+ runtime_unlock(&runtime_sched->lock);
return out;
}
void
runtime_proc_scan(struct Workbuf** wbufp, void (*enqueue1)(struct Workbuf**, Obj))
{
- enqueue1(wbufp, (Obj){(byte*)&runtime_sched, sizeof runtime_sched, 0});
enqueue1(wbufp, (Obj){(byte*)&runtime_main_init_done, sizeof runtime_main_init_done, 0});
}
bool
runtime_gcwaiting(void)
{
- return runtime_sched.gcwaiting;
+ return runtime_sched->gcwaiting;
}
// os_beforeExit is called from os.Exit(0).
// GOMAXPROCS>1 and there is at least one other running P and local runq is empty.
// As opposed to runtime mutex we don't do passive spinning here,
// because there can be work on global runq on on other Ps.
- if (i >= ACTIVE_SPIN || runtime_ncpu <= 1 || runtime_gomaxprocs <= (int32)(runtime_sched.npidle+runtime_sched.nmspinning)+1) {
+ if (i >= ACTIVE_SPIN || runtime_ncpu <= 1 || runtime_gomaxprocs <= (int32)(runtime_sched->npidle+runtime_sched->nmspinning)+1) {
return false;
}
p = (P*)g->m->p;
projects / gcc.git / commitdiff