1 // Copyright 2009 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
17 #ifdef USING_SPLIT_STACK
19 /* FIXME: These are not declared anywhere. */
21 extern void __splitstack_getcontext(void *context
[10]);
23 extern void __splitstack_setcontext(void *context
[10]);
25 extern void *__splitstack_makecontext(size_t, void *context
[10], size_t *);
27 extern void * __splitstack_resetcontext(void *context
[10], size_t *);
29 extern void *__splitstack_find(void *, void *, size_t *, void **, void **,
32 extern void __splitstack_block_signals (int *, int *);
34 extern void __splitstack_block_signals_context (void *context
[10], int *,
39 #if defined(USING_SPLIT_STACK) && defined(LINKER_SUPPORTS_SPLIT_STACK)
40 # ifdef PTHREAD_STACK_MIN
41 # define StackMin PTHREAD_STACK_MIN
43 # define StackMin 8192
46 # define StackMin 2 * 1024 * 1024
49 uintptr runtime_stacks_sys
;
51 static void schedule(G
*);
53 static void gtraceback(G
*);
55 typedef struct Sched Sched
;
58 G runtime_g0
; // idle goroutine for m0
67 #ifndef SETCONTEXT_CLOBBERS_TLS
75 fixcontext(ucontext_t
*c
__attribute__ ((unused
)))
81 # if defined(__x86_64__) && defined(__sun__)
83 // x86_64 Solaris 10 and 11 have a bug: setcontext switches the %fs
84 // register to that of the thread which called getcontext. The effect
85 // is that the address of all __thread variables changes. This bug
86 // also affects pthread_self() and pthread_getspecific. We work
87 // around it by clobbering the context field directly to keep %fs the
90 static __thread greg_t fs
;
98 fs
= c
.uc_mcontext
.gregs
[REG_FSBASE
];
102 fixcontext(ucontext_t
* c
)
104 c
->uc_mcontext
.gregs
[REG_FSBASE
] = fs
;
109 # error unknown case for SETCONTEXT_CLOBBERS_TLS
115 // We can not always refer to the TLS variables directly. The
116 // compiler will call tls_get_addr to get the address of the variable,
117 // and it may hold it in a register across a call to schedule. When
118 // we get back from the call we may be running in a different thread,
119 // in which case the register now points to the TLS variable for a
120 // different thread. We use non-inlinable functions to avoid this
123 G
* runtime_g(void) __attribute__ ((noinline
, no_split_stack
));
131 M
* runtime_m(void) __attribute__ ((noinline
, no_split_stack
));
139 int32 runtime_gcwaiting
;
143 // The go scheduler's job is to match ready-to-run goroutines (`g's)
144 // with waiting-for-work schedulers (`m's). If there are ready g's
145 // and no waiting m's, ready() will start a new m running in a new
146 // OS thread, so that all ready g's can run simultaneously, up to a limit.
147 // For now, m's never go away.
149 // By default, Go keeps only one kernel thread (m) running user code
150 // at a single time; other threads may be blocked in the operating system.
151 // Setting the environment variable $GOMAXPROCS or calling
152 // runtime.GOMAXPROCS() will change the number of user threads
153 // allowed to execute simultaneously. $GOMAXPROCS is thus an
154 // approximation of the maximum number of cores to use.
156 // Even a program that can run without deadlock in a single process
157 // might use more m's if given the chance. For example, the prime
158 // sieve will use as many m's as there are primes (up to runtime_sched.mmax),
159 // allowing different stages of the pipeline to execute in parallel.
160 // We could revisit this choice, only kicking off new m's for blocking
161 // system calls, but that would limit the amount of parallel computation
162 // that go would try to do.
164 // In general, one could imagine all sorts of refinements to the
165 // scheduler, but the goal now is just to get something working on
171 G
*gfree
; // available g's (status == Gdead)
174 G
*ghead
; // g's waiting to run
176 int32 gwait
; // number of g's waiting to run
177 int32 gcount
; // number of g's that are alive
178 int32 grunning
; // number of g's running on cpu or in syscall
180 M
*mhead
; // m's waiting for work
181 int32 mwait
; // number of m's waiting for work
182 int32 mcount
; // number of m's that have been created
184 volatile uint32 atomic
; // atomic scheduling word (see below)
186 int32 profilehz
; // cpu profiling rate
188 bool init
; // running initialization
189 bool lockmain
; // init called runtime.LockOSThread
191 Note stopped
; // one g can set waitstop and wait here for m's to stop
194 // The atomic word in sched is an atomic uint32 that
195 // holds these fields.
197 // [15 bits] mcpu number of m's executing on cpu
198 // [15 bits] mcpumax max number of m's allowed on cpu
199 // [1 bit] waitstop some g is waiting on stopped
200 // [1 bit] gwaiting gwait != 0
202 // These fields are the information needed by entersyscall
203 // and exitsyscall to decide whether to coordinate with the
204 // scheduler. Packing them into a single machine word lets
205 // them use a fast path with a single atomic read/write and
206 // no lock/unlock. This greatly reduces contention in
207 // syscall- or cgo-heavy multithreaded programs.
209 // Except for entersyscall and exitsyscall, the manipulations
210 // to these fields only happen while holding the schedlock,
211 // so the routines holding schedlock only need to worry about
212 // what entersyscall and exitsyscall do, not the other routines
213 // (which also use the schedlock).
215 // In particular, entersyscall and exitsyscall only read mcpumax,
216 // waitstop, and gwaiting. They never write them. Thus, writes to those
217 // fields can be done (holding schedlock) without fear of write conflicts.
218 // There may still be logic conflicts: for example, the set of waitstop must
219 // be conditioned on mcpu >= mcpumax or else the wait may be a
220 // spurious sleep. The Promela model in proc.p verifies these accesses.
223 mcpuMask
= (1<<mcpuWidth
) - 1,
225 mcpumaxShift
= mcpuShift
+ mcpuWidth
,
226 waitstopShift
= mcpumaxShift
+ mcpuWidth
,
227 gwaitingShift
= waitstopShift
+1,
229 // The max value of GOMAXPROCS is constrained
230 // by the max value we can store in the bit fields
231 // of the atomic word. Reserve a few high values
232 // so that we can detect accidental decrement
234 maxgomaxprocs
= mcpuMask
- 10,
237 #define atomic_mcpu(v) (((v)>>mcpuShift)&mcpuMask)
238 #define atomic_mcpumax(v) (((v)>>mcpumaxShift)&mcpuMask)
239 #define atomic_waitstop(v) (((v)>>waitstopShift)&1)
240 #define atomic_gwaiting(v) (((v)>>gwaitingShift)&1)
243 int32 runtime_gomaxprocs
;
244 bool runtime_singleproc
;
246 static bool canaddmcpu(void);
248 // An m that is waiting for notewakeup(&m->havenextg). This may
249 // only be accessed while the scheduler lock is held. This is used to
250 // minimize the number of times we call notewakeup while the scheduler
251 // lock is held, since the m will normally move quickly to lock the
252 // scheduler itself, producing lock contention.
255 // Scheduling helpers. Sched must be locked.
256 static void gput(G
*); // put/get on ghead/gtail
257 static G
* gget(void);
258 static void mput(M
*); // put/get on mhead
260 static void gfput(G
*); // put/get on gfree
261 static G
* gfget(void);
262 static void matchmg(void); // match m's to g's
263 static void readylocked(G
*); // ready, but sched is locked
264 static void mnextg(M
*, G
*);
265 static void mcommoninit(M
*);
273 v
= runtime_sched
.atomic
;
275 w
&= ~(mcpuMask
<<mcpumaxShift
);
276 w
|= n
<<mcpumaxShift
;
277 if(runtime_cas(&runtime_sched
.atomic
, v
, w
))
282 // First function run by a new goroutine. This replaces gogocall.
288 fn
= (void (*)(void*))(g
->entry
);
293 // Switch context to a different goroutine. This is like longjmp.
294 static void runtime_gogo(G
*) __attribute__ ((noinline
));
296 runtime_gogo(G
* newg
)
298 #ifdef USING_SPLIT_STACK
299 __splitstack_setcontext(&newg
->stack_context
[0]);
302 newg
->fromgogo
= true;
303 fixcontext(&newg
->context
);
304 setcontext(&newg
->context
);
305 runtime_throw("gogo setcontext returned");
308 // Save context and call fn passing g as a parameter. This is like
309 // setjmp. Because getcontext always returns 0, unlike setjmp, we use
310 // g->fromgogo as a code. It will be true if we got here via
311 // setcontext. g == nil the first time this is called in a new m.
312 static void runtime_mcall(void (*)(G
*)) __attribute__ ((noinline
));
314 runtime_mcall(void (*pfn
)(G
*))
318 #ifndef USING_SPLIT_STACK
322 // Ensure that all registers are on the stack for the garbage
324 __builtin_unwind_init();
329 runtime_throw("runtime: mcall called on m->g0 stack");
333 #ifdef USING_SPLIT_STACK
334 __splitstack_getcontext(&g
->stack_context
[0]);
338 gp
->fromgogo
= false;
339 getcontext(&gp
->context
);
341 // When we return from getcontext, we may be running
342 // in a new thread. That means that m and g may have
343 // changed. They are global variables so we will
344 // reload them, but the addresses of m and g may be
345 // cached in our local stack frame, and those
346 // addresses may be wrong. Call functions to reload
347 // the values for this thread.
351 if(gp
->dotraceback
!= nil
)
354 if (gp
== nil
|| !gp
->fromgogo
) {
355 #ifdef USING_SPLIT_STACK
356 __splitstack_setcontext(&mp
->g0
->stack_context
[0]);
358 mp
->g0
->entry
= (byte
*)pfn
;
361 // It's OK to set g directly here because this case
362 // can not occur if we got here via a setcontext to
363 // the getcontext call just above.
366 fixcontext(&mp
->g0
->context
);
367 setcontext(&mp
->g0
->context
);
368 runtime_throw("runtime: mcall function returned");
372 // Keep trace of scavenger's goroutine for deadlock detection.
375 // The bootstrap sequence is:
379 // make & queue new G
380 // call runtime_mstart
382 // The new G calls runtime_main.
384 runtime_schedinit(void)
398 runtime_mallocinit();
405 // Allocate internal symbol table representation now,
406 // so that we don't need to call malloc when we crash.
407 // runtime_findfunc(0);
409 runtime_gomaxprocs
= 1;
410 p
= runtime_getenv("GOMAXPROCS");
411 if(p
!= nil
&& (n
= runtime_atoi(p
)) != 0) {
412 if(n
> maxgomaxprocs
)
414 runtime_gomaxprocs
= n
;
416 // wait for the main goroutine to start before taking
417 // GOMAXPROCS into account.
419 runtime_singleproc
= runtime_gomaxprocs
== 1;
421 canaddmcpu(); // mcpu++ to account for bootstrap m
422 m
->helpgc
= 1; // flag to tell schedule() to mcpu--
423 runtime_sched
.grunning
++;
425 // Can not enable GC until all roots are registered.
426 // mstats.enablegc = 1;
430 extern void main_init(void) __asm__ ("__go_init_main");
431 extern void main_main(void) __asm__ ("main.main");
433 // The main goroutine.
437 // Lock the main goroutine onto this, the main OS thread,
438 // during initialization. Most programs won't care, but a few
439 // do require certain calls to be made by the main thread.
440 // Those can arrange for main.main to run in the main thread
441 // by calling runtime.LockOSThread during initialization
442 // to preserve the lock.
443 runtime_LockOSThread();
444 // From now on, newgoroutines may use non-main threads.
445 setmcpumax(runtime_gomaxprocs
);
446 runtime_sched
.init
= true;
447 scvg
= __go_go(runtime_MHeap_Scavenger
, nil
);
449 runtime_sched
.init
= false;
450 if(!runtime_sched
.lockmain
)
451 runtime_UnlockOSThread();
453 // For gccgo we have to wait until after main is initialized
454 // to enable GC, because initializing main registers the GC
458 // The deadlock detection has false negatives.
459 // Let scvg start up, to eliminate the false negative
460 // for the trivial program func main() { select{} }.
469 // Lock the scheduler.
473 runtime_lock(&runtime_sched
);
476 // Unlock the scheduler.
484 runtime_unlock(&runtime_sched
);
486 runtime_notewakeup(&m
->havenextg
);
492 g
->status
= Gmoribund
;
497 runtime_goroutineheader(G
*g
)
516 status
= g
->waitreason
;
527 runtime_printf("goroutine %d [%s]:\n", g
->goid
, status
);
531 runtime_goroutinetrailer(G
*g
)
533 if(g
!= nil
&& g
->gopc
!= 0 && g
->goid
!= 1) {
534 struct __go_string fn
;
535 struct __go_string file
;
538 if(__go_file_line(g
->gopc
- 1, &fn
, &file
, &line
)) {
539 runtime_printf("created by %s\n", fn
.__data
);
540 runtime_printf("\t%s:%d\n", file
.__data
, line
);
546 runtime_tracebackothers(G
* volatile me
)
550 for(g
= runtime_allg
; g
!= nil
; g
= g
->alllink
) {
551 if(g
== me
|| g
->status
== Gdead
)
553 runtime_printf("\n");
554 runtime_goroutineheader(g
);
556 // Our only mechanism for doing a stack trace is
557 // _Unwind_Backtrace. And that only works for the
558 // current thread, not for other random goroutines.
559 // So we need to switch context to the goroutine, get
560 // the backtrace, and then switch back.
562 // This means that if g is running or in a syscall, we
563 // can't reliably print a stack trace. FIXME.
564 if(g
->status
== Gsyscall
|| g
->status
== Grunning
) {
565 runtime_printf("no stack trace available\n");
566 runtime_goroutinetrailer(g
);
572 #ifdef USING_SPLIT_STACK
573 __splitstack_getcontext(&me
->stack_context
[0]);
575 getcontext(&me
->context
);
583 // Do a stack trace of gp, and then restore the context to
591 runtime_traceback(nil
);
592 runtime_goroutinetrailer(gp
);
593 ret
= gp
->dotraceback
;
594 gp
->dotraceback
= nil
;
598 // Mark this g as m's idle goroutine.
599 // This functionality might be used in environments where programs
600 // are limited to a single thread, to simulate a select-driven
601 // network server. It is not exposed via the standard runtime API.
603 runtime_idlegoroutine(void)
606 runtime_throw("g is already an idle goroutine");
613 m
->id
= runtime_sched
.mcount
++;
614 m
->fastrand
= 0x49f6428aUL
+ m
->id
+ runtime_cputicks();
617 m
->mcache
= runtime_allocmcache();
619 runtime_callers(1, m
->createstack
, nelem(m
->createstack
));
621 // Add to runtime_allm so garbage collector doesn't free m
622 // when it is just in a register or thread-local storage.
623 m
->alllink
= runtime_allm
;
624 // runtime_NumCgoCall() iterates over allm w/o schedlock,
625 // so we need to publish it safely.
626 runtime_atomicstorep(&runtime_allm
, m
);
629 // Try to increment mcpu. Report whether succeeded.
636 v
= runtime_sched
.atomic
;
637 if(atomic_mcpu(v
) >= atomic_mcpumax(v
))
639 if(runtime_cas(&runtime_sched
.atomic
, v
, v
+(1<<mcpuShift
)))
644 // Put on `g' queue. Sched must be locked.
650 // If g is wired, hand it off directly.
651 if((m
= g
->lockedm
) != nil
&& canaddmcpu()) {
656 // If g is the idle goroutine for an m, hand it off.
657 if(g
->idlem
!= nil
) {
658 if(g
->idlem
->idleg
!= nil
) {
659 runtime_printf("m%d idle out of sync: g%d g%d\n",
661 g
->idlem
->idleg
->goid
, g
->goid
);
662 runtime_throw("runtime: double idle");
669 if(runtime_sched
.ghead
== nil
)
670 runtime_sched
.ghead
= g
;
672 runtime_sched
.gtail
->schedlink
= g
;
673 runtime_sched
.gtail
= g
;
676 // if it transitions to nonzero, set atomic gwaiting bit.
677 if(runtime_sched
.gwait
++ == 0)
678 runtime_xadd(&runtime_sched
.atomic
, 1<<gwaitingShift
);
681 // Report whether gget would return something.
685 return runtime_sched
.ghead
!= nil
|| m
->idleg
!= nil
;
688 // Get from `g' queue. Sched must be locked.
694 g
= runtime_sched
.ghead
;
696 runtime_sched
.ghead
= g
->schedlink
;
697 if(runtime_sched
.ghead
== nil
)
698 runtime_sched
.gtail
= nil
;
700 // if it transitions to zero, clear atomic gwaiting bit.
701 if(--runtime_sched
.gwait
== 0)
702 runtime_xadd(&runtime_sched
.atomic
, -1<<gwaitingShift
);
703 } else if(m
->idleg
!= nil
) {
710 // Put on `m' list. Sched must be locked.
714 m
->schedlink
= runtime_sched
.mhead
;
715 runtime_sched
.mhead
= m
;
716 runtime_sched
.mwait
++;
719 // Get an `m' to run `g'. Sched must be locked.
725 // if g has its own m, use it.
726 if(g
&& (m
= g
->lockedm
) != nil
)
729 // otherwise use general m pool.
730 if((m
= runtime_sched
.mhead
) != nil
){
731 runtime_sched
.mhead
= m
->schedlink
;
732 runtime_sched
.mwait
--;
737 // Mark g ready to run.
746 // Mark g ready to run. Sched is already locked.
747 // G might be running already and about to stop.
748 // The sched lock protects g->status from changing underfoot.
753 // Running on another machine.
754 // Ready it when it stops.
760 if(g
->status
== Grunnable
|| g
->status
== Grunning
) {
761 runtime_printf("goroutine %d has status %d\n", g
->goid
, g
->status
);
762 runtime_throw("bad g->status in ready");
764 g
->status
= Grunnable
;
770 // Same as readylocked but a different symbol so that
771 // debuggers can set a breakpoint here and catch all
774 newprocreadylocked(G
*g
)
779 // Pass g to m for running.
780 // Caller has already incremented mcpu.
784 runtime_sched
.grunning
++;
789 runtime_notewakeup(&mwakeup
->havenextg
);
794 // Get the next goroutine that m should run.
795 // Sched must be locked on entry, is unlocked on exit.
796 // Makes sure that at most $GOMAXPROCS g's are
797 // running on cpus (not in system calls) at any given time.
805 if(atomic_mcpu(runtime_sched
.atomic
) >= maxgomaxprocs
)
806 runtime_throw("negative mcpu");
808 // If there is a g waiting as m->nextg, the mcpu++
809 // happened before it was passed to mnextg.
810 if(m
->nextg
!= nil
) {
817 if(m
->lockedg
!= nil
) {
818 // We can only run one g, and it's not available.
819 // Make sure some other cpu is running to handle
820 // the ordinary run queue.
821 if(runtime_sched
.gwait
!= 0) {
823 // m->lockedg might have been on the queue.
824 if(m
->nextg
!= nil
) {
832 // Look for work on global queue.
833 while(haveg() && canaddmcpu()) {
836 runtime_throw("gget inconsistency");
839 mnextg(gp
->lockedm
, gp
);
842 runtime_sched
.grunning
++;
847 // The while loop ended either because the g queue is empty
848 // or because we have maxed out our m procs running go
849 // code (mcpu >= mcpumax). We need to check that
850 // concurrent actions by entersyscall/exitsyscall cannot
851 // invalidate the decision to end the loop.
853 // We hold the sched lock, so no one else is manipulating the
854 // g queue or changing mcpumax. Entersyscall can decrement
855 // mcpu, but if does so when there is something on the g queue,
856 // the gwait bit will be set, so entersyscall will take the slow path
857 // and use the sched lock. So it cannot invalidate our decision.
859 // Wait on global m queue.
863 // Look for deadlock situation.
864 // There is a race with the scavenger that causes false negatives:
865 // if the scavenger is just starting, then we have
866 // scvg != nil && grunning == 0 && gwait == 0
867 // and we do not detect a deadlock. It is possible that we should
868 // add that case to the if statement here, but it is too close to Go 1
869 // to make such a subtle change. Instead, we work around the
870 // false negative in trivial programs by calling runtime.gosched
871 // from the main goroutine just before main.main.
872 // See runtime_main above.
874 // On a related note, it is also possible that the scvg == nil case is
875 // wrong and should include gwait, but that does not happen in
876 // standard Go programs, which all start the scavenger.
878 if((scvg
== nil
&& runtime_sched
.grunning
== 0) ||
879 (scvg
!= nil
&& runtime_sched
.grunning
== 1 && runtime_sched
.gwait
== 0 &&
880 (scvg
->status
== Grunning
|| scvg
->status
== Gsyscall
))) {
881 runtime_throw("all goroutines are asleep - deadlock!");
886 runtime_noteclear(&m
->havenextg
);
888 // Stoptheworld is waiting for all but its cpu to go to stop.
889 // Entersyscall might have decremented mcpu too, but if so
890 // it will see the waitstop and take the slow path.
891 // Exitsyscall never increments mcpu beyond mcpumax.
892 v
= runtime_atomicload(&runtime_sched
.atomic
);
893 if(atomic_waitstop(v
) && atomic_mcpu(v
) <= atomic_mcpumax(v
)) {
894 // set waitstop = 0 (known to be 1)
895 runtime_xadd(&runtime_sched
.atomic
, -1<<waitstopShift
);
896 runtime_notewakeup(&runtime_sched
.stopped
);
900 runtime_notesleep(&m
->havenextg
);
904 runtime_lock(&runtime_sched
);
907 if((gp
= m
->nextg
) == nil
)
908 runtime_throw("bad m->nextg in nextgoroutine");
914 runtime_helpgc(bool *extra
)
919 // Figure out how many CPUs to use.
920 // Limited by gomaxprocs, number of actual CPUs, and MaxGcproc.
921 max
= runtime_gomaxprocs
;
922 if(max
> runtime_ncpu
)
923 max
= runtime_ncpu
> 0 ? runtime_ncpu
: 1;
927 // We're going to use one CPU no matter what.
928 // Figure out the max number of additional CPUs.
931 runtime_lock(&runtime_sched
);
933 while(n
< max
&& (mp
= mget(nil
)) != nil
) {
937 runtime_notewakeup(&mp
->havenextg
);
939 runtime_unlock(&runtime_sched
);
946 runtime_stoptheworld(void)
951 runtime_gcwaiting
= 1;
957 v
= runtime_sched
.atomic
;
958 if(atomic_mcpu(v
) <= 1)
961 // It would be unsafe for multiple threads to be using
962 // the stopped note at once, but there is only
963 // ever one thread doing garbage collection.
964 runtime_noteclear(&runtime_sched
.stopped
);
965 if(atomic_waitstop(v
))
966 runtime_throw("invalid waitstop");
968 // atomic { waitstop = 1 }, predicated on mcpu <= 1 check above
970 if(!runtime_cas(&runtime_sched
.atomic
, v
, v
+(1<<waitstopShift
)))
974 runtime_notesleep(&runtime_sched
.stopped
);
977 runtime_singleproc
= runtime_gomaxprocs
== 1;
982 runtime_starttheworld(bool extra
)
987 runtime_gcwaiting
= 0;
988 setmcpumax(runtime_gomaxprocs
);
990 if(extra
&& canaddmcpu()) {
991 // Start a new m that will (we hope) be idle
992 // and so available to help when the next
993 // garbage collection happens.
994 // canaddmcpu above did mcpu++
995 // (necessary, because m will be doing various
996 // initialization work so is definitely running),
997 // but m is not running a specific goroutine,
998 // so set the helpgc flag as a signal to m's
999 // first schedule(nil) to mcpu-- and grunning--.
1002 runtime_sched
.grunning
++;
1007 // Called to start an M.
1009 runtime_mstart(void* mp
)
1019 // Record top of stack for use by mcall.
1020 // Once we call schedule we're never coming back,
1021 // so other calls can reuse this stack space.
1022 #ifdef USING_SPLIT_STACK
1023 __splitstack_getcontext(&g
->stack_context
[0]);
1025 g
->gcinitial_sp
= &mp
;
1026 // Setting gcstack_size to 0 is a marker meaning that gcinitial_sp
1027 // is the top of the stack, not the bottom.
1028 g
->gcstack_size
= 0;
1031 getcontext(&g
->context
);
1033 if(g
->entry
!= nil
) {
1034 // Got here from mcall.
1035 void (*pfn
)(G
*) = (void (*)(G
*))g
->entry
;
1036 G
* gp
= (G
*)g
->param
;
1042 #ifdef USING_SPLIT_STACK
1044 int dont_block_signals
= 0;
1045 __splitstack_block_signals(&dont_block_signals
, nil
);
1049 // Install signal handlers; after minit so that minit can
1050 // prepare the thread to be able to handle the signals.
1051 if(m
== &runtime_m0
)
1058 typedef struct CgoThreadStart CgoThreadStart
;
1059 struct CgoThreadStart
1066 // Kick off new m's as needed (up to mcpumax).
1074 if(m
->mallocing
|| m
->gcing
)
1077 while(haveg() && canaddmcpu()) {
1080 runtime_throw("gget inconsistency");
1082 // Find the m that will run gp.
1083 if((mp
= mget(gp
)) == nil
)
1084 mp
= runtime_newm();
1089 // Create a new m. It will start off with a call to runtime_mstart.
1094 pthread_attr_t attr
;
1097 m
= runtime_malloc(sizeof(M
));
1099 m
->g0
= runtime_malg(-1, nil
, nil
);
1101 if(pthread_attr_init(&attr
) != 0)
1102 runtime_throw("pthread_attr_init");
1103 if(pthread_attr_setdetachstate(&attr
, PTHREAD_CREATE_DETACHED
) != 0)
1104 runtime_throw("pthread_attr_setdetachstate");
1106 #ifndef PTHREAD_STACK_MIN
1107 #define PTHREAD_STACK_MIN 8192
1109 if(pthread_attr_setstacksize(&attr
, PTHREAD_STACK_MIN
) != 0)
1110 runtime_throw("pthread_attr_setstacksize");
1112 if(pthread_create(&tid
, &attr
, runtime_mstart
, m
) != 0)
1113 runtime_throw("pthread_create");
1118 // One round of scheduler: find a goroutine and run it.
1119 // The argument is the goroutine that was running before
1120 // schedule was called, or nil if this is the first call.
1130 // Just finished running gp.
1132 runtime_sched
.grunning
--;
1134 // atomic { mcpu-- }
1135 v
= runtime_xadd(&runtime_sched
.atomic
, -1<<mcpuShift
);
1136 if(atomic_mcpu(v
) > maxgomaxprocs
)
1137 runtime_throw("negative mcpu in scheduler");
1142 // Shouldn't have been running!
1143 runtime_throw("bad gp->status in sched");
1145 gp
->status
= Grunnable
;
1155 runtime_memclr(&gp
->context
, sizeof gp
->context
);
1157 if(--runtime_sched
.gcount
== 0)
1161 if(gp
->readyonstop
){
1162 gp
->readyonstop
= 0;
1165 } else if(m
->helpgc
) {
1166 // Bootstrap m or new m started by starttheworld.
1167 // atomic { mcpu-- }
1168 v
= runtime_xadd(&runtime_sched
.atomic
, -1<<mcpuShift
);
1169 if(atomic_mcpu(v
) > maxgomaxprocs
)
1170 runtime_throw("negative mcpu in scheduler");
1171 // Compensate for increment in starttheworld().
1172 runtime_sched
.grunning
--;
1174 } else if(m
->nextg
!= nil
) {
1175 // New m started by matchmg.
1177 runtime_throw("invalid m state in scheduler");
1180 // Find (or wait for) g to run. Unlocks runtime_sched.
1181 gp
= nextgandunlock();
1182 gp
->readyonstop
= 0;
1183 gp
->status
= Grunning
;
1187 // Check whether the profiler needs to be turned on or off.
1188 hz
= runtime_sched
.profilehz
;
1189 if(m
->profilehz
!= hz
)
1190 runtime_resetcpuprofiler(hz
);
1195 // Enter scheduler. If g->status is Grunning,
1196 // re-queues g and runs everyone else who is waiting
1197 // before running g again. If g->status is Gmoribund,
1200 runtime_gosched(void)
1203 runtime_throw("gosched holding locks");
1205 runtime_throw("gosched of g0");
1206 runtime_mcall(schedule
);
1209 // The goroutine g is about to enter a system call.
1210 // Record that it's not using the cpu anymore.
1211 // This is called only from the go syscall library and cgocall,
1212 // not from the low-level system calls used by the runtime.
1214 // Entersyscall cannot split the stack: the runtime_gosave must
1215 // make g->sched refer to the caller's stack segment, because
1216 // entersyscall is going to return immediately after.
1217 // It's okay to call matchmg and notewakeup even after
1218 // decrementing mcpu, because we haven't released the
1219 // sched lock yet, so the garbage collector cannot be running.
1221 void runtime_entersyscall(void) __attribute__ ((no_split_stack
));
1224 runtime_entersyscall(void)
1228 if(m
->profilehz
> 0)
1229 runtime_setprof(false);
1231 // Leave SP around for gc and traceback.
1232 #ifdef USING_SPLIT_STACK
1233 g
->gcstack
= __splitstack_find(nil
, nil
, &g
->gcstack_size
,
1234 &g
->gcnext_segment
, &g
->gcnext_sp
,
1237 g
->gcnext_sp
= (byte
*) &v
;
1240 // Save the registers in the g structure so that any pointers
1241 // held in registers will be seen by the garbage collector.
1242 // We could use getcontext here, but setjmp is more efficient
1243 // because it doesn't need to save the signal mask.
1246 g
->status
= Gsyscall
;
1249 // The slow path inside the schedlock/schedunlock will get
1250 // through without stopping if it does:
1253 // waitstop && mcpu <= mcpumax not true
1254 // If we can do the same with a single atomic add,
1255 // then we can skip the locks.
1256 v
= runtime_xadd(&runtime_sched
.atomic
, -1<<mcpuShift
);
1257 if(!atomic_gwaiting(v
) && (!atomic_waitstop(v
) || atomic_mcpu(v
) > atomic_mcpumax(v
)))
1261 v
= runtime_atomicload(&runtime_sched
.atomic
);
1262 if(atomic_gwaiting(v
)) {
1264 v
= runtime_atomicload(&runtime_sched
.atomic
);
1266 if(atomic_waitstop(v
) && atomic_mcpu(v
) <= atomic_mcpumax(v
)) {
1267 runtime_xadd(&runtime_sched
.atomic
, -1<<waitstopShift
);
1268 runtime_notewakeup(&runtime_sched
.stopped
);
1274 // The goroutine g exited its system call.
1275 // Arrange for it to run on a cpu again.
1276 // This is called only from the go syscall library, not
1277 // from the low-level system calls used by the runtime.
1279 runtime_exitsyscall(void)
1285 // If we can do the mcpu++ bookkeeping and
1286 // find that we still have mcpu <= mcpumax, then we can
1287 // start executing Go code immediately, without having to
1288 // schedlock/schedunlock.
1289 // Also do fast return if any locks are held, so that
1290 // panic code can use syscalls to open a file.
1292 v
= runtime_xadd(&runtime_sched
.atomic
, (1<<mcpuShift
));
1293 if((m
->profilehz
== runtime_sched
.profilehz
&& atomic_mcpu(v
) <= atomic_mcpumax(v
)) || m
->locks
> 0) {
1294 // There's a cpu for us, so we can run.
1295 gp
->status
= Grunning
;
1296 // Garbage collector isn't running (since we are),
1297 // so okay to clear gcstack.
1298 #ifdef USING_SPLIT_STACK
1301 gp
->gcnext_sp
= nil
;
1302 runtime_memclr(gp
->gcregs
, sizeof gp
->gcregs
);
1304 if(m
->profilehz
> 0)
1305 runtime_setprof(true);
1309 // Tell scheduler to put g back on the run queue:
1310 // mostly equivalent to g->status = Grunning,
1311 // but keeps the garbage collector from thinking
1312 // that g is running right now, which it's not.
1313 gp
->readyonstop
= 1;
1315 // All the cpus are taken.
1316 // The scheduler will ready g and put this m to sleep.
1317 // When the scheduler takes g away from m,
1318 // it will undo the runtime_sched.mcpu++ above.
1321 // Gosched returned, so we're allowed to run now.
1322 // Delete the gcstack information that we left for
1323 // the garbage collector during the system call.
1324 // Must wait until now because until gosched returns
1325 // we don't know for sure that the garbage collector
1327 #ifdef USING_SPLIT_STACK
1330 gp
->gcnext_sp
= nil
;
1331 runtime_memclr(gp
->gcregs
, sizeof gp
->gcregs
);
1334 // Allocate a new g, with a stack big enough for stacksize bytes.
1336 runtime_malg(int32 stacksize
, byte
** ret_stack
, size_t* ret_stacksize
)
1340 newg
= runtime_malloc(sizeof(G
));
1341 if(stacksize
>= 0) {
1342 #if USING_SPLIT_STACK
1343 int dont_block_signals
= 0;
1345 *ret_stack
= __splitstack_makecontext(stacksize
,
1346 &newg
->stack_context
[0],
1348 __splitstack_block_signals_context(&newg
->stack_context
[0],
1349 &dont_block_signals
, nil
);
1351 *ret_stack
= runtime_mallocgc(stacksize
, FlagNoProfiling
|FlagNoGC
, 0, 0);
1352 *ret_stacksize
= stacksize
;
1353 newg
->gcinitial_sp
= *ret_stack
;
1354 newg
->gcstack_size
= stacksize
;
1355 runtime_xadd(&runtime_stacks_sys
, stacksize
);
1361 /* For runtime package testing. */
1363 void runtime_testing_entersyscall(void)
1364 __asm__("runtime.entersyscall");
1367 runtime_testing_entersyscall()
1369 runtime_entersyscall();
1372 void runtime_testing_exitsyscall(void)
1373 __asm__("runtime.exitsyscall");
1376 runtime_testing_exitsyscall()
1378 runtime_exitsyscall();
1382 __go_go(void (*fn
)(void*), void* arg
)
1390 if((newg
= gfget()) != nil
){
1391 #ifdef USING_SPLIT_STACK
1392 int dont_block_signals
= 0;
1394 sp
= __splitstack_resetcontext(&newg
->stack_context
[0],
1396 __splitstack_block_signals_context(&newg
->stack_context
[0],
1397 &dont_block_signals
, nil
);
1399 sp
= newg
->gcinitial_sp
;
1400 spsize
= newg
->gcstack_size
;
1402 runtime_throw("bad spsize in __go_go");
1403 newg
->gcnext_sp
= sp
;
1406 newg
= runtime_malg(StackMin
, &sp
, &spsize
);
1407 if(runtime_lastg
== nil
)
1408 runtime_allg
= newg
;
1410 runtime_lastg
->alllink
= newg
;
1411 runtime_lastg
= newg
;
1413 newg
->status
= Gwaiting
;
1414 newg
->waitreason
= "new goroutine";
1416 newg
->entry
= (byte
*)fn
;
1418 newg
->gopc
= (uintptr
)__builtin_return_address(0);
1420 runtime_sched
.gcount
++;
1421 runtime_sched
.goidgen
++;
1422 newg
->goid
= runtime_sched
.goidgen
;
1425 runtime_throw("nil g->stack0");
1428 // Avoid warnings about variables clobbered by
1430 byte
* volatile vsp
= sp
;
1431 size_t volatile vspsize
= spsize
;
1432 G
* volatile vnewg
= newg
;
1434 getcontext(&vnewg
->context
);
1435 vnewg
->context
.uc_stack
.ss_sp
= vsp
;
1436 #ifdef MAKECONTEXT_STACK_TOP
1437 vnewg
->context
.uc_stack
.ss_sp
+= vspsize
;
1439 vnewg
->context
.uc_stack
.ss_size
= vspsize
;
1440 makecontext(&vnewg
->context
, kickoff
, 0);
1442 newprocreadylocked(vnewg
);
1449 // Put on gfree list. Sched must be locked.
1453 g
->schedlink
= runtime_sched
.gfree
;
1454 runtime_sched
.gfree
= g
;
1457 // Get from gfree list. Sched must be locked.
1463 g
= runtime_sched
.gfree
;
1465 runtime_sched
.gfree
= g
->schedlink
;
1469 // Run all deferred functions for the current goroutine.
1475 while((d
= g
->defer
) != nil
) {
1482 g
->defer
= d
->__next
;
1487 void runtime_Goexit (void) asm ("runtime.Goexit");
1490 runtime_Goexit(void)
1496 void runtime_Gosched (void) asm ("runtime.Gosched");
1499 runtime_Gosched(void)
1504 // Implementation of runtime.GOMAXPROCS.
1505 // delete when scheduler is stronger
1507 runtime_gomaxprocsfunc(int32 n
)
1513 ret
= runtime_gomaxprocs
;
1516 if(n
> maxgomaxprocs
)
1518 runtime_gomaxprocs
= n
;
1519 if(runtime_gomaxprocs
> 1)
1520 runtime_singleproc
= false;
1521 if(runtime_gcwaiting
!= 0) {
1522 if(atomic_mcpumax(runtime_sched
.atomic
) != 1)
1523 runtime_throw("invalid mcpumax during gc");
1530 // If there are now fewer allowed procs
1531 // than procs running, stop.
1532 v
= runtime_atomicload(&runtime_sched
.atomic
);
1533 if((int32
)atomic_mcpu(v
) > n
) {
1538 // handle more procs
1545 runtime_LockOSThread(void)
1547 if(m
== &runtime_m0
&& runtime_sched
.init
) {
1548 runtime_sched
.lockmain
= true;
1556 runtime_UnlockOSThread(void)
1558 if(m
== &runtime_m0
&& runtime_sched
.init
) {
1559 runtime_sched
.lockmain
= false;
1567 runtime_lockedOSThread(void)
1569 return g
->lockedm
!= nil
&& m
->lockedg
!= nil
;
1572 // for testing of callbacks
1574 _Bool
runtime_golockedOSThread(void)
1575 asm("runtime.golockedOSThread");
1578 runtime_golockedOSThread(void)
1580 return runtime_lockedOSThread();
1583 // for testing of wire, unwire
1590 int32
runtime_NumGoroutine (void)
1591 __asm__ ("runtime.NumGoroutine");
1594 runtime_NumGoroutine()
1596 return runtime_sched
.gcount
;
1600 runtime_gcount(void)
1602 return runtime_sched
.gcount
;
1606 runtime_mcount(void)
1608 return runtime_sched
.mcount
;
1613 void (*fn
)(uintptr
*, int32
);
1618 // Called if we receive a SIGPROF signal.
1620 runtime_sigprof(uint8
*pc
__attribute__ ((unused
)),
1621 uint8
*sp
__attribute__ ((unused
)),
1622 uint8
*lr
__attribute__ ((unused
)),
1623 G
*gp
__attribute__ ((unused
)))
1627 if(prof
.fn
== nil
|| prof
.hz
== 0)
1630 runtime_lock(&prof
);
1631 if(prof
.fn
== nil
) {
1632 runtime_unlock(&prof
);
1635 n
= runtime_callers(0, prof
.pcbuf
, nelem(prof
.pcbuf
));
1637 prof
.fn(prof
.pcbuf
, n
);
1638 runtime_unlock(&prof
);
1641 // Arrange to call fn with a traceback hz times a second.
1643 runtime_setcpuprofilerate(void (*fn
)(uintptr
*, int32
), int32 hz
)
1645 // Force sane arguments.
1653 // Stop profiler on this cpu so that it is safe to lock prof.
1654 // if a profiling signal came in while we had prof locked,
1655 // it would deadlock.
1656 runtime_resetcpuprofiler(0);
1658 runtime_lock(&prof
);
1661 runtime_unlock(&prof
);
1662 runtime_lock(&runtime_sched
);
1663 runtime_sched
.profilehz
= hz
;
1664 runtime_unlock(&runtime_sched
);
1667 runtime_resetcpuprofiler(hz
);