+++ /dev/null
-// Copyright 2013 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build ignore
-
-package runtime
-
-import (
- "runtime/internal/atomic"
- "runtime/internal/sys"
- "unsafe"
-)
-
-/*
-Stack layout parameters.
-Included both by runtime (compiled via 6c) and linkers (compiled via gcc).
-
-The per-goroutine g->stackguard is set to point StackGuard bytes
-above the bottom of the stack. Each function compares its stack
-pointer against g->stackguard to check for overflow. To cut one
-instruction from the check sequence for functions with tiny frames,
-the stack is allowed to protrude StackSmall bytes below the stack
-guard. Functions with large frames don't bother with the check and
-always call morestack. The sequences are (for amd64, others are
-similar):
-
- guard = g->stackguard
- frame = function's stack frame size
- argsize = size of function arguments (call + return)
-
- stack frame size <= StackSmall:
- CMPQ guard, SP
- JHI 3(PC)
- MOVQ m->morearg, $(argsize << 32)
- CALL morestack(SB)
-
- stack frame size > StackSmall but < StackBig
- LEAQ (frame-StackSmall)(SP), R0
- CMPQ guard, R0
- JHI 3(PC)
- MOVQ m->morearg, $(argsize << 32)
- CALL morestack(SB)
-
- stack frame size >= StackBig:
- MOVQ m->morearg, $((argsize << 32) | frame)
- CALL morestack(SB)
-
-The bottom StackGuard - StackSmall bytes are important: there has
-to be enough room to execute functions that refuse to check for
-stack overflow, either because they need to be adjacent to the
-actual caller's frame (deferproc) or because they handle the imminent
-stack overflow (morestack).
-
-For example, deferproc might call malloc, which does one of the
-above checks (without allocating a full frame), which might trigger
-a call to morestack. This sequence needs to fit in the bottom
-section of the stack. On amd64, morestack's frame is 40 bytes, and
-deferproc's frame is 56 bytes. That fits well within the
-StackGuard - StackSmall bytes at the bottom.
-The linkers explore all possible call traces involving non-splitting
-functions to make sure that this limit cannot be violated.
-*/
-
-const (
- // StackSystem is a number of additional bytes to add
- // to each stack below the usual guard area for OS-specific
- // purposes like signal handling. Used on Windows, Plan 9,
- // and Darwin/ARM because they do not use a separate stack.
- _StackSystem = sys.GoosWindows*512*sys.PtrSize + sys.GoosPlan9*512 + sys.GoosDarwin*sys.GoarchArm*1024
-
- // The minimum size of stack used by Go code
- _StackMin = 2048
-
- // The minimum stack size to allocate.
- // The hackery here rounds FixedStack0 up to a power of 2.
- _FixedStack0 = _StackMin + _StackSystem
- _FixedStack1 = _FixedStack0 - 1
- _FixedStack2 = _FixedStack1 | (_FixedStack1 >> 1)
- _FixedStack3 = _FixedStack2 | (_FixedStack2 >> 2)
- _FixedStack4 = _FixedStack3 | (_FixedStack3 >> 4)
- _FixedStack5 = _FixedStack4 | (_FixedStack4 >> 8)
- _FixedStack6 = _FixedStack5 | (_FixedStack5 >> 16)
- _FixedStack = _FixedStack6 + 1
-
- // Functions that need frames bigger than this use an extra
- // instruction to do the stack split check, to avoid overflow
- // in case SP - framesize wraps below zero.
- // This value can be no bigger than the size of the unmapped
- // space at zero.
- _StackBig = 4096
-
- // The stack guard is a pointer this many bytes above the
- // bottom of the stack.
- _StackGuard = 880*sys.StackGuardMultiplier + _StackSystem
-
- // After a stack split check the SP is allowed to be this
- // many bytes below the stack guard. This saves an instruction
- // in the checking sequence for tiny frames.
- _StackSmall = 128
-
- // The maximum number of bytes that a chain of NOSPLIT
- // functions can use.
- _StackLimit = _StackGuard - _StackSystem - _StackSmall
-)
-
-// Goroutine preemption request.
-// Stored into g->stackguard0 to cause split stack check failure.
-// Must be greater than any real sp.
-// 0xfffffade in hex.
-const (
- _StackPreempt = uintptrMask & -1314
- _StackFork = uintptrMask & -1234
-)
-
-const (
- // stackDebug == 0: no logging
- // == 1: logging of per-stack operations
- // == 2: logging of per-frame operations
- // == 3: logging of per-word updates
- // == 4: logging of per-word reads
- stackDebug = 0
- stackFromSystem = 0 // allocate stacks from system memory instead of the heap
- stackFaultOnFree = 0 // old stacks are mapped noaccess to detect use after free
- stackPoisonCopy = 0 // fill stack that should not be accessed with garbage, to detect bad dereferences during copy
-
- stackCache = 1
-
- // check the BP links during traceback.
- debugCheckBP = false
-)
-
-const (
- uintptrMask = 1<<(8*sys.PtrSize) - 1
-
- // Goroutine preemption request.
- // Stored into g->stackguard0 to cause split stack check failure.
- // Must be greater than any real sp.
- // 0xfffffade in hex.
- stackPreempt = uintptrMask & -1314
-
- // Thread is forking.
- // Stored into g->stackguard0 to cause split stack check failure.
- // Must be greater than any real sp.
- stackFork = uintptrMask & -1234
-)
-
-// Global pool of spans that have free stacks.
-// Stacks are assigned an order according to size.
-// order = log_2(size/FixedStack)
-// There is a free list for each order.
-// TODO: one lock per order?
-var stackpool [_NumStackOrders]mSpanList
-var stackpoolmu mutex
-
-// Global pool of large stack spans.
-var stackLarge struct {
- lock mutex
- free [_MHeapMap_Bits]mSpanList // free lists by log_2(s.npages)
-}
-
-func stackinit() {
- if _StackCacheSize&_PageMask != 0 {
- throw("cache size must be a multiple of page size")
- }
- for i := range stackpool {
- stackpool[i].init()
- }
- for i := range stackLarge.free {
- stackLarge.free[i].init()
- }
-}
-
-// stacklog2 returns ⌊log_2(n)⌋.
-func stacklog2(n uintptr) int {
- log2 := 0
- for n > 1 {
- n >>= 1
- log2++
- }
- return log2
-}
-
-// Allocates a stack from the free pool. Must be called with
-// stackpoolmu held.
-func stackpoolalloc(order uint8) gclinkptr {
- list := &stackpool[order]
- s := list.first
- if s == nil {
- // no free stacks. Allocate another span worth.
- s = mheap_.allocStack(_StackCacheSize >> _PageShift)
- if s == nil {
- throw("out of memory")
- }
- if s.allocCount != 0 {
- throw("bad allocCount")
- }
- if s.stackfreelist.ptr() != nil {
- throw("bad stackfreelist")
- }
- for i := uintptr(0); i < _StackCacheSize; i += _FixedStack << order {
- x := gclinkptr(s.base() + i)
- x.ptr().next = s.stackfreelist
- s.stackfreelist = x
- }
- list.insert(s)
- }
- x := s.stackfreelist
- if x.ptr() == nil {
- throw("span has no free stacks")
- }
- s.stackfreelist = x.ptr().next
- s.allocCount++
- if s.stackfreelist.ptr() == nil {
- // all stacks in s are allocated.
- list.remove(s)
- }
- return x
-}
-
-// Adds stack x to the free pool. Must be called with stackpoolmu held.
-func stackpoolfree(x gclinkptr, order uint8) {
- s := mheap_.lookup(unsafe.Pointer(x))
- if s.state != _MSpanStack {
- throw("freeing stack not in a stack span")
- }
- if s.stackfreelist.ptr() == nil {
- // s will now have a free stack
- stackpool[order].insert(s)
- }
- x.ptr().next = s.stackfreelist
- s.stackfreelist = x
- s.allocCount--
- if gcphase == _GCoff && s.allocCount == 0 {
- // Span is completely free. Return it to the heap
- // immediately if we're sweeping.
- //
- // If GC is active, we delay the free until the end of
- // GC to avoid the following type of situation:
- //
- // 1) GC starts, scans a SudoG but does not yet mark the SudoG.elem pointer
- // 2) The stack that pointer points to is copied
- // 3) The old stack is freed
- // 4) The containing span is marked free
- // 5) GC attempts to mark the SudoG.elem pointer. The
- // marking fails because the pointer looks like a
- // pointer into a free span.
- //
- // By not freeing, we prevent step #4 until GC is done.
- stackpool[order].remove(s)
- s.stackfreelist = 0
- mheap_.freeStack(s)
- }
-}
-
-// stackcacherefill/stackcacherelease implement a global pool of stack segments.
-// The pool is required to prevent unlimited growth of per-thread caches.
-//
-//go:systemstack
-func stackcacherefill(c *mcache, order uint8) {
- if stackDebug >= 1 {
- print("stackcacherefill order=", order, "\n")
- }
-
- // Grab some stacks from the global cache.
- // Grab half of the allowed capacity (to prevent thrashing).
- var list gclinkptr
- var size uintptr
- lock(&stackpoolmu)
- for size < _StackCacheSize/2 {
- x := stackpoolalloc(order)
- x.ptr().next = list
- list = x
- size += _FixedStack << order
- }
- unlock(&stackpoolmu)
- c.stackcache[order].list = list
- c.stackcache[order].size = size
-}
-
-//go:systemstack
-func stackcacherelease(c *mcache, order uint8) {
- if stackDebug >= 1 {
- print("stackcacherelease order=", order, "\n")
- }
- x := c.stackcache[order].list
- size := c.stackcache[order].size
- lock(&stackpoolmu)
- for size > _StackCacheSize/2 {
- y := x.ptr().next
- stackpoolfree(x, order)
- x = y
- size -= _FixedStack << order
- }
- unlock(&stackpoolmu)
- c.stackcache[order].list = x
- c.stackcache[order].size = size
-}
-
-//go:systemstack
-func stackcache_clear(c *mcache) {
- if stackDebug >= 1 {
- print("stackcache clear\n")
- }
- lock(&stackpoolmu)
- for order := uint8(0); order < _NumStackOrders; order++ {
- x := c.stackcache[order].list
- for x.ptr() != nil {
- y := x.ptr().next
- stackpoolfree(x, order)
- x = y
- }
- c.stackcache[order].list = 0
- c.stackcache[order].size = 0
- }
- unlock(&stackpoolmu)
-}
-
-// stackalloc allocates an n byte stack.
-//
-// stackalloc must run on the system stack because it uses per-P
-// resources and must not split the stack.
-//
-//go:systemstack
-func stackalloc(n uint32) (stack, []stkbar) {
- // Stackalloc must be called on scheduler stack, so that we
- // never try to grow the stack during the code that stackalloc runs.
- // Doing so would cause a deadlock (issue 1547).
- thisg := getg()
- if thisg != thisg.m.g0 {
- throw("stackalloc not on scheduler stack")
- }
- if n&(n-1) != 0 {
- throw("stack size not a power of 2")
- }
- if stackDebug >= 1 {
- print("stackalloc ", n, "\n")
- }
-
- // Compute the size of stack barrier array.
- maxstkbar := gcMaxStackBarriers(int(n))
- nstkbar := unsafe.Sizeof(stkbar{}) * uintptr(maxstkbar)
- var stkbarSlice slice
-
- if debug.efence != 0 || stackFromSystem != 0 {
- v := sysAlloc(round(uintptr(n), _PageSize), &memstats.stacks_sys)
- if v == nil {
- throw("out of memory (stackalloc)")
- }
- top := uintptr(n) - nstkbar
- if maxstkbar != 0 {
- stkbarSlice = slice{add(v, top), 0, maxstkbar}
- }
- return stack{uintptr(v), uintptr(v) + top}, *(*[]stkbar)(unsafe.Pointer(&stkbarSlice))
- }
-
- // Small stacks are allocated with a fixed-size free-list allocator.
- // If we need a stack of a bigger size, we fall back on allocating
- // a dedicated span.
- var v unsafe.Pointer
- if stackCache != 0 && n < _FixedStack<<_NumStackOrders && n < _StackCacheSize {
- order := uint8(0)
- n2 := n
- for n2 > _FixedStack {
- order++
- n2 >>= 1
- }
- var x gclinkptr
- c := thisg.m.mcache
- if c == nil || thisg.m.preemptoff != "" || thisg.m.helpgc != 0 {
- // c == nil can happen in the guts of exitsyscall or
- // procresize. Just get a stack from the global pool.
- // Also don't touch stackcache during gc
- // as it's flushed concurrently.
- lock(&stackpoolmu)
- x = stackpoolalloc(order)
- unlock(&stackpoolmu)
- } else {
- x = c.stackcache[order].list
- if x.ptr() == nil {
- stackcacherefill(c, order)
- x = c.stackcache[order].list
- }
- c.stackcache[order].list = x.ptr().next
- c.stackcache[order].size -= uintptr(n)
- }
- v = unsafe.Pointer(x)
- } else {
- var s *mspan
- npage := uintptr(n) >> _PageShift
- log2npage := stacklog2(npage)
-
- // Try to get a stack from the large stack cache.
- lock(&stackLarge.lock)
- if !stackLarge.free[log2npage].isEmpty() {
- s = stackLarge.free[log2npage].first
- stackLarge.free[log2npage].remove(s)
- }
- unlock(&stackLarge.lock)
-
- if s == nil {
- // Allocate a new stack from the heap.
- s = mheap_.allocStack(npage)
- if s == nil {
- throw("out of memory")
- }
- }
- v = unsafe.Pointer(s.base())
- }
-
- if raceenabled {
- racemalloc(v, uintptr(n))
- }
- if msanenabled {
- msanmalloc(v, uintptr(n))
- }
- if stackDebug >= 1 {
- print(" allocated ", v, "\n")
- }
- top := uintptr(n) - nstkbar
- if maxstkbar != 0 {
- stkbarSlice = slice{add(v, top), 0, maxstkbar}
- }
- return stack{uintptr(v), uintptr(v) + top}, *(*[]stkbar)(unsafe.Pointer(&stkbarSlice))
-}
-
-// stackfree frees an n byte stack allocation at stk.
-//
-// stackfree must run on the system stack because it uses per-P
-// resources and must not split the stack.
-//
-//go:systemstack
-func stackfree(stk stack, n uintptr) {
- gp := getg()
- v := unsafe.Pointer(stk.lo)
- if n&(n-1) != 0 {
- throw("stack not a power of 2")
- }
- if stk.lo+n < stk.hi {
- throw("bad stack size")
- }
- if stackDebug >= 1 {
- println("stackfree", v, n)
- memclrNoHeapPointers(v, n) // for testing, clobber stack data
- }
- if debug.efence != 0 || stackFromSystem != 0 {
- if debug.efence != 0 || stackFaultOnFree != 0 {
- sysFault(v, n)
- } else {
- sysFree(v, n, &memstats.stacks_sys)
- }
- return
- }
- if msanenabled {
- msanfree(v, n)
- }
- if stackCache != 0 && n < _FixedStack<<_NumStackOrders && n < _StackCacheSize {
- order := uint8(0)
- n2 := n
- for n2 > _FixedStack {
- order++
- n2 >>= 1
- }
- x := gclinkptr(v)
- c := gp.m.mcache
- if c == nil || gp.m.preemptoff != "" || gp.m.helpgc != 0 {
- lock(&stackpoolmu)
- stackpoolfree(x, order)
- unlock(&stackpoolmu)
- } else {
- if c.stackcache[order].size >= _StackCacheSize {
- stackcacherelease(c, order)
- }
- x.ptr().next = c.stackcache[order].list
- c.stackcache[order].list = x
- c.stackcache[order].size += n
- }
- } else {
- s := mheap_.lookup(v)
- if s.state != _MSpanStack {
- println(hex(s.base()), v)
- throw("bad span state")
- }
- if gcphase == _GCoff {
- // Free the stack immediately if we're
- // sweeping.
- mheap_.freeStack(s)
- } else {
- // If the GC is running, we can't return a
- // stack span to the heap because it could be
- // reused as a heap span, and this state
- // change would race with GC. Add it to the
- // large stack cache instead.
- log2npage := stacklog2(s.npages)
- lock(&stackLarge.lock)
- stackLarge.free[log2npage].insert(s)
- unlock(&stackLarge.lock)
- }
- }
-}
-
-var maxstacksize uintptr = 1 << 20 // enough until runtime.main sets it for real
-
-var ptrnames = []string{
- 0: "scalar",
- 1: "ptr",
-}
-
-// Stack frame layout
-//
-// (x86)
-// +------------------+
-// | args from caller |
-// +------------------+ <- frame->argp
-// | return address |
-// +------------------+
-// | caller's BP (*) | (*) if framepointer_enabled && varp < sp
-// +------------------+ <- frame->varp
-// | locals |
-// +------------------+
-// | args to callee |
-// +------------------+ <- frame->sp
-//
-// (arm)
-// +------------------+
-// | args from caller |
-// +------------------+ <- frame->argp
-// | caller's retaddr |
-// +------------------+ <- frame->varp
-// | locals |
-// +------------------+
-// | args to callee |
-// +------------------+
-// | return address |
-// +------------------+ <- frame->sp
-
-type adjustinfo struct {
- old stack
- delta uintptr // ptr distance from old to new stack (newbase - oldbase)
- cache pcvalueCache
-
- // sghi is the highest sudog.elem on the stack.
- sghi uintptr
-}
-
-// Adjustpointer checks whether *vpp is in the old stack described by adjinfo.
-// If so, it rewrites *vpp to point into the new stack.
-func adjustpointer(adjinfo *adjustinfo, vpp unsafe.Pointer) {
- pp := (*uintptr)(vpp)
- p := *pp
- if stackDebug >= 4 {
- print(" ", pp, ":", hex(p), "\n")
- }
- if adjinfo.old.lo <= p && p < adjinfo.old.hi {
- *pp = p + adjinfo.delta
- if stackDebug >= 3 {
- print(" adjust ptr ", pp, ":", hex(p), " -> ", hex(*pp), "\n")
- }
- }
-}
-
-// Information from the compiler about the layout of stack frames.
-type bitvector struct {
- n int32 // # of bits
- bytedata *uint8
-}
-
-type gobitvector struct {
- n uintptr
- bytedata []uint8
-}
-
-func gobv(bv bitvector) gobitvector {
- return gobitvector{
- uintptr(bv.n),
- (*[1 << 30]byte)(unsafe.Pointer(bv.bytedata))[:(bv.n+7)/8],
- }
-}
-
-func ptrbit(bv *gobitvector, i uintptr) uint8 {
- return (bv.bytedata[i/8] >> (i % 8)) & 1
-}
-
-// bv describes the memory starting at address scanp.
-// Adjust any pointers contained therein.
-func adjustpointers(scanp unsafe.Pointer, cbv *bitvector, adjinfo *adjustinfo, f *_func) {
- bv := gobv(*cbv)
- minp := adjinfo.old.lo
- maxp := adjinfo.old.hi
- delta := adjinfo.delta
- num := bv.n
- // If this frame might contain channel receive slots, use CAS
- // to adjust pointers. If the slot hasn't been received into
- // yet, it may contain stack pointers and a concurrent send
- // could race with adjusting those pointers. (The sent value
- // itself can never contain stack pointers.)
- useCAS := uintptr(scanp) < adjinfo.sghi
- for i := uintptr(0); i < num; i++ {
- if stackDebug >= 4 {
- print(" ", add(scanp, i*sys.PtrSize), ":", ptrnames[ptrbit(&bv, i)], ":", hex(*(*uintptr)(add(scanp, i*sys.PtrSize))), " # ", i, " ", bv.bytedata[i/8], "\n")
- }
- if ptrbit(&bv, i) == 1 {
- pp := (*uintptr)(add(scanp, i*sys.PtrSize))
- retry:
- p := *pp
- if f != nil && 0 < p && p < minLegalPointer && debug.invalidptr != 0 {
- // Looks like a junk value in a pointer slot.
- // Live analysis wrong?
- getg().m.traceback = 2
- print("runtime: bad pointer in frame ", funcname(f), " at ", pp, ": ", hex(p), "\n")
- throw("invalid pointer found on stack")
- }
- if minp <= p && p < maxp {
- if stackDebug >= 3 {
- print("adjust ptr ", hex(p), " ", funcname(f), "\n")
- }
- if useCAS {
- ppu := (*unsafe.Pointer)(unsafe.Pointer(pp))
- if !atomic.Casp1(ppu, unsafe.Pointer(p), unsafe.Pointer(p+delta)) {
- goto retry
- }
- } else {
- *pp = p + delta
- }
- }
- }
- }
-}
-
-// Note: the argument/return area is adjusted by the callee.
-func adjustframe(frame *stkframe, arg unsafe.Pointer) bool {
- adjinfo := (*adjustinfo)(arg)
- targetpc := frame.continpc
- if targetpc == 0 {
- // Frame is dead.
- return true
- }
- f := frame.fn
- if stackDebug >= 2 {
- print(" adjusting ", funcname(f), " frame=[", hex(frame.sp), ",", hex(frame.fp), "] pc=", hex(frame.pc), " continpc=", hex(frame.continpc), "\n")
- }
- if f.entry == systemstack_switchPC {
- // A special routine at the bottom of stack of a goroutine that does an systemstack call.
- // We will allow it to be copied even though we don't
- // have full GC info for it (because it is written in asm).
- return true
- }
- if targetpc != f.entry {
- targetpc--
- }
- pcdata := pcdatavalue(f, _PCDATA_StackMapIndex, targetpc, &adjinfo.cache)
- if pcdata == -1 {
- pcdata = 0 // in prologue
- }
-
- // Adjust local variables if stack frame has been allocated.
- size := frame.varp - frame.sp
- var minsize uintptr
- switch sys.ArchFamily {
- case sys.ARM64:
- minsize = sys.SpAlign
- default:
- minsize = sys.MinFrameSize
- }
- if size > minsize {
- var bv bitvector
- stackmap := (*stackmap)(funcdata(f, _FUNCDATA_LocalsPointerMaps))
- if stackmap == nil || stackmap.n <= 0 {
- print("runtime: frame ", funcname(f), " untyped locals ", hex(frame.varp-size), "+", hex(size), "\n")
- throw("missing stackmap")
- }
- // Locals bitmap information, scan just the pointers in locals.
- if pcdata < 0 || pcdata >= stackmap.n {
- // don't know where we are
- print("runtime: pcdata is ", pcdata, " and ", stackmap.n, " locals stack map entries for ", funcname(f), " (targetpc=", targetpc, ")\n")
- throw("bad symbol table")
- }
- bv = stackmapdata(stackmap, pcdata)
- size = uintptr(bv.n) * sys.PtrSize
- if stackDebug >= 3 {
- print(" locals ", pcdata, "/", stackmap.n, " ", size/sys.PtrSize, " words ", bv.bytedata, "\n")
- }
- adjustpointers(unsafe.Pointer(frame.varp-size), &bv, adjinfo, f)
- }
-
- // Adjust saved base pointer if there is one.
- if sys.ArchFamily == sys.AMD64 && frame.argp-frame.varp == 2*sys.RegSize {
- if !framepointer_enabled {
- print("runtime: found space for saved base pointer, but no framepointer experiment\n")
- print("argp=", hex(frame.argp), " varp=", hex(frame.varp), "\n")
- throw("bad frame layout")
- }
- if stackDebug >= 3 {
- print(" saved bp\n")
- }
- if debugCheckBP {
- // Frame pointers should always point to the next higher frame on
- // the Go stack (or be nil, for the top frame on the stack).
- bp := *(*uintptr)(unsafe.Pointer(frame.varp))
- if bp != 0 && (bp < adjinfo.old.lo || bp >= adjinfo.old.hi) {
- println("runtime: found invalid frame pointer")
- print("bp=", hex(bp), " min=", hex(adjinfo.old.lo), " max=", hex(adjinfo.old.hi), "\n")
- throw("bad frame pointer")
- }
- }
- adjustpointer(adjinfo, unsafe.Pointer(frame.varp))
- }
-
- // Adjust arguments.
- if frame.arglen > 0 {
- var bv bitvector
- if frame.argmap != nil {
- bv = *frame.argmap
- } else {
- stackmap := (*stackmap)(funcdata(f, _FUNCDATA_ArgsPointerMaps))
- if stackmap == nil || stackmap.n <= 0 {
- print("runtime: frame ", funcname(f), " untyped args ", frame.argp, "+", frame.arglen, "\n")
- throw("missing stackmap")
- }
- if pcdata < 0 || pcdata >= stackmap.n {
- // don't know where we are
- print("runtime: pcdata is ", pcdata, " and ", stackmap.n, " args stack map entries for ", funcname(f), " (targetpc=", targetpc, ")\n")
- throw("bad symbol table")
- }
- bv = stackmapdata(stackmap, pcdata)
- }
- if stackDebug >= 3 {
- print(" args\n")
- }
- adjustpointers(unsafe.Pointer(frame.argp), &bv, adjinfo, nil)
- }
- return true
-}
-
-func adjustctxt(gp *g, adjinfo *adjustinfo) {
- adjustpointer(adjinfo, unsafe.Pointer(&gp.sched.ctxt))
- if !framepointer_enabled {
- return
- }
- if debugCheckBP {
- bp := gp.sched.bp
- if bp != 0 && (bp < adjinfo.old.lo || bp >= adjinfo.old.hi) {
- println("runtime: found invalid top frame pointer")
- print("bp=", hex(bp), " min=", hex(adjinfo.old.lo), " max=", hex(adjinfo.old.hi), "\n")
- throw("bad top frame pointer")
- }
- }
- adjustpointer(adjinfo, unsafe.Pointer(&gp.sched.bp))
-}
-
-func adjustdefers(gp *g, adjinfo *adjustinfo) {
- // Adjust defer argument blocks the same way we adjust active stack frames.
- tracebackdefers(gp, adjustframe, noescape(unsafe.Pointer(adjinfo)))
-
- // Adjust pointers in the Defer structs.
- // Defer structs themselves are never on the stack.
- for d := gp._defer; d != nil; d = d.link {
- adjustpointer(adjinfo, unsafe.Pointer(&d.fn))
- adjustpointer(adjinfo, unsafe.Pointer(&d.sp))
- adjustpointer(adjinfo, unsafe.Pointer(&d._panic))
- }
-}
-
-func adjustpanics(gp *g, adjinfo *adjustinfo) {
- // Panics are on stack and already adjusted.
- // Update pointer to head of list in G.
- adjustpointer(adjinfo, unsafe.Pointer(&gp._panic))
-}
-
-func adjustsudogs(gp *g, adjinfo *adjustinfo) {
- // the data elements pointed to by a SudoG structure
- // might be in the stack.
- for s := gp.waiting; s != nil; s = s.waitlink {
- adjustpointer(adjinfo, unsafe.Pointer(&s.elem))
- adjustpointer(adjinfo, unsafe.Pointer(&s.selectdone))
- }
-}
-
-func adjuststkbar(gp *g, adjinfo *adjustinfo) {
- for i := int(gp.stkbarPos); i < len(gp.stkbar); i++ {
- adjustpointer(adjinfo, unsafe.Pointer(&gp.stkbar[i].savedLRPtr))
- }
-}
-
-func fillstack(stk stack, b byte) {
- for p := stk.lo; p < stk.hi; p++ {
- *(*byte)(unsafe.Pointer(p)) = b
- }
-}
-
-func findsghi(gp *g, stk stack) uintptr {
- var sghi uintptr
- for sg := gp.waiting; sg != nil; sg = sg.waitlink {
- p := uintptr(sg.elem) + uintptr(sg.c.elemsize)
- if stk.lo <= p && p < stk.hi && p > sghi {
- sghi = p
- }
- p = uintptr(unsafe.Pointer(sg.selectdone)) + unsafe.Sizeof(sg.selectdone)
- if stk.lo <= p && p < stk.hi && p > sghi {
- sghi = p
- }
- }
- return sghi
-}
-
-// syncadjustsudogs adjusts gp's sudogs and copies the part of gp's
-// stack they refer to while synchronizing with concurrent channel
-// operations. It returns the number of bytes of stack copied.
-func syncadjustsudogs(gp *g, used uintptr, adjinfo *adjustinfo) uintptr {
- if gp.waiting == nil {
- return 0
- }
-
- // Lock channels to prevent concurrent send/receive.
- // It's important that we *only* do this for async
- // copystack; otherwise, gp may be in the middle of
- // putting itself on wait queues and this would
- // self-deadlock.
- var lastc *hchan
- for sg := gp.waiting; sg != nil; sg = sg.waitlink {
- if sg.c != lastc {
- lock(&sg.c.lock)
- }
- lastc = sg.c
- }
-
- // Adjust sudogs.
- adjustsudogs(gp, adjinfo)
-
- // Copy the part of the stack the sudogs point in to
- // while holding the lock to prevent races on
- // send/receive slots.
- var sgsize uintptr
- if adjinfo.sghi != 0 {
- oldBot := adjinfo.old.hi - used
- newBot := oldBot + adjinfo.delta
- sgsize = adjinfo.sghi - oldBot
- memmove(unsafe.Pointer(newBot), unsafe.Pointer(oldBot), sgsize)
- }
-
- // Unlock channels.
- lastc = nil
- for sg := gp.waiting; sg != nil; sg = sg.waitlink {
- if sg.c != lastc {
- unlock(&sg.c.lock)
- }
- lastc = sg.c
- }
-
- return sgsize
-}
-
-// Copies gp's stack to a new stack of a different size.
-// Caller must have changed gp status to Gcopystack.
-//
-// If sync is true, this is a self-triggered stack growth and, in
-// particular, no other G may be writing to gp's stack (e.g., via a
-// channel operation). If sync is false, copystack protects against
-// concurrent channel operations.
-func copystack(gp *g, newsize uintptr, sync bool) {
- if gp.syscallsp != 0 {
- throw("stack growth not allowed in system call")
- }
- old := gp.stack
- if old.lo == 0 {
- throw("nil stackbase")
- }
- used := old.hi - gp.sched.sp
-
- // allocate new stack
- new, newstkbar := stackalloc(uint32(newsize))
- if stackPoisonCopy != 0 {
- fillstack(new, 0xfd)
- }
- if stackDebug >= 1 {
- print("copystack gp=", gp, " [", hex(old.lo), " ", hex(old.hi-used), " ", hex(old.hi), "]/", gp.stackAlloc, " -> [", hex(new.lo), " ", hex(new.hi-used), " ", hex(new.hi), "]/", newsize, "\n")
- }
-
- // Compute adjustment.
- var adjinfo adjustinfo
- adjinfo.old = old
- adjinfo.delta = new.hi - old.hi
-
- // Adjust sudogs, synchronizing with channel ops if necessary.
- ncopy := used
- if sync {
- adjustsudogs(gp, &adjinfo)
- } else {
- // sudogs can point in to the stack. During concurrent
- // shrinking, these areas may be written to. Find the
- // highest such pointer so we can handle everything
- // there and below carefully. (This shouldn't be far
- // from the bottom of the stack, so there's little
- // cost in handling everything below it carefully.)
- adjinfo.sghi = findsghi(gp, old)
-
- // Synchronize with channel ops and copy the part of
- // the stack they may interact with.
- ncopy -= syncadjustsudogs(gp, used, &adjinfo)
- }
-
- // Copy the stack (or the rest of it) to the new location
- memmove(unsafe.Pointer(new.hi-ncopy), unsafe.Pointer(old.hi-ncopy), ncopy)
-
- // Disallow sigprof scans of this stack and block if there's
- // one in progress.
- gcLockStackBarriers(gp)
-
- // Adjust remaining structures that have pointers into stacks.
- // We have to do most of these before we traceback the new
- // stack because gentraceback uses them.
- adjustctxt(gp, &adjinfo)
- adjustdefers(gp, &adjinfo)
- adjustpanics(gp, &adjinfo)
- adjuststkbar(gp, &adjinfo)
- if adjinfo.sghi != 0 {
- adjinfo.sghi += adjinfo.delta
- }
-
- // copy old stack barriers to new stack barrier array
- newstkbar = newstkbar[:len(gp.stkbar)]
- copy(newstkbar, gp.stkbar)
-
- // Swap out old stack for new one
- gp.stack = new
- gp.stackguard0 = new.lo + _StackGuard // NOTE: might clobber a preempt request
- gp.sched.sp = new.hi - used
- oldsize := gp.stackAlloc
- gp.stackAlloc = newsize
- gp.stkbar = newstkbar
- gp.stktopsp += adjinfo.delta
-
- // Adjust pointers in the new stack.
- gentraceback(^uintptr(0), ^uintptr(0), 0, gp, 0, nil, 0x7fffffff, adjustframe, noescape(unsafe.Pointer(&adjinfo)), 0)
-
- gcUnlockStackBarriers(gp)
-
- // free old stack
- if stackPoisonCopy != 0 {
- fillstack(old, 0xfc)
- }
- stackfree(old, oldsize)
-}
-
-// round x up to a power of 2.
-func round2(x int32) int32 {
- s := uint(0)
- for 1<<s < x {
- s++
- }
- return 1 << s
-}
-
-// Called from runtime·morestack when more stack is needed.
-// Allocate larger stack and relocate to new stack.
-// Stack growth is multiplicative, for constant amortized cost.
-//
-// g->atomicstatus will be Grunning or Gscanrunning upon entry.
-// If the GC is trying to stop this g then it will set preemptscan to true.
-//
-// ctxt is the value of the context register on morestack. newstack
-// will write it to g.sched.ctxt.
-func newstack(ctxt unsafe.Pointer) {
- thisg := getg()
- // TODO: double check all gp. shouldn't be getg().
- if thisg.m.morebuf.g.ptr().stackguard0 == stackFork {
- throw("stack growth after fork")
- }
- if thisg.m.morebuf.g.ptr() != thisg.m.curg {
- print("runtime: newstack called from g=", hex(thisg.m.morebuf.g), "\n"+"\tm=", thisg.m, " m->curg=", thisg.m.curg, " m->g0=", thisg.m.g0, " m->gsignal=", thisg.m.gsignal, "\n")
- morebuf := thisg.m.morebuf
- traceback(morebuf.pc, morebuf.sp, morebuf.lr, morebuf.g.ptr())
- throw("runtime: wrong goroutine in newstack")
- }
-
- gp := thisg.m.curg
- // Write ctxt to gp.sched. We do this here instead of in
- // morestack so it has the necessary write barrier.
- gp.sched.ctxt = ctxt
-
- if thisg.m.curg.throwsplit {
- // Update syscallsp, syscallpc in case traceback uses them.
- morebuf := thisg.m.morebuf
- gp.syscallsp = morebuf.sp
- gp.syscallpc = morebuf.pc
- print("runtime: newstack sp=", hex(gp.sched.sp), " stack=[", hex(gp.stack.lo), ", ", hex(gp.stack.hi), "]\n",
- "\tmorebuf={pc:", hex(morebuf.pc), " sp:", hex(morebuf.sp), " lr:", hex(morebuf.lr), "}\n",
- "\tsched={pc:", hex(gp.sched.pc), " sp:", hex(gp.sched.sp), " lr:", hex(gp.sched.lr), " ctxt:", gp.sched.ctxt, "}\n")
-
- traceback(morebuf.pc, morebuf.sp, morebuf.lr, gp)
- throw("runtime: stack split at bad time")
- }
-
- morebuf := thisg.m.morebuf
- thisg.m.morebuf.pc = 0
- thisg.m.morebuf.lr = 0
- thisg.m.morebuf.sp = 0
- thisg.m.morebuf.g = 0
-
- // NOTE: stackguard0 may change underfoot, if another thread
- // is about to try to preempt gp. Read it just once and use that same
- // value now and below.
- preempt := atomic.Loaduintptr(&gp.stackguard0) == stackPreempt
-
- // Be conservative about where we preempt.
- // We are interested in preempting user Go code, not runtime code.
- // If we're holding locks, mallocing, or preemption is disabled, don't
- // preempt.
- // This check is very early in newstack so that even the status change
- // from Grunning to Gwaiting and back doesn't happen in this case.
- // That status change by itself can be viewed as a small preemption,
- // because the GC might change Gwaiting to Gscanwaiting, and then
- // this goroutine has to wait for the GC to finish before continuing.
- // If the GC is in some way dependent on this goroutine (for example,
- // it needs a lock held by the goroutine), that small preemption turns
- // into a real deadlock.
- if preempt {
- if thisg.m.locks != 0 || thisg.m.mallocing != 0 || thisg.m.preemptoff != "" || thisg.m.p.ptr().status != _Prunning {
- // Let the goroutine keep running for now.
- // gp->preempt is set, so it will be preempted next time.
- gp.stackguard0 = gp.stack.lo + _StackGuard
- gogo(&gp.sched) // never return
- }
- }
-
- if gp.stack.lo == 0 {
- throw("missing stack in newstack")
- }
- sp := gp.sched.sp
- if sys.ArchFamily == sys.AMD64 || sys.ArchFamily == sys.I386 {
- // The call to morestack cost a word.
- sp -= sys.PtrSize
- }
- if stackDebug >= 1 || sp < gp.stack.lo {
- print("runtime: newstack sp=", hex(sp), " stack=[", hex(gp.stack.lo), ", ", hex(gp.stack.hi), "]\n",
- "\tmorebuf={pc:", hex(morebuf.pc), " sp:", hex(morebuf.sp), " lr:", hex(morebuf.lr), "}\n",
- "\tsched={pc:", hex(gp.sched.pc), " sp:", hex(gp.sched.sp), " lr:", hex(gp.sched.lr), " ctxt:", gp.sched.ctxt, "}\n")
- }
- if sp < gp.stack.lo {
- print("runtime: gp=", gp, ", gp->status=", hex(readgstatus(gp)), "\n ")
- print("runtime: split stack overflow: ", hex(sp), " < ", hex(gp.stack.lo), "\n")
- throw("runtime: split stack overflow")
- }
-
- if preempt {
- if gp == thisg.m.g0 {
- throw("runtime: preempt g0")
- }
- if thisg.m.p == 0 && thisg.m.locks == 0 {
- throw("runtime: g is running but p is not")
- }
- // Synchronize with scang.
- casgstatus(gp, _Grunning, _Gwaiting)
- if gp.preemptscan {
- for !castogscanstatus(gp, _Gwaiting, _Gscanwaiting) {
- // Likely to be racing with the GC as
- // it sees a _Gwaiting and does the
- // stack scan. If so, gcworkdone will
- // be set and gcphasework will simply
- // return.
- }
- if !gp.gcscandone {
- // gcw is safe because we're on the
- // system stack.
- gcw := &gp.m.p.ptr().gcw
- scanstack(gp, gcw)
- if gcBlackenPromptly {
- gcw.dispose()
- }
- gp.gcscandone = true
- }
- gp.preemptscan = false
- gp.preempt = false
- casfrom_Gscanstatus(gp, _Gscanwaiting, _Gwaiting)
- // This clears gcscanvalid.
- casgstatus(gp, _Gwaiting, _Grunning)
- gp.stackguard0 = gp.stack.lo + _StackGuard
- gogo(&gp.sched) // never return
- }
-
- // Act like goroutine called runtime.Gosched.
- casgstatus(gp, _Gwaiting, _Grunning)
- gopreempt_m(gp) // never return
- }
-
- // Allocate a bigger segment and move the stack.
- oldsize := int(gp.stackAlloc)
- newsize := oldsize * 2
- if uintptr(newsize) > maxstacksize {
- print("runtime: goroutine stack exceeds ", maxstacksize, "-byte limit\n")
- throw("stack overflow")
- }
-
- // The goroutine must be executing in order to call newstack,
- // so it must be Grunning (or Gscanrunning).
- casgstatus(gp, _Grunning, _Gcopystack)
-
- // The concurrent GC will not scan the stack while we are doing the copy since
- // the gp is in a Gcopystack status.
- copystack(gp, uintptr(newsize), true)
- if stackDebug >= 1 {
- print("stack grow done\n")
- }
- casgstatus(gp, _Gcopystack, _Grunning)
- gogo(&gp.sched)
-}
-
-//go:nosplit
-func nilfunc() {
- *(*uint8)(nil) = 0
-}
-
-// adjust Gobuf as if it executed a call to fn
-// and then did an immediate gosave.
-func gostartcallfn(gobuf *gobuf, fv *funcval) {
- var fn unsafe.Pointer
- if fv != nil {
- fn = unsafe.Pointer(fv.fn)
- } else {
- fn = unsafe.Pointer(funcPC(nilfunc))
- }
- gostartcall(gobuf, fn, unsafe.Pointer(fv))
-}
-
-// Maybe shrink the stack being used by gp.
-// Called at garbage collection time.
-// gp must be stopped, but the world need not be.
-func shrinkstack(gp *g) {
- gstatus := readgstatus(gp)
- if gstatus&^_Gscan == _Gdead {
- if gp.stack.lo != 0 {
- // Free whole stack - it will get reallocated
- // if G is used again.
- stackfree(gp.stack, gp.stackAlloc)
- gp.stack.lo = 0
- gp.stack.hi = 0
- gp.stkbar = nil
- gp.stkbarPos = 0
- }
- return
- }
- if gp.stack.lo == 0 {
- throw("missing stack in shrinkstack")
- }
- if gstatus&_Gscan == 0 {
- throw("bad status in shrinkstack")
- }
-
- if debug.gcshrinkstackoff > 0 {
- return
- }
- if gp.startpc == gcBgMarkWorkerPC {
- // We're not allowed to shrink the gcBgMarkWorker
- // stack (see gcBgMarkWorker for explanation).
- return
- }
-
- oldsize := gp.stackAlloc
- newsize := oldsize / 2
- // Don't shrink the allocation below the minimum-sized stack
- // allocation.
- if newsize < _FixedStack {
- return
- }
- // Compute how much of the stack is currently in use and only
- // shrink the stack if gp is using less than a quarter of its
- // current stack. The currently used stack includes everything
- // down to the SP plus the stack guard space that ensures
- // there's room for nosplit functions.
- avail := gp.stack.hi - gp.stack.lo
- if used := gp.stack.hi - gp.sched.sp + _StackLimit; used >= avail/4 {
- return
- }
-
- // We can't copy the stack if we're in a syscall.
- // The syscall might have pointers into the stack.
- if gp.syscallsp != 0 {
- return
- }
- if sys.GoosWindows != 0 && gp.m != nil && gp.m.libcallsp != 0 {
- return
- }
-
- if stackDebug > 0 {
- print("shrinking stack ", oldsize, "->", newsize, "\n")
- }
-
- copystack(gp, newsize, false)
-}
-
-// freeStackSpans frees unused stack spans at the end of GC.
-func freeStackSpans() {
- lock(&stackpoolmu)
-
- // Scan stack pools for empty stack spans.
- for order := range stackpool {
- list := &stackpool[order]
- for s := list.first; s != nil; {
- next := s.next
- if s.allocCount == 0 {
- list.remove(s)
- s.stackfreelist = 0
- mheap_.freeStack(s)
- }
- s = next
- }
- }
-
- unlock(&stackpoolmu)
-
- // Free large stack spans.
- lock(&stackLarge.lock)
- for i := range stackLarge.free {
- for s := stackLarge.free[i].first; s != nil; {
- next := s.next
- stackLarge.free[i].remove(s)
- mheap_.freeStack(s)
- s = next
- }
- }
- unlock(&stackLarge.lock)
-}
-
-//go:nosplit
-func morestackc() {
- systemstack(func() {
- throw("attempt to execute C code on Go stack")
- })
-}
projects / gcc.git / commitdiff