1 // Copyright 2014 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
12 // Should be a built-in for unsafe.Pointer?
14 func add(p unsafe.Pointer, x uintptr) unsafe.Pointer {
15 return unsafe.Pointer(uintptr(p) + x)
18 // getg returns the pointer to the current g.
19 // The compiler rewrites calls to this function into instructions
20 // that fetch the g directly (from TLS or from the dedicated register).
23 // mcall switches from the g to the g0 stack and invokes fn(g),
24 // where g is the goroutine that made the call.
25 // mcall saves g's current PC/SP in g->sched so that it can be restored later.
26 // It is up to fn to arrange for that later execution, typically by recording
27 // g in a data structure, causing something to call ready(g) later.
28 // mcall returns to the original goroutine g later, when g has been rescheduled.
29 // fn must not return at all; typically it ends by calling schedule, to let the m
30 // run other goroutines.
32 // mcall can only be called from g stacks (not g0, not gsignal).
34 // This must NOT be go:noescape: if fn is a stack-allocated closure,
35 // fn puts g on a run queue, and g executes before fn returns, the
36 // closure will be invalidated while it is still executing.
37 func mcall(fn func(*g))
39 // systemstack runs fn on a system stack.
41 // It is common to use a func literal as the argument, in order
42 // to share inputs and outputs with the code around the call
46 // systemstack(func() {
51 // For the gc toolchain this permits running a function that requires
52 // additional stack space in a context where the stack can not be
53 // split. We don't really need additional stack space in gccgo, since
54 // stack splitting is handled separately. But to keep things looking
55 // the same, we do switch to the g0 stack here if necessary.
56 func systemstack(fn func()) {
59 if gp == mp.g0 || gp == mp.gsignal {
61 } else if gp == mp.curg {
62 fn1 := func(origg *g) {
66 mcall(*(*func(*g))(noescape(unsafe.Pointer(&fn1))))
72 var badsystemstackMsg = "fatal: systemstack called from unexpected goroutine"
75 //go:nowritebarrierrec
76 func badsystemstack() {
77 sp := stringStructOf(&badsystemstackMsg)
78 write(2, sp.str, int32(sp.len))
81 // memclrNoHeapPointers clears n bytes starting at ptr.
83 // Usually you should use typedmemclr. memclrNoHeapPointers should be
84 // used only when the caller knows that *ptr contains no heap pointers
87 // *ptr is initialized memory and its type is pointer-free, or
89 // *ptr is uninitialized memory (e.g., memory that's being reused
90 // for a new allocation) and hence contains only "junk".
92 // The (CPU-specific) implementations of this function are in memclr_*.s.
94 func memclrNoHeapPointers(ptr unsafe.Pointer, n uintptr)
96 //go:linkname reflect_memclrNoHeapPointers reflect.memclrNoHeapPointers
97 func reflect_memclrNoHeapPointers(ptr unsafe.Pointer, n uintptr) {
98 memclrNoHeapPointers(ptr, n)
101 // memmove copies n bytes from "from" to "to".
103 func memmove(to, from unsafe.Pointer, n uintptr)
105 //go:linkname reflect_memmove reflect.memmove
106 func reflect_memmove(to, from unsafe.Pointer, n uintptr) {
111 //extern __builtin_memcmp
112 func memcmp(a, b unsafe.Pointer, size uintptr) int32
114 // exported value for testing
115 var hashLoad = float32(loadFactorNum) / float32(loadFactorDen)
118 func fastrand() uint32 {
120 // Implement xorshift64+: 2 32-bit xorshift sequences added together.
121 // Shift triplet [17,7,16] was calculated as indicated in Marsaglia's
122 // Xorshift paper: https://www.jstatsoft.org/article/view/v008i14/xorshift.pdf
123 // This generator passes the SmallCrush suite, part of TestU01 framework:
124 // http://simul.iro.umontreal.ca/testu01/tu01.html
125 s1, s0 := mp.fastrand[0], mp.fastrand[1]
127 s1 = s1 ^ s0 ^ s1>>7 ^ s0>>16
128 mp.fastrand[0], mp.fastrand[1] = s0, s1
133 func fastrandn(n uint32) uint32 {
134 // This is similar to fastrand() % n, but faster.
135 // See https://lemire.me/blog/2016/06/27/a-fast-alternative-to-the-modulo-reduction/
136 return uint32(uint64(fastrand()) * uint64(n) >> 32)
139 //go:linkname sync_fastrand sync.fastrand
140 func sync_fastrand() uint32 { return fastrand() }
144 func memequal(a, b unsafe.Pointer, size uintptr) bool
146 // noescape hides a pointer from escape analysis. noescape is
147 // the identity function but escape analysis doesn't think the
148 // output depends on the input. noescape is inlined and currently
149 // compiles down to zero instructions.
152 func noescape(p unsafe.Pointer) unsafe.Pointer {
154 return unsafe.Pointer(x ^ 0)
158 func jmpdefer(fv *funcval, argp uintptr)
159 func exit1(code int32)
162 //extern __builtin_trap
167 //go:linkname reflectcall runtime.reflectcall
169 func reflectcall(fntype *functype, fn *funcval, isInterface, isMethod bool, params, results *unsafe.Pointer)
171 func procyield(cycles uint32)
173 type neverCallThisFunction struct{}
175 // goexit is the return stub at the top of every goroutine call stack.
176 // Each goroutine stack is constructed as if goexit called the
177 // goroutine's entry point function, so that when the entry point
178 // function returns, it will return to goexit, which will call goexit1
179 // to perform the actual exit.
181 // This function must never be called directly. Call goexit1 instead.
182 // gentraceback assumes that goexit terminates the stack. A direct
183 // call on the stack will cause gentraceback to stop walking the stack
184 // prematurely and if there is leftover state it may panic.
185 func goexit(neverCallThisFunction)
187 // publicationBarrier performs a store/store barrier (a "publication"
188 // or "export" barrier). Some form of synchronization is required
189 // between initializing an object and making that object accessible to
190 // another processor. Without synchronization, the initialization
191 // writes and the "publication" write may be reordered, allowing the
192 // other processor to follow the pointer and observe an uninitialized
193 // object. In general, higher-level synchronization should be used,
194 // such as locking or an atomic pointer write. publicationBarrier is
195 // for when those aren't an option, such as in the implementation of
196 // the memory manager.
198 // There's no corresponding barrier for the read side because the read
199 // side naturally has a data dependency order. All architectures that
200 // Go supports or seems likely to ever support automatically enforce
201 // data dependency ordering.
202 func publicationBarrier()
204 // getcallerpc returns the program counter (PC) of its caller's caller.
205 // getcallersp returns the stack pointer (SP) of its caller's caller.
206 // The implementation may be a compiler intrinsic; there is not
207 // necessarily code implementing this on every platform.
211 // func f(arg1, arg2, arg3 int) {
212 // pc := getcallerpc()
213 // sp := getcallersp()
216 // These two lines find the PC and SP immediately following
217 // the call to f (where f will return).
219 // The call to getcallerpc and getcallersp must be done in the
220 // frame being asked about.
222 // The result of getcallersp is correct at the time of the return,
223 // but it may be invalidated by any subsequent call to a function
224 // that might relocate the stack in order to grow or shrink it.
225 // A general rule is that the result of getcallersp should be used
226 // immediately and can only be passed to nosplit functions.
229 func getcallerpc() uintptr
232 func getcallersp() uintptr // implemented as an intrinsic on all platforms
234 func asmcgocall(fn, arg unsafe.Pointer) int32 {
239 // argp used in Defer structs when there is no argp.
240 const _NoArgs = ^uintptr(0)
242 //extern __builtin_prefetch
243 func prefetch(addr unsafe.Pointer, rw int32, locality int32)
245 func prefetcht0(addr uintptr) {
246 prefetch(unsafe.Pointer(addr), 0, 3)
249 func prefetcht1(addr uintptr) {
250 prefetch(unsafe.Pointer(addr), 0, 2)
253 func prefetcht2(addr uintptr) {
254 prefetch(unsafe.Pointer(addr), 0, 1)
257 func prefetchnta(addr uintptr) {
258 prefetch(unsafe.Pointer(addr), 0, 0)
261 // round n up to a multiple of a. a must be a power of 2.
262 func round(n, a uintptr) uintptr {
263 return (n + a - 1) &^ (a - 1)
266 // checkASM returns whether assembly runtime checks have passed.
267 func checkASM() bool {
271 func eqstring(x, y string) bool {
272 a := stringStructOf(&x)
273 b := stringStructOf(&y)
280 return memequal(a.str, b.str, uintptr(a.len))
283 // For gccgo this is in the C code.
286 //extern __go_syscall6
287 func syscall(trap uintptr, a1, a2, a3, a4, a5, a6 uintptr) uintptr
289 // For gccgo, to communicate from the C code to the Go code.
290 //go:linkname setIsCgo runtime.setIsCgo
295 // For gccgo, to communicate from the C code to the Go code.
296 //go:linkname setSupportAES runtime.setSupportAES
297 func setSupportAES(v bool) {
304 // For gccgo these are written in C.
306 func entersyscallblock()
308 // For gccgo to call from C code, so that the C code and the Go code
309 // can share the memstats variable for now.
310 //go:linkname getMstats runtime.getMstats
311 func getMstats() *mstats {
315 // Get signal trampoline, written in C.
316 func getSigtramp() uintptr
318 // The sa_handler field is generally hidden in a union, so use C accessors.
320 func getSigactionHandler(*_sigaction) uintptr
323 func setSigactionHandler(*_sigaction, uintptr)
325 // Retrieve fields from the siginfo_t and ucontext_t pointers passed
326 // to a signal handler using C, as they are often hidden in a union.
327 // Returns and, if available, PC where signal occurred.
328 func getSiginfo(*_siginfo_t, unsafe.Pointer) (sigaddr uintptr, sigpc uintptr)
330 // Implemented in C for gccgo.
331 func dumpregs(*_siginfo_t, unsafe.Pointer)
333 // Implemented in C for gccgo.
334 func setRandomNumber(uint32)
336 // Temporary for gccgo until we port proc.go.
337 //go:linkname getsched runtime.getsched
338 func getsched() *schedt {
342 // Temporary for gccgo until we port proc.go.
343 //go:linkname getCgoHasExtraM runtime.getCgoHasExtraM
344 func getCgoHasExtraM() *bool {
348 // Temporary for gccgo until we port proc.go.
349 //go:linkname getAllP runtime.getAllP
354 // Temporary for gccgo until we port proc.go.
355 //go:linkname allocg runtime.allocg
360 // Temporary for gccgo until we port the garbage collector.
361 //go:linkname getallglen runtime.getallglen
362 func getallglen() uintptr {
366 // Temporary for gccgo until we port the garbage collector.
367 //go:linkname getallg runtime.getallg
368 func getallg(i int) *g {
372 // Temporary for gccgo until we port the garbage collector.
373 //go:linkname getallm runtime.getallm
378 // Throw and rethrow an exception.
379 func throwException()
380 func rethrowException()
382 // Fetch the size and required alignment of the _Unwind_Exception type
383 // used by the stack unwinder.
384 func unwindExceptionSize() uintptr
386 // Temporary for gccgo until C code no longer needs it.
388 //go:linkname getPanicking runtime.getPanicking
389 func getPanicking() uint32 {
393 // Called by C code to set the number of CPUs.
394 //go:linkname setncpu runtime.setncpu
395 func setncpu(n int32) {
399 // Called by C code to set the page size.
400 //go:linkname setpagesize runtime.setpagesize
401 func setpagesize(s uintptr) {
402 if physPageSize == 0 {
407 // Called by C code during library initialization.
408 //go:linkname runtime_m0 runtime.runtime_m0
409 func runtime_m0() *m {
413 // Temporary for gccgo until we port mgc.go.
414 //go:linkname runtime_g0 runtime.runtime_g0
415 func runtime_g0() *g {
419 const uintptrMask = 1<<(8*sys.PtrSize) - 1
421 type bitvector struct {
426 // ptrbit returns the i'th bit in bv.
427 // ptrbit is less efficient than iterating directly over bitvector bits,
428 // and should only be used in non-performance-critical code.
429 // See adjustpointers for an example of a high-efficiency walk of a bitvector.
430 func (bv *bitvector) ptrbit(i uintptr) uint8 {
431 b := *(addb(bv.bytedata, i/8))
432 return (b >> (i % 8)) & 1
435 // bool2int returns 0 if x is false or 1 if x is true.
436 func bool2int(x bool) int {
443 // abort crashes the runtime in situations where even throw might not
444 // work. In general it should do something a debugger will recognize
445 // (e.g., an INT3 on x86). A crash in abort is recognized by the
446 // signal handler, which will attempt to tear down the runtime
450 // usestackmaps is true if stack map (precise stack scan) is enabled.
451 var usestackmaps bool
453 // probestackmaps detects whether there are stack maps.
454 //go:linkname probestackmaps runtime.probestackmaps
455 func probestackmaps() bool
457 // For the math/bits packages for gccgo.
458 //go:linkname getDivideError runtime.getDivideError
459 func getDivideError() error {
463 // For the math/bits packages for gccgo.
464 //go:linkname getOverflowError runtime.getOverflowError
465 func getOverflowError() error {