diff --git a/src/runtime/alg.go b/src/runtime/alg.go index e9ed59503f..e367bc5b2f 100644 --- a/src/runtime/alg.go +++ b/src/runtime/alg.go @@ -314,9 +314,6 @@ const hashRandomBytes = 32 var aeskeysched [hashRandomBytes]byte -//go:noescape -func get_random_data(rnd *unsafe.Pointer, n *int32) - func init() { if theGoos == "nacl" { return diff --git a/src/runtime/os1_darwin.go b/src/runtime/os1_darwin.go new file mode 100644 index 0000000000..b30ffbe830 --- /dev/null +++ b/src/runtime/os1_darwin.go @@ -0,0 +1,423 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +import "unsafe" + +//extern SigTabTT runtime·sigtab[]; + +var sigset_none = uint32(0) +var sigset_all = ^uint32(0) + +func unimplemented(name string) { + println(name, "not implemented") + *(*int)(unsafe.Pointer(uintptr(1231))) = 1231 +} + +//go:nosplit +func semawakeup(mp *m) { + mach_semrelease(uint32(mp.waitsema)) +} + +//go:nosplit +func semacreate() uintptr { + var x uintptr + onM(func() { + x = uintptr(mach_semcreate()) + }) + return x +} + +// BSD interface for threading. +func osinit() { + // bsdthread_register delayed until end of goenvs so that we + // can look at the environment first. + + // Use sysctl to fetch hw.ncpu. + mib := [2]uint32{6, 3} + out := uint32(0) + nout := unsafe.Sizeof(out) + ret := sysctl(&mib[0], 2, (*byte)(unsafe.Pointer(&out)), &nout, nil, 0) + if ret >= 0 { + ncpu = int32(out) + } +} + +var urandom_data [_HashRandomBytes]byte +var urandom_dev = []byte("/dev/random\x00") + +//go:nosplit +func get_random_data(rnd *unsafe.Pointer, rnd_len *int32) { + fd := open(&urandom_dev[0], 0 /* O_RDONLY */, 0) + if read(fd, unsafe.Pointer(&urandom_data), _HashRandomBytes) == _HashRandomBytes { + *rnd = unsafe.Pointer(&urandom_data[0]) + *rnd_len = _HashRandomBytes + } else { + *rnd = nil + *rnd_len = 0 + } + close(fd) +} + +func goenvs() { + goenvs_unix() + + // Register our thread-creation callback (see sys_darwin_{amd64,386}.s) + // but only if we're not using cgo. If we are using cgo we need + // to let the C pthread library install its own thread-creation callback. + if !iscgo { + if bsdthread_register() != 0 { + if gogetenv("DYLD_INSERT_LIBRARIES") != "" { + gothrow("runtime: bsdthread_register error (unset DYLD_INSERT_LIBRARIES)") + } + gothrow("runtime: bsdthread_register error") + } + } +} + +func newosproc(mp *m, stk unsafe.Pointer) { + mp.tls[0] = uintptr(mp.id) // so 386 asm can find it + if false { + print("newosproc stk=", stk, " m=", mp, " g=", mp.g0, " id=", mp.id, "/", int(mp.tls[0]), " ostk=", &mp, "\n") + } + + var oset uint32 + sigprocmask(_SIG_SETMASK, &sigset_all, &oset) + errno := bsdthread_create(stk, mp, mp.g0, funcPC(mstart)) + sigprocmask(_SIG_SETMASK, &oset, nil) + + if errno < 0 { + print("runtime: failed to create new OS thread (have ", mcount(), " already; errno=", -errno, ")\n") + gothrow("runtime.newosproc") + } +} + +// Called to initialize a new m (including the bootstrap m). +// Called on the parent thread (main thread in case of bootstrap), can allocate memory. +func mpreinit(mp *m) { + mp.gsignal = malg(32 * 1024) // OS X wants >= 8K + mp.gsignal.m = mp +} + +// Called to initialize a new m (including the bootstrap m). +// Called on the new thread, can not allocate memory. +func minit() { + // Initialize signal handling. + _g_ := getg() + signalstack((*byte)(unsafe.Pointer(_g_.m.gsignal.stack.lo)), 32*1024) + sigprocmask(_SIG_SETMASK, &sigset_none, nil) +} + +// Called from dropm to undo the effect of an minit. +func unminit() { + signalstack(nil, 0) +} + +// Mach IPC, to get at semaphores +// Definitions are in /usr/include/mach on a Mac. + +func macherror(r int32, fn string) { + print("mach error ", fn, ": ", r, "\n") + gothrow("mach error") +} + +const _DebugMach = false + +var zerondr machndr + +func mach_msgh_bits(a, b uint32) uint32 { + return a | b<<8 +} + +func mach_msg(h *machheader, op int32, send_size, rcv_size, rcv_name, timeout, notify uint32) int32 { + // TODO: Loop on interrupt. + return mach_msg_trap(unsafe.Pointer(h), op, send_size, rcv_size, rcv_name, timeout, notify) +} + +// Mach RPC (MIG) +const ( + _MinMachMsg = 48 + _MachReply = 100 +) + +type codemsg struct { + h machheader + ndr machndr + code int32 +} + +func machcall(h *machheader, maxsize int32, rxsize int32) int32 { + _g_ := getg() + port := _g_.m.machport + if port == 0 { + port = mach_reply_port() + _g_.m.machport = port + } + + h.msgh_bits |= mach_msgh_bits(_MACH_MSG_TYPE_COPY_SEND, _MACH_MSG_TYPE_MAKE_SEND_ONCE) + h.msgh_local_port = port + h.msgh_reserved = 0 + id := h.msgh_id + + if _DebugMach { + p := (*[10000]unsafe.Pointer)(unsafe.Pointer(h)) + print("send:\t") + var i uint32 + for i = 0; i < h.msgh_size/uint32(unsafe.Sizeof(p[0])); i++ { + print(" ", p[i]) + if i%8 == 7 { + print("\n\t") + } + } + if i%8 != 0 { + print("\n") + } + } + ret := mach_msg(h, _MACH_SEND_MSG|_MACH_RCV_MSG, h.msgh_size, uint32(maxsize), port, 0, 0) + if ret != 0 { + if _DebugMach { + print("mach_msg error ", ret, "\n") + } + return ret + } + if _DebugMach { + p := (*[10000]unsafe.Pointer)(unsafe.Pointer(h)) + var i uint32 + for i = 0; i < h.msgh_size/uint32(unsafe.Sizeof(p[0])); i++ { + print(" ", p[i]) + if i%8 == 7 { + print("\n\t") + } + } + if i%8 != 0 { + print("\n") + } + } + if h.msgh_id != id+_MachReply { + if _DebugMach { + print("mach_msg _MachReply id mismatch ", h.msgh_id, " != ", id+_MachReply, "\n") + } + return -303 // MIG_REPLY_MISMATCH + } + // Look for a response giving the return value. + // Any call can send this back with an error, + // and some calls only have return values so they + // send it back on success too. I don't quite see how + // you know it's one of these and not the full response + // format, so just look if the message is right. + c := (*codemsg)(unsafe.Pointer(h)) + if uintptr(h.msgh_size) == unsafe.Sizeof(*c) && h.msgh_bits&_MACH_MSGH_BITS_COMPLEX == 0 { + if _DebugMach { + print("mig result ", c.code, "\n") + } + return c.code + } + if h.msgh_size != uint32(rxsize) { + if _DebugMach { + print("mach_msg _MachReply size mismatch ", h.msgh_size, " != ", rxsize, "\n") + } + return -307 // MIG_ARRAY_TOO_LARGE + } + return 0 +} + +// Semaphores! + +const ( + tmach_semcreate = 3418 + rmach_semcreate = tmach_semcreate + _MachReply + + tmach_semdestroy = 3419 + rmach_semdestroy = tmach_semdestroy + _MachReply + + _KERN_ABORTED = 14 + _KERN_OPERATION_TIMED_OUT = 49 +) + +type tmach_semcreatemsg struct { + h machheader + ndr machndr + policy int32 + value int32 +} + +type rmach_semcreatemsg struct { + h machheader + body machbody + semaphore machport +} + +type tmach_semdestroymsg struct { + h machheader + body machbody + semaphore machport +} + +func mach_semcreate() uint32 { + var m [256]uint8 + tx := (*tmach_semcreatemsg)(unsafe.Pointer(&m)) + rx := (*rmach_semcreatemsg)(unsafe.Pointer(&m)) + + tx.h.msgh_bits = 0 + tx.h.msgh_size = uint32(unsafe.Sizeof(*tx)) + tx.h.msgh_remote_port = mach_task_self() + tx.h.msgh_id = tmach_semcreate + tx.ndr = zerondr + + tx.policy = 0 // 0 = SYNC_POLICY_FIFO + tx.value = 0 + + for { + r := machcall(&tx.h, int32(unsafe.Sizeof(m)), int32(unsafe.Sizeof(*rx))) + if r == 0 { + break + } + if r == _KERN_ABORTED { // interrupted + continue + } + macherror(r, "semaphore_create") + } + if rx.body.msgh_descriptor_count != 1 { + unimplemented("mach_semcreate desc count") + } + return rx.semaphore.name +} + +func mach_semdestroy(sem uint32) { + var m [256]uint8 + tx := (*tmach_semdestroymsg)(unsafe.Pointer(&m)) + + tx.h.msgh_bits = _MACH_MSGH_BITS_COMPLEX + tx.h.msgh_size = uint32(unsafe.Sizeof(*tx)) + tx.h.msgh_remote_port = mach_task_self() + tx.h.msgh_id = tmach_semdestroy + tx.body.msgh_descriptor_count = 1 + tx.semaphore.name = sem + tx.semaphore.disposition = _MACH_MSG_TYPE_MOVE_SEND + tx.semaphore._type = 0 + + for { + r := machcall(&tx.h, int32(unsafe.Sizeof(m)), 0) + if r == 0 { + break + } + if r == _KERN_ABORTED { // interrupted + continue + } + macherror(r, "semaphore_destroy") + } +} + +// The other calls have simple system call traps in sys_darwin_{amd64,386}.s + +func mach_semaphore_wait(sema uint32) int32 +func mach_semaphore_timedwait(sema, sec, nsec uint32) int32 +func mach_semaphore_signal(sema uint32) int32 +func mach_semaphore_signal_all(sema uint32) int32 + +func semasleep1(ns int64) int32 { + _g_ := getg() + + if ns >= 0 { + var nsecs int32 + secs := timediv(ns, 1000000000, &nsecs) + r := mach_semaphore_timedwait(uint32(_g_.m.waitsema), uint32(secs), uint32(nsecs)) + if r == _KERN_ABORTED || r == _KERN_OPERATION_TIMED_OUT { + return -1 + } + if r != 0 { + macherror(r, "semaphore_wait") + } + return 0 + } + + for { + r := mach_semaphore_wait(uint32(_g_.m.waitsema)) + if r == 0 { + break + } + if r == _KERN_ABORTED { // interrupted + continue + } + macherror(r, "semaphore_wait") + } + return 0 +} + +//go:nosplit +func semasleep(ns int64) int32 { + var r int32 + onM(func() { + r = semasleep1(ns) + }) + return r +} + +//go:nosplit +func mach_semrelease(sem uint32) { + for { + r := mach_semaphore_signal(sem) + if r == 0 { + break + } + if r == _KERN_ABORTED { // interrupted + continue + } + + // mach_semrelease must be completely nosplit, + // because it is called from Go code. + // If we're going to die, start that process on the m stack + // to avoid a Go stack split. + onM_signalok(func() { macherror(r, "semaphore_signal") }) + } +} + +//go:nosplit +func osyield() { + usleep(1) +} + +func memlimit() uintptr { + // NOTE(rsc): Could use getrlimit here, + // like on FreeBSD or Linux, but Darwin doesn't enforce + // ulimit -v, so it's unclear why we'd try to stay within + // the limit. + return 0 +} + +func setsig(i int32, fn uintptr, restart bool) { + var sa sigactiont + memclr(unsafe.Pointer(&sa), unsafe.Sizeof(sa)) + sa.sa_flags = _SA_SIGINFO | _SA_ONSTACK + if restart { + sa.sa_flags |= _SA_RESTART + } + sa.sa_mask = ^uint32(0) + sa.sa_tramp = unsafe.Pointer(funcPC(sigtramp)) // runtime·sigtramp's job is to call into real handler + *(*uintptr)(unsafe.Pointer(&sa.__sigaction_u)) = fn + sigaction(uint32(i), &sa, nil) +} + +func getsig(i int32) uintptr { + var sa sigactiont + memclr(unsafe.Pointer(&sa), unsafe.Sizeof(sa)) + sigaction(uint32(i), nil, &sa) + return *(*uintptr)(unsafe.Pointer(&sa.__sigaction_u)) +} + +func signalstack(p *byte, n int32) { + var st stackt + st.ss_sp = p + st.ss_size = uintptr(n) + st.ss_flags = 0 + if p == nil { + st.ss_flags = _SS_DISABLE + } + sigaltstack(&st, nil) +} + +func unblocksignals() { + sigprocmask(_SIG_SETMASK, &sigset_none, nil) +} diff --git a/src/runtime/os1_linux.go b/src/runtime/os1_linux.go new file mode 100644 index 0000000000..311d0ab1ab --- /dev/null +++ b/src/runtime/os1_linux.go @@ -0,0 +1,277 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +import "unsafe" + +var sigset_none sigset +var sigset_all sigset = sigset{^uint32(0), ^uint32(0)} + +// Linux futex. +// +// futexsleep(uint32 *addr, uint32 val) +// futexwakeup(uint32 *addr) +// +// Futexsleep atomically checks if *addr == val and if so, sleeps on addr. +// Futexwakeup wakes up threads sleeping on addr. +// Futexsleep is allowed to wake up spuriously. + +const ( + _FUTEX_WAIT = 0 + _FUTEX_WAKE = 1 +) + +// Atomically, +// if(*addr == val) sleep +// Might be woken up spuriously; that's allowed. +// Don't sleep longer than ns; ns < 0 means forever. +//go:nosplit +func futexsleep(addr *uint32, val uint32, ns int64) { + var ts timespec + + // Some Linux kernels have a bug where futex of + // FUTEX_WAIT returns an internal error code + // as an errno. Libpthread ignores the return value + // here, and so can we: as it says a few lines up, + // spurious wakeups are allowed. + if ns < 0 { + futex(unsafe.Pointer(addr), _FUTEX_WAIT, val, nil, nil, 0) + return + } + + // NOTE: tv_nsec is int64 on amd64, so this assumes a little-endian system. + ts.tv_nsec = 0 + ts.set_sec(timediv(ns, 1000000000, (*int32)(unsafe.Pointer(&ts.tv_nsec)))) + futex(unsafe.Pointer(addr), _FUTEX_WAIT, val, unsafe.Pointer(&ts), nil, 0) +} + +// If any procs are sleeping on addr, wake up at most cnt. +//go:nosplit +func futexwakeup(addr *uint32, cnt uint32) { + ret := futex(unsafe.Pointer(addr), _FUTEX_WAKE, cnt, nil, nil, 0) + if ret >= 0 { + return + } + + // I don't know that futex wakeup can return + // EAGAIN or EINTR, but if it does, it would be + // safe to loop and call futex again. + onM_signalok(func() { + print("futexwakeup addr=", addr, " returned ", ret, "\n") + }) + + *(*int32)(unsafe.Pointer(uintptr(0x1006))) = 0x1006 +} + +func getproccount() int32 { + var buf [16]uintptr + r := sched_getaffinity(0, unsafe.Sizeof(buf), &buf[0]) + n := int32(0) + for _, v := range buf[:r/ptrSize] { + for i := 0; i < 64; i++ { + n += int32(v & 1) + v >>= 1 + } + } + if n == 0 { + n = 1 + } + return n +} + +// Clone, the Linux rfork. +const ( + _CLONE_VM = 0x100 + _CLONE_FS = 0x200 + _CLONE_FILES = 0x400 + _CLONE_SIGHAND = 0x800 + _CLONE_PTRACE = 0x2000 + _CLONE_VFORK = 0x4000 + _CLONE_PARENT = 0x8000 + _CLONE_THREAD = 0x10000 + _CLONE_NEWNS = 0x20000 + _CLONE_SYSVSEM = 0x40000 + _CLONE_SETTLS = 0x80000 + _CLONE_PARENT_SETTID = 0x100000 + _CLONE_CHILD_CLEARTID = 0x200000 + _CLONE_UNTRACED = 0x800000 + _CLONE_CHILD_SETTID = 0x1000000 + _CLONE_STOPPED = 0x2000000 + _CLONE_NEWUTS = 0x4000000 + _CLONE_NEWIPC = 0x8000000 +) + +func newosproc(mp *m, stk unsafe.Pointer) { + /* + * note: strace gets confused if we use CLONE_PTRACE here. + */ + var flags int32 = _CLONE_VM | /* share memory */ + _CLONE_FS | /* share cwd, etc */ + _CLONE_FILES | /* share fd table */ + _CLONE_SIGHAND | /* share sig handler table */ + _CLONE_THREAD /* revisit - okay for now */ + + mp.tls[0] = uintptr(mp.id) // so 386 asm can find it + if false { + print("newosproc stk=", stk, " m=", mp, " g=", mp.g0, " clone=", funcPC(clone), " id=", mp.id, "/", mp.tls[0], " ostk=", &mp, "\n") + } + + // Disable signals during clone, so that the new thread starts + // with signals disabled. It will enable them in minit. + var oset sigset + rtsigprocmask(_SIG_SETMASK, &sigset_all, &oset, int32(unsafe.Sizeof(oset))) + ret := clone(flags, stk, unsafe.Pointer(mp), unsafe.Pointer(mp.g0), unsafe.Pointer(funcPC(mstart))) + rtsigprocmask(_SIG_SETMASK, &oset, nil, int32(unsafe.Sizeof(oset))) + + if ret < 0 { + print("runtime: failed to create new OS thread (have ", mcount(), " already; errno=", -ret, ")\n") + gothrow("newosproc") + } +} + +func osinit() { + ncpu = getproccount() +} + +// Random bytes initialized at startup. These come +// from the ELF AT_RANDOM auxiliary vector (vdso_linux_amd64.c). +// byte* runtime·startup_random_data; +// uint32 runtime·startup_random_data_len; + +var urandom_data [_HashRandomBytes]byte +var urandom_dev = []byte("/dev/random\x00") + +//go:nosplit +func get_random_data(rnd *unsafe.Pointer, rnd_len *int32) { + if startup_random_data != nil { + *rnd = unsafe.Pointer(startup_random_data) + *rnd_len = int32(startup_random_data_len) + return + } + fd := open(&urandom_dev[0], 0 /* O_RDONLY */, 0) + if read(fd, unsafe.Pointer(&urandom_data), _HashRandomBytes) == _HashRandomBytes { + *rnd = unsafe.Pointer(&urandom_data[0]) + *rnd_len = _HashRandomBytes + } else { + *rnd = nil + *rnd_len = 0 + } + close(fd) +} + +func goenvs() { + goenvs_unix() +} + +// Called to initialize a new m (including the bootstrap m). +// Called on the parent thread (main thread in case of bootstrap), can allocate memory. +func mpreinit(mp *m) { + mp.gsignal = malg(32 * 1024) // Linux wants >= 2K + mp.gsignal.m = mp +} + +// Called to initialize a new m (including the bootstrap m). +// Called on the new thread, can not allocate memory. +func minit() { + // Initialize signal handling. + _g_ := getg() + signalstack((*byte)(unsafe.Pointer(_g_.m.gsignal.stack.lo)), 32*1024) + rtsigprocmask(_SIG_SETMASK, &sigset_none, nil, int32(unsafe.Sizeof(sigset_none))) +} + +// Called from dropm to undo the effect of an minit. +func unminit() { + signalstack(nil, 0) +} + +func memlimit() uintptr { + /* + TODO: Convert to Go when something actually uses the result. + + Rlimit rl; + extern byte runtime·text[], runtime·end[]; + uintptr used; + + if(runtime·getrlimit(RLIMIT_AS, &rl) != 0) + return 0; + if(rl.rlim_cur >= 0x7fffffff) + return 0; + + // Estimate our VM footprint excluding the heap. + // Not an exact science: use size of binary plus + // some room for thread stacks. + used = runtime·end - runtime·text + (64<<20); + if(used >= rl.rlim_cur) + return 0; + + // If there's not at least 16 MB left, we're probably + // not going to be able to do much. Treat as no limit. + rl.rlim_cur -= used; + if(rl.rlim_cur < (16<<20)) + return 0; + + return rl.rlim_cur - used; + */ + + return 0 +} + +//#ifdef GOARCH_386 +//#define sa_handler k_sa_handler +//#endif + +func sigreturn() +func sigtramp() + +func setsig(i int32, fn uintptr, restart bool) { + var sa sigactiont + memclr(unsafe.Pointer(&sa), unsafe.Sizeof(sa)) + sa.sa_flags = _SA_SIGINFO | _SA_ONSTACK | _SA_RESTORER + if restart { + sa.sa_flags |= _SA_RESTART + } + sa.sa_mask = ^uint64(0) + // Although Linux manpage says "sa_restorer element is obsolete and + // should not be used". x86_64 kernel requires it. Only use it on + // x86. + if GOARCH == "386" || GOARCH == "amd64" { + sa.sa_restorer = funcPC(sigreturn) + } + if fn == funcPC(sighandler) { + fn = funcPC(sigtramp) + } + sa.sa_handler = fn + if rt_sigaction(uintptr(i), &sa, nil, unsafe.Sizeof(sa.sa_mask)) != 0 { + gothrow("rt_sigaction failure") + } +} + +func getsig(i int32) uintptr { + var sa sigactiont + + memclr(unsafe.Pointer(&sa), unsafe.Sizeof(sa)) + if rt_sigaction(uintptr(i), nil, &sa, unsafe.Sizeof(sa.sa_mask)) != 0 { + gothrow("rt_sigaction read failure") + } + if sa.sa_handler == funcPC(sigtramp) { + return funcPC(sighandler) + } + return sa.sa_handler +} + +func signalstack(p *byte, n int32) { + var st sigaltstackt + st.ss_sp = p + st.ss_size = uintptr(n) + st.ss_flags = 0 + if p == nil { + st.ss_flags = _SS_DISABLE + } + sigaltstack(&st, nil) +} + +func unblocksignals() { + rtsigprocmask(_SIG_SETMASK, &sigset_none, nil, int32(unsafe.Sizeof(sigset_none))) +} diff --git a/src/runtime/os2_darwin.go b/src/runtime/os2_darwin.go new file mode 100644 index 0000000000..542bd74219 --- /dev/null +++ b/src/runtime/os2_darwin.go @@ -0,0 +1,14 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +const ( + _NSIG = 32 + _SI_USER = 0 /* empirically true, but not what headers say */ + _SIG_BLOCK = 1 + _SIG_UNBLOCK = 2 + _SIG_SETMASK = 3 + _SS_DISABLE = 4 +) diff --git a/src/runtime/os2_linux.go b/src/runtime/os2_linux.go new file mode 100644 index 0000000000..eaa9f0e833 --- /dev/null +++ b/src/runtime/os2_linux.go @@ -0,0 +1,23 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +const ( + _SS_DISABLE = 2 + _NSIG = 65 + _SI_USER = 0 + _SIG_SETMASK = 2 + _RLIMIT_AS = 9 +) + +// It's hard to tease out exactly how big a Sigset is, but +// rt_sigprocmask crashes if we get it wrong, so if binaries +// are running, this is right. +type sigset [2]uint32 + +type rlimit struct { + rlim_cur uintptr + rlim_max uintptr +} diff --git a/src/runtime/os_darwin.c b/src/runtime/os_darwin.c deleted file mode 100644 index bbd29282b0..0000000000 --- a/src/runtime/os_darwin.c +++ /dev/null @@ -1,567 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -#include "runtime.h" -#include "defs_GOOS_GOARCH.h" -#include "os_GOOS.h" -#include "signal_unix.h" -#include "stack.h" -#include "textflag.h" - -extern SigTab runtime·sigtab[]; - -static Sigset sigset_none; -static Sigset sigset_all = ~(Sigset)0; - -static void -unimplemented(int8 *name) -{ - runtime·prints(name); - runtime·prints(" not implemented\n"); - *(int32*)1231 = 1231; -} - -#pragma textflag NOSPLIT -void -runtime·semawakeup(M *mp) -{ - runtime·mach_semrelease(mp->waitsema); -} - -static void -semacreate(void) -{ - g->m->scalararg[0] = runtime·mach_semcreate(); -} - -#pragma textflag NOSPLIT -uintptr -runtime·semacreate(void) -{ - uintptr x; - void (*fn)(void); - - fn = semacreate; - runtime·onM(&fn); - x = g->m->scalararg[0]; - g->m->scalararg[0] = 0; - return x; -} - -// BSD interface for threading. -void -runtime·osinit(void) -{ - // bsdthread_register delayed until end of goenvs so that we - // can look at the environment first. - - // Use sysctl to fetch hw.ncpu. - uint32 mib[2]; - uint32 out; - int32 ret; - uintptr nout; - - mib[0] = 6; - mib[1] = 3; - nout = sizeof out; - out = 0; - ret = runtime·sysctl(mib, 2, (byte*)&out, &nout, nil, 0); - if(ret >= 0) - runtime·ncpu = out; -} - -#pragma textflag NOSPLIT -void -runtime·get_random_data(byte **rnd, int32 *rnd_len) -{ - #pragma dataflag NOPTR - static byte urandom_data[HashRandomBytes]; - int32 fd; - fd = runtime·open("/dev/urandom", 0 /* O_RDONLY */, 0); - if(runtime·read(fd, urandom_data, HashRandomBytes) == HashRandomBytes) { - *rnd = urandom_data; - *rnd_len = HashRandomBytes; - } else { - *rnd = nil; - *rnd_len = 0; - } - runtime·close(fd); -} - -void -runtime·goenvs(void) -{ - runtime·goenvs_unix(); - - // Register our thread-creation callback (see sys_darwin_{amd64,386}.s) - // but only if we're not using cgo. If we are using cgo we need - // to let the C pthread library install its own thread-creation callback. - if(!runtime·iscgo) { - if(runtime·bsdthread_register() != 0) { - if(runtime·getenv("DYLD_INSERT_LIBRARIES")) - runtime·throw("runtime: bsdthread_register error (unset DYLD_INSERT_LIBRARIES)"); - runtime·throw("runtime: bsdthread_register error"); - } - } - -} - -void -runtime·newosproc(M *mp, void *stk) -{ - int32 errno; - Sigset oset; - - mp->tls[0] = mp->id; // so 386 asm can find it - if(0){ - runtime·printf("newosproc stk=%p m=%p g=%p id=%d/%d ostk=%p\n", - stk, mp, mp->g0, mp->id, (int32)mp->tls[0], &mp); - } - - runtime·sigprocmask(SIG_SETMASK, &sigset_all, &oset); - errno = runtime·bsdthread_create(stk, mp, mp->g0, runtime·mstart); - runtime·sigprocmask(SIG_SETMASK, &oset, nil); - - if(errno < 0) { - runtime·printf("runtime: failed to create new OS thread (have %d already; errno=%d)\n", runtime·mcount(), -errno); - runtime·throw("runtime.newosproc"); - } -} - -// Called to initialize a new m (including the bootstrap m). -// Called on the parent thread (main thread in case of bootstrap), can allocate memory. -void -runtime·mpreinit(M *mp) -{ - mp->gsignal = runtime·malg(32*1024); // OS X wants >=8K, Linux >=2K - mp->gsignal->m = mp; -} - -// Called to initialize a new m (including the bootstrap m). -// Called on the new thread, can not allocate memory. -void -runtime·minit(void) -{ - // Initialize signal handling. - runtime·signalstack((byte*)g->m->gsignal->stack.lo, 32*1024); - - runtime·sigprocmask(SIG_SETMASK, &sigset_none, nil); -} - -// Called from dropm to undo the effect of an minit. -void -runtime·unminit(void) -{ - runtime·signalstack(nil, 0); -} - -// Mach IPC, to get at semaphores -// Definitions are in /usr/include/mach on a Mac. - -static void -macherror(int32 r, int8 *fn) -{ - runtime·prints("mach error "); - runtime·prints(fn); - runtime·prints(": "); - runtime·printint(r); - runtime·prints("\n"); - runtime·throw("mach error"); -} - -enum -{ - DebugMach = 0 -}; - -static MachNDR zerondr; - -#define MACH_MSGH_BITS(a, b) ((a) | ((b)<<8)) - -static int32 -mach_msg(MachHeader *h, - int32 op, - uint32 send_size, - uint32 rcv_size, - uint32 rcv_name, - uint32 timeout, - uint32 notify) -{ - // TODO: Loop on interrupt. - return runtime·mach_msg_trap(h, op, send_size, rcv_size, rcv_name, timeout, notify); -} - -// Mach RPC (MIG) - -enum -{ - MinMachMsg = 48, - Reply = 100, -}; - -#pragma pack on -typedef struct CodeMsg CodeMsg; -struct CodeMsg -{ - MachHeader h; - MachNDR NDR; - int32 code; -}; -#pragma pack off - -static int32 -machcall(MachHeader *h, int32 maxsize, int32 rxsize) -{ - uint32 *p; - int32 i, ret, id; - uint32 port; - CodeMsg *c; - - if((port = g->m->machport) == 0){ - port = runtime·mach_reply_port(); - g->m->machport = port; - } - - h->msgh_bits |= MACH_MSGH_BITS(MACH_MSG_TYPE_COPY_SEND, MACH_MSG_TYPE_MAKE_SEND_ONCE); - h->msgh_local_port = port; - h->msgh_reserved = 0; - id = h->msgh_id; - - if(DebugMach){ - p = (uint32*)h; - runtime·prints("send:\t"); - for(i=0; imsgh_size/sizeof(p[0]); i++){ - runtime·prints(" "); - runtime·printpointer((void*)p[i]); - if(i%8 == 7) - runtime·prints("\n\t"); - } - if(i%8) - runtime·prints("\n"); - } - - ret = mach_msg(h, MACH_SEND_MSG|MACH_RCV_MSG, - h->msgh_size, maxsize, port, 0, 0); - if(ret != 0){ - if(DebugMach){ - runtime·prints("mach_msg error "); - runtime·printint(ret); - runtime·prints("\n"); - } - return ret; - } - - if(DebugMach){ - p = (uint32*)h; - runtime·prints("recv:\t"); - for(i=0; imsgh_size/sizeof(p[0]); i++){ - runtime·prints(" "); - runtime·printpointer((void*)p[i]); - if(i%8 == 7) - runtime·prints("\n\t"); - } - if(i%8) - runtime·prints("\n"); - } - - if(h->msgh_id != id+Reply){ - if(DebugMach){ - runtime·prints("mach_msg reply id mismatch "); - runtime·printint(h->msgh_id); - runtime·prints(" != "); - runtime·printint(id+Reply); - runtime·prints("\n"); - } - return -303; // MIG_REPLY_MISMATCH - } - - // Look for a response giving the return value. - // Any call can send this back with an error, - // and some calls only have return values so they - // send it back on success too. I don't quite see how - // you know it's one of these and not the full response - // format, so just look if the message is right. - c = (CodeMsg*)h; - if(h->msgh_size == sizeof(CodeMsg) - && !(h->msgh_bits & MACH_MSGH_BITS_COMPLEX)){ - if(DebugMach){ - runtime·prints("mig result "); - runtime·printint(c->code); - runtime·prints("\n"); - } - return c->code; - } - - if(h->msgh_size != rxsize){ - if(DebugMach){ - runtime·prints("mach_msg reply size mismatch "); - runtime·printint(h->msgh_size); - runtime·prints(" != "); - runtime·printint(rxsize); - runtime·prints("\n"); - } - return -307; // MIG_ARRAY_TOO_LARGE - } - - return 0; -} - - -// Semaphores! - -enum -{ - Tmach_semcreate = 3418, - Rmach_semcreate = Tmach_semcreate + Reply, - - Tmach_semdestroy = 3419, - Rmach_semdestroy = Tmach_semdestroy + Reply, - - // Mach calls that get interrupted by Unix signals - // return this error code. We retry them. - KERN_ABORTED = 14, - KERN_OPERATION_TIMED_OUT = 49, -}; - -typedef struct Tmach_semcreateMsg Tmach_semcreateMsg; -typedef struct Rmach_semcreateMsg Rmach_semcreateMsg; -typedef struct Tmach_semdestroyMsg Tmach_semdestroyMsg; -// Rmach_semdestroyMsg = CodeMsg - -#pragma pack on -struct Tmach_semcreateMsg -{ - MachHeader h; - MachNDR ndr; - int32 policy; - int32 value; -}; - -struct Rmach_semcreateMsg -{ - MachHeader h; - MachBody body; - MachPort semaphore; -}; - -struct Tmach_semdestroyMsg -{ - MachHeader h; - MachBody body; - MachPort semaphore; -}; -#pragma pack off - -uint32 -runtime·mach_semcreate(void) -{ - union { - Tmach_semcreateMsg tx; - Rmach_semcreateMsg rx; - uint8 pad[MinMachMsg]; - } m; - int32 r; - - m.tx.h.msgh_bits = 0; - m.tx.h.msgh_size = sizeof(m.tx); - m.tx.h.msgh_remote_port = runtime·mach_task_self(); - m.tx.h.msgh_id = Tmach_semcreate; - m.tx.ndr = zerondr; - - m.tx.policy = 0; // 0 = SYNC_POLICY_FIFO - m.tx.value = 0; - - while((r = machcall(&m.tx.h, sizeof m, sizeof(m.rx))) != 0){ - if(r == KERN_ABORTED) // interrupted - continue; - macherror(r, "semaphore_create"); - } - if(m.rx.body.msgh_descriptor_count != 1) - unimplemented("mach_semcreate desc count"); - return m.rx.semaphore.name; -} - -void -runtime·mach_semdestroy(uint32 sem) -{ - union { - Tmach_semdestroyMsg tx; - uint8 pad[MinMachMsg]; - } m; - int32 r; - - m.tx.h.msgh_bits = MACH_MSGH_BITS_COMPLEX; - m.tx.h.msgh_size = sizeof(m.tx); - m.tx.h.msgh_remote_port = runtime·mach_task_self(); - m.tx.h.msgh_id = Tmach_semdestroy; - m.tx.body.msgh_descriptor_count = 1; - m.tx.semaphore.name = sem; - m.tx.semaphore.disposition = MACH_MSG_TYPE_MOVE_SEND; - m.tx.semaphore.type = 0; - - while((r = machcall(&m.tx.h, sizeof m, 0)) != 0){ - if(r == KERN_ABORTED) // interrupted - continue; - macherror(r, "semaphore_destroy"); - } -} - -// The other calls have simple system call traps in sys_darwin_{amd64,386}.s -int32 runtime·mach_semaphore_wait(uint32 sema); -int32 runtime·mach_semaphore_timedwait(uint32 sema, uint32 sec, uint32 nsec); -int32 runtime·mach_semaphore_signal(uint32 sema); -int32 runtime·mach_semaphore_signal_all(uint32 sema); - -static void -semasleep(void) -{ - int32 r, secs, nsecs; - int64 ns; - - ns = (int64)(uint32)g->m->scalararg[0] | (int64)(uint32)g->m->scalararg[1]<<32; - g->m->scalararg[0] = 0; - g->m->scalararg[1] = 0; - - if(ns >= 0) { - secs = runtime·timediv(ns, 1000000000, &nsecs); - r = runtime·mach_semaphore_timedwait(g->m->waitsema, secs, nsecs); - if(r == KERN_ABORTED || r == KERN_OPERATION_TIMED_OUT) { - g->m->scalararg[0] = -1; - return; - } - if(r != 0) - macherror(r, "semaphore_wait"); - g->m->scalararg[0] = 0; - return; - } - while((r = runtime·mach_semaphore_wait(g->m->waitsema)) != 0) { - if(r == KERN_ABORTED) // interrupted - continue; - macherror(r, "semaphore_wait"); - } - g->m->scalararg[0] = 0; - return; -} - -#pragma textflag NOSPLIT -int32 -runtime·semasleep(int64 ns) -{ - int32 r; - void (*fn)(void); - - g->m->scalararg[0] = (uint32)ns; - g->m->scalararg[1] = (uint32)(ns>>32); - fn = semasleep; - runtime·onM(&fn); - r = g->m->scalararg[0]; - g->m->scalararg[0] = 0; - return r; -} - -static int32 mach_semrelease_errno; - -static void -mach_semrelease_fail(void) -{ - macherror(mach_semrelease_errno, "semaphore_signal"); -} - -#pragma textflag NOSPLIT -void -runtime·mach_semrelease(uint32 sem) -{ - int32 r; - void (*fn)(void); - - while((r = runtime·mach_semaphore_signal(sem)) != 0) { - if(r == KERN_ABORTED) // interrupted - continue; - - // mach_semrelease must be completely nosplit, - // because it is called from Go code. - // If we're going to die, start that process on the m stack - // to avoid a Go stack split. - // Only do that if we're actually running on the g stack. - // We might be on the gsignal stack, and if so, onM will abort. - // We use the global variable instead of scalararg because - // we might be on the gsignal stack, having interrupted a - // normal call to onM. It doesn't quite matter, since the - // program is about to die, but better to be clean. - mach_semrelease_errno = r; - fn = mach_semrelease_fail; - if(g == g->m->curg) - runtime·onM(&fn); - else - fn(); - } -} - -#pragma textflag NOSPLIT -void -runtime·osyield(void) -{ - runtime·usleep(1); -} - -uintptr -runtime·memlimit(void) -{ - // NOTE(rsc): Could use getrlimit here, - // like on FreeBSD or Linux, but Darwin doesn't enforce - // ulimit -v, so it's unclear why we'd try to stay within - // the limit. - return 0; -} - -void -runtime·setsig(int32 i, GoSighandler *fn, bool restart) -{ - SigactionT sa; - - runtime·memclr((byte*)&sa, sizeof sa); - sa.sa_flags = SA_SIGINFO|SA_ONSTACK; - if(restart) - sa.sa_flags |= SA_RESTART; - sa.sa_mask = ~(uintptr)0; - sa.sa_tramp = (void*)runtime·sigtramp; // runtime·sigtramp's job is to call into real handler - *(uintptr*)sa.__sigaction_u = (uintptr)fn; - runtime·sigaction(i, &sa, nil); -} - -GoSighandler* -runtime·getsig(int32 i) -{ - SigactionT sa; - - runtime·memclr((byte*)&sa, sizeof sa); - runtime·sigaction(i, nil, &sa); - return *(void**)sa.__sigaction_u; -} - -void -runtime·signalstack(byte *p, int32 n) -{ - StackT st; - - st.ss_sp = (void*)p; - st.ss_size = n; - st.ss_flags = 0; - if(p == nil) - st.ss_flags = SS_DISABLE; - runtime·sigaltstack(&st, nil); -} - -void -runtime·unblocksignals(void) -{ - runtime·sigprocmask(SIG_SETMASK, &sigset_none, nil); -} - -#pragma textflag NOSPLIT -int8* -runtime·signame(int32 sig) -{ - return runtime·sigtab[sig].name; -} diff --git a/src/runtime/os_darwin.go b/src/runtime/os_darwin.go index 4327ced914..fce01efef6 100644 --- a/src/runtime/os_darwin.go +++ b/src/runtime/os_darwin.go @@ -6,19 +6,35 @@ package runtime import "unsafe" -func bsdthread_create(stk, mm, gg, fn unsafe.Pointer) int32 +func bsdthread_create(stk unsafe.Pointer, mm *m, gg *g, fn uintptr) int32 func bsdthread_register() int32 + +//go:noescape func mach_msg_trap(h unsafe.Pointer, op int32, send_size, rcv_size, rcv_name, timeout, notify uint32) int32 + func mach_reply_port() uint32 func mach_task_self() uint32 func mach_thread_self() uint32 + +//go:noescape func sysctl(mib *uint32, miblen uint32, out *byte, size *uintptr, dst *byte, ndst uintptr) int32 -func sigprocmask(sig int32, new, old unsafe.Pointer) -func sigaction(mode uint32, new, old unsafe.Pointer) -func sigaltstack(new, old unsafe.Pointer) + +//go:noescape +func sigprocmask(sig uint32, new, old *uint32) + +//go:noescape +func sigaction(mode uint32, new, old *sigactiont) + +//go:noescape +func sigaltstack(new, old *stackt) + func sigtramp() -func setitimer(mode int32, new, old unsafe.Pointer) -func mach_semaphore_wait(sema uint32) int32 -func mach_semaphore_timedwait(sema, sec, nsec uint32) int32 -func mach_semaphore_signal(sema uint32) int32 -func mach_semaphore_signal_all(sema uint32) int32 + +//go:noescape +func setitimer(mode int32, new, old *itimerval) + +func raise(int32) + +// careful: cputicks is not guaranteed to be monotonic! In particular, we have +// noticed drift between cpus on certain os/arch combinations. See issue 8976. +func cputicks() int64 diff --git a/src/runtime/os_darwin.h b/src/runtime/os_darwin.h deleted file mode 100644 index e8bb45dafc..0000000000 --- a/src/runtime/os_darwin.h +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -typedef byte* kevent_udata; - -int32 runtime·bsdthread_create(void*, M*, G*, void(*)(void)); -int32 runtime·bsdthread_register(void); -int32 runtime·mach_msg_trap(MachHeader*, int32, uint32, uint32, uint32, uint32, uint32); -uint32 runtime·mach_reply_port(void); -int32 runtime·mach_semacquire(uint32, int64); -uint32 runtime·mach_semcreate(void); -void runtime·mach_semdestroy(uint32); -void runtime·mach_semrelease(uint32); -void runtime·mach_semreset(uint32); -uint32 runtime·mach_task_self(void); -uint32 runtime·mach_task_self(void); -uint32 runtime·mach_thread_self(void); -uint32 runtime·mach_thread_self(void); -int32 runtime·sysctl(uint32*, uint32, byte*, uintptr*, byte*, uintptr); - -typedef uint32 Sigset; -void runtime·sigprocmask(int32, Sigset*, Sigset*); -void runtime·unblocksignals(void); - -struct SigactionT; -void runtime·sigaction(uintptr, struct SigactionT*, struct SigactionT*); - -struct StackT; -void runtime·sigaltstack(struct StackT*, struct StackT*); -void runtime·sigtramp(void); -void runtime·sigpanic(void); -void runtime·setitimer(int32, Itimerval*, Itimerval*); - - -enum { - NSIG = 32, - SI_USER = 0, /* empirically true, but not what headers say */ - SIG_BLOCK = 1, - SIG_UNBLOCK = 2, - SIG_SETMASK = 3, - SS_DISABLE = 4, -}; diff --git a/src/runtime/os_dragonfly.c b/src/runtime/os_dragonfly.c index e372205ec8..75c43b056d 100644 --- a/src/runtime/os_dragonfly.c +++ b/src/runtime/os_dragonfly.c @@ -9,7 +9,7 @@ #include "stack.h" #include "textflag.h" -extern SigTab runtime·sigtab[]; +extern SigTabT runtime·sigtab[]; extern int32 runtime·sys_umtx_sleep(uint32*, int32, int32); extern int32 runtime·sys_umtx_wakeup(uint32*, int32); diff --git a/src/runtime/os_freebsd.c b/src/runtime/os_freebsd.c index a513cb6044..23f6e93a3d 100644 --- a/src/runtime/os_freebsd.c +++ b/src/runtime/os_freebsd.c @@ -9,7 +9,7 @@ #include "stack.h" #include "textflag.h" -extern SigTab runtime·sigtab[]; +extern SigTabT runtime·sigtab[]; extern int32 runtime·sys_umtx_op(uint32*, int32, uint32, void*, void*); // From FreeBSD's diff --git a/src/runtime/os_linux.c b/src/runtime/os_linux.c deleted file mode 100644 index 0d8ffc995f..0000000000 --- a/src/runtime/os_linux.c +++ /dev/null @@ -1,342 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -#include "runtime.h" -#include "defs_GOOS_GOARCH.h" -#include "os_GOOS.h" -#include "signal_unix.h" -#include "stack.h" -#include "textflag.h" - -extern SigTab runtime·sigtab[]; - -static Sigset sigset_none; -static Sigset sigset_all = { ~(uint32)0, ~(uint32)0 }; - -// Linux futex. -// -// futexsleep(uint32 *addr, uint32 val) -// futexwakeup(uint32 *addr) -// -// Futexsleep atomically checks if *addr == val and if so, sleeps on addr. -// Futexwakeup wakes up threads sleeping on addr. -// Futexsleep is allowed to wake up spuriously. - -enum -{ - FUTEX_WAIT = 0, - FUTEX_WAKE = 1, -}; - -// Atomically, -// if(*addr == val) sleep -// Might be woken up spuriously; that's allowed. -// Don't sleep longer than ns; ns < 0 means forever. -#pragma textflag NOSPLIT -void -runtime·futexsleep(uint32 *addr, uint32 val, int64 ns) -{ - Timespec ts; - - // Some Linux kernels have a bug where futex of - // FUTEX_WAIT returns an internal error code - // as an errno. Libpthread ignores the return value - // here, and so can we: as it says a few lines up, - // spurious wakeups are allowed. - - if(ns < 0) { - runtime·futex(addr, FUTEX_WAIT, val, nil, nil, 0); - return; - } - // NOTE: tv_nsec is int64 on amd64, so this assumes a little-endian system. - ts.tv_nsec = 0; - ts.tv_sec = runtime·timediv(ns, 1000000000LL, (int32*)&ts.tv_nsec); - runtime·futex(addr, FUTEX_WAIT, val, &ts, nil, 0); -} - -static void badfutexwakeup(void); - -// If any procs are sleeping on addr, wake up at most cnt. -#pragma textflag NOSPLIT -void -runtime·futexwakeup(uint32 *addr, uint32 cnt) -{ - int64 ret; - void (*fn)(void); - - ret = runtime·futex(addr, FUTEX_WAKE, cnt, nil, nil, 0); - if(ret >= 0) - return; - - // I don't know that futex wakeup can return - // EAGAIN or EINTR, but if it does, it would be - // safe to loop and call futex again. - g->m->ptrarg[0] = addr; - g->m->scalararg[0] = (int32)ret; // truncated but fine - fn = badfutexwakeup; - if(g == g->m->gsignal) - fn(); - else - runtime·onM(&fn); - *(int32*)0x1006 = 0x1006; -} - -static void -badfutexwakeup(void) -{ - void *addr; - int64 ret; - - addr = g->m->ptrarg[0]; - ret = (int32)g->m->scalararg[0]; - runtime·printf("futexwakeup addr=%p returned %D\n", addr, ret); -} - -extern runtime·sched_getaffinity(uintptr pid, uintptr len, uintptr *buf); -static int32 -getproccount(void) -{ - uintptr buf[16], t; - int32 r, cnt, i; - - cnt = 0; - r = runtime·sched_getaffinity(0, sizeof(buf), buf); - if(r > 0) - for(i = 0; i < r/sizeof(buf[0]); i++) { - t = buf[i]; - t = t - ((t >> 1) & 0x5555555555555555ULL); - t = (t & 0x3333333333333333ULL) + ((t >> 2) & 0x3333333333333333ULL); - cnt += (int32)((((t + (t >> 4)) & 0xF0F0F0F0F0F0F0FULL) * 0x101010101010101ULL) >> 56); - } - - return cnt ? cnt : 1; -} - -// Clone, the Linux rfork. -enum -{ - CLONE_VM = 0x100, - CLONE_FS = 0x200, - CLONE_FILES = 0x400, - CLONE_SIGHAND = 0x800, - CLONE_PTRACE = 0x2000, - CLONE_VFORK = 0x4000, - CLONE_PARENT = 0x8000, - CLONE_THREAD = 0x10000, - CLONE_NEWNS = 0x20000, - CLONE_SYSVSEM = 0x40000, - CLONE_SETTLS = 0x80000, - CLONE_PARENT_SETTID = 0x100000, - CLONE_CHILD_CLEARTID = 0x200000, - CLONE_UNTRACED = 0x800000, - CLONE_CHILD_SETTID = 0x1000000, - CLONE_STOPPED = 0x2000000, - CLONE_NEWUTS = 0x4000000, - CLONE_NEWIPC = 0x8000000, -}; - -void -runtime·newosproc(M *mp, void *stk) -{ - int32 ret; - int32 flags; - Sigset oset; - - /* - * note: strace gets confused if we use CLONE_PTRACE here. - */ - flags = CLONE_VM /* share memory */ - | CLONE_FS /* share cwd, etc */ - | CLONE_FILES /* share fd table */ - | CLONE_SIGHAND /* share sig handler table */ - | CLONE_THREAD /* revisit - okay for now */ - ; - - mp->tls[0] = mp->id; // so 386 asm can find it - if(0){ - runtime·printf("newosproc stk=%p m=%p g=%p clone=%p id=%d/%d ostk=%p\n", - stk, mp, mp->g0, runtime·clone, mp->id, (int32)mp->tls[0], &mp); - } - - // Disable signals during clone, so that the new thread starts - // with signals disabled. It will enable them in minit. - runtime·rtsigprocmask(SIG_SETMASK, &sigset_all, &oset, sizeof oset); - ret = runtime·clone(flags, stk, mp, mp->g0, runtime·mstart); - runtime·rtsigprocmask(SIG_SETMASK, &oset, nil, sizeof oset); - - if(ret < 0) { - runtime·printf("runtime: failed to create new OS thread (have %d already; errno=%d)\n", runtime·mcount(), -ret); - runtime·throw("runtime.newosproc"); - } -} - -void -runtime·osinit(void) -{ - runtime·ncpu = getproccount(); -} - -// Random bytes initialized at startup. These come -// from the ELF AT_RANDOM auxiliary vector (vdso_linux_amd64.c). -byte* runtime·startup_random_data; -uint32 runtime·startup_random_data_len; - -#pragma textflag NOSPLIT -void -runtime·get_random_data(byte **rnd, int32 *rnd_len) -{ - if(runtime·startup_random_data != nil) { - *rnd = runtime·startup_random_data; - *rnd_len = runtime·startup_random_data_len; - } else { - #pragma dataflag NOPTR - static byte urandom_data[HashRandomBytes]; - int32 fd; - fd = runtime·open("/dev/urandom", 0 /* O_RDONLY */, 0); - if(runtime·read(fd, urandom_data, HashRandomBytes) == HashRandomBytes) { - *rnd = urandom_data; - *rnd_len = HashRandomBytes; - } else { - *rnd = nil; - *rnd_len = 0; - } - runtime·close(fd); - } -} - -void -runtime·goenvs(void) -{ - runtime·goenvs_unix(); -} - -// Called to initialize a new m (including the bootstrap m). -// Called on the parent thread (main thread in case of bootstrap), can allocate memory. -void -runtime·mpreinit(M *mp) -{ - mp->gsignal = runtime·malg(32*1024); // OS X wants >=8K, Linux >=2K - mp->gsignal->m = mp; -} - -// Called to initialize a new m (including the bootstrap m). -// Called on the new thread, can not allocate memory. -void -runtime·minit(void) -{ - // Initialize signal handling. - runtime·signalstack((byte*)g->m->gsignal->stack.lo, 32*1024); - runtime·rtsigprocmask(SIG_SETMASK, &sigset_none, nil, sizeof(Sigset)); -} - -// Called from dropm to undo the effect of an minit. -void -runtime·unminit(void) -{ - runtime·signalstack(nil, 0); -} - -uintptr -runtime·memlimit(void) -{ - Rlimit rl; - extern byte runtime·text[], runtime·end[]; - uintptr used; - - if(runtime·getrlimit(RLIMIT_AS, &rl) != 0) - return 0; - if(rl.rlim_cur >= 0x7fffffff) - return 0; - - // Estimate our VM footprint excluding the heap. - // Not an exact science: use size of binary plus - // some room for thread stacks. - used = runtime·end - runtime·text + (64<<20); - if(used >= rl.rlim_cur) - return 0; - - // If there's not at least 16 MB left, we're probably - // not going to be able to do much. Treat as no limit. - rl.rlim_cur -= used; - if(rl.rlim_cur < (16<<20)) - return 0; - - return rl.rlim_cur - used; -} - -#ifdef GOARCH_386 -#define sa_handler k_sa_handler -#endif - -/* - * This assembler routine takes the args from registers, puts them on the stack, - * and calls sighandler(). - */ -extern void runtime·sigtramp(void); -extern void runtime·sigreturn(void); // calls rt_sigreturn, only used with SA_RESTORER - -void -runtime·setsig(int32 i, GoSighandler *fn, bool restart) -{ - SigactionT sa; - - runtime·memclr((byte*)&sa, sizeof sa); - sa.sa_flags = SA_ONSTACK | SA_SIGINFO | SA_RESTORER; - if(restart) - sa.sa_flags |= SA_RESTART; - sa.sa_mask = ~0ULL; - // Although Linux manpage says "sa_restorer element is obsolete and - // should not be used". x86_64 kernel requires it. Only use it on - // x86. -#ifdef GOARCH_386 - sa.sa_restorer = (void*)runtime·sigreturn; -#endif -#ifdef GOARCH_amd64 - sa.sa_restorer = (void*)runtime·sigreturn; -#endif - if(fn == runtime·sighandler) - fn = (void*)runtime·sigtramp; - sa.sa_handler = fn; - if(runtime·rt_sigaction(i, &sa, nil, sizeof(sa.sa_mask)) != 0) - runtime·throw("rt_sigaction failure"); -} - -GoSighandler* -runtime·getsig(int32 i) -{ - SigactionT sa; - - runtime·memclr((byte*)&sa, sizeof sa); - if(runtime·rt_sigaction(i, nil, &sa, sizeof(sa.sa_mask)) != 0) - runtime·throw("rt_sigaction read failure"); - if((void*)sa.sa_handler == runtime·sigtramp) - return runtime·sighandler; - return (void*)sa.sa_handler; -} - -void -runtime·signalstack(byte *p, int32 n) -{ - SigaltstackT st; - - st.ss_sp = p; - st.ss_size = n; - st.ss_flags = 0; - if(p == nil) - st.ss_flags = SS_DISABLE; - runtime·sigaltstack(&st, nil); -} - -void -runtime·unblocksignals(void) -{ - runtime·rtsigprocmask(SIG_SETMASK, &sigset_none, nil, sizeof sigset_none); -} - -#pragma textflag NOSPLIT -int8* -runtime·signame(int32 sig) -{ - return runtime·sigtab[sig].name; -} diff --git a/src/runtime/os_linux.go b/src/runtime/os_linux.go index 41123ad570..113219aab0 100644 --- a/src/runtime/os_linux.go +++ b/src/runtime/os_linux.go @@ -6,12 +6,28 @@ package runtime import "unsafe" +//go:noescape func futex(addr unsafe.Pointer, op int32, val uint32, ts, addr2 unsafe.Pointer, val3 uint32) int32 + +//go:noescape func clone(flags int32, stk, mm, gg, fn unsafe.Pointer) int32 -func rt_sigaction(sig uintptr, new, old unsafe.Pointer, size uintptr) int32 -func sigaltstack(new, old unsafe.Pointer) -func setitimer(mode int32, new, old unsafe.Pointer) -func rtsigprocmask(sig int32, new, old unsafe.Pointer, size int32) + +//go:noescape +func rt_sigaction(sig uintptr, new, old *sigactiont, size uintptr) int32 + +//go:noescape +func sigaltstack(new, old *sigaltstackt) + +//go:noescape +func setitimer(mode int32, new, old *itimerval) + +//go:noescape +func rtsigprocmask(sig uint32, new, old *sigset, size int32) + +//go:noescape func getrlimit(kind int32, limit unsafe.Pointer) int32 -func raise(sig int32) +func raise(sig uint32) + +//go:noescape func sched_getaffinity(pid, len uintptr, buf *uintptr) int32 +func osyield() diff --git a/src/runtime/os_linux.h b/src/runtime/os_linux.h deleted file mode 100644 index 75606d6152..0000000000 --- a/src/runtime/os_linux.h +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - - -// Linux-specific system calls -int32 runtime·futex(uint32*, int32, uint32, Timespec*, uint32*, uint32); -int32 runtime·clone(int32, void*, M*, G*, void(*)(void)); - -struct SigactionT; -int32 runtime·rt_sigaction(uintptr, struct SigactionT*, void*, uintptr); - -void runtime·sigaltstack(SigaltstackT*, SigaltstackT*); -void runtime·sigpanic(void); -void runtime·setitimer(int32, Itimerval*, Itimerval*); - -enum { - SS_DISABLE = 2, - NSIG = 65, - SI_USER = 0, - SIG_SETMASK = 2, - RLIMIT_AS = 9, -}; - -// It's hard to tease out exactly how big a Sigset is, but -// rt_sigprocmask crashes if we get it wrong, so if binaries -// are running, this is right. -typedef struct Sigset Sigset; -struct Sigset -{ - uint32 mask[2]; -}; -void runtime·rtsigprocmask(int32, Sigset*, Sigset*, int32); -void runtime·unblocksignals(void); - -typedef struct Rlimit Rlimit; -struct Rlimit { - uintptr rlim_cur; - uintptr rlim_max; -}; -int32 runtime·getrlimit(int32, Rlimit*); diff --git a/src/runtime/os_linux_386.c b/src/runtime/os_linux_386.c deleted file mode 100644 index dc89d04e28..0000000000 --- a/src/runtime/os_linux_386.c +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -#include "runtime.h" -#include "defs_GOOS_GOARCH.h" -#include "os_GOOS.h" -#include "textflag.h" - -#define AT_NULL 0 -#define AT_RANDOM 25 -#define AT_SYSINFO 32 -extern uint32 runtime·_vdso; - -#pragma textflag NOSPLIT -void -runtime·linux_setup_vdso(int32 argc, byte **argv) -{ - byte **envp; - uint32 *auxv; - - // skip envp to get to ELF auxiliary vector. - for(envp = &argv[argc+1]; *envp != nil; envp++) - ; - envp++; - - for(auxv=(uint32*)envp; auxv[0] != AT_NULL; auxv += 2) { - if(auxv[0] == AT_SYSINFO) { - runtime·_vdso = auxv[1]; - continue; - } - if(auxv[0] == AT_RANDOM) { - runtime·startup_random_data = (byte*)auxv[1]; - runtime·startup_random_data_len = 16; - continue; - } - } -} diff --git a/src/runtime/os_linux_386.go b/src/runtime/os_linux_386.go new file mode 100644 index 0000000000..de833ab46d --- /dev/null +++ b/src/runtime/os_linux_386.go @@ -0,0 +1,41 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +import "unsafe" + +const ( + _AT_NULL = 0 + _AT_RANDOM = 25 + _AT_SYSINFO = 32 +) + +var _vdso uint32 + +//go:nosplit +func linux_setup_vdso(argc int32, argv **byte) { + // skip over argv, envv to get to auxv + n := argc + 1 + for argv_index(argv, n) != nil { + n++ + } + n++ + auxv := (*[1 << 28]uint32)(add(unsafe.Pointer(argv), uintptr(n)*ptrSize)) + + for i := 0; auxv[i] != _AT_NULL; i += 2 { + switch auxv[i] { + case _AT_SYSINFO: + _vdso = auxv[i+1] + + case _AT_RANDOM: + startup_random_data = (*byte)(unsafe.Pointer(uintptr(auxv[i+1]))) + startup_random_data_len = 16 + } + } +} + +// careful: cputicks is not guaranteed to be monotonic! In particular, we have +// noticed drift between cpus on certain os/arch combinations. See issue 8976. +func cputicks() int64 diff --git a/src/runtime/os_linux_amd64.go b/src/runtime/os_linux_amd64.go new file mode 100644 index 0000000000..bd6a78ed1c --- /dev/null +++ b/src/runtime/os_linux_amd64.go @@ -0,0 +1,9 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +// careful: cputicks is not guaranteed to be monotonic! In particular, we have +// noticed drift between cpus on certain os/arch combinations. See issue 8976. +func cputicks() int64 diff --git a/src/runtime/os_linux_arm.c b/src/runtime/os_linux_arm.c deleted file mode 100644 index e3eda7c2d1..0000000000 --- a/src/runtime/os_linux_arm.c +++ /dev/null @@ -1,80 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -#include "runtime.h" -#include "defs_GOOS_GOARCH.h" -#include "os_GOOS.h" -#include "textflag.h" - -#define AT_NULL 0 -#define AT_PLATFORM 15 // introduced in at least 2.6.11 -#define AT_HWCAP 16 // introduced in at least 2.6.11 -#define AT_RANDOM 25 // introduced in 2.6.29 -#define HWCAP_VFP (1 << 6) // introduced in at least 2.6.11 -#define HWCAP_VFPv3 (1 << 13) // introduced in 2.6.30 -static uint32 runtime·randomNumber; -uint8 runtime·armArch = 6; // we default to ARMv6 -uint32 runtime·hwcap; // set by setup_auxv -extern uint8 runtime·goarm; // set by 5l - -void -runtime·checkgoarm(void) -{ - if(runtime·goarm > 5 && !(runtime·hwcap & HWCAP_VFP)) { - runtime·printf("runtime: this CPU has no floating point hardware, so it cannot run\n"); - runtime·printf("this GOARM=%d binary. Recompile using GOARM=5.\n", runtime·goarm); - runtime·exit(1); - } - if(runtime·goarm > 6 && !(runtime·hwcap & HWCAP_VFPv3)) { - runtime·printf("runtime: this CPU has no VFPv3 floating point hardware, so it cannot run\n"); - runtime·printf("this GOARM=%d binary. Recompile using GOARM=6.\n", runtime·goarm); - runtime·exit(1); - } -} - -#pragma textflag NOSPLIT -void -runtime·setup_auxv(int32 argc, byte **argv) -{ - byte **envp; - byte *rnd; - uint32 *auxv; - uint32 t; - - // skip envp to get to ELF auxiliary vector. - for(envp = &argv[argc+1]; *envp != nil; envp++) - ; - envp++; - - for(auxv=(uint32*)envp; auxv[0] != AT_NULL; auxv += 2) { - switch(auxv[0]) { - case AT_RANDOM: // kernel provided 16-byte worth of random data - if(auxv[1]) { - rnd = (byte*)auxv[1]; - runtime·randomNumber = rnd[4] | rnd[5]<<8 | rnd[6]<<16 | rnd[7]<<24; - } - break; - case AT_PLATFORM: // v5l, v6l, v7l - if(auxv[1]) { - t = *(uint8*)(auxv[1]+1); - if(t >= '5' && t <= '7') - runtime·armArch = t - '0'; - } - break; - case AT_HWCAP: // CPU capability bit flags - runtime·hwcap = auxv[1]; - break; - } - } -} - -#pragma textflag NOSPLIT -int64 -runtime·cputicks(void) -{ - // Currently cputicks() is used in blocking profiler and to seed runtime·fastrand1(). - // runtime·nanotime() is a poor approximation of CPU ticks that is enough for the profiler. - // runtime·randomNumber provides better seeding of fastrand1. - return runtime·nanotime() + runtime·randomNumber; -} diff --git a/src/runtime/os_linux_arm.go b/src/runtime/os_linux_arm.go new file mode 100644 index 0000000000..9b0ade6148 --- /dev/null +++ b/src/runtime/os_linux_arm.go @@ -0,0 +1,75 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +import "unsafe" + +const ( + _AT_NULL = 0 + _AT_PLATFORM = 15 // introduced in at least 2.6.11 + _AT_HWCAP = 16 // introduced in at least 2.6.11 + _AT_RANDOM = 25 // introduced in 2.6.29 + + _HWCAP_VFP = 1 << 6 // introduced in at least 2.6.11 + _HWCAP_VFPv3 = 1 << 13 // introduced in 2.6.30 +) + +var randomNumber uint32 +var armArch uint8 = 6 // we default to ARMv6 +var hwcap uint32 // set by setup_auxv +var goarm uint8 // set by 5l + +func checkgoarm() { + if goarm > 5 && hwcap&_HWCAP_VFP == 0 { + print("runtime: this CPU has no floating point hardware, so it cannot run\n") + print("this GOARM=", goarm, " binary. Recompile using GOARM=5.\n") + exit(1) + } + if goarm > 6 && hwcap&_HWCAP_VFPv3 == 0 { + print("runtime: this CPU has no VFPv3 floating point hardware, so it cannot run\n") + print("this GOARM=", goarm, " binary. Recompile using GOARM=5.\n") + exit(1) + } +} + +//go:nosplit +func setup_auxv(argc int32, argv **byte) { + // skip over argv, envv to get to auxv + n := argc + 1 + for argv_index(argv, n) != nil { + n++ + } + n++ + auxv := (*[1 << 28]uint32)(add(unsafe.Pointer(argv), uintptr(n)*ptrSize)) + + for i := 0; auxv[i] != _AT_NULL; i += 2 { + switch auxv[i] { + case _AT_RANDOM: // kernel provided 16-byte worth of random data + if auxv[i+1] != 0 { + randomNumber = *(*uint32)(unsafe.Pointer(uintptr(auxv[i+1]))) + } + + case _AT_PLATFORM: // v5l, v6l, v7l + t := *(*uint8)(unsafe.Pointer(uintptr(auxv[i+1] + 1))) + if '5' <= t && t <= '7' { + armArch = t - '0' + } + + case _AT_HWCAP: // CPU capability bit flags + hwcap = auxv[i+1] + } + } +} + +func cputicks() int64 { + // Currently cputicks() is used in blocking profiler and to seed fastrand1(). + // nanotime() is a poor approximation of CPU ticks that is enough for the profiler. + // randomNumber provides better seeding of fastrand1. + return nanotime() + int64(randomNumber) +} diff --git a/src/runtime/vdso_linux_amd64.c b/src/runtime/vdso_linux_amd64.c deleted file mode 100644 index 681340c5b6..0000000000 --- a/src/runtime/vdso_linux_amd64.c +++ /dev/null @@ -1,371 +0,0 @@ -// Copyright 2012 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -#include "runtime.h" -#include "textflag.h" - -// Look up symbols in the Linux vDSO. - -// This code was originally based on the sample Linux vDSO parser at -// https://git.kernel.org/cgit/linux/kernel/git/torvalds/linux.git/tree/Documentation/vDSO/parse_vdso.c - -// This implements the ELF dynamic linking spec at -// http://sco.com/developers/gabi/latest/ch5.dynamic.html - -// The version section is documented at -// http://refspecs.linuxfoundation.org/LSB_3.2.0/LSB-Core-generic/LSB-Core-generic/symversion.html - -#define AT_RANDOM 25 -#define AT_SYSINFO_EHDR 33 -#define AT_NULL 0 /* End of vector */ -#define PT_LOAD 1 /* Loadable program segment */ -#define PT_DYNAMIC 2 /* Dynamic linking information */ -#define DT_NULL 0 /* Marks end of dynamic section */ -#define DT_HASH 4 /* Dynamic symbol hash table */ -#define DT_STRTAB 5 /* Address of string table */ -#define DT_SYMTAB 6 /* Address of symbol table */ -#define DT_VERSYM 0x6ffffff0 -#define DT_VERDEF 0x6ffffffc - -#define VER_FLG_BASE 0x1 /* Version definition of file itself */ -#define SHN_UNDEF 0 /* Undefined section */ -#define SHT_DYNSYM 11 /* Dynamic linker symbol table */ -#define STT_FUNC 2 /* Symbol is a code object */ -#define STB_GLOBAL 1 /* Global symbol */ -#define STB_WEAK 2 /* Weak symbol */ - -/* How to extract and insert information held in the st_info field. */ -#define ELF64_ST_BIND(val) (((byte) (val)) >> 4) -#define ELF64_ST_TYPE(val) ((val) & 0xf) - -#define EI_NIDENT (16) - -typedef uint16 Elf64_Half; -typedef uint32 Elf64_Word; -typedef int32 Elf64_Sword; -typedef uint64 Elf64_Xword; -typedef int64 Elf64_Sxword; -typedef uint64 Elf64_Addr; -typedef uint64 Elf64_Off; -typedef uint16 Elf64_Section; -typedef Elf64_Half Elf64_Versym; - - -typedef struct Elf64_Sym -{ - Elf64_Word st_name; - byte st_info; - byte st_other; - Elf64_Section st_shndx; - Elf64_Addr st_value; - Elf64_Xword st_size; -} Elf64_Sym; - -typedef struct Elf64_Verdef -{ - Elf64_Half vd_version; /* Version revision */ - Elf64_Half vd_flags; /* Version information */ - Elf64_Half vd_ndx; /* Version Index */ - Elf64_Half vd_cnt; /* Number of associated aux entries */ - Elf64_Word vd_hash; /* Version name hash value */ - Elf64_Word vd_aux; /* Offset in bytes to verdaux array */ - Elf64_Word vd_next; /* Offset in bytes to next verdef entry */ -} Elf64_Verdef; - -typedef struct Elf64_Ehdr -{ - byte e_ident[EI_NIDENT]; /* Magic number and other info */ - Elf64_Half e_type; /* Object file type */ - Elf64_Half e_machine; /* Architecture */ - Elf64_Word e_version; /* Object file version */ - Elf64_Addr e_entry; /* Entry point virtual address */ - Elf64_Off e_phoff; /* Program header table file offset */ - Elf64_Off e_shoff; /* Section header table file offset */ - Elf64_Word e_flags; /* Processor-specific flags */ - Elf64_Half e_ehsize; /* ELF header size in bytes */ - Elf64_Half e_phentsize; /* Program header table entry size */ - Elf64_Half e_phnum; /* Program header table entry count */ - Elf64_Half e_shentsize; /* Section header table entry size */ - Elf64_Half e_shnum; /* Section header table entry count */ - Elf64_Half e_shstrndx; /* Section header string table index */ -} Elf64_Ehdr; - -typedef struct Elf64_Phdr -{ - Elf64_Word p_type; /* Segment type */ - Elf64_Word p_flags; /* Segment flags */ - Elf64_Off p_offset; /* Segment file offset */ - Elf64_Addr p_vaddr; /* Segment virtual address */ - Elf64_Addr p_paddr; /* Segment physical address */ - Elf64_Xword p_filesz; /* Segment size in file */ - Elf64_Xword p_memsz; /* Segment size in memory */ - Elf64_Xword p_align; /* Segment alignment */ -} Elf64_Phdr; - -typedef struct Elf64_Shdr -{ - Elf64_Word sh_name; /* Section name (string tbl index) */ - Elf64_Word sh_type; /* Section type */ - Elf64_Xword sh_flags; /* Section flags */ - Elf64_Addr sh_addr; /* Section virtual addr at execution */ - Elf64_Off sh_offset; /* Section file offset */ - Elf64_Xword sh_size; /* Section size in bytes */ - Elf64_Word sh_link; /* Link to another section */ - Elf64_Word sh_info; /* Additional section information */ - Elf64_Xword sh_addralign; /* Section alignment */ - Elf64_Xword sh_entsize; /* Entry size if section holds table */ -} Elf64_Shdr; - -typedef struct Elf64_Dyn -{ - Elf64_Sxword d_tag; /* Dynamic entry type */ - union - { - Elf64_Xword d_val; /* Integer value */ - Elf64_Addr d_ptr; /* Address value */ - } d_un; -} Elf64_Dyn; - -typedef struct Elf64_Verdaux -{ - Elf64_Word vda_name; /* Version or dependency names */ - Elf64_Word vda_next; /* Offset in bytes to next verdaux entry */ -} Elf64_Verdaux; - -typedef struct Elf64_auxv_t -{ - uint64 a_type; /* Entry type */ - union - { - uint64 a_val; /* Integer value */ - } a_un; -} Elf64_auxv_t; - - -typedef struct symbol_key { - byte* name; - int32 sym_hash; - void** var_ptr; -} symbol_key; - -typedef struct version_key { - byte* version; - int32 ver_hash; -} version_key; - -struct vdso_info { - bool valid; - - /* Load information */ - uintptr load_addr; - uintptr load_offset; /* load_addr - recorded vaddr */ - - /* Symbol table */ - Elf64_Sym *symtab; - const byte *symstrings; - Elf64_Word *bucket, *chain; - Elf64_Word nbucket, nchain; - - /* Version table */ - Elf64_Versym *versym; - Elf64_Verdef *verdef; -}; - -#pragma dataflag NOPTR -static version_key linux26 = { (byte*)"LINUX_2.6", 0x3ae75f6 }; - -// initialize with vsyscall fallbacks -#pragma dataflag NOPTR -void* runtime·__vdso_time_sym = (void*)0xffffffffff600400ULL; -#pragma dataflag NOPTR -void* runtime·__vdso_gettimeofday_sym = (void*)0xffffffffff600000ULL; -#pragma dataflag NOPTR -void* runtime·__vdso_clock_gettime_sym = (void*)0; - -#pragma dataflag NOPTR -static symbol_key sym_keys[] = { - { (byte*)"__vdso_time", 0xa33c485, &runtime·__vdso_time_sym }, - { (byte*)"__vdso_gettimeofday", 0x315ca59, &runtime·__vdso_gettimeofday_sym }, - { (byte*)"__vdso_clock_gettime", 0xd35ec75, &runtime·__vdso_clock_gettime_sym }, -}; - -static void -vdso_init_from_sysinfo_ehdr(struct vdso_info *vdso_info, Elf64_Ehdr* hdr) -{ - uint64 i; - bool found_vaddr = false; - Elf64_Phdr *pt; - Elf64_Dyn *dyn; - Elf64_Word *hash; - - vdso_info->valid = false; - vdso_info->load_addr = (uintptr) hdr; - - pt = (Elf64_Phdr*)(vdso_info->load_addr + hdr->e_phoff); - dyn = nil; - - // We need two things from the segment table: the load offset - // and the dynamic table. - for(i=0; ie_phnum; i++) { - if(pt[i].p_type == PT_LOAD && found_vaddr == false) { - found_vaddr = true; - vdso_info->load_offset = (uintptr)hdr - + (uintptr)pt[i].p_offset - - (uintptr)pt[i].p_vaddr; - } else if(pt[i].p_type == PT_DYNAMIC) { - dyn = (Elf64_Dyn*)((uintptr)hdr + pt[i].p_offset); - } - } - - if(found_vaddr == false || dyn == nil) - return; // Failed - - // Fish out the useful bits of the dynamic table. - hash = nil; - vdso_info->symstrings = nil; - vdso_info->symtab = nil; - vdso_info->versym = nil; - vdso_info->verdef = nil; - for(i=0; dyn[i].d_tag!=DT_NULL; i++) { - switch(dyn[i].d_tag) { - case DT_STRTAB: - vdso_info->symstrings = (const byte *) - ((uintptr)dyn[i].d_un.d_ptr - + vdso_info->load_offset); - break; - case DT_SYMTAB: - vdso_info->symtab = (Elf64_Sym *) - ((uintptr)dyn[i].d_un.d_ptr - + vdso_info->load_offset); - break; - case DT_HASH: - hash = (Elf64_Word *) - ((uintptr)dyn[i].d_un.d_ptr - + vdso_info->load_offset); - break; - case DT_VERSYM: - vdso_info->versym = (Elf64_Versym *) - ((uintptr)dyn[i].d_un.d_ptr - + vdso_info->load_offset); - break; - case DT_VERDEF: - vdso_info->verdef = (Elf64_Verdef *) - ((uintptr)dyn[i].d_un.d_ptr - + vdso_info->load_offset); - break; - } - } - if(vdso_info->symstrings == nil || vdso_info->symtab == nil || hash == nil) - return; // Failed - - if(vdso_info->verdef == nil) - vdso_info->versym = 0; - - // Parse the hash table header. - vdso_info->nbucket = hash[0]; - vdso_info->nchain = hash[1]; - vdso_info->bucket = &hash[2]; - vdso_info->chain = &hash[vdso_info->nbucket + 2]; - - // That's all we need. - vdso_info->valid = true; -} - -static int32 -vdso_find_version(struct vdso_info *vdso_info, version_key* ver) -{ - if(vdso_info->valid == false) { - return 0; - } - Elf64_Verdef *def = vdso_info->verdef; - while(true) { - if((def->vd_flags & VER_FLG_BASE) == 0) { - Elf64_Verdaux *aux = (Elf64_Verdaux*)((byte *)def + def->vd_aux); - if(def->vd_hash == ver->ver_hash && - runtime·strcmp(ver->version, vdso_info->symstrings + aux->vda_name) == 0) { - return def->vd_ndx & 0x7fff; - } - } - - if(def->vd_next == 0) { - break; - } - def = (Elf64_Verdef *)((byte *)def + def->vd_next); - } - return -1; // can not match any version -} - -static void -vdso_parse_symbols(struct vdso_info *vdso_info, int32 version) -{ - int32 i; - Elf64_Word chain; - Elf64_Sym *sym; - - if(vdso_info->valid == false) - return; - - for(i=0; ibucket[sym_keys[i].sym_hash % vdso_info->nbucket]; - chain != 0; chain = vdso_info->chain[chain]) { - - sym = &vdso_info->symtab[chain]; - if(ELF64_ST_TYPE(sym->st_info) != STT_FUNC) - continue; - if(ELF64_ST_BIND(sym->st_info) != STB_GLOBAL && - ELF64_ST_BIND(sym->st_info) != STB_WEAK) - continue; - if(sym->st_shndx == SHN_UNDEF) - continue; - if(runtime·strcmp(sym_keys[i].name, vdso_info->symstrings + sym->st_name) != 0) - continue; - - // Check symbol version. - if(vdso_info->versym != nil && version != 0 - && vdso_info->versym[chain] & 0x7fff != version) - continue; - - *sym_keys[i].var_ptr = (void *)(vdso_info->load_offset + sym->st_value); - break; - } - } -} - -static void -runtime·linux_setup_vdso(int32 argc, uint8** argv) -{ - struct vdso_info vdso_info; - - // skip argvc - byte **p = argv; - p = &p[argc+1]; - - // skip envp to get to ELF auxiliary vector. - for(; *p!=0; p++) {} - - // skip NULL separator - p++; - - // now, p points to auxv - Elf64_auxv_t *elf_auxv = (Elf64_auxv_t*) p; - - for(int32 i=0; elf_auxv[i].a_type!=AT_NULL; i++) { - if(elf_auxv[i].a_type == AT_SYSINFO_EHDR) { - if(elf_auxv[i].a_un.a_val == 0) { - // Something went wrong - continue; - } - vdso_init_from_sysinfo_ehdr(&vdso_info, (Elf64_Ehdr*)elf_auxv[i].a_un.a_val); - vdso_parse_symbols(&vdso_info, vdso_find_version(&vdso_info, &linux26)); - continue; - } - if(elf_auxv[i].a_type == AT_RANDOM) { - runtime·startup_random_data = (byte*)elf_auxv[i].a_un.a_val; - runtime·startup_random_data_len = 16; - continue; - } - } -} - -void (*runtime·sysargs)(int32, uint8**) = runtime·linux_setup_vdso; diff --git a/src/runtime/vdso_linux_amd64.go b/src/runtime/vdso_linux_amd64.go new file mode 100644 index 0000000000..7eb6988118 --- /dev/null +++ b/src/runtime/vdso_linux_amd64.go @@ -0,0 +1,328 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +import "unsafe" + +// Look up symbols in the Linux vDSO. + +// This code was originally based on the sample Linux vDSO parser at +// https://git.kernel.org/cgit/linux/kernel/git/torvalds/linux.git/tree/Documentation/vDSO/parse_vdso.c + +// This implements the ELF dynamic linking spec at +// http://sco.com/developers/gabi/latest/ch5.dynamic.html + +// The version section is documented at +// http://refspecs.linuxfoundation.org/LSB_3.2.0/LSB-Core-generic/LSB-Core-generic/symversion.html + +const ( + _AT_RANDOM = 25 + _AT_SYSINFO_EHDR = 33 + _AT_NULL = 0 /* End of vector */ + + _PT_LOAD = 1 /* Loadable program segment */ + _PT_DYNAMIC = 2 /* Dynamic linking information */ + + _DT_NULL = 0 /* Marks end of dynamic section */ + _DT_HASH = 4 /* Dynamic symbol hash table */ + _DT_STRTAB = 5 /* Address of string table */ + _DT_SYMTAB = 6 /* Address of symbol table */ + _DT_VERSYM = 0x6ffffff0 + _DT_VERDEF = 0x6ffffffc + + _VER_FLG_BASE = 0x1 /* Version definition of file itself */ + + _SHN_UNDEF = 0 /* Undefined section */ + + _SHT_DYNSYM = 11 /* Dynamic linker symbol table */ + + _STT_FUNC = 2 /* Symbol is a code object */ + + _STB_GLOBAL = 1 /* Global symbol */ + _STB_WEAK = 2 /* Weak symbol */ + + _EI_NIDENT = 16 +) + +/* How to extract and insert information held in the st_info field. */ +func _ELF64_ST_BIND(val byte) byte { return val >> 4 } +func _ELF64_ST_TYPE(val byte) byte { return val & 0xf } + +type elf64Sym struct { + st_name uint32 + st_info byte + st_other byte + st_shndx uint16 + st_value uint64 + st_size uint64 +} + +type elf64Verdef struct { + vd_version uint16 /* Version revision */ + vd_flags uint16 /* Version information */ + vd_ndx uint16 /* Version Index */ + vd_cnt uint16 /* Number of associated aux entries */ + vd_hash uint32 /* Version name hash value */ + vd_aux uint32 /* Offset in bytes to verdaux array */ + vd_next uint32 /* Offset in bytes to next verdef entry */ +} + +type elf64Ehdr struct { + e_ident [_EI_NIDENT]byte /* Magic number and other info */ + e_type uint16 /* Object file type */ + e_machine uint16 /* Architecture */ + e_version uint32 /* Object file version */ + e_entry uint64 /* Entry point virtual address */ + e_phoff uint64 /* Program header table file offset */ + e_shoff uint64 /* Section header table file offset */ + e_flags uint32 /* Processor-specific flags */ + e_ehsize uint16 /* ELF header size in bytes */ + e_phentsize uint16 /* Program header table entry size */ + e_phnum uint16 /* Program header table entry count */ + e_shentsize uint16 /* Section header table entry size */ + e_shnum uint16 /* Section header table entry count */ + e_shstrndx uint16 /* Section header string table index */ +} + +type elf64Phdr struct { + p_type uint32 /* Segment type */ + p_flags uint32 /* Segment flags */ + p_offset uint64 /* Segment file offset */ + p_vaddr uint64 /* Segment virtual address */ + p_paddr uint64 /* Segment physical address */ + p_filesz uint64 /* Segment size in file */ + p_memsz uint64 /* Segment size in memory */ + p_align uint64 /* Segment alignment */ +} + +type elf64Shdr struct { + sh_name uint32 /* Section name (string tbl index) */ + sh_type uint32 /* Section type */ + sh_flags uint64 /* Section flags */ + sh_addr uint64 /* Section virtual addr at execution */ + sh_offset uint64 /* Section file offset */ + sh_size uint64 /* Section size in bytes */ + sh_link uint32 /* Link to another section */ + sh_info uint32 /* Additional section information */ + sh_addralign uint64 /* Section alignment */ + sh_entsize uint64 /* Entry size if section holds table */ +} + +type elf64Dyn struct { + d_tag int64 /* Dynamic entry type */ + d_val uint64 /* Integer value */ +} + +type elf64Verdaux struct { + vda_name uint32 /* Version or dependency names */ + vda_next uint32 /* Offset in bytes to next verdaux entry */ +} + +type elf64Auxv struct { + a_type uint64 /* Entry type */ + a_val uint64 /* Integer value */ +} + +type symbol_key struct { + name string + sym_hash uint32 + ptr *uintptr +} + +type version_key struct { + version string + ver_hash uint32 +} + +type vdso_info struct { + valid bool + + /* Load information */ + load_addr uintptr + load_offset uintptr /* load_addr - recorded vaddr */ + + /* Symbol table */ + symtab *[1 << 32]elf64Sym + symstrings *[1 << 32]byte + chain []uint32 + bucket []uint32 + + /* Version table */ + versym *[1 << 32]uint16 + verdef *elf64Verdef +} + +var linux26 = version_key{"LINUX_2.6", 0x3ae75f6} + +var sym_keys = []symbol_key{ + {"__vdso_time", 0xa33c485, &__vdso_time_sym}, + {"__vdso_gettimeofday", 0x315ca59, &__vdso_gettimeofday_sym}, + {"__vdso_clock_gettime", 0xd35ec75, &__vdso_clock_gettime_sym}, +} + +// initialize with vsyscall fallbacks +var ( + __vdso_time_sym uintptr = 0xffffffffff600400 + __vdso_gettimeofday_sym uintptr = 0xffffffffff600000 + __vdso_clock_gettime_sym uintptr = 0 +) + +func vdso_init_from_sysinfo_ehdr(info *vdso_info, hdr *elf64Ehdr) { + info.valid = false + info.load_addr = uintptr(unsafe.Pointer(hdr)) + + pt := unsafe.Pointer(info.load_addr + uintptr(hdr.e_phoff)) + + // We need two things from the segment table: the load offset + // and the dynamic table. + var found_vaddr bool + var dyn *[1 << 20]elf64Dyn + for i := uint16(0); i < hdr.e_phnum; i++ { + pt := (*elf64Phdr)(add(pt, uintptr(i)*unsafe.Sizeof(elf64Phdr{}))) + switch pt.p_type { + case _PT_LOAD: + if !found_vaddr { + found_vaddr = true + info.load_offset = info.load_addr + uintptr(pt.p_offset-pt.p_vaddr) + } + + case _PT_DYNAMIC: + dyn = (*[1 << 20]elf64Dyn)(unsafe.Pointer(info.load_addr + uintptr(pt.p_offset))) + } + } + + if !found_vaddr || dyn == nil { + return // Failed + } + + // Fish out the useful bits of the dynamic table. + + var hash *[1 << 30]uint32 + hash = nil + info.symstrings = nil + info.symtab = nil + info.versym = nil + info.verdef = nil + for i := 0; dyn[i].d_tag != _DT_NULL; i++ { + dt := &dyn[i] + p := info.load_offset + uintptr(dt.d_val) + switch dt.d_tag { + case _DT_STRTAB: + info.symstrings = (*[1 << 32]byte)(unsafe.Pointer(p)) + case _DT_SYMTAB: + info.symtab = (*[1 << 32]elf64Sym)(unsafe.Pointer(p)) + case _DT_HASH: + hash = (*[1 << 30]uint32)(unsafe.Pointer(p)) + case _DT_VERSYM: + info.versym = (*[1 << 32]uint16)(unsafe.Pointer(p)) + case _DT_VERDEF: + info.verdef = (*elf64Verdef)(unsafe.Pointer(p)) + } + } + + if info.symstrings == nil || info.symtab == nil || hash == nil { + return // Failed + } + + if info.verdef == nil { + info.versym = nil + } + + // Parse the hash table header. + nbucket := hash[0] + nchain := hash[1] + info.bucket = hash[2 : 2+nbucket] + info.chain = hash[2+nbucket : 2+nbucket+nchain] + + // That's all we need. + info.valid = true +} + +func vdso_find_version(info *vdso_info, ver *version_key) int32 { + if !info.valid { + return 0 + } + + def := info.verdef + for { + if def.vd_flags&_VER_FLG_BASE == 0 { + aux := (*elf64Verdaux)(add(unsafe.Pointer(def), uintptr(def.vd_aux))) + if def.vd_hash == ver.ver_hash && ver.version == gostringnocopy(&info.symstrings[aux.vda_name]) { + return int32(def.vd_ndx & 0x7fff) + } + } + + if def.vd_next == 0 { + break + } + def = (*elf64Verdef)(add(unsafe.Pointer(def), uintptr(def.vd_next))) + } + + return -1 // can not match any version +} + +func vdso_parse_symbols(info *vdso_info, version int32) { + if !info.valid { + return + } + + for _, k := range sym_keys { + for chain := info.bucket[k.sym_hash%uint32(len(info.bucket))]; chain != 0; chain = info.chain[chain] { + sym := &info.symtab[chain] + typ := _ELF64_ST_TYPE(sym.st_info) + bind := _ELF64_ST_BIND(sym.st_info) + if typ != _STT_FUNC || bind != _STB_GLOBAL && bind != _STB_WEAK || sym.st_shndx == _SHN_UNDEF { + continue + } + if k.name != gostringnocopy(&info.symstrings[sym.st_name]) { + continue + } + + // Check symbol version. + if info.versym != nil && version != 0 && int32(info.versym[chain]&0x7fff) != version { + continue + } + + *k.ptr = info.load_offset + uintptr(sym.st_value) + break + } + } +} + +func sysargs(argc int32, argv **byte) { + n := argc + 1 + + // skip envp to get to ELF auxiliary vector. + for argv_index(argv, n) != nil { + n++ + } + + // skip NULL separator + n++ + + // now argv+n is auxv + auxv := (*[1 << 32]elf64Auxv)(add(unsafe.Pointer(argv), uintptr(n)*ptrSize)) + + for i := 0; auxv[i].a_type != _AT_NULL; i++ { + av := &auxv[i] + switch av.a_type { + case _AT_SYSINFO_EHDR: + if av.a_val == 0 { + // Something went wrong + continue + } + var info vdso_info + // TODO(rsc): I don't understand why the compiler thinks info escapes + // when passed to the three functions below. + info1 := (*vdso_info)(noescape(unsafe.Pointer(&info))) + vdso_init_from_sysinfo_ehdr(info1, (*elf64Ehdr)(unsafe.Pointer(uintptr(av.a_val)))) + vdso_parse_symbols(info1, vdso_find_version(info1, &linux26)) + + case _AT_RANDOM: + startup_random_data = (*byte)(unsafe.Pointer(uintptr(av.a_val))) + startup_random_data_len = 16 + } + } +} diff --git a/src/runtime/vdso_none.go b/src/runtime/vdso_none.go new file mode 100644 index 0000000000..ac6f8cb18d --- /dev/null +++ b/src/runtime/vdso_none.go @@ -0,0 +1,10 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !linux !amd64 + +package runtime + +func sysargs(argc int32, argv **byte) { +}