mirror of
https://github.com/golang/go
synced 2024-10-04 17:21:20 -06:00
67faca7d9c
This change breaks out most of the atomics functions in the runtime into package runtime/internal/atomic. It adds some basic support in the toolchain for runtime packages, and also modifies linux/arm atomics to remove the dependency on the runtime's mutex. The mutexes have been replaced with spinlocks. all trybots are happy! In addition to the trybots, I've tested on the darwin/arm64 builder, on the darwin/arm builder, and on a ppc64le machine. Change-Id: I6698c8e3cf3834f55ce5824059f44d00dc8e3c2f Reviewed-on: https://go-review.googlesource.com/14204 Run-TryBot: Michael Matloob <matloob@golang.org> Reviewed-by: Russ Cox <rsc@golang.org>
210 lines
4.5 KiB
Go
210 lines
4.5 KiB
Go
// Copyright 2011 The Go Authors. All rights reserved.
|
|
// Use of this source code is governed by a BSD-style
|
|
// license that can be found in the LICENSE file.
|
|
|
|
// +build dragonfly freebsd linux
|
|
|
|
package runtime
|
|
|
|
import (
|
|
"runtime/internal/atomic"
|
|
"unsafe"
|
|
)
|
|
|
|
// This implementation depends on OS-specific implementations of
|
|
//
|
|
// runtime·futexsleep(uint32 *addr, uint32 val, int64 ns)
|
|
// Atomically,
|
|
// if(*addr == val) sleep
|
|
// Might be woken up spuriously; that's allowed.
|
|
// Don't sleep longer than ns; ns < 0 means forever.
|
|
//
|
|
// runtime·futexwakeup(uint32 *addr, uint32 cnt)
|
|
// If any procs are sleeping on addr, wake up at most cnt.
|
|
|
|
const (
|
|
mutex_unlocked = 0
|
|
mutex_locked = 1
|
|
mutex_sleeping = 2
|
|
|
|
active_spin = 4
|
|
active_spin_cnt = 30
|
|
passive_spin = 1
|
|
)
|
|
|
|
// Possible lock states are mutex_unlocked, mutex_locked and mutex_sleeping.
|
|
// mutex_sleeping means that there is presumably at least one sleeping thread.
|
|
// Note that there can be spinning threads during all states - they do not
|
|
// affect mutex's state.
|
|
|
|
// We use the uintptr mutex.key and note.key as a uint32.
|
|
func key32(p *uintptr) *uint32 {
|
|
return (*uint32)(unsafe.Pointer(p))
|
|
}
|
|
|
|
func lock(l *mutex) {
|
|
gp := getg()
|
|
|
|
if gp.m.locks < 0 {
|
|
throw("runtime·lock: lock count")
|
|
}
|
|
gp.m.locks++
|
|
|
|
// Speculative grab for lock.
|
|
v := atomic.Xchg(key32(&l.key), mutex_locked)
|
|
if v == mutex_unlocked {
|
|
return
|
|
}
|
|
|
|
// wait is either MUTEX_LOCKED or MUTEX_SLEEPING
|
|
// depending on whether there is a thread sleeping
|
|
// on this mutex. If we ever change l->key from
|
|
// MUTEX_SLEEPING to some other value, we must be
|
|
// careful to change it back to MUTEX_SLEEPING before
|
|
// returning, to ensure that the sleeping thread gets
|
|
// its wakeup call.
|
|
wait := v
|
|
|
|
// On uniprocessors, no point spinning.
|
|
// On multiprocessors, spin for ACTIVE_SPIN attempts.
|
|
spin := 0
|
|
if ncpu > 1 {
|
|
spin = active_spin
|
|
}
|
|
for {
|
|
// Try for lock, spinning.
|
|
for i := 0; i < spin; i++ {
|
|
for l.key == mutex_unlocked {
|
|
if atomic.Cas(key32(&l.key), mutex_unlocked, wait) {
|
|
return
|
|
}
|
|
}
|
|
procyield(active_spin_cnt)
|
|
}
|
|
|
|
// Try for lock, rescheduling.
|
|
for i := 0; i < passive_spin; i++ {
|
|
for l.key == mutex_unlocked {
|
|
if atomic.Cas(key32(&l.key), mutex_unlocked, wait) {
|
|
return
|
|
}
|
|
}
|
|
osyield()
|
|
}
|
|
|
|
// Sleep.
|
|
v = atomic.Xchg(key32(&l.key), mutex_sleeping)
|
|
if v == mutex_unlocked {
|
|
return
|
|
}
|
|
wait = mutex_sleeping
|
|
futexsleep(key32(&l.key), mutex_sleeping, -1)
|
|
}
|
|
}
|
|
|
|
func unlock(l *mutex) {
|
|
v := atomic.Xchg(key32(&l.key), mutex_unlocked)
|
|
if v == mutex_unlocked {
|
|
throw("unlock of unlocked lock")
|
|
}
|
|
if v == mutex_sleeping {
|
|
futexwakeup(key32(&l.key), 1)
|
|
}
|
|
|
|
gp := getg()
|
|
gp.m.locks--
|
|
if gp.m.locks < 0 {
|
|
throw("runtime·unlock: lock count")
|
|
}
|
|
if gp.m.locks == 0 && gp.preempt { // restore the preemption request in case we've cleared it in newstack
|
|
gp.stackguard0 = stackPreempt
|
|
}
|
|
}
|
|
|
|
// One-time notifications.
|
|
func noteclear(n *note) {
|
|
n.key = 0
|
|
}
|
|
|
|
func notewakeup(n *note) {
|
|
old := atomic.Xchg(key32(&n.key), 1)
|
|
if old != 0 {
|
|
print("notewakeup - double wakeup (", old, ")\n")
|
|
throw("notewakeup - double wakeup")
|
|
}
|
|
futexwakeup(key32(&n.key), 1)
|
|
}
|
|
|
|
func notesleep(n *note) {
|
|
gp := getg()
|
|
if gp != gp.m.g0 {
|
|
throw("notesleep not on g0")
|
|
}
|
|
for atomic.Load(key32(&n.key)) == 0 {
|
|
gp.m.blocked = true
|
|
futexsleep(key32(&n.key), 0, -1)
|
|
gp.m.blocked = false
|
|
}
|
|
}
|
|
|
|
// May run with m.p==nil if called from notetsleep, so write barriers
|
|
// are not allowed.
|
|
//
|
|
//go:nosplit
|
|
//go:nowritebarrier
|
|
func notetsleep_internal(n *note, ns int64) bool {
|
|
gp := getg()
|
|
|
|
if ns < 0 {
|
|
for atomic.Load(key32(&n.key)) == 0 {
|
|
gp.m.blocked = true
|
|
futexsleep(key32(&n.key), 0, -1)
|
|
gp.m.blocked = false
|
|
}
|
|
return true
|
|
}
|
|
|
|
if atomic.Load(key32(&n.key)) != 0 {
|
|
return true
|
|
}
|
|
|
|
deadline := nanotime() + ns
|
|
for {
|
|
gp.m.blocked = true
|
|
futexsleep(key32(&n.key), 0, ns)
|
|
gp.m.blocked = false
|
|
if atomic.Load(key32(&n.key)) != 0 {
|
|
break
|
|
}
|
|
now := nanotime()
|
|
if now >= deadline {
|
|
break
|
|
}
|
|
ns = deadline - now
|
|
}
|
|
return atomic.Load(key32(&n.key)) != 0
|
|
}
|
|
|
|
func notetsleep(n *note, ns int64) bool {
|
|
gp := getg()
|
|
if gp != gp.m.g0 && gp.m.preemptoff != "" {
|
|
throw("notetsleep not on g0")
|
|
}
|
|
|
|
return notetsleep_internal(n, ns)
|
|
}
|
|
|
|
// same as runtime·notetsleep, but called on user g (not g0)
|
|
// calls only nosplit functions between entersyscallblock/exitsyscall
|
|
func notetsleepg(n *note, ns int64) bool {
|
|
gp := getg()
|
|
if gp == gp.m.g0 {
|
|
throw("notetsleepg on g0")
|
|
}
|
|
|
|
entersyscallblock(0)
|
|
ok := notetsleep_internal(n, ns)
|
|
exitsyscall(0)
|
|
return ok
|
|
}
|