From 296c40dbbe91db993e4c756f3b2b383d6995edbc Mon Sep 17 00:00:00 2001 From: Andy Pan Date: Fri, 26 Aug 2022 01:29:57 +0800 Subject: [PATCH] runtime: mark all methods of atomic types with go:nosplit Atomic types may be used anywhere in the runtime, so they must omit its usual stack overflow checks to avoid errors from stack splits. Change-Id: Icb497334c860bcaa284a2d5a3edc47ee89844301 Reviewed-on: https://go-review.googlesource.com/c/go/+/425484 Reviewed-by: Cuong Manh Le Run-TryBot: Ian Lance Taylor Auto-Submit: Cuong Manh Le Reviewed-by: David Chase TryBot-Result: Gopher Robot Reviewed-by: Michael Pratt --- src/runtime/internal/atomic/types.go | 72 ++++++++++++++++++++++ src/runtime/internal/atomic/types_64bit.go | 4 ++ 2 files changed, 76 insertions(+) diff --git a/src/runtime/internal/atomic/types.go b/src/runtime/internal/atomic/types.go index dbacb86704a..35d8935c70c 100644 --- a/src/runtime/internal/atomic/types.go +++ b/src/runtime/internal/atomic/types.go @@ -15,11 +15,15 @@ type Int32 struct { } // Load accesses and returns the value atomically. +// +//go:nosplit func (i *Int32) Load() int32 { return Loadint32(&i.value) } // Store updates the value atomically. +// +//go:nosplit func (i *Int32) Store(value int32) { Storeint32(&i.value, value) } @@ -28,12 +32,16 @@ func (i *Int32) Store(value int32) { // and if they're equal, swaps i's value with new. // // Returns true if the operation succeeded. +// +//go:nosplit func (i *Int32) CompareAndSwap(old, new int32) bool { return Casint32(&i.value, old, new) } // Swap replaces i's value with new, returning // i's value before the replacement. +// +//go:nosplit func (i *Int32) Swap(new int32) int32 { return Xchgint32(&i.value, new) } @@ -43,6 +51,8 @@ func (i *Int32) Swap(new int32) int32 { // // This operation wraps around in the usual // two's-complement way. +// +//go:nosplit func (i *Int32) Add(delta int32) int32 { return Xaddint32(&i.value, delta) } @@ -59,11 +69,15 @@ type Int64 struct { } // Load accesses and returns the value atomically. +// +//go:nosplit func (i *Int64) Load() int64 { return Loadint64(&i.value) } // Store updates the value atomically. +// +//go:nosplit func (i *Int64) Store(value int64) { Storeint64(&i.value, value) } @@ -72,12 +86,16 @@ func (i *Int64) Store(value int64) { // and if they're equal, swaps i's value with new. // // Returns true if the operation succeeded. +// +//go:nosplit func (i *Int64) CompareAndSwap(old, new int64) bool { return Casint64(&i.value, old, new) } // Swap replaces i's value with new, returning // i's value before the replacement. +// +//go:nosplit func (i *Int64) Swap(new int64) int64 { return Xchgint64(&i.value, new) } @@ -87,6 +105,8 @@ func (i *Int64) Swap(new int64) int64 { // // This operation wraps around in the usual // two's-complement way. +// +//go:nosplit func (i *Int64) Add(delta int64) int64 { return Xaddint64(&i.value, delta) } @@ -107,6 +127,8 @@ func (u *Uint8) Load() uint8 { } // Store updates the value atomically. +// +//go:nosplit func (u *Uint8) Store(value uint8) { Store8(&u.value, value) } @@ -116,6 +138,8 @@ func (u *Uint8) Store(value uint8) { // the result into u. // // The full process is performed atomically. +// +//go:nosplit func (u *Uint8) And(value uint8) { And8(&u.value, value) } @@ -125,6 +149,8 @@ func (u *Uint8) And(value uint8) { // the result into u. // // The full process is performed atomically. +// +//go:nosplit func (u *Uint8) Or(value uint8) { Or8(&u.value, value) } @@ -145,6 +171,8 @@ func (b *Bool) Load() bool { } // Store updates the value atomically. +// +//go:nosplit func (b *Bool) Store(value bool) { s := uint8(0) if value { @@ -182,6 +210,8 @@ func (u *Uint32) LoadAcquire() uint32 { } // Store updates the value atomically. +// +//go:nosplit func (u *Uint32) Store(value uint32) { Store(&u.value, value) } @@ -193,6 +223,8 @@ func (u *Uint32) Store(value uint32) { // on this thread can be observed to occur after it. // // WARNING: Use sparingly and with great care. +// +//go:nosplit func (u *Uint32) StoreRelease(value uint32) { StoreRel(&u.value, value) } @@ -201,6 +233,8 @@ func (u *Uint32) StoreRelease(value uint32) { // and if they're equal, swaps u's value with new. // // Returns true if the operation succeeded. +// +//go:nosplit func (u *Uint32) CompareAndSwap(old, new uint32) bool { return Cas(&u.value, old, new) } @@ -214,12 +248,16 @@ func (u *Uint32) CompareAndSwap(old, new uint32) bool { // Returns true if the operation succeeded. // // WARNING: Use sparingly and with great care. +// +//go:nosplit func (u *Uint32) CompareAndSwapRelease(old, new uint32) bool { return CasRel(&u.value, old, new) } // Swap replaces u's value with new, returning // u's value before the replacement. +// +//go:nosplit func (u *Uint32) Swap(value uint32) uint32 { return Xchg(&u.value, value) } @@ -229,6 +267,8 @@ func (u *Uint32) Swap(value uint32) uint32 { // the result into u. // // The full process is performed atomically. +// +//go:nosplit func (u *Uint32) And(value uint32) { And(&u.value, value) } @@ -238,6 +278,8 @@ func (u *Uint32) And(value uint32) { // the result into u. // // The full process is performed atomically. +// +//go:nosplit func (u *Uint32) Or(value uint32) { Or(&u.value, value) } @@ -247,6 +289,8 @@ func (u *Uint32) Or(value uint32) { // // This operation wraps around in the usual // two's-complement way. +// +//go:nosplit func (u *Uint32) Add(delta int32) uint32 { return Xadd(&u.value, delta) } @@ -270,6 +314,8 @@ func (u *Uint64) Load() uint64 { } // Store updates the value atomically. +// +//go:nosplit func (u *Uint64) Store(value uint64) { Store64(&u.value, value) } @@ -278,12 +324,16 @@ func (u *Uint64) Store(value uint64) { // and if they're equal, swaps u's value with new. // // Returns true if the operation succeeded. +// +//go:nosplit func (u *Uint64) CompareAndSwap(old, new uint64) bool { return Cas64(&u.value, old, new) } // Swap replaces u's value with new, returning // u's value before the replacement. +// +//go:nosplit func (u *Uint64) Swap(value uint64) uint64 { return Xchg64(&u.value, value) } @@ -308,6 +358,8 @@ type Uintptr struct { } // Load accesses and returns the value atomically. +// +//go:nosplit func (u *Uintptr) Load() uintptr { return Loaduintptr(&u.value) } @@ -326,6 +378,8 @@ func (u *Uintptr) LoadAcquire() uintptr { } // Store updates the value atomically. +// +//go:nosplit func (u *Uintptr) Store(value uintptr) { Storeuintptr(&u.value, value) } @@ -337,6 +391,8 @@ func (u *Uintptr) Store(value uintptr) { // on this thread can be observed to occur after it. // // WARNING: Use sparingly and with great care. +// +//go:nosplit func (u *Uintptr) StoreRelease(value uintptr) { StoreReluintptr(&u.value, value) } @@ -345,12 +401,16 @@ func (u *Uintptr) StoreRelease(value uintptr) { // and if they're equal, swaps u's value with new. // // Returns true if the operation succeeded. +// +//go:nosplit func (u *Uintptr) CompareAndSwap(old, new uintptr) bool { return Casuintptr(&u.value, old, new) } // Swap replaces u's value with new, returning // u's value before the replacement. +// +//go:nosplit func (u *Uintptr) Swap(value uintptr) uintptr { return Xchguintptr(&u.value, value) } @@ -360,6 +420,8 @@ func (u *Uintptr) Swap(value uintptr) uintptr { // // This operation wraps around in the usual // two's-complement way. +// +//go:nosplit func (u *Uintptr) Add(delta uintptr) uintptr { return Xadduintptr(&u.value, delta) } @@ -383,6 +445,8 @@ func (f *Float64) Load() float64 { } // Store updates the value atomically. +// +//go:nosplit func (f *Float64) Store(value float64) { f.u.Store(*(*uint64)(unsafe.Pointer(&value))) } @@ -414,6 +478,8 @@ func (u *UnsafePointer) Load() unsafe.Pointer { // perform a write barrier on value, and so this operation may // hide pointers from the GC. Use with care and sparingly. // It is safe to use with values not found in the Go heap. +// +//go:nosplit func (u *UnsafePointer) StoreNoWB(value unsafe.Pointer) { StorepNoWB(unsafe.Pointer(&u.value), value) } @@ -428,6 +494,8 @@ func (u *UnsafePointer) StoreNoWB(value unsafe.Pointer) { // perform a write barrier on value, and so this operation may // hide pointers from the GC. Use with care and sparingly. // It is safe to use with values not found in the Go heap. +// +//go:nosplit func (u *UnsafePointer) CompareAndSwapNoWB(old, new unsafe.Pointer) bool { return Casp1(&u.value, old, new) } @@ -450,6 +518,8 @@ func (p *Pointer[T]) Load() *T { // perform a write barrier on value, and so this operation may // hide pointers from the GC. Use with care and sparingly. // It is safe to use with values not found in the Go heap. +// +//go:nosplit func (p *Pointer[T]) StoreNoWB(value *T) { p.u.StoreNoWB(unsafe.Pointer(value)) } @@ -464,6 +534,8 @@ func (p *Pointer[T]) StoreNoWB(value *T) { // perform a write barrier on value, and so this operation may // hide pointers from the GC. Use with care and sparingly. // It is safe to use with values not found in the Go heap. +// +//go:nosplit func (p *Pointer[T]) CompareAndSwapNoWB(old, new *T) bool { return p.u.CompareAndSwapNoWB(unsafe.Pointer(old), unsafe.Pointer(new)) } diff --git a/src/runtime/internal/atomic/types_64bit.go b/src/runtime/internal/atomic/types_64bit.go index 43c1ba27096..fda2de9cc4c 100644 --- a/src/runtime/internal/atomic/types_64bit.go +++ b/src/runtime/internal/atomic/types_64bit.go @@ -13,6 +13,8 @@ package atomic // on this thread can be observed to occur before it. // // WARNING: Use sparingly and with great care. +// +//go:nosplit func (u *Uint64) LoadAcquire() uint64 { return LoadAcq64(&u.value) } @@ -24,6 +26,8 @@ func (u *Uint64) LoadAcquire() uint64 { // on this thread can be observed to occur after it. // // WARNING: Use sparingly and with great care. +// +//go:nosplit func (u *Uint64) StoreRelease(value uint64) { StoreRel64(&u.value, value) }