runtime: mark all methods of atomic types with go:nosplit

Atomic types may be used anywhere in the runtime,
so they must omit its usual stack overflow checks to avoid errors from stack splits.

Change-Id: Icb497334c860bcaa284a2d5a3edc47ee89844301
Reviewed-on: https://go-review.googlesource.com/c/go/+/425484
Reviewed-by: Cuong Manh Le <cuong.manhle.vn@gmail.com>
Run-TryBot: Ian Lance Taylor <iant@golang.org>
Auto-Submit: Cuong Manh Le <cuong.manhle.vn@gmail.com>
Reviewed-by: David Chase <drchase@google.com>
TryBot-Result: Gopher Robot <gobot@golang.org>
Reviewed-by: Michael Pratt <mpratt@google.com>

src/runtime/internal/atomic/types.go

@@ -15,11 +15,15 @@ type Int32 struct {
 }
 
 // Load accesses and returns the value atomically.
+//
+//go:nosplit
 func (i *Int32) Load() int32 {
 	return Loadint32(&i.value)
 }
 
 // Store updates the value atomically.
+//
+//go:nosplit
 func (i *Int32) Store(value int32) {
 	Storeint32(&i.value, value)
 }
@@ -28,12 +32,16 @@ func (i *Int32) Store(value int32) {
 // and if they're equal, swaps i's value with new.
 //
 // Returns true if the operation succeeded.
+//
+//go:nosplit
 func (i *Int32) CompareAndSwap(old, new int32) bool {
 	return Casint32(&i.value, old, new)
 }
 
 // Swap replaces i's value with new, returning
 // i's value before the replacement.
+//
+//go:nosplit
 func (i *Int32) Swap(new int32) int32 {
 	return Xchgint32(&i.value, new)
 }
@@ -43,6 +51,8 @@ func (i *Int32) Swap(new int32) int32 {
 //
 // This operation wraps around in the usual
 // two's-complement way.
+//
+//go:nosplit
 func (i *Int32) Add(delta int32) int32 {
 	return Xaddint32(&i.value, delta)
 }
@@ -59,11 +69,15 @@ type Int64 struct {
 }
 
 // Load accesses and returns the value atomically.
+//
+//go:nosplit
 func (i *Int64) Load() int64 {
 	return Loadint64(&i.value)
 }
 
 // Store updates the value atomically.
+//
+//go:nosplit
 func (i *Int64) Store(value int64) {
 	Storeint64(&i.value, value)
 }
@@ -72,12 +86,16 @@ func (i *Int64) Store(value int64) {
 // and if they're equal, swaps i's value with new.
 //
 // Returns true if the operation succeeded.
+//
+//go:nosplit
 func (i *Int64) CompareAndSwap(old, new int64) bool {
 	return Casint64(&i.value, old, new)
 }
 
 // Swap replaces i's value with new, returning
 // i's value before the replacement.
+//
+//go:nosplit
 func (i *Int64) Swap(new int64) int64 {
 	return Xchgint64(&i.value, new)
 }
@@ -87,6 +105,8 @@ func (i *Int64) Swap(new int64) int64 {
 //
 // This operation wraps around in the usual
 // two's-complement way.
+//
+//go:nosplit
 func (i *Int64) Add(delta int64) int64 {
 	return Xaddint64(&i.value, delta)
 }
@@ -107,6 +127,8 @@ func (u *Uint8) Load() uint8 {
 }
 
 // Store updates the value atomically.
+//
+//go:nosplit
 func (u *Uint8) Store(value uint8) {
 	Store8(&u.value, value)
 }
@@ -116,6 +138,8 @@ func (u *Uint8) Store(value uint8) {
 // the result into u.
 //
 // The full process is performed atomically.
+//
+//go:nosplit
 func (u *Uint8) And(value uint8) {
 	And8(&u.value, value)
 }
@@ -125,6 +149,8 @@ func (u *Uint8) And(value uint8) {
 // the result into u.
 //
 // The full process is performed atomically.
+//
+//go:nosplit
 func (u *Uint8) Or(value uint8) {
 	Or8(&u.value, value)
 }
@@ -145,6 +171,8 @@ func (b *Bool) Load() bool {
 }
 
 // Store updates the value atomically.
+//
+//go:nosplit
 func (b *Bool) Store(value bool) {
 	s := uint8(0)
 	if value {
@@ -182,6 +210,8 @@ func (u *Uint32) LoadAcquire() uint32 {
 }
 
 // Store updates the value atomically.
+//
+//go:nosplit
 func (u *Uint32) Store(value uint32) {
 	Store(&u.value, value)
 }
@@ -193,6 +223,8 @@ func (u *Uint32) Store(value uint32) {
 // on this thread can be observed to occur after it.
 //
 // WARNING: Use sparingly and with great care.
+//
+//go:nosplit
 func (u *Uint32) StoreRelease(value uint32) {
 	StoreRel(&u.value, value)
 }
@@ -201,6 +233,8 @@ func (u *Uint32) StoreRelease(value uint32) {
 // and if they're equal, swaps u's value with new.
 //
 // Returns true if the operation succeeded.
+//
+//go:nosplit
 func (u *Uint32) CompareAndSwap(old, new uint32) bool {
 	return Cas(&u.value, old, new)
 }
@@ -214,12 +248,16 @@ func (u *Uint32) CompareAndSwap(old, new uint32) bool {
 // Returns true if the operation succeeded.
 //
 // WARNING: Use sparingly and with great care.
+//
+//go:nosplit
 func (u *Uint32) CompareAndSwapRelease(old, new uint32) bool {
 	return CasRel(&u.value, old, new)
 }
 
 // Swap replaces u's value with new, returning
 // u's value before the replacement.
+//
+//go:nosplit
 func (u *Uint32) Swap(value uint32) uint32 {
 	return Xchg(&u.value, value)
 }
@@ -229,6 +267,8 @@ func (u *Uint32) Swap(value uint32) uint32 {
 // the result into u.
 //
 // The full process is performed atomically.
+//
+//go:nosplit
 func (u *Uint32) And(value uint32) {
 	And(&u.value, value)
 }
@@ -238,6 +278,8 @@ func (u *Uint32) And(value uint32) {
 // the result into u.
 //
 // The full process is performed atomically.
+//
+//go:nosplit
 func (u *Uint32) Or(value uint32) {
 	Or(&u.value, value)
 }
@@ -247,6 +289,8 @@ func (u *Uint32) Or(value uint32) {
 //
 // This operation wraps around in the usual
 // two's-complement way.
+//
+//go:nosplit
 func (u *Uint32) Add(delta int32) uint32 {
 	return Xadd(&u.value, delta)
 }
@@ -270,6 +314,8 @@ func (u *Uint64) Load() uint64 {
 }
 
 // Store updates the value atomically.
+//
+//go:nosplit
 func (u *Uint64) Store(value uint64) {
 	Store64(&u.value, value)
 }
@@ -278,12 +324,16 @@ func (u *Uint64) Store(value uint64) {
 // and if they're equal, swaps u's value with new.
 //
 // Returns true if the operation succeeded.
+//
+//go:nosplit
 func (u *Uint64) CompareAndSwap(old, new uint64) bool {
 	return Cas64(&u.value, old, new)
 }
 
 // Swap replaces u's value with new, returning
 // u's value before the replacement.
+//
+//go:nosplit
 func (u *Uint64) Swap(value uint64) uint64 {
 	return Xchg64(&u.value, value)
 }
@@ -308,6 +358,8 @@ type Uintptr struct {
 }
 
 // Load accesses and returns the value atomically.
+//
+//go:nosplit
 func (u *Uintptr) Load() uintptr {
 	return Loaduintptr(&u.value)
 }
@@ -326,6 +378,8 @@ func (u *Uintptr) LoadAcquire() uintptr {
 }
 
 // Store updates the value atomically.
+//
+//go:nosplit
 func (u *Uintptr) Store(value uintptr) {
 	Storeuintptr(&u.value, value)
 }
@@ -337,6 +391,8 @@ func (u *Uintptr) Store(value uintptr) {
 // on this thread can be observed to occur after it.
 //
 // WARNING: Use sparingly and with great care.
+//
+//go:nosplit
 func (u *Uintptr) StoreRelease(value uintptr) {
 	StoreReluintptr(&u.value, value)
 }
@@ -345,12 +401,16 @@ func (u *Uintptr) StoreRelease(value uintptr) {
 // and if they're equal, swaps u's value with new.
 //
 // Returns true if the operation succeeded.
+//
+//go:nosplit
 func (u *Uintptr) CompareAndSwap(old, new uintptr) bool {
 	return Casuintptr(&u.value, old, new)
 }
 
 // Swap replaces u's value with new, returning
 // u's value before the replacement.
+//
+//go:nosplit
 func (u *Uintptr) Swap(value uintptr) uintptr {
 	return Xchguintptr(&u.value, value)
 }
@@ -360,6 +420,8 @@ func (u *Uintptr) Swap(value uintptr) uintptr {
 //
 // This operation wraps around in the usual
 // two's-complement way.
+//
+//go:nosplit
 func (u *Uintptr) Add(delta uintptr) uintptr {
 	return Xadduintptr(&u.value, delta)
 }
@@ -383,6 +445,8 @@ func (f *Float64) Load() float64 {
 }
 
 // Store updates the value atomically.
+//
+//go:nosplit
 func (f *Float64) Store(value float64) {
 	f.u.Store(*(*uint64)(unsafe.Pointer(&value)))
 }
@@ -414,6 +478,8 @@ func (u *UnsafePointer) Load() unsafe.Pointer {
 // perform a write barrier on value, and so this operation may
 // hide pointers from the GC. Use with care and sparingly.
 // It is safe to use with values not found in the Go heap.
+//
+//go:nosplit
 func (u *UnsafePointer) StoreNoWB(value unsafe.Pointer) {
 	StorepNoWB(unsafe.Pointer(&u.value), value)
 }
@@ -428,6 +494,8 @@ func (u *UnsafePointer) StoreNoWB(value unsafe.Pointer) {
 // perform a write barrier on value, and so this operation may
 // hide pointers from the GC. Use with care and sparingly.
 // It is safe to use with values not found in the Go heap.
+//
+//go:nosplit
 func (u *UnsafePointer) CompareAndSwapNoWB(old, new unsafe.Pointer) bool {
 	return Casp1(&u.value, old, new)
 }
@@ -450,6 +518,8 @@ func (p *Pointer[T]) Load() *T {
 // perform a write barrier on value, and so this operation may
 // hide pointers from the GC. Use with care and sparingly.
 // It is safe to use with values not found in the Go heap.
+//
+//go:nosplit
 func (p *Pointer[T]) StoreNoWB(value *T) {
 	p.u.StoreNoWB(unsafe.Pointer(value))
 }
@@ -464,6 +534,8 @@ func (p *Pointer[T]) StoreNoWB(value *T) {
 // perform a write barrier on value, and so this operation may
 // hide pointers from the GC. Use with care and sparingly.
 // It is safe to use with values not found in the Go heap.
+//
+//go:nosplit
 func (p *Pointer[T]) CompareAndSwapNoWB(old, new *T) bool {
 	return p.u.CompareAndSwapNoWB(unsafe.Pointer(old), unsafe.Pointer(new))
 }

src/runtime/internal/atomic/types_64bit.go

@@ -13,6 +13,8 @@ package atomic
 // on this thread can be observed to occur before it.
 //
 // WARNING: Use sparingly and with great care.
+//
+//go:nosplit
 func (u *Uint64) LoadAcquire() uint64 {
 	return LoadAcq64(&u.value)
 }
@@ -24,6 +26,8 @@ func (u *Uint64) LoadAcquire() uint64 {
 // on this thread can be observed to occur after it.
 //
 // WARNING: Use sparingly and with great care.
+//
+//go:nosplit
 func (u *Uint64) StoreRelease(value uint64) {
 	StoreRel64(&u.value, value)
 }