[dev.garbage] runtime: reduce stack footprint of write barriers

This is going to hurt a bit but we'll make it better later.
Now the race detector can be run again.

I added the write barrier optimizations from
CL 183020043 to try to make it hurt a little less.

TBR=rlh
CC=golang-codereviews
https://golang.org/cl/185070043

src/run.bash

@@ -163,7 +163,7 @@ esac
 # and only on amd64, and only when cgo is enabled.
 # Delayed until here so we know whether to try external linking.
 # DISABLED until we get garbage collection working.
-case "$GOHOSTOS-$GOOS-$GOARCH-$CGO_ENABLED-XXX-DISABLED" in
+case "$GOHOSTOS-$GOOS-$GOARCH-$CGO_ENABLED" in
 linux-linux-amd64-1 | freebsd-freebsd-amd64-1 | darwin-darwin-amd64-1)
 	echo
 	echo '# Testing race detector.'

src/runtime/mgc0.go

@@ -102,19 +102,25 @@ const (
 	_PoisonStack = 0x6868686868686868 & (1<<(8*ptrSize) - 1)
 )
 
+func needwb() bool {
+	return gcphase == _GCmark || gcphase == _GCmarktermination
+}
+
 // NOTE: Really dst *unsafe.Pointer, src unsafe.Pointer,
 // but if we do that, Go inserts a write barrier on *dst = src.
 //go:nosplit
 func writebarrierptr(dst *uintptr, src uintptr) {
 	*dst = src
+	if needwb() {
 		writebarrierptr_nostore(dst, src)
+	}
 }
 
 // Like writebarrierptr, but the store has already been applied.
 // Do not reapply.
 //go:nosplit
 func writebarrierptr_nostore(dst *uintptr, src uintptr) {
-	if getg() == nil { // very low-level startup
+	if getg() == nil || !needwb() { // very low-level startup
 		return
 	}
 
@@ -162,6 +168,12 @@ func writebarrieriface(dst *[2]uintptr, src [2]uintptr) {
 
 //go:nosplit
 func writebarrierfat(typ *_type, dst, src unsafe.Pointer) {
+	if !needwb() {
+		memmove(dst, src, typ.size)
+		return
+	}
+
+	systemstack(func() {
 		mask := loadPtrMask(typ)
 		nptr := typ.size / ptrSize
 		for i := uintptr(0); i < nptr; i += 2 {
@@ -185,6 +197,7 @@ func writebarrierfat(typ *_type, dst, src unsafe.Pointer) {
 			dst = add(dst, ptrSize)
 			src = add(src, ptrSize)
 		}
+	})
 }
 
 //go:nosplit
@@ -199,6 +212,12 @@ func writebarriercopy(typ *_type, dst, src slice) int {
 	dstp := unsafe.Pointer(dst.array)
 	srcp := unsafe.Pointer(src.array)
 
+	if !needwb() {
+		memmove(dstp, srcp, uintptr(n)*typ.size)
+		return int(n)
+	}
+
+	systemstack(func() {
 		if uintptr(srcp) < uintptr(dstp) && uintptr(srcp)+uintptr(n)*typ.size > uintptr(dstp) {
 			// Overlap with src before dst.
 			// Copy backward, being careful not to move dstp/srcp
@@ -227,5 +246,6 @@ func writebarriercopy(typ *_type, dst, src slice) int {
 				srcp = add(srcp, typ.size)
 			}
 		}
+	})
 	return int(n)
 }