runtime: avoid relying on the unwinder in deferreturn

This CL changes deferreturn so that it never needs to invoke the
unwinder. Instead, in the unusual case that we recover into a frame
with pending open-coded defers, we now save the extra state needed to
find them in g.param.

Change-Id: Ied35f6c1063fee5b6044cc37b2bccd3f90682fe6
Reviewed-on: https://go-review.googlesource.com/c/go/+/515856
Reviewed-by: Keith Randall <khr@golang.org>
Reviewed-by: Keith Randall <khr@google.com>
TryBot-Result: Gopher Robot <gobot@golang.org>
Run-TryBot: Matthew Dempsky <mdempsky@google.com>

src/runtime/panic.go

@@ -635,19 +635,26 @@ func (p *_panic) start(pc uintptr, sp unsafe.Pointer) {
 	p.startPC = getcallerpc()
 	p.startSP = unsafe.Pointer(getcallersp())
 
-	if !p.deferreturn {
-		p.link = gp._panic
-		gp._panic = (*_panic)(noescape(unsafe.Pointer(p)))
-	} else {
-		// Fast path for deferreturn: if there's a pending linked defer
-		// for this frame, then we know there aren't any open-coded
-		// defers, and we don't need to find the parent frame either.
-		if d := gp._defer; d != nil && d.sp == uintptr(sp) {
-			p.sp = sp
-			return
+	if p.deferreturn {
+		p.sp = sp
+
+		if s := (*savedOpenDeferState)(gp.param); s != nil {
+			// recovery saved some state for us, so that we can resume
+			// calling open-coded defers without unwinding the stack.
+
+			gp.param = nil
+
+			p.retpc = s.retpc
+			p.deferBitsPtr = (*byte)(add(sp, s.deferBitsOffset))
+			p.slotsPtr = add(sp, s.slotsOffset)
 		}
+
+		return
 	}
 
+	p.link = gp._panic
+	gp._panic = (*_panic)(noescape(unsafe.Pointer(p)))
+
 	// Initialize state machine, and find the first frame with a defer.
 	//
 	// Note: We could use startPC and startSP here, but callers will
@@ -684,6 +691,15 @@ func (p *_panic) nextDefer() (func(), bool) {
 	for {
 		for p.deferBitsPtr != nil {
 			bits := *p.deferBitsPtr
+
+			// Check whether any open-coded defers are still pending.
+			//
+			// Note: We need to check this upfront (rather than after
+			// clearing the top bit) because it's possible that Goexit
+			// invokes a deferred call, and there were still more pending
+			// open-coded defers in the frame; but then the deferred call
+			// panic and invoked the remaining defers in the frame, before
+			// recovering and restarting the Goexit loop.
 			if bits == 0 {
 				p.deferBitsPtr = nil
 				break
@@ -730,9 +746,7 @@ func (p *_panic) nextFrame() (ok bool) {
 	gp := getg()
 	systemstack(func() {
 		var limit uintptr
-		if p.deferreturn {
-			limit = uintptr(p.fp)
-		} else if d := gp._defer; d != nil {
+		if d := gp._defer; d != nil {
 			limit = uintptr(d.sp)
 		}
 
@@ -749,22 +763,18 @@ func (p *_panic) nextFrame() (ok bool) {
 			// then we can simply loop until we find the next frame where
 			// it's non-zero.
 
-			if p.initOpenCodedDefers(u.frame.fn, unsafe.Pointer(u.frame.varp)) {
-				break // found a frame with open-coded defers
+			if u.frame.sp == limit {
+				break // found a frame with linked defers
 			}
 
-			if u.frame.sp == limit {
-				break // found a frame with linked defers, or deferreturn with no defers
+			if p.initOpenCodedDefers(u.frame.fn, unsafe.Pointer(u.frame.varp)) {
+				break // found a frame with open-coded defers
 			}
 
 			u.next()
 		}
 
-		if p.deferreturn {
-			p.lr = 0 // prevent unwinding past this frame
-		} else {
-			p.lr = u.frame.lr
-		}
+		p.lr = u.frame.lr
 		p.sp = unsafe.Pointer(u.frame.sp)
 		p.fp = unsafe.Pointer(u.frame.fp)
 
@@ -889,6 +899,7 @@ var paniclk mutex
 func recovery(gp *g) {
 	p := gp._panic
 	pc, sp := p.retpc, uintptr(p.sp)
+	p0, saveOpenDeferState := p, p.deferBitsPtr != nil && *p.deferBitsPtr != 0
 
 	// Unwind the panic stack.
 	for ; p != nil && uintptr(p.startSP) < sp; p = p.link {
@@ -913,6 +924,7 @@ func recovery(gp *g) {
 		// worthwhile though.
 		if p.goexit {
 			pc, sp = p.startPC, uintptr(p.startSP)
+			saveOpenDeferState = false // goexit is unwinding the stack anyway
 			break
 		}
 
@@ -924,6 +936,24 @@ func recovery(gp *g) {
 		gp.sig = 0
 	}
 
+	if gp.param != nil {
+		throw("unexpected gp.param")
+	}
+	if saveOpenDeferState {
+		// If we're returning to deferreturn and there are more open-coded
+		// defers for it to call, save enough state for it to be able to
+		// pick up where p0 left off.
+		gp.param = unsafe.Pointer(&savedOpenDeferState{
+			retpc: p0.retpc,
+
+			// We need to save deferBitsPtr and slotsPtr too, but those are
+			// stack pointers. To avoid issues around heap objects pointing
+			// to the stack, save them as offsets from SP.
+			deferBitsOffset: uintptr(unsafe.Pointer(p0.deferBitsPtr)) - uintptr(p0.sp),
+			slotsOffset:     uintptr(p0.slotsPtr) - uintptr(p0.sp),
+		})
+	}
+
 	// TODO(mdempsky): Currently, we rely on frames containing "defer"
 	// to end with "CALL deferreturn; RET". This allows deferreturn to
 	// finish running any pending defers in the frame.

src/runtime/runtime2.go

@@ -433,7 +433,7 @@ type g struct {
 	// param is a generic pointer parameter field used to pass
 	// values in particular contexts where other storage for the
 	// parameter would be difficult to find. It is currently used
-	// in three ways:
+	// in four ways:
 	// 1. When a channel operation wakes up a blocked goroutine, it sets param to
 	//    point to the sudog of the completed blocking operation.
 	// 2. By gcAssistAlloc1 to signal back to its caller that the goroutine completed
@@ -441,6 +441,8 @@ type g struct {
 	//    stack may have moved in the meantime.
 	// 3. By debugCallWrap to pass parameters to a new goroutine because allocating a
 	//    closure in the runtime is forbidden.
+	// 4. When a panic is recovered and control returns to the respective frame,
+	//    param may point to a savedOpenDeferState.
 	param        unsafe.Pointer
 	atomicstatus atomic.Uint32
 	stackLock    uint32 // sigprof/scang lock; TODO: fold in to atomicstatus
@@ -1041,6 +1043,15 @@ type _panic struct {
 	deferreturn bool
 }
 
+// savedOpenDeferState tracks the extra state from _panic that's
+// necessary for deferreturn to pick up where gopanic left off,
+// without needing to unwind the stack.
+type savedOpenDeferState struct {
+	retpc           uintptr
+	deferBitsOffset uintptr
+	slotsOffset     uintptr
+}
+
 // ancestorInfo records details of where a goroutine was started.
 type ancestorInfo struct {
 	pcs  []uintptr // pcs from the stack of this goroutine