diff --git a/src/runtime/panic.go b/src/runtime/panic.go index e7483b80b6a..5b7f35a0a50 100644 --- a/src/runtime/panic.go +++ b/src/runtime/panic.go @@ -635,19 +635,26 @@ func (p *_panic) start(pc uintptr, sp unsafe.Pointer) { p.startPC = getcallerpc() p.startSP = unsafe.Pointer(getcallersp()) - if !p.deferreturn { - p.link = gp._panic - gp._panic = (*_panic)(noescape(unsafe.Pointer(p))) - } else { - // Fast path for deferreturn: if there's a pending linked defer - // for this frame, then we know there aren't any open-coded - // defers, and we don't need to find the parent frame either. - if d := gp._defer; d != nil && d.sp == uintptr(sp) { - p.sp = sp - return + if p.deferreturn { + p.sp = sp + + if s := (*savedOpenDeferState)(gp.param); s != nil { + // recovery saved some state for us, so that we can resume + // calling open-coded defers without unwinding the stack. + + gp.param = nil + + p.retpc = s.retpc + p.deferBitsPtr = (*byte)(add(sp, s.deferBitsOffset)) + p.slotsPtr = add(sp, s.slotsOffset) } + + return } + p.link = gp._panic + gp._panic = (*_panic)(noescape(unsafe.Pointer(p))) + // Initialize state machine, and find the first frame with a defer. // // Note: We could use startPC and startSP here, but callers will @@ -684,6 +691,15 @@ func (p *_panic) nextDefer() (func(), bool) { for { for p.deferBitsPtr != nil { bits := *p.deferBitsPtr + + // Check whether any open-coded defers are still pending. + // + // Note: We need to check this upfront (rather than after + // clearing the top bit) because it's possible that Goexit + // invokes a deferred call, and there were still more pending + // open-coded defers in the frame; but then the deferred call + // panic and invoked the remaining defers in the frame, before + // recovering and restarting the Goexit loop. if bits == 0 { p.deferBitsPtr = nil break @@ -730,9 +746,7 @@ func (p *_panic) nextFrame() (ok bool) { gp := getg() systemstack(func() { var limit uintptr - if p.deferreturn { - limit = uintptr(p.fp) - } else if d := gp._defer; d != nil { + if d := gp._defer; d != nil { limit = uintptr(d.sp) } @@ -749,22 +763,18 @@ func (p *_panic) nextFrame() (ok bool) { // then we can simply loop until we find the next frame where // it's non-zero. - if p.initOpenCodedDefers(u.frame.fn, unsafe.Pointer(u.frame.varp)) { - break // found a frame with open-coded defers + if u.frame.sp == limit { + break // found a frame with linked defers } - if u.frame.sp == limit { - break // found a frame with linked defers, or deferreturn with no defers + if p.initOpenCodedDefers(u.frame.fn, unsafe.Pointer(u.frame.varp)) { + break // found a frame with open-coded defers } u.next() } - if p.deferreturn { - p.lr = 0 // prevent unwinding past this frame - } else { - p.lr = u.frame.lr - } + p.lr = u.frame.lr p.sp = unsafe.Pointer(u.frame.sp) p.fp = unsafe.Pointer(u.frame.fp) @@ -889,6 +899,7 @@ var paniclk mutex func recovery(gp *g) { p := gp._panic pc, sp := p.retpc, uintptr(p.sp) + p0, saveOpenDeferState := p, p.deferBitsPtr != nil && *p.deferBitsPtr != 0 // Unwind the panic stack. for ; p != nil && uintptr(p.startSP) < sp; p = p.link { @@ -913,6 +924,7 @@ func recovery(gp *g) { // worthwhile though. if p.goexit { pc, sp = p.startPC, uintptr(p.startSP) + saveOpenDeferState = false // goexit is unwinding the stack anyway break } @@ -924,6 +936,24 @@ func recovery(gp *g) { gp.sig = 0 } + if gp.param != nil { + throw("unexpected gp.param") + } + if saveOpenDeferState { + // If we're returning to deferreturn and there are more open-coded + // defers for it to call, save enough state for it to be able to + // pick up where p0 left off. + gp.param = unsafe.Pointer(&savedOpenDeferState{ + retpc: p0.retpc, + + // We need to save deferBitsPtr and slotsPtr too, but those are + // stack pointers. To avoid issues around heap objects pointing + // to the stack, save them as offsets from SP. + deferBitsOffset: uintptr(unsafe.Pointer(p0.deferBitsPtr)) - uintptr(p0.sp), + slotsOffset: uintptr(p0.slotsPtr) - uintptr(p0.sp), + }) + } + // TODO(mdempsky): Currently, we rely on frames containing "defer" // to end with "CALL deferreturn; RET". This allows deferreturn to // finish running any pending defers in the frame. diff --git a/src/runtime/runtime2.go b/src/runtime/runtime2.go index cdd8c3db7fe..54fab050eaa 100644 --- a/src/runtime/runtime2.go +++ b/src/runtime/runtime2.go @@ -433,7 +433,7 @@ type g struct { // param is a generic pointer parameter field used to pass // values in particular contexts where other storage for the // parameter would be difficult to find. It is currently used - // in three ways: + // in four ways: // 1. When a channel operation wakes up a blocked goroutine, it sets param to // point to the sudog of the completed blocking operation. // 2. By gcAssistAlloc1 to signal back to its caller that the goroutine completed @@ -441,6 +441,8 @@ type g struct { // stack may have moved in the meantime. // 3. By debugCallWrap to pass parameters to a new goroutine because allocating a // closure in the runtime is forbidden. + // 4. When a panic is recovered and control returns to the respective frame, + // param may point to a savedOpenDeferState. param unsafe.Pointer atomicstatus atomic.Uint32 stackLock uint32 // sigprof/scang lock; TODO: fold in to atomicstatus @@ -1041,6 +1043,15 @@ type _panic struct { deferreturn bool } +// savedOpenDeferState tracks the extra state from _panic that's +// necessary for deferreturn to pick up where gopanic left off, +// without needing to unwind the stack. +type savedOpenDeferState struct { + retpc uintptr + deferBitsOffset uintptr + slotsOffset uintptr +} + // ancestorInfo records details of where a goroutine was started. type ancestorInfo struct { pcs []uintptr // pcs from the stack of this goroutine