diff --git a/src/runtime/mgc.go b/src/runtime/mgc.go index 82b12b6c09..820e3782a8 100644 --- a/src/runtime/mgc.go +++ b/src/runtime/mgc.go @@ -133,26 +133,12 @@ const ( _RootFlushCaches = 4 _RootCount = 5 - debugStackBarrier = false - // sweepMinHeapDistance is a lower bound on the heap distance // (in bytes) reserved for concurrent sweeping between GC // cycles. This will be scaled by gcpercent/100. sweepMinHeapDistance = 1024 * 1024 ) -// firstStackBarrierOffset is the approximate byte offset at -// which to place the first stack barrier from the current SP. -// This is a lower bound on how much stack will have to be -// re-scanned during mark termination. Subsequent barriers are -// placed at firstStackBarrierOffset * 2^n offsets. -// -// For debugging, this can be set to 0, which will install a -// stack barrier at every frame. If you do this, you may also -// have to raise _StackMin, since the stack barrier -// bookkeeping will use a large amount of each stack. -var firstStackBarrierOffset = 1024 - // heapminimum is the minimum heap size at which to trigger GC. // For small heaps, this overrides the usual GOGC*live set rule. // diff --git a/src/runtime/mgcmark.go b/src/runtime/mgcmark.go index 44f951269b..968b0cdca0 100644 --- a/src/runtime/mgcmark.go +++ b/src/runtime/mgcmark.go @@ -487,167 +487,6 @@ func scanframeworker(frame *stkframe, unused unsafe.Pointer, gcw *gcWork) { } } -// gcMaxStackBarriers returns the maximum number of stack barriers -// that can be installed in a stack of stackSize bytes. -func gcMaxStackBarriers(stackSize int) (n int) { - if firstStackBarrierOffset == 0 { - // Special debugging case for inserting stack barriers - // at every frame. Steal half of the stack for the - // []stkbar. Technically, if the stack were to consist - // solely of return PCs we would need two thirds of - // the stack, but stealing that much breaks things and - // this doesn't happen in practice. - return stackSize / 2 / int(unsafe.Sizeof(stkbar{})) - } - - offset := firstStackBarrierOffset - for offset < stackSize { - n++ - offset *= 2 - } - return n + 1 -} - -// gcInstallStackBarrier installs a stack barrier over the return PC of frame. -//go:nowritebarrier -func gcInstallStackBarrier(gp *g, frame *stkframe) bool { - if frame.lr == 0 { - if debugStackBarrier { - print("not installing stack barrier with no LR, goid=", gp.goid, "\n") - } - return false - } - - if frame.fn.entry == cgocallback_gofuncPC { - // cgocallback_gofunc doesn't return to its LR; - // instead, its return path puts LR in g.sched.pc and - // switches back to the system stack on which - // cgocallback_gofunc was originally called. We can't - // have a stack barrier in g.sched.pc, so don't - // install one in this frame. - if debugStackBarrier { - print("not installing stack barrier over LR of cgocallback_gofunc, goid=", gp.goid, "\n") - } - return false - } - - // Save the return PC and overwrite it with stackBarrier. - var lrUintptr uintptr - if usesLR { - lrUintptr = frame.sp - } else { - lrUintptr = frame.fp - regSize - } - lrPtr := (*uintreg)(unsafe.Pointer(lrUintptr)) - if debugStackBarrier { - print("install stack barrier at ", hex(lrUintptr), " over ", hex(*lrPtr), ", goid=", gp.goid, "\n") - if uintptr(*lrPtr) != frame.lr { - print("frame.lr=", hex(frame.lr)) - throw("frame.lr differs from stack LR") - } - } - - gp.stkbar = gp.stkbar[:len(gp.stkbar)+1] - stkbar := &gp.stkbar[len(gp.stkbar)-1] - stkbar.savedLRPtr = lrUintptr - stkbar.savedLRVal = uintptr(*lrPtr) - *lrPtr = uintreg(stackBarrierPC) - return true -} - -// gcRemoveStackBarriers removes all stack barriers installed in gp's stack. -//go:nowritebarrier -func gcRemoveStackBarriers(gp *g) { - if debugStackBarrier && gp.stkbarPos != 0 { - print("hit ", gp.stkbarPos, " stack barriers, goid=", gp.goid, "\n") - } - - // Remove stack barriers that we didn't hit. - for _, stkbar := range gp.stkbar[gp.stkbarPos:] { - gcRemoveStackBarrier(gp, stkbar) - } - - // Clear recorded stack barriers so copystack doesn't try to - // adjust them. - gp.stkbarPos = 0 - gp.stkbar = gp.stkbar[:0] -} - -// gcRemoveStackBarrier removes a single stack barrier. It is the -// inverse operation of gcInstallStackBarrier. -// -// This is nosplit to ensure gp's stack does not move. -// -//go:nowritebarrier -//go:nosplit -func gcRemoveStackBarrier(gp *g, stkbar stkbar) { - if debugStackBarrier { - print("remove stack barrier at ", hex(stkbar.savedLRPtr), " with ", hex(stkbar.savedLRVal), ", goid=", gp.goid, "\n") - } - lrPtr := (*uintreg)(unsafe.Pointer(stkbar.savedLRPtr)) - if val := *lrPtr; val != uintreg(stackBarrierPC) { - printlock() - print("at *", hex(stkbar.savedLRPtr), " expected stack barrier PC ", hex(stackBarrierPC), ", found ", hex(val), ", goid=", gp.goid, "\n") - print("gp.stkbar=") - gcPrintStkbars(gp.stkbar) - print(", gp.stkbarPos=", gp.stkbarPos, ", gp.stack=[", hex(gp.stack.lo), ",", hex(gp.stack.hi), ")\n") - throw("stack barrier lost") - } - *lrPtr = uintreg(stkbar.savedLRVal) -} - -// gcPrintStkbars prints a []stkbar for debugging. -func gcPrintStkbars(stkbar []stkbar) { - print("[") - for i, s := range stkbar { - if i > 0 { - print(" ") - } - print("*", hex(s.savedLRPtr), "=", hex(s.savedLRVal)) - } - print("]") -} - -// gcUnwindBarriers marks all stack barriers up the frame containing -// sp as hit and removes them. This is used during stack unwinding for -// panic/recover and by heapBitsBulkBarrier to force stack re-scanning -// when its destination is on the stack. -// -// This is nosplit to ensure gp's stack does not move. -// -//go:nosplit -func gcUnwindBarriers(gp *g, sp uintptr) { - // On LR machines, if there is a stack barrier on the return - // from the frame containing sp, this will mark it as hit even - // though it isn't, but it's okay to be conservative. - before := gp.stkbarPos - for int(gp.stkbarPos) < len(gp.stkbar) && gp.stkbar[gp.stkbarPos].savedLRPtr < sp { - gcRemoveStackBarrier(gp, gp.stkbar[gp.stkbarPos]) - gp.stkbarPos++ - } - if debugStackBarrier && gp.stkbarPos != before { - print("skip barriers below ", hex(sp), " in goid=", gp.goid, ": ") - gcPrintStkbars(gp.stkbar[before:gp.stkbarPos]) - print("\n") - } -} - -// nextBarrierPC returns the original return PC of the next stack barrier. -// Used by getcallerpc, so it must be nosplit. -//go:nosplit -func nextBarrierPC() uintptr { - gp := getg() - return gp.stkbar[gp.stkbarPos].savedLRVal -} - -// setNextBarrierPC sets the return PC of the next stack barrier. -// Used by setcallerpc, so it must be nosplit. -//go:nosplit -func setNextBarrierPC(pc uintptr) { - gp := getg() - gp.stkbar[gp.stkbarPos].savedLRVal = pc -} - // TODO(austin): Can we consolidate the gcDrain* functions? // gcDrain scans objects in work buffers, blackening grey diff --git a/src/runtime/mstkbar.go b/src/runtime/mstkbar.go new file mode 100644 index 0000000000..815b85d66e --- /dev/null +++ b/src/runtime/mstkbar.go @@ -0,0 +1,184 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Garbage collector: stack barriers + +package runtime + +import "unsafe" + +const debugStackBarrier = false + +// firstStackBarrierOffset is the approximate byte offset at +// which to place the first stack barrier from the current SP. +// This is a lower bound on how much stack will have to be +// re-scanned during mark termination. Subsequent barriers are +// placed at firstStackBarrierOffset * 2^n offsets. +// +// For debugging, this can be set to 0, which will install a +// stack barrier at every frame. If you do this, you may also +// have to raise _StackMin, since the stack barrier +// bookkeeping will use a large amount of each stack. +var firstStackBarrierOffset = 1024 + +// gcMaxStackBarriers returns the maximum number of stack barriers +// that can be installed in a stack of stackSize bytes. +func gcMaxStackBarriers(stackSize int) (n int) { + if firstStackBarrierOffset == 0 { + // Special debugging case for inserting stack barriers + // at every frame. Steal half of the stack for the + // []stkbar. Technically, if the stack were to consist + // solely of return PCs we would need two thirds of + // the stack, but stealing that much breaks things and + // this doesn't happen in practice. + return stackSize / 2 / int(unsafe.Sizeof(stkbar{})) + } + + offset := firstStackBarrierOffset + for offset < stackSize { + n++ + offset *= 2 + } + return n + 1 +} + +// gcInstallStackBarrier installs a stack barrier over the return PC of frame. +//go:nowritebarrier +func gcInstallStackBarrier(gp *g, frame *stkframe) bool { + if frame.lr == 0 { + if debugStackBarrier { + print("not installing stack barrier with no LR, goid=", gp.goid, "\n") + } + return false + } + + if frame.fn.entry == cgocallback_gofuncPC { + // cgocallback_gofunc doesn't return to its LR; + // instead, its return path puts LR in g.sched.pc and + // switches back to the system stack on which + // cgocallback_gofunc was originally called. We can't + // have a stack barrier in g.sched.pc, so don't + // install one in this frame. + if debugStackBarrier { + print("not installing stack barrier over LR of cgocallback_gofunc, goid=", gp.goid, "\n") + } + return false + } + + // Save the return PC and overwrite it with stackBarrier. + var lrUintptr uintptr + if usesLR { + lrUintptr = frame.sp + } else { + lrUintptr = frame.fp - regSize + } + lrPtr := (*uintreg)(unsafe.Pointer(lrUintptr)) + if debugStackBarrier { + print("install stack barrier at ", hex(lrUintptr), " over ", hex(*lrPtr), ", goid=", gp.goid, "\n") + if uintptr(*lrPtr) != frame.lr { + print("frame.lr=", hex(frame.lr)) + throw("frame.lr differs from stack LR") + } + } + + gp.stkbar = gp.stkbar[:len(gp.stkbar)+1] + stkbar := &gp.stkbar[len(gp.stkbar)-1] + stkbar.savedLRPtr = lrUintptr + stkbar.savedLRVal = uintptr(*lrPtr) + *lrPtr = uintreg(stackBarrierPC) + return true +} + +// gcRemoveStackBarriers removes all stack barriers installed in gp's stack. +//go:nowritebarrier +func gcRemoveStackBarriers(gp *g) { + if debugStackBarrier && gp.stkbarPos != 0 { + print("hit ", gp.stkbarPos, " stack barriers, goid=", gp.goid, "\n") + } + + // Remove stack barriers that we didn't hit. + for _, stkbar := range gp.stkbar[gp.stkbarPos:] { + gcRemoveStackBarrier(gp, stkbar) + } + + // Clear recorded stack barriers so copystack doesn't try to + // adjust them. + gp.stkbarPos = 0 + gp.stkbar = gp.stkbar[:0] +} + +// gcRemoveStackBarrier removes a single stack barrier. It is the +// inverse operation of gcInstallStackBarrier. +// +// This is nosplit to ensure gp's stack does not move. +// +//go:nowritebarrier +//go:nosplit +func gcRemoveStackBarrier(gp *g, stkbar stkbar) { + if debugStackBarrier { + print("remove stack barrier at ", hex(stkbar.savedLRPtr), " with ", hex(stkbar.savedLRVal), ", goid=", gp.goid, "\n") + } + lrPtr := (*uintreg)(unsafe.Pointer(stkbar.savedLRPtr)) + if val := *lrPtr; val != uintreg(stackBarrierPC) { + printlock() + print("at *", hex(stkbar.savedLRPtr), " expected stack barrier PC ", hex(stackBarrierPC), ", found ", hex(val), ", goid=", gp.goid, "\n") + print("gp.stkbar=") + gcPrintStkbars(gp.stkbar) + print(", gp.stkbarPos=", gp.stkbarPos, ", gp.stack=[", hex(gp.stack.lo), ",", hex(gp.stack.hi), ")\n") + throw("stack barrier lost") + } + *lrPtr = uintreg(stkbar.savedLRVal) +} + +// gcPrintStkbars prints a []stkbar for debugging. +func gcPrintStkbars(stkbar []stkbar) { + print("[") + for i, s := range stkbar { + if i > 0 { + print(" ") + } + print("*", hex(s.savedLRPtr), "=", hex(s.savedLRVal)) + } + print("]") +} + +// gcUnwindBarriers marks all stack barriers up the frame containing +// sp as hit and removes them. This is used during stack unwinding for +// panic/recover and by heapBitsBulkBarrier to force stack re-scanning +// when its destination is on the stack. +// +// This is nosplit to ensure gp's stack does not move. +// +//go:nosplit +func gcUnwindBarriers(gp *g, sp uintptr) { + // On LR machines, if there is a stack barrier on the return + // from the frame containing sp, this will mark it as hit even + // though it isn't, but it's okay to be conservative. + before := gp.stkbarPos + for int(gp.stkbarPos) < len(gp.stkbar) && gp.stkbar[gp.stkbarPos].savedLRPtr < sp { + gcRemoveStackBarrier(gp, gp.stkbar[gp.stkbarPos]) + gp.stkbarPos++ + } + if debugStackBarrier && gp.stkbarPos != before { + print("skip barriers below ", hex(sp), " in goid=", gp.goid, ": ") + gcPrintStkbars(gp.stkbar[before:gp.stkbarPos]) + print("\n") + } +} + +// nextBarrierPC returns the original return PC of the next stack barrier. +// Used by getcallerpc, so it must be nosplit. +//go:nosplit +func nextBarrierPC() uintptr { + gp := getg() + return gp.stkbar[gp.stkbarPos].savedLRVal +} + +// setNextBarrierPC sets the return PC of the next stack barrier. +// Used by setcallerpc, so it must be nosplit. +//go:nosplit +func setNextBarrierPC(pc uintptr) { + gp := getg() + gp.stkbar[gp.stkbarPos].savedLRVal = pc +}