From 92e68c89f686d451df2d72e6b3f279dbd2b82442 Mon Sep 17 00:00:00 2001 From: Austin Clements Date: Fri, 28 Aug 2015 23:15:41 -0400 Subject: [PATCH] runtime: move stack barrier code to its own file Currently the stack barrier code is mixed in with the mark and scan code. Move all of the stack barrier related functions and variables to a new dedicated source file. There are no code modifications. Change-Id: I604603045465ef8573b9f88915d28ab6b5910903 Reviewed-on: https://go-review.googlesource.com/14050 Reviewed-by: Rick Hudson --- src/runtime/mgc.go | 14 ---- src/runtime/mgcmark.go | 161 ------------------------------------ src/runtime/mstkbar.go | 184 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 184 insertions(+), 175 deletions(-) create mode 100644 src/runtime/mstkbar.go diff --git a/src/runtime/mgc.go b/src/runtime/mgc.go index 82b12b6c09..820e3782a8 100644 --- a/src/runtime/mgc.go +++ b/src/runtime/mgc.go @@ -133,26 +133,12 @@ const ( _RootFlushCaches = 4 _RootCount = 5 - debugStackBarrier = false - // sweepMinHeapDistance is a lower bound on the heap distance // (in bytes) reserved for concurrent sweeping between GC // cycles. This will be scaled by gcpercent/100. sweepMinHeapDistance = 1024 * 1024 ) -// firstStackBarrierOffset is the approximate byte offset at -// which to place the first stack barrier from the current SP. -// This is a lower bound on how much stack will have to be -// re-scanned during mark termination. Subsequent barriers are -// placed at firstStackBarrierOffset * 2^n offsets. -// -// For debugging, this can be set to 0, which will install a -// stack barrier at every frame. If you do this, you may also -// have to raise _StackMin, since the stack barrier -// bookkeeping will use a large amount of each stack. -var firstStackBarrierOffset = 1024 - // heapminimum is the minimum heap size at which to trigger GC. // For small heaps, this overrides the usual GOGC*live set rule. // diff --git a/src/runtime/mgcmark.go b/src/runtime/mgcmark.go index 44f951269b..968b0cdca0 100644 --- a/src/runtime/mgcmark.go +++ b/src/runtime/mgcmark.go @@ -487,167 +487,6 @@ func scanframeworker(frame *stkframe, unused unsafe.Pointer, gcw *gcWork) { } } -// gcMaxStackBarriers returns the maximum number of stack barriers -// that can be installed in a stack of stackSize bytes. -func gcMaxStackBarriers(stackSize int) (n int) { - if firstStackBarrierOffset == 0 { - // Special debugging case for inserting stack barriers - // at every frame. Steal half of the stack for the - // []stkbar. Technically, if the stack were to consist - // solely of return PCs we would need two thirds of - // the stack, but stealing that much breaks things and - // this doesn't happen in practice. - return stackSize / 2 / int(unsafe.Sizeof(stkbar{})) - } - - offset := firstStackBarrierOffset - for offset < stackSize { - n++ - offset *= 2 - } - return n + 1 -} - -// gcInstallStackBarrier installs a stack barrier over the return PC of frame. -//go:nowritebarrier -func gcInstallStackBarrier(gp *g, frame *stkframe) bool { - if frame.lr == 0 { - if debugStackBarrier { - print("not installing stack barrier with no LR, goid=", gp.goid, "\n") - } - return false - } - - if frame.fn.entry == cgocallback_gofuncPC { - // cgocallback_gofunc doesn't return to its LR; - // instead, its return path puts LR in g.sched.pc and - // switches back to the system stack on which - // cgocallback_gofunc was originally called. We can't - // have a stack barrier in g.sched.pc, so don't - // install one in this frame. - if debugStackBarrier { - print("not installing stack barrier over LR of cgocallback_gofunc, goid=", gp.goid, "\n") - } - return false - } - - // Save the return PC and overwrite it with stackBarrier. - var lrUintptr uintptr - if usesLR { - lrUintptr = frame.sp - } else { - lrUintptr = frame.fp - regSize - } - lrPtr := (*uintreg)(unsafe.Pointer(lrUintptr)) - if debugStackBarrier { - print("install stack barrier at ", hex(lrUintptr), " over ", hex(*lrPtr), ", goid=", gp.goid, "\n") - if uintptr(*lrPtr) != frame.lr { - print("frame.lr=", hex(frame.lr)) - throw("frame.lr differs from stack LR") - } - } - - gp.stkbar = gp.stkbar[:len(gp.stkbar)+1] - stkbar := &gp.stkbar[len(gp.stkbar)-1] - stkbar.savedLRPtr = lrUintptr - stkbar.savedLRVal = uintptr(*lrPtr) - *lrPtr = uintreg(stackBarrierPC) - return true -} - -// gcRemoveStackBarriers removes all stack barriers installed in gp's stack. -//go:nowritebarrier -func gcRemoveStackBarriers(gp *g) { - if debugStackBarrier && gp.stkbarPos != 0 { - print("hit ", gp.stkbarPos, " stack barriers, goid=", gp.goid, "\n") - } - - // Remove stack barriers that we didn't hit. - for _, stkbar := range gp.stkbar[gp.stkbarPos:] { - gcRemoveStackBarrier(gp, stkbar) - } - - // Clear recorded stack barriers so copystack doesn't try to - // adjust them. - gp.stkbarPos = 0 - gp.stkbar = gp.stkbar[:0] -} - -// gcRemoveStackBarrier removes a single stack barrier. It is the -// inverse operation of gcInstallStackBarrier. -// -// This is nosplit to ensure gp's stack does not move. -// -//go:nowritebarrier -//go:nosplit -func gcRemoveStackBarrier(gp *g, stkbar stkbar) { - if debugStackBarrier { - print("remove stack barrier at ", hex(stkbar.savedLRPtr), " with ", hex(stkbar.savedLRVal), ", goid=", gp.goid, "\n") - } - lrPtr := (*uintreg)(unsafe.Pointer(stkbar.savedLRPtr)) - if val := *lrPtr; val != uintreg(stackBarrierPC) { - printlock() - print("at *", hex(stkbar.savedLRPtr), " expected stack barrier PC ", hex(stackBarrierPC), ", found ", hex(val), ", goid=", gp.goid, "\n") - print("gp.stkbar=") - gcPrintStkbars(gp.stkbar) - print(", gp.stkbarPos=", gp.stkbarPos, ", gp.stack=[", hex(gp.stack.lo), ",", hex(gp.stack.hi), ")\n") - throw("stack barrier lost") - } - *lrPtr = uintreg(stkbar.savedLRVal) -} - -// gcPrintStkbars prints a []stkbar for debugging. -func gcPrintStkbars(stkbar []stkbar) { - print("[") - for i, s := range stkbar { - if i > 0 { - print(" ") - } - print("*", hex(s.savedLRPtr), "=", hex(s.savedLRVal)) - } - print("]") -} - -// gcUnwindBarriers marks all stack barriers up the frame containing -// sp as hit and removes them. This is used during stack unwinding for -// panic/recover and by heapBitsBulkBarrier to force stack re-scanning -// when its destination is on the stack. -// -// This is nosplit to ensure gp's stack does not move. -// -//go:nosplit -func gcUnwindBarriers(gp *g, sp uintptr) { - // On LR machines, if there is a stack barrier on the return - // from the frame containing sp, this will mark it as hit even - // though it isn't, but it's okay to be conservative. - before := gp.stkbarPos - for int(gp.stkbarPos) < len(gp.stkbar) && gp.stkbar[gp.stkbarPos].savedLRPtr < sp { - gcRemoveStackBarrier(gp, gp.stkbar[gp.stkbarPos]) - gp.stkbarPos++ - } - if debugStackBarrier && gp.stkbarPos != before { - print("skip barriers below ", hex(sp), " in goid=", gp.goid, ": ") - gcPrintStkbars(gp.stkbar[before:gp.stkbarPos]) - print("\n") - } -} - -// nextBarrierPC returns the original return PC of the next stack barrier. -// Used by getcallerpc, so it must be nosplit. -//go:nosplit -func nextBarrierPC() uintptr { - gp := getg() - return gp.stkbar[gp.stkbarPos].savedLRVal -} - -// setNextBarrierPC sets the return PC of the next stack barrier. -// Used by setcallerpc, so it must be nosplit. -//go:nosplit -func setNextBarrierPC(pc uintptr) { - gp := getg() - gp.stkbar[gp.stkbarPos].savedLRVal = pc -} - // TODO(austin): Can we consolidate the gcDrain* functions? // gcDrain scans objects in work buffers, blackening grey diff --git a/src/runtime/mstkbar.go b/src/runtime/mstkbar.go new file mode 100644 index 0000000000..815b85d66e --- /dev/null +++ b/src/runtime/mstkbar.go @@ -0,0 +1,184 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Garbage collector: stack barriers + +package runtime + +import "unsafe" + +const debugStackBarrier = false + +// firstStackBarrierOffset is the approximate byte offset at +// which to place the first stack barrier from the current SP. +// This is a lower bound on how much stack will have to be +// re-scanned during mark termination. Subsequent barriers are +// placed at firstStackBarrierOffset * 2^n offsets. +// +// For debugging, this can be set to 0, which will install a +// stack barrier at every frame. If you do this, you may also +// have to raise _StackMin, since the stack barrier +// bookkeeping will use a large amount of each stack. +var firstStackBarrierOffset = 1024 + +// gcMaxStackBarriers returns the maximum number of stack barriers +// that can be installed in a stack of stackSize bytes. +func gcMaxStackBarriers(stackSize int) (n int) { + if firstStackBarrierOffset == 0 { + // Special debugging case for inserting stack barriers + // at every frame. Steal half of the stack for the + // []stkbar. Technically, if the stack were to consist + // solely of return PCs we would need two thirds of + // the stack, but stealing that much breaks things and + // this doesn't happen in practice. + return stackSize / 2 / int(unsafe.Sizeof(stkbar{})) + } + + offset := firstStackBarrierOffset + for offset < stackSize { + n++ + offset *= 2 + } + return n + 1 +} + +// gcInstallStackBarrier installs a stack barrier over the return PC of frame. +//go:nowritebarrier +func gcInstallStackBarrier(gp *g, frame *stkframe) bool { + if frame.lr == 0 { + if debugStackBarrier { + print("not installing stack barrier with no LR, goid=", gp.goid, "\n") + } + return false + } + + if frame.fn.entry == cgocallback_gofuncPC { + // cgocallback_gofunc doesn't return to its LR; + // instead, its return path puts LR in g.sched.pc and + // switches back to the system stack on which + // cgocallback_gofunc was originally called. We can't + // have a stack barrier in g.sched.pc, so don't + // install one in this frame. + if debugStackBarrier { + print("not installing stack barrier over LR of cgocallback_gofunc, goid=", gp.goid, "\n") + } + return false + } + + // Save the return PC and overwrite it with stackBarrier. + var lrUintptr uintptr + if usesLR { + lrUintptr = frame.sp + } else { + lrUintptr = frame.fp - regSize + } + lrPtr := (*uintreg)(unsafe.Pointer(lrUintptr)) + if debugStackBarrier { + print("install stack barrier at ", hex(lrUintptr), " over ", hex(*lrPtr), ", goid=", gp.goid, "\n") + if uintptr(*lrPtr) != frame.lr { + print("frame.lr=", hex(frame.lr)) + throw("frame.lr differs from stack LR") + } + } + + gp.stkbar = gp.stkbar[:len(gp.stkbar)+1] + stkbar := &gp.stkbar[len(gp.stkbar)-1] + stkbar.savedLRPtr = lrUintptr + stkbar.savedLRVal = uintptr(*lrPtr) + *lrPtr = uintreg(stackBarrierPC) + return true +} + +// gcRemoveStackBarriers removes all stack barriers installed in gp's stack. +//go:nowritebarrier +func gcRemoveStackBarriers(gp *g) { + if debugStackBarrier && gp.stkbarPos != 0 { + print("hit ", gp.stkbarPos, " stack barriers, goid=", gp.goid, "\n") + } + + // Remove stack barriers that we didn't hit. + for _, stkbar := range gp.stkbar[gp.stkbarPos:] { + gcRemoveStackBarrier(gp, stkbar) + } + + // Clear recorded stack barriers so copystack doesn't try to + // adjust them. + gp.stkbarPos = 0 + gp.stkbar = gp.stkbar[:0] +} + +// gcRemoveStackBarrier removes a single stack barrier. It is the +// inverse operation of gcInstallStackBarrier. +// +// This is nosplit to ensure gp's stack does not move. +// +//go:nowritebarrier +//go:nosplit +func gcRemoveStackBarrier(gp *g, stkbar stkbar) { + if debugStackBarrier { + print("remove stack barrier at ", hex(stkbar.savedLRPtr), " with ", hex(stkbar.savedLRVal), ", goid=", gp.goid, "\n") + } + lrPtr := (*uintreg)(unsafe.Pointer(stkbar.savedLRPtr)) + if val := *lrPtr; val != uintreg(stackBarrierPC) { + printlock() + print("at *", hex(stkbar.savedLRPtr), " expected stack barrier PC ", hex(stackBarrierPC), ", found ", hex(val), ", goid=", gp.goid, "\n") + print("gp.stkbar=") + gcPrintStkbars(gp.stkbar) + print(", gp.stkbarPos=", gp.stkbarPos, ", gp.stack=[", hex(gp.stack.lo), ",", hex(gp.stack.hi), ")\n") + throw("stack barrier lost") + } + *lrPtr = uintreg(stkbar.savedLRVal) +} + +// gcPrintStkbars prints a []stkbar for debugging. +func gcPrintStkbars(stkbar []stkbar) { + print("[") + for i, s := range stkbar { + if i > 0 { + print(" ") + } + print("*", hex(s.savedLRPtr), "=", hex(s.savedLRVal)) + } + print("]") +} + +// gcUnwindBarriers marks all stack barriers up the frame containing +// sp as hit and removes them. This is used during stack unwinding for +// panic/recover and by heapBitsBulkBarrier to force stack re-scanning +// when its destination is on the stack. +// +// This is nosplit to ensure gp's stack does not move. +// +//go:nosplit +func gcUnwindBarriers(gp *g, sp uintptr) { + // On LR machines, if there is a stack barrier on the return + // from the frame containing sp, this will mark it as hit even + // though it isn't, but it's okay to be conservative. + before := gp.stkbarPos + for int(gp.stkbarPos) < len(gp.stkbar) && gp.stkbar[gp.stkbarPos].savedLRPtr < sp { + gcRemoveStackBarrier(gp, gp.stkbar[gp.stkbarPos]) + gp.stkbarPos++ + } + if debugStackBarrier && gp.stkbarPos != before { + print("skip barriers below ", hex(sp), " in goid=", gp.goid, ": ") + gcPrintStkbars(gp.stkbar[before:gp.stkbarPos]) + print("\n") + } +} + +// nextBarrierPC returns the original return PC of the next stack barrier. +// Used by getcallerpc, so it must be nosplit. +//go:nosplit +func nextBarrierPC() uintptr { + gp := getg() + return gp.stkbar[gp.stkbarPos].savedLRVal +} + +// setNextBarrierPC sets the return PC of the next stack barrier. +// Used by setcallerpc, so it must be nosplit. +//go:nosplit +func setNextBarrierPC(pc uintptr) { + gp := getg() + gp.stkbar[gp.stkbarPos].savedLRVal = pc +}