runtime: enable profiling on g0

Since we now have stack information for code running on the
systemstack, we can traceback over it. To make cpu profiles useful,
add a case in gentraceback to jump over systemstack switches.

Fixes #10609.

Change-Id: I21f47fcc802c07c5d4a1ada56374314e388a6dc7
Reviewed-on: https://go-review.googlesource.com/9506
Reviewed-by: Dmitry Vyukov <dvyukov@google.com>

src/runtime/arch1_386.go

@@ -5,12 +5,11 @@
 package runtime
 
 const (
-	thechar           = '8'
-	_BigEndian        = 0
-	_CacheLineSize    = 64
-	_RuntimeGogoBytes = 64
-	_PhysPageSize     = goos_nacl*65536 + (1-goos_nacl)*4096 // 4k normally; 64k on NaCl
-	_PCQuantum        = 1
-	_Int64Align       = 4
-	hugePageSize      = 1 << 21
+	thechar        = '8'
+	_BigEndian     = 0
+	_CacheLineSize = 64
+	_PhysPageSize  = goos_nacl*65536 + (1-goos_nacl)*4096 // 4k normally; 64k on NaCl
+	_PCQuantum     = 1
+	_Int64Align    = 4
+	hugePageSize   = 1 << 21
 )

src/runtime/arch1_amd64.go

@@ -5,12 +5,11 @@
 package runtime
 
 const (
-	thechar           = '6'
-	_BigEndian        = 0
-	_CacheLineSize    = 64
-	_RuntimeGogoBytes = 80 + (goos_solaris)*16
-	_PhysPageSize     = 4096
-	_PCQuantum        = 1
-	_Int64Align       = 8
-	hugePageSize      = 1 << 21
+	thechar        = '6'
+	_BigEndian     = 0
+	_CacheLineSize = 64
+	_PhysPageSize  = 4096
+	_PCQuantum     = 1
+	_Int64Align    = 8
+	hugePageSize   = 1 << 21
 )

src/runtime/arch1_amd64p32.go

@@ -5,12 +5,11 @@
 package runtime
 
 const (
-	thechar           = '6'
-	_BigEndian        = 0
-	_CacheLineSize    = 64
-	_RuntimeGogoBytes = 64
-	_PhysPageSize     = 65536*goos_nacl + 4096*(1-goos_nacl)
-	_PCQuantum        = 1
-	_Int64Align       = 8
-	hugePageSize      = 1 << 21
+	thechar        = '6'
+	_BigEndian     = 0
+	_CacheLineSize = 64
+	_PhysPageSize  = 65536*goos_nacl + 4096*(1-goos_nacl)
+	_PCQuantum     = 1
+	_Int64Align    = 8
+	hugePageSize   = 1 << 21
 )

src/runtime/arch1_arm.go

@@ -5,12 +5,11 @@
 package runtime
 
 const (
-	thechar           = '5'
-	_BigEndian        = 0
-	_CacheLineSize    = 32
-	_RuntimeGogoBytes = 60
-	_PhysPageSize     = 65536*goos_nacl + 4096*(1-goos_nacl)
-	_PCQuantum        = 4
-	_Int64Align       = 4
-	hugePageSize      = 0
+	thechar        = '5'
+	_BigEndian     = 0
+	_CacheLineSize = 32
+	_PhysPageSize  = 65536*goos_nacl + 4096*(1-goos_nacl)
+	_PCQuantum     = 4
+	_Int64Align    = 4
+	hugePageSize   = 0
 )

src/runtime/arch1_arm64.go

@@ -5,12 +5,11 @@
 package runtime
 
 const (
-	thechar           = '7'
-	_BigEndian        = 0
-	_CacheLineSize    = 32
-	_RuntimeGogoBytes = 64
-	_PhysPageSize     = 4096*(1-goos_darwin) + 16384*goos_darwin
-	_PCQuantum        = 4
-	_Int64Align       = 8
-	hugePageSize      = 0
+	thechar        = '7'
+	_BigEndian     = 0
+	_CacheLineSize = 32
+	_PhysPageSize  = 4096*(1-goos_darwin) + 16384*goos_darwin
+	_PCQuantum     = 4
+	_Int64Align    = 8
+	hugePageSize   = 0
 )

src/runtime/arch1_ppc64.go

@@ -5,12 +5,11 @@
 package runtime
 
 const (
-	thechar           = '9'
-	_BigEndian        = 1
-	_CacheLineSize    = 64
-	_RuntimeGogoBytes = 72
-	_PhysPageSize     = 65536
-	_PCQuantum        = 4
-	_Int64Align       = 8
-	hugePageSize      = 0
+	thechar        = '9'
+	_BigEndian     = 1
+	_CacheLineSize = 64
+	_PhysPageSize  = 65536
+	_PCQuantum     = 4
+	_Int64Align    = 8
+	hugePageSize   = 0
 )

src/runtime/arch1_ppc64le.go

@@ -5,12 +5,11 @@
 package runtime
 
 const (
-	thechar           = '9'
-	_BigEndian        = 0
-	_CacheLineSize    = 64
-	_RuntimeGogoBytes = 72
-	_PhysPageSize     = 65536
-	_PCQuantum        = 4
-	_Int64Align       = 8
-	hugePageSize      = 0
+	thechar        = '9'
+	_BigEndian     = 0
+	_CacheLineSize = 64
+	_PhysPageSize  = 65536
+	_PCQuantum     = 4
+	_Int64Align    = 8
+	hugePageSize   = 0
 )

src/runtime/export_test.go

@@ -106,11 +106,6 @@ var MemclrBytes = memclrBytes
 
 var HashLoad = &hashLoad
 
-// For testing.
-func GogoBytes() int32 {
-	return _RuntimeGogoBytes
-}
-
 // entry point for testing
 func GostringW(w []uint16) (s string) {
 	systemstack(func() {

src/runtime/proc1.go

@@ -2484,11 +2484,9 @@ func sigprof(pc, sp, lr uintptr, gp *g, mp *m) {
 	mp.mallocing++
 
 	// Define that a "user g" is a user-created goroutine, and a "system g"
-	// is one that is m->g0 or m->gsignal. We've only made sure that we
-	// can unwind user g's, so exclude the system g's.
+	// is one that is m->g0 or m->gsignal.
 	//
-	// It is not quite as easy as testing gp == m->curg (the current user g)
-	// because we might be interrupted for profiling halfway through a
+	// We might be interrupted for profiling halfway through a
 	// goroutine switch. The switch involves updating three (or four) values:
 	// g, PC, SP, and (on arm) LR. The PC must be the last to be updated,
 	// because once it gets updated the new g is running.
@@ -2497,8 +2495,7 @@ func sigprof(pc, sp, lr uintptr, gp *g, mp *m) {
 	// so the update only affects g, SP, and PC. Since PC must be last, there
 	// the possible partial transitions in ordinary execution are (1) g alone is updated,
 	// (2) both g and SP are updated, and (3) SP alone is updated.
-	// If g is updated, we'll see a system g and not look closer.
-	// If SP alone is updated, we can detect the partial transition by checking
+	// If SP or g alone is updated, we can detect the partial transition by checking
 	// whether the SP is within g's stack bounds. (We could also require that SP
 	// be changed only after g, but the stack bounds check is needed by other
 	// cases, so there is no need to impose an additional requirement.)
@@ -2527,15 +2524,11 @@ func sigprof(pc, sp, lr uintptr, gp *g, mp *m) {
 	// disabled, so a profiling signal cannot arrive then anyway.
 	//
 	// Third, the common case: it may be that the switch updates g, SP, and PC
-	// separately, as in gogo.
-	//
-	// Because gogo is the only instance, we check whether the PC lies
-	// within that function, and if so, not ask for a traceback. This approach
-	// requires knowing the size of the gogo function, which we
-	// record in arch_*.h and check in runtime_test.go.
+	// separately. If the PC is within any of the functions that does this,
+	// we don't ask for a traceback. C.F. the function setsSP for more about this.
 	//
 	// There is another apparently viable approach, recorded here in case
-	// the "PC within gogo" check turns out not to be usable.
+	// the "PC within setsSP function" check turns out not to be usable.
 	// It would be possible to delay the update of either g or SP until immediately
 	// before the PC update instruction. Then, because of the stack bounds check,
 	// the only problematic interrupt point is just before that PC update instruction,
@@ -2556,28 +2549,23 @@ func sigprof(pc, sp, lr uintptr, gp *g, mp *m) {
 	// transition. We simply require that g and SP match and that the PC is not
 	// in gogo.
 	traceback := true
-	gogo := funcPC(gogo)
-	if gp == nil || gp != mp.curg ||
-		sp < gp.stack.lo || gp.stack.hi < sp ||
-		(gogo <= pc && pc < gogo+_RuntimeGogoBytes) {
+	if gp == nil || sp < gp.stack.lo || gp.stack.hi < sp || setsSP(pc) {
 		traceback = false
 	}
-
 	var stk [maxCPUProfStack]uintptr
 	n := 0
-	if traceback {
-		n = gentraceback(pc, sp, lr, gp, 0, &stk[0], len(stk), nil, nil, _TraceTrap)
+	if mp.ncgo > 0 && mp.curg != nil && mp.curg.syscallpc != 0 && mp.curg.syscallsp != 0 {
+		// Cgo, we can't unwind and symbolize arbitrary C code,
+		// so instead collect Go stack that leads to the cgo call.
+		// This is especially important on windows, since all syscalls are cgo calls.
+		n = gentraceback(mp.curg.syscallpc, mp.curg.syscallsp, 0, mp.curg, 0, &stk[0], len(stk), nil, nil, 0)
+	} else if traceback {
+		n = gentraceback(pc, sp, lr, gp, 0, &stk[0], len(stk), nil, nil, _TraceTrap|_TraceJumpStack)
 	}
 	if !traceback || n <= 0 {
 		// Normal traceback is impossible or has failed.
 		// See if it falls into several common cases.
 		n = 0
-		if mp.ncgo > 0 && mp.curg != nil && mp.curg.syscallpc != 0 && mp.curg.syscallsp != 0 {
-			// Cgo, we can't unwind and symbolize arbitrary C code,
-			// so instead collect Go stack that leads to the cgo call.
-			// This is especially important on windows, since all syscalls are cgo calls.
-			n = gentraceback(mp.curg.syscallpc, mp.curg.syscallsp, 0, mp.curg, 0, &stk[0], len(stk), nil, nil, 0)
-		}
 		if GOOS == "windows" && n == 0 && mp.libcallg != 0 && mp.libcallpc != 0 && mp.libcallsp != 0 {
 			// Libcall, i.e. runtime syscall on windows.
 			// Collect Go stack that leads to the call.
@@ -2612,6 +2600,30 @@ func sigprof(pc, sp, lr uintptr, gp *g, mp *m) {
 	mp.mallocing--
 }
 
+// Reports whether a function will set the SP
+// to an absolute value. Important that
+// we don't traceback when these are at the bottom
+// of the stack since we can't be sure that we will
+// find the caller.
+//
+// If the function is not on the bottom of the stack
+// we assume that it will have set it up so that traceback will be consistent,
+// either by being a traceback terminating function
+// or putting one on the stack at the right offset.
+func setsSP(pc uintptr) bool {
+	f := findfunc(pc)
+	if f == nil {
+		// couldn't find the function for this PC,
+		// so assume the worst and stop traceback
+		return true
+	}
+	switch f.entry {
+	case gogoPC, systemstackPC, mcallPC, morestackPC:
+		return true
+	}
+	return false
+}
+
 // Arrange to call fn with a traceback hz times a second.
 func setcpuprofilerate_m(hz int32) {
 	// Force sane arguments.

src/runtime/runtime2.go

@@ -594,8 +594,9 @@ type stkframe struct {
 }
 
 const (
-	_TraceRuntimeFrames = 1 << 0 // include frames for internal runtime functions.
-	_TraceTrap          = 1 << 1 // the initial PC, SP are from a trap, not a return PC from a call
+	_TraceRuntimeFrames = 1 << iota // include frames for internal runtime functions.
+	_TraceTrap                      // the initial PC, SP are from a trap, not a return PC from a call
+	_TraceJumpStack                 // if traceback is on a systemstack, resume trace at g that called into it
 )
 
 const (

src/runtime/runtime_test.go

@@ -6,13 +6,8 @@ package runtime_test
 
 import (
 	"io"
-	"io/ioutil"
-	"os"
-	"os/exec"
 	. "runtime"
 	"runtime/debug"
-	"strconv"
-	"strings"
 	"testing"
 	"unsafe"
 )
@@ -88,53 +83,6 @@ func BenchmarkDeferMany(b *testing.B) {
 	}
 }
 
-// The profiling signal handler needs to know whether it is executing runtime.gogo.
-// The constant RuntimeGogoBytes in arch_*.h gives the size of the function;
-// we don't have a way to obtain it from the linker (perhaps someday).
-// Test that the constant matches the size determined by 'go tool nm -S'.
-// The value reported will include the padding between runtime.gogo and the
-// next function in memory. That's fine.
-func TestRuntimeGogoBytes(t *testing.T) {
-	switch GOOS {
-	case "android", "nacl":
-		t.Skipf("skipping on %s", GOOS)
-	case "darwin":
-		switch GOARCH {
-		case "arm", "arm64":
-			t.Skipf("skipping on %s/%s, no fork", GOOS, GOARCH)
-		}
-	}
-
-	dir, err := ioutil.TempDir("", "go-build")
-	if err != nil {
-		t.Fatalf("failed to create temp directory: %v", err)
-	}
-	defer os.RemoveAll(dir)
-
-	out, err := exec.Command("go", "build", "-o", dir+"/hello", "../../test/helloworld.go").CombinedOutput()
-	if err != nil {
-		t.Fatalf("building hello world: %v\n%s", err, out)
-	}
-
-	out, err = exec.Command("go", "tool", "nm", "-size", dir+"/hello").CombinedOutput()
-	if err != nil {
-		t.Fatalf("go tool nm: %v\n%s", err, out)
-	}
-
-	for _, line := range strings.Split(string(out), "\n") {
-		f := strings.Fields(line)
-		if len(f) == 4 && f[3] == "runtime.gogo" {
-			size, _ := strconv.Atoi(f[1])
-			if GogoBytes() != int32(size) {
-				t.Fatalf("RuntimeGogoBytes = %d, should be %d", GogoBytes(), size)
-			}
-			return
-		}
-	}
-
-	t.Fatalf("go tool nm did not report size for runtime.gogo")
-}
-
 // golang.org/issue/7063
 func TestStopCPUProfilingWithProfilerOff(t *testing.T) {
 	SetCPUProfileRate(0)

src/runtime/traceback.go

@@ -46,6 +46,9 @@ var (
 	timerprocPC          uintptr
 	gcBgMarkWorkerPC     uintptr
 	systemstack_switchPC uintptr
+	systemstackPC        uintptr
+
+	gogoPC uintptr
 
 	externalthreadhandlerp uintptr // initialized elsewhere
 )
@@ -69,6 +72,10 @@ func tracebackinit() {
 	timerprocPC = funcPC(timerproc)
 	gcBgMarkWorkerPC = funcPC(gcBgMarkWorker)
 	systemstack_switchPC = funcPC(systemstack_switch)
+	systemstackPC = funcPC(systemstack)
+
+	// used by sigprof handler
+	gogoPC = funcPC(gogo)
 }
 
 // Traceback over the deferred function calls.
@@ -194,7 +201,14 @@ func gentraceback(pc0, sp0, lr0 uintptr, gp *g, skip int, pcbuf *uintptr, max in
 		// Found an actual function.
 		// Derive frame pointer and link register.
 		if frame.fp == 0 {
-			frame.fp = frame.sp + uintptr(funcspdelta(f, frame.pc))
+			// We want to jump over the systemstack switch. If we're running on the
+			// g0, this systemstack is at the top of the stack.
+			// if we're not on g0 or there's a no curg, then this is a regular call.
+			sp := frame.sp
+			if flags&_TraceJumpStack != 0 && f.entry == systemstackPC && gp == g.m.g0 && gp.m.curg != nil {
+				sp = gp.m.curg.sched.sp
+			}
+			frame.fp = sp + uintptr(funcspdelta(f, frame.pc))
 			if !usesLR {
 				// On x86, call instruction pushes return PC before entering new function.
 				frame.fp += regSize