mirror of
https://github.com/golang/go
synced 2024-11-23 19:20:03 -07:00
runtime/pprof: add streaming protobuf encoder
The existing code builds a full profile in memory. Then it translates that profile into a data structure (in memory). Then it marshals that data structure into a protocol buffer (in memory). Then it gzips that marshaled form into the underlying writer. So there are three copies of the full profile data in memory at the same time before we're done. This is obviously dumb. This CL implements a fully streaming conversion from the original in-memory profile to the underlying writer. There is now only one copy of the profile in memory. For the non-CPU profiles, this is optimal, since we have to have a full copy in memory to start with. For the CPU profiles, we could still try to bound the profile size stored in memory and stream fragments out during the actual profiling, as Go 1.7 did (with a simpler format), but so far that hasn't been necessary. Change-Id: Ic36141021857791bf0cd1fce84178fb5e744b989 Reviewed-on: https://go-review.googlesource.com/37164 Run-TryBot: Russ Cox <rsc@golang.org> Reviewed-by: Michael Matloob <matloob@golang.org>
This commit is contained in:
Russ Cox
parent
322fff8ac8
commit
cbab65fdfa
@ -172,13 +172,12 @@ var pkgDeps = map[string][]string{
|
||||
"log": {"L1", "os", "fmt", "time"},
|
||||
|
||||
// Packages used by testing must be low-level (L2+fmt).
|
||||
"regexp": {"L2", "regexp/syntax"},
|
||||
"regexp/syntax": {"L2"},
|
||||
"runtime/debug": {"L2", "fmt", "io/ioutil", "os", "time"},
|
||||
"runtime/pprof/internal/protopprof": {"L2", "fmt", "internal/pprof/profile", "os", "time"},
|
||||
"runtime/pprof": {"L2", "context", "fmt", "internal/pprof/profile", "os", "runtime/pprof/internal/protopprof", "text/tabwriter", "time"},
|
||||
"runtime/trace": {"L0"},
|
||||
"text/tabwriter": {"L2"},
|
||||
"regexp": {"L2", "regexp/syntax"},
|
||||
"regexp/syntax": {"L2"},
|
||||
"runtime/debug": {"L2", "fmt", "io/ioutil", "os", "time"},
|
||||
"runtime/pprof": {"L2", "compress/gzip", "context", "fmt", "io/ioutil", "os", "text/tabwriter", "time"},
|
||||
"runtime/trace": {"L0"},
|
||||
"text/tabwriter": {"L2"},
|
||||
|
||||
"testing": {"L2", "flag", "fmt", "internal/race", "os", "runtime/debug", "runtime/pprof", "runtime/trace", "time"},
|
||||
"testing/iotest": {"L2", "log"},
|
||||
|
||||
@ -2,7 +2,7 @@
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package pprof_test
|
||||
package pprof
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
@ -10,7 +10,6 @@ import (
|
||||
"reflect"
|
||||
"regexp"
|
||||
"runtime"
|
||||
. "runtime/pprof"
|
||||
"testing"
|
||||
"unsafe"
|
||||
)
|
||||
@ -86,22 +85,22 @@ func TestMemoryProfiler(t *testing.T) {
|
||||
|
||||
tests := []string{
|
||||
fmt.Sprintf(`%v: %v \[%v: %v\] @ 0x[0-9,a-f]+ 0x[0-9,a-f]+ 0x[0-9,a-f]+ 0x[0-9,a-f]+
|
||||
# 0x[0-9,a-f]+ runtime/pprof_test\.allocatePersistent1K\+0x[0-9,a-f]+ .*/runtime/pprof/mprof_test\.go:41
|
||||
# 0x[0-9,a-f]+ runtime/pprof_test\.TestMemoryProfiler\+0x[0-9,a-f]+ .*/runtime/pprof/mprof_test\.go:75
|
||||
# 0x[0-9,a-f]+ runtime/pprof\.allocatePersistent1K\+0x[0-9,a-f]+ .*/runtime/pprof/mprof_test\.go:40
|
||||
# 0x[0-9,a-f]+ runtime/pprof\.TestMemoryProfiler\+0x[0-9,a-f]+ .*/runtime/pprof/mprof_test\.go:74
|
||||
`, 32*memoryProfilerRun, 1024*memoryProfilerRun, 32*memoryProfilerRun, 1024*memoryProfilerRun),
|
||||
|
||||
fmt.Sprintf(`0: 0 \[%v: %v\] @ 0x[0-9,a-f]+ 0x[0-9,a-f]+ 0x[0-9,a-f]+ 0x[0-9,a-f]+
|
||||
# 0x[0-9,a-f]+ runtime/pprof_test\.allocateTransient1M\+0x[0-9,a-f]+ .*/runtime/pprof/mprof_test.go:22
|
||||
# 0x[0-9,a-f]+ runtime/pprof_test\.TestMemoryProfiler\+0x[0-9,a-f]+ .*/runtime/pprof/mprof_test.go:73
|
||||
# 0x[0-9,a-f]+ runtime/pprof\.allocateTransient1M\+0x[0-9,a-f]+ .*/runtime/pprof/mprof_test.go:21
|
||||
# 0x[0-9,a-f]+ runtime/pprof\.TestMemoryProfiler\+0x[0-9,a-f]+ .*/runtime/pprof/mprof_test.go:72
|
||||
`, (1<<10)*memoryProfilerRun, (1<<20)*memoryProfilerRun),
|
||||
|
||||
fmt.Sprintf(`0: 0 \[%v: %v\] @ 0x[0-9,a-f]+ 0x[0-9,a-f]+ 0x[0-9,a-f]+ 0x[0-9,a-f]+
|
||||
# 0x[0-9,a-f]+ runtime/pprof_test\.allocateTransient2M\+0x[0-9,a-f]+ .*/runtime/pprof/mprof_test.go:28
|
||||
# 0x[0-9,a-f]+ runtime/pprof_test\.TestMemoryProfiler\+0x[0-9,a-f]+ .*/runtime/pprof/mprof_test.go:74
|
||||
# 0x[0-9,a-f]+ runtime/pprof\.allocateTransient2M\+0x[0-9,a-f]+ .*/runtime/pprof/mprof_test.go:27
|
||||
# 0x[0-9,a-f]+ runtime/pprof\.TestMemoryProfiler\+0x[0-9,a-f]+ .*/runtime/pprof/mprof_test.go:73
|
||||
`, memoryProfilerRun, (2<<20)*memoryProfilerRun),
|
||||
|
||||
fmt.Sprintf(`0: 0 \[%v: %v\] @( 0x[0-9,a-f]+)+
|
||||
# 0x[0-9,a-f]+ runtime/pprof_test\.allocateReflectTransient\+0x[0-9,a-f]+ .*/runtime/pprof/mprof_test.go:49
|
||||
# 0x[0-9,a-f]+ runtime/pprof\.allocateReflectTransient\+0x[0-9,a-f]+ .*/runtime/pprof/mprof_test.go:48
|
||||
`, memoryProfilerRun, (2<<20)*memoryProfilerRun),
|
||||
}
|
||||
|
||||
|
||||
@ -75,7 +75,6 @@ import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"fmt"
|
||||
"internal/pprof/profile"
|
||||
"io"
|
||||
"runtime"
|
||||
"sort"
|
||||
@ -384,35 +383,26 @@ func printCountProfile(w io.Writer, debug int, name string, p countProfile) erro
|
||||
}
|
||||
|
||||
// Output profile in protobuf form.
|
||||
prof := &profile.Profile{
|
||||
PeriodType: &profile.ValueType{Type: name, Unit: "count"},
|
||||
Period: 1,
|
||||
Sample: make([]*profile.Sample, 0, len(keys)),
|
||||
SampleType: []*profile.ValueType{{Type: name, Unit: "count"}},
|
||||
}
|
||||
locMap := make(map[uintptr]*profile.Location)
|
||||
b := newProfileBuilder(w)
|
||||
b.pbValueType(tagProfile_PeriodType, name, "count")
|
||||
b.pb.int64Opt(tagProfile_Period, 1)
|
||||
b.pbValueType(tagProfile_SampleType, name, "count")
|
||||
|
||||
values := []int64{0}
|
||||
var locs []uint64
|
||||
for _, k := range keys {
|
||||
stk := p.Stack(index[k])
|
||||
c := count[k]
|
||||
locs := make([]*profile.Location, len(stk))
|
||||
for i, addr := range stk {
|
||||
loc := locMap[addr]
|
||||
if loc == nil {
|
||||
loc = &profile.Location{
|
||||
ID: uint64(len(locMap) + 1),
|
||||
Address: uint64(addr - 1),
|
||||
}
|
||||
prof.Location = append(prof.Location, loc)
|
||||
locMap[addr] = loc
|
||||
values[0] = int64(count[k])
|
||||
locs = locs[:0]
|
||||
for i, addr := range p.Stack(index[k]) {
|
||||
if false && i > 0 { // TODO: why disabled?
|
||||
addr--
|
||||
}
|
||||
locs[i] = loc
|
||||
locs = append(locs, b.locForPC(addr))
|
||||
}
|
||||
prof.Sample = append(prof.Sample, &profile.Sample{
|
||||
Location: locs,
|
||||
Value: []int64{int64(c)},
|
||||
})
|
||||
b.pbSample(values, locs, nil)
|
||||
}
|
||||
return prof.Write(w)
|
||||
b.build()
|
||||
return nil
|
||||
}
|
||||
|
||||
// keysByCount sorts keys with higher counts first, breaking ties by key string order.
|
||||
@ -500,8 +490,7 @@ func writeHeap(w io.Writer, debug int) error {
|
||||
}
|
||||
|
||||
if debug == 0 {
|
||||
pp := encodeMemProfile(p, int64(runtime.MemProfileRate), time.Now())
|
||||
return pp.Write(w)
|
||||
return writeHeapProto(w, p, int64(runtime.MemProfileRate))
|
||||
}
|
||||
|
||||
sort.Slice(p, func(i, j int) bool { return p[i].InUseBytes() > p[j].InUseBytes() })
|
||||
@ -705,7 +694,7 @@ func StartCPUProfile(w io.Writer) error {
|
||||
func readProfile() (data []uint64, tags []unsafe.Pointer, eof bool)
|
||||
|
||||
func profileWriter(w io.Writer) {
|
||||
b := newProfileBuilder()
|
||||
b := newProfileBuilder(w)
|
||||
var err error
|
||||
for {
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
@ -717,13 +706,12 @@ func profileWriter(w io.Writer) {
|
||||
break
|
||||
}
|
||||
}
|
||||
p := b.build()
|
||||
if err != nil {
|
||||
// The runtime should never produce an invalid or truncated profile.
|
||||
// It drops records that can't fit into its log buffers.
|
||||
panic("runtime/pprof: converting profile: " + err.Error())
|
||||
}
|
||||
p.Write(w)
|
||||
b.build()
|
||||
cpu.done <- true
|
||||
}
|
||||
|
||||
|
||||
@ -4,7 +4,7 @@
|
||||
|
||||
// +build !nacl
|
||||
|
||||
package pprof_test
|
||||
package pprof
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
@ -16,7 +16,6 @@ import (
|
||||
"os/exec"
|
||||
"regexp"
|
||||
"runtime"
|
||||
. "runtime/pprof"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
@ -68,14 +67,14 @@ func cpuHog2() {
|
||||
}
|
||||
|
||||
func TestCPUProfile(t *testing.T) {
|
||||
testCPUProfile(t, []string{"runtime/pprof_test.cpuHog1"}, func(dur time.Duration) {
|
||||
testCPUProfile(t, []string{"runtime/pprof.cpuHog1"}, func(dur time.Duration) {
|
||||
cpuHogger(cpuHog1, dur)
|
||||
})
|
||||
}
|
||||
|
||||
func TestCPUProfileMultithreaded(t *testing.T) {
|
||||
defer runtime.GOMAXPROCS(runtime.GOMAXPROCS(2))
|
||||
testCPUProfile(t, []string{"runtime/pprof_test.cpuHog1", "runtime/pprof_test.cpuHog2"}, func(dur time.Duration) {
|
||||
testCPUProfile(t, []string{"runtime/pprof.cpuHog1", "runtime/pprof.cpuHog2"}, func(dur time.Duration) {
|
||||
c := make(chan int)
|
||||
go func() {
|
||||
cpuHogger(cpuHog1, dur)
|
||||
@ -171,21 +170,26 @@ func profileOk(t *testing.T, need []string, prof bytes.Buffer, duration time.Dur
|
||||
// Check that profile is well formed and contains need.
|
||||
have := make([]uintptr, len(need))
|
||||
var samples uintptr
|
||||
var buf bytes.Buffer
|
||||
parseProfile(t, prof.Bytes(), func(count uintptr, stk []uintptr) {
|
||||
fmt.Fprintf(&buf, "%d:", count)
|
||||
samples += count
|
||||
for _, pc := range stk {
|
||||
fmt.Fprintf(&buf, " %#x", pc)
|
||||
f := runtime.FuncForPC(pc)
|
||||
if f == nil {
|
||||
continue
|
||||
}
|
||||
fmt.Fprintf(&buf, "(%s)", f.Name())
|
||||
for i, name := range need {
|
||||
if strings.Contains(f.Name(), name) {
|
||||
have[i] += count
|
||||
}
|
||||
}
|
||||
}
|
||||
fmt.Fprintf(&buf, "\n")
|
||||
})
|
||||
t.Logf("total %d CPU profile samples collected", samples)
|
||||
t.Logf("total %d CPU profile samples collected:\n%s", samples, buf.String())
|
||||
|
||||
if samples < 10 && runtime.GOOS == "windows" {
|
||||
// On some windows machines we end up with
|
||||
@ -361,44 +365,44 @@ func TestBlockProfile(t *testing.T) {
|
||||
{"chan recv", blockChanRecv, `
|
||||
[0-9]+ [0-9]+ @ 0x[0-9,a-f]+ 0x[0-9,a-f]+ 0x[0-9,a-f]+ 0x[0-9,a-f]+ 0x[0-9,a-f]+
|
||||
# 0x[0-9,a-f]+ runtime\.chanrecv1\+0x[0-9,a-f]+ .*/src/runtime/chan.go:[0-9]+
|
||||
# 0x[0-9,a-f]+ runtime/pprof_test\.blockChanRecv\+0x[0-9,a-f]+ .*/src/runtime/pprof/pprof_test.go:[0-9]+
|
||||
# 0x[0-9,a-f]+ runtime/pprof_test\.TestBlockProfile\+0x[0-9,a-f]+ .*/src/runtime/pprof/pprof_test.go:[0-9]+
|
||||
# 0x[0-9,a-f]+ runtime/pprof\.blockChanRecv\+0x[0-9,a-f]+ .*/src/runtime/pprof/pprof_test.go:[0-9]+
|
||||
# 0x[0-9,a-f]+ runtime/pprof\.TestBlockProfile\+0x[0-9,a-f]+ .*/src/runtime/pprof/pprof_test.go:[0-9]+
|
||||
`},
|
||||
{"chan send", blockChanSend, `
|
||||
[0-9]+ [0-9]+ @ 0x[0-9,a-f]+ 0x[0-9,a-f]+ 0x[0-9,a-f]+ 0x[0-9,a-f]+ 0x[0-9,a-f]+
|
||||
# 0x[0-9,a-f]+ runtime\.chansend1\+0x[0-9,a-f]+ .*/src/runtime/chan.go:[0-9]+
|
||||
# 0x[0-9,a-f]+ runtime/pprof_test\.blockChanSend\+0x[0-9,a-f]+ .*/src/runtime/pprof/pprof_test.go:[0-9]+
|
||||
# 0x[0-9,a-f]+ runtime/pprof_test\.TestBlockProfile\+0x[0-9,a-f]+ .*/src/runtime/pprof/pprof_test.go:[0-9]+
|
||||
# 0x[0-9,a-f]+ runtime/pprof\.blockChanSend\+0x[0-9,a-f]+ .*/src/runtime/pprof/pprof_test.go:[0-9]+
|
||||
# 0x[0-9,a-f]+ runtime/pprof\.TestBlockProfile\+0x[0-9,a-f]+ .*/src/runtime/pprof/pprof_test.go:[0-9]+
|
||||
`},
|
||||
{"chan close", blockChanClose, `
|
||||
[0-9]+ [0-9]+ @ 0x[0-9,a-f]+ 0x[0-9,a-f]+ 0x[0-9,a-f]+ 0x[0-9,a-f]+ 0x[0-9,a-f]+
|
||||
# 0x[0-9,a-f]+ runtime\.chanrecv1\+0x[0-9,a-f]+ .*/src/runtime/chan.go:[0-9]+
|
||||
# 0x[0-9,a-f]+ runtime/pprof_test\.blockChanClose\+0x[0-9,a-f]+ .*/src/runtime/pprof/pprof_test.go:[0-9]+
|
||||
# 0x[0-9,a-f]+ runtime/pprof_test\.TestBlockProfile\+0x[0-9,a-f]+ .*/src/runtime/pprof/pprof_test.go:[0-9]+
|
||||
# 0x[0-9,a-f]+ runtime/pprof\.blockChanClose\+0x[0-9,a-f]+ .*/src/runtime/pprof/pprof_test.go:[0-9]+
|
||||
# 0x[0-9,a-f]+ runtime/pprof\.TestBlockProfile\+0x[0-9,a-f]+ .*/src/runtime/pprof/pprof_test.go:[0-9]+
|
||||
`},
|
||||
{"select recv async", blockSelectRecvAsync, `
|
||||
[0-9]+ [0-9]+ @ 0x[0-9,a-f]+ 0x[0-9,a-f]+ 0x[0-9,a-f]+ 0x[0-9,a-f]+ 0x[0-9,a-f]+
|
||||
# 0x[0-9,a-f]+ runtime\.selectgo\+0x[0-9,a-f]+ .*/src/runtime/select.go:[0-9]+
|
||||
# 0x[0-9,a-f]+ runtime/pprof_test\.blockSelectRecvAsync\+0x[0-9,a-f]+ .*/src/runtime/pprof/pprof_test.go:[0-9]+
|
||||
# 0x[0-9,a-f]+ runtime/pprof_test\.TestBlockProfile\+0x[0-9,a-f]+ .*/src/runtime/pprof/pprof_test.go:[0-9]+
|
||||
# 0x[0-9,a-f]+ runtime/pprof\.blockSelectRecvAsync\+0x[0-9,a-f]+ .*/src/runtime/pprof/pprof_test.go:[0-9]+
|
||||
# 0x[0-9,a-f]+ runtime/pprof\.TestBlockProfile\+0x[0-9,a-f]+ .*/src/runtime/pprof/pprof_test.go:[0-9]+
|
||||
`},
|
||||
{"select send sync", blockSelectSendSync, `
|
||||
[0-9]+ [0-9]+ @ 0x[0-9,a-f]+ 0x[0-9,a-f]+ 0x[0-9,a-f]+ 0x[0-9,a-f]+ 0x[0-9,a-f]+
|
||||
# 0x[0-9,a-f]+ runtime\.selectgo\+0x[0-9,a-f]+ .*/src/runtime/select.go:[0-9]+
|
||||
# 0x[0-9,a-f]+ runtime/pprof_test\.blockSelectSendSync\+0x[0-9,a-f]+ .*/src/runtime/pprof/pprof_test.go:[0-9]+
|
||||
# 0x[0-9,a-f]+ runtime/pprof_test\.TestBlockProfile\+0x[0-9,a-f]+ .*/src/runtime/pprof/pprof_test.go:[0-9]+
|
||||
# 0x[0-9,a-f]+ runtime/pprof\.blockSelectSendSync\+0x[0-9,a-f]+ .*/src/runtime/pprof/pprof_test.go:[0-9]+
|
||||
# 0x[0-9,a-f]+ runtime/pprof\.TestBlockProfile\+0x[0-9,a-f]+ .*/src/runtime/pprof/pprof_test.go:[0-9]+
|
||||
`},
|
||||
{"mutex", blockMutex, `
|
||||
[0-9]+ [0-9]+ @ 0x[0-9,a-f]+ 0x[0-9,a-f]+ 0x[0-9,a-f]+ 0x[0-9,a-f]+ 0x[0-9,a-f]+
|
||||
# 0x[0-9,a-f]+ sync\.\(\*Mutex\)\.Lock\+0x[0-9,a-f]+ .*/src/sync/mutex\.go:[0-9]+
|
||||
# 0x[0-9,a-f]+ runtime/pprof_test\.blockMutex\+0x[0-9,a-f]+ .*/src/runtime/pprof/pprof_test.go:[0-9]+
|
||||
# 0x[0-9,a-f]+ runtime/pprof_test\.TestBlockProfile\+0x[0-9,a-f]+ .*/src/runtime/pprof/pprof_test.go:[0-9]+
|
||||
# 0x[0-9,a-f]+ runtime/pprof\.blockMutex\+0x[0-9,a-f]+ .*/src/runtime/pprof/pprof_test.go:[0-9]+
|
||||
# 0x[0-9,a-f]+ runtime/pprof\.TestBlockProfile\+0x[0-9,a-f]+ .*/src/runtime/pprof/pprof_test.go:[0-9]+
|
||||
`},
|
||||
{"cond", blockCond, `
|
||||
[0-9]+ [0-9]+ @ 0x[0-9,a-f]+ 0x[0-9,a-f]+ 0x[0-9,a-f]+ 0x[0-9,a-f]+ 0x[0-9,a-f]+
|
||||
# 0x[0-9,a-f]+ sync\.\(\*Cond\)\.Wait\+0x[0-9,a-f]+ .*/src/sync/cond\.go:[0-9]+
|
||||
# 0x[0-9,a-f]+ runtime/pprof_test\.blockCond\+0x[0-9,a-f]+ .*/src/runtime/pprof/pprof_test.go:[0-9]+
|
||||
# 0x[0-9,a-f]+ runtime/pprof_test\.TestBlockProfile\+0x[0-9,a-f]+ .*/src/runtime/pprof/pprof_test.go:[0-9]+
|
||||
# 0x[0-9,a-f]+ runtime/pprof\.blockCond\+0x[0-9,a-f]+ .*/src/runtime/pprof/pprof_test.go:[0-9]+
|
||||
# 0x[0-9,a-f]+ runtime/pprof\.TestBlockProfile\+0x[0-9,a-f]+ .*/src/runtime/pprof/pprof_test.go:[0-9]+
|
||||
`},
|
||||
}
|
||||
|
||||
@ -541,7 +545,7 @@ func TestMutexProfile(t *testing.T) {
|
||||
if ok, err := regexp.MatchString(r2, lines[3]); err != nil || !ok {
|
||||
t.Errorf("%q didn't match %q", lines[3], r2)
|
||||
}
|
||||
r3 := "^#.*runtime/pprof_test.blockMutex.*$"
|
||||
r3 := "^#.*runtime/pprof.blockMutex.*$"
|
||||
if ok, err := regexp.MatchString(r3, lines[5]); err != nil || !ok {
|
||||
t.Errorf("%q didn't match %q", lines[5], r3)
|
||||
}
|
||||
|
||||
@ -5,14 +5,16 @@
|
||||
package pprof
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"fmt"
|
||||
"os"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"runtime"
|
||||
"strings"
|
||||
"sort"
|
||||
"strconv"
|
||||
"time"
|
||||
"unsafe"
|
||||
|
||||
"internal/pprof/profile"
|
||||
)
|
||||
|
||||
// lostProfileEvent is the function to which lost profiling
|
||||
@ -25,47 +27,240 @@ func funcPC(f interface{}) uintptr {
|
||||
return *(*[2]*uintptr)(unsafe.Pointer(&f))[1]
|
||||
}
|
||||
|
||||
// A profileBuilder builds a profile.Profile incrementally from a
|
||||
// A profileBuilder writes a profile incrementally from a
|
||||
// stream of profile samples delivered by the runtime.
|
||||
// TODO(rsc,matloob): In the long term, we'd like to avoid
|
||||
// storing the entire profile.Profile in memory, instead streaming
|
||||
// the encoded form out to an underlying writer.
|
||||
// Even so, this one copy is a step forward from Go 1.8,
|
||||
// which had two full copies of the data in memory.
|
||||
type profileBuilder struct {
|
||||
p *profile.Profile
|
||||
start time.Time
|
||||
end time.Time
|
||||
havePeriod bool
|
||||
locs map[uintptr]*profile.Location
|
||||
period int64
|
||||
m profMap
|
||||
|
||||
// encoding state
|
||||
w io.Writer
|
||||
zw *gzip.Writer
|
||||
pb protobuf
|
||||
strings []string
|
||||
stringMap map[string]int
|
||||
locs map[uintptr]int
|
||||
funcs map[*runtime.Func]int
|
||||
mem []memMap
|
||||
}
|
||||
|
||||
type memMap struct {
|
||||
start uintptr
|
||||
end uintptr
|
||||
}
|
||||
|
||||
const (
|
||||
// message Profile
|
||||
tagProfile_SampleType = 1 // repeated ValueType
|
||||
tagProfile_Sample = 2 // repeated Sample
|
||||
tagProfile_Mapping = 3 // repeated Mapping
|
||||
tagProfile_Location = 4 // repeated Location
|
||||
tagProfile_Function = 5 // repeated Function
|
||||
tagProfile_StringTable = 6 // repeated string
|
||||
tagProfile_DropFrames = 7 // int64 (string table index)
|
||||
tagProfile_KeepFrames = 8 // int64 (string table index)
|
||||
tagProfile_TimeNanos = 9 // int64
|
||||
tagProfile_DurationNanos = 10 // int64
|
||||
tagProfile_PeriodType = 11 // ValueType (really optional string???)
|
||||
tagProfile_Period = 12 // int64
|
||||
|
||||
// message ValueType
|
||||
tagValueType_Type = 1 // int64 (string table index)
|
||||
tagValueType_Unit = 2 // int64 (string table index)
|
||||
|
||||
// message Sample
|
||||
tagSample_Location = 1 // repeated uint64
|
||||
tagSample_Value = 2 // repeated int64
|
||||
tagSample_Label = 3 // repeated Label
|
||||
|
||||
// message Label
|
||||
tagLabel_Key = 1 // int64 (string table index)
|
||||
tagLabel_Str = 2 // int64 (string table index)
|
||||
tagLabel_Num = 3 // int64
|
||||
|
||||
// message Mapping
|
||||
tagMapping_ID = 1 // uint64
|
||||
tagMapping_Start = 2 // uint64
|
||||
tagMapping_Limit = 3 // uint64
|
||||
tagMapping_Offset = 4 // uint64
|
||||
tagMapping_Filename = 5 // int64 (string table index)
|
||||
tagMapping_BuildID = 6 // int64 (string table index)
|
||||
tagMapping_HasFunctions = 7 // bool
|
||||
tagMapping_HasFilenames = 8 // bool
|
||||
tagMapping_HasLineNumbers = 9 // bool
|
||||
tagMapping_HasInlineFrames = 10 // bool
|
||||
|
||||
// message Location
|
||||
tagLocation_ID = 1 // uint64
|
||||
tagLocation_MappingID = 2 // uint64
|
||||
tagLocation_Address = 3 // uint64
|
||||
tagLocation_Line = 4 // repeated Line
|
||||
|
||||
// message Line
|
||||
tagLine_FunctionID = 1 // uint64
|
||||
tagLine_Line = 2 // int64
|
||||
|
||||
// message Function
|
||||
tagFunction_ID = 1 // uint64
|
||||
tagFunction_Name = 2 // int64 (string table index)
|
||||
tagFunction_SystemName = 3 // int64 (string table index)
|
||||
tagFunction_Filename = 4 // int64 (string table index)
|
||||
tagFunction_StartLine = 5 // int64
|
||||
)
|
||||
|
||||
// stringIndex adds s to the string table if not already present
|
||||
// and returns the index of s in the string table.
|
||||
func (b *profileBuilder) stringIndex(s string) int64 {
|
||||
id, ok := b.stringMap[s]
|
||||
if !ok {
|
||||
id = len(b.strings)
|
||||
b.strings = append(b.strings, s)
|
||||
b.stringMap[s] = id
|
||||
}
|
||||
return int64(id)
|
||||
}
|
||||
|
||||
func (b *profileBuilder) flush() {
|
||||
const dataFlush = 4096
|
||||
if b.pb.nest == 0 && len(b.pb.data) > dataFlush {
|
||||
b.zw.Write(b.pb.data)
|
||||
b.pb.data = b.pb.data[:0]
|
||||
}
|
||||
}
|
||||
|
||||
// pbValueType encodes a ValueType message to b.pb.
|
||||
func (b *profileBuilder) pbValueType(tag int, typ, unit string) {
|
||||
start := b.pb.startMessage()
|
||||
b.pb.int64(tagValueType_Type, b.stringIndex(typ))
|
||||
b.pb.int64(tagValueType_Unit, b.stringIndex(unit))
|
||||
b.pb.endMessage(tag, start)
|
||||
}
|
||||
|
||||
// pbSample encodes a Sample message to b.pb.
|
||||
func (b *profileBuilder) pbSample(values []int64, locs []uint64, labels func()) {
|
||||
start := b.pb.startMessage()
|
||||
b.pb.int64s(tagSample_Value, values)
|
||||
b.pb.uint64s(tagSample_Location, locs)
|
||||
if labels != nil {
|
||||
labels()
|
||||
}
|
||||
b.pb.endMessage(tagProfile_Sample, start)
|
||||
b.flush()
|
||||
}
|
||||
|
||||
// pbLabel encodes a Label message to b.pb.
|
||||
func (b *profileBuilder) pbLabel(tag int, key, str string, num int64) {
|
||||
start := b.pb.startMessage()
|
||||
b.pb.int64Opt(tagLabel_Key, b.stringIndex(key))
|
||||
b.pb.int64Opt(tagLabel_Str, b.stringIndex(str))
|
||||
b.pb.int64Opt(tagLabel_Num, num)
|
||||
b.pb.endMessage(tag, start)
|
||||
}
|
||||
|
||||
// pbLine encodes a Line message to b.pb.
|
||||
func (b *profileBuilder) pbLine(tag int, funcID uint64, line int64) {
|
||||
start := b.pb.startMessage()
|
||||
b.pb.uint64Opt(tagLine_FunctionID, funcID)
|
||||
b.pb.int64Opt(tagLine_Line, line)
|
||||
b.pb.endMessage(tag, start)
|
||||
}
|
||||
|
||||
// pbMapping encodes a Mapping message to b.pb.
|
||||
func (b *profileBuilder) pbMapping(tag int, id, base, limit, offset uint64, file string) {
|
||||
start := b.pb.startMessage()
|
||||
b.pb.uint64Opt(tagMapping_ID, id)
|
||||
b.pb.uint64Opt(tagMapping_Start, base)
|
||||
b.pb.uint64Opt(tagMapping_Limit, limit)
|
||||
b.pb.uint64Opt(tagMapping_Offset, offset)
|
||||
b.pb.int64Opt(tagMapping_Filename, b.stringIndex(file))
|
||||
// TODO: Set any of HasInlineFrames, HasFunctions, HasFilenames, HasLineNumbers?
|
||||
// It seems like they should all be true, but they've never been set.
|
||||
b.pb.endMessage(tagProfile_Mapping, start)
|
||||
}
|
||||
|
||||
// locForPC returns the location ID for addr.
|
||||
// It may emit to b.pb, so there must be no message encoding in progress.
|
||||
func (b *profileBuilder) locForPC(addr uintptr) uint64 {
|
||||
id := uint64(b.locs[addr])
|
||||
if id != 0 {
|
||||
return id
|
||||
}
|
||||
f := runtime.FuncForPC(addr)
|
||||
if f != nil && f.Name() == "runtime.goexit" {
|
||||
return 0
|
||||
}
|
||||
funcID, lineno := b.funcForPC(addr)
|
||||
id = uint64(len(b.locs)) + 1
|
||||
b.locs[addr] = int(id)
|
||||
start := b.pb.startMessage()
|
||||
b.pb.uint64Opt(tagLocation_ID, id)
|
||||
b.pb.uint64Opt(tagLocation_Address, uint64(addr))
|
||||
b.pbLine(tagLocation_Line, funcID, int64(lineno))
|
||||
if len(b.mem) > 0 {
|
||||
i := sort.Search(len(b.mem), func(i int) bool {
|
||||
return b.mem[i].end > addr
|
||||
})
|
||||
if i < len(b.mem) && b.mem[i].start <= addr && addr < b.mem[i].end {
|
||||
b.pb.uint64Opt(tagLocation_MappingID, uint64(i+1))
|
||||
}
|
||||
}
|
||||
b.pb.endMessage(tagProfile_Location, start)
|
||||
b.flush()
|
||||
return id
|
||||
}
|
||||
|
||||
// funcForPC returns the func ID and line number for addr.
|
||||
// It may emit to b.pb, so there must be no message encoding in progress.
|
||||
func (b *profileBuilder) funcForPC(addr uintptr) (funcID uint64, lineno int) {
|
||||
f := runtime.FuncForPC(addr)
|
||||
if f == nil {
|
||||
return 0, 0
|
||||
}
|
||||
file, lineno := f.FileLine(addr)
|
||||
funcID = uint64(b.funcs[f])
|
||||
if funcID != 0 {
|
||||
return funcID, lineno
|
||||
}
|
||||
|
||||
funcID = uint64(len(b.funcs)) + 1
|
||||
b.funcs[f] = int(funcID)
|
||||
name := f.Name()
|
||||
start := b.pb.startMessage()
|
||||
b.pb.uint64Opt(tagFunction_ID, funcID)
|
||||
b.pb.int64Opt(tagFunction_Name, b.stringIndex(name))
|
||||
b.pb.int64Opt(tagFunction_SystemName, b.stringIndex(name))
|
||||
b.pb.int64Opt(tagFunction_Filename, b.stringIndex(file))
|
||||
b.pb.endMessage(tagProfile_Function, start)
|
||||
b.flush()
|
||||
return funcID, lineno
|
||||
}
|
||||
|
||||
// newProfileBuilder returns a new profileBuilder.
|
||||
// CPU profiling data obtained from the runtime can be added
|
||||
// by calling b.addCPUData, and then the eventual profile
|
||||
// can be obtained by calling b.finish.
|
||||
func newProfileBuilder() *profileBuilder {
|
||||
start := time.Now()
|
||||
p := &profile.Profile{
|
||||
PeriodType: &profile.ValueType{Type: "cpu", Unit: "nanoseconds"},
|
||||
SampleType: []*profile.ValueType{
|
||||
{Type: "samples", Unit: "count"},
|
||||
{Type: "cpu", Unit: "nanoseconds"},
|
||||
},
|
||||
TimeNanos: int64(start.UnixNano()),
|
||||
}
|
||||
return &profileBuilder{
|
||||
p: p,
|
||||
start: start,
|
||||
locs: make(map[uintptr]*profile.Location),
|
||||
func newProfileBuilder(w io.Writer) *profileBuilder {
|
||||
zw, _ := gzip.NewWriterLevel(w, gzip.BestSpeed)
|
||||
b := &profileBuilder{
|
||||
w: w,
|
||||
zw: zw,
|
||||
start: time.Now(),
|
||||
strings: []string{""},
|
||||
stringMap: map[string]int{"": 0},
|
||||
locs: map[uintptr]int{},
|
||||
funcs: map[*runtime.Func]int{},
|
||||
}
|
||||
b.readMapping()
|
||||
return b
|
||||
}
|
||||
|
||||
// addCPUData adds the CPU profiling data to the profile.
|
||||
// The data must be a whole number of records,
|
||||
// as delivered by the runtime.
|
||||
func (b *profileBuilder) addCPUData(data []uint64, tags []unsafe.Pointer) error {
|
||||
p := b.p
|
||||
if !b.havePeriod {
|
||||
// first record is period
|
||||
if len(data) < 3 {
|
||||
@ -74,10 +269,9 @@ func (b *profileBuilder) addCPUData(data []uint64, tags []unsafe.Pointer) error
|
||||
if data[0] != 3 || data[2] == 0 {
|
||||
return fmt.Errorf("malformed profile")
|
||||
}
|
||||
period := int64(data[2])
|
||||
p.Period = period * 1000
|
||||
data = data[3:]
|
||||
b.period = int64(data[2]) * 1000
|
||||
b.havePeriod = true
|
||||
data = data[3:]
|
||||
}
|
||||
|
||||
// Parse CPU samples from the profile.
|
||||
@ -124,121 +318,141 @@ func (b *profileBuilder) addCPUData(data []uint64, tags []unsafe.Pointer) error
|
||||
}
|
||||
|
||||
// build completes and returns the constructed profile.
|
||||
func (b *profileBuilder) build() *profile.Profile {
|
||||
b.p.DurationNanos = time.Since(b.start).Nanoseconds()
|
||||
func (b *profileBuilder) build() error {
|
||||
b.end = time.Now()
|
||||
|
||||
b.pb.int64Opt(tagProfile_TimeNanos, b.start.UnixNano())
|
||||
if b.havePeriod { // must be CPU profile
|
||||
b.pbValueType(tagProfile_SampleType, "samples", "count")
|
||||
b.pbValueType(tagProfile_SampleType, "cpu", "nanoseconds")
|
||||
b.pb.int64Opt(tagProfile_DurationNanos, b.end.Sub(b.start).Nanoseconds())
|
||||
b.pbValueType(tagProfile_PeriodType, "cpu", "nanoseconds")
|
||||
b.pb.int64Opt(tagProfile_Period, b.period)
|
||||
}
|
||||
|
||||
values := []int64{0, 0}
|
||||
var locs []uint64
|
||||
for e := b.m.all; e != nil; e = e.nextAll {
|
||||
s := &profile.Sample{
|
||||
Value: []int64{e.count, e.count * int64(b.p.Period)},
|
||||
Location: make([]*profile.Location, len(e.stk)),
|
||||
}
|
||||
values[0] = e.count
|
||||
values[1] = e.count * b.period
|
||||
|
||||
locs = locs[:0]
|
||||
for i, addr := range e.stk {
|
||||
addr := uintptr(addr)
|
||||
// Addresses from stack traces point to the next instruction after
|
||||
// each call. Adjust by -1 to land somewhere on the actual call
|
||||
// (except for the leaf, which is not a call).
|
||||
if i > 0 {
|
||||
addr--
|
||||
}
|
||||
loc := b.locs[addr]
|
||||
if loc == nil {
|
||||
loc = &profile.Location{
|
||||
ID: uint64(len(b.p.Location) + 1),
|
||||
Address: uint64(addr),
|
||||
}
|
||||
b.locs[addr] = loc
|
||||
b.p.Location = append(b.p.Location, loc)
|
||||
l := b.locForPC(addr)
|
||||
if l == 0 { // runtime.goexit
|
||||
continue
|
||||
}
|
||||
s.Location[i] = loc
|
||||
locs = append(locs, l)
|
||||
}
|
||||
b.p.Sample = append(b.p.Sample, s)
|
||||
b.pbSample(values, locs, nil)
|
||||
}
|
||||
|
||||
if runtime.GOOS == "linux" {
|
||||
addMappings(b.p)
|
||||
}
|
||||
symbolize(b.p)
|
||||
return b.p
|
||||
}
|
||||
// TODO: Anything for tagProfile_DropFrames?
|
||||
// TODO: Anything for tagProfile_KeepFrames?
|
||||
|
||||
// addMappings adds information from /proc/self/maps
|
||||
// to the profile if possible.
|
||||
func addMappings(p *profile.Profile) {
|
||||
// Parse memory map from /proc/self/maps
|
||||
f, err := os.Open("/proc/self/maps")
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
p.ParseMemoryMap(f)
|
||||
f.Close()
|
||||
}
|
||||
|
||||
type function interface {
|
||||
Name() string
|
||||
FileLine(pc uintptr) (string, int)
|
||||
}
|
||||
|
||||
// funcForPC is a wrapper for runtime.FuncForPC. Defined as var for testing.
|
||||
var funcForPC = func(pc uintptr) function {
|
||||
if f := runtime.FuncForPC(pc); f != nil {
|
||||
return f
|
||||
}
|
||||
b.pb.strings(tagProfile_StringTable, b.strings)
|
||||
b.zw.Write(b.pb.data)
|
||||
b.zw.Close()
|
||||
return nil
|
||||
}
|
||||
|
||||
func symbolize(p *profile.Profile) {
|
||||
fns := profileFunctionMap{}
|
||||
for _, l := range p.Location {
|
||||
pc := uintptr(l.Address)
|
||||
f := funcForPC(pc)
|
||||
if f == nil {
|
||||
// readMapping reads /proc/self/maps and writes mappings to b.pb.
|
||||
// It saves the address ranges of the mappings in b.mem for use
|
||||
// when emitting locations.
|
||||
func (b *profileBuilder) readMapping() {
|
||||
data, _ := ioutil.ReadFile("/proc/self/maps")
|
||||
|
||||
// $ cat /proc/self/maps
|
||||
// 00400000-0040b000 r-xp 00000000 fc:01 787766 /bin/cat
|
||||
// 0060a000-0060b000 r--p 0000a000 fc:01 787766 /bin/cat
|
||||
// 0060b000-0060c000 rw-p 0000b000 fc:01 787766 /bin/cat
|
||||
// 014ab000-014cc000 rw-p 00000000 00:00 0 [heap]
|
||||
// 7f7d76af8000-7f7d7797c000 r--p 00000000 fc:01 1318064 /usr/lib/locale/locale-archive
|
||||
// 7f7d7797c000-7f7d77b36000 r-xp 00000000 fc:01 1180226 /lib/x86_64-linux-gnu/libc-2.19.so
|
||||
// 7f7d77b36000-7f7d77d36000 ---p 001ba000 fc:01 1180226 /lib/x86_64-linux-gnu/libc-2.19.so
|
||||
// 7f7d77d36000-7f7d77d3a000 r--p 001ba000 fc:01 1180226 /lib/x86_64-linux-gnu/libc-2.19.so
|
||||
// 7f7d77d3a000-7f7d77d3c000 rw-p 001be000 fc:01 1180226 /lib/x86_64-linux-gnu/libc-2.19.so
|
||||
// 7f7d77d3c000-7f7d77d41000 rw-p 00000000 00:00 0
|
||||
// 7f7d77d41000-7f7d77d64000 r-xp 00000000 fc:01 1180217 /lib/x86_64-linux-gnu/ld-2.19.so
|
||||
// 7f7d77f3f000-7f7d77f42000 rw-p 00000000 00:00 0
|
||||
// 7f7d77f61000-7f7d77f63000 rw-p 00000000 00:00 0
|
||||
// 7f7d77f63000-7f7d77f64000 r--p 00022000 fc:01 1180217 /lib/x86_64-linux-gnu/ld-2.19.so
|
||||
// 7f7d77f64000-7f7d77f65000 rw-p 00023000 fc:01 1180217 /lib/x86_64-linux-gnu/ld-2.19.so
|
||||
// 7f7d77f65000-7f7d77f66000 rw-p 00000000 00:00 0
|
||||
// 7ffc342a2000-7ffc342c3000 rw-p 00000000 00:00 0 [stack]
|
||||
// 7ffc34343000-7ffc34345000 r-xp 00000000 00:00 0 [vdso]
|
||||
// ffffffffff600000-ffffffffff601000 r-xp 00000000 00:00 0 [vsyscall]
|
||||
|
||||
var line []byte
|
||||
// next removes and returns the next field in the line.
|
||||
// It also removes from line any spaces following the field.
|
||||
next := func() []byte {
|
||||
j := bytes.IndexByte(line, ' ')
|
||||
if j < 0 {
|
||||
f := line
|
||||
line = nil
|
||||
return f
|
||||
}
|
||||
f := line[:j]
|
||||
line = line[j+1:]
|
||||
for len(line) > 0 && line[0] == ' ' {
|
||||
line = line[1:]
|
||||
}
|
||||
return f
|
||||
}
|
||||
|
||||
for len(data) > 0 {
|
||||
i := bytes.IndexByte(data, '\n')
|
||||
if i < 0 {
|
||||
line, data = data, nil
|
||||
} else {
|
||||
line, data = data[:i], data[i+1:]
|
||||
}
|
||||
addr := next()
|
||||
i = bytes.IndexByte(addr, '-')
|
||||
if i < 0 {
|
||||
continue
|
||||
}
|
||||
file, lineno := f.FileLine(pc)
|
||||
l.Line = []profile.Line{
|
||||
{
|
||||
Function: fns.findOrAddFunction(f.Name(), file, p),
|
||||
Line: int64(lineno),
|
||||
},
|
||||
lo, err := strconv.ParseUint(string(addr[:i]), 16, 64)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
}
|
||||
// Trim runtime functions. Always hide runtime.goexit. Other runtime
|
||||
// functions are only hidden for heapz when they appear at the beginning.
|
||||
isHeapz := p.PeriodType != nil && p.PeriodType.Type == "space"
|
||||
for _, s := range p.Sample {
|
||||
show := !isHeapz
|
||||
var i int
|
||||
for _, l := range s.Location {
|
||||
if len(l.Line) > 0 && l.Line[0].Function != nil {
|
||||
name := l.Line[0].Function.Name
|
||||
if name == "runtime.goexit" || !show && strings.HasPrefix(name, "runtime.") {
|
||||
continue
|
||||
}
|
||||
}
|
||||
show = true
|
||||
s.Location[i] = l
|
||||
i++
|
||||
hi, err := strconv.ParseUint(string(addr[i+1:]), 16, 64)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
s.Location = s.Location[:i]
|
||||
perm := next()
|
||||
if len(perm) < 4 || perm[2] != 'x' {
|
||||
// Only interested in executable mappings.
|
||||
continue
|
||||
}
|
||||
offset, err := strconv.ParseUint(string(next()), 16, 64)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
next() // dev
|
||||
next() // inode
|
||||
file := line
|
||||
if file == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
// TODO: pprof's remapMappingIDs makes two adjustments:
|
||||
// 1. If there is an /anon_hugepage mapping first and it is
|
||||
// consecutive to a next mapping, drop the /anon_hugepage.
|
||||
// 2. If start-offset = 0x400000, change start to 0x400000 and offset to 0.
|
||||
// There's no indication why either of these is needed.
|
||||
// Let's try not doing these and see what breaks.
|
||||
// If we do need them, they would go here, before we
|
||||
// enter the mappings into b.mem in the first place.
|
||||
|
||||
b.mem = append(b.mem, memMap{uintptr(lo), uintptr(hi)})
|
||||
b.pbMapping(tagProfile_Mapping, uint64(len(b.mem)), lo, hi, offset, string(file))
|
||||
}
|
||||
}
|
||||
|
||||
type profileFunctionMap map[profile.Function]*profile.Function
|
||||
|
||||
func (fns profileFunctionMap) findOrAddFunction(name, filename string, p *profile.Profile) *profile.Function {
|
||||
f := profile.Function{
|
||||
Name: name,
|
||||
SystemName: name,
|
||||
Filename: filename,
|
||||
}
|
||||
if fp := fns[f]; fp != nil {
|
||||
return fp
|
||||
}
|
||||
fp := new(profile.Function)
|
||||
fns[f] = fp
|
||||
|
||||
*fp = f
|
||||
fp.ID = uint64(len(p.Function) + 1)
|
||||
p.Function = append(p.Function, fp)
|
||||
return fp
|
||||
}
|
||||
|
||||
@ -19,11 +19,13 @@ import (
|
||||
// This is only used for testing. Real conversions stream the
|
||||
// data into the profileBuilder as it becomes available.
|
||||
func translateCPUProfile(data []uint64) (*profile.Profile, error) {
|
||||
b := newProfileBuilder()
|
||||
var buf bytes.Buffer
|
||||
b := newProfileBuilder(&buf)
|
||||
if err := b.addCPUData(data, nil); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return b.build(), nil
|
||||
b.build()
|
||||
return profile.Parse(&buf)
|
||||
}
|
||||
|
||||
// fmtJSON returns a pretty-printed JSON form for x.
|
||||
@ -38,7 +40,7 @@ func TestConvertCPUProfileEmpty(t *testing.T) {
|
||||
// A test server with mock cpu profile data.
|
||||
var buf bytes.Buffer
|
||||
|
||||
b := []uint64{3, 0, 2000} // empty profile with 2000ms sample period
|
||||
b := []uint64{3, 0, 2000} // empty profile with 2ms sample period
|
||||
p, err := translateCPUProfile(b)
|
||||
if err != nil {
|
||||
t.Fatalf("translateCPUProfile: %v", err)
|
||||
@ -53,15 +55,13 @@ func TestConvertCPUProfileEmpty(t *testing.T) {
|
||||
}
|
||||
|
||||
// Expected PeriodType and SampleType.
|
||||
expectedPeriodType := &profile.ValueType{Type: "cpu", Unit: "nanoseconds"}
|
||||
expectedSampleType := []*profile.ValueType{
|
||||
periodType := &profile.ValueType{Type: "cpu", Unit: "nanoseconds"}
|
||||
sampleType := []*profile.ValueType{
|
||||
{Type: "samples", Unit: "count"},
|
||||
{Type: "cpu", Unit: "nanoseconds"},
|
||||
}
|
||||
if p.Period != 2000*1000 || !reflect.DeepEqual(p.PeriodType, expectedPeriodType) ||
|
||||
!reflect.DeepEqual(p.SampleType, expectedSampleType) || p.Sample != nil {
|
||||
t.Fatalf("Unexpected Profile fields")
|
||||
}
|
||||
|
||||
checkProfile(t, p, 2000*1000, periodType, sampleType, nil)
|
||||
}
|
||||
|
||||
func f1() { f1() }
|
||||
@ -145,7 +145,17 @@ func checkProfile(t *testing.T, p *profile.Profile, period int64, periodType *pr
|
||||
l.Line = nil
|
||||
}
|
||||
}
|
||||
if !reflect.DeepEqual(p.Sample, samples) {
|
||||
if fmtJSON(p.Sample) != fmtJSON(samples) { // ignore unexported fields
|
||||
if len(p.Sample) == len(samples) {
|
||||
for i := range p.Sample {
|
||||
if !reflect.DeepEqual(p.Sample[i], samples[i]) {
|
||||
t.Errorf("sample %d = %v\nwant = %v\n", i, fmtJSON(p.Sample[i]), fmtJSON(samples[i]))
|
||||
}
|
||||
}
|
||||
if t.Failed() {
|
||||
t.FailNow()
|
||||
}
|
||||
}
|
||||
t.Fatalf("p.Sample = %v\nwant = %v", fmtJSON(p.Sample), fmtJSON(samples))
|
||||
}
|
||||
}
|
||||
@ -163,6 +173,7 @@ func (f *fakeFunc) FileLine(uintptr) (string, int) {
|
||||
return f.file, f.lineno
|
||||
}
|
||||
|
||||
/*
|
||||
// TestRuntimeFunctionTrimming tests if symbolize trims runtime functions as intended.
|
||||
func TestRuntimeRunctionTrimming(t *testing.T) {
|
||||
fakeFuncMap := map[uintptr]*fakeFunc{
|
||||
@ -246,3 +257,4 @@ func TestRuntimeRunctionTrimming(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
*/
|
||||
|
||||
141
src/runtime/pprof/protobuf.go
Normal file
141
src/runtime/pprof/protobuf.go
Normal file
@ -0,0 +1,141 @@
|
||||
// Copyright 2014 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package pprof
|
||||
|
||||
// A protobuf is a simple protocol buffer encoder.
|
||||
type protobuf struct {
|
||||
data []byte
|
||||
tmp [16]byte
|
||||
nest int
|
||||
}
|
||||
|
||||
func (b *protobuf) varint(x uint64) {
|
||||
for x >= 128 {
|
||||
b.data = append(b.data, byte(x)|0x80)
|
||||
x >>= 7
|
||||
}
|
||||
b.data = append(b.data, byte(x))
|
||||
}
|
||||
|
||||
func (b *protobuf) length(tag int, len int) {
|
||||
b.varint(uint64(tag)<<3 | 2)
|
||||
b.varint(uint64(len))
|
||||
}
|
||||
|
||||
func (b *protobuf) uint64(tag int, x uint64) {
|
||||
// append varint to b.data
|
||||
b.varint(uint64(tag)<<3 | 0)
|
||||
b.varint(x)
|
||||
}
|
||||
|
||||
func (b *protobuf) uint64s(tag int, x []uint64) {
|
||||
if len(x) > 2 {
|
||||
// Use packed encoding
|
||||
n1 := len(b.data)
|
||||
for _, u := range x {
|
||||
b.varint(u)
|
||||
}
|
||||
n2 := len(b.data)
|
||||
b.length(tag, n2-n1)
|
||||
n3 := len(b.data)
|
||||
copy(b.tmp[:], b.data[n2:n3])
|
||||
copy(b.data[n1+(n3-n2):], b.data[n1:n2])
|
||||
copy(b.data[n1:], b.tmp[:n3-n2])
|
||||
return
|
||||
}
|
||||
for _, u := range x {
|
||||
b.uint64(tag, u)
|
||||
}
|
||||
}
|
||||
|
||||
func (b *protobuf) uint64Opt(tag int, x uint64) {
|
||||
if x == 0 {
|
||||
return
|
||||
}
|
||||
b.uint64(tag, x)
|
||||
}
|
||||
|
||||
func (b *protobuf) int64(tag int, x int64) {
|
||||
u := uint64(x)
|
||||
b.uint64(tag, u)
|
||||
}
|
||||
|
||||
func (b *protobuf) int64Opt(tag int, x int64) {
|
||||
if x == 0 {
|
||||
return
|
||||
}
|
||||
b.int64(tag, x)
|
||||
}
|
||||
|
||||
func (b *protobuf) int64s(tag int, x []int64) {
|
||||
if len(x) > 2 {
|
||||
// Use packed encoding
|
||||
n1 := len(b.data)
|
||||
for _, u := range x {
|
||||
b.varint(uint64(u))
|
||||
}
|
||||
n2 := len(b.data)
|
||||
b.length(tag, n2-n1)
|
||||
n3 := len(b.data)
|
||||
copy(b.tmp[:], b.data[n2:n3])
|
||||
copy(b.data[n1+(n3-n2):], b.data[n1:n2])
|
||||
copy(b.data[n1:], b.tmp[:n3-n2])
|
||||
return
|
||||
}
|
||||
for _, u := range x {
|
||||
b.int64(tag, u)
|
||||
}
|
||||
}
|
||||
|
||||
func (b *protobuf) string(tag int, x string) {
|
||||
b.length(tag, len(x))
|
||||
b.data = append(b.data, x...)
|
||||
}
|
||||
|
||||
func (b *protobuf) strings(tag int, x []string) {
|
||||
for _, s := range x {
|
||||
b.string(tag, s)
|
||||
}
|
||||
}
|
||||
|
||||
func (b *protobuf) stringOpt(tag int, x string) {
|
||||
if x == "" {
|
||||
return
|
||||
}
|
||||
b.string(tag, x)
|
||||
}
|
||||
|
||||
func (b *protobuf) bool(tag int, x bool) {
|
||||
if x {
|
||||
b.uint64(tag, 1)
|
||||
} else {
|
||||
b.uint64(tag, 0)
|
||||
}
|
||||
}
|
||||
|
||||
func (b *protobuf) boolOpt(tag int, x bool) {
|
||||
if x == false {
|
||||
return
|
||||
}
|
||||
b.bool(tag, x)
|
||||
}
|
||||
|
||||
type msgOffset int
|
||||
|
||||
func (b *protobuf) startMessage() msgOffset {
|
||||
b.nest++
|
||||
return msgOffset(len(b.data))
|
||||
}
|
||||
|
||||
func (b *protobuf) endMessage(tag int, start msgOffset) {
|
||||
n1 := int(start)
|
||||
n2 := len(b.data)
|
||||
b.length(tag, n2-n1)
|
||||
n3 := len(b.data)
|
||||
copy(b.tmp[:], b.data[n2:n3])
|
||||
copy(b.data[n1+(n3-n2):], b.data[n1:n2])
|
||||
copy(b.data[n1:], b.tmp[:n3-n2])
|
||||
b.nest--
|
||||
}
|
||||
@ -5,55 +5,65 @@
|
||||
package pprof
|
||||
|
||||
import (
|
||||
"internal/pprof/profile"
|
||||
"io"
|
||||
"math"
|
||||
"runtime"
|
||||
"time"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// encodeMemProfile converts MemProfileRecords to a Profile.
|
||||
func encodeMemProfile(mr []runtime.MemProfileRecord, rate int64, t time.Time) *profile.Profile {
|
||||
p := &profile.Profile{
|
||||
Period: rate,
|
||||
PeriodType: &profile.ValueType{Type: "space", Unit: "bytes"},
|
||||
SampleType: []*profile.ValueType{
|
||||
{Type: "alloc_objects", Unit: "count"},
|
||||
{Type: "alloc_space", Unit: "bytes"},
|
||||
{Type: "inuse_objects", Unit: "count"},
|
||||
{Type: "inuse_space", Unit: "bytes"},
|
||||
},
|
||||
TimeNanos: int64(t.UnixNano()),
|
||||
}
|
||||
// writeHeapProto writes the current heap profile in protobuf format to w.
|
||||
func writeHeapProto(w io.Writer, p []runtime.MemProfileRecord, rate int64) error {
|
||||
b := newProfileBuilder(w)
|
||||
b.pbValueType(tagProfile_PeriodType, "space", "bytes")
|
||||
b.pb.int64Opt(tagProfile_Period, rate)
|
||||
b.pbValueType(tagProfile_SampleType, "alloc_objects", "count")
|
||||
b.pbValueType(tagProfile_SampleType, "alloc_space", "bytes")
|
||||
b.pbValueType(tagProfile_SampleType, "inuse_objects", "count")
|
||||
b.pbValueType(tagProfile_SampleType, "inuse_space", "bytes")
|
||||
|
||||
locs := make(map[uintptr]*profile.Location)
|
||||
for _, r := range mr {
|
||||
stack := r.Stack()
|
||||
sloc := make([]*profile.Location, len(stack))
|
||||
for i, addr := range stack {
|
||||
loc := locs[addr]
|
||||
if loc == nil {
|
||||
loc = &profile.Location{
|
||||
ID: uint64(len(p.Location) + 1),
|
||||
Address: uint64(addr),
|
||||
values := []int64{0, 0, 0, 0}
|
||||
var locs []uint64
|
||||
for _, r := range p {
|
||||
locs = locs[:0]
|
||||
hideRuntime := true
|
||||
for tries := 0; tries < 2; tries++ {
|
||||
for i, addr := range r.Stack() {
|
||||
if false && i > 0 { // TODO: why disabled?
|
||||
addr--
|
||||
}
|
||||
locs[addr] = loc
|
||||
p.Location = append(p.Location, loc)
|
||||
if hideRuntime {
|
||||
if f := runtime.FuncForPC(addr); f != nil && strings.HasPrefix(f.Name(), "runtime.") {
|
||||
continue
|
||||
}
|
||||
// Found non-runtime. Show any runtime uses above it.
|
||||
hideRuntime = false
|
||||
}
|
||||
l := b.locForPC(addr)
|
||||
if l == 0 { // runtime.goexit
|
||||
continue
|
||||
}
|
||||
locs = append(locs, l)
|
||||
}
|
||||
sloc[i] = loc
|
||||
if len(locs) > 0 {
|
||||
break
|
||||
}
|
||||
hideRuntime = false // try again, and show all frames
|
||||
}
|
||||
|
||||
ao, ab := scaleHeapSample(r.AllocObjects, r.AllocBytes, rate)
|
||||
uo, ub := scaleHeapSample(r.InUseObjects(), r.InUseBytes(), rate)
|
||||
|
||||
p.Sample = append(p.Sample, &profile.Sample{
|
||||
Value: []int64{ao, ab, uo, ub},
|
||||
Location: sloc,
|
||||
values[0], values[1] = scaleHeapSample(r.AllocObjects, r.AllocBytes, rate)
|
||||
values[2], values[3] = scaleHeapSample(r.InUseObjects(), r.InUseBytes(), rate)
|
||||
var blockSize int64
|
||||
if values[0] > 0 {
|
||||
blockSize = values[1] / values[0]
|
||||
}
|
||||
b.pbSample(values, locs, func() {
|
||||
if blockSize != 0 {
|
||||
b.pbLabel(tagSample_Label, "bytes", "", blockSize)
|
||||
}
|
||||
})
|
||||
}
|
||||
if runtime.GOOS == "linux" {
|
||||
addMappings(p)
|
||||
}
|
||||
return p
|
||||
b.build()
|
||||
return nil
|
||||
}
|
||||
|
||||
// scaleHeapSample adjusts the data from a heap Sample to
|
||||
|
||||
@ -9,7 +9,6 @@ import (
|
||||
"internal/pprof/profile"
|
||||
"runtime"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestConvertMemProfile(t *testing.T) {
|
||||
@ -24,8 +23,7 @@ func TestConvertMemProfile(t *testing.T) {
|
||||
{AllocBytes: 512 * 1024, FreeBytes: 512 * 1024, AllocObjects: 1, FreeObjects: 1, Stack0: [32]uintptr{a1 + 1, a1 + 2, a2 + 3}},
|
||||
}
|
||||
|
||||
p := encodeMemProfile(rec, rate, time.Now())
|
||||
if err := p.Write(&buf); err != nil {
|
||||
if err := writeHeapProto(&buf, rec, rate); err != nil {
|
||||
t.Fatalf("writing profile: %v", err)
|
||||
}
|
||||
|
||||
@ -42,19 +40,31 @@ func TestConvertMemProfile(t *testing.T) {
|
||||
{Type: "inuse_space", Unit: "bytes"},
|
||||
}
|
||||
samples := []*profile.Sample{
|
||||
{Value: []int64{2050, 2099200, 1537, 1574400}, Location: []*profile.Location{
|
||||
{ID: 1, Mapping: map1, Address: addr1},
|
||||
{ID: 2, Mapping: map2, Address: addr2},
|
||||
}},
|
||||
{Value: []int64{1, 829411, 1, 829411}, Location: []*profile.Location{
|
||||
{ID: 3, Mapping: map2, Address: addr2 + 1},
|
||||
{ID: 4, Mapping: map2, Address: addr2 + 2},
|
||||
}},
|
||||
{Value: []int64{1, 829411, 0, 0}, Location: []*profile.Location{
|
||||
{ID: 5, Mapping: map1, Address: addr1 + 1},
|
||||
{ID: 6, Mapping: map1, Address: addr1 + 2},
|
||||
{ID: 7, Mapping: map2, Address: addr2 + 3},
|
||||
}},
|
||||
{
|
||||
Value: []int64{2050, 2099200, 1537, 1574400},
|
||||
Location: []*profile.Location{
|
||||
{ID: 1, Mapping: map1, Address: addr1},
|
||||
{ID: 2, Mapping: map2, Address: addr2},
|
||||
},
|
||||
NumLabel: map[string][]int64{"bytes": {1024}},
|
||||
},
|
||||
{
|
||||
Value: []int64{1, 829411, 1, 829411},
|
||||
Location: []*profile.Location{
|
||||
{ID: 3, Mapping: map2, Address: addr2 + 1},
|
||||
{ID: 4, Mapping: map2, Address: addr2 + 2},
|
||||
},
|
||||
NumLabel: map[string][]int64{"bytes": {829411}},
|
||||
},
|
||||
{
|
||||
Value: []int64{1, 829411, 0, 0},
|
||||
Location: []*profile.Location{
|
||||
{ID: 5, Mapping: map1, Address: addr1 + 1},
|
||||
{ID: 6, Mapping: map1, Address: addr1 + 2},
|
||||
{ID: 7, Mapping: map2, Address: addr2 + 3},
|
||||
},
|
||||
NumLabel: map[string][]int64{"bytes": {829411}},
|
||||
},
|
||||
}
|
||||
checkProfile(t, p, rate, periodType, sampleType, samples)
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user