qbit/go
1
0
Fork 0
mirror of https://github.com/golang/go synced 2024-11-26 05:57:58 -07:00
Code Releases Activity

runtime/metrics: add additional allocation metrics

Browse Source 
This change adds four additional metrics to the runtime/metrics package
to fill in a few gaps with runtime.MemStats that were overlooked. The
biggest one is TotalAlloc, which is impossible to find with the
runtime/metrics package, but also add a few others for convenience and
clarity. For instance, the total number of objects allocated and freed
are technically available via allocs-by-size and frees-by-size, but it's
onerous to get them (one needs to sum the sample counts in the
histograms).

The four additional metrics are:
- /gc/heap/allocs:bytes   -- total bytes allocated (TotalAlloc)
- /gc/heap/allocs:objects -- total objects allocated (Mallocs - [tiny])
- /gc/heap/frees:bytes    -- total bytes frees (TotalAlloc-HeapAlloc)
- /gc/heap/frees:objects  -- total objects freed (Frees - [tiny])

This change also updates the descriptions of allocs-by-size and
frees-by-size to be more precise.

Change-Id: Iec8c1797a584491e3484b198f2e7f325b68954a7
Reviewed-on: https://go-review.googlesource.com/c/go/+/312431
Reviewed-by: Michael Pratt <mpratt@google.com>
Trust: Michael Knyszek <mknyszek@google.com>
Run-TryBot: Michael Knyszek <mknyszek@google.com>
TryBot-Result: Go Bot <gobot@golang.org>
This commit is contained in:
Michael Anthony Knyszek 2021-04-20 19:30:21 +00:00 committed by Michael Knyszek
parent fd09593667
commit 897baae953
4 changed files with 164 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

61
src/runtime/metrics.go
View File

@ -98,6 +98,20 @@ func initMetrics() {
}
},
},
"/gc/heap/allocs:bytes": {
deps: makeStatDepSet(heapStatsDep),
compute: func(in *statAggregate, out *metricValue) {
out.kind = metricKindUint64
out.scalar = in.heapStats.totalAllocated
},
},
"/gc/heap/allocs:objects": {
deps: makeStatDepSet(heapStatsDep),
compute: func(in *statAggregate, out *metricValue) {
out.kind = metricKindUint64
out.scalar = in.heapStats.totalAllocs
},
},
"/gc/heap/frees-by-size:bytes": {
deps: makeStatDepSet(heapStatsDep),
compute: func(in *statAggregate, out *metricValue) {
@ -110,6 +124,20 @@ func initMetrics() {
}
},
},
"/gc/heap/frees:bytes": {
deps: makeStatDepSet(heapStatsDep),
compute: func(in *statAggregate, out *metricValue) {
out.kind = metricKindUint64
out.scalar = in.heapStats.totalFreed
},
},
"/gc/heap/frees:objects": {
deps: makeStatDepSet(heapStatsDep),
compute: func(in *statAggregate, out *metricValue) {
out.kind = metricKindUint64
out.scalar = in.heapStats.totalFrees
},
},
"/gc/heap/goal:bytes": {
deps: makeStatDepSet(sysStatsDep),
compute: func(in *statAggregate, out *metricValue) {
@ -337,6 +365,22 @@ type heapStatsAggregate struct {
// numObjects is the number of live objects in the heap.
numObjects uint64
// totalAllocated is the total bytes of heap objects allocated
// over the lifetime of the program.
totalAllocated uint64
// totalFreed is the total bytes of heap objects freed
// over the lifetime of the program.
totalFreed uint64
// totalAllocs is the number of heap objects allocated over
// the lifetime of the program.
totalAllocs uint64
// totalFrees is the number of heap objects freed over
// the lifetime of the program.
totalFrees uint64
}
// compute populates the heapStatsAggregate with values from the runtime.
@ -344,13 +388,20 @@ func (a *heapStatsAggregate) compute() {
memstats.heapStats.read(&a.heapStatsDelta)
// Calculate derived stats.
a.inObjects = uint64(a.largeAlloc - a.largeFree)
a.numObjects = uint64(a.largeAllocCount - a.largeFreeCount)
a.totalAllocs = uint64(a.largeAllocCount)
a.totalFrees = uint64(a.largeFreeCount)
a.totalAllocated = uint64(a.largeAlloc)
a.totalFreed = uint64(a.largeFree)
for i := range a.smallAllocCount {
n := uint64(a.smallAllocCount[i] - a.smallFreeCount[i])
a.inObjects += n * uint64(class_to_size[i])
a.numObjects += n
na := uint64(a.smallAllocCount[i])
nf := uint64(a.smallFreeCount[i])
a.totalAllocs += na
a.totalFrees += nf
a.totalAllocated += na * uint64(class_to_size[i])
a.totalFreed += nf * uint64(class_to_size[i])
}
a.inObjects = a.totalAllocated - a.totalFreed
a.numObjects = a.totalAllocs - a.totalFrees
}
// sysStatsAggregate represents system memory stats obtained

45
src/runtime/metrics/description.go
View File

@ -70,17 +70,50 @@ var allDesc = []Description{
Cumulative: true,
},
{
Name: "/gc/heap/allocs-by-size:bytes",
Description: "Distribution of all objects allocated by approximate size.",
Kind: KindFloat64Histogram,
Name: "/gc/heap/allocs-by-size:bytes",
Description: "Distribution of heap allocations by approximate size. " +
"Note that this does not include tiny objects as defined by " +
"/gc/heap/tiny/allocs:objects, only tiny blocks.",
Kind: KindFloat64Histogram,
Cumulative: true,
},
{
Name: "/gc/heap/allocs:bytes",
Description: "Cumulative sum of memory allocated to the heap by the application.",
Kind: KindUint64,
Cumulative: true,
},
{
Name: "/gc/heap/frees-by-size:bytes",
Description: "Distribution of all objects freed by approximate size.",
Kind: KindFloat64Histogram,
Name: "/gc/heap/allocs:objects",
Description: "Cumulative count of heap allocations triggered by the application. " +
"Note that this does not include tiny objects as defined by " +
"/gc/heap/tiny/allocs:objects, only tiny blocks.",
Kind: KindUint64,
Cumulative: true,
},
{
Name: "/gc/heap/frees-by-size:bytes",
Description: "Distribution of freed heap allocations by approximate size. " +
"Note that this does not include tiny objects as defined by " +
"/gc/heap/tiny/allocs:objects, only tiny blocks.",
Kind: KindFloat64Histogram,
Cumulative: true,
},
{
Name: "/gc/heap/frees:bytes",
Description: "Cumulative sum of heap memory freed by the garbage collector.",
Kind: KindUint64,
Cumulative: true,
},
{
Name: "/gc/heap/frees:objects",
Description: "Cumulative count of heap allocations whose storage was freed " +
"by the garbage collector. " +
"Note that this does not include tiny objects as defined by " +
"/gc/heap/tiny/allocs:objects, only tiny blocks.",
Kind: KindUint64,
Cumulative: true,
},
{
Name: "/gc/heap/goal:bytes",
Description: "Heap size target for the end of the GC cycle.",

24
src/runtime/metrics/doc.go
View File

@ -61,10 +61,30 @@ Below is the full list of supported metrics, ordered lexicographically.
Count of all completed GC cycles.
/gc/heap/allocs-by-size:bytes
Distribution of all objects allocated by approximate size.
Distribution of heap allocations by approximate size.
Note that this does not include tiny objects as defined by /gc/heap/tiny/allocs:objects,
only tiny blocks.
/gc/heap/allocs:bytes
Cumulative sum of memory allocated to the heap by the application.
/gc/heap/allocs:objects
Cumulative count of heap allocations triggered by the application.
Note that this does not include tiny objects as defined by /gc/heap/tiny/allocs:objects,
only tiny blocks.
/gc/heap/frees-by-size:bytes
Distribution of all objects freed by approximate size.
Distribution of freed heap allocations by approximate size.
Note that this does not include tiny objects as defined by /gc/heap/tiny/allocs:objects,
only tiny blocks.
/gc/heap/frees:bytes
Cumulative sum of heap memory freed by the garbage collector.
/gc/heap/frees:objects
Cumulative count of heap allocations whose storage was freed by the garbage collector.
Note that this does not include tiny objects as defined by /gc/heap/tiny/allocs:objects,
only tiny blocks.
/gc/heap/goal:bytes
Heap size target for the end of the GC cycle.

52
src/runtime/metrics_test.go
View File

@ -42,6 +42,7 @@ func TestReadMetrics(t *testing.T) {
// Check to make sure the values we read line up with other values we read.
var allocsBySize *metrics.Float64Histogram
var tinyAllocs uint64
var mallocs, frees uint64
for i := range samples {
switch name := samples[i].Name; name {
case "/memory/classes/heap/free:bytes":
@ -87,6 +88,8 @@ func TestReadMetrics(t *testing.T) {
}
}
allocsBySize = hist
case "/gc/heap/allocs:bytes":
checkUint64(t, name, samples[i].Value.Uint64(), mstats.TotalAlloc)
case "/gc/heap/frees-by-size:bytes":
hist := samples[i].Value.Float64Histogram()
// Skip size class 0 in BySize, because it's always empty and not represented
@ -101,6 +104,8 @@ func TestReadMetrics(t *testing.T) {
t.Errorf("histogram counts do not match BySize for class %d: got %d, want %d", i, c, f)
}
}
case "/gc/heap/frees:bytes":
checkUint64(t, name, samples[i].Value.Uint64(), mstats.TotalAlloc-mstats.HeapAlloc)
case "/gc/heap/tiny/allocs:objects":
// Currently, MemStats adds tiny alloc count to both Mallocs AND Frees.
// The reason for this is because MemStats couldn't be extended at the time
@ -112,6 +117,13 @@ func TestReadMetrics(t *testing.T) {
// Check tiny allocation count outside of this loop, by using the allocs-by-size
// histogram in order to figure out how many large objects there are.
tinyAllocs = samples[i].Value.Uint64()
// Because the next two metrics tests are checking against Mallocs and Frees,
// we can't check them directly for the same reason: we need to account for tiny
// allocations included in Mallocs and Frees.
case "/gc/heap/allocs:objects":
mallocs = samples[i].Value.Uint64()
case "/gc/heap/frees:objects":
frees = samples[i].Value.Uint64()
case "/gc/heap/objects:objects":
checkUint64(t, name, samples[i].Value.Uint64(), mstats.HeapObjects)
case "/gc/heap/goal:bytes":
@ -131,6 +143,10 @@ func TestReadMetrics(t *testing.T) {
nonTinyAllocs += c
}
checkUint64(t, "/gc/heap/tiny/allocs:objects", tinyAllocs, mstats.Mallocs-nonTinyAllocs)
// Check allocation and free counts.
checkUint64(t, "/gc/heap/allocs:objects", mallocs, mstats.Mallocs-tinyAllocs)
checkUint64(t, "/gc/heap/frees:objects", frees, mstats.Frees-tinyAllocs)
}
func TestReadMetricsConsistency(t *testing.T) {
@ -153,8 +169,10 @@ func TestReadMetricsConsistency(t *testing.T) {
got, want uint64
}
var objects struct {
alloc, free *metrics.Float64Histogram
total uint64
alloc, free *metrics.Float64Histogram
allocs, frees uint64
allocdBytes, freedBytes uint64
total, totalBytes uint64
}
var gc struct {
numGC uint64
@ -180,10 +198,20 @@ func TestReadMetricsConsistency(t *testing.T) {
switch samples[i].Name {
case "/memory/classes/total:bytes":
totalVirtual.got = samples[i].Value.Uint64()
case "/memory/classes/heap/objects:bytes":
objects.totalBytes = samples[i].Value.Uint64()
case "/gc/heap/objects:objects":
objects.total = samples[i].Value.Uint64()
case "/gc/heap/allocs:bytes":
objects.allocdBytes = samples[i].Value.Uint64()
case "/gc/heap/allocs:objects":
objects.allocs = samples[i].Value.Uint64()
case "/gc/heap/allocs-by-size:bytes":
objects.alloc = samples[i].Value.Float64Histogram()
case "/gc/heap/frees:bytes":
objects.freedBytes = samples[i].Value.Uint64()
case "/gc/heap/frees:objects":
objects.frees = samples[i].Value.Uint64()
case "/gc/heap/frees-by-size:bytes":
objects.free = samples[i].Value.Float64Histogram()
case "/gc/cycles:gc-cycles":
@ -203,6 +231,12 @@ func TestReadMetricsConsistency(t *testing.T) {
if totalVirtual.got != totalVirtual.want {
t.Errorf(`"/memory/classes/total:bytes" does not match sum of /memory/classes/**: got %d, want %d`, totalVirtual.got, totalVirtual.want)
}
if got, want := objects.allocs-objects.frees, objects.total; got != want {
t.Errorf("mismatch between object alloc/free tallies and total: got %d, want %d", got, want)
}
if got, want := objects.allocdBytes-objects.freedBytes, objects.totalBytes; got != want {
t.Errorf("mismatch between object alloc/free tallies and total: got %d, want %d", got, want)
}
if b, c := len(objects.alloc.Buckets), len(objects.alloc.Counts); b != c+1 {
t.Errorf("allocs-by-size has wrong bucket or counts length: %d buckets, %d counts", b, c)
}
@ -222,17 +256,25 @@ func TestReadMetricsConsistency(t *testing.T) {
}
}
if !t.Failed() {
got, want := uint64(0), objects.total
var gotAlloc, gotFree uint64
want := objects.total
for i := range objects.alloc.Counts {
if objects.alloc.Counts[i] < objects.free.Counts[i] {
t.Errorf("found more allocs than frees in object dist bucket %d", i)
continue
}
got += objects.alloc.Counts[i] - objects.free.Counts[i]
gotAlloc += objects.alloc.Counts[i]
gotFree += objects.free.Counts[i]
}
if got != want {
if got := gotAlloc - gotFree; got != want {
t.Errorf("object distribution counts don't match count of live objects: got %d, want %d", got, want)
}
if gotAlloc != objects.allocs {
t.Errorf("object distribution counts don't match total allocs: got %d, want %d", gotAlloc, objects.allocs)
}
if gotFree != objects.frees {
t.Errorf("object distribution counts don't match total allocs: got %d, want %d", gotFree, objects.frees)
}
}
}
// The current GC has at least 2 pauses per GC.