From a12cc7198072f2f02599d3c640807b97f748ddc6 Mon Sep 17 00:00:00 2001 From: Josh Bleecher Snyder Date: Tue, 24 Jun 2014 08:39:30 -0700 Subject: [PATCH] testing: make benchmarking faster The number of estimated iterations required to reach the benchtime is multiplied by a safety margin (to avoid falling just short) and then rounded up to a readable number. With an accurate estimate, in the worse case, the resulting number of iterations could be 3.75x more than necessary: 1.5x for safety * 2.5x to round up (e.g. from 2eX+1 to 5eX). This CL reduces the safety margin to 1.2x. Experimentation showed a diminishing margin of return past 1.2x, although the average case continued to show improvements down to 1.05x. This CL also reduces the maximum round-up multiplier from 2.5x (from 2eX+1 to 5eX) to 2x, by allowing the number of iterations to be of the form 3eX. Both changes improve benchmark wall clock times, and the effects are cumulative. From 1.5x to 1.2x safety margin: package old s new s delta bytes 163 125 -23% encoding/json 27 21 -22% net/http 42 36 -14% runtime 463 418 -10% strings 82 65 -21% Allowing 3eX iterations: package old s new s delta bytes 163 134 -18% encoding/json 27 23 -15% net/http 42 36 -14% runtime 463 422 -9% strings 82 72 -12% Combined: package old s new s delta bytes 163 112 -31% encoding/json 27 20 -26% net/http 42 30 -29% runtime 463 346 -25% strings 82 60 -27% LGTM=crawshaw, r, rsc R=golang-codereviews, crawshaw, r, rsc CC=golang-codereviews https://golang.org/cl/105990045 --- src/pkg/testing/benchmark.go | 16 +++++++++------- src/pkg/testing/benchmark_test.go | 2 ++ 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/src/pkg/testing/benchmark.go b/src/pkg/testing/benchmark.go index 1fbf5c8615f..ffd5376844a 100644 --- a/src/pkg/testing/benchmark.go +++ b/src/pkg/testing/benchmark.go @@ -157,7 +157,7 @@ func roundDown10(n int) int { return result } -// roundUp rounds x up to a number of the form [1eX, 2eX, 5eX]. +// roundUp rounds x up to a number of the form [1eX, 2eX, 3eX, 5eX]. func roundUp(n int) int { base := roundDown10(n) switch { @@ -165,6 +165,8 @@ func roundUp(n int) int { return base case n <= (2 * base): return 2 * base + case n <= (3 * base): + return 3 * base case n <= (5 * base): return 5 * base default: @@ -180,10 +182,10 @@ func (b *B) run() BenchmarkResult { } // launch launches the benchmark function. It gradually increases the number -// of benchmark iterations until the benchmark runs for a second in order -// to get a reasonable measurement. It prints timing information in this form +// of benchmark iterations until the benchmark runs for the requested benchtime. +// It prints timing information in this form // testing.BenchmarkHello 100000 19 ns/op -// launch is run by the fun function as a separate goroutine. +// launch is run by the run function as a separate goroutine. func (b *B) launch() { // Run the benchmark for a single iteration in case it's expensive. n := 1 @@ -199,16 +201,16 @@ func (b *B) launch() { d := *benchTime for !b.failed && b.duration < d && n < 1e9 { last := n - // Predict iterations/sec. + // Predict required iterations. if b.nsPerOp() == 0 { n = 1e9 } else { n = int(d.Nanoseconds() / b.nsPerOp()) } - // Run more iterations than we think we'll need for a second (1.5x). + // Run more iterations than we think we'll need (1.2x). // Don't grow too fast in case we had timing errors previously. // Be sure to run at least one more than last time. - n = max(min(n+n/2, 100*last), last+1) + n = max(min(n+n/5, 100*last), last+1) // Round up to something easy to read. n = roundUp(n) b.runN(n) diff --git a/src/pkg/testing/benchmark_test.go b/src/pkg/testing/benchmark_test.go index f7ea64e7f1c..431bb537bd5 100644 --- a/src/pkg/testing/benchmark_test.go +++ b/src/pkg/testing/benchmark_test.go @@ -41,12 +41,14 @@ var roundUpTests = []struct { {0, 1}, {1, 1}, {2, 2}, + {3, 3}, {5, 5}, {9, 10}, {999, 1000}, {1000, 1000}, {1400, 2000}, {1700, 2000}, + {2700, 3000}, {4999, 5000}, {5000, 5000}, {5001, 10000},