mirror of
https://github.com/golang/go
synced 2024-11-11 22:20:22 -07:00
testing: add -shuffle=off|on|N to alter the execution order of tests and benchmarks
This CL adds a new flag to the testing package and the go test command which randomizes the execution order for tests and benchmarks. This can be useful for identifying unwanted dependencies between test or benchmark functions. The flag is off by default. If `-shuffle` is set to `on` then the system clock will be used as the seed value. If `-shuffle` is set to an integer N, then N will be used as the seed value. In both cases, the seed will be reported for failed runs so that they can reproduced later on. Fixes #28592 Change-Id: I62e7dfae5f63f97a0cbd7830ea844d9f7beac335 Reviewed-on: https://go-review.googlesource.com/c/go/+/310033 Run-TryBot: Ian Lance Taylor <iant@golang.org> TryBot-Result: Go Bot <gobot@golang.org> Reviewed-by: Emmanuel Odeke <emmanuel@orijtech.com> Trust: Bryan C. Mills <bcmills@google.com>
This commit is contained in:
parent
e51246c881
commit
cbb3f09047
@ -2667,6 +2667,13 @@
|
||||
// the Go tree can run a sanity check but not spend time running
|
||||
// exhaustive tests.
|
||||
//
|
||||
// -shuffle off,on,N
|
||||
// Randomize the execution order of tests and benchmarks.
|
||||
// It is off by default. If -shuffle is set to on, then it will seed
|
||||
// the randomizer using the system clock. If -shuffle is set to an
|
||||
// integer N, then N will be used as the seed value. In both cases,
|
||||
// the seed will be reported for reproducibility.
|
||||
//
|
||||
// -timeout d
|
||||
// If a test binary runs longer than duration d, panic.
|
||||
// If d is 0, the timeout is disabled.
|
||||
|
@ -28,6 +28,7 @@ var passFlagToTest = map[string]bool{
|
||||
"parallel": true,
|
||||
"run": true,
|
||||
"short": true,
|
||||
"shuffle": true,
|
||||
"timeout": true,
|
||||
"trace": true,
|
||||
"v": true,
|
||||
|
@ -272,6 +272,13 @@ control the execution of any test:
|
||||
the Go tree can run a sanity check but not spend time running
|
||||
exhaustive tests.
|
||||
|
||||
-shuffle off,on,N
|
||||
Randomize the execution order of tests and benchmarks.
|
||||
It is off by default. If -shuffle is set to on, then it will seed
|
||||
the randomizer using the system clock. If -shuffle is set to an
|
||||
integer N, then N will be used as the seed value. In both cases,
|
||||
the seed will be reported for reproducibility.
|
||||
|
||||
-timeout d
|
||||
If a test binary runs longer than duration d, panic.
|
||||
If d is 0, the timeout is disabled.
|
||||
@ -480,6 +487,7 @@ var (
|
||||
testList string // -list flag
|
||||
testO string // -o flag
|
||||
testOutputDir = base.Cwd // -outputdir flag
|
||||
testShuffle shuffleFlag // -shuffle flag
|
||||
testTimeout time.Duration // -timeout flag
|
||||
testV bool // -v flag
|
||||
testVet = vetFlag{flags: defaultVetFlags} // -vet flag
|
||||
|
@ -10,6 +10,7 @@ import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@ -68,6 +69,7 @@ func init() {
|
||||
cf.DurationVar(&testTimeout, "timeout", 10*time.Minute, "")
|
||||
cf.StringVar(&testTrace, "trace", "", "")
|
||||
cf.BoolVar(&testV, "v", false, "")
|
||||
cf.Var(&testShuffle, "shuffle", "")
|
||||
|
||||
for name, _ := range passFlagToTest {
|
||||
cf.Var(cf.Lookup(name).Value, "test."+name, "")
|
||||
@ -194,6 +196,41 @@ func (f *vetFlag) Set(value string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
type shuffleFlag struct {
|
||||
on bool
|
||||
seed *int64
|
||||
}
|
||||
|
||||
func (f *shuffleFlag) String() string {
|
||||
if !f.on {
|
||||
return "off"
|
||||
}
|
||||
if f.seed == nil {
|
||||
return "on"
|
||||
}
|
||||
return fmt.Sprintf("%d", *f.seed)
|
||||
}
|
||||
|
||||
func (f *shuffleFlag) Set(value string) error {
|
||||
if value == "off" {
|
||||
*f = shuffleFlag{on: false}
|
||||
return nil
|
||||
}
|
||||
|
||||
if value == "on" {
|
||||
*f = shuffleFlag{on: true}
|
||||
return nil
|
||||
}
|
||||
|
||||
seed, err := strconv.ParseInt(value, 10, 64)
|
||||
if err != nil {
|
||||
return fmt.Errorf(`-shuffle argument must be "on", "off", or an int64: %v`, err)
|
||||
}
|
||||
|
||||
*f = shuffleFlag{on: true, seed: &seed}
|
||||
return nil
|
||||
}
|
||||
|
||||
// testFlags processes the command line, grabbing -x and -c, rewriting known flags
|
||||
// to have "test" before them, and reading the command line for the test binary.
|
||||
// Unfortunately for us, we need to do our own flag processing because go test
|
||||
|
148
src/cmd/go/testdata/script/test_shuffle.txt
vendored
Normal file
148
src/cmd/go/testdata/script/test_shuffle.txt
vendored
Normal file
@ -0,0 +1,148 @@
|
||||
# Shuffle order of tests and benchmarks
|
||||
|
||||
# Run tests
|
||||
go test -v foo_test.go
|
||||
! stdout '-test.shuffle '
|
||||
stdout '(?s)TestOne(.*)TestTwo(.*)TestThree'
|
||||
|
||||
go test -v -shuffle=off foo_test.go
|
||||
! stdout '-test.shuffle '
|
||||
stdout '(?s)TestOne(.*)TestTwo(.*)TestThree'
|
||||
|
||||
go test -v -shuffle=42 foo_test.go
|
||||
stdout '^-test.shuffle 42'
|
||||
stdout '(?s)TestThree(.*)TestOne(.*)TestTwo'
|
||||
|
||||
go test -v -shuffle=43 foo_test.go
|
||||
stdout '^-test.shuffle 43'
|
||||
stdout '(?s)TestThree(.*)TestTwo(.*)TestOne'
|
||||
|
||||
go test -v -shuffle=44 foo_test.go
|
||||
stdout '^-test.shuffle 44'
|
||||
stdout '(?s)TestOne(.*)TestThree(.*)TestTwo'
|
||||
|
||||
go test -v -shuffle=0 foo_test.go
|
||||
stdout '^-test.shuffle 0'
|
||||
stdout '(?s)TestTwo(.*)TestOne(.*)TestThree'
|
||||
|
||||
go test -v -shuffle -1 foo_test.go
|
||||
stdout '^-test.shuffle -1'
|
||||
stdout '(?s)TestThree(.*)TestOne(.*)TestTwo'
|
||||
|
||||
go test -v -shuffle=on foo_test.go
|
||||
stdout '^-test.shuffle '
|
||||
stdout '(?s)=== RUN TestOne(.*)--- PASS: TestOne'
|
||||
stdout '(?s)=== RUN TestTwo(.*)--- PASS: TestTwo'
|
||||
stdout '(?s)=== RUN TestThree(.*)--- PASS: TestThree'
|
||||
|
||||
|
||||
# Run tests and benchmarks
|
||||
go test -v -bench=. foo_test.go
|
||||
! stdout '-test.shuffle '
|
||||
stdout '(?s)TestOne(.*)TestTwo(.*)TestThree(.*)BenchmarkOne(.*)BenchmarkTwo(.*)BenchmarkThree'
|
||||
|
||||
go test -v -bench=. -shuffle=off foo_test.go
|
||||
! stdout '-test.shuffle '
|
||||
stdout '(?s)TestOne(.*)TestTwo(.*)TestThree(.*)BenchmarkOne(.*)BenchmarkTwo(.*)BenchmarkThree'
|
||||
|
||||
go test -v -bench=. -shuffle=42 foo_test.go
|
||||
stdout '^-test.shuffle 42'
|
||||
stdout '(?s)TestThree(.*)TestOne(.*)TestTwo(.*)BenchmarkThree(.*)BenchmarkOne(.*)BenchmarkTwo'
|
||||
|
||||
go test -v -bench=. -shuffle=43 foo_test.go
|
||||
stdout '^-test.shuffle 43'
|
||||
stdout '(?s)TestThree(.*)TestTwo(.*)TestOne(.*)BenchmarkThree(.*)BenchmarkOne(.*)BenchmarkTwo'
|
||||
|
||||
go test -v -bench=. -shuffle=44 foo_test.go
|
||||
stdout '^-test.shuffle 44'
|
||||
stdout '(?s)TestOne(.*)TestThree(.*)TestTwo(.*)BenchmarkTwo(.*)BenchmarkOne(.*)BenchmarkThree'
|
||||
|
||||
go test -v -bench=. -shuffle=0 foo_test.go
|
||||
stdout '^-test.shuffle 0'
|
||||
stdout '(?s)TestTwo(.*)TestOne(.*)TestThree(.*)BenchmarkThree(.*)BenchmarkOne(.*)BenchmarkTwo'
|
||||
|
||||
go test -v -bench=. -shuffle -1 foo_test.go
|
||||
stdout '^-test.shuffle -1'
|
||||
stdout '(?s)TestThree(.*)TestOne(.*)TestTwo(.*)BenchmarkOne(.*)BenchmarkThree(.*)BenchmarkTwo'
|
||||
|
||||
go test -v -bench=. -shuffle=on foo_test.go
|
||||
stdout '^-test.shuffle '
|
||||
stdout '(?s)=== RUN TestOne(.*)--- PASS: TestOne'
|
||||
stdout '(?s)=== RUN TestTwo(.*)--- PASS: TestTwo'
|
||||
stdout '(?s)=== RUN TestThree(.*)--- PASS: TestThree'
|
||||
stdout -count=2 'BenchmarkOne'
|
||||
stdout -count=2 'BenchmarkTwo'
|
||||
stdout -count=2 'BenchmarkThree'
|
||||
|
||||
|
||||
# When running go test -count=N, each of the N runs distinct runs should maintain the same
|
||||
# shuffled order of these tests.
|
||||
go test -v -shuffle=43 -count=4 foo_test.go
|
||||
stdout '^-test.shuffle 43'
|
||||
stdout '(?s)TestThree(.*)TestTwo(.*)TestOne(.*)TestThree(.*)TestTwo(.*)TestOne(.*)TestThree(.*)TestTwo(.*)TestOne(.*)TestThree(.*)TestTwo(.*)TestOne'
|
||||
|
||||
go test -v -bench=. -shuffle=44 -count=2 foo_test.go
|
||||
stdout '^-test.shuffle 44'
|
||||
stdout '(?s)TestOne(.*)TestThree(.*)TestTwo(.*)TestOne(.*)TestThree(.*)TestTwo(.*)BenchmarkTwo(.*)BenchmarkOne(.*)BenchmarkThree(.*)'
|
||||
|
||||
|
||||
# The feature should work with test binaries as well
|
||||
go test -c
|
||||
exec ./m.test -test.shuffle=off
|
||||
! stdout '^-test.shuffle '
|
||||
|
||||
exec ./m.test -test.shuffle=on
|
||||
stdout '^-test.shuffle '
|
||||
|
||||
exec ./m.test -test.v -test.bench=. -test.shuffle=0 foo_test.go
|
||||
stdout '^-test.shuffle 0'
|
||||
stdout '(?s)TestTwo(.*)TestOne(.*)TestThree(.*)BenchmarkThree(.*)BenchmarkOne(.*)BenchmarkTwo'
|
||||
|
||||
exec ./m.test -test.v -test.bench=. -test.shuffle=123 foo_test.go
|
||||
stdout '^-test.shuffle 123'
|
||||
stdout '(?s)TestThree(.*)TestOne(.*)TestTwo(.*)BenchmarkThree(.*)BenchmarkTwo(.*)BenchmarkOne'
|
||||
|
||||
exec ./m.test -test.v -test.bench=. -test.shuffle=-1 foo_test.go
|
||||
stdout '^-test.shuffle -1'
|
||||
stdout '(?s)TestThree(.*)TestOne(.*)TestTwo(.*)BenchmarkOne(.*)BenchmarkThree(.*)BenchmarkTwo'
|
||||
|
||||
exec ./m.test -test.v -test.bench=. -test.shuffle=44 -test.count=2 foo_test.go
|
||||
stdout '^-test.shuffle 44'
|
||||
stdout '(?s)TestOne(.*)TestThree(.*)TestTwo(.*)TestOne(.*)TestThree(.*)TestTwo(.*)BenchmarkTwo(.*)BenchmarkOne(.*)BenchmarkThree(.*)'
|
||||
|
||||
|
||||
# Negative testcases for invalid input
|
||||
! go test -shuffle -count=2
|
||||
stderr 'invalid value "-count=2" for flag -shuffle: -shuffle argument must be "on", "off", or an int64: strconv.ParseInt: parsing "-count=2": invalid syntax'
|
||||
|
||||
! go test -shuffle=
|
||||
stderr '(?s)invalid value "" for flag -shuffle: -shuffle argument must be "on", "off", or an int64: strconv.ParseInt: parsing "": invalid syntax'
|
||||
|
||||
! go test -shuffle=' '
|
||||
stderr '(?s)invalid value " " for flag -shuffle: -shuffle argument must be "on", "off", or an int64: strconv.ParseInt: parsing " ": invalid syntax'
|
||||
|
||||
! go test -shuffle=true
|
||||
stderr 'invalid value "true" for flag -shuffle: -shuffle argument must be "on", "off", or an int64: strconv.ParseInt: parsing "true": invalid syntax'
|
||||
|
||||
! go test -shuffle='abc'
|
||||
stderr 'invalid value "abc" for flag -shuffle: -shuffle argument must be "on", "off", or an int64: strconv.ParseInt: parsing "abc": invalid syntax'
|
||||
|
||||
-- go.mod --
|
||||
module m
|
||||
|
||||
go 1.16
|
||||
-- foo_test.go --
|
||||
package foo
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestOne(t *testing.T) {}
|
||||
func TestTwo(t *testing.T) {}
|
||||
func TestThree(t *testing.T) {}
|
||||
|
||||
func BenchmarkOne(b *testing.B) {}
|
||||
func BenchmarkTwo(b *testing.B) {}
|
||||
func BenchmarkThree(b *testing.B) {}
|
||||
|
||||
-- foo.go --
|
||||
package foo
|
@ -503,7 +503,7 @@ var depsRules = `
|
||||
FMT, flag, math/rand
|
||||
< testing/quick;
|
||||
|
||||
FMT, flag, runtime/debug, runtime/trace, internal/sysinfo
|
||||
FMT, flag, runtime/debug, runtime/trace, internal/sysinfo, math/rand
|
||||
< testing;
|
||||
|
||||
internal/testlog, runtime/pprof, regexp
|
||||
|
17
src/math/rand/export_test.go
Normal file
17
src/math/rand/export_test.go
Normal file
@ -0,0 +1,17 @@
|
||||
// Copyright 2021 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package rand
|
||||
|
||||
func Int31nForTest(r *Rand, n int32) int32 {
|
||||
return r.int31n(n)
|
||||
}
|
||||
|
||||
func GetNormalDistributionParameters() (float64, [128]uint32, [128]float32, [128]float32) {
|
||||
return rn, kn, wn, fn
|
||||
}
|
||||
|
||||
func GetExponentialDistributionParameters() (float64, [256]uint32, [256]float32, [256]float32) {
|
||||
return re, ke, we, fe
|
||||
}
|
@ -2,9 +2,10 @@
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package rand
|
||||
package rand_test
|
||||
|
||||
import (
|
||||
. "math/rand"
|
||||
"sync"
|
||||
"testing"
|
||||
)
|
||||
|
@ -2,7 +2,7 @@
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package rand
|
||||
package rand_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
@ -11,6 +11,7 @@ import (
|
||||
"internal/testenv"
|
||||
"io"
|
||||
"math"
|
||||
. "math/rand"
|
||||
"os"
|
||||
"runtime"
|
||||
"testing"
|
||||
@ -21,6 +22,9 @@ const (
|
||||
numTestSamples = 10000
|
||||
)
|
||||
|
||||
var rn, kn, wn, fn = GetNormalDistributionParameters()
|
||||
var re, ke, we, fe = GetExponentialDistributionParameters()
|
||||
|
||||
type statsResults struct {
|
||||
mean float64
|
||||
stddev float64
|
||||
@ -503,7 +507,7 @@ func TestUniformFactorial(t *testing.T) {
|
||||
fn func() int
|
||||
}{
|
||||
{name: "Int31n", fn: func() int { return int(r.Int31n(int32(nfact))) }},
|
||||
{name: "int31n", fn: func() int { return int(r.int31n(int32(nfact))) }},
|
||||
{name: "int31n", fn: func() int { return int(Int31nForTest(r, int32(nfact))) }},
|
||||
{name: "Perm", fn: func() int { return encodePerm(r.Perm(n)) }},
|
||||
{name: "Shuffle", fn: func() int {
|
||||
// Generate permutation using Shuffle.
|
||||
|
@ -242,6 +242,7 @@ import (
|
||||
"fmt"
|
||||
"internal/race"
|
||||
"io"
|
||||
"math/rand"
|
||||
"os"
|
||||
"runtime"
|
||||
"runtime/debug"
|
||||
@ -299,6 +300,7 @@ func Init() {
|
||||
cpuListStr = flag.String("test.cpu", "", "comma-separated `list` of cpu counts to run each test with")
|
||||
parallel = flag.Int("test.parallel", runtime.GOMAXPROCS(0), "run at most `n` tests in parallel")
|
||||
testlog = flag.String("test.testlogfile", "", "write test action log to `file` (for use only by cmd/go)")
|
||||
shuffle = flag.String("test.shuffle", "off", "randomize the execution order of tests and benchmarks")
|
||||
|
||||
initBenchmarkFlags()
|
||||
}
|
||||
@ -325,6 +327,7 @@ var (
|
||||
timeout *time.Duration
|
||||
cpuListStr *string
|
||||
parallel *int
|
||||
shuffle *string
|
||||
testlog *string
|
||||
|
||||
haveExamples bool // are there examples?
|
||||
@ -1456,6 +1459,25 @@ func (m *M) Run() (code int) {
|
||||
return
|
||||
}
|
||||
|
||||
if *shuffle != "off" {
|
||||
var n int64
|
||||
var err error
|
||||
if *shuffle == "on" {
|
||||
n = time.Now().UnixNano()
|
||||
} else {
|
||||
n, err = strconv.ParseInt(*shuffle, 10, 64)
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, `testing: -shuffle should be "off", "on", or a valid integer:`, err)
|
||||
m.exitCode = 2
|
||||
return
|
||||
}
|
||||
}
|
||||
fmt.Println("-test.shuffle", n)
|
||||
rng := rand.New(rand.NewSource(n))
|
||||
rng.Shuffle(len(m.tests), func(i, j int) { m.tests[i], m.tests[j] = m.tests[j], m.tests[i] })
|
||||
rng.Shuffle(len(m.benchmarks), func(i, j int) { m.benchmarks[i], m.benchmarks[j] = m.benchmarks[j], m.benchmarks[i] })
|
||||
}
|
||||
|
||||
parseCpuList()
|
||||
|
||||
m.before()
|
||||
|
Loading…
Reference in New Issue
Block a user