From baa928a7823c0f7edce5374b449fa9275f29411a Mon Sep 17 00:00:00 2001 From: Russ Cox Date: Mon, 21 Dec 2015 16:08:57 -0500 Subject: [PATCH] cmd/dist: run various one-off tests in parallel MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Takes 15% off my all.bash run time (after this and earlier CLs, now down to 3½ from 5½ minutes). For #10571. Change-Id: Iac316ffb730c9ff0a0faa7cc3b82ed4f7e6d4361 Reviewed-on: https://go-review.googlesource.com/18088 Reviewed-by: Ian Lance Taylor --- src/cmd/dist/test.go | 315 ++++++++++++++++++++++++++----------------- 1 file changed, 190 insertions(+), 125 deletions(-) diff --git a/src/cmd/dist/test.go b/src/cmd/dist/test.go index 9ac0161653..3f07c8a54a 100644 --- a/src/cmd/dist/test.go +++ b/src/cmd/dist/test.go @@ -41,15 +41,17 @@ func cmdtest() { // tester executes cmdtest. type tester struct { - race bool - listMode bool - rebuild bool - keepGoing bool - runRxStr string - runRx *regexp.Regexp - runRxWant bool // want runRx to match (true) or not match (false) - runNames []string // tests to run, exclusive with runRx; empty means all - banner string // prefix, or "" for none + race bool + listMode bool + rebuild bool + failed bool + keepGoing bool + runRxStr string + runRx *regexp.Regexp + runRxWant bool // want runRx to match (true) or not match (false) + runNames []string // tests to run, exclusive with runRx; empty means all + banner string // prefix, or "" for none + lastHeading string // last dir heading printed goroot string goarch string @@ -62,6 +64,17 @@ type tester struct { tests []distTest timeoutScale int + + worklist []*work +} + +type work struct { + dt *distTest + cmd *exec.Cmd + start chan bool + out []byte + err error + end chan bool } // A distTest is a test run by dist test. @@ -69,7 +82,7 @@ type tester struct { type distTest struct { name string // unique test name; may be filtered with -run flag heading string // group section; this header is printed before the test is run. - fn func() error + fn func(*distTest) error } func mustEnv(k string) string { @@ -175,22 +188,14 @@ func (t *tester) run() { } } - var lastHeading string - ok := true for _, dt := range t.tests { if !t.shouldRunTest(dt.name) { t.partial = true continue } - if dt.heading != "" && lastHeading != dt.heading { - lastHeading = dt.heading - t.out(dt.heading) - } - if vflag > 0 { - fmt.Printf("# go tool dist test -run=^%s$\n", dt.name) - } - if err := dt.fn(); err != nil { - ok = false + dt := dt // dt used in background after this iteration + if err := dt.fn(&dt); err != nil { + t.failed = true if t.keepGoing { log.Printf("Failed: %v", err) } else { @@ -198,7 +203,8 @@ func (t *tester) run() { } } } - if !ok { + t.runPending(nil) + if t.failed { fmt.Println("\nFAILED") os.Exit(1) } else if t.partial { @@ -256,10 +262,11 @@ func (t *tester) registerStdTest(pkg string) { t.tests = append(t.tests, distTest{ name: testName, heading: "Testing packages.", - fn: func() error { + fn: func(dt *distTest) error { if ranGoTest { return nil } + t.runPending(dt) ranGoTest = true args := []string{ "test", @@ -288,10 +295,11 @@ func (t *tester) registerRaceBenchTest(pkg string) { t.tests = append(t.tests, distTest{ name: testName, heading: "Running benchmarks briefly.", - fn: func() error { + fn: func(dt *distTest) error { if ranGoBench { return nil } + t.runPending(dt) ranGoBench = true args := []string{ "test", @@ -355,12 +363,12 @@ func (t *tester) registerTests() { t.tests = append(t.tests, distTest{ name: testName, heading: "GOMAXPROCS=2 runtime -cpu=1,2,4", - fn: func() error { - cmd := t.dirCmd("src", "go", "test", "-short", t.timeout(300), t.tags(), "runtime", "-cpu=1,2,4") + fn: func(dt *distTest) error { + cmd := t.addCmd(dt, "src", "go", "test", "-short", t.timeout(300), t.tags(), "runtime", "-cpu=1,2,4") // We set GOMAXPROCS=2 in addition to -cpu=1,2,4 in order to test runtime bootstrap code, // creation of first goroutines and first garbage collections in the parallel setting. cmd.Env = mergeEnvLists([]string{"GOMAXPROCS=2"}, os.Environ()) - return cmd.Run() + return nil }, }) @@ -393,8 +401,9 @@ func (t *tester) registerTests() { t.tests = append(t.tests, distTest{ name: "nolibgcc:" + pkg, heading: "Testing without libgcc.", - fn: func() error { - return t.dirCmd("src", "go", "test", "-short", "-ldflags=-linkmode=internal -libgcc=none", t.tags(), pkg, "-run="+run).Run() + fn: func(dt *distTest) error { + t.addCmd(dt, "src", "go", "test", "-short", "-ldflags=-linkmode=internal -libgcc=none", t.tags(), pkg, "-run="+run) + return nil }, }) } @@ -403,8 +412,9 @@ func (t *tester) registerTests() { t.tests = append(t.tests, distTest{ name: "sync_cpu", heading: "sync -cpu=10", - fn: func() error { - return t.dirCmd("src", "go", "test", "sync", "-short", t.timeout(120), t.tags(), "-cpu=10").Run() + fn: func(dt *distTest) error { + t.addCmd(dt, "src", "go", "test", "sync", "-short", t.timeout(120), t.tags(), "-cpu=10") + return nil }, }) @@ -413,17 +423,17 @@ func (t *tester) registerTests() { t.tests = append(t.tests, distTest{ name: "cgo_stdio", heading: "../misc/cgo/stdio", - fn: func() error { - return t.dirCmd("misc/cgo/stdio", - "go", "run", filepath.Join(os.Getenv("GOROOT"), "test/run.go"), "-", ".").Run() + fn: func(dt *distTest) error { + t.addCmd(dt, "misc/cgo/stdio", "go", "run", filepath.Join(os.Getenv("GOROOT"), "test/run.go"), "-", ".") + return nil }, }) t.tests = append(t.tests, distTest{ name: "cgo_life", heading: "../misc/cgo/life", - fn: func() error { - return t.dirCmd("misc/cgo/life", - "go", "run", filepath.Join(os.Getenv("GOROOT"), "test/run.go"), "-", ".").Run() + fn: func(dt *distTest) error { + t.addCmd(dt, "misc/cgo/life", "go", "run", filepath.Join(os.Getenv("GOROOT"), "test/run.go"), "-", ".") + return nil }, }) } @@ -455,15 +465,15 @@ func (t *tester) registerTests() { t.tests = append(t.tests, distTest{ name: "testso", heading: "../misc/cgo/testso", - fn: func() error { - return t.cgoTestSO("misc/cgo/testso") + fn: func(dt *distTest) error { + return t.cgoTestSO(dt, "misc/cgo/testso") }, }) t.tests = append(t.tests, distTest{ name: "testsovar", heading: "../misc/cgo/testsovar", - fn: func() error { - return t.cgoTestSO("misc/cgo/testsovar") + fn: func(dt *distTest) error { + return t.cgoTestSO(dt, "misc/cgo/testsovar") }, }) } @@ -504,7 +514,7 @@ func (t *tester) registerTests() { continue } } - t.registerTest("shootout:"+name, "../test/bench/shootout", "time", "./timing.sh", "-test", name) + t.registerSeqTest("shootout:"+name, "../test/bench/shootout", "time", "./timing.sh", "-test", name) } } if t.goos != "android" && !t.iOS() { @@ -517,7 +527,7 @@ func (t *tester) registerTests() { t.tests = append(t.tests, distTest{ name: fmt.Sprintf("test:%d_%d", shard, nShards), heading: "../test", - fn: func() error { return t.testDirTest(shard, nShards) }, + fn: func(dt *distTest) error { return t.testDirTest(dt, shard, nShards) }, }) } } @@ -525,8 +535,9 @@ func (t *tester) registerTests() { t.tests = append(t.tests, distTest{ name: "api", heading: "API check", - fn: func() error { - return t.dirCmd("src", "go", "run", filepath.Join(t.goroot, "src/cmd/api/run.go")).Run() + fn: func(dt *distTest) error { + t.addCmd(dt, "src", "go", "run", filepath.Join(t.goroot, "src/cmd/api/run.go")) + return nil }, }) } @@ -543,7 +554,7 @@ func (t *tester) isRegisteredTestName(testName string) bool { return false } -func (t *tester) registerTest(name, dirBanner, bin string, args ...string) { +func (t *tester) registerTest1(seq bool, name, dirBanner, bin string, args ...string) { if bin == "time" && !t.haveTime { bin, args = args[0], args[1:] } @@ -553,19 +564,37 @@ func (t *tester) registerTest(name, dirBanner, bin string, args ...string) { t.tests = append(t.tests, distTest{ name: name, heading: dirBanner, - fn: func() error { - return t.dirCmd(filepath.Join(t.goroot, "src", dirBanner), bin, args...).Run() + fn: func(dt *distTest) error { + if seq { + t.runPending(dt) + return t.dirCmd(filepath.Join(t.goroot, "src", dirBanner), bin, args...).Run() + } + t.addCmd(dt, filepath.Join(t.goroot, "src", dirBanner), bin, args...) + return nil }, }) } -func (t *tester) dirCmd(dir string, bin string, args ...string) *exec.Cmd { +func (t *tester) registerTest(name, dirBanner, bin string, args ...string) { + t.registerTest1(false, name, dirBanner, bin, args...) +} + +func (t *tester) registerSeqTest(name, dirBanner, bin string, args ...string) { + t.registerTest1(true, name, dirBanner, bin, args...) +} + +func (t *tester) bgDirCmd(dir, bin string, args ...string) *exec.Cmd { cmd := exec.Command(bin, args...) if filepath.IsAbs(dir) { cmd.Dir = dir } else { cmd.Dir = filepath.Join(t.goroot, dir) } + return cmd +} + +func (t *tester) dirCmd(dir, bin string, args ...string) *exec.Cmd { + cmd := t.bgDirCmd(dir, bin, args...) cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr if vflag > 1 { @@ -574,6 +603,15 @@ func (t *tester) dirCmd(dir string, bin string, args ...string) *exec.Cmd { return cmd } +func (t *tester) addCmd(dt *distTest, dir, bin string, args ...string) *exec.Cmd { + w := &work{ + dt: dt, + cmd: t.bgDirCmd(dir, bin, args...), + } + t.worklist = append(t.worklist, w) + return w.cmd +} + func (t *tester) iOS() bool { return t.goos == "darwin" && (t.goarch == "arm" || t.goarch == "arm64") } @@ -643,7 +681,7 @@ func (t *tester) supportedBuildmode(mode string) bool { } } -func (t *tester) cgoTest() error { +func (t *tester) cgoTest(dt *distTest) error { env := mergeEnvLists([]string{"GOTRACEBACK=2"}, os.Environ()) if t.goos == "android" || t.iOS() { @@ -652,19 +690,13 @@ func (t *tester) cgoTest() error { return cmd.Run() } - cmd := t.dirCmd("misc/cgo/test", "go", "test", t.tags(), "-ldflags", "-linkmode=auto") + cmd := t.addCmd(dt, "misc/cgo/test", "go", "test", t.tags(), "-ldflags", "-linkmode=auto") cmd.Env = env - if err := cmd.Run(); err != nil { - return err - } if t.gohostos != "dragonfly" { // linkmode=internal fails on dragonfly since errno is a TLS relocation. - cmd := t.dirCmd("misc/cgo/test", "go", "test", "-ldflags", "-linkmode=internal") + cmd := t.addCmd(dt, "misc/cgo/test", "go", "test", "-ldflags", "-linkmode=internal") cmd.Env = env - if err := cmd.Run(); err != nil { - return err - } } pair := t.gohostos + "-" + t.goarch @@ -676,37 +708,24 @@ func (t *tester) cgoTest() error { if !t.extLink() { break } - cmd := t.dirCmd("misc/cgo/test", "go", "test", "-ldflags", "-linkmode=external") + cmd := t.addCmd(dt, "misc/cgo/test", "go", "test", "-ldflags", "-linkmode=external") cmd.Env = env - if err := cmd.Run(); err != nil { - return err - } - cmd = t.dirCmd("misc/cgo/test", "go", "test", "-ldflags", "-linkmode=external -s") + cmd = t.addCmd(dt, "misc/cgo/test", "go", "test", "-ldflags", "-linkmode=external -s") cmd.Env = env - if err := cmd.Run(); err != nil { - return err - } case "android-arm", "dragonfly-386", "dragonfly-amd64", "freebsd-386", "freebsd-amd64", "freebsd-arm", "linux-386", "linux-amd64", "linux-arm", "netbsd-386", "netbsd-amd64": - cmd := t.dirCmd("misc/cgo/test", "go", "test", "-ldflags", "-linkmode=external") + cmd := t.addCmd(dt, "misc/cgo/test", "go", "test", "-ldflags", "-linkmode=external") cmd.Env = env - if err := cmd.Run(); err != nil { - return err - } - cmd = t.dirCmd("misc/cgo/testtls", "go", "test", "-ldflags", "-linkmode=auto") + + cmd = t.addCmd(dt, "misc/cgo/testtls", "go", "test", "-ldflags", "-linkmode=auto") cmd.Env = env - if err := cmd.Run(); err != nil { - return err - } - cmd = t.dirCmd("misc/cgo/testtls", "go", "test", "-ldflags", "-linkmode=external") + + cmd = t.addCmd(dt, "misc/cgo/testtls", "go", "test", "-ldflags", "-linkmode=external") cmd.Env = env - if err := cmd.Run(); err != nil { - return err - } switch pair { case "netbsd-386", "netbsd-amd64": @@ -726,29 +745,17 @@ func (t *tester) cgoTest() error { if err := cmd.Run(); err != nil { fmt.Println("No support for static linking found (lacks libc.a?), skip cgo static linking test.") } else { - cmd = t.dirCmd("misc/cgo/testtls", "go", "test", "-ldflags", `-linkmode=external -extldflags "-static -pthread"`) + cmd = t.addCmd(dt, "misc/cgo/testtls", "go", "test", "-ldflags", `-linkmode=external -extldflags "-static -pthread"`) cmd.Env = env - if err := cmd.Run(); err != nil { - return err - } - cmd = t.dirCmd("misc/cgo/nocgo", "go", "test") + cmd = t.addCmd(dt, "misc/cgo/nocgo", "go", "test") cmd.Env = env - if err := cmd.Run(); err != nil { - return err - } - cmd = t.dirCmd("misc/cgo/nocgo", "go", "test", "-ldflags", `-linkmode=external`) + cmd = t.addCmd(dt, "misc/cgo/nocgo", "go", "test", "-ldflags", `-linkmode=external`) cmd.Env = env - if err := cmd.Run(); err != nil { - return err - } - cmd = t.dirCmd("misc/cgo/nocgo", "go", "test", "-ldflags", `-linkmode=external -extldflags "-static -pthread"`) + cmd = t.addCmd(dt, "misc/cgo/nocgo", "go", "test", "-ldflags", `-linkmode=external -extldflags "-static -pthread"`) cmd.Env = env - if err := cmd.Run(); err != nil { - return err - } } if pair != "freebsd-amd64" { // clang -pie fails to link misc/cgo/test @@ -759,21 +766,15 @@ func (t *tester) cgoTest() error { if err := cmd.Run(); err != nil { fmt.Println("No support for -pie found, skip cgo PIE test.") } else { - cmd = t.dirCmd("misc/cgo/test", "go", "test", "-ldflags", `-linkmode=external -extldflags "-pie"`) + cmd = t.addCmd(dt, "misc/cgo/test", "go", "test", "-ldflags", `-linkmode=external -extldflags "-pie"`) cmd.Env = env - if err := cmd.Run(); err != nil { - return fmt.Errorf("pie cgo/test: %v", err) - } - cmd = t.dirCmd("misc/cgo/testtls", "go", "test", "-ldflags", `-linkmode=external -extldflags "-pie"`) + + cmd = t.addCmd(dt, "misc/cgo/testtls", "go", "test", "-ldflags", `-linkmode=external -extldflags "-pie"`) cmd.Env = env - if err := cmd.Run(); err != nil { - return fmt.Errorf("pie cgo/testtls: %v", err) - } - cmd = t.dirCmd("misc/cgo/nocgo", "go", "test", "-ldflags", `-linkmode=external -extldflags "-pie"`) + + cmd = t.addCmd(dt, "misc/cgo/nocgo", "go", "test", "-ldflags", `-linkmode=external -extldflags "-pie"`) cmd.Env = env - if err := cmd.Run(); err != nil { - return fmt.Errorf("pie cgo/nocgo: %v", err) - } + } } } @@ -782,6 +783,78 @@ func (t *tester) cgoTest() error { return nil } +// run pending test commands, in parallel, emitting headers as appropriate. +// When finished, emit header for dt, which is going to run after the +// pending commands are done (and runPending returns). +func (t *tester) runPending(nextTest *distTest) { + worklist := t.worklist + t.worklist = nil + for _, w := range worklist { + w.start = make(chan bool) + w.end = make(chan bool) + go func(w *work) { + if !<-w.start { + w.out = []byte(fmt.Sprintf("skipped due to earlier error\n")) + } else { + w.out, w.err = w.cmd.CombinedOutput() + } + w.end <- true + }(w) + } + + started := 0 + ended := 0 + var last *distTest + for ended < len(worklist) { + for started < len(worklist) && started-ended < maxbg { + //println("start", started) + w := worklist[started] + started++ + w.start <- !t.failed || t.keepGoing + } + w := worklist[ended] + dt := w.dt + if dt.heading != "" && t.lastHeading != dt.heading { + t.lastHeading = dt.heading + t.out(dt.heading) + } + if dt != last { + // Assumes all the entries for a single dt are in one worklist. + last = w.dt + if vflag > 0 { + fmt.Printf("# go tool dist test -run=^%s$\n", dt.name) + } + } + if vflag > 1 { + errprintf("%s\n", strings.Join(w.cmd.Args, " ")) + } + //println("wait", ended) + ended++ + <-w.end + os.Stdout.Write(w.out) + if w.err != nil { + log.Printf("Failed: %v", w.err) + t.failed = true + if !t.keepGoing { + break + } + } + } + if t.failed && !t.keepGoing { + log.Fatal("FAILED") + } + + if dt := nextTest; dt != nil { + if dt.heading != "" && t.lastHeading != dt.heading { + t.lastHeading = dt.heading + t.out(dt.heading) + } + if vflag > 0 { + fmt.Printf("# go tool dist test -run=^%s$\n", dt.name) + } + } +} + func (t *tester) cgoTestSOSupported() bool { if t.goos == "android" || t.iOS() { // No exec facility on Android or iOS. @@ -798,7 +871,9 @@ func (t *tester) cgoTestSOSupported() bool { return true } -func (t *tester) cgoTestSO(testpath string) error { +func (t *tester) cgoTestSO(dt *distTest, testpath string) error { + t.runPending(dt) + dir := filepath.Join(t.goroot, testpath) // build shared object @@ -866,34 +941,24 @@ func (t *tester) raceDetectorSupported() bool { return false } -func (t *tester) raceTest() error { - if err := t.dirCmd("src", "go", "test", "-race", "-i", "runtime/race", "flag", "os/exec").Run(); err != nil { - return err - } - if err := t.dirCmd("src", "go", "test", "-race", "-run=Output", "runtime/race").Run(); err != nil { - return err - } - if err := t.dirCmd("src", "go", "test", "-race", "-short", "-run=TestParse|TestEcho", "flag", "os/exec").Run(); err != nil { - return err - } +func (t *tester) raceTest(dt *distTest) error { + t.addCmd(dt, "src", "go", "test", "-race", "-i", "runtime/race", "flag", "os/exec") + t.addCmd(dt, "src", "go", "test", "-race", "-run=Output", "runtime/race") + t.addCmd(dt, "src", "go", "test", "-race", "-short", "-run=TestParse|TestEcho", "flag", "os/exec") if t.cgoEnabled { env := mergeEnvLists([]string{"GOTRACEBACK=2"}, os.Environ()) - cmd := t.dirCmd("misc/cgo/test", "go", "test", "-race", "-short") + cmd := t.addCmd(dt, "misc/cgo/test", "go", "test", "-race", "-short") cmd.Env = env - if err := cmd.Run(); err != nil { - return err - } } if t.extLink() { // Test with external linking; see issue 9133. - if err := t.dirCmd("src", "go", "test", "-race", "-short", "-ldflags=-linkmode=external", "-run=TestParse|TestEcho", "flag", "os/exec").Run(); err != nil { - return err - } + t.addCmd(dt, "src", "go", "test", "-race", "-short", "-ldflags=-linkmode=external", "-run=TestParse|TestEcho", "flag", "os/exec") } return nil } -func (t *tester) testDirTest(shard, shards int) error { +func (t *tester) testDirTest(dt *distTest, shard, shards int) error { + t.runPending(dt) const runExe = "runtest.exe" // named exe for Windows, but harmless elsewhere cmd := t.dirCmd("test", "go", "build", "-o", runExe, "run.go") cmd.Env = mergeEnvLists([]string{"GOOS=" + t.gohostos, "GOARCH=" + t.gohostarch, "GOMAXPROCS="}, os.Environ())