cmd/dist: run various one-off tests in parallel

Takes 15% off my all.bash run time
(after this and earlier CLs, now down to 3½ from 5½ minutes).

For #10571.

Change-Id: Iac316ffb730c9ff0a0faa7cc3b82ed4f7e6d4361
Reviewed-on: https://go-review.googlesource.com/18088
Reviewed-by: Ian Lance Taylor <iant@golang.org>

src/cmd/dist/test.go

@@ -44,12 +44,14 @@ type tester struct {
 	race        bool
 	listMode    bool
 	rebuild     bool
+	failed      bool
 	keepGoing   bool
 	runRxStr    string
 	runRx       *regexp.Regexp
 	runRxWant   bool     // want runRx to match (true) or not match (false)
 	runNames    []string // tests to run, exclusive with runRx; empty means all
 	banner      string   // prefix, or "" for none
+	lastHeading string   // last dir heading printed
 
 	goroot     string
 	goarch     string
@@ -62,6 +64,17 @@ type tester struct {
 
 	tests        []distTest
 	timeoutScale int
+
+	worklist []*work
+}
+
+type work struct {
+	dt    *distTest
+	cmd   *exec.Cmd
+	start chan bool
+	out   []byte
+	err   error
+	end   chan bool
 }
 
 // A distTest is a test run by dist test.
@@ -69,7 +82,7 @@ type tester struct {
 type distTest struct {
 	name    string // unique test name; may be filtered with -run flag
 	heading string // group section; this header is printed before the test is run.
-	fn      func() error
+	fn      func(*distTest) error
 }
 
 func mustEnv(k string) string {
@@ -175,22 +188,14 @@ func (t *tester) run() {
 		}
 	}
 
-	var lastHeading string
-	ok := true
 	for _, dt := range t.tests {
 		if !t.shouldRunTest(dt.name) {
 			t.partial = true
 			continue
 		}
-		if dt.heading != "" && lastHeading != dt.heading {
+		dt := dt // dt used in background after this iteration
-			lastHeading = dt.heading
+		if err := dt.fn(&dt); err != nil {
-			t.out(dt.heading)
+			t.failed = true
-		}
-		if vflag > 0 {
-			fmt.Printf("# go tool dist test -run=^%s$\n", dt.name)
-		}
-		if err := dt.fn(); err != nil {
-			ok = false
 			if t.keepGoing {
 				log.Printf("Failed: %v", err)
 			} else {
@@ -198,7 +203,8 @@ func (t *tester) run() {
 			}
 		}
 	}
-	if !ok {
+	t.runPending(nil)
+	if t.failed {
 		fmt.Println("\nFAILED")
 		os.Exit(1)
 	} else if t.partial {
@@ -256,10 +262,11 @@ func (t *tester) registerStdTest(pkg string) {
 	t.tests = append(t.tests, distTest{
 		name:    testName,
 		heading: "Testing packages.",
-		fn: func() error {
+		fn: func(dt *distTest) error {
 			if ranGoTest {
 				return nil
 			}
+			t.runPending(dt)
 			ranGoTest = true
 			args := []string{
 				"test",
@@ -288,10 +295,11 @@ func (t *tester) registerRaceBenchTest(pkg string) {
 	t.tests = append(t.tests, distTest{
 		name:    testName,
 		heading: "Running benchmarks briefly.",
-		fn: func() error {
+		fn: func(dt *distTest) error {
 			if ranGoBench {
 				return nil
 			}
+			t.runPending(dt)
 			ranGoBench = true
 			args := []string{
 				"test",
@@ -355,12 +363,12 @@ func (t *tester) registerTests() {
 	t.tests = append(t.tests, distTest{
 		name:    testName,
 		heading: "GOMAXPROCS=2 runtime -cpu=1,2,4",
-		fn: func() error {
+		fn: func(dt *distTest) error {
-			cmd := t.dirCmd("src", "go", "test", "-short", t.timeout(300), t.tags(), "runtime", "-cpu=1,2,4")
+			cmd := t.addCmd(dt, "src", "go", "test", "-short", t.timeout(300), t.tags(), "runtime", "-cpu=1,2,4")
 			// We set GOMAXPROCS=2 in addition to -cpu=1,2,4 in order to test runtime bootstrap code,
 			// creation of first goroutines and first garbage collections in the parallel setting.
 			cmd.Env = mergeEnvLists([]string{"GOMAXPROCS=2"}, os.Environ())
-			return cmd.Run()
+			return nil
 		},
 	})
 
@@ -393,8 +401,9 @@ func (t *tester) registerTests() {
 		t.tests = append(t.tests, distTest{
 			name:    "nolibgcc:" + pkg,
 			heading: "Testing without libgcc.",
-			fn: func() error {
+			fn: func(dt *distTest) error {
-				return t.dirCmd("src", "go", "test", "-short", "-ldflags=-linkmode=internal -libgcc=none", t.tags(), pkg, "-run="+run).Run()
+				t.addCmd(dt, "src", "go", "test", "-short", "-ldflags=-linkmode=internal -libgcc=none", t.tags(), pkg, "-run="+run)
+				return nil
 			},
 		})
 	}
@@ -403,8 +412,9 @@ func (t *tester) registerTests() {
 	t.tests = append(t.tests, distTest{
 		name:    "sync_cpu",
 		heading: "sync -cpu=10",
-		fn: func() error {
+		fn: func(dt *distTest) error {
-			return t.dirCmd("src", "go", "test", "sync", "-short", t.timeout(120), t.tags(), "-cpu=10").Run()
+			t.addCmd(dt, "src", "go", "test", "sync", "-short", t.timeout(120), t.tags(), "-cpu=10")
+			return nil
 		},
 	})
 
@@ -413,17 +423,17 @@ func (t *tester) registerTests() {
 		t.tests = append(t.tests, distTest{
 			name:    "cgo_stdio",
 			heading: "../misc/cgo/stdio",
-			fn: func() error {
+			fn: func(dt *distTest) error {
-				return t.dirCmd("misc/cgo/stdio",
+				t.addCmd(dt, "misc/cgo/stdio", "go", "run", filepath.Join(os.Getenv("GOROOT"), "test/run.go"), "-", ".")
-					"go", "run", filepath.Join(os.Getenv("GOROOT"), "test/run.go"), "-", ".").Run()
+				return nil
 			},
 		})
 		t.tests = append(t.tests, distTest{
 			name:    "cgo_life",
 			heading: "../misc/cgo/life",
-			fn: func() error {
+			fn: func(dt *distTest) error {
-				return t.dirCmd("misc/cgo/life",
+				t.addCmd(dt, "misc/cgo/life", "go", "run", filepath.Join(os.Getenv("GOROOT"), "test/run.go"), "-", ".")
-					"go", "run", filepath.Join(os.Getenv("GOROOT"), "test/run.go"), "-", ".").Run()
+				return nil
 			},
 		})
 	}
@@ -455,15 +465,15 @@ func (t *tester) registerTests() {
 			t.tests = append(t.tests, distTest{
 				name:    "testso",
 				heading: "../misc/cgo/testso",
-				fn: func() error {
+				fn: func(dt *distTest) error {
-					return t.cgoTestSO("misc/cgo/testso")
+					return t.cgoTestSO(dt, "misc/cgo/testso")
 				},
 			})
 			t.tests = append(t.tests, distTest{
 				name:    "testsovar",
 				heading: "../misc/cgo/testsovar",
-				fn: func() error {
+				fn: func(dt *distTest) error {
-					return t.cgoTestSO("misc/cgo/testsovar")
+					return t.cgoTestSO(dt, "misc/cgo/testsovar")
 				},
 			})
 		}
@@ -504,7 +514,7 @@ func (t *tester) registerTests() {
 					continue
 				}
 			}
-			t.registerTest("shootout:"+name, "../test/bench/shootout", "time", "./timing.sh", "-test", name)
+			t.registerSeqTest("shootout:"+name, "../test/bench/shootout", "time", "./timing.sh", "-test", name)
 		}
 	}
 	if t.goos != "android" && !t.iOS() {
@@ -517,7 +527,7 @@ func (t *tester) registerTests() {
 			t.tests = append(t.tests, distTest{
 				name:    fmt.Sprintf("test:%d_%d", shard, nShards),
 				heading: "../test",
-				fn:      func() error { return t.testDirTest(shard, nShards) },
+				fn:      func(dt *distTest) error { return t.testDirTest(dt, shard, nShards) },
 			})
 		}
 	}
@@ -525,8 +535,9 @@ func (t *tester) registerTests() {
 		t.tests = append(t.tests, distTest{
 			name:    "api",
 			heading: "API check",
-			fn: func() error {
+			fn: func(dt *distTest) error {
-				return t.dirCmd("src", "go", "run", filepath.Join(t.goroot, "src/cmd/api/run.go")).Run()
+				t.addCmd(dt, "src", "go", "run", filepath.Join(t.goroot, "src/cmd/api/run.go"))
+				return nil
 			},
 		})
 	}
@@ -543,7 +554,7 @@ func (t *tester) isRegisteredTestName(testName string) bool {
 	return false
 }
 
-func (t *tester) registerTest(name, dirBanner, bin string, args ...string) {
+func (t *tester) registerTest1(seq bool, name, dirBanner, bin string, args ...string) {
 	if bin == "time" && !t.haveTime {
 		bin, args = args[0], args[1:]
 	}
@@ -553,19 +564,37 @@ func (t *tester) registerTest(name, dirBanner, bin string, args ...string) {
 	t.tests = append(t.tests, distTest{
 		name:    name,
 		heading: dirBanner,
-		fn: func() error {
+		fn: func(dt *distTest) error {
+			if seq {
+				t.runPending(dt)
 				return t.dirCmd(filepath.Join(t.goroot, "src", dirBanner), bin, args...).Run()
+			}
+			t.addCmd(dt, filepath.Join(t.goroot, "src", dirBanner), bin, args...)
+			return nil
 		},
 	})
 }
 
-func (t *tester) dirCmd(dir string, bin string, args ...string) *exec.Cmd {
+func (t *tester) registerTest(name, dirBanner, bin string, args ...string) {
+	t.registerTest1(false, name, dirBanner, bin, args...)
+}
+
+func (t *tester) registerSeqTest(name, dirBanner, bin string, args ...string) {
+	t.registerTest1(true, name, dirBanner, bin, args...)
+}
+
+func (t *tester) bgDirCmd(dir, bin string, args ...string) *exec.Cmd {
 	cmd := exec.Command(bin, args...)
 	if filepath.IsAbs(dir) {
 		cmd.Dir = dir
 	} else {
 		cmd.Dir = filepath.Join(t.goroot, dir)
 	}
+	return cmd
+}
+
+func (t *tester) dirCmd(dir, bin string, args ...string) *exec.Cmd {
+	cmd := t.bgDirCmd(dir, bin, args...)
 	cmd.Stdout = os.Stdout
 	cmd.Stderr = os.Stderr
 	if vflag > 1 {
@@ -574,6 +603,15 @@ func (t *tester) dirCmd(dir string, bin string, args ...string) *exec.Cmd {
 	return cmd
 }
 
+func (t *tester) addCmd(dt *distTest, dir, bin string, args ...string) *exec.Cmd {
+	w := &work{
+		dt:  dt,
+		cmd: t.bgDirCmd(dir, bin, args...),
+	}
+	t.worklist = append(t.worklist, w)
+	return w.cmd
+}
+
 func (t *tester) iOS() bool {
 	return t.goos == "darwin" && (t.goarch == "arm" || t.goarch == "arm64")
 }
@@ -643,7 +681,7 @@ func (t *tester) supportedBuildmode(mode string) bool {
 	}
 }
 
-func (t *tester) cgoTest() error {
+func (t *tester) cgoTest(dt *distTest) error {
 	env := mergeEnvLists([]string{"GOTRACEBACK=2"}, os.Environ())
 
 	if t.goos == "android" || t.iOS() {
@@ -652,19 +690,13 @@ func (t *tester) cgoTest() error {
 		return cmd.Run()
 	}
 
-	cmd := t.dirCmd("misc/cgo/test", "go", "test", t.tags(), "-ldflags", "-linkmode=auto")
+	cmd := t.addCmd(dt, "misc/cgo/test", "go", "test", t.tags(), "-ldflags", "-linkmode=auto")
 	cmd.Env = env
-	if err := cmd.Run(); err != nil {
-		return err
-	}
 
 	if t.gohostos != "dragonfly" {
 		// linkmode=internal fails on dragonfly since errno is a TLS relocation.
-		cmd := t.dirCmd("misc/cgo/test", "go", "test", "-ldflags", "-linkmode=internal")
+		cmd := t.addCmd(dt, "misc/cgo/test", "go", "test", "-ldflags", "-linkmode=internal")
 		cmd.Env = env
-		if err := cmd.Run(); err != nil {
-			return err
-		}
 	}
 
 	pair := t.gohostos + "-" + t.goarch
@@ -676,37 +708,24 @@ func (t *tester) cgoTest() error {
 		if !t.extLink() {
 			break
 		}
-		cmd := t.dirCmd("misc/cgo/test", "go", "test", "-ldflags", "-linkmode=external")
+		cmd := t.addCmd(dt, "misc/cgo/test", "go", "test", "-ldflags", "-linkmode=external")
 		cmd.Env = env
-		if err := cmd.Run(); err != nil {
+		cmd = t.addCmd(dt, "misc/cgo/test", "go", "test", "-ldflags", "-linkmode=external -s")
-			return err
-		}
-		cmd = t.dirCmd("misc/cgo/test", "go", "test", "-ldflags", "-linkmode=external -s")
 		cmd.Env = env
-		if err := cmd.Run(); err != nil {
-			return err
-		}
 	case "android-arm",
 		"dragonfly-386", "dragonfly-amd64",
 		"freebsd-386", "freebsd-amd64", "freebsd-arm",
 		"linux-386", "linux-amd64", "linux-arm",
 		"netbsd-386", "netbsd-amd64":
 
-		cmd := t.dirCmd("misc/cgo/test", "go", "test", "-ldflags", "-linkmode=external")
+		cmd := t.addCmd(dt, "misc/cgo/test", "go", "test", "-ldflags", "-linkmode=external")
 		cmd.Env = env
-		if err := cmd.Run(); err != nil {
+
-			return err
+		cmd = t.addCmd(dt, "misc/cgo/testtls", "go", "test", "-ldflags", "-linkmode=auto")
-		}
-		cmd = t.dirCmd("misc/cgo/testtls", "go", "test", "-ldflags", "-linkmode=auto")
 		cmd.Env = env
-		if err := cmd.Run(); err != nil {
+
-			return err
+		cmd = t.addCmd(dt, "misc/cgo/testtls", "go", "test", "-ldflags", "-linkmode=external")
-		}
-		cmd = t.dirCmd("misc/cgo/testtls", "go", "test", "-ldflags", "-linkmode=external")
 		cmd.Env = env
-		if err := cmd.Run(); err != nil {
-			return err
-		}
 
 		switch pair {
 		case "netbsd-386", "netbsd-amd64":
@@ -726,29 +745,17 @@ func (t *tester) cgoTest() error {
 			if err := cmd.Run(); err != nil {
 				fmt.Println("No support for static linking found (lacks libc.a?), skip cgo static linking test.")
 			} else {
-				cmd = t.dirCmd("misc/cgo/testtls", "go", "test", "-ldflags", `-linkmode=external -extldflags "-static -pthread"`)
+				cmd = t.addCmd(dt, "misc/cgo/testtls", "go", "test", "-ldflags", `-linkmode=external -extldflags "-static -pthread"`)
 				cmd.Env = env
-				if err := cmd.Run(); err != nil {
-					return err
-				}
 
-				cmd = t.dirCmd("misc/cgo/nocgo", "go", "test")
+				cmd = t.addCmd(dt, "misc/cgo/nocgo", "go", "test")
 				cmd.Env = env
-				if err := cmd.Run(); err != nil {
-					return err
-				}
 
-				cmd = t.dirCmd("misc/cgo/nocgo", "go", "test", "-ldflags", `-linkmode=external`)
+				cmd = t.addCmd(dt, "misc/cgo/nocgo", "go", "test", "-ldflags", `-linkmode=external`)
 				cmd.Env = env
-				if err := cmd.Run(); err != nil {
-					return err
-				}
 
-				cmd = t.dirCmd("misc/cgo/nocgo", "go", "test", "-ldflags", `-linkmode=external -extldflags "-static -pthread"`)
+				cmd = t.addCmd(dt, "misc/cgo/nocgo", "go", "test", "-ldflags", `-linkmode=external -extldflags "-static -pthread"`)
 				cmd.Env = env
-				if err := cmd.Run(); err != nil {
-					return err
-				}
 			}
 
 			if pair != "freebsd-amd64" { // clang -pie fails to link misc/cgo/test
@@ -759,21 +766,15 @@ func (t *tester) cgoTest() error {
 				if err := cmd.Run(); err != nil {
 					fmt.Println("No support for -pie found, skip cgo PIE test.")
 				} else {
-					cmd = t.dirCmd("misc/cgo/test", "go", "test", "-ldflags", `-linkmode=external -extldflags "-pie"`)
+					cmd = t.addCmd(dt, "misc/cgo/test", "go", "test", "-ldflags", `-linkmode=external -extldflags "-pie"`)
 					cmd.Env = env
-					if err := cmd.Run(); err != nil {
+
-						return fmt.Errorf("pie cgo/test: %v", err)
+					cmd = t.addCmd(dt, "misc/cgo/testtls", "go", "test", "-ldflags", `-linkmode=external -extldflags "-pie"`)
-					}
-					cmd = t.dirCmd("misc/cgo/testtls", "go", "test", "-ldflags", `-linkmode=external -extldflags "-pie"`)
 					cmd.Env = env
-					if err := cmd.Run(); err != nil {
+
-						return fmt.Errorf("pie cgo/testtls: %v", err)
+					cmd = t.addCmd(dt, "misc/cgo/nocgo", "go", "test", "-ldflags", `-linkmode=external -extldflags "-pie"`)
-					}
-					cmd = t.dirCmd("misc/cgo/nocgo", "go", "test", "-ldflags", `-linkmode=external -extldflags "-pie"`)
 					cmd.Env = env
-					if err := cmd.Run(); err != nil {
+
-						return fmt.Errorf("pie cgo/nocgo: %v", err)
-					}
 				}
 			}
 		}
@@ -782,6 +783,78 @@ func (t *tester) cgoTest() error {
 	return nil
 }
 
+// run pending test commands, in parallel, emitting headers as appropriate.
+// When finished, emit header for dt, which is going to run after the
+// pending commands are done (and runPending returns).
+func (t *tester) runPending(nextTest *distTest) {
+	worklist := t.worklist
+	t.worklist = nil
+	for _, w := range worklist {
+		w.start = make(chan bool)
+		w.end = make(chan bool)
+		go func(w *work) {
+			if !<-w.start {
+				w.out = []byte(fmt.Sprintf("skipped due to earlier error\n"))
+			} else {
+				w.out, w.err = w.cmd.CombinedOutput()
+			}
+			w.end <- true
+		}(w)
+	}
+
+	started := 0
+	ended := 0
+	var last *distTest
+	for ended < len(worklist) {
+		for started < len(worklist) && started-ended < maxbg {
+			//println("start", started)
+			w := worklist[started]
+			started++
+			w.start <- !t.failed || t.keepGoing
+		}
+		w := worklist[ended]
+		dt := w.dt
+		if dt.heading != "" && t.lastHeading != dt.heading {
+			t.lastHeading = dt.heading
+			t.out(dt.heading)
+		}
+		if dt != last {
+			// Assumes all the entries for a single dt are in one worklist.
+			last = w.dt
+			if vflag > 0 {
+				fmt.Printf("# go tool dist test -run=^%s$\n", dt.name)
+			}
+		}
+		if vflag > 1 {
+			errprintf("%s\n", strings.Join(w.cmd.Args, " "))
+		}
+		//println("wait", ended)
+		ended++
+		<-w.end
+		os.Stdout.Write(w.out)
+		if w.err != nil {
+			log.Printf("Failed: %v", w.err)
+			t.failed = true
+			if !t.keepGoing {
+				break
+			}
+		}
+	}
+	if t.failed && !t.keepGoing {
+		log.Fatal("FAILED")
+	}
+
+	if dt := nextTest; dt != nil {
+		if dt.heading != "" && t.lastHeading != dt.heading {
+			t.lastHeading = dt.heading
+			t.out(dt.heading)
+		}
+		if vflag > 0 {
+			fmt.Printf("# go tool dist test -run=^%s$\n", dt.name)
+		}
+	}
+}
+
 func (t *tester) cgoTestSOSupported() bool {
 	if t.goos == "android" || t.iOS() {
 		// No exec facility on Android or iOS.
@@ -798,7 +871,9 @@ func (t *tester) cgoTestSOSupported() bool {
 	return true
 }
 
-func (t *tester) cgoTestSO(testpath string) error {
+func (t *tester) cgoTestSO(dt *distTest, testpath string) error {
+	t.runPending(dt)
+
 	dir := filepath.Join(t.goroot, testpath)
 
 	// build shared object
@@ -866,34 +941,24 @@ func (t *tester) raceDetectorSupported() bool {
 	return false
 }
 
-func (t *tester) raceTest() error {
+func (t *tester) raceTest(dt *distTest) error {
-	if err := t.dirCmd("src", "go", "test", "-race", "-i", "runtime/race", "flag", "os/exec").Run(); err != nil {
+	t.addCmd(dt, "src", "go", "test", "-race", "-i", "runtime/race", "flag", "os/exec")
-		return err
+	t.addCmd(dt, "src", "go", "test", "-race", "-run=Output", "runtime/race")
-	}
+	t.addCmd(dt, "src", "go", "test", "-race", "-short", "-run=TestParse|TestEcho", "flag", "os/exec")
-	if err := t.dirCmd("src", "go", "test", "-race", "-run=Output", "runtime/race").Run(); err != nil {
-		return err
-	}
-	if err := t.dirCmd("src", "go", "test", "-race", "-short", "-run=TestParse|TestEcho", "flag", "os/exec").Run(); err != nil {
-		return err
-	}
 	if t.cgoEnabled {
 		env := mergeEnvLists([]string{"GOTRACEBACK=2"}, os.Environ())
-		cmd := t.dirCmd("misc/cgo/test", "go", "test", "-race", "-short")
+		cmd := t.addCmd(dt, "misc/cgo/test", "go", "test", "-race", "-short")
 		cmd.Env = env
-		if err := cmd.Run(); err != nil {
-			return err
-		}
 	}
 	if t.extLink() {
 		// Test with external linking; see issue 9133.
-		if err := t.dirCmd("src", "go", "test", "-race", "-short", "-ldflags=-linkmode=external", "-run=TestParse|TestEcho", "flag", "os/exec").Run(); err != nil {
+		t.addCmd(dt, "src", "go", "test", "-race", "-short", "-ldflags=-linkmode=external", "-run=TestParse|TestEcho", "flag", "os/exec")
-			return err
-		}
 	}
 	return nil
 }
 
-func (t *tester) testDirTest(shard, shards int) error {
+func (t *tester) testDirTest(dt *distTest, shard, shards int) error {
+	t.runPending(dt)
 	const runExe = "runtest.exe" // named exe for Windows, but harmless elsewhere
 	cmd := t.dirCmd("test", "go", "build", "-o", runExe, "run.go")
 	cmd.Env = mergeEnvLists([]string{"GOOS=" + t.gohostos, "GOARCH=" + t.gohostarch, "GOMAXPROCS="}, os.Environ())