From 83a6c13e7344394d32d4114c3504ffecaae37b1e Mon Sep 17 00:00:00 2001 From: qiulaidongfeng <2645477756@qq.com> Date: Sat, 23 Mar 2024 03:19:33 +0000 Subject: [PATCH] cmd/dist: avoid CPU underutilization starting from GOMAXPROCS=2 runtime This CL is doing now is: change maxbg to increase test parallelism. adjust test sequence. This CL speeds up the go tool dist test, most of the speed up is due to the fact that the three time-consuming tests cmd/internal/testdir and API check and runtime/race can be done in parallel with the GOMAXPROCS=2 runtime on a machine with enough CPU cores. In windows with an 8-core 16-thread CPU, this CL can complete all other tests before GOMAXPROCS=2 runtime -cpu=1,2,4 -quick completes. Fixes #65164 Change-Id: I56ed7031d58be3bece9f975bfc73e5c834d0a4fa GitHub-Last-Rev: 18cffb770f60399c889ceb40ef8207a2d0c488e5 GitHub-Pull-Request: golang/go#65703 Reviewed-on: https://go-review.googlesource.com/c/go/+/563916 Reviewed-by: Keith Randall Reviewed-by: Keith Randall Commit-Queue: Ian Lance Taylor Auto-Submit: Ian Lance Taylor Reviewed-by: David Chase LUCI-TryBot-Result: Go LUCI --- src/cmd/dist/test.go | 84 +++++++++++++++++++++++++++----------------- 1 file changed, 51 insertions(+), 33 deletions(-) diff --git a/src/cmd/dist/test.go b/src/cmd/dist/test.go index 95d9cab816..81bf86b688 100644 --- a/src/cmd/dist/test.go +++ b/src/cmd/dist/test.go @@ -711,24 +711,6 @@ func (t *tester) registerTests() { }) } - // Runtime CPU tests. - if !t.compileOnly && t.hasParallelism() { - for i := 1; i <= 4; i *= 2 { - t.registerTest(fmt.Sprintf("GOMAXPROCS=2 runtime -cpu=%d -quick", i), - &goTest{ - variant: "cpu" + strconv.Itoa(i), - timeout: 300 * time.Second, - cpu: strconv.Itoa(i), - short: true, - testFlags: []string{"-quick"}, - // We set GOMAXPROCS=2 in addition to -cpu=1,2,4 in order to test runtime bootstrap code, - // creation of first goroutines and first garbage collections in the parallel setting. - env: []string{"GOMAXPROCS=2"}, - pkg: "runtime", - }) - } - } - // GOEXPERIMENT=rangefunc tests if !t.compileOnly { t.registerTest("GOEXPERIMENT=rangefunc go test iter", @@ -864,10 +846,6 @@ func (t *tester) registerTests() { }) } - if t.raceDetectorSupported() { - t.registerRaceTests() - } - const cgoHeading = "Testing cgo" if t.cgoEnabled { t.registerCgoTests(cgoHeading) @@ -883,6 +861,40 @@ func (t *tester) registerTests() { }) } + // Only run the API check on fast development platforms. + // Every platform checks the API on every GOOS/GOARCH/CGO_ENABLED combination anyway, + // so we really only need to run this check once anywhere to get adequate coverage. + // To help developers avoid trybot-only failures, we try to run on typical developer machines + // which is darwin,linux,windows/amd64 and darwin/arm64. + // + // The same logic applies to the release notes that correspond to each api/next file. + if goos == "darwin" || ((goos == "linux" || goos == "windows") && goarch == "amd64") { + t.registerTest("API release note check", &goTest{variant: "check", pkg: "cmd/relnote", testFlags: []string{"-check"}}) + t.registerTest("API check", &goTest{variant: "check", pkg: "cmd/api", timeout: 5 * time.Minute, testFlags: []string{"-check"}}) + } + + // Runtime CPU tests. + if !t.compileOnly && t.hasParallelism() { + for i := 1; i <= 4; i *= 2 { + t.registerTest(fmt.Sprintf("GOMAXPROCS=2 runtime -cpu=%d -quick", i), + &goTest{ + variant: "cpu" + strconv.Itoa(i), + timeout: 300 * time.Second, + cpu: strconv.Itoa(i), + short: true, + testFlags: []string{"-quick"}, + // We set GOMAXPROCS=2 in addition to -cpu=1,2,4 in order to test runtime bootstrap code, + // creation of first goroutines and first garbage collections in the parallel setting. + env: []string{"GOMAXPROCS=2"}, + pkg: "runtime", + }) + } + } + + if t.raceDetectorSupported() { + t.registerRaceTests() + } + if goos != "android" && !t.iOS() { // Only start multiple test dir shards on builders, // where they get distributed to multiple machines. @@ -907,17 +919,6 @@ func (t *tester) registerTests() { ) } } - // Only run the API check on fast development platforms. - // Every platform checks the API on every GOOS/GOARCH/CGO_ENABLED combination anyway, - // so we really only need to run this check once anywhere to get adequate coverage. - // To help developers avoid trybot-only failures, we try to run on typical developer machines - // which is darwin,linux,windows/amd64 and darwin/arm64. - // - // The same logic applies to the release notes that correspond to each api/next file. - if goos == "darwin" || ((goos == "linux" || goos == "windows") && goarch == "amd64") { - t.registerTest("API check", &goTest{variant: "check", pkg: "cmd/api", timeout: 5 * time.Minute, testFlags: []string{"-check"}}) - t.registerTest("API release note check", &goTest{variant: "check", pkg: "cmd/relnote", testFlags: []string{"-check"}}) - } } // addTest adds an arbitrary test callback to the test list. @@ -1313,6 +1314,23 @@ func (t *tester) runPending(nextTest *distTest) { }(w) } + maxbg := maxbg + // for runtime.NumCPU() < 4 || runtime.GOMAXPROCS(0) == 1, do not change maxbg. + // Because there is not enough CPU to parallel the testing of multiple packages. + if runtime.NumCPU() > 4 && runtime.GOMAXPROCS(0) != 1 { + for _, w := range worklist { + // See go.dev/issue/65164 + // because GOMAXPROCS=2 runtime CPU usage is low, + // so increase maxbg to avoid slowing down execution with low CPU usage. + // This makes testing a single package slower, + // but testing multiple packages together faster. + if strings.Contains(w.dt.heading, "GOMAXPROCS=2 runtime") { + maxbg = runtime.NumCPU() + break + } + } + } + started := 0 ended := 0 var last *distTest