From 067d796549242bec2d33226c9da1e67f092a7be2 Mon Sep 17 00:00:00 2001 From: Katie Hockman Date: Thu, 14 Oct 2021 12:32:58 -0400 Subject: [PATCH] testing: write output to buffer when fuzzing Fixes #48709 Change-Id: Ia6376a2f792946498d6565a53605b3e6c985ea7c Reviewed-on: https://go-review.googlesource.com/c/go/+/355909 Trust: Katie Hockman Run-TryBot: Katie Hockman TryBot-Result: Go Bot Reviewed-by: Jay Conrod Reviewed-by: Roland Shoemaker Reviewed-by: Bryan C. Mills --- .../go/testdata/script/test_fuzz_minimize.txt | 20 +++++++++++-- .../script/test_fuzz_minimize_interesting.txt | 2 +- src/testing/fuzz.go | 28 +++++++++++++------ 3 files changed, 39 insertions(+), 11 deletions(-) diff --git a/src/cmd/go/testdata/script/test_fuzz_minimize.txt b/src/cmd/go/testdata/script/test_fuzz_minimize.txt index 8b11621bbd..3293e878bb 100644 --- a/src/cmd/go/testdata/script/test_fuzz_minimize.txt +++ b/src/cmd/go/testdata/script/test_fuzz_minimize.txt @@ -31,7 +31,7 @@ stdout FAIL ! go test -fuzz=FuzzMinimizeZeroLimitSet -run=FuzzMinimizeZeroLimitSet -fuzztime=10000x -fuzzminimizetime=0x . ! stdout '^ok' ! stdout 'minimizing' -stdout 'there was an Error' +stdout -count=1 'there was an Error' stdout FAIL # Test that minimization is working for recoverable errors. @@ -49,11 +49,27 @@ go run ./check_testdata FuzzMinimizerRecoverable 50 ! go test -run=FuzzMinimizerRecoverable . rm testdata +# Test that minimization is working for recoverable errors. Run it with -v this +# time to ensure the command line output still looks right. +! go test -v -fuzz=FuzzMinimizerRecoverable -run=FuzzMinimizerRecoverable -fuzztime=10000x . +! stdout '^ok' +stdout 'got the minimum size!' +# The error message that was printed should be for the one written to testdata. +stdout 'contains a non-zero byte of length 50' +stdout FAIL + +# Check that the bytes written to testdata are of length 50 (the minimum size) +go run ./check_testdata FuzzMinimizerRecoverable 50 + +# Test that re-running the minimized value causes a crash. +! go test -run=FuzzMinimizerRecoverable . +rm testdata + # Test that minimization doesn't run for non-recoverable errors. ! go test -fuzz=FuzzMinimizerNonrecoverable -run=FuzzMinimizerNonrecoverable -fuzztime=10000x . ! stdout '^ok' ! stdout 'minimizing' -stdout 'fuzzing process terminated unexpectedly: exit status 99' +stdout -count=1 'fuzzing process terminated unexpectedly: exit status 99' stdout FAIL # Check that re-running the value causes a crash. diff --git a/src/cmd/go/testdata/script/test_fuzz_minimize_interesting.txt b/src/cmd/go/testdata/script/test_fuzz_minimize_interesting.txt index fc66201eb3..8ea4cdb8a5 100644 --- a/src/cmd/go/testdata/script/test_fuzz_minimize_interesting.txt +++ b/src/cmd/go/testdata/script/test_fuzz_minimize_interesting.txt @@ -29,7 +29,7 @@ env GOCACHE=$WORK/gocache ! exec ./fuzz.test$GOEXE -test.fuzzcachedir=$GOCACHE/fuzz -test.fuzz=FuzzMinimizerCrashInMinimization -test.fuzztime=10000x -test.parallel=1 ! stdout '^ok' stdout 'got the minimum size!' -stdout 'flaky failure' +stdout -count=1 'flaky failure' stdout FAIL # Make sure the crash that was written will fail when run with go test diff --git a/src/testing/fuzz.go b/src/testing/fuzz.go index 0429f8243d..d5cb5e853f 100644 --- a/src/testing/fuzz.go +++ b/src/testing/fuzz.go @@ -5,6 +5,7 @@ package testing import ( + "bytes" "errors" "flag" "fmt" @@ -367,14 +368,14 @@ func (f *F) Fuzz(ff interface{}) { // run calls fn on a given input, as a subtest with its own T. // run is analogous to T.Run. The test filtering and cleanup works similarly. // fn is called in its own goroutine. - run := func(e corpusEntry) error { + run := func(captureOut io.Writer, e corpusEntry) (ok bool) { if e.Values == nil { // The corpusEntry must have non-nil Values in order to run the // test. If Values is nil, it is a bug in our code. panic(fmt.Sprintf("corpus file %q was not unmarshaled", e.Path)) } if shouldFailFast() { - return nil + return true } testName := f.name if e.Path != "" { @@ -405,6 +406,10 @@ func (f *F) Fuzz(ff interface{}) { }, context: f.testContext, } + if captureOut != nil { + // t.parent aliases f.common. + t.parent.w = captureOut + } t.w = indenter{&t.common} if t.chatty != nil { // TODO(#48132): adjust this to work with test2json. @@ -426,10 +431,7 @@ func (f *F) Fuzz(ff interface{}) { }) <-t.signal f.inFuzzFn = false - if t.Failed() { - return errors.New(string(f.output)) - } - return nil + return !t.Failed() } switch f.fuzzContext.mode { @@ -466,7 +468,17 @@ func (f *F) Fuzz(ff interface{}) { case fuzzWorker: // Fuzzing is enabled, and this is a worker process. Follow instructions // from the coordinator. - if err := f.fuzzContext.deps.RunFuzzWorker(run); err != nil { + if err := f.fuzzContext.deps.RunFuzzWorker(func(e corpusEntry) error { + // Don't write to f.w (which points to Stdout) if running from a + // fuzz worker. This would become very verbose, particularly during + // minimization. Return the error instead, and let the caller deal + // with the output. + var buf bytes.Buffer + if ok := run(&buf, e); !ok { + return errors.New(buf.String()) + } + return nil + }); err != nil { // Internal errors are marked with f.Fail; user code may call this too, before F.Fuzz. // The worker will exit with fuzzWorkerExitCode, indicating this is a failure // (and 'go test' should exit non-zero) but a crasher should not be recorded. @@ -479,7 +491,7 @@ func (f *F) Fuzz(ff interface{}) { for _, e := range f.corpus { name := fmt.Sprintf("%s/%s", f.name, filepath.Base(e.Path)) if _, ok, _ := f.testContext.match.fullName(nil, name); ok { - run(e) + run(f.w, e) } } }