doc/progs: rewrite test driver in Go

The tests in doc/progs appear to have been originally written
for use with the old test driver. At some later point,
they acquired their own test driver.
Both ran tests in serial.

This CL rewrites the current test driver in Go,
runs tests concurrently, and cleans up
historical artifacts from the old drivers.

The primary motivation is to speed up all.bash.
On my laptop, using tip, this CL reduces doc/progs test
wall time from 26s to 7s. The savings will remain
even when the compiler gets faster. Using Go 1.4,
this CL reduces test wall time from 15s to 4s.

Change-Id: Iae945a8490222beee76e8a2118a0d7956092f543
Reviewed-on: https://go-review.googlesource.com/8410
Reviewed-by: Brad Fitzpatrick <bradfitz@golang.org>
Run-TryBot: Josh Bleecher Snyder <josharian@gmail.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>

doc/progs/cgo1.go

@@ -1,8 +1,7 @@
-// skip
-
 // Copyright 2012 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
+
 package rand
 
 /*

doc/progs/cgo2.go

@@ -1,8 +1,7 @@
-// skip
-
 // Copyright 2012 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
+
 package rand2
 
 /*

doc/progs/cgo3.go

@@ -1,8 +1,7 @@
-// skip
-
 // Copyright 2012 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
+
 package print
 
 // #include <stdio.h>

doc/progs/cgo4.go

@@ -1,8 +1,7 @@
-// skip
-
 // Copyright 2012 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
+
 package print
 
 // #include <stdio.h>

doc/progs/defer.go

@@ -1,5 +1,3 @@
-// cmpout
-
 // Copyright 2011 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.

doc/progs/defer.out

@@ -1,3 +0,0 @@
-0
-3210
-2

doc/progs/defer2.go

@@ -1,5 +1,3 @@
-// cmpout
-
 // Copyright 2011 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.

doc/progs/defer2.out

@@ -1,12 +0,0 @@
-Calling g.
-Printing in g 0
-Printing in g 1
-Printing in g 2
-Printing in g 3
-Panicking!
-Defer in g 3
-Defer in g 2
-Defer in g 1
-Defer in g 0
-Recovered in f 4
-Returned normally from f.

doc/progs/eff_bytesize.go

@@ -1,5 +1,3 @@
-// cmpout
-
 // Copyright 2009 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.

doc/progs/eff_bytesize.out

@@ -1 +0,0 @@
-1.00YB 9.09TB

doc/progs/eff_qr.go

@@ -1,5 +1,3 @@
-// compile
-
 // Copyright 2009 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.

doc/progs/eff_sequence.go

@@ -1,5 +1,3 @@
-// cmpout
-
 // Copyright 2009 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.

doc/progs/eff_sequence.out

@@ -1 +0,0 @@
-[-1 2 6 16 44]

doc/progs/eff_unused1.go

@@ -1,5 +1,3 @@
-// skip
-
 package main
 
 import (

doc/progs/eff_unused2.go

@@ -1,5 +1,3 @@
-// compile
-
 package main
 
 import (

doc/progs/error.go

@@ -1,5 +1,3 @@
-// compile
-
 // Copyright 2011 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.

doc/progs/error2.go

@@ -1,5 +1,3 @@
-// compile
-
 // Copyright 2011 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.

doc/progs/error3.go

@@ -1,5 +1,3 @@
-// compile
-
 // Copyright 2011 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.

doc/progs/error4.go

@@ -1,5 +1,3 @@
-// compile
-
 // Copyright 2011 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.

doc/progs/go1.go

@@ -1,6 +1,3 @@
-// compile
-// this file will output a list of filenames in cwd, not suitable for cmpout
-
 // Copyright 2011 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.

doc/progs/gobs1.go

@@ -1,5 +1,3 @@
-// compile
-
 // Copyright 2011 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.

doc/progs/gobs2.go

@@ -1,5 +1,3 @@
-// compile
-
 // Copyright 2011 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.

doc/progs/image_draw.go

@@ -1,5 +1,3 @@
-// compile
-
 // Copyright 2012 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.

doc/progs/image_package1.go

@@ -1,5 +1,3 @@
-// cmpout
-
 // Copyright 2012 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.

doc/progs/image_package1.out

@@ -1 +0,0 @@
-X is 2 Y is 1

doc/progs/image_package2.go

@@ -1,5 +1,3 @@
-// cmpout
-
 // Copyright 2012 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.

doc/progs/image_package2.out

@@ -1 +0,0 @@
-3 4 false

doc/progs/image_package3.go

@@ -1,5 +1,3 @@
-// cmpout
-
 // Copyright 2012 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.

doc/progs/image_package3.out

@@ -1 +0,0 @@
-3 4 true

doc/progs/image_package4.go

@@ -1,5 +1,3 @@
-// cmpout
-
 // Copyright 2012 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.

doc/progs/image_package4.out

@@ -1 +0,0 @@
-image.Point{X:2, Y:1}

doc/progs/image_package5.go

@@ -1,5 +1,3 @@
-// cmpout
-
 // Copyright 2012 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.

doc/progs/image_package5.out

@@ -1 +0,0 @@
-{255 0 0 255}

doc/progs/image_package6.go

@@ -1,5 +1,3 @@
-// cmpout
-
 // Copyright 2012 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.

doc/progs/image_package6.out

@@ -1,2 +0,0 @@
-8 4
-true

doc/progs/interface.go

@@ -1,5 +1,3 @@
-// compile
-
 // Copyright 2012 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.

doc/progs/interface2.go

@@ -1,5 +1,3 @@
-// cmpout
-
 // Copyright 2012 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.

doc/progs/interface2.out

@@ -1 +0,0 @@
-type: float64

doc/progs/json1.go

@@ -1,5 +1,3 @@
-// run
-
 // Copyright 2012 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.

doc/progs/json2.go

@@ -1,5 +1,3 @@
-// cmpout
-
 // Copyright 2012 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.

doc/progs/json2.out

@@ -1,2 +0,0 @@
-the circle's area 24.227111172875365
-the reciprocal of i is 0.3601008282319049

doc/progs/json3.go

@@ -1,5 +1,3 @@
-// compile
-
 // Copyright 2012 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.

doc/progs/json4.go

@@ -1,5 +1,3 @@
-// run
-
 // Copyright 2012 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.

doc/progs/json5.go

@@ -1,5 +1,3 @@
-// compile
-
 // Copyright 2012 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.

doc/progs/run

@@ -1,125 +0,0 @@
-#!/usr/bin/env bash
-# Copyright 2009 The Go Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style
-# license that can be found in the LICENSE file.
-
-set -e
-
-goos=$(go env GOOS)
-
-defer_panic_recover="
-	defer
-	defer2
-"
-
-effective_go="
-	eff_bytesize
-	eff_qr
-	eff_sequence
-	eff_unused2
-"
-
-error_handling="
-	error
-	error2
-	error3
-	error4
-"
-
-law_of_reflection="
-	interface
-	interface2
-"
-
-c_go_cgo="
-	cgo1
-	cgo2
-	cgo3
-	cgo4
-"
-# cgo1 and cgo2 don't run on freebsd, srandom has a different signature
-if [ "$goos" == "freebsd" ]; then
-	c_go_cgo="cgo3 cgo4"
-fi
-# cgo1 and cgo2 don't run on netbsd, srandom has a different signature
-# cgo3 and cgo4 don't run on netbsd, since cgo cannot handle stdout correctly
-if [ "$goos" == "netbsd" ]; then
-	c_go_cgo=""
-fi
-# cgo3 and cgo4 don't run on openbsd, since cgo cannot handle stdout correctly
-if [ "$goos" == "openbsd" ]; then
-	c_go_cgo="cgo1 cgo2"
-fi
-if [ "$CGO_ENABLED" != 1 ]; then
-	c_go_cgo=""
-fi
-
-timeout="
-	timeout1
-	timeout2
-"
-
-gobs="
-	gobs1
-	gobs2
-"
-
-json="
-	json1
-	json2
-	json3
-	json4
-	json5
-"
-
-image_package="
-	image_package1
-	image_package2
-	image_package3
-	image_package4
-	image_package5
-	image_package6
-"
-
-all=$(echo $defer_panic_recover $effective_go $error_handling $law_of_reflection $c_go_cgo $timeout $gobs $json $image_package slices go1)
-
-for i in $all; do
-	go build $i.go
-done
-
-# Write to temporary file to avoid mingw bash bug.
-TMPFILE="${TMPDIR:-/tmp}/gotest3.$USER"
-
-function testit {
-	./$1 >"$TMPFILE" 2>&1 || true
-	x=$(echo $(cat "$TMPFILE")) # extra echo canonicalizes
-	if ! echo "$x" | grep "$2" > /dev/null
-	then
-		echo $1 failed: '"'$x'"' is not '"'$2'"'
-	fi
-}
-
-
-testit defer '^0 3210 2$'
-testit defer2 '^Calling g. Printing in g 0 Printing in g 1 Printing in g 2 Printing in g 3 Panicking! Defer in g 3 Defer in g 2 Defer in g 1 Defer in g 0 Recovered in f 4 Returned normally from f.$'
-
-testit eff_bytesize '^1.00YB 9.09TB$'
-testit eff_sequence '^\[-1 2 6 16 44\]$'
-
-testit go1 '^Christmas is a holiday: true Sleeping for 0.123s.*go1.go already exists$'
-
-testit interface2 "^type: float64$"
-
-testit json1 "^$"
-testit json2 "the reciprocal of i is"
-testit json3 "Age is int 6"
-testit json4 "^$"
-
-testit image_package1 "^X is 2 Y is 1$"
-testit image_package2 "^3 4 false$"
-testit image_package3 "^3 4 true$"
-testit image_package4 "^image.Point{X:2, Y:1}$"
-testit image_package5 "^{255 0 0 255}$"
-testit image_package6 "^8 4 true$"
-
-rm -f $all "$TMPFILE"

doc/progs/run.go

@@ -0,0 +1,220 @@
+// Copyright 2015 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// run runs the docs tests found in this directory.
+package main
+
+import (
+	"bytes"
+	"flag"
+	"fmt"
+	"os"
+	"os/exec"
+	"regexp"
+	"runtime"
+	"strings"
+)
+
+const usage = `go run run.go [tests]
+
+run.go runs the docs tests in this directory.
+If no tests are provided, it runs all tests.
+Tests may be specified without their .go suffix.
+`
+
+func main() {
+	flag.Usage = func() {
+		fmt.Fprintf(os.Stderr, usage)
+		flag.PrintDefaults()
+		os.Exit(2)
+	}
+
+	flag.Parse()
+	if flag.NArg() == 0 {
+		// run all tests
+		fixcgo()
+	} else {
+		// run specified tests
+		onlyTest(flag.Args()...)
+	}
+
+	// ratec limits the number of tests running concurrently.
+	// None of the tests are intensive, so don't bother
+	// trying to manually adjust for slow builders.
+	ratec := make(chan bool, runtime.NumCPU())
+	errc := make(chan error, len(tests))
+
+	for _, tt := range tests {
+		tt := tt
+		ratec <- true
+		go func() {
+			errc <- test(tt.file, tt.want)
+			<-ratec
+		}()
+	}
+
+	var rc int
+	for range tests {
+		if err := <-errc; err != nil {
+			fmt.Fprintln(os.Stderr, err)
+			rc = 1
+		}
+	}
+	os.Exit(rc)
+}
+
+// test builds the test in the given file.
+// If want is non-empty, test also runs the test
+// and checks that the output matches the regexp want.
+func test(file, want string) error {
+	// Build the program.
+	cmd := exec.Command("go", "build", file+".go")
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		return fmt.Errorf("go build %s.go failed: %v\nOutput:\n%s", file, err, out)
+	}
+	defer os.Remove(file)
+
+	// Only run the test if we have output to check.
+	if want == "" {
+		return nil
+	}
+
+	cmd = exec.Command("./" + file)
+	out, err = cmd.CombinedOutput()
+	if err != nil {
+		return fmt.Errorf("./%s failed: %v\nOutput:\n%s", file, err, out)
+	}
+
+	// Canonicalize output.
+	out = bytes.TrimRight(out, "\n")
+	out = bytes.Replace(out, []byte{'\n'}, []byte{' '}, -1)
+
+	// Check the result.
+	match, err := regexp.Match(want, out)
+	if err != nil {
+		return fmt.Errorf("failed to parse regexp %q: %v", want, err)
+	}
+	if !match {
+		return fmt.Errorf("%s.go:\n%q\ndoes not match %s", file, out, want)
+	}
+
+	return nil
+}
+
+type testcase struct {
+	file string
+	want string
+}
+
+var tests = []testcase{
+	// defer_panic_recover
+	{"defer", `^0 3210 2$`},
+	{"defer2", `^Calling g. Printing in g 0 Printing in g 1 Printing in g 2 Printing in g 3 Panicking! Defer in g 3 Defer in g 2 Defer in g 1 Defer in g 0 Recovered in f 4 Returned normally from f.$`},
+
+	// effective_go
+	{"eff_bytesize", `^1.00YB 9.09TB$`},
+	{"eff_qr", ""},
+	{"eff_sequence", `^\[-1 2 6 16 44\]$`},
+	{"eff_unused2", ""},
+
+	// error_handling
+	{"error", ""},
+	{"error2", ""},
+	{"error3", ""},
+	{"error4", ""},
+
+	// law_of_reflection
+	{"interface", ""},
+	{"interface2", `^type: float64$`},
+
+	// c_go_cgo
+	{"cgo1", ""},
+	{"cgo2", ""},
+	{"cgo3", ""},
+	{"cgo4", ""},
+
+	// timeout
+	{"timeout1", ""},
+	{"timeout2", ""},
+
+	// gobs
+	{"gobs1", ""},
+	{"gobs2", ""},
+
+	// json
+	{"json1", `^$`},
+	{"json2", `the reciprocal of i is`},
+	{"json3", `Age is int 6`},
+	{"json4", `^$`},
+	{"json5", ""},
+
+	// image_package
+	{"image_package1", `^X is 2 Y is 1$`},
+	{"image_package2", `^3 4 false$`},
+	{"image_package3", `^3 4 true$`},
+	{"image_package4", `^image.Point{X:2, Y:1}$`},
+	{"image_package5", `^{255 0 0 255}$`},
+	{"image_package6", `^8 4 true$`},
+
+	// other
+	{"go1", `^Christmas is a holiday: true Sleeping for 0.123s.*go1.go already exists$`},
+	{"slices", ""},
+}
+
+func onlyTest(files ...string) {
+	var new []testcase
+NextFile:
+	for _, file := range files {
+		file = strings.TrimSuffix(file, ".go")
+		for _, tt := range tests {
+			if tt.file == file {
+				new = append(new, tt)
+				continue NextFile
+			}
+		}
+		fmt.Fprintf(os.Stderr, "test %s.go not found\n", file)
+		os.Exit(1)
+	}
+	tests = new
+}
+
+func skipTest(file string) {
+	for i, tt := range tests {
+		if tt.file == file {
+			copy(tests[i:], tests[i+1:])
+			tests = tests[:len(tests)-1]
+			return
+		}
+	}
+	panic("delete(" + file + "): not found")
+}
+
+func fixcgo() {
+	if os.Getenv("CGO_ENABLED") != "1" {
+		skipTest("cgo1")
+		skipTest("cgo2")
+		skipTest("cgo3")
+		skipTest("cgo4")
+		return
+	}
+
+	switch runtime.GOOS {
+	case "freebsd":
+		// cgo1 and cgo2 don't run on freebsd, srandom has a different signature
+		skipTest("cgo1")
+		skipTest("cgo2")
+	case "netbsd":
+		// cgo1 and cgo2 don't run on netbsd, srandom has a different signature
+		skipTest("cgo1")
+		skipTest("cgo2")
+		// cgo3 and cgo4 don't run on netbsd, since cgo cannot handle stdout correctly
+		skipTest("cgo3")
+		skipTest("cgo4")
+	case "openbsd":
+		// cgo3 and cgo4 don't run on openbsd and solaris, since cgo cannot handle stdout correctly
+		skipTest("cgo3")
+		skipTest("cgo4")
+	}
+}

doc/progs/slices.go

@@ -1,5 +1,3 @@
-// compile
-
 // Copyright 2012 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.

doc/progs/timeout1.go

@@ -1,8 +1,7 @@
-// compile
-
 // Copyright 2012 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
+
 package timeout
 
 import (

doc/progs/timeout2.go

@@ -1,8 +1,7 @@
-// compile
-
 // Copyright 2012 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
+
 package query
 
 type Conn string

doc/progs/update.bash

@@ -1,15 +0,0 @@
-#!/usr/bin/env bash
-# Copyright 2012 The Go Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style
-# license that can be found in the LICENSE file.
-
-set -e
-
-rm -f *.out *.rej *.orig [568].out
-
-for i in *.go; do
-	if grep -q '^// cmpout$' $i; then
-		echo $i
-		go run $i &> ${i/.go/.out}
-	fi
-done

src/cmd/dist/test.go

@@ -154,7 +154,7 @@ func (t *tester) registerTests() {
 	}
 	// ranGoTest and stdMatches are state closed over by the
 	// stdlib testing func below. The tests are run sequentially,
-	// so there'no need for locks.
+	// so there's no need for locks.
 	var (
 		ranGoTest  bool
 		stdMatches []string
@@ -280,7 +280,7 @@ func (t *tester) registerTests() {
 		}
 	}
 	if t.hasBash() && t.goos != "nacl" && t.goos != "android" && !iOS {
-		t.registerTest("doc_progs", "../doc/progs", "time", "./run")
+		t.registerTest("doc_progs", "../doc/progs", "time", "go", "run", "run.go")
 		t.registerTest("wiki", "../doc/articles/wiki", "./test.bash")
 		t.registerTest("codewalk", "../doc/codewalk", "time", "./run")
 		t.registerTest("shootout", "../test/bench/shootout", "time", "./timing.sh", "-test")