cmd/go: allow generate to process invalid packages

Allow go generate to process packages that contain invalid code. Ignore
errors when loading the package, but process only files which have a
valid package clause. Set $GOPACKAGE individually for each file, based
on the package clause.

Add test script for go generate and invalid packages.

Fixes #36422

Change-Id: I91ea088346a1548ccd6678b4595a527b948331ff
Reviewed-on: https://go-review.googlesource.com/c/go/+/229097
Reviewed-by: Rob Pike <r@golang.org>
Reviewed-by: Jay Conrod <jayconrod@google.com>
Run-TryBot: Jay Conrod <jayconrod@google.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>

src/cmd/go/alldocs.go

@@ -547,6 +547,9 @@
 // tag "generate" so that files may be examined by go generate but ignored
 // during build.
 //
+// For packages with invalid code, generate processes only source files with a
+// valid package clause.
+//
 // If any generator returns an error exit status, "go generate" skips
 // all further processing for that package.
 //

src/cmd/go/internal/generate/generate.go

@@ -9,7 +9,10 @@ import (
 	"bufio"
 	"bytes"
 	"fmt"
+	"go/parser"
+	"go/token"
 	"io"
+	"io/ioutil"
 	"log"
 	"os"
 	"os/exec"
@@ -119,6 +122,9 @@ in the file, one at a time. The go generate tool also sets the build
 tag "generate" so that files may be examined by go generate but ignored
 during build.
 
+For packages with invalid code, generate processes only source files with a
+valid package clause.
+
 If any generator returns an error exit status, "go generate" skips
 all further processing for that package.
 
@@ -169,7 +175,7 @@ func runGenerate(cmd *base.Command, args []string) {
 
 	// Even if the arguments are .go files, this loop suffices.
 	printed := false
-	for _, pkg := range load.Packages(args) {
+	for _, pkg := range load.PackagesAndErrors(args) {
 		if modload.Enabled() && pkg.Module != nil && !pkg.Module.Main {
 			if !printed {
 				fmt.Fprintf(os.Stderr, "go: not generating in packages in dependency modules\n")
@@ -178,18 +184,14 @@ func runGenerate(cmd *base.Command, args []string) {
 			continue
 		}
 
-		pkgName := pkg.Name
-
 		for _, file := range pkg.InternalGoFiles() {
-			if !generate(pkgName, file) {
+			if !generate(file) {
 				break
 			}
 		}
 
-		pkgName += "_test"
-
 		for _, file := range pkg.InternalXGoFiles() {
-			if !generate(pkgName, file) {
+			if !generate(file) {
 				break
 			}
 		}
@@ -197,16 +199,23 @@ func runGenerate(cmd *base.Command, args []string) {
 }
 
 // generate runs the generation directives for a single file.
-func generate(pkg, absFile string) bool {
+func generate(absFile string) bool {
-	fd, err := os.Open(absFile)
+	src, err := ioutil.ReadFile(absFile)
 	if err != nil {
 		log.Fatalf("generate: %s", err)
 	}
-	defer fd.Close()
+
+	// Parse package clause
+	filePkg, err := parser.ParseFile(token.NewFileSet(), "", src, parser.PackageClauseOnly)
+	if err != nil {
+		// Invalid package clause - ignore file.
+		return true
+	}
+
 	g := &Generator{
-		r:        fd,
+		r:        bytes.NewReader(src),
 		path:     absFile,
-		pkg:      pkg,
+		pkg:      filePkg.Name.String(),
 		commands: make(map[string][]string),
 	}
 	return g.run()

src/cmd/go/testdata/script/generate_invalid.txt

@@ -0,0 +1,199 @@
+[short] skip
+
+# Install an echo command because Windows doesn't have it.
+env GOBIN=$WORK/tmp/bin
+go install echo.go
+env PATH=$GOBIN${:}$PATH
+
+# Test go generate for directory with no go files
+go generate ./nogo
+! stdout 'Fail'
+
+# Test go generate for package where all .go files are excluded by build
+# constraints
+go generate -v ./excluded
+! stdout 'Fail'
+! stderr 'go' # -v shouldn't list any files
+
+# Test go generate for "package" with no package clause in any file
+go generate ./nopkg
+stdout 'Success a'
+! stdout 'Fail'
+
+# Test go generate for package with inconsistent package clauses
+# $GOPACKAGE should depend on each file's package clause
+go generate ./inconsistent
+stdout 'Success a'
+stdout 'Success b'
+stdout -count=2 'Success c'
+! stdout 'Fail'
+
+# Test go generate for syntax errors before and after package clauses
+go generate ./syntax
+stdout 'Success a'
+stdout 'Success b'
+! stdout 'Fail'
+
+# Test go generate for files importing non-existent packages
+go generate ./importerr
+stdout 'Success a'
+stdout 'Success b'
+stdout 'Success c'
+
+-- echo.go --
+package main
+
+import (
+	"fmt"
+	"os"
+	"strings"
+)
+
+func main() {
+	fmt.Println(strings.Join(os.Args[1:], " "))
+	fmt.Println()
+}
+
+-- nogo/foo.txt --
+Text file in a directory without go files.
+Go generate should ignore this directory.
+//go:generate echo Fail nogo
+
+-- excluded/a.go --
+// Include a build tag that go generate should exclude.
+// Go generate should ignore this file.
+
+// +build a
+
+//go:generate echo Fail a
+
+package excluded
+
+-- excluded/b.go --
+// Include a build tag that go generate should exclude.
+// Go generate should ignore this file.
+
+//go:generate echo Fail b
+
+// +build b
+
+package excluded
+
+
+-- nopkg/a.go --
+// Go file with package clause after comment.
+// Go generate should process this file.
+
+/* Pre-comment */ package nopkg
+//go:generate echo Success a
+
+-- nopkg/b.go --
+// Go file with commented package clause.
+// Go generate should ignore this file.
+
+//package nopkg
+
+//go:generate echo Fail b
+
+-- nopkg/c.go --
+// Go file with package clause inside multiline comment.
+// Go generate should ignore this file.
+
+/*
+package nopkg
+*/
+
+//go:generate echo Fail c
+
+-- nopkg/d.go --
+// Go file with package clause inside raw string literal.
+// Go generate should ignore this file.
+
+const foo = `
+package nopkg
+`
+//go:generate echo Fail d
+
+-- nopkg/e.go --
+// Go file without package clause.
+// Go generate should ignore this file.
+
+//go:generate echo Fail e
+
+-- inconsistent/a.go --
+// Valid go file with inconsistent package name.
+// Go generate should process this file with GOPACKAGE=a
+
+package a
+//go:generate echo Success $GOPACKAGE
+
+-- inconsistent/b.go --
+// Valid go file with inconsistent package name.
+// Go generate should process this file with GOPACKAGE=b
+
+//go:generate echo Success $GOPACKAGE
+package b
+
+-- inconsistent/c.go --
+// Go file with two package clauses.
+// Go generate should process this file with GOPACKAGE=c
+
+//go:generate echo Success $GOPACKAGE
+package c
+// Invalid package clause, should be ignored:
+package cinvalid
+//go:generate echo Success $GOPACKAGE
+
+-- inconsistent/d.go --
+// Go file with invalid package name.
+// Go generate should ignore this file.
+
+package +d+
+//go:generate echo Fail $GOPACKAGE
+
+-- syntax/a.go --
+// Go file with syntax error after package clause.
+// Go generate should process this file.
+
+package syntax
+123
+//go:generate echo Success a
+
+-- syntax/b.go --
+// Go file with syntax error after package clause.
+// Go generate should process this file.
+
+package syntax; 123
+//go:generate echo Success b
+
+-- syntax/c.go --
+// Go file with syntax error before package clause.
+// Go generate should ignore this file.
+
+foo
+package syntax
+//go:generate echo Fail c
+
+-- importerr/a.go --
+// Go file which imports non-existing package.
+// Go generate should process this file.
+
+package importerr
+//go:generate echo Success a
+import "foo"
+
+-- importerr/b.go --
+// Go file which imports non-existing package.
+// Go generate should process this file.
+
+//go:generate echo Success b
+package importerr
+import "bar"
+
+-- importerr/c.go --
+// Go file which imports non-existing package.
+// Go generate should process this file.
+
+package importerr
+import "moo"
+//go:generate echo Success c