cmd/vet: add support for vet-specific export data

An upcoming change to cmd/go will enable this functionality, which
allows vet to write down information about one package for use by
later invocation of vet that analyze code importing that package.

We've intended to do this for a long time, but the build caching was
necessary to have a decent way to manage the vet-specific export data.

This is also an experiment in building scalable whole-program analyses.
In the long term we'd like to allow other analyses to be invoked this way.

Change-Id: I34e4b70445786b2e8707ff6a0c00947bf1491511
Reviewed-on: https://go-review.googlesource.com/117099
Run-TryBot: Russ Cox <rsc@golang.org>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Bryan C. Mills <bcmills@google.com>

src/cmd/vet/asmdecl.go

@@ -104,6 +104,8 @@ func init() {
 		arch.ptrSize = int(arch.sizes.Sizeof(types.Typ[types.UnsafePointer]))
 		arch.maxAlign = int(arch.sizes.Alignof(types.Typ[types.Int64]))
 	}
+
+	registerPkgCheck("asmdecl", asmCheck)
 }
 
 var (
@@ -119,7 +121,7 @@ var (
 )
 
 func asmCheck(pkg *Package) {
-	if !vet("asmdecl") {
+	if vcfg.VetxOnly {
 		return
 	}
 

src/cmd/vet/doc.go

@@ -119,22 +119,17 @@ Printf family
 
 Flag: -printf
 
-Suspicious calls to functions in the Printf family, including any functions
-with these names, disregarding case:
-	Print Printf Println
-	Fprint Fprintf Fprintln
-	Sprint Sprintf Sprintln
-	Error Errorf
-	Fatal Fatalf
-	Log Logf
-	Panic Panicf Panicln
-The -printfuncs flag can be used to redefine this list.
-If the function name ends with an 'f', the function is assumed to take
-a format descriptor string in the manner of fmt.Printf. If not, vet
-complains about arguments that look like format descriptor strings.
+Suspicious calls to fmt.Print, fmt.Printf, and related functions.
+The check applies to known functions (for example, those in package fmt)
+as well as any detected wrappers of known functions.
 
-It also checks for errors such as using a Writer as the first argument of
-Printf.
+The -printfuncs flag specifies a comma-separated list of names of
+additional known formatting functions. Each name can be of the form
+pkg.Name or pkg.Type.Name, where pkg is a complete import path,
+or else can be a case-insensitive unqualified identifier like "errorf".
+If a listed name ends in f, the function is assumed to be Printf-like,
+taking a format string before the argument list. Otherwise it is
+assumed to be Print-like, taking a list of arguments with no format string.
 
 Range loop variables
 

src/cmd/vet/main.go

@@ -4,10 +4,12 @@
 
 // Vet is a simple checker for static errors in Go source code.
 // See doc.go for more information.
+
 package main
 
 import (
 	"bytes"
+	"encoding/gob"
 	"encoding/json"
 	"flag"
 	"fmt"
@@ -24,6 +26,8 @@ import (
 	"path/filepath"
 	"strconv"
 	"strings"
+
+	"cmd/internal/objabi"
 )
 
 // Important! If you add flags here, make sure to update cmd/go/internal/vet/vetflag.go.
@@ -154,9 +158,25 @@ var (
 	// checkers is a two-level map.
 	// The outer level is keyed by a nil pointer, one of the AST vars above.
 	// The inner level is keyed by checker name.
-	checkers = make(map[ast.Node]map[string]func(*File, ast.Node))
+	checkers    = make(map[ast.Node]map[string]func(*File, ast.Node))
+	pkgCheckers = make(map[string]func(*Package))
+	exporters   = make(map[string]func() interface{})
 )
 
+// Vet can provide its own "export information"
+// about package A to future invocations of vet
+// on packages importing A. If B imports A,
+// then running "go vet B" actually invokes vet twice:
+// first, it runs vet on A, in "vetx-only" mode, which
+// skips most checks and only computes export data
+// describing A. Then it runs vet on B, making A's vetx
+// data available for consultation. The vet of B
+// computes vetx data for B in addition to its
+// usual vet checks.
+
+// register registers the named check function,
+// to be called with AST nodes of the given types.
+// The registered functions are not called in vetx-only mode.
 func register(name, usage string, fn func(*File, ast.Node), types ...ast.Node) {
 	report[name] = triStateFlag(name, unset, usage)
 	for _, typ := range types {
@@ -169,6 +189,25 @@ func register(name, usage string, fn func(*File, ast.Node), types ...ast.Node) {
 	}
 }
 
+// registerPkgCheck registers a package-level checking function,
+// to be invoked with the whole package being vetted
+// before any of the per-node handlers.
+// The registered function fn is called even in vetx-only mode
+// (see comment above), so fn must take care not to report
+// errors when vcfg.VetxOnly is true.
+func registerPkgCheck(name string, fn func(*Package)) {
+	pkgCheckers[name] = fn
+}
+
+// registerExport registers a function to return vetx export data
+// that should be saved and provided to future invocations of vet
+// when checking packages importing this one.
+// The value returned by fn should be nil or else valid to encode using gob.
+// Typically a registerExport call is paired with a call to gob.Register.
+func registerExport(name string, fn func() interface{}) {
+	exporters[name] = fn
+}
+
 // Usage is a replacement usage function for the flags package.
 func Usage() {
 	fmt.Fprintf(os.Stderr, "Usage of vet:\n")
@@ -209,6 +248,7 @@ type File struct {
 }
 
 func main() {
+	objabi.AddVersionFlag()
 	flag.Usage = Usage
 	flag.Parse()
 
@@ -295,6 +335,9 @@ type vetConfig struct {
 	ImportMap   map[string]string
 	PackageFile map[string]string
 	Standard    map[string]bool
+	PackageVetx map[string]string // map from import path to vetx data file
+	VetxOnly    bool              // only compute vetx output; don't run ordinary checks
+	VetxOutput  string            // file where vetx output should be written
 
 	SucceedOnTypecheckFailure bool
 
@@ -355,6 +398,21 @@ func doPackageCfg(cfgFile string) {
 	inittypes()
 	mustTypecheck = true
 	doPackage(vcfg.GoFiles, nil)
+	if vcfg.VetxOutput != "" {
+		out := make(map[string]interface{})
+		for name, fn := range exporters {
+			out[name] = fn()
+		}
+		var buf bytes.Buffer
+		if err := gob.NewEncoder(&buf).Encode(out); err != nil {
+			errorf("encoding vet output: %v", err)
+			return
+		}
+		if err := ioutil.WriteFile(vcfg.VetxOutput, buf.Bytes(), 0666); err != nil {
+			errorf("saving vet output: %v", err)
+			return
+		}
+	}
 }
 
 // doPackageDir analyzes the single package found in the directory, if there is one,
@@ -461,6 +519,19 @@ func doPackage(names []string, basePkg *Package) *Package {
 	}
 
 	// Check.
+	for _, file := range files {
+		file.pkg = pkg
+		file.basePkg = basePkg
+	}
+	for name, fn := range pkgCheckers {
+		if vet(name) {
+			fn(pkg)
+		}
+	}
+	if vcfg.VetxOnly {
+		return pkg
+	}
+
 	chk := make(map[ast.Node][]func(*File, ast.Node))
 	for typ, set := range checkers {
 		for name, fn := range set {
@@ -470,14 +541,11 @@ func doPackage(names []string, basePkg *Package) *Package {
 		}
 	}
 	for _, file := range files {
-		file.pkg = pkg
-		file.basePkg = basePkg
 		file.checkers = chk
 		if file.file != nil {
 			file.walkFile(file.name, file.file)
 		}
 	}
-	asmCheck(pkg)
 	return pkg
 }
 
@@ -630,3 +698,35 @@ func (f *File) gofmt(x ast.Expr) string {
 	printer.Fprint(&f.b, f.fset, x)
 	return f.b.String()
 }
+
+// imported[path][key] is previously written export data.
+var imported = make(map[string]map[string]interface{})
+
+// readVetx reads export data written by a previous
+// invocation of vet on an imported package (path).
+// The key is the name passed to registerExport
+// when the data was originally generated.
+// readVetx returns nil if the data is unavailable.
+func readVetx(path, key string) interface{} {
+	if path == "unsafe" || vcfg.ImportPath == "" {
+		return nil
+	}
+	m := imported[path]
+	if m == nil {
+		file := vcfg.PackageVetx[path]
+		if file == "" {
+			return nil
+		}
+		data, err := ioutil.ReadFile(file)
+		if err != nil {
+			return nil
+		}
+		m = make(map[string]interface{})
+		err = gob.NewDecoder(bytes.NewReader(data)).Decode(&m)
+		if err != nil {
+			return nil
+		}
+		imported[path] = m
+	}
+	return m[key]
+}