From 2f1727f1b37d7d257665debc72d4ae0b8e449748 Mon Sep 17 00:00:00 2001 From: Michael Matloob Date: Wed, 3 Oct 2018 12:41:27 -0400 Subject: [PATCH] go/packages: add basic support for overlays This allows users of go/packages to replace the contents of already existing files, to support use-cases such as unsaved files in editors. BREAKING CHANGE: This CL changes the signature of the function provided to Config.ParseFile. Change-Id: I6ce50336060832679e9f64f8d201b44651772e0b Reviewed-on: https://go-review.googlesource.com/c/139798 Reviewed-by: Alan Donovan --- go/packages/packages.go | 63 ++++++++++++++++++++++++++++++----- go/packages/packages_test.go | 64 +++++++++++++++++++++++++----------- 2 files changed, 98 insertions(+), 29 deletions(-) diff --git a/go/packages/packages.go b/go/packages/packages.go index 80bc25e23fc..92fce7585ee 100644 --- a/go/packages/packages.go +++ b/go/packages/packages.go @@ -20,6 +20,8 @@ import ( "sync" "golang.org/x/tools/go/gcexportdata" + "io/ioutil" + "path/filepath" ) // A LoadMode specifies the amount of detail to return when loading. @@ -96,12 +98,14 @@ type Config struct { // It must be safe to call ParseFile simultaneously from multiple goroutines. // If ParseFile is nil, the loader will uses parser.ParseFile. // - // Setting ParseFile to a custom implementation can allow - // providing alternate file content in order to type-check - // unsaved text editor buffers, or to selectively eliminate - // unwanted function bodies to reduce the amount of work - // done by the type checker. - ParseFile func(fset *token.FileSet, filename string) (*ast.File, error) + // ParseFile should parse the source from src and use filename only for + // recording position information. + // + // An application may supply a custom implementation of ParseFile + // to change the effective file contents or the behavior of the parser, + // or to modify the syntax tree. For example, selectively eliminating + // unwanted function bodies can significantly accelerate type checking. + ParseFile func(fset *token.FileSet, filename string, src []byte) (*ast.File, error) // If Tests is set, the loader includes not just the packages // matching a particular pattern but also any related test packages, @@ -116,6 +120,15 @@ type Config struct { // In build systems with explicit names for tests, // setting Tests may have no effect. Tests bool + + // Overlay provides a mapping of absolute file paths to file contents. + // If the file with the given path already exists, the parser will use the + // alternative file contents provided by the map. + // + // The Package.Imports map may not include packages that are imported only + // by the alternative file contents provided by Overlay. This may cause + // type-checking to fail. + Overlay map[string][]byte } // driver is the type for functions that query the build system for the @@ -380,9 +393,13 @@ func newLoader(cfg *Config) *loader { // ParseFile is required even in LoadTypes mode // because we load source if export data is missing. if ld.ParseFile == nil { - ld.ParseFile = func(fset *token.FileSet, filename string) (*ast.File, error) { + ld.ParseFile = func(fset *token.FileSet, filename string, src []byte) (*ast.File, error) { + var isrc interface{} + if src != nil { + isrc = src + } const mode = parser.AllErrors | parser.ParseComments - return parser.ParseFile(fset, filename, nil, mode) + return parser.ParseFile(fset, filename, isrc, mode) } } } @@ -743,7 +760,21 @@ func (ld *loader) parseFiles(filenames []string) ([]*ast.File, []error) { go func(i int, filename string) { ioLimit <- true // wait // ParseFile may return both an AST and an error. - parsed[i], errors[i] = ld.ParseFile(ld.Fset, filename) + var src []byte + for f, contents := range ld.Config.Overlay { + if sameFile(f, filename) { + src = contents + } + } + var err error + if src == nil { + src, err = ioutil.ReadFile(filename) + } + if err != nil { + parsed[i], errors[i] = nil, err + } else { + parsed[i], errors[i] = ld.ParseFile(ld.Fset, filename, src) + } <-ioLimit // signal wg.Done() }(i, file) @@ -772,6 +803,20 @@ func (ld *loader) parseFiles(filenames []string) ([]*ast.File, []error) { return parsed, errors } +// sameFile returns true if x and y have the same basename and denote +// the same file. +// +func sameFile(x, y string) bool { + if filepath.Base(x) == filepath.Base(y) { // (optimisation) + if xi, err := os.Stat(x); err == nil { + if yi, err := os.Stat(y); err == nil { + return os.SameFile(xi, yi) + } + } + } + return false +} + // loadFromExportData returns type information for the specified // package, loading it from an export data file on the first request. func (ld *loader) loadFromExportData(lpkg *loaderPackage) (*types.Package, error) { diff --git a/go/packages/packages_test.go b/go/packages/packages_test.go index 7f65479e4a0..555a4908732 100644 --- a/go/packages/packages_test.go +++ b/go/packages/packages_test.go @@ -763,12 +763,45 @@ func TestLoadSyntaxError(t *testing.T) { } } -// This function tests use of the ParseFile hook to supply -// alternative file contents to the parser and type-checker. -func TestLoadAllSyntaxOverlay(t *testing.T) { +// This function tests use of the ParseFile hook to modify +// the AST after parsing. +func TestParseFileModifyAST(t *testing.T) { type M = map[string]string tmp, cleanup := makeTree(t, M{ + "src/a/a.go": `package a; const A = "a" `, + }) + defer cleanup() + + parseFile := func(fset *token.FileSet, filename string, src []byte) (*ast.File, error) { + const mode = parser.AllErrors | parser.ParseComments + f, err := parser.ParseFile(fset, filename, src, mode) + // modify AST to change `const A = "a"` to `const A = "b"` + spec := f.Decls[0].(*ast.GenDecl).Specs[0].(*ast.ValueSpec) + spec.Values[0].(*ast.BasicLit).Value = `"b"` + return f, err + } + cfg := &packages.Config{ + Mode: packages.LoadAllSyntax, + Env: append(os.Environ(), "GOPATH="+tmp, "GO111MODULE=off"), + ParseFile: parseFile, + } + initial, err := packages.Load(cfg, "a") + if err != nil { + t.Error(err) + } + + // Check value of a.A has been set to "b" + a := initial[0] + got := constant(a, "A").Val().String() + if got != `"b"` { + t.Errorf("a.A: got %s, want %s", got, `"b"`) + } +} + +// This function tests config.Overlay functionality. +func TestOverlay(t *testing.T) { + tmp, cleanup := makeTree(t, map[string]string{ "src/a/a.go": `package a; import "b"; const A = "a" + b.B`, "src/b/b.go": `package b; import "c"; const B = "b" + c.C`, "src/c/c.go": `package c; const C = "c"`, @@ -777,32 +810,23 @@ func TestLoadAllSyntaxOverlay(t *testing.T) { defer cleanup() for i, test := range []struct { - overlay M + overlay map[string][]byte want string // expected value of a.A wantErrs []string }{ - {nil, `"abc"`, nil}, // default - {M{}, `"abc"`, nil}, // empty overlay - {M{filepath.Join(tmp, "src/c/c.go"): `package c; const C = "C"`}, `"abC"`, nil}, - {M{filepath.Join(tmp, "src/b/b.go"): `package b; import "c"; const B = "B" + c.C`}, `"aBc"`, nil}, - {M{filepath.Join(tmp, "src/b/b.go"): `package b; import "d"; const B = "B" + d.D`}, `unknown`, + {nil, `"abc"`, nil}, // default + {map[string][]byte{}, `"abc"`, nil}, // empty overlay + {map[string][]byte{filepath.Join(tmp, "src/c/c.go"): []byte(`package c; const C = "C"`)}, `"abC"`, nil}, + {map[string][]byte{filepath.Join(tmp, "src/b/b.go"): []byte(`package b; import "c"; const B = "B" + c.C`)}, `"aBc"`, nil}, + {map[string][]byte{filepath.Join(tmp, "src/b/b.go"): []byte(`package b; import "d"; const B = "B" + d.D`)}, `unknown`, []string{`could not import d (no metadata for d)`}}, } { - var parseFile func(fset *token.FileSet, filename string) (*ast.File, error) - if test.overlay != nil { - parseFile = func(fset *token.FileSet, filename string) (*ast.File, error) { - var src interface{} - if content, ok := test.overlay[filename]; ok { - src = content - } - const mode = parser.AllErrors | parser.ParseComments - return parser.ParseFile(fset, filename, src, mode) - } - } + var parseFile func(fset *token.FileSet, filename string, src []byte) (*ast.File, error) cfg := &packages.Config{ Mode: packages.LoadAllSyntax, Env: append(os.Environ(), "GOPATH="+tmp, "GO111MODULE=off"), ParseFile: parseFile, + Overlay: test.overlay, } initial, err := packages.Load(cfg, "a") if err != nil {