diff --git a/go/packages/golist.go b/go/packages/golist.go index 1793afbf56..99f266224b 100644 --- a/go/packages/golist.go +++ b/go/packages/golist.go @@ -27,31 +27,32 @@ type GoTooOldError struct { error } +// Fields must match go list; +// see $GOROOT/src/cmd/go/internal/load/pkg.go. +type jsonPackage struct { + ImportPath string + Dir string + Name string + Export string + GoFiles []string + CFiles []string + CgoFiles []string + SFiles []string + Imports []string + ImportMap map[string]string + Deps []string + TestGoFiles []string + TestImports []string + XTestGoFiles []string + XTestImports []string + ForTest string // q in a "p [q.test]" package, else "" + DepOnly bool +} + // golistPackages uses the "go list" command to expand the // pattern words and return metadata for the specified packages. // dir may be "" and env may be nil, as per os/exec.Command. func golistPackages(cfg *rawConfig, words ...string) ([]*rawPackage, error) { - // Fields must match go list; - // see $GOROOT/src/cmd/go/internal/load/pkg.go. - type jsonPackage struct { - ImportPath string - Dir string - Name string - Export string - GoFiles []string - CFiles []string - CgoFiles []string - SFiles []string - Imports []string - ImportMap map[string]string - Deps []string - TestGoFiles []string - TestImports []string - XTestGoFiles []string - XTestImports []string - ForTest string // q in a "p [q.test]" package, else "" - DepOnly bool - } // go list uses the following identifiers in ImportPath and Imports: // // "p" -- importable package or main (command) @@ -66,11 +67,11 @@ func golistPackages(cfg *rawConfig, words ...string) ([]*rawPackage, error) { // Run "go list" for complete // information on the specified packages. - buf, err := golist(cfg, words) + buf, err := golist(cfg, golistargs(cfg, words)) if err != nil { return nil, err } - // Decode the JSON and convert it to Package form. + // Decode the JSON and convert it to rawPackage form. var result []*rawPackage for dec := json.NewDecoder(buf); dec.More(); { p := new(jsonPackage) @@ -179,14 +180,18 @@ func absJoin(dir string, fileses ...[]string) (res []string) { return res } -// golist returns the JSON-encoded result of a "go list args..." query. -func golist(cfg *rawConfig, args []string) (*bytes.Buffer, error) { - out := new(bytes.Buffer) +func golistargs(cfg *rawConfig, words []string) []string { fullargs := []string{"list", "-e", "-json", "-cgo=true"} fullargs = append(fullargs, cfg.Flags()...) fullargs = append(fullargs, "--") - fullargs = append(fullargs, args...) - cmd := exec.CommandContext(cfg.Context, "go", fullargs...) + fullargs = append(fullargs, words...) + return fullargs +} + +// golist returns the JSON-encoded result of a "go list args..." query. +func golist(cfg *rawConfig, args []string) (*bytes.Buffer, error) { + out := new(bytes.Buffer) + cmd := exec.CommandContext(cfg.Context, "go", args...) cmd.Env = cfg.Env cmd.Dir = cfg.Dir cmd.Stdout = out diff --git a/go/packages/golist_fallback.go b/go/packages/golist_fallback.go new file mode 100644 index 0000000000..9d1e6acb7a --- /dev/null +++ b/go/packages/golist_fallback.go @@ -0,0 +1,147 @@ +package packages + +import ( + "encoding/json" + "fmt" + "golang.org/x/tools/imports" +) + +// TODO(matloob): Delete this file once Go 1.12 is released. + +// This file provides backwards compatibility support for +// loading for versions of Go earlier than 1.10.4. This support is meant to +// assist with migration to the Package API until there's +// widespread adoption of these newer Go versions. +// This support will be removed once Go 1.12 is released +// in Q1 2019. + +// TODO(matloob): Support cgo. Copy code from the loader that runs cgo. + +func golistPackagesFallback(cfg *rawConfig, words ...string) ([]*rawPackage, error) { + original, deps, err := getDeps(cfg, words...) + if err != nil { + return nil, err + } + + var result []*rawPackage + addPackage := func(p *jsonPackage) { + if p.Name == "" { + return + } + + id := p.ImportPath + pkgpath := id + + if pkgpath == "unsafe" { + p.GoFiles = nil // ignore fake unsafe.go file + } + + importMap := func(importlist []string) map[string]string { + importMap := make(map[string]string) + for _, id := range importlist { + + if id == "C" { + importMap["unsafe"] = "unsafe" + importMap["syscall"] = "syscall" + if pkgpath != "runtime/cgo" { + importMap["runtime/cgo"] = "runtime/cgo" + } + continue + } + importMap[imports.VendorlessPath(id)] = id + } + return importMap + } + + result = append(result, &rawPackage{ + ID: id, + Name: p.Name, + GoFiles: absJoin(p.Dir, p.GoFiles, p.CgoFiles), + OtherFiles: absJoin(p.Dir, p.SFiles, p.CFiles), + PkgPath: pkgpath, + Imports: importMap(p.Imports), + DepOnly: original[id] == nil, + }) + if cfg.Tests { + testID := fmt.Sprintf("%s [%s.test]", id, id) + if len(p.TestGoFiles) > 0 || len(p.XTestGoFiles) > 0 { + result = append(result, &rawPackage{ + ID: testID, + Name: p.Name, + GoFiles: absJoin(p.Dir, p.GoFiles, p.TestGoFiles, p.CgoFiles), + OtherFiles: absJoin(p.Dir, p.SFiles, p.CFiles), + PkgPath: pkgpath, + Imports: importMap(append(p.Imports, p.TestImports)), + DepOnly: original[id] == nil, + }) + } + if len(p.XTestGoFiles) > 0 { + result = append(result, &rawPackage{ + ID: fmt.Sprintf("%s_test [%s.test]", id, id), + Name: p.Name + "_test", + GoFiles: absJoin(p.Dir, p.XTestGoFiles), + PkgPath: pkgpath, + Imports: importMap(append(p.XTestImports, testID)), + DepOnly: original[id] == nil, + }) + } + } + } + + for _, pkg := range original { + addPackage(pkg) + } + if !cfg.Deps || len(deps) == 0 { + return result, nil + } + + buf, err := golist(cfg, append([]string{"list", "-e", "-json", "--"}, deps...)) + if err != nil { + return nil, err + } + + // Decode the JSON and convert it to rawPackage form. + for dec := json.NewDecoder(buf); dec.More(); { + p := new(jsonPackage) + if err := dec.Decode(p); err != nil { + return nil, fmt.Errorf("JSON decoding failed: %v", err) + } + + addPackage(p) + } + + return result, nil +} + +// getDeps runs an initial go list to determine all the dependency packages. +func getDeps(cfg *rawConfig, words ...string) (originalSet map[string]*jsonPackage, deps []string, err error) { + buf, err := golist(cfg, append([]string{"list", "-e", "-json", "--"}, words...)) + if err != nil { + return nil, nil, err + } + + depsSet := make(map[string]bool) + originalSet = make(map[string]*jsonPackage) + + // Extract deps from the JSON. + for dec := json.NewDecoder(buf); dec.More(); { + p := new(jsonPackage) + if err := dec.Decode(p); err != nil { + return nil, nil, fmt.Errorf("JSON decoding failed: %v", err) + } + + originalSet[p.ImportPath] = p + for _, dep := range p.Deps { + depsSet[dep] = true + } + } + for orig := range originalSet { + delete(depsSet, orig) + } + + deps = make([]string, 0, len(depsSet)) + for dep := range depsSet { + deps = append(deps, dep) + } + return originalSet, deps, nil +} diff --git a/go/packages/loader_fallback.go b/go/packages/loader_fallback.go deleted file mode 100644 index 4a7384db26..0000000000 --- a/go/packages/loader_fallback.go +++ /dev/null @@ -1,130 +0,0 @@ -// Copyright 2018 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// TODO(matloob): Delete this file once Go 1.12 is released. - -// This file provides backwards compatibility support for -// loading for versions of Go earlier than 1.10.4. This support is meant to -// assist with migration to the Package API until there's -// widespread adoption of these newer Go versions. -// This support will be removed once Go 1.12 is released -// in Q1 2019. - -// The support is incomplete. These are some of the missing -// features: -// - the Tests option has no behavior, and test packages are -// never returned. -// - Package.OtherFiles are always missing even if the package -// contains non-go sources. - -package packages - -import ( - "fmt" - "go/build" - "strings" - - legacy "golang.org/x/tools/go/loader" - "golang.org/x/tools/imports" -) - -func loaderFallback(dir string, env []string, patterns []string) ([]*Package, error) { - cfg := legacy.Config{} - cfg.Cwd = dir - cfg.AllowErrors = true - cfg.FromArgs(patterns, false) // test packages are not supported - - // Set build ctx - buildCtx := build.Default - for _, ev := range env { - sp := strings.Split(ev, "=") - if len(sp) != 2 { - continue - } - evar, val := sp[0], sp[1] - switch evar { - case "GOPATH": - buildCtx.GOPATH = val - case "GOROOT": - buildCtx.GOROOT = val - case "GOARCH": - buildCtx.GOARCH = val - case "GOOS": - buildCtx.GOOS = val - } - } - cfg.Build = &buildCtx - - lprog, err := cfg.Load() - if err != nil { - if err.Error() == "no initial packages were loaded" { - return nil, fmt.Errorf("packages not found") // Return same error as golist-based code - } - return nil, fmt.Errorf("failed to load packages with legacy loader: %v", err) - } - - allpkgs := make(map[string]*loaderPackage) - - initial := make(map[*legacy.PackageInfo]bool) - for _, lpkg := range lprog.InitialPackages() { - initial[lpkg] = true - } - for _, lpkg := range lprog.AllPackages { - id := lpkg.Pkg.Path() - - var goFiles []string - for _, f := range lpkg.Files { - goFiles = append(goFiles, lprog.Fset.File(f.Pos()).Name()) - } - - pkgimports := make(map[string]string) - for _, imppkg := range lpkg.Pkg.Imports() { - // TODO(matloob): Is the import path of a package always VendorlessPath(path)? - pkgimports[imports.VendorlessPath(imppkg.Path())] = imppkg.Path() - } - - allpkgs[id] = &loaderPackage{ - raw: &rawPackage{ - ID: id, - Name: lpkg.Pkg.Name(), - Imports: pkgimports, - GoFiles: goFiles, - DepOnly: !initial[lpkg], - }, - Package: &Package{ - ID: id, - Name: lpkg.Pkg.Name(), - GoFiles: goFiles, - Fset: lprog.Fset, - Syntax: lpkg.Files, - Errors: lpkg.Errors, - Types: lpkg.Pkg, - TypesInfo: &lpkg.Info, - IllTyped: !lpkg.TransitivelyErrorFree, - OtherFiles: nil, // Never set for the fallback, because we can't extract from loader. - }, - } - } - - // Do a second pass to populate imports. - for _, pkg := range allpkgs { - pkg.Imports = make(map[string]*Package) - for imppath, impid := range pkg.raw.Imports { - target, ok := allpkgs[impid] - if !ok { - // return nil, fmt.Errorf("could not load package: %v", impid) - continue - } - pkg.Imports[imppath] = target.Package - } - } - - // Grab the initial set of packages. - var packages []*Package - for _, lpkg := range lprog.InitialPackages() { - packages = append(packages, allpkgs[lpkg.Pkg.Path()].Package) - } - - return packages, nil -} diff --git a/go/packages/packages.go b/go/packages/packages.go index cd1201d54e..5a42be1f2c 100644 --- a/go/packages/packages.go +++ b/go/packages/packages.go @@ -262,7 +262,12 @@ func (ld *loader) load(patterns ...string) ([]*Package, error) { rawCfg := newRawConfig(&ld.Config) list, err := golistPackages(rawCfg, patterns...) if _, ok := err.(GoTooOldError); ok { - return loaderFallback(ld.Dir, ld.Env, patterns) + if ld.Config.Mode >= LoadTypes { + // Upgrade to LoadAllSyntax because we can't depend on the existance + // of export data. We can remove this once iancottrell's cl is in. + ld.Config.Mode = LoadAllSyntax + } + list, err = golistPackagesFallback(rawCfg, patterns...) } if err != nil { return nil, err diff --git a/go/packages/packages110_test.go b/go/packages/packages110_test.go index 74b308db99..d4005682d1 100644 --- a/go/packages/packages110_test.go +++ b/go/packages/packages110_test.go @@ -7,5 +7,5 @@ package packages_test func init() { - usesLegacyLoader = true + usesOldGolist = true } diff --git a/go/packages/packages_test.go b/go/packages/packages_test.go index a65aaacc39..bae1776080 100644 --- a/go/packages/packages_test.go +++ b/go/packages/packages_test.go @@ -25,8 +25,8 @@ import ( ) // TODO(matloob): remove this once Go 1.12 is released as we will end support -// for the loader-backed implementation then. -var usesLegacyLoader = false +// for versions of go list before Go 1.10.4. +var usesOldGolist = false // TODO(adonovan): more test cases to write: // @@ -133,7 +133,7 @@ func TestMetadataImportGraph(t *testing.T) { subdir/d_test [subdir/d.test] -> subdir/d [subdir/d.test] `[1:] - if graph != wantGraph && !usesLegacyLoader { + if graph != wantGraph && !usesOldGolist { t.Errorf("wrong import graph: got <<%s>>, want <<%s>>", graph, wantGraph) } @@ -153,8 +153,8 @@ func TestMetadataImportGraph(t *testing.T) { {"subdir/d.test", "main", "command", "0.go"}, {"unsafe", "unsafe", "package", ""}, } { - if usesLegacyLoader && test.id == "subdir/d.test" { - // Legacy Loader does not support tests. + if usesOldGolist && test.id == "subdir/d.test" { + // Legacy go list support does not create test main package. continue } p, ok := all[test.id] @@ -187,12 +187,12 @@ func TestMetadataImportGraph(t *testing.T) { t.Errorf("failed to obtain metadata for ad-hoc package: %s", err) } else { got := fmt.Sprintf("%s %s", initial[0].ID, srcs(initial[0])) - if want := "command-line-arguments [c.go]"; got != want && !usesLegacyLoader { + if want := "command-line-arguments [c.go]"; got != want && !usesOldGolist { t.Errorf("oops: got %s, want %s", got, want) } } - if usesLegacyLoader { + if usesOldGolist { // TODO(matloob): Wildcards are not yet supported. return } @@ -213,7 +213,7 @@ func TestMetadataImportGraph(t *testing.T) { } } -func TestOptionsDir_Go110(t *testing.T) { +func TestOptionsDir(t *testing.T) { tmp, cleanup := makeTree(t, map[string]string{ "src/a/a.go": `package a; const Name = "a" `, "src/a/b/b.go": `package b; const Name = "a/b"`, @@ -271,7 +271,7 @@ func (ec *errCollector) add(err error) { ec.mu.Unlock() } -func TestTypeCheckOK_Go110(t *testing.T) { +func TestTypeCheckOK(t *testing.T) { tmp, cleanup := makeTree(t, map[string]string{ "src/a/a.go": `package a; import "b"; const A = "a" + b.B`, "src/b/b.go": `package b; import "c"; const B = "b" + c.C`, @@ -307,7 +307,7 @@ func TestTypeCheckOK_Go110(t *testing.T) { t.Errorf("wrong import graph: got <<%s>>, want <<%s>>", graph, wantGraph) } - // TODO(matloob): The go/loader based support loads everything from source + // TODO(matloob): The legacy go list based support loads everything from source // because it doesn't do a build and the .a files don't exist. // Can we simulate its existance? @@ -322,8 +322,8 @@ func TestTypeCheckOK_Go110(t *testing.T) { {"d", true, false}, // export data package {"e", false, false}, // no package } { - if usesLegacyLoader && test.id == "d" || test.id == "e" { - // legacyLoader always does a whole-program load. + if usesOldGolist && test.id == "d" || test.id == "e" { + // go list always upgrades whole-program load. continue } p := all[test.id] @@ -411,8 +411,8 @@ func TestTypeCheckError(t *testing.T) { {"d", false, false, true, nil}, // missing export data {"e", false, false, false, nil}, // type info not requested (despite type error) } { - if usesLegacyLoader && test.id == "c" || test.id == "d" || test.id == "e" { - // Behavior is different for legacy loader because it always loads wholeProgram. + if usesOldGolist && test.id == "c" || test.id == "d" || test.id == "e" { + // Behavior is different for old golist because it upgrades to wholeProgram. // TODO(matloob): can we run more of this test? Can we put export data into the test GOPATH? continue } @@ -453,10 +453,6 @@ func TestTypeCheckError(t *testing.T) { // This function tests use of the ParseFile hook to supply // alternative file contents to the parser and type-checker. func TestWholeProgramOverlay(t *testing.T) { - if usesLegacyLoader { - t.Skip("not yet supported in go/loader based implementation") - } - type M = map[string]string tmp, cleanup := makeTree(t, M{ @@ -517,8 +513,8 @@ func TestWholeProgramOverlay(t *testing.T) { } func TestWholeProgramImportErrors(t *testing.T) { - if usesLegacyLoader { - t.Skip("not yet supported in go/loader based implementation") + if usesOldGolist { + t.Skip("not yet supported in pre-Go 1.10.4 golist fallback implementation") } tmp, cleanup := makeTree(t, map[string]string{