qbit/go
1
0
Fork 0
mirror of https://github.com/golang/go synced 2024-10-01 05:38:32 -06:00
Code Releases Activity

go/packages: switch fallback implementation to use go list

Browse Source 
The go/loader based implementation was too different. Instead
Just call go list twice to get all the dependencies. This immediately
enables more functionality.

Change-Id: I22883492e996133bc61664314318c5e0a9cc10dc
Reviewed-on: https://go-review.googlesource.com/125939
Reviewed-by: Suzy Mueller <suzmue@golang.org>
Reviewed-by: Ian Cottrell <iancottrell@google.com>
This commit is contained in:
Michael Matloob 2018-07-25 20:16:16 -04:00
parent 475b7a1e12
commit ceb18cec46
6 changed files with 203 additions and 180 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

61
go/packages/golist.go
View File

@ -27,31 +27,32 @@ type GoTooOldError struct {
error
}
// Fields must match go list;
// see $GOROOT/src/cmd/go/internal/load/pkg.go.
type jsonPackage struct {
ImportPath string
Dir string
Name string
Export string
GoFiles []string
CFiles []string
CgoFiles []string
SFiles []string
Imports []string
ImportMap map[string]string
Deps []string
TestGoFiles []string
TestImports []string
XTestGoFiles []string
XTestImports []string
ForTest string // q in a "p [q.test]" package, else ""
DepOnly bool
}
// golistPackages uses the "go list" command to expand the
// pattern words and return metadata for the specified packages.
// dir may be "" and env may be nil, as per os/exec.Command.
func golistPackages(cfg *rawConfig, words ...string) ([]*rawPackage, error) {
// Fields must match go list;
// see $GOROOT/src/cmd/go/internal/load/pkg.go.
type jsonPackage struct {
ImportPath string
Dir string
Name string
Export string
GoFiles []string
CFiles []string
CgoFiles []string
SFiles []string
Imports []string
ImportMap map[string]string
Deps []string
TestGoFiles []string
TestImports []string
XTestGoFiles []string
XTestImports []string
ForTest string // q in a "p [q.test]" package, else ""
DepOnly bool
}
// go list uses the following identifiers in ImportPath and Imports:
//
// "p" -- importable package or main (command)
@ -66,11 +67,11 @@ func golistPackages(cfg *rawConfig, words ...string) ([]*rawPackage, error) {
// Run "go list" for complete
// information on the specified packages.
buf, err := golist(cfg, words)
buf, err := golist(cfg, golistargs(cfg, words))
if err != nil {
return nil, err
}
// Decode the JSON and convert it to Package form.
// Decode the JSON and convert it to rawPackage form.
var result []*rawPackage
for dec := json.NewDecoder(buf); dec.More(); {
p := new(jsonPackage)
@ -179,14 +180,18 @@ func absJoin(dir string, fileses ...[]string) (res []string) {
return res
}
// golist returns the JSON-encoded result of a "go list args..." query.
func golist(cfg *rawConfig, args []string) (*bytes.Buffer, error) {
out := new(bytes.Buffer)
func golistargs(cfg *rawConfig, words []string) []string {
fullargs := []string{"list", "-e", "-json", "-cgo=true"}
fullargs = append(fullargs, cfg.Flags()...)
fullargs = append(fullargs, "--")
fullargs = append(fullargs, args...)
cmd := exec.CommandContext(cfg.Context, "go", fullargs...)
fullargs = append(fullargs, words...)
return fullargs
}
// golist returns the JSON-encoded result of a "go list args..." query.
func golist(cfg *rawConfig, args []string) (*bytes.Buffer, error) {
out := new(bytes.Buffer)
cmd := exec.CommandContext(cfg.Context, "go", args...)
cmd.Env = cfg.Env
cmd.Dir = cfg.Dir
cmd.Stdout = out

147
go/packages/golist_fallback.go Normal file
View File

@ -0,0 +1,147 @@
package packages
import (
"encoding/json"
"fmt"
"golang.org/x/tools/imports"
)
// TODO(matloob): Delete this file once Go 1.12 is released.
// This file provides backwards compatibility support for
// loading for versions of Go earlier than 1.10.4. This support is meant to
// assist with migration to the Package API until there's
// widespread adoption of these newer Go versions.
// This support will be removed once Go 1.12 is released
// in Q1 2019.
// TODO(matloob): Support cgo. Copy code from the loader that runs cgo.
func golistPackagesFallback(cfg *rawConfig, words ...string) ([]*rawPackage, error) {
original, deps, err := getDeps(cfg, words...)
if err != nil {
return nil, err
}
var result []*rawPackage
addPackage := func(p *jsonPackage) {
if p.Name == "" {
return
}
id := p.ImportPath
pkgpath := id
if pkgpath == "unsafe" {
p.GoFiles = nil // ignore fake unsafe.go file
}
importMap := func(importlist []string) map[string]string {
importMap := make(map[string]string)
for _, id := range importlist {
if id == "C" {
importMap["unsafe"] = "unsafe"
importMap["syscall"] = "syscall"
if pkgpath != "runtime/cgo" {
importMap["runtime/cgo"] = "runtime/cgo"
}
continue
}
importMap[imports.VendorlessPath(id)] = id
}
return importMap
}
result = append(result, &rawPackage{
ID: id,
Name: p.Name,
GoFiles: absJoin(p.Dir, p.GoFiles, p.CgoFiles),
OtherFiles: absJoin(p.Dir, p.SFiles, p.CFiles),
PkgPath: pkgpath,
Imports: importMap(p.Imports),
DepOnly: original[id] == nil,
})
if cfg.Tests {
testID := fmt.Sprintf("%s [%s.test]", id, id)
if len(p.TestGoFiles) > 0 || len(p.XTestGoFiles) > 0 {
result = append(result, &rawPackage{
ID: testID,
Name: p.Name,
GoFiles: absJoin(p.Dir, p.GoFiles, p.TestGoFiles, p.CgoFiles),
OtherFiles: absJoin(p.Dir, p.SFiles, p.CFiles),
PkgPath: pkgpath,
Imports: importMap(append(p.Imports, p.TestImports)),
DepOnly: original[id] == nil,
})
}
if len(p.XTestGoFiles) > 0 {
result = append(result, &rawPackage{
ID: fmt.Sprintf("%s_test [%s.test]", id, id),
Name: p.Name + "_test",
GoFiles: absJoin(p.Dir, p.XTestGoFiles),
PkgPath: pkgpath,
Imports: importMap(append(p.XTestImports, testID)),
DepOnly: original[id] == nil,
})
}
}
}
for _, pkg := range original {
addPackage(pkg)
}
if !cfg.Deps || len(deps) == 0 {
return result, nil
}
buf, err := golist(cfg, append([]string{"list", "-e", "-json", "--"}, deps...))
if err != nil {
return nil, err
}
// Decode the JSON and convert it to rawPackage form.
for dec := json.NewDecoder(buf); dec.More(); {
p := new(jsonPackage)
if err := dec.Decode(p); err != nil {
return nil, fmt.Errorf("JSON decoding failed: %v", err)
}
addPackage(p)
}
return result, nil
}
// getDeps runs an initial go list to determine all the dependency packages.
func getDeps(cfg *rawConfig, words ...string) (originalSet map[string]*jsonPackage, deps []string, err error) {
buf, err := golist(cfg, append([]string{"list", "-e", "-json", "--"}, words...))
if err != nil {
return nil, nil, err
}
depsSet := make(map[string]bool)
originalSet = make(map[string]*jsonPackage)
// Extract deps from the JSON.
for dec := json.NewDecoder(buf); dec.More(); {
p := new(jsonPackage)
if err := dec.Decode(p); err != nil {
return nil, nil, fmt.Errorf("JSON decoding failed: %v", err)
}
originalSet[p.ImportPath] = p
for _, dep := range p.Deps {
depsSet[dep] = true
}
}
for orig := range originalSet {
delete(depsSet, orig)
}
deps = make([]string, 0, len(depsSet))
for dep := range depsSet {
deps = append(deps, dep)
}
return originalSet, deps, nil
}

130
go/packages/loader_fallback.go
View File

@ -1,130 +0,0 @@
// Copyright 2018 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// TODO(matloob): Delete this file once Go 1.12 is released.
// This file provides backwards compatibility support for
// loading for versions of Go earlier than 1.10.4. This support is meant to
// assist with migration to the Package API until there's
// widespread adoption of these newer Go versions.
// This support will be removed once Go 1.12 is released
// in Q1 2019.
// The support is incomplete. These are some of the missing
// features:
// - the Tests option has no behavior, and test packages are
// never returned.
// - Package.OtherFiles are always missing even if the package
// contains non-go sources.
package packages
import (
"fmt"
"go/build"
"strings"
legacy "golang.org/x/tools/go/loader"
"golang.org/x/tools/imports"
)
func loaderFallback(dir string, env []string, patterns []string) ([]*Package, error) {
cfg := legacy.Config{}
cfg.Cwd = dir
cfg.AllowErrors = true
cfg.FromArgs(patterns, false) // test packages are not supported
// Set build ctx
buildCtx := build.Default
for _, ev := range env {
sp := strings.Split(ev, "=")
if len(sp) != 2 {
continue
}
evar, val := sp[0], sp[1]
switch evar {
case "GOPATH":
buildCtx.GOPATH = val
case "GOROOT":
buildCtx.GOROOT = val
case "GOARCH":
buildCtx.GOARCH = val
case "GOOS":
buildCtx.GOOS = val
}
}
cfg.Build = &buildCtx
lprog, err := cfg.Load()
if err != nil {
if err.Error() == "no initial packages were loaded" {
return nil, fmt.Errorf("packages not found") // Return same error as golist-based code
}
return nil, fmt.Errorf("failed to load packages with legacy loader: %v", err)
}
allpkgs := make(map[string]*loaderPackage)
initial := make(map[*legacy.PackageInfo]bool)
for _, lpkg := range lprog.InitialPackages() {
initial[lpkg] = true
}
for _, lpkg := range lprog.AllPackages {
id := lpkg.Pkg.Path()
var goFiles []string
for _, f := range lpkg.Files {
goFiles = append(goFiles, lprog.Fset.File(f.Pos()).Name())
}
pkgimports := make(map[string]string)
for _, imppkg := range lpkg.Pkg.Imports() {
// TODO(matloob): Is the import path of a package always VendorlessPath(path)?
pkgimports[imports.VendorlessPath(imppkg.Path())] = imppkg.Path()
}
allpkgs[id] = &loaderPackage{
raw: &rawPackage{
ID: id,
Name: lpkg.Pkg.Name(),
Imports: pkgimports,
GoFiles: goFiles,
DepOnly: !initial[lpkg],
},
Package: &Package{
ID: id,
Name: lpkg.Pkg.Name(),
GoFiles: goFiles,
Fset: lprog.Fset,
Syntax: lpkg.Files,
Errors: lpkg.Errors,
Types: lpkg.Pkg,
TypesInfo: &lpkg.Info,
IllTyped: !lpkg.TransitivelyErrorFree,
OtherFiles: nil, // Never set for the fallback, because we can't extract from loader.
},
}
}
// Do a second pass to populate imports.
for _, pkg := range allpkgs {
pkg.Imports = make(map[string]*Package)
for imppath, impid := range pkg.raw.Imports {
target, ok := allpkgs[impid]
if !ok {
// return nil, fmt.Errorf("could not load package: %v", impid)
continue
}
pkg.Imports[imppath] = target.Package
}
}
// Grab the initial set of packages.
var packages []*Package
for _, lpkg := range lprog.InitialPackages() {
packages = append(packages, allpkgs[lpkg.Pkg.Path()].Package)
}
return packages, nil
}

7
go/packages/packages.go
View File

@ -262,7 +262,12 @@ func (ld *loader) load(patterns ...string) ([]*Package, error) {
rawCfg := newRawConfig(&ld.Config)
list, err := golistPackages(rawCfg, patterns...)
if _, ok := err.(GoTooOldError); ok {
return loaderFallback(ld.Dir, ld.Env, patterns)
if ld.Config.Mode >= LoadTypes {
// Upgrade to LoadAllSyntax because we can't depend on the existance
// of export data. We can remove this once iancottrell's cl is in.
ld.Config.Mode = LoadAllSyntax
}
list, err = golistPackagesFallback(rawCfg, patterns...)
}
if err != nil {
return nil, err

2
go/packages/packages110_test.go
View File

@ -7,5 +7,5 @@
package packages_test
func init() {
usesLegacyLoader = true
usesOldGolist = true
}

36
go/packages/packages_test.go
View File

@ -25,8 +25,8 @@ import (
)
// TODO(matloob): remove this once Go 1.12 is released as we will end support
// for the loader-backed implementation then.
var usesLegacyLoader = false
// for versions of go list before Go 1.10.4.
var usesOldGolist = false
// TODO(adonovan): more test cases to write:
//
@ -133,7 +133,7 @@ func TestMetadataImportGraph(t *testing.T) {
subdir/d_test [subdir/d.test] -> subdir/d [subdir/d.test]
`[1:]
if graph != wantGraph && !usesLegacyLoader {
if graph != wantGraph && !usesOldGolist {
t.Errorf("wrong import graph: got <<%s>>, want <<%s>>", graph, wantGraph)
}
@ -153,8 +153,8 @@ func TestMetadataImportGraph(t *testing.T) {
{"subdir/d.test", "main", "command", "0.go"},
{"unsafe", "unsafe", "package", ""},
} {
if usesLegacyLoader && test.id == "subdir/d.test" {
// Legacy Loader does not support tests.
if usesOldGolist && test.id == "subdir/d.test" {
// Legacy go list support does not create test main package.
continue
}
p, ok := all[test.id]
@ -187,12 +187,12 @@ func TestMetadataImportGraph(t *testing.T) {
t.Errorf("failed to obtain metadata for ad-hoc package: %s", err)
} else {
got := fmt.Sprintf("%s %s", initial[0].ID, srcs(initial[0]))
if want := "command-line-arguments [c.go]"; got != want && !usesLegacyLoader {
if want := "command-line-arguments [c.go]"; got != want && !usesOldGolist {
t.Errorf("oops: got %s, want %s", got, want)
}
}
if usesLegacyLoader {
if usesOldGolist {
// TODO(matloob): Wildcards are not yet supported.
return
}
@ -213,7 +213,7 @@ func TestMetadataImportGraph(t *testing.T) {
}
}
func TestOptionsDir_Go110(t *testing.T) {
func TestOptionsDir(t *testing.T) {
tmp, cleanup := makeTree(t, map[string]string{
"src/a/a.go": `package a; const Name = "a" `,
"src/a/b/b.go": `package b; const Name = "a/b"`,
@ -271,7 +271,7 @@ func (ec *errCollector) add(err error) {
ec.mu.Unlock()
}
func TestTypeCheckOK_Go110(t *testing.T) {
func TestTypeCheckOK(t *testing.T) {
tmp, cleanup := makeTree(t, map[string]string{
"src/a/a.go": `package a; import "b"; const A = "a" + b.B`,
"src/b/b.go": `package b; import "c"; const B = "b" + c.C`,
@ -307,7 +307,7 @@ func TestTypeCheckOK_Go110(t *testing.T) {
t.Errorf("wrong import graph: got <<%s>>, want <<%s>>", graph, wantGraph)
}
// TODO(matloob): The go/loader based support loads everything from source
// TODO(matloob): The legacy go list based support loads everything from source
// because it doesn't do a build and the .a files don't exist.
// Can we simulate its existance?
@ -322,8 +322,8 @@ func TestTypeCheckOK_Go110(t *testing.T) {
{"d", true, false}, // export data package
{"e", false, false}, // no package
} {
if usesLegacyLoader && test.id == "d" || test.id == "e" {
// legacyLoader always does a whole-program load.
if usesOldGolist && test.id == "d" || test.id == "e" {
// go list always upgrades whole-program load.
continue
}
p := all[test.id]
@ -411,8 +411,8 @@ func TestTypeCheckError(t *testing.T) {
{"d", false, false, true, nil}, // missing export data
{"e", false, false, false, nil}, // type info not requested (despite type error)
} {
if usesLegacyLoader && test.id == "c" || test.id == "d" || test.id == "e" {
// Behavior is different for legacy loader because it always loads wholeProgram.
if usesOldGolist && test.id == "c" || test.id == "d" || test.id == "e" {
// Behavior is different for old golist because it upgrades to wholeProgram.
// TODO(matloob): can we run more of this test? Can we put export data into the test GOPATH?
continue
}
@ -453,10 +453,6 @@ func TestTypeCheckError(t *testing.T) {
// This function tests use of the ParseFile hook to supply
// alternative file contents to the parser and type-checker.
func TestWholeProgramOverlay(t *testing.T) {
if usesLegacyLoader {
t.Skip("not yet supported in go/loader based implementation")
}
type M = map[string]string
tmp, cleanup := makeTree(t, M{
@ -517,8 +513,8 @@ func TestWholeProgramOverlay(t *testing.T) {
}
func TestWholeProgramImportErrors(t *testing.T) {
if usesLegacyLoader {
t.Skip("not yet supported in go/loader based implementation")
if usesOldGolist {
t.Skip("not yet supported in pre-Go 1.10.4 golist fallback implementation")
}
tmp, cleanup := makeTree(t, map[string]string{