From 6eec81c74686c6d39f20af32b5ce4c9786acedba Mon Sep 17 00:00:00 2001 From: Agniva De Sarker Date: Sat, 9 May 2020 23:19:08 +0530 Subject: [PATCH] cmd/godoc: support automatic vendoring Fixes golang/go#35429 Change-Id: I060ccfbed4c3975d1ddc94fda4fadea527b29841 Reviewed-on: https://go-review.googlesource.com/c/tools/+/232958 Run-TryBot: Agniva De Sarker TryBot-Result: Gobot Gobot Reviewed-by: Dmitri Shuralyov --- cmd/godoc/main.go | 72 +++++++++++++++-------- internal/gocommand/vendor.go | 102 +++++++++++++++++++++++++++++++++ internal/imports/mod.go | 107 +++++++---------------------------- 3 files changed, 171 insertions(+), 110 deletions(-) create mode 100644 internal/gocommand/vendor.go diff --git a/cmd/godoc/main.go b/cmd/godoc/main.go index 48a658e092..7dba4d20e1 100644 --- a/cmd/godoc/main.go +++ b/cmd/godoc/main.go @@ -19,6 +19,7 @@ package main import ( "archive/zip" "bytes" + "context" "encoding/json" _ "expvar" // to serve /debug/vars "flag" @@ -44,6 +45,7 @@ import ( "golang.org/x/tools/godoc/vfs/gatefs" "golang.org/x/tools/godoc/vfs/mapfs" "golang.org/x/tools/godoc/vfs/zipfs" + "golang.org/x/tools/internal/gocommand" "golang.org/x/xerrors" ) @@ -210,28 +212,48 @@ func main() { usage() } - // Try to download dependencies that are not in the module cache in order to - // to show their documentation. - // This may fail if module downloading is disallowed (GOPROXY=off) or due to - // limited connectivity, in which case we print errors to stderr and show - // documentation only for packages that are available. - fillModuleCache(os.Stderr, goModFile) - - // Determine modules in the build list. - mods, err := buildList(goModFile) + // Detect whether to use vendor mode or not. + mainMod, vendorEnabled, err := gocommand.VendorEnabled(context.Background(), gocommand.Invocation{}, &gocommand.Runner{}) if err != nil { - fmt.Fprintf(os.Stderr, "failed to determine the build list of the main module: %v", err) + fmt.Fprintf(os.Stderr, "failed to determine if vendoring is enabled: %v", err) os.Exit(1) } + if vendorEnabled { + // Bind the root directory of the main module. + fs.Bind(path.Join("/src", mainMod.Path), gatefs.New(vfs.OS(mainMod.Dir), fsGate), "/", vfs.BindAfter) - // Bind module trees into Go root. - for _, m := range mods { - if m.Dir == "" { - // Module is not available in the module cache, skip it. - continue + // Bind the vendor directory. + // + // Note that in module mode, vendor directories in locations + // other than the main module's root directory are ignored. + // See https://golang.org/ref/mod#vendoring. + vendorDir := filepath.Join(mainMod.Dir, "vendor") + fs.Bind("/src", gatefs.New(vfs.OS(vendorDir), fsGate), "/", vfs.BindAfter) + + } else { + // Try to download dependencies that are not in the module cache in order to + // to show their documentation. + // This may fail if module downloading is disallowed (GOPROXY=off) or due to + // limited connectivity, in which case we print errors to stderr and show + // documentation only for packages that are available. + fillModuleCache(os.Stderr, goModFile) + + // Determine modules in the build list. + mods, err := buildList(goModFile) + if err != nil { + fmt.Fprintf(os.Stderr, "failed to determine the build list of the main module: %v", err) + os.Exit(1) + } + + // Bind module trees into Go root. + for _, m := range mods { + if m.Dir == "" { + // Module is not available in the module cache, skip it. + continue + } + dst := path.Join("/src", m.Path) + fs.Bind(dst, gatefs.New(vfs.OS(m.Dir), fsGate), "/", vfs.BindAfter) } - dst := path.Join("/src", m.Path) - fs.Bind(dst, gatefs.New(vfs.OS(m.Dir), fsGate), "/", vfs.BindAfter) } } else { fmt.Println("using GOPATH mode") @@ -395,7 +417,7 @@ func goMod() (string, error) { // with all dependencies of the main module in the current directory // by invoking the go command. Module download logs are streamed to w. // If there are any problems encountered, they are also written to w. -// It should only be used when operating in module mode. +// It should only be used in module mode, when vendor mode isn't on. // // See https://golang.org/cmd/go/#hdr-Download_modules_to_local_cache. func fillModuleCache(w io.Writer, goMod string) { @@ -436,9 +458,14 @@ func fillModuleCache(w io.Writer, goMod string) { } } +type mod struct { + Path string // Module path. + Dir string // Directory holding files for this module, if any. +} + // buildList determines the build list in the current directory -// by invoking the go command. It should only be used when operating -// in module mode. +// by invoking the go command. It should only be used in module mode, +// when vendor mode isn't on. // // See https://golang.org/cmd/go/#hdr-The_main_module_and_the_build_list. func buildList(goMod string) ([]mod, error) { @@ -467,11 +494,6 @@ func buildList(goMod string) ([]mod, error) { return mods, nil } -type mod struct { - Path string // Module path. - Dir string // Directory holding files for this module, if any. -} - // moduleFS is a vfs.FileSystem wrapper used when godoc is running // in module mode. It's needed so that packages inside modules are // considered to be third party. diff --git a/internal/gocommand/vendor.go b/internal/gocommand/vendor.go new file mode 100644 index 0000000000..1cd8d8473e --- /dev/null +++ b/internal/gocommand/vendor.go @@ -0,0 +1,102 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package gocommand + +import ( + "bytes" + "context" + "fmt" + "os" + "path/filepath" + "regexp" + "strings" + + "golang.org/x/mod/semver" +) + +// ModuleJSON holds information about a module. +type ModuleJSON struct { + Path string // module path + Replace *ModuleJSON // replaced by this module + Main bool // is this the main module? + Indirect bool // is this module only an indirect dependency of main module? + Dir string // directory holding files for this module, if any + GoMod string // path to go.mod file for this module, if any + GoVersion string // go version used in module +} + +var modFlagRegexp = regexp.MustCompile(`-mod[ =](\w+)`) + +// VendorEnabled reports whether vendoring is enabled. It takes a *Runner to execute Go commands +// with the supplied context.Context and Invocation. The Invocation can contain pre-defined fields, +// of which only Verb and Args are modified to run the appropriate Go command. +// Inspired by setDefaultBuildMod in modload/init.go +func VendorEnabled(ctx context.Context, inv Invocation, r *Runner) (*ModuleJSON, bool, error) { + mainMod, go114, err := getMainModuleAnd114(ctx, inv, r) + if err != nil { + return nil, false, err + } + + // We check the GOFLAGS to see if there is anything overridden or not. + inv.Verb = "env" + inv.Args = []string{"GOFLAGS"} + stdout, err := r.Run(ctx, inv) + if err != nil { + return nil, false, err + } + goflags := string(bytes.TrimSpace(stdout.Bytes())) + matches := modFlagRegexp.FindStringSubmatch(goflags) + var modFlag string + if len(matches) != 0 { + modFlag = matches[1] + } + if modFlag != "" { + // Don't override an explicit '-mod=' argument. + return mainMod, modFlag == "vendor", nil + } + if mainMod == nil || !go114 { + return mainMod, false, nil + } + // Check 1.14's automatic vendor mode. + if fi, err := os.Stat(filepath.Join(mainMod.Dir, "vendor")); err == nil && fi.IsDir() { + if mainMod.GoVersion != "" && semver.Compare("v"+mainMod.GoVersion, "v1.14") >= 0 { + // The Go version is at least 1.14, and a vendor directory exists. + // Set -mod=vendor by default. + return mainMod, true, nil + } + } + return mainMod, false, nil +} + +// getMainModuleAnd114 gets the main module's information and whether the +// go command in use is 1.14+. This is the information needed to figure out +// if vendoring should be enabled. +func getMainModuleAnd114(ctx context.Context, inv Invocation, r *Runner) (*ModuleJSON, bool, error) { + const format = `{{.Path}} +{{.Dir}} +{{.GoMod}} +{{.GoVersion}} +{{range context.ReleaseTags}}{{if eq . "go1.14"}}{{.}}{{end}}{{end}} +` + inv.Verb = "list" + inv.Args = []string{"-m", "-f", format} + stdout, err := r.Run(ctx, inv) + if err != nil { + return nil, false, err + } + + lines := strings.Split(stdout.String(), "\n") + if len(lines) < 5 { + return nil, false, fmt.Errorf("unexpected stdout: %q", stdout.String()) + } + mod := &ModuleJSON{ + Path: lines[0], + Dir: lines[1], + GoMod: lines[2], + GoVersion: lines[3], + Main: true, + } + return mod, lines[4] == "go1.14", nil +} diff --git a/internal/imports/mod.go b/internal/imports/mod.go index 69e3eecc4c..4e816e8bcf 100644 --- a/internal/imports/mod.go +++ b/internal/imports/mod.go @@ -15,7 +15,7 @@ import ( "strings" "golang.org/x/mod/module" - "golang.org/x/mod/semver" + "golang.org/x/tools/internal/gocommand" "golang.org/x/tools/internal/gopathwalk" ) @@ -24,31 +24,21 @@ import ( type ModuleResolver struct { env *ProcessEnv moduleCacheDir string - dummyVendorMod *ModuleJSON // If vendoring is enabled, the pseudo-module that represents the /vendor directory. + dummyVendorMod *gocommand.ModuleJSON // If vendoring is enabled, the pseudo-module that represents the /vendor directory. roots []gopathwalk.Root scanSema chan struct{} // scanSema prevents concurrent scans and guards scannedRoots. scannedRoots map[gopathwalk.Root]bool initialized bool - main *ModuleJSON - modsByModPath []*ModuleJSON // All modules, ordered by # of path components in module Path... - modsByDir []*ModuleJSON // ...or Dir. + main *gocommand.ModuleJSON + modsByModPath []*gocommand.ModuleJSON // All modules, ordered by # of path components in module Path... + modsByDir []*gocommand.ModuleJSON // ...or Dir. // moduleCacheCache stores information about the module cache. moduleCacheCache *dirInfoCache otherCache *dirInfoCache } -type ModuleJSON struct { - Path string // module path - Replace *ModuleJSON // replaced by this module - Main bool // is this the main module? - Indirect bool // is this module only an indirect dependency of main module? - Dir string // directory holding files for this module, if any - GoMod string // path to go.mod file for this module, if any - GoVersion string // go version used in module -} - func newModuleResolver(e *ProcessEnv) *ModuleResolver { r := &ModuleResolver{ env: e, @@ -62,7 +52,14 @@ func (r *ModuleResolver) init() error { if r.initialized { return nil } - mainMod, vendorEnabled, err := vendorEnabled(r.env) + + inv := gocommand.Invocation{ + BuildFlags: r.env.BuildFlags, + Env: r.env.env(), + Logf: r.env.Logf, + WorkingDir: r.env.WorkingDir, + } + mainMod, vendorEnabled, err := gocommand.VendorEnabled(context.TODO(), inv, r.env.GocmdRunner) if err != nil { return err } @@ -71,12 +68,12 @@ func (r *ModuleResolver) init() error { // Vendor mode is on, so all the non-Main modules are irrelevant, // and we need to search /vendor for everything. r.main = mainMod - r.dummyVendorMod = &ModuleJSON{ + r.dummyVendorMod = &gocommand.ModuleJSON{ Path: "", Dir: filepath.Join(mainMod.Dir, "vendor"), } - r.modsByModPath = []*ModuleJSON{mainMod, r.dummyVendorMod} - r.modsByDir = []*ModuleJSON{mainMod, r.dummyVendorMod} + r.modsByModPath = []*gocommand.ModuleJSON{mainMod, r.dummyVendorMod} + r.modsByDir = []*gocommand.ModuleJSON{mainMod, r.dummyVendorMod} } else { // Vendor mode is off, so run go list -m ... to find everything. r.initAllMods() @@ -106,7 +103,7 @@ func (r *ModuleResolver) init() error { if vendorEnabled { r.roots = append(r.roots, gopathwalk.Root{r.dummyVendorMod.Dir, gopathwalk.RootOther}) } else { - addDep := func(mod *ModuleJSON) { + addDep := func(mod *gocommand.ModuleJSON) { if mod.Replace == nil { // This is redundant with the cache, but we'll skip it cheaply enough. r.roots = append(r.roots, gopathwalk.Root{mod.Dir, gopathwalk.RootModuleCache}) @@ -151,7 +148,7 @@ func (r *ModuleResolver) initAllMods() error { return err } for dec := json.NewDecoder(stdout); dec.More(); { - mod := &ModuleJSON{} + mod := &gocommand.ModuleJSON{} if err := dec.Decode(mod); err != nil { return err } @@ -197,7 +194,7 @@ func (r *ModuleResolver) ClearForNewMod() { // findPackage returns the module and directory that contains the package at // the given import path, or returns nil, "" if no module is in scope. -func (r *ModuleResolver) findPackage(importPath string) (*ModuleJSON, string) { +func (r *ModuleResolver) findPackage(importPath string) (*gocommand.ModuleJSON, string) { // This can't find packages in the stdlib, but that's harmless for all // the existing code paths. for _, m := range r.modsByModPath { @@ -283,7 +280,7 @@ func (r *ModuleResolver) cacheExports(ctx context.Context, env *ProcessEnv, info // findModuleByDir returns the module that contains dir, or nil if no such // module is in scope. -func (r *ModuleResolver) findModuleByDir(dir string) *ModuleJSON { +func (r *ModuleResolver) findModuleByDir(dir string) *gocommand.ModuleJSON { // This is quite tricky and may not be correct. dir could be: // - a package in the main module. // - a replace target underneath the main module's directory. @@ -310,7 +307,7 @@ func (r *ModuleResolver) findModuleByDir(dir string) *ModuleJSON { // dirIsNestedModule reports if dir is contained in a nested module underneath // mod, not actually in mod. -func (r *ModuleResolver) dirIsNestedModule(dir string, mod *ModuleJSON) bool { +func (r *ModuleResolver) dirIsNestedModule(dir string, mod *gocommand.ModuleJSON) bool { if !strings.HasPrefix(dir, mod.Dir) { return false } @@ -490,7 +487,7 @@ func (r *ModuleResolver) scoreImportPath(ctx context.Context, path string) int { return modRelevance(mod) } -func modRelevance(mod *ModuleJSON) int { +func modRelevance(mod *gocommand.ModuleJSON) int { switch { case mod == nil: // out of scope return MaxRelevance - 4 @@ -656,63 +653,3 @@ func modulePath(mod []byte) string { } return "" // missing module path } - -var modFlagRegexp = regexp.MustCompile(`-mod[ =](\w+)`) - -// vendorEnabled indicates if vendoring is enabled. -// Inspired by setDefaultBuildMod in modload/init.go -func vendorEnabled(env *ProcessEnv) (*ModuleJSON, bool, error) { - mainMod, go114, err := getMainModuleAnd114(env) - if err != nil { - return nil, false, err - } - matches := modFlagRegexp.FindStringSubmatch(env.GOFLAGS) - var modFlag string - if len(matches) != 0 { - modFlag = matches[1] - } - if modFlag != "" { - // Don't override an explicit '-mod=' argument. - return mainMod, modFlag == "vendor", nil - } - if mainMod == nil || !go114 { - return mainMod, false, nil - } - // Check 1.14's automatic vendor mode. - if fi, err := os.Stat(filepath.Join(mainMod.Dir, "vendor")); err == nil && fi.IsDir() { - if mainMod.GoVersion != "" && semver.Compare("v"+mainMod.GoVersion, "v1.14") >= 0 { - // The Go version is at least 1.14, and a vendor directory exists. - // Set -mod=vendor by default. - return mainMod, true, nil - } - } - return mainMod, false, nil -} - -// getMainModuleAnd114 gets the main module's information and whether the -// go command in use is 1.14+. This is the information needed to figure out -// if vendoring should be enabled. -func getMainModuleAnd114(env *ProcessEnv) (*ModuleJSON, bool, error) { - const format = `{{.Path}} -{{.Dir}} -{{.GoMod}} -{{.GoVersion}} -{{range context.ReleaseTags}}{{if eq . "go1.14"}}{{.}}{{end}}{{end}} -` - stdout, err := env.invokeGo(context.TODO(), "list", "-m", "-f", format) - if err != nil { - return nil, false, nil - } - lines := strings.Split(stdout.String(), "\n") - if len(lines) < 5 { - return nil, false, fmt.Errorf("unexpected stdout: %q", stdout) - } - mod := &ModuleJSON{ - Path: lines[0], - Dir: lines[1], - GoMod: lines[2], - GoVersion: lines[3], - Main: true, - } - return mod, lines[4] == "go1.14", nil -}