cmd/godoc: support automatic vendoring

Fixes golang/go#35429

Change-Id: I060ccfbed4c3975d1ddc94fda4fadea527b29841
Reviewed-on: https://go-review.googlesource.com/c/tools/+/232958
Run-TryBot: Agniva De Sarker <agniva.quicksilver@gmail.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Dmitri Shuralyov <dmitshur@golang.org>

cmd/godoc/main.go

@@ -19,6 +19,7 @@ package main
 import (
 	"archive/zip"
 	"bytes"
+	"context"
 	"encoding/json"
 	_ "expvar" // to serve /debug/vars
 	"flag"
@@ -44,6 +45,7 @@ import (
 	"golang.org/x/tools/godoc/vfs/gatefs"
 	"golang.org/x/tools/godoc/vfs/mapfs"
 	"golang.org/x/tools/godoc/vfs/zipfs"
+	"golang.org/x/tools/internal/gocommand"
 	"golang.org/x/xerrors"
 )
 
@@ -210,28 +212,48 @@ func main() {
 			usage()
 		}
 
-		// Try to download dependencies that are not in the module cache in order to
+		// Detect whether to use vendor mode or not.
-		// to show their documentation.
+		mainMod, vendorEnabled, err := gocommand.VendorEnabled(context.Background(), gocommand.Invocation{}, &gocommand.Runner{})
-		// This may fail if module downloading is disallowed (GOPROXY=off) or due to
-		// limited connectivity, in which case we print errors to stderr and show
-		// documentation only for packages that are available.
-		fillModuleCache(os.Stderr, goModFile)
-
-		// Determine modules in the build list.
-		mods, err := buildList(goModFile)
 		if err != nil {
-			fmt.Fprintf(os.Stderr, "failed to determine the build list of the main module: %v", err)
+			fmt.Fprintf(os.Stderr, "failed to determine if vendoring is enabled: %v", err)
 			os.Exit(1)
 		}
+		if vendorEnabled {
+			// Bind the root directory of the main module.
+			fs.Bind(path.Join("/src", mainMod.Path), gatefs.New(vfs.OS(mainMod.Dir), fsGate), "/", vfs.BindAfter)
 
-		// Bind module trees into Go root.
+			// Bind the vendor directory.
-		for _, m := range mods {
+			//
-			if m.Dir == "" {
+			// Note that in module mode, vendor directories in locations
-				// Module is not available in the module cache, skip it.
+			// other than the main module's root directory are ignored.
-				continue
+			// See https://golang.org/ref/mod#vendoring.
+			vendorDir := filepath.Join(mainMod.Dir, "vendor")
+			fs.Bind("/src", gatefs.New(vfs.OS(vendorDir), fsGate), "/", vfs.BindAfter)
+
+		} else {
+			// Try to download dependencies that are not in the module cache in order to
+			// to show their documentation.
+			// This may fail if module downloading is disallowed (GOPROXY=off) or due to
+			// limited connectivity, in which case we print errors to stderr and show
+			// documentation only for packages that are available.
+			fillModuleCache(os.Stderr, goModFile)
+
+			// Determine modules in the build list.
+			mods, err := buildList(goModFile)
+			if err != nil {
+				fmt.Fprintf(os.Stderr, "failed to determine the build list of the main module: %v", err)
+				os.Exit(1)
+			}
+
+			// Bind module trees into Go root.
+			for _, m := range mods {
+				if m.Dir == "" {
+					// Module is not available in the module cache, skip it.
+					continue
+				}
+				dst := path.Join("/src", m.Path)
+				fs.Bind(dst, gatefs.New(vfs.OS(m.Dir), fsGate), "/", vfs.BindAfter)
 			}
-			dst := path.Join("/src", m.Path)
-			fs.Bind(dst, gatefs.New(vfs.OS(m.Dir), fsGate), "/", vfs.BindAfter)
 		}
 	} else {
 		fmt.Println("using GOPATH mode")
@@ -395,7 +417,7 @@ func goMod() (string, error) {
 // with all dependencies of the main module in the current directory
 // by invoking the go command. Module download logs are streamed to w.
 // If there are any problems encountered, they are also written to w.
-// It should only be used when operating in module mode.
+// It should only be used in module mode, when vendor mode isn't on.
 //
 // See https://golang.org/cmd/go/#hdr-Download_modules_to_local_cache.
 func fillModuleCache(w io.Writer, goMod string) {
@@ -436,9 +458,14 @@ func fillModuleCache(w io.Writer, goMod string) {
 	}
 }
 
+type mod struct {
+	Path string // Module path.
+	Dir  string // Directory holding files for this module, if any.
+}
+
 // buildList determines the build list in the current directory
-// by invoking the go command. It should only be used when operating
+// by invoking the go command. It should only be used in module mode,
-// in module mode.
+// when vendor mode isn't on.
 //
 // See https://golang.org/cmd/go/#hdr-The_main_module_and_the_build_list.
 func buildList(goMod string) ([]mod, error) {
@@ -467,11 +494,6 @@ func buildList(goMod string) ([]mod, error) {
 	return mods, nil
 }
 
-type mod struct {
-	Path string // Module path.
-	Dir  string // Directory holding files for this module, if any.
-}
-
 // moduleFS is a vfs.FileSystem wrapper used when godoc is running
 // in module mode. It's needed so that packages inside modules are
 // considered to be third party.

internal/gocommand/vendor.go

@@ -0,0 +1,102 @@
+// Copyright 2020 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package gocommand
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strings"
+
+	"golang.org/x/mod/semver"
+)
+
+// ModuleJSON holds information about a module.
+type ModuleJSON struct {
+	Path      string      // module path
+	Replace   *ModuleJSON // replaced by this module
+	Main      bool        // is this the main module?
+	Indirect  bool        // is this module only an indirect dependency of main module?
+	Dir       string      // directory holding files for this module, if any
+	GoMod     string      // path to go.mod file for this module, if any
+	GoVersion string      // go version used in module
+}
+
+var modFlagRegexp = regexp.MustCompile(`-mod[ =](\w+)`)
+
+// VendorEnabled reports whether vendoring is enabled. It takes a *Runner to execute Go commands
+// with the supplied context.Context and Invocation. The Invocation can contain pre-defined fields,
+// of which only Verb and Args are modified to run the appropriate Go command.
+// Inspired by setDefaultBuildMod in modload/init.go
+func VendorEnabled(ctx context.Context, inv Invocation, r *Runner) (*ModuleJSON, bool, error) {
+	mainMod, go114, err := getMainModuleAnd114(ctx, inv, r)
+	if err != nil {
+		return nil, false, err
+	}
+
+	// We check the GOFLAGS to see if there is anything overridden or not.
+	inv.Verb = "env"
+	inv.Args = []string{"GOFLAGS"}
+	stdout, err := r.Run(ctx, inv)
+	if err != nil {
+		return nil, false, err
+	}
+	goflags := string(bytes.TrimSpace(stdout.Bytes()))
+	matches := modFlagRegexp.FindStringSubmatch(goflags)
+	var modFlag string
+	if len(matches) != 0 {
+		modFlag = matches[1]
+	}
+	if modFlag != "" {
+		// Don't override an explicit '-mod=' argument.
+		return mainMod, modFlag == "vendor", nil
+	}
+	if mainMod == nil || !go114 {
+		return mainMod, false, nil
+	}
+	// Check 1.14's automatic vendor mode.
+	if fi, err := os.Stat(filepath.Join(mainMod.Dir, "vendor")); err == nil && fi.IsDir() {
+		if mainMod.GoVersion != "" && semver.Compare("v"+mainMod.GoVersion, "v1.14") >= 0 {
+			// The Go version is at least 1.14, and a vendor directory exists.
+			// Set -mod=vendor by default.
+			return mainMod, true, nil
+		}
+	}
+	return mainMod, false, nil
+}
+
+// getMainModuleAnd114 gets the main module's information and whether the
+// go command in use is 1.14+. This is the information needed to figure out
+// if vendoring should be enabled.
+func getMainModuleAnd114(ctx context.Context, inv Invocation, r *Runner) (*ModuleJSON, bool, error) {
+	const format = `{{.Path}}
+{{.Dir}}
+{{.GoMod}}
+{{.GoVersion}}
+{{range context.ReleaseTags}}{{if eq . "go1.14"}}{{.}}{{end}}{{end}}
+`
+	inv.Verb = "list"
+	inv.Args = []string{"-m", "-f", format}
+	stdout, err := r.Run(ctx, inv)
+	if err != nil {
+		return nil, false, err
+	}
+
+	lines := strings.Split(stdout.String(), "\n")
+	if len(lines) < 5 {
+		return nil, false, fmt.Errorf("unexpected stdout: %q", stdout.String())
+	}
+	mod := &ModuleJSON{
+		Path:      lines[0],
+		Dir:       lines[1],
+		GoMod:     lines[2],
+		GoVersion: lines[3],
+		Main:      true,
+	}
+	return mod, lines[4] == "go1.14", nil
+}

internal/imports/mod.go

@@ -15,7 +15,7 @@ import (
 	"strings"
 
 	"golang.org/x/mod/module"
-	"golang.org/x/mod/semver"
+	"golang.org/x/tools/internal/gocommand"
 	"golang.org/x/tools/internal/gopathwalk"
 )
 
@@ -24,31 +24,21 @@ import (
 type ModuleResolver struct {
 	env            *ProcessEnv
 	moduleCacheDir string
-	dummyVendorMod *ModuleJSON // If vendoring is enabled, the pseudo-module that represents the /vendor directory.
+	dummyVendorMod *gocommand.ModuleJSON // If vendoring is enabled, the pseudo-module that represents the /vendor directory.
 	roots          []gopathwalk.Root
 	scanSema       chan struct{} // scanSema prevents concurrent scans and guards scannedRoots.
 	scannedRoots   map[gopathwalk.Root]bool
 
 	initialized   bool
-	main          *ModuleJSON
+	main          *gocommand.ModuleJSON
-	modsByModPath []*ModuleJSON // All modules, ordered by # of path components in module Path...
+	modsByModPath []*gocommand.ModuleJSON // All modules, ordered by # of path components in module Path...
-	modsByDir     []*ModuleJSON // ...or Dir.
+	modsByDir     []*gocommand.ModuleJSON // ...or Dir.
 
 	// moduleCacheCache stores information about the module cache.
 	moduleCacheCache *dirInfoCache
 	otherCache       *dirInfoCache
 }
 
-type ModuleJSON struct {
-	Path      string      // module path
-	Replace   *ModuleJSON // replaced by this module
-	Main      bool        // is this the main module?
-	Indirect  bool        // is this module only an indirect dependency of main module?
-	Dir       string      // directory holding files for this module, if any
-	GoMod     string      // path to go.mod file for this module, if any
-	GoVersion string      // go version used in module
-}
-
 func newModuleResolver(e *ProcessEnv) *ModuleResolver {
 	r := &ModuleResolver{
 		env:      e,
@@ -62,7 +52,14 @@ func (r *ModuleResolver) init() error {
 	if r.initialized {
 		return nil
 	}
-	mainMod, vendorEnabled, err := vendorEnabled(r.env)
+
+	inv := gocommand.Invocation{
+		BuildFlags: r.env.BuildFlags,
+		Env:        r.env.env(),
+		Logf:       r.env.Logf,
+		WorkingDir: r.env.WorkingDir,
+	}
+	mainMod, vendorEnabled, err := gocommand.VendorEnabled(context.TODO(), inv, r.env.GocmdRunner)
 	if err != nil {
 		return err
 	}
@@ -71,12 +68,12 @@ func (r *ModuleResolver) init() error {
 		// Vendor mode is on, so all the non-Main modules are irrelevant,
 		// and we need to search /vendor for everything.
 		r.main = mainMod
-		r.dummyVendorMod = &ModuleJSON{
+		r.dummyVendorMod = &gocommand.ModuleJSON{
 			Path: "",
 			Dir:  filepath.Join(mainMod.Dir, "vendor"),
 		}
-		r.modsByModPath = []*ModuleJSON{mainMod, r.dummyVendorMod}
+		r.modsByModPath = []*gocommand.ModuleJSON{mainMod, r.dummyVendorMod}
-		r.modsByDir = []*ModuleJSON{mainMod, r.dummyVendorMod}
+		r.modsByDir = []*gocommand.ModuleJSON{mainMod, r.dummyVendorMod}
 	} else {
 		// Vendor mode is off, so run go list -m ... to find everything.
 		r.initAllMods()
@@ -106,7 +103,7 @@ func (r *ModuleResolver) init() error {
 	if vendorEnabled {
 		r.roots = append(r.roots, gopathwalk.Root{r.dummyVendorMod.Dir, gopathwalk.RootOther})
 	} else {
-		addDep := func(mod *ModuleJSON) {
+		addDep := func(mod *gocommand.ModuleJSON) {
 			if mod.Replace == nil {
 				// This is redundant with the cache, but we'll skip it cheaply enough.
 				r.roots = append(r.roots, gopathwalk.Root{mod.Dir, gopathwalk.RootModuleCache})
@@ -151,7 +148,7 @@ func (r *ModuleResolver) initAllMods() error {
 		return err
 	}
 	for dec := json.NewDecoder(stdout); dec.More(); {
-		mod := &ModuleJSON{}
+		mod := &gocommand.ModuleJSON{}
 		if err := dec.Decode(mod); err != nil {
 			return err
 		}
@@ -197,7 +194,7 @@ func (r *ModuleResolver) ClearForNewMod() {
 
 // findPackage returns the module and directory that contains the package at
 // the given import path, or returns nil, "" if no module is in scope.
-func (r *ModuleResolver) findPackage(importPath string) (*ModuleJSON, string) {
+func (r *ModuleResolver) findPackage(importPath string) (*gocommand.ModuleJSON, string) {
 	// This can't find packages in the stdlib, but that's harmless for all
 	// the existing code paths.
 	for _, m := range r.modsByModPath {
@@ -283,7 +280,7 @@ func (r *ModuleResolver) cacheExports(ctx context.Context, env *ProcessEnv, info
 
 // findModuleByDir returns the module that contains dir, or nil if no such
 // module is in scope.
-func (r *ModuleResolver) findModuleByDir(dir string) *ModuleJSON {
+func (r *ModuleResolver) findModuleByDir(dir string) *gocommand.ModuleJSON {
 	// This is quite tricky and may not be correct. dir could be:
 	// - a package in the main module.
 	// - a replace target underneath the main module's directory.
@@ -310,7 +307,7 @@ func (r *ModuleResolver) findModuleByDir(dir string) *ModuleJSON {
 
 // dirIsNestedModule reports if dir is contained in a nested module underneath
 // mod, not actually in mod.
-func (r *ModuleResolver) dirIsNestedModule(dir string, mod *ModuleJSON) bool {
+func (r *ModuleResolver) dirIsNestedModule(dir string, mod *gocommand.ModuleJSON) bool {
 	if !strings.HasPrefix(dir, mod.Dir) {
 		return false
 	}
@@ -490,7 +487,7 @@ func (r *ModuleResolver) scoreImportPath(ctx context.Context, path string) int {
 	return modRelevance(mod)
 }
 
-func modRelevance(mod *ModuleJSON) int {
+func modRelevance(mod *gocommand.ModuleJSON) int {
 	switch {
 	case mod == nil: // out of scope
 		return MaxRelevance - 4
@@ -656,63 +653,3 @@ func modulePath(mod []byte) string {
 	}
 	return "" // missing module path
 }
-
-var modFlagRegexp = regexp.MustCompile(`-mod[ =](\w+)`)
-
-// vendorEnabled indicates if vendoring is enabled.
-// Inspired by setDefaultBuildMod in modload/init.go
-func vendorEnabled(env *ProcessEnv) (*ModuleJSON, bool, error) {
-	mainMod, go114, err := getMainModuleAnd114(env)
-	if err != nil {
-		return nil, false, err
-	}
-	matches := modFlagRegexp.FindStringSubmatch(env.GOFLAGS)
-	var modFlag string
-	if len(matches) != 0 {
-		modFlag = matches[1]
-	}
-	if modFlag != "" {
-		// Don't override an explicit '-mod=' argument.
-		return mainMod, modFlag == "vendor", nil
-	}
-	if mainMod == nil || !go114 {
-		return mainMod, false, nil
-	}
-	// Check 1.14's automatic vendor mode.
-	if fi, err := os.Stat(filepath.Join(mainMod.Dir, "vendor")); err == nil && fi.IsDir() {
-		if mainMod.GoVersion != "" && semver.Compare("v"+mainMod.GoVersion, "v1.14") >= 0 {
-			// The Go version is at least 1.14, and a vendor directory exists.
-			// Set -mod=vendor by default.
-			return mainMod, true, nil
-		}
-	}
-	return mainMod, false, nil
-}
-
-// getMainModuleAnd114 gets the main module's information and whether the
-// go command in use is 1.14+. This is the information needed to figure out
-// if vendoring should be enabled.
-func getMainModuleAnd114(env *ProcessEnv) (*ModuleJSON, bool, error) {
-	const format = `{{.Path}}
-{{.Dir}}
-{{.GoMod}}
-{{.GoVersion}}
-{{range context.ReleaseTags}}{{if eq . "go1.14"}}{{.}}{{end}}{{end}}
-`
-	stdout, err := env.invokeGo(context.TODO(), "list", "-m", "-f", format)
-	if err != nil {
-		return nil, false, nil
-	}
-	lines := strings.Split(stdout.String(), "\n")
-	if len(lines) < 5 {
-		return nil, false, fmt.Errorf("unexpected stdout: %q", stdout)
-	}
-	mod := &ModuleJSON{
-		Path:      lines[0],
-		Dir:       lines[1],
-		GoMod:     lines[2],
-		GoVersion: lines[3],
-		Main:      true,
-	}
-	return mod, lines[4] == "go1.14", nil
-}