[release-branch.go1.10] cmd/go: add minimal module-awareness for legacy operation

We want authors to be able to publish code that works with both
the current standard go command and the planned new go command
support for modules. If authors have tagged their code v2 or later,
semantic import versioning means the import paths must include a
v2 path element after the path prefix naming the module.
One option for making this convention compatible with original go get
is to move code into a v2 subdirectory of the root.
That makes sense for some authors, but many authors would prefer
not to move all the code into a v2 subdirectory for a transition and
then move it back up once we everyone has a module-aware go command.

Instead, this CL teaches the old (non-module-aware) go command
a tiny amount about modules and their import paths, to expand
the options for authors who want to publish compatible packages.
If an author has a v2 of a package, say my/thing/v2/sub/pkg,
in the my/thing repo's sub/pkg subdirectory (no v2 in the file system path),
then old go get continues to import that package as my/thing/sub/pkg.
But when go get is processing code in any module (code in a tree with
a go.mod file) and encounters a path like my/thing/v2/sub/pkg,
it will check to see if my/thing/go.mod says "module my/thing/v2".
If so, the go command will read the import my/thing/v2/sub/pkg
as if it said my/thing/sub/pkg, which is the correct "old" import path
for the package in question.

This CL will be back-ported to Go 1.10 and Go 1.9 as well.

Once users have updated to the latest Go point releases containing
this new logic, authors will be able to update to using modules
within their own repos, including using semantic import paths
with vN path elements, and old go get will still be able to consume
those repositories.

This CL also makes "go get" ignore meta go-import lines using
the new "mod" VCS type. This allows a package to specify both
a "mod" type and a "git" type, to present more efficient module
access to module-aware go but still present a Git repo to the old
"go get".

Fixes #24751.
Fixes #25069.

Backport fixes #25139.

Change-Id: I378955613a0d63834d4f50f121f4db7e4d87dc0a
Reviewed-on: https://go-review.googlesource.com/114500
Run-TryBot: Russ Cox <rsc@golang.org>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Bryan C. Mills <bcmills@google.com>

src/cmd/go/internal/get/discovery.go

@@ -55,6 +55,13 @@ func parseMetaGoImports(r io.Reader) (imports []metaImport, err error) {
 			continue
 		}
 		if f := strings.Fields(attrValue(e.Attr, "content")); len(f) == 3 {
+			// Ignore VCS type "mod", which is new Go modules.
+			// This code is for old go get and must ignore the new mod lines.
+			// Otherwise matchGoImport will complain about two
+			// different metaImport lines for the same Prefix.
+			if f[1] == "mod" {
+				continue
+			}
 			imports = append(imports, metaImport{
 				Prefix:   f[0],
 				VCS:      f[1],

src/cmd/go/internal/get/get.go

@@ -209,7 +209,7 @@ var downloadRootCache = map[string]bool{}
 // download runs the download half of the get command
 // for the package named by the argument.
 func download(arg string, parent *load.Package, stk *load.ImportStack, mode int) {
-	if mode&load.UseVendor != 0 {
+	if mode&load.ResolveImport != 0 {
 		// Caller is responsible for expanding vendor paths.
 		panic("internal error: download mode has useVendor set")
 	}

src/cmd/go/internal/get/pkg_test.go

@@ -47,6 +47,20 @@ var parseMetaGoImportsTests = []struct {
 			{"baz/quux", "git", "http://github.com/rsc/baz/quux"},
 		},
 	},
+	{
+		`<meta name="go-import" content="foo/bar git https://github.com/rsc/foo/bar">
+		<meta name="go-import" content="foo/bar mod http://github.com/rsc/baz/quux">`,
+		[]metaImport{
+			{"foo/bar", "git", "https://github.com/rsc/foo/bar"},
+		},
+	},
+	{
+		`<meta name="go-import" content="foo/bar mod http://github.com/rsc/baz/quux">
+		<meta name="go-import" content="foo/bar git https://github.com/rsc/foo/bar">`,
+		[]metaImport{
+			{"foo/bar", "git", "https://github.com/rsc/foo/bar"},
+		},
+	},
 	{
 		`<head>
 		<meta name="go-import" content="foo/bar git https://github.com/rsc/foo/bar">

src/cmd/go/internal/list/list.go

@@ -218,8 +218,8 @@ func runList(cmd *base.Command, args []string) {
 
 	for _, pkg := range pkgs {
 		// Show vendor-expanded paths in listing
-		pkg.TestImports = pkg.Vendored(pkg.TestImports)
-		pkg.XTestImports = pkg.Vendored(pkg.XTestImports)
+		pkg.TestImports = pkg.Resolve(pkg.TestImports)
+		pkg.XTestImports = pkg.Resolve(pkg.XTestImports)
 
 		do(&pkg.PackagePublic)
 	}

src/cmd/go/internal/load/pkg.go

@@ -6,6 +6,7 @@
 package load
 
 import (
+	"bytes"
 	"fmt"
 	"go/build"
 	"go/token"
@@ -14,6 +15,7 @@ import (
 	pathpkg "path"
 	"path/filepath"
 	"sort"
+	"strconv"
 	"strings"
 	"unicode"
 	"unicode/utf8"
@@ -168,7 +170,7 @@ func (e *NoGoError) Error() string {
 	return "no Go files in " + e.Package.Dir
 }
 
-// Vendored returns the vendor-resolved version of imports,
+// Resolve returns the resolved version of imports,
 // which should be p.TestImports or p.XTestImports, NOT p.Imports.
 // The imports in p.TestImports and p.XTestImports are not recursively
 // loaded during the initial load of p, so they list the imports found in
@@ -178,14 +180,14 @@ func (e *NoGoError) Error() string {
 // can produce better error messages if it starts with the original paths.
 // The initial load of p loads all the non-test imports and rewrites
 // the vendored paths, so nothing should ever call p.vendored(p.Imports).
-func (p *Package) Vendored(imports []string) []string {
+func (p *Package) Resolve(imports []string) []string {
 	if len(imports) > 0 && len(p.Imports) > 0 && &imports[0] == &p.Imports[0] {
-		panic("internal error: p.vendored(p.Imports) called")
+		panic("internal error: p.Resolve(p.Imports) called")
 	}
 	seen := make(map[string]bool)
 	var all []string
 	for _, path := range imports {
-		path = VendoredImportPath(p, path)
+		path = ResolveImportPath(p, path)
 		if !seen[path] {
 			seen[path] = true
 			all = append(all, path)
@@ -380,16 +382,16 @@ func makeImportValid(r rune) rune {
 
 // Mode flags for loadImport and download (in get.go).
 const (
-	// UseVendor means that loadImport should do vendor expansion
-	// (provided the vendoring experiment is enabled).
-	// That is, useVendor means that the import path came from
-	// a source file and has not been vendor-expanded yet.
-	// Every import path should be loaded initially with useVendor,
-	// and then the expanded version (with the /vendor/ in it) gets
-	// recorded as the canonical import path. At that point, future loads
-	// of that package must not pass useVendor, because
+	// ResolveImport means that loadImport should do import path expansion.
+	// That is, ResolveImport means that the import path came from
+	// a source file and has not been expanded yet to account for
+	// vendoring or possible module adjustment.
+	// Every import path should be loaded initially with ResolveImport,
+	// and then the expanded version (for example with the /vendor/ in it)
+	// gets recorded as the canonical import path. At that point, future loads
+	// of that package must not pass ResolveImport, because
 	// disallowVendor will reject direct use of paths containing /vendor/.
-	UseVendor = 1 << iota
+	ResolveImport = 1 << iota
 
 	// GetTestDeps is for download (part of "go get") and indicates
 	// that test dependencies should be fetched too.
@@ -414,12 +416,12 @@ func LoadImport(path, srcDir string, parent *Package, stk *ImportStack, importPo
 	isLocal := build.IsLocalImport(path)
 	if isLocal {
 		importPath = dirToImportPath(filepath.Join(srcDir, path))
-	} else if mode&UseVendor != 0 {
-		// We do our own vendor resolution, because we want to
+	} else if mode&ResolveImport != 0 {
+		// We do our own path resolution, because we want to
 		// find out the key to use in packageCache without the
 		// overhead of repeated calls to buildContext.Import.
 		// The code is also needed in a few other places anyway.
-		path = VendoredImportPath(parent, path)
+		path = ResolveImportPath(parent, path)
 		importPath = path
 	}
 
@@ -436,7 +438,7 @@ func LoadImport(path, srcDir string, parent *Package, stk *ImportStack, importPo
 		// Import always returns bp != nil, even if an error occurs,
 		// in order to return partial information.
 		buildMode := build.ImportComment
-		if mode&UseVendor == 0 || path != origPath {
+		if mode&ResolveImport == 0 || path != origPath {
 			// Not vendoring, or we already found the vendored path.
 			buildMode |= build.IgnoreVendor
 		}
@@ -467,7 +469,7 @@ func LoadImport(path, srcDir string, parent *Package, stk *ImportStack, importPo
 	if perr := disallowInternal(srcDir, p, stk); perr != p {
 		return setErrorPos(perr, importPos)
 	}
-	if mode&UseVendor != 0 {
+	if mode&ResolveImport != 0 {
 		if perr := disallowVendor(srcDir, origPath, p, stk); perr != p {
 			return setErrorPos(perr, importPos)
 		}
@@ -526,7 +528,50 @@ func isDir(path string) bool {
 	return result
 }
 
-// VendoredImportPath returns the expansion of path when it appears in parent.
+// ResolveImportPath returns the true meaning of path when it appears in parent.
+// There are two different resolutions applied.
+// First, there is Go 1.5 vendoring (golang.org/s/go15vendor).
+// If vendor expansion doesn't trigger, then the path is also subject to
+// Go 1.11 vgo legacy conversion (golang.org/issue/25069).
+func ResolveImportPath(parent *Package, path string) (found string) {
+	found = VendoredImportPath(parent, path)
+	if found != path {
+		return found
+	}
+	return ModuleImportPath(parent, path)
+}
+
+// dirAndRoot returns the source directory and workspace root
+// for the package p, guaranteeing that root is a path prefix of dir.
+func dirAndRoot(p *Package) (dir, root string) {
+	dir = filepath.Clean(p.Dir)
+	root = filepath.Join(p.Root, "src")
+	if !str.HasFilePathPrefix(dir, root) || p.ImportPath != "command-line-arguments" && filepath.Join(root, p.ImportPath) != dir {
+		// Look for symlinks before reporting error.
+		dir = expandPath(dir)
+		root = expandPath(root)
+	}
+
+	if !str.HasFilePathPrefix(dir, root) || len(dir) <= len(root) || dir[len(root)] != filepath.Separator || p.ImportPath != "command-line-arguments" && !p.Internal.Local && filepath.Join(root, p.ImportPath) != dir {
+		base.Fatalf("unexpected directory layout:\n"+
+			"	import path: %s\n"+
+			"	root: %s\n"+
+			"	dir: %s\n"+
+			"	expand root: %s\n"+
+			"	expand dir: %s\n"+
+			"	separator: %s",
+			p.ImportPath,
+			filepath.Join(p.Root, "src"),
+			filepath.Clean(p.Dir),
+			root,
+			dir,
+			string(filepath.Separator))
+	}
+
+	return dir, root
+}
+
+// VendoredImportPath returns the vendor-expansion of path when it appears in parent.
 // If parent is x/y/z, then path might expand to x/y/z/vendor/path, x/y/vendor/path,
 // x/vendor/path, vendor/path, or else stay path if none of those exist.
 // VendoredImportPath returns the expanded path or, if no expansion is found, the original.
@@ -535,29 +580,7 @@ func VendoredImportPath(parent *Package, path string) (found string) {
 		return path
 	}
 
-	dir := filepath.Clean(parent.Dir)
-	root := filepath.Join(parent.Root, "src")
-	if !str.HasFilePathPrefix(dir, root) || parent.ImportPath != "command-line-arguments" && filepath.Join(root, parent.ImportPath) != dir {
-		// Look for symlinks before reporting error.
-		dir = expandPath(dir)
-		root = expandPath(root)
-	}
-
-	if !str.HasFilePathPrefix(dir, root) || len(dir) <= len(root) || dir[len(root)] != filepath.Separator || parent.ImportPath != "command-line-arguments" && !parent.Internal.Local && filepath.Join(root, parent.ImportPath) != dir {
-		base.Fatalf("unexpected directory layout:\n"+
-			"	import path: %s\n"+
-			"	root: %s\n"+
-			"	dir: %s\n"+
-			"	expand root: %s\n"+
-			"	expand dir: %s\n"+
-			"	separator: %s",
-			parent.ImportPath,
-			filepath.Join(parent.Root, "src"),
-			filepath.Clean(parent.Dir),
-			root,
-			dir,
-			string(filepath.Separator))
-	}
+	dir, root := dirAndRoot(parent)
 
 	vpath := "vendor/" + path
 	for i := len(dir); i >= len(root); i-- {
@@ -601,6 +624,164 @@ func VendoredImportPath(parent *Package, path string) (found string) {
 	return path
 }
 
+var (
+	modulePrefix   = []byte("\nmodule ")
+	goModPathCache = make(map[string]string)
+)
+
+// goModPath returns the module path in the go.mod in dir, if any.
+func goModPath(dir string) (path string) {
+	path, ok := goModPathCache[dir]
+	if ok {
+		return path
+	}
+	defer func() {
+		goModPathCache[dir] = path
+	}()
+
+	data, err := ioutil.ReadFile(filepath.Join(dir, "go.mod"))
+	if err != nil {
+		return ""
+	}
+	var i int
+	if bytes.HasPrefix(data, modulePrefix[1:]) {
+		i = 0
+	} else {
+		i = bytes.Index(data, modulePrefix)
+		if i < 0 {
+			return ""
+		}
+		i++
+	}
+	line := data[i:]
+
+	// Cut line at \n, drop trailing \r if present.
+	if j := bytes.IndexByte(line, '\n'); j >= 0 {
+		line = line[:j]
+	}
+	if line[len(line)-1] == '\r' {
+		line = line[:len(line)-1]
+	}
+	line = line[len("module "):]
+
+	// If quoted, unquote.
+	path = strings.TrimSpace(string(line))
+	if path != "" && path[0] == '"' {
+		s, err := strconv.Unquote(path)
+		if err != nil {
+			return ""
+		}
+		path = s
+	}
+	return path
+}
+
+// findVersionElement returns the slice indices of the final version element /vN in path.
+// If there is no such element, it returns -1, -1.
+func findVersionElement(path string) (i, j int) {
+	j = len(path)
+	for i = len(path) - 1; i >= 0; i-- {
+		if path[i] == '/' {
+			if isVersionElement(path[i:j]) {
+				return i, j
+			}
+			j = i
+		}
+	}
+	return -1, -1
+}
+
+// isVersionElement reports whether s is a well-formed path version element:
+// v2, v3, v10, etc, but not v0, v05, v1.
+func isVersionElement(s string) bool {
+	if len(s) < 3 || s[0] != '/' || s[1] != 'v' || s[2] == '0' || s[2] == '1' && len(s) == 3 {
+		return false
+	}
+	for i := 2; i < len(s); i++ {
+		if s[i] < '0' || '9' < s[i] {
+			return false
+		}
+	}
+	return true
+}
+
+// ModuleImportPath translates import paths found in go modules
+// back down to paths that can be resolved in ordinary builds.
+//
+// Define “new” code as code with a go.mod file in the same directory
+// or a parent directory. If an import in new code says x/y/v2/z but
+// x/y/v2/z does not exist and x/y/go.mod says “module x/y/v2”,
+// then go build will read the import as x/y/z instead.
+// See golang.org/issue/25069.
+func ModuleImportPath(parent *Package, path string) (found string) {
+	if parent == nil || parent.Root == "" {
+		return path
+	}
+
+	// If there are no vN elements in path, leave it alone.
+	// (The code below would do the same, but only after
+	// some other file system accesses that we can avoid
+	// here by returning early.)
+	if i, _ := findVersionElement(path); i < 0 {
+		return path
+	}
+
+	dir, root := dirAndRoot(parent)
+
+	// Consider dir and parents, up to and including root.
+	for i := len(dir); i >= len(root); i-- {
+		if i < len(dir) && dir[i] != filepath.Separator {
+			continue
+		}
+		if goModPath(dir[:i]) != "" {
+			goto HaveGoMod
+		}
+	}
+	// This code is not in a tree with a go.mod,
+	// so apply no changes to the path.
+	return path
+
+HaveGoMod:
+	// This import is in a tree with a go.mod.
+	// Allow it to refer to code in GOPATH/src/x/y/z as x/y/v2/z
+	// if GOPATH/src/x/y/go.mod says module "x/y/v2",
+
+	// If x/y/v2/z exists, use it unmodified.
+	if bp, _ := cfg.BuildContext.Import(path, "", build.IgnoreVendor); bp.Dir != "" {
+		return path
+	}
+
+	// Otherwise look for a go.mod supplying a version element.
+	// Some version-like elements may appear in paths but not
+	// be module versions; we skip over those to look for module
+	// versions. For example the module m/v2 might have a
+	// package m/v2/api/v1/foo.
+	limit := len(path)
+	for limit > 0 {
+		i, j := findVersionElement(path[:limit])
+		if i < 0 {
+			return path
+		}
+		if bp, _ := cfg.BuildContext.Import(path[:i], "", build.IgnoreVendor); bp.Dir != "" {
+			if mpath := goModPath(bp.Dir); mpath != "" {
+				// Found a valid go.mod file, so we're stopping the search.
+				// If the path is m/v2/p and we found m/go.mod that says
+				// "module m/v2", then we return "m/p".
+				if mpath == path[:j] {
+					return path[:i] + path[j:]
+				}
+				// Otherwise just return the original path.
+				// We didn't find anything worth rewriting,
+				// and the go.mod indicates that we should
+				// not consider parent directories.
+				return path
+			}
+		}
+		limit = i
+	}
+	return path
+}
+
 // hasGoFiles reports whether dir contains any files with names ending in .go.
 // For a vendor check we must exclude directories that contain no .go files.
 // Otherwise it is not possible to vendor just a/b/c and still import the
@@ -1059,7 +1240,7 @@ func (p *Package) load(stk *ImportStack, bp *build.Package, err error) {
 		if path == "C" {
 			continue
 		}
-		p1 := LoadImport(path, p.Dir, p, stk, p.Internal.Build.ImportPos[path], UseVendor)
+		p1 := LoadImport(path, p.Dir, p, stk, p.Internal.Build.ImportPos[path], ResolveImport)
 		if p.Standard && p.Error == nil && !p1.Standard && p1.Error == nil {
 			p.Error = &PackageError{
 				ImportStack: stk.Copy(),
@@ -1570,7 +1751,7 @@ func TestPackagesFor(p *Package, forceTest bool) (ptest, pxtest *Package, err er
 	stk.Push(p.ImportPath + " (test)")
 	rawTestImports := str.StringList(p.TestImports)
 	for i, path := range p.TestImports {
-		p1 := LoadImport(path, p.Dir, p, &stk, p.Internal.Build.TestImportPos[path], UseVendor)
+		p1 := LoadImport(path, p.Dir, p, &stk, p.Internal.Build.TestImportPos[path], ResolveImport)
 		if p1.Error != nil {
 			return nil, nil, p1.Error
 		}
@@ -1598,7 +1779,7 @@ func TestPackagesFor(p *Package, forceTest bool) (ptest, pxtest *Package, err er
 	pxtestNeedsPtest := false
 	rawXTestImports := str.StringList(p.XTestImports)
 	for i, path := range p.XTestImports {
-		p1 := LoadImport(path, p.Dir, p, &stk, p.Internal.Build.XTestImportPos[path], UseVendor)
+		p1 := LoadImport(path, p.Dir, p, &stk, p.Internal.Build.XTestImportPos[path], ResolveImport)
 		if p1.Error != nil {
 			return nil, nil, p1.Error
 		}

src/cmd/go/internal/test/test.go

@@ -606,10 +606,10 @@ func runTest(cmd *base.Command, args []string) {
 			for _, path := range p.Imports {
 				deps[path] = true
 			}
-			for _, path := range p.Vendored(p.TestImports) {
+			for _, path := range p.Resolve(p.TestImports) {
 				deps[path] = true
 			}
-			for _, path := range p.Vendored(p.XTestImports) {
+			for _, path := range p.Resolve(p.XTestImports) {
 				deps[path] = true
 			}
 		}

src/cmd/go/testdata/modlegacy/src/new/go.mod

@@ -0,0 +1 @@
+module "new/v2"

src/cmd/go/testdata/modlegacy/src/new/new.go

@@ -0,0 +1,3 @@
+package new
+
+import _ "new/v2/p2"

src/cmd/go/testdata/modlegacy/src/new/p1/p1.go

@@ -0,0 +1,7 @@
+package p1
+
+import _ "old/p2"
+import _ "new/v2"
+import _ "new/v2/p2"
+import _ "new/sub/v2/x/v1/y" // v2 is module, v1 is directory in module
+import _ "new/sub/inner/x"   // new/sub/inner/go.mod overrides new/sub/go.mod

src/cmd/go/testdata/modlegacy/src/new/p2/p2.go

@@ -0,0 +1 @@
+package p2

src/cmd/go/testdata/modlegacy/src/new/sub/go.mod

@@ -0,0 +1 @@
+module new/sub/v2

src/cmd/go/testdata/modlegacy/src/new/sub/inner/go.mod

@@ -0,0 +1 @@
+module new/sub/inner

src/cmd/go/testdata/modlegacy/src/new/sub/inner/x/x.go

@@ -0,0 +1 @@
+package x

src/cmd/go/testdata/modlegacy/src/new/sub/x/v1/y/y.go

@@ -0,0 +1 @@
+package y

src/cmd/go/testdata/modlegacy/src/old/p1/p1.go

@@ -0,0 +1,5 @@
+package p1
+
+import _ "old/p2"
+import _ "new/p1"
+import _ "new"

src/cmd/go/testdata/modlegacy/src/old/p2/p2.go

@@ -0,0 +1 @@
+package p2

src/cmd/go/vendor_test.go

@@ -328,3 +328,34 @@ func TestVendor12156(t *testing.T) {
 	tg.grepStderrNot("panic", "panicked")
 	tg.grepStderr(`cannot find package "x"`, "wrong error")
 }
+
+// Module legacy support does path rewriting very similar to vendoring.
+
+func TestModLegacy(t *testing.T) {
+	tg := testgo(t)
+	defer tg.cleanup()
+	tg.setenv("GOPATH", filepath.Join(tg.pwd(), "testdata/modlegacy"))
+	tg.run("list", "-f", "{{.Imports}}", "old/p1")
+	tg.grepStdout("new/p1", "old/p1 should import new/p1")
+	tg.run("list", "-f", "{{.Imports}}", "new/p1")
+	tg.grepStdout("new/p2", "new/p1 should import new/p2 (not new/v2/p2)")
+	tg.grepStdoutNot("new/v2", "new/p1 should NOT import new/v2*")
+	tg.grepStdout("new/sub/x/v1/y", "new/p1 should import new/sub/x/v1/y (not new/sub/v2/x/v1/y)")
+	tg.grepStdoutNot("new/sub/v2", "new/p1 should NOT import new/sub/v2*")
+	tg.grepStdout("new/sub/inner/x", "new/p1 should import new/sub/inner/x (no rewrites)")
+	tg.run("build", "old/p1", "new/p1")
+}
+
+func TestModLegacyGet(t *testing.T) {
+	testenv.MustHaveExternalNetwork(t)
+
+	tg := testgo(t)
+	defer tg.cleanup()
+	tg.makeTempdir()
+	tg.setenv("GOPATH", tg.path("."))
+	tg.run("get", "vcs-test.golang.org/git/modlegacy1-old.git/p1")
+	tg.run("list", "-f", "{{.Deps}}", "vcs-test.golang.org/git/modlegacy1-old.git/p1")
+	tg.grepStdout("new.git/p2", "old/p1 should depend on new/p2")
+	tg.grepStdoutNot("new.git/v2/p2", "old/p1 should NOT depend on new/v2/p2")
+	tg.run("build", "vcs-test.golang.org/git/modlegacy1-old.git/p1", "vcs-test.golang.org/git/modlegacy1-new.git/p1")
+}