From 979e1376096a4c1c7525f33dd4e76414f63c93fb Mon Sep 17 00:00:00 2001 From: Russ Cox Date: Thu, 29 Oct 2020 10:57:38 -0400 Subject: [PATCH] cmd/go: add GOVCS setting to control version control usage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The go command runs commands like git and hg to download modules. In the past, we have had problems with security bugs in version control systems becoming security bugs in “go get”. The original modules draft design removed use of these commands entirely, saying: > We want to move away from invoking version control tools such as bzr, > fossil, git, hg, and svn to download source code. These fragment the > ecosystem: packages developed using Bazaar or Fossil, for example, are > effectively unavailable to users who cannot or choose not to install > these tools. The version control tools have also been a source of > exciting security problems. It would be good to move them outside the > security perimeter. The removal of these commands was not possible in the end: being able to fetch directly from Git repos is too important, especially for closed source. But the security exposure has not gone away. We remain vulnerable to problems in VCS systems, especially the less scrutinized ones. This change adds a GOVCS setting to let users control which version control systems are allowed by default. It also changes the default allowed version control systems to git and hg for public code and any version control system for private code (import path or module path matched by the GOPRIVATE setting). See the changes in alldocs.go for detailed documentation. See #41730 for proposal and discussion. Fixes #41730. [Replay of CL 266420. See changes from Patch Set 1 for updates to fix a few long tests.] Change-Id: I4fe93804548956c42aea985368b4571bdb220f48 Reviewed-on: https://go-review.googlesource.com/c/go/+/267888 Trust: Russ Cox Run-TryBot: Russ Cox TryBot-Result: Go Bot Reviewed-by: Dmitri Shuralyov --- src/cmd/go/alldocs.go | 177 +++++++++++++++++------- src/cmd/go/go_test.go | 8 ++ src/cmd/go/internal/cfg/cfg.go | 1 + src/cmd/go/internal/envcmd/env.go | 1 + src/cmd/go/internal/help/helpdoc.go | 2 +- src/cmd/go/internal/modfetch/fetch.go | 13 +- src/cmd/go/internal/modget/get.go | 76 ++++++++++ src/cmd/go/internal/modload/help.go | 2 +- src/cmd/go/internal/vcs/vcs.go | 144 +++++++++++++++++++ src/cmd/go/internal/vcs/vcs_test.go | 104 ++++++++++++++ src/cmd/go/main.go | 3 +- src/cmd/go/script_test.go | 1 + src/cmd/go/testdata/script/govcs.txt | 174 +++++++++++++++++++++++ src/cmd/go/testdata/script/mod_auth.txt | 1 + src/internal/cfg/cfg.go | 1 + 15 files changed, 652 insertions(+), 56 deletions(-) create mode 100644 src/cmd/go/testdata/script/govcs.txt diff --git a/src/cmd/go/alldocs.go b/src/cmd/go/alldocs.go index 6e0d8337bfb..e68fa55d09b 100644 --- a/src/cmd/go/alldocs.go +++ b/src/cmd/go/alldocs.go @@ -49,10 +49,11 @@ // modules modules, module versions, and more // module-get module-aware go get // module-auth module authentication using go.sum -// module-private module configuration for non-public modules // packages package lists and patterns +// private configuration for downloading non-public code // testflag testing flags // testfunc testing functions +// vcs controlling version control with GOVCS // // Use "go help " for more information about that topic. // @@ -1799,7 +1800,7 @@ // Comma-separated list of glob patterns (in the syntax of Go's path.Match) // of module path prefixes that should always be fetched directly // or that should not be compared against the checksum database. -// See 'go help module-private'. +// See 'go help private'. // GOROOT // The root of the go tree. // GOSUMDB @@ -2869,7 +2870,7 @@ // followed by a pipe character, indicating it is safe to fall back on any error. // // The GOPRIVATE and GONOPROXY environment variables allow bypassing -// the proxy for selected modules. See 'go help module-private' for details. +// the proxy for selected modules. See 'go help private' for details. // // No matter the source of the modules, the go command checks downloads against // known checksums, to detect unexpected changes in the content of any specific @@ -2989,52 +2990,7 @@ // accepted, at the cost of giving up the security guarantee of verified repeatable // downloads for all modules. A better way to bypass the checksum database // for specific modules is to use the GOPRIVATE or GONOSUMDB environment -// variables. See 'go help module-private' for details. -// -// The 'go env -w' command (see 'go help env') can be used to set these variables -// for future go command invocations. -// -// -// Module configuration for non-public modules -// -// The go command defaults to downloading modules from the public Go module -// mirror at proxy.golang.org. It also defaults to validating downloaded modules, -// regardless of source, against the public Go checksum database at sum.golang.org. -// These defaults work well for publicly available source code. -// -// The GOPRIVATE environment variable controls which modules the go command -// considers to be private (not available publicly) and should therefore not use the -// proxy or checksum database. The variable is a comma-separated list of -// glob patterns (in the syntax of Go's path.Match) of module path prefixes. -// For example, -// -// GOPRIVATE=*.corp.example.com,rsc.io/private -// -// causes the go command to treat as private any module with a path prefix -// matching either pattern, including git.corp.example.com/xyzzy, rsc.io/private, -// and rsc.io/private/quux. -// -// The GOPRIVATE environment variable may be used by other tools as well to -// identify non-public modules. For example, an editor could use GOPRIVATE -// to decide whether to hyperlink a package import to a godoc.org page. -// -// For fine-grained control over module download and validation, the GONOPROXY -// and GONOSUMDB environment variables accept the same kind of glob list -// and override GOPRIVATE for the specific decision of whether to use the proxy -// and checksum database, respectively. -// -// For example, if a company ran a module proxy serving private modules, -// users would configure go using: -// -// GOPRIVATE=*.corp.example.com -// GOPROXY=proxy.example.com -// GONOPROXY=none -// -// This would tell the go command and other tools that modules beginning with -// a corp.example.com subdomain are private but that the company proxy should -// be used for downloading both public and private modules, because -// GONOPROXY has been set to a pattern that won't match any modules, -// overriding GOPRIVATE. +// variables. See 'go help private' for details. // // The 'go env -w' command (see 'go help env') can be used to set these variables // for future go command invocations. @@ -3124,6 +3080,56 @@ // by the go tool, as are directories named "testdata". // // +// Configuration for downloading non-public code +// +// The go command defaults to downloading modules from the public Go module +// mirror at proxy.golang.org. It also defaults to validating downloaded modules, +// regardless of source, against the public Go checksum database at sum.golang.org. +// These defaults work well for publicly available source code. +// +// The GOPRIVATE environment variable controls which modules the go command +// considers to be private (not available publicly) and should therefore not use the +// proxy or checksum database. The variable is a comma-separated list of +// glob patterns (in the syntax of Go's path.Match) of module path prefixes. +// For example, +// +// GOPRIVATE=*.corp.example.com,rsc.io/private +// +// causes the go command to treat as private any module with a path prefix +// matching either pattern, including git.corp.example.com/xyzzy, rsc.io/private, +// and rsc.io/private/quux. +// +// The GOPRIVATE environment variable may be used by other tools as well to +// identify non-public modules. For example, an editor could use GOPRIVATE +// to decide whether to hyperlink a package import to a godoc.org page. +// +// For fine-grained control over module download and validation, the GONOPROXY +// and GONOSUMDB environment variables accept the same kind of glob list +// and override GOPRIVATE for the specific decision of whether to use the proxy +// and checksum database, respectively. +// +// For example, if a company ran a module proxy serving private modules, +// users would configure go using: +// +// GOPRIVATE=*.corp.example.com +// GOPROXY=proxy.example.com +// GONOPROXY=none +// +// This would tell the go command and other tools that modules beginning with +// a corp.example.com subdomain are private but that the company proxy should +// be used for downloading both public and private modules, because +// GONOPROXY has been set to a pattern that won't match any modules, +// overriding GOPRIVATE. +// +// The GOPRIVATE variable is also used to define the "public" and "private" +// patterns for the GOVCS variable; see 'go help vcs'. For that usage, +// GOPRIVATE applies even in GOPATH mode. In that case, it matches import paths +// instead of module paths. +// +// The 'go env -w' command (see 'go help env') can be used to set these variables +// for future go command invocations. +// +// // Testing flags // // The 'go test' command takes both flags that apply to 'go test' itself @@ -3416,4 +3422,77 @@ // See the documentation of the testing package for more information. // // +// Controlling version control with GOVCS +// +// The 'go get' command can run version control commands like git +// to download imported code. This functionality is critical to the decentralized +// Go package ecosystem, in which code can be imported from any server, +// but it is also a potential security problem, if a malicious server finds a +// way to cause the invoked version control command to run unintended code. +// +// To balance the functionality and security concerns, the 'go get' command +// by default will only use git and hg to download code from public servers. +// But it will use any known version control system (bzr, fossil, git, hg, svn) +// to download code from private servers, defined as those hosting packages +// matching the GOPRIVATE variable (see 'go help private'). The rationale behind +// allowing only Git and Mercurial is that these two systems have had the most +// attention to issues of being run as clients of untrusted servers. In contrast, +// Bazaar, Fossil, and Subversion have primarily been used in trusted, +// authenticated environments and are not as well scrutinized as attack surfaces. +// +// The version control command restrictions only apply when using direct version +// control access to download code. When downloading modules from a proxy, +// 'go get' uses the proxy protocol instead, which is always permitted. +// By default, the 'go get' command uses the Go module mirror (proxy.golang.org) +// for public packages and only falls back to version control for private +// packages or when the mirror refuses to serve a public package (typically for +// legal reasons). Therefore, clients can still access public code served from +// Bazaar, Fossil, or Subversion repositories by default, because those downloads +// use the Go module mirror, which takes on the security risk of running the +// version control commands, using a custom sandbox. +// +// The GOVCS variable can be used to change the allowed version control systems +// for specific packages (identified by a module or import path). +// The GOVCS variable applies both when using modules and when using GOPATH. +// When using modules, the patterns match against the module path. +// When using GOPATH, the patterns match against the import path +// corresponding to the root of the version control repository. +// +// The general form of the GOVCS setting is a comma-separated list of +// pattern:vcslist rules. The pattern is a glob pattern that must match +// one or more leading elements of the module or import path. The vcslist +// is a pipe-separated list of allowed version control commands, or "all" +// to allow use of any known command, or "off" to allow nothing. +// The earliest matching pattern in the list applies, even if later patterns +// might also match. +// +// For example, consider: +// +// GOVCS=github.com:git,evil.com:off,*:git|hg +// +// With this setting, code with an module or import path beginning with +// github.com/ can only use git; paths on evil.com cannot use any version +// control command, and all other paths (* matches everything) can use +// only git or hg. +// +// The special patterns "public" and "private" match public and private +// module or import paths. A path is private if it matches the GOPRIVATE +// variable; otherwise it is public. +// +// If no rules in the GOVCS variable match a particular module or import path, +// the 'go get' command applies its default rule, which can now be summarized +// in GOVCS notation as 'public:git|hg,private:all'. +// +// To allow unfettered use of any version control system for any package, use: +// +// GOVCS=*:all +// +// To disable all use of version control, use: +// +// GOVCS=*:off +// +// The 'go env -w' command (see 'go help env') can be used to set the GOVCS +// variable for future go command invocations. +// +// package main diff --git a/src/cmd/go/go_test.go b/src/cmd/go/go_test.go index 91d5884036d..ee0cd8e2c72 100644 --- a/src/cmd/go/go_test.go +++ b/src/cmd/go/go_test.go @@ -35,6 +35,14 @@ import ( "cmd/internal/sys" ) +func init() { + // GOVCS defaults to public:git|hg,private:all, + // which breaks many tests here - they can't use non-git, non-hg VCS at all! + // Change to fully permissive. + // The tests of the GOVCS setting itself are in ../../testdata/script/govcs.txt. + os.Setenv("GOVCS", "*:all") +} + var ( canRace = false // whether we can run the race detector canCgo = false // whether we can use cgo diff --git a/src/cmd/go/internal/cfg/cfg.go b/src/cmd/go/internal/cfg/cfg.go index 67d581f6e6d..9bc48132ae2 100644 --- a/src/cmd/go/internal/cfg/cfg.go +++ b/src/cmd/go/internal/cfg/cfg.go @@ -268,6 +268,7 @@ var ( GONOPROXY = envOr("GONOPROXY", GOPRIVATE) GONOSUMDB = envOr("GONOSUMDB", GOPRIVATE) GOINSECURE = Getenv("GOINSECURE") + GOVCS = Getenv("GOVCS") ) var SumdbDir = gopathDir("pkg/sumdb") diff --git a/src/cmd/go/internal/envcmd/env.go b/src/cmd/go/internal/envcmd/env.go index 557e4189213..d65ace879db 100644 --- a/src/cmd/go/internal/envcmd/env.go +++ b/src/cmd/go/internal/envcmd/env.go @@ -87,6 +87,7 @@ func MkEnv() []cfg.EnvVar { {Name: "GOSUMDB", Value: cfg.GOSUMDB}, {Name: "GOTMPDIR", Value: cfg.Getenv("GOTMPDIR")}, {Name: "GOTOOLDIR", Value: base.ToolDir}, + {Name: "GOVCS", Value: cfg.GOVCS}, } if work.GccgoBin != "" { diff --git a/src/cmd/go/internal/help/helpdoc.go b/src/cmd/go/internal/help/helpdoc.go index 8dfabbaa4a0..50cf911407e 100644 --- a/src/cmd/go/internal/help/helpdoc.go +++ b/src/cmd/go/internal/help/helpdoc.go @@ -526,7 +526,7 @@ General-purpose environment variables: Comma-separated list of glob patterns (in the syntax of Go's path.Match) of module path prefixes that should always be fetched directly or that should not be compared against the checksum database. - See 'go help module-private'. + See 'go help private'. GOROOT The root of the go tree. GOSUMDB diff --git a/src/cmd/go/internal/modfetch/fetch.go b/src/cmd/go/internal/modfetch/fetch.go index 25e9fb62c13..a3e2cd1f9d6 100644 --- a/src/cmd/go/internal/modfetch/fetch.go +++ b/src/cmd/go/internal/modfetch/fetch.go @@ -848,16 +848,16 @@ the checksum database is not consulted, and all unrecognized modules are accepted, at the cost of giving up the security guarantee of verified repeatable downloads for all modules. A better way to bypass the checksum database for specific modules is to use the GOPRIVATE or GONOSUMDB environment -variables. See 'go help module-private' for details. +variables. See 'go help private' for details. The 'go env -w' command (see 'go help env') can be used to set these variables for future go command invocations. `, } -var HelpModulePrivate = &base.Command{ - UsageLine: "module-private", - Short: "module configuration for non-public modules", +var HelpPrivate = &base.Command{ + UsageLine: "private", + Short: "configuration for downloading non-public code", Long: ` The go command defaults to downloading modules from the public Go module mirror at proxy.golang.org. It also defaults to validating downloaded modules, @@ -898,6 +898,11 @@ be used for downloading both public and private modules, because GONOPROXY has been set to a pattern that won't match any modules, overriding GOPRIVATE. +The GOPRIVATE variable is also used to define the "public" and "private" +patterns for the GOVCS variable; see 'go help vcs'. For that usage, +GOPRIVATE applies even in GOPATH mode. In that case, it matches import paths +instead of module paths. + The 'go env -w' command (see 'go help env') can be used to set these variables for future go command invocations. `, diff --git a/src/cmd/go/internal/modget/get.go b/src/cmd/go/internal/modget/get.go index f7598884365..6ab242944ac 100644 --- a/src/cmd/go/internal/modget/get.go +++ b/src/cmd/go/internal/modget/get.go @@ -195,6 +195,82 @@ Usage: ` + CmdGet.UsageLine + ` ` + CmdGet.Long, } +var HelpVCS = &base.Command{ + UsageLine: "vcs", + Short: "controlling version control with GOVCS", + Long: ` +The 'go get' command can run version control commands like git +to download imported code. This functionality is critical to the decentralized +Go package ecosystem, in which code can be imported from any server, +but it is also a potential security problem, if a malicious server finds a +way to cause the invoked version control command to run unintended code. + +To balance the functionality and security concerns, the 'go get' command +by default will only use git and hg to download code from public servers. +But it will use any known version control system (bzr, fossil, git, hg, svn) +to download code from private servers, defined as those hosting packages +matching the GOPRIVATE variable (see 'go help private'). The rationale behind +allowing only Git and Mercurial is that these two systems have had the most +attention to issues of being run as clients of untrusted servers. In contrast, +Bazaar, Fossil, and Subversion have primarily been used in trusted, +authenticated environments and are not as well scrutinized as attack surfaces. + +The version control command restrictions only apply when using direct version +control access to download code. When downloading modules from a proxy, +'go get' uses the proxy protocol instead, which is always permitted. +By default, the 'go get' command uses the Go module mirror (proxy.golang.org) +for public packages and only falls back to version control for private +packages or when the mirror refuses to serve a public package (typically for +legal reasons). Therefore, clients can still access public code served from +Bazaar, Fossil, or Subversion repositories by default, because those downloads +use the Go module mirror, which takes on the security risk of running the +version control commands, using a custom sandbox. + +The GOVCS variable can be used to change the allowed version control systems +for specific packages (identified by a module or import path). +The GOVCS variable applies both when using modules and when using GOPATH. +When using modules, the patterns match against the module path. +When using GOPATH, the patterns match against the import path +corresponding to the root of the version control repository. + +The general form of the GOVCS setting is a comma-separated list of +pattern:vcslist rules. The pattern is a glob pattern that must match +one or more leading elements of the module or import path. The vcslist +is a pipe-separated list of allowed version control commands, or "all" +to allow use of any known command, or "off" to allow nothing. +The earliest matching pattern in the list applies, even if later patterns +might also match. + +For example, consider: + + GOVCS=github.com:git,evil.com:off,*:git|hg + +With this setting, code with an module or import path beginning with +github.com/ can only use git; paths on evil.com cannot use any version +control command, and all other paths (* matches everything) can use +only git or hg. + +The special patterns "public" and "private" match public and private +module or import paths. A path is private if it matches the GOPRIVATE +variable; otherwise it is public. + +If no rules in the GOVCS variable match a particular module or import path, +the 'go get' command applies its default rule, which can now be summarized +in GOVCS notation as 'public:git|hg,private:all'. + +To allow unfettered use of any version control system for any package, use: + + GOVCS=*:all + +To disable all use of version control, use: + + GOVCS=*:off + +The 'go env -w' command (see 'go help env') can be used to set the GOVCS +variable for future go command invocations. +`, +} + var ( getD = CmdGet.Flag.Bool("d", false, "") getF = CmdGet.Flag.Bool("f", false, "") diff --git a/src/cmd/go/internal/modload/help.go b/src/cmd/go/internal/modload/help.go index 0d2dd28e5f3..d81dfd56fb5 100644 --- a/src/cmd/go/internal/modload/help.go +++ b/src/cmd/go/internal/modload/help.go @@ -365,7 +365,7 @@ list if the error is a 404 or 410 HTTP response or if the current proxy is followed by a pipe character, indicating it is safe to fall back on any error. The GOPRIVATE and GONOPROXY environment variables allow bypassing -the proxy for selected modules. See 'go help module-private' for details. +the proxy for selected modules. See 'go help private' for details. No matter the source of the modules, the go command checks downloads against known checksums, to detect unexpected changes in the content of any specific diff --git a/src/cmd/go/internal/vcs/vcs.go b/src/cmd/go/internal/vcs/vcs.go index 7812afd4888..e7bef9f591a 100644 --- a/src/cmd/go/internal/vcs/vcs.go +++ b/src/cmd/go/internal/vcs/vcs.go @@ -22,7 +22,10 @@ import ( "cmd/go/internal/base" "cmd/go/internal/cfg" + "cmd/go/internal/search" "cmd/go/internal/web" + + "golang.org/x/mod/module" ) // A vcsCmd describes how to use a version control system @@ -591,12 +594,146 @@ func FromDir(dir, srcRoot string) (vcs *Cmd, root string, err error) { } if vcsRet != nil { + if err := checkGOVCS(vcsRet, rootRet); err != nil { + return nil, "", err + } return vcsRet, rootRet, nil } return nil, "", fmt.Errorf("directory %q is not using a known version control system", origDir) } +// A govcsRule is a single GOVCS rule like private:hg|svn. +type govcsRule struct { + pattern string + allowed []string +} + +// A govcsConfig is a full GOVCS configuration. +type govcsConfig []govcsRule + +func parseGOVCS(s string) (govcsConfig, error) { + s = strings.TrimSpace(s) + if s == "" { + return nil, nil + } + var cfg govcsConfig + have := make(map[string]string) + for _, item := range strings.Split(s, ",") { + item = strings.TrimSpace(item) + if item == "" { + return nil, fmt.Errorf("empty entry in GOVCS") + } + i := strings.Index(item, ":") + if i < 0 { + return nil, fmt.Errorf("malformed entry in GOVCS (missing colon): %q", item) + } + pattern, list := strings.TrimSpace(item[:i]), strings.TrimSpace(item[i+1:]) + if pattern == "" { + return nil, fmt.Errorf("empty pattern in GOVCS: %q", item) + } + if list == "" { + return nil, fmt.Errorf("empty VCS list in GOVCS: %q", item) + } + if search.IsRelativePath(pattern) { + return nil, fmt.Errorf("relative pattern not allowed in GOVCS: %q", pattern) + } + if old := have[pattern]; old != "" { + return nil, fmt.Errorf("unreachable pattern in GOVCS: %q after %q", item, old) + } + have[pattern] = item + allowed := strings.Split(list, "|") + for i, a := range allowed { + a = strings.TrimSpace(a) + if a == "" { + return nil, fmt.Errorf("empty VCS name in GOVCS: %q", item) + } + allowed[i] = a + } + cfg = append(cfg, govcsRule{pattern, allowed}) + } + return cfg, nil +} + +func (c *govcsConfig) allow(path string, private bool, vcs string) bool { + for _, rule := range *c { + match := false + switch rule.pattern { + case "private": + match = private + case "public": + match = !private + default: + // Note: rule.pattern is known to be comma-free, + // so MatchPrefixPatterns is only matching a single pattern for us. + match = module.MatchPrefixPatterns(rule.pattern, path) + } + if !match { + continue + } + for _, allow := range rule.allowed { + if allow == vcs || allow == "all" { + return true + } + } + return false + } + + // By default, nothing is allowed. + return false +} + +var ( + govcs govcsConfig + govcsErr error + govcsOnce sync.Once +) + +// defaultGOVCS is the default setting for GOVCS. +// Setting GOVCS adds entries ahead of these but does not remove them. +// (They are appended to the parsed GOVCS setting.) +// +// The rationale behind allowing only Git and Mercurial is that +// these two systems have had the most attention to issues +// of being run as clients of untrusted servers. In contrast, +// Bazaar, Fossil, and Subversion have primarily been used +// in trusted, authenticated environments and are not as well +// scrutinized as attack surfaces. +// +// See golang.org/issue/41730 for details. +var defaultGOVCS = govcsConfig{ + {"private", []string{"all"}}, + {"public", []string{"git", "hg"}}, +} + +func checkGOVCS(vcs *Cmd, root string) error { + if vcs == vcsMod { + // Direct module (proxy protocol) fetches don't + // involve an external version control system + // and are always allowed. + return nil + } + + govcsOnce.Do(func() { + govcs, govcsErr = parseGOVCS(os.Getenv("GOVCS")) + govcs = append(govcs, defaultGOVCS...) + }) + if govcsErr != nil { + return govcsErr + } + + private := module.MatchPrefixPatterns(cfg.GOPRIVATE, root) + if !govcs.allow(root, private, vcs.Cmd) { + what := "public" + if private { + what = "private" + } + return fmt.Errorf("GOVCS disallows using %s for %s %s", vcs.Cmd, what, root) + } + + return nil +} + // CheckNested checks for an incorrectly-nested VCS-inside-VCS // situation for dir, checking parents up until srcRoot. func CheckNested(vcs *Cmd, dir, srcRoot string) error { @@ -733,6 +870,9 @@ func repoRootFromVCSPaths(importPath string, security web.SecurityMode, vcsPaths if vcs == nil { return nil, fmt.Errorf("unknown version control system %q", match["vcs"]) } + if err := checkGOVCS(vcs, match["root"]); err != nil { + return nil, err + } var repoURL string if !srv.schemelessRepo { repoURL = match["repo"] @@ -857,6 +997,10 @@ func repoRootForImportDynamic(importPath string, mod ModuleMode, security web.Se } } + if err := checkGOVCS(vcs, mmi.Prefix); err != nil { + return nil, err + } + rr := &RepoRoot{ Repo: mmi.RepoRoot, Root: mmi.Prefix, diff --git a/src/cmd/go/internal/vcs/vcs_test.go b/src/cmd/go/internal/vcs/vcs_test.go index 5b874204f1e..72d74a01e30 100644 --- a/src/cmd/go/internal/vcs/vcs_test.go +++ b/src/cmd/go/internal/vcs/vcs_test.go @@ -11,11 +11,20 @@ import ( "os" "path" "path/filepath" + "strings" "testing" "cmd/go/internal/web" ) +func init() { + // GOVCS defaults to public:git|hg,private:all, + // which breaks many tests here - they can't use non-git, non-hg VCS at all! + // Change to fully permissive. + // The tests of the GOVCS setting itself are in ../../testdata/script/govcs.txt. + os.Setenv("GOVCS", "*:all") +} + // Test that RepoRootForImportPath determines the correct RepoRoot for a given importPath. // TODO(cmang): Add tests for SVN and BZR. func TestRepoRootForImportPath(t *testing.T) { @@ -473,3 +482,98 @@ func TestValidateRepoRoot(t *testing.T) { } } } + +var govcsTests = []struct { + govcs string + path string + vcs string + ok bool +}{ + {"private:all", "is-public.com/foo", "zzz", false}, + {"private:all", "is-private.com/foo", "zzz", true}, + {"public:all", "is-public.com/foo", "zzz", true}, + {"public:all", "is-private.com/foo", "zzz", false}, + {"public:all,private:none", "is-public.com/foo", "zzz", true}, + {"public:all,private:none", "is-private.com/foo", "zzz", false}, + {"*:all", "is-public.com/foo", "zzz", true}, + {"golang.org:git", "golang.org/x/text", "zzz", false}, + {"golang.org:git", "golang.org/x/text", "git", true}, + {"golang.org:zzz", "golang.org/x/text", "zzz", true}, + {"golang.org:zzz", "golang.org/x/text", "git", false}, + {"golang.org:zzz", "golang.org/x/text", "zzz", true}, + {"golang.org:zzz", "golang.org/x/text", "git", false}, + {"golang.org:git|hg", "golang.org/x/text", "hg", true}, + {"golang.org:git|hg", "golang.org/x/text", "git", true}, + {"golang.org:git|hg", "golang.org/x/text", "zzz", false}, + {"golang.org:all", "golang.org/x/text", "hg", true}, + {"golang.org:all", "golang.org/x/text", "git", true}, + {"golang.org:all", "golang.org/x/text", "zzz", true}, + {"other.xyz/p:none,golang.org/x:git", "other.xyz/p/x", "git", false}, + {"other.xyz/p:none,golang.org/x:git", "unexpected.com", "git", false}, + {"other.xyz/p:none,golang.org/x:git", "golang.org/x/text", "zzz", false}, + {"other.xyz/p:none,golang.org/x:git", "golang.org/x/text", "git", true}, + {"other.xyz/p:none,golang.org/x:zzz", "golang.org/x/text", "zzz", true}, + {"other.xyz/p:none,golang.org/x:zzz", "golang.org/x/text", "git", false}, + {"other.xyz/p:none,golang.org/x:git|hg", "golang.org/x/text", "hg", true}, + {"other.xyz/p:none,golang.org/x:git|hg", "golang.org/x/text", "git", true}, + {"other.xyz/p:none,golang.org/x:git|hg", "golang.org/x/text", "zzz", false}, + {"other.xyz/p:none,golang.org/x:all", "golang.org/x/text", "hg", true}, + {"other.xyz/p:none,golang.org/x:all", "golang.org/x/text", "git", true}, + {"other.xyz/p:none,golang.org/x:all", "golang.org/x/text", "zzz", true}, + {"other.xyz/p:none,golang.org/x:git", "golang.org/y/text", "zzz", false}, + {"other.xyz/p:none,golang.org/x:git", "golang.org/y/text", "git", false}, + {"other.xyz/p:none,golang.org/x:zzz", "golang.org/y/text", "zzz", false}, + {"other.xyz/p:none,golang.org/x:zzz", "golang.org/y/text", "git", false}, + {"other.xyz/p:none,golang.org/x:git|hg", "golang.org/y/text", "hg", false}, + {"other.xyz/p:none,golang.org/x:git|hg", "golang.org/y/text", "git", false}, + {"other.xyz/p:none,golang.org/x:git|hg", "golang.org/y/text", "zzz", false}, + {"other.xyz/p:none,golang.org/x:all", "golang.org/y/text", "hg", false}, + {"other.xyz/p:none,golang.org/x:all", "golang.org/y/text", "git", false}, + {"other.xyz/p:none,golang.org/x:all", "golang.org/y/text", "zzz", false}, +} + +func TestGOVCS(t *testing.T) { + for _, tt := range govcsTests { + cfg, err := parseGOVCS(tt.govcs) + if err != nil { + t.Errorf("parseGOVCS(%q): %v", tt.govcs, err) + continue + } + private := strings.HasPrefix(tt.path, "is-private") + ok := cfg.allow(tt.path, private, tt.vcs) + if ok != tt.ok { + t.Errorf("parseGOVCS(%q).allow(%q, %v, %q) = %v, want %v", + tt.govcs, tt.path, private, tt.vcs, ok, tt.ok) + } + } +} + +var govcsErrors = []struct { + s string + err string +}{ + {`,`, `empty entry in GOVCS`}, + {`,x`, `empty entry in GOVCS`}, + {`x,`, `malformed entry in GOVCS (missing colon): "x"`}, + {`x:y,`, `empty entry in GOVCS`}, + {`x`, `malformed entry in GOVCS (missing colon): "x"`}, + {`x:`, `empty VCS list in GOVCS: "x:"`}, + {`x:|`, `empty VCS name in GOVCS: "x:|"`}, + {`x:y|`, `empty VCS name in GOVCS: "x:y|"`}, + {`x:|y`, `empty VCS name in GOVCS: "x:|y"`}, + {`x:y,z:`, `empty VCS list in GOVCS: "z:"`}, + {`x:y,z:|`, `empty VCS name in GOVCS: "z:|"`}, + {`x:y,z:|w`, `empty VCS name in GOVCS: "z:|w"`}, + {`x:y,z:w|`, `empty VCS name in GOVCS: "z:w|"`}, + {`x:y,z:w||v`, `empty VCS name in GOVCS: "z:w||v"`}, + {`x:y,x:z`, `unreachable pattern in GOVCS: "x:z" after "x:y"`}, +} + +func TestGOVCSErrors(t *testing.T) { + for _, tt := range govcsErrors { + _, err := parseGOVCS(tt.s) + if err == nil || !strings.Contains(err.Error(), tt.err) { + t.Errorf("parseGOVCS(%s): err=%v, want %v", tt.s, err, tt.err) + } + } +} diff --git a/src/cmd/go/main.go b/src/cmd/go/main.go index 37bb7d6d270..9cc44da84db 100644 --- a/src/cmd/go/main.go +++ b/src/cmd/go/main.go @@ -75,10 +75,11 @@ func init() { modload.HelpModules, modget.HelpModuleGet, modfetch.HelpModuleAuth, - modfetch.HelpModulePrivate, help.HelpPackages, + modfetch.HelpPrivate, test.HelpTestflag, test.HelpTestfunc, + modget.HelpVCS, } } diff --git a/src/cmd/go/script_test.go b/src/cmd/go/script_test.go index e301f2fb069..b1d1499038f 100644 --- a/src/cmd/go/script_test.go +++ b/src/cmd/go/script_test.go @@ -135,6 +135,7 @@ func (ts *testScript) setup() { "GOSUMDB=" + testSumDBVerifierKey, "GONOPROXY=", "GONOSUMDB=", + "GOVCS=*:all", "PWD=" + ts.cd, tempEnvName() + "=" + filepath.Join(ts.workdir, "tmp"), "devnull=" + os.DevNull, diff --git a/src/cmd/go/testdata/script/govcs.txt b/src/cmd/go/testdata/script/govcs.txt new file mode 100644 index 00000000000..35f092ee493 --- /dev/null +++ b/src/cmd/go/testdata/script/govcs.txt @@ -0,0 +1,174 @@ +env GO111MODULE=on +env proxy=$GOPROXY +env GOPROXY=direct + +# GOVCS stops go get +env GOVCS='*:none' +! go get github.com/google/go-cmp +stderr 'go get: GOVCS disallows using git for public github.com/google/go-cmp' +env GOPRIVATE='github.com/google' +! go get github.com/google/go-cmp +stderr 'go get: GOVCS disallows using git for private github.com/google/go-cmp' + +# public pattern works +env GOPRIVATE='github.com/google' +env GOVCS='public:all,private:none' +! go get github.com/google/go-cmp +stderr 'go get: GOVCS disallows using git for private github.com/google/go-cmp' + +# private pattern works +env GOPRIVATE='hubgit.com/google' +env GOVCS='private:all,public:none' +! go get github.com/google/go-cmp +stderr 'go get: GOVCS disallows using git for public github.com/google/go-cmp' + +# other patterns work (for more patterns, see TestGOVCS) +env GOPRIVATE= +env GOVCS='github.com:svn|hg' +! go get github.com/google/go-cmp +stderr 'go get: GOVCS disallows using git for public github.com/google/go-cmp' +env GOVCS='github.com/google/go-cmp/inner:git,github.com:svn|hg' +! go get github.com/google/go-cmp +stderr 'go get: GOVCS disallows using git for public github.com/google/go-cmp' + +# bad patterns are reported (for more bad patterns, see TestGOVCSErrors) +env GOVCS='git' +! go get github.com/google/go-cmp +stderr 'go get github.com/google/go-cmp: malformed entry in GOVCS \(missing colon\): "git"' + +env GOVCS=github.com:hg,github.com:git +! go get github.com/google/go-cmp +stderr 'go get github.com/google/go-cmp: unreachable pattern in GOVCS: "github.com:git" after "github.com:hg"' + +# bad GOVCS patterns do not stop commands that do not need to check VCS +go list +env GOPROXY=$proxy +go get -d rsc.io/quote # ok because used proxy +env GOPROXY=direct + +# svn is disallowed by default +env GOPRIVATE= +env GOVCS= +! go get rsc.io/nonexist.svn/hello +stderr 'go get rsc.io/nonexist.svn/hello: GOVCS disallows using svn for public rsc.io/nonexist.svn' + +# fossil is disallowed by default +env GOPRIVATE= +env GOVCS= +! go get rsc.io/nonexist.fossil/hello +stderr 'go get rsc.io/nonexist.fossil/hello: GOVCS disallows using fossil for public rsc.io/nonexist.fossil' + +# bzr is disallowed by default +env GOPRIVATE= +env GOVCS= +! go get rsc.io/nonexist.bzr/hello +stderr 'go get rsc.io/nonexist.bzr/hello: GOVCS disallows using bzr for public rsc.io/nonexist.bzr' + +# git is OK by default +env GOVCS= +env GONOSUMDB='*' +[net] [exec:git] [!short] go get rsc.io/sampler + +# hg is OK by default +env GOVCS= +env GONOSUMDB='*' +[net] [exec:hg] [!short] go get vcs-test.golang.org/go/custom-hg-hello + +# git can be disallowed +env GOVCS=public:hg +! go get rsc.io/nonexist.git/hello +stderr 'go get rsc.io/nonexist.git/hello: GOVCS disallows using git for public rsc.io/nonexist.git' + +# hg can be disallowed +env GOVCS=public:git +! go get rsc.io/nonexist.hg/hello +stderr 'go get rsc.io/nonexist.hg/hello: GOVCS disallows using hg for public rsc.io/nonexist.hg' + +# Repeat in GOPATH mode. Error texts slightly different. + +env GO111MODULE=off + +# GOVCS stops go get +env GOVCS='*:none' +! go get github.com/google/go-cmp +stderr 'package github.com/google/go-cmp: GOVCS disallows using git for public github.com/google/go-cmp' +env GOPRIVATE='github.com/google' +! go get github.com/google/go-cmp +stderr 'package github.com/google/go-cmp: GOVCS disallows using git for private github.com/google/go-cmp' + +# public pattern works +env GOPRIVATE='github.com/google' +env GOVCS='public:all,private:none' +! go get github.com/google/go-cmp +stderr 'package github.com/google/go-cmp: GOVCS disallows using git for private github.com/google/go-cmp' + +# private pattern works +env GOPRIVATE='hubgit.com/google' +env GOVCS='private:all,public:none' +! go get github.com/google/go-cmp +stderr 'package github.com/google/go-cmp: GOVCS disallows using git for public github.com/google/go-cmp' + +# other patterns work (for more patterns, see TestGOVCS) +env GOPRIVATE= +env GOVCS='github.com:svn|hg' +! go get github.com/google/go-cmp +stderr 'package github.com/google/go-cmp: GOVCS disallows using git for public github.com/google/go-cmp' +env GOVCS='github.com/google/go-cmp/inner:git,github.com:svn|hg' +! go get github.com/google/go-cmp +stderr 'package github.com/google/go-cmp: GOVCS disallows using git for public github.com/google/go-cmp' + +# bad patterns are reported (for more bad patterns, see TestGOVCSErrors) +env GOVCS='git' +! go get github.com/google/go-cmp +stderr 'package github.com/google/go-cmp: malformed entry in GOVCS \(missing colon\): "git"' + +env GOVCS=github.com:hg,github.com:git +! go get github.com/google/go-cmp +stderr 'package github.com/google/go-cmp: unreachable pattern in GOVCS: "github.com:git" after "github.com:hg"' + +# bad GOVCS patterns do not stop commands that do not need to check VCS +go list + +# svn is disallowed by default +env GOPRIVATE= +env GOVCS= +! go get rsc.io/nonexist.svn/hello +stderr 'package rsc.io/nonexist.svn/hello: GOVCS disallows using svn for public rsc.io/nonexist.svn' + +# fossil is disallowed by default +env GOPRIVATE= +env GOVCS= +! go get rsc.io/nonexist.fossil/hello +stderr 'package rsc.io/nonexist.fossil/hello: GOVCS disallows using fossil for public rsc.io/nonexist.fossil' + +# bzr is disallowed by default +env GOPRIVATE= +env GOVCS= +! go get rsc.io/nonexist.bzr/hello +stderr 'package rsc.io/nonexist.bzr/hello: GOVCS disallows using bzr for public rsc.io/nonexist.bzr' + +# git is OK by default +env GOVCS= +env GONOSUMDB='*' +[net] [exec:git] [!short] go get rsc.io/sampler + +# hg is OK by default +env GOVCS= +env GONOSUMDB='*' +[net] [exec:hg] [!short] go get vcs-test.golang.org/go/custom-hg-hello + +# git can be disallowed +env GOVCS=public:hg +! go get rsc.io/nonexist.git/hello +stderr 'package rsc.io/nonexist.git/hello: GOVCS disallows using git for public rsc.io/nonexist.git' + +# hg can be disallowed +env GOVCS=public:git +! go get rsc.io/nonexist.hg/hello +stderr 'package rsc.io/nonexist.hg/hello: GOVCS disallows using hg for public rsc.io/nonexist.hg' + +-- go.mod -- +module m + +-- p.go -- +package p diff --git a/src/cmd/go/testdata/script/mod_auth.txt b/src/cmd/go/testdata/script/mod_auth.txt index 544acbc1f86..d8ea5867d6b 100644 --- a/src/cmd/go/testdata/script/mod_auth.txt +++ b/src/cmd/go/testdata/script/mod_auth.txt @@ -3,6 +3,7 @@ env GO111MODULE=on env GOPROXY=direct env GOSUMDB=off +env GOVCS='*:off' # Without credentials, downloading a module from a path that requires HTTPS # basic auth should fail. diff --git a/src/internal/cfg/cfg.go b/src/internal/cfg/cfg.go index bdbe9df3e75..553021374d5 100644 --- a/src/internal/cfg/cfg.go +++ b/src/internal/cfg/cfg.go @@ -58,6 +58,7 @@ const KnownEnv = ` GOSUMDB GOTMPDIR GOTOOLDIR + GOVCS GOWASM GO_EXTLINK_ENABLED PKG_CONFIG