diff --git a/src/cmd/go/alldocs.go b/src/cmd/go/alldocs.go index 43f62e822dc..9f1b4880b96 100644 --- a/src/cmd/go/alldocs.go +++ b/src/cmd/go/alldocs.go @@ -47,8 +47,9 @@ // importpath import path syntax // modules modules, module versions, and more // module-get module-aware go get -// packages package lists and patterns // module-auth module authentication using go.sum +// module-private module configuration for non-public modules +// packages package lists and patterns // testflag testing flags // testfunc testing functions // @@ -1568,19 +1569,16 @@ // For more details see: 'go help gopath'. // GOPROXY // URL of Go module proxy. See 'go help modules'. -// GONOPROXY +// GOPRIVATE, GONOPROXY, GONOSUMDB // Comma-separated list of glob patterns (in the syntax of Go's path.Match) -// of module path prefixes that should always be fetched directly, ignoring -// the GOPROXY setting. See 'go help modules'. +// of module path prefixes that should always be fetched directly +// or that should not be compared against the checksum database. +// See 'go help module-private'. +// GOROOT +// The root of the go tree. // GOSUMDB // The name of checksum database to use and optionally its public key and // URL. See 'go help module-auth'. -// GONOSUMDB -// Comma-separated list of glob patterns (in the syntax of Go's path.Match) -// of module path prefixes that should not be compared against the checksum -// database. See 'go help module-auth'. -// GOROOT -// The root of the go tree. // GOTMPDIR // The directory where the go command will write // temporary source files, packages, and binaries. @@ -2598,19 +2596,8 @@ // to cause a direct connection to be attempted at that point in the search. // Any proxies listed after "direct" are never consulted. // -// The GONOPROXY environment variable is a comma-separated list of -// glob patterns (in the syntax of Go's path.Match) of module path prefixes -// that should always be fetched directly, ignoring the GOPROXY setting. -// For example, -// -// GONOPROXY=*.corp.example.com,rsc.io/private -// -// forces a direct connection to download modules with path prefixes matching -// either pattern, including "git.corp.example.com/xyzzy", "rsc.io/private", -// and "rsc.io/private/quux". -// -// The 'go env -w' command (see 'go help env') can be used to set these variables -// for future go command invocations. +// The GOPRIVATE and GONOPROXY environment variables allow bypassing +// the proxy for selected modules. See 'go help module-private' for details. // // No matter the source of the modules, the go command checks downloads against // known checksums, to detect unexpected changes in the content of any specific @@ -2642,6 +2629,137 @@ // are still ignored. // // +// Module authentication using go.sum +// +// The go command tries to authenticate every downloaded module, +// checking that the bits downloaded for a specific module version today +// match bits downloaded yesterday. This ensures repeatable builds +// and detects introduction of unexpected changes, malicious or not. +// +// In each module's root, alongside go.mod, the go command maintains +// a file named go.sum containing the cryptographic checksums of the +// module's dependencies. +// +// The form of each line in go.sum is three fields: +// +// [/go.mod] +// +// Each known module version results in two lines in the go.sum file. +// The first line gives the hash of the module version's file tree. +// The second line appends "/go.mod" to the version and gives the hash +// of only the module version's (possibly synthesized) go.mod file. +// The go.mod-only hash allows downloading and authenticating a +// module version's go.mod file, which is needed to compute the +// dependency graph, without also downloading all the module's source code. +// +// The hash begins with an algorithm prefix of the form "h:". +// The only defined algorithm prefix is "h1:", which uses SHA-256. +// +// Module authentication failures +// +// The go command maintains a cache of downloaded packages and computes +// and records the cryptographic checksum of each package at download time. +// In normal operation, the go command checks the main module's go.sum file +// against these precomputed checksums instead of recomputing them on +// each command invocation. The 'go mod verify' command checks that +// the cached copies of module downloads still match both their recorded +// checksums and the entries in go.sum. +// +// In day-to-day development, the checksum of a given module version +// should never change. Each time a dependency is used by a given main +// module, the go command checks its local cached copy, freshly +// downloaded or not, against the main module's go.sum. If the checksums +// don't match, the go command reports the mismatch as a security error +// and refuses to run the build. When this happens, proceed with caution: +// code changing unexpectedly means today's build will not match +// yesterday's, and the unexpected change may not be beneficial. +// +// If the go command reports a mismatch in go.sum, the downloaded code +// for the reported module version does not match the one used in a +// previous build of the main module. It is important at that point +// to find out what the right checksum should be, to decide whether +// go.sum is wrong or the downloaded code is wrong. Usually go.sum is right: +// you want to use the same code you used yesterday. +// +// If a downloaded module is not yet included in go.sum and it is a publicly +// available module, the go command consults the Go checksum database to fetch +// the expected go.sum lines. If the downloaded code does not match those +// lines, the go command reports the mismatch and exits. Note that the +// database is not consulted for module versions already listed in go.sum. +// +// If a go.sum mismatch is reported, it is always worth investigating why +// the code downloaded today differs from what was downloaded yesterday. +// +// The GOSUMDB environment variable identifies the name of checksum database +// to use and optionally its public key and URL, as in: +// +// GOSUMDB="sum.golang.org" +// GOSUMDB="sum.golang.org+" +// GOSUMDB="sum.golang.org+ https://sum.golang.org" +// +// The go command knows the public key of sum.golang.org; use of any other +// database requires giving the public key explicitly. The URL defaults to +// "https://" followed by the database name. +// +// GOSUMDB defaults to "sum.golang.org", the Go checksum database run by Google. +// See https://sum.golang.org/privacy for the service's privacy policy. +// +// If GOSUMDB is set to "off", or if "go get" is invoked with the -insecure flag, +// the checksum database is not consulted, and all unrecognized modules are +// accepted, at the cost of giving up the security guarantee of verified repeatable +// downloads for all modules. A better way to bypass the checksum database +// for specific modules is to use the GOPRIVATE or GONOSUMDB environment +// variables. See 'go help module-private' for details. +// +// The 'go env -w' command (see 'go help env') can be used to set these variables +// for future go command invocations. +// +// +// Module configuration for non-public modules +// +// The go command defaults to downloading modules from the public Go module +// mirror at proxy.golang.org. It also defaults to validating downloaded modules, +// regardless of source, against the public Go checksum database at sum.golang.org. +// These defaults work well for publicly available source code. +// +// The GOPRIVATE environment variable controls which modules the go command +// considers to be private (not available publicly) and should therefore not use the +// proxy or checksum database. The variable is a comma-separated list of +// glob patterns (in the syntax of Go's path.Match) of module path prefixes. +// For example, +// +// GOPRIVATE=*.corp.example.com,rsc.io/private +// +// causes the go command to treat as private any module with a path prefix +// matching either pattern, including git.corp.example.com/xyzzy, rsc.io/private, +// and rsc.io/private/quux. +// +// The GOPRIVATE environment variable may be used by other tools as well to +// identify non-public modules. For example, an editor could use GOPRIVATE +// to decide whether to hyperlink a package import to a godoc.org page. +// +// For fine-grained control over module download and validation, the GONOPROXY +// and GONOSUMDB environment variables accept the same kind of glob list +// and override GOPRIVATE for the specific decision of whether to use the proxy +// and checksum database, respectively. +// +// For example, if a company ran a module proxy serving private modules, +// users would configure go using: +// +// GOPRIVATE=*.corp.example.com +// GOPROXY=proxy.example.com +// GONOPROXY=none +// +// This would tell the go comamnd and other tools that modules beginning with +// a corp.example.com subdomain are private but that the company proxy should +// be used for downloading both public and private modules, because +// GONOPROXY has been set to a pattern that won't match any modules, +// overriding GOPRIVATE. +// +// The 'go env -w' command (see 'go help env') can be used to set these variables +// for future go command invocations. +// +// // Package lists and patterns // // Many commands apply to a set of packages: @@ -2726,102 +2844,6 @@ // by the go tool, as are directories named "testdata". // // -// Module authentication using go.sum -// -// The go command tries to authenticate every downloaded module, -// checking that the bits downloaded for a specific module version today -// match bits downloaded yesterday. This ensures repeatable builds -// and detects introduction of unexpected changes, malicious or not. -// -// In each module's root, alongside go.mod, the go command maintains -// a file named go.sum containing the cryptographic checksums of the -// module's dependencies. -// -// The form of each line in go.sum is three fields: -// -// [/go.mod] -// -// Each known module version results in two lines in the go.sum file. -// The first line gives the hash of the module version's file tree. -// The second line appends "/go.mod" to the version and gives the hash -// of only the module version's (possibly synthesized) go.mod file. -// The go.mod-only hash allows downloading and authenticating a -// module version's go.mod file, which is needed to compute the -// dependency graph, without also downloading all the module's source code. -// -// The hash begins with an algorithm prefix of the form "h:". -// The only defined algorithm prefix is "h1:", which uses SHA-256. -// -// Module authentication failures -// -// The go command maintains a cache of downloaded packages and computes -// and records the cryptographic checksum of each package at download time. -// In normal operation, the go command checks the main module's go.sum file -// against these precomputed checksums instead of recomputing them on -// each command invocation. The 'go mod verify' command checks that -// the cached copies of module downloads still match both their recorded -// checksums and the entries in go.sum. -// -// In day-to-day development, the checksum of a given module version -// should never change. Each time a dependency is used by a given main -// module, the go command checks its local cached copy, freshly -// downloaded or not, against the main module's go.sum. If the checksums -// don't match, the go command reports the mismatch as a security error -// and refuses to run the build. When this happens, proceed with caution: -// code changing unexpectedly means today's build will not match -// yesterday's, and the unexpected change may not be beneficial. -// -// If the go command reports a mismatch in go.sum, the downloaded code -// for the reported module version does not match the one used in a -// previous build of the main module. It is important at that point -// to find out what the right checksum should be, to decide whether -// go.sum is wrong or the downloaded code is wrong. Usually go.sum is right: -// you want to use the same code you used yesterday. -// -// If a downloaded module is not yet included in go.sum and it is a publicly -// available module, the go command consults the Go checksum database to fetch -// the expected go.sum lines. If the downloaded code does not match those -// lines, the go command reports the mismatch and exits. Note that the -// database is not consulted for module versions already listed in go.sum. -// -// If a go.sum mismatch is reported, it is always worth investigating why -// the code downloaded today differs from what was downloaded yesterday. -// -// The GOSUMDB environment variable identifies the name of checksum database -// to use and optionally its public key and URL, as in: -// -// GOSUMDB="sum.golang.org" -// GOSUMDB="sum.golang.org+" -// GOSUMDB="sum.golang.org+ https://sum.golang.org" -// -// The go command knows the public key of sum.golang.org; use of any other -// database requires giving the public key explicitly. The URL defaults to -// "https://" followed by the database name. -// -// GOSUMDB defaults to "sum.golang.org", the Go checksum database run by Google. -// See https://sum.golang.org/privacy for the service's privacy policy. -// -// If GOSUMDB is set to "off", or if "go get" is invoked with the -insecure flag, -// the checksum database is not consulted, and all unrecognized modules are -// accepted, at the cost of giving up the security guarantee of verified repeatable -// downloads for all modules. A better way to bypass the checksum database -// for specific modules is to use the GONOSUMDB environment variable. -// -// The GONOSUMDB environment variable is a comma-separated list of -// glob patterns (in the syntax of Go's path.Match) of module path prefixes -// that should not be compared against the checksum database. -// For example, -// -// GONOSUMDB=*.corp.example.com,rsc.io/private -// -// disables checksum database lookups for modules with path prefixes matching -// either pattern, including "git.corp.example.com/xyzzy", "rsc.io/private", -// and "rsc.io/private/quux". -// -// The 'go env -w' command (see 'go help env') can be used to set these variables -// for future go command invocations. -// -// // Testing flags // // The 'go test' command takes both flags that apply to 'go test' itself diff --git a/src/cmd/go/internal/cfg/cfg.go b/src/cmd/go/internal/cfg/cfg.go index c3c9c978a5d..a0b51a72c33 100644 --- a/src/cmd/go/internal/cfg/cfg.go +++ b/src/cmd/go/internal/cfg/cfg.go @@ -265,6 +265,7 @@ var knownEnv = ` GOOS GOPATH GOPPC64 + GOPRIVATE GOPROXY GOROOT GOSUMDB @@ -291,30 +292,13 @@ var ( GOPPC64 = envOr("GOPPC64", fmt.Sprintf("%s%d", "power", objabi.GOPPC64)) GOWASM = envOr("GOWASM", fmt.Sprint(objabi.GOWASM)) - GOPROXY = goproxy() - GOSUMDB = gosumdb() - GONOPROXY = Getenv("GONOPROXY") - GONOSUMDB = Getenv("GONOSUMDB") + GOPROXY = envOr("GOPROXY", "https://proxy.golang.org,direct") + GOSUMDB = envOr("GOSUMDB", "sum.golang.org") + GOPRIVATE = Getenv("GOPRIVATE") + GONOPROXY = envOr("GONOPROXY", GOPRIVATE) + GONOSUMDB = envOr("GONOSUMDB", GOPRIVATE) ) -func goproxy() string { - v := Getenv("GOPROXY") - if v != "" { - return v - } - - return "https://proxy.golang.org,direct" -} - -func gosumdb() string { - v := Getenv("GOSUMDB") - if v != "" { - return v - } - - return "sum.golang.org" -} - // GetArchEnv returns the name and setting of the // GOARCH-specific architecture environment variable. // If the current architecture has no GOARCH-specific variable, diff --git a/src/cmd/go/internal/envcmd/env.go b/src/cmd/go/internal/envcmd/env.go index b3d12dd681e..17852deed1e 100644 --- a/src/cmd/go/internal/envcmd/env.go +++ b/src/cmd/go/internal/envcmd/env.go @@ -79,6 +79,7 @@ func MkEnv() []cfg.EnvVar { {Name: "GONOSUMDB", Value: cfg.GONOSUMDB}, {Name: "GOOS", Value: cfg.Goos}, {Name: "GOPATH", Value: cfg.BuildContext.GOPATH}, + {Name: "GOPRIVATE", Value: cfg.GOPRIVATE}, {Name: "GOPROXY", Value: cfg.GOPROXY}, {Name: "GOROOT", Value: cfg.GOROOT}, {Name: "GOSUMDB", Value: cfg.GOSUMDB}, diff --git a/src/cmd/go/internal/help/helpdoc.go b/src/cmd/go/internal/help/helpdoc.go index 2cdd76c6cd4..c2b5fb4b830 100644 --- a/src/cmd/go/internal/help/helpdoc.go +++ b/src/cmd/go/internal/help/helpdoc.go @@ -510,19 +510,16 @@ General-purpose environment variables: For more details see: 'go help gopath'. GOPROXY URL of Go module proxy. See 'go help modules'. - GONOPROXY + GOPRIVATE, GONOPROXY, GONOSUMDB Comma-separated list of glob patterns (in the syntax of Go's path.Match) - of module path prefixes that should always be fetched directly, ignoring - the GOPROXY setting. See 'go help modules'. + of module path prefixes that should always be fetched directly + or that should not be compared against the checksum database. + See 'go help module-private'. + GOROOT + The root of the go tree. GOSUMDB The name of checksum database to use and optionally its public key and URL. See 'go help module-auth'. - GONOSUMDB - Comma-separated list of glob patterns (in the syntax of Go's path.Match) - of module path prefixes that should not be compared against the checksum - database. See 'go help module-auth'. - GOROOT - The root of the go tree. GOTMPDIR The directory where the go command will write temporary source files, packages, and binaries. diff --git a/src/cmd/go/internal/modfetch/fetch.go b/src/cmd/go/internal/modfetch/fetch.go index 94cb0d3a199..bc1d35e690b 100644 --- a/src/cmd/go/internal/modfetch/fetch.go +++ b/src/cmd/go/internal/modfetch/fetch.go @@ -631,7 +631,7 @@ For more information, see 'go help module-auth'. ` -var HelpSum = &base.Command{ +var HelpModuleAuth = &base.Command{ UsageLine: "module-auth", Short: "module authentication using go.sum", Long: ` @@ -712,18 +712,56 @@ If GOSUMDB is set to "off", or if "go get" is invoked with the -insecure flag, the checksum database is not consulted, and all unrecognized modules are accepted, at the cost of giving up the security guarantee of verified repeatable downloads for all modules. A better way to bypass the checksum database -for specific modules is to use the GONOSUMDB environment variable. - -The GONOSUMDB environment variable is a comma-separated list of -glob patterns (in the syntax of Go's path.Match) of module path prefixes -that should not be compared against the checksum database. -For example, - - GONOSUMDB=*.corp.example.com,rsc.io/private - -disables checksum database lookups for modules with path prefixes matching -either pattern, including "git.corp.example.com/xyzzy", "rsc.io/private", -and "rsc.io/private/quux". +for specific modules is to use the GOPRIVATE or GONOSUMDB environment +variables. See 'go help module-private' for details. + +The 'go env -w' command (see 'go help env') can be used to set these variables +for future go command invocations. +`, +} + +var HelpModulePrivate = &base.Command{ + UsageLine: "module-private", + Short: "module configuration for non-public modules", + Long: ` +The go command defaults to downloading modules from the public Go module +mirror at proxy.golang.org. It also defaults to validating downloaded modules, +regardless of source, against the public Go checksum database at sum.golang.org. +These defaults work well for publicly available source code. + +The GOPRIVATE environment variable controls which modules the go command +considers to be private (not available publicly) and should therefore not use the +proxy or checksum database. The variable is a comma-separated list of +glob patterns (in the syntax of Go's path.Match) of module path prefixes. +For example, + + GOPRIVATE=*.corp.example.com,rsc.io/private + +causes the go command to treat as private any module with a path prefix +matching either pattern, including git.corp.example.com/xyzzy, rsc.io/private, +and rsc.io/private/quux. + +The GOPRIVATE environment variable may be used by other tools as well to +identify non-public modules. For example, an editor could use GOPRIVATE +to decide whether to hyperlink a package import to a godoc.org page. + +For fine-grained control over module download and validation, the GONOPROXY +and GONOSUMDB environment variables accept the same kind of glob list +and override GOPRIVATE for the specific decision of whether to use the proxy +and checksum database, respectively. + +For example, if a company ran a module proxy serving private modules, +users would configure go using: + + GOPRIVATE=*.corp.example.com + GOPROXY=proxy.example.com + GONOPROXY=none + +This would tell the go comamnd and other tools that modules beginning with +a corp.example.com subdomain are private but that the company proxy should +be used for downloading both public and private modules, because +GONOPROXY has been set to a pattern that won't match any modules, +overriding GOPRIVATE. The 'go env -w' command (see 'go help env') can be used to set these variables for future go command invocations. diff --git a/src/cmd/go/internal/modfetch/repo.go b/src/cmd/go/internal/modfetch/repo.go index 92e977646f6..95351269dbe 100644 --- a/src/cmd/go/internal/modfetch/repo.go +++ b/src/cmd/go/internal/modfetch/repo.go @@ -241,8 +241,8 @@ func lookup(proxy, path string) (r Repo, err error) { var ( errModVendor = errors.New("module lookup disabled by -mod=vendor") errProxyOff = errors.New("module lookup disabled by GOPROXY=off") - errNoproxy error = notExistError("disabled by GONOPROXY") - errUseProxy error = notExistError("path does not match GONOPROXY") + errNoproxy error = notExistError("disabled by GOPRIVATE/GONOPROXY") + errUseProxy error = notExistError("path does not match GOPRIVATE/GONOPROXY") ) func lookupDirect(path string) (Repo, error) { diff --git a/src/cmd/go/internal/modload/help.go b/src/cmd/go/internal/modload/help.go index 5630524ca26..9ce8dfb33b3 100644 --- a/src/cmd/go/internal/modload/help.go +++ b/src/cmd/go/internal/modload/help.go @@ -348,19 +348,8 @@ HTTP response. The string "direct" may appear in the proxy list, to cause a direct connection to be attempted at that point in the search. Any proxies listed after "direct" are never consulted. -The GONOPROXY environment variable is a comma-separated list of -glob patterns (in the syntax of Go's path.Match) of module path prefixes -that should always be fetched directly, ignoring the GOPROXY setting. -For example, - - GONOPROXY=*.corp.example.com,rsc.io/private - -forces a direct connection to download modules with path prefixes matching -either pattern, including "git.corp.example.com/xyzzy", "rsc.io/private", -and "rsc.io/private/quux". - -The 'go env -w' command (see 'go help env') can be used to set these variables -for future go command invocations. +The GOPRIVATE and GONOPROXY environment variables allow bypassing +the proxy for selected modules. See 'go help module-private' for details. No matter the source of the modules, the go command checks downloads against known checksums, to detect unexpected changes in the content of any specific diff --git a/src/cmd/go/main.go b/src/cmd/go/main.go index 0207862d0b0..73da736882f 100644 --- a/src/cmd/go/main.go +++ b/src/cmd/go/main.go @@ -71,8 +71,9 @@ func init() { help.HelpImportPath, modload.HelpModules, modget.HelpModuleGet, + modfetch.HelpModuleAuth, + modfetch.HelpModulePrivate, help.HelpPackages, - modfetch.HelpSum, test.HelpTestflag, test.HelpTestfunc, } diff --git a/src/cmd/go/script_test.go b/src/cmd/go/script_test.go index c169a4ceec4..87331599f64 100644 --- a/src/cmd/go/script_test.go +++ b/src/cmd/go/script_test.go @@ -111,6 +111,7 @@ func (ts *testScript) setup() { "GOOS=" + runtime.GOOS, "GOPATH=" + filepath.Join(ts.workdir, "gopath"), "GOPROXY=" + proxyURL, + "GOPRIVATE=", "GOROOT=" + testGOROOT, "GOSUMDB=" + testSumDBVerifierKey, "GONOPROXY=", diff --git a/src/cmd/go/testdata/script/mod_gonoproxy.txt b/src/cmd/go/testdata/script/mod_gonoproxy.txt index f2eb4efb6b7..2bd94cdee04 100644 --- a/src/cmd/go/testdata/script/mod_gonoproxy.txt +++ b/src/cmd/go/testdata/script/mod_gonoproxy.txt @@ -1,7 +1,7 @@ env GO111MODULE=on env sumdb=$GOSUMDB env proxy=$GOPROXY -env GOPROXY GONOPROXY GOSUMDB GONOSUMDB +env GOPRIVATE GOPROXY GONOPROXY GOSUMDB GONOSUMDB env dbname=localhost.localdev/sumdb # disagree with sumdb fails @@ -13,20 +13,25 @@ stderr 'SECURITY ERROR' # but GONOSUMDB bypasses sumdb, for rsc.io/quote, rsc.io/sampler, golang.org/x/text env GONOSUMDB='*/quote,*/*mple*,golang.org/x' go get rsc.io/quote +rm go.sum +env GOPRIVATE='*/quote,*/*mple*,golang.org/x' +env GONOPROXY=none # that is, proxy all despite GOPRIVATE +go get rsc.io/quote # and GONOPROXY bypasses proxy [!net] skip [!exec:git] skip +env GOPRIVATE=none env GONOPROXY='*/fortune' ! go get rsc.io/fortune # does not exist in real world, only on test proxy stderr 'git ls-remote' env GOSUMDB= -env GONOPROXY='*/x' +env GONOPROXY= +env GOPRIVATE='*/x' go get golang.org/x/text go list -m all ! stdout 'text.*v0.0.0-2017' # should not have the version from the proxy - -- go.mod.orig -- module m