qbit/go
1
0
Fork 0
mirror of https://github.com/golang/go synced 2024-11-26 18:16:48 -07:00
Code Releases Activity

cmd/go: stop tests from using network during -short

Browse Source 
It turned out that "go get" was using the network to look up
https://github.com?go-get=1 while resolving github.com/google/go-cmp,
and that is not the fastest page to load.
Stop that lookup by adjusting the path prefixes in the vcs table.

It also turned out that "go get" was using the network to look up
https://rsc.io?go-get=1 while resolving https://rsc.io/nonexist.svn.
That's a bit more defensible maybe, since rsc.io is not a known VCS host.
But for tests we really want to avoid the network entirely, so this CL
adds a special case in repoRootFromVCSPaths that returns a hard error
for plain "rsc.io" instead of doing the web fetch.

To keep us honest in the future, I added two automatically-set env
variables TESTGONETWORK=panic and TESTGOVCS=panic.
These cause the go command to panic rather than make a network request
or invoke a VCS command.

go test -short cmd/go now passes with these checks.

This reduced the time spent in go test -short cmd/go on my
Google workstation from 154s to 30s. (Yay network firewalls.)

Change-Id: I49207fca7f901fa011765fb984dc9cec8b691f11
Reviewed-on: https://go-review.googlesource.com/c/go/+/274441
Trust: Russ Cox <rsc@golang.org>
Trust: Jay Conrod <jayconrod@google.com>
Run-TryBot: Russ Cox <rsc@golang.org>
Reviewed-by: Jay Conrod <jayconrod@google.com>
This commit is contained in:
Russ Cox 2020-12-02 10:14:53 -05:00
parent 3d913a9266
commit ac38af2f3d
5 changed files with 67 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
src/cmd/go/internal/modfetch/codehost/codehost.go
View File

@ -264,6 +264,9 @@ func RunWithStdin(dir string, stdin io.Reader, cmdline ...interface{}) ([]byte,
}
cmd := str.StringList(cmdline...)
if os.Getenv("TESTGOVCS") == "panic" {
panic(fmt.Sprintf("use of vcs: %v", cmd))
}
if cfg.BuildX {
text := new(strings.Builder)
if dir != "" {

101
src/cmd/go/internal/vcs/vcs.go
View File

@ -23,6 +23,7 @@ import (
"cmd/go/internal/base"
"cmd/go/internal/cfg"
"cmd/go/internal/search"
"cmd/go/internal/str"
"cmd/go/internal/web"
"golang.org/x/mod/module"
@ -539,7 +540,7 @@ func (v *Cmd) TagSync(dir, tag string) error {
// A vcsPath describes how to convert an import path into a
// version control system and repository name.
type vcsPath struct {
prefix string // prefix this description applies to
pathPrefix string // prefix this description applies to
regexp *lazyregexp.Regexp // compiled pattern for import path
repo string // repository to use (expand with match of re)
vcs string // version control system to use (expand with match of re)
@ -826,6 +827,20 @@ var errUnknownSite = errors.New("dynamic lookup required to find mapping")
// repoRootFromVCSPaths attempts to map importPath to a repoRoot
// using the mappings defined in vcsPaths.
func repoRootFromVCSPaths(importPath string, security web.SecurityMode, vcsPaths []*vcsPath) (*RepoRoot, error) {
if str.HasPathPrefix(importPath, "example.net") {
// TODO(rsc): This should not be necessary, but it's required to keep
// tests like ../../testdata/script/mod_get_extra.txt from using the network.
// That script has everything it needs in the replacement set, but it is still
// doing network calls.
return nil, fmt.Errorf("no modules on example.net")
}
if importPath == "rsc.io" {
// This special case allows tests like ../../testdata/script/govcs.txt
// to avoid making any network calls. The module lookup for a path
// like rsc.io/nonexist.svn/foo needs to not make a network call for
// a lookup on rsc.io.
return nil, fmt.Errorf("rsc.io is not a module")
}
// A common error is to use https://packagepath because that's what
// hg and git require. Diagnose this helpfully.
if prefix := httpPrefix(importPath); prefix != "" {
@ -834,20 +849,20 @@ func repoRootFromVCSPaths(importPath string, security web.SecurityMode, vcsPaths
return nil, fmt.Errorf("%q not allowed in import path", prefix+"//")
}
for _, srv := range vcsPaths {
if !strings.HasPrefix(importPath, srv.prefix) {
if !str.HasPathPrefix(importPath, srv.pathPrefix) {
continue
}
m := srv.regexp.FindStringSubmatch(importPath)
if m == nil {
if srv.prefix != "" {
return nil, importErrorf(importPath, "invalid %s import path %q", srv.prefix, importPath)
if srv.pathPrefix != "" {
return nil, importErrorf(importPath, "invalid %s import path %q", srv.pathPrefix, importPath)
}
continue
}
// Build map of named subexpression matches for expand.
match := map[string]string{
"prefix": srv.prefix,
"prefix": srv.pathPrefix + "/",
"import": importPath,
}
for i, name := range srv.regexp.SubexpNames() {
@ -1098,18 +1113,6 @@ type metaImport struct {
Prefix, VCS, RepoRoot string
}
// pathPrefix reports whether sub is a prefix of s,
// only considering entire path components.
func pathPrefix(s, sub string) bool {
// strings.HasPrefix is necessary but not sufficient.
if !strings.HasPrefix(s, sub) {
return false
}
// The remainder after the prefix must either be empty or start with a slash.
rem := s[len(sub):]
return rem == "" || rem[0] == '/'
}
// A ImportMismatchError is returned where metaImport/s are present
// but none match our import path.
type ImportMismatchError struct {
@ -1133,7 +1136,7 @@ func matchGoImport(imports []metaImport, importPath string) (metaImport, error)
errImportMismatch := ImportMismatchError{importPath: importPath}
for i, im := range imports {
if !pathPrefix(importPath, im.Prefix) {
if !str.HasPathPrefix(importPath, im.Prefix) {
errImportMismatch.mismatches = append(errImportMismatch.mismatches, im.Prefix)
continue
}
@ -1175,52 +1178,52 @@ func expand(match map[string]string, s string) string {
var vcsPaths = []*vcsPath{
// Github
{
prefix: "github.com/",
regexp: lazyregexp.New(`^(?P<root>github\.com/[A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+)(/[A-Za-z0-9_.\-]+)*$`),
vcs: "git",
repo: "https://{root}",
check: noVCSSuffix,
pathPrefix: "github.com",
regexp: lazyregexp.New(`^(?P<root>github\.com/[A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+)(/[A-Za-z0-9_.\-]+)*$`),
vcs: "git",
repo: "https://{root}",
check: noVCSSuffix,
},
// Bitbucket
{
prefix: "bitbucket.org/",
regexp: lazyregexp.New(`^(?P<root>bitbucket\.org/(?P<bitname>[A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+))(/[A-Za-z0-9_.\-]+)*$`),
repo: "https://{root}",
check: bitbucketVCS,
pathPrefix: "bitbucket.org",
regexp: lazyregexp.New(`^(?P<root>bitbucket\.org/(?P<bitname>[A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+))(/[A-Za-z0-9_.\-]+)*$`),
repo: "https://{root}",
check: bitbucketVCS,
},
// IBM DevOps Services (JazzHub)
{
prefix: "hub.jazz.net/git/",
regexp: lazyregexp.New(`^(?P<root>hub\.jazz\.net/git/[a-z0-9]+/[A-Za-z0-9_.\-]+)(/[A-Za-z0-9_.\-]+)*$`),
vcs: "git",
repo: "https://{root}",
check: noVCSSuffix,
pathPrefix: "hub.jazz.net/git",
regexp: lazyregexp.New(`^(?P<root>hub\.jazz\.net/git/[a-z0-9]+/[A-Za-z0-9_.\-]+)(/[A-Za-z0-9_.\-]+)*$`),
vcs: "git",
repo: "https://{root}",
check: noVCSSuffix,
},
// Git at Apache
{
prefix: "git.apache.org/",
regexp: lazyregexp.New(`^(?P<root>git\.apache\.org/[a-z0-9_.\-]+\.git)(/[A-Za-z0-9_.\-]+)*$`),
vcs: "git",
repo: "https://{root}",
pathPrefix: "git.apache.org",
regexp: lazyregexp.New(`^(?P<root>git\.apache\.org/[a-z0-9_.\-]+\.git)(/[A-Za-z0-9_.\-]+)*$`),
vcs: "git",
repo: "https://{root}",
},
// Git at OpenStack
{
prefix: "git.openstack.org/",
regexp: lazyregexp.New(`^(?P<root>git\.openstack\.org/[A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+)(\.git)?(/[A-Za-z0-9_.\-]+)*$`),
vcs: "git",
repo: "https://{root}",
pathPrefix: "git.openstack.org",
regexp: lazyregexp.New(`^(?P<root>git\.openstack\.org/[A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+)(\.git)?(/[A-Za-z0-9_.\-]+)*$`),
vcs: "git",
repo: "https://{root}",
},
// chiselapp.com for fossil
{
prefix: "chiselapp.com/",
regexp: lazyregexp.New(`^(?P<root>chiselapp\.com/user/[A-Za-z0-9]+/repository/[A-Za-z0-9_.\-]+)$`),
vcs: "fossil",
repo: "https://{root}",
pathPrefix: "chiselapp.com",
regexp: lazyregexp.New(`^(?P<root>chiselapp\.com/user/[A-Za-z0-9]+/repository/[A-Za-z0-9_.\-]+)$`),
vcs: "fossil",
repo: "https://{root}",
},
// General syntax for any server.
@ -1238,11 +1241,11 @@ var vcsPaths = []*vcsPath{
var vcsPathsAfterDynamic = []*vcsPath{
// Launchpad. See golang.org/issue/11436.
{
prefix: "launchpad.net/",
regexp: lazyregexp.New(`^(?P<root>launchpad\.net/((?P<project>[A-Za-z0-9_.\-]+)(?P<series>/[A-Za-z0-9_.\-]+)?|~[A-Za-z0-9_.\-]+/(\+junk|[A-Za-z0-9_.\-]+)/[A-Za-z0-9_.\-]+))(/[A-Za-z0-9_.\-]+)*$`),
vcs: "bzr",
repo: "https://{root}",
check: launchpadVCS,
pathPrefix: "launchpad.net",
regexp: lazyregexp.New(`^(?P<root>launchpad\.net/((?P<project>[A-Za-z0-9_.\-]+)(?P<series>/[A-Za-z0-9_.\-]+)?|~[A-Za-z0-9_.\-]+/(\+junk|[A-Za-z0-9_.\-]+)/[A-Za-z0-9_.\-]+))(/[A-Za-z0-9_.\-]+)*$`),
vcs: "bzr",
repo: "https://{root}",
check: launchpadVCS,
},
}

7
src/cmd/go/internal/web/http.go
View File

@ -80,6 +80,13 @@ func get(security SecurityMode, url *urlpkg.URL) (*Response, error) {
return res, nil
}
if url.Host == "localhost.localdev" {
return nil, fmt.Errorf("no such host localhost.localdev")
}
if os.Getenv("TESTGONETWORK") == "panic" && !strings.HasPrefix(url.Host, "127.0.0.1") && !strings.HasPrefix(url.Host, "0.0.0.0") {
panic("use of network: " + url.String())
}
fetch := func(url *urlpkg.URL) (*urlpkg.URL, *http.Response, error) {
// Note: The -v build flag does not mean "print logging information",
// despite its historical misuse for this in GOPATH-based go get.

3
src/cmd/go/script_test.go
View File

@ -142,6 +142,9 @@ func (ts *testScript) setup() {
"goversion=" + goVersion(ts),
":=" + string(os.PathListSeparator),
}
if !testenv.HasExternalNetwork() {
ts.env = append(ts.env, "TESTGONETWORK=panic", "TESTGOVCS=panic")
}
if runtime.GOOS == "plan9" {
ts.env = append(ts.env, "path="+testBin+string(filepath.ListSeparator)+os.Getenv("path"))

6
src/cmd/go/testdata/script/mod_sumdb_proxy.txt vendored
View File

@ -17,14 +17,12 @@ rm $GOPATH/pkg/mod/cache/download/sumdb
rm go.sum
# direct access fails (because localhost.localdev does not exist)
# The text of the error message is hard to predict because some DNS servers
# will resolve unknown domains like localhost.localdev to a real IP
# to serve ads.
# web.get is providing the error message - there's no actual network access.
cp go.mod.orig go.mod
env GOSUMDB=$sumdb
env GOPROXY=direct
! go get -d rsc.io/fortune@v1.0.0
stderr 'verifying.*localhost.localdev'
stderr 'verifying module: rsc.io/fortune@v1.0.0: .*: no such host localhost.localdev'
rm $GOPATH/pkg/mod/cache/download/sumdb
rm go.sum