qbit/go
1
0
Fork 0
mirror of https://github.com/golang/go synced 2024-11-18 16:54:43 -07:00
Code Releases Activity

imports: limit local disk concurrency, avoid reads in non-Go directories

Browse Source 
If $GOPATH was large, or $GOPATH was $HOME and $HOME/src had
many files, the unbounded concurrency in loadPkgIndex/loadPkg
could make the operating system unhappy with so many
threads. (sigh once again for no async file IO and needing
threads for file operations)

In addition, don't call go/build.Context.Import on directories
that we've already determined to have no go files in them.
It's just a waste of time. Makes it about 3x faster on my
machine with hot caches and a big $HOME/src.

Fixes golang/go#7731

LGTM=iant, adg
R=golang-codereviews, iant, adg
CC=david.crawshaw, golang-codereviews
https://golang.org/cl/85670044
This commit is contained in:
Brad Fitzpatrick 2014-04-08 16:43:52 -07:00
parent fa0f6bd591
commit 87f95283ac
1 changed files with 33 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
imports/fix.go
View File

@ -174,6 +174,16 @@ var pkgIndex struct {
m map[string][]pkg // shortname => []pkg, e.g "http" => "net/http"
}
// gate is a semaphore for limiting concurrency.
type gate chan bool
func (g gate) enter() { g <- true }
func (g gate) leave() { <-g }
// fsgate protects the OS & filesystem from too much concurrency.
// Too much disk I/O -> too many threads -> swapping and bad scheduling.
var fsgate = make(gate, 8)
func loadPkgIndex() {
pkgIndex.Lock()
pkgIndex.m = make(map[string][]pkg)
@ -181,13 +191,16 @@ func loadPkgIndex() {
var wg sync.WaitGroup
for _, path := range build.Default.SrcDirs() {
fsgate.enter()
f, err := os.Open(path)
if err != nil {
fsgate.leave()
fmt.Fprint(os.Stderr, err)
continue
}
children, err := f.Readdir(-1)
f.Close()
fsgate.leave()
if err != nil {
fmt.Fprint(os.Stderr, err)
continue
@ -207,16 +220,10 @@ func loadPkgIndex() {
func loadPkg(wg *sync.WaitGroup, root, pkgrelpath string) {
importpath := filepath.ToSlash(pkgrelpath)
shortName := importPathToName(importpath)
dir := filepath.Join(root, importpath)
pkgIndex.Lock()
pkgIndex.m[shortName] = append(pkgIndex.m[shortName], pkg{
importpath: importpath,
dir: dir,
})
pkgIndex.Unlock()
fsgate.enter()
defer fsgate.leave()
pkgDir, err := os.Open(dir)
if err != nil {
return
@ -226,6 +233,11 @@ func loadPkg(wg *sync.WaitGroup, root, pkgrelpath string) {
if err != nil {
return
}
// hasGo tracks whether a directory actually appears to be a
// Go source code directory. If $GOPATH == $HOME, and
// $HOME/src has lots of other large non-Go projects in it,
// then the calls to importPathToName below can be expensive.
hasGo := false
for _, child := range children {
name := child.Name()
if name == "" {
@ -234,6 +246,9 @@ func loadPkg(wg *sync.WaitGroup, root, pkgrelpath string) {
if c := name[0]; c == '.' || ('0' <= c && c <= '9') {
continue
}
if strings.HasSuffix(name, ".go") {
hasGo = true
}
if child.IsDir() {
wg.Add(1)
go func(root, name string) {
@ -242,6 +257,16 @@ func loadPkg(wg *sync.WaitGroup, root, pkgrelpath string) {
}(root, filepath.Join(importpath, name))
}
}
if hasGo {
shortName := importPathToName(importpath)
pkgIndex.Lock()
pkgIndex.m[shortName] = append(pkgIndex.m[shortName], pkg{
importpath: importpath,
dir: dir,
})
pkgIndex.Unlock()
}
}
// loadExports returns a list exports for a package.