1
0
mirror of https://github.com/golang/go synced 2024-11-19 05:24:42 -07:00
go/godoc/corpus.go
Brad Fitzpatrick 964f0f559c godoc: feed indexer concurrently, add selective indexing hook, tests
On big corpuses, the indexer was spending most of its time waiting
for filesystem operations (especially with network filesystems)
and not actually indexing.  This keeps the filesystem busy and indexer
running in different goroutines.

Also, add a hook to let godoc hosts disable indexing of certain
directories.

And finally, start adding tests for godoc, which required
fleshing out (and testing) the mapfs code.

R=golang-dev, adg, bgarcia
CC=golang-dev
https://golang.org/cl/21520045
2013-11-05 09:35:58 -05:00

127 lines
3.7 KiB
Go

// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package godoc
import (
"errors"
pathpkg "path"
"time"
"code.google.com/p/go.tools/godoc/util"
"code.google.com/p/go.tools/godoc/vfs"
)
// A Corpus holds all the state related to serving and indexing a
// collection of Go code.
//
// Construct a new Corpus with NewCorpus, then modify options,
// then call its Init method.
type Corpus struct {
fs vfs.FileSystem
// Verbose logging.
Verbose bool
// IndexEnabled controls whether indexing is enabled.
IndexEnabled bool
// IndexFiles specifies a glob pattern specifying index files.
// If not empty, the index is read from these files in sorted
// order.
IndexFiles string
// IndexThrottle specifies the indexing throttle value
// between 0.0 and 1.0. At 0.0, the indexer always sleeps.
// At 1.0, the indexer never sleeps. Because 0.0 is useless
// and redundant with setting IndexEnabled to false, the
// zero value for IndexThrottle means 0.9.
IndexThrottle float64
// MaxResults optionally specifies the maximum results for indexing.
// The default is 1000.
MaxResults int
// SummarizePackage optionally specifies a function to
// summarize a package. It exists as an optimization to
// avoid reading files to parse package comments.
//
// If SummarizePackage returns false for ok, the caller
// ignores all return values and parses the files in the package
// as if SummarizePackage were nil.
//
// If showList is false, the package is hidden from the
// package listing.
SummarizePackage func(pkg string) (summary string, showList, ok bool)
// IndexDirectory optionally specifies a function to determine
// whether the provided directory should be indexed. The dir
// will be of the form "/src/cmd/6a", "/doc/play",
// "/src/pkg/io", etc.
// If nil, all directories are indexed if indexing is enabled.
IndexDirectory func(dir string) bool
testDir string // TODO(bradfitz,adg): migrate old godoc flag? looks unused.
// Send a value on this channel to trigger a metadata refresh.
// It is buffered so that if a signal is not lost if sent
// during a refresh.
refreshMetadataSignal chan bool
// file system information
fsTree util.RWValue // *Directory tree of packages, updated with each sync (but sync code is removed now)
fsModified util.RWValue // timestamp of last call to invalidateIndex
docMetadata util.RWValue // mapping from paths to *Metadata
// SearchIndex is the search index in use.
searchIndex util.RWValue
}
// NewCorpus returns a new Corpus from a filesystem.
// Set any options on Corpus before calling the Corpus.Init method.
func NewCorpus(fs vfs.FileSystem) *Corpus {
c := &Corpus{
fs: fs,
refreshMetadataSignal: make(chan bool, 1),
MaxResults: 1000,
IndexEnabled: true,
}
return c
}
func (c *Corpus) CurrentIndex() (*Index, time.Time) {
v, t := c.searchIndex.Get()
idx, _ := v.(*Index)
return idx, t
}
func (c *Corpus) FSModifiedTime() time.Time {
_, ts := c.fsModified.Get()
return ts
}
// Init initializes Corpus, once options on Corpus are set.
// It must be called before any subsequent method calls.
func (c *Corpus) Init() error {
// TODO(bradfitz): do this in a goroutine because newDirectory might block for a long time?
// It used to be sometimes done in a goroutine before, at least in HTTP server mode.
if err := c.initFSTree(); err != nil {
return err
}
c.updateMetadata()
go c.refreshMetadataLoop()
return nil
}
func (c *Corpus) initFSTree() error {
dir := c.newDirectory(pathpkg.Join("/", c.testDir), -1)
if dir == nil {
return errors.New("godoc: corpus fstree is nil")
}
c.fsTree.Set(dir)
c.invalidateIndex()
return nil
}