mirror of
https://github.com/golang/go
synced 2024-11-21 21:44:40 -07:00
godoc: added systematic throttling to indexing goroutine
- implemented stand-alone Throttle mechanism - added new flag -index_throttle to godoc - index throttling enables index creation when running godoc on app engine as it keeps godoc responsive R=rsc, dsymonds, adg CC=golang-dev https://golang.org/cl/4963043
This commit is contained in:
parent
77db5ff501
commit
3a1f29beec
@ -18,6 +18,7 @@ GOFILES=\
|
||||
parser.go\
|
||||
snippet.go\
|
||||
spec.go\
|
||||
throttle.go\
|
||||
utils.go\
|
||||
zip.go\
|
||||
|
||||
|
@ -50,6 +50,11 @@ The flags are:
|
||||
-index
|
||||
enable identifier and full text search index
|
||||
(no search box is shown if -index is not set)
|
||||
-index_throttle=0.75
|
||||
index throttle value; a value of 0 means no time is allocated
|
||||
to the indexer (the indexer will never finish), a value of 1.0
|
||||
means that index creation is running at full throttle (other
|
||||
goroutines may get no time while the index is built)
|
||||
-maxresults=10000
|
||||
maximum number of full text search results shown
|
||||
(no full text index is built if maxresults <= 0)
|
||||
|
@ -63,8 +63,9 @@ var (
|
||||
templateDir = flag.String("templates", "", "directory containing alternate template files")
|
||||
|
||||
// search index
|
||||
indexEnabled = flag.Bool("index", false, "enable search index")
|
||||
maxResults = flag.Int("maxresults", 10000, "maximum number of full text search results shown")
|
||||
indexEnabled = flag.Bool("index", false, "enable search index")
|
||||
maxResults = flag.Int("maxresults", 10000, "maximum number of full text search results shown")
|
||||
indexThrottle = flag.Float64("index_throttle", 0.75, "index throttle value; 0.0 = no time allocated, 1.0 = full throttle")
|
||||
|
||||
// file system mapping
|
||||
fs FileSystem // the underlying file system for godoc
|
||||
@ -1148,7 +1149,7 @@ func indexer() {
|
||||
log.Printf("updating index...")
|
||||
}
|
||||
start := time.Nanoseconds()
|
||||
index := NewIndex(fsDirnames(), *maxResults > 0)
|
||||
index := NewIndex(fsDirnames(), *maxResults > 0, *indexThrottle)
|
||||
stop := time.Nanoseconds()
|
||||
searchIndex.set(index)
|
||||
if *verbose {
|
||||
|
@ -736,8 +736,9 @@ func canonical(w string) string { return strings.ToLower(w) }
|
||||
// NewIndex creates a new index for the .go files
|
||||
// in the directories given by dirnames.
|
||||
//
|
||||
func NewIndex(dirnames <-chan string, fulltextIndex bool) *Index {
|
||||
func NewIndex(dirnames <-chan string, fulltextIndex bool, throttle float64) *Index {
|
||||
var x Indexer
|
||||
th := NewThrottle(throttle, 0.1e9) // run at least 0.1s at a time
|
||||
|
||||
// initialize Indexer
|
||||
x.fset = token.NewFileSet()
|
||||
@ -753,6 +754,7 @@ func NewIndex(dirnames <-chan string, fulltextIndex bool) *Index {
|
||||
if !f.IsDirectory() {
|
||||
x.visitFile(dirname, f, fulltextIndex)
|
||||
}
|
||||
th.Throttle()
|
||||
}
|
||||
}
|
||||
|
||||
@ -778,6 +780,7 @@ func NewIndex(dirnames <-chan string, fulltextIndex bool) *Index {
|
||||
Others: others,
|
||||
}
|
||||
wlist = append(wlist, &wordPair{canonical(w), w})
|
||||
th.Throttle()
|
||||
}
|
||||
x.stats.Words = len(words)
|
||||
|
||||
|
88
src/cmd/godoc/throttle.go
Normal file
88
src/cmd/godoc/throttle.go
Normal file
@ -0,0 +1,88 @@
|
||||
// Copyright 2011 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package main
|
||||
|
||||
import "time"
|
||||
|
||||
// A Throttle permits throttling of a goroutine by
|
||||
// calling the Throttle method repeatedly.
|
||||
//
|
||||
type Throttle struct {
|
||||
f float64 // f = (1-r)/r for 0 < r < 1
|
||||
tm int64 // minimum run time slice; >= 0
|
||||
tr int64 // accumulated time running
|
||||
ts int64 // accumulated time stopped
|
||||
tt int64 // earliest throttle time (= time Throttle returned + tm)
|
||||
}
|
||||
|
||||
// NewThrottle creates a new Throttle with a throttle value r and
|
||||
// a minimum allocated run time slice of tm nanoseconds:
|
||||
//
|
||||
// r == 0: "empty" throttle; the goroutine is always sleeping
|
||||
// r == 1: full throttle; the goroutine is never sleeping
|
||||
//
|
||||
// A value of r == 0.6 throttles a goroutine such that it runs
|
||||
// approx. 60% of the time, and sleeps approx. 40% of the time.
|
||||
// Values of r < 0 or r > 1 are clamped down to values between 0 and 1.
|
||||
// Values of tm < 0 are set to 0.
|
||||
//
|
||||
func NewThrottle(r float64, tm int64) *Throttle {
|
||||
var f float64
|
||||
switch {
|
||||
case r <= 0:
|
||||
f = -1 // indicates always sleep
|
||||
case r >= 1:
|
||||
f = 0 // assume r == 1 (never sleep)
|
||||
default:
|
||||
// 0 < r < 1
|
||||
f = (1 - r) / r
|
||||
}
|
||||
if tm < 0 {
|
||||
tm = 0
|
||||
}
|
||||
return &Throttle{f: f, tm: tm, tt: time.Nanoseconds() + tm}
|
||||
}
|
||||
|
||||
// Throttle calls time.Sleep such that over time the ratio tr/ts between
|
||||
// accumulated run (tr) and sleep times (ts) approximates the value 1/(1-r)
|
||||
// where r is the throttle value. Throttle returns immediately (w/o sleeping)
|
||||
// if less than tm ns have passed since the last call to Throttle.
|
||||
//
|
||||
func (p *Throttle) Throttle() {
|
||||
if p.f < 0 {
|
||||
select {} // always sleep
|
||||
}
|
||||
|
||||
t0 := time.Nanoseconds()
|
||||
if t0 < p.tt {
|
||||
return // keep running (minimum time slice not exhausted yet)
|
||||
}
|
||||
|
||||
// accumulate running time
|
||||
p.tr += t0 - (p.tt - p.tm)
|
||||
|
||||
// compute sleep time
|
||||
// Over time we want:
|
||||
//
|
||||
// tr/ts = r/(1-r)
|
||||
//
|
||||
// Thus:
|
||||
//
|
||||
// ts = tr*f with f = (1-r)/r
|
||||
//
|
||||
// After some incremental run time δr added to the total run time
|
||||
// tr, the incremental sleep-time δs to get to the same ratio again
|
||||
// after waking up from time.Sleep is:
|
||||
if δs := int64(float64(p.tr)*p.f) - p.ts; δs > 0 {
|
||||
time.Sleep(δs)
|
||||
}
|
||||
|
||||
// accumulate (actual) sleep time
|
||||
t1 := time.Nanoseconds()
|
||||
p.ts += t1 - t0
|
||||
|
||||
// set earliest next throttle time
|
||||
p.tt = t1 + p.tm
|
||||
}
|
Loading…
Reference in New Issue
Block a user