// Copyright 2009 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package main import ( "bytes" "flag" "fmt" "go/ast" "go/doc" "go/parser" "go/printer" "go/token" "http" "io" "io/ioutil" "log" "os" pathutil "path" "regexp" "runtime" "sort" "strings" "template" "time" ) // ---------------------------------------------------------------------------- // Globals type delayTime struct { RWValue } func (dt *delayTime) backoff(max int) { dt.mutex.Lock() v := dt.value.(int) * 2 if v > max { v = max } dt.value = v // don't change dt.timestamp - calling backoff indicates an error condition dt.mutex.Unlock() } var ( verbose = flag.Bool("v", false, "verbose mode") // file system roots // TODO(gri) consider the invariant that goroot always end in '/' goroot = flag.String("goroot", runtime.GOROOT(), "Go root directory") testDir = flag.String("testdir", "", "Go root subdirectory - for testing only (faster startups)") pkgPath = flag.String("path", "", "additional package directories (colon-separated)") filter = flag.String("filter", "", "filter file containing permitted package directory paths") filterMin = flag.Int("filter_minutes", 0, "filter file update interval in minutes; disabled if <= 0") filterDelay delayTime // actual filter update interval in minutes; usually filterDelay == filterMin, but filterDelay may back off exponentially // layout control tabwidth = flag.Int("tabwidth", 4, "tab width") showTimestamps = flag.Bool("timestamps", true, "show timestamps with directory listings") maxResults = flag.Int("maxresults", 10000, "maximum number of full text search results shown") // file system mapping fsMap Mapping // user-defined mapping fsTree RWValue // *Directory tree of packages, updated with each sync pathFilter RWValue // filter used when building fsMap directory trees fsModified RWValue // timestamp of last call to invalidateIndex // http handlers fileServer http.Handler // default file server cmdHandler httpHandler pkgHandler httpHandler ) func initHandlers() { fsMap.Init(*pkgPath) fileServer = http.FileServer(*goroot, "") cmdHandler = httpHandler{"/cmd/", pathutil.Join(*goroot, "src/cmd"), false} pkgHandler = httpHandler{"/pkg/", pathutil.Join(*goroot, "src/pkg"), true} } func registerPublicHandlers(mux *http.ServeMux) { mux.Handle(cmdHandler.pattern, &cmdHandler) mux.Handle(pkgHandler.pattern, &pkgHandler) mux.HandleFunc("/doc/codewalk/", codewalk) mux.HandleFunc("/search", search) mux.HandleFunc("/", serveFile) } func initFSTree() { fsTree.set(newDirectory(pathutil.Join(*goroot, *testDir), nil, -1)) invalidateIndex() } // ---------------------------------------------------------------------------- // Directory filters // isParentOf returns true if p is a parent of (or the same as) q // where p and q are directory paths. func isParentOf(p, q string) bool { n := len(p) return strings.HasPrefix(q, p) && (len(q) <= n || q[n] == '/') } func setPathFilter(list []string) { if len(list) == 0 { pathFilter.set(nil) return } // len(list) > 0 pathFilter.set(func(path string) bool { // list is sorted in increasing order and for each path all its children are removed i := sort.Search(len(list), func(i int) bool { return list[i] > path }) // Now we have list[i-1] <= path < list[i]. // Path may be a child of list[i-1] or a parent of list[i]. return i > 0 && isParentOf(list[i-1], path) || i < len(list) && isParentOf(path, list[i]) }) } func getPathFilter() func(string) bool { f, _ := pathFilter.get() if f != nil { return f.(func(string) bool) } return nil } // readDirList reads a file containing a newline-separated list // of directory paths and returns the list of paths. func readDirList(filename string) ([]string, os.Error) { contents, err := ioutil.ReadFile(filename) if err != nil { return nil, err } // create a sorted list of valid directory names filter := func(path string) bool { d, err := os.Lstat(path) return err == nil && isPkgDir(d) } list := canonicalizePaths(strings.Split(string(contents), "\n", -1), filter) // for each parent path, remove all it's children q // (requirement for binary search to work when filtering) i := 0 for _, q := range list { if i == 0 || !isParentOf(list[i-1], q) { list[i] = q i++ } } return list[0:i], nil } // updateMappedDirs computes the directory tree for // each user-defined file system mapping. If a filter // is provided, it is used to filter directories. // func updateMappedDirs(filter func(string) bool) { if !fsMap.IsEmpty() { fsMap.Iterate(func(path string, value *RWValue) bool { value.set(newDirectory(path, filter, -1)) return true }) invalidateIndex() } } func updateFilterFile() { updateMappedDirs(nil) // no filter for accuracy // collect directory tree leaf node paths var buf bytes.Buffer fsMap.Iterate(func(_ string, value *RWValue) bool { v, _ := value.get() if v != nil && v.(*Directory) != nil { v.(*Directory).writeLeafs(&buf) } return true }) // update filter file if err := writeFileAtomically(*filter, buf.Bytes()); err != nil { log.Printf("writeFileAtomically(%s): %s", *filter, err) filterDelay.backoff(24 * 60) // back off exponentially, but try at least once a day } else { filterDelay.set(*filterMin) // revert to regular filter update schedule } } func initDirTrees() { // setup initial path filter if *filter != "" { list, err := readDirList(*filter) if err != nil { log.Printf("%s", err) } else if len(list) == 0 { log.Printf("no directory paths in file %s", *filter) } setPathFilter(list) } go updateMappedDirs(getPathFilter()) // use filter for speed // start filter update goroutine, if enabled. if *filter != "" && *filterMin > 0 { filterDelay.set(*filterMin) // initial filter update delay go func() { for { if *verbose { log.Printf("start update of %s", *filter) } updateFilterFile() delay, _ := filterDelay.get() if *verbose { log.Printf("next filter update in %dmin", delay.(int)) } time.Sleep(int64(delay.(int)) * 60e9) } }() } } // ---------------------------------------------------------------------------- // Path mapping func absolutePath(path, defaultRoot string) string { abspath := fsMap.ToAbsolute(path) if abspath == "" { // no user-defined mapping found; use default mapping abspath = pathutil.Join(defaultRoot, path) } return abspath } func relativePath(path string) string { relpath := fsMap.ToRelative(path) if relpath == "" { // prefix must end in '/' prefix := *goroot if len(prefix) > 0 && prefix[len(prefix)-1] != '/' { prefix += "/" } if strings.HasPrefix(path, prefix) { // no user-defined mapping found; use default mapping relpath = path[len(prefix):] } } // Only if path is an invalid absolute path is relpath == "" // at this point. This should never happen since absolute paths // are only created via godoc for files that do exist. However, // it is ok to return ""; it will simply provide a link to the // top of the pkg or src directories. return relpath } // ---------------------------------------------------------------------------- // Tab conversion var spaces = []byte(" ") // 16 spaces seems like a good number const ( indenting = iota collecting ) // A tconv is an io.Writer filter for converting leading tabs into spaces. type tconv struct { output io.Writer state int // indenting or collecting indent int // valid if state == indenting } func (p *tconv) writeIndent() (err os.Error) { i := p.indent for i > len(spaces) { i -= len(spaces) if _, err = p.output.Write(spaces); err != nil { return } } _, err = p.output.Write(spaces[0:i]) return } func (p *tconv) Write(data []byte) (n int, err os.Error) { pos := 0 // valid if p.state == collecting var b byte for n, b = range data { switch p.state { case indenting: switch b { case '\t', '\v': p.indent += *tabwidth case '\n': p.indent = 0 if _, err = p.output.Write(data[n : n+1]); err != nil { return } case ' ': p.indent++ default: p.state = collecting pos = n if err = p.writeIndent(); err != nil { return } } case collecting: if b == '\n' { p.state = indenting p.indent = 0 if _, err = p.output.Write(data[pos : n+1]); err != nil { return } } } } n = len(data) if p.state == collecting { _, err = p.output.Write(data[pos:]) } return } // ---------------------------------------------------------------------------- // Templates // Write an AST-node to w; optionally html-escaped. func writeNode(w io.Writer, fset *token.FileSet, node interface{}, html bool) { mode := printer.TabIndent | printer.UseSpaces if html { mode |= printer.GenHTML } // convert trailing tabs into spaces using a tconv filter // to ensure a good outcome in most browsers (there may still // be tabs in comments and strings, but converting those into // the right number of spaces is much harder) (&printer.Config{mode, *tabwidth, nil}).Fprint(&tconv{output: w}, fset, node) } // Write text to w; optionally html-escaped. func writeText(w io.Writer, text []byte, html bool) { if html { template.HTMLEscape(w, text) return } w.Write(text) } // Write anything to w; optionally html-escaped. func writeAny(w io.Writer, fset *token.FileSet, html bool, x interface{}) { switch v := x.(type) { case []byte: writeText(w, v, html) case string: writeText(w, []byte(v), html) case ast.Decl, ast.Expr, ast.Stmt, *ast.File: writeNode(w, fset, x, html) default: if html { var buf bytes.Buffer fmt.Fprint(&buf, x) writeText(w, buf.Bytes(), true) } else { fmt.Fprint(w, x) } } } func fileset(x []interface{}) *token.FileSet { if len(x) > 1 { if fset, ok := x[1].(*token.FileSet); ok { return fset } } return nil } // Template formatter for "html" format. func htmlFmt(w io.Writer, format string, x ...interface{}) { writeAny(w, fileset(x), true, x[0]) } // Template formatter for "html-esc" format. func htmlEscFmt(w io.Writer, format string, x ...interface{}) { var buf bytes.Buffer writeAny(&buf, fileset(x), false, x[0]) template.HTMLEscape(w, buf.Bytes()) } // Template formatter for "html-comment" format. func htmlCommentFmt(w io.Writer, format string, x ...interface{}) { var buf bytes.Buffer writeAny(&buf, fileset(x), false, x[0]) // TODO(gri) Provide list of words (e.g. function parameters) // to be emphasized by ToHTML. doc.ToHTML(w, buf.Bytes(), nil) // does html-escaping } // Template formatter for "" (default) format. func textFmt(w io.Writer, format string, x ...interface{}) { writeAny(w, fileset(x), false, x[0]) } // Template formatter for "urlquery-esc" format. func urlQueryEscFmt(w io.Writer, format string, x ...interface{}) { var buf bytes.Buffer writeAny(&buf, fileset(x), false, x[0]) template.HTMLEscape(w, []byte(http.URLEscape(string(buf.Bytes())))) } // Template formatter for the various "url-xxx" formats excluding url-esc. func urlFmt(w io.Writer, format string, x ...interface{}) { var path string var line int var low, high int // selection // determine path and position info, if any type positioner interface { Pos() token.Pos End() token.Pos } switch t := x[0].(type) { case string: path = t case positioner: fset := fileset(x) if p := t.Pos(); p.IsValid() { pos := fset.Position(p) path = pos.Filename line = pos.Line low = pos.Offset } if p := t.End(); p.IsValid() { high = fset.Position(p).Offset } default: // we should never reach here, but be resilient // and assume the position is invalid (empty path, // and line 0) log.Printf("INTERNAL ERROR: urlFmt(%s) without a string or positioner", format) } // map path relpath := relativePath(path) // convert to relative URLs so that they can also // be used as relative file names in .txt templates switch format { default: // we should never reach here, but be resilient // and assume the url-pkg format instead log.Printf("INTERNAL ERROR: urlFmt(%s)", format) fallthrough case "url-pkg": // because of the irregular mapping under goroot // we need to correct certain relative paths if strings.HasPrefix(relpath, "src/pkg/") { relpath = relpath[len("src/pkg/"):] } template.HTMLEscape(w, []byte(pkgHandler.pattern[1:]+relpath)) // remove trailing '/' for relative URL case "url-src": template.HTMLEscape(w, []byte(relpath)) case "url-pos": template.HTMLEscape(w, []byte(relpath)) // selection ranges are of form "s=low:high" if low < high { fmt.Fprintf(w, "?s=%d:%d", low, high) // if we have a selection, position the page // such that the selection is a bit below the top line -= 10 if line < 1 { line = 1 } } // line id's in html-printed source are of the // form "L%d" where %d stands for the line number if line > 0 { fmt.Fprintf(w, "#L%d", line) } } } // The strings in infoKinds must be properly html-escaped. var infoKinds = [nKinds]string{ PackageClause: "package clause", ImportDecl: "import decl", ConstDecl: "const decl", TypeDecl: "type decl", VarDecl: "var decl", FuncDecl: "func decl", MethodDecl: "method decl", Use: "use", } // Template formatter for "infoKind" format. func infoKindFmt(w io.Writer, format string, x ...interface{}) { fmt.Fprintf(w, infoKinds[x[0].(SpotKind)]) // infoKind entries are html-escaped } // Template formatter for "infoLine" format. func infoLineFmt(w io.Writer, format string, x ...interface{}) { info := x[0].(SpotInfo) line := info.Lori() if info.IsIndex() { index, _ := searchIndex.get() if index != nil { line = index.(*Index).Snippet(line).Line } else { // no line information available because // we don't have an index - this should // never happen; be conservative and don't // crash line = 0 } } fmt.Fprintf(w, "%d", line) } // Template formatter for "infoSnippet" format. func infoSnippetFmt(w io.Writer, format string, x ...interface{}) { info := x[0].(SpotInfo) text := []byte(`no snippet text available`) if info.IsIndex() { index, _ := searchIndex.get() // no escaping of snippet text needed; // snippet text is escaped when generated text = index.(*Index).Snippet(info.Lori()).Text } w.Write(text) } // Template formatter for "padding" format. func paddingFmt(w io.Writer, format string, x ...interface{}) { for i := x[0].(int); i > 0; i-- { fmt.Fprint(w, `