godoc: actually include files from previous CL

This stuff was deleted from cmd/godoc, and is
moving into pkg godoc.

R=golang-dev, adg
CC=golang-dev
https://golang.org/cl/11425043

godoc/godoc.go

@@ -0,0 +1,482 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package godoc is a work-in-progress (2013-07-17) package to
+// begin splitting up the godoc binary into multiple pieces.
+//
+// This package comment will evolve over time as this package splits
+// into smaller pieces.
+package godoc
+
+import (
+	"bytes"
+	"fmt"
+	"go/ast"
+	"go/doc"
+	"go/format"
+	"go/printer"
+	"go/token"
+	"io"
+	"log"
+	"os"
+	pathpkg "path"
+	"regexp"
+	"strings"
+	"text/template"
+	"time"
+	"unicode"
+	"unicode/utf8"
+
+	"code.google.com/p/go.tools/godoc/util"
+	"code.google.com/p/go.tools/godoc/vfs"
+)
+
+// FS is the file system that godoc reads from and serves.
+// It is a virtual file system that operates on slash-separated paths,
+// and its root corresponds to the Go distribution root: /src/pkg
+// holds the source tree, and so on.  This means that the URLs served by
+// the godoc server are the same as the paths in the virtual file
+// system, which helps keep things simple.
+var FS = vfs.NameSpace{}
+
+// Old flags
+var (
+	// DeclLinks controls whether identifers are linked to their declaration.
+	DeclLinks = true
+
+	// ShowExamples controls whether to show examples in command-line mode.
+	// TODO(bradfitz,adg): delete this flag
+	ShowExamples = false
+
+	// ShowPlayground controls whether to enable the playground in
+	// the web interface.
+	// TODO(bradfitz,adg): delete this flag
+	ShowPlayground = false
+
+	IndexEnabled = false
+
+	ShowTimestamps = false
+
+	Verbose = false
+
+	TabWidth = 4
+
+	// regular expression matching note markers to show
+	NotesRx = "BUG"
+)
+
+// SearchIndex is the search index in use.
+var SearchIndex util.RWValue
+
+// Fake relative package path for built-ins. Documentation for all globals
+// (not just exported ones) will be shown for packages in this directory.
+const BuiltinPkgPath = "builtin"
+
+// FuncMap defines template functions used in godoc templates.
+//
+// Convention: template function names ending in "_html" or "_url" produce
+//             HTML- or URL-escaped strings; all other function results may
+//             require explicit escaping in the template.
+var FuncMap = template.FuncMap{
+	// various helpers
+	"filename": filenameFunc,
+	"repeat":   strings.Repeat,
+
+	// access to FileInfos (directory listings)
+	"fileInfoName": fileInfoNameFunc,
+	"fileInfoTime": fileInfoTimeFunc,
+
+	// access to search result information
+	"infoKind_html":    infoKind_htmlFunc,
+	"infoLine":         infoLineFunc,
+	"infoSnippet_html": infoSnippet_htmlFunc,
+
+	// formatting of AST nodes
+	"node":         nodeFunc,
+	"node_html":    node_htmlFunc,
+	"comment_html": comment_htmlFunc,
+	"comment_text": comment_textFunc,
+
+	// support for URL attributes
+	"pkgLink":     pkgLinkFunc,
+	"srcLink":     srcLinkFunc,
+	"posLink_url": posLink_urlFunc,
+
+	// formatting of Examples
+	"example_html":   example_htmlFunc,
+	"example_text":   example_textFunc,
+	"example_name":   example_nameFunc,
+	"example_suffix": example_suffixFunc,
+
+	// formatting of Notes
+	"noteTitle": noteTitle,
+}
+
+func filenameFunc(path string) string {
+	_, localname := pathpkg.Split(path)
+	return localname
+}
+
+func fileInfoNameFunc(fi os.FileInfo) string {
+	name := fi.Name()
+	if fi.IsDir() {
+		name += "/"
+	}
+	return name
+}
+
+func fileInfoTimeFunc(fi os.FileInfo) string {
+	if t := fi.ModTime(); t.Unix() != 0 {
+		return t.Local().String()
+	}
+	return "" // don't return epoch if time is obviously not set
+}
+
+// The strings in infoKinds must be properly html-escaped.
+var infoKinds = [nKinds]string{
+	PackageClause: "package clause",
+	ImportDecl:    "import decl",
+	ConstDecl:     "const decl",
+	TypeDecl:      "type decl",
+	VarDecl:       "var decl",
+	FuncDecl:      "func decl",
+	MethodDecl:    "method decl",
+	Use:           "use",
+}
+
+func infoKind_htmlFunc(info SpotInfo) string {
+	return infoKinds[info.Kind()] // infoKind entries are html-escaped
+}
+
+func infoLineFunc(info SpotInfo) int {
+	line := info.Lori()
+	if info.IsIndex() {
+		index, _ := SearchIndex.Get()
+		if index != nil {
+			line = index.(*Index).Snippet(line).Line
+		} else {
+			// no line information available because
+			// we don't have an index - this should
+			// never happen; be conservative and don't
+			// crash
+			line = 0
+		}
+	}
+	return line
+}
+
+func infoSnippet_htmlFunc(info SpotInfo) string {
+	if info.IsIndex() {
+		index, _ := SearchIndex.Get()
+		// Snippet.Text was HTML-escaped when it was generated
+		return index.(*Index).Snippet(info.Lori()).Text
+	}
+	return `<span class="alert">no snippet text available</span>`
+}
+
+func nodeFunc(info *PageInfo, node interface{}) string {
+	var buf bytes.Buffer
+	writeNode(&buf, info.FSet, node)
+	return buf.String()
+}
+
+func node_htmlFunc(info *PageInfo, node interface{}, linkify bool) string {
+	var buf1 bytes.Buffer
+	writeNode(&buf1, info.FSet, node)
+
+	var buf2 bytes.Buffer
+	if n, _ := node.(ast.Node); n != nil && linkify && DeclLinks {
+		LinkifyText(&buf2, buf1.Bytes(), n)
+	} else {
+		FormatText(&buf2, buf1.Bytes(), -1, true, "", nil)
+	}
+
+	return buf2.String()
+}
+
+func comment_htmlFunc(comment string) string {
+	var buf bytes.Buffer
+	// TODO(gri) Provide list of words (e.g. function parameters)
+	//           to be emphasized by ToHTML.
+	doc.ToHTML(&buf, comment, nil) // does html-escaping
+	return buf.String()
+}
+
+// punchCardWidth is the number of columns of fixed-width
+// characters to assume when wrapping text.  Very few people
+// use terminals or cards smaller than 80 characters, so 80 it is.
+// We do not try to sniff the environment or the tty to adapt to
+// the situation; instead, by using a constant we make sure that
+// godoc always produces the same output regardless of context,
+// a consistency that is lost otherwise.  For example, if we sniffed
+// the environment or tty, then http://golang.org/pkg/math/?m=text
+// would depend on the width of the terminal where godoc started,
+// which is clearly bogus.  More generally, the Unix tools that behave
+// differently when writing to a tty than when writing to a file have
+// a history of causing confusion (compare `ls` and `ls | cat`), and we
+// want to avoid that mistake here.
+const punchCardWidth = 80
+
+func comment_textFunc(comment, indent, preIndent string) string {
+	var buf bytes.Buffer
+	doc.ToText(&buf, comment, indent, preIndent, punchCardWidth-2*len(indent))
+	return buf.String()
+}
+
+type PageInfo struct {
+	Dirname string // directory containing the package
+	Err     error  // error or nil
+
+	// package info
+	FSet     *token.FileSet         // nil if no package documentation
+	PDoc     *doc.Package           // nil if no package documentation
+	Examples []*doc.Example         // nil if no example code
+	Notes    map[string][]*doc.Note // nil if no package Notes
+	PAst     *ast.File              // nil if no AST with package exports
+	IsMain   bool                   // true for package main
+
+	// directory info
+	Dirs    *DirList  // nil if no directory information
+	DirTime time.Time // directory time stamp
+	DirFlat bool      // if set, show directory in a flat (non-indented) manner
+}
+
+func (info *PageInfo) IsEmpty() bool {
+	return info.Err != nil || info.PAst == nil && info.PDoc == nil && info.Dirs == nil
+}
+
+func pkgLinkFunc(path string) string {
+	relpath := path[1:]
+	// because of the irregular mapping under goroot
+	// we need to correct certain relative paths
+	relpath = strings.TrimPrefix(relpath, "src/pkg/")
+	return PkgHandler.pattern[1:] + relpath // remove trailing '/' for relative URL
+}
+
+// n must be an ast.Node or a *doc.Note
+func posLink_urlFunc(info *PageInfo, n interface{}) string {
+	var pos, end token.Pos
+
+	switch n := n.(type) {
+	case ast.Node:
+		pos = n.Pos()
+		end = n.End()
+	case *doc.Note:
+		pos = n.Pos
+		end = n.End
+	default:
+		panic(fmt.Sprintf("wrong type for posLink_url template formatter: %T", n))
+	}
+
+	var relpath string
+	var line int
+	var low, high int // selection offset range
+
+	if pos.IsValid() {
+		p := info.FSet.Position(pos)
+		relpath = p.Filename
+		line = p.Line
+		low = p.Offset
+	}
+	if end.IsValid() {
+		high = info.FSet.Position(end).Offset
+	}
+
+	var buf bytes.Buffer
+	template.HTMLEscape(&buf, []byte(relpath))
+	// selection ranges are of form "s=low:high"
+	if low < high {
+		fmt.Fprintf(&buf, "?s=%d:%d", low, high) // no need for URL escaping
+		// if we have a selection, position the page
+		// such that the selection is a bit below the top
+		line -= 10
+		if line < 1 {
+			line = 1
+		}
+	}
+	// line id's in html-printed source are of the
+	// form "L%d" where %d stands for the line number
+	if line > 0 {
+		fmt.Fprintf(&buf, "#L%d", line) // no need for URL escaping
+	}
+
+	return buf.String()
+}
+
+func srcLinkFunc(s string) string {
+	return pathpkg.Clean("/" + s)
+}
+
+func example_textFunc(info *PageInfo, funcName, indent string) string {
+	if !ShowExamples {
+		return ""
+	}
+
+	var buf bytes.Buffer
+	first := true
+	for _, eg := range info.Examples {
+		name := stripExampleSuffix(eg.Name)
+		if name != funcName {
+			continue
+		}
+
+		if !first {
+			buf.WriteString("\n")
+		}
+		first = false
+
+		// print code
+		cnode := &printer.CommentedNode{Node: eg.Code, Comments: eg.Comments}
+		var buf1 bytes.Buffer
+		writeNode(&buf1, info.FSet, cnode)
+		code := buf1.String()
+		// Additional formatting if this is a function body.
+		if n := len(code); n >= 2 && code[0] == '{' && code[n-1] == '}' {
+			// remove surrounding braces
+			code = code[1 : n-1]
+			// unindent
+			code = strings.Replace(code, "\n    ", "\n", -1)
+		}
+		code = strings.Trim(code, "\n")
+		code = strings.Replace(code, "\n", "\n\t", -1)
+
+		buf.WriteString(indent)
+		buf.WriteString("Example:\n\t")
+		buf.WriteString(code)
+		buf.WriteString("\n")
+	}
+	return buf.String()
+}
+
+func example_htmlFunc(info *PageInfo, funcName string) string {
+	var buf bytes.Buffer
+	for _, eg := range info.Examples {
+		name := stripExampleSuffix(eg.Name)
+
+		if name != funcName {
+			continue
+		}
+
+		// print code
+		cnode := &printer.CommentedNode{Node: eg.Code, Comments: eg.Comments}
+		code := node_htmlFunc(info, cnode, true)
+		out := eg.Output
+		wholeFile := true
+
+		// Additional formatting if this is a function body.
+		if n := len(code); n >= 2 && code[0] == '{' && code[n-1] == '}' {
+			wholeFile = false
+			// remove surrounding braces
+			code = code[1 : n-1]
+			// unindent
+			code = strings.Replace(code, "\n    ", "\n", -1)
+			// remove output comment
+			if loc := exampleOutputRx.FindStringIndex(code); loc != nil {
+				code = strings.TrimSpace(code[:loc[0]])
+			}
+		}
+
+		// Write out the playground code in standard Go style
+		// (use tabs, no comment highlight, etc).
+		play := ""
+		if eg.Play != nil && ShowPlayground {
+			var buf bytes.Buffer
+			if err := format.Node(&buf, info.FSet, eg.Play); err != nil {
+				log.Print(err)
+			} else {
+				play = buf.String()
+			}
+		}
+
+		// Drop output, as the output comment will appear in the code.
+		if wholeFile && play == "" {
+			out = ""
+		}
+
+		if ExampleHTML == nil {
+			out = ""
+			return ""
+		}
+
+		err := ExampleHTML.Execute(&buf, struct {
+			Name, Doc, Code, Play, Output string
+		}{eg.Name, eg.Doc, code, play, out})
+		if err != nil {
+			log.Print(err)
+		}
+	}
+	return buf.String()
+}
+
+// example_nameFunc takes an example function name and returns its display
+// name. For example, "Foo_Bar_quux" becomes "Foo.Bar (Quux)".
+func example_nameFunc(s string) string {
+	name, suffix := splitExampleName(s)
+	// replace _ with . for method names
+	name = strings.Replace(name, "_", ".", 1)
+	// use "Package" if no name provided
+	if name == "" {
+		name = "Package"
+	}
+	return name + suffix
+}
+
+// example_suffixFunc takes an example function name and returns its suffix in
+// parenthesized form. For example, "Foo_Bar_quux" becomes " (Quux)".
+func example_suffixFunc(name string) string {
+	_, suffix := splitExampleName(name)
+	return suffix
+}
+
+func noteTitle(note string) string {
+	return strings.Title(strings.ToLower(note))
+}
+
+func startsWithUppercase(s string) bool {
+	r, _ := utf8.DecodeRuneInString(s)
+	return unicode.IsUpper(r)
+}
+
+var exampleOutputRx = regexp.MustCompile(`(?i)//[[:space:]]*output:`)
+
+// stripExampleSuffix strips lowercase braz in Foo_braz or Foo_Bar_braz from name
+// while keeping uppercase Braz in Foo_Braz.
+func stripExampleSuffix(name string) string {
+	if i := strings.LastIndex(name, "_"); i != -1 {
+		if i < len(name)-1 && !startsWithUppercase(name[i+1:]) {
+			name = name[:i]
+		}
+	}
+	return name
+}
+
+func splitExampleName(s string) (name, suffix string) {
+	i := strings.LastIndex(s, "_")
+	if 0 <= i && i < len(s)-1 && !startsWithUppercase(s[i+1:]) {
+		name = s[:i]
+		suffix = " (" + strings.Title(s[i+1:]) + ")"
+		return
+	}
+	name = s
+	return
+}
+
+// Write an AST node to w.
+func writeNode(w io.Writer, fset *token.FileSet, x interface{}) {
+	// convert trailing tabs into spaces using a tconv filter
+	// to ensure a good outcome in most browsers (there may still
+	// be tabs in comments and strings, but converting those into
+	// the right number of spaces is much harder)
+	//
+	// TODO(gri) rethink printer flags - perhaps tconv can be eliminated
+	//           with an another printer mode (which is more efficiently
+	//           implemented in the printer than here with another layer)
+	mode := printer.TabIndent | printer.UseSpaces
+	err := (&printer.Config{Mode: mode, Tabwidth: TabWidth}).Fprint(&tconv{output: w}, fset, x)
+	if err != nil {
+		log.Print(err)
+	}
+}
+
+var WriteNode = writeNode

godoc/meta.go

@@ -0,0 +1,149 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package godoc
+
+import (
+	"bytes"
+	"encoding/json"
+	"log"
+	pathpkg "path"
+	"strings"
+	"time"
+
+	"code.google.com/p/go.tools/godoc/vfs"
+)
+
+var (
+	doctype   = []byte("<!DOCTYPE ")
+	jsonStart = []byte("<!--{")
+	jsonEnd   = []byte("}-->")
+)
+
+// ----------------------------------------------------------------------------
+// Documentation Metadata
+
+// TODO(adg): why are some exported and some aren't? -brad
+type Metadata struct {
+	Title    string
+	Subtitle string
+	Template bool   // execute as template
+	Path     string // canonical path for this page
+	filePath string // filesystem path relative to goroot
+}
+
+func (m *Metadata) FilePath() string { return m.filePath }
+
+// extractMetadata extracts the Metadata from a byte slice.
+// It returns the Metadata value and the remaining data.
+// If no metadata is present the original byte slice is returned.
+//
+func extractMetadata(b []byte) (meta Metadata, tail []byte, err error) {
+	tail = b
+	if !bytes.HasPrefix(b, jsonStart) {
+		return
+	}
+	end := bytes.Index(b, jsonEnd)
+	if end < 0 {
+		return
+	}
+	b = b[len(jsonStart)-1 : end+1] // drop leading <!-- and include trailing }
+	if err = json.Unmarshal(b, &meta); err != nil {
+		return
+	}
+	tail = tail[end+len(jsonEnd):]
+	return
+}
+
+// UpdateMetadata scans $GOROOT/doc for HTML files, reads their metadata,
+// and updates the DocMetadata map.
+func UpdateMetadata() {
+	metadata := make(map[string]*Metadata)
+	var scan func(string) // scan is recursive
+	scan = func(dir string) {
+		fis, err := FS.ReadDir(dir)
+		if err != nil {
+			log.Println("updateMetadata:", err)
+			return
+		}
+		for _, fi := range fis {
+			name := pathpkg.Join(dir, fi.Name())
+			if fi.IsDir() {
+				scan(name) // recurse
+				continue
+			}
+			if !strings.HasSuffix(name, ".html") {
+				continue
+			}
+			// Extract metadata from the file.
+			b, err := vfs.ReadFile(FS, name)
+			if err != nil {
+				log.Printf("updateMetadata %s: %v", name, err)
+				continue
+			}
+			meta, _, err := extractMetadata(b)
+			if err != nil {
+				log.Printf("updateMetadata: %s: %v", name, err)
+				continue
+			}
+			// Store relative filesystem path in Metadata.
+			meta.filePath = name
+			if meta.Path == "" {
+				// If no Path, canonical path is actual path.
+				meta.Path = meta.filePath
+			}
+			// Store under both paths.
+			metadata[meta.Path] = &meta
+			metadata[meta.filePath] = &meta
+		}
+	}
+	scan("/doc")
+	DocMetadata.Set(metadata)
+}
+
+// MetadataFor returns the *Metadata for a given relative path or nil if none
+// exists.
+//
+func MetadataFor(relpath string) *Metadata {
+	if m, _ := DocMetadata.Get(); m != nil {
+		meta := m.(map[string]*Metadata)
+		// If metadata for this relpath exists, return it.
+		if p := meta[relpath]; p != nil {
+			return p
+		}
+		// Try with or without trailing slash.
+		if strings.HasSuffix(relpath, "/") {
+			relpath = relpath[:len(relpath)-1]
+		} else {
+			relpath = relpath + "/"
+		}
+		return meta[relpath]
+	}
+	return nil
+}
+
+// Send a value on this channel to trigger a metadata refresh.
+// It is buffered so that if a signal is not lost if sent during a refresh.
+//
+var refreshMetadataSignal = make(chan bool, 1)
+
+// refreshMetadata sends a signal to update DocMetadata. If a refresh is in
+// progress the metadata will be refreshed again afterward.
+//
+func refreshMetadata() {
+	select {
+	case refreshMetadataSignal <- true:
+	default:
+	}
+}
+
+// RefreshMetadataLoop runs forever, updating DocMetadata when the underlying
+// file system changes. It should be launched in a goroutine.
+func RefreshMetadataLoop() {
+	for {
+		<-refreshMetadataSignal
+		UpdateMetadata()
+		time.Sleep(10 * time.Second) // at most once every 10 seconds
+	}
+}

godoc/page.go

@@ -0,0 +1,61 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package godoc
+
+import (
+	"log"
+	"net/http"
+	"runtime"
+	"text/template"
+)
+
+// Page describes the contents of the top-level godoc webpage.
+type Page struct {
+	Title    string
+	Tabtitle string
+	Subtitle string
+	Query    string
+	Body     []byte
+
+	// filled in by servePage
+	SearchBox  bool
+	Playground bool
+	Version    string
+}
+
+var (
+	DirlistHTML,
+	ErrorHTML,
+	ExampleHTML,
+	GodocHTML,
+	PackageHTML,
+	PackageText,
+	SearchHTML,
+	SearchText,
+	SearchDescXML *template.Template
+)
+
+func ServePage(w http.ResponseWriter, page Page) {
+	if page.Tabtitle == "" {
+		page.Tabtitle = page.Title
+	}
+	page.SearchBox = IndexEnabled
+	page.Playground = ShowPlayground
+	page.Version = runtime.Version()
+	if err := GodocHTML.Execute(w, page); err != nil && err != http.ErrBodyNotAllowed {
+		// Only log if there's an error that's not about writing on HEAD requests.
+		// See Issues 5451 and 5454.
+		log.Printf("GodocHTML.Execute: %s", err)
+	}
+}
+
+func ServeError(w http.ResponseWriter, r *http.Request, relpath string, err error) {
+	w.WriteHeader(http.StatusNotFound)
+	ServePage(w, Page{
+		Title:    "File " + relpath,
+		Subtitle: relpath,
+		Body:     applyTemplate(ErrorHTML, "errorHTML", err), // err may contain an absolute path!
+	})
+}

godoc/server.go

@@ -0,0 +1,609 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package godoc
+
+import (
+	"bytes"
+	"fmt"
+	"go/ast"
+	"go/build"
+	"go/doc"
+	"go/token"
+	htmlpkg "html"
+	"io"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"os"
+	pathpkg "path"
+	"path/filepath"
+	"regexp"
+	"strings"
+	"text/template"
+	"time"
+
+	"code.google.com/p/go.tools/godoc/util"
+	"code.google.com/p/go.tools/godoc/vfs"
+	"code.google.com/p/go.tools/godoc/vfs/httpfs"
+)
+
+// TODO(bradfitz,adg): these are moved from godoc.go globals.
+// Clean this up.
+var (
+	FileServer http.Handler // default file server
+	CmdHandler Server
+	PkgHandler Server
+
+	// file system information
+	FSTree      util.RWValue // *Directory tree of packages, updated with each sync (but sync code is removed now)
+	FSModified  util.RWValue // timestamp of last call to invalidateIndex
+	DocMetadata util.RWValue // mapping from paths to *Metadata
+)
+
+func InitHandlers(fs vfs.FileSystem) {
+	FileServer = http.FileServer(httpfs.New(fs))
+	CmdHandler = Server{"/cmd/", "/src/cmd"}
+	PkgHandler = Server{"/pkg/", "/src/pkg"}
+}
+
+// Server is a godoc server.
+type Server struct {
+	pattern string // url pattern; e.g. "/pkg/"
+	fsRoot  string // file system root to which the pattern is mapped
+}
+
+func (s *Server) FSRoot() string { return s.fsRoot }
+
+func (s *Server) RegisterWithMux(mux *http.ServeMux) {
+	mux.Handle(s.pattern, s)
+}
+
+// getPageInfo returns the PageInfo for a package directory abspath. If the
+// parameter genAST is set, an AST containing only the package exports is
+// computed (PageInfo.PAst), otherwise package documentation (PageInfo.Doc)
+// is extracted from the AST. If there is no corresponding package in the
+// directory, PageInfo.PAst and PageInfo.PDoc are nil. If there are no sub-
+// directories, PageInfo.Dirs is nil. If an error occurred, PageInfo.Err is
+// set to the respective error but the error is not logged.
+//
+func (h *Server) GetPageInfo(abspath, relpath string, mode PageInfoMode) *PageInfo {
+	info := &PageInfo{Dirname: abspath}
+
+	// Restrict to the package files that would be used when building
+	// the package on this system.  This makes sure that if there are
+	// separate implementations for, say, Windows vs Unix, we don't
+	// jumble them all together.
+	// Note: Uses current binary's GOOS/GOARCH.
+	// To use different pair, such as if we allowed the user to choose,
+	// set ctxt.GOOS and ctxt.GOARCH before calling ctxt.ImportDir.
+	ctxt := build.Default
+	ctxt.IsAbsPath = pathpkg.IsAbs
+	ctxt.ReadDir = fsReadDir
+	ctxt.OpenFile = fsOpenFile
+	pkginfo, err := ctxt.ImportDir(abspath, 0)
+	// continue if there are no Go source files; we still want the directory info
+	if _, nogo := err.(*build.NoGoError); err != nil && !nogo {
+		info.Err = err
+		return info
+	}
+
+	// collect package files
+	pkgname := pkginfo.Name
+	pkgfiles := append(pkginfo.GoFiles, pkginfo.CgoFiles...)
+	if len(pkgfiles) == 0 {
+		// Commands written in C have no .go files in the build.
+		// Instead, documentation may be found in an ignored file.
+		// The file may be ignored via an explicit +build ignore
+		// constraint (recommended), or by defining the package
+		// documentation (historic).
+		pkgname = "main" // assume package main since pkginfo.Name == ""
+		pkgfiles = pkginfo.IgnoredGoFiles
+	}
+
+	// get package information, if any
+	if len(pkgfiles) > 0 {
+		// build package AST
+		fset := token.NewFileSet()
+		files, err := parseFiles(fset, abspath, pkgfiles)
+		if err != nil {
+			info.Err = err
+			return info
+		}
+
+		// ignore any errors - they are due to unresolved identifiers
+		pkg, _ := ast.NewPackage(fset, files, poorMansImporter, nil)
+
+		// extract package documentation
+		info.FSet = fset
+		if mode&ShowSource == 0 {
+			// show extracted documentation
+			var m doc.Mode
+			if mode&NoFiltering != 0 {
+				m = doc.AllDecls
+			}
+			if mode&AllMethods != 0 {
+				m |= doc.AllMethods
+			}
+			info.PDoc = doc.New(pkg, pathpkg.Clean(relpath), m) // no trailing '/' in importpath
+
+			// collect examples
+			testfiles := append(pkginfo.TestGoFiles, pkginfo.XTestGoFiles...)
+			files, err = parseFiles(fset, abspath, testfiles)
+			if err != nil {
+				log.Println("parsing examples:", err)
+			}
+			info.Examples = collectExamples(pkg, files)
+
+			// collect any notes that we want to show
+			if info.PDoc.Notes != nil {
+				// could regexp.Compile only once per godoc, but probably not worth it
+				if rx, err := regexp.Compile(NotesRx); err == nil {
+					for m, n := range info.PDoc.Notes {
+						if rx.MatchString(m) {
+							if info.Notes == nil {
+								info.Notes = make(map[string][]*doc.Note)
+							}
+							info.Notes[m] = n
+						}
+					}
+				}
+			}
+
+		} else {
+			// show source code
+			// TODO(gri) Consider eliminating export filtering in this mode,
+			//           or perhaps eliminating the mode altogether.
+			if mode&NoFiltering == 0 {
+				packageExports(fset, pkg)
+			}
+			info.PAst = ast.MergePackageFiles(pkg, 0)
+		}
+		info.IsMain = pkgname == "main"
+	}
+
+	// get directory information, if any
+	var dir *Directory
+	var timestamp time.Time
+	if tree, ts := FSTree.Get(); tree != nil && tree.(*Directory) != nil {
+		// directory tree is present; lookup respective directory
+		// (may still fail if the file system was updated and the
+		// new directory tree has not yet been computed)
+		dir = tree.(*Directory).lookup(abspath)
+		timestamp = ts
+	}
+	if dir == nil {
+		// no directory tree present (too early after startup or
+		// command-line mode); compute one level for this page
+		// note: cannot use path filter here because in general
+		//       it doesn't contain the FSTree path
+		dir = NewDirectory(abspath, 1)
+		timestamp = time.Now()
+	}
+	info.Dirs = dir.listing(true)
+	info.DirTime = timestamp
+	info.DirFlat = mode&FlatDir != 0
+
+	return info
+}
+
+func (h *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	if redirect(w, r) {
+		return
+	}
+
+	relpath := pathpkg.Clean(r.URL.Path[len(h.pattern):])
+	abspath := pathpkg.Join(h.fsRoot, relpath)
+	mode := GetPageInfoMode(r)
+	if relpath == BuiltinPkgPath {
+		mode = NoFiltering
+	}
+	info := h.GetPageInfo(abspath, relpath, mode)
+	if info.Err != nil {
+		log.Print(info.Err)
+		ServeError(w, r, relpath, info.Err)
+		return
+	}
+
+	if mode&NoHTML != 0 {
+		ServeText(w, applyTemplate(PackageText, "packageText", info))
+		return
+	}
+
+	var tabtitle, title, subtitle string
+	switch {
+	case info.PAst != nil:
+		tabtitle = info.PAst.Name.Name
+	case info.PDoc != nil:
+		tabtitle = info.PDoc.Name
+	default:
+		tabtitle = info.Dirname
+		title = "Directory "
+		if ShowTimestamps {
+			subtitle = "Last update: " + info.DirTime.String()
+		}
+	}
+	if title == "" {
+		if info.IsMain {
+			// assume that the directory name is the command name
+			_, tabtitle = pathpkg.Split(relpath)
+			title = "Command "
+		} else {
+			title = "Package "
+		}
+	}
+	title += tabtitle
+
+	// special cases for top-level package/command directories
+	switch tabtitle {
+	case "/src/pkg":
+		tabtitle = "Packages"
+	case "/src/cmd":
+		tabtitle = "Commands"
+	}
+
+	ServePage(w, Page{
+		Title:    title,
+		Tabtitle: tabtitle,
+		Subtitle: subtitle,
+		Body:     applyTemplate(PackageHTML, "packageHTML", info),
+	})
+}
+
+type PageInfoMode uint
+
+const (
+	NoFiltering PageInfoMode = 1 << iota // do not filter exports
+	AllMethods                           // show all embedded methods
+	ShowSource                           // show source code, do not extract documentation
+	NoHTML                               // show result in textual form, do not generate HTML
+	FlatDir                              // show directory in a flat (non-indented) manner
+)
+
+// modeNames defines names for each PageInfoMode flag.
+var modeNames = map[string]PageInfoMode{
+	"all":     NoFiltering,
+	"methods": AllMethods,
+	"src":     ShowSource,
+	"text":    NoHTML,
+	"flat":    FlatDir,
+}
+
+// GetPageInfoMode computes the PageInfoMode flags by analyzing the request
+// URL form value "m". It is value is a comma-separated list of mode names
+// as defined by modeNames (e.g.: m=src,text).
+func GetPageInfoMode(r *http.Request) PageInfoMode {
+	var mode PageInfoMode
+	for _, k := range strings.Split(r.FormValue("m"), ",") {
+		if m, found := modeNames[strings.TrimSpace(k)]; found {
+			mode |= m
+		}
+	}
+	return AdjustPageInfoMode(r, mode)
+}
+
+// AdjustPageInfoMode allows specialized versions of godoc to adjust
+// PageInfoMode by overriding this variable.
+var AdjustPageInfoMode = func(_ *http.Request, mode PageInfoMode) PageInfoMode {
+	return mode
+}
+
+// fsReadDir implements ReadDir for the go/build package.
+func fsReadDir(dir string) ([]os.FileInfo, error) {
+	return FS.ReadDir(filepath.ToSlash(dir))
+}
+
+// fsOpenFile implements OpenFile for the go/build package.
+func fsOpenFile(name string) (r io.ReadCloser, err error) {
+	data, err := vfs.ReadFile(FS, filepath.ToSlash(name))
+	if err != nil {
+		return nil, err
+	}
+	return ioutil.NopCloser(bytes.NewReader(data)), nil
+}
+
+// poorMansImporter returns a (dummy) package object named
+// by the last path component of the provided package path
+// (as is the convention for packages). This is sufficient
+// to resolve package identifiers without doing an actual
+// import. It never returns an error.
+//
+func poorMansImporter(imports map[string]*ast.Object, path string) (*ast.Object, error) {
+	pkg := imports[path]
+	if pkg == nil {
+		// note that strings.LastIndex returns -1 if there is no "/"
+		pkg = ast.NewObj(ast.Pkg, path[strings.LastIndex(path, "/")+1:])
+		pkg.Data = ast.NewScope(nil) // required by ast.NewPackage for dot-import
+		imports[path] = pkg
+	}
+	return pkg, nil
+}
+
+// globalNames returns a set of the names declared by all package-level
+// declarations. Method names are returned in the form Receiver_Method.
+func globalNames(pkg *ast.Package) map[string]bool {
+	names := make(map[string]bool)
+	for _, file := range pkg.Files {
+		for _, decl := range file.Decls {
+			addNames(names, decl)
+		}
+	}
+	return names
+}
+
+// collectExamples collects examples for pkg from testfiles.
+func collectExamples(pkg *ast.Package, testfiles map[string]*ast.File) []*doc.Example {
+	var files []*ast.File
+	for _, f := range testfiles {
+		files = append(files, f)
+	}
+
+	var examples []*doc.Example
+	globals := globalNames(pkg)
+	for _, e := range doc.Examples(files...) {
+		name := stripExampleSuffix(e.Name)
+		if name == "" || globals[name] {
+			examples = append(examples, e)
+		} else {
+			log.Printf("skipping example 'Example%s' because '%s' is not a known function or type", e.Name, e.Name)
+		}
+	}
+
+	return examples
+}
+
+// addNames adds the names declared by decl to the names set.
+// Method names are added in the form ReceiverTypeName_Method.
+func addNames(names map[string]bool, decl ast.Decl) {
+	switch d := decl.(type) {
+	case *ast.FuncDecl:
+		name := d.Name.Name
+		if d.Recv != nil {
+			var typeName string
+			switch r := d.Recv.List[0].Type.(type) {
+			case *ast.StarExpr:
+				typeName = r.X.(*ast.Ident).Name
+			case *ast.Ident:
+				typeName = r.Name
+			}
+			name = typeName + "_" + name
+		}
+		names[name] = true
+	case *ast.GenDecl:
+		for _, spec := range d.Specs {
+			switch s := spec.(type) {
+			case *ast.TypeSpec:
+				names[s.Name.Name] = true
+			case *ast.ValueSpec:
+				for _, id := range s.Names {
+					names[id.Name] = true
+				}
+			}
+		}
+	}
+}
+
+// packageExports is a local implementation of ast.PackageExports
+// which correctly updates each package file's comment list.
+// (The ast.PackageExports signature is frozen, hence the local
+// implementation).
+//
+func packageExports(fset *token.FileSet, pkg *ast.Package) {
+	for _, src := range pkg.Files {
+		cmap := ast.NewCommentMap(fset, src, src.Comments)
+		ast.FileExports(src)
+		src.Comments = cmap.Filter(src).Comments()
+	}
+}
+
+func applyTemplate(t *template.Template, name string, data interface{}) []byte {
+	var buf bytes.Buffer
+	if err := t.Execute(&buf, data); err != nil {
+		log.Printf("%s.Execute: %s", name, err)
+	}
+	return buf.Bytes()
+}
+
+func redirect(w http.ResponseWriter, r *http.Request) (redirected bool) {
+	canonical := pathpkg.Clean(r.URL.Path)
+	if !strings.HasSuffix(canonical, "/") {
+		canonical += "/"
+	}
+	if r.URL.Path != canonical {
+		url := *r.URL
+		url.Path = canonical
+		http.Redirect(w, r, url.String(), http.StatusMovedPermanently)
+		redirected = true
+	}
+	return
+}
+
+func redirectFile(w http.ResponseWriter, r *http.Request) (redirected bool) {
+	c := pathpkg.Clean(r.URL.Path)
+	c = strings.TrimRight(c, "/")
+	if r.URL.Path != c {
+		url := *r.URL
+		url.Path = c
+		http.Redirect(w, r, url.String(), http.StatusMovedPermanently)
+		redirected = true
+	}
+	return
+}
+
+func serveTextFile(w http.ResponseWriter, r *http.Request, abspath, relpath, title string) {
+	src, err := vfs.ReadFile(FS, abspath)
+	if err != nil {
+		log.Printf("ReadFile: %s", err)
+		ServeError(w, r, relpath, err)
+		return
+	}
+
+	if r.FormValue("m") == "text" {
+		ServeText(w, src)
+		return
+	}
+
+	var buf bytes.Buffer
+	buf.WriteString("<pre>")
+	FormatText(&buf, src, 1, pathpkg.Ext(abspath) == ".go", r.FormValue("h"), RangeSelection(r.FormValue("s")))
+	buf.WriteString("</pre>")
+	fmt.Fprintf(&buf, `<p><a href="/%s?m=text">View as plain text</a></p>`, htmlpkg.EscapeString(relpath))
+
+	ServePage(w, Page{
+		Title:    title + " " + relpath,
+		Tabtitle: relpath,
+		Body:     buf.Bytes(),
+	})
+}
+
+func serveDirectory(w http.ResponseWriter, r *http.Request, abspath, relpath string) {
+	if redirect(w, r) {
+		return
+	}
+
+	list, err := FS.ReadDir(abspath)
+	if err != nil {
+		ServeError(w, r, relpath, err)
+		return
+	}
+
+	ServePage(w, Page{
+		Title:    "Directory " + relpath,
+		Tabtitle: relpath,
+		Body:     applyTemplate(DirlistHTML, "dirlistHTML", list),
+	})
+}
+
+func ServeHTMLDoc(w http.ResponseWriter, r *http.Request, abspath, relpath string) {
+	// get HTML body contents
+	src, err := vfs.ReadFile(FS, abspath)
+	if err != nil {
+		log.Printf("ReadFile: %s", err)
+		ServeError(w, r, relpath, err)
+		return
+	}
+
+	// if it begins with "<!DOCTYPE " assume it is standalone
+	// html that doesn't need the template wrapping.
+	if bytes.HasPrefix(src, doctype) {
+		w.Write(src)
+		return
+	}
+
+	// if it begins with a JSON blob, read in the metadata.
+	meta, src, err := extractMetadata(src)
+	if err != nil {
+		log.Printf("decoding metadata %s: %v", relpath, err)
+	}
+
+	// evaluate as template if indicated
+	if meta.Template {
+		tmpl, err := template.New("main").Funcs(TemplateFuncs).Parse(string(src))
+		if err != nil {
+			log.Printf("parsing template %s: %v", relpath, err)
+			ServeError(w, r, relpath, err)
+			return
+		}
+		var buf bytes.Buffer
+		if err := tmpl.Execute(&buf, nil); err != nil {
+			log.Printf("executing template %s: %v", relpath, err)
+			ServeError(w, r, relpath, err)
+			return
+		}
+		src = buf.Bytes()
+	}
+
+	// if it's the language spec, add tags to EBNF productions
+	if strings.HasSuffix(abspath, "go_spec.html") {
+		var buf bytes.Buffer
+		Linkify(&buf, src)
+		src = buf.Bytes()
+	}
+
+	ServePage(w, Page{
+		Title:    meta.Title,
+		Subtitle: meta.Subtitle,
+		Body:     src,
+	})
+}
+
+func serveFile(w http.ResponseWriter, r *http.Request) {
+	relpath := r.URL.Path
+
+	// Check to see if we need to redirect or serve another file.
+	if m := MetadataFor(relpath); m != nil {
+		if m.Path != relpath {
+			// Redirect to canonical path.
+			http.Redirect(w, r, m.Path, http.StatusMovedPermanently)
+			return
+		}
+		// Serve from the actual filesystem path.
+		relpath = m.filePath
+	}
+
+	abspath := relpath
+	relpath = relpath[1:] // strip leading slash
+
+	switch pathpkg.Ext(relpath) {
+	case ".html":
+		if strings.HasSuffix(relpath, "/index.html") {
+			// We'll show index.html for the directory.
+			// Use the dir/ version as canonical instead of dir/index.html.
+			http.Redirect(w, r, r.URL.Path[0:len(r.URL.Path)-len("index.html")], http.StatusMovedPermanently)
+			return
+		}
+		ServeHTMLDoc(w, r, abspath, relpath)
+		return
+
+	case ".go":
+		serveTextFile(w, r, abspath, relpath, "Source file")
+		return
+	}
+
+	dir, err := FS.Lstat(abspath)
+	if err != nil {
+		log.Print(err)
+		ServeError(w, r, relpath, err)
+		return
+	}
+
+	if dir != nil && dir.IsDir() {
+		if redirect(w, r) {
+			return
+		}
+		if index := pathpkg.Join(abspath, "index.html"); util.IsTextFile(FS, index) {
+			ServeHTMLDoc(w, r, index, index)
+			return
+		}
+		serveDirectory(w, r, abspath, relpath)
+		return
+	}
+
+	if util.IsTextFile(FS, abspath) {
+		if redirectFile(w, r) {
+			return
+		}
+		serveTextFile(w, r, abspath, relpath, "Text file")
+		return
+	}
+
+	FileServer.ServeHTTP(w, r)
+}
+
+func serveSearchDesc(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("Content-Type", "application/opensearchdescription+xml")
+	data := map[string]interface{}{
+		"BaseURL": fmt.Sprintf("http://%s", r.Host),
+	}
+	if err := SearchDescXML.Execute(w, &data); err != nil && err != http.ErrBodyNotAllowed {
+		// Only log if there's an error that's not about writing on HEAD requests.
+		// See Issues 5451 and 5454.
+		log.Printf("searchDescXML.Execute: %s", err)
+	}
+}
+
+func ServeText(w http.ResponseWriter, text []byte) {
+	w.Header().Set("Content-Type", "text/plain; charset=utf-8")
+	w.Write(text)
+}

godoc/spot.go

@@ -0,0 +1,66 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package godoc
+
+// ----------------------------------------------------------------------------
+// SpotInfo
+
+// A SpotInfo value describes a particular identifier spot in a given file;
+// It encodes three values: the SpotKind (declaration or use), a line or
+// snippet index "lori", and whether it's a line or index.
+//
+// The following encoding is used:
+//
+//   bits    32   4    1       0
+//   value    [lori|kind|isIndex]
+//
+type SpotInfo uint32
+
+// SpotKind describes whether an identifier is declared (and what kind of
+// declaration) or used.
+type SpotKind uint32
+
+const (
+	PackageClause SpotKind = iota
+	ImportDecl
+	ConstDecl
+	TypeDecl
+	VarDecl
+	FuncDecl
+	MethodDecl
+	Use
+	nKinds
+)
+
+func init() {
+	// sanity check: if nKinds is too large, the SpotInfo
+	// accessor functions may need to be updated
+	if nKinds > 8 {
+		panic("internal error: nKinds > 8")
+	}
+}
+
+// makeSpotInfo makes a SpotInfo.
+func makeSpotInfo(kind SpotKind, lori int, isIndex bool) SpotInfo {
+	// encode lori: bits [4..32)
+	x := SpotInfo(lori) << 4
+	if int(x>>4) != lori {
+		// lori value doesn't fit - since snippet indices are
+		// most certainly always smaller then 1<<28, this can
+		// only happen for line numbers; give it no line number (= 0)
+		x = 0
+	}
+	// encode kind: bits [1..4)
+	x |= SpotInfo(kind) << 1
+	// encode isIndex: bit 0
+	if isIndex {
+		x |= 1
+	}
+	return x
+}
+
+func (x SpotInfo) Kind() SpotKind { return SpotKind(x >> 1 & 7) }
+func (x SpotInfo) Lori() int      { return int(x >> 4) }
+func (x SpotInfo) IsIndex() bool  { return x&1 != 0 }

godoc/tab.go

@@ -0,0 +1,77 @@
+// TODO(bradfitz,adg): move to util
+
+package godoc
+
+import "io"
+
+var spaces = []byte("                                ") // 32 spaces seems like a good number
+
+const (
+	indenting = iota
+	collecting
+)
+
+// A tconv is an io.Writer filter for converting leading tabs into spaces.
+type tconv struct {
+	output io.Writer
+	state  int // indenting or collecting
+	indent int // valid if state == indenting
+}
+
+func (p *tconv) writeIndent() (err error) {
+	i := p.indent
+	for i >= len(spaces) {
+		i -= len(spaces)
+		if _, err = p.output.Write(spaces); err != nil {
+			return
+		}
+	}
+	// i < len(spaces)
+	if i > 0 {
+		_, err = p.output.Write(spaces[0:i])
+	}
+	return
+}
+
+func (p *tconv) Write(data []byte) (n int, err error) {
+	if len(data) == 0 {
+		return
+	}
+	pos := 0 // valid if p.state == collecting
+	var b byte
+	for n, b = range data {
+		switch p.state {
+		case indenting:
+			switch b {
+			case '\t':
+				p.indent += TabWidth
+			case '\n':
+				p.indent = 0
+				if _, err = p.output.Write(data[n : n+1]); err != nil {
+					return
+				}
+			case ' ':
+				p.indent++
+			default:
+				p.state = collecting
+				pos = n
+				if err = p.writeIndent(); err != nil {
+					return
+				}
+			}
+		case collecting:
+			if b == '\n' {
+				p.state = indenting
+				p.indent = 0
+				if _, err = p.output.Write(data[pos : n+1]); err != nil {
+					return
+				}
+			}
+		}
+	}
+	n = len(data)
+	if pos < n && p.state == collecting {
+		_, err = p.output.Write(data[pos:])
+	}
+	return
+}