go/printer: remove notion of "Styler", remove HTML mode

Neither gofmt nor godoc are making use of a Styler (for
token-specific formatting) anymore. Stylers interacted in complicated
ways with HTML-escaping which was why the printer needed an HTML mode
in the first place.

godoc now uses a more powerful and general text formatting
function that does HTML escaping, text selection, and can
handle token-specific formatting if so desired (currently
used only for comments).

As a consequence, cleaned up uses of go/printer in godoc;
simplified the various write utility functions, and also
removed the need for the "html" template format (in favor of
html-esc which now does the same and is used more pervasively).

Applied gofmt -w src misc to verify no changes occured,
and tested godoc manually.

There should be no visible changes except that (type) code
snippets presented for godoc package documentation now
uses the same formatting as for general source code and
thus comments get the comment-specific color here as well
(not the case at the moment).

(TODO: godoc needs a good automatic test suite).

R=rsc
CC=golang-dev
https://golang.org/cl/4152042

lib/godoc/package.html

@@ -5,7 +5,7 @@
 -->
 
 {.section PAst}
-	<pre>{@ FSet|html}</pre>
+	<pre>{@ FSet|html-esc}</pre>
 {.end}
 {.section PDoc}
 	<!-- PackageName is printed as title by the top-level template -->
@@ -31,46 +31,45 @@
 		<h2 id="Constants">Constants</h2>
 		{.repeated section @}
 			{Doc|html-comment}
-			<pre>{Decl FSet|html}</pre>
+			<pre>{Decl FSet|html-esc}</pre>
 		{.end}
 	{.end}
 	{.section Vars}
 		<h2 id="Variables">Variables</h2>
 		{.repeated section @}
 			{Doc|html-comment}
-			<pre>{Decl FSet|html}</pre>
+			<pre>{Decl FSet|html-esc}</pre>
 		{.end}
 	{.end}
 	{.section Funcs}
 		{.repeated section @}
 			{# Name is a string - no need for FSet}
 			<h2 id="{Name|html-esc}">func <a href="/{Decl FSet|url-pos}">{Name|html-esc}</a></h2>
-			<p><code>{Decl FSet|html}</code></p>
+			<p><code>{Decl FSet|html-esc}</code></p>
 			{Doc|html-comment}
 		{.end}
 	{.end}
 	{.section Types}
 		{.repeated section @}
-			{# Type.Name is a string - no need for FSet}
 			<h2 id="{Type.Name FSet|html-esc}">type <a href="/{Decl FSet|url-pos}">{Type.Name FSet|html-esc}</a></h2>
 			{Doc|html-comment}
-			<p><pre>{Decl FSet|html}</pre></p>
+			<p><pre>{Decl FSet|html-esc}</pre></p>
 			{.repeated section Consts}
 				{Doc|html-comment}
-				<pre>{Decl FSet|html}</pre>
+				<pre>{Decl FSet|html-esc}</pre>
 			{.end}
 			{.repeated section Vars}
 				{Doc|html-comment}
-				<pre>{Decl FSet|html}</pre>
+				<pre>{Decl FSet|html-esc}</pre>
 			{.end}
 			{.repeated section Factories}
 				<h3 id="{Type.Name FSet|html-esc}.{Name|html-esc}">func <a href="/{Decl FSet|url-pos}">{Name|html-esc}</a></h3>
-				<p><code>{Decl FSet|html}</code></p>
+				<p><code>{Decl FSet|html-esc}</code></p>
 				{Doc|html-comment}
 			{.end}
 			{.repeated section Methods}
-				<h3 id="{Type.Name FSet|html-esc}.{Name|html-esc}">func ({Recv FSet|html}) <a href="/{Decl FSet|url-pos}">{Name|html-esc}</a></h3>
+				<h3 id="{Type.Name FSet|html-esc}.{Name|html-esc}">func ({Recv FSet|html-esc}) <a href="/{Decl FSet|url-pos}">{Name|html-esc}</a></h3>
-				<p><code>{Decl FSet|html}</code></p>
+				<p><code>{Decl FSet|html-esc}</code></p>
 				{Doc|html-comment}
 			{.end}
 		{.end}

src/cmd/godoc/format.go

@@ -11,7 +11,6 @@
 package main
 
 import (
-	"bytes"
 	"fmt"
 	"go/scanner"
 	"go/token"
@@ -335,12 +334,12 @@ func selectionTag(w io.Writer, text []byte, selections int) {
 }
 
 
-// FormatText HTML-escapes text and returns it wrapped in <pre> tags.
+// FormatText HTML-escapes text and writes it to w.
-// Conscutive text segments are wrapped in HTML spans (with tags as
+// Consecutive text segments are wrapped in HTML spans (with tags as
 // defined by startTags and endTag) as follows:
 //
-//	- if line >= 0, line numbers are printed before each line, starting
+//	- if line >= 0, line number (ln) spans are inserted before each line,
-//	  with the value of line
+//	  starting with the value of line
 //	- if the text is Go source, comments get the "comment" span class
 //	- each occurrence of the regular expression pattern gets the "highlight"
 //	  span class
@@ -349,10 +348,7 @@ func selectionTag(w io.Writer, text []byte, selections int) {
 // Comments, highlights, and selections may overlap arbitrarily; the respective
 // HTML span classes are specified in the startTags variable.
 //
-func FormatText(text []byte, line int, goSource bool, pattern string, selection Selection) []byte {
+func FormatText(w io.Writer, text []byte, line int, goSource bool, pattern string, selection Selection) {
-	var buf bytes.Buffer
-	buf.WriteString("<pre>\n")
-
 	var comments, highlights Selection
 	if goSource {
 		comments = commentSelection(text)
@@ -370,11 +366,8 @@ func FormatText(text []byte, line int, goSource bool, pattern string, selection
 				}
 			}
 		}
-		FormatSelections(&buf, text, lineTag, lineSelection(text), selectionTag, comments, highlights, selection)
+		FormatSelections(w, text, lineTag, lineSelection(text), selectionTag, comments, highlights, selection)
 	} else {
-		template.HTMLEscape(&buf, text)
+		template.HTMLEscape(w, text)
 	}
-
-	buf.WriteString("</pre>\n")
-	return buf.Bytes()
 }

src/cmd/godoc/godoc.go

@@ -346,47 +346,47 @@ func (p *tconv) Write(data []byte) (n int, err os.Error) {
 // ----------------------------------------------------------------------------
 // Templates
 
-// Write an AST-node to w; optionally html-escaped.
+// Write an AST node to w.
-func writeNode(w io.Writer, fset *token.FileSet, node interface{}, html bool) {
+func writeNode(w io.Writer, fset *token.FileSet, x interface{}) {
-	mode := printer.TabIndent | printer.UseSpaces
-	if html {
-		mode |= printer.GenHTML
-	}
 	// convert trailing tabs into spaces using a tconv filter
 	// to ensure a good outcome in most browsers (there may still
 	// be tabs in comments and strings, but converting those into
 	// the right number of spaces is much harder)
-	(&printer.Config{mode, *tabwidth, nil}).Fprint(&tconv{output: w}, fset, node)
+	mode := printer.TabIndent | printer.UseSpaces
+	(&printer.Config{mode, *tabwidth}).Fprint(&tconv{output: w}, fset, x)
 }
 
 
-// Write text to w; optionally html-escaped.
+// Write anything to w.
-func writeText(w io.Writer, text []byte, html bool) {
+func writeAny(w io.Writer, fset *token.FileSet, x interface{}) {
-	if html {
-		template.HTMLEscape(w, text)
-		return
-	}
-	w.Write(text)
-}
-
-
-// Write anything to w; optionally html-escaped.
-func writeAny(w io.Writer, fset *token.FileSet, html bool, x interface{}) {
 	switch v := x.(type) {
 	case []byte:
-		writeText(w, v, html)
+		w.Write(v)
 	case string:
-		writeText(w, []byte(v), html)
+		w.Write([]byte(v))
 	case ast.Decl, ast.Expr, ast.Stmt, *ast.File:
-		writeNode(w, fset, x, html)
+		writeNode(w, fset, x)
 	default:
-		if html {
-			var buf bytes.Buffer
-			fmt.Fprint(&buf, x)
-			writeText(w, buf.Bytes(), true)
-		} else {
 		fmt.Fprint(w, x)
 	}
+}
+
+
+// Write anything html-escaped to w.
+func writeAnyHTML(w io.Writer, fset *token.FileSet, x interface{}) {
+	switch v := x.(type) {
+	case []byte:
+		template.HTMLEscape(w, v)
+	case string:
+		template.HTMLEscape(w, []byte(v))
+	case ast.Decl, ast.Expr, ast.Stmt, *ast.File:
+		var buf bytes.Buffer
+		writeNode(&buf, fset, x)
+		FormatText(w, buf.Bytes(), -1, true, "", nil)
+	default:
+		var buf bytes.Buffer
+		fmt.Fprint(&buf, x)
+		template.HTMLEscape(w, buf.Bytes())
 	}
 }
 
@@ -401,24 +401,16 @@ func fileset(x []interface{}) *token.FileSet {
 }
 
 
-// Template formatter for "html" format.
-func htmlFmt(w io.Writer, format string, x ...interface{}) {
-	writeAny(w, fileset(x), true, x[0])
-}
-
-
 // Template formatter for "html-esc" format.
 func htmlEscFmt(w io.Writer, format string, x ...interface{}) {
-	var buf bytes.Buffer
+	writeAnyHTML(w, fileset(x), x[0])
-	writeAny(&buf, fileset(x), false, x[0])
-	template.HTMLEscape(w, buf.Bytes())
 }
 
 
 // Template formatter for "html-comment" format.
 func htmlCommentFmt(w io.Writer, format string, x ...interface{}) {
 	var buf bytes.Buffer
-	writeAny(&buf, fileset(x), false, x[0])
+	writeAny(&buf, fileset(x), x[0])
 	// TODO(gri) Provide list of words (e.g. function parameters)
 	//           to be emphasized by ToHTML.
 	doc.ToHTML(w, buf.Bytes(), nil) // does html-escaping
@@ -427,14 +419,14 @@ func htmlCommentFmt(w io.Writer, format string, x ...interface{}) {
 
 // Template formatter for "" (default) format.
 func textFmt(w io.Writer, format string, x ...interface{}) {
-	writeAny(w, fileset(x), false, x[0])
+	writeAny(w, fileset(x), x[0])
 }
 
 
 // Template formatter for "urlquery-esc" format.
 func urlQueryEscFmt(w io.Writer, format string, x ...interface{}) {
 	var buf bytes.Buffer
-	writeAny(&buf, fileset(x), false, x[0])
+	writeAny(&buf, fileset(x), x[0])
 	template.HTMLEscape(w, []byte(http.URLEscape(string(buf.Bytes()))))
 }
 
@@ -603,7 +595,6 @@ func numlinesFmt(w io.Writer, format string, x ...interface{}) {
 
 var fmap = template.FormatterMap{
 	"":             textFmt,
-	"html":         htmlFmt,
 	"html-esc":     htmlEscFmt,
 	"html-comment": htmlCommentFmt,
 	"urlquery-esc": urlQueryEscFmt,
@@ -775,8 +766,12 @@ func serveTextFile(w http.ResponseWriter, r *http.Request, abspath, relpath, tit
 		return
 	}
 
-	contents := FormatText(src, 1, pathutil.Ext(abspath) == ".go", r.FormValue("h"), rangeSelection(r.FormValue("s")))
+	var buf bytes.Buffer
-	servePage(w, title+" "+relpath, "", "", contents)
+	buf.WriteString("<pre>")
+	FormatText(&buf, src, 1, pathutil.Ext(abspath) == ".go", r.FormValue("h"), rangeSelection(r.FormValue("s")))
+	buf.WriteString("</pre>")
+
+	servePage(w, title+" "+relpath, "", "", buf.Bytes())
 }
 
 

src/cmd/godoc/main.go

@@ -367,7 +367,11 @@ func main() {
 				if i > 0 {
 					fmt.Println()
 				}
-				writeAny(os.Stdout, info.FSet, *html, d)
+				if *html {
+					writeAnyHTML(os.Stdout, info.FSet, d)
+				} else {
+					writeAny(os.Stdout, info.FSet, d)
+				}
 				fmt.Println()
 			}
 			return

src/cmd/godoc/snippet.go

@@ -25,9 +25,14 @@ type Snippet struct {
 
 func newSnippet(fset *token.FileSet, decl ast.Decl, id *ast.Ident) *Snippet {
 	// TODO instead of pretty-printing the node, should use the original source instead
-	var buf bytes.Buffer
+	var buf1 bytes.Buffer
-	writeNode(&buf, fset, decl, false)
+	writeNode(&buf1, fset, decl)
-	return &Snippet{fset.Position(id.Pos()).Line, FormatText(buf.Bytes(), -1, true, id.Name, nil)}
+	// wrap text with <pre> tag
+	var buf2 bytes.Buffer
+	buf2.WriteString("<pre>")
+	FormatText(&buf2, buf1.Bytes(), -1, true, id.Name, nil)
+	buf2.WriteString("</pre>")
+	return &Snippet{fset.Position(id.Pos()).Line, buf2.Bytes()}
 }
 
 

src/cmd/gofmt/gofmt.go

@@ -114,7 +114,7 @@ func processFile(f *os.File) os.Error {
 	}
 
 	var buf bytes.Buffer
-	_, err = (&printer.Config{printerMode, *tabWidth, nil}).Fprint(&buf, fset, file)
+	_, err = (&printer.Config{printerMode, *tabWidth}).Fprint(&buf, fset, file)
 	if err != nil {
 		return err
 	}

src/pkg/go/printer/printer.go

@@ -46,12 +46,6 @@ var (
 	htabs     = []byte("\t\t\t\t\t\t\t\t")
 	newlines  = []byte("\n\n\n\n\n\n\n\n") // more than the max determined by nlines
 	formfeeds = []byte("\f\f\f\f\f\f\f\f") // more than the max determined by nlines
-
-	esc_quot = []byte(""") // shorter than """
-	esc_apos = []byte("'") // shorter than "'"
-	esc_amp  = []byte("&")
-	esc_lt   = []byte("<")
-	esc_gt   = []byte(">")
 )
 
 
@@ -161,8 +155,8 @@ func (p *printer) write0(data []byte) {
 
 
 // write interprets data and writes it to p.output. It inserts indentation
-// after a line break unless in a tabwriter escape sequence, and it HTML-
+// after a line break unless in a tabwriter escape sequence.
-// escapes characters if GenHTML is set. It updates p.pos as a side-effect.
+// It updates p.pos as a side-effect.
 //
 func (p *printer) write(data []byte) {
 	i0 := 0
@@ -195,36 +189,6 @@ func (p *printer) write(data []byte) {
 			// next segment start
 			i0 = i + 1
 
-		case '"', '\'', '&', '<', '>':
-			if p.Mode&GenHTML != 0 {
-				// write segment ending in b
-				p.write0(data[i0:i])
-
-				// write HTML-escaped b
-				var esc []byte
-				switch b {
-				case '"':
-					esc = esc_quot
-				case '\'':
-					esc = esc_apos
-				case '&':
-					esc = esc_amp
-				case '<':
-					esc = esc_lt
-				case '>':
-					esc = esc_gt
-				}
-				p.write0(esc)
-
-				// update p.pos
-				d := i + 1 - i0
-				p.pos.Offset += d
-				p.pos.Column += d
-
-				// next segment start
-				i0 = i + 1
-			}
-
 		case tabwriter.Escape:
 			p.mode ^= inLiteral
 
@@ -257,29 +221,13 @@ func (p *printer) writeNewlines(n int, useFF bool) {
 }
 
 
-func (p *printer) writeTaggedItem(data []byte, tag HTMLTag) {
-	// write start tag, if any
-	// (no html-escaping and no p.pos update for tags - use write0)
-	if tag.Start != "" {
-		p.write0([]byte(tag.Start))
-	}
-	p.write(data)
-	// write end tag, if any
-	if tag.End != "" {
-		p.write0([]byte(tag.End))
-	}
-}
-
-
 // writeItem writes data at position pos. data is the text corresponding to
 // a single lexical token, but may also be comment text. pos is the actual
 // (or at least very accurately estimated) position of the data in the original
-// source text. If tags are present and GenHTML is set, the tags are written
+// source text. writeItem updates p.last to the position immediately following
-// before and after the data. writeItem updates p.last to the position
+// the data.
-// immediately following the data.
 //
-func (p *printer) writeItem(pos token.Position, data []byte, tag HTMLTag) {
+func (p *printer) writeItem(pos token.Position, data []byte) {
-	fileChanged := false
 	if pos.IsValid() {
 		// continue with previous position if we don't have a valid pos
 		if p.last.IsValid() && p.last.Filename != pos.Filename {
@@ -289,7 +237,6 @@ func (p *printer) writeItem(pos token.Position, data []byte, tag HTMLTag) {
 			p.indent = 0
 			p.mode = 0
 			p.buffer = p.buffer[0:0]
-			fileChanged = true
 		}
 		p.pos = pos
 	}
@@ -298,18 +245,7 @@ func (p *printer) writeItem(pos token.Position, data []byte, tag HTMLTag) {
 		_, filename := path.Split(pos.Filename)
 		p.write0([]byte(fmt.Sprintf("[%s:%d:%d]", filename, pos.Line, pos.Column)))
 	}
-	if p.Mode&GenHTML != 0 {
-		// write line tag if on a new line
-		// TODO(gri): should write line tags on each line at the start
-		//            will be more useful (e.g. to show line numbers)
-		if p.Styler != nil && (pos.Line != p.lastTaggedLine || fileChanged) {
-			p.writeTaggedItem(p.Styler.LineTag(pos.Line))
-			p.lastTaggedLine = pos.Line
-		}
-		p.writeTaggedItem(data, tag)
-	} else {
 	p.write(data)
-	}
 	p.last = p.pos
 }
 
@@ -419,21 +355,10 @@ func (p *printer) writeCommentPrefix(pos, next token.Position, isFirst, isKeywor
 func (p *printer) writeCommentLine(comment *ast.Comment, pos token.Position, line []byte) {
 	// line must pass through unchanged, bracket it with tabwriter.Escape
 	line = bytes.Join([][]byte{esc, line, esc}, nil)
-
+	p.writeItem(pos, line)
-	// apply styler, if any
-	var tag HTMLTag
-	if p.Styler != nil {
-		line, tag = p.Styler.Comment(comment, line)
-	}
-
-	p.writeItem(pos, line, tag)
 }
 
 
-// TODO(gri): Similar (but not quite identical) functionality for
-//            comment processing can be found in go/doc/comment.go.
-//            Perhaps this can be factored eventually.
-
 // Split comment text into lines
 func split(text []byte) [][]byte {
 	// count lines (comment text never ends in a newline)
@@ -802,7 +727,6 @@ func (p *printer) print(args ...interface{}) {
 	for _, f := range args {
 		next := p.pos // estimated position of next item
 		var data []byte
-		var tag HTMLTag
 		var tok token.Token
 
 		switch x := f.(type) {
@@ -827,27 +751,17 @@ func (p *printer) print(args ...interface{}) {
 			p.buffer = p.buffer[0 : i+1]
 			p.buffer[i] = x
 		case *ast.Ident:
-			if p.Styler != nil {
-				data, tag = p.Styler.Ident(x)
-			} else {
 			data = []byte(x.Name)
-			}
 			tok = token.IDENT
 		case *ast.BasicLit:
-			if p.Styler != nil {
-				data, tag = p.Styler.BasicLit(x)
-			} else {
-				data = x.Value
-			}
 			// escape all literals so they pass through unchanged
 			// (note that valid Go programs cannot contain
 			// tabwriter.Escape bytes since they do not appear in
 			// legal UTF-8 sequences)
-			escData := make([]byte, 0, len(data)+2)
+			data = make([]byte, 0, len(x.Value)+2)
-			escData = append(escData, tabwriter.Escape)
+			data = append(data, tabwriter.Escape)
-			escData = append(escData, data...)
+			data = append(data, x.Value...)
-			escData = append(escData, tabwriter.Escape)
+			data = append(data, tabwriter.Escape)
-			data = escData
 			tok = x.Kind
 			// If we have a raw string that spans multiple lines and
 			// the opening quote (`) is on a line preceded only by
@@ -877,11 +791,7 @@ func (p *printer) print(args ...interface{}) {
 				p.buffer = p.buffer[0:1]
 				p.buffer[0] = ' '
 			}
-			if p.Styler != nil {
-				data, tag = p.Styler.Token(x)
-			} else {
 			data = []byte(s)
-			}
 			tok = x
 		case token.Pos:
 			if x.IsValid() {
@@ -904,7 +814,7 @@ func (p *printer) print(args ...interface{}) {
 			// before
 			p.writeNewlines(next.Line-p.pos.Line, droppedFF)
 
-			p.writeItem(next, data, tag)
+			p.writeItem(next, data)
 		}
 	}
 }
@@ -1064,36 +974,16 @@ func (p *trimmer) Write(data []byte) (n int, err os.Error) {
 
 // General printing is controlled with these Config.Mode flags.
 const (
-	GenHTML   uint = 1 << iota // generate HTML
+	RawFormat uint = 1 << iota // do not use a tabwriter; if set, UseSpaces is ignored
-	RawFormat                  // do not use a tabwriter; if set, UseSpaces is ignored
 	TabIndent                  // use tabs for indentation independent of UseSpaces
 	UseSpaces                  // use spaces instead of tabs for alignment
 )
 
 
-// An HTMLTag specifies a start and end tag.
-type HTMLTag struct {
-	Start, End string // empty if tags are absent
-}
-
-
-// A Styler specifies formatting of line tags and elementary Go words.
-// A format consists of text and a (possibly empty) surrounding HTML tag.
-//
-type Styler interface {
-	LineTag(line int) ([]byte, HTMLTag)
-	Comment(c *ast.Comment, line []byte) ([]byte, HTMLTag)
-	BasicLit(x *ast.BasicLit) ([]byte, HTMLTag)
-	Ident(id *ast.Ident) ([]byte, HTMLTag)
-	Token(tok token.Token) ([]byte, HTMLTag)
-}
-
-
 // A Config node controls the output of Fprint.
 type Config struct {
 	Mode     uint // default: 0
 	Tabwidth int  // default: 8
-	Styler   Styler // default: nil
 }
 
 
@@ -1121,9 +1011,6 @@ func (cfg *Config) Fprint(output io.Writer, fset *token.FileSet, node interface{
 		}
 
 		twmode := tabwriter.DiscardEmptyColumns
-		if cfg.Mode&GenHTML != 0 {
-			twmode |= tabwriter.FilterHTML
-		}
 		if cfg.Mode&TabIndent != 0 {
 			minwidth = 0
 			twmode |= tabwriter.TabIndent