template: support string, int and float literals

This enables customizing the behavior of formatters
with logic such as {"template"|import} or even
{Field1 Field2 "%.2f 0x%X"|printf}

Thanks to Roger Peppe for some debate on this.

R=golang-dev, r, r
CC=golang-dev
https://golang.org/cl/4536059

src/pkg/template/template.go

@@ -76,6 +76,10 @@
 	executed sequentially, with each formatter receiving the bytes
 	emitted by the one to its left.
 
+	As well as field names, one may use literals with Go syntax.
+	Integer, floating-point, and string literals are supported.
+	Raw strings may not span newlines.
+
 	The delimiter strings get their default value, "{" and "}", from
 	JSON-template.  They may be set to any non-empty, space-free
 	string using the SetDelims method.  Their value can be printed
@@ -91,6 +95,7 @@ import (
 	"io/ioutil"
 	"os"
 	"reflect"
+	"strconv"
 	"strings"
 	"unicode"
 	"utf8"
@@ -151,10 +156,13 @@ type literalElement struct {
 // A variable invocation to be evaluated
 type variableElement struct {
 	linenum int
-	word    []string // The fields in the invocation.
+	args    []interface{} // The fields and literals in the invocation.
 	fmts    []string      // Names of formatters to apply. len(fmts) > 0
 }
 
+// A variableElement arg to be evaluated as a field name
+type fieldName string
+
 // A .section block, possibly with a .or
 type sectionElement struct {
 	linenum int    // of .section itself
@@ -245,6 +253,31 @@ func equal(s []byte, n int, t []byte) bool {
 	return true
 }
 
+// isQuote returns true if c is a string- or character-delimiting quote character.
+func isQuote(c byte) bool {
+	return c == '"' || c == '`' || c == '\''
+}
+
+// endQuote returns the end quote index for the quoted string that
+// starts at n, or -1 if no matching end quote is found before the end
+// of the line.
+func endQuote(s []byte, n int) int {
+	quote := s[n]
+	for n++; n < len(s); n++ {
+		switch s[n] {
+		case '\\':
+			if quote == '"' || quote == '\'' {
+				n++
+			}
+		case '\n':
+			return -1
+		case quote:
+			return n
+		}
+	}
+	return -1
+}
+
 // nextItem returns the next item from the input buffer.  If the returned
 // item is empty, we are at EOF.  The item will be either a
 // delimited string or a non-empty string between delimited
@@ -282,6 +315,14 @@ func (t *Template) nextItem() []byte {
 			if t.buf[i] == '\n' {
 				break
 			}
+			if isQuote(t.buf[i]) {
+				i = endQuote(t.buf, i)
+				if i == -1 {
+					t.parseError("unmatched quote")
+					return nil
+				}
+				continue
+			}
 			if equal(t.buf, i, t.rdelim) {
 				i += len(t.rdelim)
 				right = i
@@ -333,23 +374,33 @@ func (t *Template) nextItem() []byte {
 	return item
 }
 
-// Turn a byte array into a white-space-split array of strings.
+// Turn a byte array into a white-space-split array of strings,
+// taking into account quoted strings.
 func words(buf []byte) []string {
 	s := make([]string, 0, 5)
-	p := 0 // position in buf
+	for i := 0; i < len(buf); {
-	// one word per loop
+		// One word per loop
-	for i := 0; ; i++ {
+		for i < len(buf) && white(buf[i]) {
-		// skip white space
+			i++
-		for ; p < len(buf) && white(buf[p]); p++ {
 		}
-		// grab word
+		if i == len(buf) {
-		start := p
-		for ; p < len(buf) && !white(buf[p]); p++ {
-		}
-		if start == p { // no text left
 			break
 		}
-		s = append(s, string(buf[start:p]))
+		// Got a word
+		start := i
+		if isQuote(buf[i]) {
+			i = endQuote(buf, i)
+			if i < 0 {
+				i = len(buf)
+			} else {
+				i++
+			}
+		} else {
+			for i < len(buf) && !white(buf[i]) {
+				i++
+			}
+		}
+		s = append(s, string(buf[start:i]))
 	}
 	return s
 }
@@ -381,11 +432,17 @@ func (t *Template) analyze(item []byte) (tok int, w []string) {
 		t.parseError("empty directive")
 		return
 	}
-	if len(w) > 0 && w[0][0] != '.' {
+	first := w[0]
+	if first[0] != '.' {
 		tok = tokVariable
 		return
 	}
-	switch w[0] {
+	if len(first) > 1 && first[1] >= '0' && first[1] <= '9' {
+		// Must be a float.
+		tok = tokVariable
+		return
+	}
+	switch first {
 	case ".meta-left", ".meta-right", ".space", ".tab":
 		tok = tokLiteral
 		return
@@ -447,6 +504,37 @@ func (t *Template) newVariable(words []string) *variableElement {
 		formatters = strings.Split(lastWord[bar+1:], "|", -1)
 	}
 
+	args := make([]interface{}, len(words))
+
+	// Build argument list, processing any literals
+	for i, word := range words {
+		var lerr os.Error
+		switch word[0] {
+		case '"', '`', '\'':
+			v, err := strconv.Unquote(word)
+			if err == nil && word[0] == '\'' {
+				args[i] = []int(v)[0]
+			} else {
+				args[i], lerr = v, err
+			}
+
+		case '.', '+', '-', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
+			v, err := strconv.Btoi64(word, 0)
+			if err == nil {
+				args[i] = v
+			} else {
+				v, err := strconv.Atof64(word)
+				args[i], lerr = v, err
+			}
+
+		default:
+			args[i] = fieldName(word)
+		}
+		if lerr != nil {
+			t.parseError("invalid literal: %q: %s", word, lerr)
+		}
+	}
+
 	// We could remember the function address here and avoid the lookup later,
 	// but it's more dynamic to let the user change the map contents underfoot.
 	// We do require the name to be present, though.
@@ -457,7 +545,8 @@ func (t *Template) newVariable(words []string) *variableElement {
 			t.parseError("unknown formatter: %q", f)
 		}
 	}
-	return &variableElement{t.linenum, words, formatters}
+
+	return &variableElement{t.linenum, args, formatters}
 }
 
 // Grab the next item.  If it's simple, just append it to the template.
@@ -753,7 +842,7 @@ func (t *Template) varValue(name string, st *state) reflect.Value {
 func (t *Template) format(wr io.Writer, fmt string, val []interface{}, v *variableElement, st *state) {
 	fn := t.formatter(fmt)
 	if fn == nil {
-		t.execError(st, v.linenum, "missing formatter %s for variable %s", fmt, v.word[0])
+		t.execError(st, v.linenum, "missing formatter %s for variable", fmt)
 	}
 	fn(wr, fmt, val...)
 }
@@ -761,12 +850,15 @@ func (t *Template) format(wr io.Writer, fmt string, val []interface{}, v *variab
 // Evaluate a variable, looking up through the parent if necessary.
 // If it has a formatter attached ({var|formatter}) run that too.
 func (t *Template) writeVariable(v *variableElement, st *state) {
-	// Turn the words of the invocation into values.
+	// Resolve field names
-	val := make([]interface{}, len(v.word))
+	val := make([]interface{}, len(v.args))
-	for i, word := range v.word {
+	for i, arg := range v.args {
-		val[i] = t.varValue(word, st).Interface()
+		if name, ok := arg.(fieldName); ok {
+			val[i] = t.varValue(string(name), st).Interface()
+		} else {
+			val[i] = arg
+		}
 	}
-
 	for i, fmt := range v.fmts[:len(v.fmts)-1] {
 		b := &st.buf[i&1]
 		b.Reset()

src/pkg/template/template_test.go

@@ -94,10 +94,15 @@ func multiword(w io.Writer, format string, value ...interface{}) {
 	}
 }
 
+func printf(w io.Writer, format string, v ...interface{}) {
+	io.WriteString(w, fmt.Sprintf(v[0].(string), v[1:]...))
+}
+
 var formatters = FormatterMap{
 	"uppercase": writer(uppercase),
 	"+1":        writer(plus1),
 	"multiword": multiword,
+	"printf":    printf,
 }
 
 var tests = []*Test{
@@ -138,6 +143,36 @@ var tests = []*Test{
 		out: "nil pointer: <nil>=77\n",
 	},
 
+	&Test{
+		in: `{"Strings" ":"} {""} {"\t\u0123 \x23\\"} {"\"}{\\"}`,
+
+		out: "Strings:  \t\u0123 \x23\\ \"}{\\",
+	},
+
+	&Test{
+		in: "{`Raw strings` `:`} {``} {`\\t\\u0123 \\x23\\`} {`}{\\`}",
+
+		out: "Raw strings:  \\t\\u0123 \\x23\\ }{\\",
+	},
+
+	&Test{
+		in: "Characters: {'a'} {'\\u0123'} {' '} {'}'} {'{'}",
+
+		out: "Characters: 97 291 32 125 123",
+	},
+
+	&Test{
+		in: "Integers: {1} {-2} {+42} {0777} {0x0a}",
+
+		out: "Integers: 1 -2 42 511 10",
+	},
+
+	&Test{
+		in: "Floats: {.5} {-.5} {1.1} {-2.2} {+42.1} {1e10} {1.2e-3} {1.2e3} {-1.2e3}",
+
+		out: "Floats: 0.5 -0.5 1.1 -2.2 42.1 1e+10 0.0012 1200 -1200",
+	},
+
 	// Method at top level
 	&Test{
 		in: "ptrmethod={PointerMethod}\n",
@@ -723,6 +758,10 @@ var formatterTests = []Test{
 		in:  "{Integer|||||}", // empty string is a valid formatter
 		out: "77",
 	},
+	{
+		in:  `{"%.02f 0x%02X" 1.1 10|printf}`,
+		out: "1.10 0x0A",
+	},
 }
 
 func TestFormatters(t *testing.T) {