First cut at the parser for the new template package.

This is not a full grammar, but the pieces are there to implement whatever we converge on.

R=rsc
CC=golang-dev
https://golang.org/cl/4629053

src/pkg/exp/template/Makefile

@@ -7,5 +7,6 @@ include ../../../Make.inc
 TARG=template
 GOFILES=\
 	lex.go\
+	parse.go\
 
 include ../../../Make.pkg

src/pkg/exp/template/lex.go

@@ -17,22 +17,77 @@ type item struct {
 	val string
 }
 
+func (i item) String() string {
+	switch i.typ {
+	case itemEOF:
+		return "EOF"
+	case itemError:
+		return i.val
+	}
+	if len(i.val) > 10 {
+		return fmt.Sprintf("%.10q...", i.val)
+	}
+	return fmt.Sprintf("%q", i.val)
+}
+
 // itemType identifies the type of lex item.
 type itemType int
 
 const (
-	itemError      itemType = iota // error occurred; value is text of error
-	itemText                       // plain text
-	itemLeftMeta                   // left meta-string
-	itemRightMeta                  // right meta-string
-	itemPipe                       // pipe symbol
-	itemIdentifier                 // alphanumeric identifier
-	itemNumber                     // number
-	itemRawString                  // raw quoted string (includes quotes)
-	itemString                     // quoted string (includes quotes)
+	itemError itemType = iota // error occurred; value is text of error
+	itemDot                   // the cursor, spelled '.'.
 	itemEOF
+	itemElse       // else keyword
+	itemEnd        // end keyword
+	itemField      // alphanumeric identifier, starting with '.'.
+	itemIdentifier // alphanumeric identifier
+	itemIf         // if keyword
+	itemLeftMeta   // left meta-string
+	itemNumber     // number
+	itemPipe       // pipe symbol
+	itemRange      // range keyword
+	itemRawString  // raw quoted string (includes quotes)
+	itemRightMeta  // right meta-string
+	itemString     // quoted string (includes quotes)
+	itemText       // plain text
 )
 
+// Make the types prettyprint.
+var itemName = map[itemType]string{
+	itemError:      "error",
+	itemDot:        ".",
+	itemEOF:        "EOF",
+	itemElse:       "else",
+	itemEnd:        "end",
+	itemField:      "field",
+	itemIdentifier: "identifier",
+	itemIf:         "if",
+	itemLeftMeta:   "left meta",
+	itemNumber:     "number",
+	itemPipe:       "pipe",
+	itemRange:      "range",
+	itemRawString:  "raw string",
+	itemRightMeta:  "rightMeta",
+	itemString:     "string",
+	itemText:       "text",
+}
+
+func (i itemType) String() string {
+	s := itemName[i]
+	if s == "" {
+		return fmt.Sprintf("item%d", int(i))
+	}
+	return s
+}
+
+var key = map[string]itemType{
+	".":     itemDot,
+	"else":  itemElse,
+	"end":   itemEnd,
+	"if":    itemIf,
+	"range": itemRange,
+}
+
 const eof = -1
 
 // stateFn represents the state of the scanner as a function that returns the next state.
@@ -51,6 +106,7 @@ type lexer struct {
 // next returns the next rune in the input.
 func (l *lexer) next() (rune int) {
 	if l.pos >= len(l.input) {
+		l.width = 0
 		return eof
 	}
 	rune, l.width = utf8.DecodeRuneInString(l.input[l.pos:])
@@ -72,12 +128,11 @@ func (l *lexer) backup() {
 
 // emit passes an item back to the client.
 func (l *lexer) emit(t itemType) {
-	start := l.start
+	l.items <- item{t, l.input[l.start:l.pos]}
 	l.start = l.pos
-	l.items <- item{t, l.input[start:l.pos]}
 }
 
-// ignore discards whatever input is before this point.
+// ignore skips over the pending input before this point.
 func (l *lexer) ignore() {
 	l.start = l.pos
 }
@@ -106,13 +161,12 @@ func (l *lexer) lineNumber() int {
 
 // error returns an error token and terminates the scan by passing
 // back a nil pointer that will be the next state, terminating l.run.
-func (l *lexer) error(format string, args ...interface{}) stateFn {
-	format = fmt.Sprintf("%s:%d %s", l.name, l.lineNumber(), format)
+func (l *lexer) errorf(format string, args ...interface{}) stateFn {
 	l.items <- item{itemError, fmt.Sprintf(format, args...)}
 	return nil
 }
 
-// run lexes the input by execute state functions until nil.
+// run lexes the input by executing state functions until nil.
 func (l *lexer) run() {
 	for state := lexText; state != nil; {
 		state = state(l)
@@ -121,14 +175,14 @@ func (l *lexer) run() {
 }
 
 // lex launches a new scanner and returns the channel of items.
-func lex(name, input string) chan item {
+func lex(name, input string) (*lexer, chan item) {
 	l := &lexer{
 		name:  name,
 		input: input,
 		items: make(chan item),
 	}
 	go l.run()
-	return l.items
+	return l, l.items
 }
 
 // state functions
@@ -182,7 +236,7 @@ func lexInsideAction(l *lexer) stateFn {
 		}
 		switch r := l.next(); {
 		case r == eof || r == '\n':
-			return l.error("unclosed action")
+			return l.errorf("unclosed action")
 		case isSpace(r):
 			l.ignore()
 		case r == '|':
@@ -191,20 +245,29 @@ func lexInsideAction(l *lexer) stateFn {
 			return lexQuote
 		case r == '`':
 			return lexRawQuote
-		case r == '+' || r == '-' || r == '.' || ('0' <= r && r <= '9'):
+		case r == '.':
+			// special look-ahead for ".field" so we don't break l.backup().
+			if l.pos < len(l.input) {
+				r := l.input[l.pos]
+				if r < '0' || '9' < r {
+					return lexIdentifier // itemDot comes from the keyword table.
+				}
+			}
+			fallthrough // '.' can start a number.
+		case r == '+' || r == '-' || ('0' <= r && r <= '9'):
 			l.backup()
 			return lexNumber
 		case isAlphaNumeric(r):
 			l.backup()
 			return lexIdentifier
 		default:
-			return l.error("unrecognized character in action: %#U", r)
+			return l.errorf("unrecognized character in action: %#U", r)
 		}
 	}
 	return nil
 }
 
-// lexIdentifier scans an alphanumeric.
+// lexIdentifier scans an alphanumeric or field.
 func lexIdentifier(l *lexer) stateFn {
 Loop:
 	for {
@@ -213,7 +276,15 @@ Loop:
 			// absorb
 		default:
 			l.backup()
-			l.emit(itemIdentifier)
+			word := l.input[l.start:l.pos]
+			switch {
+			case key[word] != itemError:
+				l.emit(key[word])
+			case word[0] == '.':
+				l.emit(itemField)
+			default:
+				l.emit(itemIdentifier)
+			}
 			break Loop
 		}
 	}
@@ -246,7 +317,7 @@ func lexNumber(l *lexer) stateFn {
 	// Next thing mustn't be alphanumeric.
 	if isAlphaNumeric(l.peek()) {
 		l.next()
-		return l.error("bad number syntax: %q", l.input[l.start:l.pos])
+		return l.errorf("bad number syntax: %q", l.input[l.start:l.pos])
 	}
 	l.emit(itemNumber)
 	return lexInsideAction
@@ -263,7 +334,7 @@ Loop:
 			}
 			fallthrough
 		case eof, '\n':
-			return l.error("unterminated quoted string")
+			return l.errorf("unterminated quoted string")
 		case '"':
 			break Loop
 		}
@@ -278,7 +349,7 @@ Loop:
 	for {
 		switch l.next() {
 		case eof, '\n':
-			return l.error("unterminated raw quoted string")
+			return l.errorf("unterminated raw quoted string")
 		case '`':
 			break Loop
 		}

src/pkg/exp/template/lex_test.go

@@ -5,33 +5,10 @@
 package template
 
 import (
-	"fmt"
 	"reflect"
 	"testing"
 )
 
-// Make the types prettyprint.
-var itemName = map[itemType]string{
-	itemError:      "Error",
-	itemText:       "Text",
-	itemLeftMeta:   "LeftMeta",
-	itemRightMeta:  "RightMeta",
-	itemPipe:       "Pipe",
-	itemIdentifier: "Identifier",
-	itemNumber:     "Number",
-	itemRawString:  "RawString",
-	itemString:     "String",
-	itemEOF:        "EOF",
-}
-
-func (i itemType) String() string {
-	s := itemName[i]
-	if s == "" {
-		return fmt.Sprintf("item%d", int(i))
-	}
-	return s
-}
-
 type lexTest struct {
 	name  string
 	input string
@@ -42,6 +19,7 @@ var (
 	tEOF      = item{itemEOF, ""}
 	tLeft     = item{itemLeftMeta, "{{"}
 	tRight    = item{itemRightMeta, "}}"}
+	tRange    = item{itemRange, "range"}
 	tPipe     = item{itemPipe, "|"}
 	tFor      = item{itemIdentifier, "for"}
 	tQuote    = item{itemString, `"abc \n\t\" "`}
@@ -68,6 +46,25 @@ var lexTests = []lexTest{
 		tRight,
 		tEOF,
 	}},
+	{"dots", "{{.x . .2 .x.y }}", []item{
+		tLeft,
+		{itemField, ".x"},
+		{itemDot, "."},
+		{itemNumber, ".2"},
+		{itemField, ".x"},
+		{itemField, ".y"},
+		tRight,
+		tEOF,
+	}},
+	{"keywords", "{{range if else end}}", []item{
+		tLeft,
+		{itemRange, "range"},
+		{itemIf, "if"},
+		{itemElse, "else"},
+		{itemEnd, "end"},
+		tRight,
+		tEOF,
+	}},
 	{"pipeline", `intro {{echo hi 1.2 |noargs|args 1 "hi"}} outro`, []item{
 		{itemText, "intro "},
 		tLeft,
@@ -88,29 +85,35 @@ var lexTests = []lexTest{
 	{"badchar", "#{{#}}", []item{
 		{itemText, "#"},
 		tLeft,
-		{itemError, "badchar:1 unrecognized character in action: U+0023 '#'"},
+		{itemError, "unrecognized character in action: U+0023 '#'"},
 	}},
 	{"unclosed action", "{{\n}}", []item{
 		tLeft,
-		{itemError, "unclosed action:2 unclosed action"},
+		{itemError, "unclosed action"},
+	}},
+	{"EOF in action", "{{range", []item{
+		tLeft,
+		tRange,
+		{itemError, "unclosed action"},
 	}},
 	{"unclosed quote", "{{\"\n\"}}", []item{
 		tLeft,
-		{itemError, "unclosed quote:2 unterminated quoted string"},
+		{itemError, "unterminated quoted string"},
 	}},
 	{"unclosed raw quote", "{{`xx\n`}}", []item{
 		tLeft,
-		{itemError, "unclosed raw quote:2 unterminated raw quoted string"},
+		{itemError, "unterminated raw quoted string"},
 	}},
 	{"bad number", "{{3k}}", []item{
 		tLeft,
-		{itemError, `bad number:1 bad number syntax: "3k"`},
+		{itemError, `bad number syntax: "3k"`},
 	}},
 }
 
 // collect gathers the emitted items into a slice.
 func collect(t *lexTest) (items []item) {
-	for i := range lex(t.name, t.input) {
+	_, tokens := lex(t.name, t.input)
+	for i := range tokens {
 		items = append(items, i)
 	}
 	return
@@ -120,7 +123,7 @@ func TestLex(t *testing.T) {
 	for _, test := range lexTests {
 		items := collect(&test)
 		if !reflect.DeepEqual(items, test.items) {
-			t.Errorf("%s: got\n\t%v; expected\n\t%v", test.name, items, test.items)
+			t.Errorf("%s: got\n\t%v\nexpected\n\t%v", test.name, items, test.items)
 		}
 	}
 }

src/pkg/exp/template/parse.go

@@ -0,0 +1,518 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package template
+
+import (
+	"bytes"
+	"fmt"
+	"os"
+	"runtime"
+	"strconv"
+)
+
+// Template is the representation of a parsed template.
+type Template struct {
+	// TODO: At the moment, these are all internal to parsing.
+	name     string
+	root     *listNode
+	lex      *lexer
+	tokens   chan item
+	token    item // token lookahead for parser
+	havePeek bool
+}
+
+// next returns the next token.
+func (t *Template) next() item {
+	if t.havePeek {
+		t.havePeek = false
+	} else {
+		t.token = <-t.tokens
+	}
+	return t.token
+}
+
+// backup backs the input stream up one token.
+func (t *Template) backup() {
+	t.havePeek = true
+}
+
+// peek returns but does not consume the next token.
+func (t *Template) peek() item {
+	if t.havePeek {
+		return t.token
+	}
+	t.token = <-t.tokens
+	t.havePeek = true
+	return t.token
+}
+
+// A node is an element in the parse tree. The interface is trivial.
+type node interface {
+	typ() nodeType
+	String() string
+}
+
+type nodeType int
+
+func (t nodeType) typ() nodeType {
+	return t
+}
+
+const (
+	nodeText nodeType = iota
+	nodeAction
+	nodeCommand
+	nodeElse
+	nodeEnd
+	nodeField
+	nodeIdentifier
+	nodeNumber
+	nodeRange
+	nodeString
+)
+
+// Nodes.
+
+// listNode holds a sequence of nodes.
+type listNode struct {
+	nodeType
+	nodes []node
+}
+
+func newList() *listNode {
+	return &listNode{nodeType: nodeText}
+}
+
+func (l *listNode) append(n node) {
+	l.nodes = append(l.nodes, n)
+}
+
+func (l *listNode) String() string {
+	b := new(bytes.Buffer)
+	fmt.Fprint(b, "[")
+	for _, n := range l.nodes {
+		fmt.Fprint(b, n)
+	}
+	fmt.Fprint(b, "]")
+	return b.String()
+}
+
+// textNode holds plain text.
+type textNode struct {
+	nodeType
+	text string
+}
+
+func newText(text string) *textNode {
+	return &textNode{nodeType: nodeText, text: text}
+}
+
+func (t *textNode) String() string {
+	return fmt.Sprintf("(text: %q)", t.text)
+}
+
+// actionNode holds an action (something bounded by metacharacters).
+type actionNode struct {
+	nodeType
+	pipeline []*commandNode
+}
+
+func newAction() *actionNode {
+	return &actionNode{nodeType: nodeAction}
+}
+
+func (a *actionNode) append(command *commandNode) {
+	a.pipeline = append(a.pipeline, command)
+}
+
+func (a *actionNode) String() string {
+	return fmt.Sprintf("(action: %v)", a.pipeline)
+}
+
+// commandNode holds a command (a pipeline inside an evaluating action).
+type commandNode struct {
+	nodeType
+	args []node // identifier, string, or number
+}
+
+func newCommand() *commandNode {
+	return &commandNode{nodeType: nodeCommand}
+}
+
+func (c *commandNode) append(arg node) {
+	c.args = append(c.args, arg)
+}
+
+func (c *commandNode) String() string {
+	return fmt.Sprintf("(command: %v)", c.args)
+}
+
+// identifierNode holds an identifier.
+type identifierNode struct {
+	nodeType
+	ident string
+}
+
+func newIdentifier(ident string) *identifierNode {
+	return &identifierNode{nodeType: nodeIdentifier, ident: ident}
+}
+
+func (i *identifierNode) String() string {
+	return fmt.Sprintf("I=%s", i.ident)
+}
+
+// fieldNode holds a field (identifier starting with '.'). The period is dropped from the ident.
+type fieldNode struct {
+	nodeType
+	ident string
+}
+
+func newField(ident string) *fieldNode {
+	return &fieldNode{nodeType: nodeField, ident: ident[1:]} //drop period
+}
+
+func (f *fieldNode) String() string {
+	return fmt.Sprintf("F=.%s", f.ident)
+}
+
+// numberNode holds a number, signed or unsigned, integer, floating, or imaginary.
+// The value is parsed and stored under all the types that can represent the value
+// (although for simplicity -0 is not considered a valid unsigned integer).
+// This simulates in a small amount of code the behavior of Go's ideal constants.
+// TODO: booleans, complex numbers.
+type numberNode struct {
+	nodeType
+	isInt     bool // number has an integral value
+	isUint    bool // number has an unsigned integral value
+	isFloat   bool // number has a floating-point value
+	imaginary bool // number is imaginary
+	int64          // the signed integer value
+	uint64         // the unsigned integer value
+	float64        // the positive floating-point value
+	text      string
+}
+
+func newNumber(text string) (*numberNode, os.Error) {
+	n := &numberNode{nodeType: nodeNumber, text: text}
+	// Imaginary constants can only be floating-point.
+	if len(text) > 0 && text[len(text)-1] == 'i' {
+		f, err := strconv.Atof64(text[:len(text)-1])
+		if err == nil {
+			n.imaginary = true
+			n.isFloat = true
+			n.float64 = f
+			return n, nil
+		}
+	}
+	// Do integer test first so we get 0x123 etc.
+	u, err := strconv.Btoui64(text, 0) // will fail for -0; tough.
+	if err == nil {
+		n.isUint = true
+		n.uint64 = u
+	}
+	i, err := strconv.Btoi64(text, 0)
+	if err == nil {
+		n.isInt = true
+		n.int64 = i
+	}
+	// If an integer extraction succeeded, promote the float.
+	if n.isInt {
+		n.isFloat = true
+		n.float64 = float64(n.int64)
+	} else if n.isUint {
+		n.isFloat = true
+		n.float64 = float64(n.uint64)
+	} else {
+		f, err := strconv.Atof64(text)
+		if err == nil {
+			n.isFloat = true
+			n.float64 = f
+			// If a floating-point extraction succeeded, extract the int if needed.
+			if !n.isInt && float64(int64(f)) == f {
+				n.isInt = true
+				n.int64 = int64(f)
+			}
+			if !n.isUint && float64(uint64(f)) == f {
+				n.isUint = true
+				n.uint64 = uint64(f)
+			}
+		}
+	}
+	if !n.isInt && !n.isUint && !n.isFloat {
+		return nil, fmt.Errorf("illegal number syntax: %q", text)
+	}
+	return n, nil
+}
+
+func (n *numberNode) String() string {
+	return fmt.Sprintf("N=%s", n.text)
+}
+
+// stringNode holds a quoted string.
+type stringNode struct {
+	nodeType
+	text string
+}
+
+func newString(text string) *stringNode {
+	return &stringNode{nodeType: nodeString, text: text}
+}
+
+func (s *stringNode) String() string {
+	return fmt.Sprintf("S=%#q", s.text)
+}
+
+// endNode represents an {{end}} action. It is represented by a nil pointer.
+type endNode bool
+
+func newEnd() *endNode {
+	return nil
+}
+
+func (e *endNode) typ() nodeType {
+	return nodeEnd
+}
+
+func (e *endNode) String() string {
+	return "{{end}}"
+}
+
+// elseNode represents an {{else}} action. It is represented by a nil pointer.
+type elseNode bool
+
+func newElse() *elseNode {
+	return nil
+}
+
+func (e *elseNode) typ() nodeType {
+	return nodeElse
+}
+
+func (e *elseNode) String() string {
+	return "{{else}}"
+}
+
+// rangeNode represents an {{range}} action and its commands.
+type rangeNode struct {
+	nodeType
+	field    node
+	list     *listNode
+	elseList *listNode
+}
+
+func newRange(field node, list *listNode) *rangeNode {
+	return &rangeNode{nodeType: nodeRange, field: field, list: list}
+}
+
+func (r *rangeNode) String() string {
+	if r.elseList != nil {
+		return fmt.Sprintf("({{range %s}} %s {{else}} %s)", r.field, r.list, r.elseList)
+	}
+	return fmt.Sprintf("({{range %s}} %s)", r.field, r.list)
+}
+
+// Parsing.
+
+// New allocates a new template with the given name.
+func New(name string) *Template {
+	return &Template{
+		name: name,
+	}
+}
+
+// errorf formats the error and terminates processing.
+func (t *Template) errorf(format string, args ...interface{}) {
+	format = fmt.Sprintf("template: %s:%d: %s", t.name, t.lex.lineNumber(), format)
+	panic(fmt.Errorf(format, args...))
+}
+
+// error terminates processing.
+func (t *Template) error(err os.Error) {
+	t.errorf("%s", err)
+}
+
+// expect consumes the next token and guarantees it has the required type.
+func (t *Template) expect(expected itemType, context string) item {
+	token := t.next()
+	if token.typ != expected {
+		t.errorf("expected %s in %s; got %s", expected, context, token)
+	}
+	return token
+}
+
+// unexpected complains about the token and terminates processing.
+func (t *Template) unexpected(token item, context string) {
+	t.errorf("unexpected %s in %s", token, context)
+}
+
+// Parse parses the template definition string and constructs an efficient representation of the template.
+func (t *Template) Parse(s string) (err os.Error) {
+	t.lex, t.tokens = lex(t.name, s)
+	defer func() {
+		e := recover()
+		if e != nil {
+			if _, ok := e.(runtime.Error); ok {
+				panic(e)
+			}
+			err = e.(os.Error)
+		}
+		return
+	}()
+	var next node
+	t.root, next = t.itemList(true)
+	if next != nil {
+		t.errorf("unexpected %s", next)
+	}
+	return nil
+}
+
+// itemList:
+//	textOrAction*
+// Terminates at EOF and at {{end}} or {{else}}, which is returned separately.
+// The toEOF flag tells whether we expect to reach EOF.
+func (t *Template) itemList(toEOF bool) (list *listNode, next node) {
+	list = newList()
+	for t.peek().typ != itemEOF {
+		n := t.textOrAction()
+		switch n.typ() {
+		case nodeEnd, nodeElse:
+			return list, n
+		}
+		list.append(n)
+	}
+	if !toEOF {
+		t.unexpected(t.next(), "input")
+	}
+	return list, nil
+}
+
+// textOrAction:
+//	text | action
+func (t *Template) textOrAction() node {
+	switch token := t.next(); token.typ {
+	case itemText:
+		return newText(token.val)
+	case itemLeftMeta:
+		return t.action()
+	default:
+		t.unexpected(token, "input")
+	}
+	return nil
+}
+
+// Action:
+//	control
+//	command ("|" command)*
+// Left meta is past. Now get actions.
+func (t *Template) action() (n node) {
+	action := newAction()
+	switch token := t.next(); token.typ {
+	case itemRange:
+		return t.rangeControl()
+	case itemElse:
+		return t.elseControl()
+	case itemEnd:
+		return t.endControl()
+	}
+	t.backup()
+Loop:
+	for {
+		switch token := t.next(); token.typ {
+		case itemRightMeta:
+			break Loop
+		case itemIdentifier, itemField:
+			t.backup()
+			cmd, err := t.command()
+			if err != nil {
+				t.error(err)
+			}
+			action.append(cmd)
+		default:
+			t.unexpected(token, "command")
+		}
+	}
+	return action
+}
+
+// Range:
+//	{{range field}} itemList {{end}}
+//	{{range field}} itemList {{else}} itemList {{end}}
+// Range keyword is past.
+func (t *Template) rangeControl() node {
+	field := t.expect(itemField, "range")
+	t.expect(itemRightMeta, "range")
+	list, next := t.itemList(false)
+	r := newRange(newField(field.val), list)
+	switch next.typ() {
+	case nodeEnd: //done
+	case nodeElse:
+		elseList, next := t.itemList(false)
+		if next.typ() != nodeEnd {
+			t.errorf("expected end; found %s", next)
+		}
+		r.elseList = elseList
+	}
+	return r
+}
+
+// End:
+//	{{end}}
+// End keyword is past.
+func (t *Template) endControl() node {
+	t.expect(itemRightMeta, "end")
+	return newEnd()
+}
+
+// Else:
+//	{{else}}
+// Else keyword is past.
+func (t *Template) elseControl() node {
+	t.expect(itemRightMeta, "else")
+	return newElse()
+}
+
+// command:
+// space-separated arguments up to a pipeline character or right metacharacter.
+// we consume the pipe character but leave the right meta to terminate the action.
+func (t *Template) command() (*commandNode, os.Error) {
+	cmd := newCommand()
+Loop:
+	for {
+		switch token := t.next(); token.typ {
+		case itemRightMeta:
+			t.backup()
+			break Loop
+		case itemPipe:
+			break Loop
+		case itemError:
+			return nil, os.NewError(token.val)
+		case itemIdentifier:
+			cmd.append(newIdentifier(token.val))
+		case itemField:
+			cmd.append(newField(token.val))
+		case itemNumber:
+			if len(cmd.args) == 0 {
+				t.errorf("command cannot be %q", token.val)
+			}
+			number, err := newNumber(token.val)
+			if err != nil {
+				t.error(err)
+			}
+			cmd.append(number)
+		case itemString, itemRawString:
+			if len(cmd.args) == 0 {
+				t.errorf("command cannot be %q", token.val)
+			}
+			s, err := strconv.Unquote(token.val)
+			if err != nil {
+				return nil, err
+			}
+			cmd.append(newString(s))
+		default:
+			t.unexpected(token, "command")
+		}
+	}
+	return cmd, nil
+}

src/pkg/exp/template/parse_test.go

@@ -0,0 +1,175 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package template
+
+import (
+	"fmt"
+	"testing"
+)
+
+const dumpErrors = true
+
+type numberTest struct {
+	text      string
+	isInt     bool
+	isUint    bool
+	isFloat   bool
+	imaginary bool
+	int64
+	uint64
+	float64
+}
+
+var numberTests = []numberTest{
+	// basics
+	{"0", true, true, true, false, 0, 0, 0},
+	{"73", true, true, true, false, 73, 73, 73},
+	{"-73", true, false, true, false, -73, 0, -73},
+	{"+73", true, false, true, false, 73, 0, 73},
+	{"100", true, true, true, false, 100, 100, 100},
+	{"1e9", true, true, true, false, 1e9, 1e9, 1e9},
+	{"-1e9", true, false, true, false, -1e9, 0, -1e9},
+	{"-1.2", false, false, true, false, 0, 0, -1.2},
+	{"1e19", false, true, true, false, 0, 1e19, 1e19},
+	{"-1e19", false, false, true, false, 0, 0, -1e19},
+	{"4i", false, false, true, true, 0, 0, 4},
+	// funny bases
+	{"0123", true, true, true, false, 0123, 0123, 0123},
+	{"-0x0", true, false, true, false, 0, 0, 0},
+	{"0xdeadbeef", true, true, true, false, 0xdeadbeef, 0xdeadbeef, 0xdeadbeef},
+	// some broken syntax
+	{text: "+-2"},
+	{text: "0x123."},
+	{text: "1e."},
+	{text: "0xi."},
+}
+
+func TestNumberParse(t *testing.T) {
+	for _, test := range numberTests {
+		n, err := newNumber(test.text)
+		ok := test.isInt || test.isUint || test.isFloat
+		if ok && err != nil {
+			t.Errorf("unexpected error for %q", test.text)
+			continue
+		}
+		if !ok && err == nil {
+			t.Errorf("expected error for %q", test.text)
+			continue
+		}
+		if !ok {
+			continue
+		}
+		if n.imaginary != test.imaginary {
+			t.Errorf("imaginary incorrect for %q; should be %t", test.text, test.imaginary)
+		}
+		if test.isInt {
+			if !n.isInt {
+				t.Errorf("expected integer for %q", test.text)
+			}
+			if n.int64 != test.int64 {
+				t.Errorf("int64 for %q should be %d is %d", test.text, test.int64, n.int64)
+			}
+		} else if n.isInt {
+			t.Errorf("did not expect integer for %q", test.text)
+		}
+		if test.isUint {
+			if !n.isUint {
+				t.Errorf("expected unsigned integer for %q", test.text)
+			}
+			if n.uint64 != test.uint64 {
+				t.Errorf("uint64 for %q should be %d is %d", test.text, test.uint64, n.uint64)
+			}
+		} else if n.isUint {
+			t.Errorf("did not expect unsigned integer for %q", test.text)
+		}
+		if test.isFloat {
+			if !n.isFloat {
+				t.Errorf("expected float for %q", test.text)
+			}
+			if n.float64 != test.float64 {
+				t.Errorf("float64 for %q should be %g is %g", test.text, test.float64, n.float64)
+			}
+		} else if n.isFloat {
+			t.Errorf("did not expect float for %q", test.text)
+		}
+	}
+}
+
+func num(s string) *numberNode {
+	n, err := newNumber(s)
+	if err != nil {
+		panic(err)
+	}
+	return n
+}
+
+type parseTest struct {
+	name   string
+	input  string
+	ok     bool
+	result string
+}
+
+const (
+	noError  = true
+	hasError = false
+)
+
+var parseTests = []parseTest{
+	{"empty", "", noError,
+		`[]`},
+	{"spaces", " \t\n", noError,
+		`[(text: " \t\n")]`},
+	{"text", "some text", noError,
+		`[(text: "some text")]`},
+	{"emptyMeta", "{{}}", noError,
+		`[(action: [])]`},
+	{"simple command", "{{hello}}", noError,
+		`[(action: [(command: [I=hello])])]`},
+	{"multi-word command", "{{hello world}}", noError,
+		`[(action: [(command: [I=hello I=world])])]`},
+	{"multi-word command with number", "{{hello 80}}", noError,
+		`[(action: [(command: [I=hello N=80])])]`},
+	{"multi-word command with string", "{{hello `quoted text`}}", noError,
+		"[(action: [(command: [I=hello S=`quoted text`])])]"},
+	{"pipeline", "{{hello|world}}", noError,
+		`[(action: [(command: [I=hello]) (command: [I=world])])]`},
+	{"simple range", "{{range .x}}hello{{end}}", noError,
+		`[({{range F=.x}} [(text: "hello")])]`},
+	{"nested range", "{{range .x}}hello{{range .y}}goodbye{{end}}{{end}}", noError,
+		`[({{range F=.x}} [(text: "hello")({{range F=.y}} [(text: "goodbye")])])]`},
+	{"range with else", "{{range .x}}true{{else}}false{{end}}", noError,
+		`[({{range F=.x}} [(text: "true")] {{else}} [(text: "false")])]`},
+	// Errors.
+	{"unclosed action", "hello{{range", hasError, ""},
+	{"not a field", "hello{{range x}}{{end}}", hasError, ""},
+	{"missing end", "hello{{range .x}}", hasError, ""},
+	{"missing end after else", "hello{{range .x}}{{else}}", hasError, ""},
+}
+
+func TestParse(t *testing.T) {
+	for _, test := range parseTests {
+		tmpl := New(test.name)
+		err := tmpl.Parse(test.input)
+		switch {
+		case err == nil && !test.ok:
+			t.Errorf("%q: expected error; got none", test.name)
+			continue
+		case err != nil && test.ok:
+			t.Errorf("%q: unexpected error: %v", test.name, err)
+			continue
+		case err != nil && !test.ok:
+			// expected error, got one
+			if dumpErrors {
+				fmt.Printf("%s: %s\n\t%s\n", test.name, test.input, err)
+			}
+			continue
+		}
+		result := tmpl.root.String()
+		if result != test.result {
+			t.Errorf("%s=(%q): got\n\t%v\nexpected\n\t%v", test.name, test.input, result, test.result)
+		}
+	}
+}