From 9050550c12e2d09cf8f0c22a270cfa90120cdf6d Mon Sep 17 00:00:00 2001 From: Rob Pike Date: Mon, 24 Sep 2012 13:23:15 +1000 Subject: [PATCH] text/template: allow .Field access to parenthesized expressions Change the grammar so that field access is a proper operator. This introduces a new node, ChainNode, into the public (but actually internal) API of text/template/parse. For compatibility, we only use the new node type for the specific construct, which was not parseable before. Therefore this should be backward-compatible. Before, .X.Y was a token in the lexer; this CL breaks it out into .Y applied to .X. But for compatibility we mush them back together before delivering. One day we might remove that hack; it's the simple TODO in parse.go/operand. This change also provides grammatical distinction between f and (f) which might permit function values later, but not now. Fixes #3999. R=golang-dev, dsymonds, gri, rsc, mikesamuel CC=golang-dev https://golang.org/cl/6494119 --- src/pkg/html/template/escape_test.go | 5 + src/pkg/text/template/doc.go | 4 +- src/pkg/text/template/exec.go | 19 ++- src/pkg/text/template/exec_test.go | 23 ++++ src/pkg/text/template/parse/lex.go | 64 +++++++-- src/pkg/text/template/parse/lex_test.go | 34 +++-- src/pkg/text/template/parse/node.go | 51 +++++++- src/pkg/text/template/parse/parse.go | 151 ++++++++++++++-------- src/pkg/text/template/parse/parse_test.go | 7 +- 9 files changed, 269 insertions(+), 89 deletions(-) diff --git a/src/pkg/html/template/escape_test.go b/src/pkg/html/template/escape_test.go index ce12c1795c2..0d08101ecff 100644 --- a/src/pkg/html/template/escape_test.go +++ b/src/pkg/html/template/escape_test.go @@ -1539,6 +1539,11 @@ func TestEnsurePipelineContains(t *testing.T) { ".X | urlquery | html | print", []string{"urlquery", "html"}, }, + { + "{{($).X | html | print}}", + "($).X | urlquery | html | print", + []string{"urlquery", "html"}, + }, } for i, test := range tests { tmpl := template.Must(template.New("test").Parse(test.input)) diff --git a/src/pkg/text/template/doc.go b/src/pkg/text/template/doc.go index 224775c46c1..807914c24c8 100644 --- a/src/pkg/text/template/doc.go +++ b/src/pkg/text/template/doc.go @@ -148,8 +148,10 @@ An argument is a simple value, denoted by one of the following. The result is the value of invoking the function, fun(). The return types and values behave as in methods. Functions and function names are described below. - - Parentheses may be used for grouping, as in + - A parenthesized instance of one the above, for grouping. The result + may be accessed by a field or map key invocation. print (.F1 arg1) (.F2 arg2) + (.StructValuedMethod "arg").Field Arguments may evaluate to any type; if they are pointers the implementation automatically indirects to the base type when required. diff --git a/src/pkg/text/template/exec.go b/src/pkg/text/template/exec.go index 1739a86179d..5e127d7db4c 100644 --- a/src/pkg/text/template/exec.go +++ b/src/pkg/text/template/exec.go @@ -315,9 +315,15 @@ func (s *state) evalCommand(dot reflect.Value, cmd *parse.CommandNode, final ref switch n := firstWord.(type) { case *parse.FieldNode: return s.evalFieldNode(dot, n, cmd.Args, final) + case *parse.ChainNode: + return s.evalChainNode(dot, n, cmd.Args, final) case *parse.IdentifierNode: // Must be a function. return s.evalFunction(dot, n.Ident, cmd.Args, final) + case *parse.PipeNode: + // Parenthesized pipeline. The arguments are all inside the pipeline; final is ignored. + // TODO: is this right? + return s.evalPipeline(dot, n) case *parse.VariableNode: return s.evalVariableNode(dot, n, cmd.Args, final) } @@ -367,6 +373,15 @@ func (s *state) evalFieldNode(dot reflect.Value, field *parse.FieldNode, args [] return s.evalFieldChain(dot, dot, field.Ident, args, final) } +func (s *state) evalChainNode(dot reflect.Value, chain *parse.ChainNode, args []parse.Node, final reflect.Value) reflect.Value { + // (pipe).Field1.Field2 has pipe as .Node, fields as .Field. Eval the pipeline, then the fields. + pipe := s.evalArg(dot, nil, chain.Node) + if len(chain.Field) == 0 { + s.errorf("internal error: no fields in evalChainNode") + } + return s.evalFieldChain(dot, pipe, chain.Field, args, final) +} + func (s *state) evalVariableNode(dot reflect.Value, v *parse.VariableNode, args []parse.Node, final reflect.Value) reflect.Value { // $x.Field has $x as the first ident, Field as the second. Eval the var, then the fields. value := s.varValue(v.Ident[0]) @@ -521,13 +536,13 @@ func canBeNil(typ reflect.Type) bool { // validateType guarantees that the value is valid and assignable to the type. func (s *state) validateType(value reflect.Value, typ reflect.Type) reflect.Value { if !value.IsValid() { - if canBeNil(typ) { + if typ == nil || canBeNil(typ) { // An untyped nil interface{}. Accept as a proper nil value. return reflect.Zero(typ) } s.errorf("invalid value; expected %s", typ) } - if !value.Type().AssignableTo(typ) { + if typ != nil && !value.Type().AssignableTo(typ) { if value.Kind() == reflect.Interface && !value.IsNil() { value = value.Elem() if value.Type().AssignableTo(typ) { diff --git a/src/pkg/text/template/exec_test.go b/src/pkg/text/template/exec_test.go index 7f60dcafa51..0835d31f79d 100644 --- a/src/pkg/text/template/exec_test.go +++ b/src/pkg/text/template/exec_test.go @@ -340,6 +340,12 @@ var execTests = []execTest{ // Parenthesized expressions {"parens in pipeline", "{{printf `%d %d %d` (1) (2 | add 3) (add 4 (add 5 6))}}", "1 5 15", tVal, true}, + // Parenthesized expressions with field accesses + {"parens: $ in paren", "{{($).X}}", "x", tVal, true}, + {"parens: $.GetU in paren", "{{($.GetU).V}}", "v", tVal, true}, + {"parens: $ in paren in pipe", "{{($ | echo).X}}", "x", tVal, true}, + {"parens: spaces and args", `{{(makemap "up" "down" "left" "right").left}}`, "right", tVal, true}, + // If. {"if true", "{{if true}}TRUE{{end}}", "TRUE", tVal, true}, {"if false", "{{if false}}TRUE{{else}}FALSE{{end}}", "FALSE", tVal, true}, @@ -535,6 +541,21 @@ func add(args ...int) int { return sum } +func echo(arg interface{}) interface{} { + return arg +} + +func makemap(arg ...string) map[string]string { + if len(arg)%2 != 0 { + panic("bad makemap") + } + m := make(map[string]string) + for i := 0; i < len(arg); i += 2 { + m[arg[i]] = arg[i+1] + } + return m +} + func stringer(s fmt.Stringer) string { return s.String() } @@ -545,6 +566,8 @@ func testExecute(execTests []execTest, template *Template, t *testing.T) { "add": add, "count": count, "dddArg": dddArg, + "echo": echo, + "makemap": makemap, "oneArg": oneArg, "typeOf": typeOf, "vfunc": vfunc, diff --git a/src/pkg/text/template/parse/lex.go b/src/pkg/text/template/parse/lex.go index ddf4d3a54b9..dd7a71335d8 100644 --- a/src/pkg/text/template/parse/lex.go +++ b/src/pkg/text/template/parse/lex.go @@ -43,8 +43,8 @@ const ( itemComplex // complex constant (1+2i); imaginary is just a number itemColonEquals // colon-equals (':=') introducing a declaration itemEOF - itemField // alphanumeric identifier, starting with '.', possibly chained ('.x.y') - itemIdentifier // alphanumeric identifier + itemField // alphanumeric identifier starting with '.' + itemIdentifier // alphanumeric identifier not starting with '.' itemLeftDelim // left action delimiter itemLeftParen // '(' inside action itemNumber // simple number, including imaginary @@ -286,7 +286,7 @@ func lexInsideAction(l *lexer) stateFn { case r == '`': return lexRawQuote case r == '$': - return lexIdentifier + return lexVariable case r == '\'': return lexChar case r == '.': @@ -294,7 +294,7 @@ func lexInsideAction(l *lexer) stateFn { if l.pos < len(l.input) { r := l.input[l.pos] if r < '0' || '9' < r { - return lexIdentifier // itemDot comes from the keyword table. + return lexField } } fallthrough // '.' can start a number. @@ -334,15 +334,13 @@ func lexSpace(l *lexer) stateFn { return lexInsideAction } -// lexIdentifier scans an alphanumeric or field. +// lexIdentifier scans an alphanumeric. func lexIdentifier(l *lexer) stateFn { Loop: for { switch r := l.next(); { case isAlphaNumeric(r): // absorb. - case r == '.' && (l.input[l.start] == '.' || l.input[l.start] == '$'): - // field chaining; absorb into one token. default: l.backup() word := l.input[l.start:l.pos] @@ -354,8 +352,6 @@ Loop: l.emit(key[word]) case word[0] == '.': l.emit(itemField) - case word[0] == '$': - l.emit(itemVariable) case word == "true", word == "false": l.emit(itemBool) default: @@ -367,17 +363,59 @@ Loop: return lexInsideAction } +// lexField scans a field: .Alphanumeric. +// The . has been scanned. +func lexField(l *lexer) stateFn { + return lexFieldOrVariable(l, itemField) +} + +// lexVariable scans a Variable: $Alphanumeric. +// The $ has been scanned. +func lexVariable(l *lexer) stateFn { + if l.atTerminator() { // Nothing interesting follows -> "$". + l.emit(itemVariable) + return lexInsideAction + } + return lexFieldOrVariable(l, itemVariable) +} + +// lexVariable scans a field or variable: [.$]Alphanumeric. +// The . or $ has been scanned. +func lexFieldOrVariable(l *lexer, typ itemType) stateFn { + if l.atTerminator() { // Nothing interesting follows -> "." or "$". + if typ == itemVariable { + l.emit(itemVariable) + } else { + l.emit(itemDot) + } + return lexInsideAction + } + var r rune + for { + r = l.next() + if !isAlphaNumeric(r) { + l.backup() + break + } + } + if !l.atTerminator() { + return l.errorf("bad character %#U", r) + } + l.emit(typ) + return lexInsideAction +} + // atTerminator reports whether the input is at valid termination character to -// appear after an identifier. Mostly to catch cases like "$x+2" not being -// acceptable without a space, in case we decide one day to implement -// arithmetic. +// appear after an identifier. Breaks .X.Y into two pieces. Also catches cases +// like "$x+2" not being acceptable without a space, in case we decide one +// day to implement arithmetic. func (l *lexer) atTerminator() bool { r := l.peek() if isSpace(r) || isEndOfLine(r) { return true } switch r { - case eof, ',', '|', ':', ')', '(': + case eof, '.', ',', '|', ':', ')', '(': return true } // Does r start the delimiter? This can be ambiguous (with delim=="//", $x/2 will diff --git a/src/pkg/text/template/parse/lex_test.go b/src/pkg/text/template/parse/lex_test.go index d0d033406cf..d2264c991c9 100644 --- a/src/pkg/text/template/parse/lex_test.go +++ b/src/pkg/text/template/parse/lex_test.go @@ -61,10 +61,12 @@ var ( tEOF = item{itemEOF, 0, ""} tFor = item{itemIdentifier, 0, "for"} tLeft = item{itemLeftDelim, 0, "{{"} + tLpar = item{itemLeftParen, 0, "("} tPipe = item{itemPipe, 0, "|"} tQuote = item{itemString, 0, `"abc \n\t\" "`} tRange = item{itemRange, 0, "range"} tRight = item{itemRightDelim, 0, "}}"} + tRpar = item{itemRightParen, 0, ")"} tSpace = item{itemSpace, 0, " "} raw = "`" + `abc\n\t\" ` + "`" tRawQuote = item{itemRawString, 0, raw} @@ -90,11 +92,11 @@ var lexTests = []lexTest{ }}, {"parens", "{{((3))}}", []item{ tLeft, - {itemLeftParen, 0, "("}, - {itemLeftParen, 0, "("}, + tLpar, + tLpar, {itemNumber, 0, "3"}, - {itemRightParen, 0, ")"}, - {itemRightParen, 0, ")"}, + tRpar, + tRpar, tRight, tEOF, }}, @@ -160,7 +162,7 @@ var lexTests = []lexTest{ tRight, tEOF, }}, - {"dots", "{{.x . .2 .x.y}}", []item{ + {"dots", "{{.x . .2 .x.y.z}}", []item{ tLeft, {itemField, 0, ".x"}, tSpace, @@ -168,7 +170,9 @@ var lexTests = []lexTest{ tSpace, {itemNumber, 0, ".2"}, tSpace, - {itemField, 0, ".x.y"}, + {itemField, 0, ".x"}, + {itemField, 0, ".y"}, + {itemField, 0, ".z"}, tRight, tEOF, }}, @@ -202,13 +206,14 @@ var lexTests = []lexTest{ tSpace, {itemVariable, 0, "$"}, tSpace, - {itemVariable, 0, "$var.Field"}, + {itemVariable, 0, "$var"}, + {itemField, 0, ".Field"}, tSpace, {itemField, 0, ".Method"}, tRight, tEOF, }}, - {"variable invocation ", "{{$x 23}}", []item{ + {"variable invocation", "{{$x 23}}", []item{ tLeft, {itemVariable, 0, "$x"}, tSpace, @@ -261,6 +266,15 @@ var lexTests = []lexTest{ tRight, tEOF, }}, + {"field of parenthesized expression", "{{(.X).Y}}", []item{ + tLeft, + tLpar, + {itemField, 0, ".X"}, + tRpar, + {itemField, 0, ".Y"}, + tRight, + tEOF, + }}, // errors {"badchar", "#{{\x01}}", []item{ {itemText, 0, "#"}, @@ -294,14 +308,14 @@ var lexTests = []lexTest{ }}, {"unclosed paren", "{{(3}}", []item{ tLeft, - {itemLeftParen, 0, "("}, + tLpar, {itemNumber, 0, "3"}, {itemError, 0, `unclosed left paren`}, }}, {"extra right paren", "{{3)}}", []item{ tLeft, {itemNumber, 0, "3"}, - {itemRightParen, 0, ")"}, + tRpar, {itemError, 0, `unexpected right paren U+0029 ')'`}, }}, diff --git a/src/pkg/text/template/parse/node.go b/src/pkg/text/template/parse/node.go index e6d106f3b77..39e3b320bcb 100644 --- a/src/pkg/text/template/parse/node.go +++ b/src/pkg/text/template/parse/node.go @@ -34,8 +34,9 @@ func (t NodeType) Type() NodeType { const ( NodeText NodeType = iota // Plain text. - NodeAction // A simple action such as field evaluation. + NodeAction // A non-control action such as a field evaluation. NodeBool // A boolean constant. + NodeChain // A sequence of field accesses. NodeCommand // An element of a pipeline. NodeDot // The cursor, dot. nodeElse // An else action. Not added to tree. @@ -168,7 +169,7 @@ func (p *PipeNode) Copy() Node { // ActionNode holds an action (something bounded by delimiters). // Control actions have their own nodes; ActionNode represents simple -// ones such as field evaluations. +// ones such as field evaluations and parenthesized pipelines. type ActionNode struct { NodeType Line int // The line number in the input. @@ -248,11 +249,11 @@ func (i *IdentifierNode) Copy() Node { return NewIdentifier(i.Ident) } -// VariableNode holds a list of variable names. The dollar sign is -// part of the name. +// VariableNode holds a list of variable names, possibly with chained field +// accesses. The dollar sign is part of the (first) name. type VariableNode struct { NodeType - Ident []string // Variable names in lexical order. + Ident []string // Variable name and fields in lexical order. } func newVariable(ident string) *VariableNode { @@ -337,6 +338,46 @@ func (f *FieldNode) Copy() Node { return &FieldNode{NodeType: NodeField, Ident: append([]string{}, f.Ident...)} } +// ChainNode holds a term followed by a chain of field accesses (identifier starting with '.'). +// The names may be chained ('.x.y'). +// The periods are dropped from each ident. +type ChainNode struct { + NodeType + Node Node + Field []string // The identifiers in lexical order. +} + +func newChain(node Node) *ChainNode { + return &ChainNode{NodeType: NodeChain, Node: node} +} + +// Add adds the named field (which should start with a period) to the end of the chain. +func (c *ChainNode) Add(field string) { + if len(field) == 0 || field[0] != '.' { + panic("no dot in field") + } + field = field[1:] // Remove leading dot. + if field == "" { + panic("empty field") + } + c.Field = append(c.Field, field) +} + +func (c *ChainNode) String() string { + s := c.Node.String() + if _, ok := c.Node.(*PipeNode); ok { + s = "(" + s + ")" + } + for _, field := range c.Field { + s += "." + field + } + return s +} + +func (c *ChainNode) Copy() Node { + return &ChainNode{NodeType: NodeChain, Node: c.Node, Field: append([]string{}, c.Field...)} +} + // BoolNode holds a boolean constant. type BoolNode struct { NodeType diff --git a/src/pkg/text/template/parse/parse.go b/src/pkg/text/template/parse/parse.go index c52e41d166d..9e2af12ad7f 100644 --- a/src/pkg/text/template/parse/parse.go +++ b/src/pkg/text/template/parse/parse.go @@ -353,8 +353,7 @@ func (t *Tree) action() (n Node) { } // Pipeline: -// field or command -// pipeline "|" pipeline +// declarations? command ('|' command)* func (t *Tree) pipeline(context string) (pipe *PipeNode) { var decl []*VariableNode // Are there declarations? @@ -369,9 +368,6 @@ func (t *Tree) pipeline(context string) (pipe *PipeNode) { if next := t.peekNonSpace(); next.typ == itemColonEquals || (next.typ == itemChar && next.val == ",") { t.nextNonSpace() variable := newVariable(v.val) - if len(variable.Ident) != 1 { - t.errorf("illegal variable in declaration: %s", v.val) - } decl = append(decl, variable) t.vars = append(t.vars, v.val) if next.typ == itemChar && next.val == "," { @@ -400,7 +396,7 @@ func (t *Tree) pipeline(context string) (pipe *PipeNode) { } return case itemBool, itemCharConstant, itemComplex, itemDot, itemField, itemIdentifier, - itemNumber, itemNil, itemRawString, itemString, itemVariable: + itemNumber, itemNil, itemRawString, itemString, itemVariable, itemLeftParen: t.backup() pipe.append(t.command()) default: @@ -494,57 +490,29 @@ func (t *Tree) templateControl() Node { } // command: +// operand (space operand)* // space-separated arguments up to a pipeline character or right delimiter. // we consume the pipe character but leave the right delim to terminate the action. func (t *Tree) command() *CommandNode { cmd := newCommand() -Loop: for { - switch token := t.nextNonSpace(); token.typ { - case itemRightDelim, itemRightParen: - t.backup() - break Loop - case itemPipe: - break Loop - case itemLeftParen: - p := t.pipeline("parenthesized expression") - if t.nextNonSpace().typ != itemRightParen { - t.errorf("missing right paren in parenthesized expression") - } - cmd.append(p) + t.peekNonSpace() // skip leading spaces. + operand := t.operand() + if operand != nil { + cmd.append(operand) + } + switch token := t.next(); token.typ { + case itemSpace: + continue case itemError: t.errorf("%s", token.val) - case itemIdentifier: - if !t.hasFunction(token.val) { - t.errorf("function %q not defined", token.val) - } - cmd.append(NewIdentifier(token.val)) - case itemDot: - cmd.append(newDot()) - case itemNil: - cmd.append(newNil()) - case itemVariable: - cmd.append(t.useVar(token.val)) - case itemField: - cmd.append(newField(token.val)) - case itemBool: - cmd.append(newBool(token.val == "true")) - case itemCharConstant, itemComplex, itemNumber: - number, err := newNumber(token.val, token.typ) - if err != nil { - t.error(err) - } - cmd.append(number) - case itemString, itemRawString: - s, err := strconv.Unquote(token.val) - if err != nil { - t.error(err) - } - cmd.append(newString(token.val, s)) + case itemRightDelim, itemRightParen: + t.backup() + case itemPipe: default: - t.unexpected(token, "command") + t.errorf("unexpected %s in operand; missing space?", token) } - t.terminate() + break } if len(cmd.Args) == 0 { t.errorf("empty command") @@ -552,15 +520,86 @@ Loop: return cmd } -// terminate checks that the next token terminates an argument. This guarantees -// that arguments are space-separated, for example that (2)3 does not parse. -func (t *Tree) terminate() { - token := t.peek() - switch token.typ { - case itemChar, itemPipe, itemRightDelim, itemRightParen, itemSpace: - return +// operand: +// term .Field* +// An operand is a space-separated component of a command, +// a term possibly followed by field accesses. +// A nil return means the next item is not an operand. +func (t *Tree) operand() Node { + node := t.term() + if node == nil { + return nil } - t.unexpected(token, "argument list (missing space?)") + if t.peek().typ == itemField { + chain := newChain(node) + for t.peek().typ == itemField { + chain.Add(t.next().val) + } + // Compatibility with original API: If the term is of type NodeField + // or NodeVariable, just put more fields on the original. + // Otherwise, keep the Chain node. + // TODO: Switch to Chains always when we can. + switch node.Type() { + case NodeField: + node = newField(chain.String()) + case NodeVariable: + node = newVariable(chain.String()) + default: + node = chain + } + } + return node +} + +// term: +// literal (number, string, nil, boolean) +// function (identifier) +// . +// .Field +// $ +// '(' pipeline ')' +// A term is a simple "expression". +// A nil return means the next item is not a term. +func (t *Tree) term() Node { + switch token := t.nextNonSpace(); token.typ { + case itemError: + t.errorf("%s", token.val) + case itemIdentifier: + if !t.hasFunction(token.val) { + t.errorf("function %q not defined", token.val) + } + return NewIdentifier(token.val) + case itemDot: + return newDot() + case itemNil: + return newNil() + case itemVariable: + return t.useVar(token.val) + case itemField: + return newField(token.val) + case itemBool: + return newBool(token.val == "true") + case itemCharConstant, itemComplex, itemNumber: + number, err := newNumber(token.val, token.typ) + if err != nil { + t.error(err) + } + return number + case itemLeftParen: + pipe := t.pipeline("parenthesized pipeline") + if token := t.next(); token.typ != itemRightParen { + t.errorf("unclosed right paren: unexpected %s", token) + } + return pipe + case itemString, itemRawString: + s, err := strconv.Unquote(token.val) + if err != nil { + t.error(err) + } + return newString(token.val, s) + } + t.backup() + return nil } // hasFunction reports if a function name exists in the Tree's maps. diff --git a/src/pkg/text/template/parse/parse_test.go b/src/pkg/text/template/parse/parse_test.go index 4be4ca077d4..0f75c33e777 100644 --- a/src/pkg/text/template/parse/parse_test.go +++ b/src/pkg/text/template/parse/parse_test.go @@ -188,6 +188,8 @@ var parseTests = []parseTest{ `{{$x := .X | .Y}}`}, {"nested pipeline", "{{.X (.Y .Z) (.A | .B .C) (.E)}}", noError, `{{.X (.Y .Z) (.A | .B .C) (.E)}}`}, + {"field applied to parentheses", "{{(.Y .Z).Field}}", noError, + `{{(.Y .Z).Field}}`}, {"simple if", "{{if .X}}hello{{end}}", noError, `{{if .X}}"hello"{{end}}`}, {"if with else", "{{if .X}}true{{else}}false{{end}}", noError, @@ -370,8 +372,9 @@ var errorTests = []parseTest{ "{{range .X}}", hasError, `unexpected EOF`}, {"variable", - "{{$a.b := 23}}", - hasError, `illegal variable in declaration`}, + // Declare $x so it's defined, to avoid that error, and then check we don't parse a declaration. + "{{$x := 23}}{{with $x.y := 3}}{{$x 23}}{{end}}", + hasError, `unexpected ":="`}, {"multidecl", "{{$a,$b,$c := 23}}", hasError, `too many declarations`},