mirror of
https://github.com/golang/go
synced 2024-11-21 22:24:40 -07:00
godoc: use scanner instead of go/scanner for ebnf processing of spec
Also: Fewer calls to flush for faster processing (once per identifier or error instead of once per token). R=golang-dev, r CC=golang-dev https://golang.org/cl/5236041
This commit is contained in:
parent
af2070598f
commit
f198bbc850
@ -680,7 +680,7 @@ func serveHTMLDoc(w http.ResponseWriter, r *http.Request, abspath, relpath strin
|
||||
// if it's the language spec, add tags to EBNF productions
|
||||
if strings.HasSuffix(abspath, "go_spec.html") {
|
||||
var buf bytes.Buffer
|
||||
linkify(&buf, src)
|
||||
Linkify(&buf, src)
|
||||
src = buf.Bytes()
|
||||
}
|
||||
|
||||
|
@ -2,118 +2,103 @@
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package main
|
||||
|
||||
// This file contains the mechanism to "linkify" html source
|
||||
// text containing EBNF sections (as found in go_spec.html).
|
||||
// The result is the input source text with the EBNF sections
|
||||
// modified such that identifiers are linked to the respective
|
||||
// definitions.
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"go/scanner"
|
||||
"go/token"
|
||||
"io"
|
||||
"scanner"
|
||||
)
|
||||
|
||||
type ebnfParser struct {
|
||||
out io.Writer // parser output
|
||||
src []byte // parser source
|
||||
file *token.File // for position information
|
||||
out io.Writer // parser output
|
||||
src []byte // parser input
|
||||
scanner scanner.Scanner
|
||||
prev int // offset of previous token
|
||||
pos token.Pos // token position
|
||||
tok token.Token // one token look-ahead
|
||||
lit string // token literal
|
||||
prev int // offset of previous token
|
||||
pos int // offset of current token
|
||||
tok int // one token look-ahead
|
||||
lit string // token literal
|
||||
}
|
||||
|
||||
func (p *ebnfParser) flush() {
|
||||
offs := p.file.Offset(p.pos)
|
||||
p.out.Write(p.src[p.prev:offs])
|
||||
p.prev = offs
|
||||
p.out.Write(p.src[p.prev:p.pos])
|
||||
p.prev = p.pos
|
||||
}
|
||||
|
||||
func (p *ebnfParser) next() {
|
||||
if p.pos.IsValid() {
|
||||
p.flush()
|
||||
}
|
||||
p.pos, p.tok, p.lit = p.scanner.Scan()
|
||||
if p.tok.IsKeyword() {
|
||||
// TODO Should keyword mapping always happen outside scanner?
|
||||
// Or should there be a flag to scanner to enable keyword mapping?
|
||||
p.tok = token.IDENT
|
||||
}
|
||||
p.tok = p.scanner.Scan()
|
||||
p.pos = p.scanner.Position.Offset
|
||||
p.lit = p.scanner.TokenText()
|
||||
}
|
||||
|
||||
func (p *ebnfParser) Error(pos token.Position, msg string) {
|
||||
fmt.Fprintf(p.out, `<span class="alert">error: %s</span>`, msg)
|
||||
func (p *ebnfParser) printf(format string, args ...interface{}) {
|
||||
p.flush()
|
||||
fmt.Fprintf(p.out, format, args...)
|
||||
}
|
||||
|
||||
func (p *ebnfParser) errorExpected(pos token.Pos, msg string) {
|
||||
msg = "expected " + msg
|
||||
if pos == p.pos {
|
||||
// the error happened at the current position;
|
||||
// make the error message more specific
|
||||
msg += ", found '" + p.tok.String() + "'"
|
||||
if p.tok.IsLiteral() {
|
||||
msg += " " + p.lit
|
||||
}
|
||||
}
|
||||
p.Error(p.file.Position(pos), msg)
|
||||
func (p *ebnfParser) errorExpected(msg string) {
|
||||
p.printf(`<span class="highlight">error: expected %s, found %s</span>`, msg, scanner.TokenString(p.tok))
|
||||
}
|
||||
|
||||
func (p *ebnfParser) expect(tok token.Token) token.Pos {
|
||||
pos := p.pos
|
||||
func (p *ebnfParser) expect(tok int) {
|
||||
if p.tok != tok {
|
||||
p.errorExpected(pos, "'"+tok.String()+"'")
|
||||
p.errorExpected(scanner.TokenString(tok))
|
||||
}
|
||||
p.next() // make progress in any case
|
||||
return pos
|
||||
}
|
||||
|
||||
func (p *ebnfParser) parseIdentifier(def bool) {
|
||||
name := p.lit
|
||||
p.expect(token.IDENT)
|
||||
if def {
|
||||
fmt.Fprintf(p.out, `<a id="%s">%s</a>`, name, name)
|
||||
if p.tok == scanner.Ident {
|
||||
name := p.lit
|
||||
if def {
|
||||
p.printf(`<a id="%s">%s</a>`, name, name)
|
||||
} else {
|
||||
p.printf(`<a href="#%s" class="noline">%s</a>`, name, name)
|
||||
}
|
||||
p.prev += len(name) // skip identifier when printing next time
|
||||
p.next()
|
||||
} else {
|
||||
fmt.Fprintf(p.out, `<a href="#%s" class="noline">%s</a>`, name, name)
|
||||
p.expect(scanner.Ident)
|
||||
}
|
||||
p.prev += len(name) // skip identifier when calling flush
|
||||
}
|
||||
|
||||
func (p *ebnfParser) parseTerm() bool {
|
||||
switch p.tok {
|
||||
case token.IDENT:
|
||||
case scanner.Ident:
|
||||
p.parseIdentifier(false)
|
||||
|
||||
case token.STRING:
|
||||
case scanner.String:
|
||||
p.next()
|
||||
const ellipsis = "…" // U+2026, the horizontal ellipsis character
|
||||
if p.tok == token.ILLEGAL && p.lit == ellipsis {
|
||||
const ellipsis = '…' // U+2026, the horizontal ellipsis character
|
||||
if p.tok == ellipsis {
|
||||
p.next()
|
||||
p.expect(token.STRING)
|
||||
p.expect(scanner.String)
|
||||
}
|
||||
|
||||
case token.LPAREN:
|
||||
case '(':
|
||||
p.next()
|
||||
p.parseExpression()
|
||||
p.expect(token.RPAREN)
|
||||
p.expect(')')
|
||||
|
||||
case token.LBRACK:
|
||||
case '[':
|
||||
p.next()
|
||||
p.parseExpression()
|
||||
p.expect(token.RBRACK)
|
||||
p.expect(']')
|
||||
|
||||
case token.LBRACE:
|
||||
case '{':
|
||||
p.next()
|
||||
p.parseExpression()
|
||||
p.expect(token.RBRACE)
|
||||
p.expect('}')
|
||||
|
||||
default:
|
||||
return false
|
||||
return false // no term found
|
||||
}
|
||||
|
||||
return true
|
||||
@ -121,7 +106,7 @@ func (p *ebnfParser) parseTerm() bool {
|
||||
|
||||
func (p *ebnfParser) parseSequence() {
|
||||
if !p.parseTerm() {
|
||||
p.errorExpected(p.pos, "term")
|
||||
p.errorExpected("term")
|
||||
}
|
||||
for p.parseTerm() {
|
||||
}
|
||||
@ -130,7 +115,7 @@ func (p *ebnfParser) parseSequence() {
|
||||
func (p *ebnfParser) parseExpression() {
|
||||
for {
|
||||
p.parseSequence()
|
||||
if p.tok != token.OR {
|
||||
if p.tok != '|' {
|
||||
break
|
||||
}
|
||||
p.next()
|
||||
@ -139,23 +124,22 @@ func (p *ebnfParser) parseExpression() {
|
||||
|
||||
func (p *ebnfParser) parseProduction() {
|
||||
p.parseIdentifier(true)
|
||||
p.expect(token.ASSIGN)
|
||||
if p.tok != token.PERIOD {
|
||||
p.expect('=')
|
||||
if p.tok != '.' {
|
||||
p.parseExpression()
|
||||
}
|
||||
p.expect(token.PERIOD)
|
||||
p.expect('.')
|
||||
}
|
||||
|
||||
func (p *ebnfParser) parse(fset *token.FileSet, out io.Writer, src []byte) {
|
||||
func (p *ebnfParser) parse(out io.Writer, src []byte) {
|
||||
// initialize ebnfParser
|
||||
p.out = out
|
||||
p.src = src
|
||||
p.file = fset.AddFile("", fset.Base(), len(src))
|
||||
p.scanner.Init(p.file, src, p, scanner.AllowIllegalChars)
|
||||
p.scanner.Init(bytes.NewBuffer(src))
|
||||
p.next() // initializes pos, tok, lit
|
||||
|
||||
// process source
|
||||
for p.tok != token.EOF {
|
||||
for p.tok != scanner.EOF {
|
||||
p.parseProduction()
|
||||
}
|
||||
p.flush()
|
||||
@ -167,32 +151,29 @@ var (
|
||||
closeTag = []byte(`</pre>`)
|
||||
)
|
||||
|
||||
func linkify(out io.Writer, src []byte) {
|
||||
fset := token.NewFileSet()
|
||||
func Linkify(out io.Writer, src []byte) {
|
||||
for len(src) > 0 {
|
||||
n := len(src)
|
||||
|
||||
// i: beginning of EBNF text (or end of source)
|
||||
i := bytes.Index(src, openTag)
|
||||
if i < 0 {
|
||||
i = n - len(openTag)
|
||||
i = len(src) - len(openTag)
|
||||
}
|
||||
i += len(openTag)
|
||||
|
||||
// j: end of EBNF text (or end of source)
|
||||
j := bytes.Index(src[i:n], closeTag) // close marker
|
||||
j := bytes.Index(src[i:], closeTag) // close marker
|
||||
if j < 0 {
|
||||
j = n - i
|
||||
j = len(src) - i
|
||||
}
|
||||
j += i
|
||||
|
||||
// write text before EBNF
|
||||
out.Write(src[0:i])
|
||||
// parse and write EBNF
|
||||
// process EBNF
|
||||
var p ebnfParser
|
||||
p.parse(fset, out, src[i:j])
|
||||
p.parse(out, src[i:j])
|
||||
|
||||
// advance
|
||||
src = src[j:n]
|
||||
src = src[j:]
|
||||
}
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user