qbit/go
1
0
Fork 0
mirror of https://github.com/golang/go synced 2024-09-25 09:10:14 -06:00
Code Releases Activity

go/scanner: Make Init take a *token.File instead of a *token.FileSet.

Browse Source 
Until now, each scan of a file added a new file to the file set.
With this change, a file can be re-scanned using the same *token.File
w/o changing the file set. Eventually this will enable the re-use of
cached source code in godoc (for the fulltext index). At the moment,
source files are read over and over again from disk.

This is the first step in that direction.

R=r, rsc, r2
CC=golang-dev
https://golang.org/cl/4001041
This commit is contained in:
Robert Griesemer 2011-01-13 17:20:26 -08:00
parent b814cca3c3
commit bff2c207e4
8 changed files with 42 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/cmd/godoc/format.go
View File

@ -239,7 +239,9 @@ func lineSelection(text []byte) Selection {
//
func commentSelection(src []byte) Selection {
var s scanner.Scanner
file := s.Init(token.NewFileSet(), "", src, nil, scanner.ScanComments+scanner.InsertSemis)
fset := token.NewFileSet()
file := fset.AddFile("", fset.Base(), len(src))
s.Init(file, src, nil, scanner.ScanComments+scanner.InsertSemis)
return func() (seg []int) {
for {
pos, tok, lit := s.Scan()

3
src/cmd/godoc/index.go
View File

@ -817,7 +817,8 @@ func (x *Index) LookupWord(w string) (match *LookupResult, alt *AltWords) {
func isIdentifier(s string) bool {
var S scanner.Scanner
S.Init(token.NewFileSet(), "", []byte(s), nil, 0)
fset := token.NewFileSet()
S.Init(fset.AddFile("", fset.Base(), len(s)), []byte(s), nil, 0)
if _, tok, _ := S.Scan(); tok == token.IDENT {
_, tok, _ := S.Scan()
return tok == token.EOF

3
src/cmd/godoc/spec.go
View File

@ -156,7 +156,8 @@ func (p *ebnfParser) parse(fset *token.FileSet, out io.Writer, src []byte) {
// initialize ebnfParser
p.out = out
p.src = src
p.file = p.scanner.Init(fset, "", src, p, 0)
p.file = fset.AddFile("", fset.Base(), len(src))
p.scanner.Init(p.file, src, p, 0)
p.next() // initializes pos, tok, lit
// process source

2
src/pkg/ebnf/parser.go
View File

@ -177,7 +177,7 @@ func (p *parser) parse(fset *token.FileSet, filename string, src []byte) Grammar
// initialize parser
p.fset = fset
p.ErrorVector.Reset()
p.scanner.Init(fset, filename, src, p, 0)
p.scanner.Init(fset.AddFile(filename, fset.Base(), len(src)), src, p, 0)
p.next() // initializes pos, tok, lit
grammar := make(Grammar)

5
src/pkg/exp/datafmt/parser.go
View File

@ -42,8 +42,9 @@ func (p *parser) next() {
func (p *parser) init(fset *token.FileSet, filename string, src []byte) {
p.ErrorVector.Reset()
p.file = p.scanner.Init(fset, filename, src, p, scanner.AllowIllegalChars) // return '@' as token.ILLEGAL w/o error message
p.next() // initializes pos, tok, lit
p.file = fset.AddFile(filename, fset.Base(), len(src))
p.scanner.Init(p.file, src, p, scanner.AllowIllegalChars) // return '@' as token.ILLEGAL w/o error message
p.next() // initializes pos, tok, lit
p.packs = make(map[string]string)
p.rules = make(map[string]expr)
}

5
src/pkg/go/parser/parser.go
View File

@ -69,8 +69,9 @@ func scannerMode(mode uint) uint {
}
func (p *parser) init(fset *token.FileSet, filename string, src []byte, mode uint) {
p.file = p.scanner.Init(fset, filename, src, p, scannerMode(mode))
func (p *parser) init(file *token.File, src []byte, mode uint) {
p.file = file
p.scanner.Init(p.file, src, p, scannerMode(mode))
p.mode = mode
p.trace = mode&Trace != 0 // for convenience (p.trace is used frequently)
p.next()

28
src/pkg/go/scanner/scanner.go
View File

@ -96,24 +96,28 @@ const (
InsertSemis // automatically insert semicolons
)
// TODO(gri) Would it be better to simply provide *token.File to Init
// instead of fset, and filename, and then return the file?
// It could cause an error/panic if the provided file.Size()
// doesn't match len(src).
// Init prepares the scanner S to tokenize the text src. It sets the
// scanner at the beginning of the source text, adds a new file with
// the given filename to the file set fset, and returns that file.
// Init prepares the scanner S to tokenize the text src by setting the
// scanner at the beginning of src. The scanner uses the file set file
// for position information and it adds line information for each line.
// It is ok to re-use the same file when re-scanning the same file as
// line information which is already present is ignored. Init causes a
// panic if the file size does not match the src size.
//
// Calls to Scan will use the error handler err if they encounter a
// syntax error and err is not nil. Also, for each error encountered,
// the Scanner field ErrorCount is incremented by one. The mode parameter
// determines how comments, illegal characters, and semicolons are handled.
//
func (S *Scanner) Init(fset *token.FileSet, filename string, src []byte, err ErrorHandler, mode uint) *token.File {
// Note that Init may call err if there is an error in the first character
// of the file.
//
func (S *Scanner) Init(file *token.File, src []byte, err ErrorHandler, mode uint) {
// Explicitly initialize all fields since a scanner may be reused.
S.file = fset.AddFile(filename, fset.Base(), len(src))
S.dir, _ = path.Split(filename)
if file.Size() != len(src) {
panic("file size does not match src len")
}
S.file = file
S.dir, _ = path.Split(file.Name())
S.src = src
S.err = err
S.mode = mode
@ -126,8 +130,6 @@ func (S *Scanner) Init(fset *token.FileSet, filename string, src []byte, err Err
S.ErrorCount = 0
S.next()
return S.file
}

21
src/pkg/go/scanner/scanner_test.go
View File

@ -228,7 +228,7 @@ func TestScan(t *testing.T) {
// verify scan
var s Scanner
s.Init(fset, "", []byte(src), &testErrorHandler{t}, ScanComments)
s.Init(fset.AddFile("", fset.Base(), len(src)), []byte(src), &testErrorHandler{t}, ScanComments)
index := 0
epos := token.Position{"", 0, 1, 1} // expected position
for {
@ -273,7 +273,8 @@ func TestScan(t *testing.T) {
func checkSemi(t *testing.T, line string, mode uint) {
var S Scanner
file := S.Init(fset, "TestSemis", []byte(line), nil, mode)
file := fset.AddFile("TestSemis", fset.Base(), len(line))
S.Init(file, []byte(line), nil, mode)
pos, tok, lit := S.Scan()
for tok != token.EOF {
if tok == token.ILLEGAL {
@ -476,7 +477,8 @@ func TestLineComments(t *testing.T) {
// verify scan
var S Scanner
file := S.Init(fset, "dir/TestLineComments", []byte(src), nil, 0)
file := fset.AddFile("dir/TestLineComments", fset.Base(), len(src))
S.Init(file, []byte(src), nil, 0)
for _, s := range segments {
p, _, lit := S.Scan()
pos := file.Position(p)
@ -495,7 +497,8 @@ func TestInit(t *testing.T) {
// 1st init
src1 := "if true { }"
f1 := s.Init(fset, "", []byte(src1), nil, 0)
f1 := fset.AddFile("src1", fset.Base(), len(src1))
s.Init(f1, []byte(src1), nil, 0)
if f1.Size() != len(src1) {
t.Errorf("bad file size: got %d, expected %d", f1.Size(), len(src1))
}
@ -508,7 +511,8 @@ func TestInit(t *testing.T) {
// 2nd init
src2 := "go true { ]"
f2 := s.Init(fset, "", []byte(src2), nil, 0)
f2 := fset.AddFile("src2", fset.Base(), len(src2))
s.Init(f2, []byte(src2), nil, 0)
if f2.Size() != len(src2) {
t.Errorf("bad file size: got %d, expected %d", f2.Size(), len(src2))
}
@ -527,7 +531,8 @@ func TestIllegalChars(t *testing.T) {
var s Scanner
const src = "*?*$*@*"
file := s.Init(fset, "", []byte(src), &testErrorHandler{t}, AllowIllegalChars)
file := fset.AddFile("", fset.Base(), len(src))
s.Init(file, []byte(src), &testErrorHandler{t}, AllowIllegalChars)
for offs, ch := range src {
pos, tok, lit := s.Scan()
if poffs := file.Offset(pos); poffs != offs {
@ -556,7 +561,7 @@ func TestStdErrorHander(t *testing.T) {
v := new(ErrorVector)
var s Scanner
s.Init(fset, "File1", []byte(src), v, 0)
s.Init(fset.AddFile("File1", fset.Base(), len(src)), []byte(src), v, 0)
for {
if _, tok, _ := s.Scan(); tok == token.EOF {
break
@ -604,7 +609,7 @@ func (h *errorCollector) Error(pos token.Position, msg string) {
func checkError(t *testing.T, src string, tok token.Token, pos int, err string) {
var s Scanner
var h errorCollector
s.Init(fset, "", []byte(src), &h, ScanComments)
s.Init(fset.AddFile("", fset.Base(), len(src)), []byte(src), &h, ScanComments)
_, tok0, _ := s.Scan()
_, tok1, _ := s.Scan()
if tok0 != tok {