qbit/go
1
0
Fork 0
mirror of https://github.com/golang/go synced 2024-11-05 11:56:12 -07:00
Code Releases Activity

scanner: invalidate scanner.Position when no token is present

Browse Source 
scanner.Position is the position of the most recently
scanned token. Make sure it is invalid if there is no
token scanned and update corresponding comment. This
is particularly important when reporting errors.

Fixes #2371.

R=r
CC=golang-dev
https://golang.org/cl/5294041
This commit is contained in:
Robert Griesemer 2011-10-17 16:35:12 -07:00
parent e40d6e066a
commit df219d5197
2 changed files with 20 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
src/pkg/scanner/scanner.go
View File

@ -164,9 +164,12 @@ type Scanner struct {
// for values ch > ' '). The field may be changed at any time.
Whitespace uint64
// Current token position. The Offset, Line, and Column fields
// are set by Scan(); the Filename field is left untouched by the
// Scanner.
// Start position of most recently scanned token; set by Scan.
// Calling Init or Next invalidates the position (Line == 0).
// The Filename field is always left untouched by the Scanner.
// If an error is reported (via Error) and Position is invalid,
// the scanner is not inside a token. Call Pos to obtain an error
// position in that case.
Position
}
@ -201,6 +204,7 @@ func (s *Scanner) Init(src io.Reader) *Scanner {
s.ErrorCount = 0
s.Mode = GoTokens
s.Whitespace = GoWhitespace
s.Line = 0 // invalidate token position
return s
}
@ -302,6 +306,7 @@ func (s *Scanner) next() int {
// get the current position.
func (s *Scanner) Next() int {
s.tokPos = -1 // don't collect token text
s.Line = 0 // invalidate token position
ch := s.Peek()
s.ch = s.next()
return ch
@ -323,7 +328,11 @@ func (s *Scanner) error(msg string) {
s.Error(s, msg)
return
}
fmt.Fprintf(os.Stderr, "%s: %s\n", s.Position, msg)
pos := s.Position
if !pos.IsValid() {
pos = s.Pos()
}
fmt.Fprintf(os.Stderr, "%s: %s\n", pos, msg)
}
func (s *Scanner) scanIdentifier() int {
@ -520,6 +529,7 @@ func (s *Scanner) Scan() int {
// reset token text position
s.tokPos = -1
s.Line = 0
redo:
// skip white space

15
src/pkg/scanner/scanner_test.go
View File

@ -100,11 +100,8 @@ var tokenList = []token{
{Ident, "_abc_123_"},
{Ident, "_äöü"},
{Ident, "_本"},
// TODO for unknown reasons these fail when checking the literals
/*
token{Ident, "äöü"},
token{Ident, "本"},
*/
{Ident, "äöü"},
{Ident, "本"},
{Ident, "a۰۱۸"},
{Ident, "foo६४"},
{Ident, "bar９８７６"},
@ -365,14 +362,14 @@ func TestScanNext(t *testing.T) {
checkTok(t, s, 1, s.Scan(), Ident, "if")
checkTok(t, s, 1, s.Scan(), Ident, "a")
checkTok(t, s, 1, s.Scan(), '=', "=")
checkTok(t, s, 1, s.Next(), '=', "")
checkTok(t, s, 1, s.Next(), ' ', "")
checkTok(t, s, 1, s.Next(), 'b', "")
checkTok(t, s, 0, s.Next(), '=', "")
checkTok(t, s, 0, s.Next(), ' ', "")
checkTok(t, s, 0, s.Next(), 'b', "")
checkTok(t, s, 1, s.Scan(), Ident, "cd")
checkTok(t, s, 1, s.Scan(), '{', "{")
checkTok(t, s, 2, s.Scan(), Ident, "a")
checkTok(t, s, 2, s.Scan(), '+', "+")
checkTok(t, s, 2, s.Next(), '=', "")
checkTok(t, s, 0, s.Next(), '=', "")
checkTok(t, s, 2, s.Scan(), Ident, "c")
checkTok(t, s, 3, s.Scan(), '}', "}")
checkTok(t, s, 3, s.Scan(), -1, "")