qbit/go
1
0
Fork 0
mirror of https://github.com/golang/go synced 2024-11-12 05:40:22 -07:00
Code Releases Activity

scanner.go documentation

Browse Source 
R=r
DELTA=22  (8 added, 3 deleted, 11 changed)
OCL=25947
CL=25955
This commit is contained in:
Robert Griesemer 2009-03-09 17:16:42 -07:00
parent cdd9539362
commit 2aa77352fc
1 changed files with 18 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
src/lib/go/scanner.go
View File

@ -2,8 +2,6 @@
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package scanner
// A Go scanner. Takes a []byte as source which can then be
// tokenized through repeated calls to the Scan() function.
//
@ -23,6 +21,8 @@ package scanner
// println(pos, token.TokenString(tok), string(lit));
// }
// }
//
package scanner
import (
"utf8";
@ -36,12 +36,17 @@ import (
// If a syntax error is encountered, Error() is called with the exact
// token position (the byte position of the token in the source) and the
// error message.
//
type ErrorHandler interface {
Error(pos int, msg string);
}
// A Scanner holds the scanner's internal state while processing
// a given text. It can be allocated as part of another data
// structure but must be initialized via Init() before use.
// See also the package comment for a sample use.
//
type Scanner struct {
// immutable state
src []byte; // source
@ -94,13 +99,12 @@ func (S *Scanner) next() {
}
// Initialize the scanner.
// Init() prepares the scanner S to tokenize the text src. Calls to Scan()
// will use the error handler err if they encounter a syntax error. The boolean
// scan_comments specifies whether newline characters and comments should be
// recognized and returned by Scan as token.COMMENT. If scan_comments is false,
// they are treated as white space and ignored.
//
// The error handler (err) is called when an illegal token is encountered.
// If scan_comments is set to true, newline characters ('\n') and comments
// are recognized as token.COMMENT, otherwise they are treated as white
// space and ignored.
func (S *Scanner) Init(src []byte, err ErrorHandler, scan_comments bool) {
S.src = src;
S.err = err;
@ -397,10 +401,11 @@ func (S *Scanner) switch4(tok0, tok1, ch2, tok2, tok3 int) int {
}
// Scans the next token. Returns the token byte position in the source,
// its token value, and the corresponding literal text if the token is
// an identifier or basic type literal (token.IsLiteral(tok) == true).
// Scan() scans the next token and returns the token byte position in the
// source, its token value, and the corresponding literal text if the token
// is an identifier, basic type literal (token.IsLiteral(tok) == true), or
// comment.
//
func (S *Scanner) Scan() (pos, tok int, lit []byte) {
scan_again:
S.skipWhitespace();