qbit/go
1
0
Fork 0
mirror of https://github.com/golang/go synced 2024-11-25 02:07:58 -07:00
Code Releases Activity

go/scanner: remove AllowIllegalChars mode

Browse Source 
This mode was needed before for clients of
the go/scanner that were parsing non-Go code.
All those clients have been moved to scanner
or have been deleted from the library.

R=r
CC=golang-dev
https://golang.org/cl/5232051
This commit is contained in:
Robert Griesemer 2011-10-11 22:28:56 -07:00
parent e9ba607bf5
commit a0d335c31d
2 changed files with 7 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
src/pkg/go/scanner/scanner.go
View File

@ -90,9 +90,8 @@ func (S *Scanner) next() {
// They control scanner behavior. // They control scanner behavior.
// //
const ( const (
ScanComments = 1 << iota // return comments as COMMENT tokens ScanComments = 1 << iota // return comments as COMMENT tokens
AllowIllegalChars // do not report an error for illegal chars InsertSemis // automatically insert semicolons
InsertSemis // automatically insert semicolons
) )
// Init prepares the scanner S to tokenize the text src by setting the // Init prepares the scanner S to tokenize the text src by setting the
@ -652,9 +651,7 @@ scanAgain:
case '|': case '|':
tok = S.switch3(token.OR, token.OR_ASSIGN, '|', token.LOR) tok = S.switch3(token.OR, token.OR_ASSIGN, '|', token.LOR)
default: default:
if S.mode&AllowIllegalChars == 0 { S.error(offs, fmt.Sprintf("illegal character %#U", ch))
S.error(offs, fmt.Sprintf("illegal character %#U", ch))
}
insertSemi = S.insertSemi // preserve insertSemi info insertSemi = S.insertSemi // preserve insertSemi info
} }
} }

29
src/pkg/go/scanner/scanner_test.go
View File

@ -420,14 +420,14 @@ var lines = []string{
func TestSemis(t *testing.T) { func TestSemis(t *testing.T) {
for _, line := range lines { for _, line := range lines {
checkSemi(t, line, AllowIllegalChars|InsertSemis) checkSemi(t, line, InsertSemis)
checkSemi(t, line, AllowIllegalChars|InsertSemis|ScanComments) checkSemi(t, line, InsertSemis|ScanComments)
// if the input ended in newlines, the input must tokenize the // if the input ended in newlines, the input must tokenize the
// same with or without those newlines // same with or without those newlines
for i := len(line) - 1; i >= 0 && line[i] == '\n'; i-- { for i := len(line) - 1; i >= 0 && line[i] == '\n'; i-- {
checkSemi(t, line[0:i], AllowIllegalChars|InsertSemis) checkSemi(t, line[0:i], InsertSemis)
checkSemi(t, line[0:i], AllowIllegalChars|InsertSemis|ScanComments) checkSemi(t, line[0:i], InsertSemis|ScanComments)
} }
} }
} }
@ -529,27 +529,6 @@ func TestInit(t *testing.T) {
} }
} }
func TestIllegalChars(t *testing.T) {
var s Scanner
const src = "*?*$*@*"
file := fset.AddFile("", fset.Base(), len(src))
s.Init(file, []byte(src), &testErrorHandler{t}, AllowIllegalChars)
for offs, ch := range src {
pos, tok, lit := s.Scan()
if poffs := file.Offset(pos); poffs != offs {
t.Errorf("bad position for %s: got %d, expected %d", lit, poffs, offs)
}
if tok == token.ILLEGAL && lit != string(ch) {
t.Errorf("bad token: got %s, expected %s", lit, string(ch))
}
}
if s.ErrorCount != 0 {
t.Errorf("found %d errors", s.ErrorCount)
}
}
func TestStdErrorHander(t *testing.T) { func TestStdErrorHander(t *testing.T) {
const src = "@\n" + // illegal character, cause an error const src = "@\n" + // illegal character, cause an error
"@ @\n" + // two errors on the same line "@ @\n" + // two errors on the same line