mirror of
https://github.com/golang/go
synced 2024-11-22 00:44:39 -07:00
go/scanner: remove AllowIllegalChars mode
This mode was needed before for clients of the go/scanner that were parsing non-Go code. All those clients have been moved to scanner or have been deleted from the library. R=r CC=golang-dev https://golang.org/cl/5232051
This commit is contained in:
parent
e9ba607bf5
commit
a0d335c31d
@ -91,7 +91,6 @@ func (S *Scanner) next() {
|
||||
//
|
||||
const (
|
||||
ScanComments = 1 << iota // return comments as COMMENT tokens
|
||||
AllowIllegalChars // do not report an error for illegal chars
|
||||
InsertSemis // automatically insert semicolons
|
||||
)
|
||||
|
||||
@ -652,9 +651,7 @@ scanAgain:
|
||||
case '|':
|
||||
tok = S.switch3(token.OR, token.OR_ASSIGN, '|', token.LOR)
|
||||
default:
|
||||
if S.mode&AllowIllegalChars == 0 {
|
||||
S.error(offs, fmt.Sprintf("illegal character %#U", ch))
|
||||
}
|
||||
insertSemi = S.insertSemi // preserve insertSemi info
|
||||
}
|
||||
}
|
||||
|
@ -420,14 +420,14 @@ var lines = []string{
|
||||
|
||||
func TestSemis(t *testing.T) {
|
||||
for _, line := range lines {
|
||||
checkSemi(t, line, AllowIllegalChars|InsertSemis)
|
||||
checkSemi(t, line, AllowIllegalChars|InsertSemis|ScanComments)
|
||||
checkSemi(t, line, InsertSemis)
|
||||
checkSemi(t, line, InsertSemis|ScanComments)
|
||||
|
||||
// if the input ended in newlines, the input must tokenize the
|
||||
// same with or without those newlines
|
||||
for i := len(line) - 1; i >= 0 && line[i] == '\n'; i-- {
|
||||
checkSemi(t, line[0:i], AllowIllegalChars|InsertSemis)
|
||||
checkSemi(t, line[0:i], AllowIllegalChars|InsertSemis|ScanComments)
|
||||
checkSemi(t, line[0:i], InsertSemis)
|
||||
checkSemi(t, line[0:i], InsertSemis|ScanComments)
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -529,27 +529,6 @@ func TestInit(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestIllegalChars(t *testing.T) {
|
||||
var s Scanner
|
||||
|
||||
const src = "*?*$*@*"
|
||||
file := fset.AddFile("", fset.Base(), len(src))
|
||||
s.Init(file, []byte(src), &testErrorHandler{t}, AllowIllegalChars)
|
||||
for offs, ch := range src {
|
||||
pos, tok, lit := s.Scan()
|
||||
if poffs := file.Offset(pos); poffs != offs {
|
||||
t.Errorf("bad position for %s: got %d, expected %d", lit, poffs, offs)
|
||||
}
|
||||
if tok == token.ILLEGAL && lit != string(ch) {
|
||||
t.Errorf("bad token: got %s, expected %s", lit, string(ch))
|
||||
}
|
||||
}
|
||||
|
||||
if s.ErrorCount != 0 {
|
||||
t.Errorf("found %d errors", s.ErrorCount)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStdErrorHander(t *testing.T) {
|
||||
const src = "@\n" + // illegal character, cause an error
|
||||
"@ @\n" + // two errors on the same line
|
||||
|
Loading…
Reference in New Issue
Block a user