report an error for illegal octal numbers instead of treating them as floats

added more test cases
some capitalization cleanups

R=rsc
CC=golang-dev
https://golang.org/cl/180085

src/pkg/go/scanner/scanner.go

@@ -271,10 +271,11 @@ func (S *Scanner) scanMantissa(base int) {
 }
 
 
-func (S *Scanner) scanNumber(seen_decimal_point bool) token.Token {
+func (S *Scanner) scanNumber(pos token.Position, seenDecimalPoint bool) token.Token {
+	// digitVal(S.ch) < 10
 	tok := token.INT
 
-	if seen_decimal_point {
+	if seenDecimalPoint {
 		tok = token.FLOAT
 		S.scanMantissa(10)
 		goto exponent
@@ -289,23 +290,29 @@ func (S *Scanner) scanNumber(seen_decimal_point bool) token.Token {
 			S.scanMantissa(16)
 		} else {
 			// octal int or float
+			seenDecimalDigit := false
 			S.scanMantissa(8)
-			if digitVal(S.ch) < 10 || S.ch == '.' || S.ch == 'e' || S.ch == 'E' {
-				// float
-				tok = token.FLOAT
-				goto mantissa
+			if S.ch == '8' || S.ch == '9' {
+				// illegal octal int or float
+				seenDecimalDigit = true
+				S.scanMantissa(10)
+			}
+			if S.ch == '.' || S.ch == 'e' || S.ch == 'E' {
+				goto fraction
 			}
 			// octal int
+			if seenDecimalDigit {
+				S.error(pos, "illegal octal number")
+			}
 		}
 		goto exit
 	}
 
-mantissa:
 	// decimal int or float
 	S.scanMantissa(10)
 
+fraction:
 	if S.ch == '.' {
-		// float
 		tok = token.FLOAT
 		S.next()
 		S.scanMantissa(10)
@@ -313,7 +320,6 @@ mantissa:
 
 exponent:
 	if S.ch == 'e' || S.ch == 'E' {
-		// float
 		tok = token.FLOAT
 		S.next()
 		if S.ch == '-' || S.ch == '+' {
@@ -503,7 +509,7 @@ scanAgain:
 		}
 	case digitVal(ch) < 10:
 		insertSemi = true
-		tok = S.scanNumber(false)
+		tok = S.scanNumber(pos, false)
 	default:
 		S.next() // always make progress
 		switch ch {
@@ -532,7 +538,7 @@ scanAgain:
 		case '.':
 			if digitVal(S.ch) < 10 {
 				insertSemi = true
-				tok = S.scanNumber(true)
+				tok = S.scanNumber(pos, true)
 			} else if S.ch == '.' {
 				S.next()
 				if S.ch == '.' {

src/pkg/go/scanner/scanner_test.go

@@ -164,16 +164,16 @@ var tokens = [...]elt{
 
 const whitespace = "  \t  \n\n\n" // to separate tokens
 
-type TestErrorHandler struct {
+type testErrorHandler struct {
 	t *testing.T
 }
 
-func (h *TestErrorHandler) Error(pos token.Position, msg string) {
+func (h *testErrorHandler) Error(pos token.Position, msg string) {
 	h.t.Errorf("Error() called (msg = %s)", msg)
 }
 
 
-func NewlineCount(s string) int {
+func newlineCount(s string) int {
 	n := 0
 	for i := 0; i < len(s); i++ {
 		if s[i] == '\n' {
@@ -207,12 +207,12 @@ func TestScan(t *testing.T) {
 	for _, e := range tokens {
 		src += e.lit + whitespace
 	}
-	whitespace_linecount := NewlineCount(whitespace)
+	whitespace_linecount := newlineCount(whitespace)
 
 	// verify scan
 	index := 0
 	epos := token.Position{"", 0, 1, 1}
-	nerrors := Tokenize("", strings.Bytes(src), &TestErrorHandler{t}, ScanComments,
+	nerrors := Tokenize("", strings.Bytes(src), &testErrorHandler{t}, ScanComments,
 		func(pos token.Position, tok token.Token, litb []byte) bool {
 			e := elt{token.EOF, "", special}
 			if index < len(tokens) {
@@ -234,7 +234,7 @@ func TestScan(t *testing.T) {
 				t.Errorf("bad class for %q: got %d, expected %d", lit, tokenclass(tok), e.class)
 			}
 			epos.Offset += len(lit) + len(whitespace)
-			epos.Line += NewlineCount(lit) + whitespace_linecount
+			epos.Line += newlineCount(lit) + whitespace_linecount
 			if tok == token.COMMENT && litb[1] == '/' {
 				// correct for unaccounted '/n' in //-style comment
 				epos.Offset++
@@ -249,11 +249,6 @@ func TestScan(t *testing.T) {
 }
 
 
-func getTok(_ token.Position, tok token.Token, _ []byte) token.Token {
-	return tok
-}
-
-
 func checkSemi(t *testing.T, line string, mode uint) {
 	var S Scanner
 	S.Init("TestSemis", strings.Bytes(line), nil, mode)
@@ -485,7 +480,7 @@ func TestIllegalChars(t *testing.T) {
 	var s Scanner
 
 	const src = "*?*$*@*"
-	s.Init("", strings.Bytes(src), &TestErrorHandler{t}, AllowIllegalChars)
+	s.Init("", strings.Bytes(src), &testErrorHandler{t}, AllowIllegalChars)
 	for offs, ch := range src {
 		pos, tok, lit := s.Scan()
 		if pos.Offset != offs {
@@ -540,3 +535,74 @@ func TestStdErrorHander(t *testing.T) {
 		t.Errorf("found %d errors, expected %d", v.ErrorCount(), nerrors)
 	}
 }
+
+
+type errorCollector struct {
+	cnt int            // number of errors encountered
+	msg string         // last error message encountered
+	pos token.Position // last error position encountered
+}
+
+
+func (h *errorCollector) Error(pos token.Position, msg string) {
+	h.cnt++
+	h.msg = msg
+	h.pos = pos
+}
+
+
+func checkError(t *testing.T, src string, tok token.Token, err string) {
+	var s Scanner
+	var h errorCollector
+	s.Init("", strings.Bytes(src), &h, ScanComments)
+	_, tok0, _ := s.Scan()
+	_, tok1, _ := s.Scan()
+	if tok0 != tok {
+		t.Errorf("%q: got %s, expected %s", src, tok0, tok)
+	}
+	if tok1 != token.EOF {
+		t.Errorf("%q: got %s, expected EOF", src, tok1)
+	}
+	cnt := 0
+	if err != "" {
+		cnt = 1
+	}
+	if h.cnt != cnt {
+		t.Errorf("%q: got cnt %d, expected %d", src, h.cnt, cnt)
+	}
+	if h.msg != err {
+		t.Errorf("%q: got msg %q, expected %q", src, h.msg, err)
+	}
+	if h.pos.Offset != 0 {
+		t.Errorf("%q: got offset %d, expected 0", src, h.pos.Offset)
+	}
+}
+
+
+type srcerr struct {
+	src string
+	tok token.Token
+	err string
+}
+
+var errors = []srcerr{
+	srcerr{"\"\"", token.STRING, ""},
+	srcerr{"\"", token.STRING, "string not terminated"},
+	srcerr{"/**/", token.COMMENT, ""},
+	srcerr{"/*", token.COMMENT, "comment not terminated"},
+	srcerr{"//\n", token.COMMENT, ""},
+	srcerr{"//", token.COMMENT, "comment not terminated"},
+	srcerr{"077", token.INT, ""},
+	srcerr{"078.", token.FLOAT, ""},
+	srcerr{"07801234567.", token.FLOAT, ""},
+	srcerr{"078e0", token.FLOAT, ""},
+	srcerr{"078", token.INT, "illegal octal number"},
+	srcerr{"07800000009", token.INT, "illegal octal number"},
+}
+
+
+func TestScanErrors(t *testing.T) {
+	for _, e := range errors {
+		checkError(t, e.src, e.tok, e.err)
+	}
+}