go/scanner: faster scanning

Optimize some common cases.

benchmark            old ns/op    new ns/op    delta
BenchmarkScanFile       718907       667960   -7.09%

benchmark             old MB/s     new MB/s  speedup
BenchmarkScanFile        23.03        25.51    1.11x

R=r
CC=golang-dev
https://golang.org/cl/6454150

src/pkg/go/scanner/scanner.go

@@ -572,14 +572,25 @@ scanAgain:
 	// determine token value
 	insertSemi := false
 	switch ch := s.ch; {
-	case isLetter(ch):
+	case 'a' <= ch && ch <= 'z':
+		// literals start with a lower-case letter
 		lit = s.scanIdentifier()
-		tok = token.Lookup(lit)
+		if len(lit) > 1 {
-		switch tok {
+			// keywords are longer than one letter - avoid lookup otherwise
-		case token.IDENT, token.BREAK, token.CONTINUE, token.FALLTHROUGH, token.RETURN:
+			tok = token.Lookup(lit)
+			switch tok {
+			case token.IDENT, token.BREAK, token.CONTINUE, token.FALLTHROUGH, token.RETURN:
+				insertSemi = true
+			}
+		} else {
 			insertSemi = true
+			tok = token.IDENT
 		}
-	case digitVal(ch) < 10:
+	case 'A' <= ch && ch <= 'Z' || ch == '_':
+		insertSemi = true
+		tok = token.IDENT
+		lit = s.scanIdentifier()
+	case '0' <= ch && ch <= '9':
 		insertSemi = true
 		tok, lit = s.scanNumber(false)
 	default:
@@ -612,7 +623,7 @@ scanAgain:
 		case ':':
 			tok = s.switch2(token.COLON, token.DEFINE)
 		case '.':
-			if digitVal(s.ch) < 10 {
+			if '0' <= s.ch && s.ch <= '9' {
 				insertSemi = true
 				tok, lit = s.scanNumber(true)
 			} else if s.ch == '.' {
@@ -704,10 +715,17 @@ scanAgain:
 		case '|':
 			tok = s.switch3(token.OR, token.OR_ASSIGN, '|', token.LOR)
 		default:
-			s.error(s.file.Offset(pos), fmt.Sprintf("illegal character %#U", ch))
+			if isLetter(ch) {
-			insertSemi = s.insertSemi // preserve insertSemi info
+				// handle any letters we might have missed
-			tok = token.ILLEGAL
+				insertSemi = true
-			lit = string(ch)
+				tok = token.IDENT
+				s.scanIdentifier()
+			} else {
+				s.error(s.file.Offset(pos), fmt.Sprintf("illegal character %#U", ch))
+				insertSemi = s.insertSemi // preserve insertSemi info
+				tok = token.ILLEGAL
+				lit = string(ch)
+			}
 		}
 	}
 	if s.mode&dontInsertSemis == 0 {

src/pkg/go/scanner/scanner_test.go

@@ -6,6 +6,7 @@ package scanner
 
 import (
 	"go/token"
+	"io/ioutil"
 	"os"
 	"path/filepath"
 	"runtime"
@@ -705,7 +706,7 @@ func BenchmarkScan(b *testing.B) {
 	file := fset.AddFile("", fset.Base(), len(source))
 	var s Scanner
 	b.StartTimer()
-	for i := b.N - 1; i >= 0; i-- {
+	for i := 0; i < b.N; i++ {
 		s.Init(file, source, nil, ScanComments)
 		for {
 			_, tok, _ := s.Scan()
@@ -715,3 +716,26 @@ func BenchmarkScan(b *testing.B) {
 		}
 	}
 }
+
+func BenchmarkScanFile(b *testing.B) {
+	b.StopTimer()
+	const filename = "scanner.go"
+	src, err := ioutil.ReadFile(filename)
+	if err != nil {
+		panic(err)
+	}
+	fset := token.NewFileSet()
+	file := fset.AddFile(filename, fset.Base(), len(src))
+	b.SetBytes(int64(len(src)))
+	var s Scanner
+	b.StartTimer()
+	for i := 0; i < b.N; i++ {
+		s.Init(file, src, nil, ScanComments)
+		for {
+			_, tok, _ := s.Scan()
+			if tok == token.EOF {
+				break
+			}
+		}
+	}
+}