go/scanner: remove Tokenize() - was only used in tests

R=r
CC=golang-dev
https://golang.org/cl/3415042

src/pkg/go/scanner/scanner.go

@@ -4,7 +4,18 @@
 
 // A scanner for Go source text. Takes a []byte as source which can
 // then be tokenized through repeated calls to the Scan function.
-// For a sample use of a scanner, see the implementation of Tokenize.
+// Typical use:
+//
+//	var s Scanner
+//      fset := token.NewFileSet()  // position information is relative to fset
+//	s.Init(fset, filename, src, nil /* no error handler */, 0)
+//      for {
+//		pos, tok, lit := s.Scan()
+//		if tok == token.EOF {
+//			break
+//		}
+//		// do something here with pos, tok, and lit
+//	}
 //
 package scanner
 
@@ -19,8 +30,7 @@ import (
 
 // A Scanner holds the scanner's internal state while processing
 // a given text.  It can be allocated as part of another data
-// structure but must be initialized via Init before use. For
+// structure but must be initialized via Init before use.
-// a sample use, see the implementation of Tokenize.
 //
 type Scanner struct {
 	// immutable state
@@ -692,19 +702,3 @@ scanAgain:
 	}
 	return S.file.Pos(offs), tok, S.src[offs:S.offset]
 }
-
-
-// Tokenize calls a function f with the token position, token value, and token
-// text for each token in the source src. The other parameters have the same
-// meaning as for the Init function. Tokenize keeps scanning until f returns
-// false (usually when the token value is token.EOF). The result is the number
-// of errors encountered.
-//
-func Tokenize(set *token.FileSet, filename string, src []byte, err ErrorHandler, mode uint, f func(pos token.Pos, tok token.Token, lit []byte) bool) int {
-	var s Scanner
-	s.Init(set, filename, src, err, mode)
-	for f(s.Scan()) {
-		// action happens in f
-	}
-	return s.ErrorCount
-}

src/pkg/go/scanner/scanner_test.go

@@ -227,10 +227,12 @@ func TestScan(t *testing.T) {
 	whitespace_linecount := newlineCount(whitespace)
 
 	// verify scan
+	var s Scanner
+	s.Init(fset, "", []byte(src), &testErrorHandler{t}, ScanComments)
 	index := 0
 	epos := token.Position{"", 0, 1, 1} // expected position
-	nerrors := Tokenize(fset, "", []byte(src), &testErrorHandler{t}, ScanComments,
+	for {
-		func(pos token.Pos, tok token.Token, litb []byte) bool {
+		pos, tok, litb := s.Scan()
 		e := elt{token.EOF, "", special}
 		if index < len(tokens) {
 			e = tokens[index]
@@ -259,10 +261,12 @@ func TestScan(t *testing.T) {
 			epos.Line++
 		}
 		index++
-			return tok != token.EOF
+		if tok == token.EOF {
-		})
+			break
-	if nerrors != 0 {
+		}
-		t.Errorf("found %d errors", nerrors)
+	}
+	if s.ErrorCount != 0 {
+		t.Errorf("found %d errors", s.ErrorCount)
 	}
 }
 
@@ -551,10 +555,13 @@ func TestStdErrorHander(t *testing.T) {
 		"@ @ @" // original file, line 1 again
 
 	v := new(ErrorVector)
-	nerrors := Tokenize(fset, "File1", []byte(src), v, 0,
+	var s Scanner
-		func(pos token.Pos, tok token.Token, litb []byte) bool {
+	s.Init(fset, "File1", []byte(src), v, 0)
-			return tok != token.EOF
+	for {
-		})
+		if _, tok, _ := s.Scan(); tok == token.EOF {
+			break
+		}
+	}
 
 	list := v.GetErrorList(Raw)
 	if len(list) != 9 {
@@ -574,8 +581,8 @@ func TestStdErrorHander(t *testing.T) {
 		PrintError(os.Stderr, list)
 	}
 
-	if v.ErrorCount() != nerrors {
+	if v.ErrorCount() != s.ErrorCount {
-		t.Errorf("found %d errors, expected %d", v.ErrorCount(), nerrors)
+		t.Errorf("found %d errors, expected %d", v.ErrorCount(), s.ErrorCount)
 	}
 }