From 20bdfba32590c3dcce8885df875dc56a84b2d269 Mon Sep 17 00:00:00 2001 From: Rob Findley Date: Mon, 21 Jun 2021 12:52:17 -0400 Subject: [PATCH] go/scanner: fall back to next() when encountering 0 bytes in parseIdentifier CL 308611 optimized parseIdentifier for ASCII, but inadvertently skipped error handling for 0 bytes. Don't take the optimized path when encountering 0. Fixes #46855 Change-Id: Ic584e077eb74c012611fefa20eb71ca09c81b3c7 Reviewed-on: https://go-review.googlesource.com/c/go/+/329790 Trust: Robert Findley Run-TryBot: Robert Findley TryBot-Result: Go Bot Reviewed-by: Robert Griesemer --- src/go/scanner/scanner.go | 2 +- src/go/scanner/scanner_test.go | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/go/scanner/scanner.go b/src/go/scanner/scanner.go index 29cbf39721f..f08e28cdd6b 100644 --- a/src/go/scanner/scanner.go +++ b/src/go/scanner/scanner.go @@ -373,7 +373,7 @@ func (s *Scanner) scanIdentifier() string { continue } s.rdOffset += rdOffset - if b < utf8.RuneSelf { + if 0 < b && b < utf8.RuneSelf { // Optimization: we've encountered an ASCII character that's not a letter // or number. Avoid the call into s.next() and corresponding set up. // diff --git a/src/go/scanner/scanner_test.go b/src/go/scanner/scanner_test.go index ac8d2577169..db123c32e01 100644 --- a/src/go/scanner/scanner_test.go +++ b/src/go/scanner/scanner_test.go @@ -812,6 +812,8 @@ var errors = []struct { {"//\ufeff", token.COMMENT, 2, "//\ufeff", "illegal byte order mark"}, // only first BOM is ignored {"'\ufeff" + `'`, token.CHAR, 1, "'\ufeff" + `'`, "illegal byte order mark"}, // only first BOM is ignored {`"` + "abc\ufeffdef" + `"`, token.STRING, 4, `"` + "abc\ufeffdef" + `"`, "illegal byte order mark"}, // only first BOM is ignored + {"abc\x00def", token.IDENT, 3, "abc", "illegal character NUL"}, + {"abc\x00", token.IDENT, 3, "abc", "illegal character NUL"}, } func TestScanErrors(t *testing.T) {