cmd/yacc: allow utf-8 token values

Also clean up the code and allow \U.
Fixes #3007.

R=golang-dev, rsc, 0xjnml
CC=golang-dev
https://golang.org/cl/6492105

src/cmd/yacc/units.y

@@ -76,7 +76,7 @@ var vflag bool
 
 %type	<node>	prog expr expr0 expr1 expr2 expr3 expr4
 
-%token	<vval>	VAL
+%token	<vval>	VÄL // dieresis to test UTF-8
 %token	<vvar>	VAR
 %token	<numb>	_SUP // tests leading underscore in token name
 %%
@@ -199,7 +199,7 @@ expr0:
 			$$ = $1.node
 		}
 	}
-|	VAL
+|	VÄL
 	{
 		$$ = one
 		$$.vval = $1
@@ -275,7 +275,7 @@ numb:
 		f = 0
 	}
 	yylval.vval = f
-	return VAL
+	return VÄL
 }
 
 func (UnitsLex) Error(s string) {

src/cmd/yacc/yacc.go

@@ -52,6 +52,7 @@ import (
 	"os"
 	"strings"
 	"unicode"
+	"unicode/utf8"
 )
 
 // the following are adjustable
@@ -326,7 +327,6 @@ var resrv = []Resrv{
 var zznewstate = 0
 
 const EOF = -1
-const UTFmax = 0x3f
 
 func main() {
 
@@ -719,8 +719,8 @@ func moreprod() {
 }
 
 //
-// define s to be a terminal if t=0
-// or a nonterminal if t=1
+// define s to be a terminal if nt==0
+// or a nonterminal if nt==1
 //
 func defin(nt int, s string) int {
 	val := 0
@@ -753,12 +753,18 @@ func defin(nt int, s string) int {
 
 	// establish value for token
 	// single character literal
-	if s[0] == ' ' && len(s) == 1+1 {
-		val = int(s[1])
-	} else if s[0] == ' ' && s[1] == '\\' { // escape sequence
-		if len(s) == 2+1 {
+	if s[0] == ' ' {
+		s = s[1:]
+		r, size := utf8.DecodeRuneInString(s)
+		if r == utf8.RuneError && size == 1 {
+			errorf("invalid UTF-8 sequence %q", s)
+		}
+		val = int(r)
+		if val == '\\' { // escape sequence
+			switch {
+			case len(s) == 2:
 				// single character escape sequence
-			switch s[2] {
+				switch s[1] {
 				case '\'':
 					val = '\''
 				case '"':
@@ -769,6 +775,8 @@ func defin(nt int, s string) int {
 					val = '\a'
 				case 'b':
 					val = '\b'
+				case 'f':
+					val = '\f'
 				case 'n':
 					val = '\n'
 				case 'r':
@@ -778,11 +786,12 @@ func defin(nt int, s string) int {
 				case 'v':
 					val = '\v'
 				default:
-				errorf("invalid escape %v", s[1:3])
+					errorf("invalid escape %s", s)
 				}
-		} else if s[2] == 'u' && len(s) == 2+1+4 { // \unnnn sequence
+			case s[1] == 'u' && len(s) == 2+4, // \unnnn sequence
+				s[1] == 'U' && len(s) == 2+8: // \Unnnnnnnn sequence
 				val = 0
-			s = s[3:]
+				s = s[2:]
 				for s != "" {
 					c := int(s[0])
 					switch {
@@ -793,16 +802,17 @@ func defin(nt int, s string) int {
 					case c >= 'A' && c <= 'F':
 						c -= 'A' - 10
 					default:
-					errorf("illegal \\unnnn construction")
+						errorf(`illegal \u or \U construction`)
 					}
 					val = val*16 + c
 					s = s[1:]
 				}
-			if val == 0 {
-				errorf("'\\u0000' is illegal")
+			default:
+				errorf("invalid escape %s", s)
 			}
-		} else {
-			errorf("unknown escape")
+		}
+		if val == 0 {
+			errorf("token value 0 is illegal")
 		}
 	} else {
 		val = extval