mirror of
https://github.com/golang/go
synced 2024-11-24 23:57:57 -07:00
csv, gob, json, mail, mime, xml: use rune
Nothing terribly interesting here. R=golang-dev, r, borman CC=golang-dev https://golang.org/cl/5315043
This commit is contained in:
parent
9f6d036f33
commit
b50a847c3c
@ -101,8 +101,8 @@ var (
|
|||||||
//
|
//
|
||||||
// If TrimLeadingSpace is true, leading white space in a field is ignored.
|
// If TrimLeadingSpace is true, leading white space in a field is ignored.
|
||||||
type Reader struct {
|
type Reader struct {
|
||||||
Comma int // Field delimiter (set to ',' by NewReader)
|
Comma rune // Field delimiter (set to ',' by NewReader)
|
||||||
Comment int // Comment character for start of line
|
Comment rune // Comment character for start of line
|
||||||
FieldsPerRecord int // Number of expected fields per record
|
FieldsPerRecord int // Number of expected fields per record
|
||||||
LazyQuotes bool // Allow lazy quotes
|
LazyQuotes bool // Allow lazy quotes
|
||||||
TrailingComma bool // Allow trailing comma
|
TrailingComma bool // Allow trailing comma
|
||||||
@ -173,23 +173,23 @@ func (r *Reader) ReadAll() (records [][]string, err os.Error) {
|
|||||||
// readRune reads one rune from r, folding \r\n to \n and keeping track
|
// readRune reads one rune from r, folding \r\n to \n and keeping track
|
||||||
// of how far into the line we have read. r.column will point to the start
|
// of how far into the line we have read. r.column will point to the start
|
||||||
// of this rune, not the end of this rune.
|
// of this rune, not the end of this rune.
|
||||||
func (r *Reader) readRune() (int, os.Error) {
|
func (r *Reader) readRune() (rune, os.Error) {
|
||||||
rune, _, err := r.r.ReadRune()
|
r1, _, err := r.r.ReadRune()
|
||||||
|
|
||||||
// Handle \r\n here. We make the simplifying assumption that
|
// Handle \r\n here. We make the simplifying assumption that
|
||||||
// anytime \r is followed by \n that it can be folded to \n.
|
// anytime \r is followed by \n that it can be folded to \n.
|
||||||
// We will not detect files which contain both \r\n and bare \n.
|
// We will not detect files which contain both \r\n and bare \n.
|
||||||
if rune == '\r' {
|
if r1 == '\r' {
|
||||||
rune, _, err = r.r.ReadRune()
|
r1, _, err = r.r.ReadRune()
|
||||||
if err == nil {
|
if err == nil {
|
||||||
if rune != '\n' {
|
if r1 != '\n' {
|
||||||
r.r.UnreadRune()
|
r.r.UnreadRune()
|
||||||
rune = '\r'
|
r1 = '\r'
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
r.column++
|
r.column++
|
||||||
return rune, err
|
return r1, err
|
||||||
}
|
}
|
||||||
|
|
||||||
// unreadRune puts the last rune read from r back.
|
// unreadRune puts the last rune read from r back.
|
||||||
@ -199,13 +199,13 @@ func (r *Reader) unreadRune() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// skip reads runes up to and including the rune delim or until error.
|
// skip reads runes up to and including the rune delim or until error.
|
||||||
func (r *Reader) skip(delim int) os.Error {
|
func (r *Reader) skip(delim rune) os.Error {
|
||||||
for {
|
for {
|
||||||
rune, err := r.readRune()
|
r1, err := r.readRune()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if rune == delim {
|
if r1 == delim {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -224,12 +224,12 @@ func (r *Reader) parseRecord() (fields []string, err os.Error) {
|
|||||||
// If we are support comments and it is the comment character
|
// If we are support comments and it is the comment character
|
||||||
// then skip to the end of line.
|
// then skip to the end of line.
|
||||||
|
|
||||||
rune, _, err := r.r.ReadRune()
|
r1, _, err := r.r.ReadRune()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
if r.Comment != 0 && rune == r.Comment {
|
if r.Comment != 0 && r1 == r.Comment {
|
||||||
return nil, r.skip('\n')
|
return nil, r.skip('\n')
|
||||||
}
|
}
|
||||||
r.r.UnreadRune()
|
r.r.UnreadRune()
|
||||||
@ -252,10 +252,10 @@ func (r *Reader) parseRecord() (fields []string, err os.Error) {
|
|||||||
// parseField parses the next field in the record. The read field is
|
// parseField parses the next field in the record. The read field is
|
||||||
// located in r.field. Delim is the first character not part of the field
|
// located in r.field. Delim is the first character not part of the field
|
||||||
// (r.Comma or '\n').
|
// (r.Comma or '\n').
|
||||||
func (r *Reader) parseField() (haveField bool, delim int, err os.Error) {
|
func (r *Reader) parseField() (haveField bool, delim rune, err os.Error) {
|
||||||
r.field.Reset()
|
r.field.Reset()
|
||||||
|
|
||||||
rune, err := r.readRune()
|
r1, err := r.readRune()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
// If we have EOF and are not at the start of a line
|
// If we have EOF and are not at the start of a line
|
||||||
// then we return the empty field. We have already
|
// then we return the empty field. We have already
|
||||||
@ -267,30 +267,30 @@ func (r *Reader) parseField() (haveField bool, delim int, err os.Error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if r.TrimLeadingSpace {
|
if r.TrimLeadingSpace {
|
||||||
for rune != '\n' && unicode.IsSpace(rune) {
|
for r1 != '\n' && unicode.IsSpace(r1) {
|
||||||
rune, err = r.readRune()
|
r1, err = r.readRune()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false, 0, err
|
return false, 0, err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
switch rune {
|
switch r1 {
|
||||||
case r.Comma:
|
case r.Comma:
|
||||||
// will check below
|
// will check below
|
||||||
|
|
||||||
case '\n':
|
case '\n':
|
||||||
// We are a trailing empty field or a blank line
|
// We are a trailing empty field or a blank line
|
||||||
if r.column == 0 {
|
if r.column == 0 {
|
||||||
return false, rune, nil
|
return false, r1, nil
|
||||||
}
|
}
|
||||||
return true, rune, nil
|
return true, r1, nil
|
||||||
|
|
||||||
case '"':
|
case '"':
|
||||||
// quoted field
|
// quoted field
|
||||||
Quoted:
|
Quoted:
|
||||||
for {
|
for {
|
||||||
rune, err = r.readRune()
|
r1, err = r.readRune()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if err == os.EOF {
|
if err == os.EOF {
|
||||||
if r.LazyQuotes {
|
if r.LazyQuotes {
|
||||||
@ -300,16 +300,16 @@ func (r *Reader) parseField() (haveField bool, delim int, err os.Error) {
|
|||||||
}
|
}
|
||||||
return false, 0, err
|
return false, 0, err
|
||||||
}
|
}
|
||||||
switch rune {
|
switch r1 {
|
||||||
case '"':
|
case '"':
|
||||||
rune, err = r.readRune()
|
r1, err = r.readRune()
|
||||||
if err != nil || rune == r.Comma {
|
if err != nil || r1 == r.Comma {
|
||||||
break Quoted
|
break Quoted
|
||||||
}
|
}
|
||||||
if rune == '\n' {
|
if r1 == '\n' {
|
||||||
return true, rune, nil
|
return true, r1, nil
|
||||||
}
|
}
|
||||||
if rune != '"' {
|
if r1 != '"' {
|
||||||
if !r.LazyQuotes {
|
if !r.LazyQuotes {
|
||||||
r.column--
|
r.column--
|
||||||
return false, 0, r.error(ErrQuote)
|
return false, 0, r.error(ErrQuote)
|
||||||
@ -321,21 +321,21 @@ func (r *Reader) parseField() (haveField bool, delim int, err os.Error) {
|
|||||||
r.line++
|
r.line++
|
||||||
r.column = -1
|
r.column = -1
|
||||||
}
|
}
|
||||||
r.field.WriteRune(rune)
|
r.field.WriteRune(r1)
|
||||||
}
|
}
|
||||||
|
|
||||||
default:
|
default:
|
||||||
// unquoted field
|
// unquoted field
|
||||||
for {
|
for {
|
||||||
r.field.WriteRune(rune)
|
r.field.WriteRune(r1)
|
||||||
rune, err = r.readRune()
|
r1, err = r.readRune()
|
||||||
if err != nil || rune == r.Comma {
|
if err != nil || r1 == r.Comma {
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
if rune == '\n' {
|
if r1 == '\n' {
|
||||||
return true, rune, nil
|
return true, r1, nil
|
||||||
}
|
}
|
||||||
if !r.LazyQuotes && rune == '"' {
|
if !r.LazyQuotes && r1 == '"' {
|
||||||
return false, 0, r.error(ErrBareQuote)
|
return false, 0, r.error(ErrBareQuote)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -353,20 +353,20 @@ func (r *Reader) parseField() (haveField bool, delim int, err os.Error) {
|
|||||||
// are at the end of the line (being mindful
|
// are at the end of the line (being mindful
|
||||||
// of trimming spaces).
|
// of trimming spaces).
|
||||||
c := r.column
|
c := r.column
|
||||||
rune, err = r.readRune()
|
r1, err = r.readRune()
|
||||||
if r.TrimLeadingSpace {
|
if r.TrimLeadingSpace {
|
||||||
for rune != '\n' && unicode.IsSpace(rune) {
|
for r1 != '\n' && unicode.IsSpace(r1) {
|
||||||
rune, err = r.readRune()
|
r1, err = r.readRune()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if err == os.EOF || rune == '\n' {
|
if err == os.EOF || r1 == '\n' {
|
||||||
r.column = c // report the comma
|
r.column = c // report the comma
|
||||||
return false, 0, r.error(ErrTrailingComma)
|
return false, 0, r.error(ErrTrailingComma)
|
||||||
}
|
}
|
||||||
r.unreadRune()
|
r.unreadRune()
|
||||||
}
|
}
|
||||||
return true, rune, nil
|
return true, r1, nil
|
||||||
}
|
}
|
||||||
|
@ -17,8 +17,8 @@ var readTests = []struct {
|
|||||||
UseFieldsPerRecord bool // false (default) means FieldsPerRecord is -1
|
UseFieldsPerRecord bool // false (default) means FieldsPerRecord is -1
|
||||||
|
|
||||||
// These fields are copied into the Reader
|
// These fields are copied into the Reader
|
||||||
Comma int
|
Comma rune
|
||||||
Comment int
|
Comment rune
|
||||||
FieldsPerRecord int
|
FieldsPerRecord int
|
||||||
LazyQuotes bool
|
LazyQuotes bool
|
||||||
TrailingComma bool
|
TrailingComma bool
|
||||||
|
@ -23,7 +23,7 @@ import (
|
|||||||
//
|
//
|
||||||
// If UseCRLF is true, the Writer ends each record with \r\n instead of \n.
|
// If UseCRLF is true, the Writer ends each record with \r\n instead of \n.
|
||||||
type Writer struct {
|
type Writer struct {
|
||||||
Comma int // Field delimiter (set to to ',' by NewWriter)
|
Comma rune // Field delimiter (set to to ',' by NewWriter)
|
||||||
UseCRLF bool // True to use \r\n as the line terminator
|
UseCRLF bool // True to use \r\n as the line terminator
|
||||||
w *bufio.Writer
|
w *bufio.Writer
|
||||||
}
|
}
|
||||||
@ -58,8 +58,8 @@ func (w *Writer) Write(record []string) (err os.Error) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, rune := range field {
|
for _, r1 := range field {
|
||||||
switch rune {
|
switch r1 {
|
||||||
case '"':
|
case '"':
|
||||||
_, err = w.w.WriteString(`""`)
|
_, err = w.w.WriteString(`""`)
|
||||||
case '\r':
|
case '\r':
|
||||||
@ -73,7 +73,7 @@ func (w *Writer) Write(record []string) (err os.Error) {
|
|||||||
err = w.w.WriteByte('\n')
|
err = w.w.WriteByte('\n')
|
||||||
}
|
}
|
||||||
default:
|
default:
|
||||||
_, err = w.w.WriteRune(rune)
|
_, err = w.w.WriteRune(r1)
|
||||||
}
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return
|
return
|
||||||
@ -117,6 +117,6 @@ func (w *Writer) fieldNeedsQuotes(field string) bool {
|
|||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
|
|
||||||
rune, _ := utf8.DecodeRuneInString(field)
|
r1, _ := utf8.DecodeRuneInString(field)
|
||||||
return unicode.IsSpace(rune)
|
return unicode.IsSpace(r1)
|
||||||
}
|
}
|
||||||
|
@ -606,14 +606,14 @@ func TestSliceReusesMemory(t *testing.T) {
|
|||||||
}
|
}
|
||||||
// general slice
|
// general slice
|
||||||
{
|
{
|
||||||
x := []int("abcd")
|
x := []rune("abcd")
|
||||||
enc := NewEncoder(buf)
|
enc := NewEncoder(buf)
|
||||||
err := enc.Encode(x)
|
err := enc.Encode(x)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Errorf("ints: encode: %s", err)
|
t.Errorf("ints: encode: %s", err)
|
||||||
}
|
}
|
||||||
// Decode into y, which is big enough.
|
// Decode into y, which is big enough.
|
||||||
y := []int("ABCDE")
|
y := []rune("ABCDE")
|
||||||
addr := &y[0]
|
addr := &y[0]
|
||||||
dec := NewDecoder(buf)
|
dec := NewDecoder(buf)
|
||||||
err = dec.Decode(&y)
|
err = dec.Decode(&y)
|
||||||
|
@ -805,15 +805,15 @@ func (d *decodeState) literalInterface() interface{} {
|
|||||||
|
|
||||||
// getu4 decodes \uXXXX from the beginning of s, returning the hex value,
|
// getu4 decodes \uXXXX from the beginning of s, returning the hex value,
|
||||||
// or it returns -1.
|
// or it returns -1.
|
||||||
func getu4(s []byte) int {
|
func getu4(s []byte) rune {
|
||||||
if len(s) < 6 || s[0] != '\\' || s[1] != 'u' {
|
if len(s) < 6 || s[0] != '\\' || s[1] != 'u' {
|
||||||
return -1
|
return -1
|
||||||
}
|
}
|
||||||
rune, err := strconv.Btoui64(string(s[2:6]), 16)
|
r, err := strconv.Btoui64(string(s[2:6]), 16)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return -1
|
return -1
|
||||||
}
|
}
|
||||||
return int(rune)
|
return rune(r)
|
||||||
}
|
}
|
||||||
|
|
||||||
// unquote converts a quoted JSON string literal s into an actual string t.
|
// unquote converts a quoted JSON string literal s into an actual string t.
|
||||||
@ -843,8 +843,8 @@ func unquoteBytes(s []byte) (t []byte, ok bool) {
|
|||||||
r++
|
r++
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
rune, size := utf8.DecodeRune(s[r:])
|
rr, size := utf8.DecodeRune(s[r:])
|
||||||
if rune == utf8.RuneError && size == 1 {
|
if rr == utf8.RuneError && size == 1 {
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
r += size
|
r += size
|
||||||
@ -899,23 +899,23 @@ func unquoteBytes(s []byte) (t []byte, ok bool) {
|
|||||||
w++
|
w++
|
||||||
case 'u':
|
case 'u':
|
||||||
r--
|
r--
|
||||||
rune := getu4(s[r:])
|
rr := getu4(s[r:])
|
||||||
if rune < 0 {
|
if rr < 0 {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
r += 6
|
r += 6
|
||||||
if utf16.IsSurrogate(rune) {
|
if utf16.IsSurrogate(rr) {
|
||||||
rune1 := getu4(s[r:])
|
rr1 := getu4(s[r:])
|
||||||
if dec := utf16.DecodeRune(rune, rune1); dec != unicode.ReplacementChar {
|
if dec := utf16.DecodeRune(rr, rr1); dec != unicode.ReplacementChar {
|
||||||
// A valid pair; consume.
|
// A valid pair; consume.
|
||||||
r += 6
|
r += 6
|
||||||
w += utf8.EncodeRune(b[w:], dec)
|
w += utf8.EncodeRune(b[w:], dec)
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
// Invalid surrogate; fall back to replacement rune.
|
// Invalid surrogate; fall back to replacement rune.
|
||||||
rune = unicode.ReplacementChar
|
rr = unicode.ReplacementChar
|
||||||
}
|
}
|
||||||
w += utf8.EncodeRune(b[w:], rune)
|
w += utf8.EncodeRune(b[w:], rr)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Quote, control characters are invalid.
|
// Quote, control characters are invalid.
|
||||||
@ -930,9 +930,9 @@ func unquoteBytes(s []byte) (t []byte, ok bool) {
|
|||||||
|
|
||||||
// Coerce to well-formed UTF-8.
|
// Coerce to well-formed UTF-8.
|
||||||
default:
|
default:
|
||||||
rune, size := utf8.DecodeRune(s[r:])
|
rr, size := utf8.DecodeRune(s[r:])
|
||||||
r += size
|
r += size
|
||||||
w += utf8.EncodeRune(b[w:], rune)
|
w += utf8.EncodeRune(b[w:], rr)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return b[0:w], true
|
return b[0:w], true
|
||||||
|
@ -243,7 +243,7 @@ func TestHTMLEscape(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func noSpace(c int) int {
|
func noSpace(c rune) rune {
|
||||||
if isSpace(c) {
|
if isSpace(c) {
|
||||||
return -1
|
return -1
|
||||||
}
|
}
|
||||||
|
@ -176,7 +176,7 @@ func (s *scanner) popParseState() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func isSpace(c int) bool {
|
func isSpace(c rune) bool {
|
||||||
return c == ' ' || c == '\t' || c == '\r' || c == '\n'
|
return c == ' ' || c == '\t' || c == '\r' || c == '\n'
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -261,13 +261,13 @@ func genValue(n int) interface{} {
|
|||||||
|
|
||||||
func genString(stddev float64) string {
|
func genString(stddev float64) string {
|
||||||
n := int(math.Abs(rand.NormFloat64()*stddev + stddev/2))
|
n := int(math.Abs(rand.NormFloat64()*stddev + stddev/2))
|
||||||
c := make([]int, n)
|
c := make([]rune, n)
|
||||||
for i := range c {
|
for i := range c {
|
||||||
f := math.Abs(rand.NormFloat64()*64 + 32)
|
f := math.Abs(rand.NormFloat64()*64 + 32)
|
||||||
if f > 0x10ffff {
|
if f > 0x10ffff {
|
||||||
f = 0x10ffff
|
f = 0x10ffff
|
||||||
}
|
}
|
||||||
c[i] = int(f)
|
c[i] = rune(f)
|
||||||
}
|
}
|
||||||
return string(c)
|
return string(c)
|
||||||
}
|
}
|
||||||
|
@ -115,7 +115,7 @@ Input:
|
|||||||
|
|
||||||
func nonSpace(b []byte) bool {
|
func nonSpace(b []byte) bool {
|
||||||
for _, c := range b {
|
for _, c := range b {
|
||||||
if !isSpace(int(c)) {
|
if !isSpace(rune(c)) {
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -454,7 +454,7 @@ func decodeRFC2047Word(s string) (string, os.Error) {
|
|||||||
case "iso-8859-1":
|
case "iso-8859-1":
|
||||||
b := new(bytes.Buffer)
|
b := new(bytes.Buffer)
|
||||||
for _, c := range dec {
|
for _, c := range dec {
|
||||||
b.WriteRune(int(c))
|
b.WriteRune(rune(c))
|
||||||
}
|
}
|
||||||
return b.String(), nil
|
return b.String(), nil
|
||||||
case "utf-8":
|
case "utf-8":
|
||||||
|
@ -10,16 +10,16 @@ import (
|
|||||||
|
|
||||||
// isTSpecial returns true if rune is in 'tspecials' as defined by RFC
|
// isTSpecial returns true if rune is in 'tspecials' as defined by RFC
|
||||||
// 1521 and RFC 2045.
|
// 1521 and RFC 2045.
|
||||||
func isTSpecial(rune int) bool {
|
func isTSpecial(r rune) bool {
|
||||||
return strings.IndexRune(`()<>@,;:\"/[]?=`, rune) != -1
|
return strings.IndexRune(`()<>@,;:\"/[]?=`, r) != -1
|
||||||
}
|
}
|
||||||
|
|
||||||
// IsTokenChar returns true if rune is in 'token' as defined by RFC
|
// IsTokenChar returns true if rune is in 'token' as defined by RFC
|
||||||
// 1521 and RFC 2045.
|
// 1521 and RFC 2045.
|
||||||
func IsTokenChar(rune int) bool {
|
func IsTokenChar(r rune) bool {
|
||||||
// token := 1*<any (US-ASCII) CHAR except SPACE, CTLs,
|
// token := 1*<any (US-ASCII) CHAR except SPACE, CTLs,
|
||||||
// or tspecials>
|
// or tspecials>
|
||||||
return rune > 0x20 && rune < 0x7f && !isTSpecial(rune)
|
return r > 0x20 && r < 0x7f && !isTSpecial(r)
|
||||||
}
|
}
|
||||||
|
|
||||||
// IsToken returns true if s is a 'token' as as defined by RFC 1521
|
// IsToken returns true if s is a 'token' as as defined by RFC 1521
|
||||||
@ -32,14 +32,14 @@ func IsToken(s string) bool {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// IsQText returns true if rune is in 'qtext' as defined by RFC 822.
|
// IsQText returns true if rune is in 'qtext' as defined by RFC 822.
|
||||||
func IsQText(rune int) bool {
|
func IsQText(r int) bool {
|
||||||
// CHAR = <any ASCII character> ; ( 0-177, 0.-127.)
|
// CHAR = <any ASCII character> ; ( 0-177, 0.-127.)
|
||||||
// qtext = <any CHAR excepting <">, ; => may be folded
|
// qtext = <any CHAR excepting <">, ; => may be folded
|
||||||
// "\" & CR, and including
|
// "\" & CR, and including
|
||||||
// linear-white-space>
|
// linear-white-space>
|
||||||
switch rune {
|
switch r {
|
||||||
case '"', '\\', '\r':
|
case '"', '\\', '\r':
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
return rune < 0x80
|
return r < 0x80
|
||||||
}
|
}
|
||||||
|
@ -199,8 +199,8 @@ func decode2231Enc(v string) string {
|
|||||||
return encv
|
return encv
|
||||||
}
|
}
|
||||||
|
|
||||||
func isNotTokenChar(rune int) bool {
|
func isNotTokenChar(r rune) bool {
|
||||||
return !IsTokenChar(rune)
|
return !IsTokenChar(r)
|
||||||
}
|
}
|
||||||
|
|
||||||
// consumeToken consumes a token from the beginning of provided
|
// consumeToken consumes a token from the beginning of provided
|
||||||
@ -228,24 +228,25 @@ func consumeValue(v string) (value, rest string) {
|
|||||||
return consumeToken(v)
|
return consumeToken(v)
|
||||||
}
|
}
|
||||||
|
|
||||||
leadQuote := int(v[0])
|
leadQuote := rune(v[0])
|
||||||
|
|
||||||
// parse a quoted-string
|
// parse a quoted-string
|
||||||
rest = v[1:] // consume the leading quote
|
rest = v[1:] // consume the leading quote
|
||||||
buffer := new(bytes.Buffer)
|
buffer := new(bytes.Buffer)
|
||||||
var idx, rune int
|
var idx int
|
||||||
|
var r rune
|
||||||
var nextIsLiteral bool
|
var nextIsLiteral bool
|
||||||
for idx, rune = range rest {
|
for idx, r = range rest {
|
||||||
switch {
|
switch {
|
||||||
case nextIsLiteral:
|
case nextIsLiteral:
|
||||||
buffer.WriteRune(rune)
|
buffer.WriteRune(r)
|
||||||
nextIsLiteral = false
|
nextIsLiteral = false
|
||||||
case rune == leadQuote:
|
case r == leadQuote:
|
||||||
return buffer.String(), rest[idx+1:]
|
return buffer.String(), rest[idx+1:]
|
||||||
case rune == '\\':
|
case r == '\\':
|
||||||
nextIsLiteral = true
|
nextIsLiteral = true
|
||||||
case rune != '\r' && rune != '\n':
|
case r != '\r' && r != '\n':
|
||||||
buffer.WriteRune(rune)
|
buffer.WriteRune(r)
|
||||||
default:
|
default:
|
||||||
return "", v
|
return "", v
|
||||||
}
|
}
|
||||||
|
@ -18,32 +18,32 @@ func quoteWith(s string, quote byte, ASCIIonly bool) string {
|
|||||||
var buf bytes.Buffer
|
var buf bytes.Buffer
|
||||||
buf.WriteByte(quote)
|
buf.WriteByte(quote)
|
||||||
for width := 0; len(s) > 0; s = s[width:] {
|
for width := 0; len(s) > 0; s = s[width:] {
|
||||||
rune := int(s[0])
|
r := rune(s[0])
|
||||||
width = 1
|
width = 1
|
||||||
if rune >= utf8.RuneSelf {
|
if r >= utf8.RuneSelf {
|
||||||
rune, width = utf8.DecodeRuneInString(s)
|
r, width = utf8.DecodeRuneInString(s)
|
||||||
}
|
}
|
||||||
if width == 1 && rune == utf8.RuneError {
|
if width == 1 && r == utf8.RuneError {
|
||||||
buf.WriteString(`\x`)
|
buf.WriteString(`\x`)
|
||||||
buf.WriteByte(lowerhex[s[0]>>4])
|
buf.WriteByte(lowerhex[s[0]>>4])
|
||||||
buf.WriteByte(lowerhex[s[0]&0xF])
|
buf.WriteByte(lowerhex[s[0]&0xF])
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if rune == int(quote) || rune == '\\' { // always backslashed
|
if r == rune(quote) || r == '\\' { // always backslashed
|
||||||
buf.WriteByte('\\')
|
buf.WriteByte('\\')
|
||||||
buf.WriteByte(byte(rune))
|
buf.WriteByte(byte(r))
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if ASCIIonly {
|
if ASCIIonly {
|
||||||
if rune <= unicode.MaxASCII && unicode.IsPrint(rune) {
|
if r <= unicode.MaxASCII && unicode.IsPrint(r) {
|
||||||
buf.WriteRune(rune)
|
buf.WriteRune(r)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
} else if unicode.IsPrint(rune) {
|
} else if unicode.IsPrint(r) {
|
||||||
buf.WriteRune(rune)
|
buf.WriteRune(r)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
switch rune {
|
switch r {
|
||||||
case '\a':
|
case '\a':
|
||||||
buf.WriteString(`\a`)
|
buf.WriteString(`\a`)
|
||||||
case '\b':
|
case '\b':
|
||||||
@ -60,22 +60,22 @@ func quoteWith(s string, quote byte, ASCIIonly bool) string {
|
|||||||
buf.WriteString(`\v`)
|
buf.WriteString(`\v`)
|
||||||
default:
|
default:
|
||||||
switch {
|
switch {
|
||||||
case rune < ' ':
|
case r < ' ':
|
||||||
buf.WriteString(`\x`)
|
buf.WriteString(`\x`)
|
||||||
buf.WriteByte(lowerhex[s[0]>>4])
|
buf.WriteByte(lowerhex[s[0]>>4])
|
||||||
buf.WriteByte(lowerhex[s[0]&0xF])
|
buf.WriteByte(lowerhex[s[0]&0xF])
|
||||||
case rune > unicode.MaxRune:
|
case r > unicode.MaxRune:
|
||||||
rune = 0xFFFD
|
r = 0xFFFD
|
||||||
fallthrough
|
fallthrough
|
||||||
case rune < 0x10000:
|
case r < 0x10000:
|
||||||
buf.WriteString(`\u`)
|
buf.WriteString(`\u`)
|
||||||
for s := 12; s >= 0; s -= 4 {
|
for s := 12; s >= 0; s -= 4 {
|
||||||
buf.WriteByte(lowerhex[rune>>uint(s)&0xF])
|
buf.WriteByte(lowerhex[r>>uint(s)&0xF])
|
||||||
}
|
}
|
||||||
default:
|
default:
|
||||||
buf.WriteString(`\U`)
|
buf.WriteString(`\U`)
|
||||||
for s := 28; s >= 0; s -= 4 {
|
for s := 28; s >= 0; s -= 4 {
|
||||||
buf.WriteByte(lowerhex[rune>>uint(s)&0xF])
|
buf.WriteByte(lowerhex[r>>uint(s)&0xF])
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -130,8 +130,8 @@ func CanBackquote(s string) bool {
|
|||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
|
|
||||||
func unhex(b byte) (v int, ok bool) {
|
func unhex(b byte) (v rune, ok bool) {
|
||||||
c := int(b)
|
c := rune(b)
|
||||||
switch {
|
switch {
|
||||||
case '0' <= c && c <= '9':
|
case '0' <= c && c <= '9':
|
||||||
return c - '0', true
|
return c - '0', true
|
||||||
@ -157,7 +157,7 @@ func unhex(b byte) (v int, ok bool) {
|
|||||||
// If set to a single quote, it permits the sequence \' and disallows unescaped '.
|
// If set to a single quote, it permits the sequence \' and disallows unescaped '.
|
||||||
// If set to a double quote, it permits \" and disallows unescaped ".
|
// If set to a double quote, it permits \" and disallows unescaped ".
|
||||||
// If set to zero, it does not permit either escape and allows both quote characters to appear unescaped.
|
// If set to zero, it does not permit either escape and allows both quote characters to appear unescaped.
|
||||||
func UnquoteChar(s string, quote byte) (value int, multibyte bool, tail string, err os.Error) {
|
func UnquoteChar(s string, quote byte) (value rune, multibyte bool, tail string, err os.Error) {
|
||||||
// easy cases
|
// easy cases
|
||||||
switch c := s[0]; {
|
switch c := s[0]; {
|
||||||
case c == quote && (quote == '\'' || quote == '"'):
|
case c == quote && (quote == '\'' || quote == '"'):
|
||||||
@ -167,7 +167,7 @@ func UnquoteChar(s string, quote byte) (value int, multibyte bool, tail string,
|
|||||||
r, size := utf8.DecodeRuneInString(s)
|
r, size := utf8.DecodeRuneInString(s)
|
||||||
return r, true, s[size:], nil
|
return r, true, s[size:], nil
|
||||||
case c != '\\':
|
case c != '\\':
|
||||||
return int(s[0]), false, s[1:], nil
|
return rune(s[0]), false, s[1:], nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// hard case: c is backslash
|
// hard case: c is backslash
|
||||||
@ -203,7 +203,7 @@ func UnquoteChar(s string, quote byte) (value int, multibyte bool, tail string,
|
|||||||
case 'U':
|
case 'U':
|
||||||
n = 8
|
n = 8
|
||||||
}
|
}
|
||||||
v := 0
|
var v rune
|
||||||
if len(s) < n {
|
if len(s) < n {
|
||||||
err = os.EINVAL
|
err = os.EINVAL
|
||||||
return
|
return
|
||||||
@ -229,13 +229,13 @@ func UnquoteChar(s string, quote byte) (value int, multibyte bool, tail string,
|
|||||||
value = v
|
value = v
|
||||||
multibyte = true
|
multibyte = true
|
||||||
case '0', '1', '2', '3', '4', '5', '6', '7':
|
case '0', '1', '2', '3', '4', '5', '6', '7':
|
||||||
v := int(c) - '0'
|
v := rune(c) - '0'
|
||||||
if len(s) < 2 {
|
if len(s) < 2 {
|
||||||
err = os.EINVAL
|
err = os.EINVAL
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
for j := 0; j < 2; j++ { // one digit already; two more
|
for j := 0; j < 2; j++ { // one digit already; two more
|
||||||
x := int(s[j]) - '0'
|
x := rune(s[j]) - '0'
|
||||||
if x < 0 || x > 7 {
|
if x < 0 || x > 7 {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@ -254,7 +254,7 @@ func UnquoteChar(s string, quote byte) (value int, multibyte bool, tail string,
|
|||||||
err = os.EINVAL
|
err = os.EINVAL
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
value = int(c)
|
value = rune(c)
|
||||||
default:
|
default:
|
||||||
err = os.EINVAL
|
err = os.EINVAL
|
||||||
return
|
return
|
||||||
@ -293,7 +293,7 @@ func Unquote(s string) (t string, err os.Error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Is it trivial? Avoid allocation.
|
// Is it trivial? Avoid allocation.
|
||||||
if strings.Index(s, `\`) < 0 && strings.IndexRune(s, int(quote)) < 0 {
|
if strings.Index(s, `\`) < 0 && strings.IndexRune(s, rune(quote)) < 0 {
|
||||||
switch quote {
|
switch quote {
|
||||||
case '"':
|
case '"':
|
||||||
return s, nil
|
return s, nil
|
||||||
|
@ -206,7 +206,7 @@ func fieldName(original string) string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
return strings.Map(
|
return strings.Map(
|
||||||
func(x int) int {
|
func(x rune) rune {
|
||||||
if x == '_' || unicode.IsDigit(x) || unicode.IsLetter(x) {
|
if x == '_' || unicode.IsDigit(x) || unicode.IsLetter(x) {
|
||||||
return unicode.ToLower(x)
|
return unicode.ToLower(x)
|
||||||
}
|
}
|
||||||
|
@ -960,13 +960,13 @@ Input:
|
|||||||
// Decide whether the given rune is in the XML Character Range, per
|
// Decide whether the given rune is in the XML Character Range, per
|
||||||
// the Char production of http://www.xml.com/axml/testaxml.htm,
|
// the Char production of http://www.xml.com/axml/testaxml.htm,
|
||||||
// Section 2.2 Characters.
|
// Section 2.2 Characters.
|
||||||
func isInCharacterRange(rune int) (inrange bool) {
|
func isInCharacterRange(r rune) (inrange bool) {
|
||||||
return rune == 0x09 ||
|
return r == 0x09 ||
|
||||||
rune == 0x0A ||
|
r == 0x0A ||
|
||||||
rune == 0x0D ||
|
r == 0x0D ||
|
||||||
rune >= 0x20 && rune <= 0xDF77 ||
|
r >= 0x20 && r <= 0xDF77 ||
|
||||||
rune >= 0xE000 && rune <= 0xFFFD ||
|
r >= 0xE000 && r <= 0xFFFD ||
|
||||||
rune >= 0x10000 && rune <= 0x10FFFF
|
r >= 0x10000 && r <= 0x10FFFF
|
||||||
}
|
}
|
||||||
|
|
||||||
// Get name space name: name with a : stuck in the middle.
|
// Get name space name: name with a : stuck in the middle.
|
||||||
@ -1690,7 +1690,7 @@ func procInstEncoding(s string) string {
|
|||||||
if v[0] != '\'' && v[0] != '"' {
|
if v[0] != '\'' && v[0] != '"' {
|
||||||
return ""
|
return ""
|
||||||
}
|
}
|
||||||
idx = strings.IndexRune(v[1:], int(v[0]))
|
idx = strings.IndexRune(v[1:], rune(v[0]))
|
||||||
if idx == -1 {
|
if idx == -1 {
|
||||||
return ""
|
return ""
|
||||||
}
|
}
|
||||||
|
Loading…
Reference in New Issue
Block a user