json: add struct tag option to wrap literals in strings

Since JavaScript doesn't have [u]int64 types, some JSON APIs
encode such types as strings to avoid losing precision.

This adds a new struct tag option ",string" to cause
fields to be wrapped in JSON strings on encoding
and unwrapped from strings when decoding.

R=rsc, gustavo
CC=golang-dev
https://golang.org/cl/4918051

src/pkg/json/Makefile

@@ -11,5 +11,6 @@ GOFILES=\
 	indent.go\
 	scanner.go\
 	stream.go\
+	tags.go\
 
 include ../../Make.pkg

src/pkg/json/decode.go

@@ -140,6 +140,7 @@ type decodeState struct {
 	scan       scanner
 	nextscan   scanner // for calls to nextValue
 	savedError os.Error
+	tempstr    string // scratch space to avoid some allocations
 }
 
 // errPhase is used for errors that should not happen unless
@@ -470,6 +471,8 @@ func (d *decodeState) object(v reflect.Value) {
 
 		// Figure out field corresponding to key.
 		var subv reflect.Value
+		destring := false // whether the value is wrapped in a string to be decoded first
+
 		if mv.IsValid() {
 			elemType := mv.Type().Elem()
 			if !mapElem.IsValid() {
@@ -486,7 +489,8 @@ func (d *decodeState) object(v reflect.Value) {
 			if isValidTag(key) {
 				for i := 0; i < sv.NumField(); i++ {
 					f = st.Field(i)
-					if tagName(f.Tag.Get("json")) == key {
+					tagName, _ := parseTag(f.Tag.Get("json"))
+					if tagName == key {
 						ok = true
 						break
 					}
@@ -508,6 +512,8 @@ func (d *decodeState) object(v reflect.Value) {
 				} else {
 					subv = sv.FieldByIndex(f.Index)
 				}
+				_, opts := parseTag(f.Tag.Get("json"))
+				destring = opts.Contains("string")
 			}
 		}
 
@@ -520,8 +526,12 @@ func (d *decodeState) object(v reflect.Value) {
 		}
 
 		// Read value.
-		d.value(subv)
+		if destring {
-
+			d.value(reflect.ValueOf(&d.tempstr))
+			d.literalStore([]byte(d.tempstr), subv)
+		} else {
+			d.value(subv)
+		}
 		// Write value back to map;
 		// if using struct, subv points into struct already.
 		if mv.IsValid() {
@@ -550,8 +560,12 @@ func (d *decodeState) literal(v reflect.Value) {
 	// Scan read one byte too far; back up.
 	d.off--
 	d.scan.undo(op)
-	item := d.data[start:d.off]
 
+	d.literalStore(d.data[start:d.off], v)
+}
+
+// literalStore decodes a literal stored in item into v.
+func (d *decodeState) literalStore(item []byte, v reflect.Value) {
 	// Check for unmarshaler.
 	wantptr := item[0] == 'n' // null
 	unmarshaler, pv := d.indirect(v, wantptr)
@@ -918,13 +932,3 @@ func unquoteBytes(s []byte) (t []byte, ok bool) {
 	}
 	return b[0:w], true
 }
-
-// tagName extracts the field name part out of the "json" struct tag
-// value. The json struct tag format is an optional name, followed by
-// zero or more ",option" values.
-func tagName(v string) string {
-	if idx := strings.Index(v, ","); idx != -1 {
-		return v[:idx]
-	}
-	return v
-}

src/pkg/json/decode_test.go

@@ -265,6 +265,8 @@ type All struct {
 	Foo  string `json:"bar"`
 	Foo2 string `json:"bar2,dummyopt"`
 
+	IntStr int64 `json:",string"`
+
 	PBool    *bool
 	PInt     *int
 	PInt8    *int8
@@ -333,6 +335,7 @@ var allValue = All{
 	Float64: 15.1,
 	Foo:     "foo",
 	Foo2:    "foo2",
+	IntStr:  42,
 	String:  "16",
 	Map: map[string]Small{
 		"17": {Tag: "tag17"},
@@ -394,6 +397,7 @@ var allValueIndent = `{
 	"Float64": 15.1,
 	"bar": "foo",
 	"bar2": "foo2",
+	"IntStr": "42",
 	"PBool": null,
 	"PInt": null,
 	"PInt8": null,
@@ -485,6 +489,7 @@ var pallValueIndent = `{
 	"Float64": 0,
 	"bar": "",
 	"bar2": "",
+        "IntStr": "0",
 	"PBool": true,
 	"PInt": 2,
 	"PInt8": 3,

src/pkg/json/encode.go

@@ -14,7 +14,6 @@ import (
 	"runtime"
 	"sort"
 	"strconv"
-	"strings"
 	"unicode"
 	"utf8"
 )
@@ -59,6 +58,12 @@ import (
 //   // Note the leading comma.
 //   Field int `json:",omitempty"`
 //
+// The "string" option signals that a field is stored as JSON inside a
+// JSON-encoded string.  This extra level of encoding is sometimes
+// used when communicating with JavaScript programs:
+//
+//    Int64String int64 `json:",string"`
+//
 // The key name will be used if it's a non-empty string consisting of
 // only Unicode letters, digits, dollar signs, hyphens, and underscores.
 //
@@ -221,6 +226,12 @@ func isEmptyValue(v reflect.Value) bool {
 }
 
 func (e *encodeState) reflectValue(v reflect.Value) {
+	e.reflectValueQuoted(v, false)
+}
+
+// reflectValueQuoted writes the value in v to the output.
+// If quoted is true, the serialization is wrapped in a JSON string.
+func (e *encodeState) reflectValueQuoted(v reflect.Value, quoted bool) {
 	if !v.IsValid() {
 		e.WriteString("null")
 		return
@@ -238,26 +249,39 @@ func (e *encodeState) reflectValue(v reflect.Value) {
 		return
 	}
 
+	writeString := (*encodeState).WriteString
+	if quoted {
+		writeString = (*encodeState).string
+	}
+
 	switch v.Kind() {
 	case reflect.Bool:
 		x := v.Bool()
 		if x {
-			e.WriteString("true")
+			writeString(e, "true")
 		} else {
-			e.WriteString("false")
+			writeString(e, "false")
 		}
 
 	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
-		e.WriteString(strconv.Itoa64(v.Int()))
+		writeString(e, strconv.Itoa64(v.Int()))
 
 	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
-		e.WriteString(strconv.Uitoa64(v.Uint()))
+		writeString(e, strconv.Uitoa64(v.Uint()))
 
 	case reflect.Float32, reflect.Float64:
-		e.WriteString(strconv.FtoaN(v.Float(), 'g', -1, v.Type().Bits()))
+		writeString(e, strconv.FtoaN(v.Float(), 'g', -1, v.Type().Bits()))
 
 	case reflect.String:
-		e.string(v.String())
+		if quoted {
+			sb, err := Marshal(v.String())
+			if err != nil {
+				e.error(err)
+			}
+			e.string(string(sb))
+		} else {
+			e.string(v.String())
+		}
 
 	case reflect.Struct:
 		e.WriteByte('{')
@@ -269,17 +293,14 @@ func (e *encodeState) reflectValue(v reflect.Value) {
 			if f.PkgPath != "" {
 				continue
 			}
-			tag, omitEmpty := f.Name, false
+			tag, omitEmpty, quoted := f.Name, false, false
 			if tv := f.Tag.Get("json"); tv != "" {
-				ss := strings.SplitN(tv, ",", 2)
+				name, opts := parseTag(tv)
-				if isValidTag(ss[0]) {
+				if isValidTag(name) {
-					tag = ss[0]
+					tag = name
-				}
-				if len(ss) > 1 {
-					// Currently the only option is omitempty,
-					// so parsing is trivial.
-					omitEmpty = ss[1] == "omitempty"
 				}
+				omitEmpty = opts.Contains("omitempty")
+				quoted = opts.Contains("string")
 			}
 			fieldValue := v.Field(i)
 			if omitEmpty && isEmptyValue(fieldValue) {
@@ -292,7 +313,7 @@ func (e *encodeState) reflectValue(v reflect.Value) {
 			}
 			e.string(tag)
 			e.WriteByte(':')
-			e.reflectValue(fieldValue)
+			e.reflectValueQuoted(fieldValue, quoted)
 		}
 		e.WriteByte('}')
 
@@ -380,7 +401,8 @@ func (sv stringValues) Swap(i, j int)      { sv[i], sv[j] = sv[j], sv[i] }
 func (sv stringValues) Less(i, j int) bool { return sv.get(i) < sv.get(j) }
 func (sv stringValues) get(i int) string   { return sv[i].String() }
 
-func (e *encodeState) string(s string) {
+func (e *encodeState) string(s string) (int, os.Error) {
+	len0 := e.Len()
 	e.WriteByte('"')
 	start := 0
 	for i := 0; i < len(s); {
@@ -425,4 +447,5 @@ func (e *encodeState) string(s string) {
 		e.WriteString(s[start:])
 	}
 	e.WriteByte('"')
+	return e.Len() - len0, nil
 }

src/pkg/json/encode_test.go

@@ -5,6 +5,8 @@
 package json
 
 import (
+	"bytes"
+	"reflect"
 	"testing"
 )
 
@@ -42,3 +44,39 @@ func TestOmitEmpty(t *testing.T) {
 		t.Errorf(" got: %s\nwant: %s\n", got, optionalsExpected)
 	}
 }
+
+type StringTag struct {
+	BoolStr bool   `json:",string"`
+	IntStr  int64  `json:",string"`
+	StrStr  string `json:",string"`
+}
+
+var stringTagExpected = `{
+ "BoolStr": "true",
+ "IntStr": "42",
+ "StrStr": "\"xzbit\""
+}`
+
+func TestStringTag(t *testing.T) {
+	var s StringTag
+	s.BoolStr = true
+	s.IntStr = 42
+	s.StrStr = "xzbit"
+	got, err := MarshalIndent(&s, "", " ")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got := string(got); got != stringTagExpected {
+		t.Fatalf(" got: %s\nwant: %s\n", got, stringTagExpected)
+	}
+
+	// Verify that it round-trips.
+	var s2 StringTag
+	err = NewDecoder(bytes.NewBuffer(got)).Decode(&s2)
+	if err != nil {
+		t.Fatalf("Decode: %v", err)
+	}
+	if !reflect.DeepEqual(s, s2) {
+		t.Fatalf("decode didn't match.\nsource: %#v\nEncoded as:\n%s\ndecode: %#v", s, string(got), s2)
+	}
+}

src/pkg/json/tags.go

@@ -0,0 +1,44 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package json
+
+import (
+	"strings"
+)
+
+// tagOptions is the string following a comma in a struct field's "json"
+// tag, or the empty string. It does not include the leading comma.
+type tagOptions string
+
+// parseTag splits a struct field's json tag into its name and
+// comma-separated options.
+func parseTag(tag string) (string, tagOptions) {
+	if idx := strings.Index(tag, ","); idx != -1 {
+		return tag[:idx], tagOptions(tag[idx+1:])
+	}
+	return tag, tagOptions("")
+}
+
+// Contains returns whether checks that a comma-separated list of options
+// contains a particular substr flag. substr must be surrounded by a
+// string boundary or commas.
+func (o tagOptions) Contains(optionName string) bool {
+	if len(o) == 0 {
+		return false
+	}
+	s := string(o)
+	for s != "" {
+		var next string
+		i := strings.Index(s, ",")
+		if i >= 0 {
+			s, next = s[:i], s[i+1:]
+		}
+		if s == optionName {
+			return true
+		}
+		s = next
+	}
+	return false
+}

src/pkg/json/tags_test.go

@@ -0,0 +1,28 @@
+// Copyright 2011 The Go Authors.  All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package json
+
+import (
+	"testing"
+)
+
+func TestTagParsing(t *testing.T) {
+	name, opts := parseTag("field,foobar,foo")
+	if name != "field" {
+		t.Fatalf("name = %q, want field", name)
+	}
+	for _, tt := range []struct {
+		opt  string
+		want bool
+	}{
+		{"foobar", true},
+		{"foo", true},
+		{"bar", false},
+	} {
+		if opts.Contains(tt.opt) != tt.want {
+			t.Errorf("Contains(%q) = %v", tt.opt, !tt.want)
+		}
+	}
+}