From 0e8384af653e40f47dcfdb79c65f81586c9d7ee9 Mon Sep 17 00:00:00 2001 From: Russ Cox Date: Tue, 27 Apr 2010 10:46:37 -0700 Subject: [PATCH] json: streaming R=r, cw CC=golang-dev https://golang.org/cl/952041 --- src/pkg/json/Makefile | 1 + src/pkg/json/stream.go | 185 ++++++++++++++++++++++++++++++++++++ src/pkg/json/stream_test.go | 122 ++++++++++++++++++++++++ 3 files changed, 308 insertions(+) create mode 100644 src/pkg/json/stream.go create mode 100644 src/pkg/json/stream_test.go diff --git a/src/pkg/json/Makefile b/src/pkg/json/Makefile index 5371aee7f27..970b84dc087 100644 --- a/src/pkg/json/Makefile +++ b/src/pkg/json/Makefile @@ -12,5 +12,6 @@ GOFILES=\ indent.go\ parse.go\ scanner.go\ + stream.go\ include ../../Make.pkg diff --git a/src/pkg/json/stream.go b/src/pkg/json/stream.go new file mode 100644 index 00000000000..d4fb3466079 --- /dev/null +++ b/src/pkg/json/stream.go @@ -0,0 +1,185 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package json + +import ( + "bytes" + "io" + "os" +) + +// A Decoder reads and decodes JSON objects from an input stream. +type Decoder struct { + r io.Reader + buf []byte + d decodeState + scan scanner + err os.Error +} + +// NewDecoder returns a new decoder that reads from r. +func NewDecoder(r io.Reader) *Decoder { + return &Decoder{r: r} +} + +// Decode reads the next JSON-encoded value from the +// connection and stores it in the value pointed to by v. +// +// See the documentation for Unmarshal for details about +// the conversion of JSON into a Go value. +func (dec *Decoder) Decode(v interface{}) os.Error { + if dec.err != nil { + return dec.err + } + + n, err := dec.readValue() + if err != nil { + return err + } + + // Don't save err from unmarshal into dec.err: + // the connection is still usable since we read a complete JSON + // object from it before the error happened. + dec.d.init(dec.buf[0:n]) + err = dec.d.unmarshal(v) + + // Slide rest of data down. + rest := copy(dec.buf, dec.buf[n:]) + dec.buf = dec.buf[0:rest] + + return err +} + +// readValue reads a JSON value into dec.buf. +// It returns the length of the encoding. +func (dec *Decoder) readValue() (int, os.Error) { + dec.scan.reset() + + scanp := 0 + var err os.Error +Input: + for { + // Look in the buffer for a new value. + for i, c := range dec.buf[scanp:] { + v := dec.scan.step(&dec.scan, int(c)) + if v == scanEnd { + scanp += i + break Input + } + // scanEnd is delayed one byte. + // We might block trying to get that byte from src, + // so instead invent a space byte. + if v == scanEndObject && dec.scan.step(&dec.scan, ' ') == scanEnd { + scanp += i + 1 + break Input + } + if v == scanError { + dec.err = dec.scan.err + return 0, dec.scan.err + } + } + scanp = len(dec.buf) + + // Did the last read have an error? + // Delayed until now to allow buffer scan. + if err != nil { + if err == os.EOF { + if dec.scan.step(&dec.scan, ' ') == scanEnd { + break Input + } + if nonSpace(dec.buf) { + err = io.ErrUnexpectedEOF + } + } + dec.err = err + return 0, err + } + + // Make room to read more into the buffer. + const minRead = 512 + if cap(dec.buf)-len(dec.buf) < minRead { + newBuf := make([]byte, len(dec.buf), 2*cap(dec.buf)+minRead) + copy(newBuf, dec.buf) + dec.buf = newBuf + } + + // Read. Delay error for next iteration (after scan). + var n int + n, err = dec.r.Read(dec.buf[len(dec.buf):cap(dec.buf)]) + dec.buf = dec.buf[0 : len(dec.buf)+n] + } + return scanp, nil +} + +func nonSpace(b []byte) bool { + for _, c := range b { + if !isSpace(int(c)) { + return true + } + } + return false +} + +// An Encoder writes JSON objects to an output stream. +type Encoder struct { + w io.Writer + e encodeState + err os.Error +} + +// NewEncoder returns a new encoder that writes to w. +func NewEncoder(w io.Writer) *Encoder { + return &Encoder{w: w} +} + +// Encode writes the JSON encoding of v to the connection. +// +// See the documentation for Marshal for details about the +// conversion of Go values to JSON. +func (enc *Encoder) Encode(v interface{}) os.Error { + if enc.err != nil { + return enc.err + } + enc.e.Reset() + err := enc.e.marshal(v) + if err != nil { + return err + } + + // Terminate each value with a newline. + // This makes the output look a little nicer + // when debugging, and some kind of space + // is required if the encoded value was a number, + // so that the reader knows there aren't more + // digits coming. + enc.e.WriteByte('\n') + + if _, err = enc.w.Write(enc.e.Bytes()); err != nil { + enc.err = err + } + return err +} + +// RawMessage is a raw encoded JSON object. +// It implements Marshaler and Unmarshaler and can +// be used to delay JSON decoding or precompute a JSON encoding. +type RawMessage []byte + +// MarshalJSON returns *m as the JSON encoding of m. +func (m *RawMessage) MarshalJSON() ([]byte, os.Error) { + return *m, nil +} + +// UnmarshalJSON sets *m to a copy of data. +func (m *RawMessage) UnmarshalJSON(data []byte) os.Error { + if m == nil { + return os.NewError("json.RawMessage: UnmarshalJSON on nil pointer") + } + *m = bytes.Add((*m)[0:0], data) + return nil +} + +var _ Marshaler = (*RawMessage)(nil) +var _ Unmarshaler = (*RawMessage)(nil) diff --git a/src/pkg/json/stream_test.go b/src/pkg/json/stream_test.go new file mode 100644 index 00000000000..86d01429027 --- /dev/null +++ b/src/pkg/json/stream_test.go @@ -0,0 +1,122 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package json + +import ( + "bytes" + "reflect" + "testing" +) + +// Test values for the stream test. +// One of each JSON kind. +var streamTest = []interface{}{ + float64(0.1), + "hello", + nil, + true, + false, + []interface{}{"a", "b", "c"}, + map[string]interface{}{"K": "Kelvin", "ß": "long s"}, + float64(3.14), // another value to make sure something can follow map +} + +var streamEncoded = `0.1 +"hello" +null +true +false +["a","b","c"] +{"ß":"long s","K":"Kelvin"} +3.14 +` + +func TestEncoder(t *testing.T) { + for i := 0; i <= len(streamTest); i++ { + var buf bytes.Buffer + enc := NewEncoder(&buf) + for j, v := range streamTest[0:i] { + if err := enc.Encode(v); err != nil { + t.Fatalf("encode #%d: %v", j, err) + } + } + if have, want := buf.String(), nlines(streamEncoded, i); have != want { + t.Errorf("encoding %d items: mismatch", i) + diff(t, []byte(have), []byte(want)) + break + } + } +} + +func TestDecoder(t *testing.T) { + for i := 0; i <= len(streamTest); i++ { + // Use stream without newlines as input, + // just to stress the decoder even more. + // Our test input does not include back-to-back numbers. + // Otherwise stripping the newlines would + // merge two adjacent JSON values. + var buf bytes.Buffer + for _, c := range nlines(streamEncoded, i) { + if c != '\n' { + buf.WriteRune(c) + } + } + out := make([]interface{}, i) + dec := NewDecoder(&buf) + for j := range out { + if err := dec.Decode(&out[j]); err != nil { + t.Fatalf("decode #%d/%d: %v", j, i, err) + } + } + if !reflect.DeepEqual(out, streamTest[0:i]) { + t.Errorf("decoding %d items: mismatch") + for j := range out { + if !reflect.DeepEqual(out[j], streamTest[j]) { + t.Errorf("#%d: have %v want %v", out[j], streamTest[j]) + } + } + break + } + } +} + +func nlines(s string, n int) string { + if n <= 0 { + return "" + } + for i, c := range s { + if c == '\n' { + if n--; n == 0 { + return s[0 : i+1] + } + } + } + return s +} + +func TestRawMessage(t *testing.T) { + // TODO(rsc): Should not need the * in *RawMessage + var data struct { + X float64 + Id *RawMessage + Y float32 + } + const raw = `["\u0056",null]` + const msg = `{"X":0.1,"Id":["\u0056",null],"Y":0.2}` + err := Unmarshal([]byte(msg), &data) + if err != nil { + t.Fatalf("Unmarshal: %v", err) + } + if string(*data.Id) != raw { + t.Fatalf("Raw mismatch: have %#q want %#q", []byte(*data.Id), raw) + } + b, err := Marshal(&data) + if err != nil { + t.Fatalf("Marshal: %v", err) + } + if string(b) != msg { + t.Fatalf("Marshal: have %#q want %#q", b, msg) + } +}