gob: better debugging, commentary

Re-implement the debugging helper to be independent of the existing
implementation.  This is preparatory to a rewrite to clean up issue 1416.
Include a definition of the grammar of the data stream.

R=rsc
CC=golang-dev
https://golang.org/cl/3970045

src/pkg/gob/codec_test.go

@@ -1307,6 +1307,31 @@ func TestUnexportedFields(t *testing.T) {
 	}
 }
 
+var singletons = []interface{}{
+	true,
+	7,
+	3.2,
+	"hello",
+	[3]int{11, 22, 33},
+	[]float32{0.5, 0.25, 0.125},
+	map[string]int{"one": 1, "two": 2},
+}
+
+func TestDebugSingleton(t *testing.T) {
+	if debugFunc == nil {
+		return
+	}
+	b := new(bytes.Buffer)
+	// Accumulate a number of values and print them out all at once.
+	for _, x := range singletons {
+		err := NewEncoder(b).Encode(x)
+		if err != nil {
+			t.Fatal("encode:", err)
+		}
+	}
+	debugFunc(b)
+}
+
 // A type that won't be defined in the gob until we send it in an interface value.
 type OnTheFly struct {
 	A int
@@ -1325,7 +1350,7 @@ type DT struct {
 	S     []string
 }
 
-func TestDebug(t *testing.T) {
+func TestDebugStruct(t *testing.T) {
 	if debugFunc == nil {
 		return
 	}

src/pkg/gob/debug.go

@@ -2,309 +2,683 @@ package gob
 
 // This file is not normally included in the gob package.  Used only for debugging the package itself.
 // Add debug.go to the files listed in the Makefile to add Debug to the gob package.
+// Except for reading uints, it is an implementation of a reader that is independent of
+// the one implemented by Decoder.
 
 import (
 	"bytes"
 	"fmt"
 	"io"
 	"os"
-	"reflect"
+	"strings"
-	"runtime"
+	"sync"
 )
 
-var dump = false // If true, print the remaining bytes in the input buffer at each item.
+var dumpBytes = false // If true, print the remaining bytes in the input buffer at each item.
 
 // Init installs the debugging facility. If this file is not compiled in the
-// package, the test in codec_test.go is a no-op.
+// package, the tests in codec_test.go are no-ops.
 func init() {
 	debugFunc = Debug
 }
 
+var (
+	blanks = bytes.Repeat([]byte{' '}, 3*10)
+	empty  = []byte(": <empty>\n")
+	tabs   = strings.Repeat("\t", 100)
+)
+
+// tab indents itself when printed.
+type tab int
+
+func (t tab) String() string {
+	n := int(t)
+	if n > len(tabs) {
+		n = len(tabs)
+	}
+	return tabs[0:n]
+}
+
+func (t tab) print() {
+	fmt.Fprint(os.Stderr, t)
+}
+
+// A peekReader wraps an io.Reader, allowing one to peek ahead to see
+// what's coming without stealing the data from the client of the Reader.
+type peekReader struct {
+	r    io.Reader
+	data []byte // read-ahead data
+}
+
+// newPeekReader returns a peekReader that wraps r.
+func newPeekReader(r io.Reader) *peekReader {
+	return &peekReader{r: r}
+}
+
+// Read is the usual method. It will first take data that has been read ahead.
+func (p *peekReader) Read(b []byte) (n int, err os.Error) {
+	if len(p.data) == 0 {
+		return p.r.Read(b)
+	}
+	// Satisfy what's possible from the read-ahead data.
+	n = copy(b, p.data)
+	// Move data down to beginning of slice, to avoid endless growth
+	copy(p.data, p.data[n:])
+	p.data = p.data[:len(p.data)-n]
+	return
+}
+
+// peek returns as many bytes as possible from the unread
+// portion of the stream, up to the length of b.
+func (p *peekReader) peek(b []byte) (n int, err os.Error) {
+	if len(p.data) > 0 {
+		n = copy(b, p.data)
+		if n == len(b) {
+			return
+		}
+		b = b[n:]
+	}
+	if len(b) == 0 {
+		return
+	}
+	m, e := io.ReadFull(p.r, b)
+	if m > 0 {
+		p.data = append(p.data, b[:m]...)
+	}
+	n += m
+	if e == io.ErrUnexpectedEOF {
+		// That means m > 0 but we reached EOF. If we got data
+		// we won't complain about not being able to peek enough.
+		if n > 0 {
+			e = nil
+		} else {
+			e = os.EOF
+		}
+	}
+	return n, e
+}
+
+// dump prints the next nBytes of the input.
+// It arranges to print the output aligned from call to
+// call, to make it easy to see what has been consumed.
+func (deb *debugger) dump(nBytes int, format string, args ...interface{}) {
+	if !dumpBytes {
+		return
+	}
+	fmt.Fprintf(os.Stderr, format+" ", args...)
+	if nBytes < 0 {
+		fmt.Fprintf(os.Stderr, "nbytes is negative! %d\n", nBytes)
+		return
+	}
+	data := make([]byte, nBytes)
+	n, _ := deb.r.peek(data)
+	if n == 0 {
+		os.Stderr.Write(empty)
+		return
+	}
+	b := new(bytes.Buffer)
+	fmt.Fprint(b, "{\n")
+	// Blanks until first byte
+	lineLength := 0
+	if n := len(data); n%10 != 0 {
+		lineLength = 10 - n%10
+		fmt.Fprintf(b, "\t%s", blanks[:lineLength*3])
+	}
+	// 10 bytes per line
+	for len(data) > 0 {
+		if lineLength == 0 {
+			fmt.Fprint(b, "\t")
+		}
+		m := 10 - lineLength
+		lineLength = 0
+		if m > len(data) {
+			m = len(data)
+		}
+		fmt.Fprintf(b, "% x\n", data[:m])
+		data = data[m:]
+	}
+	fmt.Fprint(b, "}\n")
+	os.Stderr.Write(b.Bytes())
+}
+
+type debugger struct {
+	mutex    sync.Mutex
+	r        *peekReader
+	wireType map[typeId]*wireType
+	tmp      []byte // scratch space for decoding uints.
+}
+
 // Debug prints a human-readable representation of the gob data read from r.
 func Debug(r io.Reader) {
-	defer func() {
+	fmt.Fprintln(os.Stderr, "Start of debugging")
-		if e := recover(); e != nil {
+	deb := &debugger{
-			if _, ok := e.(runtime.Error); ok {
+		r:        newPeekReader(r),
-				panic(e)
+		wireType: make(map[typeId]*wireType),
-			}
+		tmp:      make([]byte, 16),
-			fmt.Printf("error during debugging: %v\n", e)
+	}
-		}
+	deb.gobStream()
-	}()
-	NewDecoder(r).debug()
 }
 
-// debugRecv is like recv but prints what it sees.
+// toInt turns an encoded uint64 into an int, according to the marshaling rules.
-func (dec *Decoder) debugRecv() {
+func toInt(x uint64) int64 {
-	if dec.byteBuffer != nil && dec.byteBuffer.Len() != 0 {
+	i := int64(x >> 1)
-		fmt.Printf("error in recv: %d bytes left in input buffer\n", dec.byteBuffer.Len())
+	if x&1 != 0 {
-		return
+		i = ^i
-	}
-	// Read a count.
-	var nbytes uint64
-	nbytes, dec.err = decodeUintReader(dec.r, dec.countBuf[0:])
-	if dec.err != nil {
-		fmt.Printf("receiver error on count: %s\n", dec.err)
-		return
-	}
-	// Allocate the buffer.
-	if nbytes > uint64(len(dec.buf)) {
-		dec.buf = make([]byte, nbytes+1000)
-	}
-	dec.byteBuffer = bytes.NewBuffer(dec.buf[0:nbytes])
-
-	// Read the data
-	_, dec.err = io.ReadFull(dec.r, dec.buf[0:nbytes])
-	if dec.err != nil {
-		fmt.Printf("receiver error on data: %s\n", dec.err)
-		if dec.err == os.EOF {
-			dec.err = io.ErrUnexpectedEOF
-		}
-		return
-	}
-	if dump {
-		fmt.Printf("received %d bytes:\n\t% x\n", nbytes, dec.byteBuffer.Bytes())
 	}
+	return i
 }
 
+// readInt returns the next int, which must be present,
+// and the number of bytes it consumed.
+// Don't call this if you could be at EOF.
+func (deb *debugger) readInt() (i int64, w int) {
+	var u uint64
+	u, w = deb.readUint()
+	return toInt(u), w
+}
 
-// debug is like Decode but just prints what it finds.  It should be safe even for corrupted data.
+// readUint returns the next uint, which must be present.
-func (dec *Decoder) debug() {
+// and the number of bytes it consumed.
+// Don't call this if you could be at EOF.
+// TODO: handle errors better.
+func (deb *debugger) readUint() (x uint64, w int) {
+	n, w, err := decodeUintReader(deb.r, deb.tmp)
+	if err != nil {
+		errorf("debug: read error: %s", err)
+	}
+	return n, w
+}
+
+// GobStream:
+//	DelimitedMessage* (until EOF)
+func (deb *debugger) gobStream() {
 	// Make sure we're single-threaded through here.
-	dec.mutex.Lock()
+	deb.mutex.Lock()
-	defer dec.mutex.Unlock()
+	defer deb.mutex.Unlock()
 
-	dec.err = nil
+	for deb.delimitedMessage(0) {
-	dec.debugRecv()
-	if dec.err != nil {
-		return
 	}
-	dec.debugFromBuffer(0, false)
 }
 
-// printFromBuffer prints the next value.  The buffer contains data, but it may
+// DelimitedMessage:
-// be a type descriptor and we may need to load more data to see the value;
+//	uint(lengthOfMessage) Message
-// printType takes care of that.
+func (deb *debugger) delimitedMessage(indent tab) bool {
-func (dec *Decoder) debugFromBuffer(indent int, countPresent bool) {
+	for {
-	for dec.state.b.Len() > 0 {
+		n := deb.loadBlock(true)
-		// Receive a type id.
+		if n < 0 {
-		id := typeId(dec.state.decodeInt())
+			return false
-
-		// Is it a new type?
-		if id < 0 { // 0 is the error state, handled above
-			// If the id is negative, we have a type.
-			dec.debugRecvType(-id)
-			if dec.err != nil {
-				break
-			}
-			continue
 		}
+		deb.dump(int(n), "Message of length %d", n)
+		deb.message(indent, n)
+	}
+	return true
+}
 
-		// No, it's a value.
+// loadBlock preps us to read a message
-		// Make sure the type has been defined already or is a builtin type (for
+// of the length specified next in the input. It returns
-		// top-level singleton values).
+// the length of the block. The argument tells whether
-		if dec.wireType[id] == nil && builtinIdToType[id] == nil {
+// an EOF is acceptable now.  If it is and one is found,
-			dec.err = errBadType
+// the return value is negative.
+func (deb *debugger) loadBlock(eofOK bool) int {
+	n64, _, err := decodeUintReader(deb.r, deb.tmp)
+	if err != nil {
+		if eofOK && err == os.EOF {
+			return -1
+		}
+		errorf("debug: unexpected error: %s", err)
+	}
+	n := int(n64)
+	if n < 0 {
+		errorf("huge value for message length: %d", n64)
+	}
+	return n
+}
+
+// Message:
+//	TypeSequence TypedValue
+// TypeSequence
+//	(TypeDefinition DelimitedTypeDefinition*)?
+// DelimitedTypeDefinition:
+//	uint(lengthOfTypeDefinition) TypeDefinition
+// TypedValue:
+//	int(typeId) Value
+func (deb *debugger) message(indent tab, n int) bool {
+	for {
+		// Convert the uint64 to a signed integer typeId
+		uid, w := deb.readInt()
+		id := typeId(uid)
+		n -= w
+		deb.dump(n, "type id=%d", id)
+		if id < 0 {
+			n -= deb.typeDefinition(indent, -id, n)
+			n = deb.loadBlock(false)
+			deb.dump(n, "Message of length %d", n)
+			continue
+		} else {
+			deb.value(indent, id, n)
 			break
 		}
-		if countPresent {
-			dec.state.decodeUint()
-		}
-		dec.debugPrint(indent, id)
-		break
 	}
+	return true
 }
 
-func (dec *Decoder) debugRecvType(id typeId) {
+// TypeDefinition:
-	// Have we already seen this type?  That's an error
+//	[int(-typeId) (already read)] encodingOfWireType
-	if _, alreadySeen := dec.wireType[id]; alreadySeen {
+func (deb *debugger) typeDefinition(indent tab, id typeId, n int) int {
-		dec.err = os.ErrorString("gob: duplicate type received")
+	deb.dump(n, "type definition for id %d", id)
-		return
+	// Encoding is of a wireType. Decode the structure as usual
+	fieldNum := -1
+	m := 0
+
+	// Closures to make it easy to scan.
+
+	// Read a uint from the input
+	getUint := func() uint {
+		i, w := deb.readUint()
+		m += w
+		n -= w
+		return uint(i)
+	}
+	// Read an int from the input
+	getInt := func() int {
+		i, w := deb.readInt()
+		m += w
+		n -= w
+		return int(i)
+	}
+	// Read a string from the input
+	getString := func() string {
+		u, w := deb.readUint()
+		x := int(u)
+		m += w
+		n -= w
+		b := make([]byte, x)
+		nb, _ := deb.r.Read(b)
+		if nb != x {
+			errorf("corrupted type")
+		}
+		m += x
+		n -= x
+		return string(b)
+	}
+	// Read a typeId from the input
+	getTypeId := func() typeId {
+		return typeId(getInt())
+	}
+	// Read a delta from the input.
+	getDelta := func(expect int) int {
+		u, w := deb.readUint()
+		m += w
+		n -= w
+		delta := int(u)
+		if delta < 0 || (expect >= 0 && delta != expect) {
+			errorf("gob decode: corrupted type: delta %d expected %d", delta, expect)
+		}
+		return int(u)
+	}
+	// Read a CommonType from the input
+	common := func() CommonType {
+		fieldNum := -1
+		name := ""
+		id := typeId(0)
+		for {
+			delta := getDelta(-1)
+			if delta == 0 {
+				break
+			}
+			fieldNum += delta
+			switch fieldNum {
+			case 0:
+				name = getString()
+			case 1:
+				// Id typeId
+				id = getTypeId()
+			default:
+				errorf("corrupted CommonType")
+			}
+		}
+		return CommonType{name, id}
 	}
 
-	// Type:
 	wire := new(wireType)
-	dec.err = dec.decode(tWireType, reflect.NewValue(wire))
+	// A wireType defines a single field.
-	if dec.err == nil {
+	delta := getDelta(-1)
-		printWireType(wire)
+	fieldNum += delta
-	}
+	switch fieldNum {
-	// Remember we've seen this type.
+	case 0: // array type, one field of {{Common}, elem, length}
-	dec.wireType[id] = wire
+		// Field number 0 is CommonType
+		getDelta(1)
+		com := common()
+		// Field number 1 is type Id of elem
+		getDelta(1)
+		id := getTypeId()
+		// Field number 3 is length
+		getDelta(1)
+		length := getInt()
+		wire.ArrayT = &arrayType{com, id, length}
 
-	// Load the next parcel.
+	case 1: // slice type, one field of {{Common}, elem}
-	dec.debugRecv()
+		// Field number 0 is CommonType
-}
+		getDelta(1)
+		com := common()
+		// Field number 1 is type Id of elem
+		getDelta(1)
+		id := getTypeId()
+		wire.SliceT = &sliceType{com, id}
 
-func printWireType(wire *wireType) {
+	case 2: // struct type, one field of {{Common}, []fieldType}
-	fmt.Printf("type definition {\n")
+		// Field number 0 is CommonType
-	switch {
+		getDelta(1)
-	case wire.ArrayT != nil:
+		com := common()
-		printCommonType("array", &wire.ArrayT.CommonType)
+		// Field number 1 is slice of FieldType
-		fmt.Printf("\tlen %d\n\telemid %d\n", wire.ArrayT.Len, wire.ArrayT.Elem)
+		getDelta(1)
-	case wire.MapT != nil:
+		numField := int(getUint())
-		printCommonType("map", &wire.MapT.CommonType)
+		field := make([]*fieldType, numField)
-		fmt.Printf("\tkeyid %d\n", wire.MapT.Key)
+		for i := 0; i < numField; i++ {
-		fmt.Printf("\telemid %d\n", wire.MapT.Elem)
+			field[i] = new(fieldType)
-	case wire.SliceT != nil:
+			getDelta(1) // field 0 of fieldType: name
-		printCommonType("slice", &wire.SliceT.CommonType)
+			field[i].Name = getString()
-		fmt.Printf("\telemid %d\n", wire.SliceT.Elem)
+			getDelta(1) // field 1 of fieldType: id
-	case wire.StructT != nil:
+			field[i].Id = getTypeId()
-		printCommonType("struct", &wire.StructT.CommonType)
+			getDelta(0) // end of fieldType
-		for i, field := range wire.StructT.Field {
-			fmt.Printf("\tfield %d:\t%s\tid=%d\n", i, field.Name, field.Id)
 		}
+		wire.StructT = &structType{com, field}
+
+	case 3: // map type, one field of {{Common}, key, elem}
+		// Field number 0 is CommonType
+		getDelta(1)
+		com := common()
+		// Field number 1 is type Id of key
+		getDelta(1)
+		keyId := getTypeId()
+		wire.SliceT = &sliceType{com, id}
+		// Field number 2 is type Id of elem
+		getDelta(1)
+		elemId := getTypeId()
+		wire.MapT = &mapType{com, keyId, elemId}
+	default:
+		errorf("bad field in type %d", fieldNum)
 	}
-	fmt.Printf("}\n")
+	deb.printWireType(indent, wire)
+	getDelta(0) // end inner type (arrayType, etc.)
+	getDelta(0) // end wireType
+	// Remember we've seen this type.
+	deb.wireType[id] = wire
+	return m
 }
 
-func printCommonType(kind string, common *CommonType) {
+
-	fmt.Printf("\t%s %q\n\tid: %d\n", kind, common.Name, common.Id)
+// Value:
+//	ConcreteValue | InterfaceValue
+func (deb *debugger) value(indent tab, id typeId, n int) int {
+	if id == tInterface {
+		return deb.interfaceValue(indent, n)
+	}
+	return deb.concreteValue(indent, id, n)
 }
 
-func (dec *Decoder) debugPrint(indent int, id typeId) {
+
-	wire, ok := dec.wireType[id]
+// ConcreteValue:
+//	SingletonValue | StructValue
+func (deb *debugger) concreteValue(indent tab, id typeId, n int) int {
+	wire, ok := deb.wireType[id]
 	if ok && wire.StructT != nil {
-		dec.debugStruct(indent+1, id, wire)
+		return deb.structValue(indent, id, n)
-	} else {
-		dec.debugSingle(indent+1, id, wire)
 	}
+	return deb.singletonValue(indent, id, n)
 }
 
-func (dec *Decoder) debugSingle(indent int, id typeId, wire *wireType) {
+// SingletonValue:
+//	int(0) FieldValue
+func (deb *debugger) singletonValue(indent tab, id typeId, n int) int {
+	deb.dump(n, "Singleton value")
 	// is it a builtin type?
+	wire := deb.wireType[id]
 	_, ok := builtinIdToType[id]
 	if !ok && wire == nil {
-		errorf("type id %d not defined\n", id)
+		errorf("type id %d not defined", id)
 	}
-	dec.state.decodeUint()
+	m, w := deb.readInt()
-	dec.printItem(indent, id)
+	if m != 0 {
+		errorf("expected zero; got %d", n)
+	}
+	return w + deb.fieldValue(indent, id, n-w)
 }
 
-func (dec *Decoder) printItem(indent int, id typeId) {
+// InterfaceValue:
-	if dump {
+//	NilInterfaceValue | NonNilInterfaceValue
-		fmt.Printf("print item %d bytes: % x\n", dec.state.b.Len(), dec.state.b.Bytes())
+func (deb *debugger) interfaceValue(indent tab, n int) int {
+	deb.dump(n, "Start of interface value")
+	nameLen, w := deb.readUint()
+	n -= w
+	if n == 0 {
+		return w + deb.nilInterfaceValue(indent)
 	}
+	return w + deb.nonNilInterfaceValue(indent, int(nameLen), n)
+}
+
+// NilInterfaceValue:
+//	uint(0) [already read]
+func (deb *debugger) nilInterfaceValue(indent tab) int {
+	fmt.Fprintf(os.Stderr, "%snil interface\n", indent)
+	return 0
+}
+
+
+// NonNilInterfaceValue:
+//	ConcreteTypeName TypeSequence InterfaceContents
+// ConcreteTypeName:
+//	uint(lengthOfName) [already read=n] name
+// InterfaceContents:
+//	int(concreteTypeId) DelimitedValue
+// DelimitedValue:
+//	uint(length) Value
+func (deb *debugger) nonNilInterfaceValue(indent tab, nameLen, n int) int {
+	// ConcreteTypeName
+	b := make([]byte, nameLen)
+	deb.r.Read(b) // TODO: CHECK THESE READS!!
+	w := nameLen
+	n -= nameLen
+	name := string(b)
+	fmt.Fprintf(os.Stderr, "%sinterface value, type %q length %d\n", indent, name, n)
+
+	for {
+		x, width := deb.readInt()
+		n -= w
+		w += width
+		id := typeId(x)
+		if id < 0 {
+			deb.typeDefinition(indent, -id, n)
+			n = deb.loadBlock(false)
+			deb.dump(n, "Message of length %d", n)
+		} else {
+			// DelimitedValue
+			x, width := deb.readUint() // in case we want to ignore the value; we don't.
+			n -= w
+			w += width
+			fmt.Fprintf(os.Stderr, "%sinterface value, type %q id=%d; length %d\n", indent, name, id, x)
+			ZZ := w + deb.value(indent, id, int(x))
+			return ZZ
+		}
+	}
+	panic("not reached")
+}
+
+// printCommonType prints a common type; used by printWireType.
+func (deb *debugger) printCommonType(indent tab, kind string, common *CommonType) {
+	indent.print()
+	fmt.Fprintf(os.Stderr, "%s %q id=%d\n", kind, common.Name, common.Id)
+}
+
+// printWireType prints the contents of a wireType.
+func (deb *debugger) printWireType(indent tab, wire *wireType) {
+	fmt.Fprintf(os.Stderr, "%stype definition {\n", indent)
+	indent++
+	switch {
+	case wire.ArrayT != nil:
+		deb.printCommonType(indent, "array", &wire.ArrayT.CommonType)
+		fmt.Fprintf(os.Stderr, "%slen %d\n", indent+1, wire.ArrayT.Len)
+		fmt.Fprintf(os.Stderr, "%selemid %d\n", indent+1, wire.ArrayT.Elem)
+	case wire.MapT != nil:
+		deb.printCommonType(indent, "map", &wire.MapT.CommonType)
+		fmt.Fprintf(os.Stderr, "%skey id=%d\n", indent+1, wire.MapT.Key)
+		fmt.Fprintf(os.Stderr, "%selem id=%d\n", indent+1, wire.MapT.Elem)
+	case wire.SliceT != nil:
+		deb.printCommonType(indent, "slice", &wire.SliceT.CommonType)
+		fmt.Fprintf(os.Stderr, "%selem id=%d\n", indent+1, wire.SliceT.Elem)
+	case wire.StructT != nil:
+		deb.printCommonType(indent, "struct", &wire.StructT.CommonType)
+		for i, field := range wire.StructT.Field {
+			fmt.Fprintf(os.Stderr, "%sfield %d:\t%s\tid=%d\n", indent+1, i, field.Name, field.Id)
+		}
+	}
+	indent--
+	fmt.Fprintf(os.Stderr, "%s}\n", indent)
+}
+
+// fieldValue prints a value of any type, such as a struct field.
+func (deb *debugger) fieldValue(indent tab, id typeId, n int) int {
 	_, ok := builtinIdToType[id]
 	if ok {
-		dec.printBuiltin(indent, id)
+		return deb.printBuiltin(indent, id, n)
-		return
 	}
-	wire, ok := dec.wireType[id]
+	wire, ok := deb.wireType[id]
 	if !ok {
-		errorf("type id %d not defined\n", id)
+		errorf("type id %d not defined", id)
 	}
 	switch {
 	case wire.ArrayT != nil:
-		dec.printArray(indent, wire)
+		return deb.arrayValue(indent, wire, n)
 	case wire.MapT != nil:
-		dec.printMap(indent, wire)
+		return deb.mapValue(indent, wire, n)
 	case wire.SliceT != nil:
-		dec.printSlice(indent, wire)
+		return deb.sliceValue(indent, wire, n)
 	case wire.StructT != nil:
-		dec.debugStruct(indent, id, wire)
+		return deb.structValue(indent, id, n)
 	}
+	panic("unreached")
 }
 
-func (dec *Decoder) printArray(indent int, wire *wireType) {
+// printBuiltin prints a value not of a fundamental type, that is,
-	elemId := wire.ArrayT.Elem
+// one whose type is known to gobs at bootstrap time.
-	n := int(dec.state.decodeUint())
+// That includes interfaces, although they may require
-	for i := 0; i < n && dec.err == nil; i++ {
+// more unpacking down the line.
-		dec.printItem(indent, elemId)
+func (deb *debugger) printBuiltin(indent tab, id typeId, n int) int {
-	}
-	if n != wire.ArrayT.Len {
-		tab(indent)
-		fmt.Printf("(wrong length for array: %d should be %d)\n", n, wire.ArrayT.Len)
-	}
-}
-
-func (dec *Decoder) printMap(indent int, wire *wireType) {
-	keyId := wire.MapT.Key
-	elemId := wire.MapT.Elem
-	n := int(dec.state.decodeUint())
-	for i := 0; i < n && dec.err == nil; i++ {
-		dec.printItem(indent, keyId)
-		dec.printItem(indent+1, elemId)
-	}
-}
-
-func (dec *Decoder) printSlice(indent int, wire *wireType) {
-	elemId := wire.SliceT.Elem
-	n := int(dec.state.decodeUint())
-	for i := 0; i < n && dec.err == nil; i++ {
-		dec.printItem(indent, elemId)
-	}
-}
-
-func (dec *Decoder) printBuiltin(indent int, id typeId) {
-	tab(indent)
 	switch id {
 	case tBool:
-		if dec.state.decodeInt() == 0 {
+		x, w := deb.readInt()
-			fmt.Printf("false\n")
+		if x == 0 {
+			fmt.Fprintf(os.Stderr, "%sfalse\n", indent)
 		} else {
-			fmt.Printf("true\n")
+			fmt.Fprintf(os.Stderr, "%strue\n", indent)
 		}
+		return w
 	case tInt:
-		fmt.Printf("%d\n", dec.state.decodeInt())
+		x, w := deb.readInt()
+		fmt.Fprintf(os.Stderr, "%s%d\n", indent, x)
+		return w
 	case tUint:
-		fmt.Printf("%d\n", dec.state.decodeUint())
+		x, w := deb.readInt()
+		fmt.Fprintf(os.Stderr, "%s%d\n", indent, x)
+		return w
 	case tFloat:
-		fmt.Printf("%g\n", floatFromBits(dec.state.decodeUint()))
+		x, w := deb.readUint()
+		fmt.Fprintf(os.Stderr, "%s%g\n", indent, floatFromBits(x))
+		return w
 	case tBytes:
-		b := make([]byte, dec.state.decodeUint())
+		x, w := deb.readUint()
-		dec.state.b.Read(b)
+		b := make([]byte, x)
-		fmt.Printf("% x\n", b)
+		deb.r.Read(b)
+		fmt.Fprintf(os.Stderr, "%s{% x}=%q\n", indent, b, b)
+		return w + int(x)
 	case tString:
-		b := make([]byte, dec.state.decodeUint())
+		x, w := deb.readUint()
-		dec.state.b.Read(b)
+		b := make([]byte, x)
-		fmt.Printf("%q\n", b)
+		deb.r.Read(b)
+		fmt.Fprintf(os.Stderr, "%s%q\n", indent, b)
+		return w + int(x)
 	case tInterface:
-		b := make([]byte, dec.state.decodeUint())
+		return deb.interfaceValue(indent, n)
-		dec.state.b.Read(b)
-		if len(b) == 0 {
-			fmt.Printf("nil interface")
-		} else {
-			fmt.Printf("interface value; type %q\n", b)
-			dec.debugFromBuffer(indent, true)
-		}
 	default:
 		fmt.Print("unknown\n")
 	}
+	panic("unknown builtin")
 }
 
-func (dec *Decoder) debugStruct(indent int, id typeId, wire *wireType) {
+
-	tab(indent)
+// ArrayValue:
-	fmt.Printf("%s struct {\n", id.name())
+//	uint(n) Value*n
+func (deb *debugger) arrayValue(indent tab, wire *wireType, n int) int {
+	elemId := wire.ArrayT.Elem
+	u, w := deb.readUint()
+	length := int(u)
+	for i := 0; i < length; i++ {
+		w += deb.fieldValue(indent, elemId, n-w)
+	}
+	if length != wire.ArrayT.Len {
+		fmt.Fprintf(os.Stderr, "%s(wrong length for array: %d should be %d)\n", indent, length, wire.ArrayT.Len)
+	}
+	return w
+}
+
+// MapValue:
+//	uint(n) (Value Value)*n  [n (key, value) pairs]
+func (deb *debugger) mapValue(indent tab, wire *wireType, n int) int {
+	keyId := wire.MapT.Key
+	elemId := wire.MapT.Elem
+	u, w := deb.readUint()
+	length := int(u)
+	for i := 0; i < length; i++ {
+		w += deb.fieldValue(indent+1, keyId, n-w)
+		w += deb.fieldValue(indent+1, elemId, n-w)
+	}
+	return w
+}
+
+// SliceValue:
+//	uint(n) (n Values)
+func (deb *debugger) sliceValue(indent tab, wire *wireType, n int) int {
+	elemId := wire.SliceT.Elem
+	u, w := deb.readUint()
+	length := int(u)
+	for i := 0; i < length; i++ {
+		w += deb.fieldValue(indent, elemId, n-w)
+	}
+	return w
+}
+
+// StructValue:
+//	(int(fieldDelta) FieldValue)*
+func (deb *debugger) structValue(indent tab, id typeId, n int) int {
+	deb.dump(n, "Start of struct value of %q id=%d\n<<\n", id.name(), id)
+	fmt.Fprintf(os.Stderr, "%s%s struct {\n", indent, id.name())
+	wire, ok := deb.wireType[id]
+	if !ok {
+		errorf("type id %d not defined", id)
+	}
 	strct := wire.StructT
-	state := newDecodeState(dec, dec.state.b)
+	fieldNum := -1
-	state.fieldnum = -1
+	indent++
-	for dec.err == nil {
+	w := 0
-		delta := int(state.decodeUint())
+	for {
-		if delta < 0 {
+		delta, wid := deb.readUint()
-			errorf("gob decode: corrupted data: negative delta")
+		w += wid
-		}
+		n -= wid
 		if delta == 0 { // struct terminator is zero delta fieldnum
 			break
 		}
-		fieldNum := state.fieldnum + delta
+		fieldNum += int(delta)
 		if fieldNum < 0 || fieldNum >= len(strct.Field) {
-			errorf("field number out of range")
+			deb.dump(n, "field number out of range: prevField=%d delta=%d", fieldNum-int(delta), delta)
 			break
 		}
-		tab(indent)
+		fmt.Fprintf(os.Stderr, "%sfield %d:\t%s\n", indent, fieldNum, wire.StructT.Field[fieldNum].Name)
-		fmt.Printf("%s(%d):\n", wire.StructT.Field[fieldNum].Name, fieldNum)
+		wid = deb.fieldValue(indent+1, strct.Field[fieldNum].Id, n)
-		dec.printItem(indent+1, strct.Field[fieldNum].Id)
+		w += wid
-		state.fieldnum = fieldNum
+		n -= wid
-	}
-	tab(indent)
-	fmt.Printf(" } // end %s struct\n", id.name())
-}
-
-func tab(indent int) {
-	for i, w := 0, 0; i < indent; i += w {
-		w = 10
-		if i+w > indent {
-			w = indent - i
-		}
-		fmt.Print("\t\t\t\t\t\t\t\t\t\t"[:w])
 	}
+	indent--
+	fmt.Fprintf(os.Stderr, "%s} // end %s struct\n", indent, id.name())
+	deb.dump(n, ">> End of struct value of type %d %q", id, id.name())
+	return w
 }

src/pkg/gob/decode.go

@@ -49,14 +49,15 @@ func overflow(name string) os.ErrorString {
 
 // decodeUintReader reads an encoded unsigned integer from an io.Reader.
 // Used only by the Decoder to read the message length.
-func decodeUintReader(r io.Reader, buf []byte) (x uint64, err os.Error) {
+func decodeUintReader(r io.Reader, buf []byte) (x uint64, width int, err os.Error) {
-	_, err = r.Read(buf[0:1])
+	width = 1
+	_, err = r.Read(buf[0:width])
 	if err != nil {
 		return
 	}
 	b := buf[0]
 	if b <= 0x7f {
-		return uint64(b), nil
+		return uint64(b), width, nil
 	}
 	nb := -int(int8(b))
 	if nb > uint64Size {
@@ -75,6 +76,7 @@ func decodeUintReader(r io.Reader, buf []byte) (x uint64, err os.Error) {
 	for i := 0; i < n; i++ {
 		x <<= 8
 		x |= uint64(buf[i])
+		width++
 	}
 	return
 }

src/pkg/gob/decoder.go

@@ -58,7 +58,7 @@ func (dec *Decoder) recvType(id typeId) {
 	dec.wireType[id] = wire
 
 	// Load the next parcel.
-	dec.recv()
+	dec.recvMessage()
 }
 
 // Decode reads the next value from the connection and stores
@@ -76,23 +76,28 @@ func (dec *Decoder) Decode(e interface{}) os.Error {
 	return dec.DecodeValue(value)
 }
 
-// recv reads the next count-delimited item from the input. It is the converse
+// recvMessage reads the next count-delimited item from the input. It is the converse
-// of Encoder.send.
+// of Encoder.writeMessage.
-func (dec *Decoder) recv() {
+func (dec *Decoder) recvMessage() {
 	// Read a count.
 	var nbytes uint64
-	nbytes, dec.err = decodeUintReader(dec.r, dec.countBuf[0:])
+	nbytes, _, dec.err = decodeUintReader(dec.r, dec.countBuf[0:])
 	if dec.err != nil {
 		return
 	}
+	dec.readMessage(int(nbytes), dec.r)
+}
+
+// readMessage reads the next nbytes bytes from the input.
+func (dec *Decoder) readMessage(nbytes int, r io.Reader) {
 	// Allocate the buffer.
-	if nbytes > uint64(len(dec.buf)) {
+	if nbytes > len(dec.buf) {
 		dec.buf = make([]byte, nbytes+1000)
 	}
 	dec.byteBuffer = bytes.NewBuffer(dec.buf[0:nbytes])
 
 	// Read the data
-	_, dec.err = io.ReadFull(dec.r, dec.buf[0:nbytes])
+	_, dec.err = io.ReadFull(r, dec.buf[0:nbytes])
 	if dec.err != nil {
 		if dec.err == os.EOF {
 			dec.err = io.ErrUnexpectedEOF
@@ -103,7 +108,7 @@ func (dec *Decoder) recv() {
 
 // decodeValueFromBuffer grabs the next value from the input. The Decoder's
 // buffer already contains data.  If the next item in the buffer is a type
-// descriptor, it may be necessary to reload the buffer, but recvType does that.
+// descriptor, it will be necessary to reload the buffer; recvType does that.
 func (dec *Decoder) decodeValueFromBuffer(value reflect.Value, ignoreInterfaceValue, countPresent bool) {
 	for dec.state.b.Len() > 0 {
 		// Receive a type id.
@@ -150,7 +155,7 @@ func (dec *Decoder) DecodeValue(value reflect.Value) os.Error {
 	defer dec.mutex.Unlock()
 
 	dec.err = nil
-	dec.recv()
+	dec.recvMessage()
 	if dec.err != nil {
 		return dec.err
 	}

src/pkg/gob/doc.go

@@ -219,6 +219,54 @@ be predefined or be defined before the value in the stream.
 */
 package gob
 
+/*
+Grammar:
+
+Tokens starting with a lower case letter are terminals; int(n)
+and uint(n) represent the signed/unsigned encodings of the value n.
+
+GobStream:
+	DelimitedMessage*
+DelimitedMessage:
+	uint(lengthOfMessage) Message
+Message:
+	TypeSequence TypedValue
+TypeSequence
+	(TypeDefinition DelimitedTypeDefinition*)?
+DelimitedTypeDefinition:
+	uint(lengthOfTypeDefinition) TypeDefinition
+TypedValue:
+	int(typeId) Value
+TypeDefinition:
+	int(-typeId) encodingOfWireType
+Value:
+	ConcreteValue | InterfaceValue
+ConcreteValue:
+	SingletonValue | StructValue
+SingletonValue:
+	int(0) FieldValue
+InterfaceValue:
+	NilInterfaceValue | NonNilInterfaceValue
+NilInterfaceValue:
+	uint(0)
+NonNilInterfaceValue:
+	ConcreteTypeName TypeSequence InterfaceContents
+ConcreteTypeName:
+	uint(lengthOfName) [already read=n] name
+InterfaceContents:
+	int(concreteTypeId) DelimitedValue
+DelimitedValue:
+	uint(length) Value
+ArrayValue:
+	uint(n) Value*n [n elements]
+MapValue:
+	uint(n) (Value Value)*n  [n (key, value) pairs]
+SliceValue:
+	uint(n) Value*n [n elements]
+StructValue:
+	(uint(fieldDelta) FieldValue)*
+*/
+
 /*
 For implementers and the curious, here is an encoded example.  Given
 	type Point struct {x, y int}