1
0
mirror of https://github.com/golang/go synced 2024-11-21 19:04:44 -07:00

encoding/gob: reduce decoder memory

Gob decoding reads a whole message into memory and then
copies it into a bytes.Buffer. For large messages this wastes
an entire copy of the message. In this CL, we use a staging
buffer to avoid the large temporary.

Update #2539
RSS drops to 775MB from 1GB.
Active memory drops to 858317048 from 1027878136,
essentially the size of one copy of the input file.

R=dsymonds, nigeltao
CC=golang-dev
https://golang.org/cl/6392057
This commit is contained in:
Rob Pike 2012-07-12 20:53:17 -07:00
parent 1255a6302d
commit 37519d950d

View File

@ -87,21 +87,38 @@ func (dec *Decoder) recvMessage() bool {
// readMessage reads the next nbytes bytes from the input.
func (dec *Decoder) readMessage(nbytes int) {
// Allocate the buffer.
if cap(dec.tmp) < nbytes {
dec.tmp = make([]byte, nbytes+100) // room to grow
// Allocate the dec.tmp buffer, up to 10KB.
const maxBuf = 10 * 1024
nTmp := nbytes
if nTmp > maxBuf {
nTmp = maxBuf
}
dec.tmp = dec.tmp[:nbytes]
if cap(dec.tmp) < nTmp {
nAlloc := nTmp + 100 // A little extra for growth.
if nAlloc > maxBuf {
nAlloc = maxBuf
}
dec.tmp = make([]byte, nAlloc)
}
dec.tmp = dec.tmp[:nTmp]
// Read the data
_, dec.err = io.ReadFull(dec.r, dec.tmp)
if dec.err != nil {
if dec.err == io.EOF {
dec.err = io.ErrUnexpectedEOF
dec.buf.Grow(nbytes)
for nbytes > 0 {
if nbytes < nTmp {
dec.tmp = dec.tmp[:nbytes]
}
return
var nRead int
nRead, dec.err = io.ReadFull(dec.r, dec.tmp)
if dec.err != nil {
if dec.err == io.EOF {
dec.err = io.ErrUnexpectedEOF
}
return
}
dec.buf.Write(dec.tmp)
nbytes -= nRead
}
dec.buf.Write(dec.tmp)
}
// toInt turns an encoded uint64 into an int, according to the marshaling rules.