compress/flate: fix overflow on 2GB input. Reset hashOffset every 16 MB.

This bug has been introduced in the following revision:

changeset:   11404:26dceba5c610
user:        Ivan Krasin <krasin@golang.org>
date:        Mon Jan 23 09:19:39 2012 -0500
summary:     compress/flate: reduce memory pressure at cost of additional arithmetic operation.

This is the review page for that CL: https://golang.org/cl/5555070/

R=rsc, imkrasin
CC=golang-dev
https://golang.org/cl/6249067

src/pkg/compress/flate/deflate.go

@@ -32,6 +32,7 @@ const (
 	hashSize            = 1 << hashBits
 	hashMask            = (1 << hashBits) - 1
 	hashShift           = (hashBits + minMatchLength - 1) / minMatchLength
+	maxHashOffset       = 1 << 24
 
 	skipNever = math.MaxInt32
 )
@@ -106,6 +107,25 @@ func (d *compressor) fillDeflate(b []byte) int {
 			d.blockStart = math.MaxInt32
 		}
 		d.hashOffset += windowSize
+		if d.hashOffset > maxHashOffset {
+			delta := d.hashOffset - 1
+			d.hashOffset -= delta
+			d.chainHead -= delta
+			for i, v := range d.hashPrev {
+				if v > delta {
+					d.hashPrev[i] -= delta
+				} else {
+					d.hashPrev[i] = 0
+				}
+			}
+			for i, v := range d.hashHead {
+				if v > delta {
+					d.hashHead[i] -= delta
+				} else {
+					d.hashHead[i] = 0
+				}
+			}
+		}
 	}
 	n := copy(d.window[d.windowEnd:], b)
 	d.windowEnd += n

src/pkg/compress/flate/deflate_test.go

@@ -94,6 +94,50 @@ func TestDeflate(t *testing.T) {
 	}
 }
 
+// A sparseReader returns a stream consisting of 0s followed by 1<<16 1s.
+// This tests missing hash references in a very large input.
+type sparseReader struct {
+	l   int64
+	cur int64
+}
+
+func (r *sparseReader) Read(b []byte) (n int, err error) {
+	if r.cur >= r.l {
+		return 0, io.EOF
+	}
+	n = len(b)
+	cur := r.cur + int64(n)
+	if cur > r.l {
+		n -= int(cur - r.l)
+		cur = r.l
+	}
+	for i := range b[0:n] {
+		if r.cur+int64(i) >= r.l-1<<16 {
+			b[i] = 1
+		} else {
+			b[i] = 0
+		}
+	}
+	r.cur = cur
+	return
+}
+
+func TestVeryLongSparseChunk(t *testing.T) {
+	if testing.Short() {
+		t.Logf("skipping sparse chunk during short test")
+		return
+	}
+	w, err := NewWriter(ioutil.Discard, 1)
+	if err != nil {
+		t.Errorf("NewWriter: %v", err)
+		return
+	}
+	if _, err = io.Copy(w, &sparseReader{l: 23E8}); err != nil {
+		t.Errorf("Compress failed: %v", err)
+		return
+	}
+}
+
 type syncBuffer struct {
 	buf    bytes.Buffer
 	mu     sync.RWMutex