mirror of
https://github.com/golang/go
synced 2024-11-12 03:00:22 -07:00
compress/zlib: add FDICT flag in Reader/Writer
R=bradfitzgo, rsc, bradfitzwork CC=golang-dev https://golang.org/cl/4406046
This commit is contained in:
parent
a12d70f60d
commit
bfb486875a
@ -36,7 +36,7 @@ const zlibDeflate = 8
|
||||
|
||||
var ChecksumError os.Error = os.ErrorString("zlib checksum error")
|
||||
var HeaderError os.Error = os.ErrorString("invalid zlib header")
|
||||
var UnsupportedError os.Error = os.ErrorString("unsupported zlib format")
|
||||
var DictionaryError os.Error = os.ErrorString("invalid zlib dictionary")
|
||||
|
||||
type reader struct {
|
||||
r flate.Reader
|
||||
@ -50,6 +50,12 @@ type reader struct {
|
||||
// The implementation buffers input and may read more data than necessary from r.
|
||||
// It is the caller's responsibility to call Close on the ReadCloser when done.
|
||||
func NewReader(r io.Reader) (io.ReadCloser, os.Error) {
|
||||
return NewReaderDict(r, nil)
|
||||
}
|
||||
|
||||
// NewReaderDict is like NewReader but uses a preset dictionary.
|
||||
// NewReaderDict ignores the dictionary if the compressed data does not refer to it.
|
||||
func NewReaderDict(r io.Reader, dict []byte) (io.ReadCloser, os.Error) {
|
||||
z := new(reader)
|
||||
if fr, ok := r.(flate.Reader); ok {
|
||||
z.r = fr
|
||||
@ -65,11 +71,19 @@ func NewReader(r io.Reader) (io.ReadCloser, os.Error) {
|
||||
return nil, HeaderError
|
||||
}
|
||||
if z.scratch[1]&0x20 != 0 {
|
||||
// BUG(nigeltao): The zlib package does not implement the FDICT flag.
|
||||
return nil, UnsupportedError
|
||||
_, err = io.ReadFull(z.r, z.scratch[0:4])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
checksum := uint32(z.scratch[0])<<24 | uint32(z.scratch[1])<<16 | uint32(z.scratch[2])<<8 | uint32(z.scratch[3])
|
||||
if checksum != adler32.Checksum(dict) {
|
||||
return nil, DictionaryError
|
||||
}
|
||||
z.decompressor = flate.NewReaderDict(z.r, dict)
|
||||
} else {
|
||||
z.decompressor = flate.NewReader(z.r)
|
||||
}
|
||||
z.digest = adler32.New()
|
||||
z.decompressor = flate.NewReader(z.r)
|
||||
return z, nil
|
||||
}
|
||||
|
||||
|
@ -15,6 +15,7 @@ type zlibTest struct {
|
||||
desc string
|
||||
raw string
|
||||
compressed []byte
|
||||
dict []byte
|
||||
err os.Error
|
||||
}
|
||||
|
||||
@ -27,6 +28,7 @@ var zlibTests = []zlibTest{
|
||||
"",
|
||||
[]byte{0x78, 0x9c, 0x03, 0x00, 0x00, 0x00, 0x00, 0x01},
|
||||
nil,
|
||||
nil,
|
||||
},
|
||||
{
|
||||
"goodbye",
|
||||
@ -37,23 +39,27 @@ var zlibTests = []zlibTest{
|
||||
0x01, 0x00, 0x28, 0xa5, 0x05, 0x5e,
|
||||
},
|
||||
nil,
|
||||
nil,
|
||||
},
|
||||
{
|
||||
"bad header",
|
||||
"",
|
||||
[]byte{0x78, 0x9f, 0x03, 0x00, 0x00, 0x00, 0x00, 0x01},
|
||||
nil,
|
||||
HeaderError,
|
||||
},
|
||||
{
|
||||
"bad checksum",
|
||||
"",
|
||||
[]byte{0x78, 0x9c, 0x03, 0x00, 0x00, 0x00, 0x00, 0xff},
|
||||
nil,
|
||||
ChecksumError,
|
||||
},
|
||||
{
|
||||
"not enough data",
|
||||
"",
|
||||
[]byte{0x78, 0x9c, 0x03, 0x00, 0x00, 0x00},
|
||||
nil,
|
||||
io.ErrUnexpectedEOF,
|
||||
},
|
||||
{
|
||||
@ -64,6 +70,33 @@ var zlibTests = []zlibTest{
|
||||
0x78, 0x9c, 0xff,
|
||||
},
|
||||
nil,
|
||||
nil,
|
||||
},
|
||||
{
|
||||
"dictionary",
|
||||
"Hello, World!\n",
|
||||
[]byte{
|
||||
0x78, 0xbb, 0x1c, 0x32, 0x04, 0x27, 0xf3, 0x00,
|
||||
0xb1, 0x75, 0x20, 0x1c, 0x45, 0x2e, 0x00, 0x24,
|
||||
0x12, 0x04, 0x74,
|
||||
},
|
||||
[]byte{
|
||||
0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x20, 0x57, 0x6f, 0x72, 0x6c, 0x64, 0x0a,
|
||||
},
|
||||
nil,
|
||||
},
|
||||
{
|
||||
"wrong dictionary",
|
||||
"",
|
||||
[]byte{
|
||||
0x78, 0xbb, 0x1c, 0x32, 0x04, 0x27, 0xf3, 0x00,
|
||||
0xb1, 0x75, 0x20, 0x1c, 0x45, 0x2e, 0x00, 0x24,
|
||||
0x12, 0x04, 0x74,
|
||||
},
|
||||
[]byte{
|
||||
0x48, 0x65, 0x6c, 0x6c,
|
||||
},
|
||||
DictionaryError,
|
||||
},
|
||||
}
|
||||
|
||||
@ -71,7 +104,7 @@ func TestDecompressor(t *testing.T) {
|
||||
b := new(bytes.Buffer)
|
||||
for _, tt := range zlibTests {
|
||||
in := bytes.NewBuffer(tt.compressed)
|
||||
zlib, err := NewReader(in)
|
||||
zlib, err := NewReaderDict(in, tt.dict)
|
||||
if err != nil {
|
||||
if err != tt.err {
|
||||
t.Errorf("%s: NewReader: %s", tt.desc, err)
|
||||
|
@ -21,56 +21,80 @@ const (
|
||||
DefaultCompression = flate.DefaultCompression
|
||||
)
|
||||
|
||||
type writer struct {
|
||||
// A Writer takes data written to it and writes the compressed
|
||||
// form of that data to an underlying writer (see NewWriter).
|
||||
type Writer struct {
|
||||
w io.Writer
|
||||
compressor io.WriteCloser
|
||||
compressor *flate.Writer
|
||||
digest hash.Hash32
|
||||
err os.Error
|
||||
scratch [4]byte
|
||||
}
|
||||
|
||||
// NewWriter calls NewWriterLevel with the default compression level.
|
||||
func NewWriter(w io.Writer) (io.WriteCloser, os.Error) {
|
||||
func NewWriter(w io.Writer) (*Writer, os.Error) {
|
||||
return NewWriterLevel(w, DefaultCompression)
|
||||
}
|
||||
|
||||
// NewWriterLevel creates a new io.WriteCloser that satisfies writes by compressing data written to w.
|
||||
// NewWriterLevel calls NewWriterDict with no dictionary.
|
||||
func NewWriterLevel(w io.Writer, level int) (*Writer, os.Error) {
|
||||
return NewWriterDict(w, level, nil)
|
||||
}
|
||||
|
||||
// NewWriterDict creates a new io.WriteCloser that satisfies writes by compressing data written to w.
|
||||
// It is the caller's responsibility to call Close on the WriteCloser when done.
|
||||
// level is the compression level, which can be DefaultCompression, NoCompression,
|
||||
// or any integer value between BestSpeed and BestCompression (inclusive).
|
||||
func NewWriterLevel(w io.Writer, level int) (io.WriteCloser, os.Error) {
|
||||
z := new(writer)
|
||||
// dict is the preset dictionary to compress with, or nil to use no dictionary.
|
||||
func NewWriterDict(w io.Writer, level int, dict []byte) (*Writer, os.Error) {
|
||||
z := new(Writer)
|
||||
// ZLIB has a two-byte header (as documented in RFC 1950).
|
||||
// The first four bits is the CINFO (compression info), which is 7 for the default deflate window size.
|
||||
// The next four bits is the CM (compression method), which is 8 for deflate.
|
||||
z.scratch[0] = 0x78
|
||||
// The next two bits is the FLEVEL (compression level). The four values are:
|
||||
// 0=fastest, 1=fast, 2=default, 3=best.
|
||||
// The next bit, FDICT, is unused, in this implementation.
|
||||
// The next bit, FDICT, is set if a dictionary is given.
|
||||
// The final five FCHECK bits form a mod-31 checksum.
|
||||
switch level {
|
||||
case 0, 1:
|
||||
z.scratch[1] = 0x01
|
||||
z.scratch[1] = 0 << 6
|
||||
case 2, 3, 4, 5:
|
||||
z.scratch[1] = 0x5e
|
||||
z.scratch[1] = 1 << 6
|
||||
case 6, -1:
|
||||
z.scratch[1] = 0x9c
|
||||
z.scratch[1] = 2 << 6
|
||||
case 7, 8, 9:
|
||||
z.scratch[1] = 0xda
|
||||
z.scratch[1] = 3 << 6
|
||||
default:
|
||||
return nil, os.NewError("level out of range")
|
||||
}
|
||||
if dict != nil {
|
||||
z.scratch[1] |= 1 << 5
|
||||
}
|
||||
z.scratch[1] += uint8(31 - (uint16(z.scratch[0])<<8+uint16(z.scratch[1]))%31)
|
||||
_, err := w.Write(z.scratch[0:2])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if dict != nil {
|
||||
// The next four bytes are the Adler-32 checksum of the dictionary.
|
||||
checksum := adler32.Checksum(dict)
|
||||
z.scratch[0] = uint8(checksum >> 24)
|
||||
z.scratch[1] = uint8(checksum >> 16)
|
||||
z.scratch[2] = uint8(checksum >> 8)
|
||||
z.scratch[3] = uint8(checksum >> 0)
|
||||
_, err = w.Write(z.scratch[0:4])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
z.w = w
|
||||
z.compressor = flate.NewWriter(w, level)
|
||||
z.digest = adler32.New()
|
||||
return z, nil
|
||||
}
|
||||
|
||||
func (z *writer) Write(p []byte) (n int, err os.Error) {
|
||||
func (z *Writer) Write(p []byte) (n int, err os.Error) {
|
||||
if z.err != nil {
|
||||
return 0, z.err
|
||||
}
|
||||
@ -86,8 +110,17 @@ func (z *writer) Write(p []byte) (n int, err os.Error) {
|
||||
return
|
||||
}
|
||||
|
||||
// Flush flushes the underlying compressor.
|
||||
func (z *Writer) Flush() os.Error {
|
||||
if z.err != nil {
|
||||
return z.err
|
||||
}
|
||||
z.err = z.compressor.Flush()
|
||||
return z.err
|
||||
}
|
||||
|
||||
// Calling Close does not close the wrapped io.Writer originally passed to NewWriter.
|
||||
func (z *writer) Close() os.Error {
|
||||
func (z *Writer) Close() os.Error {
|
||||
if z.err != nil {
|
||||
return z.err
|
||||
}
|
||||
|
@ -16,13 +16,19 @@ var filenames = []string{
|
||||
"../testdata/pi.txt",
|
||||
}
|
||||
|
||||
// Tests that compressing and then decompressing the given file at the given compression level
|
||||
// Tests that compressing and then decompressing the given file at the given compression level and dictionary
|
||||
// yields equivalent bytes to the original file.
|
||||
func testFileLevel(t *testing.T, fn string, level int) {
|
||||
func testFileLevelDict(t *testing.T, fn string, level int, d string) {
|
||||
// Read dictionary, if given.
|
||||
var dict []byte
|
||||
if d != "" {
|
||||
dict = []byte(d)
|
||||
}
|
||||
|
||||
// Read the file, as golden output.
|
||||
golden, err := os.Open(fn)
|
||||
if err != nil {
|
||||
t.Errorf("%s (level=%d): %v", fn, level, err)
|
||||
t.Errorf("%s (level=%d, dict=%q): %v", fn, level, d, err)
|
||||
return
|
||||
}
|
||||
defer golden.Close()
|
||||
@ -30,7 +36,7 @@ func testFileLevel(t *testing.T, fn string, level int) {
|
||||
// Read the file again, and push it through a pipe that compresses at the write end, and decompresses at the read end.
|
||||
raw, err := os.Open(fn)
|
||||
if err != nil {
|
||||
t.Errorf("%s (level=%d): %v", fn, level, err)
|
||||
t.Errorf("%s (level=%d, dict=%q): %v", fn, level, d, err)
|
||||
return
|
||||
}
|
||||
piper, pipew := io.Pipe()
|
||||
@ -38,9 +44,9 @@ func testFileLevel(t *testing.T, fn string, level int) {
|
||||
go func() {
|
||||
defer raw.Close()
|
||||
defer pipew.Close()
|
||||
zlibw, err := NewWriterLevel(pipew, level)
|
||||
zlibw, err := NewWriterDict(pipew, level, dict)
|
||||
if err != nil {
|
||||
t.Errorf("%s (level=%d): %v", fn, level, err)
|
||||
t.Errorf("%s (level=%d, dict=%q): %v", fn, level, d, err)
|
||||
return
|
||||
}
|
||||
defer zlibw.Close()
|
||||
@ -48,7 +54,7 @@ func testFileLevel(t *testing.T, fn string, level int) {
|
||||
for {
|
||||
n, err0 := raw.Read(b[0:])
|
||||
if err0 != nil && err0 != os.EOF {
|
||||
t.Errorf("%s (level=%d): %v", fn, level, err0)
|
||||
t.Errorf("%s (level=%d, dict=%q): %v", fn, level, d, err0)
|
||||
return
|
||||
}
|
||||
_, err1 := zlibw.Write(b[0:n])
|
||||
@ -57,7 +63,7 @@ func testFileLevel(t *testing.T, fn string, level int) {
|
||||
return
|
||||
}
|
||||
if err1 != nil {
|
||||
t.Errorf("%s (level=%d): %v", fn, level, err1)
|
||||
t.Errorf("%s (level=%d, dict=%q): %v", fn, level, d, err1)
|
||||
return
|
||||
}
|
||||
if err0 == os.EOF {
|
||||
@ -65,9 +71,9 @@ func testFileLevel(t *testing.T, fn string, level int) {
|
||||
}
|
||||
}
|
||||
}()
|
||||
zlibr, err := NewReader(piper)
|
||||
zlibr, err := NewReaderDict(piper, dict)
|
||||
if err != nil {
|
||||
t.Errorf("%s (level=%d): %v", fn, level, err)
|
||||
t.Errorf("%s (level=%d, dict=%q): %v", fn, level, d, err)
|
||||
return
|
||||
}
|
||||
defer zlibr.Close()
|
||||
@ -76,20 +82,20 @@ func testFileLevel(t *testing.T, fn string, level int) {
|
||||
b0, err0 := ioutil.ReadAll(golden)
|
||||
b1, err1 := ioutil.ReadAll(zlibr)
|
||||
if err0 != nil {
|
||||
t.Errorf("%s (level=%d): %v", fn, level, err0)
|
||||
t.Errorf("%s (level=%d, dict=%q): %v", fn, level, d, err0)
|
||||
return
|
||||
}
|
||||
if err1 != nil {
|
||||
t.Errorf("%s (level=%d): %v", fn, level, err1)
|
||||
t.Errorf("%s (level=%d, dict=%q): %v", fn, level, d, err1)
|
||||
return
|
||||
}
|
||||
if len(b0) != len(b1) {
|
||||
t.Errorf("%s (level=%d): length mismatch %d versus %d", fn, level, len(b0), len(b1))
|
||||
t.Errorf("%s (level=%d, dict=%q): length mismatch %d versus %d", fn, level, d, len(b0), len(b1))
|
||||
return
|
||||
}
|
||||
for i := 0; i < len(b0); i++ {
|
||||
if b0[i] != b1[i] {
|
||||
t.Errorf("%s (level=%d): mismatch at %d, 0x%02x versus 0x%02x\n", fn, level, i, b0[i], b1[i])
|
||||
t.Errorf("%s (level=%d, dict=%q): mismatch at %d, 0x%02x versus 0x%02x\n", fn, level, d, i, b0[i], b1[i])
|
||||
return
|
||||
}
|
||||
}
|
||||
@ -97,10 +103,21 @@ func testFileLevel(t *testing.T, fn string, level int) {
|
||||
|
||||
func TestWriter(t *testing.T) {
|
||||
for _, fn := range filenames {
|
||||
testFileLevel(t, fn, DefaultCompression)
|
||||
testFileLevel(t, fn, NoCompression)
|
||||
testFileLevelDict(t, fn, DefaultCompression, "")
|
||||
testFileLevelDict(t, fn, NoCompression, "")
|
||||
for level := BestSpeed; level <= BestCompression; level++ {
|
||||
testFileLevel(t, fn, level)
|
||||
testFileLevelDict(t, fn, level, "")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriterDict(t *testing.T) {
|
||||
const dictionary = "0123456789."
|
||||
for _, fn := range filenames {
|
||||
testFileLevelDict(t, fn, DefaultCompression, dictionary)
|
||||
testFileLevelDict(t, fn, NoCompression, dictionary)
|
||||
for level := BestSpeed; level <= BestCompression; level++ {
|
||||
testFileLevelDict(t, fn, level, dictionary)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user