From b05c6fe23a84e8e5962bd36fed9ccb145b281166 Mon Sep 17 00:00:00 2001 From: David Symonds Date: Mon, 8 Jun 2009 23:22:56 -0700 Subject: [PATCH] A basic tar package. R=rsc APPROVED=rsc DELTA=371 (370 added, 0 deleted, 1 changed) OCL=30029 CL=30084 --- src/lib/Make.deps | 3 +- src/lib/Makefile | 4 + src/lib/archive/tar/Makefile | 60 ++++++ src/lib/archive/tar/testdata/small.txt | 1 + src/lib/archive/tar/testdata/small2.txt | 1 + src/lib/archive/tar/testdata/test.tar | Bin 0 -> 3072 bytes src/lib/archive/tar/untar.go | 242 ++++++++++++++++++++++++ src/lib/archive/tar/untar_test.go | 69 +++++++ 8 files changed, 379 insertions(+), 1 deletion(-) create mode 100644 src/lib/archive/tar/Makefile create mode 100644 src/lib/archive/tar/testdata/small.txt create mode 100644 src/lib/archive/tar/testdata/small2.txt create mode 100644 src/lib/archive/tar/testdata/test.tar create mode 100644 src/lib/archive/tar/untar.go create mode 100644 src/lib/archive/tar/untar_test.go diff --git a/src/lib/Make.deps b/src/lib/Make.deps index 77c4089a00f..dd83e8b1cb5 100644 --- a/src/lib/Make.deps +++ b/src/lib/Make.deps @@ -1,7 +1,8 @@ +archive/tar.install: bufio.install bytes.install io.install os.install strconv.install bignum.install: fmt.install bufio.install: io.install os.install utf8.install bytes.install: utf8.install -compress/flate.install: bufio.install fmt.install io.install os.install strconv.install +compress/flate.install: bufio.install io.install os.install strconv.install compress/gzip.install: bufio.install compress/flate.install hash.install hash/crc32.install io.install os.install container/list.install: container/vector.install: diff --git a/src/lib/Makefile b/src/lib/Makefile index 8b6b9a8eed6..036a82e38b3 100644 --- a/src/lib/Makefile +++ b/src/lib/Makefile @@ -12,6 +12,7 @@ all: install DIRS=\ + archive/tar\ bignum\ bufio\ bytes\ @@ -65,8 +66,11 @@ DIRS=\ utf8\ TEST=\ + archive/tar\ bignum\ bufio\ + compress/flate\ + compress/gzip\ container/list\ container/vector\ crypto/aes\ diff --git a/src/lib/archive/tar/Makefile b/src/lib/archive/tar/Makefile new file mode 100644 index 00000000000..579ed4c3517 --- /dev/null +++ b/src/lib/archive/tar/Makefile @@ -0,0 +1,60 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +# DO NOT EDIT. Automatically generated by gobuild. +# gobuild -m >Makefile + +D=/archive/ + +include $(GOROOT)/src/Make.$(GOARCH) +AR=gopack + +default: packages + +clean: + rm -rf *.[$(OS)] *.a [$(OS)].out _obj + +test: packages + gotest + +coverage: packages + gotest + 6cov -g `pwd` | grep -v '_test\.go:' + +%.$O: %.go + $(GC) -I_obj $*.go + +%.$O: %.c + $(CC) $*.c + +%.$O: %.s + $(AS) $*.s + +O1=\ + untar.$O\ + + +phases: a1 +_obj$D/tar.a: phases + +a1: $(O1) + $(AR) grc _obj$D/tar.a untar.$O + rm -f $(O1) + + +newpkg: clean + mkdir -p _obj$D + $(AR) grc _obj$D/tar.a + +$(O1): newpkg +$(O2): a1 + +nuke: clean + rm -f $(GOROOT)/pkg/$(GOOS)_$(GOARCH)$D/tar.a + +packages: _obj$D/tar.a + +install: packages + test -d $(GOROOT)/pkg && mkdir -p $(GOROOT)/pkg/$(GOOS)_$(GOARCH)$D + cp _obj$D/tar.a $(GOROOT)/pkg/$(GOOS)_$(GOARCH)$D/tar.a diff --git a/src/lib/archive/tar/testdata/small.txt b/src/lib/archive/tar/testdata/small.txt new file mode 100644 index 00000000000..b249bfc518a --- /dev/null +++ b/src/lib/archive/tar/testdata/small.txt @@ -0,0 +1 @@ +Kilts \ No newline at end of file diff --git a/src/lib/archive/tar/testdata/small2.txt b/src/lib/archive/tar/testdata/small2.txt new file mode 100644 index 00000000000..394ee3ecd0e --- /dev/null +++ b/src/lib/archive/tar/testdata/small2.txt @@ -0,0 +1 @@ +Google.com diff --git a/src/lib/archive/tar/testdata/test.tar b/src/lib/archive/tar/testdata/test.tar new file mode 100644 index 0000000000000000000000000000000000000000..fc899dc8dc2ad9952f5c5f67a0c76ca2d87249e9 GIT binary patch literal 3072 zcmeHH%L>9U5Ztq0(Jv@FdDKtv;8zq|ijXv5BIw^6DOh^2UK)_HbK1>@VQ0c5`qsHR zJrb1zXEcV16&lMRW}rdtKd=NSXg->JkvP|Esp4`g&CK_h+FMmo7oR?iU7RP&svn2t z!9Ke4)upeR_aRYKtT+(g`B!B>fS>t?p7IZ9V9LKWlK+)w+iY|SVQ_tY3I4Ddrx1w) M;($0H4*b6ZFWOBnBLDyZ literal 0 HcmV?d00001 diff --git a/src/lib/archive/tar/untar.go b/src/lib/archive/tar/untar.go new file mode 100644 index 00000000000..300c0f932c6 --- /dev/null +++ b/src/lib/archive/tar/untar.go @@ -0,0 +1,242 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// The tar package implements access to tar archives. +// It aims to cover most of the variations, including those produced +// by GNU and BSD tars (not yet started). +// +// References: +// http://www.freebsd.org/cgi/man.cgi?query=tar&sektion=5 +// http://www.gnu.org/software/tar/manual/html_node/Standard.html +package tar + +// TODO(dsymonds): +// - Make it seekable. +// - Extensions. + +import ( + "bufio"; + "bytes"; + "io"; + "os"; + "strconv"; +) + +var ( + HeaderError os.Error = os.ErrorString("invalid tar header"); +) + +// A tar archive consists of a sequence of files. +// A Reader provides sequential access to the contents of a tar archive. +// The Next method advances to the next file in the archive (including the first), +// and then it can be treated as an io.Reader to access the file's data. +// +// Example: +// tr := NewTarReader(r); +// for { +// hdr, err := tr.Next(); +// if err != nil { +// // handle error +// } +// if hdr == nil { +// // end of tar archive +// break +// } +// io.Copy(tr, somewhere); +// } +type Reader struct { + r io.Reader; + err os.Error; + nb int64; // number of unread bytes for current file entry + pad int64; // amount of padding (ignored) after current file entry +} + +// A Header represents a single header in a tar archive. +// Only some fields may be populated. +type Header struct { + Name string; + Mode int64; + Uid int64; + Gid int64; + Size int64; + Mtime int64; + Typeflag byte; + Linkname string; + Uname string; + Gname string; + Devmajor int64; + Devminor int64; +} + +func (tr *Reader) skipUnread() +func (tr *Reader) readHeader() *Header + +// NewReader creates a new Reader reading the given io.Reader. +func NewReader(r io.Reader) *Reader { + return &Reader{ r: r } +} + +// Next advances to the next entry in the tar archive. +func (tr *Reader) Next() (*Header, os.Error) { + var hdr *Header; + if tr.err == nil { + tr.skipUnread(); + } + if tr.err == nil { + hdr = tr.readHeader(); + } + return hdr, tr.err +} + +const ( + blockSize = 512; + + // Types + TypeReg = '0'; + TypeRegA = '\x00'; + TypeLink = '1'; + TypeSymlink = '2'; + TypeChar = '3'; + TypeBlock = '4'; + TypeDir = '5'; + TypeFifo = '6'; + TypeCont = '7'; + TypeXHeader = 'x'; + TypeXGlobalHeader = 'g'; +) + +var zeroBlock = make([]byte, blockSize); + +// Parse bytes as a NUL-terminated C-style string. +// If a NUL byte is not found then the whole slice is returned as a string. +func cString(b []byte) string { + n := 0; + for n < len(b) && b[n] != 0 { + n++; + } + return string(b[0:n]) +} + +func (tr *Reader) octalNumber(b []byte) int64 { + x, err := strconv.Btoui64(cString(b), 8); + if err != nil { + tr.err = err; + } + return int64(x) +} + +type ignoreWriter struct {} +func (ignoreWriter) Write(b []byte) (n int, err os.Error) { + return len(b), nil +} + +type seeker interface { + Seek(offset int64, whence int) (ret int64, err os.Error); +} + +// Skip any unread bytes in the existing file entry, as well as any alignment padding. +func (tr *Reader) skipUnread() { + nr := tr.nb + tr.pad; // number of bytes to skip + + var n int64; + if sr, ok := tr.r.(seeker); ok { + n, tr.err = sr.Seek(nr, 1); + } else { + n, tr.err = io.Copyn(tr.r, ignoreWriter{}, nr); + } + tr.nb, tr.pad = 0, 0; +} + +func (tr *Reader) verifyChecksum(header []byte) bool { + given := tr.octalNumber(header[148:156]); + if tr.err != nil { + return false + } + + var computed int64; + for i := 0; i < len(header); i++ { + if i == 148 { + // The chksum field is special: it should be treated as space bytes. + computed += ' ' * 8; + i += 7; + continue + } + computed += int64(header[i]); + } + + return given == computed +} + +type slicer []byte +func (s *slicer) next(n int) (b []byte) { + b, *s = s[0:n], s[n:len(s)]; + return +} + +func (tr *Reader) readHeader() *Header { + header := make([]byte, blockSize); + var n int; + if n, tr.err = io.FullRead(tr.r, header); tr.err != nil { + return nil + } + + // Two blocks of zero bytes marks the end of the archive. + if bytes.Equal(header, zeroBlock[0:blockSize]) { + if n, tr.err = io.FullRead(tr.r, header); tr.err != nil { + return nil + } + if !bytes.Equal(header, zeroBlock[0:blockSize]) { + tr.err = HeaderError; + } + return nil + } + + if !tr.verifyChecksum(header) { + tr.err = HeaderError; + return nil + } + + // Unpack + hdr := new(Header); + s := slicer(header); + + // TODO(dsymonds): The format of the header depends on the value of magic (hdr[257:262]), + // so use that value to do the correct parsing below. + + hdr.Name = cString(s.next(100)); + hdr.Mode = tr.octalNumber(s.next(8)); + hdr.Uid = tr.octalNumber(s.next(8)); + hdr.Gid = tr.octalNumber(s.next(8)); + hdr.Size = tr.octalNumber(s.next(12)); + hdr.Mtime = tr.octalNumber(s.next(12)); + s.next(8); // chksum + hdr.Typeflag = s.next(1)[0]; + hdr.Linkname = cString(s.next(100)); + s.next(8); // magic, version + + if tr.err != nil { + tr.err = HeaderError; + return nil + } + + // Maximum value of hdr.Size is 64 GB (12 octal digits), + // so there's no risk of int64 overflowing. + tr.nb = int64(hdr.Size); + tr.pad = -tr.nb & (blockSize - 1); // blockSize is a power of two + + return hdr +} + +// Read reads from the current entry in the tar archive. +// It returns 0, nil when it reaches the end of that entry, +// until Next is called to advance to the next entry. +func (tr *Reader) Read(b []uint8) (n int, err os.Error) { + if int64(len(b)) > tr.nb { + b = b[0:tr.nb]; + } + n, err = tr.r.Read(b); + tr.nb -= int64(n); + tr.err = err; + return +} diff --git a/src/lib/archive/tar/untar_test.go b/src/lib/archive/tar/untar_test.go new file mode 100644 index 00000000000..a9c92dbf05b --- /dev/null +++ b/src/lib/archive/tar/untar_test.go @@ -0,0 +1,69 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package tar + +import ( + "archive/tar"; + "bytes"; + "fmt"; + "io"; + "os"; + "testing"; +) + +func TestUntar(t *testing.T) { + f, err := os.Open("testdata/test.tar", os.O_RDONLY, 0444); + if err != nil { + t.Fatalf("Unexpected error: %v", err); + } + defer f.Close(); + + tr := NewReader(f); + + // First file + hdr, err := tr.Next(); + if err != nil || hdr == nil { + t.Fatalf("Didn't get first file: %v", err); + } + if hdr.Name != "small.txt" { + t.Errorf(`hdr.Name = %q, want "small.txt"`, hdr.Name); + } + if hdr.Mode != 0640 { + t.Errorf("hdr.Mode = %v, want 0640", hdr.Mode); + } + if hdr.Size != 5 { + t.Errorf("hdr.Size = %v, want 5", hdr.Size); + } + + // Read the first four bytes; Next() should skip the last one. + buf := make([]byte, 4); + if n, err := io.FullRead(tr, buf); err != nil { + t.Fatalf("Unexpected error: %v", err); + } + if expected := io.StringBytes("Kilt"); !bytes.Equal(buf, expected) { + t.Errorf("Contents = %v, want %v", buf, expected); + } + + // Second file + hdr, err = tr.Next(); + if err != nil { + t.Fatalf("Didn't get second file: %v", err); + } + if hdr.Name != "small2.txt" { + t.Errorf(`hdr.Name = %q, want "small2.txt"`, hdr.Name); + } + if hdr.Mode != 0640 { + t.Errorf("hdr.Mode = %v, want 0640", hdr.Mode); + } + if hdr.Size != 11 { + t.Errorf("hdr.Size = %v, want 11", hdr.Size); + } + + + hdr, err = tr.Next(); + if hdr != nil || err != nil { + t.Fatalf("Unexpected third file or error: %v", err); + } +}