From 4371e094578cdb463a09ca57777078f36feb3002 Mon Sep 17 00:00:00 2001 From: "Pascal S. de Kloe" Date: Mon, 7 Mar 2011 11:11:21 -0500 Subject: [PATCH] hash: new FNV-1a implementation R=agl1, rsc CC=golang-dev https://golang.org/cl/4257042 --- src/pkg/Makefile | 1 + src/pkg/hash/fnv/Makefile | 11 +++ src/pkg/hash/fnv/fnv.go | 133 ++++++++++++++++++++++++++++ src/pkg/hash/fnv/fnv_test.go | 167 +++++++++++++++++++++++++++++++++++ 4 files changed, 312 insertions(+) create mode 100644 src/pkg/hash/fnv/Makefile create mode 100644 src/pkg/hash/fnv/fnv.go create mode 100644 src/pkg/hash/fnv/fnv_test.go diff --git a/src/pkg/Makefile b/src/pkg/Makefile index 9de2bd2f7d..8f7c564c63 100644 --- a/src/pkg/Makefile +++ b/src/pkg/Makefile @@ -94,6 +94,7 @@ DIRS=\ hash/adler32\ hash/crc32\ hash/crc64\ + hash/fnv\ html\ http\ http/cgi\ diff --git a/src/pkg/hash/fnv/Makefile b/src/pkg/hash/fnv/Makefile new file mode 100644 index 0000000000..4c8a4ecf03 --- /dev/null +++ b/src/pkg/hash/fnv/Makefile @@ -0,0 +1,11 @@ +# Copyright 2011 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=hash/fnv +GOFILES=\ + fnv.go\ + +include ../../../Make.pkg diff --git a/src/pkg/hash/fnv/fnv.go b/src/pkg/hash/fnv/fnv.go new file mode 100644 index 0000000000..66ab5a635b --- /dev/null +++ b/src/pkg/hash/fnv/fnv.go @@ -0,0 +1,133 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// The fnv package implements FNV-1 and FNV-1a, +// non-cryptographic hash functions created by +// Glenn Fowler, Landon Curt Noll, and Phong Vo. +// See http://isthe.com/chongo/tech/comp/fnv/. +package fnv + +import ( + "encoding/binary" + "hash" + "os" + "unsafe" +) + +type ( + sum32 uint32 + sum32a uint32 + sum64 uint64 + sum64a uint64 +) + +const ( + offset32 = 2166136261 + offset64 = 14695981039346656037 + prime32 = 16777619 + prime64 = 1099511628211 +) + +// New32 returns a new 32-bit FNV-1 hash.Hash. +func New32() hash.Hash32 { + var s sum32 = offset32 + return &s +} + +// New32a returns a new 32-bit FNV-1a hash.Hash. +func New32a() hash.Hash32 { + var s sum32a = offset32 + return &s +} + +// New64 returns a new 64-bit FNV-1 hash.Hash. +func New64() hash.Hash64 { + var s sum64 = offset64 + return &s +} + +// New64a returns a new 64-bit FNV-1a hash.Hash. +func New64a() hash.Hash64 { + var s sum64a = offset64 + return &s +} + +func (s *sum32) Reset() { *s = offset32 } +func (s *sum32a) Reset() { *s = offset32 } +func (s *sum64) Reset() { *s = offset64 } +func (s *sum64a) Reset() { *s = offset64 } + +func (s *sum32) Sum32() uint32 { return uint32(*s) } +func (s *sum32a) Sum32() uint32 { return uint32(*s) } +func (s *sum64) Sum64() uint64 { return uint64(*s) } +func (s *sum64a) Sum64() uint64 { return uint64(*s) } + +func (s *sum32) Write(data []byte) (int, os.Error) { + hash := *s + for _, c := range data { + hash *= prime32 + hash ^= sum32(c) + } + *s = hash + return len(data), nil +} + +func (s *sum32a) Write(data []byte) (int, os.Error) { + hash := *s + for _, c := range data { + hash ^= sum32a(c) + hash *= prime32 + } + *s = hash + return len(data), nil +} + +func (s *sum64) Write(data []byte) (int, os.Error) { + hash := *s + for _, c := range data { + hash *= prime64 + hash ^= sum64(c) + } + *s = hash + return len(data), nil +} + +func (s *sum64a) Write(data []byte) (int, os.Error) { + hash := *s + for _, c := range data { + hash ^= sum64a(c) + hash *= prime64 + } + *s = hash + return len(data), nil +} + +func (s *sum32) Size() int { return unsafe.Sizeof(*s) } +func (s *sum32a) Size() int { return unsafe.Sizeof(*s) } +func (s *sum64) Size() int { return unsafe.Sizeof(*s) } +func (s *sum64a) Size() int { return unsafe.Sizeof(*s) } + +func (s *sum32) Sum() []byte { + a := make([]byte, unsafe.Sizeof(*s)) + binary.BigEndian.PutUint32(a, uint32(*s)) + return a +} + +func (s *sum32a) Sum() []byte { + a := make([]byte, unsafe.Sizeof(*s)) + binary.BigEndian.PutUint32(a, uint32(*s)) + return a +} + +func (s *sum64) Sum() []byte { + a := make([]byte, unsafe.Sizeof(*s)) + binary.BigEndian.PutUint64(a, uint64(*s)) + return a +} + +func (s *sum64a) Sum() []byte { + a := make([]byte, unsafe.Sizeof(*s)) + binary.BigEndian.PutUint64(a, uint64(*s)) + return a +} diff --git a/src/pkg/hash/fnv/fnv_test.go b/src/pkg/hash/fnv/fnv_test.go new file mode 100644 index 0000000000..3ea3fe6f12 --- /dev/null +++ b/src/pkg/hash/fnv/fnv_test.go @@ -0,0 +1,167 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package fnv + +import ( + "bytes" + "encoding/binary" + "hash" + "testing" +) + +const testDataSize = 40 + +type golden struct { + sum []byte + text string +} + +var golden32 = []golden{ + {[]byte{0x81, 0x1c, 0x9d, 0xc5}, ""}, + {[]byte{0x05, 0x0c, 0x5d, 0x7e}, "a"}, + {[]byte{0x70, 0x77, 0x2d, 0x38}, "ab"}, + {[]byte{0x43, 0x9c, 0x2f, 0x4b}, "abc"}, +} + +var golden32a = []golden{ + {[]byte{0x81, 0x1c, 0x9d, 0xc5}, ""}, + {[]byte{0xe4, 0x0c, 0x29, 0x2c}, "a"}, + {[]byte{0x4d, 0x25, 0x05, 0xca}, "ab"}, + {[]byte{0x1a, 0x47, 0xe9, 0x0b}, "abc"}, +} + +var golden64 = []golden{ + {[]byte{0xcb, 0xf2, 0x9c, 0xe4, 0x84, 0x22, 0x23, 0x25}, ""}, + {[]byte{0xaf, 0x63, 0xbd, 0x4c, 0x86, 0x01, 0xb7, 0xbe}, "a"}, + {[]byte{0x08, 0x32, 0x67, 0x07, 0xb4, 0xeb, 0x37, 0xb8}, "ab"}, + {[]byte{0xd8, 0xdc, 0xca, 0x18, 0x6b, 0xaf, 0xad, 0xcb}, "abc"}, +} + +var golden64a = []golden{ + {[]byte{0xcb, 0xf2, 0x9c, 0xe4, 0x84, 0x22, 0x23, 0x25}, ""}, + {[]byte{0xaf, 0x63, 0xdc, 0x4c, 0x86, 0x01, 0xec, 0x8c}, "a"}, + {[]byte{0x08, 0x9c, 0x44, 0x07, 0xb5, 0x45, 0x98, 0x6a}, "ab"}, + {[]byte{0xe7, 0x1f, 0xa2, 0x19, 0x05, 0x41, 0x57, 0x4b}, "abc"}, +} + +func TestGolden32(t *testing.T) { + testGolden(t, New32(), golden32) +} + +func TestGolden32a(t *testing.T) { + testGolden(t, New32a(), golden32a) +} + +func TestGolden64(t *testing.T) { + testGolden(t, New64(), golden64) +} + +func TestGolden64a(t *testing.T) { + testGolden(t, New64a(), golden64a) +} + +func testGolden(t *testing.T, hash hash.Hash, gold []golden) { + for _, g := range gold { + hash.Reset() + done, error := hash.Write([]byte(g.text)) + if error != nil { + t.Fatalf("write error: %s", error) + } + if done != len(g.text) { + t.Fatalf("wrote only %d out of %d bytes", done, len(g.text)) + } + if actual := hash.Sum(); !bytes.Equal(g.sum, actual) { + t.Errorf("hash(%q) = 0x%x want 0x%x", g.text, actual, g.sum) + } + } +} + +func TestIntegrity32(t *testing.T) { + testIntegrity(t, New32()) +} + +func TestIntegrity32a(t *testing.T) { + testIntegrity(t, New32a()) +} + +func TestIntegrity64(t *testing.T) { + testIntegrity(t, New64()) +} + +func TestIntegrity64a(t *testing.T) { + testIntegrity(t, New64a()) +} + +func testIntegrity(t *testing.T, h hash.Hash) { + data := []byte{'1', '2', 3, 4, 5} + h.Write(data) + sum := h.Sum() + + if size := h.Size(); size != len(sum) { + t.Fatalf("Size()=%d but len(Sum())=%d", size, len(sum)) + } + + if a := h.Sum(); !bytes.Equal(sum, a) { + t.Fatalf("first Sum()=0x%x, second Sum()=0x%x", sum, a) + } + + h.Reset() + h.Write(data) + if a := h.Sum(); !bytes.Equal(sum, a) { + t.Fatalf("Sum()=0x%x, but after Reset() Sum()=0x%x", sum, a) + } + + h.Reset() + h.Write(data[:2]) + h.Write(data[2:]) + if a := h.Sum(); !bytes.Equal(sum, a) { + t.Fatalf("Sum()=0x%x, but with partial writes, Sum()=0x%x", sum, a) + } + + switch h.Size() { + case 4: + sum32 := h.(hash.Hash32).Sum32() + if sum32 != binary.BigEndian.Uint32(sum) { + t.Fatalf("Sum()=0x%x, but Sum32()=0x%x", sum, sum32) + } + case 8: + sum64 := h.(hash.Hash64).Sum64() + if sum64 != binary.BigEndian.Uint64(sum) { + t.Fatalf("Sum()=0x%x, but Sum64()=0x%x", sum, sum64) + } + } +} + +func Benchmark32(b *testing.B) { + benchmark(b, New32()) +} + +func Benchmark32a(b *testing.B) { + benchmark(b, New32a()) +} + +func Benchmark64(b *testing.B) { + benchmark(b, New64()) +} + +func Benchmark64a(b *testing.B) { + benchmark(b, New64a()) +} + +func benchmark(b *testing.B, h hash.Hash) { + b.ResetTimer() + b.SetBytes(testDataSize) + data := make([]byte, testDataSize) + for i, _ := range data { + data[i] = byte(i + 'a') + } + + b.StartTimer() + for todo := b.N; todo != 0; todo-- { + h.Reset() + h.Write(data) + h.Sum() + } +}