qbit/go
1
0
Fork 0
mirror of https://github.com/golang/go synced 2024-11-15 02:30:31 -07:00
Code Releases Activity

runtime,hash/maphash: reuse hashSets to save memory pressure

Browse Source 
Might help with OOMs on 32-bit platforms

Change-Id: Idd5129c61ecdfeedd5a9a18fce85dbba27cab946
Reviewed-on: https://go-review.googlesource.com/c/go/+/574475
Reviewed-by: Emmanuel Odeke <emmanuel@orijtech.com>
Reviewed-by: Keith Randall <khr@google.com>
LUCI-TryBot-Result: Go LUCI <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>
Reviewed-by: Than McIntosh <thanm@google.com>
This commit is contained in:
Keith Randall 2024-03-26 09:23:16 -07:00 committed by Keith Randall
parent d3e5e9fdf6
commit b47f2febea
2 changed files with 60 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

54
src/hash/maphash/smhasher_test.go
View File

@ -112,6 +112,8 @@ func (s *hashSet) check(t *testing.T) {
if float64(collisions) > expected+SLOP*(3*stddev+1) { if float64(collisions) > expected+SLOP*(3*stddev+1) {
t.Errorf("unexpected number of collisions: got=%d mean=%f stddev=%f", collisions, expected, stddev) t.Errorf("unexpected number of collisions: got=%d mean=%f stddev=%f", collisions, expected, stddev)
} }
// Reset for reuse
s.list = s.list[:0]
} }
// a string plus adding zeros must make distinct hashes // a string plus adding zeros must make distinct hashes
@ -212,8 +214,8 @@ func TestSmhasherCyclic(t *testing.T) {
r := rand.New(rand.NewSource(1234)) r := rand.New(rand.NewSource(1234))
const REPEAT = 8 const REPEAT = 8
const N = 1000000 const N = 1000000
for n := 4; n <= 12; n++ {
h := newHashSet() h := newHashSet()
for n := 4; n <= 12; n++ {
b := make([]byte, REPEAT*n) b := make([]byte, REPEAT*n)
for i := 0; i < N; i++ { for i := 0; i < N; i++ {
b[0] = byte(i * 79 % 97) b[0] = byte(i * 79 % 97)
@ -238,18 +240,18 @@ func TestSmhasherSparse(t *testing.T) {
if testing.Short() { if testing.Short() {
t.Skip("Skipping in short mode") t.Skip("Skipping in short mode")
} }
sparse(t, 32, 6)
sparse(t, 40, 6)
sparse(t, 48, 5)
sparse(t, 56, 5)
sparse(t, 64, 5)
sparse(t, 96, 4)
sparse(t, 256, 3)
sparse(t, 2048, 2)
}
func sparse(t *testing.T, n int, k int) {
b := make([]byte, n/8)
h := newHashSet() h := newHashSet()
sparse(t, h, 32, 6)
sparse(t, h, 40, 6)
sparse(t, h, 48, 5)
sparse(t, h, 56, 5)
sparse(t, h, 64, 5)
sparse(t, h, 96, 4)
sparse(t, h, 256, 3)
sparse(t, h, 2048, 2)
}
func sparse(t *testing.T, h *hashSet, n int, k int) {
b := make([]byte, n/8)
setbits(h, b, 0, k) setbits(h, b, 0, k)
h.check(t) h.check(t)
} }
@ -276,15 +278,15 @@ func TestSmhasherPermutation(t *testing.T) {
if testing.Short() { if testing.Short() {
t.Skip("Skipping in short mode") t.Skip("Skipping in short mode")
} }
permutation(t, []uint32{0, 1, 2, 3, 4, 5, 6, 7}, 8)
permutation(t, []uint32{0, 1 << 29, 2 << 29, 3 << 29, 4 << 29, 5 << 29, 6 << 29, 7 << 29}, 8)
permutation(t, []uint32{0, 1}, 20)
permutation(t, []uint32{0, 1 << 31}, 20)
permutation(t, []uint32{0, 1, 2, 3, 4, 5, 6, 7, 1 << 29, 2 << 29, 3 << 29, 4 << 29, 5 << 29, 6 << 29, 7 << 29}, 6)
}
func permutation(t *testing.T, s []uint32, n int) {
b := make([]byte, n*4)
h := newHashSet() h := newHashSet()
permutation(t, h, []uint32{0, 1, 2, 3, 4, 5, 6, 7}, 8)
permutation(t, h, []uint32{0, 1 << 29, 2 << 29, 3 << 29, 4 << 29, 5 << 29, 6 << 29, 7 << 29}, 8)
permutation(t, h, []uint32{0, 1}, 20)
permutation(t, h, []uint32{0, 1 << 31}, 20)
permutation(t, h, []uint32{0, 1, 2, 3, 4, 5, 6, 7, 1 << 29, 2 << 29, 3 << 29, 4 << 29, 5 << 29, 6 << 29, 7 << 29}, 6)
}
func permutation(t *testing.T, h *hashSet, s []uint32, n int) {
b := make([]byte, n*4)
genPerm(h, b, s, 0) genPerm(h, b, s, 0)
h.check(t) h.check(t)
} }
@ -418,8 +420,8 @@ func windowed(t *testing.T, k key) {
} }
const BITS = 16 const BITS = 16
for r := 0; r < k.bits(); r++ {
h := newHashSet() h := newHashSet()
for r := 0; r < k.bits(); r++ {
for i := 0; i < 1<<BITS; i++ { for i := 0; i < 1<<BITS; i++ {
k.clear() k.clear()
for j := 0; j < BITS; j++ { for j := 0; j < BITS; j++ {
@ -438,18 +440,18 @@ func TestSmhasherText(t *testing.T) {
if testing.Short() { if testing.Short() {
t.Skip("Skipping in short mode") t.Skip("Skipping in short mode")
} }
text(t, "Foo", "Bar") h := newHashSet()
text(t, "FooBar", "") text(t, h, "Foo", "Bar")
text(t, "", "FooBar") text(t, h, "FooBar", "")
text(t, h, "", "FooBar")
} }
func text(t *testing.T, prefix, suffix string) { func text(t *testing.T, h *hashSet, prefix, suffix string) {
const N = 4 const N = 4
const S = "ABCDEFGHIJKLMNOPQRSTabcdefghijklmnopqrst0123456789" const S = "ABCDEFGHIJKLMNOPQRSTabcdefghijklmnopqrst0123456789"
const L = len(S) const L = len(S)
b := make([]byte, len(prefix)+N+len(suffix)) b := make([]byte, len(prefix)+N+len(suffix))
copy(b, prefix) copy(b, prefix)
copy(b[len(prefix)+N:], suffix) copy(b[len(prefix)+N:], suffix)
h := newHashSet()
c := b[len(prefix):] c := b[len(prefix):]
for i := 0; i < L; i++ { for i := 0; i < L; i++ {
c[0] = S[i] c[0] = S[i]

62
src/runtime/hash_test.go
View File

@ -121,6 +121,8 @@ func (s *HashSet) check(t *testing.T) {
if float64(collisions) > expected+SLOP*(3*stddev+1) { if float64(collisions) > expected+SLOP*(3*stddev+1) {
t.Errorf("unexpected number of collisions: got=%d mean=%f stddev=%f threshold=%f", collisions, expected, stddev, expected+SLOP*(3*stddev+1)) t.Errorf("unexpected number of collisions: got=%d mean=%f stddev=%f threshold=%f", collisions, expected, stddev, expected+SLOP*(3*stddev+1))
} }
// Reset for reuse
s.list = s.list[:0]
} }
// a string plus adding zeros must make distinct hashes // a string plus adding zeros must make distinct hashes
@ -230,8 +232,8 @@ func TestSmhasherCyclic(t *testing.T) {
r := rand.New(rand.NewSource(1234)) r := rand.New(rand.NewSource(1234))
const REPEAT = 8 const REPEAT = 8
const N = 1000000 const N = 1000000
for n := 4; n <= 12; n++ {
h := newHashSet() h := newHashSet()
for n := 4; n <= 12; n++ {
b := make([]byte, REPEAT*n) b := make([]byte, REPEAT*n)
for i := 0; i < N; i++ { for i := 0; i < N; i++ {
b[0] = byte(i * 79 % 97) b[0] = byte(i * 79 % 97)
@ -256,18 +258,18 @@ func TestSmhasherSparse(t *testing.T) {
if testing.Short() { if testing.Short() {
t.Skip("Skipping in short mode") t.Skip("Skipping in short mode")
} }
sparse(t, 32, 6)
sparse(t, 40, 6)
sparse(t, 48, 5)
sparse(t, 56, 5)
sparse(t, 64, 5)
sparse(t, 96, 4)
sparse(t, 256, 3)
sparse(t, 2048, 2)
}
func sparse(t *testing.T, n int, k int) {
b := make([]byte, n/8)
h := newHashSet() h := newHashSet()
sparse(t, h, 32, 6)
sparse(t, h, 40, 6)
sparse(t, h, 48, 5)
sparse(t, h, 56, 5)
sparse(t, h, 64, 5)
sparse(t, h, 96, 4)
sparse(t, h, 256, 3)
sparse(t, h, 2048, 2)
}
func sparse(t *testing.T, h *HashSet, n int, k int) {
b := make([]byte, n/8)
setbits(h, b, 0, k) setbits(h, b, 0, k)
h.check(t) h.check(t)
} }
@ -297,15 +299,15 @@ func TestSmhasherPermutation(t *testing.T) {
if race.Enabled { if race.Enabled {
t.Skip("Too long for race mode") t.Skip("Too long for race mode")
} }
permutation(t, []uint32{0, 1, 2, 3, 4, 5, 6, 7}, 8)
permutation(t, []uint32{0, 1 << 29, 2 << 29, 3 << 29, 4 << 29, 5 << 29, 6 << 29, 7 << 29}, 8)
permutation(t, []uint32{0, 1}, 20)
permutation(t, []uint32{0, 1 << 31}, 20)
permutation(t, []uint32{0, 1, 2, 3, 4, 5, 6, 7, 1 << 29, 2 << 29, 3 << 29, 4 << 29, 5 << 29, 6 << 29, 7 << 29}, 6)
}
func permutation(t *testing.T, s []uint32, n int) {
b := make([]byte, n*4)
h := newHashSet() h := newHashSet()
permutation(t, h, []uint32{0, 1, 2, 3, 4, 5, 6, 7}, 8)
permutation(t, h, []uint32{0, 1 << 29, 2 << 29, 3 << 29, 4 << 29, 5 << 29, 6 << 29, 7 << 29}, 8)
permutation(t, h, []uint32{0, 1}, 20)
permutation(t, h, []uint32{0, 1 << 31}, 20)
permutation(t, h, []uint32{0, 1, 2, 3, 4, 5, 6, 7, 1 << 29, 2 << 29, 3 << 29, 4 << 29, 5 << 29, 6 << 29, 7 << 29}, 6)
}
func permutation(t *testing.T, h *HashSet, s []uint32, n int) {
b := make([]byte, n*4)
genPerm(h, b, s, 0) genPerm(h, b, s, 0)
h.check(t) h.check(t)
} }
@ -542,14 +544,15 @@ func TestSmhasherWindowed(t *testing.T) {
if race.Enabled { if race.Enabled {
t.Skip("Too long for race mode") t.Skip("Too long for race mode")
} }
h := newHashSet()
t.Logf("32 bit keys") t.Logf("32 bit keys")
windowed(t, &Int32Key{}) windowed(t, h, &Int32Key{})
t.Logf("64 bit keys") t.Logf("64 bit keys")
windowed(t, &Int64Key{}) windowed(t, h, &Int64Key{})
t.Logf("string keys") t.Logf("string keys")
windowed(t, &BytesKey{make([]byte, 128)}) windowed(t, h, &BytesKey{make([]byte, 128)})
} }
func windowed(t *testing.T, k Key) { func windowed(t *testing.T, h *HashSet, k Key) {
if GOARCH == "wasm" { if GOARCH == "wasm" {
t.Skip("Too slow on wasm") t.Skip("Too slow on wasm")
} }
@ -566,7 +569,6 @@ func windowed(t *testing.T, k Key) {
const BITS = 16 const BITS = 16
for r := 0; r < k.bits(); r++ { for r := 0; r < k.bits(); r++ {
h := newHashSet()
for i := 0; i < 1<<BITS; i++ { for i := 0; i < 1<<BITS; i++ {
k.clear() k.clear()
for j := 0; j < BITS; j++ { for j := 0; j < BITS; j++ {
@ -585,18 +587,18 @@ func TestSmhasherText(t *testing.T) {
if testing.Short() { if testing.Short() {
t.Skip("Skipping in short mode") t.Skip("Skipping in short mode")
} }
text(t, "Foo", "Bar") h := newHashSet()
text(t, "FooBar", "") text(t, h, "Foo", "Bar")
text(t, "", "FooBar") text(t, h, "FooBar", "")
text(t, h, "", "FooBar")
} }
func text(t *testing.T, prefix, suffix string) { func text(t *testing.T, h *HashSet, prefix, suffix string) {
const N = 4 const N = 4
const S = "ABCDEFGHIJKLMNOPQRSTabcdefghijklmnopqrst0123456789" const S = "ABCDEFGHIJKLMNOPQRSTabcdefghijklmnopqrst0123456789"
const L = len(S) const L = len(S)
b := make([]byte, len(prefix)+N+len(suffix)) b := make([]byte, len(prefix)+N+len(suffix))
copy(b, prefix) copy(b, prefix)
copy(b[len(prefix)+N:], suffix) copy(b[len(prefix)+N:], suffix)
h := newHashSet()
c := b[len(prefix):] c := b[len(prefix):]
for i := 0; i < L; i++ { for i := 0; i < L; i++ {
c[0] = S[i] c[0] = S[i]