From b47f2febea5c570fef4a5c27a46473f511fbdaa3 Mon Sep 17 00:00:00 2001 From: Keith Randall Date: Tue, 26 Mar 2024 09:23:16 -0700 Subject: [PATCH] runtime,hash/maphash: reuse hashSets to save memory pressure Might help with OOMs on 32-bit platforms Change-Id: Idd5129c61ecdfeedd5a9a18fce85dbba27cab946 Reviewed-on: https://go-review.googlesource.com/c/go/+/574475 Reviewed-by: Emmanuel Odeke Reviewed-by: Keith Randall LUCI-TryBot-Result: Go LUCI Reviewed-by: Than McIntosh --- src/hash/maphash/smhasher_test.go | 54 ++++++++++++++------------- src/runtime/hash_test.go | 62 ++++++++++++++++--------------- 2 files changed, 60 insertions(+), 56 deletions(-) diff --git a/src/hash/maphash/smhasher_test.go b/src/hash/maphash/smhasher_test.go index f34cea8e80..28cdae0444 100644 --- a/src/hash/maphash/smhasher_test.go +++ b/src/hash/maphash/smhasher_test.go @@ -112,6 +112,8 @@ func (s *hashSet) check(t *testing.T) { if float64(collisions) > expected+SLOP*(3*stddev+1) { t.Errorf("unexpected number of collisions: got=%d mean=%f stddev=%f", collisions, expected, stddev) } + // Reset for reuse + s.list = s.list[:0] } // a string plus adding zeros must make distinct hashes @@ -212,8 +214,8 @@ func TestSmhasherCyclic(t *testing.T) { r := rand.New(rand.NewSource(1234)) const REPEAT = 8 const N = 1000000 + h := newHashSet() for n := 4; n <= 12; n++ { - h := newHashSet() b := make([]byte, REPEAT*n) for i := 0; i < N; i++ { b[0] = byte(i * 79 % 97) @@ -238,18 +240,18 @@ func TestSmhasherSparse(t *testing.T) { if testing.Short() { t.Skip("Skipping in short mode") } - sparse(t, 32, 6) - sparse(t, 40, 6) - sparse(t, 48, 5) - sparse(t, 56, 5) - sparse(t, 64, 5) - sparse(t, 96, 4) - sparse(t, 256, 3) - sparse(t, 2048, 2) -} -func sparse(t *testing.T, n int, k int) { - b := make([]byte, n/8) h := newHashSet() + sparse(t, h, 32, 6) + sparse(t, h, 40, 6) + sparse(t, h, 48, 5) + sparse(t, h, 56, 5) + sparse(t, h, 64, 5) + sparse(t, h, 96, 4) + sparse(t, h, 256, 3) + sparse(t, h, 2048, 2) +} +func sparse(t *testing.T, h *hashSet, n int, k int) { + b := make([]byte, n/8) setbits(h, b, 0, k) h.check(t) } @@ -276,15 +278,15 @@ func TestSmhasherPermutation(t *testing.T) { if testing.Short() { t.Skip("Skipping in short mode") } - permutation(t, []uint32{0, 1, 2, 3, 4, 5, 6, 7}, 8) - permutation(t, []uint32{0, 1 << 29, 2 << 29, 3 << 29, 4 << 29, 5 << 29, 6 << 29, 7 << 29}, 8) - permutation(t, []uint32{0, 1}, 20) - permutation(t, []uint32{0, 1 << 31}, 20) - permutation(t, []uint32{0, 1, 2, 3, 4, 5, 6, 7, 1 << 29, 2 << 29, 3 << 29, 4 << 29, 5 << 29, 6 << 29, 7 << 29}, 6) -} -func permutation(t *testing.T, s []uint32, n int) { - b := make([]byte, n*4) h := newHashSet() + permutation(t, h, []uint32{0, 1, 2, 3, 4, 5, 6, 7}, 8) + permutation(t, h, []uint32{0, 1 << 29, 2 << 29, 3 << 29, 4 << 29, 5 << 29, 6 << 29, 7 << 29}, 8) + permutation(t, h, []uint32{0, 1}, 20) + permutation(t, h, []uint32{0, 1 << 31}, 20) + permutation(t, h, []uint32{0, 1, 2, 3, 4, 5, 6, 7, 1 << 29, 2 << 29, 3 << 29, 4 << 29, 5 << 29, 6 << 29, 7 << 29}, 6) +} +func permutation(t *testing.T, h *hashSet, s []uint32, n int) { + b := make([]byte, n*4) genPerm(h, b, s, 0) h.check(t) } @@ -418,8 +420,8 @@ func windowed(t *testing.T, k key) { } const BITS = 16 + h := newHashSet() for r := 0; r < k.bits(); r++ { - h := newHashSet() for i := 0; i < 1< expected+SLOP*(3*stddev+1) { t.Errorf("unexpected number of collisions: got=%d mean=%f stddev=%f threshold=%f", collisions, expected, stddev, expected+SLOP*(3*stddev+1)) } + // Reset for reuse + s.list = s.list[:0] } // a string plus adding zeros must make distinct hashes @@ -230,8 +232,8 @@ func TestSmhasherCyclic(t *testing.T) { r := rand.New(rand.NewSource(1234)) const REPEAT = 8 const N = 1000000 + h := newHashSet() for n := 4; n <= 12; n++ { - h := newHashSet() b := make([]byte, REPEAT*n) for i := 0; i < N; i++ { b[0] = byte(i * 79 % 97) @@ -256,18 +258,18 @@ func TestSmhasherSparse(t *testing.T) { if testing.Short() { t.Skip("Skipping in short mode") } - sparse(t, 32, 6) - sparse(t, 40, 6) - sparse(t, 48, 5) - sparse(t, 56, 5) - sparse(t, 64, 5) - sparse(t, 96, 4) - sparse(t, 256, 3) - sparse(t, 2048, 2) -} -func sparse(t *testing.T, n int, k int) { - b := make([]byte, n/8) h := newHashSet() + sparse(t, h, 32, 6) + sparse(t, h, 40, 6) + sparse(t, h, 48, 5) + sparse(t, h, 56, 5) + sparse(t, h, 64, 5) + sparse(t, h, 96, 4) + sparse(t, h, 256, 3) + sparse(t, h, 2048, 2) +} +func sparse(t *testing.T, h *HashSet, n int, k int) { + b := make([]byte, n/8) setbits(h, b, 0, k) h.check(t) } @@ -297,15 +299,15 @@ func TestSmhasherPermutation(t *testing.T) { if race.Enabled { t.Skip("Too long for race mode") } - permutation(t, []uint32{0, 1, 2, 3, 4, 5, 6, 7}, 8) - permutation(t, []uint32{0, 1 << 29, 2 << 29, 3 << 29, 4 << 29, 5 << 29, 6 << 29, 7 << 29}, 8) - permutation(t, []uint32{0, 1}, 20) - permutation(t, []uint32{0, 1 << 31}, 20) - permutation(t, []uint32{0, 1, 2, 3, 4, 5, 6, 7, 1 << 29, 2 << 29, 3 << 29, 4 << 29, 5 << 29, 6 << 29, 7 << 29}, 6) -} -func permutation(t *testing.T, s []uint32, n int) { - b := make([]byte, n*4) h := newHashSet() + permutation(t, h, []uint32{0, 1, 2, 3, 4, 5, 6, 7}, 8) + permutation(t, h, []uint32{0, 1 << 29, 2 << 29, 3 << 29, 4 << 29, 5 << 29, 6 << 29, 7 << 29}, 8) + permutation(t, h, []uint32{0, 1}, 20) + permutation(t, h, []uint32{0, 1 << 31}, 20) + permutation(t, h, []uint32{0, 1, 2, 3, 4, 5, 6, 7, 1 << 29, 2 << 29, 3 << 29, 4 << 29, 5 << 29, 6 << 29, 7 << 29}, 6) +} +func permutation(t *testing.T, h *HashSet, s []uint32, n int) { + b := make([]byte, n*4) genPerm(h, b, s, 0) h.check(t) } @@ -542,14 +544,15 @@ func TestSmhasherWindowed(t *testing.T) { if race.Enabled { t.Skip("Too long for race mode") } + h := newHashSet() t.Logf("32 bit keys") - windowed(t, &Int32Key{}) + windowed(t, h, &Int32Key{}) t.Logf("64 bit keys") - windowed(t, &Int64Key{}) + windowed(t, h, &Int64Key{}) t.Logf("string keys") - windowed(t, &BytesKey{make([]byte, 128)}) + windowed(t, h, &BytesKey{make([]byte, 128)}) } -func windowed(t *testing.T, k Key) { +func windowed(t *testing.T, h *HashSet, k Key) { if GOARCH == "wasm" { t.Skip("Too slow on wasm") } @@ -566,7 +569,6 @@ func windowed(t *testing.T, k Key) { const BITS = 16 for r := 0; r < k.bits(); r++ { - h := newHashSet() for i := 0; i < 1<