diff --git a/src/runtime/map.go b/src/runtime/map.go index 3e368f929fa..617e88faa45 100644 --- a/src/runtime/map.go +++ b/src/runtime/map.go @@ -89,11 +89,12 @@ const ( // Each bucket (including its overflow buckets, if any) will have either all or none of its // entries in the evacuated* states (except during the evacuate() method, which only happens // during map writes and thus no one else can observe the map during that time). - empty = 0 // cell is empty - evacuatedEmpty = 1 // cell is empty, bucket is evacuated. + emptyRest = 0 // this cell is empty, and there are no more non-empty cells at higher indexes or overflows. + emptyOne = 1 // this cell is empty evacuatedX = 2 // key/value is valid. Entry has been evacuated to first half of larger table. evacuatedY = 3 // same as above, but evacuated to second half of larger table. - minTopHash = 4 // minimum tophash for a normal filled cell. + evacuatedEmpty = 4 // cell is empty, bucket is evacuated. + minTopHash = 5 // minimum tophash for a normal filled cell. // flags iterator = 1 // there may be an iterator using buckets @@ -105,6 +106,11 @@ const ( noCheck = 1<<(8*sys.PtrSize) - 1 ) +// isEmpty reports whether the given tophash array entry represents an empty bucket entry. +func isEmpty(x uint8) bool { + return x <= emptyOne +} + // A header for a Go map. type hmap struct { // Note: the format of the hmap is also encoded in cmd/compile/internal/gc/reflect.go. @@ -197,7 +203,7 @@ func tophash(hash uintptr) uint8 { func evacuated(b *bmap) bool { h := b.tophash[0] - return h > empty && h < minTopHash + return h > emptyOne && h < minTopHash } func (b *bmap) overflow(t *maptype) *bmap { @@ -418,9 +424,13 @@ func mapaccess1(t *maptype, h *hmap, key unsafe.Pointer) unsafe.Pointer { } } top := tophash(hash) +bucketloop: for ; b != nil; b = b.overflow(t) { for i := uintptr(0); i < bucketCnt; i++ { if b.tophash[i] != top { + if b.tophash[i] == emptyRest { + break bucketloop + } continue } k := add(unsafe.Pointer(b), dataOffset+i*uintptr(t.keysize)) @@ -470,9 +480,13 @@ func mapaccess2(t *maptype, h *hmap, key unsafe.Pointer) (unsafe.Pointer, bool) } } top := tophash(hash) +bucketloop: for ; b != nil; b = b.overflow(t) { for i := uintptr(0); i < bucketCnt; i++ { if b.tophash[i] != top { + if b.tophash[i] == emptyRest { + break bucketloop + } continue } k := add(unsafe.Pointer(b), dataOffset+i*uintptr(t.keysize)) @@ -511,9 +525,13 @@ func mapaccessK(t *maptype, h *hmap, key unsafe.Pointer) (unsafe.Pointer, unsafe } } top := tophash(hash) +bucketloop: for ; b != nil; b = b.overflow(t) { for i := uintptr(0); i < bucketCnt; i++ { if b.tophash[i] != top { + if b.tophash[i] == emptyRest { + break bucketloop + } continue } k := add(unsafe.Pointer(b), dataOffset+i*uintptr(t.keysize)) @@ -587,14 +605,18 @@ again: var inserti *uint8 var insertk unsafe.Pointer var val unsafe.Pointer +bucketloop: for { for i := uintptr(0); i < bucketCnt; i++ { if b.tophash[i] != top { - if b.tophash[i] == empty && inserti == nil { + if isEmpty(b.tophash[i]) && inserti == nil { inserti = &b.tophash[i] insertk = add(unsafe.Pointer(b), dataOffset+i*uintptr(t.keysize)) val = add(unsafe.Pointer(b), dataOffset+bucketCnt*uintptr(t.keysize)+i*uintptr(t.valuesize)) } + if b.tophash[i] == emptyRest { + break bucketloop + } continue } k := add(unsafe.Pointer(b), dataOffset+i*uintptr(t.keysize)) @@ -694,6 +716,9 @@ search: for ; b != nil; b = b.overflow(t) { for i := uintptr(0); i < bucketCnt; i++ { if b.tophash[i] != top { + if b.tophash[i] == emptyRest { + break search + } continue } k := add(unsafe.Pointer(b), dataOffset+i*uintptr(t.keysize)) @@ -718,7 +743,8 @@ search: } else { memclrNoHeapPointers(v, t.elem.size) } - b.tophash[i] = empty + b.tophash[i] = emptyOne + // TODO: set up emptyRest here. h.count-- break search } @@ -833,7 +859,9 @@ next: } for ; i < bucketCnt; i++ { offi := (i + it.offset) & (bucketCnt - 1) - if b.tophash[offi] == empty || b.tophash[offi] == evacuatedEmpty { + if isEmpty(b.tophash[offi]) || b.tophash[offi] == evacuatedEmpty { + // TODO: emptyRest is hard to use here, as we start iterating + // in the middle of a bucket. It's feasible, just tricky. continue } k := add(unsafe.Pointer(b), dataOffset+uintptr(offi)*uintptr(t.keysize)) @@ -1092,7 +1120,7 @@ func evacuate(t *maptype, h *hmap, oldbucket uintptr) { v := add(k, bucketCnt*uintptr(t.keysize)) for i := 0; i < bucketCnt; i, k, v = i+1, add(k, uintptr(t.keysize)), add(v, uintptr(t.valuesize)) { top := b.tophash[i] - if top == empty { + if isEmpty(top) { b.tophash[i] = evacuatedEmpty continue } @@ -1129,7 +1157,7 @@ func evacuate(t *maptype, h *hmap, oldbucket uintptr) { } } - if evacuatedX+1 != evacuatedY { + if evacuatedX+1 != evacuatedY || evacuatedX^1 != evacuatedY { throw("bad evacuatedN") } diff --git a/src/runtime/map_benchmark_test.go b/src/runtime/map_benchmark_test.go index 5681d5eeb8c..d37dadcb569 100644 --- a/src/runtime/map_benchmark_test.go +++ b/src/runtime/map_benchmark_test.go @@ -5,6 +5,7 @@ package runtime_test import ( "fmt" + "math/rand" "strconv" "strings" "testing" @@ -206,6 +207,67 @@ func BenchmarkIntMap(b *testing.B) { } } +func BenchmarkMapFirst(b *testing.B) { + for n := 1; n <= 16; n++ { + b.Run(fmt.Sprintf("%d", n), func(b *testing.B) { + m := make(map[int]bool) + for i := 0; i < n; i++ { + m[i] = true + } + b.ResetTimer() + for i := 0; i < b.N; i++ { + _ = m[0] + } + }) + } +} +func BenchmarkMapMid(b *testing.B) { + for n := 1; n <= 16; n++ { + b.Run(fmt.Sprintf("%d", n), func(b *testing.B) { + m := make(map[int]bool) + for i := 0; i < n; i++ { + m[i] = true + } + b.ResetTimer() + for i := 0; i < b.N; i++ { + _ = m[n>>1] + } + }) + } +} +func BenchmarkMapLast(b *testing.B) { + for n := 1; n <= 16; n++ { + b.Run(fmt.Sprintf("%d", n), func(b *testing.B) { + m := make(map[int]bool) + for i := 0; i < n; i++ { + m[i] = true + } + b.ResetTimer() + for i := 0; i < b.N; i++ { + _ = m[n-1] + } + }) + } +} + +func BenchmarkMapCycle(b *testing.B) { + // Arrange map entries to be a permuation, so that + // we hit all entries, and one lookup is data dependent + // on the previous lookup. + const N = 3127 + p := rand.New(rand.NewSource(1)).Perm(N) + m := map[int]int{} + for i := 0; i < N; i++ { + m[i] = p[i] + } + b.ResetTimer() + j := 0 + for i := 0; i < b.N; i++ { + j = m[j] + } + sink = uint64(j) +} + // Accessing the same keys in a row. func benchmarkRepeatedLookup(b *testing.B, lookupKeySize int) { m := make(map[string]bool) diff --git a/src/runtime/map_fast32.go b/src/runtime/map_fast32.go index 671558545a2..063a5cbe3a6 100644 --- a/src/runtime/map_fast32.go +++ b/src/runtime/map_fast32.go @@ -41,7 +41,7 @@ func mapaccess1_fast32(t *maptype, h *hmap, key uint32) unsafe.Pointer { } for ; b != nil; b = b.overflow(t) { for i, k := uintptr(0), b.keys(); i < bucketCnt; i, k = i+1, add(k, 4) { - if *(*uint32)(k) == key && b.tophash[i] != empty { + if *(*uint32)(k) == key && !isEmpty(b.tophash[i]) { return add(unsafe.Pointer(b), dataOffset+bucketCnt*4+i*uintptr(t.valuesize)) } } @@ -81,7 +81,7 @@ func mapaccess2_fast32(t *maptype, h *hmap, key uint32) (unsafe.Pointer, bool) { } for ; b != nil; b = b.overflow(t) { for i, k := uintptr(0), b.keys(); i < bucketCnt; i, k = i+1, add(k, 4) { - if *(*uint32)(k) == key && b.tophash[i] != empty { + if *(*uint32)(k) == key && !isEmpty(b.tophash[i]) { return add(unsafe.Pointer(b), dataOffset+bucketCnt*4+i*uintptr(t.valuesize)), true } } @@ -120,13 +120,17 @@ again: var inserti uintptr var insertk unsafe.Pointer +bucketloop: for { for i := uintptr(0); i < bucketCnt; i++ { - if b.tophash[i] == empty { + if isEmpty(b.tophash[i]) { if insertb == nil { inserti = i insertb = b } + if b.tophash[i] == emptyRest { + break bucketloop + } continue } k := *((*uint32)(add(unsafe.Pointer(b), dataOffset+i*4))) @@ -206,13 +210,17 @@ again: var inserti uintptr var insertk unsafe.Pointer +bucketloop: for { for i := uintptr(0); i < bucketCnt; i++ { - if b.tophash[i] == empty { + if isEmpty(b.tophash[i]) { if insertb == nil { inserti = i insertb = b } + if b.tophash[i] == emptyRest { + break bucketloop + } continue } k := *((*unsafe.Pointer)(add(unsafe.Pointer(b), dataOffset+i*4))) @@ -286,7 +294,7 @@ func mapdelete_fast32(t *maptype, h *hmap, key uint32) { search: for ; b != nil; b = b.overflow(t) { for i, k := uintptr(0), b.keys(); i < bucketCnt; i, k = i+1, add(k, 4) { - if key != *(*uint32)(k) || b.tophash[i] == empty { + if key != *(*uint32)(k) || isEmpty(b.tophash[i]) { continue } // Only clear key if there are pointers in it. @@ -299,7 +307,8 @@ search: } else { memclrNoHeapPointers(v, t.elem.size) } - b.tophash[i] = empty + b.tophash[i] = emptyOne + // TODO: emptyRest? h.count-- break search } @@ -350,7 +359,7 @@ func evacuate_fast32(t *maptype, h *hmap, oldbucket uintptr) { v := add(k, bucketCnt*4) for i := 0; i < bucketCnt; i, k, v = i+1, add(k, 4), add(v, uintptr(t.valuesize)) { top := b.tophash[i] - if top == empty { + if isEmpty(top) { b.tophash[i] = evacuatedEmpty continue } diff --git a/src/runtime/map_fast64.go b/src/runtime/map_fast64.go index 164a4dd1cef..8270cf7b7d5 100644 --- a/src/runtime/map_fast64.go +++ b/src/runtime/map_fast64.go @@ -41,7 +41,7 @@ func mapaccess1_fast64(t *maptype, h *hmap, key uint64) unsafe.Pointer { } for ; b != nil; b = b.overflow(t) { for i, k := uintptr(0), b.keys(); i < bucketCnt; i, k = i+1, add(k, 8) { - if *(*uint64)(k) == key && b.tophash[i] != empty { + if *(*uint64)(k) == key && !isEmpty(b.tophash[i]) { return add(unsafe.Pointer(b), dataOffset+bucketCnt*8+i*uintptr(t.valuesize)) } } @@ -81,7 +81,7 @@ func mapaccess2_fast64(t *maptype, h *hmap, key uint64) (unsafe.Pointer, bool) { } for ; b != nil; b = b.overflow(t) { for i, k := uintptr(0), b.keys(); i < bucketCnt; i, k = i+1, add(k, 8) { - if *(*uint64)(k) == key && b.tophash[i] != empty { + if *(*uint64)(k) == key && !isEmpty(b.tophash[i]) { return add(unsafe.Pointer(b), dataOffset+bucketCnt*8+i*uintptr(t.valuesize)), true } } @@ -120,13 +120,17 @@ again: var inserti uintptr var insertk unsafe.Pointer +bucketloop: for { for i := uintptr(0); i < bucketCnt; i++ { - if b.tophash[i] == empty { + if isEmpty(b.tophash[i]) { if insertb == nil { insertb = b inserti = i } + if b.tophash[i] == emptyRest { + break bucketloop + } continue } k := *((*uint64)(add(unsafe.Pointer(b), dataOffset+i*8))) @@ -206,13 +210,17 @@ again: var inserti uintptr var insertk unsafe.Pointer +bucketloop: for { for i := uintptr(0); i < bucketCnt; i++ { - if b.tophash[i] == empty { + if isEmpty(b.tophash[i]) { if insertb == nil { insertb = b inserti = i } + if b.tophash[i] == emptyRest { + break bucketloop + } continue } k := *((*unsafe.Pointer)(add(unsafe.Pointer(b), dataOffset+i*8))) @@ -286,7 +294,7 @@ func mapdelete_fast64(t *maptype, h *hmap, key uint64) { search: for ; b != nil; b = b.overflow(t) { for i, k := uintptr(0), b.keys(); i < bucketCnt; i, k = i+1, add(k, 8) { - if key != *(*uint64)(k) || b.tophash[i] == empty { + if key != *(*uint64)(k) || isEmpty(b.tophash[i]) { continue } // Only clear key if there are pointers in it. @@ -299,7 +307,8 @@ search: } else { memclrNoHeapPointers(v, t.elem.size) } - b.tophash[i] = empty + b.tophash[i] = emptyOne + //TODO: emptyRest h.count-- break search } @@ -350,7 +359,7 @@ func evacuate_fast64(t *maptype, h *hmap, oldbucket uintptr) { v := add(k, bucketCnt*8) for i := 0; i < bucketCnt; i, k, v = i+1, add(k, 8), add(v, uintptr(t.valuesize)) { top := b.tophash[i] - if top == empty { + if isEmpty(top) { b.tophash[i] = evacuatedEmpty continue } diff --git a/src/runtime/map_faststr.go b/src/runtime/map_faststr.go index bee62dfb03b..8f505f90a60 100644 --- a/src/runtime/map_faststr.go +++ b/src/runtime/map_faststr.go @@ -28,7 +28,10 @@ func mapaccess1_faststr(t *maptype, h *hmap, ky string) unsafe.Pointer { // short key, doing lots of comparisons is ok for i, kptr := uintptr(0), b.keys(); i < bucketCnt; i, kptr = i+1, add(kptr, 2*sys.PtrSize) { k := (*stringStruct)(kptr) - if k.len != key.len || b.tophash[i] == empty { + if k.len != key.len || isEmpty(b.tophash[i]) { + if b.tophash[i] == emptyRest { + break + } continue } if k.str == key.str || memequal(k.str, key.str, uintptr(key.len)) { @@ -41,7 +44,10 @@ func mapaccess1_faststr(t *maptype, h *hmap, ky string) unsafe.Pointer { keymaybe := uintptr(bucketCnt) for i, kptr := uintptr(0), b.keys(); i < bucketCnt; i, kptr = i+1, add(kptr, 2*sys.PtrSize) { k := (*stringStruct)(kptr) - if k.len != key.len || b.tophash[i] == empty { + if k.len != key.len || isEmpty(b.tophash[i]) { + if b.tophash[i] == emptyRest { + break + } continue } if k.str == key.str { @@ -117,7 +123,10 @@ func mapaccess2_faststr(t *maptype, h *hmap, ky string) (unsafe.Pointer, bool) { // short key, doing lots of comparisons is ok for i, kptr := uintptr(0), b.keys(); i < bucketCnt; i, kptr = i+1, add(kptr, 2*sys.PtrSize) { k := (*stringStruct)(kptr) - if k.len != key.len || b.tophash[i] == empty { + if k.len != key.len || isEmpty(b.tophash[i]) { + if b.tophash[i] == emptyRest { + break + } continue } if k.str == key.str || memequal(k.str, key.str, uintptr(key.len)) { @@ -130,7 +139,10 @@ func mapaccess2_faststr(t *maptype, h *hmap, ky string) (unsafe.Pointer, bool) { keymaybe := uintptr(bucketCnt) for i, kptr := uintptr(0), b.keys(); i < bucketCnt; i, kptr = i+1, add(kptr, 2*sys.PtrSize) { k := (*stringStruct)(kptr) - if k.len != key.len || b.tophash[i] == empty { + if k.len != key.len || isEmpty(b.tophash[i]) { + if b.tophash[i] == emptyRest { + break + } continue } if k.str == key.str { @@ -220,13 +232,17 @@ again: var inserti uintptr var insertk unsafe.Pointer +bucketloop: for { for i := uintptr(0); i < bucketCnt; i++ { if b.tophash[i] != top { - if b.tophash[i] == empty && insertb == nil { + if isEmpty(b.tophash[i]) && insertb == nil { insertb = b inserti = i } + if b.tophash[i] == emptyRest { + break bucketloop + } continue } k := (*stringStruct)(add(unsafe.Pointer(b), dataOffset+i*2*sys.PtrSize)) @@ -320,7 +336,8 @@ search: } else { memclrNoHeapPointers(v, t.elem.size) } - b.tophash[i] = empty + b.tophash[i] = emptyOne + // TODO: emptyRest h.count-- break search } @@ -371,7 +388,7 @@ func evacuate_faststr(t *maptype, h *hmap, oldbucket uintptr) { v := add(k, bucketCnt*2*sys.PtrSize) for i := 0; i < bucketCnt; i, k, v = i+1, add(k, 2*sys.PtrSize), add(v, uintptr(t.valuesize)) { top := b.tophash[i] - if top == empty { + if isEmpty(top) { b.tophash[i] = evacuatedEmpty continue }