qbit/go
1
0
Fork 0
mirror of https://github.com/golang/go synced 2024-11-19 16:24:45 -07:00
Code Releases Activity

runtime: refactor insertion slot tracking for fast hashmap functions

Browse Source 
* Avoid calculating insertk until needed.
* Avoid a pointer into b.tophash and just track the insertion index.
  This avoids b.tophash being marked as escaping to heap.
* Calculate val only once at the end of the mapassign functions.

Function sizes decrease slightly, e.g. for mapassign_faststr:
before "".mapassign_faststr STEXT size=1166 args=0x28 locals=0x78
after  "".mapassign_faststr STEXT size=1080 args=0x28 locals=0x68

name                     old time/op  new time/op  delta
MapAssign/Int32/256-4    19.4ns ± 4%  19.5ns ±11%     ~     (p=0.973 n=20+20)
MapAssign/Int32/65536-4  32.5ns ± 2%  32.4ns ± 3%     ~     (p=0.078 n=20+19)
MapAssign/Int64/256-4    20.3ns ± 6%  17.6ns ± 5%  -13.01%  (p=0.000 n=20+20)
MapAssign/Int64/65536-4  33.3ns ± 2%  33.3ns ± 1%     ~     (p=0.444 n=20+20)
MapAssign/Str/256-4      22.3ns ± 3%  22.4ns ± 3%     ~     (p=0.343 n=20+20)
MapAssign/Str/65536-4    44.9ns ± 1%  43.9ns ± 1%   -2.39%  (p=0.000 n=20+19)

Change-Id: I2627bb8a961d366d9473b5922fa129176319eb22
Reviewed-on: https://go-review.googlesource.com/74870
Run-TryBot: Martin Möhrmann <moehrmann@google.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Keith Randall <khr@golang.org>
This commit is contained in:
Martin Möhrmann 2017-09-07 05:57:44 +02:00
parent 2f1f607b21
commit 8585f9fdb1
1 changed files with 45 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

87
src/runtime/hashmap_fast.go
View File

@ -374,16 +374,16 @@ again:
} }
b := (*bmap)(unsafe.Pointer(uintptr(h.buckets) + bucket*uintptr(t.bucketsize))) b := (*bmap)(unsafe.Pointer(uintptr(h.buckets) + bucket*uintptr(t.bucketsize)))
var inserti *uint8 var insertb *bmap
var inserti uintptr
var insertk unsafe.Pointer var insertk unsafe.Pointer
var val unsafe.Pointer
for { for {
for i := uintptr(0); i < bucketCnt; i++ { for i := uintptr(0); i < bucketCnt; i++ {
if b.tophash[i] == empty { if b.tophash[i] == empty {
if inserti == nil { if insertb == nil {
inserti = &b.tophash[i] inserti = i
insertk = add(unsafe.Pointer(b), dataOffset+i*4) insertb = b
val = add(unsafe.Pointer(b), dataOffset+bucketCnt*4+i*uintptr(t.valuesize))
} }
continue continue
} }
@ -391,7 +391,8 @@ again:
if k != key { if k != key {
continue continue
} }
val = add(unsafe.Pointer(b), dataOffset+bucketCnt*4+i*uintptr(t.valuesize)) inserti = i
insertb = b
goto done goto done
} }
ovf := b.overflow(t) ovf := b.overflow(t)
@ -410,14 +411,14 @@ again:
goto again // Growing the table invalidates everything, so try again goto again // Growing the table invalidates everything, so try again
} }
if inserti == nil { if insertb == nil {
// all current buckets are full, allocate a new one. // all current buckets are full, allocate a new one.
newb := h.newoverflow(t, b) insertb = h.newoverflow(t, b)
inserti = &newb.tophash[0] inserti = 0 // not necessary, but avoids needlessly spilling inserti
insertk = add(unsafe.Pointer(newb), dataOffset)
val = add(insertk, bucketCnt*4)
} }
insertb.tophash[inserti&(bucketCnt-1)] = tophash(hash) // mask inserti to avoid bounds checks
insertk = add(unsafe.Pointer(insertb), dataOffset+inserti*4)
// store new key at insert position // store new key at insert position
if sys.PtrSize == 4 && t.key.kind&kindNoPointers == 0 && writeBarrier.enabled { if sys.PtrSize == 4 && t.key.kind&kindNoPointers == 0 && writeBarrier.enabled {
writebarrierptr((*uintptr)(insertk), uintptr(key)) writebarrierptr((*uintptr)(insertk), uintptr(key))
@ -425,10 +426,10 @@ again:
*(*uint32)(insertk) = key *(*uint32)(insertk) = key
} }
*inserti = tophash(hash)
h.count++ h.count++
done: done:
val := add(unsafe.Pointer(insertb), dataOffset+bucketCnt*4+inserti*uintptr(t.valuesize))
if h.flags&hashWriting == 0 { if h.flags&hashWriting == 0 {
throw("concurrent map writes") throw("concurrent map writes")
} }
@ -463,16 +464,16 @@ again:
} }
b := (*bmap)(unsafe.Pointer(uintptr(h.buckets) + bucket*uintptr(t.bucketsize))) b := (*bmap)(unsafe.Pointer(uintptr(h.buckets) + bucket*uintptr(t.bucketsize)))
var inserti *uint8 var insertb *bmap
var inserti uintptr
var insertk unsafe.Pointer var insertk unsafe.Pointer
var val unsafe.Pointer
for { for {
for i := uintptr(0); i < bucketCnt; i++ { for i := uintptr(0); i < bucketCnt; i++ {
if b.tophash[i] == empty { if b.tophash[i] == empty {
if inserti == nil { if insertb == nil {
inserti = &b.tophash[i] insertb = b
insertk = add(unsafe.Pointer(b), dataOffset+i*8) inserti = i
val = add(unsafe.Pointer(b), dataOffset+bucketCnt*8+i*uintptr(t.valuesize))
} }
continue continue
} }
@ -480,7 +481,8 @@ again:
if k != key { if k != key {
continue continue
} }
val = add(unsafe.Pointer(b), dataOffset+bucketCnt*8+i*uintptr(t.valuesize)) insertb = b
inserti = i
goto done goto done
} }
ovf := b.overflow(t) ovf := b.overflow(t)
@ -499,14 +501,14 @@ again:
goto again // Growing the table invalidates everything, so try again goto again // Growing the table invalidates everything, so try again
} }
if inserti == nil { if insertb == nil {
// all current buckets are full, allocate a new one. // all current buckets are full, allocate a new one.
newb := h.newoverflow(t, b) insertb = h.newoverflow(t, b)
inserti = &newb.tophash[0] inserti = 0 // not necessary, but avoids needlessly spilling inserti
insertk = add(unsafe.Pointer(newb), dataOffset)
val = add(insertk, bucketCnt*8)
} }
insertb.tophash[inserti&(bucketCnt-1)] = tophash(hash) // mask inserti to avoid bounds checks
insertk = add(unsafe.Pointer(insertb), dataOffset+inserti*8)
// store new key at insert position // store new key at insert position
if t.key.kind&kindNoPointers == 0 && writeBarrier.enabled { if t.key.kind&kindNoPointers == 0 && writeBarrier.enabled {
if sys.PtrSize == 8 { if sys.PtrSize == 8 {
@ -520,10 +522,10 @@ again:
*(*uint64)(insertk) = key *(*uint64)(insertk) = key
} }
*inserti = tophash(hash)
h.count++ h.count++
done: done:
val := add(unsafe.Pointer(insertb), dataOffset+bucketCnt*8+inserti*uintptr(t.valuesize))
if h.flags&hashWriting == 0 { if h.flags&hashWriting == 0 {
throw("concurrent map writes") throw("concurrent map writes")
} }
@ -531,7 +533,7 @@ done:
return val return val
} }
func mapassign_faststr(t *maptype, h *hmap, ky string) unsafe.Pointer { func mapassign_faststr(t *maptype, h *hmap, s string) unsafe.Pointer {
if h == nil { if h == nil {
panic(plainError("assignment to entry in nil map")) panic(plainError("assignment to entry in nil map"))
} }
@ -542,8 +544,8 @@ func mapassign_faststr(t *maptype, h *hmap, ky string) unsafe.Pointer {
if h.flags&hashWriting != 0 { if h.flags&hashWriting != 0 {
throw("concurrent map writes") throw("concurrent map writes")
} }
key := stringStructOf(&ky) key := stringStructOf(&s)
hash := t.key.alg.hash(noescape(unsafe.Pointer(&ky)), uintptr(h.hash0)) hash := t.key.alg.hash(noescape(unsafe.Pointer(&s)), uintptr(h.hash0))
// Set hashWriting after calling alg.hash for consistency with mapassign. // Set hashWriting after calling alg.hash for consistency with mapassign.
h.flags |= hashWriting h.flags |= hashWriting
@ -560,16 +562,16 @@ again:
b := (*bmap)(unsafe.Pointer(uintptr(h.buckets) + bucket*uintptr(t.bucketsize))) b := (*bmap)(unsafe.Pointer(uintptr(h.buckets) + bucket*uintptr(t.bucketsize)))
top := tophash(hash) top := tophash(hash)
var inserti *uint8 var insertb *bmap
var inserti uintptr
var insertk unsafe.Pointer var insertk unsafe.Pointer
var val unsafe.Pointer
for { for {
for i := uintptr(0); i < bucketCnt; i++ { for i := uintptr(0); i < bucketCnt; i++ {
if b.tophash[i] != top { if b.tophash[i] != top {
if b.tophash[i] == empty && inserti == nil { if b.tophash[i] == empty && insertb == nil {
inserti = &b.tophash[i] insertb = b
insertk = add(unsafe.Pointer(b), dataOffset+i*2*sys.PtrSize) inserti = i
val = add(unsafe.Pointer(b), dataOffset+bucketCnt*2*sys.PtrSize+i*uintptr(t.valuesize))
} }
continue continue
} }
@ -581,7 +583,8 @@ again:
continue continue
} }
// already have a mapping for key. Update it. // already have a mapping for key. Update it.
val = add(unsafe.Pointer(b), dataOffset+bucketCnt*2*sys.PtrSize+i*uintptr(t.valuesize)) inserti = i
insertb = b
goto done goto done
} }
ovf := b.overflow(t) ovf := b.overflow(t)
@ -600,20 +603,20 @@ again:
goto again // Growing the table invalidates everything, so try again goto again // Growing the table invalidates everything, so try again
} }
if inserti == nil { if insertb == nil {
// all current buckets are full, allocate a new one. // all current buckets are full, allocate a new one.
newb := h.newoverflow(t, b) insertb = h.newoverflow(t, b)
inserti = &newb.tophash[0] inserti = 0 // not necessary, but avoids needlessly spilling inserti
insertk = add(unsafe.Pointer(newb), dataOffset)
val = add(insertk, bucketCnt*2*sys.PtrSize)
} }
insertb.tophash[inserti&(bucketCnt-1)] = top // mask inserti to avoid bounds checks
insertk = add(unsafe.Pointer(insertb), dataOffset+inserti*2*sys.PtrSize)
// store new key at insert position // store new key at insert position
*((*stringStruct)(insertk)) = *key *((*stringStruct)(insertk)) = *key
*inserti = top
h.count++ h.count++
done: done:
val := add(unsafe.Pointer(insertb), dataOffset+bucketCnt*2*sys.PtrSize+inserti*uintptr(t.valuesize))
if h.flags&hashWriting == 0 { if h.flags&hashWriting == 0 {
throw("concurrent map writes") throw("concurrent map writes")
} }