From 54e9406ffbabadb2eaf2cf1f3b2297f8a3998157 Mon Sep 17 00:00:00 2001 From: Dmitriy Vyukov Date: Mon, 8 Aug 2011 09:35:32 -0400 Subject: [PATCH] runtime: add more specialized type algorithms The change adds specialized type algorithms for slices and types of size 8/16/32/64/128. It significantly accelerates chan and map operations for most builtin types as well as user structs. benchmark old,ns/op new,ns/op BenchmarkChanUncontended 226 94 (on Intel Xeon E5620, 2.4GHz, Linux 64 bit) R=golang-dev, rsc CC=golang-dev https://golang.org/cl/4815087 --- src/cmd/gc/go.h | 12 ++- src/cmd/gc/subr.c | 30 +++++++- src/pkg/runtime/runtime.c | 158 +++++++++++++++++++++++++++++++++++--- src/pkg/runtime/runtime.h | 12 ++- 4 files changed, 196 insertions(+), 16 deletions(-) diff --git a/src/cmd/gc/go.h b/src/cmd/gc/go.h index 44912729267..da0fb5146e8 100644 --- a/src/cmd/gc/go.h +++ b/src/cmd/gc/go.h @@ -40,7 +40,17 @@ enum ASTRING, AINTER, ANILINTER, - AMEMWORD, + ASLICE, + AMEM8, + AMEM16, + AMEM32, + AMEM64, + AMEM128, + ANOEQ8, + ANOEQ16, + ANOEQ32, + ANOEQ64, + ANOEQ128, BADWIDTH = -1000000000, }; diff --git a/src/cmd/gc/subr.c b/src/cmd/gc/subr.c index 96675be3fd6..1a05d43d0ec 100644 --- a/src/cmd/gc/subr.c +++ b/src/cmd/gc/subr.c @@ -493,8 +493,16 @@ algtype(Type *t) if(issimple[t->etype] || isptr[t->etype] || t->etype == TCHAN || t->etype == TFUNC || t->etype == TMAP) { - if(t->width == widthptr) - a = AMEMWORD; + if(t->width == 1) + a = AMEM8; + else if(t->width == 2) + a = AMEM16; + else if(t->width == 4) + a = AMEM32; + else if(t->width == 8) + a = AMEM64; + else if(t->width == 16) + a = AMEM128; else a = AMEM; // just bytes (int, ptr, etc) } else if(t->etype == TSTRING) @@ -503,8 +511,22 @@ algtype(Type *t) a = ANILINTER; // nil interface else if(t->etype == TINTER) a = AINTER; // interface - else - a = ANOEQ; // just bytes, but no hash/eq + else if(isslice(t)) + a = ASLICE; // slice + else { + if(t->width == 1) + a = ANOEQ8; + else if(t->width == 2) + a = ANOEQ16; + else if(t->width == 4) + a = ANOEQ32; + else if(t->width == 8) + a = ANOEQ64; + else if(t->width == 16) + a = ANOEQ128; + else + a = ANOEQ; // just bytes, but no hash/eq + } return a; } diff --git a/src/pkg/runtime/runtime.c b/src/pkg/runtime/runtime.c index 57c08733064..49aba7da018 100644 --- a/src/pkg/runtime/runtime.c +++ b/src/pkg/runtime/runtime.c @@ -398,21 +398,110 @@ memcopy(uint32 s, void *a, void *b) } static uint32 -memwordequal(uint32 s, void *a, void *b) +memequal8(uint32 s, uint8 *a, uint8 *b) { USED(s); - return *(uintptr*)(a) == *(uintptr*)(b); + return *a == *b; } static void -memwordcopy(uint32 s, void *a, void *b) +memcopy8(uint32 s, uint8 *a, uint8 *b) { USED(s); - if (b == nil) { - *(uintptr*)(a) = 0; + if(b == nil) { + *a = 0; return; } - *(uintptr*)(a) = *(uintptr*)(b); + *a = *b; +} + +static uint32 +memequal16(uint32 s, uint16 *a, uint16 *b) +{ + USED(s); + return *a == *b; +} + +static void +memcopy16(uint32 s, uint16 *a, uint16 *b) +{ + USED(s); + if(b == nil) { + *a = 0; + return; + } + *a = *b; +} + +static uint32 +memequal32(uint32 s, uint32 *a, uint32 *b) +{ + USED(s); + return *a == *b; +} + +static void +memcopy32(uint32 s, uint32 *a, uint32 *b) +{ + USED(s); + if(b == nil) { + *a = 0; + return; + } + *a = *b; +} + +static uint32 +memequal64(uint32 s, uint64 *a, uint64 *b) +{ + USED(s); + return *a == *b; +} + +static void +memcopy64(uint32 s, uint64 *a, uint64 *b) +{ + USED(s); + if(b == nil) { + *a = 0; + return; + } + *a = *b; +} + +static uint32 +memequal128(uint32 s, uint64 *a, uint64 *b) +{ + USED(s); + return a[0] == b[0] && a[1] == b[1]; +} + +static void +memcopy128(uint32 s, uint64 *a, uint64 *b) +{ + USED(s); + if(b == nil) { + a[0] = 0; + a[1] = 0; + return; + } + a[0] = b[0]; + a[1] = b[1]; +} + +static void +slicecopy(uint32 s, Slice *a, Slice *b) +{ + USED(s); + if(b == nil) { + a->array = 0; + a->len = 0; + a->cap = 0; + return; + } + a->array = b->array; + a->len = b->len; + a->cap = b->cap; } static uintptr @@ -441,6 +530,19 @@ strprint(uint32 s, String *a) runtime·printstring(*a); } +static void +strcopy(uint32 s, String *a, String *b) +{ + USED(s); + if(b == nil) { + a->str = 0; + a->len = 0; + return; + } + a->str = b->str; + a->len = b->len; +} + static uintptr interhash(uint32 s, Iface *a) { @@ -462,6 +564,19 @@ interequal(uint32 s, Iface *a, Iface *b) return runtime·ifaceeq_c(*a, *b); } +static void +intercopy(uint32 s, Iface *a, Iface *b) +{ + USED(s); + if(b == nil) { + a->tab = 0; + a->data = 0; + return; + } + a->tab = b->tab; + a->data = b->data; +} + static uintptr nilinterhash(uint32 s, Eface *a) { @@ -483,6 +598,19 @@ nilinterequal(uint32 s, Eface *a, Eface *b) return runtime·efaceeq_c(*a, *b); } +static void +nilintercopy(uint32 s, Eface *a, Eface *b) +{ + USED(s); + if(b == nil) { + a->type = 0; + a->data = 0; + return; + } + a->type = b->type; + a->data = b->data; +} + uintptr runtime·nohash(uint32 s, void *a) { @@ -507,10 +635,20 @@ runtime·algarray[] = { [AMEM] { memhash, memequal, memprint, memcopy }, [ANOEQ] { runtime·nohash, runtime·noequal, memprint, memcopy }, -[ASTRING] { strhash, strequal, strprint, memcopy }, -[AINTER] { interhash, interequal, interprint, memcopy }, -[ANILINTER] { nilinterhash, nilinterequal, nilinterprint, memcopy }, -[AMEMWORD] { memhash, memwordequal, memprint, memwordcopy }, +[ASTRING] { strhash, strequal, strprint, strcopy }, +[AINTER] { interhash, interequal, interprint, intercopy }, +[ANILINTER] { nilinterhash, nilinterequal, nilinterprint, nilintercopy }, +[ASLICE] { runtime·nohash, runtime·noequal, memprint, slicecopy }, +[AMEM8] { memhash, memequal8, memprint, memcopy8 }, +[AMEM16] { memhash, memequal16, memprint, memcopy16 }, +[AMEM32] { memhash, memequal32, memprint, memcopy32 }, +[AMEM64] { memhash, memequal64, memprint, memcopy64 }, +[AMEM128] { memhash, memequal128, memprint, memcopy128 }, +[ANOEQ8] { runtime·nohash, runtime·noequal, memprint, memcopy8 }, +[ANOEQ16] { runtime·nohash, runtime·noequal, memprint, memcopy16 }, +[ANOEQ32] { runtime·nohash, runtime·noequal, memprint, memcopy32 }, +[ANOEQ64] { runtime·nohash, runtime·noequal, memprint, memcopy64 }, +[ANOEQ128] { runtime·nohash, runtime·noequal, memprint, memcopy128 }, }; int64 diff --git a/src/pkg/runtime/runtime.h b/src/pkg/runtime/runtime.h index d2e4378b59c..8c5403f444c 100644 --- a/src/pkg/runtime/runtime.h +++ b/src/pkg/runtime/runtime.h @@ -335,7 +335,17 @@ enum ASTRING, AINTER, ANILINTER, - AMEMWORD, + ASLICE, + AMEM8, + AMEM16, + AMEM32, + AMEM64, + AMEM128, + ANOEQ8, + ANOEQ16, + ANOEQ32, + ANOEQ64, + ANOEQ128, Amax };