From ca8354843ef9f30207efd0a40bb6c53e7ba86892 Mon Sep 17 00:00:00 2001 From: Carlo Alberto Ferraris Date: Tue, 13 Nov 2018 15:34:22 +0900 Subject: [PATCH] sync: allow inlining the Once.Do fast path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Using Once.Do is now extremely cheap because the fast path is just an inlined atomic load of a variable that is written only once and a conditional jump. This is very beneficial for Once.Do because, due to its nature, the fast path will be used for every call after the first one. In a attempt to mimize code size increase, reorder the fields so that the pointer to Once is also the pointer to Once.done, that is the only field used in the hot path. This allows to use more compact instruction encodings or less instructions in the hot path (that is inlined at every callsite). name old time/op new time/op delta Once 4.54ns ± 0% 2.06ns ± 0% -54.59% (p=0.000 n=19+16) Once-4 1.18ns ± 0% 0.55ns ± 0% -53.39% (p=0.000 n=15+16) Once-16 0.53ns ± 0% 0.17ns ± 0% -67.92% (p=0.000 n=18+17) linux/amd64 bin/go 14675861 (previous commit 14663387, +12474/+0.09%) Change-Id: Ie2708103ab473787875d66746d2f20f1d90a6916 Reviewed-on: https://go-review.googlesource.com/c/go/+/152697 Run-TryBot: Brad Fitzpatrick TryBot-Result: Gobot Gobot Reviewed-by: Brad Fitzpatrick Reviewed-by: Austin Clements --- src/sync/once.go | 16 ++++++++++++---- test/inline_sync.go | 7 +++++++ 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/src/sync/once.go b/src/sync/once.go index d8ef952ea54..84761970dd8 100644 --- a/src/sync/once.go +++ b/src/sync/once.go @@ -10,8 +10,13 @@ import ( // Once is an object that will perform exactly one action. type Once struct { - m Mutex + // done indicates whether the action has been performed. + // It is first in the struct because it is used in the hot path. + // The hot path is inlined at every call site. + // Placing done first allows more compact instructions on some architectures (amd64/x86), + // and fewer instructions (to calculate offset) on other architectures. done uint32 + m Mutex } // Do calls the function f if and only if Do is being called for the @@ -33,10 +38,13 @@ type Once struct { // without calling f. // func (o *Once) Do(f func()) { - if atomic.LoadUint32(&o.done) == 1 { - return + if atomic.LoadUint32(&o.done) == 0 { + // Outlined slow-path to allow inlining of the fast-path. + o.doSlow(f) } - // Slow-path. +} + +func (o *Once) doSlow(f func()) { o.m.Lock() defer o.m.Unlock() if o.done == 0 { diff --git a/test/inline_sync.go b/test/inline_sync.go index a14f58c4320..3473b92b4ac 100644 --- a/test/inline_sync.go +++ b/test/inline_sync.go @@ -31,3 +31,10 @@ func small6() { // ERROR "can inline small6" // the Lock fast path should be inlined mutex.Lock() // ERROR "inlining call to sync\.\(\*Mutex\)\.Lock" "&sync\.m\.state escapes to heap" } + +var once *sync.Once + +func small7() { // ERROR "can inline small7" + // the Do fast path should be inlined + once.Do(small5) // ERROR "inlining call to sync\.\(\*Once\)\.Do" "&sync\.o\.done escapes to heap" +}