From 6633bb2aa7c8ab53dc6cc8a4ef8c4fef7a439cee Mon Sep 17 00:00:00 2001 From: Carlos Eduardo Seo Date: Thu, 4 Jan 2018 18:23:27 -0200 Subject: [PATCH] cmd/compile/internal/ppc64, runtime internal/atomic, sync/atomic: implement faster atomics for ppc64x MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This change implements faster atomics for ppc64x based on the ISA 2.07B, Appendix B.2 recommendations, replacing SYNC/ISYNC by LWSYNC in some cases. Updates #21348 name old time/op new time/op delta Cond1-16 955ns 856ns -10.33% Cond2-16 2.38µs 2.03µs -14.59% Cond4-16 5.90µs 5.44µs -7.88% Cond8-16 12.1µs 11.1µs -8.42% Cond16-16 27.0µs 25.1µs -7.04% Cond32-16 59.1µs 55.5µs -6.14% LoadMostlyHits/*sync_test.DeepCopyMap-16 22.1ns 24.1ns +9.02% LoadMostlyHits/*sync_test.RWMutexMap-16 252ns 249ns -1.20% LoadMostlyHits/*sync.Map-16 16.2ns 16.3ns ~ LoadMostlyMisses/*sync_test.DeepCopyMap-16 22.3ns 22.6ns ~ LoadMostlyMisses/*sync_test.RWMutexMap-16 249ns 247ns -0.51% LoadMostlyMisses/*sync.Map-16 12.7ns 12.7ns ~ LoadOrStoreBalanced/*sync_test.RWMutexMap-16 1.27µs 1.17µs -7.54% LoadOrStoreBalanced/*sync.Map-16 1.12µs 1.10µs -2.35% LoadOrStoreUnique/*sync_test.RWMutexMap-16 1.75µs 1.68µs -3.84% LoadOrStoreUnique/*sync.Map-16 2.07µs 1.97µs -5.13% LoadOrStoreCollision/*sync_test.DeepCopyMap-16 15.8ns 15.9ns ~ LoadOrStoreCollision/*sync_test.RWMutexMap-16 496ns 424ns -14.48% LoadOrStoreCollision/*sync.Map-16 6.07ns 6.07ns ~ Range/*sync_test.DeepCopyMap-16 1.65µs 1.64µs ~ Range/*sync_test.RWMutexMap-16 278µs 288µs +3.75% Range/*sync.Map-16 2.00µs 2.01µs ~ AdversarialAlloc/*sync_test.DeepCopyMap-16 3.45µs 3.44µs ~ AdversarialAlloc/*sync_test.RWMutexMap-16 226ns 227ns ~ AdversarialAlloc/*sync.Map-16 1.09µs 1.07µs -2.36% AdversarialDelete/*sync_test.DeepCopyMap-16 553ns 550ns -0.57% AdversarialDelete/*sync_test.RWMutexMap-16 273ns 274ns ~ AdversarialDelete/*sync.Map-16 247ns 249ns ~ UncontendedSemaphore-16 79.0ns 65.5ns -17.11% ContendedSemaphore-16 112ns 97ns -13.77% MutexUncontended-16 3.34ns 2.51ns -24.69% Mutex-16 266ns 191ns -28.26% MutexSlack-16 226ns 159ns -29.55% MutexWork-16 377ns 338ns -10.14% MutexWorkSlack-16 335ns 308ns -8.20% MutexNoSpin-16 196ns 184ns -5.91% MutexSpin-16 710ns 666ns -6.21% Once-16 1.29ns 1.29ns ~ Pool-16 8.64ns 8.71ns ~ PoolOverflow-16 1.60µs 1.44µs -10.25% SemaUncontended-16 5.39ns 4.42ns -17.96% SemaSyntNonblock-16 539ns 483ns -10.42% SemaSyntBlock-16 413ns 354ns -14.20% SemaWorkNonblock-16 305ns 258ns -15.36% SemaWorkBlock-16 266ns 229ns -14.06% RWMutexUncontended-16 12.9ns 9.7ns -24.80% RWMutexWrite100-16 203ns 147ns -27.47% RWMutexWrite10-16 177ns 119ns -32.74% RWMutexWorkWrite100-16 435ns 403ns -7.39% RWMutexWorkWrite10-16 642ns 611ns -4.79% WaitGroupUncontended-16 4.67ns 3.70ns -20.92% WaitGroupAddDone-16 402ns 355ns -11.54% WaitGroupAddDoneWork-16 208ns 250ns +20.09% WaitGroupWait-16 1.21ns 1.21ns ~ WaitGroupWaitWork-16 5.91ns 5.87ns -0.81% WaitGroupActuallyWait-16 92.2ns 85.8ns -6.91% Updates #21348 Change-Id: Ibb9b271d11b308264103829e176c6d9fe8f867d3 Reviewed-on: https://go-review.googlesource.com/95175 Run-TryBot: Lynn Boger TryBot-Result: Gobot Gobot Reviewed-by: Lynn Boger --- src/cmd/compile/internal/ppc64/ssa.go | 51 ++++++++++++------------ src/runtime/internal/atomic/asm_ppc64x.s | 24 +++++------ src/sync/atomic/asm_ppc64x.s | 18 ++++----- 3 files changed, 43 insertions(+), 50 deletions(-) diff --git a/src/cmd/compile/internal/ppc64/ssa.go b/src/cmd/compile/internal/ppc64/ssa.go index 7a2e2c1878..8d843f0756 100644 --- a/src/cmd/compile/internal/ppc64/ssa.go +++ b/src/cmd/compile/internal/ppc64/ssa.go @@ -154,16 +154,17 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { case ssa.OpPPC64LoweredAtomicAnd8, ssa.OpPPC64LoweredAtomicOr8: - // SYNC + // LWSYNC // LBAR (Rarg0), Rtmp // AND/OR Rarg1, Rtmp // STBCCC Rtmp, (Rarg0) // BNE -3(PC) - // ISYNC r0 := v.Args[0].Reg() r1 := v.Args[1].Reg() - psync := s.Prog(ppc64.ASYNC) - psync.To.Type = obj.TYPE_NONE + // LWSYNC - Assuming shared data not write-through-required nor + // caching-inhibited. See Appendix B.2.2.2 in the ISA 2.07b. + plwsync := s.Prog(ppc64.ALWSYNC) + plwsync.To.Type = obj.TYPE_NONE p := s.Prog(ppc64.ALBAR) p.From.Type = obj.TYPE_MEM p.From.Reg = r0 @@ -183,17 +184,14 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { p3 := s.Prog(ppc64.ABNE) p3.To.Type = obj.TYPE_BRANCH gc.Patch(p3, p) - pisync := s.Prog(ppc64.AISYNC) - pisync.To.Type = obj.TYPE_NONE case ssa.OpPPC64LoweredAtomicAdd32, ssa.OpPPC64LoweredAtomicAdd64: - // SYNC + // LWSYNC // LDAR/LWAR (Rarg0), Rout // ADD Rarg1, Rout // STDCCC/STWCCC Rout, (Rarg0) // BNE -3(PC) - // ISYNC // MOVW Rout,Rout (if Add32) ld := ppc64.ALDAR st := ppc64.ASTDCCC @@ -204,9 +202,10 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { r0 := v.Args[0].Reg() r1 := v.Args[1].Reg() out := v.Reg0() - // SYNC - psync := s.Prog(ppc64.ASYNC) - psync.To.Type = obj.TYPE_NONE + // LWSYNC - Assuming shared data not write-through-required nor + // caching-inhibited. See Appendix B.2.2.2 in the ISA 2.07b. + plwsync := s.Prog(ppc64.ALWSYNC) + plwsync.To.Type = obj.TYPE_NONE // LDAR or LWAR p := s.Prog(ld) p.From.Type = obj.TYPE_MEM @@ -229,9 +228,6 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { p4 := s.Prog(ppc64.ABNE) p4.To.Type = obj.TYPE_BRANCH gc.Patch(p4, p) - // ISYNC - pisync := s.Prog(ppc64.AISYNC) - pisync.To.Type = obj.TYPE_NONE // Ensure a 32 bit result if v.Op == ssa.OpPPC64LoweredAtomicAdd32 { @@ -244,7 +240,7 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { case ssa.OpPPC64LoweredAtomicExchange32, ssa.OpPPC64LoweredAtomicExchange64: - // SYNC + // LWSYNC // LDAR/LWAR (Rarg0), Rout // STDCCC/STWCCC Rout, (Rarg0) // BNE -2(PC) @@ -258,9 +254,10 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { r0 := v.Args[0].Reg() r1 := v.Args[1].Reg() out := v.Reg0() - // SYNC - psync := s.Prog(ppc64.ASYNC) - psync.To.Type = obj.TYPE_NONE + // LWSYNC - Assuming shared data not write-through-required nor + // caching-inhibited. See Appendix B.2.2.2 in the ISA 2.07b. + plwsync := s.Prog(ppc64.ALWSYNC) + plwsync.To.Type = obj.TYPE_NONE // LDAR or LWAR p := s.Prog(ld) p.From.Type = obj.TYPE_MEM @@ -342,14 +339,14 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { case ssa.OpPPC64LoweredAtomicCas64, ssa.OpPPC64LoweredAtomicCas32: - // SYNC + // LWSYNC // loop: // LDAR (Rarg0), Rtmp // CMP Rarg1, Rtmp // BNE fail // STDCCC Rarg2, (Rarg0) // BNE loop - // ISYNC + // LWSYNC // MOVD $1, Rout // BR end // fail: @@ -367,9 +364,10 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { r1 := v.Args[1].Reg() r2 := v.Args[2].Reg() out := v.Reg0() - // SYNC - psync := s.Prog(ppc64.ASYNC) - psync.To.Type = obj.TYPE_NONE + // LWSYNC - Assuming shared data not write-through-required nor + // caching-inhibited. See Appendix B.2.2.2 in the ISA 2.07b. + plwsync1 := s.Prog(ppc64.ALWSYNC) + plwsync1.To.Type = obj.TYPE_NONE // LDAR or LWAR p := s.Prog(ld) p.From.Type = obj.TYPE_MEM @@ -395,9 +393,10 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { p4 := s.Prog(ppc64.ABNE) p4.To.Type = obj.TYPE_BRANCH gc.Patch(p4, p) - // ISYNC - pisync := s.Prog(ppc64.AISYNC) - pisync.To.Type = obj.TYPE_NONE + // LWSYNC - Assuming shared data not write-through-required nor + // caching-inhibited. See Appendix B.2.1.1 in the ISA 2.07b. + plwsync2 := s.Prog(ppc64.ALWSYNC) + plwsync2.To.Type = obj.TYPE_NONE // return true p5 := s.Prog(ppc64.AMOVD) p5.From.Type = obj.TYPE_CONST diff --git a/src/runtime/internal/atomic/asm_ppc64x.s b/src/runtime/internal/atomic/asm_ppc64x.s index 7117aef158..a2ed4adc91 100644 --- a/src/runtime/internal/atomic/asm_ppc64x.s +++ b/src/runtime/internal/atomic/asm_ppc64x.s @@ -17,7 +17,7 @@ TEXT runtime∕internal∕atomic·Cas(SB), NOSPLIT, $0-17 MOVD ptr+0(FP), R3 MOVWZ old+8(FP), R4 MOVWZ new+12(FP), R5 - SYNC + LWSYNC cas_again: LWAR (R3), R6 CMPW R6, R4 @@ -25,7 +25,7 @@ cas_again: STWCCC R5, (R3) BNE cas_again MOVD $1, R3 - ISYNC + LWSYNC MOVB R3, ret+16(FP) RET cas_fail: @@ -44,7 +44,7 @@ TEXT runtime∕internal∕atomic·Cas64(SB), NOSPLIT, $0-25 MOVD ptr+0(FP), R3 MOVD old+8(FP), R4 MOVD new+16(FP), R5 - SYNC + LWSYNC cas64_again: LDAR (R3), R6 CMP R6, R4 @@ -52,7 +52,7 @@ cas64_again: STDCCC R5, (R3) BNE cas64_again MOVD $1, R3 - ISYNC + LWSYNC MOVB R3, ret+24(FP) RET cas64_fail: @@ -97,31 +97,29 @@ TEXT runtime∕internal∕atomic·Casp1(SB), NOSPLIT, $0-25 TEXT runtime∕internal∕atomic·Xadd(SB), NOSPLIT, $0-20 MOVD ptr+0(FP), R4 MOVW delta+8(FP), R5 - SYNC + LWSYNC LWAR (R4), R3 ADD R5, R3 STWCCC R3, (R4) BNE -3(PC) - ISYNC MOVW R3, ret+16(FP) RET TEXT runtime∕internal∕atomic·Xadd64(SB), NOSPLIT, $0-24 MOVD ptr+0(FP), R4 MOVD delta+8(FP), R5 - SYNC + LWSYNC LDAR (R4), R3 ADD R5, R3 STDCCC R3, (R4) BNE -3(PC) - ISYNC MOVD R3, ret+16(FP) RET TEXT runtime∕internal∕atomic·Xchg(SB), NOSPLIT, $0-20 MOVD ptr+0(FP), R4 MOVW new+8(FP), R5 - SYNC + LWSYNC LWAR (R4), R3 STWCCC R5, (R4) BNE -2(PC) @@ -132,7 +130,7 @@ TEXT runtime∕internal∕atomic·Xchg(SB), NOSPLIT, $0-20 TEXT runtime∕internal∕atomic·Xchg64(SB), NOSPLIT, $0-24 MOVD ptr+0(FP), R4 MOVD new+8(FP), R5 - SYNC + LWSYNC LDAR (R4), R3 STDCCC R5, (R4) BNE -2(PC) @@ -165,24 +163,22 @@ TEXT runtime∕internal∕atomic·Store64(SB), NOSPLIT, $0-16 TEXT runtime∕internal∕atomic·Or8(SB), NOSPLIT, $0-9 MOVD ptr+0(FP), R3 MOVBZ val+8(FP), R4 - SYNC + LWSYNC again: LBAR (R3), R6 OR R4, R6 STBCCC R6, (R3) BNE again - ISYNC RET // void runtime∕internal∕atomic·And8(byte volatile*, byte); TEXT runtime∕internal∕atomic·And8(SB), NOSPLIT, $0-9 MOVD ptr+0(FP), R3 MOVBZ val+8(FP), R4 - SYNC + LWSYNC again: LBAR (R3),R6 AND R4,R6 STBCCC R6,(R3) BNE again - ISYNC RET diff --git a/src/sync/atomic/asm_ppc64x.s b/src/sync/atomic/asm_ppc64x.s index 44e26698b4..dc93ed8e1d 100644 --- a/src/sync/atomic/asm_ppc64x.s +++ b/src/sync/atomic/asm_ppc64x.s @@ -12,7 +12,7 @@ TEXT ·SwapInt32(SB),NOSPLIT,$0-20 TEXT ·SwapUint32(SB),NOSPLIT,$0-20 MOVD addr+0(FP), R3 MOVW new+8(FP), R4 - SYNC + LWSYNC LWAR (R3), R5 STWCCC R4, (R3) BNE -2(PC) @@ -26,7 +26,7 @@ TEXT ·SwapInt64(SB),NOSPLIT,$0-24 TEXT ·SwapUint64(SB),NOSPLIT,$0-24 MOVD addr+0(FP), R3 MOVD new+8(FP), R4 - SYNC + LWSYNC LDAR (R3), R5 STDCCC R4, (R3) BNE -2(PC) @@ -44,13 +44,13 @@ TEXT ·CompareAndSwapUint32(SB),NOSPLIT,$0-17 MOVD addr+0(FP), R3 MOVW old+8(FP), R4 MOVW new+12(FP), R5 - SYNC + LWSYNC LWAR (R3), R6 CMPW R6, R4 BNE 7(PC) STWCCC R5, (R3) BNE -4(PC) - ISYNC + LWSYNC MOVD $1, R3 MOVB R3, swapped+16(FP) RET @@ -67,13 +67,13 @@ TEXT ·CompareAndSwapUint64(SB),NOSPLIT,$0-25 MOVD addr+0(FP), R3 MOVD old+8(FP), R4 MOVD new+16(FP), R5 - SYNC + LWSYNC LDAR (R3), R6 CMP R6, R4 BNE 7(PC) STDCCC R5, (R3) BNE -4(PC) - ISYNC + LWSYNC MOVD $1, R3 MOVB R3, swapped+24(FP) RET @@ -86,12 +86,11 @@ TEXT ·AddInt32(SB),NOSPLIT,$0-20 TEXT ·AddUint32(SB),NOSPLIT,$0-20 MOVD addr+0(FP), R3 MOVW delta+8(FP), R4 - SYNC + LWSYNC LWAR (R3), R5 ADD R4, R5 STWCCC R5, (R3) BNE -3(PC) - ISYNC MOVW R5, new+16(FP) RET @@ -104,12 +103,11 @@ TEXT ·AddInt64(SB),NOSPLIT,$0-24 TEXT ·AddUint64(SB),NOSPLIT,$0-24 MOVD addr+0(FP), R3 MOVD delta+8(FP), R4 - SYNC + LWSYNC LDAR (R3), R5 ADD R4, R5 STDCCC R5, (R3) BNE -3(PC) - ISYNC MOVD R5, new+16(FP) RET