From dbd8af74723d2c98cbdcc70f7e2801f69b57ac5b Mon Sep 17 00:00:00 2001 From: Carlos Eduardo Seo Date: Thu, 23 Aug 2018 20:16:19 -0300 Subject: [PATCH] runtime: add support for VDSO on ppc64x for use in walltime/nanotime This change adds support for VDSO on ppc64x, making it possible to avoid a syscall in walltime and nanotime. BenchmarkClockVDSOAndFallbackPaths/vDSO-192 20000000 66.0 ns/op BenchmarkClockVDSOAndFallbackPaths/Fallback-192 1000000 1456 ns/op Change-Id: I3373bd804b6f122961de3ae9d034e6ccf35748e6 Reviewed-on: https://go-review.googlesource.com/131135 Run-TryBot: Lynn Boger TryBot-Result: Gobot Gobot Reviewed-by: Lynn Boger --- src/runtime/os_linux_novdso.go | 2 +- src/runtime/sys_linux_ppc64x.s | 94 ++++++++++++++++++++++++++++---- src/runtime/vdso_elf64.go | 2 +- src/runtime/vdso_in_none.go | 2 +- src/runtime/vdso_linux.go | 7 ++- src/runtime/vdso_linux_ppc64x.go | 25 +++++++++ src/runtime/vdso_linux_test.go | 2 +- 7 files changed, 118 insertions(+), 16 deletions(-) create mode 100644 src/runtime/vdso_linux_ppc64x.go diff --git a/src/runtime/os_linux_novdso.go b/src/runtime/os_linux_novdso.go index ee4a7a95c2a..e54c1c4dc13 100644 --- a/src/runtime/os_linux_novdso.go +++ b/src/runtime/os_linux_novdso.go @@ -3,7 +3,7 @@ // license that can be found in the LICENSE file. // +build linux -// +build !386,!amd64,!arm,!arm64 +// +build !386,!amd64,!arm,!arm64,!ppc64,!ppc64le package runtime diff --git a/src/runtime/sys_linux_ppc64x.s b/src/runtime/sys_linux_ppc64x.s index 483cb8ef9ae..075adf23684 100644 --- a/src/runtime/sys_linux_ppc64x.s +++ b/src/runtime/sys_linux_ppc64x.s @@ -154,21 +154,87 @@ TEXT runtime·mincore(SB),NOSPLIT|NOFRAME,$0-28 // func walltime() (sec int64, nsec int32) TEXT runtime·walltime(SB),NOSPLIT,$16 - MOVD $0, R3 // CLOCK_REALTIME - MOVD $0(R1), R4 - SYSCALL $SYS_clock_gettime - MOVD 0(R1), R3 // sec - MOVD 8(R1), R5 // nsec + MOVD R1, R15 // R15 is unchanged by C code + MOVD g_m(g), R21 // R21 = m + + MOVD $0, R3 // CLOCK_REALTIME + + MOVD runtime·vdsoClockgettimeSym(SB), R12 // Check for VDSO availability + CMP R12, R0 + BEQ fallback + + // Set vdsoPC and vdsoSP for SIGPROF traceback. + MOVD LR, R14 + MOVD R14, m_vdsoPC(R21) + MOVD R15, m_vdsoSP(R21) + + MOVD m_curg(R21), R6 + CMP g, R6 + BNE noswitch + + MOVD m_g0(R21), R7 + MOVD (g_sched+gobuf_sp)(R7), R1 // Set SP to g0 stack + +noswitch: + SUB $16, R1 // Space for results + RLDICR $0, R1, $59, R1 // Align for C code + MOVD R12, CTR + MOVD R1, R4 + BL (CTR) // Call from VDSO + MOVD $0, R0 // Restore R0 + MOVD R0, m_vdsoSP(R21) // Clear vdsoSP + MOVD 0(R1), R3 // sec + MOVD 8(R1), R5 // nsec + MOVD R15, R1 // Restore SP + +finish: MOVD R3, sec+0(FP) MOVW R5, nsec+8(FP) RET + // Syscall fallback +fallback: + ADD $32, R1, R4 + SYSCALL $SYS_clock_gettime + MOVD 32(R1), R3 + MOVD 40(R1), R5 + JMP finish + TEXT runtime·nanotime(SB),NOSPLIT,$16 - MOVW $1, R3 // CLOCK_MONOTONIC - MOVD $0(R1), R4 - SYSCALL $SYS_clock_gettime - MOVD 0(R1), R3 // sec - MOVD 8(R1), R5 // nsec + MOVD $1, R3 // CLOCK_MONOTONIC + + MOVD R1, R15 // R15 is unchanged by C code + MOVD g_m(g), R21 // R21 = m + + MOVD runtime·vdsoClockgettimeSym(SB), R12 // Check for VDSO availability + CMP R12, R0 + BEQ fallback + + // Set vdsoPC and vdsoSP for SIGPROF traceback. + MOVD LR, R14 // R14 is unchanged by C code + MOVD R14, m_vdsoPC(R21) + MOVD R15, m_vdsoSP(R21) + + MOVD m_curg(R21), R6 + CMP g, R6 + BNE noswitch + + MOVD m_g0(R21), R7 + MOVD (g_sched+gobuf_sp)(R7), R1 // Set SP to g0 stack + +noswitch: + SUB $16, R1 // Space for results + RLDICR $0, R1, $59, R1 // Align for C code + MOVD R12, CTR + MOVD R1, R4 + BL (CTR) // Call from VDSO + MOVD $0, R0 // Restore R0 + MOVD $0, m_vdsoSP(R21) // Clear vdsoSP + MOVD 0(R1), R3 // sec + MOVD 8(R1), R5 // nsec + MOVD R15, R1 // Restore SP + +finish: // sec is in R3, nsec in R5 // return nsec in R3 MOVD $1000000000, R4 @@ -177,6 +243,14 @@ TEXT runtime·nanotime(SB),NOSPLIT,$16 MOVD R3, ret+0(FP) RET + // Syscall fallback +fallback: + ADD $32, R1, R4 + SYSCALL $SYS_clock_gettime + MOVD 32(R1), R3 + MOVD 48(R1), R5 + JMP finish + TEXT runtime·rtsigprocmask(SB),NOSPLIT|NOFRAME,$0-28 MOVW how+0(FP), R3 MOVD new+8(FP), R4 diff --git a/src/runtime/vdso_elf64.go b/src/runtime/vdso_elf64.go index 85102500658..7c9bd962779 100644 --- a/src/runtime/vdso_elf64.go +++ b/src/runtime/vdso_elf64.go @@ -3,7 +3,7 @@ // license that can be found in the LICENSE file. // +build linux -// +build amd64 arm64 +// +build amd64 arm64 ppc64 ppc64le package runtime diff --git a/src/runtime/vdso_in_none.go b/src/runtime/vdso_in_none.go index 34cfac56d12..f2d6bb55d9c 100644 --- a/src/runtime/vdso_in_none.go +++ b/src/runtime/vdso_in_none.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build linux,!386,!amd64,!arm,!arm64 !linux +// +build linux,!386,!amd64,!arm,!arm64,!ppc64,!ppc64le !linux package runtime diff --git a/src/runtime/vdso_linux.go b/src/runtime/vdso_linux.go index f6a285efd73..9827874beab 100644 --- a/src/runtime/vdso_linux.go +++ b/src/runtime/vdso_linux.go @@ -3,7 +3,7 @@ // license that can be found in the LICENSE file. // +build linux -// +build 386 amd64 arm arm64 +// +build 386 amd64 arm arm64 ppc64 ppc64le package runtime @@ -42,6 +42,8 @@ const ( _STT_FUNC = 2 /* Symbol is a code object */ + _STT_NOTYPE = 0 /* Symbol type is not specified */ + _STB_GLOBAL = 1 /* Global symbol */ _STB_WEAK = 2 /* Weak symbol */ @@ -212,7 +214,8 @@ func vdsoParseSymbols(info *vdsoInfo, version int32) { sym := &info.symtab[symIndex] typ := _ELF_ST_TYPE(sym.st_info) bind := _ELF_ST_BIND(sym.st_info) - if typ != _STT_FUNC || bind != _STB_GLOBAL && bind != _STB_WEAK || sym.st_shndx == _SHN_UNDEF { + // On ppc64x, VDSO functions are of type _STT_NOTYPE. + if typ != _STT_FUNC && typ != _STT_NOTYPE || bind != _STB_GLOBAL && bind != _STB_WEAK || sym.st_shndx == _SHN_UNDEF { return false } if k.name != gostringnocopy(&info.symstrings[sym.st_name]) { diff --git a/src/runtime/vdso_linux_ppc64x.go b/src/runtime/vdso_linux_ppc64x.go new file mode 100644 index 00000000000..f30946e4c5b --- /dev/null +++ b/src/runtime/vdso_linux_ppc64x.go @@ -0,0 +1,25 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build linux +// +build ppc64 ppc64le + +package runtime + +const ( + // vdsoArrayMax is the byte-size of a maximally sized array on this architecture. + // See cmd/compile/internal/ppc64/galign.go arch.MAXWIDTH initialization. + vdsoArrayMax = 1<<50 - 1 +) + +var vdsoLinuxVersion = vdsoVersionKey{"LINUX_2.6.15", 0x75fcba5} + +var vdsoSymbolKeys = []vdsoSymbolKey{ + {"__kernel_clock_gettime", 0xb0cd725, 0xdfa941fd, &vdsoClockgettimeSym}, +} + +// initialize with vsyscall fallbacks +var ( + vdsoClockgettimeSym uintptr = 0 +) diff --git a/src/runtime/vdso_linux_test.go b/src/runtime/vdso_linux_test.go index b5221f90b71..ad083c61b44 100644 --- a/src/runtime/vdso_linux_test.go +++ b/src/runtime/vdso_linux_test.go @@ -3,7 +3,7 @@ // license that can be found in the LICENSE file. // +build linux -// +build 386 amd64 arm arm64 +// +build 386 amd64 arm arm64 ppc64 ppc64le package runtime_test