mirror of
https://github.com/golang/go
synced 2024-11-23 16:50:06 -07:00
runtime/race: implement race detector for ppc64le
This adds the support to enable the race detector for ppc64le. Added runtime/race_ppc64le.s to manage the calls from Go to the LLVM tsan functions, mostly converting from the Go ABI to the PPC64 ABI expected by Clang generated code. Changed racewalk.go to call racefuncenterfp instead of racefuncenter on ppc64le to allow the caller pc to be obtained in the asm code before calling the tsan version. Changed the set up code for racecallbackthunk so it doesn't use the autogenerated save and restore of the link register since that sequence uses registers inconsistent with the normal ppc64 ABI. Made various changes to recognize that race is supported for ppc64le. Ensured that tls_g is updated and accessible from race_linux_ppc64le.s so that the race ctx can be obtained and passed to tsan functions. This enables the race tests for ppc64le in cmd/dist/test.go and increases the timeout when running the benchmarks with the -race option to avoid timing out. Updates #24354, #23731 Change-Id: Ib97dc7ac313e6313c836dc7d2fb698f9d8fba3ef Reviewed-on: https://go-review.googlesource.com/107935 Run-TryBot: Lynn Boger <laboger@linux.vnet.ibm.com> TryBot-Result: Gobot Gobot <gobot@golang.org> Reviewed-by: Bryan C. Mills <bcmills@google.com> Reviewed-by: Brad Fitzpatrick <bradfitz@golang.org>
This commit is contained in:
Lynn Boger
Brad Fitzpatrick
parent
1de0dcfc7b
commit
9e9ff565cd
@ -141,6 +141,7 @@ var runtimeDecls = [...]struct {
|
||||
{"uint32tofloat64", funcTag, 109},
|
||||
{"complex128div", funcTag, 110},
|
||||
{"racefuncenter", funcTag, 111},
|
||||
{"racefuncenterfp", funcTag, 5},
|
||||
{"racefuncexit", funcTag, 5},
|
||||
{"raceread", funcTag, 111},
|
||||
{"racewrite", funcTag, 111},
|
||||
|
||||
@ -183,6 +183,7 @@ func complex128div(num complex128, den complex128) (quo complex128)
|
||||
|
||||
// race detection
|
||||
func racefuncenter(uintptr)
|
||||
func racefuncenterfp()
|
||||
func racefuncexit()
|
||||
func raceread(uintptr)
|
||||
func racewrite(uintptr)
|
||||
|
||||
@ -7,6 +7,7 @@ package gc
|
||||
import (
|
||||
"cmd/compile/internal/types"
|
||||
"cmd/internal/src"
|
||||
"cmd/internal/sys"
|
||||
)
|
||||
|
||||
// The racewalk pass is currently handled in two parts.
|
||||
@ -58,17 +59,23 @@ func instrument(fn *Node) {
|
||||
lno := lineno
|
||||
lineno = src.NoXPos
|
||||
|
||||
// nodpc is the PC of the caller as extracted by
|
||||
// getcallerpc. We use -widthptr(FP) for x86.
|
||||
// BUG: this will not work on arm.
|
||||
nodpc := nodfp.copy()
|
||||
nodpc.Type = types.Types[TUINTPTR]
|
||||
nodpc.Xoffset = int64(-Widthptr)
|
||||
fn.Func.Dcl = append(fn.Func.Dcl, nodpc)
|
||||
|
||||
fn.Func.Enter.Prepend(mkcall("racefuncenter", nil, nil, nodpc))
|
||||
fn.Func.Exit.Append(mkcall("racefuncexit", nil, nil))
|
||||
if thearch.LinkArch.Arch == sys.ArchPPC64LE {
|
||||
fn.Func.Enter.Prepend(mkcall("racefuncenterfp", nil, nil))
|
||||
fn.Func.Exit.Append(mkcall("racefuncexit", nil, nil))
|
||||
} else {
|
||||
|
||||
// nodpc is the PC of the caller as extracted by
|
||||
// getcallerpc. We use -widthptr(FP) for x86.
|
||||
// BUG: This only works for amd64. This will not
|
||||
// work on arm or others that might support
|
||||
// race in the future.
|
||||
nodpc := nodfp.copy()
|
||||
nodpc.Type = types.Types[TUINTPTR]
|
||||
nodpc.Xoffset = int64(-Widthptr)
|
||||
fn.Func.Dcl = append(fn.Func.Dcl, nodpc)
|
||||
fn.Func.Enter.Prepend(mkcall("racefuncenter", nil, nil, nodpc))
|
||||
fn.Func.Exit.Append(mkcall("racefuncexit", nil, nil))
|
||||
}
|
||||
lineno = lno
|
||||
}
|
||||
}
|
||||
|
||||
6
src/cmd/dist/test.go
vendored
6
src/cmd/dist/test.go
vendored
@ -312,7 +312,6 @@ func (t *tester) registerStdTest(pkg string) {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
args := []string{
|
||||
"test",
|
||||
short(),
|
||||
@ -355,7 +354,8 @@ func (t *tester) registerRaceBenchTest(pkg string) {
|
||||
"test",
|
||||
short(),
|
||||
"-race",
|
||||
"-run=^$", // nothing. only benchmarks.
|
||||
t.timeout(1200), // longer timeout for race with benchmarks
|
||||
"-run=^$", // nothing. only benchmarks.
|
||||
"-benchtime=.1s",
|
||||
"-cpu=4",
|
||||
}
|
||||
@ -1318,7 +1318,7 @@ func (t *tester) raceDetectorSupported() bool {
|
||||
case "linux", "darwin", "freebsd", "windows":
|
||||
// The race detector doesn't work on Alpine Linux:
|
||||
// golang.org/issue/14481
|
||||
return t.cgoEnabled && goarch == "amd64" && gohostos == goos && !isAlpineLinux()
|
||||
return t.cgoEnabled && (goarch == "amd64" || goarch == "ppc64le") && gohostos == goos && !isAlpineLinux()
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
@ -43,11 +43,16 @@ func instrumentInit() {
|
||||
fmt.Fprintf(os.Stderr, "-msan is not supported on %s/%s\n", cfg.Goos, cfg.Goarch)
|
||||
os.Exit(2)
|
||||
}
|
||||
if cfg.BuildRace && (cfg.Goarch != "amd64" || cfg.Goos != "linux" && cfg.Goos != "freebsd" && cfg.Goos != "darwin" && cfg.Goos != "windows") {
|
||||
fmt.Fprintf(os.Stderr, "go %s: -race is only supported on linux/amd64, freebsd/amd64, darwin/amd64 and windows/amd64\n", flag.Args()[0])
|
||||
os.Exit(2)
|
||||
if cfg.BuildRace {
|
||||
platform := cfg.Goos + "/" + cfg.Goarch
|
||||
switch platform {
|
||||
default:
|
||||
fmt.Fprintf(os.Stderr, "go %s: -race is only supported on linux/amd64, linux/ppc64le, freebsd/amd64, darwin/amd64 and windows/amd64\n", flag.Args()[0])
|
||||
os.Exit(2)
|
||||
case "linux/amd64", "linux/ppc64le", "freebsd/amd64", "darwin/amd64", "windows/amd64":
|
||||
// race supported on these platforms
|
||||
}
|
||||
}
|
||||
|
||||
mode := "race"
|
||||
if cfg.BuildMSan {
|
||||
mode = "msan"
|
||||
|
||||
@ -502,7 +502,10 @@ func preprocess(ctxt *obj.Link, cursym *obj.LSym, newprog obj.ProgAlloc) {
|
||||
q = c.stacksplit(q, autosize) // emit split check
|
||||
}
|
||||
|
||||
if autosize != 0 {
|
||||
// Special handling of the racecall thunk. Assume that its asm code will
|
||||
// save the link register and update the stack, since that code is
|
||||
// called directly from C/C++ and can't clobber REGTMP (R31).
|
||||
if autosize != 0 && c.cursym.Name != "runtime.racecallbackthunk" {
|
||||
// Save the link register and update the SP. MOVDU is used unless
|
||||
// the frame size is too large. The link register must be saved
|
||||
// even for non-empty leaf functions so that traceback works.
|
||||
@ -678,7 +681,7 @@ func preprocess(ctxt *obj.Link, cursym *obj.LSym, newprog obj.ProgAlloc) {
|
||||
retTarget := p.To.Sym
|
||||
|
||||
if c.cursym.Func.Text.Mark&LEAF != 0 {
|
||||
if autosize == 0 {
|
||||
if autosize == 0 || c.cursym.Name == "runtime.racecallbackthunk" {
|
||||
p.As = ABR
|
||||
p.From = obj.Addr{}
|
||||
if retTarget == nil {
|
||||
@ -747,8 +750,8 @@ func preprocess(ctxt *obj.Link, cursym *obj.LSym, newprog obj.ProgAlloc) {
|
||||
p.Link = q
|
||||
p = q
|
||||
}
|
||||
|
||||
if autosize != 0 {
|
||||
prev := p
|
||||
if autosize != 0 && c.cursym.Name != "runtime.racecallbackthunk" {
|
||||
q = c.newprog()
|
||||
q.As = AADD
|
||||
q.Pos = p.Pos
|
||||
@ -759,7 +762,8 @@ func preprocess(ctxt *obj.Link, cursym *obj.LSym, newprog obj.ProgAlloc) {
|
||||
q.Spadj = -autosize
|
||||
|
||||
q.Link = p.Link
|
||||
p.Link = q
|
||||
prev.Link = q
|
||||
prev = q
|
||||
}
|
||||
|
||||
q1 = c.newprog()
|
||||
@ -776,7 +780,7 @@ func preprocess(ctxt *obj.Link, cursym *obj.LSym, newprog obj.ProgAlloc) {
|
||||
q1.Spadj = +autosize
|
||||
|
||||
q1.Link = q.Link
|
||||
q.Link = q1
|
||||
prev.Link = q1
|
||||
case AADD:
|
||||
if p.To.Type == obj.TYPE_REG && p.To.Reg == REGSP && p.From.Type == obj.TYPE_CONST {
|
||||
p.Spadj = int32(-p.From.Offset)
|
||||
|
||||
@ -196,6 +196,13 @@ func mustLinkExternal(ctxt *Link) (res bool, reason string) {
|
||||
return true, objabi.GOARCH + " does not support internal cgo"
|
||||
}
|
||||
|
||||
// When the race flag is set, the LLVM tsan relocatable file is linked
|
||||
// into the final binary, which means external linking is required because
|
||||
// internal linking does not support it.
|
||||
if *flagRace && ctxt.Arch.InFamily(sys.PPC64) {
|
||||
return true, "race on ppc64le"
|
||||
}
|
||||
|
||||
// Some build modes require work the internal linker cannot do (yet).
|
||||
switch ctxt.BuildMode {
|
||||
case BuildModeCArchive:
|
||||
|
||||
@ -9,7 +9,7 @@
|
||||
set -e
|
||||
|
||||
function usage {
|
||||
echo 'race detector is only supported on linux/amd64, freebsd/amd64 and darwin/amd64' 1>&2
|
||||
echo 'race detector is only supported on linux/amd64, linux/ppc64le, freebsd/amd64 and darwin/amd64' 1>&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
@ -21,7 +21,7 @@ case $(uname) in
|
||||
fi
|
||||
;;
|
||||
"Linux")
|
||||
if [ $(uname -m) != "x86_64" ]; then
|
||||
if [ $(uname -m) != "x86_64" ] && [ $(uname -m) != "ppc64le" ]; then
|
||||
usage
|
||||
fi
|
||||
;;
|
||||
|
||||
@ -24,6 +24,7 @@ TEXT runtime·rt0_go(SB),NOSPLIT,$0
|
||||
// create istack out of the given (operating system) stack.
|
||||
// _cgo_init may update stackguard.
|
||||
MOVD $runtime·g0(SB), g
|
||||
BL runtime·save_g(SB)
|
||||
MOVD $(-64*1024), R31
|
||||
ADD R31, R1, R3
|
||||
MOVD R3, g_stackguard0(g)
|
||||
|
||||
@ -292,6 +292,7 @@ var racearenastart uintptr
|
||||
var racearenaend uintptr
|
||||
|
||||
func racefuncenter(uintptr)
|
||||
func racefuncenterfp()
|
||||
func racefuncexit()
|
||||
func racereadrangepc1(uintptr, uintptr, uintptr)
|
||||
func racewriterangepc1(uintptr, uintptr, uintptr)
|
||||
|
||||
@ -2,7 +2,7 @@
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
// +build race,linux,amd64 race,freebsd,amd64 race,darwin,amd64 race,windows,amd64
|
||||
// +build race,linux,amd64 race,freebsd,amd64 race,darwin,amd64 race,windows,amd64 race,linux,ppc64le
|
||||
|
||||
package race
|
||||
|
||||
|
||||
568
src/runtime/race_ppc64le.s
Normal file
568
src/runtime/race_ppc64le.s
Normal file
@ -0,0 +1,568 @@
|
||||
// Copyright 2018 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
// +build race
|
||||
|
||||
#include "go_asm.h"
|
||||
#include "go_tls.h"
|
||||
#include "funcdata.h"
|
||||
#include "textflag.h"
|
||||
|
||||
// The following functions allow calling the clang-compiled race runtime directly
|
||||
// from Go code without going all the way through cgo.
|
||||
// First, it's much faster (up to 50% speedup for real Go programs).
|
||||
// Second, it eliminates race-related special cases from cgocall and scheduler.
|
||||
// Third, in long-term it will allow to remove cyclic runtime/race dependency on cmd/go.
|
||||
|
||||
// A brief recap of the ppc64le calling convention.
|
||||
// Arguments are passed in R3, R4, R5 ...
|
||||
// SP must be 16-byte aligned.
|
||||
|
||||
// Note that for ppc64x, LLVM follows the standard ABI and
|
||||
// expects arguments in registers, so these functions move
|
||||
// the arguments from storage to the registers expected
|
||||
// by the ABI.
|
||||
|
||||
// When calling from Go to Clang tsan code:
|
||||
// R3 is the 1st argument and is usually the ThreadState*
|
||||
// R4-? are the 2nd, 3rd, 4th, etc. arguments
|
||||
|
||||
// When calling racecalladdr:
|
||||
// R8 is the call target address
|
||||
|
||||
// The race ctx is passed in R3 and loaded in
|
||||
// racecalladdr.
|
||||
//
|
||||
// The sequence used to get the race ctx:
|
||||
// MOVD runtime·tls_g(SB), R10 // offset to TLS
|
||||
// MOVD 0(R13)(R10*1), g // R13=TLS for this thread, g = R30
|
||||
// MOVD g_racectx(g), R3 // racectx == ThreadState
|
||||
|
||||
// func runtime·RaceRead(addr uintptr)
|
||||
// Called from instrumented Go code
|
||||
TEXT runtime·raceread(SB), NOSPLIT, $0-8
|
||||
MOVD addr+0(FP), R4
|
||||
MOVD LR, R5 // caller of this?
|
||||
// void __tsan_read(ThreadState *thr, void *addr, void *pc);
|
||||
MOVD $__tsan_read(SB), R8
|
||||
BR racecalladdr<>(SB)
|
||||
|
||||
TEXT runtime·RaceRead(SB), NOSPLIT, $0-8
|
||||
BR runtime·raceread(SB)
|
||||
|
||||
// void runtime·racereadpc(void *addr, void *callpc, void *pc)
|
||||
TEXT runtime·racereadpc(SB), NOSPLIT, $0-24
|
||||
MOVD addr+0(FP), R4
|
||||
MOVD callpc+8(FP), R5
|
||||
MOVD pc+16(FP), R6
|
||||
// void __tsan_read_pc(ThreadState *thr, void *addr, void *callpc, void *pc);
|
||||
MOVD $__tsan_read_pc(SB), R8
|
||||
BR racecalladdr<>(SB)
|
||||
|
||||
// func runtime·RaceWrite(addr uintptr)
|
||||
// Called from instrumented Go code
|
||||
TEXT runtime·racewrite(SB), NOSPLIT, $0-8
|
||||
MOVD addr+0(FP), R4
|
||||
MOVD LR, R5 // caller has set LR via BL inst
|
||||
// void __tsan_write(ThreadState *thr, void *addr, void *pc);
|
||||
MOVD $__tsan_write(SB), R8
|
||||
BR racecalladdr<>(SB)
|
||||
|
||||
TEXT runtime·RaceWrite(SB), NOSPLIT, $0-8
|
||||
JMP runtime·racewrite(SB)
|
||||
|
||||
// void runtime·racewritepc(void *addr, void *callpc, void *pc)
|
||||
TEXT runtime·racewritepc(SB), NOSPLIT, $0-24
|
||||
MOVD addr+0(FP), R4
|
||||
MOVD callpc+8(FP), R5
|
||||
MOVD pc+16(FP), R6
|
||||
// void __tsan_write_pc(ThreadState *thr, void *addr, void *callpc, void *pc);
|
||||
MOVD $__tsan_write_pc(SB), R8
|
||||
BR racecalladdr<>(SB)
|
||||
|
||||
// func runtime·RaceReadRange(addr, size uintptr)
|
||||
// Called from instrumented Go code.
|
||||
TEXT runtime·racereadrange(SB), NOSPLIT, $0-16
|
||||
MOVD addr+0(FP), R4
|
||||
MOVD size+8(FP), R5
|
||||
MOVD LR, R6
|
||||
// void __tsan_read_range(ThreadState *thr, void *addr, uintptr size, void *pc);
|
||||
MOVD $__tsan_read_range(SB), R8
|
||||
BR racecalladdr<>(SB)
|
||||
|
||||
// void runtime·racereadrangepc1(void *addr, uintptr sz, void *pc)
|
||||
TEXT runtime·racereadrangepc1(SB), NOSPLIT, $0-24
|
||||
MOVD addr+0(FP), R4
|
||||
MOVD size+8(FP), R5
|
||||
MOVD pc+16(FP), R6
|
||||
ADD $4, R6 // tsan wants return addr
|
||||
// void __tsan_read_range(ThreadState *thr, void *addr, uintptr size, void *pc);
|
||||
MOVD $__tsan_read_range(SB), R8
|
||||
BR racecalladdr<>(SB)
|
||||
|
||||
TEXT runtime·RaceReadRange(SB), NOSPLIT, $0-24
|
||||
BR runtime·racereadrange(SB)
|
||||
|
||||
// func runtime·RaceWriteRange(addr, size uintptr)
|
||||
// Called from instrumented Go code.
|
||||
TEXT runtime·racewriterange(SB), NOSPLIT, $0-16
|
||||
MOVD addr+0(FP), R4
|
||||
MOVD size+8(FP), R5
|
||||
MOVD LR, R6
|
||||
// void __tsan_write_range(ThreadState *thr, void *addr, uintptr size, void *pc);
|
||||
MOVD $__tsan_write_range(SB), R8
|
||||
BR racecalladdr<>(SB)
|
||||
|
||||
TEXT runtime·RaceWriteRange(SB), NOSPLIT, $0-16
|
||||
BR runtime·racewriterange(SB)
|
||||
|
||||
// void runtime·racewriterangepc1(void *addr, uintptr sz, void *pc)
|
||||
// Called from instrumented Go code
|
||||
TEXT runtime·racewriterangepc1(SB), NOSPLIT, $0-24
|
||||
MOVD addr+0(FP), R4
|
||||
MOVD size+8(FP), R5
|
||||
MOVD pc+16(FP), R6
|
||||
ADD $4, R6 // add 4 to inst offset?
|
||||
// void __tsan_write_range(ThreadState *thr, void *addr, uintptr size, void *pc);
|
||||
MOVD $__tsan_write_range(SB), R8
|
||||
BR racecalladdr<>(SB)
|
||||
|
||||
// Call a __tsan function from Go code.
|
||||
// R8 = tsan function address
|
||||
// R3 = *ThreadState a.k.a. g_racectx from g
|
||||
// R4 = addr passed to __tsan function
|
||||
//
|
||||
// Otherwise, setup goroutine context and invoke racecall. Other arguments already set.
|
||||
TEXT racecalladdr<>(SB), NOSPLIT, $0-0
|
||||
MOVD runtime·tls_g(SB), R10
|
||||
MOVD 0(R13)(R10*1), g
|
||||
MOVD g_racectx(g), R3 // goroutine context
|
||||
// Check that addr is within [arenastart, arenaend) or within [racedatastart, racedataend).
|
||||
MOVD runtime·racearenastart(SB), R9
|
||||
CMP R4, R9
|
||||
BLT data
|
||||
MOVD runtime·racearenaend(SB), R9
|
||||
CMP R4, R9
|
||||
BLT call
|
||||
data:
|
||||
MOVD runtime·racedatastart(SB), R9
|
||||
CMP R4, R9
|
||||
BLT ret
|
||||
MOVD runtime·racedataend(SB), R9
|
||||
CMP R4, R9
|
||||
BGT ret
|
||||
call:
|
||||
// Careful!! racecall will save LR on its
|
||||
// stack, which is OK as long as racecalladdr
|
||||
// doesn't change in a way that generates a stack.
|
||||
// racecall should return to the caller of
|
||||
// recalladdr.
|
||||
BR racecall<>(SB)
|
||||
ret:
|
||||
RET
|
||||
|
||||
// func runtime·racefuncenterfp()
|
||||
// Called from instrumented Go code.
|
||||
// Like racefuncenter but doesn't pass an arg, uses the caller pc
|
||||
// from the first slot on the stack.
|
||||
TEXT runtime·racefuncenterfp(SB), NOSPLIT, $0-0
|
||||
MOVD 0(R1), R8
|
||||
BR racefuncenter<>(SB)
|
||||
|
||||
// func runtime·racefuncenter(pc uintptr)
|
||||
// Called from instrumented Go code.
|
||||
// Not used now since gc/racewalk.go doesn't pass the
|
||||
// correct caller pc and racefuncenterfp can do it.
|
||||
TEXT runtime·racefuncenter(SB), NOSPLIT, $0-8
|
||||
MOVD callpc+0(FP), R8
|
||||
BR racefuncenter<>(SB)
|
||||
|
||||
// Common code for racefuncenter/racefuncenterfp
|
||||
// R11 = caller's return address
|
||||
TEXT racefuncenter<>(SB), NOSPLIT, $0-0
|
||||
MOVD runtime·tls_g(SB), R10
|
||||
MOVD 0(R13)(R10*1), g
|
||||
MOVD g_racectx(g), R3 // goroutine racectx aka *ThreadState
|
||||
MOVD R8, R4 // caller pc set by caller in R8
|
||||
// void __tsan_func_enter(ThreadState *thr, void *pc);
|
||||
MOVD $__tsan_func_enter(SB), R8
|
||||
BR racecall<>(SB)
|
||||
RET
|
||||
|
||||
// func runtime·racefuncexit()
|
||||
// Called from Go instrumented code.
|
||||
TEXT runtime·racefuncexit(SB), NOSPLIT, $0-0
|
||||
MOVD runtime·tls_g(SB), R10
|
||||
MOVD 0(R13)(R10*1), g
|
||||
MOVD g_racectx(g), R3 // goroutine racectx aka *ThreadState
|
||||
// void __tsan_func_exit(ThreadState *thr);
|
||||
MOVD $__tsan_func_exit(SB), R8
|
||||
BR racecall<>(SB)
|
||||
|
||||
// Atomic operations for sync/atomic package.
|
||||
// Some use the __tsan versions instead
|
||||
// R6 = addr of arguments passed to this function
|
||||
// R3, R4, R5 set in racecallatomic
|
||||
|
||||
// Load atomic in tsan
|
||||
TEXT sync∕atomic·LoadInt32(SB), NOSPLIT, $0-0
|
||||
// void __tsan_go_atomic32_load(ThreadState *thr, uptr cpc, uptr pc, u8 *a);
|
||||
MOVD $__tsan_go_atomic32_load(SB), R8
|
||||
ADD $32, R1, R6 // addr of caller's 1st arg
|
||||
BR racecallatomic<>(SB)
|
||||
RET
|
||||
|
||||
TEXT sync∕atomic·LoadInt64(SB), NOSPLIT, $0-0
|
||||
// void __tsan_go_atomic64_load(ThreadState *thr, uptr cpc, uptr pc, u8 *a);
|
||||
MOVD $__tsan_go_atomic64_load(SB), R8
|
||||
ADD $32, R1, R6 // addr of caller's 1st arg
|
||||
BR racecallatomic<>(SB)
|
||||
RET
|
||||
|
||||
TEXT sync∕atomic·LoadUint32(SB), NOSPLIT, $0-0
|
||||
BR sync∕atomic·LoadInt32(SB)
|
||||
|
||||
TEXT sync∕atomic·LoadUint64(SB), NOSPLIT, $0-0
|
||||
BR sync∕atomic·LoadInt64(SB)
|
||||
|
||||
TEXT sync∕atomic·LoadUintptr(SB), NOSPLIT, $0-0
|
||||
BR sync∕atomic·LoadInt64(SB)
|
||||
|
||||
TEXT sync∕atomic·LoadPointer(SB), NOSPLIT, $0-0
|
||||
BR sync∕atomic·LoadInt64(SB)
|
||||
|
||||
// Store atomic in tsan
|
||||
TEXT sync∕atomic·StoreInt32(SB), NOSPLIT, $0-0
|
||||
// void __tsan_go_atomic32_store(ThreadState *thr, uptr cpc, uptr pc, u8 *a);
|
||||
MOVD $__tsan_go_atomic32_store(SB), R8
|
||||
ADD $32, R1, R6 // addr of caller's 1st arg
|
||||
BR racecallatomic<>(SB)
|
||||
|
||||
TEXT sync∕atomic·StoreInt64(SB), NOSPLIT, $0-0
|
||||
// void __tsan_go_atomic64_store(ThreadState *thr, uptr cpc, uptr pc, u8 *a);
|
||||
MOVD $__tsan_go_atomic64_store(SB), R8
|
||||
ADD $32, R1, R6 // addr of caller's 1st arg
|
||||
BR racecallatomic<>(SB)
|
||||
|
||||
TEXT sync∕atomic·StoreUint32(SB), NOSPLIT, $0-0
|
||||
BR sync∕atomic·StoreInt32(SB)
|
||||
|
||||
TEXT sync∕atomic·StoreUint64(SB), NOSPLIT, $0-0
|
||||
BR sync∕atomic·StoreInt64(SB)
|
||||
|
||||
TEXT sync∕atomic·StoreUintptr(SB), NOSPLIT, $0-0
|
||||
BR sync∕atomic·StoreInt64(SB)
|
||||
|
||||
// Swap in tsan
|
||||
TEXT sync∕atomic·SwapInt32(SB), NOSPLIT, $0-0
|
||||
// void __tsan_go_atomic32_exchange(ThreadState *thr, uptr cpc, uptr pc, u8 *a);
|
||||
MOVD $__tsan_go_atomic32_exchange(SB), R8
|
||||
ADD $32, R1, R6 // addr of caller's 1st arg
|
||||
BR racecallatomic<>(SB)
|
||||
|
||||
TEXT sync∕atomic·SwapInt64(SB), NOSPLIT, $0-0
|
||||
// void __tsan_go_atomic64_exchange(ThreadState *thr, uptr cpc, uptr pc, u8 *a)
|
||||
MOVD $__tsan_go_atomic64_exchange(SB), R8
|
||||
ADD $32, R1, R6 // addr of caller's 1st arg
|
||||
BR racecallatomic<>(SB)
|
||||
|
||||
TEXT sync∕atomic·SwapUint32(SB), NOSPLIT, $0-0
|
||||
BR sync∕atomic·SwapInt32(SB)
|
||||
|
||||
TEXT sync∕atomic·SwapUint64(SB), NOSPLIT, $0-0
|
||||
BR sync∕atomic·SwapInt64(SB)
|
||||
|
||||
TEXT sync∕atomic·SwapUintptr(SB), NOSPLIT, $0-0
|
||||
BR sync∕atomic·SwapInt64(SB)
|
||||
|
||||
// Add atomic in tsan
|
||||
TEXT sync∕atomic·AddInt32(SB), NOSPLIT, $0-0
|
||||
// void __tsan_go_atomic32_fetch_add(ThreadState *thr, uptr cpc, uptr pc, u8 *a);
|
||||
MOVD $__tsan_go_atomic32_fetch_add(SB), R8
|
||||
ADD $64, R1, R6 // addr of caller's 1st arg
|
||||
BL racecallatomic<>(SB)
|
||||
// The tsan fetch_add result is not as expected by Go,
|
||||
// so the 'add' must be added to the result.
|
||||
MOVW add+8(FP), R3 // The tsa fetch_add does not return the
|
||||
MOVW ret+16(FP), R4 // result as expected by go, so fix it.
|
||||
ADD R3, R4, R3
|
||||
MOVW R3, ret+16(FP)
|
||||
RET
|
||||
|
||||
TEXT sync∕atomic·AddInt64(SB), NOSPLIT, $0-0
|
||||
// void __tsan_go_atomic64_fetch_add(ThreadState *thr, uptr cpc, uptr pc, u8 *a);
|
||||
MOVD $__tsan_go_atomic64_fetch_add(SB), R8
|
||||
ADD $64, R1, R6 // addr of caller's 1st arg
|
||||
BL racecallatomic<>(SB)
|
||||
// The tsan fetch_add result is not as expected by Go,
|
||||
// so the 'add' must be added to the result.
|
||||
MOVD add+8(FP), R3
|
||||
MOVD ret+16(FP), R4
|
||||
ADD R3, R4, R3
|
||||
MOVD R3, ret+16(FP)
|
||||
RET
|
||||
|
||||
TEXT sync∕atomic·AddUint32(SB), NOSPLIT, $0-0
|
||||
BR sync∕atomic·AddInt32(SB)
|
||||
|
||||
TEXT sync∕atomic·AddUint64(SB), NOSPLIT, $0-0
|
||||
BR sync∕atomic·AddInt64(SB)
|
||||
|
||||
TEXT sync∕atomic·AddUintptr(SB), NOSPLIT, $0-0
|
||||
BR sync∕atomic·AddInt64(SB)
|
||||
|
||||
// CompareAndSwap in tsan
|
||||
TEXT sync∕atomic·CompareAndSwapInt32(SB), NOSPLIT, $0-0
|
||||
// void __tsan_go_atomic32_compare_exchange(
|
||||
// ThreadState *thr, uptr cpc, uptr pc, u8 *a)
|
||||
MOVD $__tsan_go_atomic32_compare_exchange(SB), R8
|
||||
ADD $32, R1, R6 // addr of caller's 1st arg
|
||||
BR racecallatomic<>(SB)
|
||||
|
||||
TEXT sync∕atomic·CompareAndSwapInt64(SB), NOSPLIT, $0-0
|
||||
// void __tsan_go_atomic32_compare_exchange(
|
||||
// ThreadState *thr, uptr cpc, uptr pc, u8 *a)
|
||||
MOVD $__tsan_go_atomic64_compare_exchange(SB), R8
|
||||
ADD $32, R1, R6 // addr of caller's 1st arg
|
||||
BR racecallatomic<>(SB)
|
||||
|
||||
TEXT sync∕atomic·CompareAndSwapUint32(SB), NOSPLIT, $0-0
|
||||
BR sync∕atomic·CompareAndSwapInt32(SB)
|
||||
|
||||
TEXT sync∕atomic·CompareAndSwapUint64(SB), NOSPLIT, $0-0
|
||||
BR sync∕atomic·CompareAndSwapInt64(SB)
|
||||
|
||||
TEXT sync∕atomic·CompareAndSwapUintptr(SB), NOSPLIT, $0-0
|
||||
BR sync∕atomic·CompareAndSwapInt64(SB)
|
||||
|
||||
// Common function used to call tsan's atomic functions
|
||||
// R3 = *ThreadState
|
||||
// R4 = TODO: What's this supposed to be?
|
||||
// R5 = caller pc
|
||||
// R6 = addr of incoming arg list
|
||||
// R8 contains addr of target function.
|
||||
TEXT racecallatomic<>(SB), NOSPLIT, $0-0
|
||||
// Trigger SIGSEGV early if address passed to atomic function is bad.
|
||||
MOVD (R6), R7 // 1st arg is addr
|
||||
MOVD (R7), R9 // segv here if addr is bad
|
||||
// Check that addr is within [arenastart, arenaend) or within [racedatastart, racedataend).
|
||||
MOVD runtime·racearenastart(SB), R9
|
||||
CMP R7, R9
|
||||
BLT racecallatomic_data
|
||||
MOVD runtime·racearenaend(SB), R9
|
||||
CMP R7, R9
|
||||
BLT racecallatomic_ok
|
||||
racecallatomic_data:
|
||||
MOVD runtime·racedatastart(SB), R9
|
||||
CMP R7, R9
|
||||
BLT racecallatomic_ignore
|
||||
MOVD runtime·racedataend(SB), R9
|
||||
CMP R7, R9
|
||||
BGE racecallatomic_ignore
|
||||
racecallatomic_ok:
|
||||
// Addr is within the good range, call the atomic function.
|
||||
MOVD runtime·tls_g(SB), R10
|
||||
MOVD 0(R13)(R10*1), g
|
||||
MOVD g_racectx(g), R3 // goroutine racectx aka *ThreadState
|
||||
MOVD R8, R5 // pc is the function called
|
||||
MOVD (R1), R4 // caller pc from stack
|
||||
BL racecall<>(SB) // BL needed to maintain stack consistency
|
||||
RET //
|
||||
racecallatomic_ignore:
|
||||
// Addr is outside the good range.
|
||||
// Call __tsan_go_ignore_sync_begin to ignore synchronization during the atomic op.
|
||||
// An attempt to synchronize on the address would cause crash.
|
||||
MOVD R8, R15 // save the original function
|
||||
MOVD R6, R17 // save the original arg list addr
|
||||
MOVD $__tsan_go_ignore_sync_begin(SB), R8 // func addr to call
|
||||
MOVD runtime·tls_g(SB), R10
|
||||
MOVD 0(R13)(R10*1), g
|
||||
MOVD g_racectx(g), R3 // goroutine context
|
||||
BL racecall<>(SB)
|
||||
MOVD R15, R8 // restore the original function
|
||||
MOVD R17, R6 // restore arg list addr
|
||||
// Call the atomic function.
|
||||
// racecall will call LLVM race code which might clobber r30 (g)
|
||||
MOVD runtime·tls_g(SB), R10
|
||||
MOVD 0(R13)(R10*1), g
|
||||
|
||||
MOVD g_racectx(g), R3
|
||||
MOVD R8, R4 // pc being called same TODO as above
|
||||
MOVD (R1), R5 // caller pc from latest LR
|
||||
BL racecall<>(SB)
|
||||
// Call __tsan_go_ignore_sync_end.
|
||||
MOVD $__tsan_go_ignore_sync_end(SB), R8
|
||||
MOVD g_racectx(g), R3 // goroutine context g should sitll be good?
|
||||
BL racecall<>(SB)
|
||||
RET
|
||||
|
||||
// void runtime·racecall(void(*f)(...), ...)
|
||||
// Calls C function f from race runtime and passes up to 4 arguments to it.
|
||||
// The arguments are never heap-object-preserving pointers, so we pretend there are no arguments.
|
||||
TEXT runtime·racecall(SB), NOSPLIT, $0-0
|
||||
MOVD fn+0(FP), R8
|
||||
MOVD arg0+8(FP), R3
|
||||
MOVD arg1+16(FP), R4
|
||||
MOVD arg2+24(FP), R5
|
||||
MOVD arg3+32(FP), R6
|
||||
JMP racecall<>(SB)
|
||||
|
||||
// Finds g0 and sets its stack
|
||||
// Arguments were loaded for call from Go to C
|
||||
TEXT racecall<>(SB), NOSPLIT, $0-0
|
||||
// Set the LR slot for the ppc64 ABI
|
||||
MOVD LR, R10
|
||||
MOVD R10, 0(R1) // Go expectation
|
||||
MOVD R10, 16(R1) // C ABI
|
||||
// Get info from the current goroutine
|
||||
MOVD runtime·tls_g(SB), R10 // g offset in TLS
|
||||
MOVD 0(R13)(R10*1), g // R13 = current TLS
|
||||
MOVD g_m(g), R7 // m for g
|
||||
MOVD R1, R16 // callee-saved, preserved across C call
|
||||
MOVD m_g0(R7), R10 // g0 for m
|
||||
CMP R10, g // same g0?
|
||||
BEQ call // already on g0
|
||||
MOVD (g_sched+gobuf_sp)(R10), R1 // switch R1
|
||||
call:
|
||||
MOVD R8, CTR // R8 = caller addr
|
||||
MOVD R8, R12 // expected by PPC64 ABI
|
||||
BL (CTR)
|
||||
XOR R0, R0 // clear R0 on return from Clang
|
||||
MOVD R16, R1 // restore R1; R16 nonvol in Clang
|
||||
MOVD runtime·tls_g(SB), R10 // find correct g
|
||||
MOVD 0(R13)(R10*1), g
|
||||
MOVD 16(R1), R10 // LR was saved away, restore for return
|
||||
MOVD R10, LR
|
||||
RET
|
||||
|
||||
// C->Go callback thunk that allows to call runtime·racesymbolize from C code.
|
||||
// Direct Go->C race call has only switched SP, finish g->g0 switch by setting correct g.
|
||||
// The overall effect of Go->C->Go call chain is similar to that of mcall.
|
||||
// RARG0 contains command code. RARG1 contains command-specific context.
|
||||
// See racecallback for command codes.
|
||||
TEXT runtime·racecallbackthunk(SB), NOSPLIT, $-8
|
||||
// Handle command raceGetProcCmd (0) here.
|
||||
// First, code below assumes that we are on curg, while raceGetProcCmd
|
||||
// can be executed on g0. Second, it is called frequently, so will
|
||||
// benefit from this fast path.
|
||||
XOR R0, R0 // clear R0 since we came from C code
|
||||
CMP R3, $0
|
||||
BNE rest
|
||||
// g0 TODO: Don't modify g here since R30 is nonvolatile
|
||||
MOVD g, R9
|
||||
MOVD runtime·tls_g(SB), R10
|
||||
MOVD 0(R13)(R10*1), g
|
||||
MOVD g_m(g), R3
|
||||
MOVD m_p(R3), R3
|
||||
MOVD p_racectx(R3), R3
|
||||
MOVD R3, (R4)
|
||||
MOVD R9, g // restore R30 ??
|
||||
RET
|
||||
|
||||
// This is all similar to what cgo does
|
||||
// Save registers according to the ppc64 ABI
|
||||
rest:
|
||||
MOVD LR, R10 // save link register
|
||||
MOVD R10, 16(R1)
|
||||
MOVW CR, R10
|
||||
MOVW R10, 8(R1)
|
||||
MOVDU R1, -336(R1) // Allocate frame needed for register save area
|
||||
|
||||
MOVD R14, 40(R1)
|
||||
MOVD R15, 48(R1)
|
||||
MOVD R16, 56(R1)
|
||||
MOVD R17, 64(R1)
|
||||
MOVD R18, 72(R1)
|
||||
MOVD R19, 80(R1)
|
||||
MOVD R20, 88(R1)
|
||||
MOVD R21, 96(R1)
|
||||
MOVD R22, 104(R1)
|
||||
MOVD R23, 112(R1)
|
||||
MOVD R24, 120(R1)
|
||||
MOVD R25, 128(R1)
|
||||
MOVD R26, 136(R1)
|
||||
MOVD R27, 144(R1)
|
||||
MOVD R28, 152(R1)
|
||||
MOVD R29, 160(R1)
|
||||
MOVD g, 168(R1) // R30
|
||||
MOVD R31, 176(R1)
|
||||
FMOVD F14, 184(R1)
|
||||
FMOVD F15, 192(R1)
|
||||
FMOVD F16, 200(R1)
|
||||
FMOVD F17, 208(R1)
|
||||
FMOVD F18, 216(R1)
|
||||
FMOVD F19, 224(R1)
|
||||
FMOVD F20, 232(R1)
|
||||
FMOVD F21, 240(R1)
|
||||
FMOVD F22, 248(R1)
|
||||
FMOVD F23, 256(R1)
|
||||
FMOVD F24, 264(R1)
|
||||
FMOVD F25, 272(R1)
|
||||
FMOVD F26, 280(R1)
|
||||
FMOVD F27, 288(R1)
|
||||
FMOVD F28, 296(R1)
|
||||
FMOVD F29, 304(R1)
|
||||
FMOVD F30, 312(R1)
|
||||
FMOVD F31, 320(R1)
|
||||
|
||||
MOVD runtime·tls_g(SB), R10
|
||||
MOVD 0(R13)(R10*1), g
|
||||
|
||||
MOVD g_m(g), R7
|
||||
MOVD m_g0(R7), g // set g = m-> g0
|
||||
MOVD R3, cmd+0(FP) // can't use R1 here ?? use input args and assumer caller expects those?
|
||||
MOVD R4, ctx+8(FP) // can't use R1 here ??
|
||||
BL runtime·racecallback(SB)
|
||||
// All registers are clobbered after Go code, reload.
|
||||
MOVD runtime·tls_g(SB), R10
|
||||
MOVD 0(R13)(R10*1), g
|
||||
|
||||
MOVD g_m(g), R7
|
||||
MOVD m_curg(R7), g // restore g = m->curg
|
||||
MOVD 40(R1), R14
|
||||
MOVD 48(R1), R15
|
||||
MOVD 56(R1), R16
|
||||
MOVD 64(R1), R17
|
||||
MOVD 72(R1), R18
|
||||
MOVD 80(R1), R19
|
||||
MOVD 88(R1), R20
|
||||
MOVD 96(R1), R21
|
||||
MOVD 104(R1), R22
|
||||
MOVD 112(R1), R23
|
||||
MOVD 120(R1), R24
|
||||
MOVD 128(R1), R25
|
||||
MOVD 136(R1), R26
|
||||
MOVD 144(R1), R27
|
||||
MOVD 152(R1), R28
|
||||
MOVD 160(R1), R29
|
||||
MOVD 168(R1), g // R30
|
||||
MOVD 176(R1), R31
|
||||
FMOVD 184(R1), F14
|
||||
FMOVD 192(R1), F15
|
||||
FMOVD 200(R1), F16
|
||||
FMOVD 208(R1), F17
|
||||
FMOVD 216(R1), F18
|
||||
FMOVD 224(R1), F19
|
||||
FMOVD 232(R1), F20
|
||||
FMOVD 240(R1), F21
|
||||
FMOVD 248(R1), F22
|
||||
FMOVD 256(R1), F23
|
||||
FMOVD 264(R1), F24
|
||||
FMOVD 272(R1), F25
|
||||
FMOVD 280(R1), F26
|
||||
FMOVD 288(R1), F27
|
||||
FMOVD 296(R1), F28
|
||||
FMOVD 304(R1), F29
|
||||
FMOVD 312(R1), F30
|
||||
FMOVD 320(R1), F31
|
||||
|
||||
ADD $336, R1
|
||||
MOVD 8(R1), R10
|
||||
MOVFL R10, $0xff // Restore of CR
|
||||
MOVD 16(R1), R10 // needed?
|
||||
MOVD R10, LR
|
||||
RET
|
||||
|
||||
// tls_g, g value for each thread in TLS
|
||||
GLOBL runtime·tls_g+0(SB), TLSBSS+DUPOK, $8
|
||||
@ -46,4 +46,4 @@ TEXT runtime·load_g(SB),NOSPLIT|NOFRAME,$0-0
|
||||
MOVD 0(R13)(R31*1), g
|
||||
RET
|
||||
|
||||
GLOBL runtime·tls_g+0(SB), TLSBSS, $8
|
||||
GLOBL runtime·tls_g+0(SB), TLSBSS+DUPOK, $8
|
||||
|
||||
Loading…
Reference in New Issue
Block a user