2012-10-07 12:05:32 -06:00
|
|
|
// Copyright 2011 The Go Authors. All rights reserved.
|
|
|
|
// Use of this source code is governed by a BSD-style
|
|
|
|
// license that can be found in the LICENSE file.
|
|
|
|
|
|
|
|
// Implementation of the race detector API.
|
|
|
|
// +build race
|
|
|
|
|
|
|
|
#include "runtime.h"
|
|
|
|
#include "arch_GOARCH.h"
|
|
|
|
#include "malloc.h"
|
|
|
|
#include "race.h"
|
2014-01-04 09:43:17 -07:00
|
|
|
#include "type.h"
|
|
|
|
#include "typekind.h"
|
2014-03-06 12:48:30 -07:00
|
|
|
|
|
|
|
// Race runtime functions called via runtime·racecall.
|
|
|
|
void __tsan_init(void);
|
|
|
|
void __tsan_fini(void);
|
|
|
|
void __tsan_map_shadow(void);
|
|
|
|
void __tsan_finalizer_goroutine(void);
|
|
|
|
void __tsan_go_start(void);
|
|
|
|
void __tsan_go_end(void);
|
|
|
|
void __tsan_malloc(void);
|
|
|
|
void __tsan_acquire(void);
|
|
|
|
void __tsan_release(void);
|
|
|
|
void __tsan_release_merge(void);
|
|
|
|
|
|
|
|
// Mimic what cmd/cgo would do.
|
|
|
|
#pragma cgo_import_static __tsan_init
|
|
|
|
#pragma cgo_import_static __tsan_fini
|
|
|
|
#pragma cgo_import_static __tsan_map_shadow
|
|
|
|
#pragma cgo_import_static __tsan_finalizer_goroutine
|
|
|
|
#pragma cgo_import_static __tsan_go_start
|
|
|
|
#pragma cgo_import_static __tsan_go_end
|
|
|
|
#pragma cgo_import_static __tsan_malloc
|
|
|
|
#pragma cgo_import_static __tsan_acquire
|
|
|
|
#pragma cgo_import_static __tsan_release
|
|
|
|
#pragma cgo_import_static __tsan_release_merge
|
|
|
|
|
|
|
|
// These are called from race_amd64.s.
|
|
|
|
#pragma cgo_import_static __tsan_read
|
|
|
|
#pragma cgo_import_static __tsan_read_pc
|
|
|
|
#pragma cgo_import_static __tsan_read_range
|
|
|
|
#pragma cgo_import_static __tsan_write
|
|
|
|
#pragma cgo_import_static __tsan_write_pc
|
|
|
|
#pragma cgo_import_static __tsan_write_range
|
|
|
|
#pragma cgo_import_static __tsan_func_enter
|
|
|
|
#pragma cgo_import_static __tsan_func_exit
|
2012-10-07 12:05:32 -06:00
|
|
|
|
|
|
|
extern byte noptrdata[];
|
|
|
|
extern byte enoptrbss[];
|
2014-03-06 12:48:30 -07:00
|
|
|
|
|
|
|
// start/end of heap for race_amd64.s
|
|
|
|
uintptr runtime·racearenastart;
|
|
|
|
uintptr runtime·racearenaend;
|
2012-10-07 12:05:32 -06:00
|
|
|
|
2014-03-06 12:48:30 -07:00
|
|
|
void runtime·racefuncenter(void *callpc);
|
|
|
|
void runtime·racefuncexit(void);
|
|
|
|
void runtime·racereadrangepc1(void *addr, uintptr sz, void *pc);
|
|
|
|
void runtime·racewriterangepc1(void *addr, uintptr sz, void *pc);
|
|
|
|
void runtime·racesymbolizethunk(void*);
|
2012-10-07 12:05:32 -06:00
|
|
|
|
2014-03-06 12:48:30 -07:00
|
|
|
// racecall allows calling an arbitrary function f from C race runtime
|
|
|
|
// with up to 4 uintptr arguments.
|
|
|
|
void runtime·racecall(void(*f)(void), ...);
|
2013-08-19 13:06:46 -06:00
|
|
|
|
2013-02-06 00:40:54 -07:00
|
|
|
uintptr
|
2012-10-07 12:05:32 -06:00
|
|
|
runtime·raceinit(void)
|
|
|
|
{
|
2013-04-03 16:11:34 -06:00
|
|
|
uintptr racectx, start, size;
|
2012-11-16 09:06:11 -07:00
|
|
|
|
2014-03-06 12:48:30 -07:00
|
|
|
// cgo is required to initialize libc, which is used by race runtime
|
|
|
|
if(!runtime·iscgo)
|
|
|
|
runtime·throw("raceinit: race build must use cgo");
|
|
|
|
runtime·racecall(__tsan_init, &racectx, runtime·racesymbolizethunk);
|
2013-04-03 16:11:34 -06:00
|
|
|
// Round data segment to page boundaries, because it's used in mmap().
|
|
|
|
start = (uintptr)noptrdata & ~(PageSize-1);
|
|
|
|
size = ROUND((uintptr)enoptrbss - start, PageSize);
|
2014-03-06 12:48:30 -07:00
|
|
|
runtime·racecall(__tsan_map_shadow, start, size);
|
2013-02-06 00:40:54 -07:00
|
|
|
return racectx;
|
2012-10-07 12:05:32 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
void
|
|
|
|
runtime·racefini(void)
|
|
|
|
{
|
2014-03-06 12:48:30 -07:00
|
|
|
runtime·racecall(__tsan_fini);
|
2012-10-07 12:05:32 -06:00
|
|
|
}
|
|
|
|
|
2012-11-07 01:48:58 -07:00
|
|
|
void
|
|
|
|
runtime·racemapshadow(void *addr, uintptr size)
|
|
|
|
{
|
2014-03-06 12:48:30 -07:00
|
|
|
if(runtime·racearenastart == 0)
|
|
|
|
runtime·racearenastart = (uintptr)addr;
|
|
|
|
if(runtime·racearenaend < (uintptr)addr+size)
|
|
|
|
runtime·racearenaend = (uintptr)addr+size;
|
|
|
|
runtime·racecall(__tsan_map_shadow, addr, size);
|
2012-10-07 12:05:32 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
void
|
2013-08-12 11:48:19 -06:00
|
|
|
runtime·racemalloc(void *p, uintptr sz)
|
2012-10-07 12:05:32 -06:00
|
|
|
{
|
2014-03-06 12:48:30 -07:00
|
|
|
runtime·racecall(__tsan_malloc, p, sz);
|
2012-10-07 12:05:32 -06:00
|
|
|
}
|
|
|
|
|
2013-02-06 00:40:54 -07:00
|
|
|
uintptr
|
|
|
|
runtime·racegostart(void *pc)
|
2012-10-07 12:05:32 -06:00
|
|
|
{
|
2013-02-06 00:40:54 -07:00
|
|
|
uintptr racectx;
|
|
|
|
|
2014-03-06 12:48:30 -07:00
|
|
|
runtime·racecall(__tsan_go_start, g->racectx, &racectx, pc);
|
2013-02-06 00:40:54 -07:00
|
|
|
return racectx;
|
2012-10-07 12:05:32 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
void
|
2013-02-06 00:40:54 -07:00
|
|
|
runtime·racegoend(void)
|
2012-10-07 12:05:32 -06:00
|
|
|
{
|
2014-03-06 12:48:30 -07:00
|
|
|
runtime·racecall(__tsan_go_end, g->racectx);
|
2013-01-29 17:55:02 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
void
|
2013-06-13 06:38:44 -06:00
|
|
|
runtime·racewriterangepc(void *addr, uintptr sz, void *callpc, void *pc)
|
2013-01-29 17:55:02 -07:00
|
|
|
{
|
2014-03-06 12:48:30 -07:00
|
|
|
if(callpc != nil)
|
|
|
|
runtime·racefuncenter(callpc);
|
|
|
|
runtime·racewriterangepc1(addr, sz, pc);
|
|
|
|
if(callpc != nil)
|
|
|
|
runtime·racefuncexit();
|
2013-01-29 17:55:02 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
void
|
2013-06-13 06:38:44 -06:00
|
|
|
runtime·racereadrangepc(void *addr, uintptr sz, void *callpc, void *pc)
|
2013-01-29 17:55:02 -07:00
|
|
|
{
|
2014-03-06 12:48:30 -07:00
|
|
|
if(callpc != nil)
|
|
|
|
runtime·racefuncenter(callpc);
|
|
|
|
runtime·racereadrangepc1(addr, sz, pc);
|
|
|
|
if(callpc != nil)
|
|
|
|
runtime·racefuncexit();
|
2013-01-29 17:55:02 -07:00
|
|
|
}
|
|
|
|
|
2014-01-04 09:43:17 -07:00
|
|
|
void
|
|
|
|
runtime·racewriteobjectpc(void *addr, Type *t, void *callpc, void *pc)
|
|
|
|
{
|
|
|
|
uint8 kind;
|
|
|
|
|
|
|
|
kind = t->kind & ~KindNoPointers;
|
|
|
|
if(kind == KindArray || kind == KindStruct)
|
2014-03-06 12:48:30 -07:00
|
|
|
runtime·racewriterangepc(addr, t->size, callpc, pc);
|
2014-01-04 09:43:17 -07:00
|
|
|
else
|
2014-03-06 12:48:30 -07:00
|
|
|
runtime·racewritepc(addr, callpc, pc);
|
2014-01-04 09:43:17 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
void
|
|
|
|
runtime·racereadobjectpc(void *addr, Type *t, void *callpc, void *pc)
|
|
|
|
{
|
|
|
|
uint8 kind;
|
|
|
|
|
|
|
|
kind = t->kind & ~KindNoPointers;
|
|
|
|
if(kind == KindArray || kind == KindStruct)
|
2014-03-06 12:48:30 -07:00
|
|
|
runtime·racereadrangepc(addr, t->size, callpc, pc);
|
2014-01-04 09:43:17 -07:00
|
|
|
else
|
2014-03-06 12:48:30 -07:00
|
|
|
runtime·racereadpc(addr, callpc, pc);
|
2014-01-04 09:43:17 -07:00
|
|
|
}
|
|
|
|
|
2012-10-07 12:05:32 -06:00
|
|
|
void
|
|
|
|
runtime·raceacquire(void *addr)
|
|
|
|
{
|
|
|
|
runtime·raceacquireg(g, addr);
|
|
|
|
}
|
|
|
|
|
|
|
|
void
|
|
|
|
runtime·raceacquireg(G *gp, void *addr)
|
|
|
|
{
|
|
|
|
if(g->raceignore)
|
|
|
|
return;
|
2014-03-06 12:48:30 -07:00
|
|
|
runtime·racecall(__tsan_acquire, gp->racectx, addr);
|
2012-10-07 12:05:32 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
void
|
|
|
|
runtime·racerelease(void *addr)
|
|
|
|
{
|
|
|
|
runtime·racereleaseg(g, addr);
|
|
|
|
}
|
|
|
|
|
|
|
|
void
|
|
|
|
runtime·racereleaseg(G *gp, void *addr)
|
|
|
|
{
|
|
|
|
if(g->raceignore)
|
|
|
|
return;
|
2014-03-06 12:48:30 -07:00
|
|
|
runtime·racecall(__tsan_release, gp->racectx, addr);
|
2012-10-07 12:05:32 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
void
|
|
|
|
runtime·racereleasemerge(void *addr)
|
|
|
|
{
|
|
|
|
runtime·racereleasemergeg(g, addr);
|
|
|
|
}
|
|
|
|
|
|
|
|
void
|
|
|
|
runtime·racereleasemergeg(G *gp, void *addr)
|
|
|
|
{
|
|
|
|
if(g->raceignore)
|
|
|
|
return;
|
2014-03-06 12:48:30 -07:00
|
|
|
runtime·racecall(__tsan_release_merge, gp->racectx, addr);
|
2012-10-07 12:05:32 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
void
|
|
|
|
runtime·racefingo(void)
|
|
|
|
{
|
2014-03-06 12:48:30 -07:00
|
|
|
runtime·racecall(__tsan_finalizer_goroutine, g->racectx);
|
2012-10-07 12:05:32 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
// func RaceAcquire(addr unsafe.Pointer)
|
|
|
|
void
|
|
|
|
runtime·RaceAcquire(void *addr)
|
|
|
|
{
|
|
|
|
runtime·raceacquire(addr);
|
|
|
|
}
|
|
|
|
|
|
|
|
// func RaceRelease(addr unsafe.Pointer)
|
|
|
|
void
|
|
|
|
runtime·RaceRelease(void *addr)
|
|
|
|
{
|
|
|
|
runtime·racerelease(addr);
|
|
|
|
}
|
|
|
|
|
|
|
|
// func RaceReleaseMerge(addr unsafe.Pointer)
|
|
|
|
void
|
|
|
|
runtime·RaceReleaseMerge(void *addr)
|
|
|
|
{
|
|
|
|
runtime·racereleasemerge(addr);
|
|
|
|
}
|
|
|
|
|
|
|
|
// func RaceSemacquire(s *uint32)
|
2013-05-18 16:49:23 -06:00
|
|
|
void
|
|
|
|
runtime·RaceSemacquire(uint32 *s)
|
2012-10-07 12:05:32 -06:00
|
|
|
{
|
net: add special netFD mutex
The mutex, fdMutex, handles locking and lifetime of sysfd,
and serializes Read and Write methods.
This allows to strip 2 sync.Mutex.Lock calls,
2 sync.Mutex.Unlock calls, 1 defer and some amount
of misc overhead from every network operation.
On linux/amd64, Intel E5-2690:
benchmark old ns/op new ns/op delta
BenchmarkTCP4Persistent 9595 9454 -1.47%
BenchmarkTCP4Persistent-2 8978 8772 -2.29%
BenchmarkTCP4ConcurrentReadWrite 4900 4625 -5.61%
BenchmarkTCP4ConcurrentReadWrite-2 2603 2500 -3.96%
In general it strips 70-500 ns from every network operation depending
on processor model. On my relatively new E5-2690 it accounts to ~5%
of network op cost.
Fixes #6074.
R=golang-dev, bradfitz, alex.brainman, iant, mikioh.mikioh
CC=golang-dev
https://golang.org/cl/12418043
2013-08-09 11:43:00 -06:00
|
|
|
runtime·semacquire(s, false);
|
2012-10-07 12:05:32 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
// func RaceSemrelease(s *uint32)
|
2013-05-18 16:49:23 -06:00
|
|
|
void
|
|
|
|
runtime·RaceSemrelease(uint32 *s)
|
2012-10-07 12:05:32 -06:00
|
|
|
{
|
|
|
|
runtime·semrelease(s);
|
|
|
|
}
|
|
|
|
|
|
|
|
// func RaceDisable()
|
2013-05-18 16:49:23 -06:00
|
|
|
void
|
|
|
|
runtime·RaceDisable(void)
|
2012-10-07 12:05:32 -06:00
|
|
|
{
|
|
|
|
g->raceignore++;
|
|
|
|
}
|
|
|
|
|
|
|
|
// func RaceEnable()
|
2013-05-18 16:49:23 -06:00
|
|
|
void
|
|
|
|
runtime·RaceEnable(void)
|
2012-10-07 12:05:32 -06:00
|
|
|
{
|
|
|
|
g->raceignore--;
|
|
|
|
}
|
|
|
|
|
2014-03-06 12:48:30 -07:00
|
|
|
typedef struct SymbolizeContext SymbolizeContext;
|
|
|
|
struct SymbolizeContext
|
2012-10-07 12:05:32 -06:00
|
|
|
{
|
2014-03-06 12:48:30 -07:00
|
|
|
uintptr pc;
|
|
|
|
int8* func;
|
|
|
|
int8* file;
|
|
|
|
uintptr line;
|
|
|
|
uintptr off;
|
|
|
|
uintptr res;
|
|
|
|
};
|
|
|
|
|
|
|
|
// Callback from C into Go, runs on g0.
|
|
|
|
void
|
|
|
|
runtime·racesymbolize(SymbolizeContext *ctx)
|
|
|
|
{
|
|
|
|
Func *f;
|
|
|
|
String file;
|
|
|
|
|
|
|
|
f = runtime·findfunc(ctx->pc);
|
|
|
|
if(f == nil) {
|
|
|
|
ctx->func = "??";
|
|
|
|
ctx->file = "-";
|
|
|
|
ctx->line = 0;
|
|
|
|
ctx->off = ctx->pc;
|
|
|
|
ctx->res = 1;
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
ctx->func = runtime·funcname(f);
|
|
|
|
ctx->line = runtime·funcline(f, ctx->pc, &file);
|
|
|
|
ctx->file = (int8*)file.str; // assume zero-terminated
|
|
|
|
ctx->off = ctx->pc - f->entry;
|
|
|
|
ctx->res = 1;
|
2012-10-07 12:05:32 -06:00
|
|
|
}
|