2011-06-28 13:09:53 -06:00
|
|
|
// Copyright 2009 The Go Authors. All rights reserved.
|
|
|
|
// Use of this source code is governed by a BSD-style
|
|
|
|
// license that can be found in the LICENSE file.
|
|
|
|
|
|
|
|
#include "runtime.h"
|
2012-04-05 08:47:43 -06:00
|
|
|
#include "arch_GOARCH.h"
|
|
|
|
|
|
|
|
static union {
|
|
|
|
Lock l;
|
|
|
|
byte pad [CacheLineSize];
|
|
|
|
} locktab[57];
|
|
|
|
|
|
|
|
#define LOCK(addr) (&locktab[((uintptr)(addr)>>3)%nelem(locktab)].l)
|
2011-06-28 13:09:53 -06:00
|
|
|
|
2011-07-15 09:27:16 -06:00
|
|
|
// Atomic add and return new value.
|
|
|
|
#pragma textflag 7
|
|
|
|
uint32
|
|
|
|
runtime·xadd(uint32 volatile *val, int32 delta)
|
|
|
|
{
|
|
|
|
uint32 oval, nval;
|
|
|
|
|
|
|
|
for(;;){
|
|
|
|
oval = *val;
|
|
|
|
nval = oval + delta;
|
|
|
|
if(runtime·cas(val, oval, nval))
|
|
|
|
return nval;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
runtime: improve Linux mutex
The implementation is hybrid active/passive spin/blocking mutex.
The design minimizes amount of context switches and futex calls.
The idea is that all critical sections in runtime are intentially
small, so pure blocking mutex behaves badly causing
a lot of context switches, thread parking/unparking and kernel calls.
Note that some synthetic benchmarks become somewhat slower,
that's due to increased contention on other data structures,
it should not affect programs that do any real work.
On 2 x Intel E5620, 8 HT cores, 2.4GHz
benchmark old ns/op new ns/op delta
BenchmarkSelectContended 521.00 503.00 -3.45%
BenchmarkSelectContended-2 661.00 320.00 -51.59%
BenchmarkSelectContended-4 1139.00 629.00 -44.78%
BenchmarkSelectContended-8 2870.00 878.00 -69.41%
BenchmarkSelectContended-16 5276.00 818.00 -84.50%
BenchmarkChanContended 112.00 103.00 -8.04%
BenchmarkChanContended-2 631.00 174.00 -72.42%
BenchmarkChanContended-4 682.00 272.00 -60.12%
BenchmarkChanContended-8 1601.00 520.00 -67.52%
BenchmarkChanContended-16 3100.00 372.00 -88.00%
BenchmarkChanSync 253.00 239.00 -5.53%
BenchmarkChanSync-2 5030.00 4648.00 -7.59%
BenchmarkChanSync-4 4826.00 4694.00 -2.74%
BenchmarkChanSync-8 4778.00 4713.00 -1.36%
BenchmarkChanSync-16 5289.00 4710.00 -10.95%
BenchmarkChanProdCons0 273.00 254.00 -6.96%
BenchmarkChanProdCons0-2 599.00 400.00 -33.22%
BenchmarkChanProdCons0-4 1168.00 659.00 -43.58%
BenchmarkChanProdCons0-8 2831.00 1057.00 -62.66%
BenchmarkChanProdCons0-16 4197.00 1037.00 -75.29%
BenchmarkChanProdCons10 150.00 140.00 -6.67%
BenchmarkChanProdCons10-2 607.00 268.00 -55.85%
BenchmarkChanProdCons10-4 1137.00 404.00 -64.47%
BenchmarkChanProdCons10-8 2115.00 828.00 -60.85%
BenchmarkChanProdCons10-16 4283.00 855.00 -80.04%
BenchmarkChanProdCons100 117.00 110.00 -5.98%
BenchmarkChanProdCons100-2 558.00 218.00 -60.93%
BenchmarkChanProdCons100-4 722.00 287.00 -60.25%
BenchmarkChanProdCons100-8 1840.00 431.00 -76.58%
BenchmarkChanProdCons100-16 3394.00 448.00 -86.80%
BenchmarkChanProdConsWork0 2014.00 1996.00 -0.89%
BenchmarkChanProdConsWork0-2 1207.00 1127.00 -6.63%
BenchmarkChanProdConsWork0-4 1913.00 611.00 -68.06%
BenchmarkChanProdConsWork0-8 3016.00 949.00 -68.53%
BenchmarkChanProdConsWork0-16 4320.00 1154.00 -73.29%
BenchmarkChanProdConsWork10 1906.00 1897.00 -0.47%
BenchmarkChanProdConsWork10-2 1123.00 1033.00 -8.01%
BenchmarkChanProdConsWork10-4 1076.00 571.00 -46.93%
BenchmarkChanProdConsWork10-8 2748.00 1096.00 -60.12%
BenchmarkChanProdConsWork10-16 4600.00 1105.00 -75.98%
BenchmarkChanProdConsWork100 1884.00 1852.00 -1.70%
BenchmarkChanProdConsWork100-2 1235.00 1146.00 -7.21%
BenchmarkChanProdConsWork100-4 1217.00 619.00 -49.14%
BenchmarkChanProdConsWork100-8 1534.00 509.00 -66.82%
BenchmarkChanProdConsWork100-16 4126.00 918.00 -77.75%
BenchmarkSyscall 34.40 33.30 -3.20%
BenchmarkSyscall-2 160.00 121.00 -24.38%
BenchmarkSyscall-4 131.00 136.00 +3.82%
BenchmarkSyscall-8 139.00 131.00 -5.76%
BenchmarkSyscall-16 161.00 168.00 +4.35%
BenchmarkSyscallWork 950.00 950.00 +0.00%
BenchmarkSyscallWork-2 481.00 480.00 -0.21%
BenchmarkSyscallWork-4 268.00 270.00 +0.75%
BenchmarkSyscallWork-8 156.00 169.00 +8.33%
BenchmarkSyscallWork-16 188.00 184.00 -2.13%
BenchmarkSemaSyntNonblock 36.40 35.60 -2.20%
BenchmarkSemaSyntNonblock-2 81.40 45.10 -44.59%
BenchmarkSemaSyntNonblock-4 126.00 108.00 -14.29%
BenchmarkSemaSyntNonblock-8 112.00 112.00 +0.00%
BenchmarkSemaSyntNonblock-16 110.00 112.00 +1.82%
BenchmarkSemaSyntBlock 35.30 35.30 +0.00%
BenchmarkSemaSyntBlock-2 118.00 124.00 +5.08%
BenchmarkSemaSyntBlock-4 105.00 108.00 +2.86%
BenchmarkSemaSyntBlock-8 101.00 111.00 +9.90%
BenchmarkSemaSyntBlock-16 112.00 118.00 +5.36%
BenchmarkSemaWorkNonblock 810.00 811.00 +0.12%
BenchmarkSemaWorkNonblock-2 476.00 414.00 -13.03%
BenchmarkSemaWorkNonblock-4 238.00 228.00 -4.20%
BenchmarkSemaWorkNonblock-8 140.00 126.00 -10.00%
BenchmarkSemaWorkNonblock-16 117.00 116.00 -0.85%
BenchmarkSemaWorkBlock 810.00 811.00 +0.12%
BenchmarkSemaWorkBlock-2 454.00 466.00 +2.64%
BenchmarkSemaWorkBlock-4 243.00 241.00 -0.82%
BenchmarkSemaWorkBlock-8 145.00 137.00 -5.52%
BenchmarkSemaWorkBlock-16 132.00 123.00 -6.82%
BenchmarkContendedSemaphore 123.00 102.00 -17.07%
BenchmarkContendedSemaphore-2 34.80 34.90 +0.29%
BenchmarkContendedSemaphore-4 34.70 34.80 +0.29%
BenchmarkContendedSemaphore-8 34.70 34.70 +0.00%
BenchmarkContendedSemaphore-16 34.80 34.70 -0.29%
BenchmarkMutex 26.80 26.00 -2.99%
BenchmarkMutex-2 108.00 45.20 -58.15%
BenchmarkMutex-4 103.00 127.00 +23.30%
BenchmarkMutex-8 109.00 147.00 +34.86%
BenchmarkMutex-16 102.00 152.00 +49.02%
BenchmarkMutexSlack 27.00 26.90 -0.37%
BenchmarkMutexSlack-2 149.00 165.00 +10.74%
BenchmarkMutexSlack-4 121.00 209.00 +72.73%
BenchmarkMutexSlack-8 101.00 158.00 +56.44%
BenchmarkMutexSlack-16 97.00 129.00 +32.99%
BenchmarkMutexWork 792.00 794.00 +0.25%
BenchmarkMutexWork-2 407.00 409.00 +0.49%
BenchmarkMutexWork-4 220.00 209.00 -5.00%
BenchmarkMutexWork-8 267.00 160.00 -40.07%
BenchmarkMutexWork-16 315.00 300.00 -4.76%
BenchmarkMutexWorkSlack 792.00 793.00 +0.13%
BenchmarkMutexWorkSlack-2 406.00 404.00 -0.49%
BenchmarkMutexWorkSlack-4 225.00 212.00 -5.78%
BenchmarkMutexWorkSlack-8 268.00 136.00 -49.25%
BenchmarkMutexWorkSlack-16 300.00 300.00 +0.00%
BenchmarkRWMutexWrite100 27.10 27.00 -0.37%
BenchmarkRWMutexWrite100-2 33.10 40.80 +23.26%
BenchmarkRWMutexWrite100-4 113.00 88.10 -22.04%
BenchmarkRWMutexWrite100-8 119.00 95.30 -19.92%
BenchmarkRWMutexWrite100-16 148.00 109.00 -26.35%
BenchmarkRWMutexWrite10 29.60 29.40 -0.68%
BenchmarkRWMutexWrite10-2 111.00 61.40 -44.68%
BenchmarkRWMutexWrite10-4 270.00 208.00 -22.96%
BenchmarkRWMutexWrite10-8 204.00 185.00 -9.31%
BenchmarkRWMutexWrite10-16 261.00 190.00 -27.20%
BenchmarkRWMutexWorkWrite100 1040.00 1036.00 -0.38%
BenchmarkRWMutexWorkWrite100-2 593.00 580.00 -2.19%
BenchmarkRWMutexWorkWrite100-4 470.00 365.00 -22.34%
BenchmarkRWMutexWorkWrite100-8 468.00 289.00 -38.25%
BenchmarkRWMutexWorkWrite100-16 604.00 374.00 -38.08%
BenchmarkRWMutexWorkWrite10 951.00 951.00 +0.00%
BenchmarkRWMutexWorkWrite10-2 1001.00 928.00 -7.29%
BenchmarkRWMutexWorkWrite10-4 1555.00 1006.00 -35.31%
BenchmarkRWMutexWorkWrite10-8 2085.00 1171.00 -43.84%
BenchmarkRWMutexWorkWrite10-16 2082.00 1614.00 -22.48%
R=rsc, iant, msolo, fw, iant
CC=golang-dev
https://golang.org/cl/4711045
2011-07-29 10:44:06 -06:00
|
|
|
#pragma textflag 7
|
|
|
|
uint32
|
|
|
|
runtime·xchg(uint32 volatile* addr, uint32 v)
|
|
|
|
{
|
|
|
|
uint32 old;
|
|
|
|
|
|
|
|
for(;;) {
|
|
|
|
old = *addr;
|
|
|
|
if(runtime·cas(addr, old, v))
|
|
|
|
return old;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
#pragma textflag 7
|
|
|
|
void
|
|
|
|
runtime·procyield(uint32 cnt)
|
|
|
|
{
|
|
|
|
uint32 volatile i;
|
|
|
|
|
|
|
|
for(i = 0; i < cnt; i++) {
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2011-06-28 13:09:53 -06:00
|
|
|
#pragma textflag 7
|
|
|
|
uint32
|
|
|
|
runtime·atomicload(uint32 volatile* addr)
|
|
|
|
{
|
|
|
|
return runtime·xadd(addr, 0);
|
|
|
|
}
|
2011-07-13 12:22:41 -06:00
|
|
|
|
|
|
|
#pragma textflag 7
|
|
|
|
void*
|
|
|
|
runtime·atomicloadp(void* volatile* addr)
|
|
|
|
{
|
|
|
|
return (void*)runtime·xadd((uint32 volatile*)addr, 0);
|
|
|
|
}
|
|
|
|
|
|
|
|
#pragma textflag 7
|
|
|
|
void
|
|
|
|
runtime·atomicstorep(void* volatile* addr, void* v)
|
|
|
|
{
|
|
|
|
void *old;
|
|
|
|
|
|
|
|
for(;;) {
|
|
|
|
old = *addr;
|
|
|
|
if(runtime·casp(addr, old, v))
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
}
|
2011-07-29 11:47:24 -06:00
|
|
|
|
|
|
|
#pragma textflag 7
|
|
|
|
void
|
|
|
|
runtime·atomicstore(uint32 volatile* addr, uint32 v)
|
|
|
|
{
|
|
|
|
uint32 old;
|
|
|
|
|
|
|
|
for(;;) {
|
|
|
|
old = *addr;
|
|
|
|
if(runtime·cas(addr, old, v))
|
|
|
|
return;
|
|
|
|
}
|
2012-04-05 08:47:43 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
#pragma textflag 7
|
|
|
|
bool
|
|
|
|
runtime·cas64(uint64 volatile *addr, uint64 *old, uint64 new)
|
|
|
|
{
|
|
|
|
bool res;
|
|
|
|
|
|
|
|
runtime·lock(LOCK(addr));
|
|
|
|
if(*addr == *old) {
|
|
|
|
*addr = new;
|
|
|
|
res = true;
|
|
|
|
} else {
|
|
|
|
*old = *addr;
|
|
|
|
res = false;
|
|
|
|
}
|
|
|
|
runtime·unlock(LOCK(addr));
|
|
|
|
return res;
|
|
|
|
}
|
|
|
|
|
|
|
|
#pragma textflag 7
|
|
|
|
uint64
|
|
|
|
runtime·xadd64(uint64 volatile *addr, int64 delta)
|
|
|
|
{
|
|
|
|
uint64 res;
|
|
|
|
|
|
|
|
runtime·lock(LOCK(addr));
|
|
|
|
res = *addr + delta;
|
|
|
|
*addr = res;
|
|
|
|
runtime·unlock(LOCK(addr));
|
|
|
|
return res;
|
|
|
|
}
|
|
|
|
|
|
|
|
#pragma textflag 7
|
|
|
|
uint64
|
|
|
|
runtime·atomicload64(uint64 volatile *addr)
|
|
|
|
{
|
|
|
|
uint64 res;
|
|
|
|
|
|
|
|
runtime·lock(LOCK(addr));
|
|
|
|
res = *addr;
|
|
|
|
runtime·unlock(LOCK(addr));
|
|
|
|
return res;
|
|
|
|
}
|
|
|
|
|
|
|
|
#pragma textflag 7
|
|
|
|
void
|
|
|
|
runtime·atomicstore64(uint64 volatile *addr, uint64 v)
|
|
|
|
{
|
|
|
|
runtime·lock(LOCK(addr));
|
|
|
|
*addr = v;
|
|
|
|
runtime·unlock(LOCK(addr));
|
|
|
|
}
|