2016-03-01 15:57:46 -07:00
|
|
|
// Copyright 2010 The Go Authors. All rights reserved.
|
2010-10-25 18:55:50 -06:00
|
|
|
// Use of this source code is governed by a BSD-style
|
|
|
|
// license that can be found in the LICENSE file.
|
|
|
|
|
|
|
|
// Export guts for testing.
|
|
|
|
|
|
|
|
package runtime
|
|
|
|
|
2015-11-02 12:09:24 -07:00
|
|
|
import (
|
|
|
|
"runtime/internal/atomic"
|
2015-11-11 10:39:30 -07:00
|
|
|
"runtime/internal/sys"
|
2015-11-02 12:09:24 -07:00
|
|
|
"unsafe"
|
|
|
|
)
|
2014-08-21 11:10:45 -06:00
|
|
|
|
2010-10-25 18:55:50 -06:00
|
|
|
var Fadd64 = fadd64
|
|
|
|
var Fsub64 = fsub64
|
|
|
|
var Fmul64 = fmul64
|
|
|
|
var Fdiv64 = fdiv64
|
|
|
|
var F64to32 = f64to32
|
|
|
|
var F32to64 = f32to64
|
|
|
|
var Fcmp64 = fcmp64
|
|
|
|
var Fintto64 = fintto64
|
|
|
|
var F64toint = f64toint
|
2015-04-30 17:03:31 -06:00
|
|
|
var Sqrt = sqrt
|
2011-07-19 09:01:17 -06:00
|
|
|
|
|
|
|
var Entersyscall = entersyscall
|
|
|
|
var Exitsyscall = exitsyscall
|
2014-08-27 21:32:49 -06:00
|
|
|
var LockedOSThread = lockedOSThread
|
2015-11-02 12:09:24 -07:00
|
|
|
var Xadduintptr = atomic.Xadduintptr
|
2012-04-12 01:49:25 -06:00
|
|
|
|
2015-04-17 18:27:07 -06:00
|
|
|
var FuncPC = funcPC
|
|
|
|
|
2015-09-14 15:03:45 -06:00
|
|
|
var Fastlog2 = fastlog2
|
|
|
|
|
2012-04-12 01:49:25 -06:00
|
|
|
type LFNode struct {
|
2014-10-27 13:57:07 -06:00
|
|
|
Next uint64
|
2012-04-12 01:49:25 -06:00
|
|
|
Pushcnt uintptr
|
|
|
|
}
|
|
|
|
|
2014-08-21 11:10:45 -06:00
|
|
|
func LFStackPush(head *uint64, node *LFNode) {
|
2014-11-11 15:07:54 -07:00
|
|
|
lfstackpush(head, (*lfnode)(unsafe.Pointer(node)))
|
2014-08-21 11:10:45 -06:00
|
|
|
}
|
2012-04-12 01:49:25 -06:00
|
|
|
|
2014-08-21 11:10:45 -06:00
|
|
|
func LFStackPop(head *uint64) *LFNode {
|
2014-11-11 15:07:54 -07:00
|
|
|
return (*LFNode)(unsafe.Pointer(lfstackpop(head)))
|
2014-08-21 11:10:45 -06:00
|
|
|
}
|
2012-05-11 00:50:03 -06:00
|
|
|
|
2014-09-05 15:59:31 -06:00
|
|
|
func GCMask(x interface{}) (ret []byte) {
|
[dev.cc] runtime: delete scalararg, ptrarg; rename onM to systemstack
Scalararg and ptrarg are not "signal safe".
Go code filling them out can be interrupted by a signal,
and then the signal handler runs, and if it also ends up
in Go code that uses scalararg or ptrarg, now the old
values have been smashed.
For the pieces of code that do need to run in a signal handler,
we introduced onM_signalok, which is really just onM
except that the _signalok is meant to convey that the caller
asserts that scalarg and ptrarg will be restored to their old
values after the call (instead of the usual behavior, zeroing them).
Scalararg and ptrarg are also untyped and therefore error-prone.
Go code can always pass a closure instead of using scalararg
and ptrarg; they were only really necessary for C code.
And there's no more C code.
For all these reasons, delete scalararg and ptrarg, converting
the few remaining references to use closures.
Once those are gone, there is no need for a distinction between
onM and onM_signalok, so replace both with a single function
equivalent to the current onM_signalok (that is, it can be called
on any of the curg, g0, and gsignal stacks).
The name onM and the phrase 'm stack' are misnomers,
because on most system an M has two system stacks:
the main thread stack and the signal handling stack.
Correct the misnomer by naming the replacement function systemstack.
Fix a few references to "M stack" in code.
The main motivation for this change is to eliminate scalararg/ptrarg.
Rick and I have already seen them cause problems because
the calling sequence m.ptrarg[0] = p is a heap pointer assignment,
so it gets a write barrier. The write barrier also uses onM, so it has
all the same problems as if it were being invoked by a signal handler.
We worked around this by saving and restoring the old values
and by calling onM_signalok, but there's no point in keeping this nice
home for bugs around any longer.
This CL also changes funcline to return the file name as a result
instead of filling in a passed-in *string. (The *string signature is
left over from when the code was written in and called from C.)
That's arguably an unrelated change, except that once I had done
the ptrarg/scalararg/onM cleanup I started getting false positives
about the *string argument escaping (not allowed in package runtime).
The compiler is wrong, but the easiest fix is to write the code like
Go code instead of like C code. I am a bit worried that the compiler
is wrong because of some use of uninitialized memory in the escape
analysis. If that's the reason, it will go away when we convert the
compiler to Go. (And if not, we'll debug it the next time.)
LGTM=khr
R=r, khr
CC=austin, golang-codereviews, iant, rlh
https://golang.org/cl/174950043
2014-11-12 12:54:31 -07:00
|
|
|
systemstack(func() {
|
2015-04-27 22:28:47 -06:00
|
|
|
ret = getgcmask(x)
|
2014-09-05 15:59:31 -06:00
|
|
|
})
|
|
|
|
return
|
|
|
|
}
|
2014-07-29 01:01:02 -06:00
|
|
|
|
2014-09-06 11:07:23 -06:00
|
|
|
func RunSchedLocalQueueTest() {
|
2016-03-12 16:41:08 -07:00
|
|
|
_p_ := new(p)
|
|
|
|
gs := make([]g, len(_p_.runq))
|
|
|
|
for i := 0; i < len(_p_.runq); i++ {
|
|
|
|
if g, _ := runqget(_p_); g != nil {
|
|
|
|
throw("runq is not empty initially")
|
|
|
|
}
|
|
|
|
for j := 0; j < i; j++ {
|
|
|
|
runqput(_p_, &gs[i], false)
|
|
|
|
}
|
|
|
|
for j := 0; j < i; j++ {
|
|
|
|
if g, _ := runqget(_p_); g != &gs[i] {
|
|
|
|
print("bad element at iter ", i, "/", j, "\n")
|
|
|
|
throw("bad element")
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if g, _ := runqget(_p_); g != nil {
|
|
|
|
throw("runq is not empty afterwards")
|
|
|
|
}
|
|
|
|
}
|
2014-09-06 11:07:23 -06:00
|
|
|
}
|
2016-03-12 16:41:08 -07:00
|
|
|
|
2014-09-06 11:07:23 -06:00
|
|
|
func RunSchedLocalQueueStealTest() {
|
2016-03-12 16:41:08 -07:00
|
|
|
p1 := new(p)
|
|
|
|
p2 := new(p)
|
|
|
|
gs := make([]g, len(p1.runq))
|
|
|
|
for i := 0; i < len(p1.runq); i++ {
|
|
|
|
for j := 0; j < i; j++ {
|
|
|
|
gs[j].sig = 0
|
|
|
|
runqput(p1, &gs[j], false)
|
|
|
|
}
|
|
|
|
gp := runqsteal(p2, p1, true)
|
|
|
|
s := 0
|
|
|
|
if gp != nil {
|
|
|
|
s++
|
|
|
|
gp.sig++
|
|
|
|
}
|
|
|
|
for {
|
|
|
|
gp, _ = runqget(p2)
|
|
|
|
if gp == nil {
|
|
|
|
break
|
|
|
|
}
|
|
|
|
s++
|
|
|
|
gp.sig++
|
|
|
|
}
|
|
|
|
for {
|
|
|
|
gp, _ = runqget(p1)
|
|
|
|
if gp == nil {
|
|
|
|
break
|
|
|
|
}
|
|
|
|
gp.sig++
|
|
|
|
}
|
|
|
|
for j := 0; j < i; j++ {
|
|
|
|
if gs[j].sig != 1 {
|
|
|
|
print("bad element ", j, "(", gs[j].sig, ") at iter ", i, "\n")
|
|
|
|
throw("bad element")
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if s != i/2 && s != i/2+1 {
|
|
|
|
print("bad steal ", s, ", want ", i/2, " or ", i/2+1, ", iter ", i, "\n")
|
|
|
|
throw("bad steal")
|
|
|
|
}
|
|
|
|
}
|
2014-09-06 11:07:23 -06:00
|
|
|
}
|
2013-09-06 17:23:46 -06:00
|
|
|
|
|
|
|
var StringHash = stringHash
|
|
|
|
var BytesHash = bytesHash
|
|
|
|
var Int32Hash = int32Hash
|
|
|
|
var Int64Hash = int64Hash
|
2014-08-07 13:33:20 -06:00
|
|
|
var EfaceHash = efaceHash
|
|
|
|
var IfaceHash = ifaceHash
|
2014-08-20 15:02:11 -06:00
|
|
|
var MemclrBytes = memclrBytes
|
2013-09-13 12:19:23 -06:00
|
|
|
|
2013-10-04 14:54:03 -06:00
|
|
|
var HashLoad = &hashLoad
|
2014-02-06 18:43:22 -07:00
|
|
|
|
2014-09-05 16:01:09 -06:00
|
|
|
// entry point for testing
|
|
|
|
func GostringW(w []uint16) (s string) {
|
[dev.cc] runtime: delete scalararg, ptrarg; rename onM to systemstack
Scalararg and ptrarg are not "signal safe".
Go code filling them out can be interrupted by a signal,
and then the signal handler runs, and if it also ends up
in Go code that uses scalararg or ptrarg, now the old
values have been smashed.
For the pieces of code that do need to run in a signal handler,
we introduced onM_signalok, which is really just onM
except that the _signalok is meant to convey that the caller
asserts that scalarg and ptrarg will be restored to their old
values after the call (instead of the usual behavior, zeroing them).
Scalararg and ptrarg are also untyped and therefore error-prone.
Go code can always pass a closure instead of using scalararg
and ptrarg; they were only really necessary for C code.
And there's no more C code.
For all these reasons, delete scalararg and ptrarg, converting
the few remaining references to use closures.
Once those are gone, there is no need for a distinction between
onM and onM_signalok, so replace both with a single function
equivalent to the current onM_signalok (that is, it can be called
on any of the curg, g0, and gsignal stacks).
The name onM and the phrase 'm stack' are misnomers,
because on most system an M has two system stacks:
the main thread stack and the signal handling stack.
Correct the misnomer by naming the replacement function systemstack.
Fix a few references to "M stack" in code.
The main motivation for this change is to eliminate scalararg/ptrarg.
Rick and I have already seen them cause problems because
the calling sequence m.ptrarg[0] = p is a heap pointer assignment,
so it gets a write barrier. The write barrier also uses onM, so it has
all the same problems as if it were being invoked by a signal handler.
We worked around this by saving and restoring the old values
and by calling onM_signalok, but there's no point in keeping this nice
home for bugs around any longer.
This CL also changes funcline to return the file name as a result
instead of filling in a passed-in *string. (The *string signature is
left over from when the code was written in and called from C.)
That's arguably an unrelated change, except that once I had done
the ptrarg/scalararg/onM cleanup I started getting false positives
about the *string argument escaping (not allowed in package runtime).
The compiler is wrong, but the easiest fix is to write the code like
Go code instead of like C code. I am a bit worried that the compiler
is wrong because of some use of uninitialized memory in the escape
analysis. If that's the reason, it will go away when we convert the
compiler to Go. (And if not, we'll debug it the next time.)
LGTM=khr
R=r, khr
CC=austin, golang-codereviews, iant, rlh
https://golang.org/cl/174950043
2014-11-12 12:54:31 -07:00
|
|
|
systemstack(func() {
|
2014-09-05 16:01:09 -06:00
|
|
|
s = gostringw(&w[0])
|
|
|
|
})
|
|
|
|
return
|
|
|
|
}
|
2014-09-11 17:53:34 -06:00
|
|
|
|
|
|
|
var Gostringnocopy = gostringnocopy
|
|
|
|
var Maxstring = &maxstring
|
2015-01-08 14:34:05 -07:00
|
|
|
|
2015-11-11 10:39:30 -07:00
|
|
|
type Uintreg sys.Uintreg
|
2015-03-02 21:16:48 -07:00
|
|
|
|
|
|
|
var Open = open
|
2015-04-13 17:37:04 -06:00
|
|
|
var Close = closefd
|
2015-03-02 21:16:48 -07:00
|
|
|
var Read = read
|
|
|
|
var Write = write
|
2015-03-03 11:55:22 -07:00
|
|
|
|
|
|
|
func Envs() []string { return envs }
|
|
|
|
func SetEnvs(e []string) { envs = e }
|
2015-04-16 15:32:18 -06:00
|
|
|
|
2015-11-11 10:39:30 -07:00
|
|
|
var BigEndian = sys.BigEndian
|
2015-05-02 20:59:35 -06:00
|
|
|
|
|
|
|
// For benchmarking.
|
|
|
|
|
|
|
|
func BenchSetType(n int, x interface{}) {
|
2015-10-21 13:12:25 -06:00
|
|
|
e := *efaceOf(&x)
|
2015-05-02 20:59:35 -06:00
|
|
|
t := e._type
|
|
|
|
var size uintptr
|
|
|
|
var p unsafe.Pointer
|
|
|
|
switch t.kind & kindMask {
|
2015-10-12 17:01:51 -06:00
|
|
|
case kindPtr:
|
2015-05-02 20:59:35 -06:00
|
|
|
t = (*ptrtype)(unsafe.Pointer(t)).elem
|
|
|
|
size = t.size
|
|
|
|
p = e.data
|
2015-10-12 17:01:51 -06:00
|
|
|
case kindSlice:
|
2015-05-02 20:59:35 -06:00
|
|
|
slice := *(*struct {
|
|
|
|
ptr unsafe.Pointer
|
|
|
|
len, cap uintptr
|
|
|
|
})(e.data)
|
|
|
|
t = (*slicetype)(unsafe.Pointer(t)).elem
|
|
|
|
size = t.size * slice.len
|
|
|
|
p = slice.ptr
|
|
|
|
}
|
|
|
|
allocSize := roundupsize(size)
|
|
|
|
systemstack(func() {
|
|
|
|
for i := 0; i < n; i++ {
|
|
|
|
heapBitsSetType(uintptr(p), allocSize, size, t)
|
|
|
|
}
|
|
|
|
})
|
|
|
|
}
|
2015-05-15 12:23:23 -06:00
|
|
|
|
2015-11-11 10:39:30 -07:00
|
|
|
const PtrSize = sys.PtrSize
|
2015-07-29 22:46:42 -06:00
|
|
|
|
|
|
|
var TestingAssertE2I2GC = &testingAssertE2I2GC
|
2015-08-24 11:35:49 -06:00
|
|
|
var TestingAssertE2T2GC = &testingAssertE2T2GC
|
2015-08-05 09:35:28 -06:00
|
|
|
|
|
|
|
var ForceGCPeriod = &forcegcperiod
|
2016-01-14 14:43:40 -07:00
|
|
|
|
|
|
|
// SetTracebackEnv is like runtime/debug.SetTraceback, but it raises
|
|
|
|
// the "environment" traceback level, so later calls to
|
|
|
|
// debug.SetTraceback (e.g., from testing timeouts) can't lower it.
|
|
|
|
func SetTracebackEnv(level string) {
|
|
|
|
setTraceback(level)
|
|
|
|
traceback_env = traceback_cache
|
|
|
|
}
|
2016-03-17 14:28:04 -06:00
|
|
|
|
|
|
|
var ReadUnaligned32 = readUnaligned32
|
|
|
|
var ReadUnaligned64 = readUnaligned64
|
runtime: fix pagesInUse accounting
When we grow the heap, we create a temporary "in use" span for the
memory acquired from the OS and then free that span to link it into
the heap. Hence, we (1) increase pagesInUse when we make the temporary
span so that (2) freeing the span will correctly decrease it.
However, currently step (1) increases pagesInUse by the number of
pages requested from the heap, while step (2) decreases it by the
number of pages requested from the OS (the size of the temporary
span). These aren't necessarily the same, since we round up the number
of pages we request from the OS, so steps 1 and 2 don't necessarily
cancel out like they're supposed to. Over time, this can add up and
cause pagesInUse to underflow and wrap around to 2^64. The garbage
collector computes the sweep ratio from this, so if this happens, the
sweep ratio becomes effectively infinite, causing the first allocation
on each P in a sweep cycle to sweep the entire heap. This makes
sweeping effectively STW.
Fix this by increasing pagesInUse in step 1 by the number of pages
requested from the OS, so that the two steps correctly cancel out. We
add a test that checks that the running total matches the actual state
of the heap.
Fixes #15022. For 1.6.x.
Change-Id: Iefd9d6abe37d0d447cbdbdf9941662e4f18eeffc
Reviewed-on: https://go-review.googlesource.com/21280
Run-TryBot: Austin Clements <austin@google.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Russ Cox <rsc@golang.org>
2016-03-29 10:28:24 -06:00
|
|
|
|
|
|
|
func CountPagesInUse() (pagesInUse, counted uintptr) {
|
|
|
|
stopTheWorld("CountPagesInUse")
|
|
|
|
|
|
|
|
pagesInUse = uintptr(mheap_.pagesInUse)
|
|
|
|
|
|
|
|
for _, s := range h_allspans {
|
|
|
|
if s.state == mSpanInUse {
|
|
|
|
counted += s.npages
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
startTheWorld()
|
|
|
|
|
|
|
|
return
|
|
|
|
}
|