2009-01-26 18:37:05 -07:00
|
|
|
// Copyright 2009 The Go Authors. All rights reserved.
|
|
|
|
// Use of this source code is governed by a BSD-style
|
|
|
|
// license that can be found in the LICENSE file.
|
|
|
|
|
2011-02-02 21:03:47 -07:00
|
|
|
// Garbage collector.
|
2009-01-26 18:37:05 -07:00
|
|
|
|
|
|
|
#include "runtime.h"
|
|
|
|
#include "malloc.h"
|
runtime: stack split + garbage collection bug
The g->sched.sp saved stack pointer and the
g->stackbase and g->stackguard stack bounds
can change even while "the world is stopped",
because a goroutine has to call functions (and
therefore might split its stack) when exiting a
system call to check whether the world is stopped
(and if so, wait until the world continues).
That means the garbage collector cannot access
those values safely (without a race) for goroutines
executing system calls. Instead, save a consistent
triple in g->gcsp, g->gcstack, g->gcguard during
entersyscall and have the garbage collector refer
to those.
The old code was occasionally seeing (because of
the race) an sp and stk that did not correspond to
each other, so that stk - sp was not the number of
stack bytes following sp. In that case, if sp < stk
then the call scanblock(sp, stk - sp) scanned too
many bytes (anything between the two pointers,
which pointed into different allocation blocks).
If sp > stk then stk - sp wrapped around.
On 32-bit, stk - sp is a uintptr (uint32) converted
to int64 in the call to scanblock, so a large (~4G)
but positive number. Scanblock would try to scan
that many bytes and eventually fault accessing
unmapped memory. On 64-bit, stk - sp is a uintptr (uint64)
promoted to int64 in the call to scanblock, so a negative
number. Scanblock would not scan anything, possibly
causing in-use blocks to be freed.
In short, 32-bit platforms would have seen either
ineffective garbage collection or crashes during garbage
collection, while 64-bit platforms would have seen
either ineffective or incorrect garbage collection.
You can see the invalid arguments to scanblock in the
stack traces in issue 1620.
Fixes #1620.
Fixes #1746.
R=iant, r
CC=golang-dev
https://golang.org/cl/4437075
2011-04-27 21:21:12 -06:00
|
|
|
#include "stack.h"
|
2009-01-26 18:37:05 -07:00
|
|
|
|
|
|
|
enum {
|
2011-02-02 21:03:47 -07:00
|
|
|
Debug = 0,
|
|
|
|
UseCas = 1,
|
|
|
|
PtrSize = sizeof(void*),
|
|
|
|
|
|
|
|
// Four bits per word (see #defines below).
|
|
|
|
wordsPerBitmapWord = sizeof(void*)*8/4,
|
|
|
|
bitShift = sizeof(void*)*8/4,
|
2009-01-26 18:37:05 -07:00
|
|
|
};
|
|
|
|
|
2011-02-02 21:03:47 -07:00
|
|
|
// Bits in per-word bitmap.
|
|
|
|
// #defines because enum might not be able to hold the values.
|
|
|
|
//
|
|
|
|
// Each word in the bitmap describes wordsPerBitmapWord words
|
|
|
|
// of heap memory. There are 4 bitmap bits dedicated to each heap word,
|
|
|
|
// so on a 64-bit system there is one bitmap word per 16 heap words.
|
|
|
|
// The bits in the word are packed together by type first, then by
|
|
|
|
// heap location, so each 64-bit bitmap word consists of, from top to bottom,
|
|
|
|
// the 16 bitSpecial bits for the corresponding heap words, then the 16 bitMarked bits,
|
|
|
|
// then the 16 bitNoPointers/bitBlockBoundary bits, then the 16 bitAllocated bits.
|
|
|
|
// This layout makes it easier to iterate over the bits of a given type.
|
|
|
|
//
|
|
|
|
// The bitmap starts at mheap.arena_start and extends *backward* from
|
|
|
|
// there. On a 64-bit system the off'th word in the arena is tracked by
|
|
|
|
// the off/16+1'th word before mheap.arena_start. (On a 32-bit system,
|
|
|
|
// the only difference is that the divisor is 8.)
|
|
|
|
//
|
|
|
|
// To pull out the bits corresponding to a given pointer p, we use:
|
|
|
|
//
|
|
|
|
// off = p - (uintptr*)mheap.arena_start; // word offset
|
|
|
|
// b = (uintptr*)mheap.arena_start - off/wordsPerBitmapWord - 1;
|
|
|
|
// shift = off % wordsPerBitmapWord
|
|
|
|
// bits = *b >> shift;
|
|
|
|
// /* then test bits & bitAllocated, bits & bitMarked, etc. */
|
|
|
|
//
|
|
|
|
#define bitAllocated ((uintptr)1<<(bitShift*0))
|
|
|
|
#define bitNoPointers ((uintptr)1<<(bitShift*1)) /* when bitAllocated is set */
|
|
|
|
#define bitMarked ((uintptr)1<<(bitShift*2)) /* when bitAllocated is set */
|
|
|
|
#define bitSpecial ((uintptr)1<<(bitShift*3)) /* when bitAllocated is set - has finalizer or being profiled */
|
|
|
|
#define bitBlockBoundary ((uintptr)1<<(bitShift*1)) /* when bitAllocated is NOT set */
|
|
|
|
|
|
|
|
#define bitMask (bitBlockBoundary | bitAllocated | bitMarked | bitSpecial)
|
|
|
|
|
|
|
|
static uint64 nlookup;
|
|
|
|
static uint64 nsizelookup;
|
|
|
|
static uint64 naddrlookup;
|
|
|
|
static int32 gctrace;
|
|
|
|
|
|
|
|
typedef struct Workbuf Workbuf;
|
|
|
|
struct Workbuf
|
2010-09-07 07:57:22 -06:00
|
|
|
{
|
2011-02-02 21:03:47 -07:00
|
|
|
Workbuf *next;
|
|
|
|
uintptr nw;
|
|
|
|
byte *w[2048-2];
|
2010-09-07 07:57:22 -06:00
|
|
|
};
|
|
|
|
|
2009-08-20 17:09:38 -06:00
|
|
|
extern byte data[];
|
2009-01-26 18:37:05 -07:00
|
|
|
extern byte etext[];
|
|
|
|
extern byte end[];
|
|
|
|
|
2010-03-26 15:15:30 -06:00
|
|
|
static G *fing;
|
|
|
|
static Finalizer *finq;
|
2010-04-07 21:38:02 -06:00
|
|
|
static int32 fingwait;
|
|
|
|
|
2010-03-26 15:15:30 -06:00
|
|
|
static void runfinq(void);
|
2011-02-02 21:03:47 -07:00
|
|
|
static Workbuf* getempty(Workbuf*);
|
|
|
|
static Workbuf* getfull(Workbuf*);
|
|
|
|
|
|
|
|
// scanblock scans a block of n bytes starting at pointer b for references
|
|
|
|
// to other objects, scanning any it finds recursively until there are no
|
|
|
|
// unscanned objects left. Instead of using an explicit recursion, it keeps
|
|
|
|
// a work list in the Workbuf* structures and loops in the main function
|
|
|
|
// body. Keeping an explicit work list is easier on the stack allocator and
|
|
|
|
// more efficient.
|
2009-01-26 18:37:05 -07:00
|
|
|
static void
|
2010-09-07 07:57:22 -06:00
|
|
|
scanblock(byte *b, int64 n)
|
2009-01-26 18:37:05 -07:00
|
|
|
{
|
2011-02-02 21:03:47 -07:00
|
|
|
byte *obj, *arena_start, *p;
|
2009-01-26 18:37:05 -07:00
|
|
|
void **vp;
|
2011-02-02 21:03:47 -07:00
|
|
|
uintptr size, *bitp, bits, shift, i, j, x, xbits, off;
|
|
|
|
MSpan *s;
|
|
|
|
PageID k;
|
|
|
|
void **bw, **w, **ew;
|
|
|
|
Workbuf *wbuf;
|
2010-09-07 07:57:22 -06:00
|
|
|
|
runtime: stack split + garbage collection bug
The g->sched.sp saved stack pointer and the
g->stackbase and g->stackguard stack bounds
can change even while "the world is stopped",
because a goroutine has to call functions (and
therefore might split its stack) when exiting a
system call to check whether the world is stopped
(and if so, wait until the world continues).
That means the garbage collector cannot access
those values safely (without a race) for goroutines
executing system calls. Instead, save a consistent
triple in g->gcsp, g->gcstack, g->gcguard during
entersyscall and have the garbage collector refer
to those.
The old code was occasionally seeing (because of
the race) an sp and stk that did not correspond to
each other, so that stk - sp was not the number of
stack bytes following sp. In that case, if sp < stk
then the call scanblock(sp, stk - sp) scanned too
many bytes (anything between the two pointers,
which pointed into different allocation blocks).
If sp > stk then stk - sp wrapped around.
On 32-bit, stk - sp is a uintptr (uint32) converted
to int64 in the call to scanblock, so a large (~4G)
but positive number. Scanblock would try to scan
that many bytes and eventually fault accessing
unmapped memory. On 64-bit, stk - sp is a uintptr (uint64)
promoted to int64 in the call to scanblock, so a negative
number. Scanblock would not scan anything, possibly
causing in-use blocks to be freed.
In short, 32-bit platforms would have seen either
ineffective garbage collection or crashes during garbage
collection, while 64-bit platforms would have seen
either ineffective or incorrect garbage collection.
You can see the invalid arguments to scanblock in the
stack traces in issue 1620.
Fixes #1620.
Fixes #1746.
R=iant, r
CC=golang-dev
https://golang.org/cl/4437075
2011-04-27 21:21:12 -06:00
|
|
|
if((int64)(uintptr)n != n || n < 0) {
|
|
|
|
runtime·printf("scanblock %p %D\n", b, n);
|
|
|
|
runtime·throw("scanblock");
|
|
|
|
}
|
|
|
|
|
2011-02-02 21:03:47 -07:00
|
|
|
// Memory arena parameters.
|
|
|
|
arena_start = runtime·mheap.arena_start;
|
|
|
|
|
|
|
|
wbuf = nil; // current work buffer
|
|
|
|
ew = nil; // end of work buffer
|
|
|
|
bw = nil; // beginning of work buffer
|
|
|
|
w = nil; // current pointer into work buffer
|
|
|
|
|
|
|
|
// Align b to a word boundary.
|
|
|
|
off = (uintptr)b & (PtrSize-1);
|
|
|
|
if(off != 0) {
|
|
|
|
b += PtrSize - off;
|
|
|
|
n -= PtrSize - off;
|
|
|
|
}
|
2010-09-07 07:57:22 -06:00
|
|
|
|
2011-02-02 21:03:47 -07:00
|
|
|
for(;;) {
|
|
|
|
// Each iteration scans the block b of length n, queueing pointers in
|
|
|
|
// the work buffer.
|
2010-09-07 07:57:22 -06:00
|
|
|
if(Debug > 1)
|
runtime: ,s/[a-zA-Z0-9_]+/runtime·&/g, almost
Prefix all external symbols in runtime by runtime·,
to avoid conflicts with possible symbols of the same
name in linked-in C libraries. The obvious conflicts
are printf, malloc, and free, but hide everything to
avoid future pain.
The symbols left alone are:
** known to cgo **
_cgo_free
_cgo_malloc
libcgo_thread_start
initcgo
ncgocall
** known to linker **
_rt0_$GOARCH
_rt0_$GOARCH_$GOOS
text
etext
data
end
pclntab
epclntab
symtab
esymtab
** known to C compiler **
_divv
_modv
_div64by32
etc (arch specific)
Tested on darwin/386, darwin/amd64, linux/386, linux/amd64.
Built (but not tested) for freebsd/386, freebsd/amd64, linux/arm, windows/386.
R=r, PeterGo
CC=golang-dev
https://golang.org/cl/2899041
2010-11-04 12:00:19 -06:00
|
|
|
runtime·printf("scanblock %p %D\n", b, n);
|
2011-02-02 21:03:47 -07:00
|
|
|
|
2010-09-07 07:57:22 -06:00
|
|
|
vp = (void**)b;
|
|
|
|
n /= PtrSize;
|
|
|
|
for(i=0; i<n; i++) {
|
2011-02-02 21:03:47 -07:00
|
|
|
obj = (byte*)vp[i];
|
|
|
|
|
|
|
|
// Words outside the arena cannot be pointers.
|
|
|
|
if((byte*)obj < arena_start || (byte*)obj >= runtime·mheap.arena_used)
|
2010-09-07 07:57:22 -06:00
|
|
|
continue;
|
2011-02-02 21:03:47 -07:00
|
|
|
|
|
|
|
// obj may be a pointer to a live object.
|
|
|
|
// Try to find the beginning of the object.
|
|
|
|
|
|
|
|
// Round down to word boundary.
|
|
|
|
obj = (void*)((uintptr)obj & ~((uintptr)PtrSize-1));
|
|
|
|
|
|
|
|
// Find bits for this word.
|
|
|
|
off = (uintptr*)obj - (uintptr*)arena_start;
|
|
|
|
bitp = (uintptr*)arena_start - off/wordsPerBitmapWord - 1;
|
|
|
|
shift = off % wordsPerBitmapWord;
|
|
|
|
xbits = *bitp;
|
|
|
|
bits = xbits >> shift;
|
|
|
|
|
|
|
|
// Pointing at the beginning of a block?
|
|
|
|
if((bits & (bitAllocated|bitBlockBoundary)) != 0)
|
|
|
|
goto found;
|
|
|
|
|
|
|
|
// Pointing just past the beginning?
|
|
|
|
// Scan backward a little to find a block boundary.
|
|
|
|
for(j=shift; j-->0; ) {
|
|
|
|
if(((xbits>>j) & (bitAllocated|bitBlockBoundary)) != 0) {
|
|
|
|
obj = (byte*)obj - (shift-j)*PtrSize;
|
|
|
|
shift = j;
|
|
|
|
bits = xbits>>shift;
|
|
|
|
goto found;
|
2010-04-22 18:52:22 -06:00
|
|
|
}
|
2009-01-26 18:37:05 -07:00
|
|
|
}
|
2011-02-02 21:03:47 -07:00
|
|
|
|
|
|
|
// Otherwise consult span table to find beginning.
|
|
|
|
// (Manually inlined copy of MHeap_LookupMaybe.)
|
|
|
|
nlookup++;
|
|
|
|
naddrlookup++;
|
|
|
|
k = (uintptr)obj>>PageShift;
|
|
|
|
x = k;
|
|
|
|
if(sizeof(void*) == 8)
|
|
|
|
x -= (uintptr)arena_start>>PageShift;
|
|
|
|
s = runtime·mheap.map[x];
|
|
|
|
if(s == nil || k < s->start || k - s->start >= s->npages || s->state != MSpanInUse)
|
|
|
|
continue;
|
|
|
|
p = (byte*)((uintptr)s->start<<PageShift);
|
|
|
|
if(s->sizeclass == 0) {
|
|
|
|
obj = p;
|
|
|
|
} else {
|
|
|
|
if((byte*)obj >= (byte*)s->limit)
|
|
|
|
continue;
|
|
|
|
size = runtime·class_to_size[s->sizeclass];
|
|
|
|
int32 i = ((byte*)obj - p)/size;
|
|
|
|
obj = p+i*size;
|
|
|
|
}
|
|
|
|
|
|
|
|
// Now that we know the object header, reload bits.
|
|
|
|
off = (uintptr*)obj - (uintptr*)arena_start;
|
|
|
|
bitp = (uintptr*)arena_start - off/wordsPerBitmapWord - 1;
|
|
|
|
shift = off % wordsPerBitmapWord;
|
|
|
|
xbits = *bitp;
|
|
|
|
bits = xbits >> shift;
|
|
|
|
|
|
|
|
found:
|
|
|
|
// Now we have bits, bitp, and shift correct for
|
|
|
|
// obj pointing at the base of the object.
|
|
|
|
// If not allocated or already marked, done.
|
|
|
|
if((bits & bitAllocated) == 0 || (bits & bitMarked) != 0)
|
|
|
|
continue;
|
|
|
|
*bitp |= bitMarked<<shift;
|
|
|
|
|
|
|
|
// If object has no pointers, don't need to scan further.
|
|
|
|
if((bits & bitNoPointers) != 0)
|
|
|
|
continue;
|
|
|
|
|
|
|
|
// If buffer is full, get a new one.
|
|
|
|
if(w >= ew) {
|
|
|
|
wbuf = getempty(wbuf);
|
|
|
|
bw = wbuf->w;
|
|
|
|
w = bw;
|
|
|
|
ew = bw + nelem(wbuf->w);
|
|
|
|
}
|
|
|
|
*w++ = obj;
|
|
|
|
}
|
|
|
|
|
|
|
|
// Done scanning [b, b+n). Prepare for the next iteration of
|
|
|
|
// the loop by setting b and n to the parameters for the next block.
|
|
|
|
|
|
|
|
// Fetch b from the work buffers.
|
|
|
|
if(w <= bw) {
|
|
|
|
// Emptied our buffer: refill.
|
|
|
|
wbuf = getfull(wbuf);
|
|
|
|
if(wbuf == nil)
|
|
|
|
break;
|
|
|
|
bw = wbuf->w;
|
|
|
|
ew = wbuf->w + nelem(wbuf->w);
|
|
|
|
w = bw+wbuf->nw;
|
2009-01-26 18:37:05 -07:00
|
|
|
}
|
2011-02-02 21:03:47 -07:00
|
|
|
b = *--w;
|
|
|
|
|
|
|
|
// Figure out n = size of b. Start by loading bits for b.
|
|
|
|
off = (uintptr*)b - (uintptr*)arena_start;
|
|
|
|
bitp = (uintptr*)arena_start - off/wordsPerBitmapWord - 1;
|
|
|
|
shift = off % wordsPerBitmapWord;
|
|
|
|
xbits = *bitp;
|
|
|
|
bits = xbits >> shift;
|
|
|
|
|
|
|
|
// Might be small; look for nearby block boundary.
|
|
|
|
// A block boundary is marked by either bitBlockBoundary
|
|
|
|
// or bitAllocated being set (see notes near their definition).
|
|
|
|
enum {
|
|
|
|
boundary = bitBlockBoundary|bitAllocated
|
|
|
|
};
|
|
|
|
// Look for a block boundary both after and before b
|
|
|
|
// in the same bitmap word.
|
|
|
|
//
|
|
|
|
// A block boundary j words after b is indicated by
|
|
|
|
// bits>>j & boundary
|
|
|
|
// assuming shift+j < bitShift. (If shift+j >= bitShift then
|
|
|
|
// we'll be bleeding other bit types like bitMarked into our test.)
|
|
|
|
// Instead of inserting the conditional shift+j < bitShift into the loop,
|
|
|
|
// we can let j range from 1 to bitShift as long as we first
|
|
|
|
// apply a mask to keep only the bits corresponding
|
|
|
|
// to shift+j < bitShift aka j < bitShift-shift.
|
|
|
|
bits &= (boundary<<(bitShift-shift)) - boundary;
|
|
|
|
|
|
|
|
// A block boundary j words before b is indicated by
|
|
|
|
// xbits>>(shift-j) & boundary
|
|
|
|
// (assuming shift >= j). There is no cleverness here
|
|
|
|
// avoid the test, because when j gets too large the shift
|
|
|
|
// turns negative, which is undefined in C.
|
|
|
|
|
|
|
|
for(j=1; j<bitShift; j++) {
|
|
|
|
if(((bits>>j)&boundary) != 0 || shift>=j && ((xbits>>(shift-j))&boundary) != 0) {
|
|
|
|
n = j*PtrSize;
|
|
|
|
goto scan;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Fall back to asking span about size class.
|
|
|
|
// (Manually inlined copy of MHeap_Lookup.)
|
|
|
|
nlookup++;
|
|
|
|
nsizelookup++;
|
|
|
|
x = (uintptr)b>>PageShift;
|
|
|
|
if(sizeof(void*) == 8)
|
|
|
|
x -= (uintptr)arena_start>>PageShift;
|
|
|
|
s = runtime·mheap.map[x];
|
|
|
|
if(s->sizeclass == 0)
|
|
|
|
n = s->npages<<PageShift;
|
|
|
|
else
|
|
|
|
n = runtime·class_to_size[s->sizeclass];
|
|
|
|
scan:;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
static struct {
|
|
|
|
Workbuf *full;
|
|
|
|
Workbuf *empty;
|
|
|
|
byte *chunk;
|
|
|
|
uintptr nchunk;
|
|
|
|
} work;
|
|
|
|
|
|
|
|
// Get an empty work buffer off the work.empty list,
|
|
|
|
// allocating new buffers as needed.
|
|
|
|
static Workbuf*
|
|
|
|
getempty(Workbuf *b)
|
|
|
|
{
|
|
|
|
if(b != nil) {
|
|
|
|
b->nw = nelem(b->w);
|
|
|
|
b->next = work.full;
|
|
|
|
work.full = b;
|
|
|
|
}
|
|
|
|
b = work.empty;
|
|
|
|
if(b != nil) {
|
|
|
|
work.empty = b->next;
|
|
|
|
return b;
|
|
|
|
}
|
|
|
|
|
|
|
|
if(work.nchunk < sizeof *b) {
|
|
|
|
work.nchunk = 1<<20;
|
|
|
|
work.chunk = runtime·SysAlloc(work.nchunk);
|
|
|
|
}
|
|
|
|
b = (Workbuf*)work.chunk;
|
|
|
|
work.chunk += sizeof *b;
|
|
|
|
work.nchunk -= sizeof *b;
|
|
|
|
return b;
|
|
|
|
}
|
|
|
|
|
|
|
|
// Get a full work buffer off the work.full list, or return nil.
|
|
|
|
static Workbuf*
|
|
|
|
getfull(Workbuf *b)
|
|
|
|
{
|
|
|
|
if(b != nil) {
|
|
|
|
b->nw = 0;
|
|
|
|
b->next = work.empty;
|
|
|
|
work.empty = b;
|
2009-01-26 18:37:05 -07:00
|
|
|
}
|
2011-02-02 21:03:47 -07:00
|
|
|
b = work.full;
|
|
|
|
if(b != nil)
|
|
|
|
work.full = b->next;
|
|
|
|
return b;
|
2009-01-26 18:37:05 -07:00
|
|
|
}
|
|
|
|
|
2011-02-02 21:03:47 -07:00
|
|
|
// Scanstack calls scanblock on each of gp's stack segments.
|
2009-01-26 18:37:05 -07:00
|
|
|
static void
|
2009-07-27 15:16:28 -06:00
|
|
|
scanstack(G *gp)
|
2009-01-26 18:37:05 -07:00
|
|
|
{
|
runtime: stack split + garbage collection bug
The g->sched.sp saved stack pointer and the
g->stackbase and g->stackguard stack bounds
can change even while "the world is stopped",
because a goroutine has to call functions (and
therefore might split its stack) when exiting a
system call to check whether the world is stopped
(and if so, wait until the world continues).
That means the garbage collector cannot access
those values safely (without a race) for goroutines
executing system calls. Instead, save a consistent
triple in g->gcsp, g->gcstack, g->gcguard during
entersyscall and have the garbage collector refer
to those.
The old code was occasionally seeing (because of
the race) an sp and stk that did not correspond to
each other, so that stk - sp was not the number of
stack bytes following sp. In that case, if sp < stk
then the call scanblock(sp, stk - sp) scanned too
many bytes (anything between the two pointers,
which pointed into different allocation blocks).
If sp > stk then stk - sp wrapped around.
On 32-bit, stk - sp is a uintptr (uint32) converted
to int64 in the call to scanblock, so a large (~4G)
but positive number. Scanblock would try to scan
that many bytes and eventually fault accessing
unmapped memory. On 64-bit, stk - sp is a uintptr (uint64)
promoted to int64 in the call to scanblock, so a negative
number. Scanblock would not scan anything, possibly
causing in-use blocks to be freed.
In short, 32-bit platforms would have seen either
ineffective garbage collection or crashes during garbage
collection, while 64-bit platforms would have seen
either ineffective or incorrect garbage collection.
You can see the invalid arguments to scanblock in the
stack traces in issue 1620.
Fixes #1620.
Fixes #1746.
R=iant, r
CC=golang-dev
https://golang.org/cl/4437075
2011-04-27 21:21:12 -06:00
|
|
|
int32 n;
|
2009-01-26 18:37:05 -07:00
|
|
|
Stktop *stk;
|
runtime: stack split + garbage collection bug
The g->sched.sp saved stack pointer and the
g->stackbase and g->stackguard stack bounds
can change even while "the world is stopped",
because a goroutine has to call functions (and
therefore might split its stack) when exiting a
system call to check whether the world is stopped
(and if so, wait until the world continues).
That means the garbage collector cannot access
those values safely (without a race) for goroutines
executing system calls. Instead, save a consistent
triple in g->gcsp, g->gcstack, g->gcguard during
entersyscall and have the garbage collector refer
to those.
The old code was occasionally seeing (because of
the race) an sp and stk that did not correspond to
each other, so that stk - sp was not the number of
stack bytes following sp. In that case, if sp < stk
then the call scanblock(sp, stk - sp) scanned too
many bytes (anything between the two pointers,
which pointed into different allocation blocks).
If sp > stk then stk - sp wrapped around.
On 32-bit, stk - sp is a uintptr (uint32) converted
to int64 in the call to scanblock, so a large (~4G)
but positive number. Scanblock would try to scan
that many bytes and eventually fault accessing
unmapped memory. On 64-bit, stk - sp is a uintptr (uint64)
promoted to int64 in the call to scanblock, so a negative
number. Scanblock would not scan anything, possibly
causing in-use blocks to be freed.
In short, 32-bit platforms would have seen either
ineffective garbage collection or crashes during garbage
collection, while 64-bit platforms would have seen
either ineffective or incorrect garbage collection.
You can see the invalid arguments to scanblock in the
stack traces in issue 1620.
Fixes #1620.
Fixes #1746.
R=iant, r
CC=golang-dev
https://golang.org/cl/4437075
2011-04-27 21:21:12 -06:00
|
|
|
byte *sp, *guard;
|
2009-01-26 18:37:05 -07:00
|
|
|
|
runtime: stack split + garbage collection bug
The g->sched.sp saved stack pointer and the
g->stackbase and g->stackguard stack bounds
can change even while "the world is stopped",
because a goroutine has to call functions (and
therefore might split its stack) when exiting a
system call to check whether the world is stopped
(and if so, wait until the world continues).
That means the garbage collector cannot access
those values safely (without a race) for goroutines
executing system calls. Instead, save a consistent
triple in g->gcsp, g->gcstack, g->gcguard during
entersyscall and have the garbage collector refer
to those.
The old code was occasionally seeing (because of
the race) an sp and stk that did not correspond to
each other, so that stk - sp was not the number of
stack bytes following sp. In that case, if sp < stk
then the call scanblock(sp, stk - sp) scanned too
many bytes (anything between the two pointers,
which pointed into different allocation blocks).
If sp > stk then stk - sp wrapped around.
On 32-bit, stk - sp is a uintptr (uint32) converted
to int64 in the call to scanblock, so a large (~4G)
but positive number. Scanblock would try to scan
that many bytes and eventually fault accessing
unmapped memory. On 64-bit, stk - sp is a uintptr (uint64)
promoted to int64 in the call to scanblock, so a negative
number. Scanblock would not scan anything, possibly
causing in-use blocks to be freed.
In short, 32-bit platforms would have seen either
ineffective garbage collection or crashes during garbage
collection, while 64-bit platforms would have seen
either ineffective or incorrect garbage collection.
You can see the invalid arguments to scanblock in the
stack traces in issue 1620.
Fixes #1620.
Fixes #1746.
R=iant, r
CC=golang-dev
https://golang.org/cl/4437075
2011-04-27 21:21:12 -06:00
|
|
|
stk = (Stktop*)gp->stackbase;
|
|
|
|
guard = gp->stackguard;
|
|
|
|
|
|
|
|
if(gp == g) {
|
|
|
|
// Scanning our own stack: start at &gp.
|
2009-07-27 15:16:28 -06:00
|
|
|
sp = (byte*)&gp;
|
runtime: stack split + garbage collection bug
The g->sched.sp saved stack pointer and the
g->stackbase and g->stackguard stack bounds
can change even while "the world is stopped",
because a goroutine has to call functions (and
therefore might split its stack) when exiting a
system call to check whether the world is stopped
(and if so, wait until the world continues).
That means the garbage collector cannot access
those values safely (without a race) for goroutines
executing system calls. Instead, save a consistent
triple in g->gcsp, g->gcstack, g->gcguard during
entersyscall and have the garbage collector refer
to those.
The old code was occasionally seeing (because of
the race) an sp and stk that did not correspond to
each other, so that stk - sp was not the number of
stack bytes following sp. In that case, if sp < stk
then the call scanblock(sp, stk - sp) scanned too
many bytes (anything between the two pointers,
which pointed into different allocation blocks).
If sp > stk then stk - sp wrapped around.
On 32-bit, stk - sp is a uintptr (uint32) converted
to int64 in the call to scanblock, so a large (~4G)
but positive number. Scanblock would try to scan
that many bytes and eventually fault accessing
unmapped memory. On 64-bit, stk - sp is a uintptr (uint64)
promoted to int64 in the call to scanblock, so a negative
number. Scanblock would not scan anything, possibly
causing in-use blocks to be freed.
In short, 32-bit platforms would have seen either
ineffective garbage collection or crashes during garbage
collection, while 64-bit platforms would have seen
either ineffective or incorrect garbage collection.
You can see the invalid arguments to scanblock in the
stack traces in issue 1620.
Fixes #1620.
Fixes #1746.
R=iant, r
CC=golang-dev
https://golang.org/cl/4437075
2011-04-27 21:21:12 -06:00
|
|
|
} else {
|
|
|
|
// Scanning another goroutine's stack.
|
|
|
|
// The goroutine is usually asleep (the world is stopped).
|
2009-07-27 15:16:28 -06:00
|
|
|
sp = gp->sched.sp;
|
runtime: stack split + garbage collection bug
The g->sched.sp saved stack pointer and the
g->stackbase and g->stackguard stack bounds
can change even while "the world is stopped",
because a goroutine has to call functions (and
therefore might split its stack) when exiting a
system call to check whether the world is stopped
(and if so, wait until the world continues).
That means the garbage collector cannot access
those values safely (without a race) for goroutines
executing system calls. Instead, save a consistent
triple in g->gcsp, g->gcstack, g->gcguard during
entersyscall and have the garbage collector refer
to those.
The old code was occasionally seeing (because of
the race) an sp and stk that did not correspond to
each other, so that stk - sp was not the number of
stack bytes following sp. In that case, if sp < stk
then the call scanblock(sp, stk - sp) scanned too
many bytes (anything between the two pointers,
which pointed into different allocation blocks).
If sp > stk then stk - sp wrapped around.
On 32-bit, stk - sp is a uintptr (uint32) converted
to int64 in the call to scanblock, so a large (~4G)
but positive number. Scanblock would try to scan
that many bytes and eventually fault accessing
unmapped memory. On 64-bit, stk - sp is a uintptr (uint64)
promoted to int64 in the call to scanblock, so a negative
number. Scanblock would not scan anything, possibly
causing in-use blocks to be freed.
In short, 32-bit platforms would have seen either
ineffective garbage collection or crashes during garbage
collection, while 64-bit platforms would have seen
either ineffective or incorrect garbage collection.
You can see the invalid arguments to scanblock in the
stack traces in issue 1620.
Fixes #1620.
Fixes #1746.
R=iant, r
CC=golang-dev
https://golang.org/cl/4437075
2011-04-27 21:21:12 -06:00
|
|
|
|
|
|
|
// The exception is that if gp->status == Gsyscall, the goroutine
|
|
|
|
// is about to enter or might have just exited a system call, in
|
|
|
|
// which case it may be executing code such as schedlock and
|
|
|
|
// may have needed to start a new stack segment.
|
|
|
|
// Use the stack segment and stack pointer at the time of
|
|
|
|
// the entersyscall.
|
|
|
|
if(g->gcstack != nil) {
|
|
|
|
stk = (Stktop*)gp->gcstack;
|
|
|
|
sp = gp->gcsp;
|
|
|
|
guard = gp->gcguard;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2010-02-03 17:31:34 -07:00
|
|
|
if(Debug > 1)
|
runtime: ,s/[a-zA-Z0-9_]+/runtime·&/g, almost
Prefix all external symbols in runtime by runtime·,
to avoid conflicts with possible symbols of the same
name in linked-in C libraries. The obvious conflicts
are printf, malloc, and free, but hide everything to
avoid future pain.
The symbols left alone are:
** known to cgo **
_cgo_free
_cgo_malloc
libcgo_thread_start
initcgo
ncgocall
** known to linker **
_rt0_$GOARCH
_rt0_$GOARCH_$GOOS
text
etext
data
end
pclntab
epclntab
symtab
esymtab
** known to C compiler **
_divv
_modv
_div64by32
etc (arch specific)
Tested on darwin/386, darwin/amd64, linux/386, linux/amd64.
Built (but not tested) for freebsd/386, freebsd/amd64, linux/arm, windows/386.
R=r, PeterGo
CC=golang-dev
https://golang.org/cl/2899041
2010-11-04 12:00:19 -06:00
|
|
|
runtime·printf("scanstack %d %p\n", gp->goid, sp);
|
runtime: stack split + garbage collection bug
The g->sched.sp saved stack pointer and the
g->stackbase and g->stackguard stack bounds
can change even while "the world is stopped",
because a goroutine has to call functions (and
therefore might split its stack) when exiting a
system call to check whether the world is stopped
(and if so, wait until the world continues).
That means the garbage collector cannot access
those values safely (without a race) for goroutines
executing system calls. Instead, save a consistent
triple in g->gcsp, g->gcstack, g->gcguard during
entersyscall and have the garbage collector refer
to those.
The old code was occasionally seeing (because of
the race) an sp and stk that did not correspond to
each other, so that stk - sp was not the number of
stack bytes following sp. In that case, if sp < stk
then the call scanblock(sp, stk - sp) scanned too
many bytes (anything between the two pointers,
which pointed into different allocation blocks).
If sp > stk then stk - sp wrapped around.
On 32-bit, stk - sp is a uintptr (uint32) converted
to int64 in the call to scanblock, so a large (~4G)
but positive number. Scanblock would try to scan
that many bytes and eventually fault accessing
unmapped memory. On 64-bit, stk - sp is a uintptr (uint64)
promoted to int64 in the call to scanblock, so a negative
number. Scanblock would not scan anything, possibly
causing in-use blocks to be freed.
In short, 32-bit platforms would have seen either
ineffective garbage collection or crashes during garbage
collection, while 64-bit platforms would have seen
either ineffective or incorrect garbage collection.
You can see the invalid arguments to scanblock in the
stack traces in issue 1620.
Fixes #1620.
Fixes #1746.
R=iant, r
CC=golang-dev
https://golang.org/cl/4437075
2011-04-27 21:21:12 -06:00
|
|
|
n = 0;
|
2009-01-26 18:37:05 -07:00
|
|
|
while(stk) {
|
runtime: stack split + garbage collection bug
The g->sched.sp saved stack pointer and the
g->stackbase and g->stackguard stack bounds
can change even while "the world is stopped",
because a goroutine has to call functions (and
therefore might split its stack) when exiting a
system call to check whether the world is stopped
(and if so, wait until the world continues).
That means the garbage collector cannot access
those values safely (without a race) for goroutines
executing system calls. Instead, save a consistent
triple in g->gcsp, g->gcstack, g->gcguard during
entersyscall and have the garbage collector refer
to those.
The old code was occasionally seeing (because of
the race) an sp and stk that did not correspond to
each other, so that stk - sp was not the number of
stack bytes following sp. In that case, if sp < stk
then the call scanblock(sp, stk - sp) scanned too
many bytes (anything between the two pointers,
which pointed into different allocation blocks).
If sp > stk then stk - sp wrapped around.
On 32-bit, stk - sp is a uintptr (uint32) converted
to int64 in the call to scanblock, so a large (~4G)
but positive number. Scanblock would try to scan
that many bytes and eventually fault accessing
unmapped memory. On 64-bit, stk - sp is a uintptr (uint64)
promoted to int64 in the call to scanblock, so a negative
number. Scanblock would not scan anything, possibly
causing in-use blocks to be freed.
In short, 32-bit platforms would have seen either
ineffective garbage collection or crashes during garbage
collection, while 64-bit platforms would have seen
either ineffective or incorrect garbage collection.
You can see the invalid arguments to scanblock in the
stack traces in issue 1620.
Fixes #1620.
Fixes #1746.
R=iant, r
CC=golang-dev
https://golang.org/cl/4437075
2011-04-27 21:21:12 -06:00
|
|
|
if(sp < guard-StackGuard || (byte*)stk < sp) {
|
|
|
|
runtime·printf("scanstack inconsistent: g%d#%d sp=%p not in [%p,%p]\n", gp->goid, n, sp, guard-StackGuard, stk);
|
|
|
|
runtime·throw("scanstack");
|
|
|
|
}
|
2010-09-07 07:57:22 -06:00
|
|
|
scanblock(sp, (byte*)stk - sp);
|
2009-06-17 16:12:16 -06:00
|
|
|
sp = stk->gobuf.sp;
|
runtime: stack split + garbage collection bug
The g->sched.sp saved stack pointer and the
g->stackbase and g->stackguard stack bounds
can change even while "the world is stopped",
because a goroutine has to call functions (and
therefore might split its stack) when exiting a
system call to check whether the world is stopped
(and if so, wait until the world continues).
That means the garbage collector cannot access
those values safely (without a race) for goroutines
executing system calls. Instead, save a consistent
triple in g->gcsp, g->gcstack, g->gcguard during
entersyscall and have the garbage collector refer
to those.
The old code was occasionally seeing (because of
the race) an sp and stk that did not correspond to
each other, so that stk - sp was not the number of
stack bytes following sp. In that case, if sp < stk
then the call scanblock(sp, stk - sp) scanned too
many bytes (anything between the two pointers,
which pointed into different allocation blocks).
If sp > stk then stk - sp wrapped around.
On 32-bit, stk - sp is a uintptr (uint32) converted
to int64 in the call to scanblock, so a large (~4G)
but positive number. Scanblock would try to scan
that many bytes and eventually fault accessing
unmapped memory. On 64-bit, stk - sp is a uintptr (uint64)
promoted to int64 in the call to scanblock, so a negative
number. Scanblock would not scan anything, possibly
causing in-use blocks to be freed.
In short, 32-bit platforms would have seen either
ineffective garbage collection or crashes during garbage
collection, while 64-bit platforms would have seen
either ineffective or incorrect garbage collection.
You can see the invalid arguments to scanblock in the
stack traces in issue 1620.
Fixes #1620.
Fixes #1746.
R=iant, r
CC=golang-dev
https://golang.org/cl/4437075
2011-04-27 21:21:12 -06:00
|
|
|
guard = stk->stackguard;
|
2009-06-17 16:12:16 -06:00
|
|
|
stk = (Stktop*)stk->stackbase;
|
runtime: stack split + garbage collection bug
The g->sched.sp saved stack pointer and the
g->stackbase and g->stackguard stack bounds
can change even while "the world is stopped",
because a goroutine has to call functions (and
therefore might split its stack) when exiting a
system call to check whether the world is stopped
(and if so, wait until the world continues).
That means the garbage collector cannot access
those values safely (without a race) for goroutines
executing system calls. Instead, save a consistent
triple in g->gcsp, g->gcstack, g->gcguard during
entersyscall and have the garbage collector refer
to those.
The old code was occasionally seeing (because of
the race) an sp and stk that did not correspond to
each other, so that stk - sp was not the number of
stack bytes following sp. In that case, if sp < stk
then the call scanblock(sp, stk - sp) scanned too
many bytes (anything between the two pointers,
which pointed into different allocation blocks).
If sp > stk then stk - sp wrapped around.
On 32-bit, stk - sp is a uintptr (uint32) converted
to int64 in the call to scanblock, so a large (~4G)
but positive number. Scanblock would try to scan
that many bytes and eventually fault accessing
unmapped memory. On 64-bit, stk - sp is a uintptr (uint64)
promoted to int64 in the call to scanblock, so a negative
number. Scanblock would not scan anything, possibly
causing in-use blocks to be freed.
In short, 32-bit platforms would have seen either
ineffective garbage collection or crashes during garbage
collection, while 64-bit platforms would have seen
either ineffective or incorrect garbage collection.
You can see the invalid arguments to scanblock in the
stack traces in issue 1620.
Fixes #1620.
Fixes #1746.
R=iant, r
CC=golang-dev
https://golang.org/cl/4437075
2011-04-27 21:21:12 -06:00
|
|
|
n++;
|
2009-01-26 18:37:05 -07:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2011-02-02 21:03:47 -07:00
|
|
|
// Markfin calls scanblock on the blocks that have finalizers:
|
|
|
|
// the things pointed at cannot be freed until the finalizers have run.
|
2010-03-26 15:15:30 -06:00
|
|
|
static void
|
|
|
|
markfin(void *v)
|
|
|
|
{
|
|
|
|
uintptr size;
|
|
|
|
|
|
|
|
size = 0;
|
2011-02-02 21:03:47 -07:00
|
|
|
if(!runtime·mlookup(v, &v, &size, nil) || !runtime·blockspecial(v))
|
runtime: ,s/[a-zA-Z0-9_]+/runtime·&/g, almost
Prefix all external symbols in runtime by runtime·,
to avoid conflicts with possible symbols of the same
name in linked-in C libraries. The obvious conflicts
are printf, malloc, and free, but hide everything to
avoid future pain.
The symbols left alone are:
** known to cgo **
_cgo_free
_cgo_malloc
libcgo_thread_start
initcgo
ncgocall
** known to linker **
_rt0_$GOARCH
_rt0_$GOARCH_$GOOS
text
etext
data
end
pclntab
epclntab
symtab
esymtab
** known to C compiler **
_divv
_modv
_div64by32
etc (arch specific)
Tested on darwin/386, darwin/amd64, linux/386, linux/amd64.
Built (but not tested) for freebsd/386, freebsd/amd64, linux/arm, windows/386.
R=r, PeterGo
CC=golang-dev
https://golang.org/cl/2899041
2010-11-04 12:00:19 -06:00
|
|
|
runtime·throw("mark - finalizer inconsistency");
|
2011-02-02 21:03:47 -07:00
|
|
|
|
2010-03-26 15:15:30 -06:00
|
|
|
// do not mark the finalizer block itself. just mark the things it points at.
|
2010-09-07 07:57:22 -06:00
|
|
|
scanblock(v, size);
|
2010-03-26 15:15:30 -06:00
|
|
|
}
|
|
|
|
|
2011-02-02 21:03:47 -07:00
|
|
|
// Mark
|
2009-01-26 18:37:05 -07:00
|
|
|
static void
|
|
|
|
mark(void)
|
|
|
|
{
|
2010-01-06 20:24:11 -07:00
|
|
|
G *gp;
|
2009-01-26 18:37:05 -07:00
|
|
|
|
2009-12-07 16:52:14 -07:00
|
|
|
// mark data+bss.
|
runtime: ,s/[a-zA-Z0-9_]+/runtime·&/g, almost
Prefix all external symbols in runtime by runtime·,
to avoid conflicts with possible symbols of the same
name in linked-in C libraries. The obvious conflicts
are printf, malloc, and free, but hide everything to
avoid future pain.
The symbols left alone are:
** known to cgo **
_cgo_free
_cgo_malloc
libcgo_thread_start
initcgo
ncgocall
** known to linker **
_rt0_$GOARCH
_rt0_$GOARCH_$GOOS
text
etext
data
end
pclntab
epclntab
symtab
esymtab
** known to C compiler **
_divv
_modv
_div64by32
etc (arch specific)
Tested on darwin/386, darwin/amd64, linux/386, linux/amd64.
Built (but not tested) for freebsd/386, freebsd/amd64, linux/arm, windows/386.
R=r, PeterGo
CC=golang-dev
https://golang.org/cl/2899041
2010-11-04 12:00:19 -06:00
|
|
|
// skip runtime·mheap itself, which has no interesting pointers
|
2009-12-07 16:52:14 -07:00
|
|
|
// and is mostly zeroed and would not otherwise be paged in.
|
runtime: ,s/[a-zA-Z0-9_]+/runtime·&/g, almost
Prefix all external symbols in runtime by runtime·,
to avoid conflicts with possible symbols of the same
name in linked-in C libraries. The obvious conflicts
are printf, malloc, and free, but hide everything to
avoid future pain.
The symbols left alone are:
** known to cgo **
_cgo_free
_cgo_malloc
libcgo_thread_start
initcgo
ncgocall
** known to linker **
_rt0_$GOARCH
_rt0_$GOARCH_$GOOS
text
etext
data
end
pclntab
epclntab
symtab
esymtab
** known to C compiler **
_divv
_modv
_div64by32
etc (arch specific)
Tested on darwin/386, darwin/amd64, linux/386, linux/amd64.
Built (but not tested) for freebsd/386, freebsd/amd64, linux/arm, windows/386.
R=r, PeterGo
CC=golang-dev
https://golang.org/cl/2899041
2010-11-04 12:00:19 -06:00
|
|
|
scanblock(data, (byte*)&runtime·mheap - data);
|
|
|
|
scanblock((byte*)(&runtime·mheap+1), end - (byte*)(&runtime·mheap+1));
|
2009-01-26 18:37:05 -07:00
|
|
|
|
|
|
|
// mark stacks
|
runtime: ,s/[a-zA-Z0-9_]+/runtime·&/g, almost
Prefix all external symbols in runtime by runtime·,
to avoid conflicts with possible symbols of the same
name in linked-in C libraries. The obvious conflicts
are printf, malloc, and free, but hide everything to
avoid future pain.
The symbols left alone are:
** known to cgo **
_cgo_free
_cgo_malloc
libcgo_thread_start
initcgo
ncgocall
** known to linker **
_rt0_$GOARCH
_rt0_$GOARCH_$GOOS
text
etext
data
end
pclntab
epclntab
symtab
esymtab
** known to C compiler **
_divv
_modv
_div64by32
etc (arch specific)
Tested on darwin/386, darwin/amd64, linux/386, linux/amd64.
Built (but not tested) for freebsd/386, freebsd/amd64, linux/arm, windows/386.
R=r, PeterGo
CC=golang-dev
https://golang.org/cl/2899041
2010-11-04 12:00:19 -06:00
|
|
|
for(gp=runtime·allg; gp!=nil; gp=gp->alllink) {
|
2009-01-26 18:37:05 -07:00
|
|
|
switch(gp->status){
|
|
|
|
default:
|
runtime: ,s/[a-zA-Z0-9_]+/runtime·&/g, almost
Prefix all external symbols in runtime by runtime·,
to avoid conflicts with possible symbols of the same
name in linked-in C libraries. The obvious conflicts
are printf, malloc, and free, but hide everything to
avoid future pain.
The symbols left alone are:
** known to cgo **
_cgo_free
_cgo_malloc
libcgo_thread_start
initcgo
ncgocall
** known to linker **
_rt0_$GOARCH
_rt0_$GOARCH_$GOOS
text
etext
data
end
pclntab
epclntab
symtab
esymtab
** known to C compiler **
_divv
_modv
_div64by32
etc (arch specific)
Tested on darwin/386, darwin/amd64, linux/386, linux/amd64.
Built (but not tested) for freebsd/386, freebsd/amd64, linux/arm, windows/386.
R=r, PeterGo
CC=golang-dev
https://golang.org/cl/2899041
2010-11-04 12:00:19 -06:00
|
|
|
runtime·printf("unexpected G.status %d\n", gp->status);
|
|
|
|
runtime·throw("mark - bad status");
|
2009-01-26 18:37:05 -07:00
|
|
|
case Gdead:
|
|
|
|
break;
|
|
|
|
case Grunning:
|
2010-01-06 20:24:11 -07:00
|
|
|
if(gp != g)
|
runtime: ,s/[a-zA-Z0-9_]+/runtime·&/g, almost
Prefix all external symbols in runtime by runtime·,
to avoid conflicts with possible symbols of the same
name in linked-in C libraries. The obvious conflicts
are printf, malloc, and free, but hide everything to
avoid future pain.
The symbols left alone are:
** known to cgo **
_cgo_free
_cgo_malloc
libcgo_thread_start
initcgo
ncgocall
** known to linker **
_rt0_$GOARCH
_rt0_$GOARCH_$GOOS
text
etext
data
end
pclntab
epclntab
symtab
esymtab
** known to C compiler **
_divv
_modv
_div64by32
etc (arch specific)
Tested on darwin/386, darwin/amd64, linux/386, linux/amd64.
Built (but not tested) for freebsd/386, freebsd/amd64, linux/arm, windows/386.
R=r, PeterGo
CC=golang-dev
https://golang.org/cl/2899041
2010-11-04 12:00:19 -06:00
|
|
|
runtime·throw("mark - world not stopped");
|
2009-01-26 18:37:05 -07:00
|
|
|
scanstack(gp);
|
|
|
|
break;
|
|
|
|
case Grunnable:
|
|
|
|
case Gsyscall:
|
|
|
|
case Gwaiting:
|
|
|
|
scanstack(gp);
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2010-03-26 15:15:30 -06:00
|
|
|
// mark things pointed at by objects with finalizers
|
runtime: ,s/[a-zA-Z0-9_]+/runtime·&/g, almost
Prefix all external symbols in runtime by runtime·,
to avoid conflicts with possible symbols of the same
name in linked-in C libraries. The obvious conflicts
are printf, malloc, and free, but hide everything to
avoid future pain.
The symbols left alone are:
** known to cgo **
_cgo_free
_cgo_malloc
libcgo_thread_start
initcgo
ncgocall
** known to linker **
_rt0_$GOARCH
_rt0_$GOARCH_$GOOS
text
etext
data
end
pclntab
epclntab
symtab
esymtab
** known to C compiler **
_divv
_modv
_div64by32
etc (arch specific)
Tested on darwin/386, darwin/amd64, linux/386, linux/amd64.
Built (but not tested) for freebsd/386, freebsd/amd64, linux/arm, windows/386.
R=r, PeterGo
CC=golang-dev
https://golang.org/cl/2899041
2010-11-04 12:00:19 -06:00
|
|
|
runtime·walkfintab(markfin);
|
2010-03-26 15:15:30 -06:00
|
|
|
}
|
2010-02-03 17:31:34 -07:00
|
|
|
|
2011-02-02 21:03:47 -07:00
|
|
|
// Sweep frees or calls finalizers for blocks not marked in the mark phase.
|
|
|
|
// It clears the mark bits in preparation for the next GC round.
|
2010-02-03 17:31:34 -07:00
|
|
|
static void
|
2011-02-02 21:03:47 -07:00
|
|
|
sweep(void)
|
2010-02-03 17:31:34 -07:00
|
|
|
{
|
2011-02-02 21:03:47 -07:00
|
|
|
MSpan *s;
|
|
|
|
int32 cl, n, npages;
|
|
|
|
uintptr size;
|
2010-02-10 15:59:39 -07:00
|
|
|
byte *p;
|
|
|
|
MCache *c;
|
2010-03-26 15:15:30 -06:00
|
|
|
Finalizer *f;
|
2010-02-03 17:31:34 -07:00
|
|
|
|
2011-02-02 21:03:47 -07:00
|
|
|
for(s = runtime·mheap.allspans; s != nil; s = s->allnext) {
|
|
|
|
if(s->state != MSpanInUse)
|
|
|
|
continue;
|
|
|
|
|
|
|
|
p = (byte*)(s->start << PageShift);
|
|
|
|
cl = s->sizeclass;
|
|
|
|
if(cl == 0) {
|
|
|
|
size = s->npages<<PageShift;
|
|
|
|
n = 1;
|
|
|
|
} else {
|
|
|
|
// Chunk full of small blocks.
|
|
|
|
size = runtime·class_to_size[cl];
|
|
|
|
npages = runtime·class_to_allocnpages[cl];
|
|
|
|
n = (npages << PageShift) / size;
|
2010-02-03 17:31:34 -07:00
|
|
|
}
|
2011-02-02 21:03:47 -07:00
|
|
|
|
|
|
|
// sweep through n objects of given size starting at p.
|
|
|
|
for(; n > 0; n--, p += size) {
|
|
|
|
uintptr off, *bitp, shift, bits;
|
2010-02-10 15:59:39 -07:00
|
|
|
|
2011-02-02 21:03:47 -07:00
|
|
|
off = (uintptr*)p - (uintptr*)runtime·mheap.arena_start;
|
|
|
|
bitp = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1;
|
|
|
|
shift = off % wordsPerBitmapWord;
|
|
|
|
bits = *bitp>>shift;
|
|
|
|
|
|
|
|
if((bits & bitAllocated) == 0)
|
|
|
|
continue;
|
|
|
|
|
|
|
|
if((bits & bitMarked) != 0) {
|
|
|
|
*bitp &= ~(bitMarked<<shift);
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
|
|
|
if((bits & bitSpecial) != 0) {
|
|
|
|
// Special means it has a finalizer or is being profiled.
|
|
|
|
f = runtime·getfinalizer(p, 1);
|
|
|
|
if(f != nil) {
|
|
|
|
f->arg = p;
|
|
|
|
f->next = finq;
|
|
|
|
finq = f;
|
|
|
|
continue;
|
|
|
|
}
|
runtime: ,s/[a-zA-Z0-9_]+/runtime·&/g, almost
Prefix all external symbols in runtime by runtime·,
to avoid conflicts with possible symbols of the same
name in linked-in C libraries. The obvious conflicts
are printf, malloc, and free, but hide everything to
avoid future pain.
The symbols left alone are:
** known to cgo **
_cgo_free
_cgo_malloc
libcgo_thread_start
initcgo
ncgocall
** known to linker **
_rt0_$GOARCH
_rt0_$GOARCH_$GOOS
text
etext
data
end
pclntab
epclntab
symtab
esymtab
** known to C compiler **
_divv
_modv
_div64by32
etc (arch specific)
Tested on darwin/386, darwin/amd64, linux/386, linux/amd64.
Built (but not tested) for freebsd/386, freebsd/amd64, linux/arm, windows/386.
R=r, PeterGo
CC=golang-dev
https://golang.org/cl/2899041
2010-11-04 12:00:19 -06:00
|
|
|
runtime·MProf_Free(p, size);
|
2011-02-02 21:03:47 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
// Mark freed; restore block boundary bit.
|
|
|
|
*bitp = (*bitp & ~(bitMask<<shift)) | (bitBlockBoundary<<shift);
|
|
|
|
|
|
|
|
if(s->sizeclass == 0) {
|
|
|
|
// Free large span.
|
|
|
|
runtime·unmarkspan(p, 1<<PageShift);
|
|
|
|
*(uintptr*)p = 1; // needs zeroing
|
|
|
|
runtime·MHeap_Free(&runtime·mheap, s, 1);
|
|
|
|
} else {
|
|
|
|
// Free small object.
|
|
|
|
c = m->mcache;
|
|
|
|
if(size > sizeof(uintptr))
|
|
|
|
((uintptr*)p)[1] = 1; // mark as "needs to be zeroed"
|
|
|
|
mstats.by_size[s->sizeclass].nfree++;
|
|
|
|
runtime·MCache_Free(c, p, s->sizeclass, size);
|
|
|
|
}
|
2010-02-10 15:59:39 -07:00
|
|
|
mstats.alloc -= size;
|
2011-01-19 11:41:42 -07:00
|
|
|
mstats.nfree++;
|
2009-01-26 18:37:05 -07:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Semaphore, not Lock, so that the goroutine
|
|
|
|
// reschedules when there is contention rather
|
|
|
|
// than spinning.
|
|
|
|
static uint32 gcsema = 1;
|
|
|
|
|
|
|
|
// Initialized from $GOGC. GOGC=off means no gc.
|
|
|
|
//
|
|
|
|
// Next gc is after we've allocated an extra amount of
|
|
|
|
// memory proportional to the amount already in use.
|
|
|
|
// If gcpercent=100 and we're using 4M, we'll gc again
|
|
|
|
// when we get to 8M. This keeps the gc cost in linear
|
|
|
|
// proportion to the allocation cost. Adjusting gcpercent
|
|
|
|
// just changes the linear constant (and also the amount of
|
|
|
|
// extra memory used).
|
|
|
|
static int32 gcpercent = -2;
|
|
|
|
|
2010-03-08 15:15:44 -07:00
|
|
|
static void
|
|
|
|
stealcache(void)
|
|
|
|
{
|
|
|
|
M *m;
|
|
|
|
|
runtime: ,s/[a-zA-Z0-9_]+/runtime·&/g, almost
Prefix all external symbols in runtime by runtime·,
to avoid conflicts with possible symbols of the same
name in linked-in C libraries. The obvious conflicts
are printf, malloc, and free, but hide everything to
avoid future pain.
The symbols left alone are:
** known to cgo **
_cgo_free
_cgo_malloc
libcgo_thread_start
initcgo
ncgocall
** known to linker **
_rt0_$GOARCH
_rt0_$GOARCH_$GOOS
text
etext
data
end
pclntab
epclntab
symtab
esymtab
** known to C compiler **
_divv
_modv
_div64by32
etc (arch specific)
Tested on darwin/386, darwin/amd64, linux/386, linux/amd64.
Built (but not tested) for freebsd/386, freebsd/amd64, linux/arm, windows/386.
R=r, PeterGo
CC=golang-dev
https://golang.org/cl/2899041
2010-11-04 12:00:19 -06:00
|
|
|
for(m=runtime·allm; m; m=m->alllink)
|
|
|
|
runtime·MCache_ReleaseAll(m->mcache);
|
2010-03-08 15:15:44 -07:00
|
|
|
}
|
|
|
|
|
2010-09-07 07:57:22 -06:00
|
|
|
static void
|
|
|
|
cachestats(void)
|
|
|
|
{
|
|
|
|
M *m;
|
|
|
|
MCache *c;
|
|
|
|
|
runtime: ,s/[a-zA-Z0-9_]+/runtime·&/g, almost
Prefix all external symbols in runtime by runtime·,
to avoid conflicts with possible symbols of the same
name in linked-in C libraries. The obvious conflicts
are printf, malloc, and free, but hide everything to
avoid future pain.
The symbols left alone are:
** known to cgo **
_cgo_free
_cgo_malloc
libcgo_thread_start
initcgo
ncgocall
** known to linker **
_rt0_$GOARCH
_rt0_$GOARCH_$GOOS
text
etext
data
end
pclntab
epclntab
symtab
esymtab
** known to C compiler **
_divv
_modv
_div64by32
etc (arch specific)
Tested on darwin/386, darwin/amd64, linux/386, linux/amd64.
Built (but not tested) for freebsd/386, freebsd/amd64, linux/arm, windows/386.
R=r, PeterGo
CC=golang-dev
https://golang.org/cl/2899041
2010-11-04 12:00:19 -06:00
|
|
|
for(m=runtime·allm; m; m=m->alllink) {
|
2010-09-07 07:57:22 -06:00
|
|
|
c = m->mcache;
|
|
|
|
mstats.heap_alloc += c->local_alloc;
|
|
|
|
c->local_alloc = 0;
|
|
|
|
mstats.heap_objects += c->local_objects;
|
|
|
|
c->local_objects = 0;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2009-01-26 18:37:05 -07:00
|
|
|
void
|
runtime: ,s/[a-zA-Z0-9_]+/runtime·&/g, almost
Prefix all external symbols in runtime by runtime·,
to avoid conflicts with possible symbols of the same
name in linked-in C libraries. The obvious conflicts
are printf, malloc, and free, but hide everything to
avoid future pain.
The symbols left alone are:
** known to cgo **
_cgo_free
_cgo_malloc
libcgo_thread_start
initcgo
ncgocall
** known to linker **
_rt0_$GOARCH
_rt0_$GOARCH_$GOOS
text
etext
data
end
pclntab
epclntab
symtab
esymtab
** known to C compiler **
_divv
_modv
_div64by32
etc (arch specific)
Tested on darwin/386, darwin/amd64, linux/386, linux/amd64.
Built (but not tested) for freebsd/386, freebsd/amd64, linux/arm, windows/386.
R=r, PeterGo
CC=golang-dev
https://golang.org/cl/2899041
2010-11-04 12:00:19 -06:00
|
|
|
runtime·gc(int32 force)
|
2009-01-26 18:37:05 -07:00
|
|
|
{
|
2011-02-02 21:03:47 -07:00
|
|
|
int64 t0, t1, t2, t3;
|
|
|
|
uint64 heap0, heap1, obj0, obj1;
|
2009-01-26 18:37:05 -07:00
|
|
|
byte *p;
|
2010-03-26 15:15:30 -06:00
|
|
|
Finalizer *fp;
|
2009-01-26 18:37:05 -07:00
|
|
|
|
|
|
|
// The gc is turned off (via enablegc) until
|
|
|
|
// the bootstrap has completed.
|
|
|
|
// Also, malloc gets called in the guts
|
|
|
|
// of a number of libraries that might be
|
|
|
|
// holding locks. To avoid priority inversion
|
|
|
|
// problems, don't bother trying to run gc
|
|
|
|
// while holding a lock. The next mallocgc
|
|
|
|
// without a lock will do the gc instead.
|
runtime: ,s/[a-zA-Z0-9_]+/runtime·&/g, almost
Prefix all external symbols in runtime by runtime·,
to avoid conflicts with possible symbols of the same
name in linked-in C libraries. The obvious conflicts
are printf, malloc, and free, but hide everything to
avoid future pain.
The symbols left alone are:
** known to cgo **
_cgo_free
_cgo_malloc
libcgo_thread_start
initcgo
ncgocall
** known to linker **
_rt0_$GOARCH
_rt0_$GOARCH_$GOOS
text
etext
data
end
pclntab
epclntab
symtab
esymtab
** known to C compiler **
_divv
_modv
_div64by32
etc (arch specific)
Tested on darwin/386, darwin/amd64, linux/386, linux/amd64.
Built (but not tested) for freebsd/386, freebsd/amd64, linux/arm, windows/386.
R=r, PeterGo
CC=golang-dev
https://golang.org/cl/2899041
2010-11-04 12:00:19 -06:00
|
|
|
if(!mstats.enablegc || m->locks > 0 || runtime·panicking)
|
2009-01-26 18:37:05 -07:00
|
|
|
return;
|
|
|
|
|
|
|
|
if(gcpercent == -2) { // first time through
|
runtime: ,s/[a-zA-Z0-9_]+/runtime·&/g, almost
Prefix all external symbols in runtime by runtime·,
to avoid conflicts with possible symbols of the same
name in linked-in C libraries. The obvious conflicts
are printf, malloc, and free, but hide everything to
avoid future pain.
The symbols left alone are:
** known to cgo **
_cgo_free
_cgo_malloc
libcgo_thread_start
initcgo
ncgocall
** known to linker **
_rt0_$GOARCH
_rt0_$GOARCH_$GOOS
text
etext
data
end
pclntab
epclntab
symtab
esymtab
** known to C compiler **
_divv
_modv
_div64by32
etc (arch specific)
Tested on darwin/386, darwin/amd64, linux/386, linux/amd64.
Built (but not tested) for freebsd/386, freebsd/amd64, linux/arm, windows/386.
R=r, PeterGo
CC=golang-dev
https://golang.org/cl/2899041
2010-11-04 12:00:19 -06:00
|
|
|
p = runtime·getenv("GOGC");
|
2009-01-26 18:37:05 -07:00
|
|
|
if(p == nil || p[0] == '\0')
|
|
|
|
gcpercent = 100;
|
runtime: ,s/[a-zA-Z0-9_]+/runtime·&/g, almost
Prefix all external symbols in runtime by runtime·,
to avoid conflicts with possible symbols of the same
name in linked-in C libraries. The obvious conflicts
are printf, malloc, and free, but hide everything to
avoid future pain.
The symbols left alone are:
** known to cgo **
_cgo_free
_cgo_malloc
libcgo_thread_start
initcgo
ncgocall
** known to linker **
_rt0_$GOARCH
_rt0_$GOARCH_$GOOS
text
etext
data
end
pclntab
epclntab
symtab
esymtab
** known to C compiler **
_divv
_modv
_div64by32
etc (arch specific)
Tested on darwin/386, darwin/amd64, linux/386, linux/amd64.
Built (but not tested) for freebsd/386, freebsd/amd64, linux/arm, windows/386.
R=r, PeterGo
CC=golang-dev
https://golang.org/cl/2899041
2010-11-04 12:00:19 -06:00
|
|
|
else if(runtime·strcmp(p, (byte*)"off") == 0)
|
2009-01-26 18:37:05 -07:00
|
|
|
gcpercent = -1;
|
|
|
|
else
|
runtime: ,s/[a-zA-Z0-9_]+/runtime·&/g, almost
Prefix all external symbols in runtime by runtime·,
to avoid conflicts with possible symbols of the same
name in linked-in C libraries. The obvious conflicts
are printf, malloc, and free, but hide everything to
avoid future pain.
The symbols left alone are:
** known to cgo **
_cgo_free
_cgo_malloc
libcgo_thread_start
initcgo
ncgocall
** known to linker **
_rt0_$GOARCH
_rt0_$GOARCH_$GOOS
text
etext
data
end
pclntab
epclntab
symtab
esymtab
** known to C compiler **
_divv
_modv
_div64by32
etc (arch specific)
Tested on darwin/386, darwin/amd64, linux/386, linux/amd64.
Built (but not tested) for freebsd/386, freebsd/amd64, linux/arm, windows/386.
R=r, PeterGo
CC=golang-dev
https://golang.org/cl/2899041
2010-11-04 12:00:19 -06:00
|
|
|
gcpercent = runtime·atoi(p);
|
2011-02-02 21:03:47 -07:00
|
|
|
|
|
|
|
p = runtime·getenv("GOGCTRACE");
|
|
|
|
if(p != nil)
|
|
|
|
gctrace = runtime·atoi(p);
|
2009-01-26 18:37:05 -07:00
|
|
|
}
|
2009-06-05 11:59:37 -06:00
|
|
|
if(gcpercent < 0)
|
2009-01-26 18:37:05 -07:00
|
|
|
return;
|
|
|
|
|
runtime: ,s/[a-zA-Z0-9_]+/runtime·&/g, almost
Prefix all external symbols in runtime by runtime·,
to avoid conflicts with possible symbols of the same
name in linked-in C libraries. The obvious conflicts
are printf, malloc, and free, but hide everything to
avoid future pain.
The symbols left alone are:
** known to cgo **
_cgo_free
_cgo_malloc
libcgo_thread_start
initcgo
ncgocall
** known to linker **
_rt0_$GOARCH
_rt0_$GOARCH_$GOOS
text
etext
data
end
pclntab
epclntab
symtab
esymtab
** known to C compiler **
_divv
_modv
_div64by32
etc (arch specific)
Tested on darwin/386, darwin/amd64, linux/386, linux/amd64.
Built (but not tested) for freebsd/386, freebsd/amd64, linux/arm, windows/386.
R=r, PeterGo
CC=golang-dev
https://golang.org/cl/2899041
2010-11-04 12:00:19 -06:00
|
|
|
runtime·semacquire(&gcsema);
|
2011-02-02 21:03:47 -07:00
|
|
|
if(!force && mstats.heap_alloc < mstats.next_gc) {
|
|
|
|
runtime·semrelease(&gcsema);
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
runtime: ,s/[a-zA-Z0-9_]+/runtime·&/g, almost
Prefix all external symbols in runtime by runtime·,
to avoid conflicts with possible symbols of the same
name in linked-in C libraries. The obvious conflicts
are printf, malloc, and free, but hide everything to
avoid future pain.
The symbols left alone are:
** known to cgo **
_cgo_free
_cgo_malloc
libcgo_thread_start
initcgo
ncgocall
** known to linker **
_rt0_$GOARCH
_rt0_$GOARCH_$GOOS
text
etext
data
end
pclntab
epclntab
symtab
esymtab
** known to C compiler **
_divv
_modv
_div64by32
etc (arch specific)
Tested on darwin/386, darwin/amd64, linux/386, linux/amd64.
Built (but not tested) for freebsd/386, freebsd/amd64, linux/arm, windows/386.
R=r, PeterGo
CC=golang-dev
https://golang.org/cl/2899041
2010-11-04 12:00:19 -06:00
|
|
|
t0 = runtime·nanotime();
|
2011-02-02 21:03:47 -07:00
|
|
|
nlookup = 0;
|
|
|
|
nsizelookup = 0;
|
|
|
|
naddrlookup = 0;
|
|
|
|
|
2009-08-14 21:33:20 -06:00
|
|
|
m->gcing = 1;
|
runtime: ,s/[a-zA-Z0-9_]+/runtime·&/g, almost
Prefix all external symbols in runtime by runtime·,
to avoid conflicts with possible symbols of the same
name in linked-in C libraries. The obvious conflicts
are printf, malloc, and free, but hide everything to
avoid future pain.
The symbols left alone are:
** known to cgo **
_cgo_free
_cgo_malloc
libcgo_thread_start
initcgo
ncgocall
** known to linker **
_rt0_$GOARCH
_rt0_$GOARCH_$GOOS
text
etext
data
end
pclntab
epclntab
symtab
esymtab
** known to C compiler **
_divv
_modv
_div64by32
etc (arch specific)
Tested on darwin/386, darwin/amd64, linux/386, linux/amd64.
Built (but not tested) for freebsd/386, freebsd/amd64, linux/arm, windows/386.
R=r, PeterGo
CC=golang-dev
https://golang.org/cl/2899041
2010-11-04 12:00:19 -06:00
|
|
|
runtime·stoptheworld();
|
|
|
|
if(runtime·mheap.Lock.key != 0)
|
|
|
|
runtime·throw("runtime·mheap locked during gc");
|
2011-02-02 21:03:47 -07:00
|
|
|
|
|
|
|
cachestats();
|
|
|
|
heap0 = mstats.heap_alloc;
|
|
|
|
obj0 = mstats.nmalloc - mstats.nfree;
|
|
|
|
|
|
|
|
mark();
|
|
|
|
t1 = runtime·nanotime();
|
|
|
|
sweep();
|
|
|
|
t2 = runtime·nanotime();
|
|
|
|
stealcache();
|
|
|
|
|
|
|
|
mstats.next_gc = mstats.heap_alloc+mstats.heap_alloc*gcpercent/100;
|
2009-06-15 22:31:56 -06:00
|
|
|
m->gcing = 0;
|
2010-02-10 01:00:12 -07:00
|
|
|
|
2010-02-03 17:31:34 -07:00
|
|
|
m->locks++; // disable gc during the mallocs in newproc
|
2010-03-26 15:15:30 -06:00
|
|
|
fp = finq;
|
|
|
|
if(fp != nil) {
|
|
|
|
// kick off or wake up goroutine to run queued finalizers
|
|
|
|
if(fing == nil)
|
2011-03-02 11:42:02 -07:00
|
|
|
fing = runtime·newproc1((byte*)runfinq, nil, 0, 0, runtime·gc);
|
2010-04-07 21:38:02 -06:00
|
|
|
else if(fingwait) {
|
|
|
|
fingwait = 0;
|
runtime: ,s/[a-zA-Z0-9_]+/runtime·&/g, almost
Prefix all external symbols in runtime by runtime·,
to avoid conflicts with possible symbols of the same
name in linked-in C libraries. The obvious conflicts
are printf, malloc, and free, but hide everything to
avoid future pain.
The symbols left alone are:
** known to cgo **
_cgo_free
_cgo_malloc
libcgo_thread_start
initcgo
ncgocall
** known to linker **
_rt0_$GOARCH
_rt0_$GOARCH_$GOOS
text
etext
data
end
pclntab
epclntab
symtab
esymtab
** known to C compiler **
_divv
_modv
_div64by32
etc (arch specific)
Tested on darwin/386, darwin/amd64, linux/386, linux/amd64.
Built (but not tested) for freebsd/386, freebsd/amd64, linux/arm, windows/386.
R=r, PeterGo
CC=golang-dev
https://golang.org/cl/2899041
2010-11-04 12:00:19 -06:00
|
|
|
runtime·ready(fing);
|
2010-04-07 21:38:02 -06:00
|
|
|
}
|
2010-02-03 17:31:34 -07:00
|
|
|
}
|
|
|
|
m->locks--;
|
|
|
|
|
2011-02-02 21:03:47 -07:00
|
|
|
cachestats();
|
|
|
|
heap1 = mstats.heap_alloc;
|
|
|
|
obj1 = mstats.nmalloc - mstats.nfree;
|
|
|
|
|
|
|
|
t3 = runtime·nanotime();
|
|
|
|
mstats.pause_ns[mstats.numgc%nelem(mstats.pause_ns)] = t3 - t0;
|
|
|
|
mstats.pause_total_ns += t3 - t0;
|
2010-02-08 15:32:22 -07:00
|
|
|
mstats.numgc++;
|
|
|
|
if(mstats.debuggc)
|
2011-02-02 21:03:47 -07:00
|
|
|
runtime·printf("pause %D\n", t3-t0);
|
|
|
|
|
|
|
|
if(gctrace) {
|
|
|
|
runtime·printf("gc%d: %D+%D+%D ms %D -> %D MB %D -> %D (%D-%D) objects %D pointer lookups (%D size, %D addr)\n",
|
|
|
|
mstats.numgc, (t1-t0)/1000000, (t2-t1)/1000000, (t3-t2)/1000000,
|
|
|
|
heap0>>20, heap1>>20, obj0, obj1,
|
|
|
|
mstats.nmalloc, mstats.nfree,
|
|
|
|
nlookup, nsizelookup, naddrlookup);
|
|
|
|
}
|
|
|
|
|
runtime: ,s/[a-zA-Z0-9_]+/runtime·&/g, almost
Prefix all external symbols in runtime by runtime·,
to avoid conflicts with possible symbols of the same
name in linked-in C libraries. The obvious conflicts
are printf, malloc, and free, but hide everything to
avoid future pain.
The symbols left alone are:
** known to cgo **
_cgo_free
_cgo_malloc
libcgo_thread_start
initcgo
ncgocall
** known to linker **
_rt0_$GOARCH
_rt0_$GOARCH_$GOOS
text
etext
data
end
pclntab
epclntab
symtab
esymtab
** known to C compiler **
_divv
_modv
_div64by32
etc (arch specific)
Tested on darwin/386, darwin/amd64, linux/386, linux/amd64.
Built (but not tested) for freebsd/386, freebsd/amd64, linux/arm, windows/386.
R=r, PeterGo
CC=golang-dev
https://golang.org/cl/2899041
2010-11-04 12:00:19 -06:00
|
|
|
runtime·semrelease(&gcsema);
|
|
|
|
runtime·starttheworld();
|
2010-03-26 15:15:30 -06:00
|
|
|
|
|
|
|
// give the queued finalizers, if any, a chance to run
|
|
|
|
if(fp != nil)
|
runtime: ,s/[a-zA-Z0-9_]+/runtime·&/g, almost
Prefix all external symbols in runtime by runtime·,
to avoid conflicts with possible symbols of the same
name in linked-in C libraries. The obvious conflicts
are printf, malloc, and free, but hide everything to
avoid future pain.
The symbols left alone are:
** known to cgo **
_cgo_free
_cgo_malloc
libcgo_thread_start
initcgo
ncgocall
** known to linker **
_rt0_$GOARCH
_rt0_$GOARCH_$GOOS
text
etext
data
end
pclntab
epclntab
symtab
esymtab
** known to C compiler **
_divv
_modv
_div64by32
etc (arch specific)
Tested on darwin/386, darwin/amd64, linux/386, linux/amd64.
Built (but not tested) for freebsd/386, freebsd/amd64, linux/arm, windows/386.
R=r, PeterGo
CC=golang-dev
https://golang.org/cl/2899041
2010-11-04 12:00:19 -06:00
|
|
|
runtime·gosched();
|
2011-02-02 21:03:47 -07:00
|
|
|
|
|
|
|
if(gctrace > 1 && !force)
|
|
|
|
runtime·gc(1);
|
2010-03-26 15:15:30 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
static void
|
|
|
|
runfinq(void)
|
|
|
|
{
|
|
|
|
Finalizer *f, *next;
|
|
|
|
byte *frame;
|
|
|
|
|
|
|
|
for(;;) {
|
|
|
|
// There's no need for a lock in this section
|
|
|
|
// because it only conflicts with the garbage
|
|
|
|
// collector, and the garbage collector only
|
|
|
|
// runs when everyone else is stopped, and
|
|
|
|
// runfinq only stops at the gosched() or
|
|
|
|
// during the calls in the for loop.
|
|
|
|
f = finq;
|
|
|
|
finq = nil;
|
|
|
|
if(f == nil) {
|
2010-04-07 21:38:02 -06:00
|
|
|
fingwait = 1;
|
2010-03-26 15:15:30 -06:00
|
|
|
g->status = Gwaiting;
|
runtime: ,s/[a-zA-Z0-9_]+/runtime·&/g, almost
Prefix all external symbols in runtime by runtime·,
to avoid conflicts with possible symbols of the same
name in linked-in C libraries. The obvious conflicts
are printf, malloc, and free, but hide everything to
avoid future pain.
The symbols left alone are:
** known to cgo **
_cgo_free
_cgo_malloc
libcgo_thread_start
initcgo
ncgocall
** known to linker **
_rt0_$GOARCH
_rt0_$GOARCH_$GOOS
text
etext
data
end
pclntab
epclntab
symtab
esymtab
** known to C compiler **
_divv
_modv
_div64by32
etc (arch specific)
Tested on darwin/386, darwin/amd64, linux/386, linux/amd64.
Built (but not tested) for freebsd/386, freebsd/amd64, linux/arm, windows/386.
R=r, PeterGo
CC=golang-dev
https://golang.org/cl/2899041
2010-11-04 12:00:19 -06:00
|
|
|
runtime·gosched();
|
2010-03-26 15:15:30 -06:00
|
|
|
continue;
|
|
|
|
}
|
|
|
|
for(; f; f=next) {
|
|
|
|
next = f->next;
|
runtime: ,s/[a-zA-Z0-9_]+/runtime·&/g, almost
Prefix all external symbols in runtime by runtime·,
to avoid conflicts with possible symbols of the same
name in linked-in C libraries. The obvious conflicts
are printf, malloc, and free, but hide everything to
avoid future pain.
The symbols left alone are:
** known to cgo **
_cgo_free
_cgo_malloc
libcgo_thread_start
initcgo
ncgocall
** known to linker **
_rt0_$GOARCH
_rt0_$GOARCH_$GOOS
text
etext
data
end
pclntab
epclntab
symtab
esymtab
** known to C compiler **
_divv
_modv
_div64by32
etc (arch specific)
Tested on darwin/386, darwin/amd64, linux/386, linux/amd64.
Built (but not tested) for freebsd/386, freebsd/amd64, linux/arm, windows/386.
R=r, PeterGo
CC=golang-dev
https://golang.org/cl/2899041
2010-11-04 12:00:19 -06:00
|
|
|
frame = runtime·mal(sizeof(uintptr) + f->nret);
|
2010-03-26 15:15:30 -06:00
|
|
|
*(void**)frame = f->arg;
|
|
|
|
reflect·call((byte*)f->fn, frame, sizeof(uintptr) + f->nret);
|
runtime: ,s/[a-zA-Z0-9_]+/runtime·&/g, almost
Prefix all external symbols in runtime by runtime·,
to avoid conflicts with possible symbols of the same
name in linked-in C libraries. The obvious conflicts
are printf, malloc, and free, but hide everything to
avoid future pain.
The symbols left alone are:
** known to cgo **
_cgo_free
_cgo_malloc
libcgo_thread_start
initcgo
ncgocall
** known to linker **
_rt0_$GOARCH
_rt0_$GOARCH_$GOOS
text
etext
data
end
pclntab
epclntab
symtab
esymtab
** known to C compiler **
_divv
_modv
_div64by32
etc (arch specific)
Tested on darwin/386, darwin/amd64, linux/386, linux/amd64.
Built (but not tested) for freebsd/386, freebsd/amd64, linux/arm, windows/386.
R=r, PeterGo
CC=golang-dev
https://golang.org/cl/2899041
2010-11-04 12:00:19 -06:00
|
|
|
runtime·free(frame);
|
2010-03-26 15:15:30 -06:00
|
|
|
f->fn = nil;
|
|
|
|
f->arg = nil;
|
|
|
|
f->next = nil;
|
runtime: ,s/[a-zA-Z0-9_]+/runtime·&/g, almost
Prefix all external symbols in runtime by runtime·,
to avoid conflicts with possible symbols of the same
name in linked-in C libraries. The obvious conflicts
are printf, malloc, and free, but hide everything to
avoid future pain.
The symbols left alone are:
** known to cgo **
_cgo_free
_cgo_malloc
libcgo_thread_start
initcgo
ncgocall
** known to linker **
_rt0_$GOARCH
_rt0_$GOARCH_$GOOS
text
etext
data
end
pclntab
epclntab
symtab
esymtab
** known to C compiler **
_divv
_modv
_div64by32
etc (arch specific)
Tested on darwin/386, darwin/amd64, linux/386, linux/amd64.
Built (but not tested) for freebsd/386, freebsd/amd64, linux/arm, windows/386.
R=r, PeterGo
CC=golang-dev
https://golang.org/cl/2899041
2010-11-04 12:00:19 -06:00
|
|
|
runtime·free(f);
|
2010-03-26 15:15:30 -06:00
|
|
|
}
|
runtime: ,s/[a-zA-Z0-9_]+/runtime·&/g, almost
Prefix all external symbols in runtime by runtime·,
to avoid conflicts with possible symbols of the same
name in linked-in C libraries. The obvious conflicts
are printf, malloc, and free, but hide everything to
avoid future pain.
The symbols left alone are:
** known to cgo **
_cgo_free
_cgo_malloc
libcgo_thread_start
initcgo
ncgocall
** known to linker **
_rt0_$GOARCH
_rt0_$GOARCH_$GOOS
text
etext
data
end
pclntab
epclntab
symtab
esymtab
** known to C compiler **
_divv
_modv
_div64by32
etc (arch specific)
Tested on darwin/386, darwin/amd64, linux/386, linux/amd64.
Built (but not tested) for freebsd/386, freebsd/amd64, linux/arm, windows/386.
R=r, PeterGo
CC=golang-dev
https://golang.org/cl/2899041
2010-11-04 12:00:19 -06:00
|
|
|
runtime·gc(1); // trigger another gc to clean up the finalized objects, if possible
|
2010-03-26 15:15:30 -06:00
|
|
|
}
|
2009-01-26 18:37:05 -07:00
|
|
|
}
|
2011-02-02 21:03:47 -07:00
|
|
|
|
|
|
|
// mark the block at v of size n as allocated.
|
|
|
|
// If noptr is true, mark it as having no pointers.
|
|
|
|
void
|
|
|
|
runtime·markallocated(void *v, uintptr n, bool noptr)
|
|
|
|
{
|
2011-02-16 11:21:20 -07:00
|
|
|
uintptr *b, obits, bits, off, shift;
|
2011-02-02 21:03:47 -07:00
|
|
|
|
|
|
|
if(0)
|
|
|
|
runtime·printf("markallocated %p+%p\n", v, n);
|
|
|
|
|
|
|
|
if((byte*)v+n > (byte*)runtime·mheap.arena_used || (byte*)v < runtime·mheap.arena_start)
|
|
|
|
runtime·throw("markallocated: bad pointer");
|
|
|
|
|
|
|
|
off = (uintptr*)v - (uintptr*)runtime·mheap.arena_start; // word offset
|
|
|
|
b = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1;
|
|
|
|
shift = off % wordsPerBitmapWord;
|
|
|
|
|
2011-02-16 11:21:20 -07:00
|
|
|
for(;;) {
|
|
|
|
obits = *b;
|
|
|
|
bits = (obits & ~(bitMask<<shift)) | (bitAllocated<<shift);
|
|
|
|
if(noptr)
|
|
|
|
bits |= bitNoPointers<<shift;
|
|
|
|
if(runtime·gomaxprocs == 1) {
|
|
|
|
*b = bits;
|
|
|
|
break;
|
|
|
|
} else {
|
|
|
|
// gomaxprocs > 1: use atomic op
|
|
|
|
if(runtime·casp((void**)b, (void*)obits, (void*)bits))
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
2011-02-02 21:03:47 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
// mark the block at v of size n as freed.
|
|
|
|
void
|
|
|
|
runtime·markfreed(void *v, uintptr n)
|
|
|
|
{
|
2011-02-16 11:21:20 -07:00
|
|
|
uintptr *b, obits, bits, off, shift;
|
2011-02-02 21:03:47 -07:00
|
|
|
|
|
|
|
if(0)
|
|
|
|
runtime·printf("markallocated %p+%p\n", v, n);
|
|
|
|
|
|
|
|
if((byte*)v+n > (byte*)runtime·mheap.arena_used || (byte*)v < runtime·mheap.arena_start)
|
|
|
|
runtime·throw("markallocated: bad pointer");
|
|
|
|
|
|
|
|
off = (uintptr*)v - (uintptr*)runtime·mheap.arena_start; // word offset
|
|
|
|
b = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1;
|
|
|
|
shift = off % wordsPerBitmapWord;
|
|
|
|
|
2011-02-16 11:21:20 -07:00
|
|
|
for(;;) {
|
|
|
|
obits = *b;
|
|
|
|
bits = (obits & ~(bitMask<<shift)) | (bitBlockBoundary<<shift);
|
|
|
|
if(runtime·gomaxprocs == 1) {
|
|
|
|
*b = bits;
|
|
|
|
break;
|
|
|
|
} else {
|
|
|
|
// gomaxprocs > 1: use atomic op
|
|
|
|
if(runtime·casp((void**)b, (void*)obits, (void*)bits))
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
2011-02-02 21:03:47 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
// check that the block at v of size n is marked freed.
|
|
|
|
void
|
|
|
|
runtime·checkfreed(void *v, uintptr n)
|
|
|
|
{
|
|
|
|
uintptr *b, bits, off, shift;
|
|
|
|
|
|
|
|
if(!runtime·checking)
|
|
|
|
return;
|
|
|
|
|
|
|
|
if((byte*)v+n > (byte*)runtime·mheap.arena_used || (byte*)v < runtime·mheap.arena_start)
|
|
|
|
return; // not allocated, so okay
|
|
|
|
|
|
|
|
off = (uintptr*)v - (uintptr*)runtime·mheap.arena_start; // word offset
|
|
|
|
b = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1;
|
|
|
|
shift = off % wordsPerBitmapWord;
|
|
|
|
|
|
|
|
bits = *b>>shift;
|
|
|
|
if((bits & bitAllocated) != 0) {
|
|
|
|
runtime·printf("checkfreed %p+%p: off=%p have=%p\n",
|
|
|
|
v, n, off, bits & bitMask);
|
|
|
|
runtime·throw("checkfreed: not freed");
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// mark the span of memory at v as having n blocks of the given size.
|
|
|
|
// if leftover is true, there is left over space at the end of the span.
|
|
|
|
void
|
|
|
|
runtime·markspan(void *v, uintptr size, uintptr n, bool leftover)
|
|
|
|
{
|
|
|
|
uintptr *b, off, shift;
|
|
|
|
byte *p;
|
|
|
|
|
|
|
|
if((byte*)v+size*n > (byte*)runtime·mheap.arena_used || (byte*)v < runtime·mheap.arena_start)
|
|
|
|
runtime·throw("markspan: bad pointer");
|
|
|
|
|
|
|
|
p = v;
|
|
|
|
if(leftover) // mark a boundary just past end of last block too
|
|
|
|
n++;
|
|
|
|
for(; n-- > 0; p += size) {
|
2011-02-16 11:21:20 -07:00
|
|
|
// Okay to use non-atomic ops here, because we control
|
|
|
|
// the entire span, and each bitmap word has bits for only
|
|
|
|
// one span, so no other goroutines are changing these
|
|
|
|
// bitmap words.
|
2011-02-02 21:03:47 -07:00
|
|
|
off = (uintptr*)p - (uintptr*)runtime·mheap.arena_start; // word offset
|
|
|
|
b = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1;
|
|
|
|
shift = off % wordsPerBitmapWord;
|
|
|
|
*b = (*b & ~(bitMask<<shift)) | (bitBlockBoundary<<shift);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// unmark the span of memory at v of length n bytes.
|
|
|
|
void
|
|
|
|
runtime·unmarkspan(void *v, uintptr n)
|
|
|
|
{
|
|
|
|
uintptr *p, *b, off;
|
|
|
|
|
|
|
|
if((byte*)v+n > (byte*)runtime·mheap.arena_used || (byte*)v < runtime·mheap.arena_start)
|
|
|
|
runtime·throw("markspan: bad pointer");
|
|
|
|
|
|
|
|
p = v;
|
|
|
|
off = p - (uintptr*)runtime·mheap.arena_start; // word offset
|
|
|
|
if(off % wordsPerBitmapWord != 0)
|
|
|
|
runtime·throw("markspan: unaligned pointer");
|
|
|
|
b = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1;
|
|
|
|
n /= PtrSize;
|
|
|
|
if(n%wordsPerBitmapWord != 0)
|
|
|
|
runtime·throw("unmarkspan: unaligned length");
|
2011-02-16 11:21:20 -07:00
|
|
|
// Okay to use non-atomic ops here, because we control
|
|
|
|
// the entire span, and each bitmap word has bits for only
|
|
|
|
// one span, so no other goroutines are changing these
|
|
|
|
// bitmap words.
|
2011-02-02 21:03:47 -07:00
|
|
|
n /= wordsPerBitmapWord;
|
|
|
|
while(n-- > 0)
|
|
|
|
*b-- = 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
bool
|
|
|
|
runtime·blockspecial(void *v)
|
|
|
|
{
|
|
|
|
uintptr *b, off, shift;
|
|
|
|
|
|
|
|
off = (uintptr*)v - (uintptr*)runtime·mheap.arena_start;
|
|
|
|
b = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1;
|
|
|
|
shift = off % wordsPerBitmapWord;
|
|
|
|
|
|
|
|
return (*b & (bitSpecial<<shift)) != 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
void
|
|
|
|
runtime·setblockspecial(void *v)
|
|
|
|
{
|
2011-02-16 11:21:20 -07:00
|
|
|
uintptr *b, off, shift, bits, obits;
|
2011-02-02 21:03:47 -07:00
|
|
|
|
|
|
|
off = (uintptr*)v - (uintptr*)runtime·mheap.arena_start;
|
|
|
|
b = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1;
|
|
|
|
shift = off % wordsPerBitmapWord;
|
|
|
|
|
2011-02-16 11:21:20 -07:00
|
|
|
for(;;) {
|
|
|
|
obits = *b;
|
|
|
|
bits = obits | (bitSpecial<<shift);
|
|
|
|
if(runtime·gomaxprocs == 1) {
|
|
|
|
*b = bits;
|
|
|
|
break;
|
|
|
|
} else {
|
|
|
|
// gomaxprocs > 1: use atomic op
|
|
|
|
if(runtime·casp((void**)b, (void*)obits, (void*)bits))
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
2011-02-02 21:03:47 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
void
|
|
|
|
runtime·MHeap_MapBits(MHeap *h)
|
|
|
|
{
|
|
|
|
// Caller has added extra mappings to the arena.
|
|
|
|
// Add extra mappings of bitmap words as needed.
|
|
|
|
// We allocate extra bitmap pieces in chunks of bitmapChunk.
|
|
|
|
enum {
|
|
|
|
bitmapChunk = 8192
|
|
|
|
};
|
|
|
|
uintptr n;
|
|
|
|
|
|
|
|
n = (h->arena_used - h->arena_start) / wordsPerBitmapWord;
|
|
|
|
n = (n+bitmapChunk-1) & ~(bitmapChunk-1);
|
|
|
|
if(h->bitmap_mapped >= n)
|
|
|
|
return;
|
|
|
|
|
|
|
|
runtime·SysMap(h->arena_start - n, n - h->bitmap_mapped);
|
|
|
|
h->bitmap_mapped = n;
|
|
|
|
}
|