2009-01-26 18:37:05 -07:00
|
|
|
// Copyright 2009 The Go Authors. All rights reserved.
|
|
|
|
// Use of this source code is governed by a BSD-style
|
|
|
|
// license that can be found in the LICENSE file.
|
|
|
|
|
|
|
|
// Garbage collector -- step 0.
|
|
|
|
//
|
|
|
|
// Stop the world, mark and sweep garbage collector.
|
|
|
|
// NOT INTENDED FOR PRODUCTION USE.
|
|
|
|
//
|
|
|
|
// A mark and sweep collector provides a way to exercise
|
|
|
|
// and test the memory allocator and the stack walking machinery
|
|
|
|
// without also needing to get reference counting
|
|
|
|
// exactly right.
|
|
|
|
|
|
|
|
#include "runtime.h"
|
|
|
|
#include "malloc.h"
|
|
|
|
|
|
|
|
enum {
|
|
|
|
Debug = 0
|
|
|
|
};
|
|
|
|
|
2009-08-20 17:09:38 -06:00
|
|
|
extern byte data[];
|
2009-01-26 18:37:05 -07:00
|
|
|
extern byte etext[];
|
|
|
|
extern byte end[];
|
|
|
|
|
2010-03-26 15:15:30 -06:00
|
|
|
static G *fing;
|
|
|
|
static Finalizer *finq;
|
2010-02-03 17:31:34 -07:00
|
|
|
static void sweepblock(byte*, int64, uint32*, int32);
|
2010-03-26 15:15:30 -06:00
|
|
|
static void runfinq(void);
|
2010-02-03 17:31:34 -07:00
|
|
|
|
2009-05-26 18:39:25 -06:00
|
|
|
enum {
|
|
|
|
PtrSize = sizeof(void*)
|
|
|
|
};
|
|
|
|
|
2009-01-26 18:37:05 -07:00
|
|
|
static void
|
|
|
|
scanblock(int32 depth, byte *b, int64 n)
|
|
|
|
{
|
|
|
|
int32 off;
|
|
|
|
void *obj;
|
|
|
|
uintptr size;
|
2010-02-10 01:00:12 -07:00
|
|
|
uint32 *refp, ref;
|
2009-01-26 18:37:05 -07:00
|
|
|
void **vp;
|
|
|
|
int64 i;
|
|
|
|
|
2010-02-03 17:31:34 -07:00
|
|
|
if(Debug > 1)
|
2009-01-26 18:37:05 -07:00
|
|
|
printf("%d scanblock %p %D\n", depth, b, n);
|
2009-05-26 18:39:25 -06:00
|
|
|
off = (uint32)(uintptr)b & (PtrSize-1);
|
2009-01-26 18:37:05 -07:00
|
|
|
if(off) {
|
2009-05-26 18:39:25 -06:00
|
|
|
b += PtrSize - off;
|
|
|
|
n -= PtrSize - off;
|
2009-01-26 18:37:05 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
vp = (void**)b;
|
2009-05-26 18:39:25 -06:00
|
|
|
n /= PtrSize;
|
2009-01-26 18:37:05 -07:00
|
|
|
for(i=0; i<n; i++) {
|
2009-12-03 18:22:23 -07:00
|
|
|
obj = vp[i];
|
|
|
|
if(obj == nil || (byte*)obj < mheap.min || (byte*)obj >= mheap.max)
|
|
|
|
continue;
|
2010-02-10 22:23:08 -07:00
|
|
|
if(mlookup(obj, &obj, &size, nil, &refp)) {
|
2010-02-10 01:00:12 -07:00
|
|
|
ref = *refp;
|
2010-03-23 21:48:23 -06:00
|
|
|
switch(ref & ~RefFlags) {
|
2010-02-10 01:00:12 -07:00
|
|
|
case RefNone:
|
2010-02-03 17:31:34 -07:00
|
|
|
if(Debug > 1)
|
2009-01-26 18:37:05 -07:00
|
|
|
printf("%d found at %p: ", depth, &vp[i]);
|
2010-03-23 21:48:23 -06:00
|
|
|
*refp = RefSome | (ref & RefFlags);
|
2010-02-10 01:00:12 -07:00
|
|
|
if(!(ref & RefNoPointers))
|
|
|
|
scanblock(depth+1, obj, size);
|
|
|
|
break;
|
2009-01-26 18:37:05 -07:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
static void
|
2009-07-27 15:16:28 -06:00
|
|
|
scanstack(G *gp)
|
2009-01-26 18:37:05 -07:00
|
|
|
{
|
|
|
|
Stktop *stk;
|
|
|
|
byte *sp;
|
|
|
|
|
2010-01-06 20:24:11 -07:00
|
|
|
if(gp == g)
|
2009-07-27 15:16:28 -06:00
|
|
|
sp = (byte*)&gp;
|
|
|
|
else
|
|
|
|
sp = gp->sched.sp;
|
2010-02-03 17:31:34 -07:00
|
|
|
if(Debug > 1)
|
|
|
|
printf("scanstack %d %p\n", gp->goid, sp);
|
2009-07-27 15:16:28 -06:00
|
|
|
stk = (Stktop*)gp->stackbase;
|
2009-01-26 18:37:05 -07:00
|
|
|
while(stk) {
|
|
|
|
scanblock(0, sp, (byte*)stk - sp);
|
2009-06-17 16:12:16 -06:00
|
|
|
sp = stk->gobuf.sp;
|
|
|
|
stk = (Stktop*)stk->stackbase;
|
2009-01-26 18:37:05 -07:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2010-03-26 15:15:30 -06:00
|
|
|
static void
|
|
|
|
markfin(void *v)
|
|
|
|
{
|
|
|
|
uintptr size;
|
|
|
|
uint32 *refp;
|
|
|
|
|
|
|
|
size = 0;
|
|
|
|
refp = nil;
|
|
|
|
if(!mlookup(v, &v, &size, nil, &refp) || !(*refp & RefHasFinalizer))
|
|
|
|
throw("mark - finalizer inconsistency");
|
|
|
|
|
|
|
|
// do not mark the finalizer block itself. just mark the things it points at.
|
|
|
|
scanblock(1, v, size);
|
|
|
|
}
|
|
|
|
|
2009-01-26 18:37:05 -07:00
|
|
|
static void
|
|
|
|
mark(void)
|
|
|
|
{
|
2010-01-06 20:24:11 -07:00
|
|
|
G *gp;
|
2009-01-26 18:37:05 -07:00
|
|
|
|
2009-12-07 16:52:14 -07:00
|
|
|
// mark data+bss.
|
|
|
|
// skip mheap itself, which has no interesting pointers
|
|
|
|
// and is mostly zeroed and would not otherwise be paged in.
|
|
|
|
scanblock(0, data, (byte*)&mheap - data);
|
|
|
|
scanblock(0, (byte*)(&mheap+1), end - (byte*)(&mheap+1));
|
2009-01-26 18:37:05 -07:00
|
|
|
|
|
|
|
// mark stacks
|
|
|
|
for(gp=allg; gp!=nil; gp=gp->alllink) {
|
|
|
|
switch(gp->status){
|
|
|
|
default:
|
|
|
|
printf("unexpected G.status %d\n", gp->status);
|
|
|
|
throw("mark - bad status");
|
|
|
|
case Gdead:
|
|
|
|
break;
|
|
|
|
case Grunning:
|
2010-01-06 20:24:11 -07:00
|
|
|
if(gp != g)
|
2009-01-26 18:37:05 -07:00
|
|
|
throw("mark - world not stopped");
|
|
|
|
scanstack(gp);
|
|
|
|
break;
|
|
|
|
case Grunnable:
|
|
|
|
case Gsyscall:
|
|
|
|
case Gwaiting:
|
|
|
|
scanstack(gp);
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2010-03-26 15:15:30 -06:00
|
|
|
// mark things pointed at by objects with finalizers
|
|
|
|
walkfintab(markfin);
|
|
|
|
}
|
2010-02-03 17:31:34 -07:00
|
|
|
|
2010-03-26 15:15:30 -06:00
|
|
|
// free RefNone, free & queue finalizers for RefNone|RefHasFinalizer, reset RefSome
|
2010-02-03 17:31:34 -07:00
|
|
|
static void
|
2010-03-26 15:15:30 -06:00
|
|
|
sweepspan(MSpan *s)
|
2010-02-03 17:31:34 -07:00
|
|
|
{
|
2010-02-10 15:59:39 -07:00
|
|
|
int32 n, npages, size;
|
|
|
|
byte *p;
|
|
|
|
uint32 ref, *gcrefp, *gcrefep;
|
|
|
|
MCache *c;
|
2010-03-26 15:15:30 -06:00
|
|
|
Finalizer *f;
|
2010-02-03 17:31:34 -07:00
|
|
|
|
2010-02-10 15:59:39 -07:00
|
|
|
p = (byte*)(s->start << PageShift);
|
|
|
|
if(s->sizeclass == 0) {
|
|
|
|
// Large block.
|
|
|
|
ref = s->gcref0;
|
2010-03-26 15:15:30 -06:00
|
|
|
switch(ref & ~(RefFlags^RefHasFinalizer)) {
|
2010-02-10 15:59:39 -07:00
|
|
|
case RefNone:
|
|
|
|
// Free large object.
|
|
|
|
mstats.alloc -= s->npages<<PageShift;
|
|
|
|
runtime_memclr(p, s->npages<<PageShift);
|
2010-03-23 21:48:23 -06:00
|
|
|
if(ref & RefProfiled)
|
|
|
|
MProf_Free(p, s->npages<<PageShift);
|
2010-02-10 15:59:39 -07:00
|
|
|
s->gcref0 = RefFree;
|
2010-03-08 15:15:44 -07:00
|
|
|
MHeap_Free(&mheap, s, 1);
|
2010-02-10 15:59:39 -07:00
|
|
|
break;
|
2010-03-26 15:15:30 -06:00
|
|
|
case RefNone|RefHasFinalizer:
|
|
|
|
f = getfinalizer(p, 1);
|
|
|
|
if(f == nil)
|
|
|
|
throw("finalizer inconsistency");
|
|
|
|
f->arg = p;
|
|
|
|
f->next = finq;
|
|
|
|
finq = f;
|
|
|
|
ref &= ~RefHasFinalizer;
|
2010-02-10 15:59:39 -07:00
|
|
|
// fall through
|
|
|
|
case RefSome:
|
2010-03-26 15:15:30 -06:00
|
|
|
case RefSome|RefHasFinalizer:
|
2010-03-23 21:48:23 -06:00
|
|
|
s->gcref0 = RefNone | (ref&RefFlags);
|
2010-02-10 15:59:39 -07:00
|
|
|
break;
|
2010-02-03 17:31:34 -07:00
|
|
|
}
|
2010-02-10 15:59:39 -07:00
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
// Chunk full of small blocks.
|
|
|
|
MGetSizeClassInfo(s->sizeclass, &size, &npages, &n);
|
|
|
|
gcrefp = s->gcref;
|
|
|
|
gcrefep = s->gcref + n;
|
|
|
|
for(; gcrefp < gcrefep; gcrefp++, p += size) {
|
|
|
|
ref = *gcrefp;
|
|
|
|
if(ref < RefNone) // RefFree or RefStack
|
|
|
|
continue;
|
2010-03-26 15:15:30 -06:00
|
|
|
switch(ref & ~(RefFlags^RefHasFinalizer)) {
|
2010-02-10 15:59:39 -07:00
|
|
|
case RefNone:
|
|
|
|
// Free small object.
|
2010-03-23 21:48:23 -06:00
|
|
|
if(ref & RefProfiled)
|
|
|
|
MProf_Free(p, size);
|
2010-02-10 15:59:39 -07:00
|
|
|
*gcrefp = RefFree;
|
|
|
|
c = m->mcache;
|
|
|
|
if(size > sizeof(uintptr))
|
|
|
|
((uintptr*)p)[1] = 1; // mark as "needs to be zeroed"
|
|
|
|
mstats.alloc -= size;
|
|
|
|
mstats.by_size[s->sizeclass].nfree++;
|
|
|
|
MCache_Free(c, p, s->sizeclass, size);
|
|
|
|
break;
|
2010-03-26 15:15:30 -06:00
|
|
|
case RefNone|RefHasFinalizer:
|
|
|
|
f = getfinalizer(p, 1);
|
|
|
|
if(f == nil)
|
|
|
|
throw("finalizer inconsistency");
|
|
|
|
f->arg = p;
|
|
|
|
f->next = finq;
|
|
|
|
finq = f;
|
|
|
|
ref &= ~RefHasFinalizer;
|
2010-02-10 15:59:39 -07:00
|
|
|
// fall through
|
|
|
|
case RefSome:
|
2010-03-26 15:15:30 -06:00
|
|
|
case RefSome|RefHasFinalizer:
|
2010-03-23 21:48:23 -06:00
|
|
|
*gcrefp = RefNone | (ref&RefFlags);
|
2010-02-10 15:59:39 -07:00
|
|
|
break;
|
2009-01-26 18:37:05 -07:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
static void
|
|
|
|
sweep(void)
|
|
|
|
{
|
2009-01-28 16:22:16 -07:00
|
|
|
MSpan *s;
|
2009-01-26 18:37:05 -07:00
|
|
|
|
2009-01-28 16:22:16 -07:00
|
|
|
for(s = mheap.allspans; s != nil; s = s->allnext)
|
2010-02-10 15:59:39 -07:00
|
|
|
if(s->state == MSpanInUse)
|
2010-03-26 15:15:30 -06:00
|
|
|
sweepspan(s);
|
2009-01-26 18:37:05 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
// Semaphore, not Lock, so that the goroutine
|
|
|
|
// reschedules when there is contention rather
|
|
|
|
// than spinning.
|
|
|
|
static uint32 gcsema = 1;
|
|
|
|
|
|
|
|
// Initialized from $GOGC. GOGC=off means no gc.
|
|
|
|
//
|
|
|
|
// Next gc is after we've allocated an extra amount of
|
|
|
|
// memory proportional to the amount already in use.
|
|
|
|
// If gcpercent=100 and we're using 4M, we'll gc again
|
|
|
|
// when we get to 8M. This keeps the gc cost in linear
|
|
|
|
// proportion to the allocation cost. Adjusting gcpercent
|
|
|
|
// just changes the linear constant (and also the amount of
|
|
|
|
// extra memory used).
|
|
|
|
static int32 gcpercent = -2;
|
|
|
|
|
2010-03-08 15:15:44 -07:00
|
|
|
static void
|
|
|
|
stealcache(void)
|
|
|
|
{
|
|
|
|
M *m;
|
|
|
|
|
|
|
|
for(m=allm; m; m=m->alllink)
|
|
|
|
MCache_ReleaseAll(m->mcache);
|
|
|
|
}
|
|
|
|
|
2009-01-26 18:37:05 -07:00
|
|
|
void
|
|
|
|
gc(int32 force)
|
|
|
|
{
|
2010-02-08 15:32:22 -07:00
|
|
|
int64 t0, t1;
|
2009-01-26 18:37:05 -07:00
|
|
|
byte *p;
|
2010-03-26 15:15:30 -06:00
|
|
|
Finalizer *fp;
|
2009-01-26 18:37:05 -07:00
|
|
|
|
|
|
|
// The gc is turned off (via enablegc) until
|
|
|
|
// the bootstrap has completed.
|
|
|
|
// Also, malloc gets called in the guts
|
|
|
|
// of a number of libraries that might be
|
|
|
|
// holding locks. To avoid priority inversion
|
|
|
|
// problems, don't bother trying to run gc
|
|
|
|
// while holding a lock. The next mallocgc
|
|
|
|
// without a lock will do the gc instead.
|
|
|
|
if(!mstats.enablegc || m->locks > 0 || panicking)
|
|
|
|
return;
|
|
|
|
|
|
|
|
if(gcpercent == -2) { // first time through
|
|
|
|
p = getenv("GOGC");
|
|
|
|
if(p == nil || p[0] == '\0')
|
|
|
|
gcpercent = 100;
|
|
|
|
else if(strcmp(p, (byte*)"off") == 0)
|
|
|
|
gcpercent = -1;
|
|
|
|
else
|
|
|
|
gcpercent = atoi(p);
|
|
|
|
}
|
2009-06-05 11:59:37 -06:00
|
|
|
if(gcpercent < 0)
|
2009-01-26 18:37:05 -07:00
|
|
|
return;
|
|
|
|
|
|
|
|
semacquire(&gcsema);
|
2010-02-08 15:32:22 -07:00
|
|
|
t0 = nanotime();
|
2009-08-14 21:33:20 -06:00
|
|
|
m->gcing = 1;
|
2009-01-26 18:37:05 -07:00
|
|
|
stoptheworld();
|
|
|
|
if(mheap.Lock.key != 0)
|
|
|
|
throw("mheap locked during gc");
|
2010-03-08 15:15:44 -07:00
|
|
|
if(force || mstats.heap_alloc >= mstats.next_gc) {
|
2009-01-26 18:37:05 -07:00
|
|
|
mark();
|
|
|
|
sweep();
|
2010-03-08 15:15:44 -07:00
|
|
|
stealcache();
|
|
|
|
mstats.next_gc = mstats.heap_alloc+mstats.heap_alloc*gcpercent/100;
|
2009-01-26 18:37:05 -07:00
|
|
|
}
|
2009-06-15 22:31:56 -06:00
|
|
|
m->gcing = 0;
|
2010-02-10 01:00:12 -07:00
|
|
|
|
2010-02-03 17:31:34 -07:00
|
|
|
m->locks++; // disable gc during the mallocs in newproc
|
2010-03-26 15:15:30 -06:00
|
|
|
fp = finq;
|
|
|
|
if(fp != nil) {
|
|
|
|
// kick off or wake up goroutine to run queued finalizers
|
|
|
|
if(fing == nil)
|
|
|
|
fing = newproc1((byte*)runfinq, nil, 0, 0);
|
|
|
|
else if(fing->status == Gwaiting)
|
|
|
|
ready(fing);
|
2010-02-03 17:31:34 -07:00
|
|
|
}
|
|
|
|
m->locks--;
|
|
|
|
|
2010-02-08 15:32:22 -07:00
|
|
|
t1 = nanotime();
|
|
|
|
mstats.numgc++;
|
|
|
|
mstats.pause_ns += t1 - t0;
|
|
|
|
if(mstats.debuggc)
|
|
|
|
printf("pause %D\n", t1-t0);
|
2009-08-14 21:33:20 -06:00
|
|
|
semrelease(&gcsema);
|
2010-01-09 10:47:45 -07:00
|
|
|
starttheworld();
|
2010-03-26 15:15:30 -06:00
|
|
|
|
|
|
|
// give the queued finalizers, if any, a chance to run
|
|
|
|
if(fp != nil)
|
|
|
|
gosched();
|
|
|
|
}
|
|
|
|
|
|
|
|
static void
|
|
|
|
runfinq(void)
|
|
|
|
{
|
|
|
|
Finalizer *f, *next;
|
|
|
|
byte *frame;
|
|
|
|
|
|
|
|
for(;;) {
|
|
|
|
// There's no need for a lock in this section
|
|
|
|
// because it only conflicts with the garbage
|
|
|
|
// collector, and the garbage collector only
|
|
|
|
// runs when everyone else is stopped, and
|
|
|
|
// runfinq only stops at the gosched() or
|
|
|
|
// during the calls in the for loop.
|
|
|
|
f = finq;
|
|
|
|
finq = nil;
|
|
|
|
if(f == nil) {
|
|
|
|
g->status = Gwaiting;
|
|
|
|
gosched();
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
for(; f; f=next) {
|
|
|
|
next = f->next;
|
|
|
|
frame = mal(sizeof(uintptr) + f->nret);
|
|
|
|
*(void**)frame = f->arg;
|
|
|
|
reflect·call((byte*)f->fn, frame, sizeof(uintptr) + f->nret);
|
|
|
|
free(frame);
|
|
|
|
f->fn = nil;
|
|
|
|
f->arg = nil;
|
|
|
|
f->next = nil;
|
|
|
|
}
|
|
|
|
gc(1); // trigger another gc to clean up the finalized objects, if possible
|
|
|
|
}
|
2009-01-26 18:37:05 -07:00
|
|
|
}
|