qbit/go
1
0
Fork 0
mirror of https://github.com/golang/go synced 2024-10-04 06:21:23 -06:00
Code Releases Activity

runtime: move stack scanning into the parallel mark phase

Browse Source 
This change reduces the cost of the stack scanning by frames.
It moves the stack scanning from the serial root enumeration
phase to the parallel tracing phase.  The output that follows
are timings for the issue 6482 benchmark

Baseline

BenchmarkGoroutineSelect	      50	 108027405 ns/op
BenchmarkGoroutineBlocking	      50	  89573332 ns/op
BenchmarkGoroutineForRange	      20	  95614116 ns/op
BenchmarkGoroutineIdle		      20	 122809512 ns/op

Stack scan by frames, non-parallel

BenchmarkGoroutineSelect	      20	 297138929 ns/op
BenchmarkGoroutineBlocking	      20	 301137599 ns/op
BenchmarkGoroutineForRange	      10	 312499469 ns/op
BenchmarkGoroutineIdle		      10	 209428876 ns/op

Stack scan by frames, parallel

BenchmarkGoroutineSelect	      20	 183938431 ns/op
BenchmarkGoroutineBlocking	      20	 170109999 ns/op
BenchmarkGoroutineForRange	      20	 179628882 ns/op
BenchmarkGoroutineIdle		      20	 157541498 ns/op

The remaining performance disparity is due to inefficiencies
in gentraceback and its callees.  The effect was isolated by
using a parallel stack scan where scanstack was modified to do
a conservative scan of the stack segments without gentraceback
followed by a call of gentrackback with a no-op callback.

The output that follows are the top-10 most frequent tops of
stacks as determined by the Linux perf record facility.

Baseline

+  25.19%  gc.test  gc.test            [.] runtime.xchg
+  19.00%  gc.test  gc.test            [.] scanblock
+   8.53%  gc.test  gc.test            [.] scanstack
+   8.46%  gc.test  gc.test            [.] flushptrbuf
+   5.08%  gc.test  gc.test            [.] procresize
+   3.57%  gc.test  gc.test            [.] runtime.chanrecv
+   2.94%  gc.test  gc.test            [.] dequeue
+   2.74%  gc.test  gc.test            [.] addroots
+   2.25%  gc.test  gc.test            [.] runtime.ready
+   1.33%  gc.test  gc.test            [.] runtime.cas64

Gentraceback

+  18.12%  gc.test  gc.test             [.] runtime.xchg
+  14.68%  gc.test  gc.test             [.] scanblock
+   8.20%  gc.test  gc.test             [.] runtime.gentraceback
+   7.38%  gc.test  gc.test             [.] flushptrbuf
+   6.84%  gc.test  gc.test             [.] scanstack
+   5.92%  gc.test  gc.test             [.] runtime.findfunc
+   3.62%  gc.test  gc.test             [.] procresize
+   3.15%  gc.test  gc.test             [.] readvarint
+   1.92%  gc.test  gc.test             [.] addroots
+   1.87%  gc.test  gc.test             [.] runtime.chanrecv

R=golang-dev, dvyukov, rsc
CC=golang-dev
https://golang.org/cl/17410043
This commit is contained in:
Carl Shapiro 2013-12-03 14:12:55 -08:00
parent 24699fb05c
commit 0368a7ceb6
1 changed files with 191 additions and 133 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

324
src/pkg/runtime/mgc0.c
View File

@ -19,7 +19,7 @@ enum {
Debug = 0, Debug = 0,
DebugMark = 0, // run second pass to check mark DebugMark = 0, // run second pass to check mark
CollectStats = 0, CollectStats = 0,
ScanStackByFrames = 0, ScanStackByFrames = 1,
IgnorePreciseGC = 0, IgnorePreciseGC = 0,
// Four bits per word (see #defines below). // Four bits per word (see #defines below).
@ -151,6 +151,7 @@ static Workbuf* getfull(Workbuf*);
static void putempty(Workbuf*); static void putempty(Workbuf*);
static Workbuf* handoff(Workbuf*); static Workbuf* handoff(Workbuf*);
static void gchelperstart(void); static void gchelperstart(void);
static void scanstack(G* gp, void *scanbuf);
static struct { static struct {
uint64 full; // lock-free list of full blocks uint64 full; // lock-free list of full blocks
@ -176,6 +177,7 @@ static struct {
enum { enum {
GC_DEFAULT_PTR = GC_NUM_INSTR, GC_DEFAULT_PTR = GC_NUM_INSTR,
GC_CHAN, GC_CHAN,
GC_G_PTR,
GC_NUM_INSTR2 GC_NUM_INSTR2
}; };
@ -317,6 +319,24 @@ struct PtrTarget
uintptr ti; uintptr ti;
}; };
typedef struct Scanbuf Scanbuf;
struct Scanbuf
{
struct {
PtrTarget *begin;
PtrTarget *end;
PtrTarget *pos;
} ptr;
struct {
Obj *begin;
Obj *end;
Obj *pos;
} obj;
Workbuf *wbuf;
Obj *wp;
uintptr nobj;
};
typedef struct BufferList BufferList; typedef struct BufferList BufferList;
struct BufferList struct BufferList
{ {
@ -350,7 +370,7 @@ static void enqueue(Obj obj, Workbuf **_wbuf, Obj **_wp, uintptr *_nobj);
// flushptrbuf // flushptrbuf
// (find block start, mark and enqueue) // (find block start, mark and enqueue)
static void static void
flushptrbuf(PtrTarget *ptrbuf, PtrTarget **ptrbufpos, Obj **_wp, Workbuf **_wbuf, uintptr *_nobj) flushptrbuf(Scanbuf *sbuf)
{ {
byte *p, *arena_start, *obj; byte *p, *arena_start, *obj;
uintptr size, *bitp, bits, shift, j, x, xbits, off, nobj, ti, n; uintptr size, *bitp, bits, shift, j, x, xbits, off, nobj, ti, n;
@ -358,17 +378,19 @@ flushptrbuf(PtrTarget *ptrbuf, PtrTarget **ptrbufpos, Obj **_wp, Workbuf **_wbuf
PageID k; PageID k;
Obj *wp; Obj *wp;
Workbuf *wbuf; Workbuf *wbuf;
PtrTarget *ptrbuf;
PtrTarget *ptrbuf_end; PtrTarget *ptrbuf_end;
arena_start = runtime·mheap.arena_start; arena_start = runtime·mheap.arena_start;
wp = *_wp; wp = sbuf->wp;
wbuf = *_wbuf; wbuf = sbuf->wbuf;
nobj = *_nobj; nobj = sbuf->nobj;
ptrbuf_end = *ptrbufpos; ptrbuf = sbuf->ptr.begin;
n = ptrbuf_end - ptrbuf; ptrbuf_end = sbuf->ptr.pos;
*ptrbufpos = ptrbuf; n = ptrbuf_end - sbuf->ptr.begin;
sbuf->ptr.pos = sbuf->ptr.begin;
if(CollectStats) { if(CollectStats) {
runtime·xadd64(&gcstats.ptr.sum, n); runtime·xadd64(&gcstats.ptr.sum, n);
@ -514,25 +536,27 @@ flushptrbuf(PtrTarget *ptrbuf, PtrTarget **ptrbufpos, Obj **_wp, Workbuf **_wbuf
} }
} }
*_wp = wp; sbuf->wp = wp;
*_wbuf = wbuf; sbuf->wbuf = wbuf;
*_nobj = nobj; sbuf->nobj = nobj;
} }
static void static void
flushobjbuf(Obj *objbuf, Obj **objbufpos, Obj **_wp, Workbuf **_wbuf, uintptr *_nobj) flushobjbuf(Scanbuf *sbuf)
{ {
uintptr nobj, off; uintptr nobj, off;
Obj *wp, obj; Obj *wp, obj;
Workbuf *wbuf; Workbuf *wbuf;
Obj *objbuf;
Obj *objbuf_end; Obj *objbuf_end;
wp = *_wp; wp = sbuf->wp;
wbuf = *_wbuf; wbuf = sbuf->wbuf;
nobj = *_nobj; nobj = sbuf->nobj;
objbuf_end = *objbufpos; objbuf = sbuf->obj.begin;
*objbufpos = objbuf; objbuf_end = sbuf->obj.pos;
sbuf->obj.pos = sbuf->obj.begin;
while(objbuf < objbuf_end) { while(objbuf < objbuf_end) {
obj = *objbuf++; obj = *objbuf++;
@ -570,9 +594,9 @@ flushobjbuf(Obj *objbuf, Obj **objbufpos, Obj **_wp, Workbuf **_wbuf, uintptr *_
wp = wbuf->obj + nobj; wp = wbuf->obj + nobj;
} }
*_wp = wp; sbuf->wp = wp;
*_wbuf = wbuf; sbuf->wbuf = wbuf;
*_nobj = nobj; sbuf->nobj = nobj;
} }
// Program that scans the whole block and treats every block element as a potential pointer // Program that scans the whole block and treats every block element as a potential pointer
@ -581,6 +605,9 @@ static uintptr defaultProg[2] = {PtrSize, GC_DEFAULT_PTR};
// Hchan program // Hchan program
static uintptr chanProg[2] = {0, GC_CHAN}; static uintptr chanProg[2] = {0, GC_CHAN};
// G* program
static uintptr gptrProg[2] = {0, GC_G_PTR};
// Local variables of a program fragment or loop // Local variables of a program fragment or loop
typedef struct Frame Frame; typedef struct Frame Frame;
struct Frame { struct Frame {
@ -666,8 +693,7 @@ scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking)
Slice *sliceptr; Slice *sliceptr;
Frame *stack_ptr, stack_top, stack[GC_STACK_CAPACITY+4]; Frame *stack_ptr, stack_top, stack[GC_STACK_CAPACITY+4];
BufferList *scanbuffers; BufferList *scanbuffers;
PtrTarget *ptrbuf, *ptrbuf_end, *ptrbufpos; Scanbuf sbuf;
Obj *objbuf, *objbuf_end, *objbufpos;
Eface *eface; Eface *eface;
Iface *iface; Iface *iface;
Hchan *chan; Hchan *chan;
@ -681,21 +707,22 @@ scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking)
arena_used = runtime·mheap.arena_used; arena_used = runtime·mheap.arena_used;
stack_ptr = stack+nelem(stack)-1; stack_ptr = stack+nelem(stack)-1;
precise_type = false; precise_type = false;
nominal_size = 0; nominal_size = 0;
// Allocate ptrbuf // Initialize sbuf
{ scanbuffers = &bufferList[m->helpgc];
scanbuffers = &bufferList[m->helpgc];
ptrbuf = &scanbuffers->ptrtarget[0];
ptrbuf_end = &scanbuffers->ptrtarget[0] + nelem(scanbuffers->ptrtarget);
objbuf = &scanbuffers->obj[0];
objbuf_end = &scanbuffers->obj[0] + nelem(scanbuffers->obj);
}
ptrbufpos = ptrbuf; sbuf.ptr.begin = sbuf.ptr.pos = &scanbuffers->ptrtarget[0];
objbufpos = objbuf; sbuf.ptr.end = sbuf.ptr.begin + nelem(scanbuffers->ptrtarget);
sbuf.obj.begin = sbuf.obj.pos = &scanbuffers->obj[0];
sbuf.obj.end = sbuf.obj.begin + nelem(scanbuffers->obj);
sbuf.wbuf = wbuf;
sbuf.wp = wp;
sbuf.nobj = nobj;
// (Silence the compiler) // (Silence the compiler)
chan = nil; chan = nil;
@ -713,7 +740,7 @@ scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking)
if(CollectStats) { if(CollectStats) {
runtime·xadd64(&gcstats.nbytes, n); runtime·xadd64(&gcstats.nbytes, n);
runtime·xadd64(&gcstats.obj.sum, nobj); runtime·xadd64(&gcstats.obj.sum, sbuf.nobj);
runtime·xadd64(&gcstats.obj.cnt, 1); runtime·xadd64(&gcstats.obj.cnt, 1);
} }
@ -839,9 +866,9 @@ scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking)
// eface->type // eface->type
t = eface->type; t = eface->type;
if((void*)t >= arena_start && (void*)t < arena_used) { if((void*)t >= arena_start && (void*)t < arena_used) {
*ptrbufpos++ = (PtrTarget){t, 0}; *sbuf.ptr.pos++ = (PtrTarget){t, 0};
if(ptrbufpos == ptrbuf_end) if(sbuf.ptr.pos == sbuf.ptr.end)
flushptrbuf(ptrbuf, &ptrbufpos, &wp, &wbuf, &nobj); flushptrbuf(&sbuf);
} }
// eface->data // eface->data
@ -868,9 +895,9 @@ scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking)
// iface->tab // iface->tab
if((void*)iface->tab >= arena_start && (void*)iface->tab < arena_used) { if((void*)iface->tab >= arena_start && (void*)iface->tab < arena_used) {
*ptrbufpos++ = (PtrTarget){iface->tab, (uintptr)itabtype->gc}; *sbuf.ptr.pos++ = (PtrTarget){iface->tab, (uintptr)itabtype->gc};
if(ptrbufpos == ptrbuf_end) if(sbuf.ptr.pos == sbuf.ptr.end)
flushptrbuf(ptrbuf, &ptrbufpos, &wp, &wbuf, &nobj); flushptrbuf(&sbuf);
} }
// iface->data // iface->data
@ -895,9 +922,9 @@ scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking)
obj = *(byte**)stack_top.b; obj = *(byte**)stack_top.b;
stack_top.b += PtrSize; stack_top.b += PtrSize;
if(obj >= arena_start && obj < arena_used) { if(obj >= arena_start && obj < arena_used) {
*ptrbufpos++ = (PtrTarget){obj, 0}; *sbuf.ptr.pos++ = (PtrTarget){obj, 0};
if(ptrbufpos == ptrbuf_end) if(sbuf.ptr.pos == sbuf.ptr.end)
flushptrbuf(ptrbuf, &ptrbufpos, &wp, &wbuf, &nobj); flushptrbuf(&sbuf);
} }
} }
goto next_block; goto next_block;
@ -926,7 +953,7 @@ scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking)
if(*(byte**)i != nil) { if(*(byte**)i != nil) {
// Found a value that may be a pointer. // Found a value that may be a pointer.
// Do a rescan of the entire block. // Do a rescan of the entire block.
enqueue((Obj){b, n, 0}, &wbuf, &wp, &nobj); enqueue((Obj){b, n, 0}, &sbuf.wbuf, &sbuf.wp, &sbuf.nobj);
if(CollectStats) { if(CollectStats) {
runtime·xadd64(&gcstats.rescan, 1); runtime·xadd64(&gcstats.rescan, 1);
runtime·xadd64(&gcstats.rescanbytes, n); runtime·xadd64(&gcstats.rescanbytes, n);
@ -972,9 +999,9 @@ scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking)
objti = pc[3]; objti = pc[3];
pc += 4; pc += 4;
*objbufpos++ = (Obj){obj, size, objti}; *sbuf.obj.pos++ = (Obj){obj, size, objti};
if(objbufpos == objbuf_end) if(sbuf.obj.pos == sbuf.obj.end)
flushobjbuf(objbuf, &objbufpos, &wp, &wbuf, &nobj); flushobjbuf(&sbuf);
continue; continue;
case GC_CHAN_PTR: case GC_CHAN_PTR:
@ -1007,10 +1034,10 @@ scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking)
// in-use part of the circular buffer is scanned. // in-use part of the circular buffer is scanned.
// (Channel routines zero the unused part, so the current // (Channel routines zero the unused part, so the current
// code does not lead to leaks, it's just a little inefficient.) // code does not lead to leaks, it's just a little inefficient.)
*objbufpos++ = (Obj){(byte*)chan+runtime·Hchansize, chancap*chantype->elem->size, *sbuf.obj.pos++ = (Obj){(byte*)chan+runtime·Hchansize, chancap*chantype->elem->size,
(uintptr)chantype->elem->gc | PRECISE | LOOP}; (uintptr)chantype->elem->gc | PRECISE | LOOP};
if(objbufpos == objbuf_end) if(sbuf.obj.pos == sbuf.obj.end)
flushobjbuf(objbuf, &objbufpos, &wp, &wbuf, &nobj); flushobjbuf(&sbuf);
} }
} }
if(chan_ret == nil) if(chan_ret == nil)
@ -1018,15 +1045,20 @@ scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking)
pc = chan_ret; pc = chan_ret;
continue; continue;
case GC_G_PTR:
obj = (void*)stack_top.b;
scanstack(obj, &sbuf);
goto next_block;
default: default:
runtime·throw("scanblock: invalid GC instruction"); runtime·throw("scanblock: invalid GC instruction");
return; return;
} }
if(obj >= arena_start && obj < arena_used) { if(obj >= arena_start && obj < arena_used) {
*ptrbufpos++ = (PtrTarget){obj, objti}; *sbuf.ptr.pos++ = (PtrTarget){obj, objti};
if(ptrbufpos == ptrbuf_end) if(sbuf.ptr.pos == sbuf.ptr.end)
flushptrbuf(ptrbuf, &ptrbufpos, &wp, &wbuf, &nobj); flushptrbuf(&sbuf);
} }
} }
@ -1034,34 +1066,32 @@ scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking)
// Done scanning [b, b+n). Prepare for the next iteration of // Done scanning [b, b+n). Prepare for the next iteration of
// the loop by setting b, n, ti to the parameters for the next block. // the loop by setting b, n, ti to the parameters for the next block.
if(nobj == 0) { if(sbuf.nobj == 0) {
flushptrbuf(ptrbuf, &ptrbufpos, &wp, &wbuf, &nobj); flushptrbuf(&sbuf);
flushobjbuf(objbuf, &objbufpos, &wp, &wbuf, &nobj); flushobjbuf(&sbuf);
if(nobj == 0) { if(sbuf.nobj == 0) {
if(!keepworking) { if(!keepworking) {
if(wbuf) if(sbuf.wbuf)
putempty(wbuf); putempty(sbuf.wbuf);
goto endscan; return;
} }
// Emptied our buffer: refill. // Emptied our buffer: refill.
wbuf = getfull(wbuf); sbuf.wbuf = getfull(sbuf.wbuf);
if(wbuf == nil) if(sbuf.wbuf == nil)
goto endscan; return;
nobj = wbuf->nobj; sbuf.nobj = sbuf.wbuf->nobj;
wp = wbuf->obj + wbuf->nobj; sbuf.wp = sbuf.wbuf->obj + sbuf.wbuf->nobj;
} }
} }
// Fetch b from the work buffer. // Fetch b from the work buffer.
--wp; --sbuf.wp;
b = wp->p; b = sbuf.wp->p;
n = wp->n; n = sbuf.wp->n;
ti = wp->ti; ti = sbuf.wp->ti;
nobj--; sbuf.nobj--;
} }
endscan:;
} }
// debug_scanblock is the debug copy of scanblock. // debug_scanblock is the debug copy of scanblock.
@ -1340,7 +1370,7 @@ struct BitVector
// Scans an interface data value when the interface type indicates // Scans an interface data value when the interface type indicates
// that it is a pointer. // that it is a pointer.
static void static void
scaninterfacedata(uintptr bits, byte *scanp, bool afterprologue) scaninterfacedata(uintptr bits, byte *scanp, bool afterprologue, Scanbuf *sbuf)
{ {
Itab *tab; Itab *tab;
Type *type; Type *type;
@ -1356,12 +1386,14 @@ scaninterfacedata(uintptr bits, byte *scanp, bool afterprologue)
return; return;
} }
} }
addroot((Obj){scanp+PtrSize, PtrSize, 0}); *sbuf->obj.pos++ = (Obj){scanp+PtrSize, PtrSize, 0};
if(sbuf->obj.pos == sbuf->obj.end)
flushobjbuf(sbuf);
} }
// Starting from scanp, scans words corresponding to set bits. // Starting from scanp, scans words corresponding to set bits.
static void static void
scanbitvector(byte *scanp, BitVector *bv, bool afterprologue) scanbitvector(byte *scanp, BitVector *bv, bool afterprologue, Scanbuf *sbuf)
{ {
uintptr word, bits; uintptr word, bits;
uint32 *wordp; uint32 *wordp;
@ -1378,71 +1410,28 @@ scanbitvector(byte *scanp, BitVector *bv, bool afterprologue)
for(; i > 0; i--) { for(; i > 0; i--) {
bits = word & 3; bits = word & 3;
if(bits != BitsNoPointer && *(void**)scanp != nil) if(bits != BitsNoPointer && *(void**)scanp != nil)
if(bits == BitsPointer) if(bits == BitsPointer) {
addroot((Obj){scanp, PtrSize, 0}); *sbuf->obj.pos++ = (Obj){scanp, PtrSize, 0};
else if(sbuf->obj.pos == sbuf->obj.end)
scaninterfacedata(bits, scanp, afterprologue); flushobjbuf(sbuf);
} else
scaninterfacedata(bits, scanp, afterprologue, sbuf);
word >>= BitsPerPointer; word >>= BitsPerPointer;
scanp += PtrSize; scanp += PtrSize;
} }
} }
} }
// Scan a stack frame: local variables and function arguments/results.
static void
addframeroots(Stkframe *frame, void*)
{
Func *f;
BitVector *args, *locals;
uintptr size;
bool afterprologue;
f = frame->fn;
// Scan local variables if stack frame has been allocated.
// Use pointer information if known.
afterprologue = (frame->varp > (byte*)frame->sp);
if(afterprologue) {
locals = runtime·funcdata(f, FUNCDATA_GCLocals);
if(locals == nil) {
// No locals information, scan everything.
size = frame->varp - (byte*)frame->sp;
addroot((Obj){frame->varp - size, size, 0});
} else if(locals->n < 0) {
// Locals size information, scan just the
// locals.
size = -locals->n;
addroot((Obj){frame->varp - size, size, 0});
} else if(locals->n > 0) {
// Locals bitmap information, scan just the
// pointers in locals.
size = (locals->n*PtrSize) / BitsPerPointer;
scanbitvector(frame->varp - size, locals, afterprologue);
}
}
// Scan arguments.
// Use pointer information if known.
args = runtime·funcdata(f, FUNCDATA_GCArgs);
if(args != nil && args->n > 0)
scanbitvector(frame->argp, args, false);
else
addroot((Obj){frame->argp, frame->arglen, 0});
}
static void static void
addstackroots(G *gp) addstackroots(G *gp)
{ {
M *mp; M *mp;
int32 n; int32 n;
Stktop *stk; Stktop *stk;
uintptr sp, guard, pc, lr; uintptr sp, guard;
void *base; void *base;
uintptr size; uintptr size;
stk = (Stktop*)gp->stackbase;
guard = gp->stackguard;
if(gp == g) if(gp == g)
runtime·throw("can't scan our own stack"); runtime·throw("can't scan our own stack");
if((mp = gp->m) != nil && mp->helpgc) if((mp = gp->m) != nil && mp->helpgc)
@ -1454,28 +1443,24 @@ addstackroots(G *gp)
// Use the stack segment and stack pointer at the time of // Use the stack segment and stack pointer at the time of
// the system call instead, since that won't change underfoot. // the system call instead, since that won't change underfoot.
sp = gp->syscallsp; sp = gp->syscallsp;
pc = gp->syscallpc;
lr = 0;
stk = (Stktop*)gp->syscallstack; stk = (Stktop*)gp->syscallstack;
guard = gp->syscallguard; guard = gp->syscallguard;
} else { } else {
// Scanning another goroutine's stack. // Scanning another goroutine's stack.
// The goroutine is usually asleep (the world is stopped). // The goroutine is usually asleep (the world is stopped).
sp = gp->sched.sp; sp = gp->sched.sp;
pc = gp->sched.pc; stk = (Stktop*)gp->stackbase;
lr = gp->sched.lr; guard = gp->stackguard;
// For function about to start, context argument is a root too. // For function about to start, context argument is a root too.
if(gp->sched.ctxt != 0 && runtime·mlookup(gp->sched.ctxt, &base, &size, nil)) if(gp->sched.ctxt != 0 && runtime·mlookup(gp->sched.ctxt, &base, &size, nil))
addroot((Obj){base, size, 0}); addroot((Obj){base, size, 0});
} }
if(ScanStackByFrames) { if(ScanStackByFrames) {
USED(sp);
USED(stk); USED(stk);
USED(guard); USED(guard);
runtime·gentraceback(pc, sp, lr, gp, 0, nil, 0x7fffffff, addframeroots, nil, false); addroot((Obj){(byte*)gp, PtrSize, (uintptr)gptrProg});
} else { } else {
USED(lr);
USED(pc);
n = 0; n = 0;
while(stk) { while(stk) {
if(sp < guard-StackGuard || (uintptr)stk < sp) { if(sp < guard-StackGuard || (uintptr)stk < sp) {
@ -1491,6 +1476,79 @@ addstackroots(G *gp)
} }
} }
static void
scanframe(Stkframe *frame, void *arg)
{
BitVector *args, *locals;
Scanbuf *sbuf;
uintptr size;
bool afterprologue;
sbuf = arg;
// Scan local variables if stack frame has been allocated.
// Use pointer information if known.
afterprologue = (frame->varp > (byte*)frame->sp);
if(afterprologue) {
locals = runtime·funcdata(frame->fn, FUNCDATA_GCLocals);
if(locals == nil) {
// No locals information, scan everything.
size = frame->varp - (byte*)frame->sp;
*sbuf->obj.pos++ = (Obj){frame->varp - size, size, 0};
if(sbuf->obj.pos == sbuf->obj.end)
flushobjbuf(sbuf);
} else if(locals->n < 0) {
// Locals size information, scan just the
// locals.
size = -locals->n;
*sbuf->obj.pos++ = (Obj){frame->varp - size, size, 0};
if(sbuf->obj.pos == sbuf->obj.end)
flushobjbuf(sbuf);
} else if(locals->n > 0) {
// Locals bitmap information, scan just the
// pointers in locals.
size = (locals->n*PtrSize) / BitsPerPointer;
scanbitvector(frame->varp - size, locals, afterprologue, sbuf);
}
}
// Scan arguments.
// Use pointer information if known.
args = runtime·funcdata(frame->fn, FUNCDATA_GCArgs);
if(args != nil && args->n > 0)
scanbitvector(frame->argp, args, false, sbuf);
else {
*sbuf->obj.pos++ = (Obj){frame->argp, frame->arglen, 0};
if(sbuf->obj.pos == sbuf->obj.end)
flushobjbuf(sbuf);
}
}
static void
scanstack(G* gp, void *scanbuf)
{
uintptr pc;
uintptr sp;
uintptr lr;
if(gp->syscallstack != (uintptr)nil) {
// Scanning another goroutine that is about to enter or might
// have just exited a system call. It may be executing code such
// as schedlock and may have needed to start a new stack segment.
// Use the stack segment and stack pointer at the time of
// the system call instead, since that won't change underfoot.
sp = gp->syscallsp;
pc = gp->syscallpc;
lr = 0;
} else {
// Scanning another goroutine's stack.
// The goroutine is usually asleep (the world is stopped).
sp = gp->sched.sp;
pc = gp->sched.pc;
lr = gp->sched.lr;
}
runtime·gentraceback(pc, sp, lr, gp, 0, nil, 0x7fffffff, scanframe, scanbuf, false);
}
static void static void
addfinroots(void *v) addfinroots(void *v)
{ {