qbit/go
1
0
Fork 0
mirror of https://github.com/golang/go synced 2024-11-19 12:54:45 -07:00
Code Releases Activity

[dev.garbage] runtime: fix a few checkmark bugs

Browse Source 
- Some sequencing issues with stopping the first gc_m round
at the right place to set up correctly for the second round.

- atomicxor8 is not idempotent; avoid xor.

- Maintain BitsDead type bits correctly; see long comment added.

- Enable checkmark phase by default for now.

LGTM=rlh
R=rlh
CC=golang-codereviews
https://golang.org/cl/171090043
This commit is contained in:
Russ Cox 2014-11-05 13:37:34 -05:00
parent e1f7c3f82f
commit 0584312e7d
1 changed files with 111 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

142
src/runtime/mgc0.c
View File

@ -257,7 +257,7 @@ WorkData runtime·work;
// encoding, otherwise one uses the bitMarked bit in the lower two bits
// of the nibble.
static bool checkmark = false;
static bool gccheckmarkenable = false;
static bool gccheckmarkenable = true;
// Is address b in the known heap. If it doesn't have a valid gcmap
// returns false. For example pointers into stacks will return false.
@ -292,7 +292,7 @@ slottombits(byte *obj, Markbits *mbits)
mbits->shift = (off % wordsPerBitmapByte) * gcBits;
mbits->xbits = *mbits->bitp;
mbits->bits = (mbits->xbits >> mbits->shift) & bitMask;
mbits->tbits = (mbits->xbits >> mbits->shift) & bitPtrMask;
mbits->tbits = ((mbits->xbits >> mbits->shift) & bitPtrMask) >> 2;
}
// b is a pointer into the heap.
@ -354,13 +354,13 @@ objectstart(byte *b, Markbits *mbits)
// Slow for now as we serialize this, since this is on a debug path
// speed is not critical at this point.
static Mutex xorlock;
static Mutex andlock;
static void
atomicxor8(byte *src, byte val)
atomicand8(byte *src, byte val)
{
runtime·lock(&xorlock);
*src = *src^val;
runtime·unlock(&xorlock);
runtime·lock(&andlock);
*src = *src&val;
runtime·unlock(&andlock);
}
// Mark using the checkmark scheme.
@ -369,7 +369,16 @@ docheckmark(Markbits *mbits)
{
// xor 01 moves 01(scalar unmarked) to 00(scalar marked)
// and 10(pointer unmarked) to 11(pointer marked)
atomicxor8(mbits->bitp, BitsCheckMarkXor<<mbits->shift<<2);
if(mbits->tbits == BitsScalar)
atomicand8(mbits->bitp, ~(byte)(BitsCheckMarkXor<<mbits->shift<<2));
else if(mbits->tbits == BitsPointer)
runtime·atomicor8(mbits->bitp, BitsCheckMarkXor<<mbits->shift<<2);
// reload bits for ischeckmarked
mbits->xbits = *mbits->bitp;
mbits->bits = (mbits->xbits >> mbits->shift) & bitMask;
mbits->tbits = ((mbits->xbits >> mbits->shift) & bitPtrMask) >> 2;
return;
}
@ -434,10 +443,15 @@ greyobject(byte *obj, Markbits *mbits, Workbuf *wbuf)
if(checkmark) {
if(!ismarked(mbits)) {
runtime·printf("runtime:greyobject: checkmarks finds unexpected unmarked object obj=%p, mbits->bits=%x, *mbits->bitp=%x\n", obj, mbits->bits, *mbits->bitp);
runtime·throw("checkmark found unmarked object");
}
if(ischeckmarked(mbits))
return wbuf;
docheckmark(mbits);
if(!ischeckmarked(mbits)) {
runtime·printf("mbits xbits=%x bits=%x tbits=%x shift=%d\n", mbits->xbits, mbits->bits, mbits->tbits, mbits->shift);
runtime·throw("docheckmark and ischeckmarked disagree");
}
} else {
// If marked we have nothing to do.
if((mbits->bits&bitMarked) != 0)
@ -1670,9 +1684,6 @@ clearcheckmarkbitsspan(MSpan *s)
uintptr size, off, step;
byte *p, *bitp, *arena_start, b;
if(!checkmark)
runtime·throw("clearcheckmarkbitsspan: checkmark not set.");
if(s->state != MSpanInUse) {
runtime·printf("runtime:clearcheckmarkbitsspan: state=%d\n",
s->state);
@ -1700,19 +1711,65 @@ clearcheckmarkbitsspan(MSpan *s)
bitp = arena_start - off/wordsPerBitmapByte - 1;
step = size/(PtrSize*wordsPerBitmapByte);
// The type bit values are:
// 00 - BitsDead, for us BitsScalarMarked
// 01 - BitsScalar
// 10 - BitsPointer
// 11 - unused, for us BitsPointerMarked
//
// When called to prepare for the checkmark phase (checkmark==1),
// we change BitsDead to BitsScalar, so that there are no BitsScalarMarked
// type bits anywhere.
//
// The checkmark phase marks by changing BitsScalar to BitsScalarMarked
// and BitsPointer to BitsPointerMarked.
//
// When called to clean up after the checkmark phase (checkmark==0),
// we unmark by changing BitsScalarMarked back to BitsScalar and
// BitsPointerMarked back to BitsPointer.
//
// There are two problems with the scheme as just described.
// First, the setup rewrites BitsDead to BitsScalar, but the type bits
// following a BitsDead are uninitialized and must not be used.
// Second, objects that are free are expected to have their type
// bits zeroed (BitsDead), so in the cleanup we need to restore
// any BitsDeads that were there originally.
//
// In a one-word object (8-byte allocation on 64-bit system),
// there is no difference between BitsScalar and BitsDead, because
// neither is a pointer and there are no more words in the object,
// so using BitsScalar during the checkmark is safe and mapping
// both back to BitsDead during cleanup is also safe.
//
// In a larger object, we need to be more careful. During setup,
// if the type of the first word is BitsDead, we change it to BitsScalar
// (as we must) but also initialize the type of the second
// word to BitsDead, so that a scan during the checkmark phase
// will still stop before seeing the uninitialized type bits in the
// rest of the object. The sequence 'BitsScalar BitsDead' never
// happens in real type bitmaps - BitsDead is always as early
// as possible, so immediately after the last BitsPointer.
// During cleanup, if we see a BitsScalar, we can check to see if it
// is followed by BitsDead. If so, it was originally BitsDead and
// we can change it back.
if(step == 0) {
// updating top and bottom nibbles, all boundaries
for(i=0; i<n/2; i++, bitp--) {
if((*bitp & bitBoundary) != bitBoundary)
runtime·throw("missing bitBoundary");
b = (*bitp & bitPtrMask)>>2;
if(b == BitsScalarMarked || b == BitsPointerMarked)
if(!checkmark && (b == BitsScalar || b == BitsScalarMarked))
*bitp &= ~0x0c; // convert to BitsDead
else if(b == BitsScalarMarked || b == BitsPointerMarked)
*bitp ^= BitsCheckMarkXor<<2;
if(((*bitp>>gcBits) & bitBoundary) != bitBoundary)
runtime·throw("missing bitBoundary");
b = ((*bitp>>gcBits) & bitPtrMask)>>2;
if(b == BitsScalarMarked || b == BitsPointerMarked)
if(!checkmark && (b == BitsScalar || b == BitsScalarMarked))
*bitp &= ~0xc0; // convert to BitsDead
else if(b == BitsScalarMarked || b == BitsPointerMarked)
*bitp ^= BitsCheckMarkXor<<(2+gcBits);
}
} else {
@ -1721,7 +1778,19 @@ clearcheckmarkbitsspan(MSpan *s)
if((*bitp & bitBoundary) != bitBoundary)
runtime·throw("missing bitBoundary");
b = (*bitp & bitPtrMask)>>2;
if(b == BitsScalarMarked || b == BitsPointerMarked)
if(checkmark && b == BitsDead) {
// move BitsDead into second word.
// set bits to BitsScalar in preparation for checkmark phase.
*bitp &= ~0xc0;
*bitp |= BitsScalar<<2;
} else if(!checkmark && (b == BitsScalar || b == BitsScalarMarked) && (*bitp & 0xc0) == 0) {
// Cleaning up after checkmark phase.
// First word is scalar or dead (we forgot)
// and second word is dead.
// First word might as well be dead too.
*bitp &= ~0x0c;
} else if(b == BitsScalarMarked || b == BitsPointerMarked)
*bitp ^= BitsCheckMarkXor<<2;
}
}
@ -1763,10 +1832,9 @@ runtime·gccheckmark_m(void)
checkmark = true;
clearcheckmarkbits(); // Converts BitsDead to BitsScalar.
runtime·gc_m();
runtime·gc_m(); // turns off checkmark
// Work done, fixed up the GC bitmap to remove the checkmark bits.
clearcheckmarkbits();
checkmark = false;
}
// checkmarkenable is initially false
@ -2016,6 +2084,16 @@ gc(struct gc_args *args)
// Free the old cached mark array if necessary.
if(runtime·work.spans != nil && runtime·work.spans != runtime·mheap.allspans)
runtime·SysFree(runtime·work.spans, runtime·work.nspan*sizeof(runtime·work.spans[0]), &mstats.other_sys);
if(gccheckmarkenable) {
if(!checkmark) {
// first half of two-pass; don't set up sweep
runtime·unlock(&runtime·mheap.lock);
return;
}
checkmark = false; // done checking marks
}
// Cache the current array for sweeping.
runtime·mheap.gcspans = runtime·mheap.allspans;
runtime·mheap.sweepgen += 2;
@ -2025,24 +2103,22 @@ gc(struct gc_args *args)
runtime·sweep.spanidx = 0;
runtime·unlock(&runtime·mheap.lock);
// Start the sweep after the checkmark phase if there is one.
if(!gccheckmarkenable || checkmark) {
if(ConcurrentSweep && !args->eagersweep) {
runtime·lock(&runtime·gclock);
if(runtime·sweep.g == nil)
runtime·sweep.g = runtime·newproc1(&bgsweepv, nil, 0, 0, gc);
else if(runtime·sweep.parked) {
runtime·sweep.parked = false;
runtime·ready(runtime·sweep.g);
}
runtime·unlock(&runtime·gclock);
} else {
// Sweep all spans eagerly.
while(runtime·sweepone() != -1)
runtime·sweep.npausesweep++;
// Do an additional mProf_GC, because all 'free' events are now real as well.
runtime·mProf_GC();
if(ConcurrentSweep && !args->eagersweep) {
runtime·lock(&runtime·gclock);
if(runtime·sweep.g == nil)
runtime·sweep.g = runtime·newproc1(&bgsweepv, nil, 0, 0, gc);
else if(runtime·sweep.parked) {
runtime·sweep.parked = false;
runtime·ready(runtime·sweep.g);
}
runtime·unlock(&runtime·gclock);
} else {
// Sweep all spans eagerly.
while(runtime·sweepone() != -1)
runtime·sweep.npausesweep++;
// Do an additional mProf_GC, because all 'free' events are now real as well.
runtime·mProf_GC();
}
runtime·mProf_GC();