From acb03b8028bdbb5aa6bc3813b7de62a7202eb65e Mon Sep 17 00:00:00 2001
From: Dmitriy Vyukov <dvyukov@google.com>
Date: Wed, 7 May 2014 19:32:34 +0400
Subject: [PATCH] runtime: optimize markspan Increases throughput by 2x on a
 memory hungry program on 8-node NUMA machine.

LGTM=rsc
R=rsc
CC=golang-codereviews
https://golang.org/cl/100230043
---
 src/pkg/runtime/mgc0.c | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/src/pkg/runtime/mgc0.c b/src/pkg/runtime/mgc0.c
index 70c0c933ad..1ba0c0ee4a 100644
--- a/src/pkg/runtime/mgc0.c
+++ b/src/pkg/runtime/mgc0.c
@@ -2785,7 +2785,7 @@ runtime·checkfreed(void *v, uintptr n)
 void
 runtime·markspan(void *v, uintptr size, uintptr n, bool leftover)
 {
-	uintptr *b, off, shift, i;
+	uintptr *b, *b0, off, shift, i, x;
 	byte *p;
 
 	if((byte*)v+size*n > (byte*)runtime·mheap.arena_used || (byte*)v < runtime·mheap.arena_start)
@@ -2804,6 +2804,9 @@ runtime·markspan(void *v, uintptr size, uintptr n, bool leftover)
 	p = v;
 	if(leftover)	// mark a boundary just past end of last block too
 		n++;
+
+	b0 = nil;
+	x = 0;
 	for(; n-- > 0; p += size) {
 		// Okay to use non-atomic ops here, because we control
 		// the entire span, and each bitmap word has bits for only
@@ -2812,8 +2815,15 @@ runtime·markspan(void *v, uintptr size, uintptr n, bool leftover)
 		off = (uintptr*)p - (uintptr*)runtime·mheap.arena_start;  // word offset
 		b = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1;
 		shift = off % wordsPerBitmapWord;
-		*b = (*b & ~(bitMask<<shift)) | (bitAllocated<<shift);
+		if(b0 != b) {
+			if(b0 != nil)
+				*b0 = x;
+			b0 = b;
+			x = 0;
+		}
+		x |= bitAllocated<<shift;
 	}
+	*b0 = x;
 }
 
 // unmark the span of memory at v of length n bytes.