diff --git a/src/runtime/runtime.c b/src/runtime/runtime.c index 766f16f6d48..5dd4336e794 100644 --- a/src/runtime/runtime.c +++ b/src/runtime/runtime.c @@ -63,7 +63,7 @@ enum MAP_SHARED = 0x0001, MAP_PRIVATE = 0x0002, MAP_FIXED = 0x0010, - MAP_ANON = 0x1000, + MAP_ANON = 0x1000, // not on Linux - TODO(rsc) }; void diff --git a/src/runtime/runtime.h b/src/runtime/runtime.h index 5f2ad18b172..842ac8ed2b2 100644 --- a/src/runtime/runtime.h +++ b/src/runtime/runtime.h @@ -15,6 +15,7 @@ typedef signed long long int int64; typedef unsigned long long int uint64; typedef float float32; typedef double float64; +typedef uint64 uintptr; /* * get rid of C types @@ -69,6 +70,10 @@ enum true = 1, false = 0, }; +enum +{ + SmallFreeClasses = 168, // number of small free lists in malloc +}; /* * structures @@ -103,7 +108,7 @@ struct Array { // must not move anything byte* array; // actual data uint32 nel; // number of elements - uint32 cap; // allocate3d number of elements + uint32 cap; // allocated number of elements byte b[8]; // actual array - may not be contig }; struct Gobuf @@ -152,6 +157,7 @@ struct M M* schedlink; Mem mem; uint32 machport; // Return address for Mach IPC (OS X) + void* freelist[SmallFreeClasses]; }; struct Stktop { diff --git a/usr/rsc/mem/Makefile b/usr/rsc/mem/Makefile new file mode 100644 index 00000000000..8f2eace2483 --- /dev/null +++ b/usr/rsc/mem/Makefile @@ -0,0 +1,33 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +CC=6c -w +GC=6g +LD=6l +O=6 + +TARG=testrandom testrepeat testsizetoclass + +default: $(TARG) + +%.$O: %.c malloc.h + $(CC) $*.c + +%.$O: %.go + $(GC) $*.go + +OFILES=\ + allocator.$O\ + malloc.$O\ + pagemap.$O\ + triv.$O\ + +testrandom.$O: allocator.$O +testrepeat.$O: allocator.$O + +test%: test%.$O $(OFILES) + $(LD) -o $@ $^ + +clean: + rm -f *.$O $(TARG) diff --git a/usr/rsc/mem/allocator.go b/usr/rsc/mem/allocator.go new file mode 100644 index 00000000000..da624fcd5c5 --- /dev/null +++ b/usr/rsc/mem/allocator.go @@ -0,0 +1,12 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package allocator + +export func free(*byte) +export func malloc(int) *byte +export func memset(*byte, int, int) +export var footprint int64 +export var frozen bool +export func testsizetoclass() diff --git a/usr/rsc/mem/malloc.c b/usr/rsc/mem/malloc.c new file mode 100644 index 00000000000..f5461cddf71 --- /dev/null +++ b/usr/rsc/mem/malloc.c @@ -0,0 +1,443 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// General C malloc/free, but intended for Go. +// Same design as tcmalloc: +// see https://www/eng/designdocs/tcmalloc/tcmalloc.html + +// TODO: +// * Central free lists. +// * Thread cache stealing. +// * Return memory to the OS. +// * Memory footprint during testrandom is too big. +// * Need to coalesce adjacent free spans. +// +// *** Some way to avoid the ``malloc overflows the stack +// during the stack overflow malloc'' problem. + +#include "malloc.h" + +typedef struct Span Span; +typedef struct Central Central; + +// A Span contains metadata about a range of pages. +enum { + SpanInUse = 0, // span has been handed out by allocator + SpanFree = 1, // span is in central free list +}; +struct Span +{ + Span *next; // in free lists + byte *base; // first byte in span + uintptr length; // number of pages in span + int32 cl; + int32 state; // state (enum above) +// int ref; // reference count if state == SpanInUse (for GC) +// void *type; // object type if state == SpanInUse (for GC) +}; + +// The Central cache contains a list of free spans, +// as well as free lists of small blocks. +struct Central +{ + Lock; + Span *free[256]; + Span *large; // free spans >= MaxPage pages +}; + +static Central central; +static PageMap spanmap; + +// Insert a new span into the map. +static void +insertspan(Span *s) +{ + int32 i; + uintptr base; + + // TODO: This is likely too slow for large spans. + base = (uintptr)s->base >> PageShift; + for(i=0; ilength; i++) + pminsert(&spanmap, base+i, s); +} + +// Record that a span has gotten smaller. +static void +shrinkspan(Span *s, int32 newlength) +{ + int32 i; + uintptr base; + + // TODO: This is unnecessary, because an insertspan is next. + base = (uintptr)s->base >> PageShift; + for(i=newlength; ilength; i++) + pminsert(&spanmap, base+i, nil); + + s->length = newlength; +} + +// Find the span for a given pointer. +static Span* +spanofptr(void *v) +{ + return pmlookup(&spanmap, (uintptr)v >> PageShift); +} + +static void freespan(Span*); + +// Allocate a span of at least n pages. +static Span* +allocspan(int32 npage) +{ + Span *s, **l, *s1; + int32 allocnpage, i; + + // Look in the n-page free lists for big enough n. + for(i=npage; inext; + goto havespan; + } + } + + // Look in the large list, which has large runs of pages. + for(l=¢ral.large; (s=*l) != nil; l=&s->next) { + if(s->length >= npage) { + *l = s->next; + s->next = nil; +if(s->length > npage) { +prints("Chop span"); +sys·printint(s->length); +prints(" for "); +sys·printint(npage); +prints("\n"); +} + goto havespan; + } + } + + // Otherwise we need more memory. + // TODO: Could try to release this lock while asking for memory. + s = trivalloc(sizeof *s); + allocnpage = npage; + if(allocnpage < (1<<20>>PageShift)) // TODO: Tune + allocnpage = (1<<20>>PageShift); + s->length = allocnpage; +prints("New span "); +sys·printint(allocnpage); +prints(" for "); +sys·printint(npage); +prints("\n"); + s->base = trivalloc(allocnpage<length > npage) { + s1 = trivalloc(sizeof *s); + s1->base = s->base + (npage << PageShift); + s1->length = s->length - npage; + shrinkspan(s, npage); + insertspan(s1); + freespan(s1); + } + s->state = SpanInUse; + return s; +} + +// Free a span. +static void +freespan(Span *s) +{ + Span **l; + Span *ss; + + s->state = SpanFree; + if(s->length < nelem(central.free)) { + s->next = central.free[s->length]; + central.free[s->length] = s; + } else { + // Keep central.large sorted in + // increasing size for best-fit allocation. + for(l = ¢ral.large; (ss=*l) != nil; l=&ss->next) + if(ss->length >= s->length) + break; + s->next = *l; + *l = s; + } +} + +// Small objects are kept on per-size free lists in the M. +// There are SmallFreeClasses (defined in runtime.h) different lists. +static int32 classtosize[SmallFreeClasses] = { + /* + seq 8 8 127 | sed 's/$/,/' | fmt + seq 128 16 255 | sed 's/$/,/' | fmt + seq 256 32 511 | sed 's/$/,/' | fmt + seq 512 64 1023 | sed 's/$/,/' | fmt + seq 1024 128 2047 | sed 's/$/,/' | fmt + seq 2048 256 32768 | sed 's/$/,/' | fmt + */ + 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, + 128, 144, 160, 176, 192, 208, 224, 240, + 256, 288, 320, 352, 384, 416, 448, 480, + 512, 576, 640, 704, 768, 832, 896, 960, + 1024, 1152, 1280, 1408, 1536, 1664, 1792, 1920, + 2048, 2304, 2560, 2816, 3072, 3328, 3584, 3840, 4096, 4352, 4608, + 4864, 5120, 5376, 5632, 5888, 6144, 6400, 6656, 6912, 7168, 7424, + 7680, 7936, 8192, 8448, 8704, 8960, 9216, 9472, 9728, 9984, 10240, + 10496, 10752, 11008, 11264, 11520, 11776, 12032, 12288, 12544, + 12800, 13056, 13312, 13568, 13824, 14080, 14336, 14592, 14848, + 15104, 15360, 15616, 15872, 16128, 16384, 16640, 16896, 17152, + 17408, 17664, 17920, 18176, 18432, 18688, 18944, 19200, 19456, + 19712, 19968, 20224, 20480, 20736, 20992, 21248, 21504, 21760, + 22016, 22272, 22528, 22784, 23040, 23296, 23552, 23808, 24064, + 24320, 24576, 24832, 25088, 25344, 25600, 25856, 26112, 26368, + 26624, 26880, 27136, 27392, 27648, 27904, 28160, 28416, 28672, + 28928, 29184, 29440, 29696, 29952, 30208, 30464, 30720, 30976, + 31232, 31488, 31744, 32000, 32256, 32512, 32768, +}; +enum { + LargeSize = 32768 +}; + +// Trigger compile error if nelem(classtosize) != SmallFreeClasses. +static int32 zzz1[SmallFreeClasses-nelem(classtosize)+1]; +static int32 zzz2[nelem(classtosize)-SmallFreeClasses+1]; + +static int32 +sizetoclass(int32 siz) +{ + if(siz <= 0) + return 0; + if(siz <= 128) + return (siz-1) >> 3; + if(siz <= 256) + return ((siz-1) >> 4) + 8; + if(siz <= 512) + return ((siz-1) >> 5) + 16; + if(siz <= 1024) + return ((siz-1) >> 6) + 24; + if(siz <= 2048) + return ((siz-1) >> 7) + 32; + if(siz <= 32768) + return ((siz-1) >> 8) + 40; + throw("sizetoclass - invalid size"); + return -1; +} + +void +allocator·testsizetoclass(void) +{ + int32 i, n; + + n = 0; + for(i=0; i 1<<20) { + chunk = 1<<20; + } + chunk = (chunk+PageMask) & ~PageMask; + s = allocspan(chunk>>PageShift); +prints("New Class "); +sys·printint(cl); +prints("\n"); + s->state = SpanInUse; + s->cl = cl; + siz = classtosize[cl]; + n = chunk/siz; + p = s->base; + for(i=0; i= SmallFreeClasses) + throw("allocsmall - invalid class"); + + // try m-local cache. + p = m->freelist[cl]; + if(p == nil) { + // otherwise grab some blocks from central cache. + lock(¢ral); + p = centralgrab(cl, &n); + // TODO: update local counters using n + unlock(¢ral); + } + + // advance linked list. + m->freelist[cl] = *p; + + // Blocks on free list are zeroed except for + // the linked list pointer that we just used. Zero it. + *p = 0; + + return p; +} + +// Allocate large object of np pages. +void* +alloclarge(int32 np) +{ + Span *s; + + lock(¢ral); +//prints("Alloc span "); +//sys·printint(np); +//prints("\n"); + s = allocspan(np); + unlock(¢ral); + s->state = SpanInUse; + s->cl = -1; + return s->base; +} + +// Allocate object of n bytes. +void* +alloc(int32 n) +{ + int32 cl, np; + + if(n < LargeSize) { + cl = sizetoclass(n); + if(cl < 0 || cl >= SmallFreeClasses) { + sys·printint(n); + prints(" -> "); + sys·printint(cl); + prints("\n"); + throw("alloc - logic error"); + } + return allocsmall(sizetoclass(n)); + } + + // count number of pages; careful about overflow for big n. + np = (n>>PageShift) + (((n&PageMask)+PageMask)>>PageShift); + return alloclarge(np); +} + +void +allocator·malloc(int32 n, byte *out) +{ + out = alloc(n); + FLUSH(&out); +} + +// Free object with base pointer v. +void +free(void *v) +{ + void **p; + Span *s; + int32 siz, off; + + s = spanofptr(v); + if(s->state != SpanInUse) + throw("free - invalid pointer1"); + + // Big object should be s->base. + if(s->cl < 0) { + if(v != s->base) + throw("free - invalid pointer2"); + // TODO: For large spans, maybe just return the + // memory to the operating system and let it zero it. + sys·memclr(s->base, s->length << PageShift); +//prints("Free big "); +//sys·printint(s->length); +//prints("\n"); + lock(¢ral); + freespan(s); + unlock(¢ral); + return; + } + + // Small object should be aligned properly. + siz = classtosize[s->cl]; + off = (byte*)v - (byte*)s->base; + if(off%siz) + throw("free - invalid pointer3"); + + // Zero and add to free list. + sys·memclr(v, siz); + p = v; + *p = m->freelist[s->cl]; + m->freelist[s->cl] = p; +} + +void +allocator·free(byte *v) +{ + free(v); +} + +void +allocator·memset(byte *v, int32 c, int32 n) +{ + int32 i; + + for(i=0; ilevel0[0]; + for(i=0; i> (PMBits - PMLevelBits)) & PMLevelMask; + pn <<= PMLevelBits; + + // Walk down using index. + v = v[x]; + if(v == nil) + return nil; + } + return v; +} + +// Set the entry for page number pn in m to s. +// Return the old value. +void* +pminsert(PageMap *m, uintptr pn, void *value) +{ + int32 i, x; + void **v, **l; + + l = nil; // shut up 6c + v = &m->level0[0]; + for(i=0; i> (PMBits - PMLevelBits)) & PMLevelMask; + pn <<= PMLevelBits; + + // Walk down using index, but remember location of pointer. + l = &v[x]; + v = *l; + + // Allocate new level if needed. + if(v == nil && i < PMLevels-1) { + v = trivalloc(PMLevelSize * sizeof v[0]); + *l = v; + } + } + + // Record new value and return old. + *l = value; + return v; +} diff --git a/usr/rsc/mem/testrandom.go b/usr/rsc/mem/testrandom.go new file mode 100644 index 00000000000..7115afd6285 --- /dev/null +++ b/usr/rsc/mem/testrandom.go @@ -0,0 +1,62 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "allocator"; + "rand" +) + +var footprint int64; +var allocated int64; +func bigger() { + if footprint < allocator.footprint { + footprint = allocator.footprint; + println("Footprint", footprint, " for ", allocated); + if footprint > 1e9 { + panicln("too big"); + } + } +} + +// Prime the data structures by allocating one of +// each block in order. After this, there should be +// little reason to ask for more memory from the OS. +func prime() { + for i := 0; i < 16; i++ { + b := allocator.malloc(1<> (11 + rand.urand32() % 20); + base := allocator.malloc(siz); + blocks[b].base = base; + blocks[b].siz = siz; + allocated += int64(siz); + // println("Alloc", siz, base); + allocator.memset(base, 0xbb, siz); + bigger(); + } +} diff --git a/usr/rsc/mem/testrepeat.go b/usr/rsc/mem/testrepeat.go new file mode 100644 index 00000000000..caa9653d1eb --- /dev/null +++ b/usr/rsc/mem/testrepeat.go @@ -0,0 +1,37 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "allocator" +) + +var footprint int64 +func bigger() { + if footprint < allocator.footprint { + footprint = allocator.footprint; + println("Footprint", footprint); + } +} + +func main() { + for i := 0; i < 1<<16; i++ { + for j := 1; j <= 1<<22; j<<=1 { + if i == 0 { + println("First alloc:", j); + } + b := allocator.malloc(j); + allocator.free(b); + bigger(); + } + if i%(1<<10) == 0 { + println(i); + } + if i == 0 { + println("Primed", i); + allocator.frozen = true; + } + } +} diff --git a/usr/rsc/mem/testsizetoclass.go b/usr/rsc/mem/testsizetoclass.go new file mode 100644 index 00000000000..f92d843fa7e --- /dev/null +++ b/usr/rsc/mem/testsizetoclass.go @@ -0,0 +1,11 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import "allocator" + +func main() { + allocator.testsizetoclass() +} diff --git a/usr/rsc/mem/triv.c b/usr/rsc/mem/triv.c new file mode 100644 index 00000000000..631e93a0940 --- /dev/null +++ b/usr/rsc/mem/triv.c @@ -0,0 +1,64 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Trivial base allocator. + +#include "malloc.h" + +// TODO: The call to sys·mmap should be a call to an assembly +// function sys·mmapnew that takes only a size parameter. +enum +{ + PROT_NONE = 0x00, + PROT_READ = 0x01, + PROT_WRITE = 0x02, + PROT_EXEC = 0x04, + + MAP_FILE = 0x0000, + MAP_SHARED = 0x0001, + MAP_PRIVATE = 0x0002, + MAP_FIXED = 0x0010, + MAP_ANON = 0x1000, +}; + +// Allocate and return zeroed memory. +// Simple allocator for small things like Span structures, +// and also used to grab large amounts of memory for +// the real allocator to hand out. +enum +{ + Round = 15, +}; +void* +trivalloc(int32 size) +{ + static byte *p; + static int32 n; + byte *v; + + if(allocator·frozen) + throw("allocator frozen"); + +//prints("Newmem: "); +//sys·printint(size); +//prints("\n"); + + if(size < 4096) { // TODO: Tune constant. + size = (size + Round) & ~Round; + if(size > n) { + n = 1<<20; // TODO: Tune constant. + p = sys·mmap(nil, n, PROT_READ|PROT_WRITE, MAP_ANON|MAP_PRIVATE, 0, 0); + allocator·footprint += n; + } + v = p; + p += size; + return v; + } + if(size & PageMask) + size += (1<