2008-12-18 16:42:28 -07:00
|
|
|
// Copyright 2009 The Go Authors. All rights reserved.
|
|
|
|
// Use of this source code is governed by a BSD-style
|
|
|
|
// license that can be found in the LICENSE file.
|
|
|
|
|
|
|
|
// See malloc.h for overview.
|
|
|
|
//
|
|
|
|
// TODO(rsc): double-check stats.
|
|
|
|
|
2010-02-03 17:31:34 -07:00
|
|
|
package runtime
|
2008-12-18 16:42:28 -07:00
|
|
|
#include "runtime.h"
|
|
|
|
#include "malloc.h"
|
2009-03-30 01:01:07 -06:00
|
|
|
#include "defs.h"
|
2010-02-03 17:31:34 -07:00
|
|
|
#include "type.h"
|
2008-12-18 16:42:28 -07:00
|
|
|
|
|
|
|
MHeap mheap;
|
|
|
|
MStats mstats;
|
|
|
|
|
2010-03-24 10:40:09 -06:00
|
|
|
extern volatile int32 ·MemProfileRate;
|
|
|
|
|
2010-03-23 21:48:23 -06:00
|
|
|
// Same algorithm from chan.c, but a different
|
|
|
|
// instance of the static uint32 x.
|
|
|
|
// Not protected by a lock - let the threads use
|
|
|
|
// the same random number if they like.
|
|
|
|
static uint32
|
|
|
|
fastrand1(void)
|
|
|
|
{
|
|
|
|
static uint32 x = 0x49f6428aUL;
|
|
|
|
|
|
|
|
x += x;
|
|
|
|
if(x & 0x80000000L)
|
|
|
|
x ^= 0x88888eefUL;
|
|
|
|
return x;
|
|
|
|
}
|
|
|
|
|
2008-12-18 16:42:28 -07:00
|
|
|
// Allocate an object of at least size bytes.
|
|
|
|
// Small objects are allocated from the per-thread cache's free lists.
|
|
|
|
// Large objects (> 32 kB) are allocated straight from the heap.
|
|
|
|
void*
|
2010-03-23 21:48:23 -06:00
|
|
|
mallocgc(uintptr size, uint32 refflag, int32 dogc, int32 zeroed, int32 skip_depth)
|
2008-12-18 16:42:28 -07:00
|
|
|
{
|
2010-03-24 10:40:09 -06:00
|
|
|
int32 sizeclass, rate;
|
2008-12-18 16:42:28 -07:00
|
|
|
MCache *c;
|
|
|
|
uintptr npages;
|
|
|
|
MSpan *s;
|
|
|
|
void *v;
|
2009-01-26 18:37:05 -07:00
|
|
|
uint32 *ref;
|
2008-12-18 16:42:28 -07:00
|
|
|
|
2010-01-12 11:03:02 -07:00
|
|
|
if(gcwaiting && g != m->g0 && m->locks == 0)
|
2010-01-09 10:47:45 -07:00
|
|
|
gosched();
|
2008-12-19 04:13:39 -07:00
|
|
|
if(m->mallocing)
|
2009-05-29 14:31:53 -06:00
|
|
|
throw("malloc/free - deadlock");
|
2008-12-19 04:13:39 -07:00
|
|
|
m->mallocing = 1;
|
2008-12-18 16:42:28 -07:00
|
|
|
if(size == 0)
|
|
|
|
size = 1;
|
|
|
|
|
2009-12-03 18:22:23 -07:00
|
|
|
mstats.nmalloc++;
|
2008-12-18 16:42:28 -07:00
|
|
|
if(size <= MaxSmallSize) {
|
|
|
|
// Allocate from mcache free lists.
|
|
|
|
sizeclass = SizeToClass(size);
|
|
|
|
size = class_to_size[sizeclass];
|
|
|
|
c = m->mcache;
|
2010-02-10 01:00:12 -07:00
|
|
|
v = MCache_Alloc(c, sizeclass, size, zeroed);
|
2008-12-18 16:42:28 -07:00
|
|
|
if(v == nil)
|
2008-12-19 04:13:39 -07:00
|
|
|
throw("out of memory");
|
2008-12-18 16:42:28 -07:00
|
|
|
mstats.alloc += size;
|
2010-02-08 15:32:22 -07:00
|
|
|
mstats.total_alloc += size;
|
|
|
|
mstats.by_size[sizeclass].nmalloc++;
|
2010-02-10 22:23:08 -07:00
|
|
|
|
|
|
|
if(!mlookup(v, nil, nil, nil, &ref)) {
|
|
|
|
printf("malloc %D; mlookup failed\n", (uint64)size);
|
|
|
|
throw("malloc mlookup");
|
|
|
|
}
|
|
|
|
*ref = RefNone | refflag;
|
2008-12-19 04:13:39 -07:00
|
|
|
} else {
|
|
|
|
// TODO(rsc): Report tracebacks for very large allocations.
|
|
|
|
|
|
|
|
// Allocate directly from heap.
|
|
|
|
npages = size >> PageShift;
|
|
|
|
if((size & PageMask) != 0)
|
|
|
|
npages++;
|
2010-03-08 15:15:44 -07:00
|
|
|
s = MHeap_Alloc(&mheap, npages, 0, 1);
|
2008-12-19 04:13:39 -07:00
|
|
|
if(s == nil)
|
|
|
|
throw("out of memory");
|
2010-03-23 21:48:23 -06:00
|
|
|
size = npages<<PageShift;
|
|
|
|
mstats.alloc += size;
|
|
|
|
mstats.total_alloc += size;
|
2008-12-19 04:13:39 -07:00
|
|
|
v = (void*)(s->start << PageShift);
|
2008-12-18 16:42:28 -07:00
|
|
|
|
2010-02-10 22:23:08 -07:00
|
|
|
// setup for mark sweep
|
|
|
|
s->gcref0 = RefNone | refflag;
|
2010-03-23 21:48:23 -06:00
|
|
|
ref = &s->gcref0;
|
2009-06-04 22:09:06 -06:00
|
|
|
}
|
2009-01-26 18:37:05 -07:00
|
|
|
|
2008-12-19 04:13:39 -07:00
|
|
|
m->mallocing = 0;
|
2009-12-03 18:22:23 -07:00
|
|
|
|
2010-03-24 10:40:09 -06:00
|
|
|
if(!(refflag & RefNoProfiling) && (rate = ·MemProfileRate) > 0) {
|
|
|
|
if(size >= rate)
|
|
|
|
goto profile;
|
|
|
|
if(m->mcache->next_sample > size)
|
|
|
|
m->mcache->next_sample -= size;
|
|
|
|
else {
|
|
|
|
// pick next profile time
|
|
|
|
if(rate > 0x3fffffff) // make 2*rate not overflow
|
|
|
|
rate = 0x3fffffff;
|
|
|
|
m->mcache->next_sample = fastrand1() % (2*rate);
|
|
|
|
profile:
|
2010-03-23 21:48:23 -06:00
|
|
|
*ref |= RefProfiled;
|
|
|
|
MProf_Malloc(skip_depth+1, v, size);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2010-03-08 15:15:44 -07:00
|
|
|
if(dogc && mstats.heap_alloc >= mstats.next_gc)
|
2009-12-03 18:22:23 -07:00
|
|
|
gc(0);
|
2008-12-19 04:13:39 -07:00
|
|
|
return v;
|
2008-12-18 16:42:28 -07:00
|
|
|
}
|
|
|
|
|
2009-01-26 18:37:05 -07:00
|
|
|
void*
|
2009-12-03 18:22:23 -07:00
|
|
|
malloc(uintptr size)
|
2009-01-26 18:37:05 -07:00
|
|
|
{
|
2010-03-23 21:48:23 -06:00
|
|
|
return mallocgc(size, 0, 0, 1, 1);
|
2009-01-26 18:37:05 -07:00
|
|
|
}
|
|
|
|
|
2008-12-18 16:42:28 -07:00
|
|
|
// Free the object whose base pointer is v.
|
|
|
|
void
|
|
|
|
free(void *v)
|
|
|
|
{
|
|
|
|
int32 sizeclass, size;
|
|
|
|
MSpan *s;
|
|
|
|
MCache *c;
|
2010-03-23 21:48:23 -06:00
|
|
|
uint32 prof, *ref;
|
2008-12-18 16:42:28 -07:00
|
|
|
|
2009-01-09 17:22:13 -07:00
|
|
|
if(v == nil)
|
|
|
|
return;
|
|
|
|
|
2009-05-29 14:31:53 -06:00
|
|
|
if(m->mallocing)
|
|
|
|
throw("malloc/free - deadlock");
|
|
|
|
m->mallocing = 1;
|
|
|
|
|
2010-02-10 22:23:08 -07:00
|
|
|
if(!mlookup(v, nil, nil, &s, &ref)) {
|
2010-02-03 17:31:34 -07:00
|
|
|
printf("free %p: not an allocated block\n", v);
|
2009-06-04 22:09:06 -06:00
|
|
|
throw("free mlookup");
|
2010-02-03 17:31:34 -07:00
|
|
|
}
|
2010-03-23 21:48:23 -06:00
|
|
|
prof = *ref & RefProfiled;
|
2009-01-26 18:37:05 -07:00
|
|
|
*ref = RefFree;
|
|
|
|
|
2008-12-18 16:42:28 -07:00
|
|
|
// Find size class for v.
|
2010-02-10 22:23:08 -07:00
|
|
|
sizeclass = s->sizeclass;
|
2008-12-18 16:42:28 -07:00
|
|
|
if(sizeclass == 0) {
|
2010-02-10 22:23:08 -07:00
|
|
|
// Large object.
|
2010-03-23 21:48:23 -06:00
|
|
|
if(prof)
|
|
|
|
MProf_Free(v, s->npages<<PageShift);
|
2010-02-10 22:23:08 -07:00
|
|
|
mstats.alloc -= s->npages<<PageShift;
|
|
|
|
runtime_memclr(v, s->npages<<PageShift);
|
2010-03-08 15:15:44 -07:00
|
|
|
MHeap_Free(&mheap, s, 1);
|
2010-02-10 22:23:08 -07:00
|
|
|
} else {
|
|
|
|
// Small object.
|
|
|
|
c = m->mcache;
|
|
|
|
size = class_to_size[sizeclass];
|
|
|
|
if(size > sizeof(uintptr))
|
|
|
|
((uintptr*)v)[1] = 1; // mark as "needs to be zeroed"
|
2010-03-23 21:48:23 -06:00
|
|
|
if(prof)
|
|
|
|
MProf_Free(v, size);
|
2010-02-10 22:23:08 -07:00
|
|
|
mstats.alloc -= size;
|
|
|
|
mstats.by_size[sizeclass].nfree++;
|
|
|
|
MCache_Free(c, v, sizeclass, size);
|
2008-12-18 16:42:28 -07:00
|
|
|
}
|
2009-05-29 14:31:53 -06:00
|
|
|
m->mallocing = 0;
|
2008-12-18 16:42:28 -07:00
|
|
|
}
|
|
|
|
|
2009-01-26 18:37:05 -07:00
|
|
|
int32
|
2010-02-10 22:23:08 -07:00
|
|
|
mlookup(void *v, byte **base, uintptr *size, MSpan **sp, uint32 **ref)
|
2008-12-19 04:13:39 -07:00
|
|
|
{
|
2009-02-06 15:41:21 -07:00
|
|
|
uintptr n, nobj, i;
|
2009-02-11 18:54:03 -07:00
|
|
|
byte *p;
|
2008-12-19 04:13:39 -07:00
|
|
|
MSpan *s;
|
|
|
|
|
2009-12-03 18:22:23 -07:00
|
|
|
mstats.nlookup++;
|
2009-01-26 18:37:05 -07:00
|
|
|
s = MHeap_LookupMaybe(&mheap, (uintptr)v>>PageShift);
|
2010-02-10 22:23:08 -07:00
|
|
|
if(sp)
|
|
|
|
*sp = s;
|
2008-12-19 04:13:39 -07:00
|
|
|
if(s == nil) {
|
2009-01-26 18:37:05 -07:00
|
|
|
if(base)
|
|
|
|
*base = nil;
|
|
|
|
if(size)
|
|
|
|
*size = 0;
|
|
|
|
if(ref)
|
|
|
|
*ref = 0;
|
|
|
|
return 0;
|
2008-12-19 04:13:39 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
p = (byte*)((uintptr)s->start<<PageShift);
|
|
|
|
if(s->sizeclass == 0) {
|
|
|
|
// Large object.
|
2009-01-26 18:37:05 -07:00
|
|
|
if(base)
|
|
|
|
*base = p;
|
|
|
|
if(size)
|
|
|
|
*size = s->npages<<PageShift;
|
|
|
|
if(ref)
|
|
|
|
*ref = &s->gcref0;
|
|
|
|
return 1;
|
2008-12-19 04:13:39 -07:00
|
|
|
}
|
|
|
|
|
2009-02-06 15:41:21 -07:00
|
|
|
if((byte*)v >= (byte*)s->gcref) {
|
|
|
|
// pointers into the gc ref counts
|
|
|
|
// do not count as pointers.
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2008-12-19 04:13:39 -07:00
|
|
|
n = class_to_size[s->sizeclass];
|
2009-01-26 18:37:05 -07:00
|
|
|
i = ((byte*)v - p)/n;
|
|
|
|
if(base)
|
|
|
|
*base = p + i*n;
|
|
|
|
if(size)
|
|
|
|
*size = n;
|
2010-02-10 01:00:12 -07:00
|
|
|
|
|
|
|
// good for error checking, but expensive
|
|
|
|
if(0) {
|
|
|
|
nobj = (s->npages << PageShift) / (n + RefcountOverhead);
|
|
|
|
if((byte*)s->gcref < p || (byte*)(s->gcref+nobj) > p+(s->npages<<PageShift)) {
|
|
|
|
printf("odd span state=%d span=%p base=%p sizeclass=%d n=%D size=%D npages=%D\n",
|
|
|
|
s->state, s, p, s->sizeclass, (uint64)nobj, (uint64)n, (uint64)s->npages);
|
|
|
|
printf("s->base sizeclass %d v=%p base=%p gcref=%p blocksize=%D nobj=%D size=%D end=%p end=%p\n",
|
|
|
|
s->sizeclass, v, p, s->gcref, (uint64)s->npages<<PageShift,
|
|
|
|
(uint64)nobj, (uint64)n, s->gcref + nobj, p+(s->npages<<PageShift));
|
|
|
|
throw("bad gcref");
|
|
|
|
}
|
2009-01-26 18:37:05 -07:00
|
|
|
}
|
|
|
|
if(ref)
|
|
|
|
*ref = &s->gcref[i];
|
|
|
|
|
|
|
|
return 1;
|
2008-12-19 04:13:39 -07:00
|
|
|
}
|
|
|
|
|
2008-12-18 16:42:28 -07:00
|
|
|
MCache*
|
|
|
|
allocmcache(void)
|
|
|
|
{
|
2010-03-29 14:06:26 -06:00
|
|
|
MCache *c;
|
|
|
|
|
|
|
|
c = FixAlloc_Alloc(&mheap.cachealloc);
|
|
|
|
mstats.mcache_inuse = mheap.cachealloc.inuse;
|
|
|
|
mstats.mcache_sys = mheap.cachealloc.sys;
|
|
|
|
return c;
|
2008-12-18 16:42:28 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
void
|
|
|
|
mallocinit(void)
|
|
|
|
{
|
|
|
|
InitSizes();
|
|
|
|
MHeap_Init(&mheap, SysAlloc);
|
|
|
|
m->mcache = allocmcache();
|
|
|
|
|
|
|
|
// See if it works.
|
|
|
|
free(malloc(1));
|
|
|
|
}
|
|
|
|
|
2008-12-19 04:13:39 -07:00
|
|
|
// Runtime stubs.
|
|
|
|
|
|
|
|
void*
|
2010-03-23 21:48:23 -06:00
|
|
|
mal(uintptr n)
|
|
|
|
{
|
|
|
|
return mallocgc(n, 0, 1, 1, 2);
|
|
|
|
}
|
|
|
|
|
|
|
|
void*
|
|
|
|
malx(uintptr n, int32 skip_delta)
|
2008-12-19 04:13:39 -07:00
|
|
|
{
|
2010-03-23 21:48:23 -06:00
|
|
|
return mallocgc(n, 0, 1, 1, 2+skip_delta);
|
2008-12-19 04:13:39 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
// Stack allocator uses malloc/free most of the time,
|
|
|
|
// but if we're in the middle of malloc and need stack,
|
|
|
|
// we have to do something else to avoid deadlock.
|
|
|
|
// In that case, we fall back on a fixed-size free-list
|
|
|
|
// allocator, assuming that inside malloc all the stack
|
|
|
|
// frames are small, so that all the stack allocations
|
|
|
|
// will be a single size, the minimum (right now, 5k).
|
|
|
|
struct {
|
|
|
|
Lock;
|
|
|
|
FixAlloc;
|
|
|
|
} stacks;
|
|
|
|
|
|
|
|
void*
|
|
|
|
stackalloc(uint32 n)
|
|
|
|
{
|
|
|
|
void *v;
|
2009-01-26 18:37:05 -07:00
|
|
|
uint32 *ref;
|
2008-12-19 04:13:39 -07:00
|
|
|
|
2009-06-15 22:31:56 -06:00
|
|
|
if(m->mallocing || m->gcing) {
|
2008-12-19 04:13:39 -07:00
|
|
|
lock(&stacks);
|
|
|
|
if(stacks.size == 0)
|
2009-01-28 16:22:16 -07:00
|
|
|
FixAlloc_Init(&stacks, n, SysAlloc, nil, nil);
|
2008-12-19 04:13:39 -07:00
|
|
|
if(stacks.size != n) {
|
2009-03-30 01:01:07 -06:00
|
|
|
printf("stackalloc: in malloc, size=%D want %d", (uint64)stacks.size, n);
|
2008-12-19 04:13:39 -07:00
|
|
|
throw("stackalloc");
|
|
|
|
}
|
|
|
|
v = FixAlloc_Alloc(&stacks);
|
2010-03-29 14:06:26 -06:00
|
|
|
mstats.stacks_inuse = stacks.inuse;
|
|
|
|
mstats.stacks_sys = stacks.sys;
|
2008-12-19 04:13:39 -07:00
|
|
|
unlock(&stacks);
|
|
|
|
return v;
|
|
|
|
}
|
2010-03-23 21:48:23 -06:00
|
|
|
v = mallocgc(n, RefNoProfiling, 0, 0, 0);
|
2010-02-10 22:23:08 -07:00
|
|
|
if(!mlookup(v, nil, nil, nil, &ref))
|
2009-06-04 22:09:06 -06:00
|
|
|
throw("stackalloc mlookup");
|
2009-01-26 18:37:05 -07:00
|
|
|
*ref = RefStack;
|
|
|
|
return v;
|
2008-12-19 04:13:39 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
void
|
|
|
|
stackfree(void *v)
|
|
|
|
{
|
2009-06-15 22:31:56 -06:00
|
|
|
if(m->mallocing || m->gcing) {
|
2008-12-19 04:13:39 -07:00
|
|
|
lock(&stacks);
|
|
|
|
FixAlloc_Free(&stacks, v);
|
2010-03-29 14:06:26 -06:00
|
|
|
mstats.stacks_inuse = stacks.inuse;
|
|
|
|
mstats.stacks_sys = stacks.sys;
|
2008-12-19 04:13:39 -07:00
|
|
|
unlock(&stacks);
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
free(v);
|
|
|
|
}
|
2009-06-30 21:01:50 -06:00
|
|
|
|
|
|
|
func Alloc(n uintptr) (p *byte) {
|
|
|
|
p = malloc(n);
|
|
|
|
}
|
|
|
|
|
|
|
|
func Free(p *byte) {
|
|
|
|
free(p);
|
|
|
|
}
|
|
|
|
|
|
|
|
func Lookup(p *byte) (base *byte, size uintptr) {
|
2010-02-10 22:23:08 -07:00
|
|
|
mlookup(p, &base, &size, nil, nil);
|
2009-06-30 21:01:50 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
func GC() {
|
|
|
|
gc(1);
|
|
|
|
}
|
2010-02-03 17:31:34 -07:00
|
|
|
|
|
|
|
func SetFinalizer(obj Eface, finalizer Eface) {
|
|
|
|
byte *base;
|
|
|
|
uintptr size;
|
|
|
|
FuncType *ft;
|
2010-02-08 22:41:54 -07:00
|
|
|
int32 i, nret;
|
|
|
|
Type *t;
|
2010-02-10 01:00:12 -07:00
|
|
|
|
2010-02-03 17:31:34 -07:00
|
|
|
if(obj.type == nil) {
|
|
|
|
printf("runtime.SetFinalizer: first argument is nil interface\n");
|
|
|
|
throw:
|
|
|
|
throw("runtime.SetFinalizer");
|
|
|
|
}
|
|
|
|
if(obj.type->kind != KindPtr) {
|
|
|
|
printf("runtime.SetFinalizer: first argument is %S, not pointer\n", *obj.type->string);
|
|
|
|
goto throw;
|
|
|
|
}
|
2010-02-10 22:23:08 -07:00
|
|
|
if(!mlookup(obj.data, &base, &size, nil, nil) || obj.data != base) {
|
2010-02-03 17:31:34 -07:00
|
|
|
printf("runtime.SetFinalizer: pointer not at beginning of allocated block\n");
|
|
|
|
goto throw;
|
|
|
|
}
|
2010-02-08 22:41:54 -07:00
|
|
|
nret = 0;
|
2010-02-03 17:31:34 -07:00
|
|
|
if(finalizer.type != nil) {
|
|
|
|
if(finalizer.type->kind != KindFunc) {
|
|
|
|
badfunc:
|
|
|
|
printf("runtime.SetFinalizer: second argument is %S, not func(%S)\n", *finalizer.type->string, *obj.type->string);
|
|
|
|
goto throw;
|
|
|
|
}
|
|
|
|
ft = (FuncType*)finalizer.type;
|
2010-02-08 22:41:54 -07:00
|
|
|
if(ft->dotdotdot || ft->in.len != 1 || *(Type**)ft->in.array != obj.type)
|
2010-02-03 17:31:34 -07:00
|
|
|
goto badfunc;
|
2010-02-10 01:00:12 -07:00
|
|
|
|
2010-02-08 22:41:54 -07:00
|
|
|
// compute size needed for return parameters
|
|
|
|
for(i=0; i<ft->out.len; i++) {
|
|
|
|
t = ((Type**)ft->out.array)[i];
|
|
|
|
nret = (nret + t->align - 1) & ~(t->align - 1);
|
|
|
|
nret += t->size;
|
|
|
|
}
|
|
|
|
nret = (nret + sizeof(void*)-1) & ~(sizeof(void*)-1);
|
|
|
|
|
2010-03-26 15:15:30 -06:00
|
|
|
if(getfinalizer(obj.data, 0)) {
|
2010-02-03 17:31:34 -07:00
|
|
|
printf("runtime.SetFinalizer: finalizer already set");
|
|
|
|
goto throw;
|
|
|
|
}
|
|
|
|
}
|
2010-02-08 22:41:54 -07:00
|
|
|
addfinalizer(obj.data, finalizer.data, nret);
|
2010-02-03 17:31:34 -07:00
|
|
|
}
|