2008-07-14 15:34:27 -06:00
|
|
|
// Copyright 2009 The Go Authors. All rights reserved.
|
|
|
|
// Use of this source code is governed by a BSD-style
|
|
|
|
// license that can be found in the LICENSE file.
|
|
|
|
|
|
|
|
#include "runtime.h"
|
2010-01-06 18:58:55 -07:00
|
|
|
#include "defs.h"
|
2009-01-26 18:37:05 -07:00
|
|
|
#include "malloc.h"
|
2010-01-06 18:58:55 -07:00
|
|
|
#include "os.h"
|
2008-07-14 15:34:27 -06:00
|
|
|
|
2010-03-29 22:48:22 -06:00
|
|
|
static void unwindstack(G*, byte*);
|
|
|
|
|
2008-08-04 17:43:49 -06:00
|
|
|
typedef struct Sched Sched;
|
|
|
|
|
|
|
|
M m0;
|
|
|
|
G g0; // idle goroutine for m0
|
|
|
|
|
2008-07-14 15:34:27 -06:00
|
|
|
static int32 debug = 0;
|
|
|
|
|
2010-01-09 10:47:45 -07:00
|
|
|
int32 gcwaiting;
|
|
|
|
|
2008-08-05 15:18:47 -06:00
|
|
|
// Go scheduler
|
|
|
|
//
|
|
|
|
// The go scheduler's job is to match ready-to-run goroutines (`g's)
|
|
|
|
// with waiting-for-work schedulers (`m's). If there are ready gs
|
|
|
|
// and no waiting ms, ready() will start a new m running in a new
|
|
|
|
// OS thread, so that all ready gs can run simultaneously, up to a limit.
|
|
|
|
// For now, ms never go away.
|
|
|
|
//
|
2009-11-10 20:59:22 -07:00
|
|
|
// By default, Go keeps only one kernel thread (m) running user code
|
|
|
|
// at a single time; other threads may be blocked in the operating system.
|
|
|
|
// Setting the environment variable $GOMAXPROCS or calling
|
|
|
|
// runtime.GOMAXPROCS() will change the number of user threads
|
|
|
|
// allowed to execute simultaneously. $GOMAXPROCS is thus an
|
|
|
|
// approximation of the maximum number of cores to use.
|
2008-08-05 15:18:47 -06:00
|
|
|
//
|
|
|
|
// Even a program that can run without deadlock in a single process
|
|
|
|
// might use more ms if given the chance. For example, the prime
|
|
|
|
// sieve will use as many ms as there are primes (up to sched.mmax),
|
|
|
|
// allowing different stages of the pipeline to execute in parallel.
|
|
|
|
// We could revisit this choice, only kicking off new ms for blocking
|
|
|
|
// system calls, but that would limit the amount of parallel computation
|
|
|
|
// that go would try to do.
|
|
|
|
//
|
|
|
|
// In general, one could imagine all sorts of refinements to the
|
|
|
|
// scheduler, but the goal now is just to get something working on
|
|
|
|
// Linux and OS X.
|
|
|
|
|
2008-08-04 17:43:49 -06:00
|
|
|
struct Sched {
|
|
|
|
Lock;
|
2008-08-05 15:18:47 -06:00
|
|
|
|
|
|
|
G *gfree; // available gs (status == Gdead)
|
2008-08-05 15:21:42 -06:00
|
|
|
|
2008-08-05 15:18:47 -06:00
|
|
|
G *ghead; // gs waiting to run
|
|
|
|
G *gtail;
|
|
|
|
int32 gwait; // number of gs waiting to run
|
|
|
|
int32 gcount; // number of gs that are alive
|
2008-08-05 15:21:42 -06:00
|
|
|
|
2008-08-05 15:18:47 -06:00
|
|
|
M *mhead; // ms waiting for work
|
|
|
|
int32 mwait; // number of ms waiting for work
|
2008-11-25 17:48:10 -07:00
|
|
|
int32 mcount; // number of ms that have been created
|
|
|
|
int32 mcpu; // number of ms executing on cpu
|
|
|
|
int32 mcpumax; // max number of ms allowed on cpu
|
2008-12-05 16:24:18 -07:00
|
|
|
int32 gomaxprocs;
|
2008-11-25 17:48:10 -07:00
|
|
|
int32 msyscall; // number of ms in system calls
|
2008-08-05 15:21:42 -06:00
|
|
|
|
2008-08-05 15:18:47 -06:00
|
|
|
int32 predawn; // running initialization, don't run new gs.
|
2008-12-05 16:24:18 -07:00
|
|
|
|
|
|
|
Note stopped; // one g can wait here for ms to stop
|
2008-12-08 18:14:08 -07:00
|
|
|
int32 waitstop; // after setting this flag
|
2008-08-04 17:43:49 -06:00
|
|
|
};
|
|
|
|
|
|
|
|
Sched sched;
|
|
|
|
|
2008-08-05 15:18:47 -06:00
|
|
|
// Scheduling helpers. Sched must be locked.
|
|
|
|
static void gput(G*); // put/get on ghead/gtail
|
|
|
|
static G* gget(void);
|
|
|
|
static void mput(M*); // put/get on mhead
|
2009-07-13 18:28:39 -06:00
|
|
|
static M* mget(G*);
|
2008-08-05 15:18:47 -06:00
|
|
|
static void gfput(G*); // put/get on gfree
|
|
|
|
static G* gfget(void);
|
2008-11-25 17:48:10 -07:00
|
|
|
static void matchmg(void); // match ms to gs
|
2008-08-05 15:18:47 -06:00
|
|
|
static void readylocked(G*); // ready, but sched is locked
|
2009-07-13 18:28:39 -06:00
|
|
|
static void mnextg(M*, G*);
|
2008-08-05 15:18:47 -06:00
|
|
|
|
|
|
|
// Scheduler loop.
|
|
|
|
static void scheduler(void);
|
|
|
|
|
2008-09-18 16:56:46 -06:00
|
|
|
// The bootstrap sequence is:
|
|
|
|
//
|
|
|
|
// call osinit
|
|
|
|
// call schedinit
|
|
|
|
// make & queue new G
|
|
|
|
// call mstart
|
|
|
|
//
|
|
|
|
// The new G does:
|
|
|
|
//
|
|
|
|
// call main·init_function
|
|
|
|
// call initdone
|
|
|
|
// call main·main
|
2008-08-05 15:18:47 -06:00
|
|
|
void
|
|
|
|
schedinit(void)
|
|
|
|
{
|
|
|
|
int32 n;
|
|
|
|
byte *p;
|
2009-10-12 11:26:38 -06:00
|
|
|
|
2009-10-09 16:35:33 -06:00
|
|
|
allm = m;
|
2010-03-24 10:40:09 -06:00
|
|
|
m->nomemprof++;
|
2008-08-05 15:21:42 -06:00
|
|
|
|
2008-12-18 16:42:28 -07:00
|
|
|
mallocinit();
|
2009-01-16 15:58:14 -07:00
|
|
|
goargs();
|
|
|
|
|
2009-12-07 16:52:14 -07:00
|
|
|
// For debugging:
|
2008-12-19 04:13:39 -07:00
|
|
|
// Allocate internal symbol table representation now,
|
|
|
|
// so that we don't need to call malloc when we crash.
|
2009-12-07 16:52:14 -07:00
|
|
|
// findfunc(0);
|
2008-12-18 16:42:28 -07:00
|
|
|
|
2008-12-05 16:24:18 -07:00
|
|
|
sched.gomaxprocs = 1;
|
2008-09-17 14:49:23 -06:00
|
|
|
p = getenv("GOMAXPROCS");
|
2008-08-05 15:18:47 -06:00
|
|
|
if(p != nil && (n = atoi(p)) != 0)
|
2008-12-05 16:24:18 -07:00
|
|
|
sched.gomaxprocs = n;
|
|
|
|
sched.mcpumax = sched.gomaxprocs;
|
2008-08-05 15:18:47 -06:00
|
|
|
sched.mcount = 1;
|
|
|
|
sched.predawn = 1;
|
2010-03-24 10:40:09 -06:00
|
|
|
|
|
|
|
m->nomemprof--;
|
2008-08-05 15:18:47 -06:00
|
|
|
}
|
|
|
|
|
2008-09-18 16:56:46 -06:00
|
|
|
// Called after main·init_function; main·main will be called on return.
|
2008-08-05 15:18:47 -06:00
|
|
|
void
|
2008-09-18 16:56:46 -06:00
|
|
|
initdone(void)
|
2008-08-05 15:18:47 -06:00
|
|
|
{
|
|
|
|
// Let's go.
|
|
|
|
sched.predawn = 0;
|
2009-01-26 18:37:05 -07:00
|
|
|
mstats.enablegc = 1;
|
2008-08-05 15:18:47 -06:00
|
|
|
|
|
|
|
// If main·init_function started other goroutines,
|
|
|
|
// kick off new ms to handle them, like ready
|
|
|
|
// would have, had it not been pre-dawn.
|
2008-11-25 17:48:10 -07:00
|
|
|
lock(&sched);
|
|
|
|
matchmg();
|
|
|
|
unlock(&sched);
|
2008-08-05 15:18:47 -06:00
|
|
|
}
|
|
|
|
|
2008-07-14 15:34:27 -06:00
|
|
|
void
|
2009-05-08 16:21:41 -06:00
|
|
|
goexit(void)
|
2008-07-14 15:34:27 -06:00
|
|
|
{
|
2008-08-05 15:18:47 -06:00
|
|
|
g->status = Gmoribund;
|
2009-05-08 16:21:41 -06:00
|
|
|
gosched();
|
2008-07-14 15:34:27 -06:00
|
|
|
}
|
|
|
|
|
2008-07-28 12:29:41 -06:00
|
|
|
void
|
|
|
|
tracebackothers(G *me)
|
|
|
|
{
|
|
|
|
G *g;
|
|
|
|
|
|
|
|
for(g = allg; g != nil; g = g->alllink) {
|
2008-08-04 17:43:49 -06:00
|
|
|
if(g == me || g->status == Gdead)
|
2008-07-28 12:29:41 -06:00
|
|
|
continue;
|
2009-11-17 15:42:08 -07:00
|
|
|
printf("\ngoroutine %d [%d]:\n", g->goid, g->status);
|
2010-04-05 13:51:09 -06:00
|
|
|
traceback(g->sched.pc, g->sched.sp, 0, g);
|
2008-07-28 12:29:41 -06:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2008-08-05 15:18:47 -06:00
|
|
|
// Put on `g' queue. Sched must be locked.
|
2008-08-04 17:43:49 -06:00
|
|
|
static void
|
2008-08-05 15:18:47 -06:00
|
|
|
gput(G *g)
|
2008-07-14 15:34:27 -06:00
|
|
|
{
|
2009-07-13 18:28:39 -06:00
|
|
|
M *m;
|
|
|
|
|
|
|
|
// If g is wired, hand it off directly.
|
2010-06-29 18:47:27 -06:00
|
|
|
if(sched.mcpu < sched.mcpumax && (m = g->lockedm) != nil) {
|
2009-07-13 18:28:39 -06:00
|
|
|
mnextg(m, g);
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
2008-08-05 15:18:47 -06:00
|
|
|
g->schedlink = nil;
|
|
|
|
if(sched.ghead == nil)
|
|
|
|
sched.ghead = g;
|
2008-08-04 17:43:49 -06:00
|
|
|
else
|
2008-08-05 15:18:47 -06:00
|
|
|
sched.gtail->schedlink = g;
|
|
|
|
sched.gtail = g;
|
|
|
|
sched.gwait++;
|
|
|
|
}
|
|
|
|
|
|
|
|
// Get from `g' queue. Sched must be locked.
|
|
|
|
static G*
|
|
|
|
gget(void)
|
|
|
|
{
|
|
|
|
G *g;
|
2008-08-05 15:21:42 -06:00
|
|
|
|
2008-08-05 15:18:47 -06:00
|
|
|
g = sched.ghead;
|
|
|
|
if(g){
|
|
|
|
sched.ghead = g->schedlink;
|
|
|
|
if(sched.ghead == nil)
|
|
|
|
sched.gtail = nil;
|
|
|
|
sched.gwait--;
|
|
|
|
}
|
|
|
|
return g;
|
|
|
|
}
|
|
|
|
|
|
|
|
// Put on `m' list. Sched must be locked.
|
|
|
|
static void
|
|
|
|
mput(M *m)
|
|
|
|
{
|
|
|
|
m->schedlink = sched.mhead;
|
|
|
|
sched.mhead = m;
|
|
|
|
sched.mwait++;
|
|
|
|
}
|
|
|
|
|
2009-07-13 18:28:39 -06:00
|
|
|
// Get an `m' to run `g'. Sched must be locked.
|
2008-08-05 15:18:47 -06:00
|
|
|
static M*
|
2009-07-13 18:28:39 -06:00
|
|
|
mget(G *g)
|
2008-08-05 15:18:47 -06:00
|
|
|
{
|
|
|
|
M *m;
|
2008-08-05 15:21:42 -06:00
|
|
|
|
2009-07-13 18:28:39 -06:00
|
|
|
// if g has its own m, use it.
|
|
|
|
if((m = g->lockedm) != nil)
|
|
|
|
return m;
|
|
|
|
|
|
|
|
// otherwise use general m pool.
|
|
|
|
if((m = sched.mhead) != nil){
|
2008-08-05 15:18:47 -06:00
|
|
|
sched.mhead = m->schedlink;
|
|
|
|
sched.mwait--;
|
|
|
|
}
|
|
|
|
return m;
|
|
|
|
}
|
|
|
|
|
|
|
|
// Mark g ready to run.
|
2008-08-04 17:43:49 -06:00
|
|
|
void
|
|
|
|
ready(G *g)
|
|
|
|
{
|
|
|
|
lock(&sched);
|
2008-08-05 15:18:47 -06:00
|
|
|
readylocked(g);
|
|
|
|
unlock(&sched);
|
|
|
|
}
|
|
|
|
|
2008-09-24 15:13:07 -06:00
|
|
|
// Mark g ready to run. Sched is already locked.
|
|
|
|
// G might be running already and about to stop.
|
|
|
|
// The sched lock protects g->status from changing underfoot.
|
2008-08-05 15:18:47 -06:00
|
|
|
static void
|
|
|
|
readylocked(G *g)
|
|
|
|
{
|
2008-09-24 15:13:07 -06:00
|
|
|
if(g->m){
|
|
|
|
// Running on another machine.
|
|
|
|
// Ready it when it stops.
|
|
|
|
g->readyonstop = 1;
|
|
|
|
return;
|
|
|
|
}
|
2008-08-05 15:18:47 -06:00
|
|
|
|
|
|
|
// Mark runnable.
|
2010-04-21 17:27:41 -06:00
|
|
|
if(g->status == Grunnable || g->status == Grunning || g->status == Grecovery)
|
2008-08-05 15:18:47 -06:00
|
|
|
throw("bad g->status in ready");
|
2008-08-04 17:43:49 -06:00
|
|
|
g->status = Grunnable;
|
2008-08-05 15:18:47 -06:00
|
|
|
|
2008-11-25 17:48:10 -07:00
|
|
|
gput(g);
|
|
|
|
if(!sched.predawn)
|
|
|
|
matchmg();
|
2008-08-05 15:18:47 -06:00
|
|
|
}
|
2008-08-04 17:43:49 -06:00
|
|
|
|
2009-09-18 10:11:19 -06:00
|
|
|
static void
|
|
|
|
nop(void)
|
|
|
|
{
|
|
|
|
}
|
|
|
|
|
2009-08-31 19:10:11 -06:00
|
|
|
// Same as readylocked but a different symbol so that
|
|
|
|
// debuggers can set a breakpoint here and catch all
|
|
|
|
// new goroutines.
|
|
|
|
static void
|
|
|
|
newprocreadylocked(G *g)
|
|
|
|
{
|
2009-09-18 10:11:19 -06:00
|
|
|
nop(); // avoid inlining in 6l
|
2009-08-31 19:10:11 -06:00
|
|
|
readylocked(g);
|
|
|
|
}
|
|
|
|
|
2009-07-13 18:28:39 -06:00
|
|
|
// Pass g to m for running.
|
|
|
|
static void
|
|
|
|
mnextg(M *m, G *g)
|
|
|
|
{
|
|
|
|
sched.mcpu++;
|
|
|
|
m->nextg = g;
|
|
|
|
if(m->waitnextg) {
|
|
|
|
m->waitnextg = 0;
|
|
|
|
notewakeup(&m->havenextg);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2008-08-05 15:18:47 -06:00
|
|
|
// Get the next goroutine that m should run.
|
|
|
|
// Sched must be locked on entry, is unlocked on exit.
|
2008-11-25 17:48:10 -07:00
|
|
|
// Makes sure that at most $GOMAXPROCS gs are
|
|
|
|
// running on cpus (not in system calls) at any given time.
|
2008-08-05 15:18:47 -06:00
|
|
|
static G*
|
|
|
|
nextgandunlock(void)
|
2008-08-04 17:43:49 -06:00
|
|
|
{
|
|
|
|
G *gp;
|
|
|
|
|
2009-07-13 18:28:39 -06:00
|
|
|
if(sched.mcpu < 0)
|
|
|
|
throw("negative sched.mcpu");
|
|
|
|
|
|
|
|
// If there is a g waiting as m->nextg,
|
|
|
|
// mnextg took care of the sched.mcpu++.
|
2008-11-25 17:48:10 -07:00
|
|
|
if(m->nextg != nil) {
|
|
|
|
gp = m->nextg;
|
|
|
|
m->nextg = nil;
|
|
|
|
unlock(&sched);
|
|
|
|
return gp;
|
|
|
|
}
|
|
|
|
|
2009-07-13 18:28:39 -06:00
|
|
|
if(m->lockedg != nil) {
|
|
|
|
// We can only run one g, and it's not available.
|
|
|
|
// Make sure some other cpu is running to handle
|
|
|
|
// the ordinary run queue.
|
|
|
|
if(sched.gwait != 0)
|
|
|
|
matchmg();
|
|
|
|
} else {
|
|
|
|
// Look for work on global queue.
|
|
|
|
while(sched.mcpu < sched.mcpumax && (gp=gget()) != nil) {
|
|
|
|
if(gp->lockedm) {
|
|
|
|
mnextg(gp->lockedm, gp);
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
sched.mcpu++; // this m will run gp
|
|
|
|
unlock(&sched);
|
|
|
|
return gp;
|
2008-11-25 17:48:10 -07:00
|
|
|
}
|
2009-07-13 18:28:39 -06:00
|
|
|
// Otherwise, wait on global m queue.
|
|
|
|
mput(m);
|
2008-07-14 15:34:27 -06:00
|
|
|
}
|
2008-11-25 17:48:10 -07:00
|
|
|
if(sched.mcpu == 0 && sched.msyscall == 0)
|
2008-09-26 15:10:26 -06:00
|
|
|
throw("all goroutines are asleep - deadlock!");
|
2008-08-05 15:18:47 -06:00
|
|
|
m->nextg = nil;
|
2009-07-13 18:28:39 -06:00
|
|
|
m->waitnextg = 1;
|
2008-08-05 15:18:47 -06:00
|
|
|
noteclear(&m->havenextg);
|
2009-01-27 15:01:20 -07:00
|
|
|
if(sched.waitstop && sched.mcpu <= sched.mcpumax) {
|
2008-12-08 18:14:08 -07:00
|
|
|
sched.waitstop = 0;
|
|
|
|
notewakeup(&sched.stopped);
|
|
|
|
}
|
2008-08-05 15:18:47 -06:00
|
|
|
unlock(&sched);
|
2008-08-05 15:21:42 -06:00
|
|
|
|
2008-08-05 15:18:47 -06:00
|
|
|
notesleep(&m->havenextg);
|
|
|
|
if((gp = m->nextg) == nil)
|
|
|
|
throw("bad m->nextg in nextgoroutine");
|
|
|
|
m->nextg = nil;
|
2008-08-04 17:43:49 -06:00
|
|
|
return gp;
|
2008-07-14 15:34:27 -06:00
|
|
|
}
|
|
|
|
|
2008-12-05 16:24:18 -07:00
|
|
|
// TODO(rsc): Remove. This is only temporary,
|
|
|
|
// for the mark and sweep collector.
|
|
|
|
void
|
|
|
|
stoptheworld(void)
|
|
|
|
{
|
|
|
|
lock(&sched);
|
2010-01-09 10:47:45 -07:00
|
|
|
gcwaiting = 1;
|
2008-12-05 16:24:18 -07:00
|
|
|
sched.mcpumax = 1;
|
|
|
|
while(sched.mcpu > 1) {
|
2010-04-06 14:48:31 -06:00
|
|
|
// It would be unsafe for multiple threads to be using
|
|
|
|
// the stopped note at once, but there is only
|
|
|
|
// ever one thread doing garbage collection,
|
|
|
|
// so this is okay.
|
2008-12-05 16:24:18 -07:00
|
|
|
noteclear(&sched.stopped);
|
2008-12-08 18:14:08 -07:00
|
|
|
sched.waitstop = 1;
|
2008-12-05 16:24:18 -07:00
|
|
|
unlock(&sched);
|
|
|
|
notesleep(&sched.stopped);
|
|
|
|
lock(&sched);
|
|
|
|
}
|
|
|
|
unlock(&sched);
|
|
|
|
}
|
|
|
|
|
|
|
|
// TODO(rsc): Remove. This is only temporary,
|
|
|
|
// for the mark and sweep collector.
|
|
|
|
void
|
|
|
|
starttheworld(void)
|
|
|
|
{
|
|
|
|
lock(&sched);
|
2010-01-09 10:47:45 -07:00
|
|
|
gcwaiting = 0;
|
2008-12-05 16:24:18 -07:00
|
|
|
sched.mcpumax = sched.gomaxprocs;
|
|
|
|
matchmg();
|
|
|
|
unlock(&sched);
|
|
|
|
}
|
|
|
|
|
2008-09-18 16:56:46 -06:00
|
|
|
// Called to start an M.
|
|
|
|
void
|
|
|
|
mstart(void)
|
|
|
|
{
|
2010-01-06 20:24:11 -07:00
|
|
|
if(g != m->g0)
|
2009-11-18 17:51:59 -07:00
|
|
|
throw("bad mstart");
|
2008-12-18 16:42:28 -07:00
|
|
|
if(m->mcache == nil)
|
|
|
|
m->mcache = allocmcache();
|
2008-09-18 16:56:46 -06:00
|
|
|
minit();
|
|
|
|
scheduler();
|
|
|
|
}
|
|
|
|
|
2009-10-03 11:37:12 -06:00
|
|
|
// When running with cgo, we call libcgo_thread_start
|
|
|
|
// to start threads for us so that we can play nicely with
|
|
|
|
// foreign code.
|
|
|
|
void (*libcgo_thread_start)(void*);
|
|
|
|
|
|
|
|
typedef struct CgoThreadStart CgoThreadStart;
|
|
|
|
struct CgoThreadStart
|
|
|
|
{
|
|
|
|
M *m;
|
|
|
|
G *g;
|
|
|
|
void (*fn)(void);
|
|
|
|
};
|
|
|
|
|
|
|
|
// Kick off new ms as needed (up to mcpumax).
|
2008-11-25 17:48:10 -07:00
|
|
|
// There are already `other' other cpus that will
|
|
|
|
// start looking for goroutines shortly.
|
|
|
|
// Sched is locked.
|
|
|
|
static void
|
|
|
|
matchmg(void)
|
|
|
|
{
|
|
|
|
G *g;
|
|
|
|
|
2009-11-17 23:00:30 -07:00
|
|
|
if(m->mallocing || m->gcing)
|
2009-11-17 15:42:08 -07:00
|
|
|
return;
|
2008-11-25 17:48:10 -07:00
|
|
|
while(sched.mcpu < sched.mcpumax && (g = gget()) != nil){
|
2009-11-17 15:42:08 -07:00
|
|
|
M *m;
|
|
|
|
|
2009-07-13 18:28:39 -06:00
|
|
|
// Find the m that will run g.
|
|
|
|
if((m = mget(g)) == nil){
|
2009-01-26 18:37:05 -07:00
|
|
|
m = malloc(sizeof(M));
|
2009-10-09 16:35:33 -06:00
|
|
|
// Add to allm so garbage collector doesn't free m
|
|
|
|
// when it is just in a register (R14 on amd64).
|
|
|
|
m->alllink = allm;
|
|
|
|
allm = m;
|
2008-11-25 17:48:10 -07:00
|
|
|
m->id = sched.mcount++;
|
2009-10-03 11:37:12 -06:00
|
|
|
|
|
|
|
if(libcgo_thread_start != nil) {
|
|
|
|
CgoThreadStart ts;
|
2010-02-10 01:00:12 -07:00
|
|
|
// pthread_create will make us a stack.
|
|
|
|
m->g0 = malg(-1);
|
2009-10-03 11:37:12 -06:00
|
|
|
ts.m = m;
|
|
|
|
ts.g = m->g0;
|
|
|
|
ts.fn = mstart;
|
|
|
|
runcgo(libcgo_thread_start, &ts);
|
2010-02-10 01:00:12 -07:00
|
|
|
} else {
|
|
|
|
m->g0 = malg(8192);
|
2009-10-03 11:37:12 -06:00
|
|
|
newosproc(m, m->g0, m->g0->stackbase, mstart);
|
2010-02-10 01:00:12 -07:00
|
|
|
}
|
2008-11-25 17:48:10 -07:00
|
|
|
}
|
2009-07-13 18:28:39 -06:00
|
|
|
mnextg(m, g);
|
2008-11-25 17:48:10 -07:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2008-08-05 15:18:47 -06:00
|
|
|
// Scheduler loop: find g to run, run it, repeat.
|
|
|
|
static void
|
2008-08-02 23:34:04 -06:00
|
|
|
scheduler(void)
|
2008-07-14 15:34:27 -06:00
|
|
|
{
|
|
|
|
G* gp;
|
2008-08-04 17:43:49 -06:00
|
|
|
|
|
|
|
lock(&sched);
|
2009-06-17 16:12:16 -06:00
|
|
|
if(gosave(&m->sched) != 0){
|
2009-10-03 11:37:12 -06:00
|
|
|
gp = m->curg;
|
2010-03-31 12:46:01 -06:00
|
|
|
if(gp->status == Grecovery) {
|
|
|
|
// switched to scheduler to get stack unwound.
|
|
|
|
// don't go through the full scheduling logic.
|
|
|
|
Defer *d;
|
|
|
|
|
|
|
|
d = gp->defer;
|
|
|
|
gp->defer = d->link;
|
|
|
|
|
|
|
|
// unwind to the stack frame with d->sp in it.
|
|
|
|
unwindstack(gp, d->sp);
|
2010-04-08 14:24:53 -06:00
|
|
|
|
2010-03-31 12:46:01 -06:00
|
|
|
// make the deferproc for this d return again,
|
|
|
|
// this time returning 1. function will jump to
|
|
|
|
// standard return epilogue.
|
|
|
|
// the -2*sizeof(uintptr) makes up for the
|
|
|
|
// two extra words that are on the stack at
|
|
|
|
// each call to deferproc.
|
|
|
|
// (the pc we're returning to does pop pop
|
|
|
|
// before it tests the return value.)
|
2010-04-05 13:51:09 -06:00
|
|
|
gp->sched.sp = getcallersp(d->sp - 2*sizeof(uintptr));
|
2010-03-31 12:46:01 -06:00
|
|
|
gp->sched.pc = d->pc;
|
2010-04-21 17:27:41 -06:00
|
|
|
gp->status = Grunning;
|
2010-03-31 12:46:01 -06:00
|
|
|
free(d);
|
|
|
|
gogo(&gp->sched, 1);
|
|
|
|
}
|
2009-10-03 11:37:12 -06:00
|
|
|
|
2008-09-18 16:56:46 -06:00
|
|
|
// Jumped here via gosave/gogo, so didn't
|
2008-08-05 15:18:47 -06:00
|
|
|
// execute lock(&sched) above.
|
|
|
|
lock(&sched);
|
2008-09-26 15:10:26 -06:00
|
|
|
|
2008-09-18 16:56:46 -06:00
|
|
|
if(sched.predawn)
|
|
|
|
throw("init sleeping");
|
2008-08-05 15:18:47 -06:00
|
|
|
|
2009-10-03 11:37:12 -06:00
|
|
|
// Just finished running gp.
|
2008-09-24 15:13:07 -06:00
|
|
|
gp->m = nil;
|
2008-11-25 17:48:10 -07:00
|
|
|
sched.mcpu--;
|
2009-07-13 18:28:39 -06:00
|
|
|
|
|
|
|
if(sched.mcpu < 0)
|
|
|
|
throw("sched.mcpu < 0 in scheduler");
|
2008-08-04 17:43:49 -06:00
|
|
|
switch(gp->status){
|
2008-08-05 15:18:47 -06:00
|
|
|
case Grunnable:
|
2008-08-04 17:43:49 -06:00
|
|
|
case Gdead:
|
2008-08-05 15:18:47 -06:00
|
|
|
// Shouldn't have been running!
|
|
|
|
throw("bad gp->status in sched");
|
2008-08-04 17:43:49 -06:00
|
|
|
case Grunning:
|
2008-08-05 15:18:47 -06:00
|
|
|
gp->status = Grunnable;
|
|
|
|
gput(gp);
|
2008-08-04 17:43:49 -06:00
|
|
|
break;
|
2008-08-05 15:18:47 -06:00
|
|
|
case Gmoribund:
|
|
|
|
gp->status = Gdead;
|
2009-07-13 18:28:39 -06:00
|
|
|
if(gp->lockedm) {
|
|
|
|
gp->lockedm = nil;
|
|
|
|
m->lockedg = nil;
|
|
|
|
}
|
2010-03-29 22:48:22 -06:00
|
|
|
unwindstack(gp, nil);
|
2009-08-26 16:26:09 -06:00
|
|
|
gfput(gp);
|
2008-08-05 15:18:47 -06:00
|
|
|
if(--sched.gcount == 0)
|
2009-05-08 16:21:41 -06:00
|
|
|
exit(0);
|
2008-08-04 17:43:49 -06:00
|
|
|
break;
|
|
|
|
}
|
2008-09-24 15:13:07 -06:00
|
|
|
if(gp->readyonstop){
|
|
|
|
gp->readyonstop = 0;
|
|
|
|
readylocked(gp);
|
|
|
|
}
|
2008-08-04 17:43:49 -06:00
|
|
|
}
|
|
|
|
|
2008-08-05 15:18:47 -06:00
|
|
|
// Find (or wait for) g to run. Unlocks sched.
|
|
|
|
gp = nextgandunlock();
|
2008-09-24 15:13:07 -06:00
|
|
|
gp->readyonstop = 0;
|
2008-08-04 17:43:49 -06:00
|
|
|
gp->status = Grunning;
|
2008-07-14 15:34:27 -06:00
|
|
|
m->curg = gp;
|
2008-09-24 15:13:07 -06:00
|
|
|
gp->m = m;
|
2009-06-17 16:12:16 -06:00
|
|
|
if(gp->sched.pc == (byte*)goexit) // kickoff
|
|
|
|
gogocall(&gp->sched, (void(*)(void))gp->entry);
|
|
|
|
gogo(&gp->sched, 1);
|
2008-07-14 15:34:27 -06:00
|
|
|
}
|
|
|
|
|
2008-08-05 15:18:47 -06:00
|
|
|
// Enter scheduler. If g->status is Grunning,
|
|
|
|
// re-queues g and runs everyone else who is waiting
|
|
|
|
// before running g again. If g->status is Gmoribund,
|
|
|
|
// kills off g.
|
2008-08-02 23:34:04 -06:00
|
|
|
void
|
2009-05-08 16:21:41 -06:00
|
|
|
gosched(void)
|
2008-08-02 23:34:04 -06:00
|
|
|
{
|
2010-01-12 11:03:02 -07:00
|
|
|
if(m->locks != 0)
|
|
|
|
throw("gosched holding locks");
|
2010-01-06 20:24:11 -07:00
|
|
|
if(g == m->g0)
|
2009-01-26 18:37:05 -07:00
|
|
|
throw("gosched of g0");
|
2009-06-17 16:12:16 -06:00
|
|
|
if(gosave(&g->sched) == 0)
|
|
|
|
gogo(&m->sched, 1);
|
2008-08-02 23:34:04 -06:00
|
|
|
}
|
|
|
|
|
2008-11-25 17:48:10 -07:00
|
|
|
// The goroutine g is about to enter a system call.
|
|
|
|
// Record that it's not using the cpu anymore.
|
2010-04-09 14:30:35 -06:00
|
|
|
// This is called only from the go syscall library and cgocall,
|
|
|
|
// not from the low-level system calls used by the runtime.
|
2010-08-13 00:26:54 -06:00
|
|
|
// Entersyscall cannot split the stack: the gosave must
|
|
|
|
// make g->sched refer to the caller's stack pointer.
|
|
|
|
#pragma textflag 7
|
2008-11-25 17:48:10 -07:00
|
|
|
void
|
2010-01-25 19:52:55 -07:00
|
|
|
·entersyscall(void)
|
2008-08-05 15:18:47 -06:00
|
|
|
{
|
2008-11-25 17:48:10 -07:00
|
|
|
lock(&sched);
|
2010-05-06 23:07:03 -06:00
|
|
|
// Leave SP around for gc and traceback.
|
|
|
|
// Do before notewakeup so that gc
|
|
|
|
// never sees Gsyscall with wrong stack.
|
|
|
|
gosave(&g->sched);
|
2009-07-21 20:43:27 -06:00
|
|
|
if(sched.predawn) {
|
|
|
|
unlock(&sched);
|
|
|
|
return;
|
|
|
|
}
|
2008-12-05 16:24:18 -07:00
|
|
|
g->status = Gsyscall;
|
2008-11-25 17:48:10 -07:00
|
|
|
sched.mcpu--;
|
|
|
|
sched.msyscall++;
|
|
|
|
if(sched.gwait != 0)
|
|
|
|
matchmg();
|
2009-01-27 15:01:20 -07:00
|
|
|
if(sched.waitstop && sched.mcpu <= sched.mcpumax) {
|
|
|
|
sched.waitstop = 0;
|
|
|
|
notewakeup(&sched.stopped);
|
|
|
|
}
|
2008-11-25 17:48:10 -07:00
|
|
|
unlock(&sched);
|
|
|
|
}
|
|
|
|
|
|
|
|
// The goroutine g exited its system call.
|
|
|
|
// Arrange for it to run on a cpu again.
|
|
|
|
// This is called only from the go syscall library, not
|
|
|
|
// from the low-level system calls used by the runtime.
|
|
|
|
void
|
2010-01-25 19:52:55 -07:00
|
|
|
·exitsyscall(void)
|
2008-11-25 17:48:10 -07:00
|
|
|
{
|
|
|
|
lock(&sched);
|
2009-07-21 20:43:27 -06:00
|
|
|
if(sched.predawn) {
|
|
|
|
unlock(&sched);
|
|
|
|
return;
|
|
|
|
}
|
2008-11-25 17:48:10 -07:00
|
|
|
sched.msyscall--;
|
|
|
|
sched.mcpu++;
|
|
|
|
// Fast path - if there's room for this m, we're done.
|
|
|
|
if(sched.mcpu <= sched.mcpumax) {
|
2009-12-14 20:06:20 -07:00
|
|
|
g->status = Grunning;
|
2008-11-25 17:48:10 -07:00
|
|
|
unlock(&sched);
|
|
|
|
return;
|
2008-08-05 15:18:47 -06:00
|
|
|
}
|
2009-12-14 20:06:20 -07:00
|
|
|
// Tell scheduler to put g back on the run queue:
|
|
|
|
// mostly equivalent to g->status = Grunning,
|
|
|
|
// but keeps the garbage collector from thinking
|
|
|
|
// that g is running right now, which it's not.
|
|
|
|
g->readyonstop = 1;
|
2008-11-25 17:48:10 -07:00
|
|
|
unlock(&sched);
|
2008-08-05 15:21:42 -06:00
|
|
|
|
2008-11-25 17:48:10 -07:00
|
|
|
// Slow path - all the cpus are taken.
|
|
|
|
// The scheduler will ready g and put this m to sleep.
|
2009-07-13 18:28:39 -06:00
|
|
|
// When the scheduler takes g away from m,
|
2008-11-25 17:48:10 -07:00
|
|
|
// it will undo the sched.mcpu++ above.
|
2009-05-08 16:21:41 -06:00
|
|
|
gosched();
|
2008-08-05 15:18:47 -06:00
|
|
|
}
|
|
|
|
|
2010-04-09 14:30:35 -06:00
|
|
|
// Start scheduling g1 again for a cgo callback.
|
|
|
|
void
|
|
|
|
startcgocallback(G* g1)
|
|
|
|
{
|
|
|
|
lock(&sched);
|
|
|
|
g1->status = Grunning;
|
|
|
|
sched.msyscall--;
|
|
|
|
sched.mcpu++;
|
|
|
|
unlock(&sched);
|
|
|
|
}
|
|
|
|
|
|
|
|
// Stop scheduling g1 after a cgo callback.
|
|
|
|
void
|
|
|
|
endcgocallback(G* g1)
|
|
|
|
{
|
|
|
|
lock(&sched);
|
|
|
|
g1->status = Gsyscall;
|
|
|
|
sched.mcpu--;
|
|
|
|
sched.msyscall++;
|
|
|
|
unlock(&sched);
|
|
|
|
}
|
|
|
|
|
2009-04-01 01:26:00 -06:00
|
|
|
/*
|
|
|
|
* stack layout parameters.
|
|
|
|
* known to linkers.
|
|
|
|
*
|
|
|
|
* g->stackguard is set to point StackGuard bytes
|
|
|
|
* above the bottom of the stack. each function
|
|
|
|
* compares its stack pointer against g->stackguard
|
|
|
|
* to check for overflow. to cut one instruction from
|
|
|
|
* the check sequence for functions with tiny frames,
|
|
|
|
* the stack is allowed to protrude StackSmall bytes
|
|
|
|
* below the stack guard. functions with large frames
|
|
|
|
* don't bother with the check and always call morestack.
|
|
|
|
* the sequences are:
|
|
|
|
*
|
2009-04-02 17:41:53 -06:00
|
|
|
* guard = g->stackguard
|
|
|
|
* frame = function's stack frame size
|
|
|
|
* argsize = size of function arguments (call + return)
|
|
|
|
*
|
2009-04-01 01:26:00 -06:00
|
|
|
* stack frame size <= StackSmall:
|
|
|
|
* CMPQ guard, SP
|
|
|
|
* JHI 3(PC)
|
2009-04-13 16:22:36 -06:00
|
|
|
* MOVQ m->morearg, $(argsize << 32)
|
2009-04-01 01:26:00 -06:00
|
|
|
* CALL sys.morestack(SB)
|
|
|
|
*
|
|
|
|
* stack frame size > StackSmall but < StackBig
|
|
|
|
* LEAQ (frame-StackSmall)(SP), R0
|
|
|
|
* CMPQ guard, R0
|
|
|
|
* JHI 3(PC)
|
2009-04-13 16:22:36 -06:00
|
|
|
* MOVQ m->morearg, $(argsize << 32)
|
2009-04-01 01:26:00 -06:00
|
|
|
* CALL sys.morestack(SB)
|
|
|
|
*
|
|
|
|
* stack frame size >= StackBig:
|
2009-04-13 16:22:36 -06:00
|
|
|
* MOVQ m->morearg, $((argsize << 32) | frame)
|
2009-04-01 01:26:00 -06:00
|
|
|
* CALL sys.morestack(SB)
|
|
|
|
*
|
|
|
|
* the bottom StackGuard - StackSmall bytes are important:
|
|
|
|
* there has to be enough room to execute functions that
|
|
|
|
* refuse to check for stack overflow, either because they
|
|
|
|
* need to be adjacent to the actual caller's frame (sys.deferproc)
|
|
|
|
* or because they handle the imminent stack overflow (sys.morestack).
|
|
|
|
*
|
|
|
|
* for example, sys.deferproc might call malloc,
|
|
|
|
* which does one of the above checks (without allocating a full frame),
|
|
|
|
* which might trigger a call to sys.morestack.
|
|
|
|
* this sequence needs to fit in the bottom section of the stack.
|
|
|
|
* on amd64, sys.morestack's frame is 40 bytes, and
|
|
|
|
* sys.deferproc's frame is 56 bytes. that fits well within
|
|
|
|
* the StackGuard - StackSmall = 128 bytes at the bottom.
|
|
|
|
* there may be other sequences lurking or yet to be written
|
|
|
|
* that require more stack. sys.morestack checks to make sure
|
|
|
|
* the stack has not completely overflowed and should
|
|
|
|
* catch such sequences.
|
|
|
|
*/
|
|
|
|
enum
|
|
|
|
{
|
|
|
|
// byte offset of stack guard (g->stackguard) above bottom of stack.
|
|
|
|
StackGuard = 256,
|
|
|
|
|
|
|
|
// checked frames are allowed to protrude below the guard by
|
|
|
|
// this many bytes. this saves an instruction in the checking
|
|
|
|
// sequence when the stack frame is tiny.
|
|
|
|
StackSmall = 128,
|
|
|
|
|
|
|
|
// extra space in the frame (beyond the function for which
|
|
|
|
// the frame is allocated) is assumed not to be much bigger
|
|
|
|
// than this amount. it may not be used efficiently if it is.
|
|
|
|
StackBig = 4096,
|
|
|
|
};
|
2008-07-14 15:34:27 -06:00
|
|
|
|
|
|
|
void
|
|
|
|
oldstack(void)
|
|
|
|
{
|
2009-06-17 16:12:16 -06:00
|
|
|
Stktop *top, old;
|
2009-04-13 16:22:36 -06:00
|
|
|
uint32 args;
|
2008-07-14 15:34:27 -06:00
|
|
|
byte *sp;
|
2009-06-17 16:12:16 -06:00
|
|
|
G *g1;
|
2010-03-04 16:34:25 -07:00
|
|
|
static int32 goid;
|
2008-07-14 15:34:27 -06:00
|
|
|
|
2009-06-17 16:12:16 -06:00
|
|
|
//printf("oldstack m->cret=%p\n", m->cret);
|
2008-07-14 15:34:27 -06:00
|
|
|
|
2009-06-17 16:12:16 -06:00
|
|
|
g1 = m->curg;
|
|
|
|
top = (Stktop*)g1->stackbase;
|
2008-07-14 15:34:27 -06:00
|
|
|
sp = (byte*)top;
|
2009-06-17 16:12:16 -06:00
|
|
|
old = *top;
|
|
|
|
args = old.args;
|
2009-04-13 16:22:36 -06:00
|
|
|
if(args > 0) {
|
|
|
|
sp -= args;
|
2009-07-08 19:16:09 -06:00
|
|
|
mcpy(top->fp, sp, args);
|
2008-07-14 15:34:27 -06:00
|
|
|
}
|
2010-03-04 16:34:25 -07:00
|
|
|
goid = old.gobuf.g->goid; // fault if g is bad, before gogo
|
2008-07-14 15:34:27 -06:00
|
|
|
|
2010-03-29 22:48:22 -06:00
|
|
|
if(old.free)
|
|
|
|
stackfree(g1->stackguard - StackGuard);
|
2009-06-17 16:12:16 -06:00
|
|
|
g1->stackbase = old.stackbase;
|
|
|
|
g1->stackguard = old.stackguard;
|
2008-07-14 15:34:27 -06:00
|
|
|
|
2009-06-17 16:12:16 -06:00
|
|
|
gogo(&old.gobuf, m->cret);
|
2008-12-04 09:30:54 -07:00
|
|
|
}
|
|
|
|
|
2008-07-14 15:34:27 -06:00
|
|
|
void
|
|
|
|
newstack(void)
|
|
|
|
{
|
2009-04-01 01:26:00 -06:00
|
|
|
int32 frame, args;
|
2008-07-14 15:34:27 -06:00
|
|
|
Stktop *top;
|
|
|
|
byte *stk, *sp;
|
2009-06-17 16:12:16 -06:00
|
|
|
G *g1;
|
|
|
|
Gobuf label;
|
2010-03-29 22:48:22 -06:00
|
|
|
bool free;
|
2008-07-14 15:34:27 -06:00
|
|
|
|
2009-06-17 16:12:16 -06:00
|
|
|
frame = m->moreframe;
|
|
|
|
args = m->moreargs;
|
2010-03-29 22:48:22 -06:00
|
|
|
g1 = m->curg;
|
2010-04-05 13:51:09 -06:00
|
|
|
|
|
|
|
|
2010-03-29 22:48:22 -06:00
|
|
|
if(frame == 1 && args > 0 && m->morebuf.sp - sizeof(Stktop) - args - 32 > g1->stackguard) {
|
|
|
|
// special case: called from reflect.call (frame == 1)
|
|
|
|
// to call code with an arbitrary argument size,
|
|
|
|
// and we have enough space on the current stack.
|
|
|
|
// the new Stktop* is necessary to unwind, but
|
|
|
|
// we don't need to create a new segment.
|
|
|
|
top = (Stktop*)(m->morebuf.sp - sizeof(*top));
|
|
|
|
stk = g1->stackguard - StackGuard;
|
|
|
|
free = false;
|
|
|
|
} else {
|
|
|
|
// allocate new segment.
|
|
|
|
if(frame == 1) // failed reflect.call hint
|
|
|
|
frame = 0;
|
|
|
|
frame += args;
|
|
|
|
if(frame < StackBig)
|
|
|
|
frame = StackBig;
|
|
|
|
frame += 1024; // room for more functions, Stktop.
|
|
|
|
stk = stackalloc(frame);
|
|
|
|
top = (Stktop*)(stk+frame-sizeof(*top));
|
|
|
|
free = true;
|
|
|
|
}
|
2010-02-10 01:00:12 -07:00
|
|
|
|
2009-07-08 19:16:09 -06:00
|
|
|
//printf("newstack frame=%d args=%d morepc=%p morefp=%p gobuf=%p, %p newstk=%p\n", frame, args, m->morepc, m->morefp, g->sched.pc, g->sched.sp, stk);
|
2008-07-14 15:34:27 -06:00
|
|
|
|
2009-06-17 16:12:16 -06:00
|
|
|
top->stackbase = g1->stackbase;
|
|
|
|
top->stackguard = g1->stackguard;
|
|
|
|
top->gobuf = m->morebuf;
|
2009-07-08 19:16:09 -06:00
|
|
|
top->fp = m->morefp;
|
2009-06-17 16:12:16 -06:00
|
|
|
top->args = args;
|
2010-03-29 22:48:22 -06:00
|
|
|
top->free = free;
|
2010-03-31 12:46:01 -06:00
|
|
|
|
|
|
|
// copy flag from panic
|
|
|
|
top->panic = g1->ispanic;
|
|
|
|
g1->ispanic = false;
|
2008-07-14 15:34:27 -06:00
|
|
|
|
2009-06-17 16:12:16 -06:00
|
|
|
g1->stackbase = (byte*)top;
|
|
|
|
g1->stackguard = stk + StackGuard;
|
2008-07-14 15:34:27 -06:00
|
|
|
|
|
|
|
sp = (byte*)top;
|
2009-04-01 01:26:00 -06:00
|
|
|
if(args > 0) {
|
|
|
|
sp -= args;
|
2009-07-08 19:16:09 -06:00
|
|
|
mcpy(sp, m->morefp, args);
|
2008-07-14 15:34:27 -06:00
|
|
|
}
|
|
|
|
|
2009-06-17 16:12:16 -06:00
|
|
|
// Continue as if lessstack had just called m->morepc
|
|
|
|
// (the PC that decided to grow the stack).
|
|
|
|
label.sp = sp;
|
2010-01-25 19:52:55 -07:00
|
|
|
label.pc = (byte*)·lessstack;
|
2009-06-17 16:12:16 -06:00
|
|
|
label.g = m->curg;
|
|
|
|
gogocall(&label, m->morepc);
|
2008-07-14 15:34:27 -06:00
|
|
|
|
|
|
|
*(int32*)345 = 123; // never return
|
|
|
|
}
|
|
|
|
|
2009-04-01 01:26:00 -06:00
|
|
|
G*
|
|
|
|
malg(int32 stacksize)
|
|
|
|
{
|
|
|
|
G *g;
|
|
|
|
byte *stk;
|
|
|
|
|
|
|
|
g = malloc(sizeof(G));
|
2010-02-10 01:00:12 -07:00
|
|
|
if(stacksize >= 0) {
|
|
|
|
stk = stackalloc(stacksize + StackGuard);
|
|
|
|
g->stack0 = stk;
|
|
|
|
g->stackguard = stk + StackGuard;
|
|
|
|
g->stackbase = stk + StackGuard + stacksize - sizeof(Stktop);
|
|
|
|
runtime_memclr(g->stackbase, sizeof(Stktop));
|
|
|
|
}
|
2009-04-01 01:26:00 -06:00
|
|
|
return g;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Newproc and deferproc need to be textflag 7
|
|
|
|
* (no possible stack split when nearing overflow)
|
|
|
|
* because they assume that the arguments to fn
|
|
|
|
* are available sequentially beginning at &arg0.
|
|
|
|
* If a stack split happened, only the one word
|
|
|
|
* arg0 would be copied. It's okay if any functions
|
|
|
|
* they call split the stack below the newproc frame.
|
|
|
|
*/
|
|
|
|
#pragma textflag 7
|
|
|
|
void
|
2010-03-04 16:34:25 -07:00
|
|
|
·newproc(int32 siz, byte* fn, ...)
|
2010-02-08 22:41:54 -07:00
|
|
|
{
|
2010-03-04 16:34:25 -07:00
|
|
|
newproc1(fn, (byte*)(&fn+1), siz, 0);
|
2010-02-08 22:41:54 -07:00
|
|
|
}
|
|
|
|
|
2010-03-26 15:15:30 -06:00
|
|
|
G*
|
2010-02-08 22:41:54 -07:00
|
|
|
newproc1(byte *fn, byte *argp, int32 narg, int32 nret)
|
2009-04-01 01:26:00 -06:00
|
|
|
{
|
2010-02-10 01:00:12 -07:00
|
|
|
byte *sp;
|
2009-04-01 01:26:00 -06:00
|
|
|
G *newg;
|
2010-02-08 22:41:54 -07:00
|
|
|
int32 siz;
|
2009-04-01 01:26:00 -06:00
|
|
|
|
2010-02-08 22:41:54 -07:00
|
|
|
//printf("newproc1 %p %p narg=%d nret=%d\n", fn, argp, narg, nret);
|
|
|
|
siz = narg + nret;
|
2009-04-01 01:26:00 -06:00
|
|
|
siz = (siz+7) & ~7;
|
|
|
|
if(siz > 1024)
|
2010-01-25 19:52:55 -07:00
|
|
|
throw("runtime.newproc: too many args");
|
2009-04-01 01:26:00 -06:00
|
|
|
|
|
|
|
lock(&sched);
|
|
|
|
|
|
|
|
if((newg = gfget()) != nil){
|
|
|
|
newg->status = Gwaiting;
|
2010-03-29 22:48:22 -06:00
|
|
|
if(newg->stackguard - StackGuard != newg->stack0)
|
|
|
|
throw("invalid stack in newg");
|
2009-04-01 01:26:00 -06:00
|
|
|
} else {
|
|
|
|
newg = malg(4096);
|
|
|
|
newg->status = Gwaiting;
|
|
|
|
newg->alllink = allg;
|
|
|
|
allg = newg;
|
|
|
|
}
|
|
|
|
|
2010-02-10 01:00:12 -07:00
|
|
|
sp = newg->stackbase;
|
2009-04-01 01:26:00 -06:00
|
|
|
sp -= siz;
|
2010-02-08 22:41:54 -07:00
|
|
|
mcpy(sp, argp, narg);
|
2009-04-01 01:26:00 -06:00
|
|
|
|
2009-06-17 16:12:16 -06:00
|
|
|
newg->sched.sp = sp;
|
|
|
|
newg->sched.pc = (byte*)goexit;
|
|
|
|
newg->sched.g = newg;
|
|
|
|
newg->entry = fn;
|
2009-04-01 01:26:00 -06:00
|
|
|
|
|
|
|
sched.gcount++;
|
|
|
|
goidgen++;
|
|
|
|
newg->goid = goidgen;
|
|
|
|
|
2009-08-31 19:10:11 -06:00
|
|
|
newprocreadylocked(newg);
|
2009-04-01 01:26:00 -06:00
|
|
|
unlock(&sched);
|
|
|
|
|
2010-03-26 15:15:30 -06:00
|
|
|
return newg;
|
2009-04-01 01:26:00 -06:00
|
|
|
//printf(" goid=%d\n", newg->goid);
|
|
|
|
}
|
|
|
|
|
|
|
|
#pragma textflag 7
|
2010-03-31 12:46:01 -06:00
|
|
|
uintptr
|
2010-03-04 16:34:25 -07:00
|
|
|
·deferproc(int32 siz, byte* fn, ...)
|
2009-04-01 01:26:00 -06:00
|
|
|
{
|
|
|
|
Defer *d;
|
|
|
|
|
|
|
|
d = malloc(sizeof(*d) + siz - sizeof(d->args));
|
|
|
|
d->fn = fn;
|
2010-03-04 16:34:25 -07:00
|
|
|
d->sp = (byte*)(&fn+1);
|
2009-04-01 01:26:00 -06:00
|
|
|
d->siz = siz;
|
2010-03-31 12:46:01 -06:00
|
|
|
d->pc = ·getcallerpc(&siz);
|
2009-04-01 01:26:00 -06:00
|
|
|
mcpy(d->args, d->sp, d->siz);
|
|
|
|
|
|
|
|
d->link = g->defer;
|
|
|
|
g->defer = d;
|
2010-03-31 12:46:01 -06:00
|
|
|
|
|
|
|
// deferproc returns 0 normally.
|
|
|
|
// a deferred func that stops a panic
|
|
|
|
// makes the deferproc return 1.
|
|
|
|
// the code the compiler generates always
|
|
|
|
// checks the return value and jumps to the
|
|
|
|
// end of the function if deferproc returns != 0.
|
|
|
|
return 0;
|
2009-04-01 01:26:00 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
#pragma textflag 7
|
|
|
|
void
|
2010-01-25 19:52:55 -07:00
|
|
|
·deferreturn(uintptr arg0)
|
2009-04-01 01:26:00 -06:00
|
|
|
{
|
|
|
|
Defer *d;
|
2009-06-03 00:02:12 -06:00
|
|
|
byte *sp, *fn;
|
2009-04-01 01:26:00 -06:00
|
|
|
|
|
|
|
d = g->defer;
|
|
|
|
if(d == nil)
|
|
|
|
return;
|
2010-04-05 13:51:09 -06:00
|
|
|
sp = getcallersp(&arg0);
|
2009-04-01 01:26:00 -06:00
|
|
|
if(d->sp != sp)
|
|
|
|
return;
|
|
|
|
mcpy(d->sp, d->args, d->siz);
|
|
|
|
g->defer = d->link;
|
2009-06-03 00:02:12 -06:00
|
|
|
fn = d->fn;
|
2009-04-01 01:26:00 -06:00
|
|
|
free(d);
|
2009-06-03 00:02:12 -06:00
|
|
|
jmpdefer(fn, sp);
|
2010-03-29 22:48:22 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
static void
|
|
|
|
rundefer(void)
|
|
|
|
{
|
|
|
|
Defer *d;
|
|
|
|
|
|
|
|
while((d = g->defer) != nil) {
|
|
|
|
g->defer = d->link;
|
|
|
|
reflect·call(d->fn, d->args, d->siz);
|
|
|
|
free(d);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Free stack frames until we hit the last one
|
|
|
|
// or until we find the one that contains the sp.
|
|
|
|
static void
|
|
|
|
unwindstack(G *gp, byte *sp)
|
|
|
|
{
|
|
|
|
Stktop *top;
|
|
|
|
byte *stk;
|
|
|
|
|
|
|
|
// Must be called from a different goroutine, usually m->g0.
|
|
|
|
if(g == gp)
|
|
|
|
throw("unwindstack on self");
|
|
|
|
|
|
|
|
while((top = (Stktop*)gp->stackbase) != nil && top->stackbase != nil) {
|
|
|
|
stk = gp->stackguard - StackGuard;
|
|
|
|
if(stk <= sp && sp < gp->stackbase)
|
|
|
|
break;
|
|
|
|
gp->stackbase = top->stackbase;
|
|
|
|
gp->stackguard = top->stackguard;
|
|
|
|
free(stk);
|
|
|
|
}
|
2010-04-08 14:24:53 -06:00
|
|
|
|
|
|
|
if(sp != nil && (sp < gp->stackguard - StackGuard || gp->stackbase < sp)) {
|
|
|
|
printf("recover: %p not in [%p, %p]\n", sp, gp->stackguard - StackGuard, gp->stackbase);
|
|
|
|
throw("bad unwindstack");
|
|
|
|
}
|
2010-03-29 22:48:22 -06:00
|
|
|
}
|
|
|
|
|
2010-03-31 12:46:01 -06:00
|
|
|
static void
|
|
|
|
printpanics(Panic *p)
|
|
|
|
{
|
|
|
|
if(p->link) {
|
|
|
|
printpanics(p->link);
|
|
|
|
printf("\t");
|
|
|
|
}
|
|
|
|
printf("panic: ");
|
2010-03-31 16:55:10 -06:00
|
|
|
·printany(p->arg);
|
2010-03-31 12:46:01 -06:00
|
|
|
if(p->recovered)
|
|
|
|
printf(" [recovered]");
|
|
|
|
printf("\n");
|
|
|
|
}
|
|
|
|
|
|
|
|
void
|
|
|
|
·panic(Eface e)
|
|
|
|
{
|
|
|
|
Defer *d;
|
|
|
|
Panic *p;
|
|
|
|
|
|
|
|
p = mal(sizeof *p);
|
|
|
|
p->arg = e;
|
|
|
|
p->link = g->panic;
|
|
|
|
p->stackbase = g->stackbase;
|
|
|
|
g->panic = p;
|
|
|
|
|
|
|
|
for(;;) {
|
|
|
|
d = g->defer;
|
|
|
|
if(d == nil)
|
|
|
|
break;
|
|
|
|
// take defer off list in case of recursive panic
|
|
|
|
g->defer = d->link;
|
|
|
|
g->ispanic = true; // rock for newstack, where reflect.call ends up
|
|
|
|
reflect·call(d->fn, d->args, d->siz);
|
|
|
|
if(p->recovered) {
|
|
|
|
g->panic = p->link;
|
|
|
|
free(p);
|
|
|
|
// put recovering defer back on list
|
|
|
|
// for scheduler to find.
|
|
|
|
d->link = g->defer;
|
|
|
|
g->defer = d;
|
|
|
|
g->status = Grecovery;
|
|
|
|
gosched();
|
|
|
|
throw("recovery failed"); // gosched should not return
|
|
|
|
}
|
|
|
|
free(d);
|
|
|
|
}
|
|
|
|
|
|
|
|
// ran out of deferred calls - old-school panic now
|
|
|
|
fd = 2;
|
|
|
|
printpanics(g->panic);
|
|
|
|
panic(0);
|
|
|
|
}
|
|
|
|
|
|
|
|
#pragma textflag 7 /* no split, or else g->stackguard is not the stack for fp */
|
|
|
|
void
|
|
|
|
·recover(byte *fp, Eface ret)
|
|
|
|
{
|
|
|
|
Stktop *top, *oldtop;
|
|
|
|
Panic *p;
|
|
|
|
|
2010-04-06 14:48:31 -06:00
|
|
|
fp = getcallersp(fp);
|
|
|
|
|
2010-03-31 12:46:01 -06:00
|
|
|
// Must be a panic going on.
|
|
|
|
if((p = g->panic) == nil || p->recovered)
|
|
|
|
goto nomatch;
|
|
|
|
|
|
|
|
// Frame must be at the top of the stack segment,
|
|
|
|
// because each deferred call starts a new stack
|
|
|
|
// segment as a side effect of using reflect.call.
|
|
|
|
// (There has to be some way to remember the
|
|
|
|
// variable argument frame size, and the segment
|
|
|
|
// code already takes care of that for us, so we
|
|
|
|
// reuse it.)
|
|
|
|
//
|
|
|
|
// As usual closures complicate things: the fp that
|
|
|
|
// the closure implementation function claims to have
|
|
|
|
// is where the explicit arguments start, after the
|
|
|
|
// implicit pointer arguments and PC slot.
|
|
|
|
// If we're on the first new segment for a closure,
|
|
|
|
// then fp == top - top->args is correct, but if
|
|
|
|
// the closure has its own big argument frame and
|
|
|
|
// allocated a second segment (see below),
|
|
|
|
// the fp is slightly above top - top->args.
|
|
|
|
// That condition can't happen normally though
|
|
|
|
// (stack pointer go down, not up), so we can accept
|
|
|
|
// any fp between top and top - top->args as
|
|
|
|
// indicating the top of the segment.
|
|
|
|
top = (Stktop*)g->stackbase;
|
|
|
|
if(fp < (byte*)top - top->args || (byte*)top < fp)
|
|
|
|
goto nomatch;
|
|
|
|
|
|
|
|
// The deferred call makes a new segment big enough
|
|
|
|
// for the argument frame but not necessarily big
|
|
|
|
// enough for the function's local frame (size unknown
|
|
|
|
// at the time of the call), so the function might have
|
|
|
|
// made its own segment immediately. If that's the
|
|
|
|
// case, back top up to the older one, the one that
|
|
|
|
// reflect.call would have made for the panic.
|
|
|
|
//
|
|
|
|
// The fp comparison here checks that the argument
|
|
|
|
// frame that was copied during the split (the top->args
|
|
|
|
// bytes above top->fp) abuts the old top of stack.
|
|
|
|
// This is a correct test for both closure and non-closure code.
|
|
|
|
oldtop = (Stktop*)top->stackbase;
|
|
|
|
if(oldtop != nil && top->fp == (byte*)oldtop - top->args)
|
|
|
|
top = oldtop;
|
|
|
|
|
|
|
|
// Now we have the segment that was created to
|
|
|
|
// run this call. It must have been marked as a panic segment.
|
|
|
|
if(!top->panic)
|
|
|
|
goto nomatch;
|
|
|
|
|
|
|
|
// Okay, this is the top frame of a deferred call
|
|
|
|
// in response to a panic. It can see the panic argument.
|
|
|
|
p->recovered = 1;
|
|
|
|
ret = p->arg;
|
|
|
|
FLUSH(&ret);
|
|
|
|
return;
|
|
|
|
|
|
|
|
nomatch:
|
|
|
|
ret.type = nil;
|
|
|
|
ret.data = nil;
|
|
|
|
FLUSH(&ret);
|
|
|
|
}
|
|
|
|
|
|
|
|
|
2010-03-29 22:48:22 -06:00
|
|
|
// Put on gfree list. Sched must be locked.
|
|
|
|
static void
|
|
|
|
gfput(G *g)
|
|
|
|
{
|
|
|
|
if(g->stackguard - StackGuard != g->stack0)
|
|
|
|
throw("invalid stack in gfput");
|
|
|
|
g->schedlink = sched.gfree;
|
|
|
|
sched.gfree = g;
|
|
|
|
}
|
|
|
|
|
|
|
|
// Get from gfree list. Sched must be locked.
|
|
|
|
static G*
|
|
|
|
gfget(void)
|
|
|
|
{
|
|
|
|
G *g;
|
|
|
|
|
|
|
|
g = sched.gfree;
|
|
|
|
if(g)
|
|
|
|
sched.gfree = g->schedlink;
|
|
|
|
return g;
|
|
|
|
}
|
2008-12-04 09:30:54 -07:00
|
|
|
|
2009-05-08 16:21:41 -06:00
|
|
|
void
|
2010-01-25 19:52:55 -07:00
|
|
|
·Breakpoint(void)
|
2009-05-08 16:21:41 -06:00
|
|
|
{
|
|
|
|
breakpoint();
|
|
|
|
}
|
|
|
|
|
|
|
|
void
|
2010-01-25 19:52:55 -07:00
|
|
|
·Goexit(void)
|
2009-05-08 16:21:41 -06:00
|
|
|
{
|
2010-03-29 22:48:22 -06:00
|
|
|
rundefer();
|
2009-05-08 16:21:41 -06:00
|
|
|
goexit();
|
|
|
|
}
|
|
|
|
|
|
|
|
void
|
2010-01-25 19:52:55 -07:00
|
|
|
·Gosched(void)
|
2009-05-08 16:21:41 -06:00
|
|
|
{
|
|
|
|
gosched();
|
|
|
|
}
|
|
|
|
|
2009-07-13 18:28:39 -06:00
|
|
|
void
|
2010-01-25 19:52:55 -07:00
|
|
|
·LockOSThread(void)
|
2009-07-13 18:28:39 -06:00
|
|
|
{
|
|
|
|
if(sched.predawn)
|
|
|
|
throw("cannot wire during init");
|
|
|
|
m->lockedg = g;
|
|
|
|
g->lockedm = m;
|
|
|
|
}
|
|
|
|
|
2009-08-06 14:07:05 -06:00
|
|
|
// delete when scheduler is stronger
|
2010-05-06 12:50:47 -06:00
|
|
|
int32
|
|
|
|
gomaxprocsfunc(int32 n)
|
2009-08-06 14:07:05 -06:00
|
|
|
{
|
2010-05-06 12:50:47 -06:00
|
|
|
int32 ret;
|
2009-08-06 14:07:05 -06:00
|
|
|
|
|
|
|
lock(&sched);
|
2010-05-06 12:50:47 -06:00
|
|
|
ret = sched.gomaxprocs;
|
|
|
|
if (n <= 0)
|
|
|
|
n = ret;
|
2009-08-06 14:07:05 -06:00
|
|
|
sched.gomaxprocs = n;
|
|
|
|
sched.mcpumax = n;
|
2010-04-06 14:48:31 -06:00
|
|
|
// handle fewer procs?
|
|
|
|
if(sched.mcpu > sched.mcpumax) {
|
2009-08-06 14:07:05 -06:00
|
|
|
unlock(&sched);
|
2010-04-06 14:48:31 -06:00
|
|
|
// just give up the cpu.
|
|
|
|
// we'll only get rescheduled once the
|
|
|
|
// number has come down.
|
|
|
|
gosched();
|
2010-05-06 12:50:47 -06:00
|
|
|
return ret;
|
2009-08-06 14:07:05 -06:00
|
|
|
}
|
|
|
|
// handle more procs
|
|
|
|
matchmg();
|
|
|
|
unlock(&sched);
|
2010-05-06 12:50:47 -06:00
|
|
|
return ret;
|
2009-08-06 14:07:05 -06:00
|
|
|
}
|
|
|
|
|
2009-07-13 18:28:39 -06:00
|
|
|
void
|
2010-01-25 19:52:55 -07:00
|
|
|
·UnlockOSThread(void)
|
2009-07-13 18:28:39 -06:00
|
|
|
{
|
|
|
|
m->lockedg = nil;
|
|
|
|
g->lockedm = nil;
|
|
|
|
}
|
|
|
|
|
|
|
|
// for testing of wire, unwire
|
|
|
|
void
|
2010-01-25 19:52:55 -07:00
|
|
|
·mid(uint32 ret)
|
2009-07-13 18:28:39 -06:00
|
|
|
{
|
|
|
|
ret = m->id;
|
|
|
|
FLUSH(&ret);
|
|
|
|
}
|