diff --git a/src/cmd/api/goapi.go b/src/cmd/api/goapi.go index ce806fc95d9..2d6dde2665c 100644 --- a/src/cmd/api/goapi.go +++ b/src/cmd/api/goapi.go @@ -405,6 +405,9 @@ func (w *Walker) parseFile(dir, file string) (*ast.File, error) { " sudog struct{};" + " waitq struct{};" + " wincallbackcontext struct{};" + + " keventt struct{};" + + " timespec struct{};" + + " epollevent struct{};" + "); " + "const (" + " cb_max = 2000;" + diff --git a/src/pkg/runtime/asm_386.s b/src/pkg/runtime/asm_386.s index 3c46b40feec..87dd1fa0f5c 100644 --- a/src/pkg/runtime/asm_386.s +++ b/src/pkg/runtime/asm_386.s @@ -598,6 +598,9 @@ TEXT runtime·xchgp(SB), NOSPLIT, $0-12 MOVL AX, ret+8(FP) RET +TEXT runtime·xchguintptr(SB), NOSPLIT, $0-12 + JMP runtime·xchg(SB) + TEXT runtime·procyield(SB),NOSPLIT,$0-0 MOVL cycles+0(FP), AX again: diff --git a/src/pkg/runtime/asm_amd64.s b/src/pkg/runtime/asm_amd64.s index eb0795ec3ac..80c2ab3c544 100644 --- a/src/pkg/runtime/asm_amd64.s +++ b/src/pkg/runtime/asm_amd64.s @@ -709,6 +709,9 @@ TEXT runtime·xchgp(SB), NOSPLIT, $0-24 MOVQ AX, ret+16(FP) RET +TEXT runtime·xchguintptr(SB), NOSPLIT, $0-24 + JMP runtime·xchg64(SB) + TEXT runtime·procyield(SB),NOSPLIT,$0-0 MOVL cycles+0(FP), AX again: diff --git a/src/pkg/runtime/asm_amd64p32.s b/src/pkg/runtime/asm_amd64p32.s index 512e9231483..a0e63b44058 100644 --- a/src/pkg/runtime/asm_amd64p32.s +++ b/src/pkg/runtime/asm_amd64p32.s @@ -666,6 +666,16 @@ TEXT runtime·xchg64(SB), NOSPLIT, $0-24 MOVQ AX, ret+16(FP) RET +TEXT runtime·xchgp(SB), NOSPLIT, $0-12 + MOVL ptr+0(FP), BX + MOVL new+4(FP), AX + XCHGL AX, 0(BX) + MOVL AX, ret+8(FP) + RET + +TEXT runtime·xchguintptr(SB), NOSPLIT, $0-12 + JMP runtime·xchg(SB) + TEXT runtime·procyield(SB),NOSPLIT,$0-0 MOVL cycles+0(FP), AX again: diff --git a/src/pkg/runtime/atomic_arm.c b/src/pkg/runtime/atomic_arm.c index c15108e9a00..5369820c090 100644 --- a/src/pkg/runtime/atomic_arm.c +++ b/src/pkg/runtime/atomic_arm.c @@ -54,6 +54,13 @@ runtime·xchgp(void* volatile* addr, void* v) } } +#pragma textflag NOSPLIT +void* +runtime·xchguintptr(void* volatile* addr, void* v) +{ + return runtime·xchg((uint32*)addr, (uint32)v); +} + #pragma textflag NOSPLIT void runtime·procyield(uint32 cnt) diff --git a/src/pkg/runtime/netpoll.go b/src/pkg/runtime/netpoll.go new file mode 100644 index 00000000000..5c410fc678c --- /dev/null +++ b/src/pkg/runtime/netpoll.go @@ -0,0 +1,448 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build darwin dragonfly freebsd linux nacl netbsd openbsd solaris windows + +package runtime + +import "unsafe" + +// Integrated network poller (platform-independent part). +// A particular implementation (epoll/kqueue) must define the following functions: +// func netpollinit() // to initialize the poller +// func netpollopen(fd uintptr, pd *pollDesc) int32 // to arm edge-triggered notifications +// and associate fd with pd. +// An implementation must call the following function to denote that the pd is ready. +// func netpollready(rg, wg **gp, pd *pollDesc, mode int32) + +// pollDesc contains 2 binary semaphores, rg and wg, to park reader and writer +// goroutines respectively. The semaphore can be in the following states: +// pdReady - io readiness notification is pending; +// a goroutine consumes the notification by changing the state to nil. +// pdWait - a goroutine prepares to park on the semaphore, but not yet parked; +// the goroutine commits to park by changing the state to G pointer, +// or, alternatively, concurrent io notification changes the state to READY, +// or, alternatively, concurrent timeout/close changes the state to nil. +// G pointer - the goroutine is blocked on the semaphore; +// io notification or timeout/close changes the state to READY or nil respectively +// and unparks the goroutine. +// nil - nothing of the above. +const ( + pdReady uintptr = 1 + pdWait uintptr = 2 +) + +const pollBlockSize = 4 * 1024 + +// Network poller descriptor. +type pollDesc struct { + link *pollDesc // in pollcache, protected by pollcache.lock + + // The lock protects pollOpen, pollSetDeadline, pollUnblock and deadlineimpl operations. + // This fully covers seq, rt and wt variables. fd is constant throughout the PollDesc lifetime. + // pollReset, pollWait, pollWaitCanceled and runtime·netpollready (IO readiness notification) + // proceed w/o taking the lock. So closing, rg, rd, wg and wd are manipulated + // in a lock-free way by all operations. + // NOTE(dvyukov): the following code uses uintptr to store *g (rg/wg), + // that will blow up when GC starts moving objects. + lock mutex // protectes the following fields + fd uintptr + closing bool + seq uintptr // protects from stale timers and ready notifications + rg uintptr // pdReady, pdWait, G waiting for read or nil + rt timer // read deadline timer (set if rt.f != nil) + rd int64 // read deadline + wg uintptr // pdReady, pdWait, G waiting for write or nil + wt timer // write deadline timer + wd int64 // write deadline + user unsafe.Pointer // user settable cookie +} + +type pollCache struct { + lock mutex + first *pollDesc + // PollDesc objects must be type-stable, + // because we can get ready notification from epoll/kqueue + // after the descriptor is closed/reused. + // Stale notifications are detected using seq variable, + // seq is incremented when deadlines are changed or descriptor is reused. +} + +var pollcache pollCache + +func netpollServerInit() { + netpollinit() +} + +func netpollOpen(fd uintptr) (*pollDesc, int) { + pd := pollcache.alloc() + lock(&pd.lock) + if pd.wg != 0 && pd.wg != pdReady { + gothrow("netpollOpen: blocked write on free descriptor") + } + if pd.rg != 0 && pd.rg != pdReady { + gothrow("netpollOpen: blocked read on free descriptor") + } + pd.fd = fd + pd.closing = false + pd.seq++ + pd.rg = 0 + pd.rd = 0 + pd.wg = 0 + pd.wd = 0 + unlock(&pd.lock) + + errno := netpollopen(fd, pd) + return pd, int(errno) +} + +func netpollClose(pd *pollDesc) { + if !pd.closing { + gothrow("runtime_pollClose: close w/o unblock") + } + if pd.wg != 0 && pd.wg != pdReady { + gothrow("netpollClose: blocked write on closing descriptor") + } + if pd.rg != 0 && pd.rg != pdReady { + gothrow("netpollClose: blocked read on closing descriptor") + } + netpollclose(uintptr(pd.fd)) + pollcache.free(pd) +} + +func (c *pollCache) free(pd *pollDesc) { + lock(&c.lock) + pd.link = c.first + c.first = pd + unlock(&c.lock) +} + +func netpollReset(pd *pollDesc, mode int) int { + err := netpollcheckerr(pd, int32(mode)) + if err != 0 { + return err + } + if mode == 'r' { + pd.rg = 0 + } else if mode == 'w' { + pd.wg = 0 + } + return 0 +} + +func netpollWait(pd *pollDesc, mode int) int { + err := netpollcheckerr(pd, int32(mode)) + if err != 0 { + return err + } + // As for now only Solaris uses level-triggered IO. + if GOOS == "solaris" { + netpollarm(pd, mode) + } + for !netpollblock(pd, int32(mode), false) { + err = netpollcheckerr(pd, int32(mode)) + if err != 0 { + return err + } + // Can happen if timeout has fired and unblocked us, + // but before we had a chance to run, timeout has been reset. + // Pretend it has not happened and retry. + } + return 0 +} + +func netpollWaitCanceled(pd *pollDesc, mode int) { + // This function is used only on windows after a failed attempt to cancel + // a pending async IO operation. Wait for ioready, ignore closing or timeouts. + for !netpollblock(pd, int32(mode), true) { + } +} + +func netpollSetDeadline(pd *pollDesc, d int64, mode int) { + lock(&pd.lock) + if pd.closing { + unlock(&pd.lock) + return + } + pd.seq++ // invalidate current timers + // Reset current timers. + if pd.rt.f != nil { + deltimer(&pd.rt) + pd.rt.f = nil + } + if pd.wt.f != nil { + deltimer(&pd.wt) + pd.wt.f = nil + } + // Setup new timers. + if d != 0 && d <= nanotime() { + d = -1 + } + if mode == 'r' || mode == 'r'+'w' { + pd.rd = d + } + if mode == 'w' || mode == 'r'+'w' { + pd.wd = d + } + if pd.rd > 0 && pd.rd == pd.wd { + pd.rt.f = netpollDeadline + pd.rt.when = pd.rd + // Copy current seq into the timer arg. + // Timer func will check the seq against current descriptor seq, + // if they differ the descriptor was reused or timers were reset. + pd.rt.arg = pd + pd.rt.seq = pd.seq + addtimer(&pd.rt) + } else { + if pd.rd > 0 { + pd.rt.f = netpollReadDeadline + pd.rt.when = pd.rd + pd.rt.arg = pd + pd.rt.seq = pd.seq + addtimer(&pd.rt) + } + if pd.wd > 0 { + pd.wt.f = netpollWriteDeadline + pd.wt.when = pd.wd + pd.wt.arg = pd + pd.wt.seq = pd.seq + addtimer(&pd.wt) + } + } + // If we set the new deadline in the past, unblock currently pending IO if any. + var rg, wg *g + atomicstorep(unsafe.Pointer(&wg), nil) // full memory barrier between stores to rd/wd and load of rg/wg in netpollunblock + if pd.rd < 0 { + rg = netpollunblock(pd, 'r', false) + } + if pd.wd < 0 { + wg = netpollunblock(pd, 'w', false) + } + unlock(&pd.lock) + if rg != nil { + goready(rg) + } + if wg != nil { + goready(wg) + } +} + +func netpollUnblock(pd *pollDesc) { + lock(&pd.lock) + if pd.closing { + gothrow("netpollUnblock: already closing") + } + pd.closing = true + pd.seq++ + var rg, wg *g + atomicstorep(unsafe.Pointer(&rg), nil) // full memory barrier between store to closing and read of rg/wg in netpollunblock + rg = netpollunblock(pd, 'r', false) + wg = netpollunblock(pd, 'w', false) + if pd.rt.f != nil { + deltimer(&pd.rt) + pd.rt.f = nil + } + if pd.wt.f != nil { + deltimer(&pd.wt) + pd.wt.f = nil + } + unlock(&pd.lock) + if rg != nil { + goready(rg) + } + if wg != nil { + goready(wg) + } +} + +func netpollfd(pd *pollDesc) uintptr { + return pd.fd +} + +func netpolluser(pd *pollDesc) *unsafe.Pointer { + return &pd.user +} + +func netpollclosing(pd *pollDesc) bool { + return pd.closing +} + +func netpolllock(pd *pollDesc) { + lock(&pd.lock) +} + +func netpollunlock(pd *pollDesc) { + lock(&pd.lock) +} + +// make pd ready, newly runnable goroutines (if any) are returned in rg/wg +func netpollready(gpp **g, pd *pollDesc, mode int32) { + var rg, wg *g + if mode == 'r' || mode == 'r'+'w' { + rg = netpollunblock(pd, 'r', true) + } + if mode == 'w' || mode == 'r'+'w' { + wg = netpollunblock(pd, 'w', true) + } + if rg != nil { + rg.schedlink = *gpp + *gpp = rg + } + if wg != nil { + wg.schedlink = *gpp + *gpp = wg + } +} + +func netpollcheckerr(pd *pollDesc, mode int32) int { + if pd.closing { + return 1 // errClosing + } + if (mode == 'r' && pd.rd < 0) || (mode == 'w' && pd.wd < 0) { + return 2 // errTimeout + } + return 0 +} + +func netpollblockcommit(gp *g, gpp unsafe.Pointer) bool { + return casuintptr((*uintptr)(gpp), pdWait, uintptr(unsafe.Pointer(gp))) +} + +// returns true if IO is ready, or false if timedout or closed +// waitio - wait only for completed IO, ignore errors +func netpollblock(pd *pollDesc, mode int32, waitio bool) bool { + gpp := &pd.rg + if mode == 'w' { + gpp = &pd.wg + } + + // set the gpp semaphore to WAIT + for { + old := *gpp + if old == pdReady { + *gpp = 0 + return true + } + if old != 0 { + gothrow("netpollblock: double wait") + } + if casuintptr(gpp, 0, pdWait) { + break + } + } + + // need to recheck error states after setting gpp to WAIT + // this is necessary because runtime_pollUnblock/runtime_pollSetDeadline/deadlineimpl + // do the opposite: store to closing/rd/wd, membarrier, load of rg/wg + if waitio || netpollcheckerr(pd, mode) == 0 { + f := netpollblockcommit + gopark(**(**unsafe.Pointer)(unsafe.Pointer(&f)), unsafe.Pointer(gpp), "IO wait") + } + // be careful to not lose concurrent READY notification + old := xchguintptr(gpp, 0) + if old > pdWait { + gothrow("netpollblock: corrupted state") + } + return old == pdReady +} + +func netpollunblock(pd *pollDesc, mode int32, ioready bool) *g { + gpp := &pd.rg + if mode == 'w' { + gpp = &pd.wg + } + + for { + old := *gpp + if old == pdReady { + return nil + } + if old == 0 && !ioready { + // Only set READY for ioready. runtime_pollWait + // will check for timeout/cancel before waiting. + return nil + } + var new uintptr + if ioready { + new = pdReady + } + if casuintptr(gpp, old, new) { + if old == pdReady || old == pdWait { + old = 0 + } + return (*g)(unsafe.Pointer(old)) + } + } +} + +func netpolldeadlineimpl(pd *pollDesc, seq uintptr, read, write bool) { + lock(&pd.lock) + // Seq arg is seq when the timer was set. + // If it's stale, ignore the timer event. + if seq != pd.seq { + // The descriptor was reused or timers were reset. + unlock(&pd.lock) + return + } + var rg *g + if read { + if pd.rd <= 0 || pd.rt.f == nil { + gothrow("netpollDeadlineImpl: inconsistent read deadline") + } + pd.rd = -1 + atomicstorep(unsafe.Pointer(&pd.rt.f), nil) // full memory barrier between store to rd and load of rg in netpollunblock + rg = netpollunblock(pd, 'r', false) + } + var wg *g + if write { + if pd.wd <= 0 || pd.wt.f == nil && !read { + gothrow("netpolldeadlineimpl: inconsistent write deadline") + } + pd.wd = -1 + atomicstorep(unsafe.Pointer(&pd.wt.f), nil) // full memory barrier between store to wd and load of wg in netpollunblock + wg = netpollunblock(pd, 'w', false) + } + unlock(&pd.lock) + if rg != nil { + goready(rg) + } + if wg != nil { + goready(wg) + } +} + +func netpollDeadline(arg interface{}, seq uintptr) { + netpolldeadlineimpl(arg.(*pollDesc), seq, true, true) +} + +func netpollReadDeadline(arg interface{}, seq uintptr) { + netpolldeadlineimpl(arg.(*pollDesc), seq, true, false) +} + +func netpollWriteDeadline(arg interface{}, seq uintptr) { + netpolldeadlineimpl(arg.(*pollDesc), seq, false, true) +} + +func (c *pollCache) alloc() *pollDesc { + lock(&c.lock) + if c.first == nil { + const pdSize = unsafe.Sizeof(pollDesc{}) + n := pollBlockSize / pdSize + if n == 0 { + n = 1 + } + // Must be in non-GC memory because can be referenced + // only from epoll/kqueue internals. + mem := persistentalloc(n*pdSize, 0, &memstats.other_sys) + for i := uintptr(0); i < n; i++ { + pd := (*pollDesc)(add(mem, i*pdSize)) + pd.link = c.first + c.first = pd + } + } + pd := c.first + c.first = pd.link + unlock(&c.lock) + return pd +} diff --git a/src/pkg/runtime/netpoll.goc b/src/pkg/runtime/netpoll.goc deleted file mode 100644 index 446e78eb01c..00000000000 --- a/src/pkg/runtime/netpoll.goc +++ /dev/null @@ -1,481 +0,0 @@ -// Copyright 2013 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build darwin dragonfly freebsd linux nacl netbsd openbsd solaris windows - -package net - -#include "runtime.h" -#include "defs_GOOS_GOARCH.h" -#include "arch_GOARCH.h" -#include "malloc.h" - -// Integrated network poller (platform-independent part). -// A particular implementation (epoll/kqueue) must define the following functions: -// void runtime·netpollinit(void); // to initialize the poller -// int32 runtime·netpollopen(uintptr fd, PollDesc *pd); // to arm edge-triggered notifications - // and associate fd with pd. -// An implementation must call the following function to denote that the pd is ready. -// void runtime·netpollready(G **gpp, PollDesc *pd, int32 mode); - -// PollDesc contains 2 binary semaphores, rg and wg, to park reader and writer -// goroutines respectively. The semaphore can be in the following states: -// READY - io readiness notification is pending; -// a goroutine consumes the notification by changing the state to nil. -// WAIT - a goroutine prepares to park on the semaphore, but not yet parked; -// the goroutine commits to park by changing the state to G pointer, -// or, alternatively, concurrent io notification changes the state to READY, -// or, alternatively, concurrent timeout/close changes the state to nil. -// G pointer - the goroutine is blocked on the semaphore; -// io notification or timeout/close changes the state to READY or nil respectively -// and unparks the goroutine. -// nil - nothing of the above. -#define READY ((G*)1) -#define WAIT ((G*)2) - -enum -{ - PollBlockSize = 4*1024, -}; - -// time.go defines the layout of this structure. -// Keep in sync with time.go. -typedef struct Timer Timer; -struct Timer -{ - intgo i; - int64 when; - int64 period; - FuncVal *fv; - Eface arg; -}; - -void runtime·addtimer(Timer*); -void runtime·deltimer(Timer*); - -struct PollDesc -{ - PollDesc* link; // in pollcache, protected by pollcache.lock - - // The lock protects pollOpen, pollSetDeadline, pollUnblock and deadlineimpl operations. - // This fully covers seq, rt and wt variables. fd is constant throughout the PollDesc lifetime. - // pollReset, pollWait, pollWaitCanceled and runtime·netpollready (IO rediness notification) - // proceed w/o taking the lock. So closing, rg, rd, wg and wd are manipulated - // in a lock-free way by all operations. - Mutex lock; // protects the following fields - uintptr fd; - bool closing; - uintptr seq; // protects from stale timers and ready notifications - G* rg; // READY, WAIT, G waiting for read or nil - Timer rt; // read deadline timer (set if rt.fv != nil) - int64 rd; // read deadline - G* wg; // READY, WAIT, G waiting for write or nil - Timer wt; // write deadline timer - int64 wd; // write deadline - void* user; // user settable cookie -}; - -static struct -{ - Mutex lock; - PollDesc* first; - // PollDesc objects must be type-stable, - // because we can get ready notification from epoll/kqueue - // after the descriptor is closed/reused. - // Stale notifications are detected using seq variable, - // seq is incremented when deadlines are changed or descriptor is reused. -} pollcache; - -static bool netpollblock(PollDesc*, int32, bool); -static G* netpollunblock(PollDesc*, int32, bool); -static void deadline(Eface); -static void readDeadline(Eface); -static void writeDeadline(Eface); -static PollDesc* allocPollDesc(void); -static intgo checkerr(PollDesc *pd, int32 mode); - -static FuncVal deadlineFn = {(void(*)(void))deadline}; -static FuncVal readDeadlineFn = {(void(*)(void))readDeadline}; -static FuncVal writeDeadlineFn = {(void(*)(void))writeDeadline}; - -// runtimeNano returns the current value of the runtime clock in nanoseconds. -func runtimeNano() (ns int64) { - ns = runtime·nanotime(); -} - -func runtime_pollServerInit() { - runtime·netpollinit(); -} - -func runtime_pollOpen(fd uintptr) (pd *PollDesc, errno int) { - pd = allocPollDesc(); - runtime·lock(&pd->lock); - if(pd->wg != nil && pd->wg != READY) - runtime·throw("runtime_pollOpen: blocked write on free descriptor"); - if(pd->rg != nil && pd->rg != READY) - runtime·throw("runtime_pollOpen: blocked read on free descriptor"); - pd->fd = fd; - pd->closing = false; - pd->seq++; - pd->rg = nil; - pd->rd = 0; - pd->wg = nil; - pd->wd = 0; - runtime·unlock(&pd->lock); - - errno = runtime·netpollopen(fd, pd); -} - -func runtime_pollClose(pd *PollDesc) { - if(!pd->closing) - runtime·throw("runtime_pollClose: close w/o unblock"); - if(pd->wg != nil && pd->wg != READY) - runtime·throw("runtime_pollClose: blocked write on closing descriptor"); - if(pd->rg != nil && pd->rg != READY) - runtime·throw("runtime_pollClose: blocked read on closing descriptor"); - runtime·netpollclose(pd->fd); - runtime·lock(&pollcache.lock); - pd->link = pollcache.first; - pollcache.first = pd; - runtime·unlock(&pollcache.lock); -} - -func runtime_pollReset(pd *PollDesc, mode int) (err int) { - err = checkerr(pd, mode); - if(err) - goto ret; - if(mode == 'r') - pd->rg = nil; - else if(mode == 'w') - pd->wg = nil; -ret: -} - -func runtime_pollWait(pd *PollDesc, mode int) (err int) { - err = checkerr(pd, mode); - if(err == 0) { - // As for now only Solaris uses level-triggered IO. - if(Solaris) - runtime·netpollarm(pd, mode); - while(!netpollblock(pd, mode, false)) { - err = checkerr(pd, mode); - if(err != 0) - break; - // Can happen if timeout has fired and unblocked us, - // but before we had a chance to run, timeout has been reset. - // Pretend it has not happened and retry. - } - } -} - -func runtime_pollWaitCanceled(pd *PollDesc, mode int) { - // This function is used only on windows after a failed attempt to cancel - // a pending async IO operation. Wait for ioready, ignore closing or timeouts. - while(!netpollblock(pd, mode, true)) - ; -} - -func runtime_pollSetDeadline(pd *PollDesc, d int64, mode int) { - G *rg, *wg; - - runtime·lock(&pd->lock); - if(pd->closing) { - runtime·unlock(&pd->lock); - return; - } - pd->seq++; // invalidate current timers - // Reset current timers. - if(pd->rt.fv) { - runtime·deltimer(&pd->rt); - pd->rt.fv = nil; - } - if(pd->wt.fv) { - runtime·deltimer(&pd->wt); - pd->wt.fv = nil; - } - // Setup new timers. - if(d != 0 && d <= runtime·nanotime()) - d = -1; - if(mode == 'r' || mode == 'r'+'w') - pd->rd = d; - if(mode == 'w' || mode == 'r'+'w') - pd->wd = d; - if(pd->rd > 0 && pd->rd == pd->wd) { - pd->rt.fv = &deadlineFn; - pd->rt.when = pd->rd; - // Copy current seq into the timer arg. - // Timer func will check the seq against current descriptor seq, - // if they differ the descriptor was reused or timers were reset. - pd->rt.arg.type = (Type*)pd->seq; - pd->rt.arg.data = pd; - runtime·addtimer(&pd->rt); - } else { - if(pd->rd > 0) { - pd->rt.fv = &readDeadlineFn; - pd->rt.when = pd->rd; - pd->rt.arg.type = (Type*)pd->seq; - pd->rt.arg.data = pd; - runtime·addtimer(&pd->rt); - } - if(pd->wd > 0) { - pd->wt.fv = &writeDeadlineFn; - pd->wt.when = pd->wd; - pd->wt.arg.type = (Type*)pd->seq; - pd->wt.arg.data = pd; - runtime·addtimer(&pd->wt); - } - } - // If we set the new deadline in the past, unblock currently pending IO if any. - rg = nil; - runtime·atomicstorep(&wg, nil); // full memory barrier between stores to rd/wd and load of rg/wg in netpollunblock - if(pd->rd < 0) - rg = netpollunblock(pd, 'r', false); - if(pd->wd < 0) - wg = netpollunblock(pd, 'w', false); - runtime·unlock(&pd->lock); - if(rg) - runtime·ready(rg); - if(wg) - runtime·ready(wg); -} - -func runtime_pollUnblock(pd *PollDesc) { - G *rg, *wg; - - runtime·lock(&pd->lock); - if(pd->closing) - runtime·throw("runtime_pollUnblock: already closing"); - pd->closing = true; - pd->seq++; - runtime·atomicstorep(&rg, nil); // full memory barrier between store to closing and read of rg/wg in netpollunblock - rg = netpollunblock(pd, 'r', false); - wg = netpollunblock(pd, 'w', false); - if(pd->rt.fv) { - runtime·deltimer(&pd->rt); - pd->rt.fv = nil; - } - if(pd->wt.fv) { - runtime·deltimer(&pd->wt); - pd->wt.fv = nil; - } - runtime·unlock(&pd->lock); - if(rg) - runtime·ready(rg); - if(wg) - runtime·ready(wg); -} - -uintptr -runtime·netpollfd(PollDesc *pd) -{ - return pd->fd; -} - -void** -runtime·netpolluser(PollDesc *pd) -{ - return &pd->user; -} - -bool -runtime·netpollclosing(PollDesc *pd) -{ - return pd->closing; -} - -void -runtime·netpolllock(PollDesc *pd) -{ - runtime·lock(&pd->lock); -} - -void -runtime·netpollunlock(PollDesc *pd) -{ - runtime·unlock(&pd->lock); -} - -// make pd ready, newly runnable goroutines (if any) are enqueued info gpp list -void -runtime·netpollready(G **gpp, PollDesc *pd, int32 mode) -{ - G *rg, *wg; - - rg = wg = nil; - if(mode == 'r' || mode == 'r'+'w') - rg = netpollunblock(pd, 'r', true); - if(mode == 'w' || mode == 'r'+'w') - wg = netpollunblock(pd, 'w', true); - if(rg) { - rg->schedlink = *gpp; - *gpp = rg; - } - if(wg) { - wg->schedlink = *gpp; - *gpp = wg; - } -} - -static intgo -checkerr(PollDesc *pd, int32 mode) -{ - if(pd->closing) - return 1; // errClosing - if((mode == 'r' && pd->rd < 0) || (mode == 'w' && pd->wd < 0)) - return 2; // errTimeout - return 0; -} - -static bool -blockcommit(G *gp, G **gpp) -{ - return runtime·casp(gpp, WAIT, gp); -} - -// returns true if IO is ready, or false if timedout or closed -// waitio - wait only for completed IO, ignore errors -static bool -netpollblock(PollDesc *pd, int32 mode, bool waitio) -{ - G **gpp, *old; - - gpp = &pd->rg; - if(mode == 'w') - gpp = &pd->wg; - - // set the gpp semaphore to WAIT - for(;;) { - old = *gpp; - if(old == READY) { - *gpp = nil; - return true; - } - if(old != nil) - runtime·throw("netpollblock: double wait"); - if(runtime·casp(gpp, nil, WAIT)) - break; - } - - // need to recheck error states after setting gpp to WAIT - // this is necessary because runtime_pollUnblock/runtime_pollSetDeadline/deadlineimpl - // do the opposite: store to closing/rd/wd, membarrier, load of rg/wg - if(waitio || checkerr(pd, mode) == 0) - runtime·park((bool(*)(G*, void*))blockcommit, gpp, runtime·gostringnocopy((byte*)"IO wait")); - // be careful to not lose concurrent READY notification - old = runtime·xchgp(gpp, nil); - if(old > WAIT) - runtime·throw("netpollblock: corrupted state"); - return old == READY; -} - -static G* -netpollunblock(PollDesc *pd, int32 mode, bool ioready) -{ - G **gpp, *old, *new; - - gpp = &pd->rg; - if(mode == 'w') - gpp = &pd->wg; - - for(;;) { - old = *gpp; - if(old == READY) - return nil; - if(old == nil && !ioready) { - // Only set READY for ioready. runtime_pollWait - // will check for timeout/cancel before waiting. - return nil; - } - new = nil; - if(ioready) - new = READY; - if(runtime·casp(gpp, old, new)) - break; - } - if(old > WAIT) - return old; // must be G* - return nil; -} - -static void -deadlineimpl(Eface arg, bool read, bool write) -{ - PollDesc *pd; - uint32 seq; - G *rg, *wg; - - pd = (PollDesc*)arg.data; - // This is the seq when the timer was set. - // If it's stale, ignore the timer event. - seq = (uintptr)arg.type; - rg = wg = nil; - runtime·lock(&pd->lock); - if(seq != pd->seq) { - // The descriptor was reused or timers were reset. - runtime·unlock(&pd->lock); - return; - } - if(read) { - if(pd->rd <= 0 || pd->rt.fv == nil) - runtime·throw("deadlineimpl: inconsistent read deadline"); - pd->rd = -1; - runtime·atomicstorep(&pd->rt.fv, nil); // full memory barrier between store to rd and load of rg in netpollunblock - rg = netpollunblock(pd, 'r', false); - } - if(write) { - if(pd->wd <= 0 || (pd->wt.fv == nil && !read)) - runtime·throw("deadlineimpl: inconsistent write deadline"); - pd->wd = -1; - runtime·atomicstorep(&pd->wt.fv, nil); // full memory barrier between store to wd and load of wg in netpollunblock - wg = netpollunblock(pd, 'w', false); - } - runtime·unlock(&pd->lock); - if(rg) - runtime·ready(rg); - if(wg) - runtime·ready(wg); -} - -static void -deadline(Eface arg) -{ - deadlineimpl(arg, true, true); -} - -static void -readDeadline(Eface arg) -{ - deadlineimpl(arg, true, false); -} - -static void -writeDeadline(Eface arg) -{ - deadlineimpl(arg, false, true); -} - -static PollDesc* -allocPollDesc(void) -{ - PollDesc *pd; - uint32 i, n; - - runtime·lock(&pollcache.lock); - if(pollcache.first == nil) { - n = PollBlockSize/sizeof(*pd); - if(n == 0) - n = 1; - // Must be in non-GC memory because can be referenced - // only from epoll/kqueue internals. - pd = runtime·persistentalloc(n*sizeof(*pd), 0, &mstats.other_sys); - for(i = 0; i < n; i++) { - pd[i].link = pollcache.first; - pollcache.first = &pd[i]; - } - } - pd = pollcache.first; - pollcache.first = pd->link; - runtime·unlock(&pollcache.lock); - return pd; -} diff --git a/src/pkg/runtime/netpoll_epoll.c b/src/pkg/runtime/netpoll_epoll.c deleted file mode 100644 index 9d6c2051556..00000000000 --- a/src/pkg/runtime/netpoll_epoll.c +++ /dev/null @@ -1,103 +0,0 @@ -// Copyright 2013 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build linux - -#include "runtime.h" -#include "defs_GOOS_GOARCH.h" - -int32 runtime·epollcreate(int32 size); -int32 runtime·epollcreate1(int32 flags); -int32 runtime·epollctl(int32 epfd, int32 op, int32 fd, EpollEvent *ev); -int32 runtime·epollwait(int32 epfd, EpollEvent *ev, int32 nev, int32 timeout); -void runtime·closeonexec(int32 fd); - -static int32 epfd = -1; // epoll descriptor - -void -runtime·netpollinit(void) -{ - epfd = runtime·epollcreate1(EPOLL_CLOEXEC); - if(epfd >= 0) - return; - epfd = runtime·epollcreate(1024); - if(epfd >= 0) { - runtime·closeonexec(epfd); - return; - } - runtime·printf("netpollinit: failed to create descriptor (%d)\n", -epfd); - runtime·throw("netpollinit: failed to create descriptor"); -} - -int32 -runtime·netpollopen(uintptr fd, PollDesc *pd) -{ - EpollEvent ev; - int32 res; - - ev.events = EPOLLIN|EPOLLOUT|EPOLLRDHUP|EPOLLET; - *(uint64*)ev.data = (uint64)(uintptr)pd; - res = runtime·epollctl(epfd, EPOLL_CTL_ADD, (int32)fd, &ev); - return -res; -} - -int32 -runtime·netpollclose(uintptr fd) -{ - EpollEvent ev; - int32 res; - - res = runtime·epollctl(epfd, EPOLL_CTL_DEL, (int32)fd, &ev); - return -res; -} - -void -runtime·netpollarm(PollDesc* pd, int32 mode) -{ - USED(pd); - USED(mode); - runtime·throw("unused"); -} - -// polls for ready network connections -// returns list of goroutines that become runnable -G* -runtime·netpoll(bool block) -{ - static int32 lasterr; - EpollEvent events[128], *ev; - int32 n, i, waitms, mode; - G *gp; - - if(epfd == -1) - return nil; - waitms = -1; - if(!block) - waitms = 0; -retry: - n = runtime·epollwait(epfd, events, nelem(events), waitms); - if(n < 0) { - if(n != -EINTR && n != lasterr) { - lasterr = n; - runtime·printf("runtime: epollwait on fd %d failed with %d\n", epfd, -n); - } - goto retry; - } - gp = nil; - for(i = 0; i < n; i++) { - ev = &events[i]; - if(ev->events == 0) - continue; - mode = 0; - if(ev->events & (EPOLLIN|EPOLLRDHUP|EPOLLHUP|EPOLLERR)) - mode += 'r'; - if(ev->events & (EPOLLOUT|EPOLLHUP|EPOLLERR)) - mode += 'w'; - if(mode) - runtime·netpollready(&gp, (void*)(uintptr)*(uint64*)ev->data, mode); - } - if(block && gp == nil) - goto retry; - return gp; -} diff --git a/src/pkg/runtime/netpoll_epoll.go b/src/pkg/runtime/netpoll_epoll.go new file mode 100644 index 00000000000..ecfc9cdde8f --- /dev/null +++ b/src/pkg/runtime/netpoll_epoll.go @@ -0,0 +1,97 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build linux + +package runtime + +import "unsafe" + +func epollcreate(size int32) int32 +func epollcreate1(flags int32) int32 + +//go:noescape +func epollctl(epfd, op, fd int32, ev *epollevent) int32 + +//go:noescape +func epollwait(epfd int32, ev *epollevent, nev, timeout int32) int32 +func closeonexec(fd int32) + +var ( + epfd int32 = -1 // epoll descriptor + netpolllasterr int32 +) + +func netpollinit() { + epfd = epollcreate1(_EPOLL_CLOEXEC) + if epfd >= 0 { + return + } + epfd = epollcreate(1024) + if epfd >= 0 { + closeonexec(epfd) + return + } + println("netpollinit: failed to create epoll descriptor", -epfd) + gothrow("netpollinit: failed to create descriptor") +} + +func netpollopen(fd uintptr, pd *pollDesc) int32 { + var ev epollevent + ev.events = _EPOLLIN | _EPOLLOUT | _EPOLLRDHUP | _EPOLLET + *(**pollDesc)(unsafe.Pointer(&ev.data)) = pd + return -epollctl(epfd, _EPOLL_CTL_ADD, int32(fd), &ev) +} + +func netpollclose(fd uintptr) int32 { + var ev epollevent + return -epollctl(epfd, _EPOLL_CTL_DEL, int32(fd), &ev) +} + +func netpollarm(pd *pollDesc, mode int) { + gothrow("unused") +} + +// polls for ready network connections +// returns list of goroutines that become runnable +func netpoll(block bool) (gp *g) { + if epfd == -1 { + return + } + waitms := int32(-1) + if !block { + waitms = 0 + } + var events [128]epollevent +retry: + n := epollwait(epfd, &events[0], int32(len(events)), waitms) + if n < 0 { + if n != -_EINTR && n != netpolllasterr { + netpolllasterr = n + println("runtime: epollwait on fd", epfd, "failed with", -n) + } + goto retry + } + for i := int32(0); i < n; i++ { + ev := &events[i] + if ev.events == 0 { + continue + } + var mode int32 + if ev.events&(_EPOLLIN|_EPOLLRDHUP|_EPOLLHUP|_EPOLLERR) != 0 { + mode += 'r' + } + if ev.events&(_EPOLLOUT|_EPOLLHUP|_EPOLLERR) != 0 { + mode += 'w' + } + if mode != 0 { + pd := *(**pollDesc)(unsafe.Pointer(&ev.data)) + netpollready((**g)(noescape(unsafe.Pointer(&gp))), pd, mode) + } + } + if block && gp == nil { + goto retry + } + return gp +} diff --git a/src/pkg/runtime/netpoll_kqueue.c b/src/pkg/runtime/netpoll_kqueue.c deleted file mode 100644 index 6b7fc869d3a..00000000000 --- a/src/pkg/runtime/netpoll_kqueue.c +++ /dev/null @@ -1,111 +0,0 @@ -// Copyright 2013 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build darwin dragonfly freebsd netbsd openbsd - -#include "runtime.h" -#include "defs_GOOS_GOARCH.h" -#include "os_GOOS.h" - -// Integrated network poller (kqueue-based implementation). - -int32 runtime·kqueue(void); -int32 runtime·kevent(int32, KeventT*, int32, KeventT*, int32, Timespec*); -void runtime·closeonexec(int32); - -static int32 kq = -1; - -void -runtime·netpollinit(void) -{ - kq = runtime·kqueue(); - if(kq < 0) { - runtime·printf("netpollinit: kqueue failed with %d\n", -kq); - runtime·throw("netpollinit: kqueue failed"); - } - runtime·closeonexec(kq); -} - -int32 -runtime·netpollopen(uintptr fd, PollDesc *pd) -{ - KeventT ev[2]; - int32 n; - - // Arm both EVFILT_READ and EVFILT_WRITE in edge-triggered mode (EV_CLEAR) - // for the whole fd lifetime. The notifications are automatically unregistered - // when fd is closed. - ev[0].ident = (uint32)fd; - ev[0].filter = EVFILT_READ; - ev[0].flags = EV_ADD|EV_CLEAR; - ev[0].fflags = 0; - ev[0].data = 0; - ev[0].udata = (kevent_udata)pd; - ev[1] = ev[0]; - ev[1].filter = EVFILT_WRITE; - n = runtime·kevent(kq, ev, 2, nil, 0, nil); - if(n < 0) - return -n; - return 0; -} - -int32 -runtime·netpollclose(uintptr fd) -{ - // Don't need to unregister because calling close() - // on fd will remove any kevents that reference the descriptor. - USED(fd); - return 0; -} - -void -runtime·netpollarm(PollDesc* pd, int32 mode) -{ - USED(pd, mode); - runtime·throw("unused"); -} - -// Polls for ready network connections. -// Returns list of goroutines that become runnable. -G* -runtime·netpoll(bool block) -{ - static int32 lasterr; - KeventT events[64], *ev; - Timespec ts, *tp; - int32 n, i, mode; - G *gp; - - if(kq == -1) - return nil; - tp = nil; - if(!block) { - ts.tv_sec = 0; - ts.tv_nsec = 0; - tp = &ts; - } - gp = nil; -retry: - n = runtime·kevent(kq, nil, 0, events, nelem(events), tp); - if(n < 0) { - if(n != -EINTR && n != lasterr) { - lasterr = n; - runtime·printf("runtime: kevent on fd %d failed with %d\n", kq, -n); - } - goto retry; - } - for(i = 0; i < n; i++) { - ev = &events[i]; - mode = 0; - if(ev->filter == EVFILT_READ) - mode += 'r'; - if(ev->filter == EVFILT_WRITE) - mode += 'w'; - if(mode) - runtime·netpollready(&gp, (PollDesc*)ev->udata, mode); - } - if(block && gp == nil) - goto retry; - return gp; -} diff --git a/src/pkg/runtime/netpoll_kqueue.go b/src/pkg/runtime/netpoll_kqueue.go new file mode 100644 index 00000000000..d6d55b97b8d --- /dev/null +++ b/src/pkg/runtime/netpoll_kqueue.go @@ -0,0 +1,101 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build darwin dragonfly freebsd netbsd openbsd + +package runtime + +// Integrated network poller (kqueue-based implementation). + +import "unsafe" + +func kqueue() int32 + +//go:noescape +func kevent(kq int32, ch *keventt, nch int32, ev *keventt, nev int32, ts *timespec) int32 +func closeonexec(fd int32) + +var ( + kq int32 = -1 + netpolllasterr int32 +) + +func netpollinit() { + kq = kqueue() + if kq < 0 { + println("netpollinit: kqueue failed with", -kq) + gothrow("netpollinit: kqueue failed") + } + closeonexec(kq) +} + +func netpollopen(fd uintptr, pd *pollDesc) int32 { + // Arm both EVFILT_READ and EVFILT_WRITE in edge-triggered mode (EV_CLEAR) + // for the whole fd lifetime. The notifications are automatically unregistered + // when fd is closed. + var ev [2]keventt + *(*uintptr)(unsafe.Pointer(&ev[0].ident)) = fd + ev[0].filter = _EVFILT_READ + ev[0].flags = _EV_ADD | _EV_CLEAR + ev[0].fflags = 0 + ev[0].data = 0 + ev[0].udata = (*byte)(unsafe.Pointer(pd)) + ev[1] = ev[0] + ev[1].filter = _EVFILT_WRITE + n := kevent(kq, &ev[0], 2, nil, 0, nil) + if n < 0 { + return -n + } + return 0 +} + +func netpollclose(fd uintptr) int32 { + // Don't need to unregister because calling close() + // on fd will remove any kevents that reference the descriptor. + return 0 +} + +func netpollarm(pd *pollDesc, mode int) { + gothrow("unused") +} + +// Polls for ready network connections. +// Returns list of goroutines that become runnable. +func netpoll(block bool) (gp *g) { + if kq == -1 { + return + } + var tp *timespec + var ts timespec + if !block { + tp = &ts + } + var events [64]keventt +retry: + n := kevent(kq, nil, 0, &events[0], int32(len(events)), tp) + if n < 0 { + if n != -_EINTR && n != netpolllasterr { + netpolllasterr = n + println("runtime: kevent on fd", kq, "failed with", -n) + } + goto retry + } + for i := 0; i < int(n); i++ { + ev := &events[i] + var mode int32 + if ev.filter == _EVFILT_READ { + mode += 'r' + } + if ev.filter == _EVFILT_WRITE { + mode += 'w' + } + if mode != 0 { + netpollready((**g)(noescape(unsafe.Pointer(&gp))), (*pollDesc)(unsafe.Pointer(ev.udata)), mode) + } + } + if block && gp == nil { + goto retry + } + return gp +} diff --git a/src/pkg/runtime/netpoll_nacl.c b/src/pkg/runtime/netpoll_nacl.c deleted file mode 100644 index b75753a23b7..00000000000 --- a/src/pkg/runtime/netpoll_nacl.c +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright 2013 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -#include "runtime.h" -#include "defs_GOOS_GOARCH.h" -#include "os_GOOS.h" - -// Fake network poller for NaCl. -// Should never be used, because NaCl network connections do not honor "SetNonblock". - -void -runtime·netpollinit(void) -{ -} - -int32 -runtime·netpollopen(uintptr fd, PollDesc *pd) -{ - USED(fd); - USED(pd); - return 0; -} - -int32 -runtime·netpollclose(uintptr fd) -{ - USED(fd); - return 0; -} - -G* -runtime·netpoll(bool block) -{ - USED(block); - return nil; -} diff --git a/src/pkg/runtime/netpoll_nacl.go b/src/pkg/runtime/netpoll_nacl.go new file mode 100644 index 00000000000..5cbc3003214 --- /dev/null +++ b/src/pkg/runtime/netpoll_nacl.go @@ -0,0 +1,26 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Fake network poller for NaCl. +// Should never be used, because NaCl network connections do not honor "SetNonblock". + +package runtime + +func netpollinit() { +} + +func netpollopen(fd uintptr, pd *pollDesc) int32 { + return 0 +} + +func netpollclose(fd uintptr) int32 { + return 0 +} + +func netpollarm(pd *pollDesc, mode int) { +} + +func netpoll(block bool) *g { + return nil +} diff --git a/src/pkg/runtime/os_darwin.go b/src/pkg/runtime/os_darwin.go index 4eefec1f643..e0f63ddb901 100644 --- a/src/pkg/runtime/os_darwin.go +++ b/src/pkg/runtime/os_darwin.go @@ -18,9 +18,6 @@ func sigaction(mode uint32, new, old unsafe.Pointer) func sigaltstack(new, old unsafe.Pointer) func sigtramp() func setitimer(mode int32, new, old unsafe.Pointer) -func kqueue() int32 -func kevent(fd int32, ev1 unsafe.Pointer, nev1 int32, ev2 unsafe.Pointer, nev2 int32, ts unsafe.Pointer) int32 -func closeonexec(fd int32) func mach_semaphore_wait(sema uint32) int32 func mach_semaphore_timedwait(sema, sec, nsec uint32) int32 func mach_semaphore_signal(sema uint32) int32 diff --git a/src/pkg/runtime/os_dragonfly.go b/src/pkg/runtime/os_dragonfly.go index 3e7fc8d4eee..cdaa06986ee 100644 --- a/src/pkg/runtime/os_dragonfly.go +++ b/src/pkg/runtime/os_dragonfly.go @@ -14,9 +14,6 @@ func setitimer(mode int32, new, old unsafe.Pointer) func sysctl(mib *uint32, miblen uint32, out *byte, size *uintptr, dst *byte, ndst uintptr) int32 func getrlimit(kind int32, limit unsafe.Pointer) int32 func raise(sig int32) -func kqueue() int32 -func kevent(fd int32, ev1 unsafe.Pointer, nev1 int32, ev2 unsafe.Pointer, nev2 int32, ts unsafe.Pointer) int32 -func closeonexec(fd int32) func sys_umtx_sleep(addr unsafe.Pointer, val, timeout int32) int32 func sys_umtx_wakeup(addr unsafe.Pointer, val int32) int32 diff --git a/src/pkg/runtime/os_freebsd.go b/src/pkg/runtime/os_freebsd.go index 14d1b7ca9d1..96964f1e1e7 100644 --- a/src/pkg/runtime/os_freebsd.go +++ b/src/pkg/runtime/os_freebsd.go @@ -14,9 +14,6 @@ func setitimer(mode int32, new, old unsafe.Pointer) func sysctl(mib *uint32, miblen uint32, out *byte, size *uintptr, dst *byte, ndst uintptr) int32 func getrlimit(kind int32, limit unsafe.Pointer) int32 func raise(sig int32) -func kqueue() int32 -func kevent(fd int32, ev1 unsafe.Pointer, nev1 int32, ev2 unsafe.Pointer, nev2 int32, ts unsafe.Pointer) int32 -func closeonexec(fd int32) func sys_umtx_op(addr unsafe.Pointer, mode int32, val uint32, ptr2, ts unsafe.Pointer) int32 const stackSystem = 0 diff --git a/src/pkg/runtime/os_linux.go b/src/pkg/runtime/os_linux.go index ba52a31e90b..a6799cd4142 100644 --- a/src/pkg/runtime/os_linux.go +++ b/src/pkg/runtime/os_linux.go @@ -14,11 +14,6 @@ func setitimer(mode int32, new, old unsafe.Pointer) func rtsigprocmask(sig int32, new, old unsafe.Pointer, size int32) func getrlimit(kind int32, limit unsafe.Pointer) int32 func raise(sig int32) -func epollcreate(size int32) int32 -func epollcreate1(flags int32) int32 -func epollctl(epfd, op, fd int32, ev unsafe.Pointer) int32 -func epollwait(epfd int32, ev unsafe.Pointer, nev, timeout int32) int32 -func closeonexec(fd int32) func sched_getaffinity(pid, len uintptr, buf *uintptr) int32 const stackSystem = 0 diff --git a/src/pkg/runtime/os_netbsd.go b/src/pkg/runtime/os_netbsd.go index 15607685e2d..8792f497e45 100644 --- a/src/pkg/runtime/os_netbsd.go +++ b/src/pkg/runtime/os_netbsd.go @@ -13,9 +13,6 @@ func sigprocmask(mode int32, new, old unsafe.Pointer) func sysctl(mib *uint32, miblen uint32, out *byte, size *uintptr, dst *byte, ndst uintptr) int32 func lwp_tramp() func raise(sig int32) -func kqueue() int32 -func kevent(fd int32, ev1 unsafe.Pointer, nev1 int32, ev2 unsafe.Pointer, nev2 int32, ts unsafe.Pointer) int32 -func closeonexec(fd int32) func getcontext(ctxt unsafe.Pointer) func lwp_create(ctxt unsafe.Pointer, flags uintptr, lwpid unsafe.Pointer) int32 func lwp_park(abstime unsafe.Pointer, unpark int32, hint, unparkhint unsafe.Pointer) int32 diff --git a/src/pkg/runtime/os_openbsd.go b/src/pkg/runtime/os_openbsd.go index 681bdde82e1..19e2b45a15c 100644 --- a/src/pkg/runtime/os_openbsd.go +++ b/src/pkg/runtime/os_openbsd.go @@ -12,9 +12,6 @@ func sigaltstack(new, old unsafe.Pointer) func sigprocmask(mode int32, new uint32) uint32 func sysctl(mib *uint32, miblen uint32, out *byte, size *uintptr, dst *byte, ndst uintptr) int32 func raise(sig int32) -func kqueue() int32 -func kevent(fd int32, ev1 unsafe.Pointer, nev1 int32, ev2 unsafe.Pointer, nev2 int32, ts unsafe.Pointer) int32 -func closeonexec(fd int32) func tfork(param unsafe.Pointer, psize uintptr, mm, gg, fn unsafe.Pointer) int32 func thrsleep(ident unsafe.Pointer, clock_id int32, tsp, lock, abort unsafe.Pointer) int32 func thrwakeup(ident unsafe.Pointer, n int32) int32 diff --git a/src/pkg/runtime/os_solaris.go b/src/pkg/runtime/os_solaris.go index 556c1504bc9..a5e6966781a 100644 --- a/src/pkg/runtime/os_solaris.go +++ b/src/pkg/runtime/os_solaris.go @@ -19,6 +19,10 @@ func tstart_sysvicall(mm unsafe.Pointer) uint32 func nanotime1() int64 func usleep1(usec uint32) func osyield1() +func netpollinit() +func netpollopen(fd uintptr, pd *pollDesc) int32 +func netpollclose(fd uintptr) int32 +func netpollarm(pd *pollDesc, mode int) type libcFunc byte diff --git a/src/pkg/runtime/os_windows.go b/src/pkg/runtime/os_windows.go index 15957b32901..6a3bfca41fc 100644 --- a/src/pkg/runtime/os_windows.go +++ b/src/pkg/runtime/os_windows.go @@ -21,6 +21,10 @@ func asmstdcall(fn unsafe.Pointer) func getlasterror() uint32 func setlasterror(err uint32) func usleep1(usec uint32) +func netpollinit() +func netpollopen(fd uintptr, pd *pollDesc) int32 +func netpollclose(fd uintptr) int32 +func netpollarm(pd *pollDesc, mode int) const stackSystem = 512 * ptrSize diff --git a/src/pkg/runtime/runtime.h b/src/pkg/runtime/runtime.h index 4cecc39a6ff..7007957b65e 100644 --- a/src/pkg/runtime/runtime.h +++ b/src/pkg/runtime/runtime.h @@ -886,12 +886,8 @@ int64 runtime·cputicks(void); int64 runtime·tickspersecond(void); void runtime·blockevent(int64, intgo); G* runtime·netpoll(bool); -void runtime·netpollinit(void); -int32 runtime·netpollopen(uintptr, PollDesc*); -int32 runtime·netpollclose(uintptr); void runtime·netpollready(G**, PollDesc*, int32); uintptr runtime·netpollfd(PollDesc*); -void runtime·netpollarm(PollDesc*, int32); void** runtime·netpolluser(PollDesc*); bool runtime·netpollclosing(PollDesc*); void runtime·netpolllock(PollDesc*); diff --git a/src/pkg/runtime/stubs.go b/src/pkg/runtime/stubs.go index 3719c7501c3..73dc2ec4434 100644 --- a/src/pkg/runtime/stubs.go +++ b/src/pkg/runtime/stubs.go @@ -237,6 +237,9 @@ func xchg64(ptr *uint64, new uint64) uint64 //go:noescape func xchgp(ptr unsafe.Pointer, new unsafe.Pointer) unsafe.Pointer +//go:noescape +func xchguintptr(ptr *uintptr, new uintptr) uintptr + //go:noescape func atomicstore(ptr *uint32, val uint32) diff --git a/src/pkg/runtime/sys_linux_386.s b/src/pkg/runtime/sys_linux_386.s index ace5a1802b9..426e05faa14 100644 --- a/src/pkg/runtime/sys_linux_386.s +++ b/src/pkg/runtime/sys_linux_386.s @@ -457,7 +457,7 @@ TEXT runtime·epollcreate1(SB),NOSPLIT,$0 MOVL AX, ret+4(FP) RET -// int32 runtime·epollctl(int32 epfd, int32 op, int32 fd, EpollEvent *ev); +// func epollctl(epfd, op, fd int32, ev *epollEvent) int TEXT runtime·epollctl(SB),NOSPLIT,$0 MOVL $255, AX MOVL epfd+0(FP), BX diff --git a/src/pkg/runtime/sys_linux_amd64.s b/src/pkg/runtime/sys_linux_amd64.s index f263ef35ae1..3ce6bfc9055 100644 --- a/src/pkg/runtime/sys_linux_amd64.s +++ b/src/pkg/runtime/sys_linux_amd64.s @@ -378,7 +378,7 @@ TEXT runtime·epollcreate1(SB),NOSPLIT,$0 MOVL AX, ret+8(FP) RET -// int32 runtime·epollctl(int32 epfd, int32 op, int32 fd, EpollEvent *ev); +// func epollctl(epfd, op, fd int32, ev *epollEvent) int TEXT runtime·epollctl(SB),NOSPLIT,$0 MOVL epfd+0(FP), DI MOVL op+4(FP), SI diff --git a/src/pkg/runtime/sys_linux_arm.s b/src/pkg/runtime/sys_linux_arm.s index 3221cdf29fa..a2abe8f25cb 100644 --- a/src/pkg/runtime/sys_linux_arm.s +++ b/src/pkg/runtime/sys_linux_arm.s @@ -424,12 +424,12 @@ TEXT runtime·epollcreate1(SB),NOSPLIT,$0 MOVW R0, ret+4(FP) RET -// int32 runtime·epollctl(int32 epfd, int32 op, int32 fd, EpollEvent *ev) +// func epollctl(epfd, op, fd int32, ev *epollEvent) int TEXT runtime·epollctl(SB),NOSPLIT,$0 - MOVW 0(FP), R0 - MOVW 4(FP), R1 - MOVW 8(FP), R2 - MOVW 12(FP), R3 + MOVW epfd+0(FP), R0 + MOVW op+4(FP), R1 + MOVW fd+8(FP), R2 + MOVW ev+12(FP), R3 MOVW $SYS_epoll_ctl, R7 SWI $0 MOVW R0, ret+16(FP) diff --git a/src/pkg/runtime/thunk.s b/src/pkg/runtime/thunk.s index 0b5963e70ad..0c057a3e230 100644 --- a/src/pkg/runtime/thunk.s +++ b/src/pkg/runtime/thunk.s @@ -11,6 +11,9 @@ #define JMP B #endif +TEXT net·runtimeNano(SB),NOSPLIT,$0-0 + JMP runtime·nanotime(SB) + TEXT time·runtimeNano(SB),NOSPLIT,$0-0 JMP runtime·nanotime(SB) @@ -74,6 +77,30 @@ TEXT reflect·chanrecv(SB), NOSPLIT, $0-0 TEXT runtime∕debug·freeOSMemory(SB), NOSPLIT, $0-0 JMP runtime·freeOSMemory(SB) +TEXT net·runtime_pollServerInit(SB),NOSPLIT,$0-0 + JMP runtime·netpollServerInit(SB) + +TEXT net·runtime_pollOpen(SB),NOSPLIT,$0-0 + JMP runtime·netpollOpen(SB) + +TEXT net·runtime_pollClose(SB),NOSPLIT,$0-0 + JMP runtime·netpollClose(SB) + +TEXT net·runtime_pollReset(SB),NOSPLIT,$0-0 + JMP runtime·netpollReset(SB) + +TEXT net·runtime_pollWait(SB),NOSPLIT,$0-0 + JMP runtime·netpollWait(SB) + +TEXT net·runtime_pollWaitCanceled(SB),NOSPLIT,$0-0 + JMP runtime·netpollWaitCanceled(SB) + +TEXT net·runtime_pollSetDeadline(SB),NOSPLIT,$0-0 + JMP runtime·netpollSetDeadline(SB) + +TEXT net·runtime_pollUnblock(SB),NOSPLIT,$0-0 + JMP runtime·netpollUnblock(SB) + TEXT syscall·setenv_c(SB), NOSPLIT, $0-0 JMP runtime·syscall_setenv_c(SB) diff --git a/src/pkg/runtime/time.go b/src/pkg/runtime/time.go index 102539bf21d..8cf9eecf834 100644 --- a/src/pkg/runtime/time.go +++ b/src/pkg/runtime/time.go @@ -9,7 +9,7 @@ package runtime import "unsafe" // Package time knows the layout of this structure. -// If this struct changes, adjust ../time/sleep.go:/runtimeTimer and netpoll.goc:/timer. +// If this struct changes, adjust ../time/sleep.go:/runtimeTimer. // For GOOS=nacl, package syscall knows the layout of this structure. // If this struct changes, adjust ../syscall/net_nacl.go:/runtimeTimer. type timer struct { @@ -20,8 +20,9 @@ type timer struct { // a well-behaved function and not block. when int64 period int64 - f func(interface{}) + f func(interface{}, uintptr) arg interface{} + seq uintptr } var timers struct { @@ -74,7 +75,7 @@ func stopTimer(t *timer) bool { // Go runtime. // Ready the goroutine arg. -func goroutineReady(arg interface{}) { +func goroutineReady(arg interface{}, seq uintptr) { goready(arg.(*g)) } @@ -185,11 +186,12 @@ func timerproc() { } f := t.f arg := t.arg + seq := t.seq unlock(&timers.lock) if raceenabled { raceacquire(unsafe.Pointer(t)) } - f(arg) + f(arg, seq) lock(&timers.lock) } if delta < 0 { diff --git a/src/pkg/syscall/net_nacl.go b/src/pkg/syscall/net_nacl.go index 07d52f45256..b5cb5303062 100644 --- a/src/pkg/syscall/net_nacl.go +++ b/src/pkg/syscall/net_nacl.go @@ -21,8 +21,9 @@ type runtimeTimer struct { i int when int64 period int64 - f func(interface{}) // NOTE: must not be closure + f func(interface{}, uintptr) // NOTE: must not be closure arg interface{} + seq uintptr } func startTimer(*runtimeTimer) @@ -49,7 +50,7 @@ func (t *timer) stop() { stopTimer(&t.r) } -func timerExpired(i interface{}) { +func timerExpired(i interface{}, seq uintptr) { t := i.(*timer) go func() { t.q.Lock() diff --git a/src/pkg/time/internal_test.go b/src/pkg/time/internal_test.go index b02292ea163..edd523bc80c 100644 --- a/src/pkg/time/internal_test.go +++ b/src/pkg/time/internal_test.go @@ -12,7 +12,7 @@ func init() { var Interrupt = interrupt var DaysIn = daysIn -func empty(arg interface{}) {} +func empty(arg interface{}, seq uintptr) {} // Test that a runtimeTimer with a duration so large it overflows // does not cause other timers to hang. diff --git a/src/pkg/time/sleep.go b/src/pkg/time/sleep.go index c7b019feb03..33c349de46b 100644 --- a/src/pkg/time/sleep.go +++ b/src/pkg/time/sleep.go @@ -17,8 +17,9 @@ type runtimeTimer struct { i int when int64 period int64 - f func(interface{}) // NOTE: must not be closure + f func(interface{}, uintptr) // NOTE: must not be closure arg interface{} + seq uintptr } // when is a helper function for setting the 'when' field of a runtimeTimer. @@ -83,7 +84,7 @@ func (t *Timer) Reset(d Duration) bool { return active } -func sendTime(c interface{}) { +func sendTime(c interface{}, seq uintptr) { // Non-blocking send of time on c. // Used in NewTimer, it cannot block anyway (buffer). // Used in NewTicker, dropping sends on the floor is @@ -117,6 +118,6 @@ func AfterFunc(d Duration, f func()) *Timer { return t } -func goFunc(arg interface{}) { +func goFunc(arg interface{}, seq uintptr) { go arg.(func())() }