2013-07-21 20:49:57 -06:00
|
|
|
// Copyright 2013 The Go Authors. All rights reserved.
|
|
|
|
// Use of this source code is governed by a BSD-style
|
|
|
|
// license that can be found in the LICENSE file.
|
|
|
|
|
|
|
|
#include "runtime.h"
|
|
|
|
#include "defs_GOOS_GOARCH.h"
|
|
|
|
#include "os_GOOS.h"
|
|
|
|
|
|
|
|
#define DWORD_MAX 0xffffffff
|
|
|
|
|
|
|
|
#pragma dynimport runtime·CreateIoCompletionPort CreateIoCompletionPort "kernel32.dll"
|
|
|
|
#pragma dynimport runtime·GetQueuedCompletionStatus GetQueuedCompletionStatus "kernel32.dll"
|
2013-08-08 07:41:57 -06:00
|
|
|
#pragma dynimport runtime·WSAGetOverlappedResult WSAGetOverlappedResult "ws2_32.dll"
|
2013-07-21 20:49:57 -06:00
|
|
|
|
|
|
|
extern void *runtime·CreateIoCompletionPort;
|
|
|
|
extern void *runtime·GetQueuedCompletionStatus;
|
2013-08-08 07:41:57 -06:00
|
|
|
extern void *runtime·WSAGetOverlappedResult;
|
2013-07-21 20:49:57 -06:00
|
|
|
|
|
|
|
#define INVALID_HANDLE_VALUE ((uintptr)-1)
|
|
|
|
|
2013-08-06 04:40:10 -06:00
|
|
|
// net_op must be the same as beginning of net.operation. Keep these in sync.
|
|
|
|
typedef struct net_op net_op;
|
|
|
|
struct net_op
|
2013-07-21 20:49:57 -06:00
|
|
|
{
|
|
|
|
// used by windows
|
|
|
|
Overlapped o;
|
|
|
|
// used by netpoll
|
2013-08-08 07:41:57 -06:00
|
|
|
PollDesc* pd;
|
2013-07-21 20:49:57 -06:00
|
|
|
int32 mode;
|
|
|
|
int32 errno;
|
|
|
|
uint32 qty;
|
|
|
|
};
|
|
|
|
|
2013-08-08 07:41:57 -06:00
|
|
|
typedef struct OverlappedEntry OverlappedEntry;
|
|
|
|
struct OverlappedEntry
|
|
|
|
{
|
|
|
|
uintptr key;
|
|
|
|
net_op* op; // In reality it's Overlapped*, but we cast it to net_op* anyway.
|
|
|
|
uintptr internal;
|
|
|
|
uint32 qty;
|
|
|
|
};
|
|
|
|
|
|
|
|
static void handlecompletion(G **gpp, net_op *o, int32 errno, uint32 qty);
|
|
|
|
|
2013-07-21 20:49:57 -06:00
|
|
|
static uintptr iocphandle = INVALID_HANDLE_VALUE; // completion port io handle
|
|
|
|
|
|
|
|
void
|
|
|
|
runtime·netpollinit(void)
|
|
|
|
{
|
2014-08-29 02:44:07 -06:00
|
|
|
iocphandle = (uintptr)runtime·stdcall4(runtime·CreateIoCompletionPort, INVALID_HANDLE_VALUE, 0, 0, DWORD_MAX);
|
2013-07-21 20:49:57 -06:00
|
|
|
if(iocphandle == 0) {
|
|
|
|
runtime·printf("netpoll: failed to create iocp handle (errno=%d)\n", runtime·getlasterror());
|
|
|
|
runtime·throw("netpoll: failed to create iocp handle");
|
|
|
|
}
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
int32
|
|
|
|
runtime·netpollopen(uintptr fd, PollDesc *pd)
|
|
|
|
{
|
|
|
|
USED(pd);
|
2014-08-29 02:44:07 -06:00
|
|
|
if(runtime·stdcall4(runtime·CreateIoCompletionPort, fd, iocphandle, 0, 0) == 0)
|
2013-07-21 20:49:57 -06:00
|
|
|
return -runtime·getlasterror();
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
int32
|
|
|
|
runtime·netpollclose(uintptr fd)
|
|
|
|
{
|
|
|
|
// nothing to do
|
|
|
|
USED(fd);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2014-02-12 11:24:29 -07:00
|
|
|
void
|
2014-02-24 20:31:01 -07:00
|
|
|
runtime·netpollarm(PollDesc* pd, int32 mode)
|
2014-02-12 11:24:29 -07:00
|
|
|
{
|
2014-02-24 20:31:01 -07:00
|
|
|
USED(pd, mode);
|
2014-02-12 11:24:29 -07:00
|
|
|
runtime·throw("unused");
|
|
|
|
}
|
|
|
|
|
2013-07-21 20:49:57 -06:00
|
|
|
// Polls for completed network IO.
|
|
|
|
// Returns list of goroutines that become runnable.
|
|
|
|
G*
|
|
|
|
runtime·netpoll(bool block)
|
|
|
|
{
|
2013-08-08 07:41:57 -06:00
|
|
|
OverlappedEntry entries[64];
|
|
|
|
uint32 wait, qty, key, flags, n, i;
|
|
|
|
int32 errno;
|
|
|
|
net_op *op;
|
2013-07-21 20:49:57 -06:00
|
|
|
G *gp;
|
|
|
|
|
|
|
|
if(iocphandle == INVALID_HANDLE_VALUE)
|
|
|
|
return nil;
|
2013-07-27 03:46:40 -06:00
|
|
|
gp = nil;
|
2013-08-08 07:41:57 -06:00
|
|
|
wait = 0;
|
|
|
|
if(block)
|
|
|
|
wait = INFINITE;
|
2013-07-27 03:46:40 -06:00
|
|
|
retry:
|
2013-08-08 07:41:57 -06:00
|
|
|
if(runtime·GetQueuedCompletionStatusEx != nil) {
|
|
|
|
n = nelem(entries) / runtime·gomaxprocs;
|
|
|
|
if(n < 8)
|
|
|
|
n = 8;
|
2014-02-11 02:41:46 -07:00
|
|
|
if(block)
|
all: remove 'extern register M *m' from runtime
The runtime has historically held two dedicated values g (current goroutine)
and m (current thread) in 'extern register' slots (TLS on x86, real registers
backed by TLS on ARM).
This CL removes the extern register m; code now uses g->m.
On ARM, this frees up the register that formerly held m (R9).
This is important for NaCl, because NaCl ARM code cannot use R9 at all.
The Go 1 macrobenchmarks (those with per-op times >= 10 µs) are unaffected:
BenchmarkBinaryTree17 5491374955 5471024381 -0.37%
BenchmarkFannkuch11 4357101311 4275174828 -1.88%
BenchmarkGobDecode 11029957 11364184 +3.03%
BenchmarkGobEncode 6852205 6784822 -0.98%
BenchmarkGzip 650795967 650152275 -0.10%
BenchmarkGunzip 140962363 141041670 +0.06%
BenchmarkHTTPClientServer 71581 73081 +2.10%
BenchmarkJSONEncode 31928079 31913356 -0.05%
BenchmarkJSONDecode 117470065 113689916 -3.22%
BenchmarkMandelbrot200 6008923 5998712 -0.17%
BenchmarkGoParse 6310917 6327487 +0.26%
BenchmarkRegexpMatchMedium_1K 114568 114763 +0.17%
BenchmarkRegexpMatchHard_1K 168977 169244 +0.16%
BenchmarkRevcomp 935294971 914060918 -2.27%
BenchmarkTemplate 145917123 148186096 +1.55%
Minux previous reported larger variations, but these were caused by
run-to-run noise, not repeatable slowdowns.
Actual code changes by Minux.
I only did the docs and the benchmarking.
LGTM=dvyukov, iant, minux
R=minux, josharian, iant, dave, bradfitz, dvyukov
CC=golang-codereviews
https://golang.org/cl/109050043
2014-06-26 09:54:39 -06:00
|
|
|
g->m->blocked = true;
|
2014-08-29 02:44:07 -06:00
|
|
|
if(runtime·stdcall6(runtime·GetQueuedCompletionStatusEx, iocphandle, (uintptr)entries, n, (uintptr)&n, wait, 0) == 0) {
|
all: remove 'extern register M *m' from runtime
The runtime has historically held two dedicated values g (current goroutine)
and m (current thread) in 'extern register' slots (TLS on x86, real registers
backed by TLS on ARM).
This CL removes the extern register m; code now uses g->m.
On ARM, this frees up the register that formerly held m (R9).
This is important for NaCl, because NaCl ARM code cannot use R9 at all.
The Go 1 macrobenchmarks (those with per-op times >= 10 µs) are unaffected:
BenchmarkBinaryTree17 5491374955 5471024381 -0.37%
BenchmarkFannkuch11 4357101311 4275174828 -1.88%
BenchmarkGobDecode 11029957 11364184 +3.03%
BenchmarkGobEncode 6852205 6784822 -0.98%
BenchmarkGzip 650795967 650152275 -0.10%
BenchmarkGunzip 140962363 141041670 +0.06%
BenchmarkHTTPClientServer 71581 73081 +2.10%
BenchmarkJSONEncode 31928079 31913356 -0.05%
BenchmarkJSONDecode 117470065 113689916 -3.22%
BenchmarkMandelbrot200 6008923 5998712 -0.17%
BenchmarkGoParse 6310917 6327487 +0.26%
BenchmarkRegexpMatchMedium_1K 114568 114763 +0.17%
BenchmarkRegexpMatchHard_1K 168977 169244 +0.16%
BenchmarkRevcomp 935294971 914060918 -2.27%
BenchmarkTemplate 145917123 148186096 +1.55%
Minux previous reported larger variations, but these were caused by
run-to-run noise, not repeatable slowdowns.
Actual code changes by Minux.
I only did the docs and the benchmarking.
LGTM=dvyukov, iant, minux
R=minux, josharian, iant, dave, bradfitz, dvyukov
CC=golang-codereviews
https://golang.org/cl/109050043
2014-06-26 09:54:39 -06:00
|
|
|
g->m->blocked = false;
|
2013-08-08 07:41:57 -06:00
|
|
|
errno = runtime·getlasterror();
|
|
|
|
if(!block && errno == WAIT_TIMEOUT)
|
2013-07-21 20:49:57 -06:00
|
|
|
return nil;
|
2013-08-08 07:41:57 -06:00
|
|
|
runtime·printf("netpoll: GetQueuedCompletionStatusEx failed (errno=%d)\n", errno);
|
|
|
|
runtime·throw("netpoll: GetQueuedCompletionStatusEx failed");
|
2013-07-21 20:49:57 -06:00
|
|
|
}
|
all: remove 'extern register M *m' from runtime
The runtime has historically held two dedicated values g (current goroutine)
and m (current thread) in 'extern register' slots (TLS on x86, real registers
backed by TLS on ARM).
This CL removes the extern register m; code now uses g->m.
On ARM, this frees up the register that formerly held m (R9).
This is important for NaCl, because NaCl ARM code cannot use R9 at all.
The Go 1 macrobenchmarks (those with per-op times >= 10 µs) are unaffected:
BenchmarkBinaryTree17 5491374955 5471024381 -0.37%
BenchmarkFannkuch11 4357101311 4275174828 -1.88%
BenchmarkGobDecode 11029957 11364184 +3.03%
BenchmarkGobEncode 6852205 6784822 -0.98%
BenchmarkGzip 650795967 650152275 -0.10%
BenchmarkGunzip 140962363 141041670 +0.06%
BenchmarkHTTPClientServer 71581 73081 +2.10%
BenchmarkJSONEncode 31928079 31913356 -0.05%
BenchmarkJSONDecode 117470065 113689916 -3.22%
BenchmarkMandelbrot200 6008923 5998712 -0.17%
BenchmarkGoParse 6310917 6327487 +0.26%
BenchmarkRegexpMatchMedium_1K 114568 114763 +0.17%
BenchmarkRegexpMatchHard_1K 168977 169244 +0.16%
BenchmarkRevcomp 935294971 914060918 -2.27%
BenchmarkTemplate 145917123 148186096 +1.55%
Minux previous reported larger variations, but these were caused by
run-to-run noise, not repeatable slowdowns.
Actual code changes by Minux.
I only did the docs and the benchmarking.
LGTM=dvyukov, iant, minux
R=minux, josharian, iant, dave, bradfitz, dvyukov
CC=golang-codereviews
https://golang.org/cl/109050043
2014-06-26 09:54:39 -06:00
|
|
|
g->m->blocked = false;
|
2013-08-08 07:41:57 -06:00
|
|
|
for(i = 0; i < n; i++) {
|
|
|
|
op = entries[i].op;
|
|
|
|
errno = 0;
|
|
|
|
qty = 0;
|
2014-08-29 02:44:07 -06:00
|
|
|
if(runtime·stdcall5(runtime·WSAGetOverlappedResult, runtime·netpollfd(op->pd), (uintptr)op, (uintptr)&qty, 0, (uintptr)&flags) == 0)
|
2013-08-08 07:41:57 -06:00
|
|
|
errno = runtime·getlasterror();
|
|
|
|
handlecompletion(&gp, op, errno, qty);
|
2013-07-21 20:49:57 -06:00
|
|
|
}
|
2013-08-08 07:41:57 -06:00
|
|
|
} else {
|
|
|
|
op = nil;
|
|
|
|
errno = 0;
|
|
|
|
qty = 0;
|
2014-02-11 02:41:46 -07:00
|
|
|
if(block)
|
all: remove 'extern register M *m' from runtime
The runtime has historically held two dedicated values g (current goroutine)
and m (current thread) in 'extern register' slots (TLS on x86, real registers
backed by TLS on ARM).
This CL removes the extern register m; code now uses g->m.
On ARM, this frees up the register that formerly held m (R9).
This is important for NaCl, because NaCl ARM code cannot use R9 at all.
The Go 1 macrobenchmarks (those with per-op times >= 10 µs) are unaffected:
BenchmarkBinaryTree17 5491374955 5471024381 -0.37%
BenchmarkFannkuch11 4357101311 4275174828 -1.88%
BenchmarkGobDecode 11029957 11364184 +3.03%
BenchmarkGobEncode 6852205 6784822 -0.98%
BenchmarkGzip 650795967 650152275 -0.10%
BenchmarkGunzip 140962363 141041670 +0.06%
BenchmarkHTTPClientServer 71581 73081 +2.10%
BenchmarkJSONEncode 31928079 31913356 -0.05%
BenchmarkJSONDecode 117470065 113689916 -3.22%
BenchmarkMandelbrot200 6008923 5998712 -0.17%
BenchmarkGoParse 6310917 6327487 +0.26%
BenchmarkRegexpMatchMedium_1K 114568 114763 +0.17%
BenchmarkRegexpMatchHard_1K 168977 169244 +0.16%
BenchmarkRevcomp 935294971 914060918 -2.27%
BenchmarkTemplate 145917123 148186096 +1.55%
Minux previous reported larger variations, but these were caused by
run-to-run noise, not repeatable slowdowns.
Actual code changes by Minux.
I only did the docs and the benchmarking.
LGTM=dvyukov, iant, minux
R=minux, josharian, iant, dave, bradfitz, dvyukov
CC=golang-codereviews
https://golang.org/cl/109050043
2014-06-26 09:54:39 -06:00
|
|
|
g->m->blocked = true;
|
2014-08-29 02:44:07 -06:00
|
|
|
if(runtime·stdcall5(runtime·GetQueuedCompletionStatus, iocphandle, (uintptr)&qty, (uintptr)&key, (uintptr)&op, wait) == 0) {
|
all: remove 'extern register M *m' from runtime
The runtime has historically held two dedicated values g (current goroutine)
and m (current thread) in 'extern register' slots (TLS on x86, real registers
backed by TLS on ARM).
This CL removes the extern register m; code now uses g->m.
On ARM, this frees up the register that formerly held m (R9).
This is important for NaCl, because NaCl ARM code cannot use R9 at all.
The Go 1 macrobenchmarks (those with per-op times >= 10 µs) are unaffected:
BenchmarkBinaryTree17 5491374955 5471024381 -0.37%
BenchmarkFannkuch11 4357101311 4275174828 -1.88%
BenchmarkGobDecode 11029957 11364184 +3.03%
BenchmarkGobEncode 6852205 6784822 -0.98%
BenchmarkGzip 650795967 650152275 -0.10%
BenchmarkGunzip 140962363 141041670 +0.06%
BenchmarkHTTPClientServer 71581 73081 +2.10%
BenchmarkJSONEncode 31928079 31913356 -0.05%
BenchmarkJSONDecode 117470065 113689916 -3.22%
BenchmarkMandelbrot200 6008923 5998712 -0.17%
BenchmarkGoParse 6310917 6327487 +0.26%
BenchmarkRegexpMatchMedium_1K 114568 114763 +0.17%
BenchmarkRegexpMatchHard_1K 168977 169244 +0.16%
BenchmarkRevcomp 935294971 914060918 -2.27%
BenchmarkTemplate 145917123 148186096 +1.55%
Minux previous reported larger variations, but these were caused by
run-to-run noise, not repeatable slowdowns.
Actual code changes by Minux.
I only did the docs and the benchmarking.
LGTM=dvyukov, iant, minux
R=minux, josharian, iant, dave, bradfitz, dvyukov
CC=golang-codereviews
https://golang.org/cl/109050043
2014-06-26 09:54:39 -06:00
|
|
|
g->m->blocked = false;
|
2013-08-08 07:41:57 -06:00
|
|
|
errno = runtime·getlasterror();
|
|
|
|
if(!block && errno == WAIT_TIMEOUT)
|
|
|
|
return nil;
|
|
|
|
if(op == nil) {
|
|
|
|
runtime·printf("netpoll: GetQueuedCompletionStatus failed (errno=%d)\n", errno);
|
|
|
|
runtime·throw("netpoll: GetQueuedCompletionStatus failed");
|
|
|
|
}
|
|
|
|
// dequeued failed IO packet, so report that
|
|
|
|
}
|
all: remove 'extern register M *m' from runtime
The runtime has historically held two dedicated values g (current goroutine)
and m (current thread) in 'extern register' slots (TLS on x86, real registers
backed by TLS on ARM).
This CL removes the extern register m; code now uses g->m.
On ARM, this frees up the register that formerly held m (R9).
This is important for NaCl, because NaCl ARM code cannot use R9 at all.
The Go 1 macrobenchmarks (those with per-op times >= 10 µs) are unaffected:
BenchmarkBinaryTree17 5491374955 5471024381 -0.37%
BenchmarkFannkuch11 4357101311 4275174828 -1.88%
BenchmarkGobDecode 11029957 11364184 +3.03%
BenchmarkGobEncode 6852205 6784822 -0.98%
BenchmarkGzip 650795967 650152275 -0.10%
BenchmarkGunzip 140962363 141041670 +0.06%
BenchmarkHTTPClientServer 71581 73081 +2.10%
BenchmarkJSONEncode 31928079 31913356 -0.05%
BenchmarkJSONDecode 117470065 113689916 -3.22%
BenchmarkMandelbrot200 6008923 5998712 -0.17%
BenchmarkGoParse 6310917 6327487 +0.26%
BenchmarkRegexpMatchMedium_1K 114568 114763 +0.17%
BenchmarkRegexpMatchHard_1K 168977 169244 +0.16%
BenchmarkRevcomp 935294971 914060918 -2.27%
BenchmarkTemplate 145917123 148186096 +1.55%
Minux previous reported larger variations, but these were caused by
run-to-run noise, not repeatable slowdowns.
Actual code changes by Minux.
I only did the docs and the benchmarking.
LGTM=dvyukov, iant, minux
R=minux, josharian, iant, dave, bradfitz, dvyukov
CC=golang-codereviews
https://golang.org/cl/109050043
2014-06-26 09:54:39 -06:00
|
|
|
g->m->blocked = false;
|
2013-08-08 07:41:57 -06:00
|
|
|
handlecompletion(&gp, op, errno, qty);
|
2013-07-21 20:49:57 -06:00
|
|
|
}
|
2013-08-08 07:41:57 -06:00
|
|
|
if(block && gp == nil)
|
|
|
|
goto retry;
|
|
|
|
return gp;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void
|
|
|
|
handlecompletion(G **gpp, net_op *op, int32 errno, uint32 qty)
|
|
|
|
{
|
|
|
|
int32 mode;
|
|
|
|
|
|
|
|
if(op == nil)
|
|
|
|
runtime·throw("netpoll: GetQueuedCompletionStatus returned op == nil");
|
|
|
|
mode = op->mode;
|
2013-07-21 20:49:57 -06:00
|
|
|
if(mode != 'r' && mode != 'w') {
|
|
|
|
runtime·printf("netpoll: GetQueuedCompletionStatus returned invalid mode=%d\n", mode);
|
|
|
|
runtime·throw("netpoll: GetQueuedCompletionStatus returned invalid mode");
|
|
|
|
}
|
2013-08-08 07:41:57 -06:00
|
|
|
op->errno = errno;
|
|
|
|
op->qty = qty;
|
|
|
|
runtime·netpollready(gpp, op->pd, mode);
|
2013-07-21 20:49:57 -06:00
|
|
|
}
|