qbit/go
1
0
Fork 0
mirror of https://github.com/golang/go synced 2024-11-19 02:44:44 -07:00
Code Releases Activity

internal/poll: make SendFile work with large files on Windows

Browse Source 
CL 192518 was a minimal simplification to get sendfile
on Windows to work with chunked files, but as I had mentioned,
I would add even more improvements.

This CL improves it by:
* If the reader is not an *io.LimitedReader, since the underlying
reader is anyways an *os.File, we fallback and stat that
file to determine the file size and then also invoke the chunked
sendFile on the underlying reader. This issue existed even
before the prior CL.
* Extracting the chunked TransmitFile logic and moving it directly
into internal/poll.SendFile.

Thus if the callers of net.sendFile don't use *io.LimitedReader,
but have a huge file (>2GiB), we can still invoke the chunked
internal/poll.SendFile on it directly.

The test case is not included in this patch as it requires
creating a 3GiB file, but that if anyone wants to view it, they
can find it at
    https://go-review.googlesource.com/c/go/+/194218/13/src/net/sendfile_windows_test.go

Updates #33193.

Change-Id: I97a67c712d558c84ced716d8df98b040cd7ed7f7
Reviewed-on: https://go-review.googlesource.com/c/go/+/194218
Run-TryBot: Emmanuel Odeke <emm.odeke@gmail.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Alex Brainman <alex.brainman@gmail.com>
This commit is contained in:
Emmanuel T Odeke 2019-09-08 12:03:42 -07:00 committed by Emmanuel Odeke
parent f73d80809b
commit b2f40019da
2 changed files with 54 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

61
src/internal/poll/sendfile_windows.go
View File

@ -4,10 +4,13 @@
package poll
import "syscall"
import (
"io"
"syscall"
)
// SendFile wraps the TransmitFile call.
func SendFile(fd *FD, src syscall.Handle, n int64) (int64, error) {
func SendFile(fd *FD, src syscall.Handle, n int64) (written int64, err error) {
if fd.kind == kindPipe {
// TransmitFile does not work with pipes
return 0, syscall.ESPIPE
@ -19,26 +22,60 @@ func SendFile(fd *FD, src syscall.Handle, n int64) (int64, error) {
defer fd.writeUnlock()
o := &fd.wop
o.qty = uint32(n)
o.handle = src
// TODO(brainman): skip calling syscall.Seek if OS allows it
curpos, err := syscall.Seek(o.handle, 0, 1)
curpos, err := syscall.Seek(o.handle, 0, io.SeekCurrent)
if err != nil {
return 0, err
}
o.o.Offset = uint32(curpos)
o.o.OffsetHigh = uint32(curpos >> 32)
if n <= 0 { // We don't know the size of the file so infer it.
// Find the number of bytes offset from curpos until the end of the file.
n, err = syscall.Seek(o.handle, -curpos, io.SeekEnd)
if err != nil {
return
}
// Now seek back to the original position.
if _, err = syscall.Seek(o.handle, curpos, io.SeekStart); err != nil {
return
}
}
// TransmitFile can be invoked in one call with at most
// 2,147,483,646 bytes: the maximum value for a 32-bit integer minus 1.
// See https://docs.microsoft.com/en-us/windows/win32/api/mswsock/nf-mswsock-transmitfile
const maxChunkSizePerCall = int64(0x7fffffff - 1)
for n > 0 {
chunkSize := maxChunkSizePerCall
if chunkSize > n {
chunkSize = n
}
o.qty = uint32(chunkSize)
o.o.Offset = uint32(curpos)
o.o.OffsetHigh = uint32(curpos >> 32)
nw, err := wsrv.ExecIO(o, func(o *operation) error {
return syscall.TransmitFile(o.fd.Sysfd, o.handle, o.qty, 0, &o.o, nil, syscall.TF_WRITE_BEHIND)
})
if err != nil {
return written, err
}
curpos += int64(nw)
done, err := wsrv.ExecIO(o, func(o *operation) error {
return syscall.TransmitFile(o.fd.Sysfd, o.handle, o.qty, 0, &o.o, nil, syscall.TF_WRITE_BEHIND)
})
if err == nil {
// Some versions of Windows (Windows 10 1803) do not set
// file position after TransmitFile completes.
// So just use Seek to set file position.
_, err = syscall.Seek(o.handle, curpos+int64(done), 0)
if _, err = syscall.Seek(o.handle, curpos, io.SeekStart); err != nil {
return written, err
}
n -= int64(nw)
written += int64(nw)
}
return int64(done), err
return
}

42
src/net/sendfile_windows.go
View File

@ -34,46 +34,14 @@ func sendFile(fd *netFD, r io.Reader) (written int64, err error, handled bool) {
return 0, nil, false
}
// TransmitFile can be invoked in one call with at most
// 2,147,483,646 bytes: the maximum value for a 32-bit integer minus 1.
// See https://docs.microsoft.com/en-us/windows/win32/api/mswsock/nf-mswsock-transmitfile
const maxChunkSizePerCall = int64(0x7fffffff - 1)
switch {
case n <= maxChunkSizePerCall:
// The file is within sendfile's limits.
written, err = doSendFile(fd, lr, f, n)
default:
// Now invoke doSendFile on the file in chunks of upto 2GiB per chunk.
for lr.N > 0 { // lr.N is decremented in every successful invocation of doSendFile.
chunkSize := maxChunkSizePerCall
if chunkSize > lr.N {
chunkSize = lr.N
}
var nw int64
nw, err = doSendFile(fd, lr, f, chunkSize)
if err != nil {
break
}
written += nw
}
written, err = poll.SendFile(&fd.pfd, syscall.Handle(f.Fd()), n)
if err != nil {
err = wrapSyscallError("transmitfile", err)
}
// If any byte was copied, regardless of any error
// encountered mid-way, handled must be set to true.
return written, err, written > 0
}
handled = written > 0
// doSendFile is a helper to invoke poll.SendFile.
// It will decrement lr.N by the number of written bytes.
func doSendFile(fd *netFD, lr *io.LimitedReader, f *os.File, remain int64) (written int64, err error) {
done, err := poll.SendFile(&fd.pfd, syscall.Handle(f.Fd()), remain)
if err != nil {
return 0, wrapSyscallError("transmitfile", err)
}
if lr != nil {
lr.N -= int64(done)
}
return int64(done), nil
return
}