2011-02-23 13:20:50 -07:00
|
|
|
// Copyright 2011 The Go Authors. All rights reserved.
|
|
|
|
// Use of this source code is governed by a BSD-style
|
|
|
|
// license that can be found in the LICENSE file.
|
|
|
|
|
2011-10-19 09:48:26 -06:00
|
|
|
// HTTP client implementation. See RFC 2616.
|
2012-10-30 14:38:01 -06:00
|
|
|
//
|
2011-10-19 09:48:26 -06:00
|
|
|
// This is the low-level Transport implementation of RoundTripper.
|
|
|
|
// The high-level interface is in client.go.
|
|
|
|
|
2011-02-23 13:20:50 -07:00
|
|
|
package http
|
|
|
|
|
|
|
|
import (
|
|
|
|
"bufio"
|
2011-04-12 10:35:07 -06:00
|
|
|
"compress/gzip"
|
2011-02-23 13:20:50 -07:00
|
|
|
"crypto/tls"
|
|
|
|
"encoding/base64"
|
2011-11-01 20:04:37 -06:00
|
|
|
"errors"
|
2011-02-23 13:20:50 -07:00
|
|
|
"fmt"
|
2011-03-23 11:38:18 -06:00
|
|
|
"io"
|
2011-05-11 20:33:15 -06:00
|
|
|
"io/ioutil"
|
2011-03-23 11:38:18 -06:00
|
|
|
"log"
|
2011-02-23 13:20:50 -07:00
|
|
|
"net"
|
2011-11-08 16:41:54 -07:00
|
|
|
"net/url"
|
2011-02-23 13:20:50 -07:00
|
|
|
"os"
|
|
|
|
"strings"
|
|
|
|
"sync"
|
2012-07-11 17:40:44 -06:00
|
|
|
"time"
|
2011-02-23 13:20:50 -07:00
|
|
|
)
|
|
|
|
|
2011-03-04 12:41:57 -07:00
|
|
|
// DefaultTransport is the default implementation of Transport and is
|
|
|
|
// used by DefaultClient. It establishes a new network connection for
|
|
|
|
// each call to Do and uses HTTP proxies as directed by the
|
|
|
|
// $HTTP_PROXY and $NO_PROXY (or $http_proxy and $no_proxy)
|
|
|
|
// environment variables.
|
2011-05-18 10:23:29 -06:00
|
|
|
var DefaultTransport RoundTripper = &Transport{Proxy: ProxyFromEnvironment}
|
2011-03-23 11:38:18 -06:00
|
|
|
|
2011-03-31 13:58:50 -06:00
|
|
|
// DefaultMaxIdleConnsPerHost is the default value of Transport's
|
|
|
|
// MaxIdleConnsPerHost.
|
|
|
|
const DefaultMaxIdleConnsPerHost = 2
|
|
|
|
|
2011-03-23 11:38:18 -06:00
|
|
|
// Transport is an implementation of RoundTripper that supports http,
|
|
|
|
// https, and http proxies (for either http or https with CONNECT).
|
|
|
|
// Transport can also cache connections for future re-use.
|
|
|
|
type Transport struct {
|
2012-08-20 03:28:27 -06:00
|
|
|
idleLk sync.Mutex
|
2011-03-23 11:38:18 -06:00
|
|
|
idleConn map[string][]*persistConn
|
2012-08-20 03:28:27 -06:00
|
|
|
altLk sync.RWMutex
|
2011-05-25 13:31:11 -06:00
|
|
|
altProto map[string]RoundTripper // nil or map of URI scheme => RoundTripper
|
2011-03-23 11:38:18 -06:00
|
|
|
|
2011-03-31 13:58:50 -06:00
|
|
|
// TODO: tunable on global max cached connections
|
|
|
|
// TODO: tunable on timeout on cached connections
|
2011-03-23 11:38:18 -06:00
|
|
|
// TODO: optional pipelining
|
|
|
|
|
2011-05-29 10:32:36 -06:00
|
|
|
// Proxy specifies a function to return a proxy for a given
|
|
|
|
// Request. If the function returns a non-nil error, the
|
|
|
|
// request is aborted with the provided error.
|
|
|
|
// If Proxy is nil or returns a nil *URL, no proxy is used.
|
2011-11-01 20:04:37 -06:00
|
|
|
Proxy func(*Request) (*url.URL, error)
|
2011-05-18 10:23:29 -06:00
|
|
|
|
2011-05-29 10:32:36 -06:00
|
|
|
// Dial specifies the dial function for creating TCP
|
|
|
|
// connections.
|
|
|
|
// If Dial is nil, net.Dial is used.
|
2011-11-01 20:04:37 -06:00
|
|
|
Dial func(net, addr string) (c net.Conn, err error)
|
2011-05-29 10:32:36 -06:00
|
|
|
|
2011-08-26 00:06:35 -06:00
|
|
|
// TLSClientConfig specifies the TLS configuration to use with
|
|
|
|
// tls.Client. If nil, the default configuration is used.
|
|
|
|
TLSClientConfig *tls.Config
|
|
|
|
|
2011-04-12 10:35:07 -06:00
|
|
|
DisableKeepAlives bool
|
|
|
|
DisableCompression bool
|
2011-03-31 13:58:50 -06:00
|
|
|
|
|
|
|
// MaxIdleConnsPerHost, if non-zero, controls the maximum idle
|
2012-11-24 14:08:17 -07:00
|
|
|
// (keep-alive) to keep per-host. If zero,
|
2011-03-31 13:58:50 -06:00
|
|
|
// DefaultMaxIdleConnsPerHost is used.
|
|
|
|
MaxIdleConnsPerHost int
|
2011-03-23 11:38:18 -06:00
|
|
|
}
|
|
|
|
|
2011-05-18 10:23:29 -06:00
|
|
|
// ProxyFromEnvironment returns the URL of the proxy to use for a
|
|
|
|
// given request, as indicated by the environment variables
|
|
|
|
// $HTTP_PROXY and $NO_PROXY (or $http_proxy and $no_proxy).
|
2012-02-29 10:52:52 -07:00
|
|
|
// An error is returned if the proxy environment is invalid.
|
|
|
|
// A nil URL and nil error are returned if no proxy is defined in the
|
|
|
|
// environment, or a proxy should not be used for the given request.
|
2011-11-01 20:04:37 -06:00
|
|
|
func ProxyFromEnvironment(req *Request) (*url.URL, error) {
|
2011-05-18 10:23:29 -06:00
|
|
|
proxy := getenvEitherCase("HTTP_PROXY")
|
|
|
|
if proxy == "" {
|
|
|
|
return nil, nil
|
|
|
|
}
|
|
|
|
if !useProxy(canonicalAddr(req.URL)) {
|
|
|
|
return nil, nil
|
|
|
|
}
|
2012-02-12 21:19:50 -07:00
|
|
|
proxyURL, err := url.Parse(proxy)
|
2012-02-29 10:52:52 -07:00
|
|
|
if err != nil || proxyURL.Scheme == "" {
|
2012-02-12 21:19:50 -07:00
|
|
|
if u, err := url.Parse("http://" + proxy); err == nil {
|
|
|
|
proxyURL = u
|
|
|
|
err = nil
|
2011-05-18 10:23:29 -06:00
|
|
|
}
|
|
|
|
}
|
2012-02-12 21:19:50 -07:00
|
|
|
if err != nil {
|
|
|
|
return nil, fmt.Errorf("invalid proxy address %q: %v", proxy, err)
|
|
|
|
}
|
2011-05-18 10:23:29 -06:00
|
|
|
return proxyURL, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// ProxyURL returns a proxy function (for use in a Transport)
|
|
|
|
// that always returns the same URL.
|
2011-11-01 20:04:37 -06:00
|
|
|
func ProxyURL(fixedURL *url.URL) func(*Request) (*url.URL, error) {
|
|
|
|
return func(*Request) (*url.URL, error) {
|
2011-08-16 21:36:02 -06:00
|
|
|
return fixedURL, nil
|
2011-05-18 10:23:29 -06:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2011-10-14 15:16:43 -06:00
|
|
|
// transportRequest is a wrapper around a *Request that adds
|
|
|
|
// optional extra headers to write.
|
|
|
|
type transportRequest struct {
|
|
|
|
*Request // original request, not to be mutated
|
|
|
|
extra Header // extra headers to write, or nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func (tr *transportRequest) extraHeaders() Header {
|
|
|
|
if tr.extra == nil {
|
|
|
|
tr.extra = make(Header)
|
|
|
|
}
|
|
|
|
return tr.extra
|
|
|
|
}
|
|
|
|
|
2011-03-23 11:38:18 -06:00
|
|
|
// RoundTrip implements the RoundTripper interface.
|
2011-11-01 20:04:37 -06:00
|
|
|
func (t *Transport) RoundTrip(req *Request) (resp *Response, err error) {
|
2011-03-11 10:54:31 -07:00
|
|
|
if req.URL == nil {
|
2011-11-01 20:04:37 -06:00
|
|
|
return nil, errors.New("http: nil Request.URL")
|
2011-03-11 10:54:31 -07:00
|
|
|
}
|
2011-10-14 15:16:43 -06:00
|
|
|
if req.Header == nil {
|
2011-11-01 20:04:37 -06:00
|
|
|
return nil, errors.New("http: nil Request.Header")
|
2011-10-14 15:16:43 -06:00
|
|
|
}
|
2011-02-23 13:20:50 -07:00
|
|
|
if req.URL.Scheme != "http" && req.URL.Scheme != "https" {
|
2012-08-20 03:28:27 -06:00
|
|
|
t.altLk.RLock()
|
2011-05-25 13:31:11 -06:00
|
|
|
var rt RoundTripper
|
|
|
|
if t.altProto != nil {
|
|
|
|
rt = t.altProto[req.URL.Scheme]
|
|
|
|
}
|
2012-08-20 03:28:27 -06:00
|
|
|
t.altLk.RUnlock()
|
2011-05-25 13:31:11 -06:00
|
|
|
if rt == nil {
|
|
|
|
return nil, &badStringError{"unsupported protocol scheme", req.URL.Scheme}
|
|
|
|
}
|
|
|
|
return rt.RoundTrip(req)
|
2011-02-23 13:20:50 -07:00
|
|
|
}
|
2011-10-14 15:16:43 -06:00
|
|
|
treq := &transportRequest{Request: req}
|
|
|
|
cm, err := t.connectMethodForRequest(treq)
|
2011-03-23 11:38:18 -06:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
// Get the cached or newly-created connection to either the
|
|
|
|
// host (for http or https), the http proxy, or the http proxy
|
|
|
|
// pre-CONNECTed to https server. In any case, we'll be ready
|
|
|
|
// to send it requests.
|
|
|
|
pconn, err := t.getConn(cm)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
2011-02-23 13:20:50 -07:00
|
|
|
}
|
|
|
|
|
2011-10-14 15:16:43 -06:00
|
|
|
return pconn.roundTrip(treq)
|
2011-03-23 11:38:18 -06:00
|
|
|
}
|
|
|
|
|
2011-05-25 13:31:11 -06:00
|
|
|
// RegisterProtocol registers a new protocol with scheme.
|
|
|
|
// The Transport will pass requests using the given scheme to rt.
|
|
|
|
// It is rt's responsibility to simulate HTTP request semantics.
|
|
|
|
//
|
|
|
|
// RegisterProtocol can be used by other packages to provide
|
|
|
|
// implementations of protocol schemes like "ftp" or "file".
|
|
|
|
func (t *Transport) RegisterProtocol(scheme string, rt RoundTripper) {
|
|
|
|
if scheme == "http" || scheme == "https" {
|
|
|
|
panic("protocol " + scheme + " already registered")
|
|
|
|
}
|
2012-08-20 03:28:27 -06:00
|
|
|
t.altLk.Lock()
|
|
|
|
defer t.altLk.Unlock()
|
2011-05-25 13:31:11 -06:00
|
|
|
if t.altProto == nil {
|
|
|
|
t.altProto = make(map[string]RoundTripper)
|
|
|
|
}
|
|
|
|
if _, exists := t.altProto[scheme]; exists {
|
|
|
|
panic("protocol " + scheme + " already registered")
|
|
|
|
}
|
|
|
|
t.altProto[scheme] = rt
|
|
|
|
}
|
|
|
|
|
2011-03-23 11:38:18 -06:00
|
|
|
// CloseIdleConnections closes any connections which were previously
|
|
|
|
// connected from previous requests but are now sitting idle in
|
|
|
|
// a "keep-alive" state. It does not interrupt any connections currently
|
|
|
|
// in use.
|
|
|
|
func (t *Transport) CloseIdleConnections() {
|
2012-08-20 03:28:27 -06:00
|
|
|
t.idleLk.Lock()
|
|
|
|
m := t.idleConn
|
|
|
|
t.idleConn = nil
|
|
|
|
t.idleLk.Unlock()
|
|
|
|
if m == nil {
|
2011-03-23 11:38:18 -06:00
|
|
|
return
|
|
|
|
}
|
2012-08-20 03:28:27 -06:00
|
|
|
for _, conns := range m {
|
2011-03-23 11:38:18 -06:00
|
|
|
for _, pconn := range conns {
|
|
|
|
pconn.close()
|
2011-02-23 13:20:50 -07:00
|
|
|
}
|
|
|
|
}
|
2011-03-23 11:38:18 -06:00
|
|
|
}
|
2011-02-23 13:20:50 -07:00
|
|
|
|
2011-03-23 11:38:18 -06:00
|
|
|
//
|
|
|
|
// Private implementation past this point.
|
|
|
|
//
|
2011-03-05 12:35:15 -07:00
|
|
|
|
2011-05-18 10:23:29 -06:00
|
|
|
func getenvEitherCase(k string) string {
|
|
|
|
if v := os.Getenv(strings.ToUpper(k)); v != "" {
|
2011-03-23 11:38:18 -06:00
|
|
|
return v
|
|
|
|
}
|
2011-05-18 10:23:29 -06:00
|
|
|
return os.Getenv(strings.ToLower(k))
|
2011-03-23 11:38:18 -06:00
|
|
|
}
|
|
|
|
|
2011-11-01 20:04:37 -06:00
|
|
|
func (t *Transport) connectMethodForRequest(treq *transportRequest) (*connectMethod, error) {
|
2011-03-23 11:38:18 -06:00
|
|
|
cm := &connectMethod{
|
2011-10-14 15:16:43 -06:00
|
|
|
targetScheme: treq.URL.Scheme,
|
|
|
|
targetAddr: canonicalAddr(treq.URL),
|
2011-03-23 11:38:18 -06:00
|
|
|
}
|
2011-05-18 10:23:29 -06:00
|
|
|
if t.Proxy != nil {
|
2011-11-01 20:04:37 -06:00
|
|
|
var err error
|
2011-10-14 15:16:43 -06:00
|
|
|
cm.proxyURL, err = t.Proxy(treq.Request)
|
2011-02-23 13:20:50 -07:00
|
|
|
if err != nil {
|
2011-05-18 10:23:29 -06:00
|
|
|
return nil, err
|
2011-02-23 13:20:50 -07:00
|
|
|
}
|
2011-03-23 11:38:18 -06:00
|
|
|
}
|
|
|
|
return cm, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// proxyAuth returns the Proxy-Authorization header to set
|
|
|
|
// on requests, if applicable.
|
|
|
|
func (cm *connectMethod) proxyAuth() string {
|
|
|
|
if cm.proxyURL == nil {
|
|
|
|
return ""
|
|
|
|
}
|
2012-01-16 19:49:05 -07:00
|
|
|
if u := cm.proxyURL.User; u != nil {
|
|
|
|
return "Basic " + base64.URLEncoding.EncodeToString([]byte(u.String()))
|
2011-03-23 11:38:18 -06:00
|
|
|
}
|
|
|
|
return ""
|
|
|
|
}
|
|
|
|
|
2012-02-13 18:48:56 -07:00
|
|
|
// putIdleConn adds pconn to the list of idle persistent connections awaiting
|
|
|
|
// a new request.
|
|
|
|
// If pconn is no longer needed or not in a good state, putIdleConn
|
|
|
|
// returns false.
|
|
|
|
func (t *Transport) putIdleConn(pconn *persistConn) bool {
|
2011-03-31 13:58:50 -06:00
|
|
|
if t.DisableKeepAlives || t.MaxIdleConnsPerHost < 0 {
|
2011-03-23 11:38:18 -06:00
|
|
|
pconn.close()
|
2012-02-13 18:48:56 -07:00
|
|
|
return false
|
2011-03-23 11:38:18 -06:00
|
|
|
}
|
|
|
|
if pconn.isBroken() {
|
2012-02-13 18:48:56 -07:00
|
|
|
return false
|
2011-03-23 11:38:18 -06:00
|
|
|
}
|
|
|
|
key := pconn.cacheKey
|
2011-03-31 13:58:50 -06:00
|
|
|
max := t.MaxIdleConnsPerHost
|
|
|
|
if max == 0 {
|
|
|
|
max = DefaultMaxIdleConnsPerHost
|
|
|
|
}
|
2012-08-20 03:28:27 -06:00
|
|
|
t.idleLk.Lock()
|
|
|
|
if t.idleConn == nil {
|
|
|
|
t.idleConn = make(map[string][]*persistConn)
|
|
|
|
}
|
2011-03-31 13:58:50 -06:00
|
|
|
if len(t.idleConn[key]) >= max {
|
2012-08-20 03:28:27 -06:00
|
|
|
t.idleLk.Unlock()
|
2011-03-31 13:58:50 -06:00
|
|
|
pconn.close()
|
2012-02-13 18:48:56 -07:00
|
|
|
return false
|
2011-03-31 13:58:50 -06:00
|
|
|
}
|
2012-07-11 17:40:44 -06:00
|
|
|
for _, exist := range t.idleConn[key] {
|
|
|
|
if exist == pconn {
|
|
|
|
log.Fatalf("dup idle pconn %p in freelist", pconn)
|
|
|
|
}
|
|
|
|
}
|
2011-03-23 11:38:18 -06:00
|
|
|
t.idleConn[key] = append(t.idleConn[key], pconn)
|
2012-08-20 03:28:27 -06:00
|
|
|
t.idleLk.Unlock()
|
2012-02-13 18:48:56 -07:00
|
|
|
return true
|
2011-03-23 11:38:18 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
func (t *Transport) getIdleConn(cm *connectMethod) (pconn *persistConn) {
|
2012-08-20 03:28:27 -06:00
|
|
|
key := cm.String()
|
|
|
|
t.idleLk.Lock()
|
|
|
|
defer t.idleLk.Unlock()
|
2011-03-23 11:38:18 -06:00
|
|
|
if t.idleConn == nil {
|
2012-08-20 03:28:27 -06:00
|
|
|
return nil
|
2011-03-23 11:38:18 -06:00
|
|
|
}
|
|
|
|
for {
|
|
|
|
pconns, ok := t.idleConn[key]
|
|
|
|
if !ok {
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
if len(pconns) == 1 {
|
|
|
|
pconn = pconns[0]
|
2011-10-18 07:56:34 -06:00
|
|
|
delete(t.idleConn, key)
|
2011-03-23 11:38:18 -06:00
|
|
|
} else {
|
|
|
|
// 2 or more cached connections; pop last
|
|
|
|
// TODO: queue?
|
|
|
|
pconn = pconns[len(pconns)-1]
|
|
|
|
t.idleConn[key] = pconns[0 : len(pconns)-1]
|
|
|
|
}
|
|
|
|
if !pconn.isBroken() {
|
|
|
|
return
|
2011-02-23 13:20:50 -07:00
|
|
|
}
|
|
|
|
}
|
2012-07-11 17:40:44 -06:00
|
|
|
panic("unreachable")
|
2011-03-23 11:38:18 -06:00
|
|
|
}
|
|
|
|
|
2011-11-01 20:04:37 -06:00
|
|
|
func (t *Transport) dial(network, addr string) (c net.Conn, err error) {
|
2011-05-29 10:32:36 -06:00
|
|
|
if t.Dial != nil {
|
|
|
|
return t.Dial(network, addr)
|
|
|
|
}
|
|
|
|
return net.Dial(network, addr)
|
|
|
|
}
|
|
|
|
|
2011-03-23 11:38:18 -06:00
|
|
|
// getConn dials and creates a new persistConn to the target as
|
|
|
|
// specified in the connectMethod. This includes doing a proxy CONNECT
|
|
|
|
// and/or setting up TLS. If this doesn't return an error, the persistConn
|
|
|
|
// is ready to write requests to.
|
2011-11-01 20:04:37 -06:00
|
|
|
func (t *Transport) getConn(cm *connectMethod) (*persistConn, error) {
|
2011-03-23 11:38:18 -06:00
|
|
|
if pc := t.getIdleConn(cm); pc != nil {
|
|
|
|
return pc, nil
|
|
|
|
}
|
2011-02-23 13:20:50 -07:00
|
|
|
|
2011-05-29 10:32:36 -06:00
|
|
|
conn, err := t.dial("tcp", cm.addr())
|
2011-02-23 13:20:50 -07:00
|
|
|
if err != nil {
|
2011-04-14 14:49:19 -06:00
|
|
|
if cm.proxyURL != nil {
|
|
|
|
err = fmt.Errorf("http: error connecting to proxy %s: %v", cm.proxyURL, err)
|
|
|
|
}
|
2011-02-23 13:20:50 -07:00
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
2011-03-23 11:38:18 -06:00
|
|
|
pa := cm.proxyAuth()
|
2011-02-23 13:20:50 -07:00
|
|
|
|
2011-03-23 11:38:18 -06:00
|
|
|
pconn := &persistConn{
|
|
|
|
t: t,
|
|
|
|
cacheKey: cm.String(),
|
|
|
|
conn: conn,
|
|
|
|
reqch: make(chan requestAndChan, 50),
|
2012-06-19 10:20:41 -06:00
|
|
|
writech: make(chan writeRequest, 50),
|
2012-07-11 17:40:44 -06:00
|
|
|
closech: make(chan struct{}),
|
2011-03-23 11:38:18 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
switch {
|
|
|
|
case cm.proxyURL == nil:
|
|
|
|
// Do nothing.
|
|
|
|
case cm.targetScheme == "http":
|
2011-10-14 15:16:43 -06:00
|
|
|
pconn.isProxy = true
|
2011-03-23 11:38:18 -06:00
|
|
|
if pa != "" {
|
2011-10-14 15:16:43 -06:00
|
|
|
pconn.mutateHeaderFunc = func(h Header) {
|
|
|
|
h.Set("Proxy-Authorization", pa)
|
2011-02-23 13:20:50 -07:00
|
|
|
}
|
|
|
|
}
|
2011-03-23 11:38:18 -06:00
|
|
|
case cm.targetScheme == "https":
|
2011-05-13 08:31:24 -06:00
|
|
|
connectReq := &Request{
|
|
|
|
Method: "CONNECT",
|
2012-01-16 19:49:05 -07:00
|
|
|
URL: &url.URL{Opaque: cm.targetAddr},
|
2011-05-13 08:31:24 -06:00
|
|
|
Host: cm.targetAddr,
|
|
|
|
Header: make(Header),
|
|
|
|
}
|
2011-03-23 11:38:18 -06:00
|
|
|
if pa != "" {
|
2011-05-13 08:31:24 -06:00
|
|
|
connectReq.Header.Set("Proxy-Authorization", pa)
|
2011-03-23 11:38:18 -06:00
|
|
|
}
|
2011-05-13 08:31:24 -06:00
|
|
|
connectReq.Write(conn)
|
2011-02-23 13:20:50 -07:00
|
|
|
|
2011-03-23 11:38:18 -06:00
|
|
|
// Read response.
|
|
|
|
// Okay to use and discard buffered reader here, because
|
|
|
|
// TLS server will not speak until spoken to.
|
|
|
|
br := bufio.NewReader(conn)
|
2011-05-13 08:31:24 -06:00
|
|
|
resp, err := ReadResponse(br, connectReq)
|
2011-03-23 11:38:18 -06:00
|
|
|
if err != nil {
|
|
|
|
conn.Close()
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
if resp.StatusCode != 200 {
|
2011-06-27 17:43:14 -06:00
|
|
|
f := strings.SplitN(resp.Status, " ", 2)
|
2011-03-23 11:38:18 -06:00
|
|
|
conn.Close()
|
2011-11-01 20:04:37 -06:00
|
|
|
return nil, errors.New(f[1])
|
2011-03-23 11:38:18 -06:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if cm.targetScheme == "https" {
|
2011-02-23 13:20:50 -07:00
|
|
|
// Initiate TLS and check remote host name against certificate.
|
2012-08-22 10:15:41 -06:00
|
|
|
cfg := t.TLSClientConfig
|
|
|
|
if cfg == nil || cfg.ServerName == "" {
|
2012-09-25 10:22:13 -06:00
|
|
|
host := cm.tlsHost()
|
2012-08-22 10:15:41 -06:00
|
|
|
if cfg == nil {
|
|
|
|
cfg = &tls.Config{ServerName: host}
|
|
|
|
} else {
|
|
|
|
clone := *cfg // shallow clone
|
|
|
|
clone.ServerName = host
|
|
|
|
cfg = &clone
|
|
|
|
}
|
|
|
|
}
|
|
|
|
conn = tls.Client(conn, cfg)
|
2011-02-23 13:20:50 -07:00
|
|
|
if err = conn.(*tls.Conn).Handshake(); err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
2011-10-21 09:14:38 -06:00
|
|
|
if t.TLSClientConfig == nil || !t.TLSClientConfig.InsecureSkipVerify {
|
|
|
|
if err = conn.(*tls.Conn).VerifyHostname(cm.tlsHost()); err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
2011-02-23 13:20:50 -07:00
|
|
|
}
|
2011-03-23 11:38:18 -06:00
|
|
|
pconn.conn = conn
|
2011-02-23 13:20:50 -07:00
|
|
|
}
|
|
|
|
|
2011-03-23 11:38:18 -06:00
|
|
|
pconn.br = bufio.NewReader(pconn.conn)
|
2011-11-03 13:35:56 -06:00
|
|
|
pconn.bw = bufio.NewWriter(pconn.conn)
|
2011-03-23 11:38:18 -06:00
|
|
|
go pconn.readLoop()
|
2012-06-19 10:20:41 -06:00
|
|
|
go pconn.writeLoop()
|
2011-03-23 11:38:18 -06:00
|
|
|
return pconn, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// useProxy returns true if requests to addr should use a proxy,
|
|
|
|
// according to the NO_PROXY or no_proxy environment variable.
|
2011-04-20 14:53:34 -06:00
|
|
|
// addr is always a canonicalAddr with a host and port.
|
2011-05-18 10:23:29 -06:00
|
|
|
func useProxy(addr string) bool {
|
2011-03-23 11:38:18 -06:00
|
|
|
if len(addr) == 0 {
|
|
|
|
return true
|
|
|
|
}
|
2011-04-20 14:53:34 -06:00
|
|
|
host, _, err := net.SplitHostPort(addr)
|
|
|
|
if err != nil {
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
if host == "localhost" {
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
if ip := net.ParseIP(host); ip != nil {
|
2011-05-16 21:21:13 -06:00
|
|
|
if ip.IsLoopback() {
|
2011-04-20 14:53:34 -06:00
|
|
|
return false
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2011-05-18 10:23:29 -06:00
|
|
|
no_proxy := getenvEitherCase("NO_PROXY")
|
2011-03-23 11:38:18 -06:00
|
|
|
if no_proxy == "*" {
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
|
|
|
|
addr = strings.ToLower(strings.TrimSpace(addr))
|
|
|
|
if hasPort(addr) {
|
|
|
|
addr = addr[:strings.LastIndex(addr, ":")]
|
2011-02-23 13:20:50 -07:00
|
|
|
}
|
|
|
|
|
2011-06-27 17:43:14 -06:00
|
|
|
for _, p := range strings.Split(no_proxy, ",") {
|
2011-03-23 11:38:18 -06:00
|
|
|
p = strings.ToLower(strings.TrimSpace(p))
|
|
|
|
if len(p) == 0 {
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
if hasPort(p) {
|
|
|
|
p = p[:strings.LastIndex(p, ":")]
|
|
|
|
}
|
|
|
|
if addr == p || (p[0] == '.' && (strings.HasSuffix(addr, p) || addr == p[1:])) {
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
|
|
|
|
// connectMethod is the map key (in its String form) for keeping persistent
|
|
|
|
// TCP connections alive for subsequent HTTP requests.
|
|
|
|
//
|
|
|
|
// A connect method may be of the following types:
|
|
|
|
//
|
|
|
|
// Cache key form Description
|
|
|
|
// ----------------- -------------------------
|
|
|
|
// ||http|foo.com http directly to server, no proxy
|
|
|
|
// ||https|foo.com https directly to server, no proxy
|
|
|
|
// http://proxy.com|https|foo.com http to proxy, then CONNECT to foo.com
|
|
|
|
// http://proxy.com|http http to proxy, http to anywhere after that
|
|
|
|
//
|
|
|
|
// Note: no support to https to the proxy yet.
|
|
|
|
//
|
|
|
|
type connectMethod struct {
|
2011-08-16 21:36:02 -06:00
|
|
|
proxyURL *url.URL // nil for no proxy, else full proxy URL
|
|
|
|
targetScheme string // "http" or "https"
|
|
|
|
targetAddr string // Not used if proxy + http targetScheme (4th example in table)
|
2011-03-23 11:38:18 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
func (ck *connectMethod) String() string {
|
|
|
|
proxyStr := ""
|
2012-05-28 11:46:51 -06:00
|
|
|
targetAddr := ck.targetAddr
|
2011-03-23 11:38:18 -06:00
|
|
|
if ck.proxyURL != nil {
|
|
|
|
proxyStr = ck.proxyURL.String()
|
2012-05-28 11:46:51 -06:00
|
|
|
if ck.targetScheme == "http" {
|
|
|
|
targetAddr = ""
|
|
|
|
}
|
2011-03-23 11:38:18 -06:00
|
|
|
}
|
2012-05-28 11:46:51 -06:00
|
|
|
return strings.Join([]string{proxyStr, ck.targetScheme, targetAddr}, "|")
|
2011-03-23 11:38:18 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
// addr returns the first hop "host:port" to which we need to TCP connect.
|
|
|
|
func (cm *connectMethod) addr() string {
|
|
|
|
if cm.proxyURL != nil {
|
|
|
|
return canonicalAddr(cm.proxyURL)
|
|
|
|
}
|
|
|
|
return cm.targetAddr
|
|
|
|
}
|
|
|
|
|
|
|
|
// tlsHost returns the host name to match against the peer's
|
|
|
|
// TLS certificate.
|
|
|
|
func (cm *connectMethod) tlsHost() string {
|
|
|
|
h := cm.targetAddr
|
|
|
|
if hasPort(h) {
|
|
|
|
h = h[:strings.LastIndex(h, ":")]
|
|
|
|
}
|
|
|
|
return h
|
|
|
|
}
|
|
|
|
|
|
|
|
// persistConn wraps a connection, usually a persistent one
|
|
|
|
// (but may be used for non-keep-alive requests as well)
|
|
|
|
type persistConn struct {
|
2011-10-14 15:16:43 -06:00
|
|
|
t *Transport
|
|
|
|
cacheKey string // its connectMethod.String()
|
|
|
|
conn net.Conn
|
2012-05-23 12:19:38 -06:00
|
|
|
closed bool // whether conn has been closed
|
2011-11-03 13:35:56 -06:00
|
|
|
br *bufio.Reader // from conn
|
|
|
|
bw *bufio.Writer // to conn
|
2012-06-19 10:20:41 -06:00
|
|
|
reqch chan requestAndChan // written by roundTrip; read by readLoop
|
|
|
|
writech chan writeRequest // written by roundTrip; read by writeLoop
|
2012-07-11 17:40:44 -06:00
|
|
|
closech chan struct{} // broadcast close when readLoop (TCP connection) closes
|
2011-10-14 15:16:43 -06:00
|
|
|
isProxy bool
|
|
|
|
|
|
|
|
// mutateHeaderFunc is an optional func to modify extra
|
|
|
|
// headers on each outbound request before it's written. (the
|
|
|
|
// original Request given to RoundTrip is not modified)
|
|
|
|
mutateHeaderFunc func(Header)
|
2011-03-23 11:38:18 -06:00
|
|
|
|
|
|
|
lk sync.Mutex // guards numExpectedResponses and broken
|
|
|
|
numExpectedResponses int
|
|
|
|
broken bool // an error has happened on this connection; marked broken so it's not reused.
|
|
|
|
}
|
|
|
|
|
|
|
|
func (pc *persistConn) isBroken() bool {
|
|
|
|
pc.lk.Lock()
|
2012-08-20 03:28:27 -06:00
|
|
|
b := pc.broken
|
|
|
|
pc.lk.Unlock()
|
|
|
|
return b
|
2011-03-23 11:38:18 -06:00
|
|
|
}
|
|
|
|
|
2011-11-01 20:04:37 -06:00
|
|
|
var remoteSideClosedFunc func(error) bool // or nil to use default
|
2011-10-14 12:31:00 -06:00
|
|
|
|
2011-11-01 20:04:37 -06:00
|
|
|
func remoteSideClosed(err error) bool {
|
2011-11-13 20:42:42 -07:00
|
|
|
if err == io.EOF {
|
2011-10-14 12:31:00 -06:00
|
|
|
return true
|
|
|
|
}
|
|
|
|
if remoteSideClosedFunc != nil {
|
|
|
|
return remoteSideClosedFunc(err)
|
|
|
|
}
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
|
2011-03-23 11:38:18 -06:00
|
|
|
func (pc *persistConn) readLoop() {
|
2012-07-11 17:40:44 -06:00
|
|
|
defer close(pc.closech)
|
2011-03-23 11:38:18 -06:00
|
|
|
alive := true
|
2011-11-03 13:35:56 -06:00
|
|
|
var lastbody io.ReadCloser // last response body, if any, read on this connection
|
|
|
|
|
2011-03-23 11:38:18 -06:00
|
|
|
for alive {
|
|
|
|
pb, err := pc.br.Peek(1)
|
2012-01-25 16:00:39 -07:00
|
|
|
|
|
|
|
pc.lk.Lock()
|
|
|
|
if pc.numExpectedResponses == 0 {
|
|
|
|
pc.closeLocked()
|
|
|
|
pc.lk.Unlock()
|
2011-12-06 17:38:02 -07:00
|
|
|
if len(pb) > 0 {
|
|
|
|
log.Printf("Unsolicited response received on idle HTTP channel starting with %q; err=%v",
|
|
|
|
string(pb), err)
|
|
|
|
}
|
2011-03-23 11:38:18 -06:00
|
|
|
return
|
|
|
|
}
|
2012-01-25 16:00:39 -07:00
|
|
|
pc.lk.Unlock()
|
2011-03-23 11:38:18 -06:00
|
|
|
|
|
|
|
rc := <-pc.reqch
|
2011-11-03 13:35:56 -06:00
|
|
|
|
|
|
|
// Advance past the previous response's body, if the
|
|
|
|
// caller hasn't done so.
|
|
|
|
if lastbody != nil {
|
|
|
|
lastbody.Close() // assumed idempotent
|
|
|
|
lastbody = nil
|
|
|
|
}
|
2012-07-11 17:40:44 -06:00
|
|
|
|
|
|
|
var resp *Response
|
|
|
|
if err == nil {
|
|
|
|
resp, err = ReadResponse(pc.br, rc.req)
|
|
|
|
}
|
2011-11-03 13:35:56 -06:00
|
|
|
|
2012-01-31 10:45:13 -07:00
|
|
|
if err != nil {
|
|
|
|
pc.close()
|
|
|
|
} else {
|
2011-12-14 12:20:21 -07:00
|
|
|
hasBody := rc.req.Method != "HEAD" && resp.ContentLength != 0
|
|
|
|
if rc.addedGzip && hasBody && resp.Header.Get("Content-Encoding") == "gzip" {
|
2011-05-11 20:33:15 -06:00
|
|
|
resp.Header.Del("Content-Encoding")
|
|
|
|
resp.Header.Del("Content-Length")
|
|
|
|
resp.ContentLength = -1
|
2011-11-03 13:35:56 -06:00
|
|
|
gzReader, zerr := gzip.NewReader(resp.Body)
|
2011-12-14 12:20:21 -07:00
|
|
|
if zerr != nil {
|
2011-05-11 20:33:15 -06:00
|
|
|
pc.close()
|
2011-11-03 13:35:56 -06:00
|
|
|
err = zerr
|
|
|
|
} else {
|
|
|
|
resp.Body = &readFirstCloseBoth{&discardOnCloseReadCloser{gzReader}, resp.Body}
|
2011-05-11 20:33:15 -06:00
|
|
|
}
|
|
|
|
}
|
|
|
|
resp.Body = &bodyEOFSignal{body: resp.Body}
|
2011-11-03 13:35:56 -06:00
|
|
|
}
|
2011-04-04 20:22:47 -06:00
|
|
|
|
2011-11-03 13:35:56 -06:00
|
|
|
if err != nil || resp.Close || rc.req.Close {
|
2011-03-23 11:38:18 -06:00
|
|
|
alive = false
|
|
|
|
}
|
2011-04-04 20:22:47 -06:00
|
|
|
|
2012-11-13 23:38:25 -07:00
|
|
|
// TODO(bradfitz): this hasBody conflicts with the defition
|
|
|
|
// above which excludes HEAD requests. Is this one
|
|
|
|
// incomplete?
|
2011-04-04 17:58:11 -06:00
|
|
|
hasBody := resp != nil && resp.ContentLength != 0
|
2011-04-04 20:22:47 -06:00
|
|
|
var waitForBodyRead chan bool
|
2012-05-21 11:39:31 -06:00
|
|
|
if hasBody {
|
|
|
|
lastbody = resp.Body
|
2012-07-11 17:40:44 -06:00
|
|
|
waitForBodyRead = make(chan bool, 1)
|
net/http: fix Transport races & deadlocks
Thanks to Dustin Sallings for exposing the most frustrating
bug ever, and for providing repro cases (which formed the
basis of the new tests in this CL), and to Dave Cheney and
Dmitry Vyukov for help debugging and fixing.
This CL depends on submited pollster CLs ffd1e075c260 (Unix)
and 14b544194509 (Windows), as well as unsubmitted 6852085.
Some operating systems (OpenBSD, NetBSD, ?) may still require
more pollster work, fixing races (Issue 4434 and
http://goo.gl/JXB6W).
Tested on linux-amd64 and darwin-amd64, both with GOMAXPROCS 1
and 4 (all combinations of which previously failed differently)
Fixes #4191
Update #4434 (related fallout from this bug)
R=dave, bradfitz, dsallings, rsc, fullung
CC=golang-dev
https://golang.org/cl/6851061
2012-11-26 14:31:02 -07:00
|
|
|
resp.Body.(*bodyEOFSignal).fn = func(err error) {
|
|
|
|
alive1 := alive
|
|
|
|
if err != nil {
|
|
|
|
alive1 = false
|
2012-02-13 18:48:56 -07:00
|
|
|
}
|
net/http: fix Transport races & deadlocks
Thanks to Dustin Sallings for exposing the most frustrating
bug ever, and for providing repro cases (which formed the
basis of the new tests in this CL), and to Dave Cheney and
Dmitry Vyukov for help debugging and fixing.
This CL depends on submited pollster CLs ffd1e075c260 (Unix)
and 14b544194509 (Windows), as well as unsubmitted 6852085.
Some operating systems (OpenBSD, NetBSD, ?) may still require
more pollster work, fixing races (Issue 4434 and
http://goo.gl/JXB6W).
Tested on linux-amd64 and darwin-amd64, both with GOMAXPROCS 1
and 4 (all combinations of which previously failed differently)
Fixes #4191
Update #4434 (related fallout from this bug)
R=dave, bradfitz, dsallings, rsc, fullung
CC=golang-dev
https://golang.org/cl/6851061
2012-11-26 14:31:02 -07:00
|
|
|
if alive1 && !pc.t.putIdleConn(pc) {
|
|
|
|
alive1 = false
|
|
|
|
}
|
|
|
|
if !alive1 || pc.isBroken() {
|
2012-05-23 12:19:38 -06:00
|
|
|
pc.close()
|
|
|
|
}
|
net/http: fix Transport races & deadlocks
Thanks to Dustin Sallings for exposing the most frustrating
bug ever, and for providing repro cases (which formed the
basis of the new tests in this CL), and to Dave Cheney and
Dmitry Vyukov for help debugging and fixing.
This CL depends on submited pollster CLs ffd1e075c260 (Unix)
and 14b544194509 (Windows), as well as unsubmitted 6852085.
Some operating systems (OpenBSD, NetBSD, ?) may still require
more pollster work, fixing races (Issue 4434 and
http://goo.gl/JXB6W).
Tested on linux-amd64 and darwin-amd64, both with GOMAXPROCS 1
and 4 (all combinations of which previously failed differently)
Fixes #4191
Update #4434 (related fallout from this bug)
R=dave, bradfitz, dsallings, rsc, fullung
CC=golang-dev
https://golang.org/cl/6851061
2012-11-26 14:31:02 -07:00
|
|
|
waitForBodyRead <- alive1
|
2012-05-21 11:39:31 -06:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if alive && !hasBody {
|
|
|
|
// When there's no response body, we immediately
|
|
|
|
// reuse the TCP connection (putIdleConn), but
|
|
|
|
// we need to prevent ClientConn.Read from
|
|
|
|
// closing the Response.Body on the next
|
|
|
|
// loop, otherwise it might close the body
|
|
|
|
// before the client code has had a chance to
|
|
|
|
// read it (even though it'll just be 0, EOF).
|
|
|
|
lastbody = nil
|
|
|
|
|
|
|
|
if !pc.t.putIdleConn(pc) {
|
|
|
|
alive = false
|
2011-04-04 20:22:47 -06:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2011-03-23 11:38:18 -06:00
|
|
|
rc.ch <- responseAndError{resp, err}
|
|
|
|
|
|
|
|
// Wait for the just-returned response body to be fully consumed
|
|
|
|
// before we race and peek on the underlying bufio reader.
|
2011-04-04 20:22:47 -06:00
|
|
|
if waitForBodyRead != nil {
|
net/http: fix Transport races & deadlocks
Thanks to Dustin Sallings for exposing the most frustrating
bug ever, and for providing repro cases (which formed the
basis of the new tests in this CL), and to Dave Cheney and
Dmitry Vyukov for help debugging and fixing.
This CL depends on submited pollster CLs ffd1e075c260 (Unix)
and 14b544194509 (Windows), as well as unsubmitted 6852085.
Some operating systems (OpenBSD, NetBSD, ?) may still require
more pollster work, fixing races (Issue 4434 and
http://goo.gl/JXB6W).
Tested on linux-amd64 and darwin-amd64, both with GOMAXPROCS 1
and 4 (all combinations of which previously failed differently)
Fixes #4191
Update #4434 (related fallout from this bug)
R=dave, bradfitz, dsallings, rsc, fullung
CC=golang-dev
https://golang.org/cl/6851061
2012-11-26 14:31:02 -07:00
|
|
|
alive = <-waitForBodyRead
|
2012-05-21 11:39:31 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
if !alive {
|
2012-05-18 11:34:37 -06:00
|
|
|
pc.close()
|
2011-03-23 11:38:18 -06:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2012-06-19 10:20:41 -06:00
|
|
|
func (pc *persistConn) writeLoop() {
|
2012-08-20 19:18:16 -06:00
|
|
|
for {
|
|
|
|
select {
|
|
|
|
case wr := <-pc.writech:
|
|
|
|
if pc.isBroken() {
|
|
|
|
wr.ch <- errors.New("http: can't write HTTP request on broken connection")
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
err := wr.req.Request.write(pc.bw, pc.isProxy, wr.req.extra)
|
|
|
|
if err == nil {
|
|
|
|
err = pc.bw.Flush()
|
|
|
|
}
|
|
|
|
if err != nil {
|
|
|
|
pc.markBroken()
|
|
|
|
}
|
|
|
|
wr.ch <- err
|
|
|
|
case <-pc.closech:
|
|
|
|
return
|
2012-06-19 10:20:41 -06:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2011-03-23 11:38:18 -06:00
|
|
|
type responseAndError struct {
|
|
|
|
res *Response
|
2011-11-01 20:04:37 -06:00
|
|
|
err error
|
2011-03-23 11:38:18 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
type requestAndChan struct {
|
|
|
|
req *Request
|
|
|
|
ch chan responseAndError
|
2011-05-11 20:33:15 -06:00
|
|
|
|
|
|
|
// did the Transport (as opposed to the client code) add an
|
|
|
|
// Accept-Encoding gzip header? only if it we set it do
|
|
|
|
// we transparently decode the gzip.
|
|
|
|
addedGzip bool
|
2011-03-23 11:38:18 -06:00
|
|
|
}
|
|
|
|
|
2012-06-19 10:20:41 -06:00
|
|
|
// A writeRequest is sent by the readLoop's goroutine to the
|
|
|
|
// writeLoop's goroutine to write a request while the read loop
|
|
|
|
// concurrently waits on both the write response and the server's
|
|
|
|
// reply.
|
|
|
|
type writeRequest struct {
|
|
|
|
req *transportRequest
|
|
|
|
ch chan<- error
|
|
|
|
}
|
|
|
|
|
2011-11-01 20:04:37 -06:00
|
|
|
func (pc *persistConn) roundTrip(req *transportRequest) (resp *Response, err error) {
|
2011-10-14 15:16:43 -06:00
|
|
|
if pc.mutateHeaderFunc != nil {
|
|
|
|
pc.mutateHeaderFunc(req.extraHeaders())
|
2011-03-23 11:38:18 -06:00
|
|
|
}
|
|
|
|
|
2011-04-12 10:35:07 -06:00
|
|
|
// Ask for a compressed version if the caller didn't set their
|
|
|
|
// own value for Accept-Encoding. We only attempted to
|
|
|
|
// uncompress the gzip stream if we were the layer that
|
|
|
|
// requested it.
|
|
|
|
requestedGzip := false
|
|
|
|
if !pc.t.DisableCompression && req.Header.Get("Accept-Encoding") == "" {
|
2012-10-30 14:38:01 -06:00
|
|
|
// Request gzip only, not deflate. Deflate is ambiguous and
|
2011-10-14 15:16:43 -06:00
|
|
|
// not as universally supported anyway.
|
2011-04-12 10:35:07 -06:00
|
|
|
// See: http://www.gzip.org/zlib/zlib_faq.html#faq38
|
|
|
|
requestedGzip = true
|
2011-10-14 15:16:43 -06:00
|
|
|
req.extraHeaders().Set("Accept-Encoding", "gzip")
|
2011-04-12 10:35:07 -06:00
|
|
|
}
|
|
|
|
|
2011-03-23 11:38:18 -06:00
|
|
|
pc.lk.Lock()
|
|
|
|
pc.numExpectedResponses++
|
|
|
|
pc.lk.Unlock()
|
|
|
|
|
2012-06-19 10:20:41 -06:00
|
|
|
// Write the request concurrently with waiting for a response,
|
|
|
|
// in case the server decides to reply before reading our full
|
|
|
|
// request body.
|
|
|
|
writeErrCh := make(chan error, 1)
|
|
|
|
pc.writech <- writeRequest{req, writeErrCh}
|
|
|
|
|
|
|
|
resc := make(chan responseAndError, 1)
|
|
|
|
pc.reqch <- requestAndChan{req.Request, resc, requestedGzip}
|
|
|
|
|
|
|
|
var re responseAndError
|
2012-07-11 17:40:44 -06:00
|
|
|
var pconnDeadCh = pc.closech
|
|
|
|
var failTicker <-chan time.Time
|
2012-06-19 10:20:41 -06:00
|
|
|
WaitResponse:
|
|
|
|
for {
|
|
|
|
select {
|
|
|
|
case err := <-writeErrCh:
|
|
|
|
if err != nil {
|
|
|
|
re = responseAndError{nil, err}
|
|
|
|
break WaitResponse
|
|
|
|
}
|
2012-07-11 17:40:44 -06:00
|
|
|
case <-pconnDeadCh:
|
|
|
|
// The persist connection is dead. This shouldn't
|
|
|
|
// usually happen (only with Connection: close responses
|
|
|
|
// with no response bodies), but if it does happen it
|
|
|
|
// means either a) the remote server hung up on us
|
|
|
|
// prematurely, or b) the readLoop sent us a response &
|
|
|
|
// closed its closech at roughly the same time, and we
|
|
|
|
// selected this case first, in which case a response
|
|
|
|
// might still be coming soon.
|
|
|
|
//
|
|
|
|
// We can't avoid the select race in b) by using a unbuffered
|
|
|
|
// resc channel instead, because then goroutines can
|
|
|
|
// leak if we exit due to other errors.
|
|
|
|
pconnDeadCh = nil // avoid spinning
|
|
|
|
failTicker = time.After(100 * time.Millisecond) // arbitrary time to wait for resc
|
|
|
|
case <-failTicker:
|
|
|
|
re = responseAndError{nil, errors.New("net/http: transport closed before response was received")}
|
|
|
|
break WaitResponse
|
2012-06-19 10:20:41 -06:00
|
|
|
case re = <-resc:
|
|
|
|
break WaitResponse
|
|
|
|
}
|
2011-03-23 11:38:18 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
pc.lk.Lock()
|
|
|
|
pc.numExpectedResponses--
|
|
|
|
pc.lk.Unlock()
|
2011-04-12 10:35:07 -06:00
|
|
|
|
2011-03-23 11:38:18 -06:00
|
|
|
return re.res, re.err
|
|
|
|
}
|
|
|
|
|
2012-06-19 10:20:41 -06:00
|
|
|
// markBroken marks a connection as broken (so it's not reused).
|
|
|
|
// It differs from close in that it doesn't close the underlying
|
|
|
|
// connection for use when it's still being read.
|
|
|
|
func (pc *persistConn) markBroken() {
|
|
|
|
pc.lk.Lock()
|
|
|
|
defer pc.lk.Unlock()
|
|
|
|
pc.broken = true
|
|
|
|
}
|
|
|
|
|
2011-03-23 11:38:18 -06:00
|
|
|
func (pc *persistConn) close() {
|
|
|
|
pc.lk.Lock()
|
|
|
|
defer pc.lk.Unlock()
|
2012-01-25 16:00:39 -07:00
|
|
|
pc.closeLocked()
|
|
|
|
}
|
|
|
|
|
|
|
|
func (pc *persistConn) closeLocked() {
|
2011-03-23 11:38:18 -06:00
|
|
|
pc.broken = true
|
2012-05-23 12:19:38 -06:00
|
|
|
if !pc.closed {
|
|
|
|
pc.conn.Close()
|
|
|
|
pc.closed = true
|
|
|
|
}
|
2011-10-14 15:16:43 -06:00
|
|
|
pc.mutateHeaderFunc = nil
|
2011-03-23 11:38:18 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
var portMap = map[string]string{
|
|
|
|
"http": "80",
|
|
|
|
"https": "443",
|
|
|
|
}
|
|
|
|
|
|
|
|
// canonicalAddr returns url.Host but always with a ":port" suffix
|
2011-08-16 21:36:02 -06:00
|
|
|
func canonicalAddr(url *url.URL) string {
|
2011-03-23 11:38:18 -06:00
|
|
|
addr := url.Host
|
|
|
|
if !hasPort(addr) {
|
|
|
|
return addr + ":" + portMap[url.Scheme]
|
2011-02-23 13:20:50 -07:00
|
|
|
}
|
2011-03-23 11:38:18 -06:00
|
|
|
return addr
|
|
|
|
}
|
2011-02-23 13:20:50 -07:00
|
|
|
|
2011-04-04 20:22:47 -06:00
|
|
|
// bodyEOFSignal wraps a ReadCloser but runs fn (if non-nil) at most
|
net/http: fix Transport races & deadlocks
Thanks to Dustin Sallings for exposing the most frustrating
bug ever, and for providing repro cases (which formed the
basis of the new tests in this CL), and to Dave Cheney and
Dmitry Vyukov for help debugging and fixing.
This CL depends on submited pollster CLs ffd1e075c260 (Unix)
and 14b544194509 (Windows), as well as unsubmitted 6852085.
Some operating systems (OpenBSD, NetBSD, ?) may still require
more pollster work, fixing races (Issue 4434 and
http://goo.gl/JXB6W).
Tested on linux-amd64 and darwin-amd64, both with GOMAXPROCS 1
and 4 (all combinations of which previously failed differently)
Fixes #4191
Update #4434 (related fallout from this bug)
R=dave, bradfitz, dsallings, rsc, fullung
CC=golang-dev
https://golang.org/cl/6851061
2012-11-26 14:31:02 -07:00
|
|
|
// once, right before its final (error-producing) Read or Close call
|
|
|
|
// returns.
|
2011-03-23 11:38:18 -06:00
|
|
|
type bodyEOFSignal struct {
|
net/http: fix Transport races & deadlocks
Thanks to Dustin Sallings for exposing the most frustrating
bug ever, and for providing repro cases (which formed the
basis of the new tests in this CL), and to Dave Cheney and
Dmitry Vyukov for help debugging and fixing.
This CL depends on submited pollster CLs ffd1e075c260 (Unix)
and 14b544194509 (Windows), as well as unsubmitted 6852085.
Some operating systems (OpenBSD, NetBSD, ?) may still require
more pollster work, fixing races (Issue 4434 and
http://goo.gl/JXB6W).
Tested on linux-amd64 and darwin-amd64, both with GOMAXPROCS 1
and 4 (all combinations of which previously failed differently)
Fixes #4191
Update #4434 (related fallout from this bug)
R=dave, bradfitz, dsallings, rsc, fullung
CC=golang-dev
https://golang.org/cl/6851061
2012-11-26 14:31:02 -07:00
|
|
|
body io.ReadCloser
|
|
|
|
mu sync.Mutex // guards closed, rerr and fn
|
|
|
|
closed bool // whether Close has been called
|
|
|
|
rerr error // sticky Read error
|
|
|
|
fn func(error) // error will be nil on Read io.EOF
|
2011-03-23 11:38:18 -06:00
|
|
|
}
|
|
|
|
|
2011-11-01 20:04:37 -06:00
|
|
|
func (es *bodyEOFSignal) Read(p []byte) (n int, err error) {
|
net/http: fix Transport races & deadlocks
Thanks to Dustin Sallings for exposing the most frustrating
bug ever, and for providing repro cases (which formed the
basis of the new tests in this CL), and to Dave Cheney and
Dmitry Vyukov for help debugging and fixing.
This CL depends on submited pollster CLs ffd1e075c260 (Unix)
and 14b544194509 (Windows), as well as unsubmitted 6852085.
Some operating systems (OpenBSD, NetBSD, ?) may still require
more pollster work, fixing races (Issue 4434 and
http://goo.gl/JXB6W).
Tested on linux-amd64 and darwin-amd64, both with GOMAXPROCS 1
and 4 (all combinations of which previously failed differently)
Fixes #4191
Update #4434 (related fallout from this bug)
R=dave, bradfitz, dsallings, rsc, fullung
CC=golang-dev
https://golang.org/cl/6851061
2012-11-26 14:31:02 -07:00
|
|
|
es.mu.Lock()
|
|
|
|
closed, rerr := es.closed, es.rerr
|
|
|
|
es.mu.Unlock()
|
|
|
|
if closed {
|
|
|
|
return 0, errors.New("http: read on closed response body")
|
2011-04-26 13:32:59 -06:00
|
|
|
}
|
net/http: fix Transport races & deadlocks
Thanks to Dustin Sallings for exposing the most frustrating
bug ever, and for providing repro cases (which formed the
basis of the new tests in this CL), and to Dave Cheney and
Dmitry Vyukov for help debugging and fixing.
This CL depends on submited pollster CLs ffd1e075c260 (Unix)
and 14b544194509 (Windows), as well as unsubmitted 6852085.
Some operating systems (OpenBSD, NetBSD, ?) may still require
more pollster work, fixing races (Issue 4434 and
http://goo.gl/JXB6W).
Tested on linux-amd64 and darwin-amd64, both with GOMAXPROCS 1
and 4 (all combinations of which previously failed differently)
Fixes #4191
Update #4434 (related fallout from this bug)
R=dave, bradfitz, dsallings, rsc, fullung
CC=golang-dev
https://golang.org/cl/6851061
2012-11-26 14:31:02 -07:00
|
|
|
if rerr != nil {
|
|
|
|
return 0, rerr
|
2011-03-23 11:38:18 -06:00
|
|
|
}
|
|
|
|
|
net/http: fix Transport races & deadlocks
Thanks to Dustin Sallings for exposing the most frustrating
bug ever, and for providing repro cases (which formed the
basis of the new tests in this CL), and to Dave Cheney and
Dmitry Vyukov for help debugging and fixing.
This CL depends on submited pollster CLs ffd1e075c260 (Unix)
and 14b544194509 (Windows), as well as unsubmitted 6852085.
Some operating systems (OpenBSD, NetBSD, ?) may still require
more pollster work, fixing races (Issue 4434 and
http://goo.gl/JXB6W).
Tested on linux-amd64 and darwin-amd64, both with GOMAXPROCS 1
and 4 (all combinations of which previously failed differently)
Fixes #4191
Update #4434 (related fallout from this bug)
R=dave, bradfitz, dsallings, rsc, fullung
CC=golang-dev
https://golang.org/cl/6851061
2012-11-26 14:31:02 -07:00
|
|
|
n, err = es.body.Read(p)
|
|
|
|
if err != nil {
|
|
|
|
es.mu.Lock()
|
|
|
|
defer es.mu.Unlock()
|
|
|
|
if es.rerr == nil {
|
|
|
|
es.rerr = err
|
|
|
|
}
|
|
|
|
es.condfn(err)
|
2011-03-23 11:38:18 -06:00
|
|
|
}
|
2011-02-23 13:20:50 -07:00
|
|
|
return
|
|
|
|
}
|
2011-04-21 17:01:29 -06:00
|
|
|
|
net/http: fix Transport races & deadlocks
Thanks to Dustin Sallings for exposing the most frustrating
bug ever, and for providing repro cases (which formed the
basis of the new tests in this CL), and to Dave Cheney and
Dmitry Vyukov for help debugging and fixing.
This CL depends on submited pollster CLs ffd1e075c260 (Unix)
and 14b544194509 (Windows), as well as unsubmitted 6852085.
Some operating systems (OpenBSD, NetBSD, ?) may still require
more pollster work, fixing races (Issue 4434 and
http://goo.gl/JXB6W).
Tested on linux-amd64 and darwin-amd64, both with GOMAXPROCS 1
and 4 (all combinations of which previously failed differently)
Fixes #4191
Update #4434 (related fallout from this bug)
R=dave, bradfitz, dsallings, rsc, fullung
CC=golang-dev
https://golang.org/cl/6851061
2012-11-26 14:31:02 -07:00
|
|
|
func (es *bodyEOFSignal) Close() error {
|
|
|
|
es.mu.Lock()
|
|
|
|
defer es.mu.Unlock()
|
|
|
|
if es.closed {
|
|
|
|
return nil
|
2012-07-11 17:40:44 -06:00
|
|
|
}
|
net/http: fix Transport races & deadlocks
Thanks to Dustin Sallings for exposing the most frustrating
bug ever, and for providing repro cases (which formed the
basis of the new tests in this CL), and to Dave Cheney and
Dmitry Vyukov for help debugging and fixing.
This CL depends on submited pollster CLs ffd1e075c260 (Unix)
and 14b544194509 (Windows), as well as unsubmitted 6852085.
Some operating systems (OpenBSD, NetBSD, ?) may still require
more pollster work, fixing races (Issue 4434 and
http://goo.gl/JXB6W).
Tested on linux-amd64 and darwin-amd64, both with GOMAXPROCS 1
and 4 (all combinations of which previously failed differently)
Fixes #4191
Update #4434 (related fallout from this bug)
R=dave, bradfitz, dsallings, rsc, fullung
CC=golang-dev
https://golang.org/cl/6851061
2012-11-26 14:31:02 -07:00
|
|
|
es.closed = true
|
|
|
|
err := es.body.Close()
|
|
|
|
es.condfn(err)
|
|
|
|
return err
|
2012-07-11 17:40:44 -06:00
|
|
|
}
|
|
|
|
|
net/http: fix Transport races & deadlocks
Thanks to Dustin Sallings for exposing the most frustrating
bug ever, and for providing repro cases (which formed the
basis of the new tests in this CL), and to Dave Cheney and
Dmitry Vyukov for help debugging and fixing.
This CL depends on submited pollster CLs ffd1e075c260 (Unix)
and 14b544194509 (Windows), as well as unsubmitted 6852085.
Some operating systems (OpenBSD, NetBSD, ?) may still require
more pollster work, fixing races (Issue 4434 and
http://goo.gl/JXB6W).
Tested on linux-amd64 and darwin-amd64, both with GOMAXPROCS 1
and 4 (all combinations of which previously failed differently)
Fixes #4191
Update #4434 (related fallout from this bug)
R=dave, bradfitz, dsallings, rsc, fullung
CC=golang-dev
https://golang.org/cl/6851061
2012-11-26 14:31:02 -07:00
|
|
|
// caller must hold es.mu.
|
|
|
|
func (es *bodyEOFSignal) condfn(err error) {
|
|
|
|
if es.fn == nil {
|
|
|
|
return
|
|
|
|
}
|
|
|
|
if err == io.EOF {
|
|
|
|
err = nil
|
|
|
|
}
|
|
|
|
es.fn(err)
|
|
|
|
es.fn = nil
|
2012-10-11 15:32:56 -06:00
|
|
|
}
|
|
|
|
|
2011-04-21 17:01:29 -06:00
|
|
|
type readFirstCloseBoth struct {
|
|
|
|
io.ReadCloser
|
|
|
|
io.Closer
|
|
|
|
}
|
|
|
|
|
2011-11-01 20:04:37 -06:00
|
|
|
func (r *readFirstCloseBoth) Close() error {
|
2011-04-21 17:01:29 -06:00
|
|
|
if err := r.ReadCloser.Close(); err != nil {
|
2011-04-27 15:23:25 -06:00
|
|
|
r.Closer.Close()
|
2011-04-21 17:01:29 -06:00
|
|
|
return err
|
|
|
|
}
|
|
|
|
if err := r.Closer.Close(); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
2011-05-11 20:33:15 -06:00
|
|
|
|
|
|
|
// discardOnCloseReadCloser consumes all its input on Close.
|
|
|
|
type discardOnCloseReadCloser struct {
|
|
|
|
io.ReadCloser
|
|
|
|
}
|
|
|
|
|
2011-11-01 20:04:37 -06:00
|
|
|
func (d *discardOnCloseReadCloser) Close() error {
|
2011-05-11 20:33:15 -06:00
|
|
|
io.Copy(ioutil.Discard, d.ReadCloser) // ignore errors; likely invalid or already closed
|
|
|
|
return d.ReadCloser.Close()
|
|
|
|
}
|