From e0a2c5d4b540934e06867710fe7137661a2a39ec Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Wed, 23 Feb 2011 12:20:50 -0800 Subject: [PATCH] http: introduce start of Client and ClientTransport Much yet to come, but this is a safe first step, introducing an in-the-future configurable Client object (where policy for cookies, auth, redirects will live) as well as introducing a ClientTransport interface for sending requests. The CL intentionally ignores everything around the creation and configuration of Clients and merely ports/wraps the old interfaces to/around Client/ClientTransport. R=rsc, dsymonds, nigeltao, bradfitzwork CC=golang-dev https://golang.org/cl/4182086 --- src/pkg/http/Makefile | 1 + src/pkg/http/client.go | 206 ++++++++++++++++---------------------- src/pkg/http/fs_test.go | 2 +- src/pkg/http/transport.go | 150 +++++++++++++++++++++++++++ 4 files changed, 240 insertions(+), 119 deletions(-) create mode 100644 src/pkg/http/transport.go diff --git a/src/pkg/http/Makefile b/src/pkg/http/Makefile index 796c98f64c7..1167d8ef6b7 100644 --- a/src/pkg/http/Makefile +++ b/src/pkg/http/Makefile @@ -18,6 +18,7 @@ GOFILES=\ server.go\ status.go\ transfer.go\ + transport.go\ url.go\ include ../../Make.pkg diff --git a/src/pkg/http/client.go b/src/pkg/http/client.go index aacebab355d..116b926433b 100644 --- a/src/pkg/http/client.go +++ b/src/pkg/http/client.go @@ -7,18 +7,37 @@ package http import ( - "bufio" "bytes" - "crypto/tls" "encoding/base64" "fmt" "io" - "net" "os" "strconv" "strings" ) +// A Client is an HTTP client. +// It is not yet possible to create custom Clients; use DefaultClient. +type Client struct { + transport ClientTransport // if nil, DefaultTransport is used +} + +// DefaultClient is the default Client and is used by Get, Head, and Post. +var DefaultClient = &Client{} + +// ClientTransport is an interface representing the ability to execute a +// single HTTP transaction, obtaining the Response for a given Request. +type ClientTransport interface { + // Do executes a single HTTP transaction, returning the Response for the + // request req. Do should not attempt to interpret the response. + // In particular, Do must return err == nil if it obtained a response, + // regardless of the response's HTTP status code. A non-nil err should + // be reserved for failure to obtain a response. Similarly, Do should + // not attempt to handle higher-level protocol details such as redirects, + // authentication, or cookies. + Do(req *Request) (resp *Response, err os.Error) +} + // Given a string of the form "host", "host:port", or "[ipv6::address]:port", // return true if the string includes a port. func hasPort(s string) bool { return strings.LastIndex(s, ":") > strings.LastIndex(s, "]") } @@ -65,19 +84,25 @@ func matchNoProxy(addr string) bool { return false } -// Send issues an HTTP request. Caller should close resp.Body when done reading it. +// Do sends an HTTP request and returns an HTTP response, following +// policy (e.g. redirects, cookies, auth) as configured on the client. +// +// Callers should close resp.Body when done reading it. +// +// Generally Get, Post, or PostForm will be used instead of Do. +func (c *Client) Do(req *Request) (resp *Response, err os.Error) { + return send(req, c.transport) +} + + +// send issues an HTTP request. Caller should close resp.Body when done reading it. // // TODO: support persistent connections (multiple requests on a single connection). // send() method is nonpublic because, when we refactor the code for persistent // connections, it may no longer make sense to have a method with this signature. -func send(req *Request) (resp *Response, err os.Error) { - if req.URL.Scheme != "http" && req.URL.Scheme != "https" { - return nil, &badStringError{"unsupported protocol scheme", req.URL.Scheme} - } - - addr := req.URL.Host - if !hasPort(addr) { - addr += ":" + req.URL.Scheme +func send(req *Request, t ClientTransport) (resp *Response, err os.Error) { + if t == nil { + t = DefaultTransport } info := req.URL.RawUserinfo if len(info) > 0 { @@ -89,108 +114,7 @@ func send(req *Request) (resp *Response, err os.Error) { } req.Header.Set("Authorization", "Basic "+string(encoded)) } - - var proxyURL *URL - proxyAuth := "" - proxy := "" - if !matchNoProxy(addr) { - proxy = os.Getenv("HTTP_PROXY") - if proxy == "" { - proxy = os.Getenv("http_proxy") - } - } - - if proxy != "" { - proxyURL, err = ParseRequestURL(proxy) - if err != nil { - return nil, os.ErrorString("invalid proxy address") - } - if proxyURL.Host == "" { - proxyURL, err = ParseRequestURL("http://" + proxy) - if err != nil { - return nil, os.ErrorString("invalid proxy address") - } - } - addr = proxyURL.Host - proxyInfo := proxyURL.RawUserinfo - if proxyInfo != "" { - enc := base64.URLEncoding - encoded := make([]byte, enc.EncodedLen(len(proxyInfo))) - enc.Encode(encoded, []byte(proxyInfo)) - proxyAuth = "Basic " + string(encoded) - } - } - - // Connect to server or proxy. - conn, err := net.Dial("tcp", "", addr) - if err != nil { - return nil, err - } - - if req.URL.Scheme == "http" { - // Include proxy http header if needed. - if proxyAuth != "" { - req.Header.Set("Proxy-Authorization", proxyAuth) - } - } else { // https - if proxyURL != nil { - // Ask proxy for direct connection to server. - // addr defaults above to ":https" but we need to use numbers - addr = req.URL.Host - if !hasPort(addr) { - addr += ":443" - } - fmt.Fprintf(conn, "CONNECT %s HTTP/1.1\r\n", addr) - fmt.Fprintf(conn, "Host: %s\r\n", addr) - if proxyAuth != "" { - fmt.Fprintf(conn, "Proxy-Authorization: %s\r\n", proxyAuth) - } - fmt.Fprintf(conn, "\r\n") - - // Read response. - // Okay to use and discard buffered reader here, because - // TLS server will not speak until spoken to. - br := bufio.NewReader(conn) - resp, err := ReadResponse(br, "CONNECT") - if err != nil { - return nil, err - } - if resp.StatusCode != 200 { - f := strings.Split(resp.Status, " ", 2) - return nil, os.ErrorString(f[1]) - } - } - - // Initiate TLS and check remote host name against certificate. - conn = tls.Client(conn, nil) - if err = conn.(*tls.Conn).Handshake(); err != nil { - return nil, err - } - h := req.URL.Host - if hasPort(h) { - h = h[:strings.LastIndex(h, ":")] - } - if err = conn.(*tls.Conn).VerifyHostname(h); err != nil { - return nil, err - } - } - - err = req.Write(conn) - if err != nil { - conn.Close() - return nil, err - } - - reader := bufio.NewReader(conn) - resp, err = ReadResponse(reader, req.Method) - if err != nil { - conn.Close() - return nil, err - } - - resp.Body = readClose{resp.Body, conn} - - return + return t.Do(req) } // True if the specified HTTP status code is one for which the Get utility should @@ -215,11 +139,31 @@ func shouldRedirect(statusCode int) bool { // input URL unless redirects were followed. // // Caller should close r.Body when done reading it. +// +// Get is a convenience wrapper around DefaultClient.Get. func Get(url string) (r *Response, finalURL string, err os.Error) { + return DefaultClient.Get(url) +} + +// Get issues a GET to the specified URL. If the response is one of the following +// redirect codes, it follows the redirect, up to a maximum of 10 redirects: +// +// 301 (Moved Permanently) +// 302 (Found) +// 303 (See Other) +// 307 (Temporary Redirect) +// +// finalURL is the URL from which the response was fetched -- identical to the +// input URL unless redirects were followed. +// +// Caller should close r.Body when done reading it. +func (c *Client) Get(url string) (r *Response, finalURL string, err os.Error) { // TODO: if/when we add cookie support, the redirected request shouldn't // necessarily supply the same cookies as the original. // TODO: set referrer header on redirects. var base *URL + // TODO: remove this hard-coded 10 and use the Client's policy + // (ClientConfig) instead. for redirect := 0; ; redirect++ { if redirect >= 10 { err = os.ErrorString("stopped after 10 redirects") @@ -236,7 +180,7 @@ func Get(url string) (r *Response, finalURL string, err os.Error) { break } url = req.URL.String() - if r, err = send(&req); err != nil { + if r, err = send(&req, c.transport); err != nil { break } if shouldRedirect(r.StatusCode) { @@ -259,7 +203,16 @@ func Get(url string) (r *Response, finalURL string, err os.Error) { // Post issues a POST to the specified URL. // // Caller should close r.Body when done reading it. +// +// Post is a wrapper around DefaultClient.Post func Post(url string, bodyType string, body io.Reader) (r *Response, err os.Error) { + return DefaultClient.Post(url, bodyType, body) +} + +// Post issues a POST to the specified URL. +// +// Caller should close r.Body when done reading it. +func (c *Client) Post(url string, bodyType string, body io.Reader) (r *Response, err os.Error) { var req Request req.Method = "POST" req.ProtoMajor = 1 @@ -276,14 +229,24 @@ func Post(url string, bodyType string, body io.Reader) (r *Response, err os.Erro return nil, err } - return send(&req) + return send(&req, c.transport) } // PostForm issues a POST to the specified URL, // with data's keys and values urlencoded as the request body. // // Caller should close r.Body when done reading it. +// +// PostForm is a wrapper around DefaultClient.PostForm func PostForm(url string, data map[string]string) (r *Response, err os.Error) { + return DefaultClient.PostForm(url, data) +} + +// PostForm issues a POST to the specified URL, +// with data's keys and values urlencoded as the request body. +// +// Caller should close r.Body when done reading it. +func (c *Client) PostForm(url string, data map[string]string) (r *Response, err os.Error) { var req Request req.Method = "POST" req.ProtoMajor = 1 @@ -302,7 +265,7 @@ func PostForm(url string, data map[string]string) (r *Response, err os.Error) { return nil, err } - return send(&req) + return send(&req, c.transport) } // TODO: remove this function when PostForm takes a multimap. @@ -315,13 +278,20 @@ func urlencode(data map[string]string) (b *bytes.Buffer) { } // Head issues a HEAD to the specified URL. +// +// Head is a wrapper around DefaultClient.Head func Head(url string) (r *Response, err os.Error) { + return DefaultClient.Head(url) +} + +// Head issues a HEAD to the specified URL. +func (c *Client) Head(url string) (r *Response, err os.Error) { var req Request req.Method = "HEAD" if req.URL, err = ParseURL(url); err != nil { return } - return send(&req) + return send(&req, c.transport) } type nopCloser struct { diff --git a/src/pkg/http/fs_test.go b/src/pkg/http/fs_test.go index b66136b1a1c..a8b67e3f08c 100644 --- a/src/pkg/http/fs_test.go +++ b/src/pkg/http/fs_test.go @@ -149,7 +149,7 @@ func TestServeFile(t *testing.T) { } func getBody(t *testing.T, req Request) (*Response, []byte) { - r, err := send(&req) + r, err := send(&req, DefaultTransport) if err != nil { t.Fatal(req.URL.String(), "send:", err) } diff --git a/src/pkg/http/transport.go b/src/pkg/http/transport.go new file mode 100644 index 00000000000..7f61962c2f1 --- /dev/null +++ b/src/pkg/http/transport.go @@ -0,0 +1,150 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package http + +import ( + "bufio" + "crypto/tls" + "encoding/base64" + "fmt" + "log" + "net" + "os" + "strings" + "sync" +) + +// DefaultTransport is the default implementation of ClientTransport +// and is used by DefaultClient. It establishes a new network connection for +// each call to Do and uses HTTP proxies as directed by the $HTTP_PROXY and +// $NO_PROXY (or $http_proxy and $no_proxy) environment variables. +var DefaultTransport ClientTransport = &transport{} + +// transport implements http.ClientTranport for the default case, +// using TCP connections to either the host or a proxy, serving +// http or https schemes. In the future this may become public +// and support options on keep-alive connection duration, pipelining +// controls, etc. For now this is simply a port of the old Go code +// client code to the http.ClientTransport interface. +type transport struct { + // TODO: keep-alives, pipelining, etc using a map from + // scheme/host to a connection. Something like: + l sync.Mutex + hostConn map[string]*ClientConn +} + +func (ct *transport) Do(req *Request) (resp *Response, err os.Error) { + if req.URL.Scheme != "http" && req.URL.Scheme != "https" { + return nil, &badStringError{"unsupported protocol scheme", req.URL.Scheme} + } + + addr := req.URL.Host + if !hasPort(addr) { + addr += ":" + req.URL.Scheme + } + + var proxyURL *URL + proxyAuth := "" + proxy := "" + if !matchNoProxy(addr) { + proxy = os.Getenv("HTTP_PROXY") + if proxy == "" { + proxy = os.Getenv("http_proxy") + } + } + + if proxy != "" { + proxyURL, err = ParseRequestURL(proxy) + if err != nil { + return nil, os.ErrorString("invalid proxy address") + } + if proxyURL.Host == "" { + proxyURL, err = ParseRequestURL("http://" + proxy) + if err != nil { + return nil, os.ErrorString("invalid proxy address") + } + } + addr = proxyURL.Host + proxyInfo := proxyURL.RawUserinfo + if proxyInfo != "" { + enc := base64.URLEncoding + encoded := make([]byte, enc.EncodedLen(len(proxyInfo))) + enc.Encode(encoded, []byte(proxyInfo)) + proxyAuth = "Basic " + string(encoded) + } + } + + // Connect to server or proxy + log.Printf("Temporary necessary log statement to work around http://code.google.com/p/go/issues/detail?id=1547") + conn, err := net.Dial("tcp", "", addr) + log.Printf("Temporary necessary log statement to work around http://code.google.com/p/go/issues/detail?id=1547") + if err != nil { + return nil, err + } + + if req.URL.Scheme == "http" { + // Include proxy http header if needed. + if proxyAuth != "" { + req.Header.Set("Proxy-Authorization", proxyAuth) + } + } else { // https + if proxyURL != nil { + // Ask proxy for direct connection to server. + // addr defaults above to ":https" but we need to use numbers + addr = req.URL.Host + if !hasPort(addr) { + addr += ":443" + } + fmt.Fprintf(conn, "CONNECT %s HTTP/1.1\r\n", addr) + fmt.Fprintf(conn, "Host: %s\r\n", addr) + if proxyAuth != "" { + fmt.Fprintf(conn, "Proxy-Authorization: %s\r\n", proxyAuth) + } + fmt.Fprintf(conn, "\r\n") + + // Read response. + // Okay to use and discard buffered reader here, because + // TLS server will not speak until spoken to. + br := bufio.NewReader(conn) + resp, err := ReadResponse(br, "CONNECT") + if err != nil { + return nil, err + } + if resp.StatusCode != 200 { + f := strings.Split(resp.Status, " ", 2) + return nil, os.ErrorString(f[1]) + } + } + + // Initiate TLS and check remote host name against certificate. + conn = tls.Client(conn, nil) + if err = conn.(*tls.Conn).Handshake(); err != nil { + return nil, err + } + h := req.URL.Host + if hasPort(h) { + h = h[:strings.LastIndex(h, ":")] + } + if err = conn.(*tls.Conn).VerifyHostname(h); err != nil { + return nil, err + } + } + + err = req.Write(conn) + if err != nil { + conn.Close() + return nil, err + } + + reader := bufio.NewReader(conn) + resp, err = ReadResponse(reader, req.Method) + if err != nil { + conn.Close() + return nil, err + } + + resp.Body = readClose{resp.Body, conn} + return +}