mirror of
https://github.com/golang/go
synced 2024-11-20 03:44:40 -07:00
http: client gzip support
R=adg, rsc, bradfitzwork CC=golang-dev https://golang.org/cl/4389048
This commit is contained in:
Brad Fitzpatrick
parent
c34aadf063
commit
92210eefb2
@ -6,6 +6,7 @@ package http
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"compress/gzip"
|
||||
"crypto/tls"
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
@ -39,8 +40,9 @@ type Transport struct {
|
||||
// TODO: tunable on timeout on cached connections
|
||||
// TODO: optional pipelining
|
||||
|
||||
IgnoreEnvironment bool // don't look at environment variables for proxy configuration
|
||||
DisableKeepAlives bool
|
||||
IgnoreEnvironment bool // don't look at environment variables for proxy configuration
|
||||
DisableKeepAlives bool
|
||||
DisableCompression bool
|
||||
|
||||
// MaxIdleConnsPerHost, if non-zero, controls the maximum idle
|
||||
// (keep-alive) to keep to keep per-host. If zero,
|
||||
@ -474,6 +476,19 @@ func (pc *persistConn) roundTrip(req *Request) (resp *Response, err os.Error) {
|
||||
pc.mutateRequestFunc(req)
|
||||
}
|
||||
|
||||
// Ask for a compressed version if the caller didn't set their
|
||||
// own value for Accept-Encoding. We only attempted to
|
||||
// uncompress the gzip stream if we were the layer that
|
||||
// requested it.
|
||||
requestedGzip := false
|
||||
if !pc.t.DisableCompression && req.Header.Get("Accept-Encoding") == "" {
|
||||
// Request gzip only, not deflate. Deflate is ambiguous and
|
||||
// as universally supported anyway.
|
||||
// See: http://www.gzip.org/zlib/zlib_faq.html#faq38
|
||||
requestedGzip = true
|
||||
req.Header.Set("Accept-Encoding", "gzip")
|
||||
}
|
||||
|
||||
pc.lk.Lock()
|
||||
pc.numExpectedResponses++
|
||||
pc.lk.Unlock()
|
||||
@ -490,6 +505,19 @@ func (pc *persistConn) roundTrip(req *Request) (resp *Response, err os.Error) {
|
||||
pc.lk.Lock()
|
||||
pc.numExpectedResponses--
|
||||
pc.lk.Unlock()
|
||||
|
||||
if re.err == nil && requestedGzip && re.res.Header.Get("Content-Encoding") == "gzip" {
|
||||
re.res.Header.Del("Content-Encoding")
|
||||
re.res.Header.Del("Content-Length")
|
||||
re.res.ContentLength = -1
|
||||
var err os.Error
|
||||
re.res.Body, err = gzip.NewReader(re.res.Body)
|
||||
if err != nil {
|
||||
pc.close()
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
return re.res, re.err
|
||||
}
|
||||
|
||||
|
||||
@ -7,6 +7,8 @@
|
||||
package http_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"fmt"
|
||||
. "http"
|
||||
"http/httptest"
|
||||
@ -24,7 +26,7 @@ var hostPortHandler = HandlerFunc(func(w ResponseWriter, r *Request) {
|
||||
if r.FormValue("close") == "true" {
|
||||
w.Header().Set("Connection", "close")
|
||||
}
|
||||
fmt.Fprintf(w, "%s", r.RemoteAddr)
|
||||
w.Write([]byte(r.RemoteAddr))
|
||||
})
|
||||
|
||||
// Two subsequent requests and verify their response is the same.
|
||||
@ -179,7 +181,7 @@ func TestTransportIdleCacheKeys(t *testing.T) {
|
||||
func TestTransportMaxPerHostIdleConns(t *testing.T) {
|
||||
ch := make(chan string)
|
||||
ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) {
|
||||
fmt.Fprintf(w, "%s", <-ch)
|
||||
w.Write([]byte(<-ch))
|
||||
}))
|
||||
defer ts.Close()
|
||||
maxIdleConns := 2
|
||||
@ -338,6 +340,7 @@ func TestTransportNilURL(t *testing.T) {
|
||||
req.Proto = "HTTP/1.1"
|
||||
req.ProtoMajor = 1
|
||||
req.ProtoMinor = 1
|
||||
req.Header = make(Header)
|
||||
|
||||
tr := &Transport{}
|
||||
res, err := tr.RoundTrip(req)
|
||||
@ -349,3 +352,99 @@ func TestTransportNilURL(t *testing.T) {
|
||||
t.Fatalf("Expected response body of %q; got %q", e, g)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTransportGzip(t *testing.T) {
|
||||
const testString = "The test string aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
|
||||
ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) {
|
||||
if g, e := r.Header.Get("Accept-Encoding"), "gzip"; g != e {
|
||||
t.Errorf("Accept-Encoding = %q, want %q", g, e)
|
||||
}
|
||||
w.Header().Set("Content-Encoding", "gzip")
|
||||
gz, _ := gzip.NewWriter(w)
|
||||
defer gz.Close()
|
||||
gz.Write([]byte(testString))
|
||||
|
||||
}))
|
||||
defer ts.Close()
|
||||
|
||||
c := &Client{Transport: &Transport{}}
|
||||
res, _, err := c.Get(ts.URL)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body, err := ioutil.ReadAll(res.Body)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if g, e := string(body), testString; g != e {
|
||||
t.Fatalf("body = %q; want %q", g, e)
|
||||
}
|
||||
if g, e := res.Header.Get("Content-Encoding"), ""; g != e {
|
||||
t.Fatalf("Content-Encoding = %q; want %q", g, e)
|
||||
}
|
||||
}
|
||||
|
||||
// TestTransportGzipRecursive sends a gzip quine and checks that the
|
||||
// client gets the same value back. This is more cute than anything,
|
||||
// but checks that we don't recurse forever, and checks that
|
||||
// Content-Encoding is removed.
|
||||
func TestTransportGzipRecursive(t *testing.T) {
|
||||
ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) {
|
||||
w.Header().Set("Content-Encoding", "gzip")
|
||||
w.Write(rgz)
|
||||
}))
|
||||
defer ts.Close()
|
||||
|
||||
c := &Client{Transport: &Transport{}}
|
||||
res, _, err := c.Get(ts.URL)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body, err := ioutil.ReadAll(res.Body)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !bytes.Equal(body, rgz) {
|
||||
t.Fatalf("Incorrect result from recursive gz:\nhave=%x\nwant=%x",
|
||||
body, rgz)
|
||||
}
|
||||
if g, e := res.Header.Get("Content-Encoding"), ""; g != e {
|
||||
t.Fatalf("Content-Encoding = %q; want %q", g, e)
|
||||
}
|
||||
}
|
||||
|
||||
// rgz is a gzip quine that uncompresses to itself.
|
||||
var rgz = []byte{
|
||||
0x1f, 0x8b, 0x08, 0x08, 0x00, 0x00, 0x00, 0x00,
|
||||
0x00, 0x00, 0x72, 0x65, 0x63, 0x75, 0x72, 0x73,
|
||||
0x69, 0x76, 0x65, 0x00, 0x92, 0xef, 0xe6, 0xe0,
|
||||
0x60, 0x00, 0x83, 0xa2, 0xd4, 0xe4, 0xd2, 0xa2,
|
||||
0xe2, 0xcc, 0xb2, 0x54, 0x06, 0x00, 0x00, 0x17,
|
||||
0x00, 0xe8, 0xff, 0x92, 0xef, 0xe6, 0xe0, 0x60,
|
||||
0x00, 0x83, 0xa2, 0xd4, 0xe4, 0xd2, 0xa2, 0xe2,
|
||||
0xcc, 0xb2, 0x54, 0x06, 0x00, 0x00, 0x17, 0x00,
|
||||
0xe8, 0xff, 0x42, 0x12, 0x46, 0x16, 0x06, 0x00,
|
||||
0x05, 0x00, 0xfa, 0xff, 0x42, 0x12, 0x46, 0x16,
|
||||
0x06, 0x00, 0x05, 0x00, 0xfa, 0xff, 0x00, 0x05,
|
||||
0x00, 0xfa, 0xff, 0x00, 0x14, 0x00, 0xeb, 0xff,
|
||||
0x42, 0x12, 0x46, 0x16, 0x06, 0x00, 0x05, 0x00,
|
||||
0xfa, 0xff, 0x00, 0x05, 0x00, 0xfa, 0xff, 0x00,
|
||||
0x14, 0x00, 0xeb, 0xff, 0x42, 0x88, 0x21, 0xc4,
|
||||
0x00, 0x00, 0x14, 0x00, 0xeb, 0xff, 0x42, 0x88,
|
||||
0x21, 0xc4, 0x00, 0x00, 0x14, 0x00, 0xeb, 0xff,
|
||||
0x42, 0x88, 0x21, 0xc4, 0x00, 0x00, 0x14, 0x00,
|
||||
0xeb, 0xff, 0x42, 0x88, 0x21, 0xc4, 0x00, 0x00,
|
||||
0x14, 0x00, 0xeb, 0xff, 0x42, 0x88, 0x21, 0xc4,
|
||||
0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0x00, 0x00,
|
||||
0x00, 0xff, 0xff, 0x00, 0x17, 0x00, 0xe8, 0xff,
|
||||
0x42, 0x88, 0x21, 0xc4, 0x00, 0x00, 0x00, 0x00,
|
||||
0xff, 0xff, 0x00, 0x00, 0x00, 0xff, 0xff, 0x00,
|
||||
0x17, 0x00, 0xe8, 0xff, 0x42, 0x12, 0x46, 0x16,
|
||||
0x06, 0x00, 0x00, 0x00, 0xff, 0xff, 0x01, 0x08,
|
||||
0x00, 0xf7, 0xff, 0x3d, 0xb1, 0x20, 0x85, 0xfa,
|
||||
0x00, 0x00, 0x00, 0x42, 0x12, 0x46, 0x16, 0x06,
|
||||
0x00, 0x00, 0x00, 0xff, 0xff, 0x01, 0x08, 0x00,
|
||||
0xf7, 0xff, 0x3d, 0xb1, 0x20, 0x85, 0xfa, 0x00,
|
||||
0x00, 0x00, 0x3d, 0xb1, 0x20, 0x85, 0xfa, 0x00,
|
||||
0x00, 0x00,
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user