http: client gzip support

R=adg, rsc, bradfitzwork
CC=golang-dev
https://golang.org/cl/4389048

src/pkg/http/transport.go

@@ -6,6 +6,7 @@ package http
 
 import (
 	"bufio"
+	"compress/gzip"
 	"crypto/tls"
 	"encoding/base64"
 	"fmt"
@@ -39,8 +40,9 @@ type Transport struct {
 	// TODO: tunable on timeout on cached connections
 	// TODO: optional pipelining
 
-	IgnoreEnvironment bool // don't look at environment variables for proxy configuration
+	IgnoreEnvironment  bool // don't look at environment variables for proxy configuration
-	DisableKeepAlives bool
+	DisableKeepAlives  bool
+	DisableCompression bool
 
 	// MaxIdleConnsPerHost, if non-zero, controls the maximum idle
 	// (keep-alive) to keep to keep per-host.  If zero,
@@ -474,6 +476,19 @@ func (pc *persistConn) roundTrip(req *Request) (resp *Response, err os.Error) {
 		pc.mutateRequestFunc(req)
 	}
 
+	// Ask for a compressed version if the caller didn't set their
+	// own value for Accept-Encoding. We only attempted to
+	// uncompress the gzip stream if we were the layer that
+	// requested it.
+	requestedGzip := false
+	if !pc.t.DisableCompression && req.Header.Get("Accept-Encoding") == "" {
+		// Request gzip only, not deflate. Deflate is ambiguous and 
+		// as universally supported anyway.
+		// See: http://www.gzip.org/zlib/zlib_faq.html#faq38
+		requestedGzip = true
+		req.Header.Set("Accept-Encoding", "gzip")
+	}
+
 	pc.lk.Lock()
 	pc.numExpectedResponses++
 	pc.lk.Unlock()
@@ -490,6 +505,19 @@ func (pc *persistConn) roundTrip(req *Request) (resp *Response, err os.Error) {
 	pc.lk.Lock()
 	pc.numExpectedResponses--
 	pc.lk.Unlock()
+
+	if re.err == nil && requestedGzip && re.res.Header.Get("Content-Encoding") == "gzip" {
+		re.res.Header.Del("Content-Encoding")
+		re.res.Header.Del("Content-Length")
+		re.res.ContentLength = -1
+		var err os.Error
+		re.res.Body, err = gzip.NewReader(re.res.Body)
+		if err != nil {
+			pc.close()
+			return nil, err
+		}
+	}
+
 	return re.res, re.err
 }
 

src/pkg/http/transport_test.go

@@ -7,6 +7,8 @@
 package http_test
 
 import (
+	"bytes"
+	"compress/gzip"
 	"fmt"
 	. "http"
 	"http/httptest"
@@ -24,7 +26,7 @@ var hostPortHandler = HandlerFunc(func(w ResponseWriter, r *Request) {
 	if r.FormValue("close") == "true" {
 		w.Header().Set("Connection", "close")
 	}
-	fmt.Fprintf(w, "%s", r.RemoteAddr)
+	w.Write([]byte(r.RemoteAddr))
 })
 
 // Two subsequent requests and verify their response is the same.
@@ -179,7 +181,7 @@ func TestTransportIdleCacheKeys(t *testing.T) {
 func TestTransportMaxPerHostIdleConns(t *testing.T) {
 	ch := make(chan string)
 	ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) {
-		fmt.Fprintf(w, "%s", <-ch)
+		w.Write([]byte(<-ch))
 	}))
 	defer ts.Close()
 	maxIdleConns := 2
@@ -338,6 +340,7 @@ func TestTransportNilURL(t *testing.T) {
 	req.Proto = "HTTP/1.1"
 	req.ProtoMajor = 1
 	req.ProtoMinor = 1
+	req.Header = make(Header)
 
 	tr := &Transport{}
 	res, err := tr.RoundTrip(req)
@@ -349,3 +352,99 @@ func TestTransportNilURL(t *testing.T) {
 		t.Fatalf("Expected response body of %q; got %q", e, g)
 	}
 }
+
+func TestTransportGzip(t *testing.T) {
+	const testString = "The test string aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
+	ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) {
+		if g, e := r.Header.Get("Accept-Encoding"), "gzip"; g != e {
+			t.Errorf("Accept-Encoding = %q, want %q", g, e)
+		}
+		w.Header().Set("Content-Encoding", "gzip")
+		gz, _ := gzip.NewWriter(w)
+		defer gz.Close()
+		gz.Write([]byte(testString))
+
+	}))
+	defer ts.Close()
+
+	c := &Client{Transport: &Transport{}}
+	res, _, err := c.Get(ts.URL)
+	if err != nil {
+		t.Fatal(err)
+	}
+	body, err := ioutil.ReadAll(res.Body)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if g, e := string(body), testString; g != e {
+		t.Fatalf("body = %q; want %q", g, e)
+	}
+	if g, e := res.Header.Get("Content-Encoding"), ""; g != e {
+		t.Fatalf("Content-Encoding = %q; want %q", g, e)
+	}
+}
+
+// TestTransportGzipRecursive sends a gzip quine and checks that the
+// client gets the same value back. This is more cute than anything,
+// but checks that we don't recurse forever, and checks that
+// Content-Encoding is removed.
+func TestTransportGzipRecursive(t *testing.T) {
+	ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) {
+		w.Header().Set("Content-Encoding", "gzip")
+		w.Write(rgz)
+	}))
+	defer ts.Close()
+
+	c := &Client{Transport: &Transport{}}
+	res, _, err := c.Get(ts.URL)
+	if err != nil {
+		t.Fatal(err)
+	}
+	body, err := ioutil.ReadAll(res.Body)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !bytes.Equal(body, rgz) {
+		t.Fatalf("Incorrect result from recursive gz:\nhave=%x\nwant=%x",
+			body, rgz)
+	}
+	if g, e := res.Header.Get("Content-Encoding"), ""; g != e {
+		t.Fatalf("Content-Encoding = %q; want %q", g, e)
+	}
+}
+
+// rgz is a gzip quine that uncompresses to itself.
+var rgz = []byte{
+	0x1f, 0x8b, 0x08, 0x08, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x72, 0x65, 0x63, 0x75, 0x72, 0x73,
+	0x69, 0x76, 0x65, 0x00, 0x92, 0xef, 0xe6, 0xe0,
+	0x60, 0x00, 0x83, 0xa2, 0xd4, 0xe4, 0xd2, 0xa2,
+	0xe2, 0xcc, 0xb2, 0x54, 0x06, 0x00, 0x00, 0x17,
+	0x00, 0xe8, 0xff, 0x92, 0xef, 0xe6, 0xe0, 0x60,
+	0x00, 0x83, 0xa2, 0xd4, 0xe4, 0xd2, 0xa2, 0xe2,
+	0xcc, 0xb2, 0x54, 0x06, 0x00, 0x00, 0x17, 0x00,
+	0xe8, 0xff, 0x42, 0x12, 0x46, 0x16, 0x06, 0x00,
+	0x05, 0x00, 0xfa, 0xff, 0x42, 0x12, 0x46, 0x16,
+	0x06, 0x00, 0x05, 0x00, 0xfa, 0xff, 0x00, 0x05,
+	0x00, 0xfa, 0xff, 0x00, 0x14, 0x00, 0xeb, 0xff,
+	0x42, 0x12, 0x46, 0x16, 0x06, 0x00, 0x05, 0x00,
+	0xfa, 0xff, 0x00, 0x05, 0x00, 0xfa, 0xff, 0x00,
+	0x14, 0x00, 0xeb, 0xff, 0x42, 0x88, 0x21, 0xc4,
+	0x00, 0x00, 0x14, 0x00, 0xeb, 0xff, 0x42, 0x88,
+	0x21, 0xc4, 0x00, 0x00, 0x14, 0x00, 0xeb, 0xff,
+	0x42, 0x88, 0x21, 0xc4, 0x00, 0x00, 0x14, 0x00,
+	0xeb, 0xff, 0x42, 0x88, 0x21, 0xc4, 0x00, 0x00,
+	0x14, 0x00, 0xeb, 0xff, 0x42, 0x88, 0x21, 0xc4,
+	0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0x00, 0x00,
+	0x00, 0xff, 0xff, 0x00, 0x17, 0x00, 0xe8, 0xff,
+	0x42, 0x88, 0x21, 0xc4, 0x00, 0x00, 0x00, 0x00,
+	0xff, 0xff, 0x00, 0x00, 0x00, 0xff, 0xff, 0x00,
+	0x17, 0x00, 0xe8, 0xff, 0x42, 0x12, 0x46, 0x16,
+	0x06, 0x00, 0x00, 0x00, 0xff, 0xff, 0x01, 0x08,
+	0x00, 0xf7, 0xff, 0x3d, 0xb1, 0x20, 0x85, 0xfa,
+	0x00, 0x00, 0x00, 0x42, 0x12, 0x46, 0x16, 0x06,
+	0x00, 0x00, 0x00, 0xff, 0xff, 0x01, 0x08, 0x00,
+	0xf7, 0xff, 0x3d, 0xb1, 0x20, 0x85, 0xfa, 0x00,
+	0x00, 0x00, 0x3d, 0xb1, 0x20, 0x85, 0xfa, 0x00,
+	0x00, 0x00,
+}