net/url: reject query values with semicolons

Semicolons are no longer valid separators, so
net/url.ParseQuery will now return an error
if any part of the query contains a semicolon.

net/http.(*Request).ParseMultipartForm has been
changed to fall through and continue parsing
even if the call to (*Request).ParseForm fails.

This change also includes a few minor refactors
to existing tests.

Fixes #25192

Change-Id: Iba3f108950fb99b9288e402c41fe71ca3a2ababd
Reviewed-on: https://go-review.googlesource.com/c/go/+/325697
Trust: Katie Hockman <katie@golang.org>
Run-TryBot: Katie Hockman <katie@golang.org>
TryBot-Result: Go Bot <gobot@golang.org>
Reviewed-by: Filippo Valsorda <filippo@golang.org>

src/net/http/request.go

@@ -1293,16 +1293,18 @@ func (r *Request) ParseForm() error {
 // its file parts are stored in memory, with the remainder stored on
 // disk in temporary files.
 // ParseMultipartForm calls ParseForm if necessary.
+// If ParseForm returns an error, ParseMultipartForm returns it but also
+// continues parsing the request body.
 // After one call to ParseMultipartForm, subsequent calls have no effect.
 func (r *Request) ParseMultipartForm(maxMemory int64) error {
 	if r.MultipartForm == multipartByReader {
 		return errors.New("http: multipart handled by MultipartReader")
 	}
+	var parseFormErr error
 	if r.Form == nil {
-		err := r.ParseForm()
-		if err != nil {
-			return err
-		}
+		// Let errors in ParseForm fall through, and just
+		// return it at the end.
+		parseFormErr = r.ParseForm()
 	}
 	if r.MultipartForm != nil {
 		return nil
@@ -1329,7 +1331,7 @@ func (r *Request) ParseMultipartForm(maxMemory int64) error {
 
 	r.MultipartForm = f
 
-	return nil
+	return parseFormErr
 }
 
 // FormValue returns the first value for the named component of the query.

src/net/http/request_test.go

@@ -32,9 +32,26 @@ func TestQuery(t *testing.T) {
 	}
 }
 
+// Issue #25192: Test that ParseForm fails but still parses the form when an URL
+// containing a semicolon is provided.
+func TestParseFormSemicolonSeparator(t *testing.T) {
+	for _, method := range []string{"POST", "PATCH", "PUT", "GET"} {
+		req, _ := NewRequest(method, "http://www.google.com/search?q=foo;q=bar&a=1",
+			strings.NewReader("q"))
+		err := req.ParseForm()
+		if err == nil {
+			t.Fatalf(`for method %s, ParseForm expected an error, got success`, method)
+		}
+		wantForm := url.Values{"a": []string{"1"}}
+		if !reflect.DeepEqual(req.Form, wantForm) {
+			t.Fatalf("for method %s, ParseForm expected req.Form = %v, want %v", method, req.Form, wantForm)
+		}
+	}
+}
+
 func TestParseFormQuery(t *testing.T) {
 	req, _ := NewRequest("POST", "http://www.google.com/search?q=foo&q=bar&both=x&prio=1&orphan=nope&empty=not",
-		strings.NewReader("z=post&both=y&prio=2&=nokey&orphan;empty=&"))
+		strings.NewReader("z=post&both=y&prio=2&=nokey&orphan&empty=&"))
 	req.Header.Set("Content-Type", "application/x-www-form-urlencoded; param=value")
 
 	if q := req.FormValue("q"); q != "foo" {
@@ -365,6 +382,18 @@ func TestMultipartRequest(t *testing.T) {
 	validateTestMultipartContents(t, req, false)
 }
 
+// Issue #25192: Test that ParseMultipartForm fails but still parses the
+// multi-part form when an URL containing a semicolon is provided.
+func TestParseMultipartFormSemicolonSeparator(t *testing.T) {
+	req := newTestMultipartRequest(t)
+	req.URL = &url.URL{RawQuery: "q=foo;q=bar"}
+	if err := req.ParseMultipartForm(25); err == nil {
+		t.Fatal("ParseMultipartForm expected error due to invalid semicolon, got nil")
+	}
+	defer req.MultipartForm.RemoveAll()
+	validateTestMultipartContents(t, req, false)
+}
+
 func TestMultipartRequestAuto(t *testing.T) {
 	// Test that FormValue and FormFile automatically invoke
 	// ParseMultipartForm and return the right values.

src/net/http/server.go

@@ -2863,6 +2863,11 @@ func (sh serverHandler) ServeHTTP(rw ResponseWriter, req *Request) {
 		handler = globalOptionsHandler{}
 	}
 	handler.ServeHTTP(rw, req)
+	if req.URL != nil && strings.Contains(req.URL.RawQuery, ";") {
+		// TODO(filippo): update this not to log if the special
+		// semicolon handler was called.
+		sh.srv.logf("http: URL query contains semicolon, which is no longer a supported separator; parts of the query may be stripped when parsed; see golang.org/issue/25192")
+	}
 }
 
 // ListenAndServe listens on the TCP network address srv.Addr and then

src/net/url/example_test.go

@@ -72,13 +72,13 @@ func ExampleURL_ResolveReference() {
 }
 
 func ExampleParseQuery() {
-	m, err := url.ParseQuery(`x=1&y=2&y=3;z`)
+	m, err := url.ParseQuery(`x=1&y=2&y=3`)
 	if err != nil {
 		log.Fatal(err)
 	}
 	fmt.Println(toJSON(m))
 	// Output:
-	// {"x":["1"], "y":["2", "3"], "z":[""]}
+	// {"x":["1"], "y":["2", "3"]}
 }
 
 func ExampleURL_EscapedPath() {

src/net/url/url.go

@@ -921,9 +921,10 @@ func (v Values) Has(key string) bool {
 // valid query parameters found; err describes the first decoding error
 // encountered, if any.
 //
-// Query is expected to be a list of key=value settings separated by
-// ampersands or semicolons. A setting without an equals sign is
-// interpreted as a key set to an empty value.
+// Query is expected to be a list of key=value settings separated by ampersands.
+// A setting without an equals sign is interpreted as a key set to an empty
+// value.
+// Settings containing a non-URL-encoded semicolon are considered invalid.
 func ParseQuery(query string) (Values, error) {
 	m := make(Values)
 	err := parseQuery(m, query)
@@ -933,11 +934,15 @@ func ParseQuery(query string) (Values, error) {
 func parseQuery(m Values, query string) (err error) {
 	for query != "" {
 		key := query
-		if i := strings.IndexAny(key, "&;"); i >= 0 {
+		if i := strings.IndexAny(key, "&"); i >= 0 {
 			key, query = key[:i], key[i+1:]
 		} else {
 			query = ""
 		}
+		if strings.Contains(key, ";") {
+			err = fmt.Errorf("invalid semicolon separator in query")
+			continue
+		}
 		if key == "" {
 			continue
 		}

src/net/url/url_test.go

@@ -1334,57 +1334,125 @@ func TestQueryValues(t *testing.T) {
 type parseTest struct {
 	query string
 	out   Values
+	ok    bool
 }
 
 var parseTests = []parseTest{
+	{
+		query: "a=1",
+		out:   Values{"a": []string{"1"}},
+		ok:    true,
+	},
 	{
 		query: "a=1&b=2",
 		out:   Values{"a": []string{"1"}, "b": []string{"2"}},
+		ok:    true,
 	},
 	{
 		query: "a=1&a=2&a=banana",
 		out:   Values{"a": []string{"1", "2", "banana"}},
+		ok:    true,
 	},
 	{
 		query: "ascii=%3Ckey%3A+0x90%3E",
 		out:   Values{"ascii": []string{"<key: 0x90>"}},
+		ok:    true,
+	}, {
+		query: "a=1;b=2",
+		out:   Values{},
+		ok:    false,
+	}, {
+		query: "a;b=1",
+		out:   Values{},
+		ok:    false,
+	}, {
+		query: "a=%3B", // hex encoding for semicolon
+		out:   Values{"a": []string{";"}},
+		ok:    true,
 	},
 	{
-		query: "a=1;b=2",
-		out:   Values{"a": []string{"1"}, "b": []string{"2"}},
+		query: "a%3Bb=1",
+		out:   Values{"a;b": []string{"1"}},
+		ok:    true,
 	},
 	{
 		query: "a=1&a=2;a=banana",
-		out:   Values{"a": []string{"1", "2", "banana"}},
+		out:   Values{"a": []string{"1"}},
+		ok:    false,
+	},
+	{
+		query: "a;b&c=1",
+		out:   Values{"c": []string{"1"}},
+		ok:    false,
+	},
+	{
+		query: "a=1&b=2;a=3&c=4",
+		out:   Values{"a": []string{"1"}, "c": []string{"4"}},
+		ok:    false,
+	},
+	{
+		query: "a=1&b=2;c=3",
+		out:   Values{"a": []string{"1"}},
+		ok:    false,
+	},
+	{
+		query: ";",
+		out:   Values{},
+		ok:    false,
+	},
+	{
+		query: "a=1;",
+		out:   Values{},
+		ok:    false,
+	},
+	{
+		query: "a=1&;",
+		out:   Values{"a": []string{"1"}},
+		ok:    false,
+	},
+	{
+		query: ";a=1&b=2",
+		out:   Values{"b": []string{"2"}},
+		ok:    false,
+	},
+	{
+		query: "a=1&b=2;",
+		out:   Values{"a": []string{"1"}},
+		ok:    false,
 	},
 }
 
 func TestParseQuery(t *testing.T) {
-	for i, test := range parseTests {
-		form, err := ParseQuery(test.query)
-		if err != nil {
-			t.Errorf("test %d: Unexpected error: %v", i, err)
-			continue
-		}
-		if len(form) != len(test.out) {
-			t.Errorf("test %d: len(form) = %d, want %d", i, len(form), len(test.out))
-		}
-		for k, evs := range test.out {
-			vs, ok := form[k]
-			if !ok {
-				t.Errorf("test %d: Missing key %q", i, k)
-				continue
+	for _, test := range parseTests {
+		t.Run(test.query, func(t *testing.T) {
+			form, err := ParseQuery(test.query)
+			if test.ok != (err == nil) {
+				want := "<error>"
+				if test.ok {
+					want = "<nil>"
+				}
+				t.Errorf("Unexpected error: %v, want %v", err, want)
 			}
-			if len(vs) != len(evs) {
-				t.Errorf("test %d: len(form[%q]) = %d, want %d", i, k, len(vs), len(evs))
-				continue
+			if len(form) != len(test.out) {
+				t.Errorf("len(form) = %d, want %d", len(form), len(test.out))
 			}
-			for j, ev := range evs {
-				if v := vs[j]; v != ev {
-					t.Errorf("test %d: form[%q][%d] = %q, want %q", i, k, j, v, ev)
+			for k, evs := range test.out {
+				vs, ok := form[k]
+				if !ok {
+					t.Errorf("Missing key %q", k)
+					continue
+				}
+				if len(vs) != len(evs) {
+					t.Errorf("len(form[%q]) = %d, want %d", k, len(vs), len(evs))
+					continue
+				}
+				for j, ev := range evs {
+					if v := vs[j]; v != ev {
+						t.Errorf("form[%q][%d] = %q, want %q", k, j, v, ev)
+					}
 				}
 			}
-		}
+		})
 	}
 }