encoding/asn1: sort order of 'SET of' components during Marshal

Per X690 Section 11.6 sort the order of SET of components when generating
DER. This CL makes no changes to Unmarshal, meaning unordered components
will still be accepted, and won't be re-ordered during parsing.

In order to sort the components a new encoder, setEncoder, which is similar
to multiEncoder is added. The functional difference is that setEncoder
encodes each component to a [][]byte, sorts the slice using a sort.Sort
interface, and then writes it out to the destination slice. The ordering
matches the output of OpenSSL.

Fixes #24254

Change-Id: Iff4560f0b8c2dce5aae616ba30226f39c10b972e

src/encoding/asn1/marshal.go

@@ -5,10 +5,12 @@
 package asn1
 
 import (
+	"bytes"
 	"errors"
 	"fmt"
 	"math/big"
 	"reflect"
+	"sort"
 	"time"
 	"unicode/utf8"
 )
@@ -78,6 +80,60 @@ func (m multiEncoder) Encode(dst []byte) {
 	}
 }
 
+type octetSorter [][]byte
+
+func (s octetSorter) Len() int {
+	return len(s)
+}
+
+func (s octetSorter) Swap(i, j int) {
+	s[i], s[j] = s[j], s[i]
+}
+
+func (s octetSorter) Less(i, j int) bool {
+	// Since we are using bytes.Compare to compare TLV encodings we
+	// don't need to right pad s[i] and s[j] to the same length as
+	// suggested in X690. If len(s[i]) < len(s[j]) the length octet of
+	// s[i], which is the first determining byte, will inherently be
+	// smaller than the length octet of s[j]. This lets us skip the
+	// padding step.
+	return bytes.Compare(s[i], s[j]) < 0
+}
+
+type setEncoder []encoder
+
+func (s setEncoder) Len() int {
+	var size int
+	for _, e := range s {
+		size += e.Len()
+	}
+	return size
+}
+
+func (s setEncoder) Encode(dst []byte) {
+	// Per X690 Section 11.6: The encodings of the component values of a
+	// set-of value shall appear in ascending order, the encodings being
+	// compared as octet strings with the shorter components being padded
+	// at their trailing end with 0-octets.
+	//
+	// First we encode each element to its TLV encoding and then use
+	// octetSort to get the ordering expected by X690 DER rules before
+	// writing the sorted encodings out to dst.
+	l := make([][]byte, len(s))
+	for i, e := range s {
+		l[i] = make([]byte, e.Len())
+		e.Encode(l[i])
+	}
+
+	sort.Sort(octetSorter(l))
+
+	var off int
+	for _, b := range l {
+		copy(dst[off:], b)
+		off += len(b)
+	}
+}
+
 type taggedEncoder struct {
 	// scratch contains temporary space for encoding the tag and length of
 	// an element in order to avoid extra allocations.
@@ -511,6 +567,9 @@ func makeBody(value reflect.Value, params fieldParameters) (e encoder, err error
 				}
 			}
 
+			if params.set {
+				return setEncoder(m), nil
+			}
 			return multiEncoder(m), nil
 		}
 	case reflect.String:

src/encoding/asn1/marshal_test.go

@@ -319,3 +319,34 @@ func BenchmarkMarshal(b *testing.B) {
 		}
 	}
 }
+
+func TestSetEncoder(t *testing.T) {
+	testStruct := struct {
+		Strings []string `asn1:"set"`
+	}{
+		Strings: []string{"a", "aa", "b", "bb", "c", "cc"},
+	}
+
+	// Expected ordering of the SET should be:
+	// a, b, c, aa, bb, cc
+
+	output, err := Marshal(testStruct)
+	if err != nil {
+		t.Errorf("%v", err)
+	}
+
+	expectedOrder := []string{"a", "b", "c", "aa", "bb", "cc"}
+	var resultStruct struct {
+		Strings []string `asn1:"set"`
+	}
+	rest, err := Unmarshal(output, &resultStruct)
+	if err != nil {
+		t.Errorf("%v", err)
+	}
+	if len(rest) != 0 {
+		t.Error("Unmarshal returned extra garbage")
+	}
+	if !reflect.DeepEqual(expectedOrder, resultStruct.Strings) {
+		t.Errorf("Unexpected SET content. got: %s, want: %s", resultStruct.Strings, resultStruct.Strings)
+	}
+}