crypto/x509: add x509omitbundledroots build tag to not embed roots

On darwin/arm64, the copy of the system roots takes 256 KiB of disk
and 560 KiB of memory after parsing them (which is retained forever in
a package global by x509/root.go). In constrained environments like
iOS NetworkExtensions where total disk+RAM is capped at 15 MiB, these
certs take 5.3% of the total allowed memory.

It turns out you can get down from 816 KiB to 110 KiB by instead
storing compressed x509 certs in the binary and lazily inflating just
the needed certs at runtime as a function of the certs presented to
you by the server, then building a custom root CertPool in the
crypto/tls.Config.VerifyPeerCertificate hook.

This then saves 706 KiB.

Arguably that should be the default Go behavior, but involves
cooperation between x509 and tls, and adds a dependency to
compress/gzip. Also, it may not be the right trade-off for everybody,
as it involves burning more CPU on new TLS connections. Most iOS apps
don't run in a NetworkExtension context limiting them to 15 MiB.

The build tag is chosen to match the existing "nethttpomithttp2".

Change-Id: I7b1c845de08b22674f81dd546e7fadc7dda68bd7
Reviewed-on: https://go-review.googlesource.com/c/go/+/229762
Run-TryBot: Brad Fitzpatrick <bradfitz@golang.org>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Filippo Valsorda <filippo@golang.org>

src/cmd/dist/test.go

@@ -459,6 +459,17 @@ func (t *tester) registerTests() {
 		})
 	}
 
+	if t.iOS() && !t.compileOnly {
+		t.tests = append(t.tests, distTest{
+			name:    "x509omitbundledroots",
+			heading: "crypto/x509 without bundled roots",
+			fn: func(dt *distTest) error {
+				t.addCmd(dt, "src", t.goTest(), t.timeout(300), "-tags=x509omitbundledroots", "-run=OmitBundledRoots", "crypto/x509")
+				return nil
+			},
+		})
+	}
+
 	if t.race {
 		return
 	}

src/crypto/x509/root_darwin_arm64.go

@@ -4,6 +4,8 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
+// +build !x509omitbundledroots
+
 package x509
 
 func loadSystemRoots() (*CertPool, error) {

src/crypto/x509/root_darwin_arm_gen.go

@@ -172,6 +172,8 @@ const header = `
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
+// +build !x509omitbundledroots
+
 package x509
 
 func loadSystemRoots() (*CertPool, error) {

src/crypto/x509/root_omit.go

@@ -0,0 +1,21 @@
+// Copyright 2020 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build darwin,arm64,x509omitbundledroots
+
+// This file provides the loadSystemRoots func when the
+// "x509omitbundledroots" build tag has disabled bundling a copy,
+// which currently on happens on darwin/arm64 (root_darwin_arm64.go).
+// This then saves 256 KiB of binary size and another 560 KiB of
+// runtime memory size retaining the parsed roots forever. Constrained
+// environments can construct minimal x509 root CertPools on the fly
+// in the crypto/tls.Config.VerifyPeerCertificate hook.
+
+package x509
+
+import "errors"
+
+func loadSystemRoots() (*CertPool, error) {
+	return nil, errors.New("x509: system root bundling disabled")
+}

src/crypto/x509/root_omit_test.go

@@ -0,0 +1,22 @@
+// Copyright 2020 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build darwin,arm64,x509omitbundledroots
+
+package x509
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestOmitBundledRoots(t *testing.T) {
+	cp, err := loadSystemRoots()
+	if err == nil {
+		t.Fatalf("loadSystemRoots = (pool %p, error %v); want non-nil error", cp, err)
+	}
+	if !strings.Contains(err.Error(), "root bundling disabled") {
+		t.Errorf("unexpected error doesn't mention bundling: %v", err)
+	}
+}