os: raise open file rlimit at startup

Some systems set an artificially low soft limit on open file count,
for compatibility with code that uses select and its hard-coded
maximum file descriptor (limited by the size of fd_set).

Go does not use select, so it should not be subject to these limits.
On some systems the limit is 256, which is very easy to run into, even
in simple programs like gofmt when they parallelize walking a file tree.

After a long discussion on go.dev/issue/46279, we decided the best
approach was for Go to raise the limit unconditionally for itself, and
then leave old software to set the limit back as needed. Code that
really wants Go to leave the limit alone can set the hard limit, which
Go of course has no choice but to respect.

Take 2, after CL 392415 was rolled back for macOS and OpenBSD failures.
The macOS failures should be handled by the new call to sysctl("kern.maxfilesperproc"),
and the OpenBSD failures are handled by skipping the test (and filing #51713).

Fixes #46279.

Change-Id: I45c81b94590b447b483018a05ae980b8f02dc5de
Reviewed-on: https://go-review.googlesource.com/c/go/+/393354
Trust: Russ Cox <rsc@golang.org>
Reviewed-by: Bryan Mills <bcmills@google.com>
Run-TryBot: Russ Cox <rsc@golang.org>
TryBot-Result: Gopher Robot <gobot@golang.org>

src/os/rlimit.go

@@ -0,0 +1,32 @@
+// Copyright 2022 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build aix || darwin || dragonfly || freebsd || linux || netbsd || openbsd || solaris
+
+package os
+
+import "syscall"
+
+// Some systems set an artificially low soft limit on open file count, for compatibility
+// with code that uses select and its hard-coded maximum file descriptor
+// (limited by the size of fd_set).
+//
+// Go does not use select, so it should not be subject to these limits.
+// On some systems the limit is 256, which is very easy to run into,
+// even in simple programs like gofmt when they parallelize walking
+// a file tree.
+//
+// After a long discussion on go.dev/issue/46279, we decided the
+// best approach was for Go to raise the limit unconditionally for itself,
+// and then leave old software to set the limit back as needed.
+// Code that really wants Go to leave the limit alone can set the hard limit,
+// which Go of course has no choice but to respect.
+func init() {
+	var lim syscall.Rlimit
+	if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &lim); err == nil && lim.Cur != lim.Max {
+		lim.Cur = lim.Max
+		adjustFileLimit(&lim)
+		syscall.Setrlimit(syscall.RLIMIT_NOFILE, &lim)
+	}
+}

src/os/rlimit_darwin.go

@@ -0,0 +1,22 @@
+// Copyright 2022 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build darwin
+
+package os
+
+import "syscall"
+
+// adjustFileLimit adds per-OS limitations on the Rlimit used for RLIMIT_NOFILE. See rlimit.go.
+func adjustFileLimit(lim *syscall.Rlimit) {
+	// On older macOS, setrlimit(RLIMIT_NOFILE, lim) with lim.Cur = infinity fails.
+	// Set to the value of kern.maxfilesperproc instead.
+	n, err := syscall.SysctlUint32("kern.maxfilesperproc")
+	if err != nil {
+		return
+	}
+	if lim.Cur > uint64(n) {
+		lim.Cur = uint64(n)
+	}
+}

src/os/rlimit_stub.go

@@ -0,0 +1,12 @@
+// Copyright 2022 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build aix || dragonfly || freebsd || linux || netbsd || openbsd || solaris
+
+package os
+
+import "syscall"
+
+// adjustFileLimit adds per-OS limitations on the Rlimit used for RLIMIT_NOFILE. See rlimit.go.
+func adjustFileLimit(lim *syscall.Rlimit) {}

src/os/rlimit_test.go

@@ -0,0 +1,37 @@
+// Copyright 2022 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package os_test
+
+import (
+	. "os"
+	"runtime"
+	"testing"
+)
+
+func TestOpenFileLimit(t *testing.T) {
+	if runtime.GOOS == "openbsd" && (runtime.GOARCH == "arm" || runtime.GOARCH == "arm64") {
+		t.Skip("broken on openbsd/arm and openbsd/arm64 builder - go.dev/issue/51713")
+	}
+
+	// For open file count,
+	// macOS sets the default soft limit to 256 and no hard limit.
+	// CentOS and Fedora set the default soft limit to 1024,
+	// with hard limits of 4096 and 524288, respectively.
+	// Check that we can open 1200 files, which proves
+	// that the rlimit is being raised appropriately on those systems.
+	var files []*File
+	for i := 0; i < 1200; i++ {
+		f, err := Open("rlimit.go")
+		if err != nil {
+			t.Error(err)
+			break
+		}
+		files = append(files, f)
+	}
+
+	for _, f := range files {
+		f.Close()
+	}
+}