all: treat all files as binary, but check in .bat with CRLF

This is a followup to CL 96495.

It should be simpler and more robust to achieve .bat files having
CRLF line endings by treating it as a binary file, like all other
files, and checking it in with the desired CRLF line endings.

A test is used to check the entire Go tree, short of directories
starting with "." and named "testdata", for any .bat files that
have anything other than strict CRLF line endings. This will help
catch any accidental modifications to existing .bat files or check
ins of new .bat files.

Importantly, this is compatible with how Gerrit serves .tar.gz files,
making it so that CRLF line endings are preserved.

The Go project is supported on many different environments, some of
which may have limited git implementations available, or none at all.
Relying on fewer git features and special rules makes it easier to
have confidence in the exact content of all files. Additionally, Go
development started in Subversion, moved to Perforce, then Mercurial,
and now uses Git.¹ Reducing its reliance on git-specific features will
help if there will be another transition in the project's future.

There are only 5 .bat files in the entire Go source tree, so a new one
being added is a rare event, and we prefer to do things in Go instead.
We still have the option of improving the experience for developers by
adding a pre-commit converter for .bat files to the git-codereview tool.

¹ https://groups.google.com/d/msg/golang-dev/sckirqOWepg/YmyT7dWJiocJ

Fixes #39391.
For #37791.

Change-Id: I6e202216322872f0307ac96f1b8d3f57cb901e6b
Reviewed-on: https://go-review.googlesource.com/c/go/+/236437
Reviewed-by: Bryan C. Mills <bcmills@google.com>

.gitattributes

@@ -1,16 +1,16 @@
 # Treat all files in the Go repo as binary, with no git magic updating
-# line endings. Windows users contributing to Go will need to use a
+# line endings. This produces predictable results in different environments.
-# modern version of git and editors capable of LF line endings.
+#
+# Windows users contributing to Go will need to use a modern version
+# of git and editors capable of LF line endings.
+#
+# Windows .bat files are known to have multiple bugs when run with LF
+# endings, and so they are checked in with CRLF endings, with a test
+# in test/winbatch.go to catch problems. (See golang.org/issue/37791.)
 #
 # We'll prevent accidental CRLF line endings from entering the repo
-# via the git-review gofmt checks.
+# via the git-codereview gofmt checks and tests.
 #
-# See golang.org/issue/9281
+# See golang.org/issue/9281.
 
 * -text
-
-# The only exception is Windows files that must absolutely be CRLF or
-# might not work. Batch files are known to have multiple bugs when run
-# with LF endings. See golang.org/issue/37791 for more information.
-
-*.bat text eol=crlf

src/make.bat

@@ -150,4 +150,3 @@ set GOBUILDFAIL=1
 if x%GOBUILDEXIT%==x1 exit %GOBUILDFAIL%
 
 :end
-

test/winbatch.go

@@ -4,8 +4,8 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// Check that batch files are maintained as CRLF files (consistent behaviour
+// Check that batch files are maintained as CRLF files (consistent
-// on all operating systems). See https://github.com/golang/go/issues/37791
+// behavior on all operating systems). See golang.org/issue/37791.
 
 package main
 
@@ -13,18 +13,56 @@ import (
 	"bytes"
 	"fmt"
 	"io/ioutil"
+	"log"
 	"os"
 	"path/filepath"
 	"runtime"
+	"strings"
 )
 
 func main() {
-	batches, _ := filepath.Glob(runtime.GOROOT() + "/src/*.bat")
+	// Ensure that the GOROOT/src/all.bat file exists and has strict CRLF line endings.
-	for _, bat := range batches {
+	enforceBatchStrictCRLF(filepath.Join(runtime.GOROOT(), "src", "all.bat"))
-		body, _ := ioutil.ReadFile(bat)
+
-		if !bytes.Contains(body, []byte("\r\n")) {
+	// Walk the entire Go repository source tree (without GOROOT/pkg),
-			fmt.Printf("Windows batch file %s does not contain CRLF line termination.\nTry running git checkout src/*.bat to fix this.\n", bat)
+	// skipping directories that start with "." and named "testdata",
+	// and ensure all .bat files found have exact CRLF line endings.
+	err := filepath.Walk(runtime.GOROOT(), func(path string, fi os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if fi.IsDir() && (strings.HasPrefix(fi.Name(), ".") || fi.Name() == "testdata") {
+			return filepath.SkipDir
+		}
+		if path == filepath.Join(runtime.GOROOT(), "pkg") {
+			// GOROOT/pkg is known to contain generated artifacts, not source code.
+			// Skip it to avoid false positives. (Also see golang.org/issue/37929.)
+			return filepath.SkipDir
+		}
+		if filepath.Ext(fi.Name()) == ".bat" {
+			enforceBatchStrictCRLF(path)
+		}
+		return nil
+	})
+	if err != nil {
+		log.Fatalln(err)
+	}
+}
+
+func enforceBatchStrictCRLF(path string) {
+	b, err := ioutil.ReadFile(path)
+	if err != nil {
+		log.Fatalln(err)
+	}
+	cr, lf := bytes.Count(b, []byte{13}), bytes.Count(b, []byte{10})
+	crlf := bytes.Count(b, []byte{13, 10})
+	if cr != crlf || lf != crlf {
+		if rel, err := filepath.Rel(runtime.GOROOT(), path); err == nil {
+			// Make the test failure more readable by showing a path relative to GOROOT.
+			path = rel
+		}
+		fmt.Printf("Windows batch file %s does not use strict CRLF line termination.\n", path)
+		fmt.Printf("Please convert it to CRLF before checking it in due to golang.org/issue/37791.\n")
 		os.Exit(1)
 	}
 }
-}