cmd/go: add //go:embed all:pattern

When //go:embed d matches directory d, it embeds the directory
tree rooted at d, but it excludes files beginning with . and _,
as well as files having problematic names that will not be packaged
into modules (names such as .git and com1).

After long discussions on #42328 and #43854, we decided to keep
the behavior of excluding . and _ files by default, but to allow the pattern
prefix 'all:' to override this default. This CL implements that change.

Note that paths like .git and com1 are still excluded, as they must be,
since they will never be packed into a module.

Fixes #43854.

Change-Id: I4f3731e14ecffd4b691fda3a0890b460027fe209
Reviewed-on: https://go-review.googlesource.com/c/go/+/359413
Trust: Russ Cox <rsc@golang.org>
Run-TryBot: Russ Cox <rsc@golang.org>
TryBot-Result: Go Bot <gobot@golang.org>
Reviewed-by: Jonathan Amsterdam <jba@google.com>
Reviewed-by: Bryan C. Mills <bcmills@google.com>

src/cmd/go/internal/load/pkg.go

@@ -2017,13 +2017,18 @@ func resolveEmbed(pkgdir string, patterns []string) (files []string, pmap map[st
 	for _, pattern = range patterns {
 		pid++
 
+		glob := pattern
+		all := strings.HasPrefix(pattern, "all:")
+		if all {
+			glob = pattern[len("all:"):]
+		}
 		// Check pattern is valid for //go:embed.
-		if _, err := path.Match(pattern, ""); err != nil || !validEmbedPattern(pattern) {
+		if _, err := path.Match(glob, ""); err != nil || !validEmbedPattern(glob) {
 			return nil, nil, fmt.Errorf("invalid pattern syntax")
 		}
 
 		// Glob to find matches.
-		match, err := fsys.Glob(pkgdir + string(filepath.Separator) + filepath.FromSlash(pattern))
+		match, err := fsys.Glob(pkgdir + string(filepath.Separator) + filepath.FromSlash(glob))
 		if err != nil {
 			return nil, nil, err
 		}
@@ -2086,7 +2091,7 @@ func resolveEmbed(pkgdir string, patterns []string) (files []string, pmap map[st
 					}
 					rel := filepath.ToSlash(path[len(pkgdir)+1:])
 					name := info.Name()
-					if path != file && (isBadEmbedName(name) || name[0] == '.' || name[0] == '_') {
+					if path != file && (isBadEmbedName(name) || ((name[0] == '.' || name[0] == '_') && !all)) {
 						// Ignore bad names, assuming they won't go into modules.
 						// Also avoid hidden files that user may not know about.
 						// See golang.org/issue/42328.

src/cmd/go/testdata/script/embed.txt

@@ -60,6 +60,18 @@ rm t/x.txt
 ! go build m/use
 stderr '^x.go:5:12: pattern [*]t: cannot embed directory t: contains no embeddable files$'
 
+# all still ignores .git and symlinks
+cp x.go3 x.go
+! go build -x
+stderr '^x.go:5:12: pattern all:t: cannot embed directory t: contains no embeddable files$'
+
+# all finds dot files and underscore files
+cp x.txt t/.x.txt
+go build -x
+rm t/.x.txt
+cp x.txt t/_x.txt
+go build -x
+
 -- x.go --
 package p
 
@@ -92,6 +104,14 @@ import "embed"
 //go:embed *t
 var X embed.FS
 
+-- x.go3 --
+package p
+
+import "embed"
+
+//go:embed all:t
+var X embed.FS
+
 -- x.txt --
 hello
 

src/embed/embed.go

@@ -80,6 +80,11 @@
 //	var content embed.FS
 //
 // The difference is that ‘image/*’ embeds ‘image/.tempfile’ while ‘image’ does not.
+// Neither embeds ‘image/dir/.tempfile’.
+//
+// If a pattern begins with the prefix ‘all:’, then the rule for walking directories is changed
+// to include those files beginning with ‘.’ or ‘_’. For example, ‘all:image’ embeds
+// both ‘image/.tempfile’ and ‘image/dir/.tempfile’.
 //
 // The //go:embed directive can be used with both exported and unexported variables,
 // depending on whether the package wants to make the data available to other packages.