diff --git a/src/cmd/go/internal/envcmd/env.go b/src/cmd/go/internal/envcmd/env.go index 10499c2d3ef..66ef5ceee3c 100644 --- a/src/cmd/go/internal/envcmd/env.go +++ b/src/cmd/go/internal/envcmd/env.go @@ -58,6 +58,7 @@ For more about environment variables, see 'go help environment'. func init() { CmdEnv.Run = runEnv // break init cycle base.AddChdirFlag(&CmdEnv.Flag) + base.AddBuildFlagsNX(&CmdEnv.Flag) } var ( diff --git a/src/cmd/go/internal/work/action.go b/src/cmd/go/internal/work/action.go index fc46d19bc4e..8beb1345d0b 100644 --- a/src/cmd/go/internal/work/action.go +++ b/src/cmd/go/internal/work/action.go @@ -33,11 +33,12 @@ import ( // It does not hold per-package state, because we // build packages in parallel, and the builder is shared. type Builder struct { - WorkDir string // the temporary work directory (ends in filepath.Separator) - actionCache map[cacheKey]*Action // a cache of already-constructed actions - mkdirCache map[string]bool // a cache of created directories - flagCache map[[2]string]bool // a cache of supported compiler flags - Print func(args ...any) (int, error) + WorkDir string // the temporary work directory (ends in filepath.Separator) + actionCache map[cacheKey]*Action // a cache of already-constructed actions + mkdirCache map[string]bool // a cache of created directories + flagCache map[[2]string]bool // a cache of supported compiler flags + gccCompilerIDCache map[string]cache.ActionID // cache for gccCompilerID + Print func(args ...any) (int, error) IsCmdList bool // running as part of go list; set p.Stale and additional fields below NeedError bool // list needs p.Error diff --git a/src/cmd/go/internal/work/buildid.go b/src/cmd/go/internal/work/buildid.go index f0b12e10369..db56714788f 100644 --- a/src/cmd/go/internal/work/buildid.go +++ b/src/cmd/go/internal/work/buildid.go @@ -17,6 +17,7 @@ import ( "cmd/go/internal/fsys" "cmd/go/internal/str" "cmd/internal/buildid" + "cmd/internal/quoted" ) // Build IDs @@ -206,14 +207,20 @@ func (b *Builder) toolID(name string) string { // In order to get reproducible builds for released compilers, we // detect a released compiler by the absence of "experimental" in the // --version output, and in that case we just use the version string. -func (b *Builder) gccToolID(name, language string) (string, error) { +// +// gccToolID also returns the underlying executable for the compiler. +// The caller assumes that stat of the exe can be used, combined with the id, +// to detect changes in the underlying compiler. The returned exe can be empty, +// which means to rely only on the id. +func (b *Builder) gccToolID(name, language string) (id, exe string, err error) { key := name + "." + language b.id.Lock() - id := b.toolIDCache[key] + id = b.toolIDCache[key] + exe = b.toolIDCache[key+".exe"] b.id.Unlock() if id != "" { - return id, nil + return id, exe, nil } // Invoke the driver with -### to see the subcommands and the @@ -225,19 +232,19 @@ func (b *Builder) gccToolID(name, language string) (string, error) { cmd.Env = append(os.Environ(), "LC_ALL=C") out, err := cmd.CombinedOutput() if err != nil { - return "", fmt.Errorf("%s: %v; output: %q", name, err, out) + return "", "", fmt.Errorf("%s: %v; output: %q", name, err, out) } version := "" lines := strings.Split(string(out), "\n") for _, line := range lines { - if fields := strings.Fields(line); len(fields) > 1 && fields[1] == "version" { + if fields := strings.Fields(line); len(fields) > 1 && fields[1] == "version" || len(fields) > 2 && fields[2] == "version" { version = line break } } if version == "" { - return "", fmt.Errorf("%s: can not find version number in %q", name, out) + return "", "", fmt.Errorf("%s: can not find version number in %q", name, out) } if !strings.Contains(version, "experimental") { @@ -248,20 +255,20 @@ func (b *Builder) gccToolID(name, language string) (string, error) { // a leading space is the compiler proper. compiler := "" for _, line := range lines { - if len(line) > 1 && line[0] == ' ' { + if strings.HasPrefix(line, " ") && !strings.HasPrefix(line, " (in-process)") { compiler = line break } } if compiler == "" { - return "", fmt.Errorf("%s: can not find compilation command in %q", name, out) + return "", "", fmt.Errorf("%s: can not find compilation command in %q", name, out) } - fields := strings.Fields(compiler) + fields, _ := quoted.Split(compiler) if len(fields) == 0 { - return "", fmt.Errorf("%s: compilation command confusion %q", name, out) + return "", "", fmt.Errorf("%s: compilation command confusion %q", name, out) } - exe := fields[0] + exe = fields[0] if !strings.ContainsAny(exe, `/\`) { if lp, err := exec.LookPath(exe); err == nil { exe = lp @@ -269,7 +276,7 @@ func (b *Builder) gccToolID(name, language string) (string, error) { } id, err = buildid.ReadFile(exe) if err != nil { - return "", err + return "", "", err } // If we can't find a build ID, use a hash. @@ -280,9 +287,10 @@ func (b *Builder) gccToolID(name, language string) (string, error) { b.id.Lock() b.toolIDCache[key] = id + b.toolIDCache[key+".exe"] = exe b.id.Unlock() - return id, nil + return id, exe, nil } // Check if assembler used by gccgo is GNU as. diff --git a/src/cmd/go/internal/work/exec.go b/src/cmd/go/internal/work/exec.go index 90d96400b84..344f409199c 100644 --- a/src/cmd/go/internal/work/exec.go +++ b/src/cmd/go/internal/work/exec.go @@ -282,21 +282,21 @@ func (b *Builder) buildActionID(a *Action) cache.ActionID { // so that the prebuilt .a files from a Go binary install // don't need to be rebuilt with the local compiler. if !p.Standard { - if ccID, err := b.gccToolID(ccExe[0], "c"); err == nil { + if ccID, _, err := b.gccToolID(ccExe[0], "c"); err == nil { fmt.Fprintf(h, "CC ID=%q\n", ccID) } } if len(p.CXXFiles)+len(p.SwigCXXFiles) > 0 { cxxExe := b.cxxExe() fmt.Fprintf(h, "CXX=%q %q\n", cxxExe, cxxflags) - if cxxID, err := b.gccToolID(cxxExe[0], "c++"); err == nil { + if cxxID, _, err := b.gccToolID(cxxExe[0], "c++"); err == nil { fmt.Fprintf(h, "CXX ID=%q\n", cxxID) } } if len(p.FFiles) > 0 { fcExe := b.fcExe() fmt.Fprintf(h, "FC=%q %q\n", fcExe, fflags) - if fcID, err := b.gccToolID(fcExe[0], "f95"); err == nil { + if fcID, _, err := b.gccToolID(fcExe[0], "f95"); err == nil { fmt.Fprintf(h, "FC ID=%q\n", fcID) } } @@ -350,7 +350,7 @@ func (b *Builder) buildActionID(a *Action) cache.ActionID { } case "gccgo": - id, err := b.gccToolID(BuildToolchain.compiler(), "go") + id, _, err := b.gccToolID(BuildToolchain.compiler(), "go") if err != nil { base.Fatalf("%v", err) } @@ -358,7 +358,7 @@ func (b *Builder) buildActionID(a *Action) cache.ActionID { fmt.Fprintf(h, "pkgpath %s\n", gccgoPkgpath(p)) fmt.Fprintf(h, "ar %q\n", BuildToolchain.(gccgoToolchain).ar()) if len(p.SFiles) > 0 { - id, _ = b.gccToolID(BuildToolchain.compiler(), "assembler-with-cpp") + id, _, _ = b.gccToolID(BuildToolchain.compiler(), "assembler-with-cpp") // Ignore error; different assembler versions // are unlikely to make any difference anyhow. fmt.Fprintf(h, "asm %q\n", id) @@ -1359,7 +1359,7 @@ func (b *Builder) printLinkerConfig(h io.Writer, p *load.Package) { // Or external linker settings and flags? case "gccgo": - id, err := b.gccToolID(BuildToolchain.linker(), "go") + id, _, err := b.gccToolID(BuildToolchain.linker(), "go") if err != nil { base.Fatalf("%v", err) } @@ -2689,31 +2689,6 @@ func (b *Builder) gccNoPie(linker []string) string { func (b *Builder) gccSupportsFlag(compiler []string, flag string) bool { key := [2]string{compiler[0], flag} - b.exec.Lock() - defer b.exec.Unlock() - if b, ok := b.flagCache[key]; ok { - return b - } - if b.flagCache == nil { - b.flagCache = make(map[[2]string]bool) - } - - tmp := os.DevNull - - // On the iOS builder the command - // $CC -Wl,--no-gc-sections -x c - -o /dev/null < /dev/null - // is failing with: - // Unable to remove existing file: Invalid argument - if runtime.GOOS == "windows" || runtime.GOOS == "ios" { - f, err := os.CreateTemp(b.WorkDir, "") - if err != nil { - return false - } - f.Close() - tmp = f.Name() - defer os.Remove(tmp) - } - // We used to write an empty C file, but that gets complicated with go // build -n. We tried using a file that does not exist, but that fails on // systems with GCC version 4.2.1; that is the last GPLv2 version of GCC, @@ -2725,6 +2700,22 @@ func (b *Builder) gccSupportsFlag(compiler []string, flag string) bool { // omit the linking step with "-c". // // Using the same CFLAGS/LDFLAGS here and for building the program. + + // On the iOS builder the command + // $CC -Wl,--no-gc-sections -x c - -o /dev/null < /dev/null + // is failing with: + // Unable to remove existing file: Invalid argument + tmp := os.DevNull + if runtime.GOOS == "windows" || runtime.GOOS == "ios" { + f, err := os.CreateTemp(b.WorkDir, "") + if err != nil { + return false + } + f.Close() + tmp = f.Name() + defer os.Remove(tmp) + } + cmdArgs := str.StringList(compiler, flag) if strings.HasPrefix(flag, "-Wl,") /* linker flag */ { ldflags, err := buildFlags("LDFLAGS", defaultCFlags, nil, checkLinkerFlags) @@ -2743,12 +2734,37 @@ func (b *Builder) gccSupportsFlag(compiler []string, flag string) bool { cmdArgs = append(cmdArgs, "-x", "c", "-", "-o", tmp) - if cfg.BuildN || cfg.BuildX { + if cfg.BuildN { b.Showcmd(b.WorkDir, "%s || true", joinUnambiguously(cmdArgs)) - if cfg.BuildN { - return false + return false + } + + // gccCompilerID acquires b.exec, so do before acquiring lock. + compilerID, cacheOK := b.gccCompilerID(compiler[0]) + + b.exec.Lock() + defer b.exec.Unlock() + if b, ok := b.flagCache[key]; ok { + return b + } + if b.flagCache == nil { + b.flagCache = make(map[[2]string]bool) + } + + // Look in build cache. + var flagID cache.ActionID + if cacheOK { + flagID = cache.Subkey(compilerID, "gccSupportsFlag "+flag) + if data, _, err := cache.Default().GetBytes(flagID); err == nil { + supported := string(data) == "true" + b.flagCache[key] = supported + return supported } } + + if cfg.BuildX { + b.Showcmd(b.WorkDir, "%s || true", joinUnambiguously(cmdArgs)) + } cmd := exec.Command(cmdArgs[0], cmdArgs[1:]...) cmd.Dir = b.WorkDir cmd.Env = append(cmd.Environ(), "LC_ALL=C") @@ -2765,10 +2781,120 @@ func (b *Builder) gccSupportsFlag(compiler []string, flag string) bool { !bytes.Contains(out, []byte("is not supported")) && !bytes.Contains(out, []byte("not recognized")) && !bytes.Contains(out, []byte("unsupported")) + + if cacheOK { + s := "false" + if supported { + s = "true" + } + cache.Default().PutBytes(flagID, []byte(s)) + } + b.flagCache[key] = supported return supported } +// statString returns a string form of an os.FileInfo, for serializing and comparison. +func statString(info os.FileInfo) string { + return fmt.Sprintf("stat %d %x %v %v\n", info.Size(), uint64(info.Mode()), info.ModTime(), info.IsDir()) +} + +// gccCompilerID returns a build cache key for the current gcc, +// as identified by running 'compiler'. +// The caller can use subkeys of the key. +// Other parts of cmd/go can use the id as a hash +// of the installed compiler version. +func (b *Builder) gccCompilerID(compiler string) (id cache.ActionID, ok bool) { + if cfg.BuildN { + b.Showcmd(b.WorkDir, "%s || true", joinUnambiguously([]string{compiler, "--version"})) + return cache.ActionID{}, false + } + + b.exec.Lock() + defer b.exec.Unlock() + + if id, ok := b.gccCompilerIDCache[compiler]; ok { + return id, ok + } + + // We hash the compiler's full path to get a cache entry key. + // That cache entry holds a validation description, + // which is of the form: + // + // filename \x00 statinfo \x00 + // ... + // compiler id + // + // If os.Stat of each filename matches statinfo, + // then the entry is still valid, and we can use the + // compiler id without any further expense. + // + // Otherwise, we compute a new validation description + // and compiler id (below). + exe, err := exec.LookPath(compiler) + if err != nil { + return cache.ActionID{}, false + } + + h := cache.NewHash("gccCompilerID") + fmt.Fprintf(h, "gccCompilerID %q", exe) + key := h.Sum() + data, _, err := cache.Default().GetBytes(key) + if err == nil && len(data) > len(id) { + stats := strings.Split(string(data[:len(data)-len(id)]), "\x00") + if len(stats)%2 != 0 { + goto Miss + } + for i := 0; i+2 <= len(stats); i++ { + info, err := os.Stat(stats[i]) + if err != nil || statString(info) != stats[i+1] { + goto Miss + } + } + copy(id[:], data[len(data)-len(id):]) + return id, true + Miss: + } + + // Validation failed. Compute a new description (in buf) and compiler ID (in h). + // For now, there are only at most two filenames in the stat information. + // The first one is the compiler executable we invoke. + // The second is the underlying compiler as reported by -v -### + // (see b.gccToolID implementation in buildid.go). + toolID, exe2, err := b.gccToolID(compiler, "c") + if err != nil { + return cache.ActionID{}, false + } + + exes := []string{exe, exe2} + str.Uniq(&exes) + fmt.Fprintf(h, "gccCompilerID %q %q\n", exes, toolID) + id = h.Sum() + + var buf bytes.Buffer + for _, exe := range exes { + if exe == "" { + continue + } + info, err := os.Stat(exe) + if err != nil { + return cache.ActionID{}, false + } + buf.WriteString(exe) + buf.WriteString("\x00") + buf.WriteString(statString(info)) + buf.WriteString("\x00") + } + buf.Write(id[:]) + + cache.Default().PutBytes(key, buf.Bytes()) + if b.gccCompilerIDCache == nil { + b.gccCompilerIDCache = make(map[string]cache.ActionID) + } + b.gccCompilerIDCache[compiler] = id + return id, true +} + // gccArchArgs returns arguments to pass to gcc based on the architecture. func (b *Builder) gccArchArgs() []string { switch cfg.Goarch { diff --git a/src/cmd/go/testdata/script/env_cache.txt b/src/cmd/go/testdata/script/env_cache.txt new file mode 100644 index 00000000000..f2af7ee623e --- /dev/null +++ b/src/cmd/go/testdata/script/env_cache.txt @@ -0,0 +1,5 @@ +# go env should caches compiler results +go env +go env -x +! stdout '\|\| true' +