qbit/go
1
0
Fork 0
mirror of https://github.com/golang/go synced 2024-11-23 19:40:08 -07:00
Code Releases Activity

cmd/go: cache compiler flag info

Browse Source 
When you run 'go env' or any command that needs to consider
what the default gcc flags are (such as 'go list net' or
'go list <any package with net as a dependency>'),
the go command runs gcc (or clang) a few times to see what
flags are available.

These runs can be quite expensive on some systems, particularly
Macs that seem to need to occasionally cache something before
gcc/clang can execute quickly.

To fix this, cache the derived information about gcc under a cache
key derived from the size and modification time of the compiler binary.
This is not foolproof, but it should be good enough.

% go install cmd/go
% time go env >/dev/null
        0.22 real         0.01 user         0.01 sys
% time go env >/dev/null
        0.03 real         0.01 user         0.01 sys
%

Fixes #50982.

Change-Id: Iba7955dd10f610f2793e1accbd2d06922f928faa
Reviewed-on: https://go-review.googlesource.com/c/go/+/392454
TryBot-Result: Gopher Robot <gobot@golang.org>
Reviewed-by: Bryan Mills <bcmills@google.com>
Run-TryBot: Russ Cox <rsc@golang.org>
Auto-Submit: Russ Cox <rsc@golang.org>
This commit is contained in:
Russ Cox 2022-02-11 17:17:54 -05:00 committed by Gopher Robot
parent cb6e4f08c2
commit db259cdd80
5 changed files with 193 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
src/cmd/go/internal/envcmd/env.go
View File

@ -58,6 +58,7 @@ For more about environment variables, see 'go help environment'.
func init() {
CmdEnv.Run = runEnv // break init cycle
base.AddChdirFlag(&CmdEnv.Flag)
base.AddBuildFlagsNX(&CmdEnv.Flag)
}
var (

11
src/cmd/go/internal/work/action.go
View File

@ -33,11 +33,12 @@ import (
// It does not hold per-package state, because we
// build packages in parallel, and the builder is shared.
type Builder struct {
WorkDir string // the temporary work directory (ends in filepath.Separator)
actionCache map[cacheKey]*Action // a cache of already-constructed actions
mkdirCache map[string]bool // a cache of created directories
flagCache map[[2]string]bool // a cache of supported compiler flags
Print func(args ...any) (int, error)
WorkDir string // the temporary work directory (ends in filepath.Separator)
actionCache map[cacheKey]*Action // a cache of already-constructed actions
mkdirCache map[string]bool // a cache of created directories
flagCache map[[2]string]bool // a cache of supported compiler flags
gccCompilerIDCache map[string]cache.ActionID // cache for gccCompilerID
Print func(args ...any) (int, error)
IsCmdList bool // running as part of go list; set p.Stale and additional fields below
NeedError bool // list needs p.Error

34
src/cmd/go/internal/work/buildid.go
View File

@ -17,6 +17,7 @@ import (
"cmd/go/internal/fsys"
"cmd/go/internal/str"
"cmd/internal/buildid"
"cmd/internal/quoted"
)
// Build IDs
@ -206,14 +207,20 @@ func (b *Builder) toolID(name string) string {
// In order to get reproducible builds for released compilers, we
// detect a released compiler by the absence of "experimental" in the
// --version output, and in that case we just use the version string.
func (b *Builder) gccToolID(name, language string) (string, error) {
//
// gccToolID also returns the underlying executable for the compiler.
// The caller assumes that stat of the exe can be used, combined with the id,
// to detect changes in the underlying compiler. The returned exe can be empty,
// which means to rely only on the id.
func (b *Builder) gccToolID(name, language string) (id, exe string, err error) {
key := name + "." + language
b.id.Lock()
id := b.toolIDCache[key]
id = b.toolIDCache[key]
exe = b.toolIDCache[key+".exe"]
b.id.Unlock()
if id != "" {
return id, nil
return id, exe, nil
}
// Invoke the driver with -### to see the subcommands and the
@ -225,19 +232,19 @@ func (b *Builder) gccToolID(name, language string) (string, error) {
cmd.Env = append(os.Environ(), "LC_ALL=C")
out, err := cmd.CombinedOutput()
if err != nil {
return "", fmt.Errorf("%s: %v; output: %q", name, err, out)
return "", "", fmt.Errorf("%s: %v; output: %q", name, err, out)
}
version := ""
lines := strings.Split(string(out), "\n")
for _, line := range lines {
if fields := strings.Fields(line); len(fields) > 1 && fields[1] == "version" {
if fields := strings.Fields(line); len(fields) > 1 && fields[1] == "version" || len(fields) > 2 && fields[2] == "version" {
version = line
break
}
}
if version == "" {
return "", fmt.Errorf("%s: can not find version number in %q", name, out)
return "", "", fmt.Errorf("%s: can not find version number in %q", name, out)
}
if !strings.Contains(version, "experimental") {
@ -248,20 +255,20 @@ func (b *Builder) gccToolID(name, language string) (string, error) {
// a leading space is the compiler proper.
compiler := ""
for _, line := range lines {
if len(line) > 1 && line[0] == ' ' {
if strings.HasPrefix(line, " ") && !strings.HasPrefix(line, " (in-process)") {
compiler = line
break
}
}
if compiler == "" {
return "", fmt.Errorf("%s: can not find compilation command in %q", name, out)
return "", "", fmt.Errorf("%s: can not find compilation command in %q", name, out)
}
fields := strings.Fields(compiler)
fields, _ := quoted.Split(compiler)
if len(fields) == 0 {
return "", fmt.Errorf("%s: compilation command confusion %q", name, out)
return "", "", fmt.Errorf("%s: compilation command confusion %q", name, out)
}
exe := fields[0]
exe = fields[0]
if !strings.ContainsAny(exe, `/\`) {
if lp, err := exec.LookPath(exe); err == nil {
exe = lp
@ -269,7 +276,7 @@ func (b *Builder) gccToolID(name, language string) (string, error) {
}
id, err = buildid.ReadFile(exe)
if err != nil {
return "", err
return "", "", err
}
// If we can't find a build ID, use a hash.
@ -280,9 +287,10 @@ func (b *Builder) gccToolID(name, language string) (string, error) {
b.id.Lock()
b.toolIDCache[key] = id
b.toolIDCache[key+".exe"] = exe
b.id.Unlock()
return id, nil
return id, exe, nil
}
// Check if assembler used by gccgo is GNU as.

194
src/cmd/go/internal/work/exec.go
View File

@ -282,21 +282,21 @@ func (b *Builder) buildActionID(a *Action) cache.ActionID {
// so that the prebuilt .a files from a Go binary install
// don't need to be rebuilt with the local compiler.
if !p.Standard {
if ccID, err := b.gccToolID(ccExe[0], "c"); err == nil {
if ccID, _, err := b.gccToolID(ccExe[0], "c"); err == nil {
fmt.Fprintf(h, "CC ID=%q\n", ccID)
}
}
if len(p.CXXFiles)+len(p.SwigCXXFiles) > 0 {
cxxExe := b.cxxExe()
fmt.Fprintf(h, "CXX=%q %q\n", cxxExe, cxxflags)
if cxxID, err := b.gccToolID(cxxExe[0], "c++"); err == nil {
if cxxID, _, err := b.gccToolID(cxxExe[0], "c++"); err == nil {
fmt.Fprintf(h, "CXX ID=%q\n", cxxID)
}
}
if len(p.FFiles) > 0 {
fcExe := b.fcExe()
fmt.Fprintf(h, "FC=%q %q\n", fcExe, fflags)
if fcID, err := b.gccToolID(fcExe[0], "f95"); err == nil {
if fcID, _, err := b.gccToolID(fcExe[0], "f95"); err == nil {
fmt.Fprintf(h, "FC ID=%q\n", fcID)
}
}
@ -350,7 +350,7 @@ func (b *Builder) buildActionID(a *Action) cache.ActionID {
}
case "gccgo":
id, err := b.gccToolID(BuildToolchain.compiler(), "go")
id, _, err := b.gccToolID(BuildToolchain.compiler(), "go")
if err != nil {
base.Fatalf("%v", err)
}
@ -358,7 +358,7 @@ func (b *Builder) buildActionID(a *Action) cache.ActionID {
fmt.Fprintf(h, "pkgpath %s\n", gccgoPkgpath(p))
fmt.Fprintf(h, "ar %q\n", BuildToolchain.(gccgoToolchain).ar())
if len(p.SFiles) > 0 {
id, _ = b.gccToolID(BuildToolchain.compiler(), "assembler-with-cpp")
id, _, _ = b.gccToolID(BuildToolchain.compiler(), "assembler-with-cpp")
// Ignore error; different assembler versions
// are unlikely to make any difference anyhow.
fmt.Fprintf(h, "asm %q\n", id)
@ -1359,7 +1359,7 @@ func (b *Builder) printLinkerConfig(h io.Writer, p *load.Package) {
// Or external linker settings and flags?
case "gccgo":
id, err := b.gccToolID(BuildToolchain.linker(), "go")
id, _, err := b.gccToolID(BuildToolchain.linker(), "go")
if err != nil {
base.Fatalf("%v", err)
}
@ -2689,31 +2689,6 @@ func (b *Builder) gccNoPie(linker []string) string {
func (b *Builder) gccSupportsFlag(compiler []string, flag string) bool {
key := [2]string{compiler[0], flag}
b.exec.Lock()
defer b.exec.Unlock()
if b, ok := b.flagCache[key]; ok {
return b
}
if b.flagCache == nil {
b.flagCache = make(map[[2]string]bool)
}
tmp := os.DevNull
// On the iOS builder the command
// $CC -Wl,--no-gc-sections -x c - -o /dev/null < /dev/null
// is failing with:
// Unable to remove existing file: Invalid argument
if runtime.GOOS == "windows" || runtime.GOOS == "ios" {
f, err := os.CreateTemp(b.WorkDir, "")
if err != nil {
return false
}
f.Close()
tmp = f.Name()
defer os.Remove(tmp)
}
// We used to write an empty C file, but that gets complicated with go
// build -n. We tried using a file that does not exist, but that fails on
// systems with GCC version 4.2.1; that is the last GPLv2 version of GCC,
@ -2725,6 +2700,22 @@ func (b *Builder) gccSupportsFlag(compiler []string, flag string) bool {
// omit the linking step with "-c".
//
// Using the same CFLAGS/LDFLAGS here and for building the program.
// On the iOS builder the command
// $CC -Wl,--no-gc-sections -x c - -o /dev/null < /dev/null
// is failing with:
// Unable to remove existing file: Invalid argument
tmp := os.DevNull
if runtime.GOOS == "windows" || runtime.GOOS == "ios" {
f, err := os.CreateTemp(b.WorkDir, "")
if err != nil {
return false
}
f.Close()
tmp = f.Name()
defer os.Remove(tmp)
}
cmdArgs := str.StringList(compiler, flag)
if strings.HasPrefix(flag, "-Wl,") /* linker flag */ {
ldflags, err := buildFlags("LDFLAGS", defaultCFlags, nil, checkLinkerFlags)
@ -2743,12 +2734,37 @@ func (b *Builder) gccSupportsFlag(compiler []string, flag string) bool {
cmdArgs = append(cmdArgs, "-x", "c", "-", "-o", tmp)
if cfg.BuildN || cfg.BuildX {
if cfg.BuildN {
b.Showcmd(b.WorkDir, "%s || true", joinUnambiguously(cmdArgs))
if cfg.BuildN {
return false
return false
}
// gccCompilerID acquires b.exec, so do before acquiring lock.
compilerID, cacheOK := b.gccCompilerID(compiler[0])
b.exec.Lock()
defer b.exec.Unlock()
if b, ok := b.flagCache[key]; ok {
return b
}
if b.flagCache == nil {
b.flagCache = make(map[[2]string]bool)
}
// Look in build cache.
var flagID cache.ActionID
if cacheOK {
flagID = cache.Subkey(compilerID, "gccSupportsFlag "+flag)
if data, _, err := cache.Default().GetBytes(flagID); err == nil {
supported := string(data) == "true"
b.flagCache[key] = supported
return supported
}
}
if cfg.BuildX {
b.Showcmd(b.WorkDir, "%s || true", joinUnambiguously(cmdArgs))
}
cmd := exec.Command(cmdArgs[0], cmdArgs[1:]...)
cmd.Dir = b.WorkDir
cmd.Env = append(cmd.Environ(), "LC_ALL=C")
@ -2765,10 +2781,120 @@ func (b *Builder) gccSupportsFlag(compiler []string, flag string) bool {
!bytes.Contains(out, []byte("is not supported")) &&
!bytes.Contains(out, []byte("not recognized")) &&
!bytes.Contains(out, []byte("unsupported"))
if cacheOK {
s := "false"
if supported {
s = "true"
}
cache.Default().PutBytes(flagID, []byte(s))
}
b.flagCache[key] = supported
return supported
}
// statString returns a string form of an os.FileInfo, for serializing and comparison.
func statString(info os.FileInfo) string {
return fmt.Sprintf("stat %d %x %v %v\n", info.Size(), uint64(info.Mode()), info.ModTime(), info.IsDir())
}
// gccCompilerID returns a build cache key for the current gcc,
// as identified by running 'compiler'.
// The caller can use subkeys of the key.
// Other parts of cmd/go can use the id as a hash
// of the installed compiler version.
func (b *Builder) gccCompilerID(compiler string) (id cache.ActionID, ok bool) {
if cfg.BuildN {
b.Showcmd(b.WorkDir, "%s || true", joinUnambiguously([]string{compiler, "--version"}))
return cache.ActionID{}, false
}
b.exec.Lock()
defer b.exec.Unlock()
if id, ok := b.gccCompilerIDCache[compiler]; ok {
return id, ok
}
// We hash the compiler's full path to get a cache entry key.
// That cache entry holds a validation description,
// which is of the form:
//
// filename \x00 statinfo \x00
// ...
// compiler id
//
// If os.Stat of each filename matches statinfo,
// then the entry is still valid, and we can use the
// compiler id without any further expense.
//
// Otherwise, we compute a new validation description
// and compiler id (below).
exe, err := exec.LookPath(compiler)
if err != nil {
return cache.ActionID{}, false
}
h := cache.NewHash("gccCompilerID")
fmt.Fprintf(h, "gccCompilerID %q", exe)
key := h.Sum()
data, _, err := cache.Default().GetBytes(key)
if err == nil && len(data) > len(id) {
stats := strings.Split(string(data[:len(data)-len(id)]), "\x00")
if len(stats)%2 != 0 {
goto Miss
}
for i := 0; i+2 <= len(stats); i++ {
info, err := os.Stat(stats[i])
if err != nil || statString(info) != stats[i+1] {
goto Miss
}
}
copy(id[:], data[len(data)-len(id):])
return id, true
Miss:
}
// Validation failed. Compute a new description (in buf) and compiler ID (in h).
// For now, there are only at most two filenames in the stat information.
// The first one is the compiler executable we invoke.
// The second is the underlying compiler as reported by -v -###
// (see b.gccToolID implementation in buildid.go).
toolID, exe2, err := b.gccToolID(compiler, "c")
if err != nil {
return cache.ActionID{}, false
}
exes := []string{exe, exe2}
str.Uniq(&exes)
fmt.Fprintf(h, "gccCompilerID %q %q\n", exes, toolID)
id = h.Sum()
var buf bytes.Buffer
for _, exe := range exes {
if exe == "" {
continue
}
info, err := os.Stat(exe)
if err != nil {
return cache.ActionID{}, false
}
buf.WriteString(exe)
buf.WriteString("\x00")
buf.WriteString(statString(info))
buf.WriteString("\x00")
}
buf.Write(id[:])
cache.Default().PutBytes(key, buf.Bytes())
if b.gccCompilerIDCache == nil {
b.gccCompilerIDCache = make(map[string]cache.ActionID)
}
b.gccCompilerIDCache[compiler] = id
return id, true
}
// gccArchArgs returns arguments to pass to gcc based on the architecture.
func (b *Builder) gccArchArgs() []string {
switch cfg.Goarch {

5
src/cmd/go/testdata/script/env_cache.txt vendored Normal file
View File

@ -0,0 +1,5 @@
# go env should caches compiler results
go env
go env -x
! stdout '\|\| true'