qbit/go
1
0
Fork 0
mirror of https://github.com/golang/go synced 2024-11-18 01:34:44 -07:00
Code Releases Activity

cmd/go/internal/work: cache cgo invocations for vet, build modes

Browse Source 
Even if we had an up-to-date package binary, we reran cgo anyway if
(1) we needed a header file for buildmode c-archive or c-shared, or
(2) we needed cgo-translated files source files for input to go vet.

Cache those outputs too, so that we can avoid cgo if possible.

Working toward exposing the cgo-generated files in go list.

Change-Id: I339ecace925d2b0adc235a17977ecadb3d636c73
Reviewed-on: https://go-review.googlesource.com/108015
Run-TryBot: Russ Cox <rsc@golang.org>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Bryan C. Mills <bcmills@google.com>
This commit is contained in:
Russ Cox 2018-04-18 21:12:19 -04:00
parent 9ccfde6ee7
commit 4826d20a09
2 changed files with 197 additions and 73 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
src/cmd/go/go_test.go
View File

@ -5250,6 +5250,31 @@ func TestCacheCoverage(t *testing.T) {
tg.run("test", "-cover", "-short", "math", "strings")
}
func TestCacheVet(t *testing.T) {
tg := testgo(t)
defer tg.cleanup()
tg.parallel()
if strings.Contains(os.Getenv("GODEBUG"), "gocacheverify") {
t.Skip("GODEBUG gocacheverify")
}
if os.Getenv("GOCACHE") == "off" {
tooSlow(t)
tg.makeTempdir()
tg.setenv("GOCACHE", tg.path("cache"))
}
// Check that second vet reuses cgo-derived inputs.
// The first command could be build instead of vet,
// except that if the cache is empty and there's a net.a
// in GOROOT/pkg, the build will not bother to regenerate
// and cache the cgo outputs, whereas vet always will.
tg.run("vet", "os/user")
tg.run("vet", "-x", "os/user")
tg.grepStderrNot(`^(clang|gcc)`, "should not have run compiler")
tg.grepStderrNot(`[\\/]cgo `, "should not have run cgo")
}
func TestIssue22588(t *testing.T) {
// Don't get confused by stderr coming from tools.
tg := testgo(t)

229
src/cmd/go/internal/work/exec.go
View File

@ -296,37 +296,37 @@ func (b *Builder) buildActionID(a *Action) cache.ActionID {
return h.Sum()
}
// build is the action for building a single package.
// Note that any new influence on this logic must be reported in b.buildActionID above as well.
func (b *Builder) build(a *Action) (err error) {
p := a.Package
cached := false
if !p.BinaryOnly {
if b.useCache(a, p, b.buildActionID(a), p.Target) {
// needCgoHeader reports whether the actions triggered by this one
// expect to be able to access the cgo-generated header file.
func needCgoHeader(a *Action) bool {
// If this build triggers a header install, run cgo to get the header.
// TODO(rsc): Once we can cache multiple file outputs from an action,
// the header should be cached, and then this awful test can be deleted.
// Need to look for install header actions depending on this action,
// or depending on a link that depends on this action.
needHeader := false
if (a.Package.UsesCgo() || a.Package.UsesSwig()) && (cfg.BuildBuildmode == "c-archive" || cfg.BuildBuildmode == "c-shared") {
for _, t1 := range a.triggers {
if t1.Mode == "install header" {
needHeader = true
goto CheckedHeader
return true
}
}
for _, t1 := range a.triggers {
for _, t2 := range t1.triggers {
if t2.Mode == "install header" {
needHeader = true
goto CheckedHeader
return true
}
}
}
}
CheckedHeader:
if b.ComputeStaleOnly || !a.needVet && !needHeader {
return false
}
// build is the action for building a single package.
// Note that any new influence on this logic must be reported in b.buildActionID above as well.
func (b *Builder) build(a *Action) (err error) {
p := a.Package
cached := false
needCgo := needCgoHeader(a)
if !p.BinaryOnly {
if b.useCache(a, p, b.buildActionID(a), p.Target) {
if b.ComputeStaleOnly || !needCgo && !a.needVet {
return nil
}
cached = true
@ -375,6 +375,10 @@ func (b *Builder) build(a *Action) (err error) {
}
objdir := a.Objdir
if cached && (!needCgo || b.loadCachedCgo(a)) && (!a.needVet || b.loadCachedVet(a)) {
return nil
}
// make target directory
dir, _ := filepath.Split(a.Target)
if dir != "" {
@ -384,7 +388,6 @@ func (b *Builder) build(a *Action) (err error) {
}
var gofiles, cgofiles, cfiles, sfiles, cxxfiles, objects, cgoObjects, pcCFLAGS, pcLDFLAGS []string
gofiles = append(gofiles, a.Package.GoFiles...)
cgofiles = append(cgofiles, a.Package.CgoFiles...)
cfiles = append(cfiles, a.Package.CFiles...)
@ -489,10 +492,13 @@ func (b *Builder) build(a *Action) (err error) {
}
cgoObjects = append(cgoObjects, outObj...)
gofiles = append(gofiles, outGo...)
switch cfg.BuildBuildmode {
case "c-archive", "c-shared":
b.cacheCgoHdr(a)
}
if cached && !a.needVet {
return nil
}
b.cacheGofiles(a, gofiles)
// Sanity check only, since Package.load already checked as well.
if len(gofiles) == 0 {
@ -500,25 +506,14 @@ func (b *Builder) build(a *Action) (err error) {
}
// Prepare Go vet config if needed.
var vcfg *vetConfig
if a.needVet {
// Pass list of absolute paths to vet,
// so that vet's error messages will use absolute paths,
// so that we can reformat them relative to the directory
// in which the go command is invoked.
vcfg = &vetConfig{
Compiler: cfg.BuildToolchainName,
Dir: a.Package.Dir,
GoFiles: mkAbsFiles(a.Package.Dir, gofiles),
ImportPath: a.Package.ImportPath,
ImportMap: make(map[string]string),
PackageFile: make(map[string]string),
}
a.vetCfg = vcfg
for i, raw := range a.Package.Internal.RawImports {
final := a.Package.Imports[i]
vcfg.ImportMap[raw] = final
buildVetConfig(a, gofiles)
}
if cached {
// The cached package file is OK, so we don't need to run the compile.
// We've only gone this far in order to prepare the vet configuration
// or cgo header, and now we have.
return nil
}
// Prepare Go import config.
@ -529,45 +524,18 @@ func (b *Builder) build(a *Action) (err error) {
// except when it doesn't.
var icfg bytes.Buffer
fmt.Fprintf(&icfg, "# import config\n")
for i, raw := range a.Package.Internal.RawImports {
final := a.Package.Imports[i]
if final != raw {
fmt.Fprintf(&icfg, "importmap %s=%s\n", raw, final)
}
}
// Compute the list of mapped imports in the vet config
// so that we can add any missing mappings below.
var vcfgMapped map[string]bool
if vcfg != nil {
vcfgMapped = make(map[string]bool)
for _, p := range vcfg.ImportMap {
vcfgMapped[p] = true
}
}
for _, a1 := range a.Deps {
p1 := a1.Package
if p1 == nil || p1.ImportPath == "" || a1.built == "" {
continue
}
fmt.Fprintf(&icfg, "packagefile %s=%s\n", p1.ImportPath, a1.built)
if vcfg != nil {
// Add import mapping if needed
// (for imports like "runtime/cgo" that appear only in generated code).
if !vcfgMapped[p1.ImportPath] {
vcfg.ImportMap[p1.ImportPath] = p1.ImportPath
}
vcfg.PackageFile[p1.ImportPath] = a1.built
}
}
if cached {
// The cached package file is OK, so we don't need to run the compile.
// We've only going through the motions to prepare the vet configuration,
// which is now complete.
return nil
}
// Compile Go.
@ -678,6 +646,97 @@ func (b *Builder) build(a *Action) (err error) {
return nil
}
func (b *Builder) cacheObjdirFile(a *Action, c *cache.Cache, name string) error {
f, err := os.Open(a.Objdir + name)
if err != nil {
return err
}
defer f.Close()
_, _, err = c.Put(cache.Subkey(a.actionID, name), f)
return err
}
func (b *Builder) loadCachedObjdirFile(a *Action, c *cache.Cache, name string) error {
entry, err := c.Get(cache.Subkey(a.actionID, name))
if err != nil {
return err
}
out := c.OutputFile(entry.OutputID)
info, err := os.Stat(out)
if err != nil || info.Size() != entry.Size {
return fmt.Errorf("not in cache")
}
return b.copyFile(a.Objdir+name, out, 0666, true)
}
func (b *Builder) cacheCgoHdr(a *Action) {
c := cache.Default()
if c == nil {
return
}
b.cacheObjdirFile(a, c, "_cgo_install.h")
}
func (b *Builder) loadCachedCgo(a *Action) bool {
c := cache.Default()
if c == nil {
return false
}
err := b.loadCachedObjdirFile(a, c, "_cgo_install.h")
return err == nil
}
func (b *Builder) cacheGofiles(a *Action, gofiles []string) {
c := cache.Default()
if c == nil {
return
}
var buf bytes.Buffer
for _, file := range gofiles {
if !strings.HasPrefix(file, a.Objdir) {
// not generated
buf.WriteString("./")
buf.WriteString(file)
buf.WriteString("\n")
continue
}
name := file[len(a.Objdir):]
buf.WriteString(name)
buf.WriteString("\n")
if err := b.cacheObjdirFile(a, c, name); err != nil {
return
}
}
c.PutBytes(cache.Subkey(a.actionID, "gofiles"), buf.Bytes())
}
func (b *Builder) loadCachedVet(a *Action) bool {
c := cache.Default()
if c == nil {
return false
}
list, _, err := c.GetBytes(cache.Subkey(a.actionID, "gofiles"))
if err != nil {
return false
}
var gofiles []string
for _, name := range strings.Split(string(list), "\n") {
if name == "" { // end of list
continue
}
if strings.HasPrefix(name, "./") {
gofiles = append(gofiles, name[2:])
continue
}
if err := b.loadCachedObjdirFile(a, c, name); err != nil {
return false
}
gofiles = append(gofiles, a.Objdir+name)
}
buildVetConfig(a, gofiles)
return true
}
type vetConfig struct {
Compiler string
Dir string
@ -689,6 +748,46 @@ type vetConfig struct {
SucceedOnTypecheckFailure bool
}
func buildVetConfig(a *Action, gofiles []string) {
// Pass list of absolute paths to vet,
// so that vet's error messages will use absolute paths,
// so that we can reformat them relative to the directory
// in which the go command is invoked.
vcfg := &vetConfig{
Compiler: cfg.BuildToolchainName,
Dir: a.Package.Dir,
GoFiles: mkAbsFiles(a.Package.Dir, gofiles),
ImportPath: a.Package.ImportPath,
ImportMap: make(map[string]string),
PackageFile: make(map[string]string),
}
a.vetCfg = vcfg
for i, raw := range a.Package.Internal.RawImports {
final := a.Package.Imports[i]
vcfg.ImportMap[raw] = final
}
// Compute the list of mapped imports in the vet config
// so that we can add any missing mappings below.
vcfgMapped := make(map[string]bool)
for _, p := range vcfg.ImportMap {
vcfgMapped[p] = true
}
for _, a1 := range a.Deps {
p1 := a1.Package
if p1 == nil || p1.ImportPath == "" || a1.built == "" {
continue
}
// Add import mapping if needed
// (for imports like "runtime/cgo" that appear only in generated code).
if !vcfgMapped[p1.ImportPath] {
vcfg.ImportMap[p1.ImportPath] = p1.ImportPath
}
vcfg.PackageFile[p1.ImportPath] = a1.built
}
}
// VetTool is the path to an alternate vet tool binary.
// The caller is expected to set it (if needed) before executing any vet actions.
var VetTool string