cmd/go: change list -compiled to populate new CompiledGoFiles list

CL 108156 added -cgo during the Go 1.11 cycle.
To avoid adding a new field to Package, it redefined the
meaning of the CgoFiles list to be the cgo output instead
of the cgo input.

This was awkward in the go command itself, since the meaning
of the list changed midway through the build.

But, worse, it is awkward to users of go list.
When gathering information about a tree of packages,
we may want the names of both the cgo inputs and the cgo outputs
(golang.org/x/tools/go/packages does, it turns out),
or when combined with -deps (CL 107776),
we may only care about one list or the other depending
on whether the package was requested explicitly or is
being returned as a dependency.

Also, it's not general enough. SWIGFiles turn into cgo files
and then end up in the list too. And maybe there will be others
in the future. What clients really want is the list of files that
are presented to the go compiler, so that they can parse
and type-check them as if they were the compiler instead.

Eliminate all this awkwardness by dropping -cgo and adding
a new -compiled that populates a new CompiledGoFiles list.

Change-Id: I5f152da17cfb2692eedde61721d01ec13067c57d
Reviewed-on: https://go-review.googlesource.com/126695
Run-TryBot: Russ Cox <rsc@golang.org>
Reviewed-by: Michael Matloob <matloob@golang.org>
Reviewed-by: Bryan C. Mills <bcmills@google.com>

src/cmd/cgo/ast.go

@@ -95,7 +95,7 @@ func (f *File) ParseGo(name string, src []byte) {
 		}
 	}
 	if !sawC {
-		error_(token.NoPos, `cannot find import "C"`)
+		error_(ast1.Package, `cannot find import "C"`)
 	}
 
 	// In ast2, strip the import "C" line.

src/cmd/cgo/util.go

@@ -99,6 +99,8 @@ func error_(pos token.Pos, msg string, args ...interface{}) {
 	nerrors++
 	if pos.IsValid() {
 		fmt.Fprintf(os.Stderr, "%s: ", fset.Position(pos).String())
+	} else {
+		fmt.Fprintf(os.Stderr, "cgo: ")
 	}
 	fmt.Fprintf(os.Stderr, msg, args...)
 	fmt.Fprintf(os.Stderr, "\n")

src/cmd/go/go_test.go

@@ -1776,7 +1776,7 @@ func TestGoListTest(t *testing.T) {
 	tg.grepStdoutNot(`^sort`, "unexpected sort")
 }
 
-func TestGoListCgo(t *testing.T) {
+func TestGoListCompiledCgo(t *testing.T) {
 	tg := testgo(t)
 	defer tg.cleanup()
 	tg.parallel()
@@ -1788,18 +1788,26 @@ func TestGoListCgo(t *testing.T) {
 		t.Skip("net does not use cgo")
 	}
 	if strings.Contains(tg.stdout.String(), tg.tempdir) {
-		t.Fatalf(".CgoFiles without -cgo unexpectedly mentioned cache %s", tg.tempdir)
+		t.Fatalf(".CgoFiles unexpectedly mentioned cache %s", tg.tempdir)
 	}
-	tg.run("list", "-cgo", "-f", `{{join .CgoFiles "\n"}}`, "net")
+	tg.run("list", "-compiled", "-f", `{{.Dir}}{{"\n"}}{{join .CompiledGoFiles "\n"}}`, "net")
 	if !strings.Contains(tg.stdout.String(), tg.tempdir) {
-		t.Fatalf(".CgoFiles with -cgo did not mention cache %s", tg.tempdir)
+		t.Fatalf(".CompiledGoFiles with -compiled did not mention cache %s", tg.tempdir)
 	}
+	dir := ""
 	for _, file := range strings.Split(tg.stdout.String(), "\n") {
 		if file == "" {
 			continue
 		}
+		if dir == "" {
+			dir = file
+			continue
+		}
+		if !strings.Contains(file, "/") && !strings.Contains(file, `\`) {
+			file = filepath.Join(dir, file)
+		}
 		if _, err := os.Stat(file); err != nil {
-			t.Fatalf("cannot find .CgoFiles result %s: %v", file, err)
+			t.Fatalf("cannot find .CompiledGoFiles result %s: %v", file, err)
 		}
 	}
 }

src/cmd/go/internal/list/list.go

@@ -67,8 +67,9 @@ to -f '{{.ImportPath}}'. The struct being passed to the template is:
 
         // Source files
         GoFiles         []string // .go source files (excluding CgoFiles, TestGoFiles, XTestGoFiles)
-        CgoFiles       []string // .go sources files that import "C"
+        CgoFiles        []string // .go source files that import "C"
-        IgnoredGoFiles []string // .go sources ignored due to build constraints
+        CompiledGoFiles []string // .go files presented to compiler (when using -compiled)
+        IgnoredGoFiles  []string // .go source files ignored due to build constraints
         CFiles          []string // .c source files
         CXXFiles        []string // .cc, .cxx and .cpp source files
         MFiles          []string // .m source files
@@ -142,9 +143,10 @@ for the go/build package's Context type.
 The -json flag causes the package data to be printed in JSON format
 instead of using the template format.
 
-The -cgo flag causes list to set CgoFiles not to the original *.go files
+The -compiled flag causes list to set CompiledGoFiles to the Go source
-importing "C" but instead to the translated files generated by the cgo
+files presented to the compiler. Typically this means that it repeats
-command.
+the files listed in GoFiles and then also adds the Go code generated
+by processing CgoFiles and SwigFiles.
 
 The -deps flag causes list to iterate over not just the named packages
 but also all their dependencies. It visits them in a depth-first post-order
@@ -184,8 +186,8 @@ are all absolute paths.
 
 By default, the lists GoFiles, CgoFiles, and so on hold names of files in Dir
 (that is, paths relative to Dir, not absolute paths).
-The extra entries added by the -cgo and -test flags are absolute paths
+The generated files added when using the -compiled and -test flags
-referring to cached copies of generated Go source files.
+are absolute paths referring to cached copies of generated Go source files.
 Although they are Go source files, the paths may not end in ".go".
 
 The -m flag causes list to list modules instead of packages.
@@ -282,7 +284,7 @@ func init() {
 }
 
 var (
-	listCgo      = CmdList.Flag.Bool("cgo", false, "")
+	listCompiled = CmdList.Flag.Bool("compiled", false, "")
 	listDeps     = CmdList.Flag.Bool("deps", false, "")
 	listE        = CmdList.Flag.Bool("e", false, "")
 	listExport   = CmdList.Flag.Bool("export", false, "")
@@ -353,8 +355,8 @@ func runList(cmd *base.Command, args []string) {
 
 	if *listM {
 		// Module mode.
-		if *listCgo {
+		if *listCompiled {
-			base.Fatalf("go list -cgo cannot be used with -m")
+			base.Fatalf("go list -compiled cannot be used with -m")
 		}
 		if *listDeps {
 			// TODO(rsc): Could make this mean something with -m.
@@ -405,8 +407,8 @@ func runList(cmd *base.Command, args []string) {
 	if cache.Default() == nil {
 		// These flags return file names pointing into the build cache,
 		// so the build cache must exist.
-		if *listCgo {
+		if *listCompiled {
-			base.Fatalf("go list -cgo requires build cache")
+			base.Fatalf("go list -compiled requires build cache")
 		}
 		if *listExport {
 			base.Fatalf("go list -export requires build cache")
@@ -479,12 +481,12 @@ func runList(cmd *base.Command, args []string) {
 
 	// Do we need to run a build to gather information?
 	needStale := *listJson || strings.Contains(*listFmt, ".Stale")
-	if needStale || *listExport || *listCgo {
+	if needStale || *listExport || *listCompiled {
 		var b work.Builder
 		b.Init()
 		b.IsCmdList = true
 		b.NeedExport = *listExport
-		b.NeedCgoFiles = *listCgo
+		b.NeedCompiledGoFiles = *listCompiled
 		a := &work.Action{}
 		// TODO: Use pkgsFilter?
 		for _, p := range pkgs {

src/cmd/go/internal/load/pkg.go

@@ -80,8 +80,9 @@ type PackagePublic struct {
 	// If you add to this list you MUST add to p.AllFiles (below) too.
 	// Otherwise file name security lists will not apply to any new additions.
 	GoFiles         []string `json:",omitempty"` // .go source files (excluding CgoFiles, TestGoFiles, XTestGoFiles)
-	CgoFiles       []string `json:",omitempty"` // .go sources files that import "C"
+	CgoFiles        []string `json:",omitempty"` // .go source files that import "C"
-	IgnoredGoFiles []string `json:",omitempty"` // .go sources ignored due to build constraints
+	CompiledGoFiles []string `json:",omitempty"` // .go output from running cgo on CgoFiles
+	IgnoredGoFiles  []string `json:",omitempty"` // .go source files ignored due to build constraints
 	CFiles          []string `json:",omitempty"` // .c source files
 	CXXFiles        []string `json:",omitempty"` // .cc, .cpp and .cxx source files
 	MFiles          []string `json:",omitempty"` // .m source files
@@ -128,6 +129,7 @@ func (p *Package) AllFiles() []string {
 	return str.StringList(
 		p.GoFiles,
 		p.CgoFiles,
+		// no p.CompiledGoFiles, because they are from GoFiles or generated by us
 		p.IgnoredGoFiles,
 		p.CFiles,
 		p.CXXFiles,

src/cmd/go/internal/work/action.go

@@ -39,7 +39,7 @@ type Builder struct {
 	IsCmdList           bool // running as part of go list; set p.Stale and additional fields below
 	NeedError           bool // list needs p.Error
 	NeedExport          bool // list needs p.Export
-	NeedCgoFiles bool // list needs p.CgoFiles to cgo-generated files, not originals
+	NeedCompiledGoFiles bool // list needs p.CompiledGoFIles
 
 	objdirSeq int // counter for NewObjdir
 	pkgSeq    int

src/cmd/go/internal/work/exec.go

@@ -345,7 +345,7 @@ const (
 	needBuild uint32 = 1 << iota
 	needCgoHdr
 	needVet
-	needCgoFiles
+	needCompiledGoFiles
 	needStale
 )
 
@@ -365,10 +365,7 @@ func (b *Builder) build(a *Action) (err error) {
 	need := bit(needBuild, !b.IsCmdList || b.NeedExport) |
 		bit(needCgoHdr, b.needCgoHdr(a)) |
 		bit(needVet, a.needVet) |
-		bit(needCgoFiles, b.NeedCgoFiles && (p.UsesCgo() || p.UsesSwig()))
+		bit(needCompiledGoFiles, b.NeedCompiledGoFiles)
-
-	// Save p.CgoFiles now, because we may modify it for go list.
-	cgofiles := append([]string{}, p.CgoFiles...)
 
 	if !p.BinaryOnly {
 		if b.useCache(a, p, b.buildActionID(a), p.Target) {
@@ -378,8 +375,8 @@ func (b *Builder) build(a *Action) (err error) {
 			if b.NeedExport {
 				p.Export = a.built
 			}
-			if need&needCgoFiles != 0 && b.loadCachedCgoFiles(a) {
+			if need&needCompiledGoFiles != 0 && b.loadCachedGoFiles(a) {
-				need &^= needCgoFiles
+				need &^= needCompiledGoFiles
 			}
 			// Otherwise, we need to write files to a.Objdir (needVet, needCgoHdr).
 			// Remember that we might have them in cache
@@ -469,11 +466,12 @@ func (b *Builder) build(a *Action) (err error) {
 		}
 	}
 
-	var gofiles, cfiles, sfiles, cxxfiles, objects, cgoObjects, pcCFLAGS, pcLDFLAGS []string
+	gofiles := str.StringList(a.Package.GoFiles)
-	gofiles = append(gofiles, a.Package.GoFiles...)
+	cgofiles := str.StringList(a.Package.CgoFiles)
-	cfiles = append(cfiles, a.Package.CFiles...)
+	cfiles := str.StringList(a.Package.CFiles)
-	sfiles = append(sfiles, a.Package.SFiles...)
+	sfiles := str.StringList(a.Package.SFiles)
-	cxxfiles = append(cxxfiles, a.Package.CXXFiles...)
+	cxxfiles := str.StringList(a.Package.CXXFiles)
+	var objects, cgoObjects, pcCFLAGS, pcLDFLAGS []string
 
 	if a.Package.UsesCgo() || a.Package.UsesSwig() {
 		if pcCFLAGS, pcLDFLAGS, err = b.getPkgConfigFlags(a.Package); err != nil {
@@ -594,11 +592,11 @@ func (b *Builder) build(a *Action) (err error) {
 		buildVetConfig(a, gofiles)
 		need &^= needVet
 	}
-	if need&needCgoFiles != 0 {
+	if need&needCompiledGoFiles != 0 {
-		if !b.loadCachedCgoFiles(a) {
+		if !b.loadCachedGoFiles(a) {
-			return fmt.Errorf("failed to cache translated CgoFiles")
+			return fmt.Errorf("failed to cache compiled Go files")
 		}
-		need &^= needCgoFiles
+		need &^= needCompiledGoFiles
 	}
 	if need == 0 {
 		// Nothing left to do.
@@ -836,7 +834,7 @@ func (b *Builder) loadCachedVet(a *Action) bool {
 	return true
 }
 
-func (b *Builder) loadCachedCgoFiles(a *Action) bool {
+func (b *Builder) loadCachedGoFiles(a *Action) bool {
 	c := cache.Default()
 	if c == nil {
 		return false
@@ -851,6 +849,7 @@ func (b *Builder) loadCachedCgoFiles(a *Action) bool {
 			continue
 		}
 		if strings.HasPrefix(name, "./") {
+			files = append(files, name[len("./"):])
 			continue
 		}
 		file, err := b.findCachedObjdirFile(a, c, name)
@@ -859,7 +858,7 @@ func (b *Builder) loadCachedCgoFiles(a *Action) bool {
 		}
 		files = append(files, file)
 	}
-	a.Package.CgoFiles = files
+	a.Package.CompiledGoFiles = files
 	return true
 }