go/packages: add cgo support to go1.10 fallback loader

This change moves the cgo support for go loader to a common
internal package that can be reused by go/packages. That
support is used to invoke cgo to process the files returned
by the fallback pre-go1.11 Go list.

The cgo processing does not propagate all of the config correctly.
We're building a go/build.Package to get the cgo command line
invocation and it might be incorrectly configured if the user's
configuration is different from go/build.Default.

Change-Id: I0de3c65cb723587e1551edeb77219614ba9650bf
Reviewed-on: https://go-review.googlesource.com/127816
Run-TryBot: Michael Matloob <matloob@golang.org>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Ian Cottrell <iancottrell@google.com>

go/internal/cgo/cgo.go

@@ -2,7 +2,7 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-package loader
+package cgo
 
 // This file handles cgo preprocessing of files containing `import "C"`.
 //
@@ -66,10 +66,10 @@ import (
 	"strings"
 )
 
-// processCgoFiles invokes the cgo preprocessor on bp.CgoFiles, parses
+// ProcessCgoFiles invokes the cgo preprocessor on bp.CgoFiles, parses
 // the output and returns the resulting ASTs.
 //
-func processCgoFiles(bp *build.Package, fset *token.FileSet, DisplayPath func(path string) string, mode parser.Mode) ([]*ast.File, error) {
+func ProcessCgoFiles(bp *build.Package, fset *token.FileSet, DisplayPath func(path string) string, mode parser.Mode) ([]*ast.File, error) {
 	tmpdir, err := ioutil.TempDir("", strings.Replace(bp.ImportPath, "/", "_", -1)+"_C")
 	if err != nil {
 		return nil, err
@@ -81,7 +81,7 @@ func processCgoFiles(bp *build.Package, fset *token.FileSet, DisplayPath func(pa
 		pkgdir = DisplayPath(pkgdir)
 	}
 
-	cgoFiles, cgoDisplayFiles, err := runCgo(bp, pkgdir, tmpdir)
+	cgoFiles, cgoDisplayFiles, err := RunCgo(bp, pkgdir, tmpdir, false)
 	if err != nil {
 		return nil, err
 	}
@@ -104,7 +104,7 @@ func processCgoFiles(bp *build.Package, fset *token.FileSet, DisplayPath func(pa
 
 var cgoRe = regexp.MustCompile(`[/\\:]`)
 
-// runCgo invokes the cgo preprocessor on bp.CgoFiles and returns two
+// RunCgo invokes the cgo preprocessor on bp.CgoFiles and returns two
 // lists of files: the resulting processed files (in temporary
 // directory tmpdir) and the corresponding names of the unprocessed files.
 //
@@ -112,7 +112,7 @@ var cgoRe = regexp.MustCompile(`[/\\:]`)
 // $GOROOT/src/cmd/go/build.go, but these features are unsupported:
 // Objective C, CGOPKGPATH, CGO_FLAGS.
 //
-func runCgo(bp *build.Package, pkgdir, tmpdir string) (files, displayFiles []string, err error) {
+func RunCgo(bp *build.Package, pkgdir, tmpdir string, useabs bool) (files, displayFiles []string, err error) {
 	cgoCPPFLAGS, _, _, _ := cflags(bp, true)
 	_, cgoexeCFLAGS, _, _ := cflags(bp, false)
 
@@ -145,9 +145,17 @@ func runCgo(bp *build.Package, pkgdir, tmpdir string) (files, displayFiles []str
 		cgoflags = append(cgoflags, "-import_syscall=false")
 	}
 
+	var cgoFiles []string = bp.CgoFiles
+	if useabs {
+		cgoFiles = make([]string, len(bp.CgoFiles))
+		for i := range cgoFiles {
+			cgoFiles[i] = filepath.Join(pkgdir, bp.CgoFiles[i])
+		}
+	}
+
 	args := stringList(
 		"go", "tool", "cgo", "-objdir", tmpdir, cgoflags, "--",
-		cgoCPPFLAGS, cgoexeCFLAGS, bp.CgoFiles,
+		cgoCPPFLAGS, cgoexeCFLAGS, cgoFiles,
 	)
 	if false {
 		log.Printf("Running cgo for package %q: %s (dir=%s)", bp.ImportPath, args, pkgdir)

go/internal/cgo/cgo_pkgconfig.go

@@ -2,7 +2,7 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-package loader
+package cgo
 
 import (
 	"errors"

go/loader/loader.go

@@ -22,6 +22,7 @@ import (
 	"time"
 
 	"golang.org/x/tools/go/ast/astutil"
+	"golang.org/x/tools/go/internal/cgo"
 )
 
 var ignoreVendor build.ImportMode
@@ -754,7 +755,7 @@ func (conf *Config) parsePackageFiles(bp *build.Package, which rune) ([]*ast.Fil
 
 	// Preprocess CgoFiles and parse the outputs (sequentially).
 	if which == 'g' && bp.CgoFiles != nil {
-		cgofiles, err := processCgoFiles(bp, conf.fset(), conf.DisplayPath, conf.ParserMode)
+		cgofiles, err := cgo.ProcessCgoFiles(bp, conf.fset(), conf.DisplayPath, conf.ParserMode)
 		if err != nil {
 			errs = append(errs, err)
 		} else {

go/packages/golist/golist_fallback.go

@@ -9,8 +9,14 @@ import (
 	"encoding/json"
 	"fmt"
 
+	"go/build"
+	"golang.org/x/tools/go/internal/cgo"
 	"golang.org/x/tools/go/packages/raw"
 	"golang.org/x/tools/imports"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
 )
 
 // TODO(matloob): Delete this file once Go 1.12 is released.
@@ -22,14 +28,14 @@ import (
 // This support will be removed once Go 1.12 is released
 // in Q1 2019.
 
-// TODO(matloob): Support cgo. Copy code from the loader that runs cgo.
-
 func golistPackagesFallback(ctx context.Context, cfg *raw.Config, words ...string) ([]string, []*raw.Package, error) {
 	original, deps, err := getDeps(ctx, cfg, words...)
 	if err != nil {
 		return nil, nil, err
 	}
 
+	var tmpdir string // used for generated cgo files
+
 	var result []*raw.Package
 	var roots []string
 	addPackage := func(p *jsonPackage) {
@@ -64,11 +70,35 @@ func golistPackagesFallback(ctx context.Context, cfg *raw.Config, words ...strin
 		if isRoot {
 			roots = append(roots, id)
 		}
+		compiledGoFiles := absJoin(p.Dir, p.GoFiles)
+		// Use a function to simplify control flow. It's just a bunch of gotos.
+		processCgo := func() bool {
+			// Suppress any cgo errors. Any relevant errors will show up in typechecking.
+			// TODO(matloob): Skip running cgo if Mode < LoadTypes.
+			if tmpdir == "" {
+				if tmpdir, err = ioutil.TempDir("", "gopackages"); err != nil {
+					return false
+				}
+			}
+			outdir := filepath.Join(tmpdir, strings.Replace(p.ImportPath, "/", "_", -1))
+			if err := os.Mkdir(outdir, 0755); err != nil {
+				return false
+			}
+			files, _, err := runCgo(p.Dir, outdir, cfg.Env)
+			if err != nil {
+				return false
+			}
+			compiledGoFiles = append(compiledGoFiles, files...)
+			return true
+		}
+		if len(p.CgoFiles) == 0 || !processCgo() {
+			compiledGoFiles = append(compiledGoFiles, absJoin(p.Dir, p.CgoFiles)...) // Punt to typechecker.
+		}
 		result = append(result, &raw.Package{
 			ID:              id,
 			Name:            p.Name,
 			GoFiles:         absJoin(p.Dir, p.GoFiles, p.CgoFiles),
-			CompiledGoFiles: absJoin(p.Dir, p.GoFiles, p.CgoFiles), // TODO(matloob): Use cgo-processed Go files instead of p.GoFiles
+			CompiledGoFiles: compiledGoFiles, // TODO(matloob): Use cgo-processed Go files instead of p.GoFiles
 			OtherFiles:      absJoin(p.Dir, p.SFiles, p.CFiles),
 			PkgPath:         pkgpath,
 			Imports:         importMap(p.Imports),
@@ -82,8 +112,8 @@ func golistPackagesFallback(ctx context.Context, cfg *raw.Config, words ...strin
 				result = append(result, &raw.Package{
 					ID:              testID,
 					Name:            p.Name,
-					GoFiles:         absJoin(p.Dir, p.GoFiles, p.TestGoFiles, p.CgoFiles),
-					CompiledGoFiles: absJoin(p.Dir, p.GoFiles, p.TestGoFiles, p.CgoFiles), // TODO(matloob): Use cgo-processed Go files instead of p.GoFiles
+					GoFiles:         absJoin(p.Dir, p.GoFiles, p.CgoFiles, p.TestGoFiles),
+					CompiledGoFiles: append(compiledGoFiles, absJoin(p.Dir, p.TestGoFiles)...), // TODO(matloob): Can there be cgo files in the tests?
 					OtherFiles:      absJoin(p.Dir, p.SFiles, p.CFiles),
 					PkgPath:         pkgpath,
 					Imports:         importMap(append(p.Imports, p.TestImports...)),
@@ -207,3 +237,27 @@ func golistargs_fallback(cfg *raw.Config, words []string) []string {
 	fullargs = append(fullargs, words...)
 	return fullargs
 }
+
+func runCgo(pkgdir, tmpdir string, env []string) (files, displayfiles []string, err error) {
+	// Use go/build to open cgo files and determine the cgo flags, etc, from them.
+	// This is tricky so it's best to avoid reimplementing as much as we can, and
+	// we plan to delete this support once Go 1.12 is released anyways.
+	// TODO(matloob): This isn't completely correct because we're using the Default
+	// context. Perhaps we should more accurately fill in the context.
+	bp, err := build.ImportDir(pkgdir, build.ImportMode(0))
+	if err != nil {
+		return nil, nil, err
+	}
+	for _, ev := range env {
+		if v := strings.TrimPrefix(ev, "CGO_CPPFLAGS"); v != ev {
+			bp.CgoCPPFLAGS = append(bp.CgoCPPFLAGS, strings.Fields(v)...)
+		} else if v := strings.TrimPrefix(ev, "CGO_CFLAGS"); v != ev {
+			bp.CgoCFLAGS = append(bp.CgoCFLAGS, strings.Fields(v)...)
+		} else if v := strings.TrimPrefix(ev, "CGO_CXXFLAGS"); v != ev {
+			bp.CgoCXXFLAGS = append(bp.CgoCXXFLAGS, strings.Fields(v)...)
+		} else if v := strings.TrimPrefix(ev, "CGO_LDFLAGS"); v != ev {
+			bp.CgoLDFLAGS = append(bp.CgoLDFLAGS, strings.Fields(v)...)
+		}
+	}
+	return cgo.RunCgo(bp, pkgdir, tmpdir, true)
+}

go/packages/stdlib_test.go

@@ -2,8 +2,6 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build go1.11
-
 package packages_test
 
 import (