go.tools/go/loader: enable cgo processing of files that import "C".

Files that import "C" are not valid Go source files and
require preprocessing.  Until now, the loader has simply
hard-coded CGO_ENABLED=0 (in effect) which causes go/build to
use build tags select pure Go implementations where they exist
(e.g. in $GOROOT).  Where they don't (e.g. arbitrary user
code) this leads to masses of spurious type errors.
(Reported by Guillaume Charmes, private correspondence.)

This change causes the loader to invoke the cgo preprocessor
on such files and to load the preprocessed files instead,
using the original names.  This means that the syntax offset
position information is garbage, although thanks to //line
directives, the line numbers at least should be good.

See comment in cgo.go for details.

This CL changes the loader's default behaviour and may make it slower.
CGO_ENABLED=0 enables the old behaviour.

Tested via stdlib_test, which now loads all standard packages
using cgo, and also exercises CGO_ENABLED=0 for "net" and "os/user".

LGTM=gri
R=gri, rsc
CC=golang-codereviews, guillaume.charmes
https://golang.org/cl/86140043

go/loader/cgo.go

@@ -0,0 +1,194 @@
+package loader
+
+// This file handles cgo preprocessing of files containing `import "C"`.
+//
+// DESIGN
+//
+// The approach taken is to run the cgo processor on the package's
+// CgoFiles and parse the output, faking the filenames of the
+// resulting ASTs so that the synthetic file containing the C types is
+// called "C" (e.g. "~/go/src/pkg/net/C") and the preprocessed files
+// have their original names (e.g. "~/go/src/pkg/net/cgo_unix.go"),
+// not the names of the actual temporary files.
+//
+// The advantage of this approach is its fidelity to 'go build'.  The
+// downside is that the token.Position.Offset for each AST node is
+// incorrect, being an offset within the temporary file.  Line numbers
+// should still be correct because of the //line comments.
+//
+// The logic of this file is mostly plundered from the 'go build'
+// tool, which also invokes the cgo preprocessor.
+//
+//
+// REJECTED ALTERNATIVE
+//
+// An alternative approach that we explored is to extend go/types'
+// Importer mechanism to provide the identity of the importing package
+// so that each time `import "C"` appears it resolves to a different
+// synthetic package containing just the objects needed in that case.
+// The loader would invoke cgo but parse only the cgo_types.go file
+// defining the package-level objects, discarding the other files
+// resulting from preprocessing.
+//
+// The benefit of this approach would have been that source-level
+// syntax information would correspond exactly to the original cgo
+// file, with no preprocessing involved, making source tools like
+// godoc, oracle, and eg happy.  However, the approach was rejected
+// due to the additional complexity it would impose on go/types.  (It
+// made for a beautiful demo, though.)
+//
+// cgo files, despite their *.go extension, are not legal Go source
+// files per the specification since they may refer to unexported
+// members of package "C" such as C.int.  Also, a function such as
+// C.getpwent has in effect two types, one matching its C type and one
+// which additionally returns (errno C.int).  The cgo preprocessor
+// uses name mangling to distinguish these two functions in the
+// processed code, but go/types would need to duplicate this logic in
+// its handling of function calls, analogous to the treatment of map
+// lookups in which y=m[k] and y,ok=m[k] are both legal.
+
+import (
+	"fmt"
+	"go/ast"
+	"go/build"
+	"go/parser"
+	"go/token"
+	"io/ioutil"
+	"log"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"regexp"
+	"strings"
+)
+
+// processCgoFiles invokes the cgo preprocessor on bp.CgoFiles, parses
+// the output and returns the resulting ASTs.
+//
+func processCgoFiles(bp *build.Package, fset *token.FileSet, mode parser.Mode) ([]*ast.File, error) {
+	tmpdir, err := ioutil.TempDir("", strings.Replace(bp.ImportPath, "/", "_", -1)+"_C")
+	if err != nil {
+		return nil, err
+	}
+	defer os.RemoveAll(tmpdir)
+
+	cgoFiles, cgoDisplayFiles, err := runCgo(bp, tmpdir)
+	if err != nil {
+		return nil, err
+	}
+	var files []*ast.File
+	for i := range cgoFiles {
+		rd, err := os.Open(cgoFiles[i])
+		if err != nil {
+			return nil, err
+		}
+		defer rd.Close()
+		display := filepath.Join(bp.Dir, cgoDisplayFiles[i])
+		f, err := parser.ParseFile(fset, display, rd, mode)
+		if err != nil {
+			return nil, err
+		}
+		files = append(files, f)
+	}
+	return files, nil
+}
+
+var cgoRe = regexp.MustCompile(`[/\\:]`)
+
+// runCgo invokes the cgo preprocessor on bp.CgoFiles and returns two
+// lists of files: the resulting processed files (in temporary
+// directory tmpdir) and the corresponding names of the unprocessed files.
+//
+// runCgo is adapted from (*builder).cgo in
+// $GOROOT/src/cmd/go/build.go, but these features are unsupported:
+// pkg-config, Objective C, CGOPKGPATH, CGO_FLAGS.
+//
+func runCgo(bp *build.Package, tmpdir string) (files, displayFiles []string, err error) {
+	cgoCPPFLAGS, _, _, _ := cflags(bp, true)
+	_, cgoexeCFLAGS, _, _ := cflags(bp, false)
+
+	if len(bp.CgoPkgConfig) > 0 {
+		return nil, nil, fmt.Errorf("cgo pkg-config not supported")
+	}
+
+	// Allows including _cgo_export.h from .[ch] files in the package.
+	cgoCPPFLAGS = append(cgoCPPFLAGS, "-I", tmpdir)
+
+	// _cgo_gotypes.go (displayed "C") contains the type definitions.
+	files = append(files, filepath.Join(tmpdir, "_cgo_gotypes.go"))
+	displayFiles = append(displayFiles, "C")
+	for _, fn := range bp.CgoFiles {
+		// "foo.cgo1.go" (displayed "foo.go") is the processed Go source.
+		f := cgoRe.ReplaceAllString(fn[:len(fn)-len("go")], "_")
+		files = append(files, filepath.Join(tmpdir, f+"cgo1.go"))
+		displayFiles = append(displayFiles, fn)
+	}
+
+	var cgoflags []string
+	if bp.Goroot && bp.ImportPath == "runtime/cgo" {
+		cgoflags = append(cgoflags, "-import_runtime_cgo=false")
+	}
+	if bp.Goroot && bp.ImportPath == "runtime/race" || bp.ImportPath == "runtime/cgo" {
+		cgoflags = append(cgoflags, "-import_syscall=false")
+	}
+
+	args := stringList(
+		"go", "tool", "cgo", "-objdir", tmpdir, cgoflags, "--",
+		cgoCPPFLAGS, cgoexeCFLAGS, bp.CgoFiles,
+	)
+	if false {
+		log.Printf("Running cgo for package %q: %s", bp.ImportPath, args)
+	}
+	cmd := exec.Command(args[0], args[1:]...)
+	cmd.Dir = bp.Dir
+	cmd.Stdout = os.Stderr
+	cmd.Stderr = os.Stderr
+	if err := cmd.Run(); err != nil {
+		return nil, nil, fmt.Errorf("cgo failed: %s: %s", args, err)
+	}
+
+	return files, displayFiles, nil
+}
+
+// -- unmodified from 'go build' ---------------------------------------
+
+// Return the flags to use when invoking the C or C++ compilers, or cgo.
+func cflags(p *build.Package, def bool) (cppflags, cflags, cxxflags, ldflags []string) {
+	var defaults string
+	if def {
+		defaults = "-g -O2"
+	}
+
+	cppflags = stringList(envList("CGO_CPPFLAGS", ""), p.CgoCPPFLAGS)
+	cflags = stringList(envList("CGO_CFLAGS", defaults), p.CgoCFLAGS)
+	cxxflags = stringList(envList("CGO_CXXFLAGS", defaults), p.CgoCXXFLAGS)
+	ldflags = stringList(envList("CGO_LDFLAGS", defaults), p.CgoLDFLAGS)
+	return
+}
+
+// envList returns the value of the given environment variable broken
+// into fields, using the default value when the variable is empty.
+func envList(key, def string) []string {
+	v := os.Getenv(key)
+	if v == "" {
+		v = def
+	}
+	return strings.Fields(v)
+}
+
+// stringList's arguments should be a sequence of string or []string values.
+// stringList flattens them into a single []string.
+func stringList(args ...interface{}) []string {
+	var x []string
+	for _, arg := range args {
+		switch arg := arg.(type) {
+		case []string:
+			x = append(x, arg...)
+		case string:
+			x = append(x, arg)
+		default:
+			panic("stringList: invalid argument")
+		}
+	}
+	return x
+}

go/loader/loader.go

@@ -186,6 +186,11 @@ type Config struct {
 
 	// If Build is non-nil, it is used to locate source packages.
 	// Otherwise &build.Default is used.
+	//
+	// By default, cgo is invoked to preprocess Go files that
+	// import the fake package "C".  This behaviour can be
+	// disabled by setting CGO_ENABLED=0 in the environment prior
+	// to startup, or by setting Build.CgoEnabled=false.
 	Build *build.Context
 
 	// If DisplayPath is non-nil, it is used to transform each
@@ -612,18 +617,15 @@ func (conf *Config) build() *build.Context {
 // then loads, parses and returns them.
 //
 // 'which' indicates which files to include:
-//    'g': include non-test *.go source files (GoFiles)
+//    'g': include non-test *.go source files (GoFiles + processed CgoFiles)
 //    't': include in-package *_test.go source files (TestGoFiles)
 //    'x': include external *_test.go source files. (XTestGoFiles)
 //
+// TODO(adonovan): don't fail just because some files contain parse errors.
+//
 func (conf *Config) parsePackageFiles(path string, which rune) ([]*ast.File, error) {
-	// Set the "!cgo" go/build tag, preferring (dummy) Go to
-	// native C implementations of net.cgoLookupHost et al.
-	ctxt := *conf.build() // copy
-	ctxt.CgoEnabled = false
-
 	// Import(srcDir="") disables local imports, e.g. import "./foo".
-	bp, err := ctxt.Import(path, "", 0)
+	bp, err := conf.build().Import(path, "", 0)
 	if _, ok := err.(*build.NoGoError); ok {
 		return nil, nil // empty directory
 	}
@@ -642,7 +644,22 @@ func (conf *Config) parsePackageFiles(path string, which rune) ([]*ast.File, err
 	default:
 		panic(which)
 	}
-	return parseFiles(conf.fset(), &ctxt, conf.DisplayPath, bp.Dir, filenames, conf.ParserMode)
+
+	files, err := parseFiles(conf.fset(), conf.build(), conf.DisplayPath, bp.Dir, filenames, conf.ParserMode)
+	if err != nil {
+		return nil, err
+	}
+
+	// Preprocess CgoFiles and parse the outputs (sequentially).
+	if which == 'g' && bp.CgoFiles != nil {
+		cgofiles, err := processCgoFiles(bp, conf.fset(), conf.ParserMode)
+		if err != nil {
+			return nil, err
+		}
+		files = append(files, cgofiles...)
+	}
+
+	return files, nil
 }
 
 // doImport imports the package denoted by path.

go/loader/stdlib_test.go

@@ -11,9 +11,12 @@ package loader_test
 // Run test with GOMAXPROCS=8.
 
 import (
+	"bytes"
 	"fmt"
 	"go/ast"
+	"go/build"
 	"go/token"
+	"io/ioutil"
 	"os"
 	"path/filepath"
 	"runtime"
@@ -133,3 +136,70 @@ func TestStdlib(t *testing.T) {
 	t.Log("Load/parse/typecheck: ", t1.Sub(t0))
 	t.Log("#MB:                  ", int64(memstats.Alloc-alloc)/1000000)
 }
+
+func TestCgoOption(t *testing.T) {
+	// Test that we can load cgo-using packages with
+	// CGO_ENABLED=[01], which causes go/build to select pure
+	// Go/native implementations, respectively, based on build
+	// tags.
+	//
+	// Each entry specifies a package-level object and the generic
+	// file expected to define it when cgo is disabled.
+	// When cgo is enabled, the exact file is not specified (since
+	// it varies by platform), but must differ from the generic one.
+	//
+	// The test also loads the actual file to verify that the
+	// object is indeed defined at that location.
+	for _, test := range []struct {
+		pkg, name, genericFile string
+	}{
+		{"net", "cgoLookupHost", "cgo_stub.go"},
+		{"os/user", "lookupId", "lookup_stubs.go"},
+	} {
+		ctxt := build.Default
+		for _, ctxt.CgoEnabled = range []bool{false, true} {
+			conf := loader.Config{Build: &ctxt}
+			conf.Import(test.pkg)
+			prog, err := conf.Load()
+			if err != nil {
+				t.Errorf("Load failed: %v", err)
+				continue
+			}
+			info := prog.Imported[test.pkg]
+			if info == nil {
+				t.Errorf("package %s not found", test.pkg)
+				continue
+			}
+			obj := info.Pkg.Scope().Lookup(test.name)
+			if obj == nil {
+				t.Errorf("no object %s.%s", test.pkg, test.name)
+				continue
+			}
+			posn := prog.Fset.Position(obj.Pos())
+			t.Logf("%s: %s (CgoEnabled=%t)", posn, obj, ctxt.CgoEnabled)
+
+			gotFile := filepath.Base(posn.Filename)
+			filesMatch := gotFile == test.genericFile
+
+			if ctxt.CgoEnabled && filesMatch {
+				t.Errorf("CGO_ENABLED=1: %s found in %s, want native file",
+					obj, gotFile)
+			} else if !ctxt.CgoEnabled && !filesMatch {
+				t.Errorf("CGO_ENABLED=0: %s found in %s, want %s",
+					obj, gotFile, test.genericFile)
+			}
+
+			// Load the file and check the object is declared at the right place.
+			b, err := ioutil.ReadFile(posn.Filename)
+			if err != nil {
+				t.Errorf("can't read %s: %s", posn.Filename, err)
+				continue
+			}
+			line := string(bytes.Split(b, []byte("\n"))[posn.Line-1])
+			ident := line[posn.Column-1:]
+			if !strings.HasPrefix(ident, test.name) {
+				t.Errorf("%s: %s not declared here (looking at %q)", posn, obj, ident)
+			}
+		}
+	}
+}