go/build: handle cgo, //build comments

R=adg
CC=golang-dev
https://golang.org/cl/5018044

src/cmd/goinstall/main.go

@@ -13,7 +13,7 @@ import (
 	"go/token"
 	"io/ioutil"
 	"os"
-	"path/filepath"
+	"path/filepath" // use for file system paths
 	"regexp"
 	"runtime"
 	"strings"
@@ -190,7 +190,7 @@ func install(pkg, parent string) {
 	}()
 
 	// Don't allow trailing '/'
-	if _, f := filepath.Split(pkg); f == "" {
+	if strings.HasSuffix(pkg, "/") {
 		errorf("%s should not have trailing '/'\n", pkg)
 		return
 	}
@@ -225,16 +225,17 @@ func install(pkg, parent string) {
 		terrorf(tree, "%s: %v\n", pkg, err)
 		return
 	}
-	dir := filepath.Join(tree.SrcDir(), pkg)
+	dir := filepath.Join(tree.SrcDir(), filepath.FromSlash(pkg))
 
 	// Install prerequisites.
-	dirInfo, err := build.ScanDir(dir, parent == "")
+	dirInfo, err := build.ScanDir(dir)
 	if err != nil {
 		terrorf(tree, "%s: %v\n", pkg, err)
 		return
 	}
-	if len(dirInfo.GoFiles)+len(dirInfo.CgoFiles) == 0 {
-		terrorf(tree, "%s: package has no files\n", pkg)
+	// We reserve package main to identify commands.
+	if parent != "" && dirInfo.Package == "main" {
+		terrorf(tree, "%s: found only package main in %s; cannot import", pkg, dir)
 		return
 	}
 	for _, p := range dirInfo.Imports {

src/cmd/goinstall/make.go

@@ -10,7 +10,7 @@ import (
 	"bytes"
 	"go/build"
 	"os"
-	"path/filepath"
+	"path" // use for import paths
 	"strings"
 	"template"
 )
@@ -44,10 +44,10 @@ func makeMakefile(dir, pkg string, tree *build.Tree, isCmd bool) ([]byte, os.Err
 	targDir := tree.PkgDir()
 	if isCmd {
 		// use the last part of the package name for targ
-		_, targ = filepath.Split(pkg)
+		_, targ = path.Split(pkg)
 		targDir = tree.BinDir()
 	}
-	dirInfo, err := build.ScanDir(dir, isCmd)
+	dirInfo, err := build.ScanDir(dir)
 	if err != nil {
 		return nil, err
 	}

src/cmd/gotest/gotest.go

@@ -165,7 +165,7 @@ func setEnvironment() {
 func getTestFileNames() {
 	names := fileNames
 	if len(names) == 0 {
-		info, err := build.ScanDir(".", true)
+		info, err := build.ScanDir(".")
 		if err != nil {
 			Fatalf("scanning directory: %v", err)
 		}

src/pkg/go/build/build_test.go

@@ -27,7 +27,7 @@ var buildPkgs = []struct {
 		&DirInfo{
 			GoFiles:      []string{"pkgtest.go"},
 			SFiles:       []string{"sqrt_" + runtime.GOARCH + ".s"},
-			PkgName:      "pkgtest",
+			Package:      "pkgtest",
 			Imports:      []string{"os"},
 			TestImports:  []string{"fmt", "pkgtest"},
 			TestGoFiles:  sortstr([]string{"sqrt_test.go", "sqrt_" + runtime.GOARCH + "_test.go"}),
@@ -38,17 +38,19 @@ var buildPkgs = []struct {
 		"go/build/cmdtest",
 		&DirInfo{
 			GoFiles: []string{"main.go"},
-			PkgName: "main",
+			Package: "main",
 			Imports: []string{"go/build/pkgtest"},
 		},
 	},
 	{
 		"go/build/cgotest",
 		&DirInfo{
-			CgoFiles: []string{"cgotest.go"},
-			CFiles:   []string{"cgotest.c"},
-			Imports:  []string{"C", "unsafe"},
-			PkgName:  "cgotest",
+			CgoFiles:     []string{"cgotest.go"},
+			CFiles:       []string{"cgotest.c"},
+			Imports:      []string{"C", "unsafe"},
+			Package:      "cgotest",
+			CgoLDFLAGS:   []string{"-lregexp"},
+			CgoPkgConfig: []string{"cairo", "moscow"},
 		},
 	},
 }
@@ -56,11 +58,11 @@ var buildPkgs = []struct {
 const cmdtestOutput = "3"
 
 func TestBuild(t *testing.T) {
+	var ctxt = Context{GOOS: "darwin", GOARCH: "amd64"}
 	for _, tt := range buildPkgs {
 		tree := Path[0] // Goroot
 		dir := filepath.Join(tree.SrcDir(), tt.dir)
-
-		info, err := ScanDir(dir, true)
+		info, err := ctxt.ScanDir(dir)
 		if err != nil {
 			t.Errorf("ScanDir(%#q): %v", tt.dir, err)
 			continue
@@ -70,6 +72,13 @@ func TestBuild(t *testing.T) {
 			continue
 		}
 
+		if tt.dir == "go/build/cgotest" {
+			// Don't actually run cgo.
+			// Among other things our test depends
+			// on pkg-config, which is not present on all systems.
+			continue
+		}
+
 		s, err := Build(tree, tt.dir, info)
 		if err != nil {
 			t.Errorf("Build(%#q): %v", tt.dir, err)

src/pkg/go/build/cgotest/cgotest.go

@@ -6,6 +6,9 @@ package cgotest
 
 /*
 char* greeting = "hello, world";
+#cgo darwin/amd64 LDFLAGS: -lregexp
+#cgo linux CFLAGS: -m32
+#cgo pkg-config: cairo moscow
 */
 // #include "cgotest.h"
 import "C"

src/pkg/go/build/dir.go

@@ -5,16 +5,22 @@
 package build
 
 import (
+	"bytes"
+	"fmt"
+	"go/ast"
+	"go/doc"
 	"go/parser"
 	"go/token"
 	"io/ioutil"
 	"log"
 	"os"
+	"path"
 	"path/filepath"
+	"runtime"
 	"sort"
 	"strconv"
 	"strings"
-	"runtime"
+	"unicode"
 )
 
 // A Context specifies the supporting context for a build.
@@ -22,14 +28,55 @@ type Context struct {
 	GOARCH string // target architecture
 	GOOS   string // target operating system
 	// TODO(rsc,adg): GOPATH
+
+	// By default, ScanDir uses the operating system's
+	// file system calls to read directories and files.
+	// Callers can override those calls to provide other
+	// ways to read data by setting ReadDir and ReadFile.
+	// ScanDir does not make any assumptions about the
+	// format of the strings dir and file: they can be
+	// slash-separated, backslash-separated, even URLs.
+
+	// ReadDir returns a slice of *os.FileInfo, sorted by Name,
+	// describing the content of the named directory.
+	// The dir argument is the argument to ScanDir.
+	// If ReadDir is nil, ScanDir uses io.ReadDir.
+	ReadDir func(dir string) (fi []*os.FileInfo, err os.Error)
+
+	// ReadFile returns the content of the file named file
+	// in the directory named dir.  The dir argument is the
+	// argument to ScanDir, and the file argument is the
+	// Name field from an *os.FileInfo returned by ReadDir.
+	// The returned path is the full name of the file, to be
+	// used in error messages.
+	//
+	// If ReadFile is nil, ScanDir uses filepath.Join(dir, file)
+	// as the path and ioutil.ReadFile to read the data.
+	ReadFile func(dir, file string) (path string, content []byte, err os.Error)
+}
+
+func (ctxt *Context) readDir(dir string) ([]*os.FileInfo, os.Error) {
+	if f := ctxt.ReadDir; f != nil {
+		return f(dir)
+	}
+	return ioutil.ReadDir(dir)
+}
+
+func (ctxt *Context) readFile(dir, file string) (string, []byte, os.Error) {
+	if f := ctxt.ReadFile; f != nil {
+		return f(dir, file)
+	}
+	p := filepath.Join(dir, file)
+	content, err := ioutil.ReadFile(p)
+	return p, content, err
 }
 
 // The DefaultContext is the default Context for builds.
 // It uses the GOARCH and GOOS environment variables
 // if set, or else the compiled code's GOARCH and GOOS.
 var DefaultContext = Context{
-	envOr("GOARCH", runtime.GOARCH),
-	envOr("GOOS", runtime.GOOS),
+	GOARCH: envOr("GOARCH", runtime.GOARCH),
+	GOOS:   envOr("GOOS", runtime.GOOS),
 }
 
 func envOr(name, def string) string {
@@ -41,36 +88,48 @@ func envOr(name, def string) string {
 }
 
 type DirInfo struct {
-	GoFiles      []string // .go files in dir (excluding CgoFiles)
-	CgoFiles     []string // .go files that import "C"
-	CFiles       []string // .c files in dir
-	SFiles       []string // .s files in dir
-	Imports      []string // All packages imported by GoFiles
-	TestImports  []string // All packages imported by (X)TestGoFiles
-	PkgName      string   // Name of package in dir
+	Package        string            // Name of package in dir
+	PackageComment *ast.CommentGroup // Package comments from GoFiles
+	ImportPath     string            // Import path of package in dir
+	Imports        []string          // All packages imported by GoFiles
+
+	// Source files
+	GoFiles  []string // .go files in dir (excluding CgoFiles)
+	CFiles   []string // .c files in dir
+	SFiles   []string // .s files in dir
+	CgoFiles []string // .go files that import "C"
+
+	// Cgo directives
+	CgoPkgConfig []string // Cgo pkg-config directives
+	CgoCFLAGS    []string // Cgo CFLAGS directives
+	CgoLDFLAGS   []string // Cgo LDFLAGS directives
+
+	// Test information
 	TestGoFiles  []string // _test.go files in package
 	XTestGoFiles []string // _test.go files outside package
+	TestImports  []string // All packages imported by (X)TestGoFiles
 }
 
 func (d *DirInfo) IsCommand() bool {
-	return d.PkgName == "main"
+	// TODO(rsc): This is at least a little bogus.
+	return d.Package == "main"
 }
 
 // ScanDir calls DefaultContext.ScanDir.
-func ScanDir(dir string, allowMain bool) (info *DirInfo, err os.Error) {
-	return DefaultContext.ScanDir(dir, allowMain)
+func ScanDir(dir string) (info *DirInfo, err os.Error) {
+	return DefaultContext.ScanDir(dir)
 }
 
 // ScanDir returns a structure with details about the Go content found
 // in the given directory. The file lists exclude:
 //
-//	- files in package main (unless allowMain is true)
+//	- files in package main (unless no other package is found)
 //	- files in package documentation
 //	- files ending in _test.go
-// 	- files starting with _ or .
+//	- files starting with _ or .
 //
-func (ctxt *Context) ScanDir(dir string, allowMain bool) (info *DirInfo, err os.Error) {
-	dirs, err := ioutil.ReadDir(dir)
+func (ctxt *Context) ScanDir(dir string) (info *DirInfo, err os.Error) {
+	dirs, err := ctxt.readDir(dir)
 	if err != nil {
 		return nil, err
 	}
@@ -80,16 +139,19 @@ func (ctxt *Context) ScanDir(dir string, allowMain bool) (info *DirInfo, err os.
 	testImported := make(map[string]bool)
 	fset := token.NewFileSet()
 	for _, d := range dirs {
+		if !d.IsRegular() {
+			continue
+		}
 		if strings.HasPrefix(d.Name, "_") ||
 			strings.HasPrefix(d.Name, ".") {
 			continue
 		}
-		if !ctxt.goodOSArch(d.Name) {
+		if !ctxt.goodOSArchFile(d.Name) {
 			continue
 		}
 
 		isTest := false
-		switch filepath.Ext(d.Name) {
+		switch path.Ext(d.Name) {
 		case ".go":
 			isTest = strings.HasSuffix(d.Name, "_test.go")
 		case ".c":
@@ -102,13 +164,22 @@ func (ctxt *Context) ScanDir(dir string, allowMain bool) (info *DirInfo, err os.
 			continue
 		}
 
-		filename := filepath.Join(dir, d.Name)
-		pf, err := parser.ParseFile(fset, filename, nil, parser.ImportsOnly)
+		filename, data, err := ctxt.readFile(dir, d.Name)
 		if err != nil {
 			return nil, err
 		}
+		pf, err := parser.ParseFile(fset, filename, data, parser.ImportsOnly|parser.ParseComments)
+		if err != nil {
+			return nil, err
+		}
+
+		// Skip if the //build comments don't match.
+		if !ctxt.shouldBuild(pf) {
+			continue
+		}
+
 		pkg := string(pf.Name.Name)
-		if pkg == "main" && !allowMain {
+		if pkg == "main" && di.Package != "" && di.Package != "main" {
 			continue
 		}
 		if pkg == "documentation" {
@@ -117,35 +188,62 @@ func (ctxt *Context) ScanDir(dir string, allowMain bool) (info *DirInfo, err os.
 		if isTest && strings.HasSuffix(pkg, "_test") {
 			pkg = pkg[:len(pkg)-len("_test")]
 		}
-		if di.PkgName == "" {
-			di.PkgName = pkg
-		} else if di.PkgName != pkg {
-			// Only if all files in the directory are in package main
-			// do we return PkgName=="main".
-			// A mix of main and another package reverts
-			// to the original (allowMain=false) behaviour.
-			if pkg == "main" || di.PkgName == "main" {
-				return ScanDir(dir, false)
-			}
-			return nil, os.NewError("multiple package names in " + dir)
+
+		if pkg != di.Package && di.Package == "main" {
+			// Found non-main package but was recording
+			// information about package main.  Reset.
+			di = DirInfo{}
 		}
-		isCgo := false
-		for _, spec := range pf.Imports {
-			quoted := string(spec.Path.Value)
-			path, err := strconv.Unquote(quoted)
-			if err != nil {
-				log.Panicf("%s: parser returned invalid quoted string: <%s>", filename, quoted)
-			}
-			if isTest {
-				testImported[path] = true
+		if di.Package == "" {
+			di.Package = pkg
+		} else if pkg != di.Package {
+			return nil, fmt.Errorf("%s: found packages %s and %s", dir, pkg, di.Package)
+		}
+		if pf.Doc != nil {
+			if di.PackageComment != nil {
+				di.PackageComment.List = append(di.PackageComment.List, pf.Doc.List...)
 			} else {
-				imported[path] = true
+				di.PackageComment = pf.Doc
 			}
-			if path == "C" {
-				if isTest {
-					return nil, os.NewError("use of cgo in test " + filename)
+		}
+
+		// Record imports and information about cgo.
+		isCgo := false
+		for _, decl := range pf.Decls {
+			d, ok := decl.(*ast.GenDecl)
+			if !ok {
+				continue
+			}
+			for _, dspec := range d.Specs {
+				spec, ok := dspec.(*ast.ImportSpec)
+				if !ok {
+					continue
+				}
+				quoted := string(spec.Path.Value)
+				path, err := strconv.Unquote(quoted)
+				if err != nil {
+					log.Panicf("%s: parser returned invalid quoted string: <%s>", filename, quoted)
+				}
+				if isTest {
+					testImported[path] = true
+				} else {
+					imported[path] = true
+				}
+				if path == "C" {
+					if isTest {
+						return nil, fmt.Errorf("%s: use of cgo in test not supported", filename)
+					}
+					cg := spec.Doc
+					if cg == nil && len(d.Specs) == 1 {
+						cg = d.Doc
+					}
+					if cg != nil {
+						if err := ctxt.saveCgo(filename, &di, cg); err != nil {
+							return nil, err
+						}
+					}
+					isCgo = true
 				}
-				isCgo = true
 			}
 		}
 		if isCgo {
@@ -160,6 +258,9 @@ func (ctxt *Context) ScanDir(dir string, allowMain bool) (info *DirInfo, err os.
 			di.GoFiles = append(di.GoFiles, d.Name)
 		}
 	}
+	if di.Package == "" {
+		return nil, fmt.Errorf("%s: no Go source files", dir)
+	}
 	di.Imports = make([]string, len(imported))
 	i := 0
 	for p := range imported {
@@ -172,13 +273,208 @@ func (ctxt *Context) ScanDir(dir string, allowMain bool) (info *DirInfo, err os.
 		di.TestImports[i] = p
 		i++
 	}
-	// File name lists are sorted because ioutil.ReadDir sorts.
+	// File name lists are sorted because ReadDir sorts.
 	sort.Strings(di.Imports)
 	sort.Strings(di.TestImports)
 	return &di, nil
 }
 
-// goodOSArch returns false if the name contains a $GOOS or $GOARCH
+// okayBuild reports whether it is okay to build this Go file,
+// based on the //build comments leading up to the package clause.
+//
+// The file is accepted only if each such line lists something
+// matching the file.  For example:
+//
+//	//build windows linux
+//
+// marks the file as applicable only on Windows and Linux.
+func (ctxt *Context) shouldBuild(pf *ast.File) bool {
+	for _, com := range pf.Comments {
+		if com.Pos() >= pf.Package {
+			break
+		}
+		for _, c := range com.List {
+			if strings.HasPrefix(c.Text, "//build") {
+				f := strings.Fields(c.Text)
+				if f[0] == "//build" {
+					ok := false
+					for _, tok := range f[1:] {
+						if ctxt.matchOSArch(tok) {
+							ok = true
+							break
+						}
+					}
+					if !ok {
+						return false // this one doesn't match
+					}
+				}
+			}
+		}
+	}
+	return true // everything matches
+}
+
+// saveCgo saves the information from the #cgo lines in the import "C" comment.
+// These lines set CFLAGS and LDFLAGS and pkg-config directives that affect
+// the way cgo's C code is built.
+//
+// TODO(rsc): This duplicates code in cgo.
+// Once the dust settles, remove this code from cgo.
+func (ctxt *Context) saveCgo(filename string, di *DirInfo, cg *ast.CommentGroup) os.Error {
+	text := doc.CommentText(cg)
+	for _, line := range strings.Split(text, "\n") {
+		orig := line
+
+		// Line is
+		//	#cgo [GOOS/GOARCH...] LDFLAGS: stuff
+		//
+		line = strings.TrimSpace(line)
+		if len(line) < 5 || line[:4] != "#cgo" || (line[4] != ' ' && line[4] != '\t') {
+			continue
+		}
+
+		// Split at colon.
+		line = strings.TrimSpace(line[4:])
+		i := strings.Index(line, ":")
+		if i < 0 {
+			return fmt.Errorf("%s: invalid #cgo line: %s", filename, orig)
+		}
+		line, argstr := line[:i], line[i+1:]
+
+		// Parse GOOS/GOARCH stuff.
+		f := strings.Fields(line)
+		if len(f) < 1 {
+			return fmt.Errorf("%s: invalid #cgo line: %s", filename, orig)
+		}
+
+		cond, verb := f[:len(f)-1], f[len(f)-1]
+		if len(cond) > 0 {
+			ok := false
+			for _, c := range cond {
+				if ctxt.matchOSArch(c) {
+					ok = true
+					break
+				}
+			}
+			if !ok {
+				continue
+			}
+		}
+
+		args, err := splitQuoted(argstr)
+		if err != nil {
+			return fmt.Errorf("%s: invalid #cgo line: %s", filename, orig)
+		}
+		for _, arg := range args {
+			if !safeName(arg) {
+				return fmt.Errorf("%s: malformed #cgo argument: %s", filename, arg)
+			}
+		}
+
+		switch verb {
+		case "CFLAGS":
+			di.CgoCFLAGS = append(di.CgoCFLAGS, args...)
+		case "LDFLAGS":
+			di.CgoLDFLAGS = append(di.CgoLDFLAGS, args...)
+		case "pkg-config":
+			di.CgoPkgConfig = append(di.CgoPkgConfig, args...)
+		default:
+			return fmt.Errorf("%s: invalid #cgo verb: %s", filename, orig)
+		}
+	}
+	return nil
+}
+
+var safeBytes = []byte("+-.,/0123456789=ABCDEFGHIJKLMNOPQRSTUVWXYZ_abcdefghijklmnopqrstuvwxyz")
+
+func safeName(s string) bool {
+	if s == "" {
+		return false
+	}
+	for i := 0; i < len(s); i++ {
+		if c := s[i]; c < 0x80 && bytes.IndexByte(safeBytes, c) < 0 {
+			return false
+		}
+	}
+	return true
+}
+
+// splitQuoted splits the string s around each instance of one or more consecutive
+// white space characters while taking into account quotes and escaping, and
+// returns an array of substrings of s or an empty list if s contains only white space.
+// Single quotes and double quotes are recognized to prevent splitting within the
+// quoted region, and are removed from the resulting substrings. If a quote in s
+// isn't closed err will be set and r will have the unclosed argument as the
+// last element.  The backslash is used for escaping.
+//
+// For example, the following string:
+//
+//     a b:"c d" 'e''f'  "g\""
+//
+// Would be parsed as:
+//
+//     []string{"a", "b:c d", "ef", `g"`}
+//
+func splitQuoted(s string) (r []string, err os.Error) {
+	var args []string
+	arg := make([]int, len(s))
+	escaped := false
+	quoted := false
+	quote := 0
+	i := 0
+	for _, rune := range s {
+		switch {
+		case escaped:
+			escaped = false
+		case rune == '\\':
+			escaped = true
+			continue
+		case quote != 0:
+			if rune == quote {
+				quote = 0
+				continue
+			}
+		case rune == '"' || rune == '\'':
+			quoted = true
+			quote = rune
+			continue
+		case unicode.IsSpace(rune):
+			if quoted || i > 0 {
+				quoted = false
+				args = append(args, string(arg[:i]))
+				i = 0
+			}
+			continue
+		}
+		arg[i] = rune
+		i++
+	}
+	if quoted || i > 0 {
+		args = append(args, string(arg[:i]))
+	}
+	if quote != 0 {
+		err = os.NewError("unclosed quote")
+	} else if escaped {
+		err = os.NewError("unfinished escaping")
+	}
+	return args, err
+}
+
+// matchOSArch returns true if the name is one of:
+//
+//	$GOOS
+//	$GOARCH
+//	$GOOS/$GOARCH
+//
+func (ctxt *Context) matchOSArch(name string) bool {
+	if name == ctxt.GOOS || name == ctxt.GOARCH {
+		return true
+	}
+	i := strings.Index(name, "/")
+	return i >= 0 && name[:i] == ctxt.GOOS && name[i+1:] == ctxt.GOARCH
+}
+
+// goodOSArchFile returns false if the name contains a $GOOS or $GOARCH
 // suffix which does not match the current system.
 // The recognized name formats are:
 //
@@ -189,7 +485,7 @@ func (ctxt *Context) ScanDir(dir string, allowMain bool) (info *DirInfo, err os.
 //     name_$(GOARCH)_test.*
 //     name_$(GOOS)_$(GOARCH)_test.*
 //
-func (ctxt *Context) goodOSArch(name string) bool {
+func (ctxt *Context) goodOSArchFile(name string) bool {
 	if dot := strings.Index(name, "."); dot != -1 {
 		name = name[:dot]
 	}

src/pkg/go/build/syslist_test.go

@@ -55,8 +55,8 @@ var tests = []GoodFileTest{
 
 func TestGoodOSArch(t *testing.T) {
 	for _, test := range tests {
-		if DefaultContext.goodOSArch(test.name) != test.result {
-			t.Fatalf("goodOSArch(%q) != %v", test.name, test.result)
+		if DefaultContext.goodOSArchFile(test.name) != test.result {
+			t.Fatalf("goodOSArchFile(%q) != %v", test.name, test.result)
 		}
 	}
 }