diff --git a/src/cmd/goinstall/main.go b/src/cmd/goinstall/main.go index 23b26e3832c..47826635723 100644 --- a/src/cmd/goinstall/main.go +++ b/src/cmd/goinstall/main.go @@ -13,7 +13,7 @@ import ( "go/token" "io/ioutil" "os" - "path/filepath" + "path/filepath" // use for file system paths "regexp" "runtime" "strings" @@ -190,7 +190,7 @@ func install(pkg, parent string) { }() // Don't allow trailing '/' - if _, f := filepath.Split(pkg); f == "" { + if strings.HasSuffix(pkg, "/") { errorf("%s should not have trailing '/'\n", pkg) return } @@ -225,16 +225,17 @@ func install(pkg, parent string) { terrorf(tree, "%s: %v\n", pkg, err) return } - dir := filepath.Join(tree.SrcDir(), pkg) + dir := filepath.Join(tree.SrcDir(), filepath.FromSlash(pkg)) // Install prerequisites. - dirInfo, err := build.ScanDir(dir, parent == "") + dirInfo, err := build.ScanDir(dir) if err != nil { terrorf(tree, "%s: %v\n", pkg, err) return } - if len(dirInfo.GoFiles)+len(dirInfo.CgoFiles) == 0 { - terrorf(tree, "%s: package has no files\n", pkg) + // We reserve package main to identify commands. + if parent != "" && dirInfo.Package == "main" { + terrorf(tree, "%s: found only package main in %s; cannot import", pkg, dir) return } for _, p := range dirInfo.Imports { diff --git a/src/cmd/goinstall/make.go b/src/cmd/goinstall/make.go index 38a70ddfd4a..7f41a913f8b 100644 --- a/src/cmd/goinstall/make.go +++ b/src/cmd/goinstall/make.go @@ -10,7 +10,7 @@ import ( "bytes" "go/build" "os" - "path/filepath" + "path" // use for import paths "strings" "template" ) @@ -44,10 +44,10 @@ func makeMakefile(dir, pkg string, tree *build.Tree, isCmd bool) ([]byte, os.Err targDir := tree.PkgDir() if isCmd { // use the last part of the package name for targ - _, targ = filepath.Split(pkg) + _, targ = path.Split(pkg) targDir = tree.BinDir() } - dirInfo, err := build.ScanDir(dir, isCmd) + dirInfo, err := build.ScanDir(dir) if err != nil { return nil, err } diff --git a/src/cmd/gotest/gotest.go b/src/cmd/gotest/gotest.go index 8e3a422329a..88c746c1b70 100644 --- a/src/cmd/gotest/gotest.go +++ b/src/cmd/gotest/gotest.go @@ -165,7 +165,7 @@ func setEnvironment() { func getTestFileNames() { names := fileNames if len(names) == 0 { - info, err := build.ScanDir(".", true) + info, err := build.ScanDir(".") if err != nil { Fatalf("scanning directory: %v", err) } diff --git a/src/pkg/go/build/build_test.go b/src/pkg/go/build/build_test.go index 592ebbd9eab..f53c7ef68da 100644 --- a/src/pkg/go/build/build_test.go +++ b/src/pkg/go/build/build_test.go @@ -27,7 +27,7 @@ var buildPkgs = []struct { &DirInfo{ GoFiles: []string{"pkgtest.go"}, SFiles: []string{"sqrt_" + runtime.GOARCH + ".s"}, - PkgName: "pkgtest", + Package: "pkgtest", Imports: []string{"os"}, TestImports: []string{"fmt", "pkgtest"}, TestGoFiles: sortstr([]string{"sqrt_test.go", "sqrt_" + runtime.GOARCH + "_test.go"}), @@ -38,17 +38,19 @@ var buildPkgs = []struct { "go/build/cmdtest", &DirInfo{ GoFiles: []string{"main.go"}, - PkgName: "main", + Package: "main", Imports: []string{"go/build/pkgtest"}, }, }, { "go/build/cgotest", &DirInfo{ - CgoFiles: []string{"cgotest.go"}, - CFiles: []string{"cgotest.c"}, - Imports: []string{"C", "unsafe"}, - PkgName: "cgotest", + CgoFiles: []string{"cgotest.go"}, + CFiles: []string{"cgotest.c"}, + Imports: []string{"C", "unsafe"}, + Package: "cgotest", + CgoLDFLAGS: []string{"-lregexp"}, + CgoPkgConfig: []string{"cairo", "moscow"}, }, }, } @@ -56,11 +58,11 @@ var buildPkgs = []struct { const cmdtestOutput = "3" func TestBuild(t *testing.T) { + var ctxt = Context{GOOS: "darwin", GOARCH: "amd64"} for _, tt := range buildPkgs { tree := Path[0] // Goroot dir := filepath.Join(tree.SrcDir(), tt.dir) - - info, err := ScanDir(dir, true) + info, err := ctxt.ScanDir(dir) if err != nil { t.Errorf("ScanDir(%#q): %v", tt.dir, err) continue @@ -70,6 +72,13 @@ func TestBuild(t *testing.T) { continue } + if tt.dir == "go/build/cgotest" { + // Don't actually run cgo. + // Among other things our test depends + // on pkg-config, which is not present on all systems. + continue + } + s, err := Build(tree, tt.dir, info) if err != nil { t.Errorf("Build(%#q): %v", tt.dir, err) diff --git a/src/pkg/go/build/cgotest/cgotest.go b/src/pkg/go/build/cgotest/cgotest.go index 93bbf06883f..f62a85dfc49 100644 --- a/src/pkg/go/build/cgotest/cgotest.go +++ b/src/pkg/go/build/cgotest/cgotest.go @@ -6,6 +6,9 @@ package cgotest /* char* greeting = "hello, world"; +#cgo darwin/amd64 LDFLAGS: -lregexp +#cgo linux CFLAGS: -m32 +#cgo pkg-config: cairo moscow */ // #include "cgotest.h" import "C" diff --git a/src/pkg/go/build/dir.go b/src/pkg/go/build/dir.go index fa4d9e913f3..e0d6f59ec84 100644 --- a/src/pkg/go/build/dir.go +++ b/src/pkg/go/build/dir.go @@ -5,16 +5,22 @@ package build import ( + "bytes" + "fmt" + "go/ast" + "go/doc" "go/parser" "go/token" "io/ioutil" "log" "os" + "path" "path/filepath" + "runtime" "sort" "strconv" "strings" - "runtime" + "unicode" ) // A Context specifies the supporting context for a build. @@ -22,14 +28,55 @@ type Context struct { GOARCH string // target architecture GOOS string // target operating system // TODO(rsc,adg): GOPATH + + // By default, ScanDir uses the operating system's + // file system calls to read directories and files. + // Callers can override those calls to provide other + // ways to read data by setting ReadDir and ReadFile. + // ScanDir does not make any assumptions about the + // format of the strings dir and file: they can be + // slash-separated, backslash-separated, even URLs. + + // ReadDir returns a slice of *os.FileInfo, sorted by Name, + // describing the content of the named directory. + // The dir argument is the argument to ScanDir. + // If ReadDir is nil, ScanDir uses io.ReadDir. + ReadDir func(dir string) (fi []*os.FileInfo, err os.Error) + + // ReadFile returns the content of the file named file + // in the directory named dir. The dir argument is the + // argument to ScanDir, and the file argument is the + // Name field from an *os.FileInfo returned by ReadDir. + // The returned path is the full name of the file, to be + // used in error messages. + // + // If ReadFile is nil, ScanDir uses filepath.Join(dir, file) + // as the path and ioutil.ReadFile to read the data. + ReadFile func(dir, file string) (path string, content []byte, err os.Error) +} + +func (ctxt *Context) readDir(dir string) ([]*os.FileInfo, os.Error) { + if f := ctxt.ReadDir; f != nil { + return f(dir) + } + return ioutil.ReadDir(dir) +} + +func (ctxt *Context) readFile(dir, file string) (string, []byte, os.Error) { + if f := ctxt.ReadFile; f != nil { + return f(dir, file) + } + p := filepath.Join(dir, file) + content, err := ioutil.ReadFile(p) + return p, content, err } // The DefaultContext is the default Context for builds. // It uses the GOARCH and GOOS environment variables // if set, or else the compiled code's GOARCH and GOOS. var DefaultContext = Context{ - envOr("GOARCH", runtime.GOARCH), - envOr("GOOS", runtime.GOOS), + GOARCH: envOr("GOARCH", runtime.GOARCH), + GOOS: envOr("GOOS", runtime.GOOS), } func envOr(name, def string) string { @@ -41,36 +88,48 @@ func envOr(name, def string) string { } type DirInfo struct { - GoFiles []string // .go files in dir (excluding CgoFiles) - CgoFiles []string // .go files that import "C" - CFiles []string // .c files in dir - SFiles []string // .s files in dir - Imports []string // All packages imported by GoFiles - TestImports []string // All packages imported by (X)TestGoFiles - PkgName string // Name of package in dir + Package string // Name of package in dir + PackageComment *ast.CommentGroup // Package comments from GoFiles + ImportPath string // Import path of package in dir + Imports []string // All packages imported by GoFiles + + // Source files + GoFiles []string // .go files in dir (excluding CgoFiles) + CFiles []string // .c files in dir + SFiles []string // .s files in dir + CgoFiles []string // .go files that import "C" + + // Cgo directives + CgoPkgConfig []string // Cgo pkg-config directives + CgoCFLAGS []string // Cgo CFLAGS directives + CgoLDFLAGS []string // Cgo LDFLAGS directives + + // Test information TestGoFiles []string // _test.go files in package XTestGoFiles []string // _test.go files outside package + TestImports []string // All packages imported by (X)TestGoFiles } func (d *DirInfo) IsCommand() bool { - return d.PkgName == "main" + // TODO(rsc): This is at least a little bogus. + return d.Package == "main" } // ScanDir calls DefaultContext.ScanDir. -func ScanDir(dir string, allowMain bool) (info *DirInfo, err os.Error) { - return DefaultContext.ScanDir(dir, allowMain) +func ScanDir(dir string) (info *DirInfo, err os.Error) { + return DefaultContext.ScanDir(dir) } // ScanDir returns a structure with details about the Go content found // in the given directory. The file lists exclude: // -// - files in package main (unless allowMain is true) +// - files in package main (unless no other package is found) // - files in package documentation // - files ending in _test.go -// - files starting with _ or . +// - files starting with _ or . // -func (ctxt *Context) ScanDir(dir string, allowMain bool) (info *DirInfo, err os.Error) { - dirs, err := ioutil.ReadDir(dir) +func (ctxt *Context) ScanDir(dir string) (info *DirInfo, err os.Error) { + dirs, err := ctxt.readDir(dir) if err != nil { return nil, err } @@ -80,16 +139,19 @@ func (ctxt *Context) ScanDir(dir string, allowMain bool) (info *DirInfo, err os. testImported := make(map[string]bool) fset := token.NewFileSet() for _, d := range dirs { + if !d.IsRegular() { + continue + } if strings.HasPrefix(d.Name, "_") || strings.HasPrefix(d.Name, ".") { continue } - if !ctxt.goodOSArch(d.Name) { + if !ctxt.goodOSArchFile(d.Name) { continue } isTest := false - switch filepath.Ext(d.Name) { + switch path.Ext(d.Name) { case ".go": isTest = strings.HasSuffix(d.Name, "_test.go") case ".c": @@ -102,13 +164,22 @@ func (ctxt *Context) ScanDir(dir string, allowMain bool) (info *DirInfo, err os. continue } - filename := filepath.Join(dir, d.Name) - pf, err := parser.ParseFile(fset, filename, nil, parser.ImportsOnly) + filename, data, err := ctxt.readFile(dir, d.Name) if err != nil { return nil, err } + pf, err := parser.ParseFile(fset, filename, data, parser.ImportsOnly|parser.ParseComments) + if err != nil { + return nil, err + } + + // Skip if the //build comments don't match. + if !ctxt.shouldBuild(pf) { + continue + } + pkg := string(pf.Name.Name) - if pkg == "main" && !allowMain { + if pkg == "main" && di.Package != "" && di.Package != "main" { continue } if pkg == "documentation" { @@ -117,35 +188,62 @@ func (ctxt *Context) ScanDir(dir string, allowMain bool) (info *DirInfo, err os. if isTest && strings.HasSuffix(pkg, "_test") { pkg = pkg[:len(pkg)-len("_test")] } - if di.PkgName == "" { - di.PkgName = pkg - } else if di.PkgName != pkg { - // Only if all files in the directory are in package main - // do we return PkgName=="main". - // A mix of main and another package reverts - // to the original (allowMain=false) behaviour. - if pkg == "main" || di.PkgName == "main" { - return ScanDir(dir, false) - } - return nil, os.NewError("multiple package names in " + dir) + + if pkg != di.Package && di.Package == "main" { + // Found non-main package but was recording + // information about package main. Reset. + di = DirInfo{} } - isCgo := false - for _, spec := range pf.Imports { - quoted := string(spec.Path.Value) - path, err := strconv.Unquote(quoted) - if err != nil { - log.Panicf("%s: parser returned invalid quoted string: <%s>", filename, quoted) - } - if isTest { - testImported[path] = true + if di.Package == "" { + di.Package = pkg + } else if pkg != di.Package { + return nil, fmt.Errorf("%s: found packages %s and %s", dir, pkg, di.Package) + } + if pf.Doc != nil { + if di.PackageComment != nil { + di.PackageComment.List = append(di.PackageComment.List, pf.Doc.List...) } else { - imported[path] = true + di.PackageComment = pf.Doc } - if path == "C" { - if isTest { - return nil, os.NewError("use of cgo in test " + filename) + } + + // Record imports and information about cgo. + isCgo := false + for _, decl := range pf.Decls { + d, ok := decl.(*ast.GenDecl) + if !ok { + continue + } + for _, dspec := range d.Specs { + spec, ok := dspec.(*ast.ImportSpec) + if !ok { + continue + } + quoted := string(spec.Path.Value) + path, err := strconv.Unquote(quoted) + if err != nil { + log.Panicf("%s: parser returned invalid quoted string: <%s>", filename, quoted) + } + if isTest { + testImported[path] = true + } else { + imported[path] = true + } + if path == "C" { + if isTest { + return nil, fmt.Errorf("%s: use of cgo in test not supported", filename) + } + cg := spec.Doc + if cg == nil && len(d.Specs) == 1 { + cg = d.Doc + } + if cg != nil { + if err := ctxt.saveCgo(filename, &di, cg); err != nil { + return nil, err + } + } + isCgo = true } - isCgo = true } } if isCgo { @@ -160,6 +258,9 @@ func (ctxt *Context) ScanDir(dir string, allowMain bool) (info *DirInfo, err os. di.GoFiles = append(di.GoFiles, d.Name) } } + if di.Package == "" { + return nil, fmt.Errorf("%s: no Go source files", dir) + } di.Imports = make([]string, len(imported)) i := 0 for p := range imported { @@ -172,13 +273,208 @@ func (ctxt *Context) ScanDir(dir string, allowMain bool) (info *DirInfo, err os. di.TestImports[i] = p i++ } - // File name lists are sorted because ioutil.ReadDir sorts. + // File name lists are sorted because ReadDir sorts. sort.Strings(di.Imports) sort.Strings(di.TestImports) return &di, nil } -// goodOSArch returns false if the name contains a $GOOS or $GOARCH +// okayBuild reports whether it is okay to build this Go file, +// based on the //build comments leading up to the package clause. +// +// The file is accepted only if each such line lists something +// matching the file. For example: +// +// //build windows linux +// +// marks the file as applicable only on Windows and Linux. +func (ctxt *Context) shouldBuild(pf *ast.File) bool { + for _, com := range pf.Comments { + if com.Pos() >= pf.Package { + break + } + for _, c := range com.List { + if strings.HasPrefix(c.Text, "//build") { + f := strings.Fields(c.Text) + if f[0] == "//build" { + ok := false + for _, tok := range f[1:] { + if ctxt.matchOSArch(tok) { + ok = true + break + } + } + if !ok { + return false // this one doesn't match + } + } + } + } + } + return true // everything matches +} + +// saveCgo saves the information from the #cgo lines in the import "C" comment. +// These lines set CFLAGS and LDFLAGS and pkg-config directives that affect +// the way cgo's C code is built. +// +// TODO(rsc): This duplicates code in cgo. +// Once the dust settles, remove this code from cgo. +func (ctxt *Context) saveCgo(filename string, di *DirInfo, cg *ast.CommentGroup) os.Error { + text := doc.CommentText(cg) + for _, line := range strings.Split(text, "\n") { + orig := line + + // Line is + // #cgo [GOOS/GOARCH...] LDFLAGS: stuff + // + line = strings.TrimSpace(line) + if len(line) < 5 || line[:4] != "#cgo" || (line[4] != ' ' && line[4] != '\t') { + continue + } + + // Split at colon. + line = strings.TrimSpace(line[4:]) + i := strings.Index(line, ":") + if i < 0 { + return fmt.Errorf("%s: invalid #cgo line: %s", filename, orig) + } + line, argstr := line[:i], line[i+1:] + + // Parse GOOS/GOARCH stuff. + f := strings.Fields(line) + if len(f) < 1 { + return fmt.Errorf("%s: invalid #cgo line: %s", filename, orig) + } + + cond, verb := f[:len(f)-1], f[len(f)-1] + if len(cond) > 0 { + ok := false + for _, c := range cond { + if ctxt.matchOSArch(c) { + ok = true + break + } + } + if !ok { + continue + } + } + + args, err := splitQuoted(argstr) + if err != nil { + return fmt.Errorf("%s: invalid #cgo line: %s", filename, orig) + } + for _, arg := range args { + if !safeName(arg) { + return fmt.Errorf("%s: malformed #cgo argument: %s", filename, arg) + } + } + + switch verb { + case "CFLAGS": + di.CgoCFLAGS = append(di.CgoCFLAGS, args...) + case "LDFLAGS": + di.CgoLDFLAGS = append(di.CgoLDFLAGS, args...) + case "pkg-config": + di.CgoPkgConfig = append(di.CgoPkgConfig, args...) + default: + return fmt.Errorf("%s: invalid #cgo verb: %s", filename, orig) + } + } + return nil +} + +var safeBytes = []byte("+-.,/0123456789=ABCDEFGHIJKLMNOPQRSTUVWXYZ_abcdefghijklmnopqrstuvwxyz") + +func safeName(s string) bool { + if s == "" { + return false + } + for i := 0; i < len(s); i++ { + if c := s[i]; c < 0x80 && bytes.IndexByte(safeBytes, c) < 0 { + return false + } + } + return true +} + +// splitQuoted splits the string s around each instance of one or more consecutive +// white space characters while taking into account quotes and escaping, and +// returns an array of substrings of s or an empty list if s contains only white space. +// Single quotes and double quotes are recognized to prevent splitting within the +// quoted region, and are removed from the resulting substrings. If a quote in s +// isn't closed err will be set and r will have the unclosed argument as the +// last element. The backslash is used for escaping. +// +// For example, the following string: +// +// a b:"c d" 'e''f' "g\"" +// +// Would be parsed as: +// +// []string{"a", "b:c d", "ef", `g"`} +// +func splitQuoted(s string) (r []string, err os.Error) { + var args []string + arg := make([]int, len(s)) + escaped := false + quoted := false + quote := 0 + i := 0 + for _, rune := range s { + switch { + case escaped: + escaped = false + case rune == '\\': + escaped = true + continue + case quote != 0: + if rune == quote { + quote = 0 + continue + } + case rune == '"' || rune == '\'': + quoted = true + quote = rune + continue + case unicode.IsSpace(rune): + if quoted || i > 0 { + quoted = false + args = append(args, string(arg[:i])) + i = 0 + } + continue + } + arg[i] = rune + i++ + } + if quoted || i > 0 { + args = append(args, string(arg[:i])) + } + if quote != 0 { + err = os.NewError("unclosed quote") + } else if escaped { + err = os.NewError("unfinished escaping") + } + return args, err +} + +// matchOSArch returns true if the name is one of: +// +// $GOOS +// $GOARCH +// $GOOS/$GOARCH +// +func (ctxt *Context) matchOSArch(name string) bool { + if name == ctxt.GOOS || name == ctxt.GOARCH { + return true + } + i := strings.Index(name, "/") + return i >= 0 && name[:i] == ctxt.GOOS && name[i+1:] == ctxt.GOARCH +} + +// goodOSArchFile returns false if the name contains a $GOOS or $GOARCH // suffix which does not match the current system. // The recognized name formats are: // @@ -189,7 +485,7 @@ func (ctxt *Context) ScanDir(dir string, allowMain bool) (info *DirInfo, err os. // name_$(GOARCH)_test.* // name_$(GOOS)_$(GOARCH)_test.* // -func (ctxt *Context) goodOSArch(name string) bool { +func (ctxt *Context) goodOSArchFile(name string) bool { if dot := strings.Index(name, "."); dot != -1 { name = name[:dot] } diff --git a/src/pkg/go/build/syslist_test.go b/src/pkg/go/build/syslist_test.go index 2e8b4c86564..d27630d758d 100644 --- a/src/pkg/go/build/syslist_test.go +++ b/src/pkg/go/build/syslist_test.go @@ -55,8 +55,8 @@ var tests = []GoodFileTest{ func TestGoodOSArch(t *testing.T) { for _, test := range tests { - if DefaultContext.goodOSArch(test.name) != test.result { - t.Fatalf("goodOSArch(%q) != %v", test.name, test.result) + if DefaultContext.goodOSArchFile(test.name) != test.result { + t.Fatalf("goodOSArchFile(%q) != %v", test.name, test.result) } } }