diff --git a/dashboard/builder/env.go b/dashboard/builder/env.go new file mode 100644 index 00000000000..0a0d558dc20 --- /dev/null +++ b/dashboard/builder/env.go @@ -0,0 +1,229 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "fmt" + "io" + "io/ioutil" + "os" + "path/filepath" + "runtime" + "strings" + + "code.google.com/p/go.tools/go/vcs" +) + +// These variables are copied from the gobuilder's environment +// to the envv of its subprocesses. +var extraEnv = []string{ + "GOARM", + + // For Unix derivatives. + "CC", + "PATH", + "TMPDIR", + "USER", + + // For Plan 9. + "objtype", + "cputype", + "path", +} + +// builderEnv represents the environment that a Builder will run tests in. +type builderEnv interface { + // setup sets up the builder environment and returns the directory to run the buildCmd in. + setup(repo *Repo, workpath, hash string, envv []string) (string, error) +} + +// goEnv represents the builderEnv for the main Go repo. +type goEnv struct { + goos, goarch string +} + +func (b *Builder) envv() []string { + if runtime.GOOS == "windows" { + return b.envvWindows() + } + + var e []string + if *buildTool == "go" { + e = []string{ + "GOOS=" + b.goos, + "GOHOSTOS=" + b.goos, + "GOARCH=" + b.goarch, + "GOHOSTARCH=" + b.goarch, + "GOROOT_FINAL=/usr/local/go", + } + } + + for _, k := range extraEnv { + if s, ok := getenvOk(k); ok { + e = append(e, k+"="+s) + } + } + return e +} + +func (b *Builder) envvWindows() []string { + var start map[string]string + if *buildTool == "go" { + start = map[string]string{ + "GOOS": b.goos, + "GOHOSTOS": b.goos, + "GOARCH": b.goarch, + "GOHOSTARCH": b.goarch, + "GOROOT_FINAL": `c:\go`, + "GOBUILDEXIT": "1", // exit all.bat with completion status. + } + } + + for _, name := range extraEnv { + if s, ok := getenvOk(name); ok { + start[name] = s + } + } + skip := map[string]bool{ + "GOBIN": true, + "GOROOT": true, + "INCLUDE": true, + "LIB": true, + } + var e []string + for name, v := range start { + e = append(e, name+"="+v) + skip[name] = true + } + for _, kv := range os.Environ() { + s := strings.SplitN(kv, "=", 2) + name := strings.ToUpper(s[0]) + switch { + case name == "": + // variables, like "=C:=C:\", just copy them + e = append(e, kv) + case !skip[name]: + e = append(e, kv) + skip[name] = true + } + } + return e +} + +// setup for a goEnv clones the main go repo to workpath/go at the provided hash +// and returns the path workpath/go/src, the location of all go build scripts. +func (env *goEnv) setup(repo *Repo, workpath, hash string, envv []string) (string, error) { + goworkpath := filepath.Join(workpath, "go") + if _, err := repo.Clone(goworkpath, hash); err != nil { + return "", fmt.Errorf("error cloning repository: %s", err) + } + return filepath.Join(goworkpath, "src"), nil +} + +// gccgoEnv represents the builderEnv for the gccgo compiler. +type gccgoEnv struct{} + +// setup for a gccgoEnv clones the gofrontend repo to workpath/go at the hash +// and clones the latest GCC branch to workpath/gcc. The gccgo sources are +// replaced with the updated sources in the gofrontend repo and gcc gets +// gets configured and built in workpath/gcc-objdir. The path to +// workpath/gcc-objdir is returned. +func (env *gccgoEnv) setup(repo *Repo, workpath, hash string, envv []string) (string, error) { + gofrontendpath := filepath.Join(workpath, "gofrontend") + gccpath := filepath.Join(workpath, "gcc") + + // get a handle to SVN vcs.Cmd for pulling down GCC. + svn := vcs.ByCmd("svn") + + if err := timeout(*cmdTimeout, func() error { + // pull down a working copy of GCC. + return svn.Create(gccpath, *gccPath) + }); err != nil { + return "", err + } + + // clone gofrontend repo at specified revision + if _, err := repo.Clone(gofrontendpath, hash); err != nil { + return "", err + } + + // remove gccpath/gcc/go/gofrontend and gcc/libgo + gccgopath := filepath.Join(gccpath, "gcc", "go", "gofrontend") + gcclibgopath := filepath.Join(gccpath, "libgo") + if err := os.RemoveAll(gccgopath); err != nil { + return "", err + } + if err := os.RemoveAll(gcclibgopath); err != nil { + return "", err + } + + // copy gofrontend and libgo to appropriate locations + if err := copyDir(filepath.Join(gofrontendpath, "go"), gccgopath); err != nil { + return "", fmt.Errorf("Failed to copy gofrontend/go to gcc/go/gofrontend: %s\n", err) + } + if err := copyDir(filepath.Join(gofrontendpath, "libgo"), gcclibgopath); err != nil { + return "", fmt.Errorf("Failed to copy gofrontend/libgo to gcc/libgo: %s\n", err) + } + + // make objdir to work in + gccobjdir := filepath.Join(workpath, "gcc-objdir") + if err := os.Mkdir(gccobjdir, mkdirPerm); err != nil { + return "", err + } + + // configure GCC with substituted gofrontend and libgo + gccConfigCmd := []string{filepath.Join(gccpath, "configure"), "--enable-languages=c,c++,go", "--disable-bootstrap"} + if _, err := runOutput(*cmdTimeout, envv, ioutil.Discard, gccobjdir, gccConfigCmd...); err != nil { + return "", fmt.Errorf("Failed to configure GCC: %s", err) + } + + // build gcc + if _, err := runOutput(*buildTimeout, envv, ioutil.Discard, gccobjdir, "make"); err != nil { + return "", fmt.Errorf("Failed to build GCC: %s", err) + } + + return gccobjdir, nil + +} + +// copyDir copies the src directory into the dst +func copyDir(src, dst string) error { + return filepath.Walk(src, func(path string, f os.FileInfo, err error) error { + dstPath := strings.Replace(path, src, dst, 1) + if f.IsDir() { + return os.Mkdir(dstPath, mkdirPerm) + } + + srcFile, err := os.Open(path) + if err != nil { + return err + } + defer srcFile.Close() + + dstFile, err := os.Create(dstPath) + if err != nil { + return err + } + + if _, err := io.Copy(dstFile, srcFile); err != nil { + return err + } + return dstFile.Close() + }) +} + +func getenvOk(k string) (v string, ok bool) { + v = os.Getenv(k) + if v != "" { + return v, true + } + keq := k + "=" + for _, kv := range os.Environ() { + if kv == keq { + return "", true + } + } + return "", false +} diff --git a/dashboard/builder/main.go b/dashboard/builder/main.go index f63db8fefff..b9386b87788 100644 --- a/dashboard/builder/main.go +++ b/dashboard/builder/main.go @@ -22,36 +22,21 @@ import ( ) const ( - codeProject = "go" - codePyScript = "misc/dashboard/googlecode_upload.py" - hgUrl = "code.google.com/p/go" - mkdirPerm = 0750 - waitInterval = 30 * time.Second // time to wait before checking for new revs - pkgBuildInterval = 24 * time.Hour // rebuild packages every 24 hours + codeProject = "go" + codePyScript = "misc/dashboard/googlecode_upload.py" + goImportPath = "code.google.com/p/go" + gofrontendImportPath = "code.google.com/p/gofrontend" + mkdirPerm = 0750 + waitInterval = 30 * time.Second // time to wait before checking for new revs + pkgBuildInterval = 24 * time.Hour // rebuild packages every 24 hours ) -// These variables are copied from the gobuilder's environment -// to the envv of its subprocesses. -var extraEnv = []string{ - "GOARM", - - // For Unix derivatives. - "CC", - "PATH", - "TMPDIR", - "USER", - - // For Plan 9. - "objtype", - "cputype", - "path", -} - type Builder struct { goroot *Repo name string goos, goarch string key string + env builderEnv } var ( @@ -60,6 +45,8 @@ var ( buildRelease = flag.Bool("release", false, "Build and upload binary release archives") buildRevision = flag.String("rev", "", "Build specified revision and exit") buildCmd = flag.String("cmd", filepath.Join(".", allCmd), "Build command (specify relative to go/src/)") + buildTool = flag.String("tool", "go", "Tool to build.") + gccPath = flag.String("gccpath", "svn://gcc.gnu.org/svn/gcc/trunk", "Path to download gcc from") failAll = flag.Bool("fail", false, "fail all builds") parallel = flag.Bool("parallel", false, "Build multiple targets in parallel") buildTimeout = flag.Duration("buildTimeout", 60*time.Minute, "Maximum time to wait for builds and tests") @@ -91,7 +78,7 @@ func main() { vcs.ShowCmd = *verbose vcs.Verbose = *verbose - rr, err := vcs.RepoRootForImportPath(hgUrl, *verbose) + rr, err := repoForTool() if err != nil { log.Fatal("Error finding repository:", err) } @@ -112,9 +99,9 @@ func main() { log.Fatalf("Error making build root (%s): %s", *buildroot, err) } var err error - goroot, err = RemoteRepo(hgUrl, rootPath) + goroot, err = RemoteRepo(goroot.Master.Root, rootPath) if err != nil { - log.Fatalf("Error creating repository with url (%s): %s", hgUrl, err) + log.Fatalf("Error creating repository with url (%s): %s", goroot.Master.Root, err) } goroot, err = goroot.Clone(goroot.Path, "tip") @@ -208,12 +195,10 @@ func NewBuilder(goroot *Repo, name string) (*Builder, error) { name: name, } - // get goos/goarch from builder string - s := strings.SplitN(b.name, "-", 3) - if len(s) >= 2 { - b.goos, b.goarch = s[0], s[1] - } else { - return nil, fmt.Errorf("unsupported builder form: %s", name) + // get builderEnv for this tool + var err error + if b.env, err = b.builderEnv(name); err != nil { + return nil, err } // read keys from keyfile @@ -235,6 +220,29 @@ func NewBuilder(goroot *Repo, name string) (*Builder, error) { return b, nil } +// builderEnv returns the builderEnv for this buildTool. +func (b *Builder) builderEnv(name string) (builderEnv, error) { + // get goos/goarch from builder string + s := strings.SplitN(b.name, "-", 3) + if len(s) < 2 { + return nil, fmt.Errorf("unsupported builder form: %s", name) + } + b.goos = s[0] + b.goarch = s[1] + + switch *buildTool { + case "go": + return &goEnv{ + goos: s[0], + goarch: s[1], + }, nil + case "gccgo": + return &gccgoEnv{}, nil + default: + return nil, fmt.Errorf("unsupported build tool: %s", *buildTool) + } +} + // buildCmd returns the build command to invoke. // Builders which contain the string '-race' in their // name will override *buildCmd and return raceCmd. @@ -270,21 +278,20 @@ func (b *Builder) buildHash(hash string) error { // create place in which to do work workpath := filepath.Join(*buildroot, b.name+"-"+hash[:12]) if err := os.Mkdir(workpath, mkdirPerm); err != nil { - //return err + return err } defer os.RemoveAll(workpath) // pull before cloning to ensure we have the revision if err := b.goroot.Pull(); err != nil { - // return err - } - - // clone repo at specified revision - if _, err := b.goroot.Clone(filepath.Join(workpath, "go"), hash); err != nil { return err } - srcDir := filepath.Join(workpath, "go", "src") + // set up builder's environment. + srcDir, err := b.env.setup(b.goroot, workpath, hash, b.envv()) + if err != nil { + return err + } // build var buildlog bytes.Buffer @@ -297,11 +304,19 @@ func (b *Builder) buildHash(hash string) error { w := io.MultiWriter(f, &buildlog) cmd := b.buildCmd() - if !filepath.IsAbs(cmd) { - cmd = filepath.Join(srcDir, cmd) + + // go's build command is a script relative to the srcDir, whereas + // gccgo's build command is usually "make check-go" in the srcDir. + if *buildTool == "go" { + if !filepath.IsAbs(cmd) { + cmd = filepath.Join(srcDir, cmd) + } } + + // make sure commands with extra arguments are handled properly + splitCmd := strings.Split(cmd, " ") startTime := time.Now() - ok, err := runOutput(*buildTimeout, b.envv(), w, srcDir, cmd) + ok, err := runOutput(*buildTimeout, b.envv(), w, srcDir, splitCmd...) runTime := time.Now().Sub(startTime) errf := func() string { if err != nil { @@ -324,8 +339,8 @@ func (b *Builder) buildHash(hash string) error { return fmt.Errorf("recordResult: %s", err) } - // build Go sub-repositories - goRoot := filepath.Join(workpath, "go") + // build sub-repositories + goRoot := filepath.Join(workpath, *buildTool) goPath := workpath b.buildSubrepos(goRoot, goPath, hash) @@ -424,65 +439,17 @@ func (b *Builder) buildSubrepo(goRoot, goPath, pkg, hash string) (string, error) return log, err } -// envv returns an environment for build/bench execution -func (b *Builder) envv() []string { - if runtime.GOOS == "windows" { - return b.envvWindows() +// repoForTool returns the correct RepoRoot for the buildTool, or an error if +// the tool is unknown. +func repoForTool() (*vcs.RepoRoot, error) { + switch *buildTool { + case "go": + return vcs.RepoRootForImportPath(goImportPath, *verbose) + case "gccgo": + return vcs.RepoRootForImportPath(gofrontendImportPath, *verbose) + default: + return nil, fmt.Errorf("unknown build tool: %s", *buildTool) } - e := []string{ - "GOOS=" + b.goos, - "GOHOSTOS=" + b.goos, - "GOARCH=" + b.goarch, - "GOHOSTARCH=" + b.goarch, - "GOROOT_FINAL=/usr/local/go", - } - for _, k := range extraEnv { - if s, ok := getenvOk(k); ok { - e = append(e, k+"="+s) - } - } - return e -} - -// windows version of envv -func (b *Builder) envvWindows() []string { - start := map[string]string{ - "GOOS": b.goos, - "GOHOSTOS": b.goos, - "GOARCH": b.goarch, - "GOHOSTARCH": b.goarch, - "GOROOT_FINAL": `c:\go`, - "GOBUILDEXIT": "1", // exit all.bat with completion status. - } - for _, name := range extraEnv { - if s, ok := getenvOk(name); ok { - start[name] = s - } - } - skip := map[string]bool{ - "GOBIN": true, - "GOROOT": true, - "INCLUDE": true, - "LIB": true, - } - var e []string - for name, v := range start { - e = append(e, name+"="+v) - skip[name] = true - } - for _, kv := range os.Environ() { - s := strings.SplitN(kv, "=", 2) - name := strings.ToUpper(s[0]) - switch { - case name == "": - // variables, like "=C:=C:\", just copy them - e = append(e, kv) - case !skip[name]: - e = append(e, kv) - skip[name] = true - } - } - return e } func isDirectory(name string) bool { @@ -653,17 +620,3 @@ func defaultBuildRoot() string { } return filepath.Join(d, "gobuilder") } - -func getenvOk(k string) (v string, ok bool) { - v = os.Getenv(k) - if v != "" { - return v, true - } - keq := k + "=" - for _, kv := range os.Environ() { - if kv == keq { - return "", true - } - } - return "", false -}