1
0
mirror of https://github.com/golang/go synced 2024-11-18 16:54:43 -07:00

go.tools/dashboard: add option to build gccgo revisions

* Added --tool flag that allows user to specify "gccgo" as the tool to build.
* Added builderEnv struct to abstract away setting up the build.
* Made envv and envvWindows methods of the builderEnv struct.
* Modified Builder.envv() to wrap envv and envvWindows
* Added internal builderEnv in Builder.

R=adg
CC=golang-dev
https://golang.org/cl/13240044
This commit is contained in:
Chris Manghane 2013-09-11 16:45:24 -07:00
parent 3fee9166d4
commit c871e361fc
2 changed files with 298 additions and 116 deletions

229
dashboard/builder/env.go Normal file
View File

@ -0,0 +1,229 @@
// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package main
import (
"fmt"
"io"
"io/ioutil"
"os"
"path/filepath"
"runtime"
"strings"
"code.google.com/p/go.tools/go/vcs"
)
// These variables are copied from the gobuilder's environment
// to the envv of its subprocesses.
var extraEnv = []string{
"GOARM",
// For Unix derivatives.
"CC",
"PATH",
"TMPDIR",
"USER",
// For Plan 9.
"objtype",
"cputype",
"path",
}
// builderEnv represents the environment that a Builder will run tests in.
type builderEnv interface {
// setup sets up the builder environment and returns the directory to run the buildCmd in.
setup(repo *Repo, workpath, hash string, envv []string) (string, error)
}
// goEnv represents the builderEnv for the main Go repo.
type goEnv struct {
goos, goarch string
}
func (b *Builder) envv() []string {
if runtime.GOOS == "windows" {
return b.envvWindows()
}
var e []string
if *buildTool == "go" {
e = []string{
"GOOS=" + b.goos,
"GOHOSTOS=" + b.goos,
"GOARCH=" + b.goarch,
"GOHOSTARCH=" + b.goarch,
"GOROOT_FINAL=/usr/local/go",
}
}
for _, k := range extraEnv {
if s, ok := getenvOk(k); ok {
e = append(e, k+"="+s)
}
}
return e
}
func (b *Builder) envvWindows() []string {
var start map[string]string
if *buildTool == "go" {
start = map[string]string{
"GOOS": b.goos,
"GOHOSTOS": b.goos,
"GOARCH": b.goarch,
"GOHOSTARCH": b.goarch,
"GOROOT_FINAL": `c:\go`,
"GOBUILDEXIT": "1", // exit all.bat with completion status.
}
}
for _, name := range extraEnv {
if s, ok := getenvOk(name); ok {
start[name] = s
}
}
skip := map[string]bool{
"GOBIN": true,
"GOROOT": true,
"INCLUDE": true,
"LIB": true,
}
var e []string
for name, v := range start {
e = append(e, name+"="+v)
skip[name] = true
}
for _, kv := range os.Environ() {
s := strings.SplitN(kv, "=", 2)
name := strings.ToUpper(s[0])
switch {
case name == "":
// variables, like "=C:=C:\", just copy them
e = append(e, kv)
case !skip[name]:
e = append(e, kv)
skip[name] = true
}
}
return e
}
// setup for a goEnv clones the main go repo to workpath/go at the provided hash
// and returns the path workpath/go/src, the location of all go build scripts.
func (env *goEnv) setup(repo *Repo, workpath, hash string, envv []string) (string, error) {
goworkpath := filepath.Join(workpath, "go")
if _, err := repo.Clone(goworkpath, hash); err != nil {
return "", fmt.Errorf("error cloning repository: %s", err)
}
return filepath.Join(goworkpath, "src"), nil
}
// gccgoEnv represents the builderEnv for the gccgo compiler.
type gccgoEnv struct{}
// setup for a gccgoEnv clones the gofrontend repo to workpath/go at the hash
// and clones the latest GCC branch to workpath/gcc. The gccgo sources are
// replaced with the updated sources in the gofrontend repo and gcc gets
// gets configured and built in workpath/gcc-objdir. The path to
// workpath/gcc-objdir is returned.
func (env *gccgoEnv) setup(repo *Repo, workpath, hash string, envv []string) (string, error) {
gofrontendpath := filepath.Join(workpath, "gofrontend")
gccpath := filepath.Join(workpath, "gcc")
// get a handle to SVN vcs.Cmd for pulling down GCC.
svn := vcs.ByCmd("svn")
if err := timeout(*cmdTimeout, func() error {
// pull down a working copy of GCC.
return svn.Create(gccpath, *gccPath)
}); err != nil {
return "", err
}
// clone gofrontend repo at specified revision
if _, err := repo.Clone(gofrontendpath, hash); err != nil {
return "", err
}
// remove gccpath/gcc/go/gofrontend and gcc/libgo
gccgopath := filepath.Join(gccpath, "gcc", "go", "gofrontend")
gcclibgopath := filepath.Join(gccpath, "libgo")
if err := os.RemoveAll(gccgopath); err != nil {
return "", err
}
if err := os.RemoveAll(gcclibgopath); err != nil {
return "", err
}
// copy gofrontend and libgo to appropriate locations
if err := copyDir(filepath.Join(gofrontendpath, "go"), gccgopath); err != nil {
return "", fmt.Errorf("Failed to copy gofrontend/go to gcc/go/gofrontend: %s\n", err)
}
if err := copyDir(filepath.Join(gofrontendpath, "libgo"), gcclibgopath); err != nil {
return "", fmt.Errorf("Failed to copy gofrontend/libgo to gcc/libgo: %s\n", err)
}
// make objdir to work in
gccobjdir := filepath.Join(workpath, "gcc-objdir")
if err := os.Mkdir(gccobjdir, mkdirPerm); err != nil {
return "", err
}
// configure GCC with substituted gofrontend and libgo
gccConfigCmd := []string{filepath.Join(gccpath, "configure"), "--enable-languages=c,c++,go", "--disable-bootstrap"}
if _, err := runOutput(*cmdTimeout, envv, ioutil.Discard, gccobjdir, gccConfigCmd...); err != nil {
return "", fmt.Errorf("Failed to configure GCC: %s", err)
}
// build gcc
if _, err := runOutput(*buildTimeout, envv, ioutil.Discard, gccobjdir, "make"); err != nil {
return "", fmt.Errorf("Failed to build GCC: %s", err)
}
return gccobjdir, nil
}
// copyDir copies the src directory into the dst
func copyDir(src, dst string) error {
return filepath.Walk(src, func(path string, f os.FileInfo, err error) error {
dstPath := strings.Replace(path, src, dst, 1)
if f.IsDir() {
return os.Mkdir(dstPath, mkdirPerm)
}
srcFile, err := os.Open(path)
if err != nil {
return err
}
defer srcFile.Close()
dstFile, err := os.Create(dstPath)
if err != nil {
return err
}
if _, err := io.Copy(dstFile, srcFile); err != nil {
return err
}
return dstFile.Close()
})
}
func getenvOk(k string) (v string, ok bool) {
v = os.Getenv(k)
if v != "" {
return v, true
}
keq := k + "="
for _, kv := range os.Environ() {
if kv == keq {
return "", true
}
}
return "", false
}

View File

@ -22,36 +22,21 @@ import (
)
const (
codeProject = "go"
codePyScript = "misc/dashboard/googlecode_upload.py"
hgUrl = "code.google.com/p/go"
mkdirPerm = 0750
waitInterval = 30 * time.Second // time to wait before checking for new revs
pkgBuildInterval = 24 * time.Hour // rebuild packages every 24 hours
codeProject = "go"
codePyScript = "misc/dashboard/googlecode_upload.py"
goImportPath = "code.google.com/p/go"
gofrontendImportPath = "code.google.com/p/gofrontend"
mkdirPerm = 0750
waitInterval = 30 * time.Second // time to wait before checking for new revs
pkgBuildInterval = 24 * time.Hour // rebuild packages every 24 hours
)
// These variables are copied from the gobuilder's environment
// to the envv of its subprocesses.
var extraEnv = []string{
"GOARM",
// For Unix derivatives.
"CC",
"PATH",
"TMPDIR",
"USER",
// For Plan 9.
"objtype",
"cputype",
"path",
}
type Builder struct {
goroot *Repo
name string
goos, goarch string
key string
env builderEnv
}
var (
@ -60,6 +45,8 @@ var (
buildRelease = flag.Bool("release", false, "Build and upload binary release archives")
buildRevision = flag.String("rev", "", "Build specified revision and exit")
buildCmd = flag.String("cmd", filepath.Join(".", allCmd), "Build command (specify relative to go/src/)")
buildTool = flag.String("tool", "go", "Tool to build.")
gccPath = flag.String("gccpath", "svn://gcc.gnu.org/svn/gcc/trunk", "Path to download gcc from")
failAll = flag.Bool("fail", false, "fail all builds")
parallel = flag.Bool("parallel", false, "Build multiple targets in parallel")
buildTimeout = flag.Duration("buildTimeout", 60*time.Minute, "Maximum time to wait for builds and tests")
@ -91,7 +78,7 @@ func main() {
vcs.ShowCmd = *verbose
vcs.Verbose = *verbose
rr, err := vcs.RepoRootForImportPath(hgUrl, *verbose)
rr, err := repoForTool()
if err != nil {
log.Fatal("Error finding repository:", err)
}
@ -112,9 +99,9 @@ func main() {
log.Fatalf("Error making build root (%s): %s", *buildroot, err)
}
var err error
goroot, err = RemoteRepo(hgUrl, rootPath)
goroot, err = RemoteRepo(goroot.Master.Root, rootPath)
if err != nil {
log.Fatalf("Error creating repository with url (%s): %s", hgUrl, err)
log.Fatalf("Error creating repository with url (%s): %s", goroot.Master.Root, err)
}
goroot, err = goroot.Clone(goroot.Path, "tip")
@ -208,12 +195,10 @@ func NewBuilder(goroot *Repo, name string) (*Builder, error) {
name: name,
}
// get goos/goarch from builder string
s := strings.SplitN(b.name, "-", 3)
if len(s) >= 2 {
b.goos, b.goarch = s[0], s[1]
} else {
return nil, fmt.Errorf("unsupported builder form: %s", name)
// get builderEnv for this tool
var err error
if b.env, err = b.builderEnv(name); err != nil {
return nil, err
}
// read keys from keyfile
@ -235,6 +220,29 @@ func NewBuilder(goroot *Repo, name string) (*Builder, error) {
return b, nil
}
// builderEnv returns the builderEnv for this buildTool.
func (b *Builder) builderEnv(name string) (builderEnv, error) {
// get goos/goarch from builder string
s := strings.SplitN(b.name, "-", 3)
if len(s) < 2 {
return nil, fmt.Errorf("unsupported builder form: %s", name)
}
b.goos = s[0]
b.goarch = s[1]
switch *buildTool {
case "go":
return &goEnv{
goos: s[0],
goarch: s[1],
}, nil
case "gccgo":
return &gccgoEnv{}, nil
default:
return nil, fmt.Errorf("unsupported build tool: %s", *buildTool)
}
}
// buildCmd returns the build command to invoke.
// Builders which contain the string '-race' in their
// name will override *buildCmd and return raceCmd.
@ -270,21 +278,20 @@ func (b *Builder) buildHash(hash string) error {
// create place in which to do work
workpath := filepath.Join(*buildroot, b.name+"-"+hash[:12])
if err := os.Mkdir(workpath, mkdirPerm); err != nil {
//return err
return err
}
defer os.RemoveAll(workpath)
// pull before cloning to ensure we have the revision
if err := b.goroot.Pull(); err != nil {
// return err
}
// clone repo at specified revision
if _, err := b.goroot.Clone(filepath.Join(workpath, "go"), hash); err != nil {
return err
}
srcDir := filepath.Join(workpath, "go", "src")
// set up builder's environment.
srcDir, err := b.env.setup(b.goroot, workpath, hash, b.envv())
if err != nil {
return err
}
// build
var buildlog bytes.Buffer
@ -297,11 +304,19 @@ func (b *Builder) buildHash(hash string) error {
w := io.MultiWriter(f, &buildlog)
cmd := b.buildCmd()
if !filepath.IsAbs(cmd) {
cmd = filepath.Join(srcDir, cmd)
// go's build command is a script relative to the srcDir, whereas
// gccgo's build command is usually "make check-go" in the srcDir.
if *buildTool == "go" {
if !filepath.IsAbs(cmd) {
cmd = filepath.Join(srcDir, cmd)
}
}
// make sure commands with extra arguments are handled properly
splitCmd := strings.Split(cmd, " ")
startTime := time.Now()
ok, err := runOutput(*buildTimeout, b.envv(), w, srcDir, cmd)
ok, err := runOutput(*buildTimeout, b.envv(), w, srcDir, splitCmd...)
runTime := time.Now().Sub(startTime)
errf := func() string {
if err != nil {
@ -324,8 +339,8 @@ func (b *Builder) buildHash(hash string) error {
return fmt.Errorf("recordResult: %s", err)
}
// build Go sub-repositories
goRoot := filepath.Join(workpath, "go")
// build sub-repositories
goRoot := filepath.Join(workpath, *buildTool)
goPath := workpath
b.buildSubrepos(goRoot, goPath, hash)
@ -424,65 +439,17 @@ func (b *Builder) buildSubrepo(goRoot, goPath, pkg, hash string) (string, error)
return log, err
}
// envv returns an environment for build/bench execution
func (b *Builder) envv() []string {
if runtime.GOOS == "windows" {
return b.envvWindows()
// repoForTool returns the correct RepoRoot for the buildTool, or an error if
// the tool is unknown.
func repoForTool() (*vcs.RepoRoot, error) {
switch *buildTool {
case "go":
return vcs.RepoRootForImportPath(goImportPath, *verbose)
case "gccgo":
return vcs.RepoRootForImportPath(gofrontendImportPath, *verbose)
default:
return nil, fmt.Errorf("unknown build tool: %s", *buildTool)
}
e := []string{
"GOOS=" + b.goos,
"GOHOSTOS=" + b.goos,
"GOARCH=" + b.goarch,
"GOHOSTARCH=" + b.goarch,
"GOROOT_FINAL=/usr/local/go",
}
for _, k := range extraEnv {
if s, ok := getenvOk(k); ok {
e = append(e, k+"="+s)
}
}
return e
}
// windows version of envv
func (b *Builder) envvWindows() []string {
start := map[string]string{
"GOOS": b.goos,
"GOHOSTOS": b.goos,
"GOARCH": b.goarch,
"GOHOSTARCH": b.goarch,
"GOROOT_FINAL": `c:\go`,
"GOBUILDEXIT": "1", // exit all.bat with completion status.
}
for _, name := range extraEnv {
if s, ok := getenvOk(name); ok {
start[name] = s
}
}
skip := map[string]bool{
"GOBIN": true,
"GOROOT": true,
"INCLUDE": true,
"LIB": true,
}
var e []string
for name, v := range start {
e = append(e, name+"="+v)
skip[name] = true
}
for _, kv := range os.Environ() {
s := strings.SplitN(kv, "=", 2)
name := strings.ToUpper(s[0])
switch {
case name == "":
// variables, like "=C:=C:\", just copy them
e = append(e, kv)
case !skip[name]:
e = append(e, kv)
skip[name] = true
}
}
return e
}
func isDirectory(name string) bool {
@ -653,17 +620,3 @@ func defaultBuildRoot() string {
}
return filepath.Join(d, "gobuilder")
}
func getenvOk(k string) (v string, ok bool) {
v = os.Getenv(k)
if v != "" {
return v, true
}
keq := k + "="
for _, kv := range os.Environ() {
if kv == keq {
return "", true
}
}
return "", false
}