From 3e8cc7f1144821d3d6123db5b9d9d2f080853967 Mon Sep 17 00:00:00 2001 From: Andrew Gerrand Date: Mon, 13 Sep 2010 10:46:17 +1000 Subject: [PATCH] misc/dashboard/builder: gobuilder, a continuous build client R=rsc CC=golang-dev https://golang.org/cl/2126042 --- misc/dashboard/builder/Makefile | 14 ++ misc/dashboard/builder/doc.go | 54 ++++++ misc/dashboard/builder/exec.go | 45 +++++ misc/dashboard/builder/hg.go | 53 ++++++ misc/dashboard/builder/http.go | 70 +++++++ misc/dashboard/builder/main.go | 320 ++++++++++++++++++++++++++++++++ 6 files changed, 556 insertions(+) create mode 100644 misc/dashboard/builder/Makefile create mode 100644 misc/dashboard/builder/doc.go create mode 100644 misc/dashboard/builder/exec.go create mode 100644 misc/dashboard/builder/hg.go create mode 100644 misc/dashboard/builder/http.go create mode 100644 misc/dashboard/builder/main.go diff --git a/misc/dashboard/builder/Makefile b/misc/dashboard/builder/Makefile new file mode 100644 index 0000000000..7270a3f425 --- /dev/null +++ b/misc/dashboard/builder/Makefile @@ -0,0 +1,14 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../src/Make.inc + +TARG=gobuilder +GOFILES=\ + exec.go\ + hg.go\ + http.go\ + main.go\ + +include ../../../src/Make.cmd diff --git a/misc/dashboard/builder/doc.go b/misc/dashboard/builder/doc.go new file mode 100644 index 0000000000..54a9adfc04 --- /dev/null +++ b/misc/dashboard/builder/doc.go @@ -0,0 +1,54 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* + +Go Builder is a continuous build client for the Go project. +It integrates with the Go Dashboard AppEngine application. + +Go Builder is intended to run continuously as a background process. + +It periodically pulls updates from the Go Mercurial repository. + +When a newer revision is found, Go Builder creates a clone of the repository, +runs all.bash, and reports build success or failure to the Go Dashboard. + +For a successful build, Go Builder will also run benchmarks +(cd $GOROOT/src/pkg; make bench) and send the results to the Go Dashboard. + +For a release revision (a change description that matches "release.YYYY-MM-DD"), +Go Builder will create a tar.gz archive of the GOROOT and deliver it to the +Go Google Code project's downloads section. + +Usage: + + gobuilder goos-goarch... + + Several goos-goarch combinations can be provided, and the builder will + build them in serial. + +Optional flags: + + -dashboard="godashboard.appspot.com": Go Dashboard Host + The location of the Go Dashboard application to which Go Builder will + report its results. + + -bench: Run benchmarks + + -release: Build and deliver binary release archive + +The key file should be located at $HOME/.gobuilder or, for a builder-specific +key, $HOME/.gobuilder-$BUILDER (eg, $HOME/.gobuilder-linux-amd64). + +The build key file is a text file of the format: + + godashboard-key + googlecode-username + googlecode-password + +If the Google Code credentials are not provided the archival step +will be skipped. + +*/ +package documentation diff --git a/misc/dashboard/builder/exec.go b/misc/dashboard/builder/exec.go new file mode 100644 index 0000000000..bdc740c103 --- /dev/null +++ b/misc/dashboard/builder/exec.go @@ -0,0 +1,45 @@ +package main + +import ( + "bytes" + "exec" + "os" +) + +// run is a simple wrapper for exec.Run/Close +func run(envv []string, dir string, argv ...string) os.Error { + bin, err := exec.LookPath(argv[0]) + if err != nil { + return err + } + p, err := exec.Run(bin, argv, envv, dir, + exec.DevNull, exec.DevNull, exec.PassThrough) + if err != nil { + return err + } + return p.Close() +} + +// runLog runs a process and returns the combined stdout/stderr +func runLog(envv []string, dir string, argv ...string) (output string, exitStatus int, err os.Error) { + bin, err := exec.LookPath(argv[0]) + if err != nil { + return + } + p, err := exec.Run(bin, argv, envv, dir, + exec.DevNull, exec.Pipe, exec.MergeWithStdout) + if err != nil { + return + } + defer p.Close() + b := new(bytes.Buffer) + _, err = b.ReadFrom(p.Stdout) + if err != nil { + return + } + w, err := p.Wait(0) + if err != nil { + return + } + return b.String(), w.WaitStatus.ExitStatus(), nil +} diff --git a/misc/dashboard/builder/hg.go b/misc/dashboard/builder/hg.go new file mode 100644 index 0000000000..b15a2e381c --- /dev/null +++ b/misc/dashboard/builder/hg.go @@ -0,0 +1,53 @@ +package main + +import ( + "os" + "strconv" + "strings" +) + +type Commit struct { + num int // mercurial revision number + node string // mercurial hash + parent string // hash of commit's parent + user string // author's Name + date string // date of commit + desc string // description +} + +// getCommit returns details about the Commit specified by the revision hash +func getCommit(rev string) (c Commit, err os.Error) { + defer func() { + if err != nil { + err = errf("getCommit: %s: %s", rev, err) + } + }() + parts, err := getCommitParts(rev) + if err != nil { + return + } + num, err := strconv.Atoi(parts[0]) + if err != nil { + return + } + parent := "" + if num > 0 { + prev := strconv.Itoa(num - 1) + if pparts, err := getCommitParts(prev); err == nil { + parent = pparts[1] + } + } + user := strings.Replace(parts[2], "<", "<", -1) + user = strings.Replace(user, ">", ">", -1) + return Commit{num, parts[1], parent, user, parts[3], parts[4]}, nil +} + +func getCommitParts(rev string) (parts []string, err os.Error) { + const format = "{rev}>{node}>{author|escape}>{date}>{desc}" + s, _, err := runLog(nil, goroot, + "hg", "log", "-r", rev, "-l", "1", "--template", format) + if err != nil { + return + } + return strings.Split(s, ">", 5), nil +} diff --git a/misc/dashboard/builder/http.go b/misc/dashboard/builder/http.go new file mode 100644 index 0000000000..02f2810617 --- /dev/null +++ b/misc/dashboard/builder/http.go @@ -0,0 +1,70 @@ +package main + +import ( + "bytes" + "encoding/base64" + "encoding/binary" + "fmt" + "http" + "os" + "regexp" +) + +// getHighWater returns the current highwater revision hash for this builder +func (b *Builder) getHighWater() (rev string, err os.Error) { + url := fmt.Sprintf("http://%s/hw-get?builder=%s", *dashboard, b.name) + r, _, err := http.Get(url) + if err != nil { + return + } + buf := new(bytes.Buffer) + _, err = buf.ReadFrom(r.Body) + if err != nil { + return + } + r.Body.Close() + return buf.String(), nil +} + +// recordResult sends build results to the dashboard +func (b *Builder) recordResult(buildLog string, c Commit) os.Error { + return httpCommand("build", map[string]string{ + "builder": b.name, + "key": b.key, + "node": c.node, + "parent": c.parent, + "user": c.user, + "date": c.date, + "desc": c.desc, + "log": buildLog, + }) +} + +// match lines like: "package.BechmarkFunc 100000 999 ns/op" +var benchmarkRegexp = regexp.MustCompile("([^\n\t ]+)[\t ]+([0-9]+)[\t ]+([0-9]+) ns/op") + +// recordBenchmarks sends benchmark results to the dashboard +func (b *Builder) recordBenchmarks(benchLog string, c Commit) os.Error { + results := benchmarkRegexp.FindAllStringSubmatch(benchLog, -1) + var buf bytes.Buffer + b64 := base64.NewEncoder(base64.StdEncoding, &buf) + for _, r := range results { + for _, s := range r[1:] { + binary.Write(b64, binary.BigEndian, uint16(len(s))) + b64.Write([]byte(s)) + } + } + b64.Close() + return httpCommand("benchmarks", map[string]string{ + "builder": b.name, + "key": b.key, + "node": c.node, + "benchmarkdata": buf.String(), + }) +} + +func httpCommand(cmd string, args map[string]string) os.Error { + url := fmt.Sprintf("http://%v/%v", *dashboard, cmd) + _, err := http.PostForm(url, args) + return err +} diff --git a/misc/dashboard/builder/main.go b/misc/dashboard/builder/main.go new file mode 100644 index 0000000000..388a262b0a --- /dev/null +++ b/misc/dashboard/builder/main.go @@ -0,0 +1,320 @@ +package main + +import ( + "container/vector" + "flag" + "fmt" + "io/ioutil" + "log" + "os" + "path" + "regexp" + "strconv" + "strings" + "time" +) + +const ( + codeProject = "go" + codePyScript = "misc/dashboard/googlecode_upload.py" + hgUrl = "https://go.googlecode.com/hg/" + waitInterval = 10e9 // time to wait before checking for new revs + mkdirPerm = 0750 +) + +type Builder struct { + name string + goos, goarch string + key string + codeUsername string + codePassword string +} + +type BenchRequest struct { + builder *Builder + commit Commit + path string +} + +var ( + dashboard = flag.String("dashboard", "godashboard.appspot.com", "Go Dashboard Host") + runBenchmarks = flag.Bool("bench", false, "Run benchmarks") + buildRelease = flag.Bool("release", false, "Build and deliver binary release archive") +) + +var ( + buildroot = path.Join(os.TempDir(), "gobuilder") + goroot = path.Join(buildroot, "goroot") + releaseRegexp = regexp.MustCompile(`^release\.[0-9\-]+`) + benchRequests vector.Vector +) + +func main() { + flag.Usage = func() { + fmt.Fprintf(os.Stderr, "usage: %s goos-goarch...\n", os.Args[0]) + flag.PrintDefaults() + os.Exit(2) + } + flag.Parse() + if len(flag.Args()) == 0 { + flag.Usage() + } + builders := make([]*Builder, len(flag.Args())) + for i, builder := range flag.Args() { + b, err := NewBuilder(builder) + if err != nil { + log.Exit(err) + } + builders[i] = b + } + if err := os.RemoveAll(buildroot); err != nil { + log.Exitf("Error removing build root (%s): %s", buildroot, err) + } + if err := os.Mkdir(buildroot, mkdirPerm); err != nil { + log.Exitf("Error making build root (%s): %s", buildroot, err) + } + if err := run(nil, buildroot, "hg", "clone", hgUrl, goroot); err != nil { + log.Exit("Error cloning repository:", err) + } + // check for new commits and build them + for { + err := run(nil, goroot, "hg", "pull", "-u") + if err != nil { + log.Stderr("hg pull failed:", err) + time.Sleep(waitInterval) + continue + } + built := false + for _, b := range builders { + if b.build() { + built = true + } + } + // only run benchmarks if we didn't build anything + // so that they don't hold up the builder queue + if !built { + // if we have no benchmarks to do, pause + if benchRequests.Len() == 0 { + time.Sleep(waitInterval) + } else { + runBenchmark(benchRequests.Pop().(BenchRequest)) + // after running one benchmark, + // continue to find and build new revisions. + } + } + } +} + +func runBenchmark(r BenchRequest) { + // run benchmarks and send to dashboard + pkg := path.Join(r.path, "go", "src", "pkg") + bin := path.Join(r.path, "go", "bin") + env := []string{ + "GOOS=" + r.builder.goos, + "GOARCH=" + r.builder.goarch, + "PATH=" + bin + ":" + os.Getenv("PATH"), + } + benchLog, _, err := runLog(env, pkg, "gomake", "bench") + if err != nil { + log.Stderr("%s gomake bench:", r.builder.name, err) + return + } + if err = r.builder.recordBenchmarks(benchLog, r.commit); err != nil { + log.Stderr("recordBenchmarks:", err) + } +} + +func NewBuilder(builder string) (*Builder, os.Error) { + b := &Builder{name: builder} + + // get goos/goarch from builder string + s := strings.Split(builder, "-", 3) + if len(s) == 2 { + b.goos, b.goarch = s[0], s[1] + } else { + return nil, errf("unsupported builder form: %s", builder) + } + + // read keys from keyfile + fn := path.Join(os.Getenv("HOME"), ".gobuildkey") + if s := fn+"-"+b.name; isFile(s) { // builder-specific file + fn = s + } + c, err := ioutil.ReadFile(fn) + if err != nil { + return nil, errf("readKeys %s (%s): %s", b.name, fn, err) + } + v := strings.Split(string(c), "\n", -1) + b.key = v[0] + if len(v) >= 3 { + b.codeUsername, b.codePassword = v[1], v[2] + } + + return b, nil +} + +// build checks for a new commit for this builder +// and builds it if one is found. +// It returns true if a build was attempted. +func (b *Builder) build() bool { + defer func() { + err := recover() + if err != nil { + log.Stderr("%s build: %s", b.name, err) + } + }() + c, err := b.nextCommit() + if err != nil { + log.Stderr(err) + return false + } + if c == nil { + return false + } + log.Stderrf("%s building %d", b.name, c.num) + err = b.buildCommit(*c) + if err != nil { + log.Stderr(err) + } + return true +} + +// nextCommit returns the next unbuilt Commit for this builder +func (b *Builder) nextCommit() (nextC *Commit, err os.Error) { + defer func() { + if err != nil { + err = errf("%s nextCommit: %s", b.name, err) + } + }() + hw, err := b.getHighWater() + if err != nil { + return + } + c, err := getCommit(hw) + if err != nil { + return + } + next := c.num + 1 + c, err = getCommit(strconv.Itoa(next)) + if err == nil || c.num == next { + return &c, nil + } + return nil, nil +} + +func (b *Builder) buildCommit(c Commit) (err os.Error) { + defer func() { + if err != nil { + err = errf("%s buildCommit: %d: %s", b.name, c.num, err) + } + }() + + // create place in which to do work + workpath := path.Join(buildroot, b.name+"-"+strconv.Itoa(c.num)) + err = os.Mkdir(workpath, mkdirPerm) + if err != nil { + return + } + benchRequested := false + defer func() { + if !benchRequested { + os.RemoveAll(workpath) + } + }() + + // clone repo at revision num (new candidate) + err = run(nil, workpath, + "hg", "clone", + "-r", strconv.Itoa(c.num), + goroot, "go") + if err != nil { + return + } + + // set up environment for build/bench execution + env := []string{ + "GOOS=" + b.goos, + "GOARCH=" + b.goarch, + "GOROOT_FINAL=/usr/local/go", + "PATH=" + os.Getenv("PATH"), + } + srcDir := path.Join(workpath, "go", "src") + + // build the release candidate + buildLog, status, err := runLog(env, srcDir, "bash", "all.bash") + if err != nil { + return errf("all.bash: %s", err) + } + if status != 0 { + // record failure + return b.recordResult(buildLog, c) + } + + // record success + if err = b.recordResult("", c); err != nil { + return errf("recordResult: %s", err) + } + + // send benchmark request if benchmarks are enabled + if *runBenchmarks { + benchRequests.Insert(0, BenchRequest{ + builder: b, + commit: c, + path: workpath, + }) + benchRequested = true + } + + // finish here if codeUsername and codePassword aren't set + if b.codeUsername == "" || b.codePassword == "" || !*buildRelease { + return + } + + // if this is a release, create tgz and upload to google code + if release := releaseRegexp.FindString(c.desc); release != "" { + // clean out build state + err = run(env, srcDir, "sh", "clean.bash", "--nopkg") + if err != nil { + return errf("clean.bash: %s", err) + } + // upload binary release + err = b.codeUpload(release) + } + + return +} + +func (b *Builder) codeUpload(release string) (err os.Error) { + defer func() { + if err != nil { + err = errf("%s codeUpload release: %s: %s", b.name, release, err) + } + }() + fn := fmt.Sprintf("%s.%s-%s.tar.gz", release, b.goos, b.goarch) + err = run(nil, "", "tar", "czf", fn, "go") + if err != nil { + return + } + return run(nil, "", "python", + path.Join(goroot, codePyScript), + "-s", release, + "-p", codeProject, + "-u", b.codeUsername, + "-w", b.codePassword, + "-l", fmt.Sprintf("%s,%s", b.goos, b.goarch), + fn) +} + +func isDirectory(name string) bool { + s, err := os.Stat(name) + return err == nil && s.IsDirectory() +} + +func isFile(name string) bool { + s, err := os.Stat(name) + return err == nil && (s.IsRegular() || s.IsSymlink()) +} + +func errf(format string, args ...interface{}) os.Error { + return os.NewError(fmt.Sprintf(format, args)) +}