mirror of
https://github.com/golang/go
synced 2024-11-24 08:20:03 -07:00
revert accidental submit of builder.go
R=rsc CC=golang-dev https://golang.org/cl/2121042
This commit is contained in:
Andrew Gerrand
parent
2b4218f321
commit
e11bcc88f5
@ -1,14 +0,0 @@
|
|||||||
# Copyright 2009 The Go Authors. All rights reserved.
|
|
||||||
# Use of this source code is governed by a BSD-style
|
|
||||||
# license that can be found in the LICENSE file.
|
|
||||||
|
|
||||||
include ../../../src/Make.inc
|
|
||||||
|
|
||||||
TARG=gobuilder
|
|
||||||
GOFILES=\
|
|
||||||
exec.go\
|
|
||||||
hg.go\
|
|
||||||
http.go\
|
|
||||||
main.go\
|
|
||||||
|
|
||||||
include ../../../src/Make.cmd
|
|
||||||
@ -1,61 +0,0 @@
|
|||||||
// Copyright 2010 The Go Authors. All rights reserved.
|
|
||||||
// Use of this source code is governed by a BSD-style
|
|
||||||
// license that can be found in the LICENSE file.
|
|
||||||
|
|
||||||
/*
|
|
||||||
|
|
||||||
Go Builder is a continuous build client for the Go project.
|
|
||||||
It integrates with the Go Dashboard AppEngine application.
|
|
||||||
|
|
||||||
Go Builder is intended to run continuously as a background process.
|
|
||||||
|
|
||||||
It periodically pulls updates from the Go Mercurial repository.
|
|
||||||
|
|
||||||
When a newer revision is found, Go Builder creates a clone of the repository,
|
|
||||||
runs all.bash, and reports build success or failure to the Go Dashboard.
|
|
||||||
|
|
||||||
For a successful build, Go Builder will also run benchmarks
|
|
||||||
(cd $GOROOT/src/pkg; make bench) and send the results to the Go Dashboard.
|
|
||||||
|
|
||||||
For release revision (a change description that matches "release.YYYY-MM-DD"),
|
|
||||||
Go Builder will create a tar.gz archive of the GOROOT and deliver it to the
|
|
||||||
Go Google Code project's downloads section.
|
|
||||||
|
|
||||||
Command-line options (and defaults):
|
|
||||||
|
|
||||||
-goarch="": $GOARCH
|
|
||||||
-goos="": $GOOS
|
|
||||||
The target architecture and operating system of this build client.
|
|
||||||
|
|
||||||
-goroot="": $GOROOT
|
|
||||||
A persistent Go checkout. Go Builder will periodically run 'hg pull -u'
|
|
||||||
from this location and use it as a source repository when cloning a
|
|
||||||
revision to be built.
|
|
||||||
|
|
||||||
-path="": Build Path
|
|
||||||
The base path in which building, testing, and archival will occur,
|
|
||||||
such as "/tmp/build". This can be considered volatile.
|
|
||||||
|
|
||||||
-keyfile="": Key File
|
|
||||||
The file containing the build key and Google Code credentials.
|
|
||||||
It is a text file of the format:
|
|
||||||
|
|
||||||
godashboard-key
|
|
||||||
googlecode-username
|
|
||||||
googlecode-password
|
|
||||||
|
|
||||||
If the Google Code credentials are not provided the archival step
|
|
||||||
will be skipped.
|
|
||||||
|
|
||||||
-host="godashboard.appspot.com": Go Dashboard Host
|
|
||||||
The location of the Go Dashboard application to which Go Builder will
|
|
||||||
report its results.
|
|
||||||
|
|
||||||
-pybin="/usr/bin/python": Python Binary
|
|
||||||
-hgbin="/usr/local/bin/hg": Mercurial Binary
|
|
||||||
These name the local Python and Mercurial binaries.
|
|
||||||
(Python is required only to run the Google Code uploader script, found
|
|
||||||
at $GOROOT/misc/dashboard/googlecode_upload.py.)
|
|
||||||
|
|
||||||
*/
|
|
||||||
package documentation
|
|
||||||
@ -1,45 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"exec"
|
|
||||||
"os"
|
|
||||||
)
|
|
||||||
|
|
||||||
// run is a simple wrapper for exec.Run/Close
|
|
||||||
func run(envv []string, dir string, argv ...string) os.Error {
|
|
||||||
bin, err := exec.LookPath(argv[0])
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
p, err := exec.Run(bin, argv, envv, dir,
|
|
||||||
exec.DevNull, exec.DevNull, exec.PassThrough)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
return p.Close()
|
|
||||||
}
|
|
||||||
|
|
||||||
// runLog runs a process and returns the combined stdout/stderr
|
|
||||||
func runLog(envv []string, dir string, argv ...string) (o string, s int, err os.Error) {
|
|
||||||
s = -1
|
|
||||||
bin, err := exec.LookPath(argv[0])
|
|
||||||
if err != nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
p, err := exec.Run(bin, argv, envv, dir,
|
|
||||||
exec.DevNull, exec.Pipe, exec.MergeWithStdout)
|
|
||||||
if err != nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
b := new(bytes.Buffer)
|
|
||||||
_, err = b.ReadFrom(p.Stdout)
|
|
||||||
if err != nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
w, err := p.Wait(0)
|
|
||||||
if err != nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
return b.String(), w.WaitStatus.ExitStatus(), nil
|
|
||||||
}
|
|
||||||
@ -1,56 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"os"
|
|
||||||
"fmt"
|
|
||||||
"strconv"
|
|
||||||
"strings"
|
|
||||||
)
|
|
||||||
|
|
||||||
type Commit struct {
|
|
||||||
num int // mercurial revision number
|
|
||||||
node string // mercurial hash
|
|
||||||
parent string // hash of commit's parent
|
|
||||||
user string // author's name and email
|
|
||||||
date string // date of commit
|
|
||||||
desc string // description
|
|
||||||
}
|
|
||||||
|
|
||||||
// getCommit returns details about the Commit specified by the revision hash
|
|
||||||
func getCommit(rev string) (c Commit, err os.Error) {
|
|
||||||
defer func() {
|
|
||||||
if err != nil {
|
|
||||||
err = os.NewError(fmt.Sprintf("getCommit: %s: %s",
|
|
||||||
rev, err))
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
parts, err := getCommitParts(rev)
|
|
||||||
if err != nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
num, err := strconv.Atoi(parts[0])
|
|
||||||
if err != nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
parent := ""
|
|
||||||
if num > 0 {
|
|
||||||
prev := strconv.Itoa(num - 1)
|
|
||||||
if pparts, err := getCommitParts(prev); err == nil {
|
|
||||||
parent = pparts[1]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
user := strings.Replace(parts[2], "<", "<", -1)
|
|
||||||
user = strings.Replace(user, ">", ">", -1)
|
|
||||||
return Commit{num, parts[1], parent, user, parts[3], parts[4]}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func getCommitParts(rev string) (parts []string, err os.Error) {
|
|
||||||
format := "{rev}>{node|escape}>{author|escape}>{date}>{desc}"
|
|
||||||
s, _, err := runLog(nil, goroot,
|
|
||||||
"hg", "log", "-r", rev, "-l", "1", "--template", format)
|
|
||||||
if err != nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
return strings.Split(s, ">", -1), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
@ -1,71 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"encoding/base64"
|
|
||||||
"encoding/binary"
|
|
||||||
"fmt"
|
|
||||||
"http"
|
|
||||||
"os"
|
|
||||||
"regexp"
|
|
||||||
)
|
|
||||||
|
|
||||||
// getHighWater returns the current highwater revision hash for this builder
|
|
||||||
func (b *Builder) getHighWater() (rev string, err os.Error) {
|
|
||||||
url := fmt.Sprintf("http://%s/hw-get?builder=%s",
|
|
||||||
*dashboardhost, b.name)
|
|
||||||
r, _, err := http.Get(url)
|
|
||||||
if err != nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
buf := new(bytes.Buffer)
|
|
||||||
_, err = buf.ReadFrom(r.Body)
|
|
||||||
if err != nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
r.Body.Close()
|
|
||||||
return buf.String(), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// recordResult sends build results to the dashboard
|
|
||||||
func (b *Builder) recordResult(buildLog string, c Commit) os.Error {
|
|
||||||
return httpCommand("build", map[string]string{
|
|
||||||
"builder": b.name,
|
|
||||||
"key": b.key,
|
|
||||||
"node": c.node,
|
|
||||||
"parent": c.parent,
|
|
||||||
"user": c.user,
|
|
||||||
"date": c.date,
|
|
||||||
"desc": c.desc,
|
|
||||||
"log": buildLog,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// match lines like: "package.BechmarkFunc 100000 999 ns/op"
|
|
||||||
var benchmarkRegexp = regexp.MustCompile("([^\n\t ]+)[\t ]+([0-9]+)[\t ]+([0-9]+) ns/op")
|
|
||||||
|
|
||||||
// recordBenchmarks sends benchmark results to the dashboard
|
|
||||||
func (b *Builder) recordBenchmarks(benchLog string, c Commit) os.Error {
|
|
||||||
results := benchmarkRegexp.FindAllStringSubmatch(benchLog, -1)
|
|
||||||
var buf bytes.Buffer
|
|
||||||
b64 := base64.NewEncoder(base64.StdEncoding, &buf)
|
|
||||||
for _, r := range results {
|
|
||||||
for _, s := range r[1:] {
|
|
||||||
binary.Write(b64, binary.BigEndian, uint16(len(s)))
|
|
||||||
b64.Write([]byte(s))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
b64.Close()
|
|
||||||
return httpCommand("benchmarks", map[string]string{
|
|
||||||
"builder": b.name,
|
|
||||||
"key": b.key,
|
|
||||||
"node": c.node,
|
|
||||||
"benchmarkdata": buf.String(),
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func httpCommand(cmd string, args map[string]string) os.Error {
|
|
||||||
url := fmt.Sprintf("http://%v/%v", *dashboardhost, cmd)
|
|
||||||
_, err := http.PostForm(url, args)
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
@ -1,259 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"flag"
|
|
||||||
"fmt"
|
|
||||||
"io/ioutil"
|
|
||||||
"log"
|
|
||||||
"os"
|
|
||||||
"path"
|
|
||||||
"regexp"
|
|
||||||
"strconv"
|
|
||||||
"strings"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
const (
|
|
||||||
codeProject = "go"
|
|
||||||
codePyScript = "misc/dashboard/googlecode_upload.py"
|
|
||||||
hgUrl = "https://go.googlecode.com/hg/"
|
|
||||||
waitInterval = 10e9 // time to wait before checking for new revs
|
|
||||||
)
|
|
||||||
|
|
||||||
var (
|
|
||||||
goroot = path.Join(os.Getenv("PWD"), "goroot")
|
|
||||||
releaseRegexp = regexp.MustCompile(`^release\.[0-9\-]+`)
|
|
||||||
dashboardhost = flag.String("dashboard", "godashboard.appspot.com",
|
|
||||||
"Godashboard Host")
|
|
||||||
)
|
|
||||||
|
|
||||||
type Builder struct {
|
|
||||||
name string
|
|
||||||
goos, goarch string
|
|
||||||
key string
|
|
||||||
codeUsername string
|
|
||||||
codePassword string
|
|
||||||
}
|
|
||||||
|
|
||||||
func main() {
|
|
||||||
flag.Parse()
|
|
||||||
builders := make(map[string]*Builder)
|
|
||||||
if len(flag.Args()) == 0{
|
|
||||||
log.Exit("No builders specified.")
|
|
||||||
}
|
|
||||||
for _, builder := range flag.Args() {
|
|
||||||
b, err := NewBuilder(builder)
|
|
||||||
if err != nil {
|
|
||||||
log.Exit(err)
|
|
||||||
}
|
|
||||||
builders[builder] = b
|
|
||||||
}
|
|
||||||
err := run(nil, "", "hg", "clone", hgUrl, goroot)
|
|
||||||
if err != nil {
|
|
||||||
log.Exit("Error cloning repository:", err)
|
|
||||||
}
|
|
||||||
// check for new commits and build them
|
|
||||||
for {
|
|
||||||
err := run(nil, goroot, "hg", "pull", "-u")
|
|
||||||
if err != nil {
|
|
||||||
log.Stderr("hg pull failed:", err)
|
|
||||||
time.Sleep(waitInterval)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
built := false
|
|
||||||
for _, b := range builders {
|
|
||||||
built = b.tryBuild() || built
|
|
||||||
}
|
|
||||||
// only wait if we didn't do anything
|
|
||||||
if !built {
|
|
||||||
time.Sleep(waitInterval)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func NewBuilder(builder string) (*Builder, os.Error) {
|
|
||||||
b := &Builder{name:builder}
|
|
||||||
|
|
||||||
// get goos/goarch from builder string
|
|
||||||
s := strings.Split(builder, "-", 3)
|
|
||||||
if len(s) == 2 {
|
|
||||||
b.goos, b.goarch = s[0], s[1]
|
|
||||||
} else {
|
|
||||||
return nil, os.NewError(fmt.Sprintf(
|
|
||||||
"unsupported builder form: %s", builder))
|
|
||||||
}
|
|
||||||
|
|
||||||
// read keys from keyfile
|
|
||||||
fn := path.Join(os.Getenv("HOME"), ".gobuildkey")
|
|
||||||
if isFile(fn + "-" + b.name) { // builder-specific file
|
|
||||||
fn += "-" + b.name
|
|
||||||
}
|
|
||||||
c, err := ioutil.ReadFile(fn)
|
|
||||||
if err != nil {
|
|
||||||
return nil, os.NewError(fmt.Sprintf("readKeys %s (%s): %s",
|
|
||||||
b.name, fn, err))
|
|
||||||
}
|
|
||||||
v := strings.Split(string(c), "\n", -1)
|
|
||||||
b.key = v[0]
|
|
||||||
if len(v) >= 3 {
|
|
||||||
b.codeUsername, b.codePassword = v[1], v[2]
|
|
||||||
}
|
|
||||||
|
|
||||||
return b, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// tryBuild checks for a new commit for this builder,
|
|
||||||
// and builds it if one is found.
|
|
||||||
// Its return value indicates whether a build happened or not.
|
|
||||||
func (b *Builder) tryBuild() bool {
|
|
||||||
c, err := b.nextCommit()
|
|
||||||
if err != nil {
|
|
||||||
log.Stderr(err)
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
if c == nil {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
log.Stderr("Building new revision: ", c.num)
|
|
||||||
err = b.build(*c)
|
|
||||||
if err != nil {
|
|
||||||
log.Stderr(err)
|
|
||||||
}
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|
||||||
// nextCommit returns the next unbuilt Commit for this builder
|
|
||||||
func (b *Builder) nextCommit() (nextC *Commit, err os.Error) {
|
|
||||||
defer func() {
|
|
||||||
if err != nil {
|
|
||||||
err = os.NewError(fmt.Sprintf(
|
|
||||||
"%s nextCommit: %s", b.name, err))
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
hw, err := b.getHighWater()
|
|
||||||
if err != nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
c, err := getCommit(hw)
|
|
||||||
if err != nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
next := c.num + 1
|
|
||||||
c, err = getCommit(strconv.Itoa(next))
|
|
||||||
if err == nil || c.num == next {
|
|
||||||
return &c, nil
|
|
||||||
}
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (b *Builder) build(c Commit) (err os.Error) {
|
|
||||||
defer func() {
|
|
||||||
if err != nil {
|
|
||||||
err = os.NewError(fmt.Sprintf(
|
|
||||||
"%s buildRev commit: %s: %s",
|
|
||||||
b.name, c.num, err))
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
|
|
||||||
// destroy old build candidate
|
|
||||||
err = run(nil, "", "rm", "-Rf", "go")
|
|
||||||
if err != nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// clone repo at revision num (new candidate)
|
|
||||||
err = run(nil, "",
|
|
||||||
"hg", "clone",
|
|
||||||
"-r", strconv.Itoa(c.num),
|
|
||||||
goroot, "go")
|
|
||||||
if err != nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// set up environment for build/bench execution
|
|
||||||
env := []string{
|
|
||||||
"GOOS=" + b.goos,
|
|
||||||
"GOARCH=" + b.goarch,
|
|
||||||
"GOROOT_FINAL=/usr/local/go",
|
|
||||||
"PATH=" + os.Getenv("PATH"),
|
|
||||||
}
|
|
||||||
srcDir := path.Join("", "go", "src")
|
|
||||||
|
|
||||||
// build the release candidate
|
|
||||||
buildLog, status, err := runLog(env, srcDir, "./all.bash")
|
|
||||||
if err != nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if status != 0 {
|
|
||||||
// record failure
|
|
||||||
return b.recordResult(buildLog, c)
|
|
||||||
}
|
|
||||||
|
|
||||||
// record success
|
|
||||||
if err = b.recordResult("", c); err != nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// run benchmarks and send to dashboard
|
|
||||||
pkgDir := path.Join(srcDir, "pkg")
|
|
||||||
benchLog, _, err := runLog(env, pkgDir, "gomake", "bench")
|
|
||||||
if err != nil {
|
|
||||||
log.Stderr("gomake bench:", err)
|
|
||||||
} else if err = b.recordBenchmarks(benchLog, c); err != nil {
|
|
||||||
log.Stderr("recordBenchmarks:", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// finish here if codeUsername and codePassword aren't set
|
|
||||||
if b.codeUsername == "" || b.codePassword == "" {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// if this is a release, create tgz and upload to google code
|
|
||||||
if release := releaseRegexp.FindString(c.desc); release != "" {
|
|
||||||
// clean out build state
|
|
||||||
err = run(env, srcDir, "sh", "clean.bash", "--nopkg")
|
|
||||||
if err != nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
// upload binary release
|
|
||||||
err = b.codeUpload(release)
|
|
||||||
if err != nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
func (b *Builder) codeUpload(release string) (err os.Error) {
|
|
||||||
defer func() {
|
|
||||||
if err != nil {
|
|
||||||
err = os.NewError(fmt.Sprintf(
|
|
||||||
"%s codeUpload release: %s: %s",
|
|
||||||
b.name, release, err))
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
fn := fmt.Sprintf("%s.%s-%s.tar.gz", release, b.goos, b.goarch)
|
|
||||||
err = run(nil, "", "tar", "czf", fn, "go")
|
|
||||||
if err != nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
return run(nil, "", "python",
|
|
||||||
path.Join(goroot, codePyScript),
|
|
||||||
"-s", release,
|
|
||||||
"-p", codeProject,
|
|
||||||
"-u", b.codeUsername,
|
|
||||||
"-w", b.codePassword,
|
|
||||||
"-l", fmt.Sprintf("%s,%s", b.goos, b.goarch),
|
|
||||||
fn)
|
|
||||||
}
|
|
||||||
|
|
||||||
func isDirectory(name string) bool {
|
|
||||||
s, err := os.Stat(name)
|
|
||||||
return err == nil && s.IsDirectory()
|
|
||||||
}
|
|
||||||
|
|
||||||
func isFile(name string) bool {
|
|
||||||
s, err := os.Stat(name)
|
|
||||||
return err == nil && (s.IsRegular() || s.IsSymlink())
|
|
||||||
}
|
|
||||||
Loading…
Reference in New Issue
Block a user