diff --git a/misc/dashboard/builder/Makefile b/misc/dashboard/builder/Makefile index cff47942aac..f1d9c549722 100644 --- a/misc/dashboard/builder/Makefile +++ b/misc/dashboard/builder/Makefile @@ -7,7 +7,6 @@ include ../../../src/Make.inc TARG=gobuilder GOFILES=\ exec.go\ - hg.go\ http.go\ main.go\ package.go\ diff --git a/misc/dashboard/builder/doc.go b/misc/dashboard/builder/doc.go index 7bb7ccbe38d..30d8fe94889 100644 --- a/misc/dashboard/builder/doc.go +++ b/misc/dashboard/builder/doc.go @@ -14,9 +14,6 @@ It periodically pulls updates from the Go Mercurial repository. When a newer revision is found, Go Builder creates a clone of the repository, runs all.bash, and reports build success or failure to the Go Dashboard. -For a successful build, Go Builder will also run benchmarks -(cd $GOROOT/src/pkg; make bench) and send the results to the Go Dashboard. - For a release revision (a change description that matches "release.YYYY-MM-DD"), Go Builder will create a tar.gz archive of the GOROOT and deliver it to the Go Google Code project's downloads section. @@ -34,8 +31,6 @@ Optional flags: The location of the Go Dashboard application to which Go Builder will report its results. - -bench: Run benchmarks - -release: Build and deliver binary release archive -rev=N: Build revision N and exit @@ -45,7 +40,7 @@ Optional flags: -v: Verbose logging -external: External package builder mode (will not report Go build - state to dashboard, issue releases, or run benchmarks) + state to dashboard or issue releases) The key file should be located at $HOME/.gobuildkey or, for a builder-specific key, $HOME/.gobuildkey-$BUILDER (eg, $HOME/.gobuildkey-linux-amd64). diff --git a/misc/dashboard/builder/exec.go b/misc/dashboard/builder/exec.go index 3c6fbdced47..988d216cec4 100644 --- a/misc/dashboard/builder/exec.go +++ b/misc/dashboard/builder/exec.go @@ -18,7 +18,7 @@ func run(envv []string, dir string, argv ...string) os.Error { if *verbose { log.Println("run", argv) } - bin, err := pathLookup(argv[0]) + bin, err := lookPath(argv[0]) if err != nil { return err } @@ -36,7 +36,7 @@ func runLog(envv []string, logfile, dir string, argv ...string) (output string, if *verbose { log.Println("runLog", argv) } - bin, err := pathLookup(argv[0]) + bin, err := lookPath(argv[0]) if err != nil { return } @@ -67,10 +67,10 @@ func runLog(envv []string, logfile, dir string, argv ...string) (output string, return b.String(), wait.WaitStatus.ExitStatus(), nil } -// Find bin in PATH if a relative or absolute path hasn't been specified -func pathLookup(s string) (string, os.Error) { - if strings.HasPrefix(s, "/") || strings.HasPrefix(s, "./") || strings.HasPrefix(s, "../") { - return s, nil +// lookPath looks for cmd in $PATH if cmd does not begin with / or ./ or ../. +func lookPath(cmd string) (string, os.Error) { + if strings.HasPrefix(cmd, "/") || strings.HasPrefix(cmd, "./") || strings.HasPrefix(cmd, "../") { + return cmd, nil } - return exec.LookPath(s) + return exec.LookPath(cmd) } diff --git a/misc/dashboard/builder/hg.go b/misc/dashboard/builder/hg.go deleted file mode 100644 index d4310845d1b..00000000000 --- a/misc/dashboard/builder/hg.go +++ /dev/null @@ -1,86 +0,0 @@ -// Copyright 2011 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package main - -import ( - "fmt" - "os" - "regexp" - "strconv" - "strings" -) - -type Commit struct { - num int // mercurial revision number - node string // mercurial hash - parent string // hash of commit's parent - user string // author's Name - date string // date of commit - desc string // description -} - -// getCommit returns details about the Commit specified by the revision hash -func getCommit(rev string) (c Commit, err os.Error) { - defer func() { - if err != nil { - err = fmt.Errorf("getCommit: %s: %s", rev, err) - } - }() - parts, err := getCommitParts(rev) - if err != nil { - return - } - num, err := strconv.Atoi(parts[0]) - if err != nil { - return - } - parent := "" - if num > 0 { - prev := strconv.Itoa(num - 1) - if pparts, err := getCommitParts(prev); err == nil { - parent = pparts[1] - } - } - user := strings.Replace(parts[2], "<", "<", -1) - user = strings.Replace(user, ">", ">", -1) - return Commit{num, parts[1], parent, user, parts[3], parts[4]}, nil -} - -func getCommitParts(rev string) (parts []string, err os.Error) { - const format = "{rev}>{node}>{author|escape}>{date}>{desc}" - s, _, err := runLog(nil, "", goroot, - "hg", "log", - "--encoding", "utf-8", - "--rev", rev, - "--limit", "1", - "--template", format, - ) - if err != nil { - return - } - return strings.Split(s, ">", 5), nil -} - -var revisionRe = regexp.MustCompile(`([0-9]+):[0-9a-f]+$`) - -// getTag fetches a Commit by finding the first hg tag that matches re. -func getTag(re *regexp.Regexp) (c Commit, tag string, err os.Error) { - o, _, err := runLog(nil, "", goroot, "hg", "tags") - for _, l := range strings.Split(o, "\n", -1) { - tag = re.FindString(l) - if tag == "" { - continue - } - s := revisionRe.FindStringSubmatch(l) - if s == nil { - err = os.NewError("couldn't find revision number") - return - } - c, err = getCommit(s[1]) - return - } - err = os.NewError("no matching tag found") - return -} diff --git a/misc/dashboard/builder/http.go b/misc/dashboard/builder/http.go index dba19ba8fd7..6749e3528e1 100644 --- a/misc/dashboard/builder/http.go +++ b/misc/dashboard/builder/http.go @@ -6,84 +6,104 @@ package main import ( "bytes" - "encoding/base64" - "encoding/binary" "fmt" "http" "json" "log" "os" - "regexp" "strconv" ) -// getHighWater returns the current highwater revision hash for this builder -func (b *Builder) getHighWater() (rev string, err os.Error) { - url := fmt.Sprintf("http://%s/hw-get?builder=%s", *dashboard, b.name) - r, _, err := http.Get(url) +type param map[string]string + +// dash runs the given method and command on the dashboard. +// If args is not nil, it is the query or post parameters. +// If resp is not nil, dash unmarshals the body as JSON into resp. +func dash(meth, cmd string, resp interface{}, args param) os.Error { + var r *http.Response + var err os.Error + if *verbose { + log.Println("dash", cmd, args) + } + cmd = "http://" + *dashboard + "/" + cmd + switch meth { + case "GET": + if args != nil { + m := make(map[string][]string) + for k, v := range args { + m[k] = []string{v} + } + cmd += "?" + http.EncodeQuery(m) + } + r, _, err = http.Get(cmd) + case "POST": + r, err = http.PostForm(cmd, args) + default: + return fmt.Errorf("unknown method %q", meth) + } if err != nil { + return err + } + defer r.Body.Close() + var buf bytes.Buffer + buf.ReadFrom(r.Body) + if resp != nil { + if err = json.Unmarshal(buf.Bytes(), resp); err != nil { + log.Printf("json unmarshal %#q: %s\n", buf.Bytes(), err) + return err + } + } + return nil +} + +func dashStatus(meth, cmd string, args param) os.Error { + var resp struct { + Status string + Error string + } + err := dash(meth, cmd, &resp, args) + if err != nil { + return err + } + if resp.Status != "OK" { + return os.NewError("/build: " + resp.Error) + } + return nil +} + +// todo returns the next hash to build. +func (b *Builder) todo() (rev string, err os.Error) { + var resp []struct{ + Hash string + } + if err = dash("GET", "todo", &resp, param{"builder": b.name}); err != nil { return } - buf := new(bytes.Buffer) - _, err = buf.ReadFrom(r.Body) - if err != nil { - return + if len(resp) > 0 { + rev = resp[0].Hash } - r.Body.Close() - return buf.String(), nil + return } // recordResult sends build results to the dashboard -func (b *Builder) recordResult(buildLog string, c Commit) os.Error { - return httpCommand("build", map[string]string{ +func (b *Builder) recordResult(buildLog string, hash string) os.Error { + return dash("POST", "build", nil, param{ "builder": b.name, "key": b.key, - "node": c.node, - "parent": c.parent, - "user": c.user, - "date": c.date, - "desc": c.desc, + "node": hash, "log": buildLog, }) } -// match lines like: "package.BechmarkFunc 100000 999 ns/op" -var benchmarkRegexp = regexp.MustCompile("([^\n\t ]+)[\t ]+([0-9]+)[\t ]+([0-9]+) ns/op") - -// recordBenchmarks sends benchmark results to the dashboard -func (b *Builder) recordBenchmarks(benchLog string, c Commit) os.Error { - results := benchmarkRegexp.FindAllStringSubmatch(benchLog, -1) - var buf bytes.Buffer - b64 := base64.NewEncoder(base64.StdEncoding, &buf) - for _, r := range results { - for _, s := range r[1:] { - binary.Write(b64, binary.BigEndian, uint16(len(s))) - b64.Write([]byte(s)) - } - } - b64.Close() - return httpCommand("benchmarks", map[string]string{ - "builder": b.name, - "key": b.key, - "node": c.node, - "benchmarkdata": buf.String(), - }) -} - -// getPackages fetches a list of package paths from the dashboard -func getPackages() (pkgs []string, err os.Error) { - r, _, err := http.Get(fmt.Sprintf("http://%v/package?fmt=json", *dashboard)) - if err != nil { - return - } - defer r.Body.Close() - d := json.NewDecoder(r.Body) +// packages fetches a list of package paths from the dashboard +func packages() (pkgs []string, err os.Error) { var resp struct { Packages []struct { Path string } } - if err = d.Decode(&resp); err != nil { + err = dash("GET", "package", &resp, param{"fmt": "json"}) + if err != nil { return } for _, p := range resp.Packages { @@ -93,24 +113,36 @@ func getPackages() (pkgs []string, err os.Error) { } // updatePackage sends package build results and info to the dashboard -func (b *Builder) updatePackage(pkg string, state bool, buildLog, info string, c Commit) os.Error { - args := map[string]string{ +func (b *Builder) updatePackage(pkg string, state bool, buildLog, info string, hash string) os.Error { + return dash("POST", "package", nil, param{ "builder": b.name, "key": b.key, "path": pkg, "state": strconv.Btoa(state), "log": buildLog, "info": info, - "go_rev": strconv.Itoa(c.num), - } - return httpCommand("package", args) + "go_rev": hash[:12], + }) } -func httpCommand(cmd string, args map[string]string) os.Error { - if *verbose { - log.Println("httpCommand", cmd, args) - } - url := fmt.Sprintf("http://%v/%v", *dashboard, cmd) - _, err := http.PostForm(url, args) - return err +// postCommit informs the dashboard of a new commit +func postCommit(key string, l *HgLog) os.Error { + return dashStatus("POST", "commit", param{ + "key": key, + "node": l.Hash, + "date": l.Date, + "user": l.Author, + "parent": l.Parent, + "desc": l.Desc, + }) +} + +// dashboardCommit returns true if the dashboard knows about hash. +func dashboardCommit(hash string) bool { + err := dashStatus("GET", "commit", param{"node": hash}) + if err != nil { + log.Printf("check %s: %s", hash, err) + return false + } + return true } diff --git a/misc/dashboard/builder/main.go b/misc/dashboard/builder/main.go index d11cbb1337d..c8583f79d9a 100644 --- a/misc/dashboard/builder/main.go +++ b/misc/dashboard/builder/main.go @@ -5,7 +5,6 @@ package main import ( - "container/vector" "flag" "fmt" "io/ioutil" @@ -16,13 +15,14 @@ import ( "strconv" "strings" "time" + "xml" ) const ( codeProject = "go" codePyScript = "misc/dashboard/googlecode_upload.py" hgUrl = "https://go.googlecode.com/hg/" - waitInterval = 10e9 // time to wait before checking for new revs + waitInterval = 30e9 // time to wait before checking for new revs mkdirPerm = 0750 pkgBuildInterval = 1e9 * 60 * 60 * 24 // rebuild packages every 24 hours ) @@ -46,16 +46,10 @@ type Builder struct { codePassword string } -type BenchRequest struct { - builder *Builder - commit Commit - path string -} - var ( buildroot = flag.String("buildroot", path.Join(os.TempDir(), "gobuilder"), "Directory under which to build") + commitFlag = flag.Bool("commit", false, "upload information about new commits") dashboard = flag.String("dashboard", "godashboard.appspot.com", "Go Dashboard Host") - runBenchmarks = flag.Bool("bench", false, "Run benchmarks") buildRelease = flag.Bool("release", false, "Build and upload binary release archives") buildRevision = flag.String("rev", "", "Build specified revision and exit") buildCmd = flag.String("cmd", "./all.bash", "Build command (specify absolute or relative to go/src/)") @@ -67,7 +61,6 @@ var ( var ( goroot string releaseRegexp = regexp.MustCompile(`^(release|weekly)\.[0-9\-.]+`) - benchRequests vector.Vector ) func main() { @@ -77,7 +70,7 @@ func main() { os.Exit(2) } flag.Parse() - if len(flag.Args()) == 0 { + if len(flag.Args()) == 0 && !*commitFlag { flag.Usage() } goroot = path.Join(*buildroot, "goroot") @@ -100,18 +93,25 @@ func main() { if err := run(nil, *buildroot, "hg", "clone", hgUrl, goroot); err != nil { log.Fatal("Error cloning repository:", err) } + + if *commitFlag { + if len(flag.Args()) == 0 { + commitWatcher() + return + } + go commitWatcher() + } // if specified, build revision and return if *buildRevision != "" { - c, err := getCommit(*buildRevision) + hash, err := fullHash(*buildRevision) if err != nil { log.Fatal("Error finding revision: ", err) } for _, b := range builders { - if err := b.buildCommit(c); err != nil { + if err := b.buildHash(hash); err != nil { log.Println(err) } - runQueuedBenchmark() } return } @@ -134,6 +134,7 @@ func main() { continue } built := false + t := time.Nanoseconds() if *parallel { done := make(chan bool) for _, b := range builders { @@ -149,46 +150,15 @@ func main() { built = b.build() || built } } - // only run benchmarks if we didn't build anything - // so that they don't hold up the builder queue + // sleep if there was nothing to build if !built { - if !runQueuedBenchmark() { - // if we have no benchmarks to do, pause - time.Sleep(waitInterval) - } - // after running one benchmark, - // continue to find and build new revisions. + time.Sleep(waitInterval) + } + // sleep if we're looping too fast. + t1 := time.Nanoseconds() - t + if t1 < waitInterval { + time.Sleep(waitInterval - t1) } - } -} - -func runQueuedBenchmark() bool { - if benchRequests.Len() == 0 { - return false - } - runBenchmark(benchRequests.Pop().(BenchRequest)) - return true -} - -func runBenchmark(r BenchRequest) { - // run benchmarks and send to dashboard - log.Println(r.builder.name, "benchmarking", r.commit.num) - defer os.RemoveAll(r.path) - pkg := path.Join(r.path, "go", "src", "pkg") - bin := path.Join(r.path, "go", "bin") - env := []string{ - "GOOS=" + r.builder.goos, - "GOARCH=" + r.builder.goarch, - "PATH=" + bin + ":" + os.Getenv("PATH"), - } - logfile := path.Join(r.path, "bench.log") - benchLog, _, err := runLog(env, logfile, pkg, "gomake", "bench") - if err != nil { - log.Println(r.builder.name, "gomake bench:", err) - return - } - if err = r.builder.recordBenchmarks(benchLog, r.commit); err != nil { - log.Println("recordBenchmarks:", err) } } @@ -235,7 +205,7 @@ func (b *Builder) buildExternal() { log.Println("hg pull failed:", err) continue } - c, tag, err := getTag(releaseRegexp) + hash, tag, err := firstTag(releaseRegexp) if err != nil { log.Println(err) continue @@ -249,8 +219,8 @@ func (b *Builder) buildExternal() { if tag == prevTag && time.Nanoseconds() < nextBuild { continue } - // buildCommit will also build the packages - if err := b.buildCommit(c); err != nil { + // build will also build the packages + if err := b.buildHash(hash); err != nil { log.Println(err) continue } @@ -269,65 +239,37 @@ func (b *Builder) build() bool { log.Println(b.name, "build:", err) } }() - c, err := b.nextCommit() + hash, err := b.todo() if err != nil { log.Println(err) return false } - if c == nil { + if hash == "" { return false } - err = b.buildCommit(*c) + err = b.buildHash(hash) if err != nil { log.Println(err) } return true } -// nextCommit returns the next unbuilt Commit for this builder -func (b *Builder) nextCommit() (nextC *Commit, err os.Error) { +func (b *Builder) buildHash(hash string) (err os.Error) { defer func() { if err != nil { - err = fmt.Errorf("%s nextCommit: %s", b.name, err) - } - }() - hw, err := b.getHighWater() - if err != nil { - return - } - c, err := getCommit(hw) - if err != nil { - return - } - next := c.num + 1 - c, err = getCommit(strconv.Itoa(next)) - if err == nil && c.num == next { - return &c, nil - } - return nil, nil -} - -func (b *Builder) buildCommit(c Commit) (err os.Error) { - defer func() { - if err != nil { - err = fmt.Errorf("%s buildCommit: %d: %s", b.name, c.num, err) + err = fmt.Errorf("%s build: %s: %s", b.name, hash, err) } }() - log.Println(b.name, "building", c.num) + log.Println(b.name, "building", hash) // create place in which to do work - workpath := path.Join(*buildroot, b.name+"-"+strconv.Itoa(c.num)) + workpath := path.Join(*buildroot, b.name+"-"+hash[:12]) err = os.Mkdir(workpath, mkdirPerm) if err != nil { return } - benchRequested := false - defer func() { - if !benchRequested { - os.RemoveAll(workpath) - } - }() + defer os.RemoveAll(workpath) // clone repo err = run(nil, workpath, "hg", "clone", goroot, "go") @@ -337,7 +279,7 @@ func (b *Builder) buildCommit(c Commit) (err os.Error) { // update to specified revision err = run(nil, path.Join(workpath, "go"), - "hg", "update", "-r", strconv.Itoa(c.num)) + "hg", "update", hash) if err != nil { return } @@ -356,36 +298,27 @@ func (b *Builder) buildCommit(c Commit) (err os.Error) { if status != 0 { return os.NewError("go build failed") } - return b.buildPackages(workpath, c) + return b.buildPackages(workpath, hash) } if status != 0 { // record failure - return b.recordResult(buildLog, c) + return b.recordResult(buildLog, hash) } // record success - if err = b.recordResult("", c); err != nil { + if err = b.recordResult("", hash); err != nil { return fmt.Errorf("recordResult: %s", err) } - // send benchmark request if benchmarks are enabled - if *runBenchmarks { - benchRequests.Insert(0, BenchRequest{ - builder: b, - commit: c, - path: workpath, - }) - benchRequested = true - } - // finish here if codeUsername and codePassword aren't set if b.codeUsername == "" || b.codePassword == "" || !*buildRelease { return } // if this is a release, create tgz and upload to google code - if release := releaseRegexp.FindString(c.desc); release != "" { + releaseHash, release, err := firstTag(releaseRegexp) + if hash == releaseHash { // clean out build state err = run(b.envv(), srcDir, "./clean.bash", "--nopkg") if err != nil { @@ -431,3 +364,209 @@ func isFile(name string) bool { s, err := os.Stat(name) return err == nil && (s.IsRegular() || s.IsSymlink()) } + +// commitWatcher polls hg for new commits and tells the dashboard about them. +func commitWatcher() { + // Create builder just to get master key. + b, err := NewBuilder("mercurial-commit") + if err != nil { + log.Fatal(err) + } + for { + if *verbose { + log.Printf("poll...") + } + commitPoll(b.key) + if *verbose { + log.Printf("sleep...") + } + time.Sleep(60e9) + } +} + +// HgLog represents a single Mercurial revision. +type HgLog struct { + Hash string + Author string + Date string + Desc string + Parent string + + // Internal metadata + added bool +} + +// logByHash is a cache of all Mercurial revisions we know about, +// indexed by full hash. +var logByHash = map[string]*HgLog{} + +// xmlLogTemplate is a template to pass to Mercurial to make +// hg log print the log in valid XML for parsing with xml.Unmarshal. +const xmlLogTemplate = ` + + {node|escape} + {parent|escape} + {author|escape} + {date} + {desc|escape} + +` + +// commitPoll pulls any new revisions from the hg server +// and tells the server about them. +func commitPoll(key string) { + // Catch unexpected panics. + defer func() { + if err := recover(); err != nil { + log.Printf("commitPoll panic: %s", err) + } + }() + + if err := run(nil, goroot, "hg", "pull"); err != nil { + log.Printf("hg pull: %v", err) + return + } + + const N = 20 // how many revisions to grab + + data, _, err := runLog(nil, "", goroot, "hg", "log", + "--encoding=utf-8", + "--limit=" + strconv.Itoa(N), + "--template=" + xmlLogTemplate, + ) + if err != nil { + log.Printf("hg log: %v", err) + return + } + + var logStruct struct { + Log []HgLog + } + err = xml.Unmarshal(strings.NewReader("" + data + ""), &logStruct) + if err != nil { + log.Printf("unmarshal hg log: %v", err) + return + } + + logs := logStruct.Log + + // Pass 1. Fill in parents and add new log entries to logsByHash. + // Empty parent means take parent from next log entry. + // Non-empty parent has form 1234:hashhashhash; we weant full hash. + for i := range logs { + l := &logs[i] + log.Printf("hg log: %s < %s\n", l.Hash, l.Parent) + if l.Parent == "" && i+1 < len(logs) { + l.Parent = logs[i+1].Hash + } else if l.Parent != "" { + l.Parent, _ = fullHash(l.Parent) + } + if l.Parent == "" { + // Can't create node without parent. + continue + } + + if logByHash[l.Hash] == nil { + // Make copy to avoid pinning entire slice when only one entry is new. + t := *l + logByHash[t.Hash] = &t + } + } + + for i := range logs { + l := &logs[i] + if l.Parent == "" { + continue + } + addCommit(l.Hash, key) + } +} + +// addCommit adds the commit with the named hash to the dashboard. +// key is the secret key for authentication to the dashboard. +// It avoids duplicate effort. +func addCommit(hash, key string) bool { + l := logByHash[hash] + if l == nil { + return false + } + if l.added { + return true + } + + // Check for already added, perhaps in an earlier run. + if dashboardCommit(hash) { + log.Printf("%s already on dashboard\n", hash) + // Record that this hash is on the dashboard, + // as must be all its parents. + for l != nil { + l.added = true + l = logByHash[l.Parent] + } + return true + } + + // Create parent first, to maintain some semblance of order. + if !addCommit(l.Parent, key) { + return false + } + + // Create commit. + if err := postCommit(key, l); err != nil { + log.Printf("faield to add %s to dashboard: %v", err) + return false + } + return true +} + +// fullHash returns the full hash for the given Mercurial revision. +func fullHash(rev string) (hash string, err os.Error) { + defer func() { + if err != nil { + err = fmt.Errorf("fullHash: %s: %s", rev, err) + } + }() + s, _, err := runLog(nil, "", goroot, + "hg", "log", + "--encoding=utf-8", + "--rev="+rev, + "--limit=1", + "--template={node}", + ) + if err != nil { + return + } + s = strings.TrimSpace(s) + if s == "" { + return "", fmt.Errorf("cannot find revision") + } + if len(s) != 20 { + return "", fmt.Errorf("hg returned invalid hash " + s) + } + return s, nil +} + +var revisionRe = regexp.MustCompile(`^([^ ]+) +[0-9]+:([0-9a-f]+)$`) + +// firstTag returns the hash and tag of the most recent tag matching re. +func firstTag(re *regexp.Regexp) (hash string, tag string, err os.Error) { + o, _, err := runLog(nil, "", goroot, "hg", "tags") + for _, l := range strings.Split(o, "\n", -1) { + if l == "" { + continue + } + s := revisionRe.FindStringSubmatch(l) + if s == nil { + err = os.NewError("couldn't find revision number") + return + } + if !re.MatchString(s[1]) { + continue + } + tag = s[1] + hash, err = fullHash(s[3]) + return + } + err = os.NewError("no matching tag found") + return +} diff --git a/misc/dashboard/builder/package.go b/misc/dashboard/builder/package.go index 6e9f9ff3962..ee65d7669d0 100644 --- a/misc/dashboard/builder/package.go +++ b/misc/dashboard/builder/package.go @@ -13,8 +13,8 @@ import ( "path" ) -func (b *Builder) buildPackages(workpath string, c Commit) os.Error { - pkgs, err := getPackages() +func (b *Builder) buildPackages(workpath string, hash string) os.Error { + pkgs, err := packages() if err != nil { return err } @@ -32,13 +32,13 @@ func (b *Builder) buildPackages(workpath string, c Commit) os.Error { built := code != 0 // get doc comment from package source - info, err := getPackageComment(p, path.Join(goroot, "pkg", p)) + info, err := packageComment(p, path.Join(goroot, "pkg", p)) if err != nil { log.Printf("goinstall %v: %v", p, err) } // update dashboard with build state + info - err = b.updatePackage(p, built, buildLog, info, c) + err = b.updatePackage(p, built, buildLog, info, hash) if err != nil { log.Printf("updatePackage %v: %v", p, err) } @@ -46,7 +46,7 @@ func (b *Builder) buildPackages(workpath string, c Commit) os.Error { return nil } -func getPackageComment(pkg, pkgpath string) (info string, err os.Error) { +func packageComment(pkg, pkgpath string) (info string, err os.Error) { fset := token.NewFileSet() pkgs, err := parser.ParseDir(fset, pkgpath, nil, parser.PackageClauseOnly|parser.ParseComments) if err != nil { diff --git a/misc/dashboard/godashboard/app.yaml b/misc/dashboard/godashboard/app.yaml index aec559dccd5..455da57f0c2 100644 --- a/misc/dashboard/godashboard/app.yaml +++ b/misc/dashboard/godashboard/app.yaml @@ -1,5 +1,5 @@ application: godashboard -version: 5 +version: 6 runtime: python api_version: 1 diff --git a/misc/dashboard/godashboard/gobuild.py b/misc/dashboard/godashboard/gobuild.py index 035bf842fb5..baddb0e9b8f 100644 --- a/misc/dashboard/godashboard/gobuild.py +++ b/misc/dashboard/godashboard/gobuild.py @@ -5,6 +5,7 @@ # This is the server part of the continuous build system for Go. It must be run # by AppEngine. +from django.utils import simplejson from google.appengine.api import mail from google.appengine.api import memcache from google.appengine.ext import db @@ -50,10 +51,6 @@ class Commit(db.Model): fail_notification_sent = db.BooleanProperty() -class Cache(db.Model): - data = db.BlobProperty() - expire = db.IntegerProperty() - # A CompressedLog contains the textual build log of a failed build. # The key name is the hex digest of the SHA256 hash of the contents. # The contents is bz2 compressed. @@ -62,23 +59,6 @@ class CompressedLog(db.Model): N = 30 -def cache_get(key): - c = Cache.get_by_key_name(key) - if c is None or c.expire < time.time(): - return None - return c.data - -def cache_set(key, val, timeout): - c = Cache(key_name = key) - c.data = val - c.expire = int(time.time() + timeout) - c.put() - -def cache_del(key): - c = Cache.get_by_key_name(key) - if c is not None: - c.delete() - def builderInfo(b): f = b.split('-', 3) goos = f[0] @@ -96,7 +76,7 @@ def builderset(): for c in results: builders.update(set(parseBuild(build)['builder'] for build in c.builds)) return builders - + class MainPage(webapp.RequestHandler): def get(self): self.response.headers['Content-Type'] = 'text/html; charset=utf-8' @@ -147,7 +127,30 @@ class MainPage(webapp.RequestHandler): path = os.path.join(os.path.dirname(__file__), 'main.html') self.response.out.write(template.render(path, values)) -class GetHighwater(webapp.RequestHandler): +# A DashboardHandler is a webapp.RequestHandler but provides +# authenticated_post - called by post after authenticating +# json - writes object in json format to response output +class DashboardHandler(webapp.RequestHandler): + def post(self): + if not auth(self.request): + self.response.set_status(403) + return + self.authenticated_post() + + def authenticated_post(self): + return + + def json(self, obj): + self.response.set_status(200) + simplejson.dump(obj, self.response.out) + return + +def auth(req): + k = req.get('key') + return k == hmac.new(key.accessKey, req.get('builder')).hexdigest() or k == key.accessKey + +# Todo serves /todo. It tells the builder which commits need to be built. +class Todo(DashboardHandler): def get(self): builder = self.request.get('builder') key = 'todo-%s' % builder @@ -155,28 +158,19 @@ class GetHighwater(webapp.RequestHandler): if response is None: # Fell out of memcache. Rebuild from datastore results. # We walk the commit list looking for nodes that have not - # been built by this builder and record the *parents* of those - # nodes, because each builder builds the revision *after* the - # one return (because we might not know about the latest - # revision). + # been built by this builder. q = Commit.all() q.order('-__key__') todo = [] - need = False first = None for c in q.fetch(N+1): if first is None: first = c - if need: - todo.append(c.node) - need = not built(c, builder) - if not todo: - todo.append(first.node) - response = ' '.join(todo) + if not built(c, builder): + todo.append({'Hash': c.node}) + response = simplejson.dumps(todo) memcache.set(key, response, 3600) self.response.set_status(200) - if self.request.get('all') != 'yes': - response = response.split()[0] self.response.out.write(response) def built(c, builder): @@ -185,22 +179,8 @@ def built(c, builder): return True return False -def auth(req): - k = req.get('key') - return k == hmac.new(key.accessKey, req.get('builder')).hexdigest() or k == key.accessKey - -class SetHighwater(webapp.RequestHandler): - def post(self): - if not auth(self.request): - self.response.set_status(403) - return - - # Allow for old builders. - # This is a no-op now: we figure out what to build based - # on the current dashboard status. - return - -class LogHandler(webapp.RequestHandler): +# Log serves /log/. It retrieves log data by content hash. +class LogHandler(DashboardHandler): def get(self): self.response.headers['Content-Type'] = 'text/plain; charset=utf-8' hash = self.request.path[5:] @@ -214,12 +194,8 @@ class LogHandler(webapp.RequestHandler): # Init creates the commit with id 0. Since this commit doesn't have a parent, # it cannot be created by Build. -class Init(webapp.RequestHandler): - def post(self): - if not auth(self.request): - self.response.set_status(403) - return - +class Init(DashboardHandler): + def authenticated_post(self): date = parseDate(self.request.get('date')) node = self.request.get('node') if not validNode(node) or date is None: @@ -239,7 +215,86 @@ class Init(webapp.RequestHandler): self.response.set_status(200) -# Build is the main command: it records the result of a new build. +# The last commit when we switched to using entity groups. +# This is the root of the new commit entity group. +RootCommitKeyName = '00000f26-f32c6f1038207c55d5780231f7484f311020747e' + +# CommitHandler serves /commit. +# A GET of /commit retrieves information about the specified commit. +# A POST of /commit creates a node for the given commit. +# If the commit already exists, the POST silently succeeds (like mkdir -p). +class CommitHandler(DashboardHandler): + def get(self): + node = self.request.get('node') + if not validNode(node): + return self.json({'Status': 'FAIL', 'Error': 'malformed node hash'}) + n = nodeByHash(node) + if n is None: + return self.json({'Status': 'FAIL', 'Error': 'unknown revision'}) + return self.json({'Status': 'OK', 'Node': nodeObj(n)}) + + def authenticated_post(self): + # Require auth with the master key, not a per-builder key. + if self.request.get('builder'): + self.response.set_status(403) + return + + node = self.request.get('node') + date = parseDate(self.request.get('date')) + user = self.request.get('user').encode('utf8') + desc = self.request.get('desc').encode('utf8') + parenthash = self.request.get('parent') + + if not validNode(node) or not validNode(parenthash) or date is None: + return self.json({'Status': 'FAIL', 'Error': 'malformed node, parent, or date'}) + + n = nodeByHash(node) + if n is None: + p = nodeByHash(parenthash) + if p is None: + return self.json({'Status': 'FAIL', 'Error': 'unknown parent'}) + + # Want to create new node in a transaction so that multiple + # requests creating it do not collide and so that multiple requests + # creating different nodes get different sequence numbers. + # All queries within a transaction must include an ancestor, + # but the original datastore objects we used for the dashboard + # have no common ancestor. Instead, we use a well-known + # root node - the last one before we switched to entity groups - + # as the as the common ancestor. + root = Commit.get_by_key_name(RootCommitKeyName) + + def add_commit(): + if nodeByHash(node, ancestor=root) is not None: + return + + # Determine number for this commit. + # Once we have created one new entry it will be lastRooted.num+1, + # but the very first commit created in this scheme will have to use + # last.num's number instead (last is likely not rooted). + q = Commit.all() + q.order('-__key__') + q.ancestor(root) + last = q.fetch(1)[0] + num = last.num+1 + + n = Commit(key_name = '%08x-%s' % (num, node), parent = root) + n.num = num + n.node = node + n.parentnode = parenthash + n.user = user + n.date = date + n.desc = desc + n.put() + db.run_in_transaction(add_commit) + n = nodeByHash(node) + if n is None: + return self.json({'Status': 'FAIL', 'Error': 'failed to create commit node'}) + + return self.json({'Status': 'OK', 'Node': nodeObj(n)}) + +# Build serves /build. +# A POST to /build records a new build result. class Build(webapp.RequestHandler): def post(self): if not auth(self.request): @@ -256,44 +311,33 @@ class Build(webapp.RequestHandler): l.log = bz2.compress(log) l.put() - date = parseDate(self.request.get('date')) - user = self.request.get('user').encode('utf8') - desc = self.request.get('desc').encode('utf8') node = self.request.get('node') - parenthash = self.request.get('parent') - if not validNode(node) or not validNode(parenthash) or date is None: - logging.error("Not valid node ('%s') or bad date (%s %s)", node, date, self.request.get('date')) + if not validNode(node): + logging.error('Invalid node %s' % (node)) self.response.set_status(500) return - q = Commit.all() - q.filter('node =', parenthash) - parent = q.get() - if parent is None: - logging.error('Cannot find parent %s of node %s' % (parenthash, node)) + n = nodeByHash(node) + if n is None: + logging.error('Cannot find node %s' % (node)) self.response.set_status(404) return - parentnum, _ = parent.key().name().split('-', 1) - nodenum = int(parentnum, 16) + 1 - - key_name = '%08x-%s' % (nodenum, node) + nn = n def add_build(): - n = Commit.get_by_key_name(key_name) + n = nodeByHash(node, ancestor=nn) if n is None: - n = Commit(key_name = key_name) - n.num = nodenum - n.node = node - n.parentnode = parenthash - n.user = user - n.date = date - n.desc = desc + logging.error('Cannot find hash in add_build: %s %s' % (builder, node)) + return + s = '%s`%s' % (builder, loghash) for i, b in enumerate(n.builds): if b.split('`', 1)[0] == builder: + # logging.error('Found result for %s %s already' % (builder, node)) n.builds[i] = s break else: + # logging.error('Added result for %s %s' % (builder, node)) n.builds.append(s) n.put() @@ -302,30 +346,7 @@ class Build(webapp.RequestHandler): key = 'todo-%s' % builder memcache.delete(key) - def mark_sent(): - n = Commit.get_by_key_name(key_name) - n.fail_notification_sent = True - n.put() - - n = Commit.get_by_key_name(key_name) - if loghash and not failed(parent, builder) and not n.fail_notification_sent: - subject = const.mail_fail_subject % (builder, desc.split("\n")[0]) - path = os.path.join(os.path.dirname(__file__), 'fail-notify.txt') - body = template.render(path, { - "builder": builder, - "node": node[:12], - "user": user, - "desc": desc, - "loghash": loghash - }) - mail.send_mail( - sender=const.mail_from, - reply_to=const.mail_fail_reply_to, - to=const.mail_fail_to, - subject=subject, - body=body - ) - db.run_in_transaction(mark_sent) + # TODO: Send mail for build breakage. self.response.set_status(200) @@ -342,6 +363,24 @@ def node(num): n = q.get() return n +def nodeByHash(hash, ancestor=None): + q = Commit.all() + q.filter('node =', hash) + if ancestor is not None: + q.ancestor(ancestor) + n = q.get() + return n + +# nodeObj returns a JSON object (ready to be passed to simplejson.dump) describing node. +def nodeObj(n): + return { + 'Hash': n.node, + 'ParentHash': n.parentnode, + 'User': n.user, + 'Date': n.date.strftime('%Y-%m-%d %H:%M %z'), + 'Desc': n.desc, + } + class FixedOffset(datetime.tzinfo): """Fixed offset in minutes east from UTC.""" @@ -417,15 +456,20 @@ def toRev(c): def byBuilder(x, y): return cmp(x['builder'], y['builder']) +# Give old builders work; otherwise they pound on the web site. +class Hwget(DashboardHandler): + def get(self): + self.response.out.write("8000\n") + # This is the URL map for the server. The first three entries are public, the # rest are only used by the builders. application = webapp.WSGIApplication( [('/', MainPage), + ('/hw-get', Hwget), ('/log/.*', LogHandler), - ('/hw-get', GetHighwater), - ('/hw-set', SetHighwater), - + ('/commit', CommitHandler), ('/init', Init), + ('/todo', Todo), ('/build', Build), ], debug=True) diff --git a/misc/dashboard/godashboard/index.yaml b/misc/dashboard/godashboard/index.yaml index 148824bb6af..4a00c4a6fe8 100644 --- a/misc/dashboard/godashboard/index.yaml +++ b/misc/dashboard/godashboard/index.yaml @@ -23,6 +23,12 @@ indexes: - name: __key__ direction: desc +- kind: Commit + ancestor: yes + properties: + - name: __key__ + direction: desc + - kind: Project properties: - name: approved