dashboard: fix for branches

In the new world, one builder runs
        gobuilder -commit
which uploads information about commits to the dashboard,
which then hands the work out to the builders by hash.
There is no assumption anymore that the commit numbers
are consistent across builders.

New builders will need to be deployed.  For now darwin-amd64
is running the new builder to test the code.

The new JSON-based protocol for handing out work via /todo
should be easy to extend if we want to add support for sending
trial CLs to the builders.

This code is already running on godashboard.appspot.com.

R=adg, dave
CC=golang-dev
https://golang.org/cl/4519047

misc/dashboard/builder/Makefile

@@ -7,7 +7,6 @@ include ../../../src/Make.inc
 TARG=gobuilder
 GOFILES=\
 	exec.go\
-	hg.go\
 	http.go\
 	main.go\
 	package.go\

misc/dashboard/builder/doc.go

@@ -14,9 +14,6 @@ It periodically pulls updates from the Go Mercurial repository.
 When a newer revision is found, Go Builder creates a clone of the repository,
 runs all.bash, and reports build success or failure to the Go Dashboard. 
 
-For a successful build, Go Builder will also run benchmarks 
-(cd $GOROOT/src/pkg; make bench) and send the results to the Go Dashboard.
-
 For a release revision (a change description that matches "release.YYYY-MM-DD"),
 Go Builder will create a tar.gz archive of the GOROOT and deliver it to the
 Go Google Code project's downloads section.
@@ -34,8 +31,6 @@ Optional flags:
     The location of the Go Dashboard application to which Go Builder will
     report its results.
 
-  -bench: Run benchmarks
-
   -release: Build and deliver binary release archive
 
   -rev=N: Build revision N and exit
@@ -45,7 +40,7 @@ Optional flags:
   -v: Verbose logging
 
   -external: External package builder mode (will not report Go build
-     state to dashboard, issue releases, or run benchmarks)
+     state to dashboard or issue releases)
 
 The key file should be located at $HOME/.gobuildkey or, for a builder-specific
 key, $HOME/.gobuildkey-$BUILDER (eg, $HOME/.gobuildkey-linux-amd64).

misc/dashboard/builder/exec.go

@@ -18,7 +18,7 @@ func run(envv []string, dir string, argv ...string) os.Error {
 	if *verbose {
 		log.Println("run", argv)
 	}
-	bin, err := pathLookup(argv[0])
+	bin, err := lookPath(argv[0])
 	if err != nil {
 		return err
 	}
@@ -36,7 +36,7 @@ func runLog(envv []string, logfile, dir string, argv ...string) (output string,
 	if *verbose {
 		log.Println("runLog", argv)
 	}
-	bin, err := pathLookup(argv[0])
+	bin, err := lookPath(argv[0])
 	if err != nil {
 		return
 	}
@@ -67,10 +67,10 @@ func runLog(envv []string, logfile, dir string, argv ...string) (output string,
 	return b.String(), wait.WaitStatus.ExitStatus(), nil
 }
 
-// Find bin in PATH if a relative or absolute path hasn't been specified
+// lookPath looks for cmd in $PATH if cmd does not begin with / or ./ or ../.
-func pathLookup(s string) (string, os.Error) {
+func lookPath(cmd string) (string, os.Error) {
-	if strings.HasPrefix(s, "/") || strings.HasPrefix(s, "./") || strings.HasPrefix(s, "../") {
+	if strings.HasPrefix(cmd, "/") || strings.HasPrefix(cmd, "./") || strings.HasPrefix(cmd, "../") {
-		return s, nil
+		return cmd, nil
 	}
-	return exec.LookPath(s)
+	return exec.LookPath(cmd)
 }

misc/dashboard/builder/hg.go

@@ -1,86 +0,0 @@
-// Copyright 2011 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package main
-
-import (
-	"fmt"
-	"os"
-	"regexp"
-	"strconv"
-	"strings"
-)
-
-type Commit struct {
-	num    int    // mercurial revision number
-	node   string // mercurial hash
-	parent string // hash of commit's parent
-	user   string // author's Name <email>
-	date   string // date of commit
-	desc   string // description
-}
-
-// getCommit returns details about the Commit specified by the revision hash
-func getCommit(rev string) (c Commit, err os.Error) {
-	defer func() {
-		if err != nil {
-			err = fmt.Errorf("getCommit: %s: %s", rev, err)
-		}
-	}()
-	parts, err := getCommitParts(rev)
-	if err != nil {
-		return
-	}
-	num, err := strconv.Atoi(parts[0])
-	if err != nil {
-		return
-	}
-	parent := ""
-	if num > 0 {
-		prev := strconv.Itoa(num - 1)
-		if pparts, err := getCommitParts(prev); err == nil {
-			parent = pparts[1]
-		}
-	}
-	user := strings.Replace(parts[2], "&lt;", "<", -1)
-	user = strings.Replace(user, "&gt;", ">", -1)
-	return Commit{num, parts[1], parent, user, parts[3], parts[4]}, nil
-}
-
-func getCommitParts(rev string) (parts []string, err os.Error) {
-	const format = "{rev}>{node}>{author|escape}>{date}>{desc}"
-	s, _, err := runLog(nil, "", goroot,
-		"hg", "log",
-		"--encoding", "utf-8",
-		"--rev", rev,
-		"--limit", "1",
-		"--template", format,
-	)
-	if err != nil {
-		return
-	}
-	return strings.Split(s, ">", 5), nil
-}
-
-var revisionRe = regexp.MustCompile(`([0-9]+):[0-9a-f]+$`)
-
-// getTag fetches a Commit by finding the first hg tag that matches re.
-func getTag(re *regexp.Regexp) (c Commit, tag string, err os.Error) {
-	o, _, err := runLog(nil, "", goroot, "hg", "tags")
-	for _, l := range strings.Split(o, "\n", -1) {
-		tag = re.FindString(l)
-		if tag == "" {
-			continue
-		}
-		s := revisionRe.FindStringSubmatch(l)
-		if s == nil {
-			err = os.NewError("couldn't find revision number")
-			return
-		}
-		c, err = getCommit(s[1])
-		return
-	}
-	err = os.NewError("no matching tag found")
-	return
-}

misc/dashboard/builder/http.go

@@ -6,84 +6,104 @@ package main
 
 import (
 	"bytes"
-	"encoding/base64"
-	"encoding/binary"
 	"fmt"
 	"http"
 	"json"
 	"log"
 	"os"
-	"regexp"
 	"strconv"
 )
 
-// getHighWater returns the current highwater revision hash for this builder
+type param map[string]string
-func (b *Builder) getHighWater() (rev string, err os.Error) {
+
-	url := fmt.Sprintf("http://%s/hw-get?builder=%s", *dashboard, b.name)
+// dash runs the given method and command on the dashboard.
-	r, _, err := http.Get(url)
+// If args is not nil, it is the query or post parameters.
+// If resp is not nil, dash unmarshals the body as JSON into resp.
+func dash(meth, cmd string, resp interface{}, args param) os.Error {
+	var r *http.Response
+	var err os.Error
+	if *verbose {
+		log.Println("dash", cmd, args)
+	}
+	cmd = "http://" + *dashboard + "/" + cmd
+	switch meth {
+	case "GET":
+		if args != nil {
+			m := make(map[string][]string)
+			for k, v := range args {
+				m[k] = []string{v}
+			}
+			cmd += "?" + http.EncodeQuery(m)
+		}
+		r, _, err = http.Get(cmd)
+	case "POST":
+		r, err = http.PostForm(cmd, args)
+	default:
+		return fmt.Errorf("unknown method %q", meth)
+	}
 	if err != nil {
+		return err
+	}
+	defer r.Body.Close()
+	var buf bytes.Buffer
+	buf.ReadFrom(r.Body)
+	if resp != nil {
+		if err = json.Unmarshal(buf.Bytes(), resp); err != nil {
+			log.Printf("json unmarshal %#q: %s\n", buf.Bytes(), err)
+			return err
+		}
+	}
+	return nil
+}
+
+func dashStatus(meth, cmd string, args param) os.Error {
+	var resp struct {
+		Status string
+		Error string
+	}
+	err := dash(meth, cmd, &resp, args)
+	if err != nil {
+		return err
+	}
+	if resp.Status != "OK" {
+		return os.NewError("/build: " + resp.Error)
+	}
+	return nil	
+}
+	
+// todo returns the next hash to build.
+func (b *Builder) todo() (rev string, err os.Error) {
+	var resp []struct{
+		Hash string
+	}
+	if err = dash("GET", "todo", &resp, param{"builder": b.name}); err != nil {
 		return
 	}
-	buf := new(bytes.Buffer)
+	if len(resp) > 0 {
-	_, err = buf.ReadFrom(r.Body)
+		rev = resp[0].Hash
-	if err != nil {
-		return
 	}
-	r.Body.Close()
+	return
-	return buf.String(), nil
 }
 
 // recordResult sends build results to the dashboard
-func (b *Builder) recordResult(buildLog string, c Commit) os.Error {
+func (b *Builder) recordResult(buildLog string, hash string) os.Error {
-	return httpCommand("build", map[string]string{
+	return dash("POST", "build", nil, param{
 		"builder": b.name,
 		"key":     b.key,
-		"node":    c.node,
+		"node":    hash,
-		"parent":  c.parent,
-		"user":    c.user,
-		"date":    c.date,
-		"desc":    c.desc,
 		"log":     buildLog,
 	})
 }
 
-// match lines like: "package.BechmarkFunc	100000	    999 ns/op"
+// packages fetches a list of package paths from the dashboard
-var benchmarkRegexp = regexp.MustCompile("([^\n\t ]+)[\t ]+([0-9]+)[\t ]+([0-9]+) ns/op")
+func packages() (pkgs []string, err os.Error) {
-
-// recordBenchmarks sends benchmark results to the dashboard
-func (b *Builder) recordBenchmarks(benchLog string, c Commit) os.Error {
-	results := benchmarkRegexp.FindAllStringSubmatch(benchLog, -1)
-	var buf bytes.Buffer
-	b64 := base64.NewEncoder(base64.StdEncoding, &buf)
-	for _, r := range results {
-		for _, s := range r[1:] {
-			binary.Write(b64, binary.BigEndian, uint16(len(s)))
-			b64.Write([]byte(s))
-		}
-	}
-	b64.Close()
-	return httpCommand("benchmarks", map[string]string{
-		"builder":       b.name,
-		"key":           b.key,
-		"node":          c.node,
-		"benchmarkdata": buf.String(),
-	})
-}
-
-// getPackages fetches a list of package paths from the dashboard
-func getPackages() (pkgs []string, err os.Error) {
-	r, _, err := http.Get(fmt.Sprintf("http://%v/package?fmt=json", *dashboard))
-	if err != nil {
-		return
-	}
-	defer r.Body.Close()
-	d := json.NewDecoder(r.Body)
 	var resp struct {
 		Packages []struct {
 			Path string
 		}
 	}
-	if err = d.Decode(&resp); err != nil {
+	err = dash("GET", "package", &resp, param{"fmt": "json"})
+	if err != nil {
 		return
 	}
 	for _, p := range resp.Packages {
@@ -93,24 +113,36 @@ func getPackages() (pkgs []string, err os.Error) {
 }
 
 // updatePackage sends package build results and info to the dashboard
-func (b *Builder) updatePackage(pkg string, state bool, buildLog, info string, c Commit) os.Error {
+func (b *Builder) updatePackage(pkg string, state bool, buildLog, info string, hash string) os.Error {
-	args := map[string]string{
+	return dash("POST", "package", nil, param{
 		"builder": b.name,
 		"key":     b.key,
 		"path":    pkg,
 		"state":   strconv.Btoa(state),
 		"log":     buildLog,
 		"info":    info,
-		"go_rev":  strconv.Itoa(c.num),
+		"go_rev":  hash[:12],
-	}
+	})
-	return httpCommand("package", args)
 }
 
-func httpCommand(cmd string, args map[string]string) os.Error {
+// postCommit informs the dashboard of a new commit
-	if *verbose {
+func postCommit(key string, l *HgLog) os.Error {
-		log.Println("httpCommand", cmd, args)
+	return dashStatus("POST", "commit", param{
+		"key": key,
+		"node": l.Hash,
+		"date": l.Date,
+		"user": l.Author,
+		"parent": l.Parent,
+		"desc": l.Desc,
+	})
 }
-	url := fmt.Sprintf("http://%v/%v", *dashboard, cmd)
+
-	_, err := http.PostForm(url, args)
+// dashboardCommit returns true if the dashboard knows about hash.
-	return err
+func dashboardCommit(hash string) bool {
+	err := dashStatus("GET", "commit", param{"node": hash})
+	if err != nil {
+		log.Printf("check %s: %s", hash, err)
+		return false
+	}
+	return true
 }

misc/dashboard/builder/main.go

@@ -5,7 +5,6 @@
 package main
 
 import (
-	"container/vector"
 	"flag"
 	"fmt"
 	"io/ioutil"
@@ -16,13 +15,14 @@ import (
 	"strconv"
 	"strings"
 	"time"
+	"xml"
 )
 
 const (
 	codeProject      = "go"
 	codePyScript     = "misc/dashboard/googlecode_upload.py"
 	hgUrl            = "https://go.googlecode.com/hg/"
-	waitInterval     = 10e9 // time to wait before checking for new revs
+	waitInterval     = 30e9 // time to wait before checking for new revs
 	mkdirPerm        = 0750
 	pkgBuildInterval = 1e9 * 60 * 60 * 24 // rebuild packages every 24 hours
 )
@@ -46,16 +46,10 @@ type Builder struct {
 	codePassword string
 }
 
-type BenchRequest struct {
-	builder *Builder
-	commit  Commit
-	path    string
-}
-
 var (
 	buildroot     = flag.String("buildroot", path.Join(os.TempDir(), "gobuilder"), "Directory under which to build")
+	commitFlag = flag.Bool("commit", false, "upload information about new commits")
 	dashboard     = flag.String("dashboard", "godashboard.appspot.com", "Go Dashboard Host")
-	runBenchmarks = flag.Bool("bench", false, "Run benchmarks")
 	buildRelease  = flag.Bool("release", false, "Build and upload binary release archives")
 	buildRevision = flag.String("rev", "", "Build specified revision and exit")
 	buildCmd      = flag.String("cmd", "./all.bash", "Build command (specify absolute or relative to go/src/)")
@@ -67,7 +61,6 @@ var (
 var (
 	goroot        string
 	releaseRegexp = regexp.MustCompile(`^(release|weekly)\.[0-9\-.]+`)
-	benchRequests vector.Vector
 )
 
 func main() {
@@ -77,7 +70,7 @@ func main() {
 		os.Exit(2)
 	}
 	flag.Parse()
-	if len(flag.Args()) == 0 {
+	if len(flag.Args()) == 0 && !*commitFlag {
 		flag.Usage()
 	}
 	goroot = path.Join(*buildroot, "goroot")
@@ -101,17 +94,24 @@ func main() {
 		log.Fatal("Error cloning repository:", err)
 	}
 	
+	if *commitFlag {
+		if len(flag.Args()) == 0 {
+			commitWatcher()
+			return
+		}
+		go commitWatcher()
+	}
+
 	// if specified, build revision and return
 	if *buildRevision != "" {
-		c, err := getCommit(*buildRevision)
+		hash, err := fullHash(*buildRevision)
 		if err != nil {
 			log.Fatal("Error finding revision: ", err)
 		}
 		for _, b := range builders {
-			if err := b.buildCommit(c); err != nil {
+			if err := b.buildHash(hash); err != nil {
 				log.Println(err)
 			}
-			runQueuedBenchmark()
 		}
 		return
 	}
@@ -134,6 +134,7 @@ func main() {
 			continue
 		}
 		built := false
+		t := time.Nanoseconds()
 		if *parallel {
 			done := make(chan bool)
 			for _, b := range builders {
@@ -149,49 +150,18 @@ func main() {
 				built = b.build() || built
 			}
 		}
-		// only run benchmarks if we didn't build anything
+		// sleep if there was nothing to build
-		// so that they don't hold up the builder queue
 		if !built {
-			if !runQueuedBenchmark() {
-				// if we have no benchmarks to do, pause
 			time.Sleep(waitInterval)
 		}
-			// after running one benchmark, 
+		// sleep if we're looping too fast.
-			// continue to find and build new revisions.
+		t1 := time.Nanoseconds() - t
+		if t1 < waitInterval {
+			time.Sleep(waitInterval - t1)
 		}
 	}
 }
 
-func runQueuedBenchmark() bool {
-	if benchRequests.Len() == 0 {
-		return false
-	}
-	runBenchmark(benchRequests.Pop().(BenchRequest))
-	return true
-}
-
-func runBenchmark(r BenchRequest) {
-	// run benchmarks and send to dashboard
-	log.Println(r.builder.name, "benchmarking", r.commit.num)
-	defer os.RemoveAll(r.path)
-	pkg := path.Join(r.path, "go", "src", "pkg")
-	bin := path.Join(r.path, "go", "bin")
-	env := []string{
-		"GOOS=" + r.builder.goos,
-		"GOARCH=" + r.builder.goarch,
-		"PATH=" + bin + ":" + os.Getenv("PATH"),
-	}
-	logfile := path.Join(r.path, "bench.log")
-	benchLog, _, err := runLog(env, logfile, pkg, "gomake", "bench")
-	if err != nil {
-		log.Println(r.builder.name, "gomake bench:", err)
-		return
-	}
-	if err = r.builder.recordBenchmarks(benchLog, r.commit); err != nil {
-		log.Println("recordBenchmarks:", err)
-	}
-}
-
 func NewBuilder(builder string) (*Builder, os.Error) {
 	b := &Builder{name: builder}
 
@@ -235,7 +205,7 @@ func (b *Builder) buildExternal() {
 			log.Println("hg pull failed:", err)
 			continue
 		}
-		c, tag, err := getTag(releaseRegexp)
+		hash, tag, err := firstTag(releaseRegexp)
 		if err != nil {
 			log.Println(err)
 			continue
@@ -249,8 +219,8 @@ func (b *Builder) buildExternal() {
 		if tag == prevTag && time.Nanoseconds() < nextBuild {
 			continue
 		}
-		// buildCommit will also build the packages
+		// build will also build the packages
-		if err := b.buildCommit(c); err != nil {
+		if err := b.buildHash(hash); err != nil {
 			log.Println(err)
 			continue
 		}
@@ -269,65 +239,37 @@ func (b *Builder) build() bool {
 			log.Println(b.name, "build:", err)
 		}
 	}()
-	c, err := b.nextCommit()
+	hash, err := b.todo()
 	if err != nil {
 		log.Println(err)
 		return false
 	}
-	if c == nil {
+	if hash == "" {
 		return false
 	}
-	err = b.buildCommit(*c)
+	err = b.buildHash(hash)
 	if err != nil {
 		log.Println(err)
 	}
 	return true
 }
 
-// nextCommit returns the next unbuilt Commit for this builder
+func (b *Builder) buildHash(hash string) (err os.Error) {
-func (b *Builder) nextCommit() (nextC *Commit, err os.Error) {
 	defer func() {
 		if err != nil {
-			err = fmt.Errorf("%s nextCommit: %s", b.name, err)
+			err = fmt.Errorf("%s build: %s: %s", b.name, hash, err)
-		}
-	}()
-	hw, err := b.getHighWater()
-	if err != nil {
-		return
-	}
-	c, err := getCommit(hw)
-	if err != nil {
-		return
-	}
-	next := c.num + 1
-	c, err = getCommit(strconv.Itoa(next))
-	if err == nil && c.num == next {
-		return &c, nil
-	}
-	return nil, nil
-}
-
-func (b *Builder) buildCommit(c Commit) (err os.Error) {
-	defer func() {
-		if err != nil {
-			err = fmt.Errorf("%s buildCommit: %d: %s", b.name, c.num, err)
 		}
 	}()
 
-	log.Println(b.name, "building", c.num)
+	log.Println(b.name, "building", hash)
 
 	// create place in which to do work
-	workpath := path.Join(*buildroot, b.name+"-"+strconv.Itoa(c.num))
+	workpath := path.Join(*buildroot, b.name+"-"+hash[:12])
 	err = os.Mkdir(workpath, mkdirPerm)
 	if err != nil {
 		return
 	}
-	benchRequested := false
+	defer os.RemoveAll(workpath)
-	defer func() {
-		if !benchRequested {
-			os.RemoveAll(workpath)
-		}
-	}()
 
 	// clone repo
 	err = run(nil, workpath, "hg", "clone", goroot, "go")
@@ -337,7 +279,7 @@ func (b *Builder) buildCommit(c Commit) (err os.Error) {
 
 	// update to specified revision
 	err = run(nil, path.Join(workpath, "go"),
-		"hg", "update", "-r", strconv.Itoa(c.num))
+		"hg", "update", hash)
 	if err != nil {
 		return
 	}
@@ -356,36 +298,27 @@ func (b *Builder) buildCommit(c Commit) (err os.Error) {
 		if status != 0 {
 			return os.NewError("go build failed")
 		}
-		return b.buildPackages(workpath, c)
+		return b.buildPackages(workpath, hash)
 	}
 
 	if status != 0 {
 		// record failure
-		return b.recordResult(buildLog, c)
+		return b.recordResult(buildLog, hash)
 	}
 
 	// record success
-	if err = b.recordResult("", c); err != nil {
+	if err = b.recordResult("", hash); err != nil {
 		return fmt.Errorf("recordResult: %s", err)
 	}
 
-	// send benchmark request if benchmarks are enabled
-	if *runBenchmarks {
-		benchRequests.Insert(0, BenchRequest{
-			builder: b,
-			commit:  c,
-			path:    workpath,
-		})
-		benchRequested = true
-	}
-
 	// finish here if codeUsername and codePassword aren't set
 	if b.codeUsername == "" || b.codePassword == "" || !*buildRelease {
 		return
 	}
 
 	// if this is a release, create tgz and upload to google code
-	if release := releaseRegexp.FindString(c.desc); release != "" {
+	releaseHash, release, err := firstTag(releaseRegexp)
+	if hash == releaseHash {
 		// clean out build state
 		err = run(b.envv(), srcDir, "./clean.bash", "--nopkg")
 		if err != nil {
@@ -431,3 +364,209 @@ func isFile(name string) bool {
 	s, err := os.Stat(name)
 	return err == nil && (s.IsRegular() || s.IsSymlink())
 }
+
+// commitWatcher polls hg for new commits and tells the dashboard about them.
+func commitWatcher() {
+	// Create builder just to get master key.
+	b, err := NewBuilder("mercurial-commit")
+	if err != nil {
+		log.Fatal(err)
+	}
+	for {
+		if *verbose {
+			log.Printf("poll...")
+		}
+		commitPoll(b.key)
+		if *verbose {
+			log.Printf("sleep...")
+		}
+		time.Sleep(60e9)
+	}
+}
+
+// HgLog represents a single Mercurial revision.
+type HgLog struct {
+	Hash string
+	Author string
+	Date string
+	Desc string
+	Parent string
+	
+	// Internal metadata
+	added bool
+}
+
+// logByHash is a cache of all Mercurial revisions we know about,
+// indexed by full hash.
+var logByHash = map[string]*HgLog{}
+
+// xmlLogTemplate is a template to pass to Mercurial to make
+// hg log print the log in valid XML for parsing with xml.Unmarshal.
+const xmlLogTemplate = `
+	<log>
+	<hash>{node|escape}</hash>
+	<parent>{parent|escape}</parent>
+	<author>{author|escape}</author>
+	<date>{date}</date>
+	<desc>{desc|escape}</desc>
+	</log>
+`
+
+// commitPoll pulls any new revisions from the hg server
+// and tells the server about them.
+func commitPoll(key string) {
+	// Catch unexpected panics.
+	defer func() {
+		if err := recover(); err != nil {
+			log.Printf("commitPoll panic: %s", err)
+		}
+	}()
+
+	if err := run(nil, goroot, "hg", "pull"); err != nil {
+		log.Printf("hg pull: %v", err)
+		return
+	}
+	
+	const N = 20   // how many revisions to grab
+
+	data, _, err := runLog(nil, "", goroot, "hg", "log",
+		"--encoding=utf-8",
+		"--limit=" + strconv.Itoa(N),
+		"--template=" + xmlLogTemplate,
+	)
+	if err != nil {
+		log.Printf("hg log: %v", err)
+		return
+	}
+	
+	var logStruct struct {
+		Log []HgLog
+	}
+	err = xml.Unmarshal(strings.NewReader("<top>" + data + "</top>"), &logStruct)
+	if err != nil {
+		log.Printf("unmarshal hg log: %v", err)
+		return
+	}
+
+	logs := logStruct.Log
+
+	// Pass 1.  Fill in parents and add new log entries to logsByHash.
+	// Empty parent means take parent from next log entry.
+	// Non-empty parent has form 1234:hashhashhash; we weant full hash.
+	for i := range logs {
+		l := &logs[i]
+		log.Printf("hg log: %s < %s\n", l.Hash, l.Parent)
+		if l.Parent == "" && i+1 < len(logs) {
+			l.Parent = logs[i+1].Hash
+		} else if l.Parent != "" {
+			l.Parent, _ = fullHash(l.Parent)
+		}
+		if l.Parent == "" {
+			// Can't create node without parent.
+			continue
+		}
+		
+		if logByHash[l.Hash] == nil {
+			// Make copy to avoid pinning entire slice when only one entry is new.
+			t := *l
+			logByHash[t.Hash] = &t
+		}
+	}
+
+	for i := range logs {
+		l := &logs[i]
+		if l.Parent == "" {
+			continue
+		}
+		addCommit(l.Hash, key)
+	}
+}
+
+// addCommit adds the commit with the named hash to the dashboard.
+// key is the secret key for authentication to the dashboard.
+// It avoids duplicate effort.
+func addCommit(hash, key string) bool {
+	l := logByHash[hash]
+	if l == nil {
+		return false
+	}
+	if l.added {
+		return true
+	}
+	
+	// Check for already added, perhaps in an earlier run.
+	if dashboardCommit(hash) {
+		log.Printf("%s already on dashboard\n", hash)
+		// Record that this hash is on the dashboard,
+		// as must be all its parents.
+		for l != nil {
+			l.added = true
+			l = logByHash[l.Parent]
+		}
+		return true
+	}
+
+	// Create parent first, to maintain some semblance of order.
+	if !addCommit(l.Parent, key) {
+		return false
+	}
+
+	// Create commit.
+	if err := postCommit(key, l); err != nil {
+		log.Printf("faield to add %s to dashboard: %v", err)
+		return false
+	}
+	return true
+}
+
+// fullHash returns the full hash for the given Mercurial revision.
+func fullHash(rev string) (hash string, err os.Error) {
+	defer func() {
+		if err != nil {
+			err = fmt.Errorf("fullHash: %s: %s", rev, err)
+		}
+	}()
+	s, _, err := runLog(nil, "", goroot,
+		"hg", "log",
+		"--encoding=utf-8",
+		"--rev="+rev,
+		"--limit=1",
+		"--template={node}",
+	)
+	if err != nil {
+		return
+	}
+	s = strings.TrimSpace(s)
+	if s == "" {
+		return "", fmt.Errorf("cannot find revision")
+	}
+	if len(s) != 20 {
+		return "", fmt.Errorf("hg returned invalid hash " + s)
+	}
+	return s, nil
+}
+
+var revisionRe = regexp.MustCompile(`^([^ ]+) +[0-9]+:([0-9a-f]+)$`)
+
+// firstTag returns the hash and tag of the most recent tag matching re.
+func firstTag(re *regexp.Regexp) (hash string, tag string, err os.Error) {
+	o, _, err := runLog(nil, "", goroot, "hg", "tags")
+	for _, l := range strings.Split(o, "\n", -1) {
+		if l == "" {
+			continue
+		}
+		s := revisionRe.FindStringSubmatch(l)
+		if s == nil {
+			err = os.NewError("couldn't find revision number")
+			return
+		}
+		if !re.MatchString(s[1]) {
+			continue
+		}
+		tag = s[1]
+		hash, err = fullHash(s[3])
+		return
+	}
+	err = os.NewError("no matching tag found")
+	return
+}

misc/dashboard/builder/package.go

@@ -13,8 +13,8 @@ import (
 	"path"
 )
 
-func (b *Builder) buildPackages(workpath string, c Commit) os.Error {
+func (b *Builder) buildPackages(workpath string, hash string) os.Error {
-	pkgs, err := getPackages()
+	pkgs, err := packages()
 	if err != nil {
 		return err
 	}
@@ -32,13 +32,13 @@ func (b *Builder) buildPackages(workpath string, c Commit) os.Error {
 		built := code != 0
 
 		// get doc comment from package source
-		info, err := getPackageComment(p, path.Join(goroot, "pkg", p))
+		info, err := packageComment(p, path.Join(goroot, "pkg", p))
 		if err != nil {
 			log.Printf("goinstall %v: %v", p, err)
 		}
 
 		// update dashboard with build state + info
-		err = b.updatePackage(p, built, buildLog, info, c)
+		err = b.updatePackage(p, built, buildLog, info, hash)
 		if err != nil {
 			log.Printf("updatePackage %v: %v", p, err)
 		}
@@ -46,7 +46,7 @@ func (b *Builder) buildPackages(workpath string, c Commit) os.Error {
 	return nil
 }
 
-func getPackageComment(pkg, pkgpath string) (info string, err os.Error) {
+func packageComment(pkg, pkgpath string) (info string, err os.Error) {
 	fset := token.NewFileSet()
 	pkgs, err := parser.ParseDir(fset, pkgpath, nil, parser.PackageClauseOnly|parser.ParseComments)
 	if err != nil {

misc/dashboard/godashboard/app.yaml

@@ -1,5 +1,5 @@
 application: godashboard
-version: 5
+version: 6
 runtime: python
 api_version: 1
 

misc/dashboard/godashboard/gobuild.py

@@ -5,6 +5,7 @@
 # This is the server part of the continuous build system for Go. It must be run
 # by AppEngine.
 
+from django.utils import simplejson
 from google.appengine.api import mail
 from google.appengine.api import memcache
 from google.appengine.ext import db
@@ -50,10 +51,6 @@ class Commit(db.Model):
 
     fail_notification_sent = db.BooleanProperty()
 
-class Cache(db.Model):
-    data = db.BlobProperty()
-    expire = db.IntegerProperty()
-
 # A CompressedLog contains the textual build log of a failed build. 
 # The key name is the hex digest of the SHA256 hash of the contents.
 # The contents is bz2 compressed.
@@ -62,23 +59,6 @@ class CompressedLog(db.Model):
 
 N = 30
 
-def cache_get(key):
-    c = Cache.get_by_key_name(key)
-    if c is None or c.expire < time.time():
-        return None
-    return c.data
-
-def cache_set(key, val, timeout):
-    c = Cache(key_name = key)
-    c.data = val
-    c.expire = int(time.time() + timeout)
-    c.put()
-
-def cache_del(key):
-    c = Cache.get_by_key_name(key)
-    if c is not None:
-        c.delete()
-
 def builderInfo(b):
     f = b.split('-', 3)
     goos = f[0]
@@ -147,7 +127,30 @@ class MainPage(webapp.RequestHandler):
         path = os.path.join(os.path.dirname(__file__), 'main.html')
         self.response.out.write(template.render(path, values))
 
-class GetHighwater(webapp.RequestHandler):
+# A DashboardHandler is a webapp.RequestHandler but provides
+#    authenticated_post - called by post after authenticating
+#    json - writes object in json format to response output
+class DashboardHandler(webapp.RequestHandler):
+    def post(self):
+        if not auth(self.request):
+            self.response.set_status(403)
+            return
+        self.authenticated_post()
+
+    def authenticated_post(self):
+        return
+    
+    def json(self, obj):
+        self.response.set_status(200)
+        simplejson.dump(obj, self.response.out)
+        return
+
+def auth(req):
+    k = req.get('key')
+    return k == hmac.new(key.accessKey, req.get('builder')).hexdigest() or k == key.accessKey
+
+# Todo serves /todo.  It tells the builder which commits need to be built.
+class Todo(DashboardHandler):
     def get(self):
         builder = self.request.get('builder')
         key = 'todo-%s' % builder
@@ -155,28 +158,19 @@ class GetHighwater(webapp.RequestHandler):
         if response is None:
             # Fell out of memcache.  Rebuild from datastore results.
             # We walk the commit list looking for nodes that have not
-            # been built by this builder and record the *parents* of those
+            # been built by this builder.
-            # nodes, because each builder builds the revision *after* the
-            # one return (because we might not know about the latest
-            # revision).
             q = Commit.all()
             q.order('-__key__')
             todo = []
-            need = False
             first = None
             for c in q.fetch(N+1):
                 if first is None:
                     first = c
-                if need:
+                if not built(c, builder):
-                    todo.append(c.node)
+                    todo.append({'Hash': c.node})
-                need = not built(c, builder)
+            response = simplejson.dumps(todo)
-            if not todo:
-                todo.append(first.node)
-            response = ' '.join(todo)
             memcache.set(key, response, 3600)
         self.response.set_status(200)
-        if self.request.get('all') != 'yes':
-            response = response.split()[0]
         self.response.out.write(response)
 
 def built(c, builder):
@@ -185,22 +179,8 @@ def built(c, builder):
             return True
     return False
 
-def auth(req):
+# Log serves /log/.  It retrieves log data by content hash.
-    k = req.get('key')
+class LogHandler(DashboardHandler):
-    return k == hmac.new(key.accessKey, req.get('builder')).hexdigest() or k == key.accessKey
-    
-class SetHighwater(webapp.RequestHandler):
-    def post(self):
-        if not auth(self.request):
-            self.response.set_status(403)
-            return
-
-        # Allow for old builders.
-        # This is a no-op now: we figure out what to build based
-        # on the current dashboard status.
-        return
-
-class LogHandler(webapp.RequestHandler):
     def get(self):
         self.response.headers['Content-Type'] = 'text/plain; charset=utf-8'
         hash = self.request.path[5:]
@@ -214,12 +194,8 @@ class LogHandler(webapp.RequestHandler):
 
 # Init creates the commit with id 0. Since this commit doesn't have a parent,
 # it cannot be created by Build.
-class Init(webapp.RequestHandler):
+class Init(DashboardHandler):
-    def post(self):
+    def authenticated_post(self):
-        if not auth(self.request):
-            self.response.set_status(403)
-            return
-
         date = parseDate(self.request.get('date'))
         node = self.request.get('node')
         if not validNode(node) or date is None:
@@ -239,7 +215,86 @@ class Init(webapp.RequestHandler):
 
         self.response.set_status(200)
 
-# Build is the main command: it records the result of a new build.
+# The last commit when we switched to using entity groups.
+# This is the root of the new commit entity group.
+RootCommitKeyName = '00000f26-f32c6f1038207c55d5780231f7484f311020747e'
+
+# CommitHandler serves /commit.
+# A GET of /commit retrieves information about the specified commit.
+# A POST of /commit creates a node for the given commit.
+# If the commit already exists, the POST silently succeeds (like mkdir -p).
+class CommitHandler(DashboardHandler):
+    def get(self):
+        node = self.request.get('node')
+        if not validNode(node):
+            return self.json({'Status': 'FAIL', 'Error': 'malformed node hash'})
+        n = nodeByHash(node)
+        if n is None:
+            return self.json({'Status': 'FAIL', 'Error': 'unknown revision'})
+        return self.json({'Status': 'OK', 'Node': nodeObj(n)})
+
+    def authenticated_post(self):
+        # Require auth with the master key, not a per-builder key.
+        if self.request.get('builder'):
+            self.response.set_status(403)
+            return
+
+        node = self.request.get('node')
+        date = parseDate(self.request.get('date'))
+        user = self.request.get('user').encode('utf8')
+        desc = self.request.get('desc').encode('utf8')
+        parenthash = self.request.get('parent')
+
+        if not validNode(node) or not validNode(parenthash) or date is None:
+            return self.json({'Status': 'FAIL', 'Error': 'malformed node, parent, or date'})
+
+        n = nodeByHash(node)
+        if n is None:
+            p = nodeByHash(parenthash)
+            if p is None:
+                return self.json({'Status': 'FAIL', 'Error': 'unknown parent'})
+
+            # Want to create new node in a transaction so that multiple
+            # requests creating it do not collide and so that multiple requests
+            # creating different nodes get different sequence numbers.
+            # All queries within a transaction must include an ancestor,
+            # but the original datastore objects we used for the dashboard
+            # have no common ancestor.  Instead, we use a well-known
+            # root node - the last one before we switched to entity groups -
+            # as the as the common ancestor.
+            root = Commit.get_by_key_name(RootCommitKeyName)
+
+            def add_commit():
+                if nodeByHash(node, ancestor=root) is not None:
+                    return
+
+                # Determine number for this commit.
+                # Once we have created one new entry it will be lastRooted.num+1,
+                # but the very first commit created in this scheme will have to use
+                # last.num's number instead (last is likely not rooted).
+                q = Commit.all()
+                q.order('-__key__')
+                q.ancestor(root)
+                last = q.fetch(1)[0]
+                num = last.num+1
+
+                n = Commit(key_name = '%08x-%s' % (num, node), parent = root)
+                n.num = num
+                n.node = node
+                n.parentnode = parenthash
+                n.user = user
+                n.date = date
+                n.desc = desc
+                n.put()
+            db.run_in_transaction(add_commit)
+            n = nodeByHash(node)
+            if n is None:
+                return self.json({'Status': 'FAIL', 'Error': 'failed to create commit node'})
+
+        return self.json({'Status': 'OK', 'Node': nodeObj(n)})
+
+# Build serves /build.
+# A POST to /build records a new build result.
 class Build(webapp.RequestHandler):
     def post(self):
         if not auth(self.request):
@@ -256,44 +311,33 @@ class Build(webapp.RequestHandler):
             l.log = bz2.compress(log)
             l.put()
 
-        date = parseDate(self.request.get('date'))
-        user = self.request.get('user').encode('utf8')
-        desc = self.request.get('desc').encode('utf8')
         node = self.request.get('node')
-        parenthash = self.request.get('parent')
+        if not validNode(node):
-        if not validNode(node) or not validNode(parenthash) or date is None:
+            logging.error('Invalid node %s' % (node))
-            logging.error("Not valid node ('%s') or bad date (%s %s)", node, date, self.request.get('date'))
             self.response.set_status(500)
             return
 
-        q = Commit.all()
+        n = nodeByHash(node)
-        q.filter('node =', parenthash)
+        if n is None:
-        parent = q.get()
+            logging.error('Cannot find node %s' % (node))
-        if parent is None:
-            logging.error('Cannot find parent %s of node %s' % (parenthash, node))
             self.response.set_status(404)
             return
-        parentnum, _ = parent.key().name().split('-', 1)
+        nn = n
-        nodenum = int(parentnum, 16) + 1
-
-        key_name = '%08x-%s' % (nodenum, node)
 
         def add_build():
-            n = Commit.get_by_key_name(key_name)
+            n = nodeByHash(node, ancestor=nn)
             if n is None:
-                n = Commit(key_name = key_name)
+                logging.error('Cannot find hash in add_build: %s %s' % (builder, node))
-                n.num = nodenum
+                return
-                n.node = node
+
-                n.parentnode = parenthash
-                n.user = user
-                n.date = date
-                n.desc = desc
             s = '%s`%s' % (builder, loghash)
             for i, b in enumerate(n.builds):
                 if b.split('`', 1)[0] == builder:
+                    # logging.error('Found result for %s %s already' % (builder, node))
                     n.builds[i] = s
                     break
             else:
+                # logging.error('Added result for %s %s' % (builder, node))
                 n.builds.append(s)
             n.put()
 
@@ -302,30 +346,7 @@ class Build(webapp.RequestHandler):
         key = 'todo-%s' % builder
         memcache.delete(key)
 
-        def mark_sent():
+        # TODO: Send mail for build breakage.
-            n = Commit.get_by_key_name(key_name)
-            n.fail_notification_sent = True
-            n.put()
-
-        n = Commit.get_by_key_name(key_name)
-        if loghash and not failed(parent, builder) and not n.fail_notification_sent:
-            subject = const.mail_fail_subject % (builder, desc.split("\n")[0])
-            path = os.path.join(os.path.dirname(__file__), 'fail-notify.txt')
-            body = template.render(path, {
-                "builder": builder,
-                "node": node[:12],
-                "user": user,
-                "desc": desc, 
-                "loghash": loghash
-            })
-            mail.send_mail(
-                sender=const.mail_from,
-                reply_to=const.mail_fail_reply_to,
-                to=const.mail_fail_to,
-                subject=subject,
-                body=body
-            )
-            db.run_in_transaction(mark_sent)
 
         self.response.set_status(200)
 
@@ -342,6 +363,24 @@ def node(num):
     n = q.get()
     return n
 
+def nodeByHash(hash, ancestor=None):
+    q = Commit.all()
+    q.filter('node =', hash)
+    if ancestor is not None:
+      q.ancestor(ancestor)
+    n = q.get()
+    return n
+
+# nodeObj returns a JSON object (ready to be passed to simplejson.dump) describing node.
+def nodeObj(n):
+    return {
+        'Hash': n.node,
+        'ParentHash': n.parentnode,
+        'User': n.user,
+        'Date': n.date.strftime('%Y-%m-%d %H:%M %z'),
+        'Desc': n.desc,
+    }
+
 class FixedOffset(datetime.tzinfo):
     """Fixed offset in minutes east from UTC."""
 
@@ -417,15 +456,20 @@ def toRev(c):
 def byBuilder(x, y):
     return cmp(x['builder'], y['builder'])
 
+# Give old builders work; otherwise they pound on the web site.
+class Hwget(DashboardHandler):
+    def get(self):
+        self.response.out.write("8000\n")
+
 # This is the URL map for the server. The first three entries are public, the
 # rest are only used by the builders.
 application = webapp.WSGIApplication(
                                      [('/', MainPage),
+                                      ('/hw-get', Hwget),
                                       ('/log/.*', LogHandler),
-                                      ('/hw-get', GetHighwater),
+                                      ('/commit', CommitHandler),
-                                      ('/hw-set', SetHighwater),
-
                                       ('/init', Init),
+                                      ('/todo', Todo),
                                       ('/build', Build),
                                      ], debug=True)
 

misc/dashboard/godashboard/index.yaml

@@ -23,6 +23,12 @@ indexes:
   - name: __key__
     direction: desc
 
+- kind: Commit
+  ancestor: yes
+  properties:
+  - name: __key__
+    direction: desc
+
 - kind: Project
   properties:
   - name: approved