dashboard: builder changes for performance dashboard

This CL moves code from code.google.com/p/dvyukov-go-perf-dashboard,
which was previously reviewed.

LGTM=adg
R=adg
CC=golang-codereviews
https://golang.org/cl/95190043

dashboard/builder/bench.go

@@ -0,0 +1,253 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"regexp"
+	"strconv"
+	"strings"
+)
+
+// benchHash benchmarks a single commit.
+func (b *Builder) benchHash(hash string, benchs []string) error {
+	if *verbose {
+		log.Println(b.name, "benchmarking", hash)
+	}
+
+	res := &PerfResult{Hash: hash, Benchmark: "meta-done"}
+
+	// Create place in which to do work.
+	workpath := filepath.Join(*buildroot, b.name+"-"+hash[:12])
+	// Prepare a workpath if we don't have one we can reuse.
+	update := false
+	if b.lastWorkpath != workpath {
+		if err := os.Mkdir(workpath, mkdirPerm); err != nil {
+			return err
+		}
+		buildLog, _, err := b.buildRepoOnHash(workpath, hash, makeCmd)
+		if err != nil {
+			removePath(workpath)
+			// record failure
+			res.Artifacts = append(res.Artifacts, PerfArtifact{"log", buildLog})
+			return b.recordPerfResult(res)
+		}
+		b.lastWorkpath = workpath
+		update = true
+	}
+
+	// Build the benchmark binary.
+	benchBin, buildLog, err := b.buildBenchmark(workpath, update)
+	if err != nil {
+		// record failure
+		res.Artifacts = append(res.Artifacts, PerfArtifact{"log", buildLog})
+		return b.recordPerfResult(res)
+	}
+
+	benchmark, procs, affinity, last := chooseBenchmark(benchBin, benchs)
+	if benchmark != "" {
+		res.Benchmark = fmt.Sprintf("%v-%v", benchmark, procs)
+		res.Metrics, res.Artifacts, res.OK = b.executeBenchmark(workpath, hash, benchBin, benchmark, procs, affinity)
+		if err = b.recordPerfResult(res); err != nil {
+			return fmt.Errorf("recordResult: %s", err)
+		}
+	}
+
+	if last {
+		// All benchmarks have beed executed, don't need workpath anymore.
+		removePath(b.lastWorkpath)
+		b.lastWorkpath = ""
+		// Notify the app.
+		res = &PerfResult{Hash: hash, Benchmark: "meta-done", OK: true}
+		if err = b.recordPerfResult(res); err != nil {
+			return fmt.Errorf("recordResult: %s", err)
+		}
+	}
+
+	return nil
+}
+
+// buildBenchmark builds the benchmark binary.
+func (b *Builder) buildBenchmark(workpath string, update bool) (benchBin, log string, err error) {
+	goroot := filepath.Join(workpath, "go")
+	gobin := filepath.Join(goroot, "bin", "go") + exeExt
+	gopath := filepath.Join(*buildroot, "gopath")
+	env := append([]string{
+		"GOROOT=" + goroot,
+		"GOPATH=" + gopath},
+		b.envv()...)
+	// First, download without installing.
+	cmd := []string{gobin, "get", "-d"}
+	if update {
+		cmd = append(cmd, "-u")
+	}
+	cmd = append(cmd, *benchPath)
+	var buildlog bytes.Buffer
+	ok, err := runOutput(*buildTimeout, env, &buildlog, workpath, cmd...)
+	if !ok || err != nil {
+		fmt.Fprintf(&buildlog, "go get -d %s failed: %s", *benchPath, err)
+		return "", buildlog.String(), err
+	}
+	// Then, build into workpath.
+	benchBin = filepath.Join(workpath, "benchbin") + exeExt
+	cmd = []string{gobin, "build", "-o", benchBin, *benchPath}
+	buildlog.Reset()
+	ok, err = runOutput(*buildTimeout, env, &buildlog, workpath, cmd...)
+	if !ok || err != nil {
+		fmt.Fprintf(&buildlog, "go build %s failed: %s", *benchPath, err)
+		return "", buildlog.String(), err
+	}
+	return benchBin, "", nil
+}
+
+// chooseBenchmark chooses the next benchmark to run
+// based on the list of available benchmarks, already executed benchmarks
+// and -benchcpu list.
+func chooseBenchmark(benchBin string, doneBenchs []string) (bench string, procs, affinity int, last bool) {
+	out, err := exec.Command(benchBin).CombinedOutput()
+	if err != nil {
+		log.Printf("Failed to query benchmark list: %v\n%s", err, out)
+		last = true
+		return
+	}
+	outStr := string(out)
+	nlIdx := strings.Index(outStr, "\n")
+	if nlIdx < 0 {
+		log.Printf("Failed to parse benchmark list (no new line): %s", outStr)
+		last = true
+		return
+	}
+	localBenchs := strings.Split(outStr[:nlIdx], ",")
+	benchsMap := make(map[string]bool)
+	for _, b := range doneBenchs {
+		benchsMap[b] = true
+	}
+	cnt := 0
+	// We want to run all benchmarks with GOMAXPROCS=1 first.
+	for i, procs1 := range benchCPU {
+		for _, bench1 := range localBenchs {
+			if benchsMap[fmt.Sprintf("%v-%v", bench1, procs1)] {
+				continue
+			}
+			cnt++
+			if cnt == 1 {
+				bench = bench1
+				procs = procs1
+				if i < len(benchAffinity) {
+					affinity = benchAffinity[i]
+				}
+			}
+		}
+	}
+	last = cnt <= 1
+	return
+}
+
+// executeBenchmark runs a single benchmark and parses its output.
+func (b *Builder) executeBenchmark(workpath, hash, benchBin, bench string, procs, affinity int) (metrics []PerfMetric, artifacts []PerfArtifact, ok bool) {
+	// Benchmarks runs mutually exclusive with other activities.
+	benchMutex.RUnlock()
+	defer benchMutex.RLock()
+	benchMutex.Lock()
+	defer benchMutex.Unlock()
+
+	log.Printf("%v executing benchmark %v-%v on %v", b.name, bench, procs, hash)
+
+	// The benchmark executes 'go build'/'go tool',
+	// so we need properly setup env.
+	env := append([]string{
+		"GOROOT=" + filepath.Join(workpath, "go"),
+		"PATH=" + filepath.Join(workpath, "go", "bin") + string(os.PathListSeparator) + os.Getenv("PATH"),
+		"GODEBUG=gctrace=1", // since Go1.2
+		"GOGCTRACE=1",       // before Go1.2
+		fmt.Sprintf("GOMAXPROCS=%v", procs)},
+		b.envv()...)
+	cmd := []string{benchBin,
+		"-bench", bench,
+		"-benchmem", strconv.Itoa(*benchMem),
+		"-benchtime", benchTime.String(),
+		"-benchnum", strconv.Itoa(*benchNum),
+		"-tmpdir", workpath}
+	if affinity != 0 {
+		cmd = append(cmd, "-affinity", strconv.Itoa(affinity))
+	}
+	benchlog := new(bytes.Buffer)
+	ok, err := runOutput(*buildTimeout, env, benchlog, workpath, cmd...)
+	if strip := benchlog.Len() - 512<<10; strip > 0 {
+		// Leave the last 512K, that part contains metrics.
+		benchlog = bytes.NewBuffer(benchlog.Bytes()[strip:])
+	}
+	artifacts = []PerfArtifact{{Type: "log", Body: benchlog.String()}}
+	if !ok || err != nil {
+		if err != nil {
+			log.Printf("Failed to execute benchmark '%v': %v", bench, err)
+			ok = false
+		}
+		return
+	}
+
+	metrics1, artifacts1, err := parseBenchmarkOutput(benchlog)
+	if err != nil {
+		log.Printf("Failed to parse benchmark output: %v", err)
+		ok = false
+		return
+	}
+	metrics = metrics1
+	artifacts = append(artifacts, artifacts1...)
+	return
+}
+
+// parseBenchmarkOutput fetches metrics and artifacts from benchmark output.
+func parseBenchmarkOutput(out io.Reader) (metrics []PerfMetric, artifacts []PerfArtifact, err error) {
+	s := bufio.NewScanner(out)
+	metricRe := regexp.MustCompile("^GOPERF-METRIC:([a-z,0-9,-]+)=([0-9]+)$")
+	fileRe := regexp.MustCompile("^GOPERF-FILE:([a-z,0-9,-]+)=(.+)$")
+	for s.Scan() {
+		ln := s.Text()
+		if ss := metricRe.FindStringSubmatch(ln); ss != nil {
+			var v uint64
+			v, err = strconv.ParseUint(ss[2], 10, 64)
+			if err != nil {
+				err = fmt.Errorf("Failed to parse metric '%v=%v': %v", ss[1], ss[2], err)
+				return
+			}
+			metrics = append(metrics, PerfMetric{Type: ss[1], Val: v})
+		} else if ss := fileRe.FindStringSubmatch(ln); ss != nil {
+			var buf []byte
+			buf, err = ioutil.ReadFile(ss[2])
+			if err != nil {
+				err = fmt.Errorf("Failed to read file '%v': %v", ss[2], err)
+				return
+			}
+			artifacts = append(artifacts, PerfArtifact{ss[1], string(buf)})
+		}
+	}
+	return
+}
+
+// needsBenchmarking determines whether the commit needs benchmarking.
+func needsBenchmarking(log *HgLog) bool {
+	// Do not benchmark branch commits, they are usually not interesting
+	// and fall out of the trunk succession.
+	if log.Branch != "" {
+		return false
+	}
+	// Do not benchmark commits that do not touch source files (e.g. CONTRIBUTORS).
+	for _, f := range strings.Split(log.Files, " ") {
+		if (strings.HasPrefix(f, "include") || strings.HasPrefix(f, "src")) &&
+			!strings.HasSuffix(f, "_test.go") && !strings.Contains(f, "testdata") {
+			return true
+		}
+	}
+	return false
+}

dashboard/builder/filemutex_flock.go

@@ -0,0 +1,66 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build linux darwin freebsd
+
+package main
+
+import (
+	"sync"
+	"syscall"
+)
+
+// FileMutex is similar to sync.RWMutex, but also synchronizes across processes.
+// This implementation is based on flock syscall.
+type FileMutex struct {
+	mu sync.RWMutex
+	fd int
+}
+
+func MakeFileMutex(filename string) *FileMutex {
+	if filename == "" {
+		return &FileMutex{fd: -1}
+	}
+	fd, err := syscall.Open(filename, syscall.O_CREAT|syscall.O_RDONLY, mkdirPerm)
+	if err != nil {
+		panic(err)
+	}
+	return &FileMutex{fd: fd}
+}
+
+func (m *FileMutex) Lock() {
+	m.mu.Lock()
+	if m.fd != -1 {
+		if err := syscall.Flock(m.fd, syscall.LOCK_EX); err != nil {
+			panic(err)
+		}
+	}
+}
+
+func (m *FileMutex) Unlock() {
+	if m.fd != -1 {
+		if err := syscall.Flock(m.fd, syscall.LOCK_UN); err != nil {
+			panic(err)
+		}
+	}
+	m.mu.Unlock()
+}
+
+func (m *FileMutex) RLock() {
+	m.mu.RLock()
+	if m.fd != -1 {
+		if err := syscall.Flock(m.fd, syscall.LOCK_SH); err != nil {
+			panic(err)
+		}
+	}
+}
+
+func (m *FileMutex) RUnlock() {
+	if m.fd != -1 {
+		if err := syscall.Flock(m.fd, syscall.LOCK_UN); err != nil {
+			panic(err)
+		}
+	}
+	m.mu.RUnlock()
+}

dashboard/builder/filemutex_local.go

@@ -0,0 +1,27 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build netbsd openbsd plan9
+
+package main
+
+import (
+	"log"
+	"sync"
+)
+
+// FileMutex is similar to sync.RWMutex, but also synchronizes across processes.
+// This implementation is a fallback that does not actually provide inter-process synchronization.
+type FileMutex struct {
+	sync.RWMutex
+}
+
+func MakeFileMutex(filename string) *FileMutex {
+	return &FileMutex{}
+}
+
+func init() {
+	log.Printf("WARNING: using fake file mutex." +
+		" Don't run more than one of these at once!!!")
+}

dashboard/builder/filemutex_windows.go

@@ -0,0 +1,105 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+	"sync"
+	"syscall"
+	"unsafe"
+)
+
+var (
+	modkernel32      = syscall.NewLazyDLL("kernel32.dll")
+	procLockFileEx   = modkernel32.NewProc("LockFileEx")
+	procUnlockFileEx = modkernel32.NewProc("UnlockFileEx")
+)
+
+const (
+	INVALID_FILE_HANDLE     = ^syscall.Handle(0)
+	LOCKFILE_EXCLUSIVE_LOCK = 2
+)
+
+func lockFileEx(h syscall.Handle, flags, reserved, locklow, lockhigh uint32, ol *syscall.Overlapped) (err error) {
+	r1, _, e1 := syscall.Syscall6(procLockFileEx.Addr(), 6, uintptr(h), uintptr(flags), uintptr(reserved), uintptr(locklow), uintptr(lockhigh), uintptr(unsafe.Pointer(ol)))
+	if r1 == 0 {
+		if e1 != 0 {
+			err = error(e1)
+		} else {
+			err = syscall.EINVAL
+		}
+	}
+	return
+}
+
+func unlockFileEx(h syscall.Handle, reserved, locklow, lockhigh uint32, ol *syscall.Overlapped) (err error) {
+	r1, _, e1 := syscall.Syscall6(procUnlockFileEx.Addr(), 5, uintptr(h), uintptr(reserved), uintptr(locklow), uintptr(lockhigh), uintptr(unsafe.Pointer(ol)), 0)
+	if r1 == 0 {
+		if e1 != 0 {
+			err = error(e1)
+		} else {
+			err = syscall.EINVAL
+		}
+	}
+	return
+}
+
+// FileMutex is similar to sync.RWMutex, but also synchronizes across processes.
+// This implementation is based on flock syscall.
+type FileMutex struct {
+	mu sync.RWMutex
+	fd syscall.Handle
+}
+
+func MakeFileMutex(filename string) *FileMutex {
+	if filename == "" {
+		return &FileMutex{fd: INVALID_FILE_HANDLE}
+	}
+	fd, err := syscall.CreateFile(&(syscall.StringToUTF16(filename)[0]), syscall.GENERIC_READ|syscall.GENERIC_WRITE,
+		syscall.FILE_SHARE_READ|syscall.FILE_SHARE_WRITE, nil, syscall.OPEN_ALWAYS, syscall.FILE_ATTRIBUTE_NORMAL, 0)
+	if err != nil {
+		panic(err)
+	}
+	return &FileMutex{fd: fd}
+}
+
+func (m *FileMutex) Lock() {
+	m.mu.Lock()
+	if m.fd != INVALID_FILE_HANDLE {
+		var ol syscall.Overlapped
+		if err := lockFileEx(m.fd, LOCKFILE_EXCLUSIVE_LOCK, 0, 1, 0, &ol); err != nil {
+			panic(err)
+		}
+	}
+}
+
+func (m *FileMutex) Unlock() {
+	if m.fd != INVALID_FILE_HANDLE {
+		var ol syscall.Overlapped
+		if err := unlockFileEx(m.fd, 0, 1, 0, &ol); err != nil {
+			panic(err)
+		}
+	}
+	m.mu.Unlock()
+}
+
+func (m *FileMutex) RLock() {
+	m.mu.RLock()
+	if m.fd != INVALID_FILE_HANDLE {
+		var ol syscall.Overlapped
+		if err := lockFileEx(m.fd, 0, 0, 1, 0, &ol); err != nil {
+			panic(err)
+		}
+	}
+}
+
+func (m *FileMutex) RUnlock() {
+	if m.fd != INVALID_FILE_HANDLE {
+		var ol syscall.Overlapped
+		if err := unlockFileEx(m.fd, 0, 1, 0, &ol); err != nil {
+			panic(err)
+		}
+	}
+	m.mu.RUnlock()
+}

dashboard/builder/http.go

@@ -89,30 +89,36 @@ func dash(meth, cmd string, args url.Values, req, resp interface{}) error {
 	return nil
 }
 
-// todo returns the next hash to build.
-func (b *Builder) todo(kind, pkg, goHash string) (rev string, err error) {
+// todo returns the next hash to build or benchmark.
+func (b *Builder) todo(kinds []string, pkg, goHash string) (kind, rev string, benchs []string, err error) {
 	args := url.Values{
-		"kind":        {kind},
 		"builder":     {b.name},
 		"packagePath": {pkg},
 		"goHash":      {goHash},
 	}
+	for _, k := range kinds {
+		args.Add("kind", k)
+	}
 	var resp *struct {
 		Kind string
 		Data struct {
-			Hash string
+			Hash        string
+			PerfResults []string
 		}
 	}
 	if err = dash("GET", "todo", args, nil, &resp); err != nil {
-		return "", err
+		return
 	}
 	if resp == nil {
-		return "", nil
+		return
 	}
-	if kind != resp.Kind {
-		return "", fmt.Errorf("expecting Kind %q, got %q", kind, resp.Kind)
+	for _, k := range kinds {
+		if k == resp.Kind {
+			return resp.Kind, resp.Data.Hash, resp.Data.PerfResults, nil
+		}
 	}
-	return resp.Data.Hash, nil
+	err = fmt.Errorf("expecting Kinds %q, got %q", kinds, resp.Kind)
+	return
 }
 
 // recordResult sends build results to the dashboard
@@ -130,18 +136,46 @@ func (b *Builder) recordResult(ok bool, pkg, hash, goHash, buildLog string, runT
 	return dash("POST", "result", args, req, nil)
 }
 
+// Result of running a single benchmark on a single commit.
+type PerfResult struct {
+	Builder   string
+	Benchmark string
+	Hash      string
+	OK        bool
+	Metrics   []PerfMetric
+	Artifacts []PerfArtifact
+}
+
+type PerfMetric struct {
+	Type string
+	Val  uint64
+}
+
+type PerfArtifact struct {
+	Type string
+	Body string
+}
+
+// recordPerfResult sends benchmarking results to the dashboard
+func (b *Builder) recordPerfResult(req *PerfResult) error {
+	req.Builder = b.name
+	args := url.Values{"key": {b.key}, "builder": {b.name}}
+	return dash("POST", "perf-result", args, req, nil)
+}
+
 func postCommit(key, pkg string, l *HgLog) error {
 	t, err := time.Parse(time.RFC3339, l.Date)
 	if err != nil {
 		return fmt.Errorf("parsing %q: %v", l.Date, t)
 	}
 	return dash("POST", "commit", url.Values{"key": {key}}, obj{
-		"PackagePath": pkg,
-		"Hash":        l.Hash,
-		"ParentHash":  l.Parent,
-		"Time":        t.Format(time.RFC3339),
-		"User":        l.Author,
-		"Desc":        l.Desc,
+		"PackagePath":       pkg,
+		"Hash":              l.Hash,
+		"ParentHash":        l.Parent,
+		"Time":              t.Format(time.RFC3339),
+		"User":              l.Author,
+		"Desc":              l.Desc,
+		"NeedsBenchmarking": l.bench,
 	}, nil)
 }
 

dashboard/builder/main.go

@@ -15,6 +15,7 @@ import (
 	"path/filepath"
 	"regexp"
 	"runtime"
+	"strconv"
 	"strings"
 	"time"
 
@@ -36,9 +37,13 @@ type Builder struct {
 	goos, goarch string
 	key          string
 	env          builderEnv
+	// Last benchmarking workpath. We reuse it, if do successive benchmarks on the same commit.
+	lastWorkpath string
 }
 
 var (
+	doBuild        = flag.Bool("build", true, "Build and test packages")
+	doBench        = flag.Bool("bench", false, "Run benchmarks")
 	buildroot      = flag.String("buildroot", defaultBuildRoot(), "Directory under which to build")
 	dashboard      = flag.String("dashboard", "build.golang.org", "Go Dashboard Host")
 	buildRelease   = flag.Bool("release", false, "Build and upload binary release archives")
@@ -47,11 +52,16 @@ var (
 	buildTool      = flag.String("tool", "go", "Tool to build.")
 	gcPath         = flag.String("gcpath", "code.google.com/p/go", "Path to download gc from")
 	gccPath        = flag.String("gccpath", "https://github.com/mirrors/gcc.git", "Path to download gcc from")
+	benchPath      = flag.String("benchpath", "code.google.com/p/go.benchmarks/bench", "Path to download benchmarks from")
 	failAll        = flag.Bool("fail", false, "fail all builds")
 	parallel       = flag.Bool("parallel", false, "Build multiple targets in parallel")
 	buildTimeout   = flag.Duration("buildTimeout", 60*time.Minute, "Maximum time to wait for builds and tests")
 	cmdTimeout     = flag.Duration("cmdTimeout", 10*time.Minute, "Maximum time to wait for an external command")
 	commitInterval = flag.Duration("commitInterval", 1*time.Minute, "Time to wait between polling for new commits (0 disables commit poller)")
+	benchNum       = flag.Int("benchnum", 5, "Run each benchmark that many times")
+	benchTime      = flag.Duration("benchtime", 5*time.Second, "Benchmarking time for a single benchmark run")
+	benchMem       = flag.Int("benchmem", 64, "Approx RSS value to aim at in benchmarks, in MB")
+	fileLock       = flag.String("filelock", "", "File to lock around benchmaring (synchronizes several builders)")
 	verbose        = flag.Bool("v", false, "verbose")
 )
 
@@ -59,12 +69,54 @@ var (
 	binaryTagRe = regexp.MustCompile(`^(release\.r|weekly\.)[0-9\-.]+`)
 	releaseRe   = regexp.MustCompile(`^release\.r[0-9\-.]+`)
 	allCmd      = "all" + suffix
+	makeCmd     = "make" + suffix
 	raceCmd     = "race" + suffix
 	cleanCmd    = "clean" + suffix
 	suffix      = defaultSuffix()
+	exeExt      = defaultExeExt()
+
+	benchCPU      = CpuList([]int{1})
+	benchAffinity = CpuList([]int{})
+	benchMutex    *FileMutex // Isolates benchmarks from other activities
 )
 
+// CpuList is used as flag.Value for -benchcpu flag.
+type CpuList []int
+
+func (cl *CpuList) String() string {
+	str := ""
+	for _, cpu := range *cl {
+		if str == "" {
+			str = strconv.Itoa(cpu)
+		} else {
+			str += fmt.Sprintf(",%v", cpu)
+		}
+	}
+	return str
+}
+
+func (cl *CpuList) Set(str string) error {
+	*cl = []int{}
+	for _, val := range strings.Split(str, ",") {
+		val = strings.TrimSpace(val)
+		if val == "" {
+			continue
+		}
+		cpu, err := strconv.Atoi(val)
+		if err != nil || cpu <= 0 {
+			return fmt.Errorf("%v is a bad value for GOMAXPROCS", val)
+		}
+		*cl = append(*cl, cpu)
+	}
+	if len(*cl) == 0 {
+		*cl = append(*cl, 1)
+	}
+	return nil
+}
+
 func main() {
+	flag.Var(&benchCPU, "benchcpu", "Comma-delimited list of GOMAXPROCS values for benchmarking")
+	flag.Var(&benchAffinity, "benchaffinity", "Comma-delimited list of affinity values for benchmarking")
 	flag.Usage = func() {
 		fmt.Fprintf(os.Stderr, "usage: %s goos-goarch...\n", os.Args[0])
 		flag.PrintDefaults()
@@ -78,6 +130,8 @@ func main() {
 	vcs.ShowCmd = *verbose
 	vcs.Verbose = *verbose
 
+	benchMutex = MakeFileMutex(*fileLock)
+
 	rr, err := repoForTool()
 	if err != nil {
 		log.Fatal("Error finding repository:", err)
@@ -139,11 +193,17 @@ func main() {
 		return
 	}
 
+	if !*doBuild && !*doBench {
+		fmt.Fprintf(os.Stderr, "Nothing to do, exiting (specify either -build or -bench or both)\n")
+		os.Exit(2)
+	}
+
 	// Start commit watcher
 	go commitWatcher(goroot)
 
 	// go continuous build mode
 	// check for new commits and build them
+	benchMutex.RLock()
 	for {
 		built := false
 		t := time.Now()
@@ -151,7 +211,7 @@ func main() {
 			done := make(chan bool)
 			for _, b := range builders {
 				go func(b *Builder) {
-					done <- b.build()
+					done <- b.buildOrBench()
 				}(b)
 			}
 			for _ = range builders {
@@ -159,13 +219,15 @@ func main() {
 			}
 		} else {
 			for _, b := range builders {
-				built = b.build() || built
+				built = b.buildOrBench() || built
 			}
 		}
 		// sleep if there was nothing to build
+		benchMutex.RUnlock()
 		if !built {
 			time.Sleep(waitInterval)
 		}
+		benchMutex.RLock()
 		// sleep if we're looping too fast.
 		dt := time.Now().Sub(t)
 		if dt < waitInterval {
@@ -256,11 +318,18 @@ func (b *Builder) buildCmd() string {
 	return *buildCmd
 }
 
-// build checks for a new commit for this builder
-// and builds it if one is found.
-// It returns true if a build was attempted.
-func (b *Builder) build() bool {
-	hash, err := b.todo("build-go-commit", "", "")
+// buildOrBench checks for a new commit for this builder
+// and builds or benchmarks it if one is found.
+// It returns true if a build/benchmark was attempted.
+func (b *Builder) buildOrBench() bool {
+	var kinds []string
+	if *doBuild {
+		kinds = append(kinds, "build-go-commit")
+	}
+	if *doBench {
+		kinds = append(kinds, "benchmark-go-commit")
+	}
+	kind, hash, benchs, err := b.todo(kinds, "", "")
 	if err != nil {
 		log.Println(err)
 		return false
@@ -268,11 +337,21 @@ func (b *Builder) build() bool {
 	if hash == "" {
 		return false
 	}
-
-	if err := b.buildHash(hash); err != nil {
-		log.Println(err)
+	switch kind {
+	case "build-go-commit":
+		if err := b.buildHash(hash); err != nil {
+			log.Println(err)
+		}
+		return true
+	case "benchmark-go-commit":
+		if err := b.benchHash(hash, benchs); err != nil {
+			log.Println(err)
+		}
+		return true
+	default:
+		log.Println("Unknown todo kind %v", kind)
+		return false
 	}
-	return true
 }
 
 func (b *Builder) buildHash(hash string) error {
@@ -283,58 +362,12 @@ func (b *Builder) buildHash(hash string) error {
 	if err := os.Mkdir(workpath, mkdirPerm); err != nil {
 		return err
 	}
-	defer os.RemoveAll(workpath)
+	defer removePath(workpath)
 
-	// pull before cloning to ensure we have the revision
-	if err := b.goroot.Pull(); err != nil {
-		return err
-	}
-
-	// set up builder's environment.
-	srcDir, err := b.env.setup(b.goroot, workpath, hash, b.envv())
+	buildLog, runTime, err := b.buildRepoOnHash(workpath, hash, b.buildCmd())
 	if err != nil {
-		return err
-	}
-
-	// build
-	var buildlog bytes.Buffer
-	logfile := filepath.Join(workpath, "build.log")
-	f, err := os.Create(logfile)
-	if err != nil {
-		return err
-	}
-	defer f.Close()
-	w := io.MultiWriter(f, &buildlog)
-
-	cmd := b.buildCmd()
-
-	// go's build command is a script relative to the srcDir, whereas
-	// gccgo's build command is usually "make check-go" in the srcDir.
-	if *buildTool == "go" {
-		if !filepath.IsAbs(cmd) {
-			cmd = filepath.Join(srcDir, cmd)
-		}
-	}
-
-	// make sure commands with extra arguments are handled properly
-	splitCmd := strings.Split(cmd, " ")
-	startTime := time.Now()
-	ok, err := runOutput(*buildTimeout, b.envv(), w, srcDir, splitCmd...)
-	runTime := time.Now().Sub(startTime)
-	errf := func() string {
-		if err != nil {
-			return fmt.Sprintf("error: %v", err)
-		}
-		if !ok {
-			return "failed"
-		}
-		return "success"
-	}
-	fmt.Fprintf(w, "Build complete, duration %v. Result: %v\n", runTime, errf())
-
-	if err != nil || !ok {
 		// record failure
-		return b.recordResult(false, "", hash, "", buildlog.String(), runTime)
+		return b.recordResult(false, "", hash, "", buildLog, runTime)
 	}
 
 	// record success
@@ -350,11 +383,74 @@ func (b *Builder) buildHash(hash string) error {
 	return nil
 }
 
+// buildRepoOnHash clones repo into workpath and builds it.
+func (b *Builder) buildRepoOnHash(workpath, hash, cmd string) (buildLog string, runTime time.Duration, err error) {
+	// Delete the previous workdir, if necessary
+	// (benchmarking code can execute several benchmarks in the same workpath).
+	if b.lastWorkpath != "" {
+		if b.lastWorkpath == workpath {
+			panic("workpath already exists: " + workpath)
+		}
+		removePath(b.lastWorkpath)
+		b.lastWorkpath = ""
+	}
+
+	// pull before cloning to ensure we have the revision
+	if err = b.goroot.Pull(); err != nil {
+		buildLog = err.Error()
+		return
+	}
+
+	// set up builder's environment.
+	srcDir, err := b.env.setup(b.goroot, workpath, hash, b.envv())
+	if err != nil {
+		buildLog = err.Error()
+		return
+	}
+
+	// build
+	var buildlog bytes.Buffer
+	logfile := filepath.Join(workpath, "build.log")
+	f, err := os.Create(logfile)
+	if err != nil {
+		buildLog = err.Error()
+		return
+	}
+	defer f.Close()
+	w := io.MultiWriter(f, &buildlog)
+
+	// go's build command is a script relative to the srcDir, whereas
+	// gccgo's build command is usually "make check-go" in the srcDir.
+	if *buildTool == "go" {
+		if !filepath.IsAbs(cmd) {
+			cmd = filepath.Join(srcDir, cmd)
+		}
+	}
+
+	// make sure commands with extra arguments are handled properly
+	splitCmd := strings.Split(cmd, " ")
+	startTime := time.Now()
+	ok, err := runOutput(*buildTimeout, b.envv(), w, srcDir, splitCmd...)
+	runTime = time.Now().Sub(startTime)
+	if !ok && err == nil {
+		err = fmt.Errorf("build failed")
+	}
+	errf := func() string {
+		if err != nil {
+			return fmt.Sprintf("error: %v", err)
+		}
+		return "success"
+	}
+	fmt.Fprintf(w, "Build complete, duration %v. Result: %v\n", runTime, errf())
+	buildLog = buildlog.String()
+	return
+}
+
 // failBuild checks for a new commit for this builder
 // and fails it if one is found.
 // It returns true if a build was "attempted".
 func (b *Builder) failBuild() bool {
-	hash, err := b.todo("build-go-commit", "", "")
+	_, hash, _, err := b.todo([]string{"build-go-commit"}, "", "")
 	if err != nil {
 		log.Println(err)
 		return false
@@ -374,7 +470,7 @@ func (b *Builder) failBuild() bool {
 func (b *Builder) buildSubrepos(goRoot, goPath, goHash string) {
 	for _, pkg := range dashboardPackages("subrepo") {
 		// get the latest todo for this package
-		hash, err := b.todo("build-package", pkg, goHash)
+		_, hash, _, err := b.todo([]string{"build-package"}, pkg, goHash)
 		if err != nil {
 			log.Printf("buildSubrepos %s: %v", pkg, err)
 			continue
@@ -406,7 +502,7 @@ func (b *Builder) buildSubrepos(goRoot, goPath, goHash string) {
 // buildSubrepo fetches the given package, updates it to the specified hash,
 // and runs 'go test -short pkg/...'. It returns the build log and any error.
 func (b *Builder) buildSubrepo(goRoot, goPath, pkg, hash string) (string, error) {
-	goTool := filepath.Join(goRoot, "bin", "go")
+	goTool := filepath.Join(goRoot, "bin", "go") + exeExt
 	env := append(b.envv(), "GOROOT="+goRoot, "GOPATH="+goPath)
 
 	// add $GOROOT/bin and $GOPATH/bin to PATH
@@ -485,6 +581,7 @@ func commitWatcher(goroot *Repo) {
 	}
 	key := b.key
 
+	benchMutex.RLock()
 	for {
 		if *verbose {
 			log.Printf("poll...")
@@ -504,10 +601,12 @@ func commitWatcher(goroot *Repo) {
 			}
 			commitPoll(pkgroot, pkg, key)
 		}
+		benchMutex.RUnlock()
 		if *verbose {
 			log.Printf("sleep...")
 		}
 		time.Sleep(*commitInterval)
+		benchMutex.RLock()
 	}
 }
 
@@ -556,6 +655,10 @@ func commitPoll(repo *Repo, pkg, key string) {
 			log.Printf("hg log %s: %s < %s\n", pkg, l.Hash, l.Parent)
 		}
 		if logByHash[l.Hash] == nil {
+			l.bench = needsBenchmarking(l)
+			// These fields are needed only for needsBenchmarking, do not waste memory.
+			l.Branch = ""
+			l.Files = ""
 			// Make copy to avoid pinning entire slice when only one entry is new.
 			t := *l
 			logByHash[t.Hash] = &t
@@ -619,6 +722,15 @@ func defaultSuffix() string {
 	}
 }
 
+func defaultExeExt() string {
+	switch runtime.GOOS {
+	case "windows":
+		return ".exe"
+	default:
+		return ""
+	}
+}
+
 // defaultBuildRoot returns default buildroot directory.
 func defaultBuildRoot() string {
 	var d string
@@ -631,3 +743,16 @@ func defaultBuildRoot() string {
 	}
 	return filepath.Join(d, "gobuilder")
 }
+
+// removePath is a more robust version of os.RemoveAll.
+// On windows, if remove fails (which can happen if test/benchmark timeouts
+// and keeps some files open) it tries to rename the dir.
+func removePath(path string) error {
+	if err := os.RemoveAll(path); err != nil {
+		if runtime.GOOS == "windows" {
+			err = os.Rename(path, filepath.Clean(path)+"_remove_me")
+		}
+		return err
+	}
+	return nil
+}

dashboard/builder/vcs.go

@@ -136,6 +136,12 @@ func (r *Repo) Log() ([]HgLog, error) {
 	if err != nil {
 		return nil, err
 	}
+	for i, log := range logStruct.Log {
+		// Let's pretend there can be only one parent.
+		if log.Parent != "" && strings.Contains(log.Parent, " ") {
+			logStruct.Log[i].Parent = strings.Split(log.Parent, " ")[0]
+		}
+	}
 	return logStruct.Log, nil
 }
 
@@ -174,19 +180,26 @@ type HgLog struct {
 	Date   string
 	Desc   string
 	Parent string
+	Branch string
+	Files  string
 
 	// Internal metadata
 	added bool
+	bench bool // needs to be benchmarked?
 }
 
 // xmlLogTemplate is a template to pass to Mercurial to make
 // hg log print the log in valid XML for parsing with xml.Unmarshal.
+// Can not escape branches and files, because it crashes python with:
+// AttributeError: 'NoneType' object has no attribute 'replace'
 const xmlLogTemplate = `
         <Log>
         <Hash>{node|escape}</Hash>
-        <Parent>{parent|escape}</Parent>
+        <Parent>{parents}</Parent>
         <Author>{author|escape}</Author>
         <Date>{date|rfc3339date}</Date>
         <Desc>{desc|escape}</Desc>
+        <Branch>{branches}</Branch>
+        <Files>{files}</Files>
         </Log>
 `