From d2a9e7164ee23fcf0d9650ec4dbd2f10e405e718 Mon Sep 17 00:00:00 2001 From: Dmitriy Vyukov Date: Tue, 13 May 2014 11:00:11 +0400 Subject: [PATCH] dashboard: builder changes for performance dashboard This CL moves code from code.google.com/p/dvyukov-go-perf-dashboard, which was previously reviewed. LGTM=adg R=adg CC=golang-codereviews https://golang.org/cl/95190043 --- dashboard/builder/bench.go | 253 +++++++++++++++++++++++++ dashboard/builder/filemutex_flock.go | 66 +++++++ dashboard/builder/filemutex_local.go | 27 +++ dashboard/builder/filemutex_windows.go | 105 ++++++++++ dashboard/builder/http.go | 64 +++++-- dashboard/builder/main.go | 251 ++++++++++++++++++------ dashboard/builder/vcs.go | 15 +- 7 files changed, 702 insertions(+), 79 deletions(-) create mode 100644 dashboard/builder/bench.go create mode 100644 dashboard/builder/filemutex_flock.go create mode 100644 dashboard/builder/filemutex_local.go create mode 100644 dashboard/builder/filemutex_windows.go diff --git a/dashboard/builder/bench.go b/dashboard/builder/bench.go new file mode 100644 index 00000000000..cb2b0278ff2 --- /dev/null +++ b/dashboard/builder/bench.go @@ -0,0 +1,253 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "bufio" + "bytes" + "fmt" + "io" + "io/ioutil" + "log" + "os" + "os/exec" + "path/filepath" + "regexp" + "strconv" + "strings" +) + +// benchHash benchmarks a single commit. +func (b *Builder) benchHash(hash string, benchs []string) error { + if *verbose { + log.Println(b.name, "benchmarking", hash) + } + + res := &PerfResult{Hash: hash, Benchmark: "meta-done"} + + // Create place in which to do work. + workpath := filepath.Join(*buildroot, b.name+"-"+hash[:12]) + // Prepare a workpath if we don't have one we can reuse. + update := false + if b.lastWorkpath != workpath { + if err := os.Mkdir(workpath, mkdirPerm); err != nil { + return err + } + buildLog, _, err := b.buildRepoOnHash(workpath, hash, makeCmd) + if err != nil { + removePath(workpath) + // record failure + res.Artifacts = append(res.Artifacts, PerfArtifact{"log", buildLog}) + return b.recordPerfResult(res) + } + b.lastWorkpath = workpath + update = true + } + + // Build the benchmark binary. + benchBin, buildLog, err := b.buildBenchmark(workpath, update) + if err != nil { + // record failure + res.Artifacts = append(res.Artifacts, PerfArtifact{"log", buildLog}) + return b.recordPerfResult(res) + } + + benchmark, procs, affinity, last := chooseBenchmark(benchBin, benchs) + if benchmark != "" { + res.Benchmark = fmt.Sprintf("%v-%v", benchmark, procs) + res.Metrics, res.Artifacts, res.OK = b.executeBenchmark(workpath, hash, benchBin, benchmark, procs, affinity) + if err = b.recordPerfResult(res); err != nil { + return fmt.Errorf("recordResult: %s", err) + } + } + + if last { + // All benchmarks have beed executed, don't need workpath anymore. + removePath(b.lastWorkpath) + b.lastWorkpath = "" + // Notify the app. + res = &PerfResult{Hash: hash, Benchmark: "meta-done", OK: true} + if err = b.recordPerfResult(res); err != nil { + return fmt.Errorf("recordResult: %s", err) + } + } + + return nil +} + +// buildBenchmark builds the benchmark binary. +func (b *Builder) buildBenchmark(workpath string, update bool) (benchBin, log string, err error) { + goroot := filepath.Join(workpath, "go") + gobin := filepath.Join(goroot, "bin", "go") + exeExt + gopath := filepath.Join(*buildroot, "gopath") + env := append([]string{ + "GOROOT=" + goroot, + "GOPATH=" + gopath}, + b.envv()...) + // First, download without installing. + cmd := []string{gobin, "get", "-d"} + if update { + cmd = append(cmd, "-u") + } + cmd = append(cmd, *benchPath) + var buildlog bytes.Buffer + ok, err := runOutput(*buildTimeout, env, &buildlog, workpath, cmd...) + if !ok || err != nil { + fmt.Fprintf(&buildlog, "go get -d %s failed: %s", *benchPath, err) + return "", buildlog.String(), err + } + // Then, build into workpath. + benchBin = filepath.Join(workpath, "benchbin") + exeExt + cmd = []string{gobin, "build", "-o", benchBin, *benchPath} + buildlog.Reset() + ok, err = runOutput(*buildTimeout, env, &buildlog, workpath, cmd...) + if !ok || err != nil { + fmt.Fprintf(&buildlog, "go build %s failed: %s", *benchPath, err) + return "", buildlog.String(), err + } + return benchBin, "", nil +} + +// chooseBenchmark chooses the next benchmark to run +// based on the list of available benchmarks, already executed benchmarks +// and -benchcpu list. +func chooseBenchmark(benchBin string, doneBenchs []string) (bench string, procs, affinity int, last bool) { + out, err := exec.Command(benchBin).CombinedOutput() + if err != nil { + log.Printf("Failed to query benchmark list: %v\n%s", err, out) + last = true + return + } + outStr := string(out) + nlIdx := strings.Index(outStr, "\n") + if nlIdx < 0 { + log.Printf("Failed to parse benchmark list (no new line): %s", outStr) + last = true + return + } + localBenchs := strings.Split(outStr[:nlIdx], ",") + benchsMap := make(map[string]bool) + for _, b := range doneBenchs { + benchsMap[b] = true + } + cnt := 0 + // We want to run all benchmarks with GOMAXPROCS=1 first. + for i, procs1 := range benchCPU { + for _, bench1 := range localBenchs { + if benchsMap[fmt.Sprintf("%v-%v", bench1, procs1)] { + continue + } + cnt++ + if cnt == 1 { + bench = bench1 + procs = procs1 + if i < len(benchAffinity) { + affinity = benchAffinity[i] + } + } + } + } + last = cnt <= 1 + return +} + +// executeBenchmark runs a single benchmark and parses its output. +func (b *Builder) executeBenchmark(workpath, hash, benchBin, bench string, procs, affinity int) (metrics []PerfMetric, artifacts []PerfArtifact, ok bool) { + // Benchmarks runs mutually exclusive with other activities. + benchMutex.RUnlock() + defer benchMutex.RLock() + benchMutex.Lock() + defer benchMutex.Unlock() + + log.Printf("%v executing benchmark %v-%v on %v", b.name, bench, procs, hash) + + // The benchmark executes 'go build'/'go tool', + // so we need properly setup env. + env := append([]string{ + "GOROOT=" + filepath.Join(workpath, "go"), + "PATH=" + filepath.Join(workpath, "go", "bin") + string(os.PathListSeparator) + os.Getenv("PATH"), + "GODEBUG=gctrace=1", // since Go1.2 + "GOGCTRACE=1", // before Go1.2 + fmt.Sprintf("GOMAXPROCS=%v", procs)}, + b.envv()...) + cmd := []string{benchBin, + "-bench", bench, + "-benchmem", strconv.Itoa(*benchMem), + "-benchtime", benchTime.String(), + "-benchnum", strconv.Itoa(*benchNum), + "-tmpdir", workpath} + if affinity != 0 { + cmd = append(cmd, "-affinity", strconv.Itoa(affinity)) + } + benchlog := new(bytes.Buffer) + ok, err := runOutput(*buildTimeout, env, benchlog, workpath, cmd...) + if strip := benchlog.Len() - 512<<10; strip > 0 { + // Leave the last 512K, that part contains metrics. + benchlog = bytes.NewBuffer(benchlog.Bytes()[strip:]) + } + artifacts = []PerfArtifact{{Type: "log", Body: benchlog.String()}} + if !ok || err != nil { + if err != nil { + log.Printf("Failed to execute benchmark '%v': %v", bench, err) + ok = false + } + return + } + + metrics1, artifacts1, err := parseBenchmarkOutput(benchlog) + if err != nil { + log.Printf("Failed to parse benchmark output: %v", err) + ok = false + return + } + metrics = metrics1 + artifacts = append(artifacts, artifacts1...) + return +} + +// parseBenchmarkOutput fetches metrics and artifacts from benchmark output. +func parseBenchmarkOutput(out io.Reader) (metrics []PerfMetric, artifacts []PerfArtifact, err error) { + s := bufio.NewScanner(out) + metricRe := regexp.MustCompile("^GOPERF-METRIC:([a-z,0-9,-]+)=([0-9]+)$") + fileRe := regexp.MustCompile("^GOPERF-FILE:([a-z,0-9,-]+)=(.+)$") + for s.Scan() { + ln := s.Text() + if ss := metricRe.FindStringSubmatch(ln); ss != nil { + var v uint64 + v, err = strconv.ParseUint(ss[2], 10, 64) + if err != nil { + err = fmt.Errorf("Failed to parse metric '%v=%v': %v", ss[1], ss[2], err) + return + } + metrics = append(metrics, PerfMetric{Type: ss[1], Val: v}) + } else if ss := fileRe.FindStringSubmatch(ln); ss != nil { + var buf []byte + buf, err = ioutil.ReadFile(ss[2]) + if err != nil { + err = fmt.Errorf("Failed to read file '%v': %v", ss[2], err) + return + } + artifacts = append(artifacts, PerfArtifact{ss[1], string(buf)}) + } + } + return +} + +// needsBenchmarking determines whether the commit needs benchmarking. +func needsBenchmarking(log *HgLog) bool { + // Do not benchmark branch commits, they are usually not interesting + // and fall out of the trunk succession. + if log.Branch != "" { + return false + } + // Do not benchmark commits that do not touch source files (e.g. CONTRIBUTORS). + for _, f := range strings.Split(log.Files, " ") { + if (strings.HasPrefix(f, "include") || strings.HasPrefix(f, "src")) && + !strings.HasSuffix(f, "_test.go") && !strings.Contains(f, "testdata") { + return true + } + } + return false +} diff --git a/dashboard/builder/filemutex_flock.go b/dashboard/builder/filemutex_flock.go new file mode 100644 index 00000000000..3c1d655f986 --- /dev/null +++ b/dashboard/builder/filemutex_flock.go @@ -0,0 +1,66 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build linux darwin freebsd + +package main + +import ( + "sync" + "syscall" +) + +// FileMutex is similar to sync.RWMutex, but also synchronizes across processes. +// This implementation is based on flock syscall. +type FileMutex struct { + mu sync.RWMutex + fd int +} + +func MakeFileMutex(filename string) *FileMutex { + if filename == "" { + return &FileMutex{fd: -1} + } + fd, err := syscall.Open(filename, syscall.O_CREAT|syscall.O_RDONLY, mkdirPerm) + if err != nil { + panic(err) + } + return &FileMutex{fd: fd} +} + +func (m *FileMutex) Lock() { + m.mu.Lock() + if m.fd != -1 { + if err := syscall.Flock(m.fd, syscall.LOCK_EX); err != nil { + panic(err) + } + } +} + +func (m *FileMutex) Unlock() { + if m.fd != -1 { + if err := syscall.Flock(m.fd, syscall.LOCK_UN); err != nil { + panic(err) + } + } + m.mu.Unlock() +} + +func (m *FileMutex) RLock() { + m.mu.RLock() + if m.fd != -1 { + if err := syscall.Flock(m.fd, syscall.LOCK_SH); err != nil { + panic(err) + } + } +} + +func (m *FileMutex) RUnlock() { + if m.fd != -1 { + if err := syscall.Flock(m.fd, syscall.LOCK_UN); err != nil { + panic(err) + } + } + m.mu.RUnlock() +} diff --git a/dashboard/builder/filemutex_local.go b/dashboard/builder/filemutex_local.go new file mode 100644 index 00000000000..36248f729d1 --- /dev/null +++ b/dashboard/builder/filemutex_local.go @@ -0,0 +1,27 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build netbsd openbsd plan9 + +package main + +import ( + "log" + "sync" +) + +// FileMutex is similar to sync.RWMutex, but also synchronizes across processes. +// This implementation is a fallback that does not actually provide inter-process synchronization. +type FileMutex struct { + sync.RWMutex +} + +func MakeFileMutex(filename string) *FileMutex { + return &FileMutex{} +} + +func init() { + log.Printf("WARNING: using fake file mutex." + + " Don't run more than one of these at once!!!") +} diff --git a/dashboard/builder/filemutex_windows.go b/dashboard/builder/filemutex_windows.go new file mode 100644 index 00000000000..1f058b2380d --- /dev/null +++ b/dashboard/builder/filemutex_windows.go @@ -0,0 +1,105 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "sync" + "syscall" + "unsafe" +) + +var ( + modkernel32 = syscall.NewLazyDLL("kernel32.dll") + procLockFileEx = modkernel32.NewProc("LockFileEx") + procUnlockFileEx = modkernel32.NewProc("UnlockFileEx") +) + +const ( + INVALID_FILE_HANDLE = ^syscall.Handle(0) + LOCKFILE_EXCLUSIVE_LOCK = 2 +) + +func lockFileEx(h syscall.Handle, flags, reserved, locklow, lockhigh uint32, ol *syscall.Overlapped) (err error) { + r1, _, e1 := syscall.Syscall6(procLockFileEx.Addr(), 6, uintptr(h), uintptr(flags), uintptr(reserved), uintptr(locklow), uintptr(lockhigh), uintptr(unsafe.Pointer(ol))) + if r1 == 0 { + if e1 != 0 { + err = error(e1) + } else { + err = syscall.EINVAL + } + } + return +} + +func unlockFileEx(h syscall.Handle, reserved, locklow, lockhigh uint32, ol *syscall.Overlapped) (err error) { + r1, _, e1 := syscall.Syscall6(procUnlockFileEx.Addr(), 5, uintptr(h), uintptr(reserved), uintptr(locklow), uintptr(lockhigh), uintptr(unsafe.Pointer(ol)), 0) + if r1 == 0 { + if e1 != 0 { + err = error(e1) + } else { + err = syscall.EINVAL + } + } + return +} + +// FileMutex is similar to sync.RWMutex, but also synchronizes across processes. +// This implementation is based on flock syscall. +type FileMutex struct { + mu sync.RWMutex + fd syscall.Handle +} + +func MakeFileMutex(filename string) *FileMutex { + if filename == "" { + return &FileMutex{fd: INVALID_FILE_HANDLE} + } + fd, err := syscall.CreateFile(&(syscall.StringToUTF16(filename)[0]), syscall.GENERIC_READ|syscall.GENERIC_WRITE, + syscall.FILE_SHARE_READ|syscall.FILE_SHARE_WRITE, nil, syscall.OPEN_ALWAYS, syscall.FILE_ATTRIBUTE_NORMAL, 0) + if err != nil { + panic(err) + } + return &FileMutex{fd: fd} +} + +func (m *FileMutex) Lock() { + m.mu.Lock() + if m.fd != INVALID_FILE_HANDLE { + var ol syscall.Overlapped + if err := lockFileEx(m.fd, LOCKFILE_EXCLUSIVE_LOCK, 0, 1, 0, &ol); err != nil { + panic(err) + } + } +} + +func (m *FileMutex) Unlock() { + if m.fd != INVALID_FILE_HANDLE { + var ol syscall.Overlapped + if err := unlockFileEx(m.fd, 0, 1, 0, &ol); err != nil { + panic(err) + } + } + m.mu.Unlock() +} + +func (m *FileMutex) RLock() { + m.mu.RLock() + if m.fd != INVALID_FILE_HANDLE { + var ol syscall.Overlapped + if err := lockFileEx(m.fd, 0, 0, 1, 0, &ol); err != nil { + panic(err) + } + } +} + +func (m *FileMutex) RUnlock() { + if m.fd != INVALID_FILE_HANDLE { + var ol syscall.Overlapped + if err := unlockFileEx(m.fd, 0, 1, 0, &ol); err != nil { + panic(err) + } + } + m.mu.RUnlock() +} diff --git a/dashboard/builder/http.go b/dashboard/builder/http.go index 1a3d15612ff..ece5df3acd4 100644 --- a/dashboard/builder/http.go +++ b/dashboard/builder/http.go @@ -89,30 +89,36 @@ func dash(meth, cmd string, args url.Values, req, resp interface{}) error { return nil } -// todo returns the next hash to build. -func (b *Builder) todo(kind, pkg, goHash string) (rev string, err error) { +// todo returns the next hash to build or benchmark. +func (b *Builder) todo(kinds []string, pkg, goHash string) (kind, rev string, benchs []string, err error) { args := url.Values{ - "kind": {kind}, "builder": {b.name}, "packagePath": {pkg}, "goHash": {goHash}, } + for _, k := range kinds { + args.Add("kind", k) + } var resp *struct { Kind string Data struct { - Hash string + Hash string + PerfResults []string } } if err = dash("GET", "todo", args, nil, &resp); err != nil { - return "", err + return } if resp == nil { - return "", nil + return } - if kind != resp.Kind { - return "", fmt.Errorf("expecting Kind %q, got %q", kind, resp.Kind) + for _, k := range kinds { + if k == resp.Kind { + return resp.Kind, resp.Data.Hash, resp.Data.PerfResults, nil + } } - return resp.Data.Hash, nil + err = fmt.Errorf("expecting Kinds %q, got %q", kinds, resp.Kind) + return } // recordResult sends build results to the dashboard @@ -130,18 +136,46 @@ func (b *Builder) recordResult(ok bool, pkg, hash, goHash, buildLog string, runT return dash("POST", "result", args, req, nil) } +// Result of running a single benchmark on a single commit. +type PerfResult struct { + Builder string + Benchmark string + Hash string + OK bool + Metrics []PerfMetric + Artifacts []PerfArtifact +} + +type PerfMetric struct { + Type string + Val uint64 +} + +type PerfArtifact struct { + Type string + Body string +} + +// recordPerfResult sends benchmarking results to the dashboard +func (b *Builder) recordPerfResult(req *PerfResult) error { + req.Builder = b.name + args := url.Values{"key": {b.key}, "builder": {b.name}} + return dash("POST", "perf-result", args, req, nil) +} + func postCommit(key, pkg string, l *HgLog) error { t, err := time.Parse(time.RFC3339, l.Date) if err != nil { return fmt.Errorf("parsing %q: %v", l.Date, t) } return dash("POST", "commit", url.Values{"key": {key}}, obj{ - "PackagePath": pkg, - "Hash": l.Hash, - "ParentHash": l.Parent, - "Time": t.Format(time.RFC3339), - "User": l.Author, - "Desc": l.Desc, + "PackagePath": pkg, + "Hash": l.Hash, + "ParentHash": l.Parent, + "Time": t.Format(time.RFC3339), + "User": l.Author, + "Desc": l.Desc, + "NeedsBenchmarking": l.bench, }, nil) } diff --git a/dashboard/builder/main.go b/dashboard/builder/main.go index af36db4f635..da912b477b4 100644 --- a/dashboard/builder/main.go +++ b/dashboard/builder/main.go @@ -15,6 +15,7 @@ import ( "path/filepath" "regexp" "runtime" + "strconv" "strings" "time" @@ -36,9 +37,13 @@ type Builder struct { goos, goarch string key string env builderEnv + // Last benchmarking workpath. We reuse it, if do successive benchmarks on the same commit. + lastWorkpath string } var ( + doBuild = flag.Bool("build", true, "Build and test packages") + doBench = flag.Bool("bench", false, "Run benchmarks") buildroot = flag.String("buildroot", defaultBuildRoot(), "Directory under which to build") dashboard = flag.String("dashboard", "build.golang.org", "Go Dashboard Host") buildRelease = flag.Bool("release", false, "Build and upload binary release archives") @@ -47,11 +52,16 @@ var ( buildTool = flag.String("tool", "go", "Tool to build.") gcPath = flag.String("gcpath", "code.google.com/p/go", "Path to download gc from") gccPath = flag.String("gccpath", "https://github.com/mirrors/gcc.git", "Path to download gcc from") + benchPath = flag.String("benchpath", "code.google.com/p/go.benchmarks/bench", "Path to download benchmarks from") failAll = flag.Bool("fail", false, "fail all builds") parallel = flag.Bool("parallel", false, "Build multiple targets in parallel") buildTimeout = flag.Duration("buildTimeout", 60*time.Minute, "Maximum time to wait for builds and tests") cmdTimeout = flag.Duration("cmdTimeout", 10*time.Minute, "Maximum time to wait for an external command") commitInterval = flag.Duration("commitInterval", 1*time.Minute, "Time to wait between polling for new commits (0 disables commit poller)") + benchNum = flag.Int("benchnum", 5, "Run each benchmark that many times") + benchTime = flag.Duration("benchtime", 5*time.Second, "Benchmarking time for a single benchmark run") + benchMem = flag.Int("benchmem", 64, "Approx RSS value to aim at in benchmarks, in MB") + fileLock = flag.String("filelock", "", "File to lock around benchmaring (synchronizes several builders)") verbose = flag.Bool("v", false, "verbose") ) @@ -59,12 +69,54 @@ var ( binaryTagRe = regexp.MustCompile(`^(release\.r|weekly\.)[0-9\-.]+`) releaseRe = regexp.MustCompile(`^release\.r[0-9\-.]+`) allCmd = "all" + suffix + makeCmd = "make" + suffix raceCmd = "race" + suffix cleanCmd = "clean" + suffix suffix = defaultSuffix() + exeExt = defaultExeExt() + + benchCPU = CpuList([]int{1}) + benchAffinity = CpuList([]int{}) + benchMutex *FileMutex // Isolates benchmarks from other activities ) +// CpuList is used as flag.Value for -benchcpu flag. +type CpuList []int + +func (cl *CpuList) String() string { + str := "" + for _, cpu := range *cl { + if str == "" { + str = strconv.Itoa(cpu) + } else { + str += fmt.Sprintf(",%v", cpu) + } + } + return str +} + +func (cl *CpuList) Set(str string) error { + *cl = []int{} + for _, val := range strings.Split(str, ",") { + val = strings.TrimSpace(val) + if val == "" { + continue + } + cpu, err := strconv.Atoi(val) + if err != nil || cpu <= 0 { + return fmt.Errorf("%v is a bad value for GOMAXPROCS", val) + } + *cl = append(*cl, cpu) + } + if len(*cl) == 0 { + *cl = append(*cl, 1) + } + return nil +} + func main() { + flag.Var(&benchCPU, "benchcpu", "Comma-delimited list of GOMAXPROCS values for benchmarking") + flag.Var(&benchAffinity, "benchaffinity", "Comma-delimited list of affinity values for benchmarking") flag.Usage = func() { fmt.Fprintf(os.Stderr, "usage: %s goos-goarch...\n", os.Args[0]) flag.PrintDefaults() @@ -78,6 +130,8 @@ func main() { vcs.ShowCmd = *verbose vcs.Verbose = *verbose + benchMutex = MakeFileMutex(*fileLock) + rr, err := repoForTool() if err != nil { log.Fatal("Error finding repository:", err) @@ -139,11 +193,17 @@ func main() { return } + if !*doBuild && !*doBench { + fmt.Fprintf(os.Stderr, "Nothing to do, exiting (specify either -build or -bench or both)\n") + os.Exit(2) + } + // Start commit watcher go commitWatcher(goroot) // go continuous build mode // check for new commits and build them + benchMutex.RLock() for { built := false t := time.Now() @@ -151,7 +211,7 @@ func main() { done := make(chan bool) for _, b := range builders { go func(b *Builder) { - done <- b.build() + done <- b.buildOrBench() }(b) } for _ = range builders { @@ -159,13 +219,15 @@ func main() { } } else { for _, b := range builders { - built = b.build() || built + built = b.buildOrBench() || built } } // sleep if there was nothing to build + benchMutex.RUnlock() if !built { time.Sleep(waitInterval) } + benchMutex.RLock() // sleep if we're looping too fast. dt := time.Now().Sub(t) if dt < waitInterval { @@ -256,11 +318,18 @@ func (b *Builder) buildCmd() string { return *buildCmd } -// build checks for a new commit for this builder -// and builds it if one is found. -// It returns true if a build was attempted. -func (b *Builder) build() bool { - hash, err := b.todo("build-go-commit", "", "") +// buildOrBench checks for a new commit for this builder +// and builds or benchmarks it if one is found. +// It returns true if a build/benchmark was attempted. +func (b *Builder) buildOrBench() bool { + var kinds []string + if *doBuild { + kinds = append(kinds, "build-go-commit") + } + if *doBench { + kinds = append(kinds, "benchmark-go-commit") + } + kind, hash, benchs, err := b.todo(kinds, "", "") if err != nil { log.Println(err) return false @@ -268,11 +337,21 @@ func (b *Builder) build() bool { if hash == "" { return false } - - if err := b.buildHash(hash); err != nil { - log.Println(err) + switch kind { + case "build-go-commit": + if err := b.buildHash(hash); err != nil { + log.Println(err) + } + return true + case "benchmark-go-commit": + if err := b.benchHash(hash, benchs); err != nil { + log.Println(err) + } + return true + default: + log.Println("Unknown todo kind %v", kind) + return false } - return true } func (b *Builder) buildHash(hash string) error { @@ -283,58 +362,12 @@ func (b *Builder) buildHash(hash string) error { if err := os.Mkdir(workpath, mkdirPerm); err != nil { return err } - defer os.RemoveAll(workpath) + defer removePath(workpath) - // pull before cloning to ensure we have the revision - if err := b.goroot.Pull(); err != nil { - return err - } - - // set up builder's environment. - srcDir, err := b.env.setup(b.goroot, workpath, hash, b.envv()) + buildLog, runTime, err := b.buildRepoOnHash(workpath, hash, b.buildCmd()) if err != nil { - return err - } - - // build - var buildlog bytes.Buffer - logfile := filepath.Join(workpath, "build.log") - f, err := os.Create(logfile) - if err != nil { - return err - } - defer f.Close() - w := io.MultiWriter(f, &buildlog) - - cmd := b.buildCmd() - - // go's build command is a script relative to the srcDir, whereas - // gccgo's build command is usually "make check-go" in the srcDir. - if *buildTool == "go" { - if !filepath.IsAbs(cmd) { - cmd = filepath.Join(srcDir, cmd) - } - } - - // make sure commands with extra arguments are handled properly - splitCmd := strings.Split(cmd, " ") - startTime := time.Now() - ok, err := runOutput(*buildTimeout, b.envv(), w, srcDir, splitCmd...) - runTime := time.Now().Sub(startTime) - errf := func() string { - if err != nil { - return fmt.Sprintf("error: %v", err) - } - if !ok { - return "failed" - } - return "success" - } - fmt.Fprintf(w, "Build complete, duration %v. Result: %v\n", runTime, errf()) - - if err != nil || !ok { // record failure - return b.recordResult(false, "", hash, "", buildlog.String(), runTime) + return b.recordResult(false, "", hash, "", buildLog, runTime) } // record success @@ -350,11 +383,74 @@ func (b *Builder) buildHash(hash string) error { return nil } +// buildRepoOnHash clones repo into workpath and builds it. +func (b *Builder) buildRepoOnHash(workpath, hash, cmd string) (buildLog string, runTime time.Duration, err error) { + // Delete the previous workdir, if necessary + // (benchmarking code can execute several benchmarks in the same workpath). + if b.lastWorkpath != "" { + if b.lastWorkpath == workpath { + panic("workpath already exists: " + workpath) + } + removePath(b.lastWorkpath) + b.lastWorkpath = "" + } + + // pull before cloning to ensure we have the revision + if err = b.goroot.Pull(); err != nil { + buildLog = err.Error() + return + } + + // set up builder's environment. + srcDir, err := b.env.setup(b.goroot, workpath, hash, b.envv()) + if err != nil { + buildLog = err.Error() + return + } + + // build + var buildlog bytes.Buffer + logfile := filepath.Join(workpath, "build.log") + f, err := os.Create(logfile) + if err != nil { + buildLog = err.Error() + return + } + defer f.Close() + w := io.MultiWriter(f, &buildlog) + + // go's build command is a script relative to the srcDir, whereas + // gccgo's build command is usually "make check-go" in the srcDir. + if *buildTool == "go" { + if !filepath.IsAbs(cmd) { + cmd = filepath.Join(srcDir, cmd) + } + } + + // make sure commands with extra arguments are handled properly + splitCmd := strings.Split(cmd, " ") + startTime := time.Now() + ok, err := runOutput(*buildTimeout, b.envv(), w, srcDir, splitCmd...) + runTime = time.Now().Sub(startTime) + if !ok && err == nil { + err = fmt.Errorf("build failed") + } + errf := func() string { + if err != nil { + return fmt.Sprintf("error: %v", err) + } + return "success" + } + fmt.Fprintf(w, "Build complete, duration %v. Result: %v\n", runTime, errf()) + buildLog = buildlog.String() + return +} + // failBuild checks for a new commit for this builder // and fails it if one is found. // It returns true if a build was "attempted". func (b *Builder) failBuild() bool { - hash, err := b.todo("build-go-commit", "", "") + _, hash, _, err := b.todo([]string{"build-go-commit"}, "", "") if err != nil { log.Println(err) return false @@ -374,7 +470,7 @@ func (b *Builder) failBuild() bool { func (b *Builder) buildSubrepos(goRoot, goPath, goHash string) { for _, pkg := range dashboardPackages("subrepo") { // get the latest todo for this package - hash, err := b.todo("build-package", pkg, goHash) + _, hash, _, err := b.todo([]string{"build-package"}, pkg, goHash) if err != nil { log.Printf("buildSubrepos %s: %v", pkg, err) continue @@ -406,7 +502,7 @@ func (b *Builder) buildSubrepos(goRoot, goPath, goHash string) { // buildSubrepo fetches the given package, updates it to the specified hash, // and runs 'go test -short pkg/...'. It returns the build log and any error. func (b *Builder) buildSubrepo(goRoot, goPath, pkg, hash string) (string, error) { - goTool := filepath.Join(goRoot, "bin", "go") + goTool := filepath.Join(goRoot, "bin", "go") + exeExt env := append(b.envv(), "GOROOT="+goRoot, "GOPATH="+goPath) // add $GOROOT/bin and $GOPATH/bin to PATH @@ -485,6 +581,7 @@ func commitWatcher(goroot *Repo) { } key := b.key + benchMutex.RLock() for { if *verbose { log.Printf("poll...") @@ -504,10 +601,12 @@ func commitWatcher(goroot *Repo) { } commitPoll(pkgroot, pkg, key) } + benchMutex.RUnlock() if *verbose { log.Printf("sleep...") } time.Sleep(*commitInterval) + benchMutex.RLock() } } @@ -556,6 +655,10 @@ func commitPoll(repo *Repo, pkg, key string) { log.Printf("hg log %s: %s < %s\n", pkg, l.Hash, l.Parent) } if logByHash[l.Hash] == nil { + l.bench = needsBenchmarking(l) + // These fields are needed only for needsBenchmarking, do not waste memory. + l.Branch = "" + l.Files = "" // Make copy to avoid pinning entire slice when only one entry is new. t := *l logByHash[t.Hash] = &t @@ -619,6 +722,15 @@ func defaultSuffix() string { } } +func defaultExeExt() string { + switch runtime.GOOS { + case "windows": + return ".exe" + default: + return "" + } +} + // defaultBuildRoot returns default buildroot directory. func defaultBuildRoot() string { var d string @@ -631,3 +743,16 @@ func defaultBuildRoot() string { } return filepath.Join(d, "gobuilder") } + +// removePath is a more robust version of os.RemoveAll. +// On windows, if remove fails (which can happen if test/benchmark timeouts +// and keeps some files open) it tries to rename the dir. +func removePath(path string) error { + if err := os.RemoveAll(path); err != nil { + if runtime.GOOS == "windows" { + err = os.Rename(path, filepath.Clean(path)+"_remove_me") + } + return err + } + return nil +} diff --git a/dashboard/builder/vcs.go b/dashboard/builder/vcs.go index 66dc251c7ff..544f7e322e3 100644 --- a/dashboard/builder/vcs.go +++ b/dashboard/builder/vcs.go @@ -136,6 +136,12 @@ func (r *Repo) Log() ([]HgLog, error) { if err != nil { return nil, err } + for i, log := range logStruct.Log { + // Let's pretend there can be only one parent. + if log.Parent != "" && strings.Contains(log.Parent, " ") { + logStruct.Log[i].Parent = strings.Split(log.Parent, " ")[0] + } + } return logStruct.Log, nil } @@ -174,19 +180,26 @@ type HgLog struct { Date string Desc string Parent string + Branch string + Files string // Internal metadata added bool + bench bool // needs to be benchmarked? } // xmlLogTemplate is a template to pass to Mercurial to make // hg log print the log in valid XML for parsing with xml.Unmarshal. +// Can not escape branches and files, because it crashes python with: +// AttributeError: 'NoneType' object has no attribute 'replace' const xmlLogTemplate = ` {node|escape} - {parent|escape} + {parents} {author|escape} {date|rfc3339date} {desc|escape} + {branches} + {files} `