From 062fee0536b77ba8edc7ebfdffc6349d95b424c2 Mon Sep 17 00:00:00 2001 From: Adam Langley Date: Tue, 26 Jan 2010 12:56:29 -0800 Subject: [PATCH] dashboard: add benchmarking support. This has actually been running for a while and gathering benchmark data. I haven't had a chance to add a UI for it yet however. R=rsc CC=golang-dev https://golang.org/cl/194082 --- misc/dashboard/README | 9 +- misc/dashboard/buildcontrol.py | 51 +++++++++- misc/dashboard/builder.sh | 17 +++- misc/dashboard/godashboard/gobuild.py | 140 +++++++++++++++++++++++++- misc/dashboard/godashboard/index.yaml | 6 ++ misc/dashboard/godashboard/key.py | 1 - 6 files changed, 212 insertions(+), 12 deletions(-) diff --git a/misc/dashboard/README b/misc/dashboard/README index 7b07a21a20..2878daef0c 100644 --- a/misc/dashboard/README +++ b/misc/dashboard/README @@ -1,6 +1,10 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + The files in this directory constitute the continuous builder: -godashboard/: An AppEngine which acts as a server +godashboard/: An AppEngine that acts as a server builder.sh, buildcontrol.sh: used by the build slaves If you wish to run a Go builder, please email golang-dev@googlegroups.com @@ -19,7 +23,8 @@ export PATH=$PATH:/gobuild/bin export BUILDER=XXX export BUILDHOST=godashboard.appspot.com -* Write ~gobuild/.gobuildkey (you need to get it from someone who knows it) +* Write the key ~gobuild/.gobuildkey (you need to get it from someone who knows + the key) * sudo apt-get install bison gcc libc6-dev ed make * cd ~gobuild diff --git a/misc/dashboard/buildcontrol.py b/misc/dashboard/buildcontrol.py index caa1a2f477..7851de731d 100644 --- a/misc/dashboard/buildcontrol.py +++ b/misc/dashboard/buildcontrol.py @@ -6,8 +6,10 @@ # This is a utility script for implementing a Go build slave. +import binascii import httplib import os +import struct import subprocess import sys import time @@ -42,10 +44,14 @@ def main(args): return doInit(args) elif args[1] == 'hwget': return doHWGet(args) + elif args[1] == 'hwset': + return doHWSet(args) elif args[1] == 'next': return doNext(args) elif args[1] == 'record': return doRecord(args) + elif args[1] == 'benchmarks': + return doBenchmarks(args) else: return usage(args[0]) @@ -55,8 +61,10 @@ def usage(name): Commands: init : init the build bot with the given commit as the first in history hwget : get the most recent revision built by the given builder + hwset : get the most recent revision built by the given builder next : get the next revision number to by built by the given builder record : record a build result + benchmarks : record benchmark numbers ''' % name) return 1 @@ -78,11 +86,21 @@ def doHWGet(args, retries = 0): if reply.status == 200: print reply.read() elif reply.status == 500 and retries < 3: + time.sleep(3) return doHWGet(args, retries = retries + 1) else: raise Failed('get-hw returned %d' % reply.status) return 0 +def doHWSet(args): + if len(args) != 4: + return usage(args[0]) + c = getCommit(args[3]) + if c is None: + fatal('Cannot get commit %s' % args[3]) + + return command('hw-set', {'builder': args[2], 'hw': c.node}) + def doNext(args): if len(args) != 3: return usage(args[0]) @@ -96,7 +114,7 @@ def doNext(args): c = getCommit(rev) next = getCommit(str(c.num + 1)) - if next is not None: + if next is not None and next.parent == c.node: print c.num + 1 else: print "" @@ -117,8 +135,32 @@ def doRecord(args): log = file(logfile, 'r').read() return command('build', {'node': c.node, 'parent': c.parent, 'date': c.date, 'user': c.user, 'desc': c.desc, 'log': log, 'builder': builder}) -if __name__ == '__main__': - sys.exit(main(sys.argv)) +def doBenchmarks(args): + if len(args) != 5: + return usage(args[0]) + builder = args[2] + rev = args[3] + c = getCommit(rev) + if c is None: + print >>sys.stderr, "Bad revision:", rev + return 1 + + benchmarks = {} + for line in file(args[4], 'r').readlines(): + if 'Benchmark' in line and 'ns/op' in line: + parts = line.split() + if parts[3] == 'ns/op': + benchmarks[parts[0]] = (parts[1], parts[2]) + + e = [] + for (name, (a, b)) in benchmarks.items(): + e.append(struct.pack('>H', len(name))) + e.append(name) + e.append(struct.pack('>H', len(a))) + e.append(a) + e.append(struct.pack('>H', len(b))) + e.append(b) + return command('benchmarks', {'node': c.node, 'builder': builder, 'benchmarkdata': binascii.b2a_base64(''.join(e))}) def encodeMultipartFormdata(fields, files): """fields is a sequence of (name, value) elements for regular form fields. @@ -191,3 +233,6 @@ def command(cmd, args, retries = 0): return command(cmd, args, retries = retries + 1) if reply.status != 200: raise Failed('Command "%s" returned %d' % (cmd, reply.status)) + +if __name__ == '__main__': + sys.exit(main(sys.argv)) diff --git a/misc/dashboard/builder.sh b/misc/dashboard/builder.sh index 4a87ed2d53..d66ba08c35 100644 --- a/misc/dashboard/builder.sh +++ b/misc/dashboard/builder.sh @@ -5,7 +5,7 @@ # license that can be found in the LICENSE file. fatal() { - echo $1 + echo $0: $1 1>&2 exit 1 } @@ -14,19 +14,19 @@ if [ ! -d go ] ; then fi if [ ! -f buildcontrol.py ] ; then - fatal "Please include buildcontrol.py in this directory" + fatal 'Please include buildcontrol.py in this directory' fi if [ "x$BUILDER" == "x" ] ; then - fatal "Please set \$BUILDER to the name of this builder" + fatal 'Please set $BUILDER to the name of this builder' fi if [ "x$BUILDHOST" == "x" ] ; then - fatal "Please set \$BUILDHOST to the hostname of the gobuild server" + fatal 'Please set $BUILDHOST to the hostname of the gobuild server' fi if [ "x$GOARCH" == "x" -o "x$GOOS" == "x" ] ; then - fatal "Please set $GOARCH and $GOOS" + fatal 'Please set $GOARCH and $GOOS' fi export PATH=$PATH:`pwd`/candidate/bin @@ -59,6 +59,13 @@ while true ; do else echo "Recording success for $rev" python ../../buildcontrol.py record $BUILDER $rev ok || fatal "Cannot record result" + echo "Running benchmarks" + cd pkg || fatal "failed to cd to pkg" + make bench > ../../benchmarks 2>&1 + if [ $? -eq 0 ] ; then + python ../../../buildcontrol.py benchmarks $BUILDER $rev ../../benchmarks || fatal "Cannot record benchmarks" + fi + cd .. || fatal "failed to cd out of pkg" fi cd ../.. || fatal "Cannot cd up" done diff --git a/misc/dashboard/godashboard/gobuild.py b/misc/dashboard/godashboard/gobuild.py index f984d920f9..de08490e8e 100644 --- a/misc/dashboard/godashboard/gobuild.py +++ b/misc/dashboard/godashboard/gobuild.py @@ -9,15 +9,17 @@ from google.appengine.ext import db from google.appengine.ext import webapp from google.appengine.ext.webapp import template from google.appengine.ext.webapp.util import run_wsgi_app +import binascii import datetime import hashlib import logging import os import re +import struct import key -# The main class of state are commit objects. One of these exists for each of +# The majority of our state are commit objects. One of these exists for each of # the commits known to the build system. Their key names are of the form # "-" . This means that a sorting by the key # name is sufficient to order the commits. @@ -39,6 +41,15 @@ class Commit(db.Model): # successful. builds = db.StringListProperty() +class Benchmark(db.Model): + name = db.StringProperty() + +class BenchmarkResult(db.Model): + num = db.IntegerProperty() + builder = db.StringProperty() + iterations = db.IntegerProperty() + nsperop = db.IntegerProperty() + # A Log contains the textual build log of a failed build. The key name is the # hex digest of the SHA256 hash of the contents. class Log(db.Model): @@ -96,6 +107,25 @@ class GetHighwater(webapp.RequestHandler): self.response.set_status(200) self.response.out.write(hw.commit) +class SetHighwater(webapp.RequestHandler): + def post(self): + if self.request.get('key') != key.accessKey: + self.response.set_status(403) + return + + builder = self.request.get('builder') + newhw = self.request.get('hw') + q = Commit.all() + q.filter('node =', newhw) + c = q.get() + if c is None: + self.response.set_status(404) + return + + hw = Highwater(key_name = 'hw-%s' % builder) + hw.commit = c.node + hw.put() + class LogHandler(webapp.RequestHandler): def get(self): self.response.headers['Content-Type'] = 'text/plain; charset=utf-8' @@ -163,6 +193,7 @@ class Build(webapp.RequestHandler): q.filter('node =', parent) p = q.get() if p is None: + logging.error('Cannot find parent %s of node %s' % (parent, node)) self.response.set_status(404) return parentnum, _ = p.key().name().split('-', 1) @@ -198,6 +229,110 @@ class Build(webapp.RequestHandler): self.response.set_status(200) +class Benchmarks(webapp.RequestHandler): + def get(self): + q = Benchmark.all() + bs = q.fetch(10000) + + self.response.set_status(200) + self.response.headers['Content-Type'] = 'application/json; charset=utf-8' + self.response.out.write('{"benchmarks": [\n') + + first = True + for b in bs: + if not first: + self.response.out.write(',"' + b.name + '"\n') + else: + self.response.out.write('"' + b.name + '"\n') + first = False + self.response.out.write(']}\n') + + def post(self): + if self.request.get('key') != key.accessKey: + self.response.set_status(403) + return + + builder = self.request.get('builder') + node = self.request.get('node') + if not validNode(node): + logging.error("Not valid node ('%s')", node) + self.response.set_status(500) + return + + benchmarkdata = self.request.get('benchmarkdata') + benchmarkdata = binascii.a2b_base64(benchmarkdata) + + def get_string(i): + l, = struct.unpack('>H', i[:2]) + s = i[2:2+l] + if len(s) != l: + return None, None + return s, i[2+l:] + + benchmarks = {} + while len(benchmarkdata) > 0: + name, benchmarkdata = get_string(benchmarkdata) + iterations_str, benchmarkdata = get_string(benchmarkdata) + time_str, benchmarkdata = get_string(benchmarkdata) + iterations = int(iterations_str) + time = int(time_str) + + benchmarks[name] = (iterations, time) + + q = Commit.all() + q.filter('node =', node) + n = q.get() + if n is None: + logging.error('Client asked for unknown commit while uploading benchmarks') + self.response.set_status(404) + return + + for (benchmark, (iterations, time)) in benchmarks.items(): + b = Benchmark.get_or_insert(benchmark.encode('base64'), name = benchmark) + r = BenchmarkResult(key_name = '%08x/builder' % n.num, parent = b, num = n.num, iterations = iterations, nsperop = time, builder = builder) + r.put() + + self.response.set_status(200) + +class GetBenchmarks(webapp.RequestHandler): + def get(self): + self.response.headers['Content-Type'] = 'application/json; charset=utf-8' + benchmark = self.request.path[12:].decode('hex').encode('base64') + + b = Benchmark.get_by_key_name(benchmark) + if b is None: + self.response.set_status(404) + return + + q = BenchmarkResult.all() + q.ancestor(b) + q.order('-__key__') + results = q.fetch(10000) + + if len(results) == 0: + self.response.set_status(404) + return + + max = -1 + min = 2000000000 + builders = set() + for r in results: + if max < r.num: + max = r.num + if min > r.num: + min = r.num + builders.add(r.builder) + + res = {} + for b in builders: + res[b] = [[-1] * ((max - min) + 1), [-1] * ((max - min) + 1)] + + for r in results: + res[r.builder][0][r.num - min] = r.iterations + res[r.builder][1][r.num - min] = r.nsperop + + self.response.out.write(str(res)) + class FixedOffset(datetime.tzinfo): """Fixed offset in minutes east from UTC.""" @@ -273,9 +408,12 @@ application = webapp.WSGIApplication( [('/', MainPage), ('/log/.*', LogHandler), ('/hw-get', GetHighwater), + ('/hw-set', SetHighwater), ('/init', Init), ('/build', Build), + ('/benchmarks', Benchmarks), + ('/benchmarks/.*', GetBenchmarks), ]) def main(): diff --git a/misc/dashboard/godashboard/index.yaml b/misc/dashboard/godashboard/index.yaml index 784d23d012..19d2f3778e 100644 --- a/misc/dashboard/godashboard/index.yaml +++ b/misc/dashboard/godashboard/index.yaml @@ -10,6 +10,12 @@ indexes: # automatically uploaded to the admin console when you next deploy # your application using appcfg.py. +- kind: BenchmarkResult + ancestor: yes + properties: + - name: __key__ + direction: desc + - kind: Commit properties: - name: __key__ diff --git a/misc/dashboard/godashboard/key.py b/misc/dashboard/godashboard/key.py index 7495709ecd..3abe410dd7 100644 --- a/misc/dashboard/godashboard/key.py +++ b/misc/dashboard/godashboard/key.py @@ -6,4 +6,3 @@ # builds). It's tranmitted in the clear but, given the low value of the target, # this should be sufficient. accessKey = "this is not the real key" -