1
0
mirror of https://github.com/golang/go synced 2024-11-21 20:04:44 -07:00

dashboard: add benchmarking support.

This has actually been running for a while and gathering benchmark
data. I haven't had a chance to add a UI for it yet however.

R=rsc
CC=golang-dev
https://golang.org/cl/194082
This commit is contained in:
Adam Langley 2010-01-26 12:56:29 -08:00 committed by Russ Cox
parent d4ca006334
commit 062fee0536
6 changed files with 212 additions and 12 deletions

View File

@ -1,6 +1,10 @@
// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
The files in this directory constitute the continuous builder: The files in this directory constitute the continuous builder:
godashboard/: An AppEngine which acts as a server godashboard/: An AppEngine that acts as a server
builder.sh, buildcontrol.sh: used by the build slaves builder.sh, buildcontrol.sh: used by the build slaves
If you wish to run a Go builder, please email golang-dev@googlegroups.com If you wish to run a Go builder, please email golang-dev@googlegroups.com
@ -19,7 +23,8 @@ export PATH=$PATH:/gobuild/bin
export BUILDER=XXX export BUILDER=XXX
export BUILDHOST=godashboard.appspot.com export BUILDHOST=godashboard.appspot.com
* Write ~gobuild/.gobuildkey (you need to get it from someone who knows it) * Write the key ~gobuild/.gobuildkey (you need to get it from someone who knows
the key)
* sudo apt-get install bison gcc libc6-dev ed make * sudo apt-get install bison gcc libc6-dev ed make
* cd ~gobuild * cd ~gobuild

View File

@ -6,8 +6,10 @@
# This is a utility script for implementing a Go build slave. # This is a utility script for implementing a Go build slave.
import binascii
import httplib import httplib
import os import os
import struct
import subprocess import subprocess
import sys import sys
import time import time
@ -42,10 +44,14 @@ def main(args):
return doInit(args) return doInit(args)
elif args[1] == 'hwget': elif args[1] == 'hwget':
return doHWGet(args) return doHWGet(args)
elif args[1] == 'hwset':
return doHWSet(args)
elif args[1] == 'next': elif args[1] == 'next':
return doNext(args) return doNext(args)
elif args[1] == 'record': elif args[1] == 'record':
return doRecord(args) return doRecord(args)
elif args[1] == 'benchmarks':
return doBenchmarks(args)
else: else:
return usage(args[0]) return usage(args[0])
@ -55,8 +61,10 @@ def usage(name):
Commands: Commands:
init <rev>: init the build bot with the given commit as the first in history init <rev>: init the build bot with the given commit as the first in history
hwget <builder>: get the most recent revision built by the given builder hwget <builder>: get the most recent revision built by the given builder
hwset <builder> <rev>: get the most recent revision built by the given builder
next <builder>: get the next revision number to by built by the given builder next <builder>: get the next revision number to by built by the given builder
record <builder> <rev> <ok|log file>: record a build result record <builder> <rev> <ok|log file>: record a build result
benchmarks <builder> <rev> <log file>: record benchmark numbers
''' % name) ''' % name)
return 1 return 1
@ -78,11 +86,21 @@ def doHWGet(args, retries = 0):
if reply.status == 200: if reply.status == 200:
print reply.read() print reply.read()
elif reply.status == 500 and retries < 3: elif reply.status == 500 and retries < 3:
time.sleep(3)
return doHWGet(args, retries = retries + 1) return doHWGet(args, retries = retries + 1)
else: else:
raise Failed('get-hw returned %d' % reply.status) raise Failed('get-hw returned %d' % reply.status)
return 0 return 0
def doHWSet(args):
if len(args) != 4:
return usage(args[0])
c = getCommit(args[3])
if c is None:
fatal('Cannot get commit %s' % args[3])
return command('hw-set', {'builder': args[2], 'hw': c.node})
def doNext(args): def doNext(args):
if len(args) != 3: if len(args) != 3:
return usage(args[0]) return usage(args[0])
@ -96,7 +114,7 @@ def doNext(args):
c = getCommit(rev) c = getCommit(rev)
next = getCommit(str(c.num + 1)) next = getCommit(str(c.num + 1))
if next is not None: if next is not None and next.parent == c.node:
print c.num + 1 print c.num + 1
else: else:
print "<none>" print "<none>"
@ -117,8 +135,32 @@ def doRecord(args):
log = file(logfile, 'r').read() log = file(logfile, 'r').read()
return command('build', {'node': c.node, 'parent': c.parent, 'date': c.date, 'user': c.user, 'desc': c.desc, 'log': log, 'builder': builder}) return command('build', {'node': c.node, 'parent': c.parent, 'date': c.date, 'user': c.user, 'desc': c.desc, 'log': log, 'builder': builder})
if __name__ == '__main__': def doBenchmarks(args):
sys.exit(main(sys.argv)) if len(args) != 5:
return usage(args[0])
builder = args[2]
rev = args[3]
c = getCommit(rev)
if c is None:
print >>sys.stderr, "Bad revision:", rev
return 1
benchmarks = {}
for line in file(args[4], 'r').readlines():
if 'Benchmark' in line and 'ns/op' in line:
parts = line.split()
if parts[3] == 'ns/op':
benchmarks[parts[0]] = (parts[1], parts[2])
e = []
for (name, (a, b)) in benchmarks.items():
e.append(struct.pack('>H', len(name)))
e.append(name)
e.append(struct.pack('>H', len(a)))
e.append(a)
e.append(struct.pack('>H', len(b)))
e.append(b)
return command('benchmarks', {'node': c.node, 'builder': builder, 'benchmarkdata': binascii.b2a_base64(''.join(e))})
def encodeMultipartFormdata(fields, files): def encodeMultipartFormdata(fields, files):
"""fields is a sequence of (name, value) elements for regular form fields. """fields is a sequence of (name, value) elements for regular form fields.
@ -191,3 +233,6 @@ def command(cmd, args, retries = 0):
return command(cmd, args, retries = retries + 1) return command(cmd, args, retries = retries + 1)
if reply.status != 200: if reply.status != 200:
raise Failed('Command "%s" returned %d' % (cmd, reply.status)) raise Failed('Command "%s" returned %d' % (cmd, reply.status))
if __name__ == '__main__':
sys.exit(main(sys.argv))

View File

@ -5,7 +5,7 @@
# license that can be found in the LICENSE file. # license that can be found in the LICENSE file.
fatal() { fatal() {
echo $1 echo $0: $1 1>&2
exit 1 exit 1
} }
@ -14,19 +14,19 @@ if [ ! -d go ] ; then
fi fi
if [ ! -f buildcontrol.py ] ; then if [ ! -f buildcontrol.py ] ; then
fatal "Please include buildcontrol.py in this directory" fatal 'Please include buildcontrol.py in this directory'
fi fi
if [ "x$BUILDER" == "x" ] ; then if [ "x$BUILDER" == "x" ] ; then
fatal "Please set \$BUILDER to the name of this builder" fatal 'Please set $BUILDER to the name of this builder'
fi fi
if [ "x$BUILDHOST" == "x" ] ; then if [ "x$BUILDHOST" == "x" ] ; then
fatal "Please set \$BUILDHOST to the hostname of the gobuild server" fatal 'Please set $BUILDHOST to the hostname of the gobuild server'
fi fi
if [ "x$GOARCH" == "x" -o "x$GOOS" == "x" ] ; then if [ "x$GOARCH" == "x" -o "x$GOOS" == "x" ] ; then
fatal "Please set $GOARCH and $GOOS" fatal 'Please set $GOARCH and $GOOS'
fi fi
export PATH=$PATH:`pwd`/candidate/bin export PATH=$PATH:`pwd`/candidate/bin
@ -59,6 +59,13 @@ while true ; do
else else
echo "Recording success for $rev" echo "Recording success for $rev"
python ../../buildcontrol.py record $BUILDER $rev ok || fatal "Cannot record result" python ../../buildcontrol.py record $BUILDER $rev ok || fatal "Cannot record result"
echo "Running benchmarks"
cd pkg || fatal "failed to cd to pkg"
make bench > ../../benchmarks 2>&1
if [ $? -eq 0 ] ; then
python ../../../buildcontrol.py benchmarks $BUILDER $rev ../../benchmarks || fatal "Cannot record benchmarks"
fi
cd .. || fatal "failed to cd out of pkg"
fi fi
cd ../.. || fatal "Cannot cd up" cd ../.. || fatal "Cannot cd up"
done done

View File

@ -9,15 +9,17 @@ from google.appengine.ext import db
from google.appengine.ext import webapp from google.appengine.ext import webapp
from google.appengine.ext.webapp import template from google.appengine.ext.webapp import template
from google.appengine.ext.webapp.util import run_wsgi_app from google.appengine.ext.webapp.util import run_wsgi_app
import binascii
import datetime import datetime
import hashlib import hashlib
import logging import logging
import os import os
import re import re
import struct
import key import key
# The main class of state are commit objects. One of these exists for each of # The majority of our state are commit objects. One of these exists for each of
# the commits known to the build system. Their key names are of the form # the commits known to the build system. Their key names are of the form
# <commit number (%08x)> "-" <hg hash>. This means that a sorting by the key # <commit number (%08x)> "-" <hg hash>. This means that a sorting by the key
# name is sufficient to order the commits. # name is sufficient to order the commits.
@ -39,6 +41,15 @@ class Commit(db.Model):
# successful. # successful.
builds = db.StringListProperty() builds = db.StringListProperty()
class Benchmark(db.Model):
name = db.StringProperty()
class BenchmarkResult(db.Model):
num = db.IntegerProperty()
builder = db.StringProperty()
iterations = db.IntegerProperty()
nsperop = db.IntegerProperty()
# A Log contains the textual build log of a failed build. The key name is the # A Log contains the textual build log of a failed build. The key name is the
# hex digest of the SHA256 hash of the contents. # hex digest of the SHA256 hash of the contents.
class Log(db.Model): class Log(db.Model):
@ -96,6 +107,25 @@ class GetHighwater(webapp.RequestHandler):
self.response.set_status(200) self.response.set_status(200)
self.response.out.write(hw.commit) self.response.out.write(hw.commit)
class SetHighwater(webapp.RequestHandler):
def post(self):
if self.request.get('key') != key.accessKey:
self.response.set_status(403)
return
builder = self.request.get('builder')
newhw = self.request.get('hw')
q = Commit.all()
q.filter('node =', newhw)
c = q.get()
if c is None:
self.response.set_status(404)
return
hw = Highwater(key_name = 'hw-%s' % builder)
hw.commit = c.node
hw.put()
class LogHandler(webapp.RequestHandler): class LogHandler(webapp.RequestHandler):
def get(self): def get(self):
self.response.headers['Content-Type'] = 'text/plain; charset=utf-8' self.response.headers['Content-Type'] = 'text/plain; charset=utf-8'
@ -163,6 +193,7 @@ class Build(webapp.RequestHandler):
q.filter('node =', parent) q.filter('node =', parent)
p = q.get() p = q.get()
if p is None: if p is None:
logging.error('Cannot find parent %s of node %s' % (parent, node))
self.response.set_status(404) self.response.set_status(404)
return return
parentnum, _ = p.key().name().split('-', 1) parentnum, _ = p.key().name().split('-', 1)
@ -198,6 +229,110 @@ class Build(webapp.RequestHandler):
self.response.set_status(200) self.response.set_status(200)
class Benchmarks(webapp.RequestHandler):
def get(self):
q = Benchmark.all()
bs = q.fetch(10000)
self.response.set_status(200)
self.response.headers['Content-Type'] = 'application/json; charset=utf-8'
self.response.out.write('{"benchmarks": [\n')
first = True
for b in bs:
if not first:
self.response.out.write(',"' + b.name + '"\n')
else:
self.response.out.write('"' + b.name + '"\n')
first = False
self.response.out.write(']}\n')
def post(self):
if self.request.get('key') != key.accessKey:
self.response.set_status(403)
return
builder = self.request.get('builder')
node = self.request.get('node')
if not validNode(node):
logging.error("Not valid node ('%s')", node)
self.response.set_status(500)
return
benchmarkdata = self.request.get('benchmarkdata')
benchmarkdata = binascii.a2b_base64(benchmarkdata)
def get_string(i):
l, = struct.unpack('>H', i[:2])
s = i[2:2+l]
if len(s) != l:
return None, None
return s, i[2+l:]
benchmarks = {}
while len(benchmarkdata) > 0:
name, benchmarkdata = get_string(benchmarkdata)
iterations_str, benchmarkdata = get_string(benchmarkdata)
time_str, benchmarkdata = get_string(benchmarkdata)
iterations = int(iterations_str)
time = int(time_str)
benchmarks[name] = (iterations, time)
q = Commit.all()
q.filter('node =', node)
n = q.get()
if n is None:
logging.error('Client asked for unknown commit while uploading benchmarks')
self.response.set_status(404)
return
for (benchmark, (iterations, time)) in benchmarks.items():
b = Benchmark.get_or_insert(benchmark.encode('base64'), name = benchmark)
r = BenchmarkResult(key_name = '%08x/builder' % n.num, parent = b, num = n.num, iterations = iterations, nsperop = time, builder = builder)
r.put()
self.response.set_status(200)
class GetBenchmarks(webapp.RequestHandler):
def get(self):
self.response.headers['Content-Type'] = 'application/json; charset=utf-8'
benchmark = self.request.path[12:].decode('hex').encode('base64')
b = Benchmark.get_by_key_name(benchmark)
if b is None:
self.response.set_status(404)
return
q = BenchmarkResult.all()
q.ancestor(b)
q.order('-__key__')
results = q.fetch(10000)
if len(results) == 0:
self.response.set_status(404)
return
max = -1
min = 2000000000
builders = set()
for r in results:
if max < r.num:
max = r.num
if min > r.num:
min = r.num
builders.add(r.builder)
res = {}
for b in builders:
res[b] = [[-1] * ((max - min) + 1), [-1] * ((max - min) + 1)]
for r in results:
res[r.builder][0][r.num - min] = r.iterations
res[r.builder][1][r.num - min] = r.nsperop
self.response.out.write(str(res))
class FixedOffset(datetime.tzinfo): class FixedOffset(datetime.tzinfo):
"""Fixed offset in minutes east from UTC.""" """Fixed offset in minutes east from UTC."""
@ -273,9 +408,12 @@ application = webapp.WSGIApplication(
[('/', MainPage), [('/', MainPage),
('/log/.*', LogHandler), ('/log/.*', LogHandler),
('/hw-get', GetHighwater), ('/hw-get', GetHighwater),
('/hw-set', SetHighwater),
('/init', Init), ('/init', Init),
('/build', Build), ('/build', Build),
('/benchmarks', Benchmarks),
('/benchmarks/.*', GetBenchmarks),
]) ])
def main(): def main():

View File

@ -10,6 +10,12 @@ indexes:
# automatically uploaded to the admin console when you next deploy # automatically uploaded to the admin console when you next deploy
# your application using appcfg.py. # your application using appcfg.py.
- kind: BenchmarkResult
ancestor: yes
properties:
- name: __key__
direction: desc
- kind: Commit - kind: Commit
properties: properties:
- name: __key__ - name: __key__

View File

@ -6,4 +6,3 @@
# builds). It's tranmitted in the clear but, given the low value of the target, # builds). It's tranmitted in the clear but, given the low value of the target,
# this should be sufficient. # this should be sufficient.
accessKey = "this is not the real key" accessKey = "this is not the real key"