From d635d846f47396efd0cb0c953c7591a0c83fca0f Mon Sep 17 00:00:00 2001 From: Adam Langley Date: Thu, 7 Jan 2010 18:45:45 -0800 Subject: [PATCH] Add builder scripts. These are the scripts behind godashboard.appspot.com. Nothing is particularly beautiful about it, but it does run. I still need to add support for per-builder keys and for running the benchmarks. R=rsc CC=golang-dev https://golang.org/cl/183153 --- misc/dashboard/README | 33 ++ misc/dashboard/buildcontrol.py | 193 ++++++++++++ misc/dashboard/builder.sh | 64 ++++ .../dashboard/godashboard/_multiprocessing.py | 5 + misc/dashboard/godashboard/app.yaml | 8 + misc/dashboard/godashboard/gobuild.py | 285 ++++++++++++++++++ misc/dashboard/godashboard/index.yaml | 16 + misc/dashboard/godashboard/key.py | 9 + misc/dashboard/godashboard/main.html | 84 ++++++ 9 files changed, 697 insertions(+) create mode 100644 misc/dashboard/README create mode 100644 misc/dashboard/buildcontrol.py create mode 100644 misc/dashboard/builder.sh create mode 100644 misc/dashboard/godashboard/_multiprocessing.py create mode 100644 misc/dashboard/godashboard/app.yaml create mode 100644 misc/dashboard/godashboard/gobuild.py create mode 100644 misc/dashboard/godashboard/index.yaml create mode 100644 misc/dashboard/godashboard/key.py create mode 100644 misc/dashboard/godashboard/main.html diff --git a/misc/dashboard/README b/misc/dashboard/README new file mode 100644 index 0000000000..7b07a21a20 --- /dev/null +++ b/misc/dashboard/README @@ -0,0 +1,33 @@ +The files in this directory constitute the continuous builder: + +godashboard/: An AppEngine which acts as a server +builder.sh, buildcontrol.sh: used by the build slaves + +If you wish to run a Go builder, please email golang-dev@googlegroups.com + + +Setting up a Go builder: + +* (Optional) create a new user 'gobuild' +* Edit ~gobuild/.bash_profile and add the following: + +export GOROOT=/gobuild/go +export GOARCH=XXX +export GOOS=XXX +export GOBIN=/gobuild/bin +export PATH=$PATH:/gobuild/bin +export BUILDER=XXX +export BUILDHOST=godashboard.appspot.com + +* Write ~gobuild/.gobuildkey (you need to get it from someone who knows it) + +* sudo apt-get install bison gcc libc6-dev ed make +* cd ~gobuild +* mkdir bin +* hg clone https://go.googlecode.com/hg/ $GOROOT +* copy builder.sh and buildcontrol.py to ~gobuild +* chmod a+x ./builder.sh ./buildcontrol.py +* cd go +* ../buildcontrol.py next $BUILDER (just to check that things are ok) +* cd .. +* ./builder.sh (You probably want to run this in a screen long term.) diff --git a/misc/dashboard/buildcontrol.py b/misc/dashboard/buildcontrol.py new file mode 100644 index 0000000000..caa1a2f477 --- /dev/null +++ b/misc/dashboard/buildcontrol.py @@ -0,0 +1,193 @@ +#!/usr/bin/env python + +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +# This is a utility script for implementing a Go build slave. + +import httplib +import os +import subprocess +import sys +import time + +buildhost = '' +buildport = -1 +buildkey = '' + +def main(args): + global buildport, buildhost, buildkey + + if len(args) < 2: + return usage(args[0]) + + if 'BUILDHOST' not in os.environ: + print >>sys.stderr, "Please set $BUILDHOST" + return + buildhost = os.environ['BUILDHOST'] + + if 'BUILDPORT' not in os.environ: + buildport = 80 + else: + buildport = int(os.environ['BUILDPORT']) + + try: + buildkey = file('%s/.gobuildkey' % os.environ['HOME'], 'r').read().strip() + except IOError: + print >>sys.stderr, "Need key in ~/.gobuildkey" + return + + if args[1] == 'init': + return doInit(args) + elif args[1] == 'hwget': + return doHWGet(args) + elif args[1] == 'next': + return doNext(args) + elif args[1] == 'record': + return doRecord(args) + else: + return usage(args[0]) + +def usage(name): + sys.stderr.write('''Usage: %s + +Commands: + init : init the build bot with the given commit as the first in history + hwget : get the most recent revision built by the given builder + next : get the next revision number to by built by the given builder + record : record a build result +''' % name) + return 1 + +def doInit(args): + if len(args) != 3: + return usage(args[0]) + c = getCommit(args[2]) + if c is None: + fatal('Cannot get commit %s' % args[2]) + + return command('init', {'node': c.node, 'date': c.date, 'user': c.user, 'desc': c.desc}) + +def doHWGet(args, retries = 0): + if len(args) != 3: + return usage(args[0]) + conn = httplib.HTTPConnection(buildhost, buildport, True) + conn.request('GET', '/hw-get?builder=%s' % args[2]); + reply = conn.getresponse() + if reply.status == 200: + print reply.read() + elif reply.status == 500 and retries < 3: + return doHWGet(args, retries = retries + 1) + else: + raise Failed('get-hw returned %d' % reply.status) + return 0 + +def doNext(args): + if len(args) != 3: + return usage(args[0]) + conn = httplib.HTTPConnection(buildhost, buildport, True) + conn.request('GET', '/hw-get?builder=%s' % args[2]); + reply = conn.getresponse() + if reply.status == 200: + rev = reply.read() + else: + raise Failed('get-hw returned %d' % reply.status) + + c = getCommit(rev) + next = getCommit(str(c.num + 1)) + if next is not None: + print c.num + 1 + else: + print "" + return 0 + +def doRecord(args): + if len(args) != 5: + return usage(args[0]) + builder = args[2] + rev = args[3] + c = getCommit(rev) + if c is None: + print >>sys.stderr, "Bad revision:", rev + return 1 + logfile = args[4] + log = '' + if logfile != 'ok': + log = file(logfile, 'r').read() + return command('build', {'node': c.node, 'parent': c.parent, 'date': c.date, 'user': c.user, 'desc': c.desc, 'log': log, 'builder': builder}) + +if __name__ == '__main__': + sys.exit(main(sys.argv)) + +def encodeMultipartFormdata(fields, files): + """fields is a sequence of (name, value) elements for regular form fields. + files is a sequence of (name, filename, value) elements for data to be uploaded as files""" + BOUNDARY = '----------ThIs_Is_tHe_bouNdaRY_$' + CRLF = '\r\n' + L = [] + for (key, value) in fields.items(): + L.append('--' + BOUNDARY) + L.append('Content-Disposition: form-data; name="%s"' % key) + L.append('') + L.append(value) + for (key, filename, value) in files: + L.append('--' + BOUNDARY) + L.append('Content-Disposition: form-data; name="%s"; filename="%s"' % (key, filename)) + L.append('Content-Type: %s' % get_content_type(filename)) + L.append('') + L.append(value) + L.append('--' + BOUNDARY + '--') + L.append('') + body = CRLF.join(L) + content_type = 'multipart/form-data; boundary=%s' % BOUNDARY + return content_type, body + +def unescapeXML(s): + return s.replace('<', '<').replace('>', '>').replace('&', '&') + +class Commit: + pass + +def getCommit(rev): + output, stderr = subprocess.Popen(['hg', 'log', '-r', rev, '-l', '1', '--template', '{rev}>{node|escape}>{author|escape}>{date}>{desc}'], stdout = subprocess.PIPE, stderr = subprocess.PIPE, close_fds = True).communicate() + if len(stderr) > 0: + return None + [n, node, user, date, desc] = output.split('>', 4) + + c = Commit() + c.num = int(n) + c.node = unescapeXML(node) + c.user = unescapeXML(user) + c.date = unescapeXML(date) + c.desc = desc + c.parent = '' + + if c.num > 0: + output, _ = subprocess.Popen(['hg', 'log', '-r', str(c.num - 1), '-l', '1', '--template', '{node}'], stdout = subprocess.PIPE, close_fds = True).communicate() + c.parent = output + + return c + +class Failed(Exception): + def __init__(self, msg): + self.msg = msg + def __str__(self): + return self.msg + +def command(cmd, args, retries = 0): + args['key'] = buildkey + contentType, body = encodeMultipartFormdata(args, []) + print body + conn = httplib.HTTPConnection(buildhost, buildport, True) + conn.request('POST', '/' + cmd, body, {'Content-Type': contentType}) + reply = conn.getresponse() + if reply.status != 200: + print "Command failed. Output:" + print reply.read() + if reply.status == 500 and retries < 3: + print "Was a 500. Waiting two seconds and trying again." + time.sleep(2) + return command(cmd, args, retries = retries + 1) + if reply.status != 200: + raise Failed('Command "%s" returned %d' % (cmd, reply.status)) diff --git a/misc/dashboard/builder.sh b/misc/dashboard/builder.sh new file mode 100644 index 0000000000..4a87ed2d53 --- /dev/null +++ b/misc/dashboard/builder.sh @@ -0,0 +1,64 @@ +#!/bin/sh + +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +fatal() { + echo $1 + exit 1 +} + +if [ ! -d go ] ; then + fatal "Please run in directory that contains a checked out repo in 'go'" +fi + +if [ ! -f buildcontrol.py ] ; then + fatal "Please include buildcontrol.py in this directory" +fi + +if [ "x$BUILDER" == "x" ] ; then + fatal "Please set \$BUILDER to the name of this builder" +fi + +if [ "x$BUILDHOST" == "x" ] ; then + fatal "Please set \$BUILDHOST to the hostname of the gobuild server" +fi + +if [ "x$GOARCH" == "x" -o "x$GOOS" == "x" ] ; then + fatal "Please set $GOARCH and $GOOS" +fi + +export PATH=$PATH:`pwd`/candidate/bin +export GOBIN=`pwd`/candidate/bin + +while true ; do + cd go || fatal "Cannot cd into 'go'" + hg pull -u || fatal "hg sync failed" + rev=`python ../buildcontrol.py next $BUILDER` + if [ $? -ne 0 ] ; then + fatal "Cannot get next revision" + fi + cd .. || fatal "Cannot cd up" + if [ "x$rev" == "x" ] ; then + sleep 10 + continue + fi + + echo "Cloning for revision $rev" + rm -Rf candidate + hg clone -r $rev go candidate || fatal "hg clone failed" + export GOROOT=`pwd`/candidate + mkdir -p candidate/bin || fatal "Cannot create candidate/bin" + cd candidate/src || fatal "Cannot cd into candidate/src" + echo "Building revision $rev" + ./all.bash > ../log 2>&1 + if [ $? -ne 0 ] ; then + echo "Recording failure for $rev" + python ../../buildcontrol.py record $BUILDER $rev ../log || fatal "Cannot record result" + else + echo "Recording success for $rev" + python ../../buildcontrol.py record $BUILDER $rev ok || fatal "Cannot record result" + fi + cd ../.. || fatal "Cannot cd up" +done diff --git a/misc/dashboard/godashboard/_multiprocessing.py b/misc/dashboard/godashboard/_multiprocessing.py new file mode 100644 index 0000000000..8c66c06596 --- /dev/null +++ b/misc/dashboard/godashboard/_multiprocessing.py @@ -0,0 +1,5 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +import multiprocessing diff --git a/misc/dashboard/godashboard/app.yaml b/misc/dashboard/godashboard/app.yaml new file mode 100644 index 0000000000..06681def10 --- /dev/null +++ b/misc/dashboard/godashboard/app.yaml @@ -0,0 +1,8 @@ +application: godashboard +version: 1 +runtime: python +api_version: 1 + +handlers: +- url: /.* + script: gobuild.py diff --git a/misc/dashboard/godashboard/gobuild.py b/misc/dashboard/godashboard/gobuild.py new file mode 100644 index 0000000000..f984d920f9 --- /dev/null +++ b/misc/dashboard/godashboard/gobuild.py @@ -0,0 +1,285 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +# This is the server part of the continuous build system for Go. It must be run +# by AppEngine. + +from google.appengine.ext import db +from google.appengine.ext import webapp +from google.appengine.ext.webapp import template +from google.appengine.ext.webapp.util import run_wsgi_app +import datetime +import hashlib +import logging +import os +import re + +import key + +# The main class of state are commit objects. One of these exists for each of +# the commits known to the build system. Their key names are of the form +# "-" . This means that a sorting by the key +# name is sufficient to order the commits. +# +# The commit numbers are purely local. They need not match up to the commit +# numbers in an hg repo. When inserting a new commit, the parent commit must be +# given and this is used to generate the new commit number. In order to create +# the first Commit object, a special command (/init) is used. +class Commit(db.Model): + num = db.IntegerProperty() # internal, monotonic counter. + node = db.StringProperty() # Hg hash + parentnode = db.StringProperty() # Hg hash + user = db.StringProperty() + date = db.DateTimeProperty() + desc = db.BlobProperty() + + # This is the list of builds. Each element is a string of the form "`" . If the log hash is empty, then the build was + # successful. + builds = db.StringListProperty() + +# A Log contains the textual build log of a failed build. The key name is the +# hex digest of the SHA256 hash of the contents. +class Log(db.Model): + log = db.BlobProperty() + +# For each builder, we store the last revision that it built. So, if it +# crashes, it knows where to start up from. The key names for these objects are +# "hw-" +class Highwater(db.Model): + commit = db.StringProperty() + +class MainPage(webapp.RequestHandler): + def get(self): + self.response.headers['Content-Type'] = 'text/html; charset=utf-8' + + q = Commit.all() + q.order('-__key__') + results = q.fetch(30) + + revs = [toRev(r) for r in results] + allbuilders = set() + + for r in revs: + for b in r['builds']: + allbuilders.add(b['builder']) + for r in revs: + have = set(x['builder'] for x in r['builds']) + need = allbuilders.difference(have) + for n in need: + r['builds'].append({'builder': n, 'log':'', 'ok': False}) + r['builds'].sort(cmp = byBuilder) + + builders = list(allbuilders) + builders.sort() + values = {"revs": revs, "builders": builders} + + path = os.path.join(os.path.dirname(__file__), 'main.html') + self.response.out.write(template.render(path, values)) + +class GetHighwater(webapp.RequestHandler): + def get(self): + builder = self.request.get('builder') + + hw = Highwater.get_by_key_name('hw-%s' % builder) + if hw is None: + # If no highwater has been recorded for this builder, we find the + # initial commit and return that. + q = Commit.all() + q.filter('num =', 0) + commitzero = q.get() + self.response.set_status(200) + self.response.out.write(commitzero.node) + return + + self.response.set_status(200) + self.response.out.write(hw.commit) + +class LogHandler(webapp.RequestHandler): + def get(self): + self.response.headers['Content-Type'] = 'text/plain; charset=utf-8' + hash = self.request.path[5:] + l = Log.get_by_key_name(hash) + if l is None: + self.response.set_status(404) + return + self.response.set_status(200) + self.response.out.write(l.log) + +# Init creates the commit with id 0. Since this commit doesn't have a parent, +# it cannot be created by Build. +class Init(webapp.RequestHandler): + def post(self): + if self.request.get('key') != key.accessKey: + self.response.set_status(403) + return + + date = parseDate(self.request.get('date')) + node = self.request.get('node') + if not validNode(node) or date is None: + logging.error("Not valid node ('%s') or bad date (%s %s)", node, date, self.request.get('date')) + self.response.set_status(500) + return + + commit = Commit(key_name = '00000000-%s' % node) + commit.num = 0 + commit.node = node + commit.parentnode = '' + commit.user = self.request.get('user') + commit.date = date + commit.desc = self.request.get('desc').encode('utf8') + + commit.put() + + self.response.set_status(200) + +# Build is the main command: it records the result of a new build. +class Build(webapp.RequestHandler): + def post(self): + if self.request.get('key') != key.accessKey: + self.response.set_status(403) + return + + builder = self.request.get('builder') + log = self.request.get('log').encode('utf-8') + + loghash = '' + if len(log) > 0: + loghash = hashlib.sha256(log).hexdigest() + l = Log(key_name = loghash) + l.log = log + l.put() + + date = parseDate(self.request.get('date')) + node = self.request.get('node') + parent = self.request.get('parent') + if not validNode(node) or not validNode(parent) or date is None: + logging.error("Not valid node ('%s') or bad date (%s %s)", node, date, self.request.get('date')) + self.response.set_status(500) + return + + q = Commit.all() + q.filter('node =', parent) + p = q.get() + if p is None: + self.response.set_status(404) + return + parentnum, _ = p.key().name().split('-', 1) + nodenum = int(parentnum, 16) + 1 + + def add_build(): + key_name = '%08x-%s' % (nodenum, node) + n = Commit.get_by_key_name(key_name) + if n is None: + n = Commit(key_name = key_name) + n.num = nodenum + n.node = node + n.parentnode = parent + n.user = self.request.get('user') + n.date = date + n.desc = self.request.get('desc').encode('utf8') + s = '%s`%s' % (builder, loghash) + for i, b in enumerate(n.builds): + if b.split('`', 1)[0] == builder: + n.builds[i] = s + break + else: + n.builds.append(s) + n.put() + + db.run_in_transaction(add_build) + + hw = Highwater.get_by_key_name('hw-%s' % builder) + if hw is None: + hw = Highwater(key_name = 'hw-%s' % builder) + hw.commit = node + hw.put() + + self.response.set_status(200) + +class FixedOffset(datetime.tzinfo): + """Fixed offset in minutes east from UTC.""" + + def __init__(self, offset): + self.__offset = datetime.timedelta(seconds = offset) + + def utcoffset(self, dt): + return self.__offset + + def tzname(self, dt): + return None + + def dst(self, dt): + return datetime.timedelta(0) + +def validNode(node): + if len(node) != 40: + return False + for x in node: + o = ord(x) + if (o < ord('0') or o > ord('9')) and (o < ord('a') or o > ord('f')): + return False + return True + +def parseDate(date): + if '-' in date: + (a, offset) = date.split('-', 1) + try: + return datetime.datetime.fromtimestamp(float(a), FixedOffset(0-int(offset))) + except ValueError: + return None + if '+' in date: + (a, offset) = date.split('+', 1) + try: + return datetime.datetime.fromtimestamp(float(a), FixedOffset(int(offset))) + except ValueError: + return None + try: + return datetime.datetime.utcfromtimestamp(float(date)) + except ValueError: + return None + +email_re = re.compile('^[^<]+<([^>]*)>$') + +def toUsername(user): + r = email_re.match(user) + if r is None: + return user + email = r.groups()[0] + return email.replace('@golang.org', '') + +def dateToShortStr(d): + return d.strftime('%a %b %d %H:%M') + +def parseBuild(build): + [builder, logblob] = build.split('`') + return {'builder': builder, 'log': logblob, 'ok': len(logblob) == 0} + +def toRev(c): + b = { "node": c.node, + "user": toUsername(c.user), + "date": dateToShortStr(c.date), + "desc": c.desc} + b['builds'] = [parseBuild(build) for build in c.builds] + return b + +def byBuilder(x, y): + return cmp(x['builder'], y['builder']) + +# This is the URL map for the server. The first three entries are public, the +# rest are only used by the builders. +application = webapp.WSGIApplication( + [('/', MainPage), + ('/log/.*', LogHandler), + ('/hw-get', GetHighwater), + + ('/init', Init), + ('/build', Build), + ]) + +def main(): + run_wsgi_app(application) + +if __name__ == "__main__": + main() diff --git a/misc/dashboard/godashboard/index.yaml b/misc/dashboard/godashboard/index.yaml new file mode 100644 index 0000000000..784d23d012 --- /dev/null +++ b/misc/dashboard/godashboard/index.yaml @@ -0,0 +1,16 @@ +indexes: + +# AUTOGENERATED + +# This index.yaml is automatically updated whenever the dev_appserver +# detects that a new type of query is run. If you want to manage the +# index.yaml file manually, remove the above marker line (the line +# saying "# AUTOGENERATED"). If you want to manage some indexes +# manually, move them above the marker line. The index.yaml file is +# automatically uploaded to the admin console when you next deploy +# your application using appcfg.py. + +- kind: Commit + properties: + - name: __key__ + direction: desc diff --git a/misc/dashboard/godashboard/key.py b/misc/dashboard/godashboard/key.py new file mode 100644 index 0000000000..7495709ecd --- /dev/null +++ b/misc/dashboard/godashboard/key.py @@ -0,0 +1,9 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +# accessKey controls private access to the build server (i.e. to record new +# builds). It's tranmitted in the clear but, given the low value of the target, +# this should be sufficient. +accessKey = "this is not the real key" + diff --git a/misc/dashboard/godashboard/main.html b/misc/dashboard/godashboard/main.html new file mode 100644 index 0000000000..ec874ce4ff --- /dev/null +++ b/misc/dashboard/godashboard/main.html @@ -0,0 +1,84 @@ + + + + Go build + + + + + + + + {% for b in builders %} + + {% endfor %} + + + + + + + {% for r in revs %} + + {% for b in r.builds %} + + {% endfor %} + + + + + + + {% endfor %} +
{{b}}
+ {% if b.ok %} + + {% else %} + {% if b.log %} + failed + {% else %} + + {% endif %} + {% endif %} + {{r.node|slice:":12"}}{{r.user|escape}}{{r.date|escape}}{{r.desc|escape}}
+ +