mirror of
https://github.com/golang/go
synced 2024-11-22 13:45:00 -07:00
12c3afc1ae
In the new world, one builder runs
gobuilder -commit
which uploads information about commits to the dashboard,
which then hands the work out to the builders by hash.
There is no assumption anymore that the commit numbers
are consistent across builders.
New builders will need to be deployed. For now darwin-amd64
is running the new builder to test the code.
The new JSON-based protocol for handing out work via /todo
should be easy to extend if we want to add support for sending
trial CLs to the builders.
This code is already running on godashboard.appspot.com.
R=adg, dave
CC=golang-dev
https://golang.org/cl/4519047
482 lines
15 KiB
Python
482 lines
15 KiB
Python
# Copyright 2009 The Go Authors. All rights reserved.
|
|
# Use of this source code is governed by a BSD-style
|
|
# license that can be found in the LICENSE file.
|
|
|
|
# This is the server part of the continuous build system for Go. It must be run
|
|
# by AppEngine.
|
|
|
|
from django.utils import simplejson
|
|
from google.appengine.api import mail
|
|
from google.appengine.api import memcache
|
|
from google.appengine.ext import db
|
|
from google.appengine.ext import webapp
|
|
from google.appengine.ext.webapp import template
|
|
from google.appengine.ext.webapp.util import run_wsgi_app
|
|
import binascii
|
|
import datetime
|
|
import hashlib
|
|
import hmac
|
|
import logging
|
|
import os
|
|
import re
|
|
import struct
|
|
import time
|
|
import bz2
|
|
|
|
# local imports
|
|
import key
|
|
import const
|
|
|
|
# The majority of our state are commit objects. One of these exists for each of
|
|
# the commits known to the build system. Their key names are of the form
|
|
# <commit number (%08x)> "-" <hg hash>. This means that a sorting by the key
|
|
# name is sufficient to order the commits.
|
|
#
|
|
# The commit numbers are purely local. They need not match up to the commit
|
|
# numbers in an hg repo. When inserting a new commit, the parent commit must be
|
|
# given and this is used to generate the new commit number. In order to create
|
|
# the first Commit object, a special command (/init) is used.
|
|
class Commit(db.Model):
|
|
num = db.IntegerProperty() # internal, monotonic counter.
|
|
node = db.StringProperty() # Hg hash
|
|
parentnode = db.StringProperty() # Hg hash
|
|
user = db.StringProperty()
|
|
date = db.DateTimeProperty()
|
|
desc = db.BlobProperty()
|
|
|
|
# This is the list of builds. Each element is a string of the form <builder
|
|
# name> '`' <log hash>. If the log hash is empty, then the build was
|
|
# successful.
|
|
builds = db.StringListProperty()
|
|
|
|
fail_notification_sent = db.BooleanProperty()
|
|
|
|
# A CompressedLog contains the textual build log of a failed build.
|
|
# The key name is the hex digest of the SHA256 hash of the contents.
|
|
# The contents is bz2 compressed.
|
|
class CompressedLog(db.Model):
|
|
log = db.BlobProperty()
|
|
|
|
N = 30
|
|
|
|
def builderInfo(b):
|
|
f = b.split('-', 3)
|
|
goos = f[0]
|
|
goarch = f[1]
|
|
note = ""
|
|
if len(f) > 2:
|
|
note = f[2]
|
|
return {'name': b, 'goos': goos, 'goarch': goarch, 'note': note}
|
|
|
|
def builderset():
|
|
q = Commit.all()
|
|
q.order('-__key__')
|
|
results = q.fetch(N)
|
|
builders = set()
|
|
for c in results:
|
|
builders.update(set(parseBuild(build)['builder'] for build in c.builds))
|
|
return builders
|
|
|
|
class MainPage(webapp.RequestHandler):
|
|
def get(self):
|
|
self.response.headers['Content-Type'] = 'text/html; charset=utf-8'
|
|
|
|
try:
|
|
page = int(self.request.get('p', 1))
|
|
if not page > 0:
|
|
raise
|
|
except:
|
|
page = 1
|
|
|
|
try:
|
|
num = int(self.request.get('n', N))
|
|
if num <= 0 or num > 200:
|
|
raise
|
|
except:
|
|
num = N
|
|
|
|
offset = (page-1) * num
|
|
|
|
q = Commit.all()
|
|
q.order('-__key__')
|
|
results = q.fetch(num, offset)
|
|
|
|
revs = [toRev(r) for r in results]
|
|
builders = {}
|
|
|
|
for r in revs:
|
|
for b in r['builds']:
|
|
builders[b['builder']] = builderInfo(b['builder'])
|
|
|
|
for r in revs:
|
|
have = set(x['builder'] for x in r['builds'])
|
|
need = set(builders.keys()).difference(have)
|
|
for n in need:
|
|
r['builds'].append({'builder': n, 'log':'', 'ok': False})
|
|
r['builds'].sort(cmp = byBuilder)
|
|
|
|
builders = list(builders.items())
|
|
builders.sort()
|
|
values = {"revs": revs, "builders": [v for k,v in builders]}
|
|
|
|
values['num'] = num
|
|
values['prev'] = page - 1
|
|
if len(results) == num:
|
|
values['next'] = page + 1
|
|
|
|
path = os.path.join(os.path.dirname(__file__), 'main.html')
|
|
self.response.out.write(template.render(path, values))
|
|
|
|
# A DashboardHandler is a webapp.RequestHandler but provides
|
|
# authenticated_post - called by post after authenticating
|
|
# json - writes object in json format to response output
|
|
class DashboardHandler(webapp.RequestHandler):
|
|
def post(self):
|
|
if not auth(self.request):
|
|
self.response.set_status(403)
|
|
return
|
|
self.authenticated_post()
|
|
|
|
def authenticated_post(self):
|
|
return
|
|
|
|
def json(self, obj):
|
|
self.response.set_status(200)
|
|
simplejson.dump(obj, self.response.out)
|
|
return
|
|
|
|
def auth(req):
|
|
k = req.get('key')
|
|
return k == hmac.new(key.accessKey, req.get('builder')).hexdigest() or k == key.accessKey
|
|
|
|
# Todo serves /todo. It tells the builder which commits need to be built.
|
|
class Todo(DashboardHandler):
|
|
def get(self):
|
|
builder = self.request.get('builder')
|
|
key = 'todo-%s' % builder
|
|
response = memcache.get(key)
|
|
if response is None:
|
|
# Fell out of memcache. Rebuild from datastore results.
|
|
# We walk the commit list looking for nodes that have not
|
|
# been built by this builder.
|
|
q = Commit.all()
|
|
q.order('-__key__')
|
|
todo = []
|
|
first = None
|
|
for c in q.fetch(N+1):
|
|
if first is None:
|
|
first = c
|
|
if not built(c, builder):
|
|
todo.append({'Hash': c.node})
|
|
response = simplejson.dumps(todo)
|
|
memcache.set(key, response, 3600)
|
|
self.response.set_status(200)
|
|
self.response.out.write(response)
|
|
|
|
def built(c, builder):
|
|
for b in c.builds:
|
|
if b.startswith(builder+'`'):
|
|
return True
|
|
return False
|
|
|
|
# Log serves /log/. It retrieves log data by content hash.
|
|
class LogHandler(DashboardHandler):
|
|
def get(self):
|
|
self.response.headers['Content-Type'] = 'text/plain; charset=utf-8'
|
|
hash = self.request.path[5:]
|
|
l = CompressedLog.get_by_key_name(hash)
|
|
if l is None:
|
|
self.response.set_status(404)
|
|
return
|
|
log = bz2.decompress(l.log)
|
|
self.response.set_status(200)
|
|
self.response.out.write(log)
|
|
|
|
# Init creates the commit with id 0. Since this commit doesn't have a parent,
|
|
# it cannot be created by Build.
|
|
class Init(DashboardHandler):
|
|
def authenticated_post(self):
|
|
date = parseDate(self.request.get('date'))
|
|
node = self.request.get('node')
|
|
if not validNode(node) or date is None:
|
|
logging.error("Not valid node ('%s') or bad date (%s %s)", node, date, self.request.get('date'))
|
|
self.response.set_status(500)
|
|
return
|
|
|
|
commit = Commit(key_name = '00000000-%s' % node)
|
|
commit.num = 0
|
|
commit.node = node
|
|
commit.parentnode = ''
|
|
commit.user = self.request.get('user').encode('utf8')
|
|
commit.date = date
|
|
commit.desc = self.request.get('desc').encode('utf8')
|
|
|
|
commit.put()
|
|
|
|
self.response.set_status(200)
|
|
|
|
# The last commit when we switched to using entity groups.
|
|
# This is the root of the new commit entity group.
|
|
RootCommitKeyName = '00000f26-f32c6f1038207c55d5780231f7484f311020747e'
|
|
|
|
# CommitHandler serves /commit.
|
|
# A GET of /commit retrieves information about the specified commit.
|
|
# A POST of /commit creates a node for the given commit.
|
|
# If the commit already exists, the POST silently succeeds (like mkdir -p).
|
|
class CommitHandler(DashboardHandler):
|
|
def get(self):
|
|
node = self.request.get('node')
|
|
if not validNode(node):
|
|
return self.json({'Status': 'FAIL', 'Error': 'malformed node hash'})
|
|
n = nodeByHash(node)
|
|
if n is None:
|
|
return self.json({'Status': 'FAIL', 'Error': 'unknown revision'})
|
|
return self.json({'Status': 'OK', 'Node': nodeObj(n)})
|
|
|
|
def authenticated_post(self):
|
|
# Require auth with the master key, not a per-builder key.
|
|
if self.request.get('builder'):
|
|
self.response.set_status(403)
|
|
return
|
|
|
|
node = self.request.get('node')
|
|
date = parseDate(self.request.get('date'))
|
|
user = self.request.get('user').encode('utf8')
|
|
desc = self.request.get('desc').encode('utf8')
|
|
parenthash = self.request.get('parent')
|
|
|
|
if not validNode(node) or not validNode(parenthash) or date is None:
|
|
return self.json({'Status': 'FAIL', 'Error': 'malformed node, parent, or date'})
|
|
|
|
n = nodeByHash(node)
|
|
if n is None:
|
|
p = nodeByHash(parenthash)
|
|
if p is None:
|
|
return self.json({'Status': 'FAIL', 'Error': 'unknown parent'})
|
|
|
|
# Want to create new node in a transaction so that multiple
|
|
# requests creating it do not collide and so that multiple requests
|
|
# creating different nodes get different sequence numbers.
|
|
# All queries within a transaction must include an ancestor,
|
|
# but the original datastore objects we used for the dashboard
|
|
# have no common ancestor. Instead, we use a well-known
|
|
# root node - the last one before we switched to entity groups -
|
|
# as the as the common ancestor.
|
|
root = Commit.get_by_key_name(RootCommitKeyName)
|
|
|
|
def add_commit():
|
|
if nodeByHash(node, ancestor=root) is not None:
|
|
return
|
|
|
|
# Determine number for this commit.
|
|
# Once we have created one new entry it will be lastRooted.num+1,
|
|
# but the very first commit created in this scheme will have to use
|
|
# last.num's number instead (last is likely not rooted).
|
|
q = Commit.all()
|
|
q.order('-__key__')
|
|
q.ancestor(root)
|
|
last = q.fetch(1)[0]
|
|
num = last.num+1
|
|
|
|
n = Commit(key_name = '%08x-%s' % (num, node), parent = root)
|
|
n.num = num
|
|
n.node = node
|
|
n.parentnode = parenthash
|
|
n.user = user
|
|
n.date = date
|
|
n.desc = desc
|
|
n.put()
|
|
db.run_in_transaction(add_commit)
|
|
n = nodeByHash(node)
|
|
if n is None:
|
|
return self.json({'Status': 'FAIL', 'Error': 'failed to create commit node'})
|
|
|
|
return self.json({'Status': 'OK', 'Node': nodeObj(n)})
|
|
|
|
# Build serves /build.
|
|
# A POST to /build records a new build result.
|
|
class Build(webapp.RequestHandler):
|
|
def post(self):
|
|
if not auth(self.request):
|
|
self.response.set_status(403)
|
|
return
|
|
|
|
builder = self.request.get('builder')
|
|
log = self.request.get('log').encode('utf-8')
|
|
|
|
loghash = ''
|
|
if len(log) > 0:
|
|
loghash = hashlib.sha256(log).hexdigest()
|
|
l = CompressedLog(key_name=loghash)
|
|
l.log = bz2.compress(log)
|
|
l.put()
|
|
|
|
node = self.request.get('node')
|
|
if not validNode(node):
|
|
logging.error('Invalid node %s' % (node))
|
|
self.response.set_status(500)
|
|
return
|
|
|
|
n = nodeByHash(node)
|
|
if n is None:
|
|
logging.error('Cannot find node %s' % (node))
|
|
self.response.set_status(404)
|
|
return
|
|
nn = n
|
|
|
|
def add_build():
|
|
n = nodeByHash(node, ancestor=nn)
|
|
if n is None:
|
|
logging.error('Cannot find hash in add_build: %s %s' % (builder, node))
|
|
return
|
|
|
|
s = '%s`%s' % (builder, loghash)
|
|
for i, b in enumerate(n.builds):
|
|
if b.split('`', 1)[0] == builder:
|
|
# logging.error('Found result for %s %s already' % (builder, node))
|
|
n.builds[i] = s
|
|
break
|
|
else:
|
|
# logging.error('Added result for %s %s' % (builder, node))
|
|
n.builds.append(s)
|
|
n.put()
|
|
|
|
db.run_in_transaction(add_build)
|
|
|
|
key = 'todo-%s' % builder
|
|
memcache.delete(key)
|
|
|
|
# TODO: Send mail for build breakage.
|
|
|
|
self.response.set_status(200)
|
|
|
|
def failed(c, builder):
|
|
for i, b in enumerate(c.builds):
|
|
p = b.split('`', 1)
|
|
if p[0] == builder:
|
|
return len(p[1]) > 0
|
|
return False
|
|
|
|
def node(num):
|
|
q = Commit.all()
|
|
q.filter('num =', num)
|
|
n = q.get()
|
|
return n
|
|
|
|
def nodeByHash(hash, ancestor=None):
|
|
q = Commit.all()
|
|
q.filter('node =', hash)
|
|
if ancestor is not None:
|
|
q.ancestor(ancestor)
|
|
n = q.get()
|
|
return n
|
|
|
|
# nodeObj returns a JSON object (ready to be passed to simplejson.dump) describing node.
|
|
def nodeObj(n):
|
|
return {
|
|
'Hash': n.node,
|
|
'ParentHash': n.parentnode,
|
|
'User': n.user,
|
|
'Date': n.date.strftime('%Y-%m-%d %H:%M %z'),
|
|
'Desc': n.desc,
|
|
}
|
|
|
|
class FixedOffset(datetime.tzinfo):
|
|
"""Fixed offset in minutes east from UTC."""
|
|
|
|
def __init__(self, offset):
|
|
self.__offset = datetime.timedelta(seconds = offset)
|
|
|
|
def utcoffset(self, dt):
|
|
return self.__offset
|
|
|
|
def tzname(self, dt):
|
|
return None
|
|
|
|
def dst(self, dt):
|
|
return datetime.timedelta(0)
|
|
|
|
def validNode(node):
|
|
if len(node) != 40:
|
|
return False
|
|
for x in node:
|
|
o = ord(x)
|
|
if (o < ord('0') or o > ord('9')) and (o < ord('a') or o > ord('f')):
|
|
return False
|
|
return True
|
|
|
|
def parseDate(date):
|
|
if '-' in date:
|
|
(a, offset) = date.split('-', 1)
|
|
try:
|
|
return datetime.datetime.fromtimestamp(float(a), FixedOffset(0-int(offset)))
|
|
except ValueError:
|
|
return None
|
|
if '+' in date:
|
|
(a, offset) = date.split('+', 1)
|
|
try:
|
|
return datetime.datetime.fromtimestamp(float(a), FixedOffset(int(offset)))
|
|
except ValueError:
|
|
return None
|
|
try:
|
|
return datetime.datetime.utcfromtimestamp(float(date))
|
|
except ValueError:
|
|
return None
|
|
|
|
email_re = re.compile('^[^<]+<([^>]*)>$')
|
|
|
|
def toUsername(user):
|
|
r = email_re.match(user)
|
|
if r is None:
|
|
return user
|
|
email = r.groups()[0]
|
|
return email.replace('@golang.org', '')
|
|
|
|
def dateToShortStr(d):
|
|
return d.strftime('%a %b %d %H:%M')
|
|
|
|
def parseBuild(build):
|
|
[builder, logblob] = build.split('`')
|
|
return {'builder': builder, 'log': logblob, 'ok': len(logblob) == 0}
|
|
|
|
def nodeInfo(c):
|
|
return {
|
|
"node": c.node,
|
|
"user": toUsername(c.user),
|
|
"date": dateToShortStr(c.date),
|
|
"desc": c.desc,
|
|
"shortdesc": c.desc.split('\n', 2)[0]
|
|
}
|
|
|
|
def toRev(c):
|
|
b = nodeInfo(c)
|
|
b['builds'] = [parseBuild(build) for build in c.builds]
|
|
return b
|
|
|
|
def byBuilder(x, y):
|
|
return cmp(x['builder'], y['builder'])
|
|
|
|
# Give old builders work; otherwise they pound on the web site.
|
|
class Hwget(DashboardHandler):
|
|
def get(self):
|
|
self.response.out.write("8000\n")
|
|
|
|
# This is the URL map for the server. The first three entries are public, the
|
|
# rest are only used by the builders.
|
|
application = webapp.WSGIApplication(
|
|
[('/', MainPage),
|
|
('/hw-get', Hwget),
|
|
('/log/.*', LogHandler),
|
|
('/commit', CommitHandler),
|
|
('/init', Init),
|
|
('/todo', Todo),
|
|
('/build', Build),
|
|
], debug=True)
|
|
|
|
def main():
|
|
run_wsgi_app(application)
|
|
|
|
if __name__ == "__main__":
|
|
main()
|
|
|