diff --git a/misc/dashboard/godashboard/app.yaml b/misc/dashboard/godashboard/app.yaml index 83611cf9018..7b77a85ccce 100644 --- a/misc/dashboard/godashboard/app.yaml +++ b/misc/dashboard/godashboard/app.yaml @@ -11,9 +11,13 @@ handlers: - url: /static static_dir: static -- url: /package.* +- url: /package script: package.py +- url: /package/daily + script: package.py + login: admin + - url: /project.* script: package.py diff --git a/misc/dashboard/godashboard/cron.yaml b/misc/dashboard/godashboard/cron.yaml new file mode 100644 index 00000000000..953b6a1cde1 --- /dev/null +++ b/misc/dashboard/godashboard/cron.yaml @@ -0,0 +1,4 @@ +cron: +- description: daily package maintenance + url: /package/daily + schedule: every 24 hours diff --git a/misc/dashboard/godashboard/package.html b/misc/dashboard/godashboard/package.html index 043080b5bfd..8a9d0a3a044 100644 --- a/misc/dashboard/godashboard/package.html +++ b/misc/dashboard/godashboard/package.html @@ -32,6 +32,20 @@ package doc comment.

+

Most Installed Packages (this week)

+ + + {% for r in by_week_count %} + + + + + + + + {% endfor %} +
last installcountbuildpathinfo
{{r.last_install|date:"Y-M-d H:i"}}{{r.week_count}}{% if r.ok %}ok{% else %} {% endif %}{{r.path}}{% if r.info %}{{r.info|escape}}{% endif %}
+

Recently Installed Packages

@@ -41,12 +55,12 @@ - + {% endfor %}
last installcountbuildpathinfo
{{r.count}} {% if r.ok %}ok{% else %} {% endif %} {{r.path}}{% if r.info %}{{r.info|escape}}{% endif %}{% if r.info %}{{r.info|escape}}{% endif %}
-

Most Installed Packages

+

Most Installed Packages (all time)

{% for r in by_count %} @@ -55,7 +69,7 @@ - + {% endfor %}
last installcountbuildpathinfo
{{r.count}} {% if r.ok %}ok{% else %} {% endif %} {{r.path}}{% if r.info %}{{r.info|escape}}{% endif %}{% if r.info %}{{r.info|escape}}{% endif %}
diff --git a/misc/dashboard/godashboard/package.py b/misc/dashboard/godashboard/package.py index 316f3867f0e..87da51424ac 100644 --- a/misc/dashboard/godashboard/package.py +++ b/misc/dashboard/godashboard/package.py @@ -5,34 +5,36 @@ # This is the server part of the package dashboard. # It must be run by App Engine. +from google.appengine.api import mail from google.appengine.api import memcache +from google.appengine.api import taskqueue +from google.appengine.api import urlfetch +from google.appengine.api import users from google.appengine.ext import db from google.appengine.ext import webapp from google.appengine.ext.webapp import template from google.appengine.ext.webapp.util import run_wsgi_app -from google.appengine.api import users -from google.appengine.api import mail -from google.appengine.api import urlfetch import datetime import logging import os import re -import urllib2 import sets +import urllib2 # local imports +from auth import auth import toutf8 import const -from auth import auth template.register_template_library('toutf8') # Storage model for package info recorded on server. -# Just path, count, and time of last install. class Package(db.Model): path = db.StringProperty() - web_url = db.StringProperty() # derived from path - count = db.IntegerProperty() + web_url = db.StringProperty() # derived from path + count = db.IntegerProperty() # grand total + week_count = db.IntegerProperty() # rolling weekly count + day_count = db.TextProperty(default='') # daily count last_install = db.DateTimeProperty() # data contributed by gobuilder @@ -40,6 +42,67 @@ class Package(db.Model): ok = db.BooleanProperty() last_ok = db.DateTimeProperty() + def get_day_count(self): + counts = {} + if not self.day_count: + return counts + for d in str(self.day_count).split('\n'): + date, count = d.split(' ') + counts[date] = int(count) + return counts + + def set_day_count(self, count): + days = [] + for day, count in count.items(): + days.append('%s %d' % (day, count)) + days.sort(reverse=True) + days = days[:28] + self.day_count = '\n'.join(days) + + def inc(self): + count = self.get_day_count() + today = str(datetime.date.today()) + count[today] = count.get(today, 0) + 1 + self.set_day_count(count) + self.update_week_count(count) + self.count += 1 + + def update_week_count(self, count=None): + if count is None: + count = self.get_day_count() + total = 0 + today = datetime.date.today() + for i in range(7): + day = str(today - datetime.timedelta(days=i)) + if day in count: + total += count[day] + self.week_count = total + + +# PackageDaily kicks off the daily package maintenance cron job +# and serves the associated task queue. +class PackageDaily(webapp.RequestHandler): + + def get(self): + # queue a task to update each package with a week_count > 0 + keys = Package.all(keys_only=True).filter('week_count >', 0) + for key in keys: + taskqueue.add(url='/package/daily', params={'key': key.name()}) + + def post(self): + # update a single package (in a task queue) + def update(key): + p = Package.get_by_key_name(key) + if not p: + return + p.update_week_count() + p.put() + key = self.request.get('key') + if not key: + return + db.run_in_transaction(update, key) + + class Project(db.Model): name = db.StringProperty(indexed=True) descr = db.StringProperty() @@ -49,6 +112,7 @@ class Project(db.Model): tags = db.ListProperty(str) approved = db.BooleanProperty(indexed=True) + re_bitbucket = re.compile(r'^(bitbucket\.org/[a-z0-9A-Z_.\-]+/[a-zA-Z0-9_.\-]+)(/[a-z0-9A-Z_.\-/]+)?$') re_googlecode = re.compile(r'^[a-z0-9\-]+\.googlecode\.com/(svn|hg)(/[a-z0-9A-Z_.\-/]+)?$') re_github = re.compile(r'^github\.com/[a-z0-9A-Z_.\-]+(/[a-z0-9A-Z_.\-]+)+$') @@ -115,29 +179,30 @@ class PackagePage(webapp.RequestHandler): html = memcache.get('view-package') if not html: + tdata = {} + + q = Package.all().filter('week_count >', 0) + q.order('-week_count') + tdata['by_week_count'] = q.fetch(50) + q = Package.all() q.order('-last_install') - by_time = q.fetch(100) + tdata['by_time'] = q.fetch(20) q = Package.all() q.order('-count') - by_count = q.fetch(100) + tdata['by_count'] = q.fetch(100) - self.response.headers['Content-Type'] = 'text/html; charset=utf-8' path = os.path.join(os.path.dirname(__file__), 'package.html') - html = template.render( - path, - {"by_time": by_time, "by_count": by_count} - ) + html = template.render(path, tdata) memcache.set('view-package', html, time=CacheTimeout) + self.response.headers['Content-Type'] = 'text/html; charset=utf-8' self.response.out.write(html) def json(self): json = memcache.get('view-package-json') if not json: - self.response.set_status(200) - self.response.headers['Content-Type'] = 'text/plain; charset=utf-8' q = Package.all() s = '{"packages": [' sep = '' @@ -147,6 +212,8 @@ class PackagePage(webapp.RequestHandler): s += '\n]}\n' json = s memcache.set('view-package-json', json, time=CacheTimeout) + self.response.set_status(200) + self.response.headers['Content-Type'] = 'text/plain; charset=utf-8' self.response.out.write(json) def can_get_url(self, url): @@ -181,14 +248,15 @@ class PackagePage(webapp.RequestHandler): return False p = Package(key_name = key, path = path, count = 0, web_url = web) - # is this the builder updating package metadata? if auth(self.request): + # builder updating package metadata p.info = self.request.get('info') p.ok = self.request.get('ok') == "true" if p.ok: p.last_ok = datetime.datetime.utcnow() else: - p.count += 1 + # goinstall reporting an install + p.inc() p.last_install = datetime.datetime.utcnow() # update package object @@ -197,7 +265,7 @@ class PackagePage(webapp.RequestHandler): def post(self): path = self.request.get('path') - ok = db.run_in_transaction(self.record_pkg, path) + ok = db.run_in_transaction(self.record_pkg, path) if ok: self.response.set_status(200) self.response.out.write('ok') @@ -347,6 +415,7 @@ class ProjectPage(webapp.RequestHandler): def main(): app = webapp.WSGIApplication([ ('/package', PackagePage), + ('/package/daily', PackageDaily), ('/project.*', ProjectPage), ], debug=True) run_wsgi_app(app)