go/misc/dashboard/godashboard/package.py

# Copyright 2010 The Go Authors. All rights reserved.
# Use of this source code is governed by a BSD-style
# license that can be found in the LICENSE file.

# This is the server part of the package dashboard.
# It must be run by App Engine.

from google.appengine.api import memcache
from google.appengine.ext import db
from google.appengine.ext import webapp
from google.appengine.ext.webapp import template
from google.appengine.ext.webapp.util import run_wsgi_app
from google.appengine.api import users
from google.appengine.api import mail
from google.appengine.api import urlfetch
import datetime
import logging
import os
import re
import urllib2
import sets

# local imports
import toutf8
import const

template.register_template_library('toutf8')

# Storage model for package info recorded on server.
# Just path, count, and time of last install.
class Package(db.Model):
    path = db.StringProperty()
    web_url = db.StringProperty()  # derived from path
    count = db.IntegerProperty()
    last_install = db.DateTimeProperty()

class Project(db.Model):
    name = db.StringProperty(indexed=True)
    descr = db.StringProperty()
    web_url = db.StringProperty()
    package = db.ReferenceProperty(Package)
    category = db.StringProperty(indexed=True)
    tags = db.ListProperty(str)
    approved = db.BooleanProperty(indexed=True)

re_bitbucket = re.compile(r'^bitbucket\.org/[a-z0-9A-Z_.\-]+/[a-z0-9A-Z_.\-]+$')
re_googlecode = re.compile(r'^[a-z0-9\-]+\.googlecode\.com/(svn|hg)$')
re_github = re.compile(r'^github\.com/[a-z0-9A-Z_.\-]+/[a-z0-9A-Z_.\-]+$')
re_launchpad = re.compile(r'^launchpad\.net/([a-z0-9A-Z_.\-]+(/[a-z0-9A-Z_.\-]+)?|~[a-z0-9A-Z_.\-]+/(\+junk|[a-z0-9A-Z_.\-]+)/[a-z0-9A-Z_.\-]+)(/[a-z0-9A-Z_.\-/]+)?$')


def vc_to_web(path):
    if re_bitbucket.match(path):
        check_url = 'http://' + path + '/?cmd=heads'
        web = 'http://' + path + '/'
    elif re_github.match(path):
        # github doesn't let you fetch the .git directory anymore.
        # fetch .git/info/refs instead, like git clone would.
        check_url = 'http://'+path+'.git/info/refs'
        web = 'http://' + path
    elif re_googlecode.match(path):
        check_url = 'http://'+path
        web = 'http://code.google.com/p/' + path[:path.index('.')]
    elif re_launchpad.match(path):
        check_url = web = 'https://'+path
    else:
        return False, False
    return web, check_url

re_bitbucket_web = re.compile(r'bitbucket\.org/([a-z0-9A-Z_.\-]+)/([a-z0-9A-Z_.\-]+)')
re_googlecode_web = re.compile(r'code.google.com/p/([a-z0-9\-]+)')
re_github_web = re.compile(r'github\.com/([a-z0-9A-Z_.\-]+)/([a-z0-9A-Z_.\-]+)')
re_launchpad_web = re.compile(r'launchpad\.net/([a-z0-9A-Z_.\-]+(/[a-z0-9A-Z_.\-]+)?|~[a-z0-9A-Z_.\-]+/(\+junk|[a-z0-9A-Z_.\-]+)/[a-z0-9A-Z_.\-]+)(/[a-z0-9A-Z_.\-/]+)?')
re_striphttp = re.compile(r'https?://(www\.)?')

def web_to_vc(url):
    url = re_striphttp.sub('', url)
    m = re_bitbucket_web.match(url)
    if m:
        return 'bitbucket.org/'+m.group(1)+'/'+m.group(2)
    m = re_github_web.match(url)
    if m:
        return 'github.com/'+m.group(1)+'/'+m.group(2)
    m = re_googlecode_web.match(url)
    if m:
        path = m.group(1)+'.googlecode.com/'
        # perform http request to path/hg to check if they're using mercurial
        vcs = 'svn'
        try:
            response = urlfetch.fetch('http://'+path+'hg', deadline=1)
            if response.status_code == 200:
                vcs = 'hg'
        except: pass
        return path + vcs
    m = re_launchpad_web.match(url)
    if m:
        return m.group(0)
    return False

MaxPathLength = 100
CacheTimeout = 3600

class PackagePage(webapp.RequestHandler):
    def get(self):
        if self.request.get('fmt') == 'json':
            return self.json()

        html = memcache.get('view-package')
        if not html:
            q = Package.all()
            q.order('-last_install')
            by_time = q.fetch(100)

            q = Package.all()
            q.order('-count')
            by_count = q.fetch(100)

            self.response.headers['Content-Type'] = 'text/html; charset=utf-8'
            path = os.path.join(os.path.dirname(__file__), 'package.html')
            html = template.render(
                path, 
                {"by_time": by_time, "by_count": by_count}
            )
            memcache.set('view-package', html, time=CacheTimeout)

        self.response.out.write(html)

    def json(self):
        json = memcache.get('view-package-json')
        if not json:
            self.response.set_status(200)
            self.response.headers['Content-Type'] = 'text/plain; charset=utf-8'
            q = Package.all()
            s = '{"packages": ['
            sep = ''
            for r in q.fetch(1000):
                s += '%s\n\t{"path": "%s", "last_install": "%s", "count": "%s"}' % (sep, r.path, r.last_install, r.count)
                sep = ','
            s += '\n]}\n'
            json = s
            memcache.set('view-package-json', json, time=CacheTimeout)
        self.response.out.write(json)

    def can_get_url(self, url):
        try:
            req = urllib2.Request(url)
            response = urllib2.urlopen(req)
            return True
        except:
            return False

    def is_valid_package_path(self, path):
        return (re_bitbucket.match(path) or
            re_googlecode.match(path) or
            re_github.match(path) or
            re_launchpad.match(path))

    def record_pkg(self, path):
        # sanity check string
        if not path or len(path) > MaxPathLength or not self.is_valid_package_path(path):
            return False

        # look in datastore
        key = 'pkg-' + path
        p = Package.get_by_key_name(key)
        if p is None:
            # not in datastore - verify URL before creating
            web, check_url = vc_to_web(path)
            if not web:
                logging.error('unrecognized path: %s', path)
                return False
            if not self.can_get_url(check_url):
                logging.error('cannot get %s', check_url)
                return False
            p = Package(key_name = key, path = path, count = 0, web_url = web)

        # update package object
        p.count += 1
        p.last_install = datetime.datetime.utcnow()
        p.put()
        return True

    def post(self):
        path = self.request.get('path')
        ok = self.record_pkg(path)
        if ok:
            self.response.set_status(200)
            self.response.out.write('ok')
        else:
            logging.error('invalid path in post: %s', path)
            self.response.set_status(500)
            self.response.out.write('not ok')

class ProjectPage(webapp.RequestHandler):

    def get(self):
        admin = users.is_current_user_admin()
        if self.request.path == "/project/login":
            self.redirect(users.create_login_url("/project"))
        elif self.request.path == "/project/logout":
            self.redirect(users.create_logout_url("/project"))
        elif self.request.path == "/project/edit" and admin:
            self.edit()
        elif self.request.path == "/project/assoc" and admin:
            self.assoc()
        else:
            self.list()

    def assoc(self):
        projects = Project.all()
        for p in projects:
            if p.package:
                continue
            path = web_to_vc(p.web_url)
            if not path:
                continue
            pkg = Package.get_by_key_name("pkg-"+path)
            if not pkg:
                self.response.out.write('no: %s %s<br>' % (p.web_url, path))
                continue
            p.package = pkg
            p.put()
            self.response.out.write('yes: %s %s<br>' % (p.web_url, path))

    def post(self):
        if self.request.path == "/project/edit":
            self.edit(True)
        else:
            data = dict(map(lambda x: (x, self.request.get(x)), ["name","descr","web_url"]))
            if reduce(lambda x, y: x or not y, data.values(), False):
                data["submitMsg"] = "You must complete all the fields."
                self.list(data)
                return
            p = Project.get_by_key_name("proj-"+data["name"])
            if p is not None:
                data["submitMsg"] = "A project by this name already exists."
                self.list(data)
                return
            p = Project(key_name="proj-"+data["name"], **data)
            p.put()
		
            path = os.path.join(os.path.dirname(__file__), 'project-notify.txt')
            mail.send_mail(
                sender=const.mail_from,
                to=const.mail_submit_to,
                subject=const.mail_submit_subject,
                body=template.render(path, {'project': p}))

            self.list({"submitMsg": "Your project has been submitted."})

    def list(self, additional_data={}):
        cache_key = 'view-project-data'
        tag = self.request.get('tag', None)
        if tag:
            cache_key += '-'+tag
        data = memcache.get(cache_key)
        admin = users.is_current_user_admin()
        if admin or not data:
            projects = Project.all().order('category').order('name')
            if not admin:
                projects = projects.filter('approved =', True)
            projects = list(projects)

            tags = sets.Set()
            for p in projects:
                for t in p.tags:
                    tags.add(t)

            if tag:
                projects = filter(lambda x: tag in x.tags, projects)

            data = {}
            data['tag'] = tag
            data['tags'] = tags
            data['projects'] = projects 
            data['admin']= admin
            if not admin:
                memcache.set(cache_key, data, time=CacheTimeout)

        for k, v in additional_data.items():
            data[k] = v

        self.response.headers['Content-Type'] = 'text/html; charset=utf-8'
        path = os.path.join(os.path.dirname(__file__), 'project.html')
        self.response.out.write(template.render(path, data))

    def edit(self, save=False):
        if save:
            name = self.request.get("orig_name")
        else:
            name = self.request.get("name")

        p = Project.get_by_key_name("proj-"+name)
        if not p:
            self.response.out.write("Couldn't find that Project.")
            return

        if save:
            if self.request.get("do") == "Delete":
                p.delete()
            else:
                pkg_name = self.request.get("package", None)
                if pkg_name:
                    pkg = Package.get_by_key_name("pkg-"+pkg_name)
                    if pkg:
                        p.package = pkg.key()
                for f in ['name', 'descr', 'web_url', 'category']:
                    setattr(p, f, self.request.get(f, None))
                p.approved = self.request.get("approved") == "1"
                p.tags = filter(lambda x: x, self.request.get("tags", "").split(","))
                p.put()
            memcache.delete('view-project-data')
            self.redirect('/project')
            return

        # get all project categories and tags
        cats, tags = sets.Set(), sets.Set()
        for r in Project.all():
            cats.add(r.category)
            for t in r.tags:
                tags.add(t)

        self.response.headers['Content-Type'] = 'text/html; charset=utf-8'
        path = os.path.join(os.path.dirname(__file__), 'project-edit.html')
        self.response.out.write(template.render(path, { 
            "taglist": tags, "catlist": cats, "p": p, "tags": ",".join(p.tags) }))

    def redirect(self, url):
        self.response.set_status(302)
        self.response.headers.add_header("Location", url)

def main():
    app = webapp.WSGIApplication([
        ('/package', PackagePage),
        ('/project.*', ProjectPage),
        ], debug=True)
    run_wsgi_app(app)

if __name__ == '__main__':
    main()