From 8421390cb9ba9583af6369c7cc4786149058b131 Mon Sep 17 00:00:00 2001 From: Andrew Gerrand Date: Mon, 27 Feb 2012 15:25:41 +1100 Subject: [PATCH] misc/dashboard: record install counts for external packages R=golang-dev, bradfitz CC=golang-dev https://golang.org/cl/5699082 --- misc/dashboard/app/app.yaml | 4 +- misc/dashboard/app/build/build.go | 52 ++++++++ misc/dashboard/app/build/pkg.go | 201 ++++++++++++++++++++++++++++++ misc/dashboard/app/cron.yaml | 4 + 4 files changed, 259 insertions(+), 2 deletions(-) create mode 100644 misc/dashboard/app/build/pkg.go create mode 100644 misc/dashboard/app/cron.yaml diff --git a/misc/dashboard/app/app.yaml b/misc/dashboard/app/app.yaml index 6e19db09c62..47b7d822a39 100644 --- a/misc/dashboard/app/app.yaml +++ b/misc/dashboard/app/app.yaml @@ -13,8 +13,8 @@ handlers: static_dir: static - url: /log/.+ script: _go_app -- url: /(|commit|packages|result|tag|todo) +- url: /(|commit|install|packages|result|tag|todo) script: _go_app -- url: /(init|buildtest|key|_ah/queue/go/delay) +- url: /(init|buildtest|key|_ah/queue/go/delay|install/cron) script: _go_app login: admin diff --git a/misc/dashboard/app/build/build.go b/misc/dashboard/app/build/build.go index c49fa8bb2af..a7b08321b46 100644 --- a/misc/dashboard/app/build/build.go +++ b/misc/dashboard/app/build/build.go @@ -12,6 +12,7 @@ import ( "fmt" "io" "io/ioutil" + "strconv" "strings" "time" @@ -27,6 +28,10 @@ type Package struct { Name string Path string // (empty for the main Go tree) NextNum int // Num of the next head Commit + + Installs int // All-time total install count + InstallsByDay []string `datastore:",noindex"` // "yyyy-mm-dd,n" + InstallsThisWeek int // Rolling weekly count } func (p *Package) String() string { @@ -41,6 +46,53 @@ func (p *Package) Key(c appengine.Context) *datastore.Key { return datastore.NewKey(c, "Package", key, 0, nil) } +const day = time.Hour * 24 + +// IncrementInstalls increments the total install count and today's install count. +// Daily install counts for dates older than 30 days are discarded. +func (p *Package) IncrementInstalls() { + c := p.dayCounts() + s := []string{} + now := time.Now() + for i := 0; i < 30; i++ { + d := now.Add(-day * time.Duration(i)).Format("2006-01-02") + n := c[d] + if i == 0 { + n++ // increment today's count + } + if n > 0 { // no need to store zeroes in the datastore + s = append(s, fmt.Sprintf("%s,%d", d, n)) + } + } + p.InstallsByDay = s + p.Installs++ + p.UpdateInstallsThisWeek() +} + +// UpdateInstallsThisWeek updates the package's InstallsThisWeek field using data +// from the InstallsByDay list. +func (p *Package) UpdateInstallsThisWeek() { + c := p.dayCounts() + n := 0 + now := time.Now() + for i := 0; i < 7; i++ { + d := now.Add(-day * time.Duration(i)).Format("2006-01-02") + n += c[d] + } + p.InstallsThisWeek = n +} + +// dayCounts explodes InstallsByDay into a map of dates to install counts. +func (p *Package) dayCounts() map[string]int { + c := make(map[string]int) + for _, d := range p.InstallsByDay { + p := strings.SplitN(d, ",", 2) + n, _ := strconv.Atoi(p[1]) + c[p[0]] = n + } + return c +} + // LastCommit returns the most recent Commit for this Package. func (p *Package) LastCommit(c appengine.Context) (*Commit, error) { var commits []*Commit diff --git a/misc/dashboard/app/build/pkg.go b/misc/dashboard/app/build/pkg.go new file mode 100644 index 00000000000..076fafe4fdf --- /dev/null +++ b/misc/dashboard/app/build/pkg.go @@ -0,0 +1,201 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package build + +import ( + "net/http" + "regexp" + "strings" + + "appengine" + "appengine/datastore" + "appengine/delay" + "appengine/urlfetch" +) + +func init() { + http.HandleFunc("/install", installHandler) + http.HandleFunc("/install/cron", installCronHandler) +} + +// installHandler serves requests from the go tool to increment the install +// count for a given package. +func installHandler(w http.ResponseWriter, r *http.Request) { + installLater.Call(appengine.NewContext(r), r.FormValue("packagePath")) +} + +// installCronHandler starts a task to update the weekly install counts for +// every external package. +func installCronHandler(w http.ResponseWriter, r *http.Request) { + c := appengine.NewContext(r) + q := datastore.NewQuery("Package").Filter("Kind=", "external").KeysOnly() + for t := q.Run(c); ; { + key, err := t.Next(nil) + if err == datastore.Done { + break + } else if err != nil { + c.Errorf("%v", err) + return + } + updateWeeklyLater.Call(c, key) + } +} + +var ( + installLater = delay.Func("install", install) + updateWeeklyLater = delay.Func("updateWeekly", updateWeekly) +) + +// install validates the provided package path, increments its install count, +// and creates the Package record if it doesn't exist. +func install(c appengine.Context, path string) { + if !validPath(c, path) { + return + } + tx := func(c appengine.Context) error { + p := &Package{Path: path, Kind: "external"} + err := datastore.Get(c, p.Key(c), p) + if err != nil && err != datastore.ErrNoSuchEntity { + return err + } + p.IncrementInstalls() + _, err = datastore.Put(c, p.Key(c), p) + return err + } + if err := datastore.RunInTransaction(c, tx, nil); err != nil { + c.Errorf("install(%q): %v", path, err) + } +} + +// updateWeekly updates the weekly count for the specified Package. +func updateWeekly(c appengine.Context, key *datastore.Key) { + tx := func(c appengine.Context) error { + p := new(Package) + if err := datastore.Get(c, key, p); err != nil { + return err + } + p.UpdateInstallsThisWeek() + _, err := datastore.Put(c, key, p) + return err + } + if err := datastore.RunInTransaction(c, tx, nil); err != nil { + c.Errorf("updateWeekly: %v", err) + } +} + +// validPath validates the specified import path by matching it against the +// vcsPath regexen and validating its existence by making an HTTP GET request +// to the remote repository. +func validPath(c appengine.Context, path string) bool { + for _, p := range vcsPaths { + if !strings.HasPrefix(path, p.prefix) { + continue + } + m := p.regexp.FindStringSubmatch(path) + if m == nil { + continue + } + if p.check == nil { + // no check function, so just say OK + return true + } + match := make(map[string]string) + for i, name := range p.regexp.SubexpNames() { + if name != "" { + match[name] = m[i] + } + } + return p.check(c, match) + } + c.Debugf("validPath(%q): matching vcsPath not found", path) + return false +} + +// A vcsPath describes how to convert an import path into a version control +// system and repository name. +// +// This is a cut down and modified version of the data structure from +// $GOROOT/src/cmd/go/vcs.go. +type vcsPath struct { + prefix string // prefix this description applies to + re string // pattern for import path + + // check should perform an HTTP request to validate the import path + check func(c appengine.Context, match map[string]string) bool + + regexp *regexp.Regexp // cached compiled form of re +} + +// vcsPaths lists the known vcs paths. +// +// This is a cut down version of the data from $GOROOT/src/cmd/go/vcs.go. +var vcsPaths = []*vcsPath{ + // Google Code - new syntax + { + prefix: "code.google.com/", + re: `^(?Pcode\.google\.com/p/(?P[a-z0-9\-]+)(\.(?P[a-z0-9\-]+))?)(/[A-Za-z0-9_.\-]+)*$`, + check: googleCodeVCS, + }, + + // Github + { + prefix: "github.com/", + re: `^(?Pgithub\.com/[A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+)(/[A-Za-z0-9_.\-]+)*$`, + check: checkRoot, + }, + + // Bitbucket + { + prefix: "bitbucket.org/", + re: `^(?Pbitbucket\.org/(?P[A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+))(/[A-Za-z0-9_.\-]+)*$`, + check: checkRoot, + }, + + // Launchpad + { + prefix: "launchpad.net/", + re: `^(?Plaunchpad\.net/((?P[A-Za-z0-9_.\-]+)(?P/[A-Za-z0-9_.\-]+)?|~[A-Za-z0-9_.\-]+/(\+junk|[A-Za-z0-9_.\-]+)/[A-Za-z0-9_.\-]+))(/[A-Za-z0-9_.\-]+)*$`, + // TODO(adg): write check function for Launchpad + }, +} + +func init() { + // Compile the regular expressions. + for i := range vcsPaths { + vcsPaths[i].regexp = regexp.MustCompile(vcsPaths[i].re) + } +} + +// googleCodeVCS performs an HTTP GET to verify that a Google Code project +// (and, optionally, a sub-repository) exists. +func googleCodeVCS(c appengine.Context, match map[string]string) bool { + u := "https://code.google.com/p/" + match["project"] + if match["subrepo"] != "" { + u += "/source/checkout?repo=" + match["subrepo"] + } + return checkURL(c, u) +} + +// checkRoot performs an HTTP GET to verify that a specific repository root +// exists (for github and bitbucket both). +func checkRoot(c appengine.Context, match map[string]string) bool { + return checkURL(c, "https://"+match["root"]) +} + +// checkURL performs an HTTP GET to the specified URL and returns whether the +// remote server returned a 2xx response. +func checkURL(c appengine.Context, u string) bool { + client := urlfetch.Client(c) + resp, err := client.Get(u) + if err != nil { + c.Errorf("checkURL(%q): %v", u, err) + return false + } + if resp.StatusCode/100 != 2 { + c.Debugf("checkURL(%q): HTTP status: %s", u, resp.Status) + return false + } + return true +} diff --git a/misc/dashboard/app/cron.yaml b/misc/dashboard/app/cron.yaml new file mode 100644 index 00000000000..b54c24cef78 --- /dev/null +++ b/misc/dashboard/app/cron.yaml @@ -0,0 +1,4 @@ +cron: +- description: update rolling package install counts + url: /install/cron + schedule: every 24 hours