dashboard: builder-facing implementation and tests

R=golang-dev, dsymonds
CC=golang-dev
https://golang.org/cl/5431048

misc/dashboard/app/app.yaml

@@ -4,5 +4,10 @@ runtime: go
 api_version: 3
 
 handlers:
+- url: /log/.+
+  script: _go_app
 - url: /(commit|tag|todo|result)
   script: _go_app
+- url: /buildtest
+  script: _go_app
+  login: admin

misc/dashboard/app/build/build.go

@@ -7,13 +7,24 @@ package build
 import (
 	"appengine"
 	"appengine/datastore"
+	"bytes"
+	"compress/gzip"
+	"crypto/sha1"
+	"fmt"
 	"http"
+	"io"
+	"json"
+	"os"
+	"strings"
 )
 
+const commitsPerPage = 20
+
 // A Package describes a package that is listed on the dashboard.
 type Package struct {
-	Name string
+	Name    string
-	Path string // (empty for the main Go tree)
+	Path    string // (empty for the main Go tree)
+	NextNum int    // Num of the next head Commit
 }
 
 func (p *Package) Key(c appengine.Context) *datastore.Key {
@@ -24,6 +35,15 @@ func (p *Package) Key(c appengine.Context) *datastore.Key {
 	return datastore.NewKey(c, "Package", key, 0, nil)
 }
 
+func GetPackage(c appengine.Context, path string) (*Package, os.Error) {
+	p := &Package{Path: path}
+	err := datastore.Get(c, p.Key(c), p)
+	if err == datastore.ErrNoSuchEntity {
+		return nil, fmt.Errorf("package %q not found", path)
+	}
+	return p, err
+}
+
 // A Commit describes an individual commit in a package.
 //
 // Each Commit entity is a descendant of its associated Package entity.
@@ -31,9 +51,9 @@ func (p *Package) Key(c appengine.Context) *datastore.Key {
 // datastore entity group.
 type Commit struct {
 	PackagePath string // (empty for Go commits)
-	Num         int    // Internal monotonic counter unique to this package.
 	Hash        string
 	ParentHash  string
+	Num         int // Internal monotonic counter unique to this package.
 
 	User string
 	Desc string `datastore:",noindex"`
@@ -47,8 +67,42 @@ type Commit struct {
 }
 
 func (com *Commit) Key(c appengine.Context) *datastore.Key {
-	key := com.PackagePath + ":" + com.Hash
+	if com.Hash == "" {
-	return datastore.NewKey(c, "Commit", key, 0, nil)
+		panic("tried Key on Commit with empty Hash")
+	}
+	p := Package{Path: com.PackagePath}
+	key := com.PackagePath + "|" + com.Hash
+	return datastore.NewKey(c, "Commit", key, 0, p.Key(c))
+}
+
+func (c *Commit) Valid() os.Error {
+	if !validHash(c.Hash) {
+		return os.NewError("invalid Hash")
+	}
+	if !validHash(c.ParentHash) {
+		return os.NewError("invalid ParentHash")
+	}
+	return nil
+}
+
+// AddResult adds the denormalized Reuslt data to the Commit's Result field.
+// It must be called from inside a datastore transaction.
+func (com *Commit) AddResult(c appengine.Context, r *Result) os.Error {
+	if err := datastore.Get(c, com.Key(c), com); err != nil {
+		return err
+	}
+	com.Result = append(com.Result, r.Data())
+	_, err := datastore.Put(c, com.Key(c), com)
+	return err
+}
+
+func (com *Commit) HasResult(builder string) bool {
+	for _, r := range com.Result {
+		if strings.SplitN(r, "|", 2)[0] == builder {
+			return true
+		}
+	}
+	return false
 }
 
 // A Result describes a build result for a Commit on an OS/architecture.
@@ -63,21 +117,50 @@ type Result struct {
 	GoHash string
 
 	OK      bool
-	Log     string `datastore:"-"`        // for JSON unmarshaling
+	Log     []byte `datastore:"-"`        // for JSON unmarshaling
 	LogHash string `datastore:",noindex"` // Key to the Log record.
 }
 
+func (r *Result) Key(c appengine.Context) *datastore.Key {
+	p := Package{Path: r.PackagePath}
+	key := r.Builder + "|" + r.PackagePath + "|" + r.Hash + "|" + r.GoHash
+	return datastore.NewKey(c, "Result", key, 0, p.Key(c))
+}
+
 func (r *Result) Data() string {
 	return fmt.Sprintf("%v|%v|%v|%v", r.Builder, r.OK, r.LogHash, r.GoHash)
 }
 
+func (r *Result) Valid() os.Error {
+	if !validHash(r.Hash) {
+		return os.NewError("invalid Hash")
+	}
+	if r.PackagePath != "" && !validHash(r.GoHash) {
+		return os.NewError("invalid GoHash")
+	}
+	return nil
+}
+
 // A Log is a gzip-compressed log file stored under the SHA1 hash of the
 // uncompressed log text.
 type Log struct {
 	CompressedLog []byte
 }
 
-// A Tag is used to keep track of the most recent weekly and release tags.
+func PutLog(c appengine.Context, text []byte) (hash string, err os.Error) {
+	h := sha1.New()
+	h.Write(text)
+	b := new(bytes.Buffer)
+	z, _ := gzip.NewWriterLevel(b, gzip.BestCompression)
+	z.Write(text)
+	z.Close()
+	hash = fmt.Sprintf("%x", h.Sum())
+	key := datastore.NewKey(c, "Log", hash, 0, nil)
+	_, err = datastore.Put(c, key, &Log{b.Bytes()})
+	return
+}
+
+// A Tag is used to keep track of the most recent Go weekly and release tags.
 // Typically there will be one Tag entity for each kind of hg tag.
 type Tag struct {
 	Kind string // "weekly", "release", or "tip"
@@ -86,7 +169,18 @@ type Tag struct {
 }
 
 func (t *Tag) Key(c appengine.Context) *datastore.Key {
-	return datastore.NewKey(c, "Tag", t.Kind, 0, nil)
+	p := &Package{Path: ""}
+	return datastore.NewKey(c, "Tag", t.Kind, 0, p.Key(c))
+}
+
+func (t *Tag) Valid() os.Error {
+	if t.Kind != "weekly" || t.Kind != "release" || t.Kind != "tip" {
+		return os.NewError("invalid Kind")
+	}
+	if !validHash(t.Hash) {
+		return os.NewError("invalid Hash")
+	}
+	return nil
 }
 
 // commitHandler records a new commit. It reads a JSON-encoded Commit value
@@ -94,16 +188,94 @@ func (t *Tag) Key(c appengine.Context) *datastore.Key {
 // commitHandler also updates the "tip" Tag for each new commit at tip.
 //
 // This handler is used by a gobuilder process in -commit mode.
-func commitHandler(w http.ResponseWriter, r *http.Request)
+func commitHandler(w http.ResponseWriter, r *http.Request) {
+	com := new(Commit)
+	defer r.Body.Close()
+	if err := json.NewDecoder(r.Body).Decode(com); err != nil {
+		logErr(w, r, err)
+		return
+	}
+	if err := com.Valid(); err != nil {
+		logErr(w, r, err)
+		return
+	}
+	tx := func(c appengine.Context) os.Error {
+		return addCommit(c, com)
+	}
+	c := appengine.NewContext(r)
+	if err := datastore.RunInTransaction(c, tx, nil); err != nil {
+		logErr(w, r, err)
+	}
+}
+
+// addCommit adds the Commit entity to the datastore and updates the tip Tag.
+// It must be run inside a datastore transaction.
+func addCommit(c appengine.Context, com *Commit) os.Error {
+	// if this commit is already in the datastore, do nothing
+	var tc Commit // temp value so we don't clobber com
+	err := datastore.Get(c, com.Key(c), &tc)
+	if err != datastore.ErrNoSuchEntity {
+		return err
+	}
+	// get the next commit number
+	p, err := GetPackage(c, com.PackagePath)
+	if err != nil {
+		return err
+	}
+	com.Num = p.NextNum
+	p.NextNum++
+	if _, err := datastore.Put(c, p.Key(c), p); err != nil {
+		return err
+	}
+	// if this isn't the first Commit test the parent commit exists
+	if com.Num > 0 {
+		n, err := datastore.NewQuery("Commit").
+			Filter("Hash =", com.ParentHash).
+			Ancestor(p.Key(c)).
+			Count(c)
+		if err != nil {
+			return err
+		}
+		if n == 0 {
+			return os.NewError("parent commit not found")
+		}
+	}
+	// update the tip Tag if this is the Go repo
+	if p.Path == "" {
+		t := &Tag{Kind: "tip", Hash: com.Hash}
+		if _, err = datastore.Put(c, t.Key(c), t); err != nil {
+			return err
+		}
+	}
+	// put the Commit
+	_, err = datastore.Put(c, com.Key(c), com)
+	return err
+}
 
 // tagHandler records a new tag. It reads a JSON-encoded Tag value from the
 // request body and updates the Tag entity for the Kind of tag provided.
 //
 // This handler is used by a gobuilder process in -commit mode.
-func tagHandler(w http.ResponseWriter, r *http.Request)
+func tagHandler(w http.ResponseWriter, r *http.Request) {
+	t := new(Tag)
+	defer r.Body.Close()
+	if err := json.NewDecoder(r.Body).Decode(t); err != nil {
+		logErr(w, r, err)
+		return
+	}
+	if err := t.Valid(); err != nil {
+		logErr(w, r, err)
+		return
+	}
+	c := appengine.NewContext(r)
+	if _, err := datastore.Put(c, t.Key(c), t); err != nil {
+		logErr(w, r, err)
+		return
+	}
+}
 
-// todoHandler returns a JSON-encoded string of the hash of the next of Commit
+// todoHandler returns the string of the hash of the next Commit to be built.
-// to be built. It expects a "builder" query parameter.
+// It expects a "builder" query parameter.
 //
 // By default it scans the first 20 Go Commits in Num-descending order and
 // returns the first one it finds that doesn't have a Result for this builder.
@@ -112,22 +284,145 @@ func tagHandler(w http.ResponseWriter, r *http.Request)
 // and scans the first 20 Commits in Num-descending order for the specified
 // packagePath and returns the first that doesn't have a Result for this builder
 // and goHash combination.
-func todoHandler(w http.ResponseWriter, r *http.Request)
+func todoHandler(w http.ResponseWriter, r *http.Request) {
+	builder := r.FormValue("builder")
+	goHash := r.FormValue("goHash")
+
+	c := appengine.NewContext(r)
+	p, err := GetPackage(c, r.FormValue("packagePath"))
+	if err != nil {
+		logErr(w, r, err)
+		return
+	}
+
+	q := datastore.NewQuery("Commit").
+		Ancestor(p.Key(c)).
+		Limit(commitsPerPage).
+		Order("-Num")
+	if goHash != "" && p.Path != "" {
+		q.Filter("GoHash =", goHash)
+	}
+	var nextHash string
+	for t := q.Run(c); ; {
+		com := new(Commit)
+		if _, err := t.Next(com); err == datastore.Done {
+			break
+		} else if err != nil {
+			logErr(w, r, err)
+			return
+		}
+		if !com.HasResult(builder) {
+			nextHash = com.Hash
+			break
+		}
+	}
+	fmt.Fprint(w, nextHash)
+}
 
 // resultHandler records a build result.
 // It reads a JSON-encoded Result value from the request body,
 // creates a new Result entity, and updates the relevant Commit entity.
 // If the Log field is not empty, resultHandler creates a new Log entity
 // and updates the LogHash field before putting the Commit entity.
-func resultHandler(w http.ResponseWriter, r *http.Request)
+func resultHandler(w http.ResponseWriter, r *http.Request) {
+	res := new(Result)
+	defer r.Body.Close()
+	if err := json.NewDecoder(r.Body).Decode(res); err != nil {
+		logErr(w, r, err)
+		return
+	}
+	if err := res.Valid(); err != nil {
+		logErr(w, r, err)
+		return
+	}
+	c := appengine.NewContext(r)
+	// store the Log text if supplied
+	if len(res.Log) > 0 {
+		hash, err := PutLog(c, res.Log)
+		if err != nil {
+			logErr(w, r, err)
+			return
+		}
+		res.LogHash = hash
+	}
+	tx := func(c appengine.Context) os.Error {
+		// check Package exists
+		if _, err := GetPackage(c, res.PackagePath); err != nil {
+			return err
+		}
+		// put Result
+		if _, err := datastore.Put(c, res.Key(c), res); err != nil {
+			return err
+		}
+		// add Result to Commit
+		com := &Commit{PackagePath: res.PackagePath, Hash: res.Hash}
+		return com.AddResult(c, res)
+	}
+	if err := datastore.RunInTransaction(c, tx, nil); err != nil {
+		logErr(w, r, err)
+	}
+}
+
+func logHandler(w http.ResponseWriter, r *http.Request) {
+	c := appengine.NewContext(r)
+	h := r.URL.Path[len("/log/"):]
+	k := datastore.NewKey(c, "Log", h, 0, nil)
+	l := new(Log)
+	if err := datastore.Get(c, k, l); err != nil {
+		logErr(w, r, err)
+		return
+	}
+	d, err := gzip.NewReader(bytes.NewBuffer(l.CompressedLog))
+	if err != nil {
+		logErr(w, r, err)
+		return
+	}
+	if _, err := io.Copy(w, d); err != nil {
+		logErr(w, r, err)
+	}
+}
 
 // AuthHandler wraps a http.HandlerFunc with a handler that validates the
 // supplied key and builder query parameters.
-func AuthHandler(http.HandlerFunc) http.HandlerFunc
+func AuthHandler(h http.HandlerFunc) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		// Put the URL Query values into r.Form to avoid parsing the
+		// request body when calling r.FormValue.
+		r.Form = r.URL.Query()
+
+		// Validate key query parameter.
+		key := r.FormValue("key")
+		if key != secretKey {
+			h := sha1.New()
+			h.Write([]byte(r.FormValue("builder") + secretKey))
+			if key != fmt.Sprintf("%x", h.Sum()) {
+				logErr(w, r, os.NewError("invalid key"))
+				return
+			}
+		}
+
+		h(w, r) // Call the original HandlerFunc.
+	}
+}
 
 func init() {
+	// authenticated handlers
 	http.HandleFunc("/commit", AuthHandler(commitHandler))
-	http.HandleFunc("/result", AuthHandler(commitHandler))
+	http.HandleFunc("/result", AuthHandler(resultHandler))
 	http.HandleFunc("/tag", AuthHandler(tagHandler))
 	http.HandleFunc("/todo", AuthHandler(todoHandler))
+
+	// public handlers
+	http.HandleFunc("/log/", logHandler)
+}
+
+func validHash(hash string) bool {
+	// TODO(adg): correctly validate a hash
+	return hash != ""
+}
+
+func logErr(w http.ResponseWriter, r *http.Request, err os.Error) {
+	appengine.NewContext(r).Errorf("Error: %v", err)
+	w.WriteHeader(http.StatusInternalServerError)
+	fmt.Fprint(w, "Error: ", err)
 }

misc/dashboard/app/build/key.go

@@ -0,0 +1,16 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package build
+
+import "appengine"
+
+// Delete this init function before deploying to production.
+func init() {
+	if !appengine.IsDevAppServer() {
+		panic("please read key.go")
+	}
+}
+
+const secretKey = "" // Important! Put a secret here before deploying!

misc/dashboard/app/build/test.go

@@ -0,0 +1,141 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package build
+
+// TODO(adg): test branches
+// TODO(adg): test non-Go packages
+// TODO(adg): test authentication
+
+import (
+	"appengine"
+	"appengine/datastore"
+	"bytes"
+	"fmt"
+	"http"
+	"http/httptest"
+	"io"
+	"json"
+	"os"
+	"url"
+)
+
+func init() {
+	http.HandleFunc("/buildtest", testHandler)
+}
+
+var testEntityKinds = []string{
+	"Package",
+	"Commit",
+	"Result",
+	"Log",
+}
+
+var testRequests = []struct {
+	path string
+	vals url.Values
+	req  interface{}
+	res  interface{}
+}{
+	{"/commit", nil, &Commit{Hash: "0001", ParentHash: "0000"}, nil},
+	{"/commit", nil, &Commit{Hash: "0002", ParentHash: "0001"}, nil},
+	{"/commit", nil, &Commit{Hash: "0003", ParentHash: "0002"}, nil},
+	{"/todo", url.Values{"builder": {"linux-386"}}, nil, "0003"},
+	{"/todo", url.Values{"builder": {"linux-amd64"}}, nil, "0003"},
+
+	{"/result", nil, &Result{Builder: "linux-386", Hash: "0001", OK: true}, nil},
+	{"/todo", url.Values{"builder": {"linux-386"}}, nil, "0003"},
+
+	{"/result", nil, &Result{Builder: "linux-386", Hash: "0002", OK: false, Log: []byte("test")}, nil},
+	{"/todo", url.Values{"builder": {"linux-386"}}, nil, "0003"},
+	{"/log/a94a8fe5ccb19ba61c4c0873d391e987982fbbd3", nil, nil, "test"},
+
+	{"/result", nil, &Result{Builder: "linux-amd64", Hash: "0003", OK: true}, nil},
+	{"/todo", url.Values{"builder": {"linux-386"}}, nil, "0003"},
+	{"/todo", url.Values{"builder": {"linux-amd64"}}, nil, "0002"},
+}
+
+var testPackages = []*Package{
+	&Package{Name: "Go", Path: ""},
+	&Package{Name: "Other", Path: "code.google.com/p/go.other"},
+}
+
+func testHandler(w http.ResponseWriter, r *http.Request) {
+	if !appengine.IsDevAppServer() {
+		fmt.Fprint(w, "These tests must be run under the dev_appserver.")
+		return
+	}
+	c := appengine.NewContext(r)
+	if err := nukeEntities(c, testEntityKinds); err != nil {
+		logErr(w, r, err)
+		return
+	}
+
+	for _, p := range testPackages {
+		if _, err := datastore.Put(c, p.Key(c), p); err != nil {
+			logErr(w, r, err)
+			return
+		}
+	}
+
+	failed := false
+	for i, t := range testRequests {
+		errorf := func(format string, args ...interface{}) {
+			fmt.Fprintf(w, "%d %s: ", i, t.path)
+			fmt.Fprintf(w, format, args...)
+			fmt.Fprintln(w)
+			failed = true
+		}
+		var body io.ReadWriter
+		if t.req != nil {
+			body = new(bytes.Buffer)
+			json.NewEncoder(body).Encode(t.req)
+		}
+		url := "http://" + appengine.DefaultVersionHostname(c) + t.path
+		if t.vals != nil {
+			url += "?" + t.vals.Encode()
+		}
+		req, err := http.NewRequest("POST", url, body)
+		if err != nil {
+			logErr(w, r, err)
+			return
+		}
+		req.Header = r.Header
+		rec := httptest.NewRecorder()
+		http.DefaultServeMux.ServeHTTP(rec, req)
+		if rec.Code != 0 && rec.Code != 200 {
+			errorf(rec.Body.String())
+		}
+		if e, ok := t.res.(string); ok {
+			g := rec.Body.String()
+			if g != e {
+				errorf("body mismatch: got %q want %q", g, e)
+			}
+		}
+	}
+	if !failed {
+		fmt.Fprint(w, "PASS")
+	}
+}
+
+func nukeEntities(c appengine.Context, kinds []string) os.Error {
+	if !appengine.IsDevAppServer() {
+		return os.NewError("can't nuke production data")
+	}
+	var keys []*datastore.Key
+	for _, kind := range kinds {
+		q := datastore.NewQuery(kind).KeysOnly()
+		for t := q.Run(c); ; {
+			k, err := t.Next(nil)
+			if err == datastore.Done {
+				break
+			}
+			if err != nil {
+				return err
+			}
+			keys = append(keys, k)
+		}
+	}
+	return datastore.DeleteMulti(c, keys)
+}