diff --git a/misc/dashboard/app/app.yaml b/misc/dashboard/app/app.yaml index 695c04e78a0..ef101b5e9b0 100644 --- a/misc/dashboard/app/app.yaml +++ b/misc/dashboard/app/app.yaml @@ -4,5 +4,10 @@ runtime: go api_version: 3 handlers: +- url: /log/.+ + script: _go_app - url: /(commit|tag|todo|result) script: _go_app +- url: /buildtest + script: _go_app + login: admin diff --git a/misc/dashboard/app/build/build.go b/misc/dashboard/app/build/build.go index 138a86bc5ed..fa415f9334a 100644 --- a/misc/dashboard/app/build/build.go +++ b/misc/dashboard/app/build/build.go @@ -7,13 +7,24 @@ package build import ( "appengine" "appengine/datastore" + "bytes" + "compress/gzip" + "crypto/sha1" + "fmt" "http" + "io" + "json" + "os" + "strings" ) +const commitsPerPage = 20 + // A Package describes a package that is listed on the dashboard. type Package struct { - Name string - Path string // (empty for the main Go tree) + Name string + Path string // (empty for the main Go tree) + NextNum int // Num of the next head Commit } func (p *Package) Key(c appengine.Context) *datastore.Key { @@ -24,6 +35,15 @@ func (p *Package) Key(c appengine.Context) *datastore.Key { return datastore.NewKey(c, "Package", key, 0, nil) } +func GetPackage(c appengine.Context, path string) (*Package, os.Error) { + p := &Package{Path: path} + err := datastore.Get(c, p.Key(c), p) + if err == datastore.ErrNoSuchEntity { + return nil, fmt.Errorf("package %q not found", path) + } + return p, err +} + // A Commit describes an individual commit in a package. // // Each Commit entity is a descendant of its associated Package entity. @@ -31,9 +51,9 @@ func (p *Package) Key(c appengine.Context) *datastore.Key { // datastore entity group. type Commit struct { PackagePath string // (empty for Go commits) - Num int // Internal monotonic counter unique to this package. Hash string ParentHash string + Num int // Internal monotonic counter unique to this package. User string Desc string `datastore:",noindex"` @@ -47,8 +67,42 @@ type Commit struct { } func (com *Commit) Key(c appengine.Context) *datastore.Key { - key := com.PackagePath + ":" + com.Hash - return datastore.NewKey(c, "Commit", key, 0, nil) + if com.Hash == "" { + panic("tried Key on Commit with empty Hash") + } + p := Package{Path: com.PackagePath} + key := com.PackagePath + "|" + com.Hash + return datastore.NewKey(c, "Commit", key, 0, p.Key(c)) +} + +func (c *Commit) Valid() os.Error { + if !validHash(c.Hash) { + return os.NewError("invalid Hash") + } + if !validHash(c.ParentHash) { + return os.NewError("invalid ParentHash") + } + return nil +} + +// AddResult adds the denormalized Reuslt data to the Commit's Result field. +// It must be called from inside a datastore transaction. +func (com *Commit) AddResult(c appengine.Context, r *Result) os.Error { + if err := datastore.Get(c, com.Key(c), com); err != nil { + return err + } + com.Result = append(com.Result, r.Data()) + _, err := datastore.Put(c, com.Key(c), com) + return err +} + +func (com *Commit) HasResult(builder string) bool { + for _, r := range com.Result { + if strings.SplitN(r, "|", 2)[0] == builder { + return true + } + } + return false } // A Result describes a build result for a Commit on an OS/architecture. @@ -63,21 +117,50 @@ type Result struct { GoHash string OK bool - Log string `datastore:"-"` // for JSON unmarshaling + Log []byte `datastore:"-"` // for JSON unmarshaling LogHash string `datastore:",noindex"` // Key to the Log record. } +func (r *Result) Key(c appengine.Context) *datastore.Key { + p := Package{Path: r.PackagePath} + key := r.Builder + "|" + r.PackagePath + "|" + r.Hash + "|" + r.GoHash + return datastore.NewKey(c, "Result", key, 0, p.Key(c)) +} + func (r *Result) Data() string { return fmt.Sprintf("%v|%v|%v|%v", r.Builder, r.OK, r.LogHash, r.GoHash) } +func (r *Result) Valid() os.Error { + if !validHash(r.Hash) { + return os.NewError("invalid Hash") + } + if r.PackagePath != "" && !validHash(r.GoHash) { + return os.NewError("invalid GoHash") + } + return nil +} + // A Log is a gzip-compressed log file stored under the SHA1 hash of the // uncompressed log text. type Log struct { CompressedLog []byte } -// A Tag is used to keep track of the most recent weekly and release tags. +func PutLog(c appengine.Context, text []byte) (hash string, err os.Error) { + h := sha1.New() + h.Write(text) + b := new(bytes.Buffer) + z, _ := gzip.NewWriterLevel(b, gzip.BestCompression) + z.Write(text) + z.Close() + hash = fmt.Sprintf("%x", h.Sum()) + key := datastore.NewKey(c, "Log", hash, 0, nil) + _, err = datastore.Put(c, key, &Log{b.Bytes()}) + return +} + +// A Tag is used to keep track of the most recent Go weekly and release tags. // Typically there will be one Tag entity for each kind of hg tag. type Tag struct { Kind string // "weekly", "release", or "tip" @@ -86,7 +169,18 @@ type Tag struct { } func (t *Tag) Key(c appengine.Context) *datastore.Key { - return datastore.NewKey(c, "Tag", t.Kind, 0, nil) + p := &Package{Path: ""} + return datastore.NewKey(c, "Tag", t.Kind, 0, p.Key(c)) +} + +func (t *Tag) Valid() os.Error { + if t.Kind != "weekly" || t.Kind != "release" || t.Kind != "tip" { + return os.NewError("invalid Kind") + } + if !validHash(t.Hash) { + return os.NewError("invalid Hash") + } + return nil } // commitHandler records a new commit. It reads a JSON-encoded Commit value @@ -94,16 +188,94 @@ func (t *Tag) Key(c appengine.Context) *datastore.Key { // commitHandler also updates the "tip" Tag for each new commit at tip. // // This handler is used by a gobuilder process in -commit mode. -func commitHandler(w http.ResponseWriter, r *http.Request) +func commitHandler(w http.ResponseWriter, r *http.Request) { + com := new(Commit) + defer r.Body.Close() + if err := json.NewDecoder(r.Body).Decode(com); err != nil { + logErr(w, r, err) + return + } + if err := com.Valid(); err != nil { + logErr(w, r, err) + return + } + tx := func(c appengine.Context) os.Error { + return addCommit(c, com) + } + c := appengine.NewContext(r) + if err := datastore.RunInTransaction(c, tx, nil); err != nil { + logErr(w, r, err) + } +} + +// addCommit adds the Commit entity to the datastore and updates the tip Tag. +// It must be run inside a datastore transaction. +func addCommit(c appengine.Context, com *Commit) os.Error { + // if this commit is already in the datastore, do nothing + var tc Commit // temp value so we don't clobber com + err := datastore.Get(c, com.Key(c), &tc) + if err != datastore.ErrNoSuchEntity { + return err + } + // get the next commit number + p, err := GetPackage(c, com.PackagePath) + if err != nil { + return err + } + com.Num = p.NextNum + p.NextNum++ + if _, err := datastore.Put(c, p.Key(c), p); err != nil { + return err + } + // if this isn't the first Commit test the parent commit exists + if com.Num > 0 { + n, err := datastore.NewQuery("Commit"). + Filter("Hash =", com.ParentHash). + Ancestor(p.Key(c)). + Count(c) + if err != nil { + return err + } + if n == 0 { + return os.NewError("parent commit not found") + } + } + // update the tip Tag if this is the Go repo + if p.Path == "" { + t := &Tag{Kind: "tip", Hash: com.Hash} + if _, err = datastore.Put(c, t.Key(c), t); err != nil { + return err + } + } + // put the Commit + _, err = datastore.Put(c, com.Key(c), com) + return err +} // tagHandler records a new tag. It reads a JSON-encoded Tag value from the // request body and updates the Tag entity for the Kind of tag provided. // // This handler is used by a gobuilder process in -commit mode. -func tagHandler(w http.ResponseWriter, r *http.Request) +func tagHandler(w http.ResponseWriter, r *http.Request) { + t := new(Tag) + defer r.Body.Close() + if err := json.NewDecoder(r.Body).Decode(t); err != nil { + logErr(w, r, err) + return + } + if err := t.Valid(); err != nil { + logErr(w, r, err) + return + } + c := appengine.NewContext(r) + if _, err := datastore.Put(c, t.Key(c), t); err != nil { + logErr(w, r, err) + return + } +} -// todoHandler returns a JSON-encoded string of the hash of the next of Commit -// to be built. It expects a "builder" query parameter. +// todoHandler returns the string of the hash of the next Commit to be built. +// It expects a "builder" query parameter. // // By default it scans the first 20 Go Commits in Num-descending order and // returns the first one it finds that doesn't have a Result for this builder. @@ -112,22 +284,145 @@ func tagHandler(w http.ResponseWriter, r *http.Request) // and scans the first 20 Commits in Num-descending order for the specified // packagePath and returns the first that doesn't have a Result for this builder // and goHash combination. -func todoHandler(w http.ResponseWriter, r *http.Request) +func todoHandler(w http.ResponseWriter, r *http.Request) { + builder := r.FormValue("builder") + goHash := r.FormValue("goHash") + + c := appengine.NewContext(r) + p, err := GetPackage(c, r.FormValue("packagePath")) + if err != nil { + logErr(w, r, err) + return + } + + q := datastore.NewQuery("Commit"). + Ancestor(p.Key(c)). + Limit(commitsPerPage). + Order("-Num") + if goHash != "" && p.Path != "" { + q.Filter("GoHash =", goHash) + } + var nextHash string + for t := q.Run(c); ; { + com := new(Commit) + if _, err := t.Next(com); err == datastore.Done { + break + } else if err != nil { + logErr(w, r, err) + return + } + if !com.HasResult(builder) { + nextHash = com.Hash + break + } + } + fmt.Fprint(w, nextHash) +} // resultHandler records a build result. // It reads a JSON-encoded Result value from the request body, // creates a new Result entity, and updates the relevant Commit entity. // If the Log field is not empty, resultHandler creates a new Log entity // and updates the LogHash field before putting the Commit entity. -func resultHandler(w http.ResponseWriter, r *http.Request) +func resultHandler(w http.ResponseWriter, r *http.Request) { + res := new(Result) + defer r.Body.Close() + if err := json.NewDecoder(r.Body).Decode(res); err != nil { + logErr(w, r, err) + return + } + if err := res.Valid(); err != nil { + logErr(w, r, err) + return + } + c := appengine.NewContext(r) + // store the Log text if supplied + if len(res.Log) > 0 { + hash, err := PutLog(c, res.Log) + if err != nil { + logErr(w, r, err) + return + } + res.LogHash = hash + } + tx := func(c appengine.Context) os.Error { + // check Package exists + if _, err := GetPackage(c, res.PackagePath); err != nil { + return err + } + // put Result + if _, err := datastore.Put(c, res.Key(c), res); err != nil { + return err + } + // add Result to Commit + com := &Commit{PackagePath: res.PackagePath, Hash: res.Hash} + return com.AddResult(c, res) + } + if err := datastore.RunInTransaction(c, tx, nil); err != nil { + logErr(w, r, err) + } +} + +func logHandler(w http.ResponseWriter, r *http.Request) { + c := appengine.NewContext(r) + h := r.URL.Path[len("/log/"):] + k := datastore.NewKey(c, "Log", h, 0, nil) + l := new(Log) + if err := datastore.Get(c, k, l); err != nil { + logErr(w, r, err) + return + } + d, err := gzip.NewReader(bytes.NewBuffer(l.CompressedLog)) + if err != nil { + logErr(w, r, err) + return + } + if _, err := io.Copy(w, d); err != nil { + logErr(w, r, err) + } +} // AuthHandler wraps a http.HandlerFunc with a handler that validates the // supplied key and builder query parameters. -func AuthHandler(http.HandlerFunc) http.HandlerFunc +func AuthHandler(h http.HandlerFunc) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + // Put the URL Query values into r.Form to avoid parsing the + // request body when calling r.FormValue. + r.Form = r.URL.Query() + + // Validate key query parameter. + key := r.FormValue("key") + if key != secretKey { + h := sha1.New() + h.Write([]byte(r.FormValue("builder") + secretKey)) + if key != fmt.Sprintf("%x", h.Sum()) { + logErr(w, r, os.NewError("invalid key")) + return + } + } + + h(w, r) // Call the original HandlerFunc. + } +} func init() { + // authenticated handlers http.HandleFunc("/commit", AuthHandler(commitHandler)) - http.HandleFunc("/result", AuthHandler(commitHandler)) + http.HandleFunc("/result", AuthHandler(resultHandler)) http.HandleFunc("/tag", AuthHandler(tagHandler)) http.HandleFunc("/todo", AuthHandler(todoHandler)) + + // public handlers + http.HandleFunc("/log/", logHandler) +} + +func validHash(hash string) bool { + // TODO(adg): correctly validate a hash + return hash != "" +} + +func logErr(w http.ResponseWriter, r *http.Request, err os.Error) { + appengine.NewContext(r).Errorf("Error: %v", err) + w.WriteHeader(http.StatusInternalServerError) + fmt.Fprint(w, "Error: ", err) } diff --git a/misc/dashboard/app/build/key.go b/misc/dashboard/app/build/key.go new file mode 100644 index 00000000000..d19902a554a --- /dev/null +++ b/misc/dashboard/app/build/key.go @@ -0,0 +1,16 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package build + +import "appengine" + +// Delete this init function before deploying to production. +func init() { + if !appengine.IsDevAppServer() { + panic("please read key.go") + } +} + +const secretKey = "" // Important! Put a secret here before deploying! diff --git a/misc/dashboard/app/build/test.go b/misc/dashboard/app/build/test.go new file mode 100644 index 00000000000..83df0529af2 --- /dev/null +++ b/misc/dashboard/app/build/test.go @@ -0,0 +1,141 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package build + +// TODO(adg): test branches +// TODO(adg): test non-Go packages +// TODO(adg): test authentication + +import ( + "appengine" + "appengine/datastore" + "bytes" + "fmt" + "http" + "http/httptest" + "io" + "json" + "os" + "url" +) + +func init() { + http.HandleFunc("/buildtest", testHandler) +} + +var testEntityKinds = []string{ + "Package", + "Commit", + "Result", + "Log", +} + +var testRequests = []struct { + path string + vals url.Values + req interface{} + res interface{} +}{ + {"/commit", nil, &Commit{Hash: "0001", ParentHash: "0000"}, nil}, + {"/commit", nil, &Commit{Hash: "0002", ParentHash: "0001"}, nil}, + {"/commit", nil, &Commit{Hash: "0003", ParentHash: "0002"}, nil}, + {"/todo", url.Values{"builder": {"linux-386"}}, nil, "0003"}, + {"/todo", url.Values{"builder": {"linux-amd64"}}, nil, "0003"}, + + {"/result", nil, &Result{Builder: "linux-386", Hash: "0001", OK: true}, nil}, + {"/todo", url.Values{"builder": {"linux-386"}}, nil, "0003"}, + + {"/result", nil, &Result{Builder: "linux-386", Hash: "0002", OK: false, Log: []byte("test")}, nil}, + {"/todo", url.Values{"builder": {"linux-386"}}, nil, "0003"}, + {"/log/a94a8fe5ccb19ba61c4c0873d391e987982fbbd3", nil, nil, "test"}, + + {"/result", nil, &Result{Builder: "linux-amd64", Hash: "0003", OK: true}, nil}, + {"/todo", url.Values{"builder": {"linux-386"}}, nil, "0003"}, + {"/todo", url.Values{"builder": {"linux-amd64"}}, nil, "0002"}, +} + +var testPackages = []*Package{ + &Package{Name: "Go", Path: ""}, + &Package{Name: "Other", Path: "code.google.com/p/go.other"}, +} + +func testHandler(w http.ResponseWriter, r *http.Request) { + if !appengine.IsDevAppServer() { + fmt.Fprint(w, "These tests must be run under the dev_appserver.") + return + } + c := appengine.NewContext(r) + if err := nukeEntities(c, testEntityKinds); err != nil { + logErr(w, r, err) + return + } + + for _, p := range testPackages { + if _, err := datastore.Put(c, p.Key(c), p); err != nil { + logErr(w, r, err) + return + } + } + + failed := false + for i, t := range testRequests { + errorf := func(format string, args ...interface{}) { + fmt.Fprintf(w, "%d %s: ", i, t.path) + fmt.Fprintf(w, format, args...) + fmt.Fprintln(w) + failed = true + } + var body io.ReadWriter + if t.req != nil { + body = new(bytes.Buffer) + json.NewEncoder(body).Encode(t.req) + } + url := "http://" + appengine.DefaultVersionHostname(c) + t.path + if t.vals != nil { + url += "?" + t.vals.Encode() + } + req, err := http.NewRequest("POST", url, body) + if err != nil { + logErr(w, r, err) + return + } + req.Header = r.Header + rec := httptest.NewRecorder() + http.DefaultServeMux.ServeHTTP(rec, req) + if rec.Code != 0 && rec.Code != 200 { + errorf(rec.Body.String()) + } + if e, ok := t.res.(string); ok { + g := rec.Body.String() + if g != e { + errorf("body mismatch: got %q want %q", g, e) + } + } + } + if !failed { + fmt.Fprint(w, "PASS") + } +} + +func nukeEntities(c appengine.Context, kinds []string) os.Error { + if !appengine.IsDevAppServer() { + return os.NewError("can't nuke production data") + } + var keys []*datastore.Key + for _, kind := range kinds { + q := datastore.NewQuery(kind).KeysOnly() + for t := q.Run(c); ; { + k, err := t.Next(nil) + if err == datastore.Done { + break + } + if err != nil { + return err + } + keys = append(keys, k) + } + } + return datastore.DeleteMulti(c, keys) +}