// Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. // +build appengine package build import ( "crypto/hmac" "crypto/md5" "encoding/json" "errors" "fmt" "net/http" "strings" "appengine" "appengine/datastore" "cache" ) const commitsPerPage = 30 // commitHandler retrieves commit data or records a new commit. // // For GET requests it returns a Commit value for the specified // packagePath and hash. // // For POST requests it reads a JSON-encoded Commit value from the request // body and creates a new Commit entity. It also updates the "tip" Tag for // each new commit at tip. // // This handler is used by a gobuilder process in -commit mode. func commitHandler(r *http.Request) (interface{}, error) { c := contextForRequest(r) com := new(Commit) if r.Method == "GET" { com.PackagePath = r.FormValue("packagePath") com.Hash = r.FormValue("hash") if err := datastore.Get(c, com.Key(c), com); err != nil { return nil, fmt.Errorf("getting Commit: %v", err) } return com, nil } if r.Method != "POST" { return nil, errBadMethod(r.Method) } if !isMasterKey(c, r.FormValue("key")) { return nil, errors.New("can only POST commits with master key") } // POST request defer r.Body.Close() if err := json.NewDecoder(r.Body).Decode(com); err != nil { return nil, fmt.Errorf("decoding Body: %v", err) } if len(com.Desc) > maxDatastoreStringLen { com.Desc = com.Desc[:maxDatastoreStringLen] } if err := com.Valid(); err != nil { return nil, fmt.Errorf("validating Commit: %v", err) } defer cache.Tick(c) tx := func(c appengine.Context) error { return addCommit(c, com) } return nil, datastore.RunInTransaction(c, tx, nil) } // addCommit adds the Commit entity to the datastore and updates the tip Tag. // It must be run inside a datastore transaction. func addCommit(c appengine.Context, com *Commit) error { var tc Commit // temp value so we don't clobber com err := datastore.Get(c, com.Key(c), &tc) if err != datastore.ErrNoSuchEntity { // if this commit is already in the datastore, do nothing if err == nil { return nil } return fmt.Errorf("getting Commit: %v", err) } // get the next commit number p, err := GetPackage(c, com.PackagePath) if err != nil { return fmt.Errorf("GetPackage: %v", err) } com.Num = p.NextNum p.NextNum++ if _, err := datastore.Put(c, p.Key(c), p); err != nil { return fmt.Errorf("putting Package: %v", err) } // if this isn't the first Commit test the parent commit exists if com.Num > 0 { n, err := datastore.NewQuery("Commit"). Filter("Hash =", com.ParentHash). Ancestor(p.Key(c)). Count(c) if err != nil { return fmt.Errorf("testing for parent Commit: %v", err) } if n == 0 { return errors.New("parent commit not found") } } // update the tip Tag if this is the Go repo and this isn't on a release branch if p.Path == "" && !strings.HasPrefix(com.Desc, "[release-branch") { t := &Tag{Kind: "tip", Hash: com.Hash} if _, err = datastore.Put(c, t.Key(c), t); err != nil { return fmt.Errorf("putting Tag: %v", err) } } // put the Commit if _, err = datastore.Put(c, com.Key(c), com); err != nil { return fmt.Errorf("putting Commit: %v", err) } return nil } // tagHandler records a new tag. It reads a JSON-encoded Tag value from the // request body and updates the Tag entity for the Kind of tag provided. // // This handler is used by a gobuilder process in -commit mode. func tagHandler(r *http.Request) (interface{}, error) { if r.Method != "POST" { return nil, errBadMethod(r.Method) } t := new(Tag) defer r.Body.Close() if err := json.NewDecoder(r.Body).Decode(t); err != nil { return nil, err } if err := t.Valid(); err != nil { return nil, err } c := contextForRequest(r) defer cache.Tick(c) _, err := datastore.Put(c, t.Key(c), t) return nil, err } // Todo is a todoHandler response. type Todo struct { Kind string // "build-go-commit" or "build-package" Data interface{} } // todoHandler returns the next action to be performed by a builder. // It expects "builder" and "kind" query parameters and returns a *Todo value. // Multiple "kind" parameters may be specified. func todoHandler(r *http.Request) (interface{}, error) { c := contextForRequest(r) now := cache.Now(c) key := "build-todo-" + r.Form.Encode() var todo *Todo if cache.Get(r, now, key, &todo) { return todo, nil } var err error builder := r.FormValue("builder") for _, kind := range r.Form["kind"] { var com *Commit switch kind { case "build-go-commit": com, err = buildTodo(c, builder, "", "") case "build-package": packagePath := r.FormValue("packagePath") goHash := r.FormValue("goHash") com, err = buildTodo(c, builder, packagePath, goHash) } if com != nil || err != nil { if com != nil { // ResultData can be large and not needed on builder. com.ResultData = []string{} } todo = &Todo{Kind: kind, Data: com} break } } if err == nil { cache.Set(r, now, key, todo) } return todo, err } // buildTodo returns the next Commit to be built (or nil if none available). // // If packagePath and goHash are empty, it scans the first 20 Go Commits in // Num-descending order and returns the first one it finds that doesn't have a // Result for this builder. // // If provided with non-empty packagePath and goHash args, it scans the first // 20 Commits in Num-descending order for the specified packagePath and // returns the first that doesn't have a Result for this builder and goHash. func buildTodo(c appengine.Context, builder, packagePath, goHash string) (*Commit, error) { p, err := GetPackage(c, packagePath) if err != nil { return nil, err } t := datastore.NewQuery("Commit"). Ancestor(p.Key(c)). Limit(commitsPerPage). Order("-Num"). Run(c) for { com := new(Commit) if _, err := t.Next(com); err == datastore.Done { break } else if err != nil { return nil, err } if com.Result(builder, goHash) == nil { return com, nil } } // Nothing left to do if this is a package (not the Go tree). if packagePath != "" { return nil, nil } // If there are no Go tree commits left to build, // see if there are any subrepo commits that need to be built at tip. // If so, ask the builder to build a go tree at the tip commit. // TODO(adg): do the same for "weekly" and "release" tags. tag, err := GetTag(c, "tip") if err != nil { return nil, err } // Check that this Go commit builds OK for this builder. // If not, don't re-build as the subrepos will never get built anyway. com, err := tag.Commit(c) if err != nil { return nil, err } if r := com.Result(builder, ""); r != nil && !r.OK { return nil, nil } pkgs, err := Packages(c, "subrepo") if err != nil { return nil, err } for _, pkg := range pkgs { com, err := pkg.LastCommit(c) if err != nil { c.Warningf("%v: no Commit found: %v", pkg, err) continue } if com.Result(builder, tag.Hash) == nil { return tag.Commit(c) } } return nil, nil } // packagesHandler returns a list of the non-Go Packages monitored // by the dashboard. func packagesHandler(r *http.Request) (interface{}, error) { kind := r.FormValue("kind") c := contextForRequest(r) now := cache.Now(c) key := "build-packages-" + kind var p []*Package if cache.Get(r, now, key, &p) { return p, nil } p, err := Packages(c, kind) if err != nil { return nil, err } cache.Set(r, now, key, p) return p, nil } // resultHandler records a build result. // It reads a JSON-encoded Result value from the request body, // creates a new Result entity, and updates the relevant Commit entity. // If the Log field is not empty, resultHandler creates a new Log entity // and updates the LogHash field before putting the Commit entity. func resultHandler(r *http.Request) (interface{}, error) { if r.Method != "POST" { return nil, errBadMethod(r.Method) } c := contextForRequest(r) res := new(Result) defer r.Body.Close() if err := json.NewDecoder(r.Body).Decode(res); err != nil { return nil, fmt.Errorf("decoding Body: %v", err) } if err := res.Valid(); err != nil { return nil, fmt.Errorf("validating Result: %v", err) } defer cache.Tick(c) // store the Log text if supplied if len(res.Log) > 0 { hash, err := PutLog(c, res.Log) if err != nil { return nil, fmt.Errorf("putting Log: %v", err) } res.LogHash = hash } tx := func(c appengine.Context) error { // check Package exists if _, err := GetPackage(c, res.PackagePath); err != nil { return fmt.Errorf("GetPackage: %v", err) } // put Result if _, err := datastore.Put(c, res.Key(c), res); err != nil { return fmt.Errorf("putting Result: %v", err) } // add Result to Commit com := &Commit{PackagePath: res.PackagePath, Hash: res.Hash} if err := com.AddResult(c, res); err != nil { return fmt.Errorf("AddResult: %v", err) } // Send build failure notifications, if necessary. // Note this must run after the call AddResult, which // populates the Commit's ResultData field. return notifyOnFailure(c, com, res.Builder) } return nil, datastore.RunInTransaction(c, tx, nil) } // logHandler displays log text for a given hash. // It handles paths like "/log/hash". func logHandler(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-type", "text/plain; charset=utf-8") c := contextForRequest(r) hash := r.URL.Path[len("/log/"):] key := datastore.NewKey(c, "Log", hash, 0, nil) l := new(Log) if err := datastore.Get(c, key, l); err != nil { logErr(w, r, err) return } b, err := l.Text() if err != nil { logErr(w, r, err) return } w.Write(b) } type dashHandler func(*http.Request) (interface{}, error) type dashResponse struct { Response interface{} Error string } // errBadMethod is returned by a dashHandler when // the request has an unsuitable method. type errBadMethod string func (e errBadMethod) Error() string { return "bad method: " + string(e) } // AuthHandler wraps a http.HandlerFunc with a handler that validates the // supplied key and builder query parameters. func AuthHandler(h dashHandler) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { c := contextForRequest(r) // Put the URL Query values into r.Form to avoid parsing the // request body when calling r.FormValue. r.Form = r.URL.Query() var err error var resp interface{} // Validate key query parameter for POST requests only. key := r.FormValue("key") builder := r.FormValue("builder") if r.Method == "POST" && !validKey(c, key, builder) { err = errors.New("invalid key: " + key) } // Call the original HandlerFunc and return the response. if err == nil { resp, err = h(r) } // Write JSON response. dashResp := &dashResponse{Response: resp} if err != nil { c.Errorf("%v", err) dashResp.Error = err.Error() } w.Header().Set("Content-Type", "application/json") if err = json.NewEncoder(w).Encode(dashResp); err != nil { c.Criticalf("encoding response: %v", err) } } } func keyHandler(w http.ResponseWriter, r *http.Request) { builder := r.FormValue("builder") if builder == "" { logErr(w, r, errors.New("must supply builder in query string")) return } c := contextForRequest(r) fmt.Fprint(w, builderKey(c, builder)) } func init() { for _, d := range dashboards { // admin handlers http.HandleFunc(d.RelPath+"init", initHandler) http.HandleFunc(d.RelPath+"key", keyHandler) // authenticated handlers http.HandleFunc(d.RelPath+"commit", AuthHandler(commitHandler)) http.HandleFunc(d.RelPath+"packages", AuthHandler(packagesHandler)) http.HandleFunc(d.RelPath+"result", AuthHandler(resultHandler)) http.HandleFunc(d.RelPath+"tag", AuthHandler(tagHandler)) http.HandleFunc(d.RelPath+"todo", AuthHandler(todoHandler)) // public handlers http.HandleFunc(d.RelPath+"log/", logHandler) } } func validHash(hash string) bool { // TODO(adg): correctly validate a hash return hash != "" } func validKey(c appengine.Context, key, builder string) bool { return isMasterKey(c, key) || key == builderKey(c, builder) } func isMasterKey(c appengine.Context, key string) bool { return appengine.IsDevAppServer() || key == secretKey(c) } func builderKey(c appengine.Context, builder string) string { h := hmac.New(md5.New, []byte(secretKey(c))) h.Write([]byte(builder)) return fmt.Sprintf("%x", h.Sum(nil)) } func logErr(w http.ResponseWriter, r *http.Request, err error) { contextForRequest(r).Errorf("Error: %v", err) w.WriteHeader(http.StatusInternalServerError) fmt.Fprint(w, "Error: ", err) } func contextForRequest(r *http.Request) appengine.Context { return dashboardForRequest(r).Context(appengine.NewContext(r)) }