// Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package build import ( "appengine" "appengine/datastore" "crypto/hmac" "fmt" "http" "json" "os" ) const commitsPerPage = 30 // defaultPackages specifies the Package records to be created by initHandler. var defaultPackages = []*Package{ &Package{Name: "Go"}, } // commitHandler retrieves commit data or records a new commit. // // For GET requests it returns a Commit value for the specified // packagePath and hash. // // For POST requests it reads a JSON-encoded Commit value from the request // body and creates a new Commit entity. It also updates the "tip" Tag for // each new commit at tip. // // This handler is used by a gobuilder process in -commit mode. func commitHandler(r *http.Request) (interface{}, os.Error) { c := appengine.NewContext(r) com := new(Commit) if r.Method == "GET" { com.PackagePath = r.FormValue("packagePath") com.Hash = r.FormValue("hash") if err := datastore.Get(c, com.Key(c), com); err != nil { return nil, fmt.Errorf("getting Commit: %v", err) } return com, nil } if r.Method != "POST" { return nil, errBadMethod(r.Method) } // POST request defer r.Body.Close() if err := json.NewDecoder(r.Body).Decode(com); err != nil { return nil, fmt.Errorf("decoding Body: %v", err) } if len(com.Desc) > maxDatastoreStringLen { com.Desc = com.Desc[:maxDatastoreStringLen] } if err := com.Valid(); err != nil { return nil, fmt.Errorf("validating Commit: %v", err) } defer invalidateCache(c) tx := func(c appengine.Context) os.Error { return addCommit(c, com) } return nil, datastore.RunInTransaction(c, tx, nil) } // addCommit adds the Commit entity to the datastore and updates the tip Tag. // It must be run inside a datastore transaction. func addCommit(c appengine.Context, com *Commit) os.Error { var tc Commit // temp value so we don't clobber com err := datastore.Get(c, com.Key(c), &tc) if err != datastore.ErrNoSuchEntity { // if this commit is already in the datastore, do nothing if err == nil { return nil } return fmt.Errorf("getting Commit: %v", err) } // get the next commit number p, err := GetPackage(c, com.PackagePath) if err != nil { return fmt.Errorf("GetPackage: %v", err) } com.Num = p.NextNum p.NextNum++ if _, err := datastore.Put(c, p.Key(c), p); err != nil { return fmt.Errorf("putting Package: %v", err) } // if this isn't the first Commit test the parent commit exists if com.Num > 0 { n, err := datastore.NewQuery("Commit"). Filter("Hash =", com.ParentHash). Ancestor(p.Key(c)). Count(c) if err != nil { return fmt.Errorf("testing for parent Commit: %v", err) } if n == 0 { return os.NewError("parent commit not found") } } // update the tip Tag if this is the Go repo if p.Path == "" { t := &Tag{Kind: "tip", Hash: com.Hash} if _, err = datastore.Put(c, t.Key(c), t); err != nil { return fmt.Errorf("putting Tag: %v", err) } } // put the Commit if _, err = datastore.Put(c, com.Key(c), com); err != nil { return fmt.Errorf("putting Commit: %v", err) } return nil } // tagHandler records a new tag. It reads a JSON-encoded Tag value from the // request body and updates the Tag entity for the Kind of tag provided. // // This handler is used by a gobuilder process in -commit mode. func tagHandler(r *http.Request) (interface{}, os.Error) { if r.Method != "POST" { return nil, errBadMethod(r.Method) } t := new(Tag) defer r.Body.Close() if err := json.NewDecoder(r.Body).Decode(t); err != nil { return nil, err } if err := t.Valid(); err != nil { return nil, err } c := appengine.NewContext(r) defer invalidateCache(c) _, err := datastore.Put(c, t.Key(c), t) return nil, err } // Todo is a todoHandler response. type Todo struct { Kind string // "build-go-commit" or "build-package" Data interface{} } // todoHandler returns the next action to be performed by a builder. // It expects "builder" and "kind" query parameters and returns a *Todo value. // Multiple "kind" parameters may be specified. func todoHandler(r *http.Request) (interface{}, os.Error) { c := appengine.NewContext(r) todoKey := r.Form.Encode() if t, ok := cachedTodo(c, todoKey); ok { c.Debugf("cache hit") return t, nil } c.Debugf("cache miss") var todo *Todo var err os.Error builder := r.FormValue("builder") for _, kind := range r.Form["kind"] { var data interface{} switch kind { case "build-go-commit": data, err = buildTodo(c, builder, "", "") case "build-package": packagePath := r.FormValue("packagePath") goHash := r.FormValue("goHash") data, err = buildTodo(c, builder, packagePath, goHash) } if data != nil || err != nil { todo = &Todo{Kind: kind, Data: data} break } } if err == nil { cacheTodo(c, todoKey, todo) } return todo, err } // buildTodo returns the next Commit to be built (or nil if none available). // // If packagePath and goHash are empty, it scans the first 20 Go Commits in // Num-descending order and returns the first one it finds that doesn't have a // Result for this builder. // // If provided with non-empty packagePath and goHash args, it scans the first // 20 Commits in Num-descending order for the specified packagePath and // returns the first that doesn't have a Result for this builder and goHash. func buildTodo(c appengine.Context, builder, packagePath, goHash string) (interface{}, os.Error) { p, err := GetPackage(c, packagePath) if err != nil { return nil, err } t := datastore.NewQuery("Commit"). Ancestor(p.Key(c)). Limit(commitsPerPage). Order("-Num"). Run(c) for { com := new(Commit) if _, err := t.Next(com); err != nil { if err == datastore.Done { err = nil } return nil, err } if com.Result(builder, goHash) == nil { return com, nil } } panic("unreachable") } // packagesHandler returns a list of the non-Go Packages monitored // by the dashboard. func packagesHandler(r *http.Request) (interface{}, os.Error) { return Packages(appengine.NewContext(r)) } // resultHandler records a build result. // It reads a JSON-encoded Result value from the request body, // creates a new Result entity, and updates the relevant Commit entity. // If the Log field is not empty, resultHandler creates a new Log entity // and updates the LogHash field before putting the Commit entity. func resultHandler(r *http.Request) (interface{}, os.Error) { if r.Method != "POST" { return nil, errBadMethod(r.Method) } c := appengine.NewContext(r) res := new(Result) defer r.Body.Close() if err := json.NewDecoder(r.Body).Decode(res); err != nil { return nil, fmt.Errorf("decoding Body: %v", err) } if err := res.Valid(); err != nil { return nil, fmt.Errorf("validating Result: %v", err) } defer invalidateCache(c) // store the Log text if supplied if len(res.Log) > 0 { hash, err := PutLog(c, res.Log) if err != nil { return nil, fmt.Errorf("putting Log: %v", err) } res.LogHash = hash } tx := func(c appengine.Context) os.Error { // check Package exists if _, err := GetPackage(c, res.PackagePath); err != nil { return fmt.Errorf("GetPackage: %v", err) } // put Result if _, err := datastore.Put(c, res.Key(c), res); err != nil { return fmt.Errorf("putting Result: %v", err) } // add Result to Commit com := &Commit{PackagePath: res.PackagePath, Hash: res.Hash} if err := com.AddResult(c, res); err != nil { return fmt.Errorf("AddResult: %v", err) } // Send build failure notifications, if necessary. // Note this must run after the call AddResult, which // populates the Commit's ResultData field. return notifyOnFailure(c, com, res.Builder) } return nil, datastore.RunInTransaction(c, tx, nil) } // logHandler displays log text for a given hash. // It handles paths like "/log/hash". func logHandler(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-type", "text/plain") c := appengine.NewContext(r) hash := r.URL.Path[len("/log/"):] key := datastore.NewKey(c, "Log", hash, 0, nil) l := new(Log) if err := datastore.Get(c, key, l); err != nil { logErr(w, r, err) return } b, err := l.Text() if err != nil { logErr(w, r, err) return } w.Write(b) } type dashHandler func(*http.Request) (interface{}, os.Error) type dashResponse struct { Response interface{} Error string } // errBadMethod is returned by a dashHandler when // the request has an unsuitable method. type errBadMethod string func (e errBadMethod) String() string { return "bad method: " + string(e) } // AuthHandler wraps a http.HandlerFunc with a handler that validates the // supplied key and builder query parameters. func AuthHandler(h dashHandler) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { c := appengine.NewContext(r) // Put the URL Query values into r.Form to avoid parsing the // request body when calling r.FormValue. r.Form = r.URL.Query() var err os.Error var resp interface{} // Validate key query parameter for POST requests only. key := r.FormValue("key") builder := r.FormValue("builder") if r.Method == "POST" && !validKey(key, builder) { err = os.NewError("invalid key: " + key) } // Call the original HandlerFunc and return the response. if err == nil { resp, err = h(r) } // Write JSON response. dashResp := &dashResponse{Response: resp} if err != nil { c.Errorf("%v", err) dashResp.Error = err.String() } w.Header().Set("Content-Type", "application/json") if err = json.NewEncoder(w).Encode(dashResp); err != nil { c.Criticalf("encoding response: %v", err) } } } func initHandler(w http.ResponseWriter, r *http.Request) { // TODO(adg): devise a better way of bootstrapping new packages c := appengine.NewContext(r) for _, p := range defaultPackages { if err := datastore.Get(c, p.Key(c), new(Package)); err == nil { continue } else if err != datastore.ErrNoSuchEntity { logErr(w, r, err) return } if _, err := datastore.Put(c, p.Key(c), p); err != nil { logErr(w, r, err) return } } fmt.Fprint(w, "OK") } func keyHandler(w http.ResponseWriter, r *http.Request) { builder := r.FormValue("builder") if builder == "" { logErr(w, r, os.NewError("must supply builder in query string")) return } fmt.Fprint(w, builderKey(builder)) } func init() { // admin handlers http.HandleFunc("/init", initHandler) http.HandleFunc("/key", keyHandler) // authenticated handlers http.HandleFunc("/commit", AuthHandler(commitHandler)) http.HandleFunc("/packages", AuthHandler(packagesHandler)) http.HandleFunc("/result", AuthHandler(resultHandler)) http.HandleFunc("/tag", AuthHandler(tagHandler)) http.HandleFunc("/todo", AuthHandler(todoHandler)) // public handlers http.HandleFunc("/log/", logHandler) } func validHash(hash string) bool { // TODO(adg): correctly validate a hash return hash != "" } func validKey(key, builder string) bool { if appengine.IsDevAppServer() { return true } if key == secretKey { return true } return key == builderKey(builder) } func builderKey(builder string) string { h := hmac.NewMD5([]byte(secretKey)) h.Write([]byte(builder)) return fmt.Sprintf("%x", h.Sum()) } func logErr(w http.ResponseWriter, r *http.Request, err os.Error) { appengine.NewContext(r).Errorf("Error: %v", err) w.WriteHeader(http.StatusInternalServerError) fmt.Fprint(w, "Error: ", err) }