go/dashboard/app/build/handler.go

// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build appengine

package build

import (
	"crypto/hmac"
	"crypto/md5"
	"encoding/json"
	"errors"
	"fmt"
	"net/http"
	"strings"
	"unicode/utf8"

	"appengine"
	"appengine/datastore"

	"cache"
)

const commitsPerPage = 30

// commitHandler retrieves commit data or records a new commit.
//
// For GET requests it returns a Commit value for the specified
// packagePath and hash.
//
// For POST requests it reads a JSON-encoded Commit value from the request
// body and creates a new Commit entity. It also updates the "tip" Tag for
// each new commit at tip.
//
// This handler is used by a gobuilder process in -commit mode.
func commitHandler(r *http.Request) (interface{}, error) {
	c := contextForRequest(r)
	com := new(Commit)

	if r.Method == "GET" {
		com.PackagePath = r.FormValue("packagePath")
		com.Hash = r.FormValue("hash")
		if err := datastore.Get(c, com.Key(c), com); err != nil {
			return nil, fmt.Errorf("getting Commit: %v", err)
		}
		return com, nil
	}
	if r.Method != "POST" {
		return nil, errBadMethod(r.Method)
	}
	if !isMasterKey(c, r.FormValue("key")) {
		return nil, errors.New("can only POST commits with master key")
	}

	// POST request
	defer r.Body.Close()
	if err := json.NewDecoder(r.Body).Decode(com); err != nil {
		return nil, fmt.Errorf("decoding Body: %v", err)
	}
	com.Desc = limitStringLength(com.Desc, maxDatastoreStringLen)
	if err := com.Valid(); err != nil {
		return nil, fmt.Errorf("validating Commit: %v", err)
	}
	defer cache.Tick(c)
	tx := func(c appengine.Context) error {
		return addCommit(c, com)
	}
	return nil, datastore.RunInTransaction(c, tx, nil)
}

// addCommit adds the Commit entity to the datastore and updates the tip Tag.
// It must be run inside a datastore transaction.
func addCommit(c appengine.Context, com *Commit) error {
	var tc Commit // temp value so we don't clobber com
	err := datastore.Get(c, com.Key(c), &tc)
	if err != datastore.ErrNoSuchEntity {
		// if this commit is already in the datastore, do nothing
		if err == nil {
			return nil
		}
		return fmt.Errorf("getting Commit: %v", err)
	}
	// get the next commit number
	p, err := GetPackage(c, com.PackagePath)
	if err != nil {
		return fmt.Errorf("GetPackage: %v", err)
	}
	com.Num = p.NextNum
	p.NextNum++
	if _, err := datastore.Put(c, p.Key(c), p); err != nil {
		return fmt.Errorf("putting Package: %v", err)
	}
	// if this isn't the first Commit test the parent commit exists
	if com.Num > 0 {
		n, err := datastore.NewQuery("Commit").
			Filter("Hash =", com.ParentHash).
			Ancestor(p.Key(c)).
			Count(c)
		if err != nil {
			return fmt.Errorf("testing for parent Commit: %v", err)
		}
		if n == 0 {
			return errors.New("parent commit not found")
		}
	}
	// update the tip Tag if this is the Go repo and this isn't on a release branch
	if p.Path == "" && !strings.HasPrefix(com.Desc, "[release-branch") {
		t := &Tag{Kind: "tip", Hash: com.Hash}
		if _, err = datastore.Put(c, t.Key(c), t); err != nil {
			return fmt.Errorf("putting Tag: %v", err)
		}
	}
	// put the Commit
	if _, err = datastore.Put(c, com.Key(c), com); err != nil {
		return fmt.Errorf("putting Commit: %v", err)
	}
	return nil
}

// tagHandler records a new tag. It reads a JSON-encoded Tag value from the
// request body and updates the Tag entity for the Kind of tag provided.
//
// This handler is used by a gobuilder process in -commit mode.
func tagHandler(r *http.Request) (interface{}, error) {
	if r.Method != "POST" {
		return nil, errBadMethod(r.Method)
	}

	t := new(Tag)
	defer r.Body.Close()
	if err := json.NewDecoder(r.Body).Decode(t); err != nil {
		return nil, err
	}
	if err := t.Valid(); err != nil {
		return nil, err
	}
	c := contextForRequest(r)
	defer cache.Tick(c)
	_, err := datastore.Put(c, t.Key(c), t)
	return nil, err
}

// Todo is a todoHandler response.
type Todo struct {
	Kind string // "build-go-commit" or "build-package"
	Data interface{}
}

// todoHandler returns the next action to be performed by a builder.
// It expects "builder" and "kind" query parameters and returns a *Todo value.
// Multiple "kind" parameters may be specified.
func todoHandler(r *http.Request) (interface{}, error) {
	c := contextForRequest(r)
	now := cache.Now(c)
	key := "build-todo-" + r.Form.Encode()
	var todo *Todo
	if cache.Get(r, now, key, &todo) {
		return todo, nil
	}
	var err error
	builder := r.FormValue("builder")
	for _, kind := range r.Form["kind"] {
		var com *Commit
		switch kind {
		case "build-go-commit":
			com, err = buildTodo(c, builder, "", "")
		case "build-package":
			packagePath := r.FormValue("packagePath")
			goHash := r.FormValue("goHash")
			com, err = buildTodo(c, builder, packagePath, goHash)
		}
		if com != nil || err != nil {
			if com != nil {
				// ResultData can be large and not needed on builder.
				com.ResultData = []string{}
			}
			todo = &Todo{Kind: kind, Data: com}
			break
		}
	}
	if err == nil {
		cache.Set(r, now, key, todo)
	}
	return todo, err
}

// buildTodo returns the next Commit to be built (or nil if none available).
//
// If packagePath and goHash are empty, it scans the first 20 Go Commits in
// Num-descending order and returns the first one it finds that doesn't have a
// Result for this builder.
//
// If provided with non-empty packagePath and goHash args, it scans the first
// 20 Commits in Num-descending order for the specified packagePath and
// returns the first that doesn't have a Result for this builder and goHash.
func buildTodo(c appengine.Context, builder, packagePath, goHash string) (*Commit, error) {
	p, err := GetPackage(c, packagePath)
	if err != nil {
		return nil, err
	}

	t := datastore.NewQuery("Commit").
		Ancestor(p.Key(c)).
		Limit(commitsPerPage).
		Order("-Num").
		Run(c)
	for {
		com := new(Commit)
		if _, err := t.Next(com); err == datastore.Done {
			break
		} else if err != nil {
			return nil, err
		}
		if com.Result(builder, goHash) == nil {
			return com, nil
		}
	}

	// Nothing left to do if this is a package (not the Go tree).
	if packagePath != "" {
		return nil, nil
	}

	// If there are no Go tree commits left to build,
	// see if there are any subrepo commits that need to be built at tip.
	// If so, ask the builder to build a go tree at the tip commit.
	// TODO(adg): do the same for "weekly" and "release" tags.

	tag, err := GetTag(c, "tip")
	if err != nil {
		return nil, err
	}

	// Check that this Go commit builds OK for this builder.
	// If not, don't re-build as the subrepos will never get built anyway.
	com, err := tag.Commit(c)
	if err != nil {
		return nil, err
	}
	if r := com.Result(builder, ""); r != nil && !r.OK {
		return nil, nil
	}

	pkgs, err := Packages(c, "subrepo")
	if err != nil {
		return nil, err
	}
	for _, pkg := range pkgs {
		com, err := pkg.LastCommit(c)
		if err != nil {
			c.Warningf("%v: no Commit found: %v", pkg, err)
			continue
		}
		if com.Result(builder, tag.Hash) == nil {
			return tag.Commit(c)
		}
	}

	return nil, nil
}

// packagesHandler returns a list of the non-Go Packages monitored
// by the dashboard.
func packagesHandler(r *http.Request) (interface{}, error) {
	kind := r.FormValue("kind")
	c := contextForRequest(r)
	now := cache.Now(c)
	key := "build-packages-" + kind
	var p []*Package
	if cache.Get(r, now, key, &p) {
		return p, nil
	}
	p, err := Packages(c, kind)
	if err != nil {
		return nil, err
	}
	cache.Set(r, now, key, p)
	return p, nil
}

// resultHandler records a build result.
// It reads a JSON-encoded Result value from the request body,
// creates a new Result entity, and updates the relevant Commit entity.
// If the Log field is not empty, resultHandler creates a new Log entity
// and updates the LogHash field before putting the Commit entity.
func resultHandler(r *http.Request) (interface{}, error) {
	if r.Method != "POST" {
		return nil, errBadMethod(r.Method)
	}

	c := contextForRequest(r)
	res := new(Result)
	defer r.Body.Close()
	if err := json.NewDecoder(r.Body).Decode(res); err != nil {
		return nil, fmt.Errorf("decoding Body: %v", err)
	}
	if err := res.Valid(); err != nil {
		return nil, fmt.Errorf("validating Result: %v", err)
	}
	defer cache.Tick(c)
	// store the Log text if supplied
	if len(res.Log) > 0 {
		hash, err := PutLog(c, res.Log)
		if err != nil {
			return nil, fmt.Errorf("putting Log: %v", err)
		}
		res.LogHash = hash
	}
	tx := func(c appengine.Context) error {
		// check Package exists
		if _, err := GetPackage(c, res.PackagePath); err != nil {
			return fmt.Errorf("GetPackage: %v", err)
		}
		// put Result
		if _, err := datastore.Put(c, res.Key(c), res); err != nil {
			return fmt.Errorf("putting Result: %v", err)
		}
		// add Result to Commit
		com := &Commit{PackagePath: res.PackagePath, Hash: res.Hash}
		if err := com.AddResult(c, res); err != nil {
			return fmt.Errorf("AddResult: %v", err)
		}
		// Send build failure notifications, if necessary.
		// Note this must run after the call AddResult, which
		// populates the Commit's ResultData field.
		return notifyOnFailure(c, com, res.Builder)
	}
	return nil, datastore.RunInTransaction(c, tx, nil)
}

// logHandler displays log text for a given hash.
// It handles paths like "/log/hash".
func logHandler(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-type", "text/plain; charset=utf-8")
	c := contextForRequest(r)
	hash := r.URL.Path[strings.LastIndex(r.URL.Path, "/")+1:]
	key := datastore.NewKey(c, "Log", hash, 0, nil)
	l := new(Log)
	if err := datastore.Get(c, key, l); err != nil {
		logErr(w, r, err)
		return
	}
	b, err := l.Text()
	if err != nil {
		logErr(w, r, err)
		return
	}
	w.Write(b)
}

type dashHandler func(*http.Request) (interface{}, error)

type dashResponse struct {
	Response interface{}
	Error    string
}

// errBadMethod is returned by a dashHandler when
// the request has an unsuitable method.
type errBadMethod string

func (e errBadMethod) Error() string {
	return "bad method: " + string(e)
}

// AuthHandler wraps a http.HandlerFunc with a handler that validates the
// supplied key and builder query parameters.
func AuthHandler(h dashHandler) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		c := contextForRequest(r)

		// Put the URL Query values into r.Form to avoid parsing the
		// request body when calling r.FormValue.
		r.Form = r.URL.Query()

		var err error
		var resp interface{}

		// Validate key query parameter for POST requests only.
		key := r.FormValue("key")
		builder := r.FormValue("builder")
		if r.Method == "POST" && !validKey(c, key, builder) {
			err = fmt.Errorf("invalid key %q for builder %q", key, builder)
		}

		// Call the original HandlerFunc and return the response.
		if err == nil {
			resp, err = h(r)
		}

		// Write JSON response.
		dashResp := &dashResponse{Response: resp}
		if err != nil {
			c.Errorf("%v", err)
			dashResp.Error = err.Error()
		}
		w.Header().Set("Content-Type", "application/json")
		if err = json.NewEncoder(w).Encode(dashResp); err != nil {
			c.Criticalf("encoding response: %v", err)
		}
	}
}

func keyHandler(w http.ResponseWriter, r *http.Request) {
	builder := r.FormValue("builder")
	if builder == "" {
		logErr(w, r, errors.New("must supply builder in query string"))
		return
	}
	c := contextForRequest(r)
	fmt.Fprint(w, builderKey(c, builder))
}

func init() {
	for _, d := range dashboards {
		// admin handlers
		http.HandleFunc(d.RelPath+"init", initHandler)
		http.HandleFunc(d.RelPath+"key", keyHandler)

		// authenticated handlers
		http.HandleFunc(d.RelPath+"commit", AuthHandler(commitHandler))
		http.HandleFunc(d.RelPath+"packages", AuthHandler(packagesHandler))
		http.HandleFunc(d.RelPath+"result", AuthHandler(resultHandler))
		http.HandleFunc(d.RelPath+"tag", AuthHandler(tagHandler))
		http.HandleFunc(d.RelPath+"todo", AuthHandler(todoHandler))

		// public handlers
		http.HandleFunc(d.RelPath+"log/", logHandler)
	}
}

func validHash(hash string) bool {
	// TODO(adg): correctly validate a hash
	return hash != ""
}

func validKey(c appengine.Context, key, builder string) bool {
	return isMasterKey(c, key) || key == builderKey(c, builder)
}

func isMasterKey(c appengine.Context, key string) bool {
	return appengine.IsDevAppServer() || key == secretKey(c)
}

func builderKey(c appengine.Context, builder string) string {
	h := hmac.New(md5.New, []byte(secretKey(c)))
	h.Write([]byte(builder))
	return fmt.Sprintf("%x", h.Sum(nil))
}

func logErr(w http.ResponseWriter, r *http.Request, err error) {
	contextForRequest(r).Errorf("Error: %v", err)
	w.WriteHeader(http.StatusInternalServerError)
	fmt.Fprint(w, "Error: ", err)
}

func contextForRequest(r *http.Request) appengine.Context {
	return dashboardForRequest(r).Context(appengine.NewContext(r))
}

// limitStringLength essentially does return s[:max],
// but it ensures that we dot not split UTF-8 rune in half.
// Otherwise appengine python scripts will break badly.
func limitStringLength(s string, max int) string {
	if len(s) <= max {
		return s
	}
	for {
		s = s[:max]
		r, size := utf8.DecodeLastRuneInString(s)
		if r != utf8.RuneError || size != 1 {
			return s
		}
		max--
	}
}