gobuilder: use new dashboard protocol

gobuilder: -commit mode for packages
gobuilder: cripple -package mode temporarily

R=golang-dev, dsymonds
CC=golang-dev
https://golang.org/cl/5450092

misc/dashboard/builder/http.go

@@ -8,96 +8,108 @@ import (
 	"bytes"
 	"encoding/json"
 	"errors"
-	"fmt"
+	"io"
 	"log"
 	"net/http"
 	"net/url"
-	"strconv"
 )
 
-type param map[string]string
+type obj map[string]interface{}
 
 // dash runs the given method and command on the dashboard.
-// If args is not nil, it is the query or post parameters.
+// If args is non-nil it is encoded as the URL query string.
-// If resp is not nil, dash unmarshals the body as JSON into resp.
+// If req is non-nil it is JSON-encoded and passed as the body of the HTTP POST.
-func dash(meth, cmd string, resp interface{}, args param) error {
+// If resp is non-nil the server's response is decoded into the value pointed
+// to by resp (resp must be a pointer).
+func dash(meth, cmd string, args url.Values, req, resp interface{}) error {
 	var r *http.Response
 	var err error
 	if *verbose {
-		log.Println("dash", cmd, args)
+		log.Println("dash", meth, cmd, args, req)
 	}
 	cmd = "http://" + *dashboard + "/" + cmd
-	vals := make(url.Values)
+	if len(args) > 0 {
-	for k, v := range args {
+		cmd += "?" + args.Encode()
-		vals.Add(k, v)
 	}
 	switch meth {
 	case "GET":
-		if q := vals.Encode(); q != "" {
+		if req != nil {
-			cmd += "?" + q
+			log.Panicf("%s to %s with req", meth, cmd)
 		}
 		r, err = http.Get(cmd)
 	case "POST":
-		r, err = http.PostForm(cmd, vals)
+		var body io.Reader
-	default:
+		if req != nil {
-		return fmt.Errorf("unknown method %q", meth)
+			b, err := json.Marshal(req)
-	}
+			if err != nil {
-	if err != nil {
+				return err
-		return err
+			}
-	}
+			body = bytes.NewBuffer(b)
-	defer r.Body.Close()
-	var buf bytes.Buffer
-	buf.ReadFrom(r.Body)
-	if resp != nil {
-		if err = json.Unmarshal(buf.Bytes(), resp); err != nil {
-			log.Printf("json unmarshal %#q: %s\n", buf.Bytes(), err)
-			return err
 		}
+		r, err = http.Post(cmd, "text/json", body)
+	default:
+		log.Panicf("%s: invalid method %q", cmd, meth)
+		panic("invalid method: " + meth)
 	}
-	return nil
-}
-
-func dashStatus(meth, cmd string, args param) error {
-	var resp struct {
-		Status string
-		Error  string
-	}
-	err := dash(meth, cmd, &resp, args)
 	if err != nil {
 		return err
 	}
-	if resp.Status != "OK" {
+
-		return errors.New("/build: " + resp.Error)
+	defer r.Body.Close()
+	body := new(bytes.Buffer)
+	if _, err := body.ReadFrom(r.Body); err != nil {
+		return err
 	}
+
+	// Read JSON-encoded Response into provided resp
+	// and return an error if present.
+	var result = struct {
+		Response interface{}
+		Error    string
+	}{
+		// Put the provided resp in here as it can be a pointer to
+		// some value we should unmarshal into.
+		Response: resp,
+	}
+	if err = json.Unmarshal(body.Bytes(), &result); err != nil {
+		log.Printf("json unmarshal %#q: %s\n", body.Bytes(), err)
+		return err
+	}
+	if result.Error != "" {
+		return errors.New(result.Error)
+	}
+
 	return nil
 }
 
 // todo returns the next hash to build.
 func (b *Builder) todo() (rev string, err error) {
-	var resp []struct {
+	// TODO(adg): handle packages
-		Hash string
+	args := url.Values{"builder": {b.name}}
-	}
+	var resp string
-	if err = dash("GET", "todo", &resp, param{"builder": b.name}); err != nil {
+	if err = dash("GET", "todo", args, nil, &resp); err != nil {
 		return
 	}
-	if len(resp) > 0 {
+	if resp != "" {
-		rev = resp[0].Hash
+		rev = resp
 	}
 	return
 }
 
 // recordResult sends build results to the dashboard
 func (b *Builder) recordResult(buildLog string, hash string) error {
-	return dash("POST", "build", nil, param{
+	// TODO(adg): handle packages
-		"builder": b.name,
+	return dash("POST", "result", url.Values{"key": {b.key}}, obj{
-		"key":     b.key,
+		"Builder": b.name,
-		"node":    hash,
+		"Hash":    hash,
-		"log":     buildLog,
+		"Log":     buildLog,
-	})
+	}, nil)
 }
 
 // packages fetches a list of package paths from the dashboard
 func packages() (pkgs []string, err error) {
+	return nil, nil
+	/* TODO(adg): un-stub this once the new package builder design is done
 	var resp struct {
 		Packages []struct {
 			Path string
@@ -111,10 +123,13 @@ func packages() (pkgs []string, err error) {
 		pkgs = append(pkgs, p.Path)
 	}
 	return
+	*/
 }
 
 // updatePackage sends package build results and info dashboard
 func (b *Builder) updatePackage(pkg string, ok bool, buildLog, info string) error {
+	return nil
+	/* TODO(adg): un-stub this once the new package builder design is done
 	return dash("POST", "package", nil, param{
 		"builder": b.name,
 		"key":     b.key,
@@ -123,26 +138,44 @@ func (b *Builder) updatePackage(pkg string, ok bool, buildLog, info string) erro
 		"log":     buildLog,
 		"info":    info,
 	})
+	*/
 }
 
-// postCommit informs the dashboard of a new commit
+func postCommit(key, pkg string, l *HgLog) bool {
-func postCommit(key string, l *HgLog) error {
+	err := dash("POST", "commit", url.Values{"key": {key}}, obj{
-	return dashStatus("POST", "commit", param{
+		"PackagePath": pkg,
-		"key":    key,
+		"Hash":        l.Hash,
-		"node":   l.Hash,
+		"ParentHash":  l.Parent,
-		"date":   l.Date,
+		// TODO(adg): l.Date as int64 unix epoch secs in Time field
-		"user":   l.Author,
+		"User": l.Author,
-		"parent": l.Parent,
+		"Desc": l.Desc,
-		"desc":   l.Desc,
+	}, nil)
-	})
-}
-
-// dashboardCommit returns true if the dashboard knows about hash.
-func dashboardCommit(hash string) bool {
-	err := dashStatus("GET", "commit", param{"node": hash})
 	if err != nil {
-		log.Printf("check %s: %s", hash, err)
+		log.Printf("failed to add %s to dashboard: %v", key, err)
 		return false
 	}
 	return true
 }
+
+func dashboardCommit(pkg, hash string) bool {
+	err := dash("GET", "commit", url.Values{
+		"packagePath": {pkg},
+		"hash":        {hash},
+	}, nil, nil)
+	return err == nil
+}
+
+func dashboardPackages() []string {
+	var resp []struct {
+		Path string
+	}
+	if err := dash("GET", "packages", nil, nil, &resp); err != nil {
+		log.Println("dashboardPackages:", err)
+		return nil
+	}
+	var pkgs []string
+	for _, r := range resp {
+		pkgs = append(pkgs, r.Path)
+	}
+	return pkgs
+}

misc/dashboard/builder/main.go

@@ -13,6 +13,7 @@ import (
 	"log"
 	"os"
 	"path"
+	"path/filepath"
 	"regexp"
 	"runtime"
 	"strconv"
@@ -93,7 +94,7 @@ func main() {
 	if err := os.Mkdir(*buildroot, mkdirPerm); err != nil {
 		log.Fatalf("Error making build root (%s): %s", *buildroot, err)
 	}
-	if err := run(nil, *buildroot, "hg", "clone", hgUrl, goroot); err != nil {
+	if err := hgClone(hgUrl, goroot); err != nil {
 		log.Fatal("Error cloning repository:", err)
 	}
 
@@ -107,7 +108,7 @@ func main() {
 
 	// if specified, build revision and return
 	if *buildRevision != "" {
-		hash, err := fullHash(*buildRevision)
+		hash, err := fullHash(goroot, *buildRevision)
 		if err != nil {
 			log.Fatal("Error finding revision: ", err)
 		}
@@ -246,7 +247,7 @@ func (b *Builder) build() bool {
 	}
 	// Look for hash locally before running hg pull.
 
-	if _, err := fullHash(hash[:12]); err != nil {
+	if _, err := fullHash(goroot, hash[:12]); err != nil {
 		// Don't have hash, so run hg pull.
 		if err := run(nil, goroot, "hg", "pull"); err != nil {
 			log.Println("hg pull failed:", err)
@@ -425,11 +426,16 @@ func commitWatcher() {
 	if err != nil {
 		log.Fatal(err)
 	}
+	key := b.key
+
 	for {
 		if *verbose {
 			log.Printf("poll...")
 		}
-		commitPoll(b.key)
+		commitPoll(key, "")
+		for _, pkg := range dashboardPackages() {
+			commitPoll(key, pkg)
+		}
 		if *verbose {
 			log.Printf("sleep...")
 		}
@@ -437,6 +443,18 @@ func commitWatcher() {
 	}
 }
 
+func hgClone(url, path string) error {
+	return run(nil, *buildroot, "hg", "clone", url, path)
+}
+
+func hgRepoExists(path string) bool {
+	fi, err := os.Stat(filepath.Join(path, ".hg"))
+	if err != nil {
+		return false
+	}
+	return fi.IsDir()
+}
+
 // HgLog represents a single Mercurial revision.
 type HgLog struct {
 	Hash   string
@@ -467,7 +485,7 @@ const xmlLogTemplate = `
 
 // commitPoll pulls any new revisions from the hg server
 // and tells the server about them.
-func commitPoll(key string) {
+func commitPoll(key, pkg string) {
 	// Catch unexpected panics.
 	defer func() {
 		if err := recover(); err != nil {
@@ -475,14 +493,29 @@ func commitPoll(key string) {
 		}
 	}()
 
-	if err := run(nil, goroot, "hg", "pull"); err != nil {
+	pkgRoot := goroot
+
+	if pkg != "" {
+		pkgRoot = path.Join(*buildroot, pkg)
+		if !hgRepoExists(pkgRoot) {
+			if err := hgClone(repoURL(pkg), pkgRoot); err != nil {
+				log.Printf("%s: hg clone failed: %v", pkg, err)
+				if err := os.RemoveAll(pkgRoot); err != nil {
+					log.Printf("%s: %v", pkg, err)
+				}
+				return
+			}
+		}
+	}
+
+	if err := run(nil, pkgRoot, "hg", "pull"); err != nil {
 		log.Printf("hg pull: %v", err)
 		return
 	}
 
 	const N = 50 // how many revisions to grab
 
-	data, _, err := runLog(nil, "", goroot, "hg", "log",
+	data, _, err := runLog(nil, "", pkgRoot, "hg", "log",
 		"--encoding=utf-8",
 		"--limit="+strconv.Itoa(N),
 		"--template="+xmlLogTemplate,
@@ -511,14 +544,11 @@ func commitPoll(key string) {
 		if l.Parent == "" && i+1 < len(logs) {
 			l.Parent = logs[i+1].Hash
 		} else if l.Parent != "" {
-			l.Parent, _ = fullHash(l.Parent)
+			l.Parent, _ = fullHash(pkgRoot, l.Parent)
 		}
-		log.Printf("hg log: %s < %s\n", l.Hash, l.Parent)
+		if *verbose {
-		if l.Parent == "" {
+			log.Printf("hg log %s: %s < %s\n", pkg, l.Hash, l.Parent)
-			// Can't create node without parent.
-			continue
 		}
-
 		if logByHash[l.Hash] == nil {
 			// Make copy to avoid pinning entire slice when only one entry is new.
 			t := *l
@@ -528,17 +558,14 @@ func commitPoll(key string) {
 
 	for i := range logs {
 		l := &logs[i]
-		if l.Parent == "" {
+		addCommit(pkg, l.Hash, key)
-			continue
-		}
-		addCommit(l.Hash, key)
 	}
 }
 
 // addCommit adds the commit with the named hash to the dashboard.
 // key is the secret key for authentication to the dashboard.
 // It avoids duplicate effort.
-func addCommit(hash, key string) bool {
+func addCommit(pkg, hash, key string) bool {
 	l := logByHash[hash]
 	if l == nil {
 		return false
@@ -548,7 +575,7 @@ func addCommit(hash, key string) bool {
 	}
 
 	// Check for already added, perhaps in an earlier run.
-	if dashboardCommit(hash) {
+	if dashboardCommit(pkg, hash) {
 		log.Printf("%s already on dashboard\n", hash)
 		// Record that this hash is on the dashboard,
 		// as must be all its parents.
@@ -560,26 +587,24 @@ func addCommit(hash, key string) bool {
 	}
 
 	// Create parent first, to maintain some semblance of order.
-	if !addCommit(l.Parent, key) {
+	if l.Parent != "" {
-		return false
+		if !addCommit(pkg, l.Parent, key) {
+			return false
+		}
 	}
 
 	// Create commit.
-	if err := postCommit(key, l); err != nil {
+	return postCommit(key, pkg, l)
-		log.Printf("failed to add %s to dashboard: %v", key, err)
-		return false
-	}
-	return true
 }
 
 // fullHash returns the full hash for the given Mercurial revision.
-func fullHash(rev string) (hash string, err error) {
+func fullHash(root, rev string) (hash string, err error) {
 	defer func() {
 		if err != nil {
 			err = fmt.Errorf("fullHash: %s: %s", rev, err)
 		}
 	}()
-	s, _, err := runLog(nil, "", goroot,
+	s, _, err := runLog(nil, "", root,
 		"hg", "log",
 		"--encoding=utf-8",
 		"--rev="+rev,
@@ -617,9 +642,21 @@ func firstTag(re *regexp.Regexp) (hash string, tag string, err error) {
 			continue
 		}
 		tag = s[1]
-		hash, err = fullHash(s[2])
+		hash, err = fullHash(goroot, s[2])
 		return
 	}
 	err = errors.New("no matching tag found")
 	return
 }
+
+var repoRe = regexp.MustCompile(`^code\.google\.com/p/([a-z0-9\-]+(\.[a-z0-9\-]+)?)(/[a-z0-9A-Z_.\-/]+)?$`)
+
+// repoURL returns the repository URL for the supplied import path.
+func repoURL(importPath string) string {
+	m := repoRe.FindStringSubmatch(importPath)
+	if len(m) < 2 {
+		log.Printf("repoURL: couldn't decipher %q", importPath)
+		return ""
+	}
+	return "https://code.google.com/p/" + m[1]
+}