From 263c955f2fff2016b5ff77d787d8e1b50555930a Mon Sep 17 00:00:00 2001 From: Andrew Gerrand Date: Mon, 5 Dec 2011 16:44:10 +1100 Subject: [PATCH] gobuilder: use new dashboard protocol gobuilder: -commit mode for packages gobuilder: cripple -package mode temporarily R=golang-dev, dsymonds CC=golang-dev https://golang.org/cl/5450092 --- misc/dashboard/builder/http.go | 163 ++++++++++++++++++++------------- misc/dashboard/builder/main.go | 95 +++++++++++++------ 2 files changed, 164 insertions(+), 94 deletions(-) diff --git a/misc/dashboard/builder/http.go b/misc/dashboard/builder/http.go index 3e2217f5412..9de54a36947 100644 --- a/misc/dashboard/builder/http.go +++ b/misc/dashboard/builder/http.go @@ -8,96 +8,108 @@ import ( "bytes" "encoding/json" "errors" - "fmt" + "io" "log" "net/http" "net/url" - "strconv" ) -type param map[string]string +type obj map[string]interface{} // dash runs the given method and command on the dashboard. -// If args is not nil, it is the query or post parameters. -// If resp is not nil, dash unmarshals the body as JSON into resp. -func dash(meth, cmd string, resp interface{}, args param) error { +// If args is non-nil it is encoded as the URL query string. +// If req is non-nil it is JSON-encoded and passed as the body of the HTTP POST. +// If resp is non-nil the server's response is decoded into the value pointed +// to by resp (resp must be a pointer). +func dash(meth, cmd string, args url.Values, req, resp interface{}) error { var r *http.Response var err error if *verbose { - log.Println("dash", cmd, args) + log.Println("dash", meth, cmd, args, req) } cmd = "http://" + *dashboard + "/" + cmd - vals := make(url.Values) - for k, v := range args { - vals.Add(k, v) + if len(args) > 0 { + cmd += "?" + args.Encode() } switch meth { case "GET": - if q := vals.Encode(); q != "" { - cmd += "?" + q + if req != nil { + log.Panicf("%s to %s with req", meth, cmd) } r, err = http.Get(cmd) case "POST": - r, err = http.PostForm(cmd, vals) - default: - return fmt.Errorf("unknown method %q", meth) - } - if err != nil { - return err - } - defer r.Body.Close() - var buf bytes.Buffer - buf.ReadFrom(r.Body) - if resp != nil { - if err = json.Unmarshal(buf.Bytes(), resp); err != nil { - log.Printf("json unmarshal %#q: %s\n", buf.Bytes(), err) - return err + var body io.Reader + if req != nil { + b, err := json.Marshal(req) + if err != nil { + return err + } + body = bytes.NewBuffer(b) } + r, err = http.Post(cmd, "text/json", body) + default: + log.Panicf("%s: invalid method %q", cmd, meth) + panic("invalid method: " + meth) } - return nil -} - -func dashStatus(meth, cmd string, args param) error { - var resp struct { - Status string - Error string - } - err := dash(meth, cmd, &resp, args) if err != nil { return err } - if resp.Status != "OK" { - return errors.New("/build: " + resp.Error) + + defer r.Body.Close() + body := new(bytes.Buffer) + if _, err := body.ReadFrom(r.Body); err != nil { + return err } + + // Read JSON-encoded Response into provided resp + // and return an error if present. + var result = struct { + Response interface{} + Error string + }{ + // Put the provided resp in here as it can be a pointer to + // some value we should unmarshal into. + Response: resp, + } + if err = json.Unmarshal(body.Bytes(), &result); err != nil { + log.Printf("json unmarshal %#q: %s\n", body.Bytes(), err) + return err + } + if result.Error != "" { + return errors.New(result.Error) + } + return nil } // todo returns the next hash to build. func (b *Builder) todo() (rev string, err error) { - var resp []struct { - Hash string - } - if err = dash("GET", "todo", &resp, param{"builder": b.name}); err != nil { + // TODO(adg): handle packages + args := url.Values{"builder": {b.name}} + var resp string + if err = dash("GET", "todo", args, nil, &resp); err != nil { return } - if len(resp) > 0 { - rev = resp[0].Hash + if resp != "" { + rev = resp } return } // recordResult sends build results to the dashboard func (b *Builder) recordResult(buildLog string, hash string) error { - return dash("POST", "build", nil, param{ - "builder": b.name, - "key": b.key, - "node": hash, - "log": buildLog, - }) + // TODO(adg): handle packages + return dash("POST", "result", url.Values{"key": {b.key}}, obj{ + "Builder": b.name, + "Hash": hash, + "Log": buildLog, + }, nil) } // packages fetches a list of package paths from the dashboard func packages() (pkgs []string, err error) { + return nil, nil + /* TODO(adg): un-stub this once the new package builder design is done var resp struct { Packages []struct { Path string @@ -111,10 +123,13 @@ func packages() (pkgs []string, err error) { pkgs = append(pkgs, p.Path) } return + */ } // updatePackage sends package build results and info dashboard func (b *Builder) updatePackage(pkg string, ok bool, buildLog, info string) error { + return nil + /* TODO(adg): un-stub this once the new package builder design is done return dash("POST", "package", nil, param{ "builder": b.name, "key": b.key, @@ -123,26 +138,44 @@ func (b *Builder) updatePackage(pkg string, ok bool, buildLog, info string) erro "log": buildLog, "info": info, }) + */ } -// postCommit informs the dashboard of a new commit -func postCommit(key string, l *HgLog) error { - return dashStatus("POST", "commit", param{ - "key": key, - "node": l.Hash, - "date": l.Date, - "user": l.Author, - "parent": l.Parent, - "desc": l.Desc, - }) -} - -// dashboardCommit returns true if the dashboard knows about hash. -func dashboardCommit(hash string) bool { - err := dashStatus("GET", "commit", param{"node": hash}) +func postCommit(key, pkg string, l *HgLog) bool { + err := dash("POST", "commit", url.Values{"key": {key}}, obj{ + "PackagePath": pkg, + "Hash": l.Hash, + "ParentHash": l.Parent, + // TODO(adg): l.Date as int64 unix epoch secs in Time field + "User": l.Author, + "Desc": l.Desc, + }, nil) if err != nil { - log.Printf("check %s: %s", hash, err) + log.Printf("failed to add %s to dashboard: %v", key, err) return false } return true } + +func dashboardCommit(pkg, hash string) bool { + err := dash("GET", "commit", url.Values{ + "packagePath": {pkg}, + "hash": {hash}, + }, nil, nil) + return err == nil +} + +func dashboardPackages() []string { + var resp []struct { + Path string + } + if err := dash("GET", "packages", nil, nil, &resp); err != nil { + log.Println("dashboardPackages:", err) + return nil + } + var pkgs []string + for _, r := range resp { + pkgs = append(pkgs, r.Path) + } + return pkgs +} diff --git a/misc/dashboard/builder/main.go b/misc/dashboard/builder/main.go index 101b77ed767..aaeedcfb608 100644 --- a/misc/dashboard/builder/main.go +++ b/misc/dashboard/builder/main.go @@ -13,6 +13,7 @@ import ( "log" "os" "path" + "path/filepath" "regexp" "runtime" "strconv" @@ -93,7 +94,7 @@ func main() { if err := os.Mkdir(*buildroot, mkdirPerm); err != nil { log.Fatalf("Error making build root (%s): %s", *buildroot, err) } - if err := run(nil, *buildroot, "hg", "clone", hgUrl, goroot); err != nil { + if err := hgClone(hgUrl, goroot); err != nil { log.Fatal("Error cloning repository:", err) } @@ -107,7 +108,7 @@ func main() { // if specified, build revision and return if *buildRevision != "" { - hash, err := fullHash(*buildRevision) + hash, err := fullHash(goroot, *buildRevision) if err != nil { log.Fatal("Error finding revision: ", err) } @@ -246,7 +247,7 @@ func (b *Builder) build() bool { } // Look for hash locally before running hg pull. - if _, err := fullHash(hash[:12]); err != nil { + if _, err := fullHash(goroot, hash[:12]); err != nil { // Don't have hash, so run hg pull. if err := run(nil, goroot, "hg", "pull"); err != nil { log.Println("hg pull failed:", err) @@ -425,11 +426,16 @@ func commitWatcher() { if err != nil { log.Fatal(err) } + key := b.key + for { if *verbose { log.Printf("poll...") } - commitPoll(b.key) + commitPoll(key, "") + for _, pkg := range dashboardPackages() { + commitPoll(key, pkg) + } if *verbose { log.Printf("sleep...") } @@ -437,6 +443,18 @@ func commitWatcher() { } } +func hgClone(url, path string) error { + return run(nil, *buildroot, "hg", "clone", url, path) +} + +func hgRepoExists(path string) bool { + fi, err := os.Stat(filepath.Join(path, ".hg")) + if err != nil { + return false + } + return fi.IsDir() +} + // HgLog represents a single Mercurial revision. type HgLog struct { Hash string @@ -467,7 +485,7 @@ const xmlLogTemplate = ` // commitPoll pulls any new revisions from the hg server // and tells the server about them. -func commitPoll(key string) { +func commitPoll(key, pkg string) { // Catch unexpected panics. defer func() { if err := recover(); err != nil { @@ -475,14 +493,29 @@ func commitPoll(key string) { } }() - if err := run(nil, goroot, "hg", "pull"); err != nil { + pkgRoot := goroot + + if pkg != "" { + pkgRoot = path.Join(*buildroot, pkg) + if !hgRepoExists(pkgRoot) { + if err := hgClone(repoURL(pkg), pkgRoot); err != nil { + log.Printf("%s: hg clone failed: %v", pkg, err) + if err := os.RemoveAll(pkgRoot); err != nil { + log.Printf("%s: %v", pkg, err) + } + return + } + } + } + + if err := run(nil, pkgRoot, "hg", "pull"); err != nil { log.Printf("hg pull: %v", err) return } const N = 50 // how many revisions to grab - data, _, err := runLog(nil, "", goroot, "hg", "log", + data, _, err := runLog(nil, "", pkgRoot, "hg", "log", "--encoding=utf-8", "--limit="+strconv.Itoa(N), "--template="+xmlLogTemplate, @@ -511,14 +544,11 @@ func commitPoll(key string) { if l.Parent == "" && i+1 < len(logs) { l.Parent = logs[i+1].Hash } else if l.Parent != "" { - l.Parent, _ = fullHash(l.Parent) + l.Parent, _ = fullHash(pkgRoot, l.Parent) } - log.Printf("hg log: %s < %s\n", l.Hash, l.Parent) - if l.Parent == "" { - // Can't create node without parent. - continue + if *verbose { + log.Printf("hg log %s: %s < %s\n", pkg, l.Hash, l.Parent) } - if logByHash[l.Hash] == nil { // Make copy to avoid pinning entire slice when only one entry is new. t := *l @@ -528,17 +558,14 @@ func commitPoll(key string) { for i := range logs { l := &logs[i] - if l.Parent == "" { - continue - } - addCommit(l.Hash, key) + addCommit(pkg, l.Hash, key) } } // addCommit adds the commit with the named hash to the dashboard. // key is the secret key for authentication to the dashboard. // It avoids duplicate effort. -func addCommit(hash, key string) bool { +func addCommit(pkg, hash, key string) bool { l := logByHash[hash] if l == nil { return false @@ -548,7 +575,7 @@ func addCommit(hash, key string) bool { } // Check for already added, perhaps in an earlier run. - if dashboardCommit(hash) { + if dashboardCommit(pkg, hash) { log.Printf("%s already on dashboard\n", hash) // Record that this hash is on the dashboard, // as must be all its parents. @@ -560,26 +587,24 @@ func addCommit(hash, key string) bool { } // Create parent first, to maintain some semblance of order. - if !addCommit(l.Parent, key) { - return false + if l.Parent != "" { + if !addCommit(pkg, l.Parent, key) { + return false + } } // Create commit. - if err := postCommit(key, l); err != nil { - log.Printf("failed to add %s to dashboard: %v", key, err) - return false - } - return true + return postCommit(key, pkg, l) } // fullHash returns the full hash for the given Mercurial revision. -func fullHash(rev string) (hash string, err error) { +func fullHash(root, rev string) (hash string, err error) { defer func() { if err != nil { err = fmt.Errorf("fullHash: %s: %s", rev, err) } }() - s, _, err := runLog(nil, "", goroot, + s, _, err := runLog(nil, "", root, "hg", "log", "--encoding=utf-8", "--rev="+rev, @@ -617,9 +642,21 @@ func firstTag(re *regexp.Regexp) (hash string, tag string, err error) { continue } tag = s[1] - hash, err = fullHash(s[2]) + hash, err = fullHash(goroot, s[2]) return } err = errors.New("no matching tag found") return } + +var repoRe = regexp.MustCompile(`^code\.google\.com/p/([a-z0-9\-]+(\.[a-z0-9\-]+)?)(/[a-z0-9A-Z_.\-/]+)?$`) + +// repoURL returns the repository URL for the supplied import path. +func repoURL(importPath string) string { + m := repoRe.FindStringSubmatch(importPath) + if len(m) < 2 { + log.Printf("repoURL: couldn't decipher %q", importPath) + return "" + } + return "https://code.google.com/p/" + m[1] +}