From 1d214a6a09ff4746cf03d1ca1a08c08e5ad3914f Mon Sep 17 00:00:00 2001 From: Chris Manghane Date: Tue, 27 Aug 2013 21:52:18 -0700 Subject: [PATCH] go.tools/dashboard: implement dashboard using vcs package. To make the dashboard more flexible with different VCS, the Repo now depends on the VCS package for running commands. * Exported Repo.Master * Modified RemoteRepo to return a repo and an error * Reimplemented all Repo methods using vcs package * Removed hgCmd * Removed repoURL * Removed scheme from hgUrl since vcs.RepoRootForImportPath decides the scheme. * Changed waitWithTimeout into timeout * Added waitForFuncWithTimeout to wrap vcs commands with a timeout. R=adg CC=golang-dev https://golang.org/cl/13201043 --- dashboard/builder/exec.go | 51 +++++++++++++----- dashboard/builder/main.go | 54 ++++++++++++------- dashboard/builder/vcs.go | 109 +++++++++++++++++++++++--------------- 3 files changed, 138 insertions(+), 76 deletions(-) diff --git a/dashboard/builder/exec.go b/dashboard/builder/exec.go index a4aabd2842..b6f02f5180 100644 --- a/dashboard/builder/exec.go +++ b/dashboard/builder/exec.go @@ -15,7 +15,7 @@ import ( ) // run is a simple wrapper for exec.Run/Close -func run(timeout time.Duration, envv []string, dir string, argv ...string) error { +func run(d time.Duration, envv []string, dir string, argv ...string) error { if *verbose { log.Println("run", argv) } @@ -26,7 +26,15 @@ func run(timeout time.Duration, envv []string, dir string, argv ...string) error if err := cmd.Start(); err != nil { return err } - return waitWithTimeout(timeout, cmd) + return timeout(d, func() error { + if err := cmd.Wait(); err != nil { + if _, ok := err.(TimeoutErr); ok { + cmd.Process.Kill() + } + return err + } + return nil + }) } // runLog runs a process and returns the combined stdout/stderr. It returns @@ -42,7 +50,7 @@ func runLog(timeout time.Duration, envv []string, dir string, argv ...string) (s // runOutput runs a process and directs any output to the supplied writer. // It returns exit status and error. The error returned is nil, if process // is started successfully, even if exit status is not successful. -func runOutput(timeout time.Duration, envv []string, out io.Writer, dir string, argv ...string) (bool, error) { +func runOutput(d time.Duration, envv []string, out io.Writer, dir string, argv ...string) (bool, error) { if *verbose { log.Println("runOutput", argv) } @@ -57,23 +65,40 @@ func runOutput(timeout time.Duration, envv []string, out io.Writer, dir string, if startErr != nil { return false, startErr } - if err := waitWithTimeout(timeout, cmd); err != nil { + + if err := timeout(d, func() error { + if err := cmd.Wait(); err != nil { + if _, ok := err.(TimeoutErr); ok { + cmd.Process.Kill() + } + return err + } + return nil + }); err != nil { return false, err } return true, nil } -func waitWithTimeout(timeout time.Duration, cmd *exec.Cmd) error { - errc := make(chan error, 1) +// timeout runs f and returns its error value, or if the function does not +// complete before the provided duration it returns a timeout error. +func timeout(d time.Duration, f func() error) error { + errc := make(chan error) go func() { - errc <- cmd.Wait() + errc <- f() }() - var err error + t := time.NewTimer(d) + defer t.Stop() select { - case <-time.After(timeout): - cmd.Process.Kill() - err = fmt.Errorf("timed out after %v", timeout) - case err = <-errc: + case <-t.C: + return fmt.Errorf("timed out after %v", d) + case err := <-errc: + return err } - return err +} + +type TimeoutErr time.Duration + +func (e TimeoutErr) Error() string { + return fmt.Sprintf("timed out after %v", time.Duration(e)) } diff --git a/dashboard/builder/main.go b/dashboard/builder/main.go index 6ef357ad49..f63db8feff 100644 --- a/dashboard/builder/main.go +++ b/dashboard/builder/main.go @@ -17,12 +17,14 @@ import ( "runtime" "strings" "time" + + "code.google.com/p/go.tools/go/vcs" ) const ( codeProject = "go" codePyScript = "misc/dashboard/googlecode_upload.py" - hgUrl = "https://code.google.com/p/go/" + hgUrl = "code.google.com/p/go" mkdirPerm = 0750 waitInterval = 30 * time.Second // time to wait before checking for new revs pkgBuildInterval = 24 * time.Hour // rebuild packages every 24 hours @@ -85,8 +87,18 @@ func main() { if len(flag.Args()) == 0 { flag.Usage() } + + vcs.ShowCmd = *verbose + vcs.Verbose = *verbose + + rr, err := vcs.RepoRootForImportPath(hgUrl, *verbose) + if err != nil { + log.Fatal("Error finding repository:", err) + } + rootPath := filepath.Join(*buildroot, "goroot") goroot := &Repo{ - Path: filepath.Join(*buildroot, "goroot"), + Path: rootPath, + Master: rr, } // set up work environment, use existing enviroment if possible @@ -100,7 +112,12 @@ func main() { log.Fatalf("Error making build root (%s): %s", *buildroot, err) } var err error - goroot, err = RemoteRepo(hgUrl).Clone(goroot.Path, "tip") + goroot, err = RemoteRepo(hgUrl, rootPath) + if err != nil { + log.Fatalf("Error creating repository with url (%s): %s", hgUrl, err) + } + + goroot, err = goroot.Clone(goroot.Path, "tip") if err != nil { log.Fatal("Error cloning repository:", err) } @@ -253,13 +270,13 @@ func (b *Builder) buildHash(hash string) error { // create place in which to do work workpath := filepath.Join(*buildroot, b.name+"-"+hash[:12]) if err := os.Mkdir(workpath, mkdirPerm); err != nil { - return err + //return err } defer os.RemoveAll(workpath) // pull before cloning to ensure we have the revision if err := b.goroot.Pull(); err != nil { - return err + // return err } // clone repo at specified revision @@ -499,8 +516,13 @@ func commitWatcher(goroot *Repo) { commitPoll(goroot, "", key) // Go sub-repositories. for _, pkg := range dashboardPackages("subrepo") { + pkgmaster, err := vcs.RepoRootForImportPath(pkg, *verbose) + if err != nil { + log.Fatalf("Error finding subrepo (%s): %s", pkg, err) + } pkgroot := &Repo{ - Path: filepath.Join(*buildroot, pkg), + Path: filepath.Join(*buildroot, pkg), + Master: pkgmaster, } commitPoll(pkgroot, pkg, key) } @@ -518,9 +540,15 @@ var logByHash = map[string]*HgLog{} // commitPoll pulls any new revisions from the hg server // and tells the server about them. func commitPoll(repo *Repo, pkg, key string) { + pkgPath := filepath.Join(*buildroot, repo.Master.Root) if !repo.Exists() { var err error - repo, err = RemoteRepo(repoURL(pkg)).Clone(repo.Path, "tip") + repo, err = RemoteRepo(pkg, pkgPath) + if err != nil { + log.Printf("Error cloning package (%s): %s", pkg, err) + } + + repo, err = repo.Clone(repo.Path, "tip") if err != nil { log.Printf("%s: hg clone failed: %v", pkg, err) if err := os.RemoveAll(repo.Path); err != nil { @@ -600,18 +628,6 @@ func addCommit(pkg, hash, key string) bool { return true } -var repoRe = regexp.MustCompile(`^code\.google\.com/p/([a-z0-9\-]+(\.[a-z0-9\-]+)?)(/[a-z0-9A-Z_.\-/]+)?$`) - -// repoURL returns the repository URL for the supplied import path. -func repoURL(importPath string) string { - m := repoRe.FindStringSubmatch(importPath) - if len(m) < 2 { - log.Printf("repoURL: couldn't decipher %q", importPath) - return "" - } - return "https://code.google.com/p/" + m[1] -} - // defaultSuffix returns file extension used for command files in // current os environment. func defaultSuffix() string { diff --git a/dashboard/builder/vcs.go b/dashboard/builder/vcs.go index 63198a34bf..02ea7b7527 100644 --- a/dashboard/builder/vcs.go +++ b/dashboard/builder/vcs.go @@ -7,25 +7,31 @@ package main import ( "encoding/xml" "fmt" - "log" "os" "path/filepath" - "strconv" "strings" "sync" + + "code.google.com/p/go.tools/go/vcs" ) // Repo represents a mercurial repository. type Repo struct { - Path string + Path string + Master *vcs.RepoRoot sync.Mutex } // RemoteRepo constructs a *Repo representing a remote repository. -func RemoteRepo(url string) *Repo { - return &Repo{ - Path: url, +func RemoteRepo(url, path string) (*Repo, error) { + rr, err := vcs.RepoRootForImportPath(url, *verbose) + if err != nil { + return nil, err } + return &Repo{ + Path: path, + Master: rr, + }, nil } // Clone clones the current Repo to a new destination @@ -33,11 +39,20 @@ func RemoteRepo(url string) *Repo { func (r *Repo) Clone(path, rev string) (*Repo, error) { r.Lock() defer r.Unlock() - if err := run(*cmdTimeout, nil, *buildroot, r.hgCmd("clone", "-r", rev, r.Path, path)...); err != nil { + + err := timeout(*cmdTimeout, func() error { + err := r.Master.VCS.CreateAtRev(path, r.Master.Repo, rev) + if err != nil { + return err + } + return r.Master.VCS.TagSync(path, "") + }) + if err != nil { return nil, err } return &Repo{ - Path: path, + Path: path, + Master: r.Master, }, nil } @@ -46,12 +61,15 @@ func (r *Repo) Clone(path, rev string) (*Repo, error) { func (r *Repo) UpdateTo(hash string) error { r.Lock() defer r.Unlock() - return run(*cmdTimeout, nil, r.Path, r.hgCmd("update", hash)...) + + return timeout(*cmdTimeout, func() error { + return r.Master.VCS.TagSync(r.Path, hash) + }) } // Exists reports whether this Repo represents a valid Mecurial repository. func (r *Repo) Exists() bool { - fi, err := os.Stat(filepath.Join(r.Path, ".hg")) + fi, err := os.Stat(filepath.Join(r.Path, "."+r.Master.VCS.Cmd)) if err != nil { return false } @@ -63,7 +81,10 @@ func (r *Repo) Exists() bool { func (r *Repo) Pull() error { r.Lock() defer r.Unlock() - return run(*cmdTimeout, nil, r.Path, r.hgCmd("pull")...) + + return timeout(*cmdTimeout, func() error { + return r.Master.VCS.Download(r.Path) + }) } // Log returns the changelog for this repository. @@ -71,25 +92,25 @@ func (r *Repo) Log() ([]HgLog, error) { if err := r.Pull(); err != nil { return nil, err } - const N = 50 // how many revisions to grab - r.Lock() defer r.Unlock() - data, _, err := runLog(*cmdTimeout, nil, r.Path, r.hgCmd("log", - "--encoding=utf-8", - "--limit="+strconv.Itoa(N), - "--template="+xmlLogTemplate)..., - ) - if err != nil { - return nil, err - } var logStruct struct { Log []HgLog } - err = xml.Unmarshal([]byte(""+data+""), &logStruct) + err := timeout(*cmdTimeout, func() error { + data, err := r.Master.VCS.Log(r.Path, xmlLogTemplate) + if err != nil { + return err + } + + err = xml.Unmarshal([]byte(""+string(data)+""), &logStruct) + if err != nil { + return fmt.Errorf("unmarshal %s log: %v", r.Master.VCS, err) + } + return nil + }) if err != nil { - log.Printf("unmarshal hg log: %v", err) return nil, err } return logStruct.Log, nil @@ -99,28 +120,28 @@ func (r *Repo) Log() ([]HgLog, error) { func (r *Repo) FullHash(rev string) (string, error) { r.Lock() defer r.Unlock() - s, _, err := runLog(*cmdTimeout, nil, r.Path, - r.hgCmd("log", - "--encoding=utf-8", - "--rev="+rev, - "--limit=1", - "--template={node}")..., - ) - if err != nil { - return "", nil - } - s = strings.TrimSpace(s) - if s == "" { - return "", fmt.Errorf("cannot find revision") - } - if len(s) != 40 { - return "", fmt.Errorf("hg returned invalid hash " + s) - } - return s, nil -} -func (r *Repo) hgCmd(args ...string) []string { - return append([]string{"hg", "--config", "extensions.codereview=!"}, args...) + var hash string + err := timeout(*cmdTimeout, func() error { + data, err := r.Master.VCS.LogAtRev(r.Path, rev, "{node}") + if err != nil { + return err + } + + s := strings.TrimSpace(string(data)) + if s == "" { + return fmt.Errorf("cannot find revision") + } + if len(s) != 40 { + return fmt.Errorf("%s returned invalid hash: %s", r.Master.VCS, s) + } + hash = s + return nil + }) + if err != nil { + return "", err + } + return hash, nil } // HgLog represents a single Mercurial revision.