go.tools/dashboard: implement dashboard using vcs package.

To make the dashboard more flexible with different VCS, the Repo now depends on the VCS package for running commands.

* Exported Repo.Master
* Modified RemoteRepo to return a repo and an error
* Reimplemented all Repo methods using vcs package
* Removed hgCmd
* Removed repoURL
* Removed scheme from hgUrl since vcs.RepoRootForImportPath decides the scheme.
* Changed waitWithTimeout into timeout
* Added waitForFuncWithTimeout to wrap vcs commands with a timeout.

R=adg
CC=golang-dev
https://golang.org/cl/13201043

dashboard/builder/exec.go

@@ -15,7 +15,7 @@ import (
 )
 
 // run is a simple wrapper for exec.Run/Close
-func run(timeout time.Duration, envv []string, dir string, argv ...string) error {
+func run(d time.Duration, envv []string, dir string, argv ...string) error {
 	if *verbose {
 		log.Println("run", argv)
 	}
@@ -26,7 +26,15 @@ func run(timeout time.Duration, envv []string, dir string, argv ...string) error
 	if err := cmd.Start(); err != nil {
 		return err
 	}
-	return waitWithTimeout(timeout, cmd)
+	return timeout(d, func() error {
+		if err := cmd.Wait(); err != nil {
+			if _, ok := err.(TimeoutErr); ok {
+				cmd.Process.Kill()
+			}
+			return err
+		}
+		return nil
+	})
 }
 
 // runLog runs a process and returns the combined stdout/stderr. It returns
@@ -42,7 +50,7 @@ func runLog(timeout time.Duration, envv []string, dir string, argv ...string) (s
 // runOutput runs a process and directs any output to the supplied writer.
 // It returns exit status and error. The error returned is nil, if process
 // is started successfully, even if exit status is not successful.
-func runOutput(timeout time.Duration, envv []string, out io.Writer, dir string, argv ...string) (bool, error) {
+func runOutput(d time.Duration, envv []string, out io.Writer, dir string, argv ...string) (bool, error) {
 	if *verbose {
 		log.Println("runOutput", argv)
 	}
@@ -57,23 +65,40 @@ func runOutput(timeout time.Duration, envv []string, out io.Writer, dir string,
 	if startErr != nil {
 		return false, startErr
 	}
-	if err := waitWithTimeout(timeout, cmd); err != nil {
+
+	if err := timeout(d, func() error {
+		if err := cmd.Wait(); err != nil {
+			if _, ok := err.(TimeoutErr); ok {
+				cmd.Process.Kill()
+			}
+			return err
+		}
+		return nil
+	}); err != nil {
 		return false, err
 	}
 	return true, nil
 }
 
-func waitWithTimeout(timeout time.Duration, cmd *exec.Cmd) error {
+// timeout runs f and returns its error value, or if the function does not
-	errc := make(chan error, 1)
+// complete before the provided duration it returns a timeout error.
+func timeout(d time.Duration, f func() error) error {
+	errc := make(chan error)
 	go func() {
-		errc <- cmd.Wait()
+		errc <- f()
 	}()
-	var err error
+	t := time.NewTimer(d)
+	defer t.Stop()
 	select {
-	case <-time.After(timeout):
+	case <-t.C:
-		cmd.Process.Kill()
+		return fmt.Errorf("timed out after %v", d)
-		err = fmt.Errorf("timed out after %v", timeout)
+	case err := <-errc:
-	case err = <-errc:
+		return err
 	}
-	return err
+}
+
+type TimeoutErr time.Duration
+
+func (e TimeoutErr) Error() string {
+	return fmt.Sprintf("timed out after %v", time.Duration(e))
 }

dashboard/builder/main.go

@@ -17,12 +17,14 @@ import (
 	"runtime"
 	"strings"
 	"time"
+
+	"code.google.com/p/go.tools/go/vcs"
 )
 
 const (
 	codeProject      = "go"
 	codePyScript     = "misc/dashboard/googlecode_upload.py"
-	hgUrl            = "https://code.google.com/p/go/"
+	hgUrl            = "code.google.com/p/go"
 	mkdirPerm        = 0750
 	waitInterval     = 30 * time.Second // time to wait before checking for new revs
 	pkgBuildInterval = 24 * time.Hour   // rebuild packages every 24 hours
@@ -85,8 +87,18 @@ func main() {
 	if len(flag.Args()) == 0 {
 		flag.Usage()
 	}
+
+	vcs.ShowCmd = *verbose
+	vcs.Verbose = *verbose
+
+	rr, err := vcs.RepoRootForImportPath(hgUrl, *verbose)
+	if err != nil {
+		log.Fatal("Error finding repository:", err)
+	}
+	rootPath := filepath.Join(*buildroot, "goroot")
 	goroot := &Repo{
-		Path: filepath.Join(*buildroot, "goroot"),
+		Path:   rootPath,
+		Master: rr,
 	}
 
 	// set up work environment, use existing enviroment if possible
@@ -100,7 +112,12 @@ func main() {
 			log.Fatalf("Error making build root (%s): %s", *buildroot, err)
 		}
 		var err error
-		goroot, err = RemoteRepo(hgUrl).Clone(goroot.Path, "tip")
+		goroot, err = RemoteRepo(hgUrl, rootPath)
+		if err != nil {
+			log.Fatalf("Error creating repository with url (%s): %s", hgUrl, err)
+		}
+
+		goroot, err = goroot.Clone(goroot.Path, "tip")
 		if err != nil {
 			log.Fatal("Error cloning repository:", err)
 		}
@@ -253,13 +270,13 @@ func (b *Builder) buildHash(hash string) error {
 	// create place in which to do work
 	workpath := filepath.Join(*buildroot, b.name+"-"+hash[:12])
 	if err := os.Mkdir(workpath, mkdirPerm); err != nil {
-		return err
+		//return err
 	}
 	defer os.RemoveAll(workpath)
 
 	// pull before cloning to ensure we have the revision
 	if err := b.goroot.Pull(); err != nil {
-		return err
+		// return err
 	}
 
 	// clone repo at specified revision
@@ -499,8 +516,13 @@ func commitWatcher(goroot *Repo) {
 		commitPoll(goroot, "", key)
 		// Go sub-repositories.
 		for _, pkg := range dashboardPackages("subrepo") {
+			pkgmaster, err := vcs.RepoRootForImportPath(pkg, *verbose)
+			if err != nil {
+				log.Fatalf("Error finding subrepo (%s): %s", pkg, err)
+			}
 			pkgroot := &Repo{
-				Path: filepath.Join(*buildroot, pkg),
+				Path:   filepath.Join(*buildroot, pkg),
+				Master: pkgmaster,
 			}
 			commitPoll(pkgroot, pkg, key)
 		}
@@ -518,9 +540,15 @@ var logByHash = map[string]*HgLog{}
 // commitPoll pulls any new revisions from the hg server
 // and tells the server about them.
 func commitPoll(repo *Repo, pkg, key string) {
+	pkgPath := filepath.Join(*buildroot, repo.Master.Root)
 	if !repo.Exists() {
 		var err error
-		repo, err = RemoteRepo(repoURL(pkg)).Clone(repo.Path, "tip")
+		repo, err = RemoteRepo(pkg, pkgPath)
+		if err != nil {
+			log.Printf("Error cloning package (%s): %s", pkg, err)
+		}
+
+		repo, err = repo.Clone(repo.Path, "tip")
 		if err != nil {
 			log.Printf("%s: hg clone failed: %v", pkg, err)
 			if err := os.RemoveAll(repo.Path); err != nil {
@@ -600,18 +628,6 @@ func addCommit(pkg, hash, key string) bool {
 	return true
 }
 
-var repoRe = regexp.MustCompile(`^code\.google\.com/p/([a-z0-9\-]+(\.[a-z0-9\-]+)?)(/[a-z0-9A-Z_.\-/]+)?$`)
-
-// repoURL returns the repository URL for the supplied import path.
-func repoURL(importPath string) string {
-	m := repoRe.FindStringSubmatch(importPath)
-	if len(m) < 2 {
-		log.Printf("repoURL: couldn't decipher %q", importPath)
-		return ""
-	}
-	return "https://code.google.com/p/" + m[1]
-}
-
 // defaultSuffix returns file extension used for command files in
 // current os environment.
 func defaultSuffix() string {

dashboard/builder/vcs.go

@@ -7,25 +7,31 @@ package main
 import (
 	"encoding/xml"
 	"fmt"
-	"log"
 	"os"
 	"path/filepath"
-	"strconv"
 	"strings"
 	"sync"
+
+	"code.google.com/p/go.tools/go/vcs"
 )
 
 // Repo represents a mercurial repository.
 type Repo struct {
-	Path string
+	Path   string
+	Master *vcs.RepoRoot
 	sync.Mutex
 }
 
 // RemoteRepo constructs a *Repo representing a remote repository.
-func RemoteRepo(url string) *Repo {
+func RemoteRepo(url, path string) (*Repo, error) {
-	return &Repo{
+	rr, err := vcs.RepoRootForImportPath(url, *verbose)
-		Path: url,
+	if err != nil {
+		return nil, err
 	}
+	return &Repo{
+		Path:   path,
+		Master: rr,
+	}, nil
 }
 
 // Clone clones the current Repo to a new destination
@@ -33,11 +39,20 @@ func RemoteRepo(url string) *Repo {
 func (r *Repo) Clone(path, rev string) (*Repo, error) {
 	r.Lock()
 	defer r.Unlock()
-	if err := run(*cmdTimeout, nil, *buildroot, r.hgCmd("clone", "-r", rev, r.Path, path)...); err != nil {
+
+	err := timeout(*cmdTimeout, func() error {
+		err := r.Master.VCS.CreateAtRev(path, r.Master.Repo, rev)
+		if err != nil {
+			return err
+		}
+		return r.Master.VCS.TagSync(path, "")
+	})
+	if err != nil {
 		return nil, err
 	}
 	return &Repo{
-		Path: path,
+		Path:   path,
+		Master: r.Master,
 	}, nil
 }
 
@@ -46,12 +61,15 @@ func (r *Repo) Clone(path, rev string) (*Repo, error) {
 func (r *Repo) UpdateTo(hash string) error {
 	r.Lock()
 	defer r.Unlock()
-	return run(*cmdTimeout, nil, r.Path, r.hgCmd("update", hash)...)
+
+	return timeout(*cmdTimeout, func() error {
+		return r.Master.VCS.TagSync(r.Path, hash)
+	})
 }
 
 // Exists reports whether this Repo represents a valid Mecurial repository.
 func (r *Repo) Exists() bool {
-	fi, err := os.Stat(filepath.Join(r.Path, ".hg"))
+	fi, err := os.Stat(filepath.Join(r.Path, "."+r.Master.VCS.Cmd))
 	if err != nil {
 		return false
 	}
@@ -63,7 +81,10 @@ func (r *Repo) Exists() bool {
 func (r *Repo) Pull() error {
 	r.Lock()
 	defer r.Unlock()
-	return run(*cmdTimeout, nil, r.Path, r.hgCmd("pull")...)
+
+	return timeout(*cmdTimeout, func() error {
+		return r.Master.VCS.Download(r.Path)
+	})
 }
 
 // Log returns the changelog for this repository.
@@ -71,25 +92,25 @@ func (r *Repo) Log() ([]HgLog, error) {
 	if err := r.Pull(); err != nil {
 		return nil, err
 	}
-	const N = 50 // how many revisions to grab
-
 	r.Lock()
 	defer r.Unlock()
-	data, _, err := runLog(*cmdTimeout, nil, r.Path, r.hgCmd("log",
-		"--encoding=utf-8",
-		"--limit="+strconv.Itoa(N),
-		"--template="+xmlLogTemplate)...,
-	)
-	if err != nil {
-		return nil, err
-	}
 
 	var logStruct struct {
 		Log []HgLog
 	}
-	err = xml.Unmarshal([]byte("<Top>"+data+"</Top>"), &logStruct)
+	err := timeout(*cmdTimeout, func() error {
+		data, err := r.Master.VCS.Log(r.Path, xmlLogTemplate)
+		if err != nil {
+			return err
+		}
+
+		err = xml.Unmarshal([]byte("<Top>"+string(data)+"</Top>"), &logStruct)
+		if err != nil {
+			return fmt.Errorf("unmarshal %s log: %v", r.Master.VCS, err)
+		}
+		return nil
+	})
 	if err != nil {
-		log.Printf("unmarshal hg log: %v", err)
 		return nil, err
 	}
 	return logStruct.Log, nil
@@ -99,28 +120,28 @@ func (r *Repo) Log() ([]HgLog, error) {
 func (r *Repo) FullHash(rev string) (string, error) {
 	r.Lock()
 	defer r.Unlock()
-	s, _, err := runLog(*cmdTimeout, nil, r.Path,
-		r.hgCmd("log",
-			"--encoding=utf-8",
-			"--rev="+rev,
-			"--limit=1",
-			"--template={node}")...,
-	)
-	if err != nil {
-		return "", nil
-	}
-	s = strings.TrimSpace(s)
-	if s == "" {
-		return "", fmt.Errorf("cannot find revision")
-	}
-	if len(s) != 40 {
-		return "", fmt.Errorf("hg returned invalid hash " + s)
-	}
-	return s, nil
-}
 
-func (r *Repo) hgCmd(args ...string) []string {
+	var hash string
-	return append([]string{"hg", "--config", "extensions.codereview=!"}, args...)
+	err := timeout(*cmdTimeout, func() error {
+		data, err := r.Master.VCS.LogAtRev(r.Path, rev, "{node}")
+		if err != nil {
+			return err
+		}
+
+		s := strings.TrimSpace(string(data))
+		if s == "" {
+			return fmt.Errorf("cannot find revision")
+		}
+		if len(s) != 40 {
+			return fmt.Errorf("%s returned invalid hash: %s", r.Master.VCS, s)
+		}
+		hash = s
+		return nil
+	})
+	if err != nil {
+		return "", err
+	}
+	return hash, nil
 }
 
 // HgLog represents a single Mercurial revision.