diff --git a/dashboard/buildlet/buildletclient.go b/dashboard/buildlet/buildletclient.go index ef9c0a283aa..53b2c6abf17 100644 --- a/dashboard/buildlet/buildletclient.go +++ b/dashboard/buildlet/buildletclient.go @@ -102,12 +102,38 @@ func (c *Client) PutTarFromURL(tarURL, dir string) error { return c.doOK(req) } +// GetTar returns a .tar.gz stream of the given directory, relative to the buildlet's work dir. +// The provided dir may be empty to get everything. +func (c *Client) GetTar(dir string) (tgz io.ReadCloser, err error) { + req, err := http.NewRequest("GET", c.URL()+"/tgz?dir="+url.QueryEscape(dir), nil) + if err != nil { + return nil, err + } + res, err := c.do(req) + if err != nil { + return nil, err + } + if res.StatusCode != http.StatusOK { + slurp, _ := ioutil.ReadAll(io.LimitReader(res.Body, 4<<10)) + res.Body.Close() + return nil, fmt.Errorf("%v; body: %s", res.Status, slurp) + } + return res.Body, nil +} + // ExecOpts are options for a remote command invocation. type ExecOpts struct { // Output is the output of stdout and stderr. // If nil, the output is discarded. Output io.Writer + // Args are the arguments to pass to the cmd given to Client.Exec. + Args []string + + // SystemLevel controls whether the command is run outside of + // the buildlet's environment. + SystemLevel bool + // OnStartExec is an optional hook that runs after the 200 OK // response from the buildlet, but before the output begins // writing to Output. @@ -122,8 +148,14 @@ type ExecOpts struct { // seen to completition. If execErr is non-nil, the remoteErr is // meaningless. func (c *Client) Exec(cmd string, opts ExecOpts) (remoteErr, execErr error) { + var mode string + if opts.SystemLevel { + mode = "sys" + } form := url.Values{ - "cmd": {cmd}, + "cmd": {cmd}, + "mode": {mode}, + "cmdArg": opts.Args, } req, err := http.NewRequest("POST", c.URL()+"/exec", strings.NewReader(form.Encode())) if err != nil { diff --git a/dashboard/cmd/buildlet/Makefile b/dashboard/cmd/buildlet/Makefile index 078a4e9ae61..3edd8f8e51a 100644 --- a/dashboard/cmd/buildlet/Makefile +++ b/dashboard/cmd/buildlet/Makefile @@ -1,10 +1,22 @@ buildlet: buildlet.go go build --tags=extdep +buildlet.darwin-amd64: buildlet.go + GOOS=darwin GOARCH=amd64 go build -o $@ --tags=extdep + cat $@ | (cd ../upload && go run --tags=extdep upload.go --public go-builder-data/$@) + +buildlet.freebsd-amd64: buildlet.go + GOOS=freebsd GOARCH=amd64 go build -o $@ --tags=extdep + cat $@ | (cd ../upload && go run --tags=extdep upload.go --public go-builder-data/$@) + buildlet.linux-amd64: buildlet.go GOOS=linux GOARCH=amd64 go build -o $@ --tags=extdep cat $@ | (cd ../upload && go run --tags=extdep upload.go --public go-builder-data/$@) +buildlet.netbsd-amd64: buildlet.go + GOOS=netbsd GOARCH=amd64 go build -o $@ --tags=extdep + cat $@ | (cd ../upload && go run --tags=extdep upload.go --public go-builder-data/$@) + buildlet.openbsd-amd64: buildlet.go GOOS=openbsd GOARCH=amd64 go build -o $@ --tags=extdep cat $@ | (cd ../upload && go run --tags=extdep upload.go --public go-builder-data/$@) @@ -17,10 +29,3 @@ buildlet.windows-amd64: buildlet.go GOOS=windows GOARCH=amd64 go build -o $@ --tags=extdep cat $@ | (cd ../upload && go run --tags=extdep upload.go --public go-builder-data/$@) -buildlet.darwin-amd64: buildlet.go - GOOS=darwin GOARCH=amd64 go build -o $@ --tags=extdep - cat $@ | (cd ../upload && go run --tags=extdep upload.go --public go-builder-data/$@) - -buildlet.netbsd-amd64: buildlet.go - GOOS=netbsd GOARCH=amd64 go build -o $@ --tags=extdep - cat $@ | (cd ../upload && go run --tags=extdep upload.go --public go-builder-data/$@) diff --git a/dashboard/cmd/buildlet/buildlet.go b/dashboard/cmd/buildlet/buildlet.go index b4d3f7ef29a..95cc2bad840 100644 --- a/dashboard/cmd/buildlet/buildlet.go +++ b/dashboard/cmd/buildlet/buildlet.go @@ -29,10 +29,11 @@ import ( "net/url" "os" "os/exec" + "path" "path/filepath" "runtime" + "strconv" "strings" - "sync" "time" "google.golang.org/cloud/compute/metadata" @@ -66,9 +67,13 @@ var osHalt func() // set by some machines func main() { flag.Parse() - if !metadata.OnGCE() && !strings.HasPrefix(*listenAddr, "localhost:") { + onGCE := metadata.OnGCE() + if !onGCE && !strings.HasPrefix(*listenAddr, "localhost:") { log.Printf("** WARNING *** This server is unsafe and offers no security. Be careful.") } + if onGCE { + fixMTU() + } if runtime.GOOS == "plan9" { // Plan 9 is too slow on GCE, so stop running run.rc after the basics. // See https://golang.org/cl/2522 and https://golang.org/issue/9491 @@ -78,7 +83,6 @@ func main() { // But no need for environment variables quite yet. os.Setenv("GOTESTONLY", "std") } - if *scratchDir == "" { dir, err := ioutil.TempDir("", "buildlet-scatch") if err != nil { @@ -86,10 +90,21 @@ func main() { } *scratchDir = dir } + // TODO(bradfitz): if this becomes more of a general tool, + // perhaps we want to remove this hard-coded here. Also, + // if/once the exec handler ever gets generic environment + // variable support, it would make sense to remove this too + // and push it to the client. This hard-codes policy. But + // that's okay for now. + os.Setenv("GOROOT_BOOTSTRAP", filepath.Join(*scratchDir, "go1.4")) + os.Setenv("WORKDIR", *scratchDir) // mostly for demos + if _, err := os.Lstat(*scratchDir); err != nil { log.Fatalf("invalid --scratchdir %q: %v", *scratchDir, err) } http.HandleFunc("/", handleRoot) + http.HandleFunc("/debug/goroutines", handleGoroutines) + http.HandleFunc("/debug/x", handleX) password := metadataValue("password") requireAuth := func(handler func(w http.ResponseWriter, r *http.Request)) http.Handler { @@ -98,6 +113,7 @@ func main() { http.Handle("/writetgz", requireAuth(handleWriteTGZ)) http.Handle("/exec", requireAuth(handleExec)) http.Handle("/halt", requireAuth(handleHalt)) + http.Handle("/tgz", requireAuth(handleGetTGZ)) // TODO: removeall tlsCert, tlsKey := metadataValue("tls-cert"), metadataValue("tls-key") @@ -179,10 +195,193 @@ func (ln tcpKeepAliveListener) Accept() (c net.Conn, err error) { return tc, nil } +func fixMTU_freebsd() error { return fixMTU_ifconfig("vtnet0") } +func fixMTU_openbsd() error { return fixMTU_ifconfig("vio0") } +func fixMTU_ifconfig(iface string) error { + out, err := exec.Command("/sbin/ifconfig", iface, "mtu", "1460").CombinedOutput() + if err != nil { + return fmt.Errorf("/sbin/ifconfig %s mtu 1460: %v, %s", iface, err, out) + } + return nil +} + +func fixMTU_plan9() error { + f, err := os.OpenFile("/net/ipifc/0/ctl", os.O_WRONLY, 0) + if err != nil { + return err + } + if _, err := io.WriteString(f, "mtu 1400\n"); err != nil { // not 1460 + f.Close() + return err + } + return f.Close() +} + +func fixMTU() { + fn, ok := map[string]func() error{ + "openbsd": fixMTU_openbsd, + "freebsd": fixMTU_freebsd, + "plan9": fixMTU_plan9, + }[runtime.GOOS] + if ok { + if err := fn(); err != nil { + log.Printf("Failed to set MTU: %v", err) + } else { + log.Printf("Adjusted MTU.") + } + } +} + +// mtuWriter is a hack for environments where we can't (or can't yet) +// fix the machine's MTU. +// Instead of telling the operating system the MTU, we just cut up our +// writes into small pieces to make sure we don't get too near the +// MTU, and we hope the kernel doesn't coalesce different flushed +// writes back together into the same TCP IP packets. +type mtuWriter struct { + rw http.ResponseWriter +} + +func (mw mtuWriter) Write(p []byte) (n int, err error) { + const mtu = 1000 // way less than 1460; since HTTP response headers might be in there too + for len(p) > 0 { + chunk := p + if len(chunk) > mtu { + chunk = p[:mtu] + } + n0, err := mw.rw.Write(chunk) + n += n0 + if n0 != len(chunk) && err == nil { + err = io.ErrShortWrite + } + if err != nil { + return n, err + } + p = p[n0:] + mw.rw.(http.Flusher).Flush() + if len(p) > 0 { + // Whitelisted operating systems: + if runtime.GOOS == "openbsd" || runtime.GOOS == "linux" { + // Nothing + } else { + // Try to prevent the kernel from Nagel-ing the IP packets + // together into one that's too large. + time.Sleep(250 * time.Millisecond) + } + } + } + return n, nil +} + func handleRoot(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/" { + http.NotFound(w, r) + return + } fmt.Fprintf(w, "buildlet running on %s-%s\n", runtime.GOOS, runtime.GOARCH) } +// unauthenticated /debug/goroutines handler +func handleGoroutines(rw http.ResponseWriter, r *http.Request) { + w := mtuWriter{rw} + log.Printf("Dumping goroutines.") + rw.Header().Set("Content-Type", "text/plain; charset=utf-8") + buf := make([]byte, 2<<20) + buf = buf[:runtime.Stack(buf, true)] + w.Write(buf) + log.Printf("Dumped goroutines.") +} + +// unauthenticated /debug/x handler, to test MTU settings. +func handleX(w http.ResponseWriter, r *http.Request) { + n, _ := strconv.Atoi(r.FormValue("n")) + if n > 1<<20 { + n = 1 << 20 + } + log.Printf("Dumping %d X.", n) + w.Header().Set("Content-Type", "text/plain; charset=utf-8") + buf := make([]byte, n) + for i := range buf { + buf[i] = 'X' + } + w.Write(buf) + log.Printf("Dumped X.") +} + +// This is a remote code execution daemon, so security is kinda pointless, but: +func validRelativeDir(dir string) bool { + if strings.Contains(dir, `\`) || path.IsAbs(dir) { + return false + } + dir = path.Clean(dir) + if strings.HasPrefix(dir, "../") || strings.HasSuffix(dir, "/..") || dir == ".." { + return false + } + return true +} + +func handleGetTGZ(rw http.ResponseWriter, r *http.Request) { + if r.Method != "GET" { + http.Error(rw, "requires GET method", http.StatusBadRequest) + return + } + dir := r.FormValue("dir") + if !validRelativeDir(dir) { + http.Error(rw, "bogus dir", http.StatusBadRequest) + return + } + zw := gzip.NewWriter(mtuWriter{rw}) + tw := tar.NewWriter(zw) + base := filepath.Join(*scratchDir, filepath.FromSlash(dir)) + err := filepath.Walk(base, func(path string, fi os.FileInfo, err error) error { + if err != nil { + return err + } + rel := strings.TrimPrefix(strings.TrimPrefix(path, base), "/") + var linkName string + if fi.Mode()&os.ModeSymlink != 0 { + linkName, err = os.Readlink(path) + if err != nil { + return err + } + } + th, err := tar.FileInfoHeader(fi, linkName) + if err != nil { + return err + } + th.Name = rel + if fi.IsDir() && !strings.HasSuffix(th.Name, "/") { + th.Name += "/" + } + if th.Name == "/" { + return nil + } + if err := tw.WriteHeader(th); err != nil { + return err + } + if fi.Mode().IsRegular() { + f, err := os.Open(path) + if err != nil { + return err + } + defer f.Close() + if _, err := io.Copy(tw, f); err != nil { + return err + } + } + return nil + }) + if err != nil { + log.Printf("Walk error: %v", err) + // Decent way to signal failure to the caller, since it'll break + // the chunked response, rather than have a valid EOF. + conn, _, _ := rw.(http.Hijacker).Hijack() + conn.Close() + } + tw.Close() + zw.Close() +} + func handleWriteTGZ(w http.ResponseWriter, r *http.Request) { var tgz io.Reader switch r.Method { @@ -213,12 +412,11 @@ func handleWriteTGZ(w http.ResponseWriter, r *http.Request) { urlParam, _ := url.ParseQuery(r.URL.RawQuery) baseDir := *scratchDir if dir := urlParam.Get("dir"); dir != "" { - dir = filepath.FromSlash(dir) - if strings.Contains(dir, "../") { - // This is a remote code execution daemon, so security is kinda pointless, but: + if !validRelativeDir(dir) { http.Error(w, "bogus dir", http.StatusBadRequest) return } + dir = filepath.FromSlash(dir) baseDir = filepath.Join(baseDir, dir) if err := os.MkdirAll(baseDir, 0755); err != nil { http.Error(w, "mkdir of base: "+err.Error(), http.StatusInternalServerError) @@ -300,6 +498,11 @@ func untar(r io.Reader, dir string) error { const hdrProcessState = "Process-State" func handleExec(w http.ResponseWriter, r *http.Request) { + cn := w.(http.CloseNotifier) + clientGone := cn.CloseNotify() + handlerDone := make(chan bool) + defer close(handlerDone) + if r.Method != "POST" { http.Error(w, "requires POST method", http.StatusBadRequest) return @@ -313,21 +516,46 @@ func handleExec(w http.ResponseWriter, r *http.Request) { w.Header().Set("Trailer", hdrProcessState) // declare it so we can set it cmdPath := r.FormValue("cmd") // required - if !validRelPath(cmdPath) { - http.Error(w, "requires 'cmd' parameter", http.StatusBadRequest) - return + absCmd := cmdPath + sysMode := r.FormValue("mode") == "sys" + if sysMode { + if cmdPath == "" { + http.Error(w, "requires 'cmd' parameter", http.StatusBadRequest) + return + } + } else { + if !validRelPath(cmdPath) { + http.Error(w, "requires 'cmd' parameter", http.StatusBadRequest) + return + } + absCmd = filepath.Join(*scratchDir, filepath.FromSlash(cmdPath)) } + if f, ok := w.(http.Flusher); ok { f.Flush() } - absCmd := filepath.Join(*scratchDir, filepath.FromSlash(cmdPath)) cmd := exec.Command(absCmd, r.PostForm["cmdArg"]...) - cmd.Dir = filepath.Dir(absCmd) - cmdOutput := &flushWriter{w: w} + if sysMode { + cmd.Dir = *scratchDir + } else { + cmd.Dir = filepath.Dir(absCmd) + } + cmdOutput := mtuWriter{w} cmd.Stdout = cmdOutput cmd.Stderr = cmdOutput - err := cmd.Run() + err := cmd.Start() + if err == nil { + go func() { + select { + case <-clientGone: + cmd.Process.Kill() + case <-handlerDone: + return + } + }() + err = cmd.Wait() + } state := "ok" if err != nil { if ps := cmd.ProcessState; ps != nil { @@ -385,23 +613,6 @@ func haltMachine() { os.Exit(0) } -// flushWriter is an io.Writer wrapper that writes to w and -// Flushes the output immediately, if w is an http.Flusher. -type flushWriter struct { - mu sync.Mutex - w http.ResponseWriter -} - -func (hw *flushWriter) Write(p []byte) (n int, err error) { - hw.mu.Lock() - defer hw.mu.Unlock() - n, err = hw.w.Write(p) - if f, ok := hw.w.(http.Flusher); ok { - f.Flush() - } - return -} - func validRelPath(p string) bool { if p == "" || strings.Contains(p, `\`) || strings.HasPrefix(p, "/") || strings.Contains(p, "../") { return false diff --git a/dashboard/cmd/gomote/create.go b/dashboard/cmd/gomote/create.go index bb25a8e73e1..31806af01db 100644 --- a/dashboard/cmd/gomote/create.go +++ b/dashboard/cmd/gomote/create.go @@ -12,6 +12,7 @@ import ( "log" "os" "sort" + "strings" "time" "golang.org/x/tools/dashboard" @@ -36,13 +37,22 @@ func create(args []string) error { conf, ok := dashboard.Builders[builderType] if !ok || !conf.UsesVM() { var valid []string + var prefixMatch []string for k, conf := range dashboard.Builders { if conf.UsesVM() { valid = append(valid, k) + if strings.HasPrefix(k, builderType) { + prefixMatch = append(prefixMatch, k) + } } } - sort.Strings(valid) - return fmt.Errorf("Invalid builder type %q. Valid options include: %q", builderType, valid) + if len(prefixMatch) == 1 { + builderType = prefixMatch[0] + conf, _ = dashboard.Builders[builderType] + } else { + sort.Strings(valid) + return fmt.Errorf("Invalid builder type %q. Valid options include: %q", builderType, valid) + } } instName := fmt.Sprintf("mote-%s-%s", username(), builderType) diff --git a/dashboard/cmd/gomote/destroy.go b/dashboard/cmd/gomote/destroy.go index ca9b4dee2fb..1f758ef818c 100644 --- a/dashboard/cmd/gomote/destroy.go +++ b/dashboard/cmd/gomote/destroy.go @@ -7,9 +7,12 @@ package main import ( + "errors" "flag" "fmt" + "log" "os" + "time" "golang.org/x/tools/dashboard/buildlet" ) @@ -32,11 +35,47 @@ func destroy(args []string) error { return err } - // First ask it to kill itself, and then tell GCE to kill it too: - shutErr := bc.Destroy() - gceErr := buildlet.DestroyVM(projTokenSource(), *proj, *zone, fmt.Sprintf("mote-%s-%s", username(), name)) - if shutErr != nil { - return shutErr + // Ask it to kill itself, and tell GCE to kill it too: + gceErrc := make(chan error, 1) + buildletErrc := make(chan error, 1) + go func() { + gceErrc <- buildlet.DestroyVM(projTokenSource(), *proj, *zone, fmt.Sprintf("mote-%s-%s", username(), name)) + }() + go func() { + buildletErrc <- bc.Destroy() + }() + timeout := time.NewTimer(5 * time.Second) + defer timeout.Stop() + + var retErr error + var gceDone, buildletDone bool + for !gceDone || !buildletDone { + select { + case err := <-gceErrc: + if err != nil { + log.Printf("GCE: %v", err) + retErr = err + } else { + log.Printf("Requested GCE delete.") + } + gceDone = true + case err := <-buildletErrc: + if err != nil { + log.Printf("Buildlet: %v", err) + retErr = err + } else { + log.Printf("Requested buildlet to shut down.") + } + buildletDone = true + case <-timeout.C: + if !buildletDone { + log.Printf("timeout asking buildlet to shut down") + } + if !gceDone { + log.Printf("timeout asking GCE to delete builder VM") + } + return errors.New("timeout") + } } - return gceErr + return retErr } diff --git a/dashboard/cmd/gomote/get.go b/dashboard/cmd/gomote/get.go new file mode 100644 index 00000000000..f78ebd3d0cf --- /dev/null +++ b/dashboard/cmd/gomote/get.go @@ -0,0 +1,44 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build extdep + +package main + +import ( + "flag" + "fmt" + "io" + "os" +) + +// get a .tar.gz +func getTar(args []string) error { + fs := flag.NewFlagSet("get", flag.ContinueOnError) + fs.Usage = func() { + fmt.Fprintln(os.Stderr, "create usage: gomote gettar [get-opts] \n") + fs.PrintDefaults() + os.Exit(1) + } + var dir string + fs.StringVar(&dir, "dir", "", "relative directory from buildlet's work dir to tar up") + + fs.Parse(args) + if fs.NArg() != 1 { + fs.Usage() + } + + name := fs.Arg(0) + bc, err := namedClient(name) + if err != nil { + return err + } + tgz, err := bc.GetTar(dir) + if err != nil { + return err + } + defer tgz.Close() + _, err = io.Copy(os.Stdout, tgz) + return err +} diff --git a/dashboard/cmd/gomote/gomote.go b/dashboard/cmd/gomote/gomote.go index 7ede49496d6..79f4a335b4f 100644 --- a/dashboard/cmd/gomote/gomote.go +++ b/dashboard/cmd/gomote/gomote.go @@ -81,6 +81,7 @@ func registerCommands() { registerCommand("run", "run a command on a buildlet", run) registerCommand("put", "put files on a buildlet", put) registerCommand("puttar", "extract a tar.gz to a buildlet", putTar) + registerCommand("gettar", "extract a tar.gz from a buildlet", getTar) } func main() { diff --git a/dashboard/cmd/gomote/list.go b/dashboard/cmd/gomote/list.go index cd3d1f8ef89..c410f124da5 100644 --- a/dashboard/cmd/gomote/list.go +++ b/dashboard/cmd/gomote/list.go @@ -48,10 +48,21 @@ func namedClient(name string) (*buildlet.Client, error) { return nil, fmt.Errorf("error listing VMs while looking up %q: %v", name, err) } wantName := fmt.Sprintf("mote-%s-%s", username(), name) + var matches []buildlet.VM for _, vm := range vms { if vm.Name == wantName { return buildlet.NewClient(vm.IPPort, vm.TLS), nil } + if strings.HasPrefix(vm.Name, wantName) { + matches = append(matches, vm) + } + } + if len(matches) == 1 { + vm := matches[0] + return buildlet.NewClient(vm.IPPort, vm.TLS), nil + } + if len(matches) > 1 { + return nil, fmt.Errorf("prefix %q is ambiguous") } return nil, fmt.Errorf("buildlet %q not running", name) } diff --git a/dashboard/cmd/gomote/run.go b/dashboard/cmd/gomote/run.go index 6be9346c69d..ff2d47c2f1f 100644 --- a/dashboard/cmd/gomote/run.go +++ b/dashboard/cmd/gomote/run.go @@ -10,6 +10,7 @@ import ( "flag" "fmt" "os" + "strings" "golang.org/x/tools/dashboard/buildlet" ) @@ -17,10 +18,12 @@ import ( func run(args []string) error { fs := flag.NewFlagSet("run", flag.ContinueOnError) fs.Usage = func() { - fmt.Fprintln(os.Stderr, "create usage: gomote run [run-opts] [args...]") + fmt.Fprintln(os.Stderr, "create usage: gomote run [run-opts] [args...]") fs.PrintDefaults() os.Exit(1) } + var sys bool + fs.BoolVar(&sys, "system", false, "run inside the system, and not inside the workdir; this is implicit if cmd starts with '/'") fs.Parse(args) if fs.NArg() < 2 { @@ -33,7 +36,9 @@ func run(args []string) error { } remoteErr, execErr := bc.Exec(cmd, buildlet.ExecOpts{ - Output: os.Stdout, + SystemLevel: sys || strings.HasPrefix(cmd, "/"), + Output: os.Stdout, + Args: fs.Args()[2:], }) if execErr != nil { return fmt.Errorf("Error trying to execute %s: %v", cmd, execErr)