1
0
mirror of https://github.com/golang/go synced 2024-11-18 18:44:42 -07:00

dashboard: buildlet client, server, and gomote enhancements

- get tarballs out of buildlets
- gomote can pass arguments to buildlet's exec handler
- buildlet's exec handler can run system-level commands
- hard-code GOROOT_BOOTSTRAP to be "go1.4" under the workdir
- adjust MTU size on GCE

etc

Change-Id: I73e18b7a5e395a889f5ee93ba9850d331ffb7812
Reviewed-on: https://go-review.googlesource.com/3052
Reviewed-by: Andrew Gerrand <adg@golang.org>
This commit is contained in:
Brad Fitzpatrick 2015-01-19 20:53:34 -08:00
parent 4a084c7791
commit b34c44b0e2
9 changed files with 406 additions and 48 deletions

View File

@ -102,12 +102,38 @@ func (c *Client) PutTarFromURL(tarURL, dir string) error {
return c.doOK(req)
}
// GetTar returns a .tar.gz stream of the given directory, relative to the buildlet's work dir.
// The provided dir may be empty to get everything.
func (c *Client) GetTar(dir string) (tgz io.ReadCloser, err error) {
req, err := http.NewRequest("GET", c.URL()+"/tgz?dir="+url.QueryEscape(dir), nil)
if err != nil {
return nil, err
}
res, err := c.do(req)
if err != nil {
return nil, err
}
if res.StatusCode != http.StatusOK {
slurp, _ := ioutil.ReadAll(io.LimitReader(res.Body, 4<<10))
res.Body.Close()
return nil, fmt.Errorf("%v; body: %s", res.Status, slurp)
}
return res.Body, nil
}
// ExecOpts are options for a remote command invocation.
type ExecOpts struct {
// Output is the output of stdout and stderr.
// If nil, the output is discarded.
Output io.Writer
// Args are the arguments to pass to the cmd given to Client.Exec.
Args []string
// SystemLevel controls whether the command is run outside of
// the buildlet's environment.
SystemLevel bool
// OnStartExec is an optional hook that runs after the 200 OK
// response from the buildlet, but before the output begins
// writing to Output.
@ -122,8 +148,14 @@ type ExecOpts struct {
// seen to completition. If execErr is non-nil, the remoteErr is
// meaningless.
func (c *Client) Exec(cmd string, opts ExecOpts) (remoteErr, execErr error) {
var mode string
if opts.SystemLevel {
mode = "sys"
}
form := url.Values{
"cmd": {cmd},
"mode": {mode},
"cmdArg": opts.Args,
}
req, err := http.NewRequest("POST", c.URL()+"/exec", strings.NewReader(form.Encode()))
if err != nil {

View File

@ -1,10 +1,22 @@
buildlet: buildlet.go
go build --tags=extdep
buildlet.darwin-amd64: buildlet.go
GOOS=darwin GOARCH=amd64 go build -o $@ --tags=extdep
cat $@ | (cd ../upload && go run --tags=extdep upload.go --public go-builder-data/$@)
buildlet.freebsd-amd64: buildlet.go
GOOS=freebsd GOARCH=amd64 go build -o $@ --tags=extdep
cat $@ | (cd ../upload && go run --tags=extdep upload.go --public go-builder-data/$@)
buildlet.linux-amd64: buildlet.go
GOOS=linux GOARCH=amd64 go build -o $@ --tags=extdep
cat $@ | (cd ../upload && go run --tags=extdep upload.go --public go-builder-data/$@)
buildlet.netbsd-amd64: buildlet.go
GOOS=netbsd GOARCH=amd64 go build -o $@ --tags=extdep
cat $@ | (cd ../upload && go run --tags=extdep upload.go --public go-builder-data/$@)
buildlet.openbsd-amd64: buildlet.go
GOOS=openbsd GOARCH=amd64 go build -o $@ --tags=extdep
cat $@ | (cd ../upload && go run --tags=extdep upload.go --public go-builder-data/$@)
@ -17,10 +29,3 @@ buildlet.windows-amd64: buildlet.go
GOOS=windows GOARCH=amd64 go build -o $@ --tags=extdep
cat $@ | (cd ../upload && go run --tags=extdep upload.go --public go-builder-data/$@)
buildlet.darwin-amd64: buildlet.go
GOOS=darwin GOARCH=amd64 go build -o $@ --tags=extdep
cat $@ | (cd ../upload && go run --tags=extdep upload.go --public go-builder-data/$@)
buildlet.netbsd-amd64: buildlet.go
GOOS=netbsd GOARCH=amd64 go build -o $@ --tags=extdep
cat $@ | (cd ../upload && go run --tags=extdep upload.go --public go-builder-data/$@)

View File

@ -29,10 +29,11 @@ import (
"net/url"
"os"
"os/exec"
"path"
"path/filepath"
"runtime"
"strconv"
"strings"
"sync"
"time"
"google.golang.org/cloud/compute/metadata"
@ -66,9 +67,13 @@ var osHalt func() // set by some machines
func main() {
flag.Parse()
if !metadata.OnGCE() && !strings.HasPrefix(*listenAddr, "localhost:") {
onGCE := metadata.OnGCE()
if !onGCE && !strings.HasPrefix(*listenAddr, "localhost:") {
log.Printf("** WARNING *** This server is unsafe and offers no security. Be careful.")
}
if onGCE {
fixMTU()
}
if runtime.GOOS == "plan9" {
// Plan 9 is too slow on GCE, so stop running run.rc after the basics.
// See https://golang.org/cl/2522 and https://golang.org/issue/9491
@ -78,7 +83,6 @@ func main() {
// But no need for environment variables quite yet.
os.Setenv("GOTESTONLY", "std")
}
if *scratchDir == "" {
dir, err := ioutil.TempDir("", "buildlet-scatch")
if err != nil {
@ -86,10 +90,21 @@ func main() {
}
*scratchDir = dir
}
// TODO(bradfitz): if this becomes more of a general tool,
// perhaps we want to remove this hard-coded here. Also,
// if/once the exec handler ever gets generic environment
// variable support, it would make sense to remove this too
// and push it to the client. This hard-codes policy. But
// that's okay for now.
os.Setenv("GOROOT_BOOTSTRAP", filepath.Join(*scratchDir, "go1.4"))
os.Setenv("WORKDIR", *scratchDir) // mostly for demos
if _, err := os.Lstat(*scratchDir); err != nil {
log.Fatalf("invalid --scratchdir %q: %v", *scratchDir, err)
}
http.HandleFunc("/", handleRoot)
http.HandleFunc("/debug/goroutines", handleGoroutines)
http.HandleFunc("/debug/x", handleX)
password := metadataValue("password")
requireAuth := func(handler func(w http.ResponseWriter, r *http.Request)) http.Handler {
@ -98,6 +113,7 @@ func main() {
http.Handle("/writetgz", requireAuth(handleWriteTGZ))
http.Handle("/exec", requireAuth(handleExec))
http.Handle("/halt", requireAuth(handleHalt))
http.Handle("/tgz", requireAuth(handleGetTGZ))
// TODO: removeall
tlsCert, tlsKey := metadataValue("tls-cert"), metadataValue("tls-key")
@ -179,10 +195,193 @@ func (ln tcpKeepAliveListener) Accept() (c net.Conn, err error) {
return tc, nil
}
func fixMTU_freebsd() error { return fixMTU_ifconfig("vtnet0") }
func fixMTU_openbsd() error { return fixMTU_ifconfig("vio0") }
func fixMTU_ifconfig(iface string) error {
out, err := exec.Command("/sbin/ifconfig", iface, "mtu", "1460").CombinedOutput()
if err != nil {
return fmt.Errorf("/sbin/ifconfig %s mtu 1460: %v, %s", iface, err, out)
}
return nil
}
func fixMTU_plan9() error {
f, err := os.OpenFile("/net/ipifc/0/ctl", os.O_WRONLY, 0)
if err != nil {
return err
}
if _, err := io.WriteString(f, "mtu 1400\n"); err != nil { // not 1460
f.Close()
return err
}
return f.Close()
}
func fixMTU() {
fn, ok := map[string]func() error{
"openbsd": fixMTU_openbsd,
"freebsd": fixMTU_freebsd,
"plan9": fixMTU_plan9,
}[runtime.GOOS]
if ok {
if err := fn(); err != nil {
log.Printf("Failed to set MTU: %v", err)
} else {
log.Printf("Adjusted MTU.")
}
}
}
// mtuWriter is a hack for environments where we can't (or can't yet)
// fix the machine's MTU.
// Instead of telling the operating system the MTU, we just cut up our
// writes into small pieces to make sure we don't get too near the
// MTU, and we hope the kernel doesn't coalesce different flushed
// writes back together into the same TCP IP packets.
type mtuWriter struct {
rw http.ResponseWriter
}
func (mw mtuWriter) Write(p []byte) (n int, err error) {
const mtu = 1000 // way less than 1460; since HTTP response headers might be in there too
for len(p) > 0 {
chunk := p
if len(chunk) > mtu {
chunk = p[:mtu]
}
n0, err := mw.rw.Write(chunk)
n += n0
if n0 != len(chunk) && err == nil {
err = io.ErrShortWrite
}
if err != nil {
return n, err
}
p = p[n0:]
mw.rw.(http.Flusher).Flush()
if len(p) > 0 {
// Whitelisted operating systems:
if runtime.GOOS == "openbsd" || runtime.GOOS == "linux" {
// Nothing
} else {
// Try to prevent the kernel from Nagel-ing the IP packets
// together into one that's too large.
time.Sleep(250 * time.Millisecond)
}
}
}
return n, nil
}
func handleRoot(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/" {
http.NotFound(w, r)
return
}
fmt.Fprintf(w, "buildlet running on %s-%s\n", runtime.GOOS, runtime.GOARCH)
}
// unauthenticated /debug/goroutines handler
func handleGoroutines(rw http.ResponseWriter, r *http.Request) {
w := mtuWriter{rw}
log.Printf("Dumping goroutines.")
rw.Header().Set("Content-Type", "text/plain; charset=utf-8")
buf := make([]byte, 2<<20)
buf = buf[:runtime.Stack(buf, true)]
w.Write(buf)
log.Printf("Dumped goroutines.")
}
// unauthenticated /debug/x handler, to test MTU settings.
func handleX(w http.ResponseWriter, r *http.Request) {
n, _ := strconv.Atoi(r.FormValue("n"))
if n > 1<<20 {
n = 1 << 20
}
log.Printf("Dumping %d X.", n)
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
buf := make([]byte, n)
for i := range buf {
buf[i] = 'X'
}
w.Write(buf)
log.Printf("Dumped X.")
}
// This is a remote code execution daemon, so security is kinda pointless, but:
func validRelativeDir(dir string) bool {
if strings.Contains(dir, `\`) || path.IsAbs(dir) {
return false
}
dir = path.Clean(dir)
if strings.HasPrefix(dir, "../") || strings.HasSuffix(dir, "/..") || dir == ".." {
return false
}
return true
}
func handleGetTGZ(rw http.ResponseWriter, r *http.Request) {
if r.Method != "GET" {
http.Error(rw, "requires GET method", http.StatusBadRequest)
return
}
dir := r.FormValue("dir")
if !validRelativeDir(dir) {
http.Error(rw, "bogus dir", http.StatusBadRequest)
return
}
zw := gzip.NewWriter(mtuWriter{rw})
tw := tar.NewWriter(zw)
base := filepath.Join(*scratchDir, filepath.FromSlash(dir))
err := filepath.Walk(base, func(path string, fi os.FileInfo, err error) error {
if err != nil {
return err
}
rel := strings.TrimPrefix(strings.TrimPrefix(path, base), "/")
var linkName string
if fi.Mode()&os.ModeSymlink != 0 {
linkName, err = os.Readlink(path)
if err != nil {
return err
}
}
th, err := tar.FileInfoHeader(fi, linkName)
if err != nil {
return err
}
th.Name = rel
if fi.IsDir() && !strings.HasSuffix(th.Name, "/") {
th.Name += "/"
}
if th.Name == "/" {
return nil
}
if err := tw.WriteHeader(th); err != nil {
return err
}
if fi.Mode().IsRegular() {
f, err := os.Open(path)
if err != nil {
return err
}
defer f.Close()
if _, err := io.Copy(tw, f); err != nil {
return err
}
}
return nil
})
if err != nil {
log.Printf("Walk error: %v", err)
// Decent way to signal failure to the caller, since it'll break
// the chunked response, rather than have a valid EOF.
conn, _, _ := rw.(http.Hijacker).Hijack()
conn.Close()
}
tw.Close()
zw.Close()
}
func handleWriteTGZ(w http.ResponseWriter, r *http.Request) {
var tgz io.Reader
switch r.Method {
@ -213,12 +412,11 @@ func handleWriteTGZ(w http.ResponseWriter, r *http.Request) {
urlParam, _ := url.ParseQuery(r.URL.RawQuery)
baseDir := *scratchDir
if dir := urlParam.Get("dir"); dir != "" {
dir = filepath.FromSlash(dir)
if strings.Contains(dir, "../") {
// This is a remote code execution daemon, so security is kinda pointless, but:
if !validRelativeDir(dir) {
http.Error(w, "bogus dir", http.StatusBadRequest)
return
}
dir = filepath.FromSlash(dir)
baseDir = filepath.Join(baseDir, dir)
if err := os.MkdirAll(baseDir, 0755); err != nil {
http.Error(w, "mkdir of base: "+err.Error(), http.StatusInternalServerError)
@ -300,6 +498,11 @@ func untar(r io.Reader, dir string) error {
const hdrProcessState = "Process-State"
func handleExec(w http.ResponseWriter, r *http.Request) {
cn := w.(http.CloseNotifier)
clientGone := cn.CloseNotify()
handlerDone := make(chan bool)
defer close(handlerDone)
if r.Method != "POST" {
http.Error(w, "requires POST method", http.StatusBadRequest)
return
@ -313,21 +516,46 @@ func handleExec(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Trailer", hdrProcessState) // declare it so we can set it
cmdPath := r.FormValue("cmd") // required
absCmd := cmdPath
sysMode := r.FormValue("mode") == "sys"
if sysMode {
if cmdPath == "" {
http.Error(w, "requires 'cmd' parameter", http.StatusBadRequest)
return
}
} else {
if !validRelPath(cmdPath) {
http.Error(w, "requires 'cmd' parameter", http.StatusBadRequest)
return
}
absCmd = filepath.Join(*scratchDir, filepath.FromSlash(cmdPath))
}
if f, ok := w.(http.Flusher); ok {
f.Flush()
}
absCmd := filepath.Join(*scratchDir, filepath.FromSlash(cmdPath))
cmd := exec.Command(absCmd, r.PostForm["cmdArg"]...)
if sysMode {
cmd.Dir = *scratchDir
} else {
cmd.Dir = filepath.Dir(absCmd)
cmdOutput := &flushWriter{w: w}
}
cmdOutput := mtuWriter{w}
cmd.Stdout = cmdOutput
cmd.Stderr = cmdOutput
err := cmd.Run()
err := cmd.Start()
if err == nil {
go func() {
select {
case <-clientGone:
cmd.Process.Kill()
case <-handlerDone:
return
}
}()
err = cmd.Wait()
}
state := "ok"
if err != nil {
if ps := cmd.ProcessState; ps != nil {
@ -385,23 +613,6 @@ func haltMachine() {
os.Exit(0)
}
// flushWriter is an io.Writer wrapper that writes to w and
// Flushes the output immediately, if w is an http.Flusher.
type flushWriter struct {
mu sync.Mutex
w http.ResponseWriter
}
func (hw *flushWriter) Write(p []byte) (n int, err error) {
hw.mu.Lock()
defer hw.mu.Unlock()
n, err = hw.w.Write(p)
if f, ok := hw.w.(http.Flusher); ok {
f.Flush()
}
return
}
func validRelPath(p string) bool {
if p == "" || strings.Contains(p, `\`) || strings.HasPrefix(p, "/") || strings.Contains(p, "../") {
return false

View File

@ -12,6 +12,7 @@ import (
"log"
"os"
"sort"
"strings"
"time"
"golang.org/x/tools/dashboard"
@ -36,14 +37,23 @@ func create(args []string) error {
conf, ok := dashboard.Builders[builderType]
if !ok || !conf.UsesVM() {
var valid []string
var prefixMatch []string
for k, conf := range dashboard.Builders {
if conf.UsesVM() {
valid = append(valid, k)
if strings.HasPrefix(k, builderType) {
prefixMatch = append(prefixMatch, k)
}
}
}
if len(prefixMatch) == 1 {
builderType = prefixMatch[0]
conf, _ = dashboard.Builders[builderType]
} else {
sort.Strings(valid)
return fmt.Errorf("Invalid builder type %q. Valid options include: %q", builderType, valid)
}
}
instName := fmt.Sprintf("mote-%s-%s", username(), builderType)
client, err := buildlet.StartNewVM(projTokenSource(), instName, builderType, buildlet.VMOpts{

View File

@ -7,9 +7,12 @@
package main
import (
"errors"
"flag"
"fmt"
"log"
"os"
"time"
"golang.org/x/tools/dashboard/buildlet"
)
@ -32,11 +35,47 @@ func destroy(args []string) error {
return err
}
// First ask it to kill itself, and then tell GCE to kill it too:
shutErr := bc.Destroy()
gceErr := buildlet.DestroyVM(projTokenSource(), *proj, *zone, fmt.Sprintf("mote-%s-%s", username(), name))
if shutErr != nil {
return shutErr
// Ask it to kill itself, and tell GCE to kill it too:
gceErrc := make(chan error, 1)
buildletErrc := make(chan error, 1)
go func() {
gceErrc <- buildlet.DestroyVM(projTokenSource(), *proj, *zone, fmt.Sprintf("mote-%s-%s", username(), name))
}()
go func() {
buildletErrc <- bc.Destroy()
}()
timeout := time.NewTimer(5 * time.Second)
defer timeout.Stop()
var retErr error
var gceDone, buildletDone bool
for !gceDone || !buildletDone {
select {
case err := <-gceErrc:
if err != nil {
log.Printf("GCE: %v", err)
retErr = err
} else {
log.Printf("Requested GCE delete.")
}
return gceErr
gceDone = true
case err := <-buildletErrc:
if err != nil {
log.Printf("Buildlet: %v", err)
retErr = err
} else {
log.Printf("Requested buildlet to shut down.")
}
buildletDone = true
case <-timeout.C:
if !buildletDone {
log.Printf("timeout asking buildlet to shut down")
}
if !gceDone {
log.Printf("timeout asking GCE to delete builder VM")
}
return errors.New("timeout")
}
}
return retErr
}

View File

@ -0,0 +1,44 @@
// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// +build extdep
package main
import (
"flag"
"fmt"
"io"
"os"
)
// get a .tar.gz
func getTar(args []string) error {
fs := flag.NewFlagSet("get", flag.ContinueOnError)
fs.Usage = func() {
fmt.Fprintln(os.Stderr, "create usage: gomote gettar [get-opts] <buildlet-name>\n")
fs.PrintDefaults()
os.Exit(1)
}
var dir string
fs.StringVar(&dir, "dir", "", "relative directory from buildlet's work dir to tar up")
fs.Parse(args)
if fs.NArg() != 1 {
fs.Usage()
}
name := fs.Arg(0)
bc, err := namedClient(name)
if err != nil {
return err
}
tgz, err := bc.GetTar(dir)
if err != nil {
return err
}
defer tgz.Close()
_, err = io.Copy(os.Stdout, tgz)
return err
}

View File

@ -81,6 +81,7 @@ func registerCommands() {
registerCommand("run", "run a command on a buildlet", run)
registerCommand("put", "put files on a buildlet", put)
registerCommand("puttar", "extract a tar.gz to a buildlet", putTar)
registerCommand("gettar", "extract a tar.gz from a buildlet", getTar)
}
func main() {

View File

@ -48,10 +48,21 @@ func namedClient(name string) (*buildlet.Client, error) {
return nil, fmt.Errorf("error listing VMs while looking up %q: %v", name, err)
}
wantName := fmt.Sprintf("mote-%s-%s", username(), name)
var matches []buildlet.VM
for _, vm := range vms {
if vm.Name == wantName {
return buildlet.NewClient(vm.IPPort, vm.TLS), nil
}
if strings.HasPrefix(vm.Name, wantName) {
matches = append(matches, vm)
}
}
if len(matches) == 1 {
vm := matches[0]
return buildlet.NewClient(vm.IPPort, vm.TLS), nil
}
if len(matches) > 1 {
return nil, fmt.Errorf("prefix %q is ambiguous")
}
return nil, fmt.Errorf("buildlet %q not running", name)
}

View File

@ -10,6 +10,7 @@ import (
"flag"
"fmt"
"os"
"strings"
"golang.org/x/tools/dashboard/buildlet"
)
@ -17,10 +18,12 @@ import (
func run(args []string) error {
fs := flag.NewFlagSet("run", flag.ContinueOnError)
fs.Usage = func() {
fmt.Fprintln(os.Stderr, "create usage: gomote run [run-opts] <buildlet-name> <cmd> [args...]")
fmt.Fprintln(os.Stderr, "create usage: gomote run [run-opts] <instance> <cmd> [args...]")
fs.PrintDefaults()
os.Exit(1)
}
var sys bool
fs.BoolVar(&sys, "system", false, "run inside the system, and not inside the workdir; this is implicit if cmd starts with '/'")
fs.Parse(args)
if fs.NArg() < 2 {
@ -33,7 +36,9 @@ func run(args []string) error {
}
remoteErr, execErr := bc.Exec(cmd, buildlet.ExecOpts{
SystemLevel: sys || strings.HasPrefix(cmd, "/"),
Output: os.Stdout,
Args: fs.Args()[2:],
})
if execErr != nil {
return fmt.Errorf("Error trying to execute %s: %v", cmd, execErr)