1
0
mirror of https://github.com/golang/go synced 2024-11-18 12:44:49 -07:00

dashboard: rearrange tree, in prep for packagification

Package main binaries now go under cmd, leaving the top level for
similarly-named packages.

I'll be moving a lot of data structures and functions out of the
commands and into common packages.

A future CL will also unify all the "+build whatever" restrictions
into one common build tag, since there will be so much package sharing
coming up. Probably they will become "+build extdep", to indicate that
they have dependencies outside of x/tools.

Change-Id: Idc198e0dfa933b5f5de8f2b581533e8d299d2293
Reviewed-on: https://go-review.googlesource.com/2738
Reviewed-by: Andrew Gerrand <adg@golang.org>
This commit is contained in:
Brad Fitzpatrick 2015-01-14 17:24:45 -08:00
parent 9f20d8c198
commit e83451b9c2
31 changed files with 30 additions and 149 deletions

5
.gitignore vendored
View File

@ -1,5 +1,2 @@
# Add no patterns to .hgignore except for files generated by the build. # Add no patterns to .gitignore except for files generated by the build.
last-change last-change
dashboard/coordinator/buildongce/client-*.dat
dashboard/coordinator/buildongce/token.dat
dashboard/coordinator/coordinator

View File

@ -4,28 +4,37 @@
The files in these directories constitute the continuous builder: The files in these directories constitute the continuous builder:
app/: a.k.a the "dashboard"; the App Engine code that runs http://build.golang.org/ app/: a.k.a the "dashboard"; the App Engine code that runs http://build.golang.org/
buildlet/: HTTP server that runs on a VM and is told what to write to disk
cmd/:
buildlet/: HTTP server that runs on a VM and is told what to write to disk
and what command to run. This is cross-compiled to different architectures and what command to run. This is cross-compiled to different architectures
and is the first program run when a builder VM comes up. It then and is the first program run when a builder VM comes up. It then
is contacted by the coordinator to do a build. Not all builders use is contacted by the coordinator to do a build. Not all builders use
the buildlet (at least not yet). the buildlet (at least not yet).
builder/: gobuilder, a Go continuous build client. The original Go builder program.
coordinator/: daemon that runs on CoreOS on Google Compute Engine and manages builder/: gobuilder, a Go continuous build client. The original Go builder program.
coordinator/: daemon that runs on CoreOS on Google Compute Engine and manages
builds using Docker containers and/or VMs as needed. builds using Docker containers and/or VMs as needed.
retrybuilds/: a Go client program to delete build results from the dashboard (app)
upload/: a Go program to upload to Google Cloud Storage. used by Makefiles elsewhere.
watcher/: a daemon that watches for new commits to the Go repository and
its sub-repositories, and notifies the dashboard of those commits.
env/: configuration files describing the environment of builders and related env/: configuration files describing the environment of builders and related
binaries. Many builders are still configured ad-hoc, without a hermetic binaries. Many builders are still configured ad-hoc, without a hermetic
environment. environment.
retrybuilds/: a Go client program to delete build results from the dashboard (app)
types/: a Go package contain common types used by other pieces. types/: a Go package contain common types used by other pieces.
upload/: a Go program to upload to Google Cloud Storage. used by Makefiles elsewhere.
watcher/: a daemon that watches for new commits to the Go repository and
its sub-repositories, and notifies the dashboard of those commits.
If you wish to run a Go builder, please email golang-dev@googlegroups.com first. If you wish to run a Go builder, please email golang-dev@googlegroups.com first.
There is documentation at https://golang.org/wiki/DashboardBuilders but There is documentation at https://golang.org/wiki/DashboardBuilders but
depending on the type of builder, we may want to run it ourselves, after you depending on the type of builder, we may want to run it ourselves, after you
prepare an environment description (resulting in a VM image) of it. See the env prepare an environment description (resulting in a VM image) of it. See the env
directory. directory.

View File

@ -55,4 +55,4 @@ If the Google Code credentials are not provided the archival step
will be skipped. will be skipped.
*/ */
package main // import "golang.org/x/tools/dashboard/builder" package main // import "golang.org/x/tools/dashboard/cmd/builder"

3
dashboard/cmd/coordinator/.gitignore vendored Normal file
View File

@ -0,0 +1,3 @@
buildongce/client-*.dat
buildongce/token.dat
coordinator

View File

@ -6,4 +6,4 @@ docker: Dockerfile
docker build -t go-commit-watcher . docker build -t go-commit-watcher .
docker-commit-watcher.tar.gz: docker docker-commit-watcher.tar.gz: docker
docker save go-commit-watcher | gzip | (cd ../../upload && go run upload.go --public go-builder-data/docker-commit-watcher.tar.gz) docker save go-commit-watcher | gzip | (cd ../../cmd/upload && go run upload.go --public go-builder-data/docker-commit-watcher.tar.gz)

View File

@ -6,7 +6,7 @@ docker: Dockerfile
docker build -t gobuilders/linux-x86-base . docker build -t gobuilders/linux-x86-base .
docker-linux.base.tar.gz: docker docker-linux.base.tar.gz: docker
docker save gobuilders/linux-x86-base | gzip | (cd ../../upload && go run upload.go --public go-builder-data/docker-linux.base.tar.gz) docker save gobuilders/linux-x86-base | gzip | (cd ../../cmd/upload && go run upload.go --public go-builder-data/docker-linux.base.tar.gz)
check: docker check: docker
docker run -e GOROOT_BOOTSTRAP=/go1.4-amd64/go gobuilders/linux-x86-base /usr/local/bin/builder -rev=20a10e7ddd1 -buildroot=/ -v -report=false linux-amd64-temp docker run -e GOROOT_BOOTSTRAP=/go1.4-amd64/go gobuilders/linux-x86-base /usr/local/bin/builder -rev=20a10e7ddd1 -buildroot=/ -v -report=false linux-amd64-temp

View File

@ -6,7 +6,7 @@ docker: Dockerfile
docker build -t gobuilders/linux-x86-clang . docker build -t gobuilders/linux-x86-clang .
docker-linux.clang.tar.gz: docker docker-linux.clang.tar.gz: docker
docker save gobuilders/linux-x86-clang | gzip | (cd ../../upload && go run upload.go --public go-builder-data/docker-linux.clang.tar.gz) docker save gobuilders/linux-x86-clang | gzip | (cd ../../cmd/upload && go run upload.go --public go-builder-data/docker-linux.clang.tar.gz)
check: docker check: docker
docker run -e GOROOT_BOOTSTRAP=/go1.4-amd64/go gobuilders/linux-x86-clang /usr/local/bin/builder -rev=20a10e7ddd1b -buildroot=/ -v -report=false linux-amd64-temp docker run -e GOROOT_BOOTSTRAP=/go1.4-amd64/go gobuilders/linux-x86-clang /usr/local/bin/builder -rev=20a10e7ddd1b -buildroot=/ -v -report=false linux-amd64-temp

View File

@ -6,7 +6,7 @@ docker: Dockerfile
docker build -t gobuilders/linux-x86-gccgo . docker build -t gobuilders/linux-x86-gccgo .
docker-linux.gccgo.tar.gz: docker docker-linux.gccgo.tar.gz: docker
docker save gobuilders/linux-x86-gccgo | gzip | (cd ../../upload && go run upload.go --public go-builder-data/docker-linux.gccgo.tar.gz) docker save gobuilders/linux-x86-gccgo | gzip | (cd ../../cmd/upload && go run upload.go --public go-builder-data/docker-linux.gccgo.tar.gz)
check: docker check: docker
docker run gobuilders/linux-x86-gccgo /usr/local/bin/builder -tool="gccgo" -rev=b9151e911a54 -v -cmd='make RUNTESTFLAGS="--target_board=unix/-m64" check-go' -report=false linux-amd64-gccgo-temp docker run gobuilders/linux-x86-gccgo /usr/local/bin/builder -tool="gccgo" -rev=b9151e911a54 -v -cmd='make RUNTESTFLAGS="--target_board=unix/-m64" check-go' -report=false linux-amd64-gccgo-temp

View File

@ -6,7 +6,7 @@ docker: Dockerfile
docker build -t gobuilders/linux-x86-nacl . docker build -t gobuilders/linux-x86-nacl .
upload: docker upload: docker
docker save gobuilders/linux-x86-nacl | gzip | (cd ../../upload && go run upload.go --public go-builder-data/docker-linux.nacl.tar.gz) docker save gobuilders/linux-x86-nacl | gzip | (cd ../../cmd/upload && go run upload.go --public go-builder-data/docker-linux.nacl.tar.gz)
check: docker check: docker
docker run gobuilders/linux-x86-nacl /usr/local/bin/builder -rev=77e96c9208d0 -buildroot=/ -v -cmd=/usr/local/bin/build-command.pl -report=false nacl-amd64p32 docker run gobuilders/linux-x86-nacl /usr/local/bin/builder -rev=77e96c9208d0 -buildroot=/ -v -cmd=/usr/local/bin/build-command.pl -report=false nacl-amd64p32

View File

@ -6,7 +6,7 @@ docker: Dockerfile
docker build -t gobuilders/linux-x86-sid . docker build -t gobuilders/linux-x86-sid .
docker-linux.sid.tar.gz: docker docker-linux.sid.tar.gz: docker
docker save gobuilders/linux-x86-sid | gzip | (cd ../../upload && go run upload.go --public go-builder-data/docker-linux.sid.tar.gz) docker save gobuilders/linux-x86-sid | gzip | (cd ../../cmd/upload && go run upload.go --public go-builder-data/docker-linux.sid.tar.gz)
check: docker check: docker
docker run -e GOROOT_BOOTSTRAP=/go1.4-amd64/go gobuilders/linux-x86-sid /usr/local/bin/builder -rev=20a10e7ddd1b -buildroot=/ -v -report=false linux-amd64-sid docker run -e GOROOT_BOOTSTRAP=/go1.4-amd64/go gobuilders/linux-x86-sid /usr/local/bin/builder -rev=20a10e7ddd1b -buildroot=/ -v -report=false linux-amd64-sid

View File

@ -1,128 +0,0 @@
// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package main // import "golang.org/x/tools/dashboard/updater"
import (
"bytes"
"encoding/json"
"encoding/xml"
"flag"
"fmt"
"io/ioutil"
"net/http"
"net/url"
"os"
"os/exec"
"strings"
)
var (
builder = flag.String("builder", "", "builder name")
key = flag.String("key", "", "builder key")
gopath = flag.String("gopath", "", "path to go repo")
dashboard = flag.String("dashboard", "build.golang.org", "Go Dashboard Host")
batch = flag.Int("batch", 100, "upload batch size")
)
// Do not benchmark beyond this commit.
// There is little sense in benchmarking till first commit,
// and the benchmark won't build anyway.
const Go1Commit = "0051c7442fed" // test/bench/shootout: update timing.log to Go 1.
// HgLog represents a single Mercurial revision.
type HgLog struct {
Hash string
Branch string
Files string
}
func main() {
flag.Parse()
logs := hgLog()
var hashes []string
ngo1 := 0
for i := range logs {
if strings.HasPrefix(logs[i].Hash, Go1Commit) {
break
}
if needsBenchmarking(&logs[i]) {
hashes = append(hashes, logs[i].Hash)
}
ngo1++
}
fmt.Printf("found %v commits, %v after Go1, %v need benchmarking\n", len(logs), ngo1, len(hashes))
for i := 0; i < len(hashes); i += *batch {
j := i + *batch
if j > len(hashes) {
j = len(hashes)
}
fmt.Printf("sending %v-%v... ", i, j)
res := postCommits(hashes[i:j])
fmt.Printf("%s\n", res)
}
}
func hgLog() []HgLog {
var out bytes.Buffer
cmd := exec.Command("hg", "log", "--encoding=utf-8", "--template", xmlLogTemplate)
cmd.Dir = *gopath
cmd.Stdout = &out
cmd.Stderr = os.Stderr
err := cmd.Run()
if err != nil {
fmt.Printf("failed to execute 'hg log': %v\n", err)
os.Exit(1)
}
var top struct{ Log []HgLog }
err = xml.Unmarshal([]byte("<Top>"+out.String()+"</Top>"), &top)
if err != nil {
fmt.Printf("failed to parse log: %v\n", err)
os.Exit(1)
}
return top.Log
}
func needsBenchmarking(log *HgLog) bool {
if log.Branch != "" {
return false
}
for _, f := range strings.Split(log.Files, " ") {
if (strings.HasPrefix(f, "include") || strings.HasPrefix(f, "src")) &&
!strings.HasSuffix(f, "_test.go") && !strings.Contains(f, "testdata") {
return true
}
}
return false
}
func postCommits(hashes []string) string {
args := url.Values{"builder": {*builder}, "key": {*key}}
cmd := fmt.Sprintf("http://%v/updatebenchmark?%v", *dashboard, args.Encode())
b, err := json.Marshal(hashes)
if err != nil {
return fmt.Sprintf("failed to encode request: %v\n", err)
}
r, err := http.Post(cmd, "text/json", bytes.NewReader(b))
if err != nil {
return fmt.Sprintf("failed to send http request: %v\n", err)
}
defer r.Body.Close()
if r.StatusCode != http.StatusOK {
return fmt.Sprintf("http request failed: %v\n", r.Status)
}
resp, err := ioutil.ReadAll(r.Body)
if err != nil {
return fmt.Sprintf("failed to read http response: %v\n", err)
}
return string(resp)
}
const xmlLogTemplate = `
<Log>
<Hash>{node|escape}</Hash>
<Branch>{branches}</Branch>
<Files>{files}</Files>
</Log>
`