mirror of
https://github.com/golang/go
synced 2024-11-18 12:44:49 -07:00
dashboard: rearrange tree, in prep for packagification
Package main binaries now go under cmd, leaving the top level for similarly-named packages. I'll be moving a lot of data structures and functions out of the commands and into common packages. A future CL will also unify all the "+build whatever" restrictions into one common build tag, since there will be so much package sharing coming up. Probably they will become "+build extdep", to indicate that they have dependencies outside of x/tools. Change-Id: Idc198e0dfa933b5f5de8f2b581533e8d299d2293 Reviewed-on: https://go-review.googlesource.com/2738 Reviewed-by: Andrew Gerrand <adg@golang.org>
This commit is contained in:
parent
9f20d8c198
commit
e83451b9c2
5
.gitignore
vendored
5
.gitignore
vendored
@ -1,5 +1,2 @@
|
|||||||
# Add no patterns to .hgignore except for files generated by the build.
|
# Add no patterns to .gitignore except for files generated by the build.
|
||||||
last-change
|
last-change
|
||||||
dashboard/coordinator/buildongce/client-*.dat
|
|
||||||
dashboard/coordinator/buildongce/token.dat
|
|
||||||
dashboard/coordinator/coordinator
|
|
||||||
|
@ -4,28 +4,37 @@
|
|||||||
|
|
||||||
The files in these directories constitute the continuous builder:
|
The files in these directories constitute the continuous builder:
|
||||||
|
|
||||||
app/: a.k.a the "dashboard"; the App Engine code that runs http://build.golang.org/
|
app/: a.k.a the "dashboard"; the App Engine code that runs http://build.golang.org/
|
||||||
buildlet/: HTTP server that runs on a VM and is told what to write to disk
|
|
||||||
|
cmd/:
|
||||||
|
|
||||||
|
buildlet/: HTTP server that runs on a VM and is told what to write to disk
|
||||||
and what command to run. This is cross-compiled to different architectures
|
and what command to run. This is cross-compiled to different architectures
|
||||||
and is the first program run when a builder VM comes up. It then
|
and is the first program run when a builder VM comes up. It then
|
||||||
is contacted by the coordinator to do a build. Not all builders use
|
is contacted by the coordinator to do a build. Not all builders use
|
||||||
the buildlet (at least not yet).
|
the buildlet (at least not yet).
|
||||||
builder/: gobuilder, a Go continuous build client. The original Go builder program.
|
|
||||||
coordinator/: daemon that runs on CoreOS on Google Compute Engine and manages
|
builder/: gobuilder, a Go continuous build client. The original Go builder program.
|
||||||
|
|
||||||
|
coordinator/: daemon that runs on CoreOS on Google Compute Engine and manages
|
||||||
builds using Docker containers and/or VMs as needed.
|
builds using Docker containers and/or VMs as needed.
|
||||||
|
|
||||||
|
retrybuilds/: a Go client program to delete build results from the dashboard (app)
|
||||||
|
|
||||||
|
upload/: a Go program to upload to Google Cloud Storage. used by Makefiles elsewhere.
|
||||||
|
|
||||||
|
watcher/: a daemon that watches for new commits to the Go repository and
|
||||||
|
its sub-repositories, and notifies the dashboard of those commits.
|
||||||
|
|
||||||
env/: configuration files describing the environment of builders and related
|
env/: configuration files describing the environment of builders and related
|
||||||
binaries. Many builders are still configured ad-hoc, without a hermetic
|
binaries. Many builders are still configured ad-hoc, without a hermetic
|
||||||
environment.
|
environment.
|
||||||
retrybuilds/: a Go client program to delete build results from the dashboard (app)
|
|
||||||
types/: a Go package contain common types used by other pieces.
|
types/: a Go package contain common types used by other pieces.
|
||||||
upload/: a Go program to upload to Google Cloud Storage. used by Makefiles elsewhere.
|
|
||||||
watcher/: a daemon that watches for new commits to the Go repository and
|
|
||||||
its sub-repositories, and notifies the dashboard of those commits.
|
|
||||||
|
|
||||||
If you wish to run a Go builder, please email golang-dev@googlegroups.com first.
|
If you wish to run a Go builder, please email golang-dev@googlegroups.com first.
|
||||||
There is documentation at https://golang.org/wiki/DashboardBuilders but
|
There is documentation at https://golang.org/wiki/DashboardBuilders but
|
||||||
depending on the type of builder, we may want to run it ourselves, after you
|
depending on the type of builder, we may want to run it ourselves, after you
|
||||||
prepare an environment description (resulting in a VM image) of it. See the env
|
prepare an environment description (resulting in a VM image) of it. See the env
|
||||||
directory.
|
directory.
|
||||||
|
|
||||||
|
|
||||||
|
@ -55,4 +55,4 @@ If the Google Code credentials are not provided the archival step
|
|||||||
will be skipped.
|
will be skipped.
|
||||||
|
|
||||||
*/
|
*/
|
||||||
package main // import "golang.org/x/tools/dashboard/builder"
|
package main // import "golang.org/x/tools/dashboard/cmd/builder"
|
3
dashboard/cmd/coordinator/.gitignore
vendored
Normal file
3
dashboard/cmd/coordinator/.gitignore
vendored
Normal file
@ -0,0 +1,3 @@
|
|||||||
|
buildongce/client-*.dat
|
||||||
|
buildongce/token.dat
|
||||||
|
coordinator
|
2
dashboard/env/commit-watcher/Makefile
vendored
2
dashboard/env/commit-watcher/Makefile
vendored
@ -6,4 +6,4 @@ docker: Dockerfile
|
|||||||
docker build -t go-commit-watcher .
|
docker build -t go-commit-watcher .
|
||||||
|
|
||||||
docker-commit-watcher.tar.gz: docker
|
docker-commit-watcher.tar.gz: docker
|
||||||
docker save go-commit-watcher | gzip | (cd ../../upload && go run upload.go --public go-builder-data/docker-commit-watcher.tar.gz)
|
docker save go-commit-watcher | gzip | (cd ../../cmd/upload && go run upload.go --public go-builder-data/docker-commit-watcher.tar.gz)
|
||||||
|
2
dashboard/env/linux-x86-base/Makefile
vendored
2
dashboard/env/linux-x86-base/Makefile
vendored
@ -6,7 +6,7 @@ docker: Dockerfile
|
|||||||
docker build -t gobuilders/linux-x86-base .
|
docker build -t gobuilders/linux-x86-base .
|
||||||
|
|
||||||
docker-linux.base.tar.gz: docker
|
docker-linux.base.tar.gz: docker
|
||||||
docker save gobuilders/linux-x86-base | gzip | (cd ../../upload && go run upload.go --public go-builder-data/docker-linux.base.tar.gz)
|
docker save gobuilders/linux-x86-base | gzip | (cd ../../cmd/upload && go run upload.go --public go-builder-data/docker-linux.base.tar.gz)
|
||||||
|
|
||||||
check: docker
|
check: docker
|
||||||
docker run -e GOROOT_BOOTSTRAP=/go1.4-amd64/go gobuilders/linux-x86-base /usr/local/bin/builder -rev=20a10e7ddd1 -buildroot=/ -v -report=false linux-amd64-temp
|
docker run -e GOROOT_BOOTSTRAP=/go1.4-amd64/go gobuilders/linux-x86-base /usr/local/bin/builder -rev=20a10e7ddd1 -buildroot=/ -v -report=false linux-amd64-temp
|
||||||
|
2
dashboard/env/linux-x86-clang/Makefile
vendored
2
dashboard/env/linux-x86-clang/Makefile
vendored
@ -6,7 +6,7 @@ docker: Dockerfile
|
|||||||
docker build -t gobuilders/linux-x86-clang .
|
docker build -t gobuilders/linux-x86-clang .
|
||||||
|
|
||||||
docker-linux.clang.tar.gz: docker
|
docker-linux.clang.tar.gz: docker
|
||||||
docker save gobuilders/linux-x86-clang | gzip | (cd ../../upload && go run upload.go --public go-builder-data/docker-linux.clang.tar.gz)
|
docker save gobuilders/linux-x86-clang | gzip | (cd ../../cmd/upload && go run upload.go --public go-builder-data/docker-linux.clang.tar.gz)
|
||||||
|
|
||||||
check: docker
|
check: docker
|
||||||
docker run -e GOROOT_BOOTSTRAP=/go1.4-amd64/go gobuilders/linux-x86-clang /usr/local/bin/builder -rev=20a10e7ddd1b -buildroot=/ -v -report=false linux-amd64-temp
|
docker run -e GOROOT_BOOTSTRAP=/go1.4-amd64/go gobuilders/linux-x86-clang /usr/local/bin/builder -rev=20a10e7ddd1b -buildroot=/ -v -report=false linux-amd64-temp
|
||||||
|
2
dashboard/env/linux-x86-gccgo/Makefile
vendored
2
dashboard/env/linux-x86-gccgo/Makefile
vendored
@ -6,7 +6,7 @@ docker: Dockerfile
|
|||||||
docker build -t gobuilders/linux-x86-gccgo .
|
docker build -t gobuilders/linux-x86-gccgo .
|
||||||
|
|
||||||
docker-linux.gccgo.tar.gz: docker
|
docker-linux.gccgo.tar.gz: docker
|
||||||
docker save gobuilders/linux-x86-gccgo | gzip | (cd ../../upload && go run upload.go --public go-builder-data/docker-linux.gccgo.tar.gz)
|
docker save gobuilders/linux-x86-gccgo | gzip | (cd ../../cmd/upload && go run upload.go --public go-builder-data/docker-linux.gccgo.tar.gz)
|
||||||
|
|
||||||
check: docker
|
check: docker
|
||||||
docker run gobuilders/linux-x86-gccgo /usr/local/bin/builder -tool="gccgo" -rev=b9151e911a54 -v -cmd='make RUNTESTFLAGS="--target_board=unix/-m64" check-go' -report=false linux-amd64-gccgo-temp
|
docker run gobuilders/linux-x86-gccgo /usr/local/bin/builder -tool="gccgo" -rev=b9151e911a54 -v -cmd='make RUNTESTFLAGS="--target_board=unix/-m64" check-go' -report=false linux-amd64-gccgo-temp
|
||||||
|
2
dashboard/env/linux-x86-nacl/Makefile
vendored
2
dashboard/env/linux-x86-nacl/Makefile
vendored
@ -6,7 +6,7 @@ docker: Dockerfile
|
|||||||
docker build -t gobuilders/linux-x86-nacl .
|
docker build -t gobuilders/linux-x86-nacl .
|
||||||
|
|
||||||
upload: docker
|
upload: docker
|
||||||
docker save gobuilders/linux-x86-nacl | gzip | (cd ../../upload && go run upload.go --public go-builder-data/docker-linux.nacl.tar.gz)
|
docker save gobuilders/linux-x86-nacl | gzip | (cd ../../cmd/upload && go run upload.go --public go-builder-data/docker-linux.nacl.tar.gz)
|
||||||
|
|
||||||
check: docker
|
check: docker
|
||||||
docker run gobuilders/linux-x86-nacl /usr/local/bin/builder -rev=77e96c9208d0 -buildroot=/ -v -cmd=/usr/local/bin/build-command.pl -report=false nacl-amd64p32
|
docker run gobuilders/linux-x86-nacl /usr/local/bin/builder -rev=77e96c9208d0 -buildroot=/ -v -cmd=/usr/local/bin/build-command.pl -report=false nacl-amd64p32
|
||||||
|
2
dashboard/env/linux-x86-sid/Makefile
vendored
2
dashboard/env/linux-x86-sid/Makefile
vendored
@ -6,7 +6,7 @@ docker: Dockerfile
|
|||||||
docker build -t gobuilders/linux-x86-sid .
|
docker build -t gobuilders/linux-x86-sid .
|
||||||
|
|
||||||
docker-linux.sid.tar.gz: docker
|
docker-linux.sid.tar.gz: docker
|
||||||
docker save gobuilders/linux-x86-sid | gzip | (cd ../../upload && go run upload.go --public go-builder-data/docker-linux.sid.tar.gz)
|
docker save gobuilders/linux-x86-sid | gzip | (cd ../../cmd/upload && go run upload.go --public go-builder-data/docker-linux.sid.tar.gz)
|
||||||
|
|
||||||
check: docker
|
check: docker
|
||||||
docker run -e GOROOT_BOOTSTRAP=/go1.4-amd64/go gobuilders/linux-x86-sid /usr/local/bin/builder -rev=20a10e7ddd1b -buildroot=/ -v -report=false linux-amd64-sid
|
docker run -e GOROOT_BOOTSTRAP=/go1.4-amd64/go gobuilders/linux-x86-sid /usr/local/bin/builder -rev=20a10e7ddd1b -buildroot=/ -v -report=false linux-amd64-sid
|
||||||
|
@ -1,128 +0,0 @@
|
|||||||
// Copyright 2013 The Go Authors. All rights reserved.
|
|
||||||
// Use of this source code is governed by a BSD-style
|
|
||||||
// license that can be found in the LICENSE file.
|
|
||||||
|
|
||||||
package main // import "golang.org/x/tools/dashboard/updater"
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"encoding/json"
|
|
||||||
"encoding/xml"
|
|
||||||
"flag"
|
|
||||||
"fmt"
|
|
||||||
"io/ioutil"
|
|
||||||
"net/http"
|
|
||||||
"net/url"
|
|
||||||
"os"
|
|
||||||
"os/exec"
|
|
||||||
"strings"
|
|
||||||
)
|
|
||||||
|
|
||||||
var (
|
|
||||||
builder = flag.String("builder", "", "builder name")
|
|
||||||
key = flag.String("key", "", "builder key")
|
|
||||||
gopath = flag.String("gopath", "", "path to go repo")
|
|
||||||
dashboard = flag.String("dashboard", "build.golang.org", "Go Dashboard Host")
|
|
||||||
batch = flag.Int("batch", 100, "upload batch size")
|
|
||||||
)
|
|
||||||
|
|
||||||
// Do not benchmark beyond this commit.
|
|
||||||
// There is little sense in benchmarking till first commit,
|
|
||||||
// and the benchmark won't build anyway.
|
|
||||||
const Go1Commit = "0051c7442fed" // test/bench/shootout: update timing.log to Go 1.
|
|
||||||
|
|
||||||
// HgLog represents a single Mercurial revision.
|
|
||||||
type HgLog struct {
|
|
||||||
Hash string
|
|
||||||
Branch string
|
|
||||||
Files string
|
|
||||||
}
|
|
||||||
|
|
||||||
func main() {
|
|
||||||
flag.Parse()
|
|
||||||
logs := hgLog()
|
|
||||||
var hashes []string
|
|
||||||
ngo1 := 0
|
|
||||||
for i := range logs {
|
|
||||||
if strings.HasPrefix(logs[i].Hash, Go1Commit) {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
if needsBenchmarking(&logs[i]) {
|
|
||||||
hashes = append(hashes, logs[i].Hash)
|
|
||||||
}
|
|
||||||
ngo1++
|
|
||||||
}
|
|
||||||
fmt.Printf("found %v commits, %v after Go1, %v need benchmarking\n", len(logs), ngo1, len(hashes))
|
|
||||||
for i := 0; i < len(hashes); i += *batch {
|
|
||||||
j := i + *batch
|
|
||||||
if j > len(hashes) {
|
|
||||||
j = len(hashes)
|
|
||||||
}
|
|
||||||
fmt.Printf("sending %v-%v... ", i, j)
|
|
||||||
res := postCommits(hashes[i:j])
|
|
||||||
fmt.Printf("%s\n", res)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func hgLog() []HgLog {
|
|
||||||
var out bytes.Buffer
|
|
||||||
cmd := exec.Command("hg", "log", "--encoding=utf-8", "--template", xmlLogTemplate)
|
|
||||||
cmd.Dir = *gopath
|
|
||||||
cmd.Stdout = &out
|
|
||||||
cmd.Stderr = os.Stderr
|
|
||||||
err := cmd.Run()
|
|
||||||
if err != nil {
|
|
||||||
fmt.Printf("failed to execute 'hg log': %v\n", err)
|
|
||||||
os.Exit(1)
|
|
||||||
}
|
|
||||||
var top struct{ Log []HgLog }
|
|
||||||
err = xml.Unmarshal([]byte("<Top>"+out.String()+"</Top>"), &top)
|
|
||||||
if err != nil {
|
|
||||||
fmt.Printf("failed to parse log: %v\n", err)
|
|
||||||
os.Exit(1)
|
|
||||||
}
|
|
||||||
return top.Log
|
|
||||||
}
|
|
||||||
|
|
||||||
func needsBenchmarking(log *HgLog) bool {
|
|
||||||
if log.Branch != "" {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
for _, f := range strings.Split(log.Files, " ") {
|
|
||||||
if (strings.HasPrefix(f, "include") || strings.HasPrefix(f, "src")) &&
|
|
||||||
!strings.HasSuffix(f, "_test.go") && !strings.Contains(f, "testdata") {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
func postCommits(hashes []string) string {
|
|
||||||
args := url.Values{"builder": {*builder}, "key": {*key}}
|
|
||||||
cmd := fmt.Sprintf("http://%v/updatebenchmark?%v", *dashboard, args.Encode())
|
|
||||||
b, err := json.Marshal(hashes)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Sprintf("failed to encode request: %v\n", err)
|
|
||||||
}
|
|
||||||
r, err := http.Post(cmd, "text/json", bytes.NewReader(b))
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Sprintf("failed to send http request: %v\n", err)
|
|
||||||
}
|
|
||||||
defer r.Body.Close()
|
|
||||||
if r.StatusCode != http.StatusOK {
|
|
||||||
return fmt.Sprintf("http request failed: %v\n", r.Status)
|
|
||||||
}
|
|
||||||
resp, err := ioutil.ReadAll(r.Body)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Sprintf("failed to read http response: %v\n", err)
|
|
||||||
}
|
|
||||||
return string(resp)
|
|
||||||
}
|
|
||||||
|
|
||||||
const xmlLogTemplate = `
|
|
||||||
<Log>
|
|
||||||
<Hash>{node|escape}</Hash>
|
|
||||||
<Branch>{branches}</Branch>
|
|
||||||
<Files>{files}</Files>
|
|
||||||
</Log>
|
|
||||||
`
|
|
Loading…
Reference in New Issue
Block a user