qbit/go
1
0
Fork 0
mirror of https://github.com/golang/go synced 2024-11-18 16:44:43 -07:00
Code Releases Activity

dashboard: server app changes for performance dashboard

Browse Source 
This CL moves code from code.google.com/p/dvyukov-go-perf-dashboard,
which was previously reviewed.
UI part will be submitted separately.

LGTM=adg
R=adg
CC=golang-codereviews
https://golang.org/cl/97260043
This commit is contained in:
Dmitriy Vyukov 2014-05-13 11:00:32 +04:00
parent d2a9e7164e
commit 828191dc1e
9 changed files with 1430 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

476
dashboard/app/build/build.go
View File

@ -14,14 +14,22 @@ import (
"fmt"
"io"
"io/ioutil"
"net/http"
"sort"
"strconv"
"strings"
"time"
"appengine"
"appengine/datastore"
"cache"
)
const maxDatastoreStringLen = 500
const (
maxDatastoreStringLen = 500
PerfRunLength = 1024
)
// A Package describes a package that is listed on the dashboard.
type Package struct {
@ -89,9 +97,10 @@ type Commit struct {
ParentHash string
Num int // Internal monotonic counter unique to this package.
User string
Desc string `datastore:",noindex"`
Time time.Time
User string
Desc string `datastore:",noindex"`
Time time.Time
NeedsBenchmarking bool
// ResultData is the Data string of each build Result for this Commit.
// For non-Go commits, only the Results for the current Go tip, weekly,
@ -99,6 +108,10 @@ type Commit struct {
// The complete data set is stored in Result entities.
ResultData []string `datastore:",noindex"`
// PerfResults holds a set of “builder|benchmark” tuples denoting
// what benchmarks have been executed on the commit.
PerfResults []string `datastore:",noindex"`
FailNotificationSent bool
}
@ -138,6 +151,28 @@ func (com *Commit) AddResult(c appengine.Context, r *Result) error {
return nil
}
// AddPerfResult remembers that the builder has run the benchmark on the commit.
// It must be called from inside a datastore transaction.
func (com *Commit) AddPerfResult(c appengine.Context, builder, benchmark string) error {
if err := datastore.Get(c, com.Key(c), com); err != nil {
return fmt.Errorf("getting Commit: %v", err)
}
if !com.NeedsBenchmarking {
return fmt.Errorf("trying to add perf result to Commit(%v) that does not require benchmarking", com.Hash)
}
s := builder + "|" + benchmark
for _, v := range com.PerfResults {
if v == s {
return nil
}
}
com.PerfResults = append(com.PerfResults, s)
if _, err := datastore.Put(c, com.Key(c), com); err != nil {
return fmt.Errorf("putting Commit: %v", err)
}
return nil
}
func trim(s []string, n int) []string {
l := min(len(s), n)
return s[len(s)-l:]
@ -207,6 +242,94 @@ func reverse(s []string) {
}
}
// A CommitRun provides summary information for commits [StartCommitNum, StartCommitNum + PerfRunLength).
// Descendant of Package.
type CommitRun struct {
PackagePath string // (empty for main repo commits)
StartCommitNum int
Hash []string `datastore:",noindex"`
User []string `datastore:",noindex"`
Desc []string `datastore:",noindex"` // Only first line.
Time []time.Time `datastore:",noindex"`
NeedsBenchmarking []bool `datastore:",noindex"`
}
func (cr *CommitRun) Key(c appengine.Context) *datastore.Key {
p := Package{Path: cr.PackagePath}
key := strconv.Itoa(cr.StartCommitNum)
return datastore.NewKey(c, "CommitRun", key, 0, p.Key(c))
}
// GetCommitRun loads and returns CommitRun that contains information
// for commit commitNum.
func GetCommitRun(c appengine.Context, commitNum int) (*CommitRun, error) {
cr := &CommitRun{StartCommitNum: commitNum / PerfRunLength * PerfRunLength}
err := datastore.Get(c, cr.Key(c), cr)
if err != nil && err != datastore.ErrNoSuchEntity {
return nil, fmt.Errorf("getting CommitRun: %v", err)
}
if len(cr.Hash) != PerfRunLength {
cr.Hash = make([]string, PerfRunLength)
cr.User = make([]string, PerfRunLength)
cr.Desc = make([]string, PerfRunLength)
cr.Time = make([]time.Time, PerfRunLength)
cr.NeedsBenchmarking = make([]bool, PerfRunLength)
}
return cr, nil
}
func (cr *CommitRun) AddCommit(c appengine.Context, com *Commit) error {
if com.Num < cr.StartCommitNum || com.Num >= cr.StartCommitNum+PerfRunLength {
return fmt.Errorf("AddCommit: commit num %v out of range [%v, %v)",
com.Num, cr.StartCommitNum, cr.StartCommitNum+PerfRunLength)
}
i := com.Num - cr.StartCommitNum
// Be careful with string lengths,
// we need to fit 1024 commits into 1 MB.
cr.Hash[i] = com.Hash
cr.User[i] = shortDesc(com.User)
cr.Desc[i] = shortDesc(com.Desc)
cr.Time[i] = com.Time
cr.NeedsBenchmarking[i] = com.NeedsBenchmarking
if _, err := datastore.Put(c, cr.Key(c), cr); err != nil {
return fmt.Errorf("putting CommitRun: %v", err)
}
return nil
}
// GetCommits returns [startCommitNum, startCommitNum+n) commits.
// Commits information is partial (obtained from CommitRun),
// do not store them back into datastore.
func GetCommits(c appengine.Context, startCommitNum, n int) ([]*Commit, error) {
if startCommitNum < 0 || n <= 0 {
return nil, fmt.Errorf("GetCommits: invalid args (%v, %v)", startCommitNum, n)
}
var res []*Commit
for n > 0 {
cr, err := GetCommitRun(c, startCommitNum)
if err != nil {
return nil, err
}
idx := startCommitNum - cr.StartCommitNum
cnt := PerfRunLength - idx
if cnt > n {
cnt = n
}
for i := idx; i < idx+cnt; i++ {
com := new(Commit)
com.Hash = cr.Hash[i]
com.User = cr.User[i]
com.Desc = cr.Desc[i]
com.Time = cr.Time[i]
com.NeedsBenchmarking = cr.NeedsBenchmarking[i]
res = append(res, com)
}
startCommitNum += cnt
n -= cnt
}
return res, nil
}
// partsToHash converts a Commit and ResultData substrings to a Result.
func partsToHash(c *Commit, p []string) *Result {
return &Result{
@ -223,9 +346,9 @@ func partsToHash(c *Commit, p []string) *Result {
//
// Each Result entity is a descendant of its associated Package entity.
type Result struct {
PackagePath string // (empty for Go commits)
Builder string // "os-arch[-note]"
Hash string
PackagePath string // (empty for Go commits)
// The Go Commit this was built against (empty for Go commits).
GoHash string
@ -259,6 +382,349 @@ func (r *Result) Data() string {
return fmt.Sprintf("%v|%v|%v|%v", r.Builder, r.OK, r.LogHash, r.GoHash)
}
// A PerfResult describes all benchmarking result for a Commit.
// Descendant of Package.
type PerfResult struct {
PackagePath string
CommitHash string
CommitNum int
Data []string `datastore:",noindex"` // "builder|benchmark|ok|metric1=val1|metric2=val2|file:log=hash|file:cpuprof=hash"
// Local cache with parsed Data.
// Maps builder->benchmark->ParsedPerfResult.
parsedData map[string]map[string]*ParsedPerfResult
}
type ParsedPerfResult struct {
OK bool
Metrics map[string]uint64
Artifacts map[string]string
}
func (r *PerfResult) Key(c appengine.Context) *datastore.Key {
p := Package{Path: r.PackagePath}
key := r.CommitHash
return datastore.NewKey(c, "PerfResult", key, 0, p.Key(c))
}
// AddResult add the benchmarking result to r.
// Existing result for the same builder/benchmark is replaced if already exists.
// Returns whether the result was already present.
func (r *PerfResult) AddResult(req *PerfRequest) bool {
present := false
str := fmt.Sprintf("%v|%v|", req.Builder, req.Benchmark)
for i, s := range r.Data {
if strings.HasPrefix(s, str) {
present = true
last := len(r.Data) - 1
r.Data[i] = r.Data[last]
r.Data = r.Data[:last]
break
}
}
ok := "ok"
if !req.OK {
ok = "false"
}
str += ok
for _, m := range req.Metrics {
str += fmt.Sprintf("|%v=%v", m.Type, m.Val)
}
for _, a := range req.Artifacts {
str += fmt.Sprintf("|file:%v=%v", a.Type, a.Body)
}
r.Data = append(r.Data, str)
r.parsedData = nil
return present
}
func (r *PerfResult) ParseData() map[string]map[string]*ParsedPerfResult {
if r.parsedData != nil {
return r.parsedData
}
res := make(map[string]map[string]*ParsedPerfResult)
for _, str := range r.Data {
ss := strings.Split(str, "|")
builder := ss[0]
bench := ss[1]
ok := ss[2]
m := res[builder]
if m == nil {
m = make(map[string]*ParsedPerfResult)
res[builder] = m
}
var p ParsedPerfResult
p.OK = ok == "ok"
p.Metrics = make(map[string]uint64)
p.Artifacts = make(map[string]string)
for _, entry := range ss[3:] {
if strings.HasPrefix(entry, "file:") {
ss1 := strings.Split(entry[len("file:"):], "=")
p.Artifacts[ss1[0]] = ss1[1]
} else {
ss1 := strings.Split(entry, "=")
val, _ := strconv.ParseUint(ss1[1], 10, 64)
p.Metrics[ss1[0]] = val
}
}
m[bench] = &p
}
r.parsedData = res
return res
}
// A PerfMetricRun entity holds a set of metric values for builder/benchmark/metric
// for commits [StartCommitNum, StartCommitNum + PerfRunLength).
// Descendant of Package.
type PerfMetricRun struct {
PackagePath string
Builder string
Benchmark string
Metric string // e.g. realtime, cputime, gc-pause
StartCommitNum int
Vals []int64 `datastore:",noindex"`
}
func (m *PerfMetricRun) Key(c appengine.Context) *datastore.Key {
p := Package{Path: m.PackagePath}
key := m.Builder + "|" + m.Benchmark + "|" + m.Metric + "|" + strconv.Itoa(m.StartCommitNum)
return datastore.NewKey(c, "PerfMetricRun", key, 0, p.Key(c))
}
// GetPerfMetricRun loads and returns PerfMetricRun that contains information
// for commit commitNum.
func GetPerfMetricRun(c appengine.Context, builder, benchmark, metric string, commitNum int) (*PerfMetricRun, error) {
startCommitNum := commitNum / PerfRunLength * PerfRunLength
m := &PerfMetricRun{Builder: builder, Benchmark: benchmark, Metric: metric, StartCommitNum: startCommitNum}
err := datastore.Get(c, m.Key(c), m)
if err != nil && err != datastore.ErrNoSuchEntity {
return nil, fmt.Errorf("getting PerfMetricRun: %v", err)
}
if len(m.Vals) != PerfRunLength {
m.Vals = make([]int64, PerfRunLength)
}
return m, nil
}
func (m *PerfMetricRun) AddMetric(c appengine.Context, commitNum int, v uint64) error {
if commitNum < m.StartCommitNum || commitNum >= m.StartCommitNum+PerfRunLength {
return fmt.Errorf("AddMetric: CommitNum %v out of range [%v, %v)",
commitNum, m.StartCommitNum, m.StartCommitNum+PerfRunLength)
}
m.Vals[commitNum-m.StartCommitNum] = int64(v)
if _, err := datastore.Put(c, m.Key(c), m); err != nil {
return fmt.Errorf("putting PerfMetricRun: %v", err)
}
return nil
}
// GetPerfMetricsForCommits returns perf metrics for builder/benchmark/metric
// and commits [startCommitNum, startCommitNum+n).
func GetPerfMetricsForCommits(c appengine.Context, builder, benchmark, metric string, startCommitNum, n int) ([]uint64, error) {
if startCommitNum < 0 || n <= 0 {
return nil, fmt.Errorf("GetPerfMetricsForCommits: invalid args (%v, %v)", startCommitNum, n)
}
var res []uint64
for n > 0 {
metrics, err := GetPerfMetricRun(c, builder, benchmark, metric, startCommitNum)
if err != nil {
return nil, err
}
idx := startCommitNum - metrics.StartCommitNum
cnt := PerfRunLength - idx
if cnt > n {
cnt = n
}
for _, v := range metrics.Vals[idx : idx+cnt] {
res = append(res, uint64(v))
}
startCommitNum += cnt
n -= cnt
}
return res, nil
}
// PerfConfig holds read-mostly configuration related to benchmarking.
// There is only one PerfConfig entity.
type PerfConfig struct {
BuilderBench []string `datastore:",noindex"` // "builder|benchmark" pairs
BuilderProcs []string `datastore:",noindex"` // "builder|proc" pairs
BenchMetric []string `datastore:",noindex"` // "benchmark|metric" pairs
NoiseLevels []string `datastore:",noindex"` // "builder|benchmark|metric1=noise1|metric2=noise2"
// Local cache of "builder|benchmark|metric" -> noise.
noise map[string]float64
}
func PerfConfigKey(c appengine.Context) *datastore.Key {
p := Package{}
return datastore.NewKey(c, "PerfConfig", "PerfConfig", 0, p.Key(c))
}
const perfConfigCacheKey = "perf-config"
func GetPerfConfig(c appengine.Context, r *http.Request) (*PerfConfig, error) {
pc := new(PerfConfig)
now := cache.Now(c)
if cache.Get(r, now, perfConfigCacheKey, pc) {
return pc, nil
}
err := datastore.Get(c, PerfConfigKey(c), pc)
if err != nil && err != datastore.ErrNoSuchEntity {
return nil, fmt.Errorf("GetPerfConfig: %v", err)
}
cache.Set(r, now, perfConfigCacheKey, pc)
return pc, nil
}
func (pc *PerfConfig) NoiseLevel(builder, benchmark, metric string) float64 {
if pc.noise == nil {
pc.noise = make(map[string]float64)
for _, str := range pc.NoiseLevels {
split := strings.Split(str, "|")
builderBench := split[0] + "|" + split[1]
for _, entry := range split[2:] {
metricValue := strings.Split(entry, "=")
noise, _ := strconv.ParseFloat(metricValue[1], 64)
pc.noise[builderBench+"|"+metricValue[0]] = noise
}
}
}
me := fmt.Sprintf("%v|%v|%v", builder, benchmark, metric)
n := pc.noise[me]
if n == 0 {
// Use a very conservative value
// until we have learned the real noise level.
n = 200
}
return n
}
// UpdatePerfConfig updates the PerfConfig entity with results of benchmarking.
// Returns whether it's a benchmark that we have not yet seem on the builder.
func UpdatePerfConfig(c appengine.Context, r *http.Request, req *PerfRequest) (newBenchmark bool, err error) {
pc, err := GetPerfConfig(c, r)
if err != nil {
return false, err
}
modified := false
add := func(arr *[]string, str string) {
for _, s := range *arr {
if s == str {
return
}
}
*arr = append(*arr, str)
modified = true
return
}
BenchProcs := strings.Split(req.Benchmark, "-")
benchmark := BenchProcs[0]
procs := "1"
if len(BenchProcs) > 1 {
procs = BenchProcs[1]
}
add(&pc.BuilderBench, req.Builder+"|"+benchmark)
newBenchmark = modified
add(&pc.BuilderProcs, req.Builder+"|"+procs)
for _, m := range req.Metrics {
add(&pc.BenchMetric, benchmark+"|"+m.Type)
}
if modified {
if _, err := datastore.Put(c, PerfConfigKey(c), pc); err != nil {
return false, fmt.Errorf("putting PerfConfig: %v", err)
}
cache.Tick(c)
}
return newBenchmark, nil
}
func collectList(all []string, idx int, second string) (res []string) {
m := make(map[string]bool)
for _, str := range all {
ss := strings.Split(str, "|")
v := ss[idx]
v2 := ss[1-idx]
if (second == "" || second == v2) && !m[v] {
m[v] = true
res = append(res, v)
}
}
sort.Strings(res)
return res
}
func (pc *PerfConfig) BuildersForBenchmark(bench string) []string {
return collectList(pc.BuilderBench, 0, bench)
}
func (pc *PerfConfig) BenchmarksForBuilder(builder string) []string {
return collectList(pc.BuilderBench, 1, builder)
}
func (pc *PerfConfig) MetricsForBenchmark(bench string) []string {
return collectList(pc.BenchMetric, 1, bench)
}
func (pc *PerfConfig) BenchmarkProcList() (res []string) {
bl := pc.BenchmarksForBuilder("")
pl := pc.ProcList("")
for _, b := range bl {
for _, p := range pl {
res = append(res, fmt.Sprintf("%v-%v", b, p))
}
}
return res
}
func (pc *PerfConfig) ProcList(builder string) []int {
ss := collectList(pc.BuilderProcs, 1, builder)
var procs []int
for _, s := range ss {
p, _ := strconv.ParseInt(s, 10, 32)
procs = append(procs, int(p))
}
sort.Ints(procs)
return procs
}
// A PerfTodo contains outstanding commits for benchmarking for a builder.
// Descendant of Package.
type PerfTodo struct {
PackagePath string // (empty for main repo commits)
Builder string
CommitNums []int `datastore:",noindex"` // LIFO queue of commits to benchmark.
}
func (todo *PerfTodo) Key(c appengine.Context) *datastore.Key {
p := Package{Path: todo.PackagePath}
key := todo.Builder
return datastore.NewKey(c, "PerfTodo", key, 0, p.Key(c))
}
// AddCommitToPerfTodo adds the commit to all existing PerfTodo entities.
func AddCommitToPerfTodo(c appengine.Context, com *Commit) error {
var todos []*PerfTodo
_, err := datastore.NewQuery("PerfTodo").
Ancestor((&Package{}).Key(c)).
GetAll(c, &todos)
if err != nil {
return fmt.Errorf("fetching PerfTodo's: %v", err)
}
for _, todo := range todos {
todo.CommitNums = append(todo.CommitNums, com.Num)
_, err = datastore.Put(c, todo.Key(c), todo)
if err != nil {
return fmt.Errorf("updating PerfTodo: %v", err)
}
}
return nil
}
// A Log is a gzip-compressed log file stored under the SHA1 hash of the
// uncompressed log text.
type Log struct {

350
dashboard/app/build/handler.go
View File

@ -44,6 +44,9 @@ func commitHandler(r *http.Request) (interface{}, error) {
if err := datastore.Get(c, com.Key(c), com); err != nil {
return nil, fmt.Errorf("getting Commit: %v", err)
}
// Strip potentially large and unnecessary fields.
com.ResultData = nil
com.PerfResults = nil
return com, nil
}
if r.Method != "POST" {
@ -115,6 +118,25 @@ func addCommit(c appengine.Context, com *Commit) error {
if _, err = datastore.Put(c, com.Key(c), com); err != nil {
return fmt.Errorf("putting Commit: %v", err)
}
if com.NeedsBenchmarking {
// add to CommitRun
cr, err := GetCommitRun(c, com.Num)
if err != nil {
return err
}
if err = cr.AddCommit(c, com); err != nil {
return err
}
// create PerfResult
res := &PerfResult{CommitHash: com.Hash, CommitNum: com.Num}
if _, err := datastore.Put(c, res.Key(c), res); err != nil {
return fmt.Errorf("putting PerfResult: %v", err)
}
// Update perf todo if necessary.
if err = AddCommitToPerfTodo(c, com); err != nil {
return err
}
}
return nil
}
@ -165,10 +187,18 @@ func todoHandler(r *http.Request) (interface{}, error) {
switch kind {
case "build-go-commit":
com, err = buildTodo(c, builder, "", "")
if com != nil {
com.PerfResults = []string{}
}
case "build-package":
packagePath := r.FormValue("packagePath")
goHash := r.FormValue("goHash")
com, err = buildTodo(c, builder, packagePath, goHash)
if com != nil {
com.PerfResults = []string{}
}
case "benchmark-go-commit":
com, err = perfTodo(c, builder)
}
if com != nil || err != nil {
if com != nil {
@ -260,6 +290,129 @@ func buildTodo(c appengine.Context, builder, packagePath, goHash string) (*Commi
return nil, nil
}
// perfTodo returns the next Commit to be benchmarked (or nil if none available).
func perfTodo(c appengine.Context, builder string) (*Commit, error) {
p := &Package{}
todo := &PerfTodo{Builder: builder}
err := datastore.Get(c, todo.Key(c), todo)
if err != nil && err != datastore.ErrNoSuchEntity {
return nil, fmt.Errorf("fetching PerfTodo: %v", err)
}
if err == datastore.ErrNoSuchEntity {
todo, err = buildPerfTodo(c, builder)
if err != nil {
return nil, err
}
}
if len(todo.CommitNums) == 0 {
return nil, nil
}
// Have commit to benchmark, fetch it.
num := todo.CommitNums[len(todo.CommitNums)-1]
t := datastore.NewQuery("Commit").
Ancestor(p.Key(c)).
Filter("Num =", num).
Limit(1).
Run(c)
com := new(Commit)
if _, err := t.Next(com); err != nil {
return nil, err
}
if !com.NeedsBenchmarking {
return nil, fmt.Errorf("commit from perf todo queue is not intended for benchmarking")
}
// Remove benchmarks from other builders.
var benchs []string
for _, b := range com.PerfResults {
bb := strings.Split(b, "|")
if bb[0] == builder && bb[1] != "meta-done" {
benchs = append(benchs, bb[1])
}
}
com.PerfResults = benchs
return com, nil
}
// buildPerfTodo creates PerfTodo for the builder with all commits. In a transaction.
func buildPerfTodo(c appengine.Context, builder string) (*PerfTodo, error) {
todo := &PerfTodo{Builder: builder}
tx := func(c appengine.Context) error {
err := datastore.Get(c, todo.Key(c), todo)
if err != nil && err != datastore.ErrNoSuchEntity {
return fmt.Errorf("fetching PerfTodo: %v", err)
}
if err == nil {
return nil
}
t := datastore.NewQuery("CommitRun").
Ancestor((&Package{}).Key(c)).
Order("-StartCommitNum").
Run(c)
var nums []int
var releaseNums []int
loop:
for {
cr := new(CommitRun)
if _, err := t.Next(cr); err == datastore.Done {
break
} else if err != nil {
return fmt.Errorf("scanning commit runs for perf todo: %v", err)
}
for i := len(cr.Hash) - 1; i >= 0; i-- {
if !cr.NeedsBenchmarking[i] || cr.Hash[i] == "" {
continue // There's nothing to see here. Move along.
}
num := cr.StartCommitNum + i
for k, v := range knownTags {
// Releases are benchmarked first, because they are important (and there are few of them).
if cr.Hash[i] == v {
releaseNums = append(releaseNums, num)
if k == "go1" {
break loop // Point of no benchmark: test/bench/shootout: update timing.log to Go 1.
}
}
}
nums = append(nums, num)
}
}
todo.CommitNums = orderPrefTodo(nums)
todo.CommitNums = append(todo.CommitNums, releaseNums...)
if _, err = datastore.Put(c, todo.Key(c), todo); err != nil {
return fmt.Errorf("putting PerfTodo: %v", err)
}
return nil
}
return todo, datastore.RunInTransaction(c, tx, nil)
}
func removeCommitFromPerfTodo(c appengine.Context, builder string, num int) error {
todo := &PerfTodo{Builder: builder}
err := datastore.Get(c, todo.Key(c), todo)
if err != nil && err != datastore.ErrNoSuchEntity {
return fmt.Errorf("fetching PerfTodo: %v", err)
}
if err == datastore.ErrNoSuchEntity {
return nil
}
for i := len(todo.CommitNums) - 1; i >= 0; i-- {
if todo.CommitNums[i] == num {
for ; i < len(todo.CommitNums)-1; i++ {
todo.CommitNums[i] = todo.CommitNums[i+1]
}
todo.CommitNums = todo.CommitNums[:i]
_, err = datastore.Put(c, todo.Key(c), todo)
if err != nil {
return fmt.Errorf("putting PerfTodo: %v", err)
}
break
}
}
return nil
}
// packagesHandler returns a list of the non-Go Packages monitored
// by the dashboard.
func packagesHandler(r *http.Request) (interface{}, error) {
@ -329,6 +482,202 @@ func resultHandler(r *http.Request) (interface{}, error) {
return nil, datastore.RunInTransaction(c, tx, nil)
}
// perf-result request payload
type PerfRequest struct {
Builder string
Benchmark string
Hash string
OK bool
Metrics []PerfMetric
Artifacts []PerfArtifact
}
type PerfMetric struct {
Type string
Val uint64
}
type PerfArtifact struct {
Type string
Body string
}
// perfResultHandler records a becnhmarking result.
func perfResultHandler(r *http.Request) (interface{}, error) {
defer r.Body.Close()
if r.Method != "POST" {
return nil, errBadMethod(r.Method)
}
req := new(PerfRequest)
if err := json.NewDecoder(r.Body).Decode(req); err != nil {
return nil, fmt.Errorf("decoding Body: %v", err)
}
c := contextForRequest(r)
defer cache.Tick(c)
// store the text files if supplied
for i, a := range req.Artifacts {
hash, err := PutLog(c, a.Body)
if err != nil {
return nil, fmt.Errorf("putting Log: %v", err)
}
req.Artifacts[i].Body = hash
}
tx := func(c appengine.Context) error {
return addPerfResult(c, r, req)
}
return nil, datastore.RunInTransaction(c, tx, nil)
}
// addPerfResult creates PerfResult and updates Commit, PerfTodo,
// PerfMetricRun and PerfConfig. Must be executed within a transaction.
func addPerfResult(c appengine.Context, r *http.Request, req *PerfRequest) error {
// check Package exists
p, err := GetPackage(c, "")
if err != nil {
return fmt.Errorf("GetPackage: %v", err)
}
// add result to Commit
com := &Commit{Hash: req.Hash}
if err := com.AddPerfResult(c, req.Builder, req.Benchmark); err != nil {
return fmt.Errorf("AddPerfResult: %v", err)
}
// add the result to PerfResult
res := &PerfResult{CommitHash: req.Hash}
if err := datastore.Get(c, res.Key(c), res); err != nil {
return fmt.Errorf("getting PerfResult: %v", err)
}
present := res.AddResult(req)
if _, err := datastore.Put(c, res.Key(c), res); err != nil {
return fmt.Errorf("putting PerfResult: %v", err)
}
// Meta-done denotes that there are no benchmarks left.
if req.Benchmark == "meta-done" {
// Don't send duplicate emails for the same commit/builder.
// And don't send emails about too old commits.
if !present && com.Num >= p.NextNum-commitsPerPage {
if err := checkPerfChanges(c, r, com, req.Builder, res); err != nil {
return err
}
}
if err := removeCommitFromPerfTodo(c, req.Builder, com.Num); err != nil {
return nil
}
return nil
}
// update PerfConfig
newBenchmark, err := UpdatePerfConfig(c, r, req)
if err != nil {
return fmt.Errorf("updating PerfConfig: %v", err)
}
if newBenchmark {
// If this is a new benchmark on the builder, delete PerfTodo.
// It will be recreated later with all commits again.
todo := &PerfTodo{Builder: req.Builder}
err = datastore.Delete(c, todo.Key(c))
if err != nil && err != datastore.ErrNoSuchEntity {
return fmt.Errorf("deleting PerfTodo: %v", err)
}
}
// add perf metrics
for _, metric := range req.Metrics {
m, err := GetPerfMetricRun(c, req.Builder, req.Benchmark, metric.Type, com.Num)
if err != nil {
return fmt.Errorf("GetPerfMetrics: %v", err)
}
if err = m.AddMetric(c, com.Num, metric.Val); err != nil {
return fmt.Errorf("AddMetric: %v", err)
}
}
return nil
}
func checkPerfChanges(c appengine.Context, r *http.Request, com *Commit, builder string, res *PerfResult) error {
pc, err := GetPerfConfig(c, r)
if err != nil {
return err
}
results := res.ParseData()[builder]
rcNewer := MakePerfResultCache(c, com, true)
rcOlder := MakePerfResultCache(c, com, false)
// Check whether we need to send failure notification email.
if results["meta-done"].OK {
// This one is successful, see if the next is failed.
nextRes, err := rcNewer.Next(com.Num)
if err != nil {
return err
}
if nextRes != nil && isPerfFailed(nextRes, builder) {
sendPerfFailMail(c, builder, nextRes)
}
} else {
// This one is failed, see if the previous is successful.
prevRes, err := rcOlder.Next(com.Num)
if err != nil {
return err
}
if prevRes != nil && !isPerfFailed(prevRes, builder) {
sendPerfFailMail(c, builder, res)
}
}
// Now see if there are any performance changes.
// Find the previous and the next results for performance comparison.
prevRes, err := rcOlder.NextForComparison(com.Num, builder)
if err != nil {
return err
}
nextRes, err := rcNewer.NextForComparison(com.Num, builder)
if err != nil {
return err
}
if results["meta-done"].OK {
// This one is successful, compare with a previous one.
if prevRes != nil {
if err := comparePerfResults(c, pc, builder, prevRes, res); err != nil {
return err
}
}
// Compare a next one with the current.
if nextRes != nil {
if err := comparePerfResults(c, pc, builder, res, nextRes); err != nil {
return err
}
}
} else {
// This one is failed, compare a previous one with a next one.
if prevRes != nil && nextRes != nil {
if err := comparePerfResults(c, pc, builder, prevRes, nextRes); err != nil {
return err
}
}
}
return nil
}
func comparePerfResults(c appengine.Context, pc *PerfConfig, builder string, prevRes, res *PerfResult) error {
changes := significantPerfChanges(pc, builder, prevRes, res)
if len(changes) == 0 {
return nil
}
com := &Commit{Hash: res.CommitHash}
if err := datastore.Get(c, com.Key(c), com); err != nil {
return fmt.Errorf("getting commit %v: %v", com.Hash, err)
}
sendPerfMailLater.Call(c, com, prevRes.CommitHash, builder, changes) // add task to queue
return nil
}
// logHandler displays log text for a given hash.
// It handles paths like "/log/hash".
func logHandler(w http.ResponseWriter, r *http.Request) {
@ -422,6 +771,7 @@ func init() {
http.HandleFunc(d.RelPath+"commit", AuthHandler(commitHandler))
http.HandleFunc(d.RelPath+"packages", AuthHandler(packagesHandler))
http.HandleFunc(d.RelPath+"result", AuthHandler(resultHandler))
http.HandleFunc(d.RelPath+"perf-result", AuthHandler(perfResultHandler))
http.HandleFunc(d.RelPath+"tag", AuthHandler(tagHandler))
http.HandleFunc(d.RelPath+"todo", AuthHandler(todoHandler))

4
dashboard/app/build/init.go
View File

@ -36,5 +36,9 @@ func initHandler(w http.ResponseWriter, r *http.Request) {
return
}
}
// Create secret key.
secretKey(c)
fmt.Fprint(w, "OK")
}

173
dashboard/app/build/notify.go
View File

@ -14,6 +14,7 @@ import (
"io/ioutil"
"net/http"
"regexp"
"sort"
"text/template"
"appengine"
@ -99,14 +100,14 @@ func notifyOnFailure(c appengine.Context, com *Commit, builder string) error {
broken = com
}
}
var err error
if broken != nil && !broken.FailNotificationSent {
c.Infof("%s is broken commit; notifying", broken.Hash)
notifyLater.Call(c, broken, builder) // add task to queue
broken.FailNotificationSent = true
_, err = datastore.Put(c, broken.Key(c), broken)
if broken == nil {
return nil
}
return err
r := broken.Result(builder, "")
if r == nil {
return fmt.Errorf("finding result for %q: %+v", builder, com)
}
return commonNotify(c, broken, builder, r.LogHash)
}
// firstMatch executes the query q and loads the first entity into v.
@ -123,27 +124,22 @@ var notifyLater = delay.Func("notify", notify)
// notify tries to update the CL for the given Commit with a failure message.
// If it doesn't succeed, it sends a failure email to golang-dev.
func notify(c appengine.Context, com *Commit, builder string) {
if !updateCL(c, com, builder) {
func notify(c appengine.Context, com *Commit, builder, logHash string) {
if !updateCL(c, com, builder, logHash) {
// Send a mail notification if the CL can't be found.
sendFailMail(c, com, builder)
sendFailMail(c, com, builder, logHash)
}
}
// updateCL updates the CL for the given Commit with a failure message
// for the given builder.
func updateCL(c appengine.Context, com *Commit, builder string) bool {
func updateCL(c appengine.Context, com *Commit, builder, logHash string) bool {
cl, err := lookupCL(c, com)
if err != nil {
c.Errorf("could not find CL for %v: %v", com.Hash, err)
return false
}
res := com.Result(builder, "")
if res == nil {
c.Errorf("finding result for %q: %+v", builder, com)
return false
}
url := fmt.Sprintf("%v?cl=%v&brokebuild=%v&log=%v", gobotBase, cl, builder, res.LogHash)
url := fmt.Sprintf("%v?cl=%v&brokebuild=%v&log=%v", gobotBase, cl, builder, logHash)
r, err := urlfetch.Client(c).Post(url, "text/plain", nil)
if err != nil {
c.Errorf("could not update CL %v: %v", cl, err)
@ -192,30 +188,65 @@ func init() {
gob.Register(&Commit{}) // for delay
}
var (
sendPerfMailLater = delay.Func("sendPerfMail", sendPerfMailFunc)
sendPerfMailTmpl = template.Must(
template.New("perf_notify.txt").
Funcs(template.FuncMap(tmplFuncs)).
ParseFiles("build/perf_notify.txt"),
)
)
func sendPerfFailMail(c appengine.Context, builder string, res *PerfResult) error {
com := &Commit{Hash: res.CommitHash}
if err := datastore.Get(c, com.Key(c), com); err != nil {
return fmt.Errorf("getting commit %v: %v", com.Hash, err)
}
logHash := ""
parsed := res.ParseData()
for _, data := range parsed[builder] {
if !data.OK {
logHash = data.Artifacts["log"]
break
}
}
if logHash == "" {
return fmt.Errorf("can not find failed result for commit %v on builder %v", com.Hash, builder)
}
return commonNotify(c, com, builder, logHash)
}
func commonNotify(c appengine.Context, com *Commit, builder, logHash string) error {
if com.FailNotificationSent {
return nil
}
c.Infof("%s is broken commit; notifying", com.Hash)
notifyLater.Call(c, com, builder, logHash) // add task to queue
com.FailNotificationSent = true
_, err := datastore.Put(c, com.Key(c), com)
return err
}
// sendFailMail sends a mail notification that the build failed on the
// provided commit and builder.
func sendFailMail(c appengine.Context, com *Commit, builder string) {
// TODO(adg): handle packages
// get Result
r := com.Result(builder, "")
if r == nil {
c.Errorf("finding result for %q: %+v", builder, com)
return
}
func sendFailMail(c appengine.Context, com *Commit, builder, logHash string) {
// get Log
k := datastore.NewKey(c, "Log", r.LogHash, 0, nil)
k := datastore.NewKey(c, "Log", logHash, 0, nil)
l := new(Log)
if err := datastore.Get(c, k, l); err != nil {
c.Errorf("finding Log record %v: %v", r.LogHash, err)
c.Errorf("finding Log record %v: %v", logHash, err)
return
}
logText, err := l.Text()
if err != nil {
c.Errorf("unpacking Log record %v: %v", logHash, err)
return
}
// prepare mail message
var body bytes.Buffer
err := sendFailMailTmpl.Execute(&body, map[string]interface{}{
"Builder": builder, "Commit": com, "Result": r, "Log": l,
err = sendFailMailTmpl.Execute(&body, map[string]interface{}{
"Builder": builder, "Commit": com, "LogHash": logHash, "LogText": logText,
"Hostname": domain,
})
if err != nil {
@ -236,3 +267,83 @@ func sendFailMail(c appengine.Context, com *Commit, builder string) {
c.Errorf("sending mail: %v", err)
}
}
type PerfChangeBenchmark struct {
Name string
Metrics []*PerfChangeMetric
}
type PerfChangeMetric struct {
Name string
Old uint64
New uint64
Delta float64
}
type PerfChangeBenchmarkSlice []*PerfChangeBenchmark
func (l PerfChangeBenchmarkSlice) Len() int { return len(l) }
func (l PerfChangeBenchmarkSlice) Swap(i, j int) { l[i], l[j] = l[j], l[i] }
func (l PerfChangeBenchmarkSlice) Less(i, j int) bool {
b1, p1 := splitBench(l[i].Name)
b2, p2 := splitBench(l[j].Name)
if b1 != b2 {
return b1 < b2
}
return p1 < p2
}
type PerfChangeMetricSlice []*PerfChangeMetric
func (l PerfChangeMetricSlice) Len() int { return len(l) }
func (l PerfChangeMetricSlice) Swap(i, j int) { l[i], l[j] = l[j], l[i] }
func (l PerfChangeMetricSlice) Less(i, j int) bool { return l[i].Name < l[j].Name }
func sendPerfMailFunc(c appengine.Context, com *Commit, prevCommitHash, builder string, changes []*PerfChange) {
// Sort the changes into the right order.
var benchmarks []*PerfChangeBenchmark
for _, ch := range changes {
// Find the benchmark.
var b *PerfChangeBenchmark
for _, b1 := range benchmarks {
if b1.Name == ch.bench {
b = b1
break
}
}
if b == nil {
b = &PerfChangeBenchmark{Name: ch.bench}
benchmarks = append(benchmarks, b)
}
b.Metrics = append(b.Metrics, &PerfChangeMetric{Name: ch.metric, Old: ch.old, New: ch.new, Delta: ch.diff})
}
for _, b := range benchmarks {
sort.Sort(PerfChangeMetricSlice(b.Metrics))
}
sort.Sort(PerfChangeBenchmarkSlice(benchmarks))
url := fmt.Sprintf("http://%v/perfdetail?commit=%v&commit0=%v&kind=builder&builder=%v", domain, com.Hash, prevCommitHash, builder)
// prepare mail message
var body bytes.Buffer
err := sendPerfMailTmpl.Execute(&body, map[string]interface{}{
"Builder": builder, "Commit": com, "Hostname": domain, "Url": url, "Benchmarks": benchmarks,
})
if err != nil {
c.Errorf("rendering perf mail template: %v", err)
return
}
subject := fmt.Sprintf("Perf changes on %s by %s", builder, shortDesc(com.Desc))
msg := &mail.Message{
Sender: mailFrom,
To: []string{failMailTo},
ReplyTo: failMailTo,
Subject: subject,
Body: body.String(),
}
// send mail
if err := mail.Send(c, msg); err != nil {
c.Errorf("sending mail: %v", err)
}
}

4
dashboard/app/build/notify.txt
View File

@ -1,9 +1,9 @@
Change {{shortHash .Commit.Hash}} broke the {{.Builder}} build:
http://{{.Hostname}}/log/{{.Result.LogHash}}
http://{{.Hostname}}/log/{{.LogHash}}
{{.Commit.Desc}}
http://code.google.com/p/go/source/detail?r={{shortHash .Commit.Hash}}
$ tail -200 < log
{{printf "%s" .Log.Text | tail 200}}
{{printf "%s" .LogText | tail 200}}

310
dashboard/app/build/perf.go Normal file
View File

@ -0,0 +1,310 @@
// Copyright 2014 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// +build appengine
package build
import (
"fmt"
"sort"
"strconv"
"strings"
"appengine"
"appengine/datastore"
)
var knownTags = map[string]string{
"go1": "0051c7442fed9c888de6617fa9239a913904d96e",
"go1.1": "d29da2ced72ba2cf48ed6a8f1ec4abc01e4c5bf1",
"go1.2": "b1edf8faa5d6cbc50c6515785df9df9c19296564",
}
var lastRelease = "go1.2"
func splitBench(benchProcs string) (string, int) {
ss := strings.Split(benchProcs, "-")
procs, _ := strconv.Atoi(ss[1])
return ss[0], procs
}
func dashPerfCommits(c appengine.Context, page int) ([]*Commit, error) {
q := datastore.NewQuery("Commit").
Ancestor((&Package{}).Key(c)).
Order("-Num").
Filter("NeedsBenchmarking =", true).
Limit(commitsPerPage).
Offset(page * commitsPerPage)
var commits []*Commit
_, err := q.GetAll(c, &commits)
if err == nil && len(commits) == 0 {
err = fmt.Errorf("no commits")
}
return commits, err
}
func perfChangeStyle(pc *PerfConfig, v float64, builder, benchmark, metric string) string {
noise := pc.NoiseLevel(builder, benchmark, metric)
if isNoise(v, noise) {
return "noise"
}
if v > 0 {
return "bad"
}
return "good"
}
func isNoise(diff, noise float64) bool {
rnoise := -100 * noise / (noise + 100)
return diff < noise && diff > rnoise
}
func perfDiff(old, new uint64) float64 {
return 100*float64(new)/float64(old) - 100
}
func isPerfFailed(res *PerfResult, builder string) bool {
data := res.ParseData()[builder]
return data != nil && data["meta-done"] != nil && !data["meta-done"].OK
}
// PerfResultCache caches a set of PerfResults so that it's easy to access them
// without lots of duplicate accesses to datastore.
// It allows to iterate over newer or older results for some base commit.
type PerfResultCache struct {
c appengine.Context
newer bool
iter *datastore.Iterator
results map[int]*PerfResult
}
func MakePerfResultCache(c appengine.Context, com *Commit, newer bool) *PerfResultCache {
p := &Package{}
q := datastore.NewQuery("PerfResult").Ancestor(p.Key(c)).Limit(100)
if newer {
q = q.Filter("CommitNum >=", com.Num).Order("CommitNum")
} else {
q = q.Filter("CommitNum <=", com.Num).Order("-CommitNum")
}
rc := &PerfResultCache{c: c, newer: newer, iter: q.Run(c), results: make(map[int]*PerfResult)}
return rc
}
func (rc *PerfResultCache) Get(commitNum int) *PerfResult {
rc.Next(commitNum) // fetch the commit, if necessary
return rc.results[commitNum]
}
// Next returns the next PerfResult for the commit commitNum.
// It does not care whether the result has any data, failed or whatever.
func (rc *PerfResultCache) Next(commitNum int) (*PerfResult, error) {
// See if we have next result in the cache.
next := -1
for ci := range rc.results {
if rc.newer {
if ci > commitNum && (next == -1 || ci < next) {
next = ci
}
} else {
if ci < commitNum && (next == -1 || ci > next) {
next = ci
}
}
}
//rc.c.Errorf("PerfResultCache.Next: num=%v next=%v", commitNum, next)
if next != -1 {
return rc.results[next], nil
}
// Fetch next result from datastore.
res := new(PerfResult)
_, err := rc.iter.Next(res)
//rc.c.Errorf("PerfResultCache.Next: fetched %v %+v", err, res)
if err == datastore.Done {
return nil, nil
}
if err != nil {
return nil, fmt.Errorf("fetching perf results: %v", err)
}
if (rc.newer && res.CommitNum < commitNum) || (!rc.newer && res.CommitNum > commitNum) {
rc.c.Errorf("PerfResultCache.Next: bad commit num")
}
rc.results[res.CommitNum] = res
return res, nil
}
// NextForComparison returns PerfResult which we need to use for performance comprison.
// It skips failed results, but does not skip results with no data.
func (rc *PerfResultCache) NextForComparison(commitNum int, builder string) (*PerfResult, error) {
for {
res, err := rc.Next(commitNum)
if err != nil {
return nil, err
}
if res == nil {
return nil, nil
}
if res.CommitNum == commitNum {
continue
}
parsed := res.ParseData()
if builder != "" {
// Comparing for a particular builder.
// This is used in perf_changes and in email notifications.
b := parsed[builder]
if b == nil || b["meta-done"] == nil {
// No results yet, must not do the comparison.
return nil, nil
}
if b["meta-done"].OK {
// Have complete results, compare.
return res, nil
}
} else {
// Comparing for all builders, find a result with at least
// one successful meta-done.
// This is used in perf_detail.
for _, benchs := range parsed {
if data := benchs["meta-done"]; data != nil && data.OK {
return res, nil
}
}
}
// Failed, try next result.
commitNum = res.CommitNum
}
}
type PerfChange struct {
builder string
bench string
metric string
old uint64
new uint64
diff float64
}
func significantPerfChanges(pc *PerfConfig, builder string, prevRes, res *PerfResult) (changes []*PerfChange) {
// First, collect all significant changes.
for builder1, benchmarks1 := range res.ParseData() {
if builder != "" && builder != builder1 {
// This is not the builder you're looking for, Luke.
continue
}
benchmarks0 := prevRes.ParseData()[builder1]
if benchmarks0 == nil {
continue
}
for benchmark, data1 := range benchmarks1 {
data0 := benchmarks0[benchmark]
if data0 == nil {
continue
}
for metric, val := range data1.Metrics {
val0 := data0.Metrics[metric]
if val0 == 0 {
continue
}
diff := perfDiff(val0, val)
noise := pc.NoiseLevel(builder, benchmark, metric)
if isNoise(diff, noise) {
continue
}
ch := &PerfChange{builder: builder, bench: benchmark, metric: metric, old: val0, new: val, diff: diff}
changes = append(changes, ch)
}
}
}
// Then, strip non-repeatable changes (flakes).
// The hypothesis is that a real change must show up with at least
// 2 different values of GOMAXPROCS.
cnt := make(map[string]int)
for _, ch := range changes {
b, _ := splitBench(ch.bench)
name := b + "|" + ch.metric
inc := 1
if ch.diff < 0 {
inc = -1
}
cnt[name] = cnt[name] + inc
}
for i := 0; i < len(changes); i++ {
ch := changes[i]
b, _ := splitBench(ch.bench)
name := b + "|" + ch.metric
if n := cnt[name]; n <= -2 || n >= 2 {
continue
}
last := len(changes) - 1
changes[i] = changes[last]
changes = changes[:last]
i--
}
return changes
}
// orderPrefTodo reorders commit nums for benchmarking todo.
// The resulting order is somewhat tricky. We want 2 things:
// 1. benchmark sequentially backwards (this provides information about most
// recent changes, and allows to estimate noise levels)
// 2. benchmark old commits in "scatter" order (this allows to quickly gather
// brief information about thousands of old commits)
// So this function interleaves the two orders.
func orderPrefTodo(nums []int) []int {
sort.Ints(nums)
n := len(nums)
pow2 := uint32(0) // next power-of-two that is >= n
npow2 := 0
for npow2 <= n {
pow2++
npow2 = 1 << pow2
}
res := make([]int, n)
resPos := n - 1 // result array is filled backwards
present := make([]bool, n) // denotes values that already present in result array
for i0, i1 := n-1, 0; i0 >= 0 || i1 < npow2; {
// i0 represents "benchmark sequentially backwards" sequence
// find the next commit that is not yet present and add it
for cnt := 0; cnt < 2; cnt++ {
for ; i0 >= 0; i0-- {
if !present[i0] {
present[i0] = true
res[resPos] = nums[i0]
resPos--
i0--
break
}
}
}
// i1 represents "scatter order" sequence
// find the next commit that is not yet present and add it
for ; i1 < npow2; i1++ {
// do the "recursive split-ordering" trick
idx := 0 // bitwise reverse of i1
for j := uint32(0); j <= pow2; j++ {
if (i1 & (1 << j)) != 0 {
idx = idx | (1 << (pow2 - j - 1))
}
}
if idx < n && !present[idx] {
present[idx] = true
res[resPos] = nums[idx]
resPos--
i1++
break
}
}
}
// The above can't possibly be correct. Do dump check.
res2 := make([]int, n)
copy(res2, res)
sort.Ints(res2)
for i := range res2 {
if res2[i] != nums[i] {
panic(fmt.Sprintf("diff at %v: expect %v, want %v\nwas: %v\n become: %v",
i, nums[i], res2[i], nums, res2))
}
}
return res
}

141
dashboard/app/build/test.go
View File

@ -32,7 +32,12 @@ func init() {
var testEntityKinds = []string{
"Package",
"Commit",
"CommitRun",
"Result",
"PerfResult",
"PerfMetricRun",
"PerfConfig",
"PerfTodo",
"Log",
}
@ -47,15 +52,16 @@ var testPackages = []*Package{
var tCommitTime = time.Now().Add(-time.Hour * 24 * 7)
func tCommit(hash, parentHash, path string) *Commit {
func tCommit(hash, parentHash, path string, bench bool) *Commit {
tCommitTime.Add(time.Hour) // each commit should have a different time
return &Commit{
PackagePath: path,
Hash: hash,
ParentHash: parentHash,
Time: tCommitTime,
User: "adg",
Desc: "change description " + hash,
PackagePath: path,
Hash: hash,
ParentHash: parentHash,
Time: tCommitTime,
User: "adg",
Desc: "change description " + hash,
NeedsBenchmarking: bench,
}
}
@ -69,9 +75,9 @@ var testRequests = []struct {
{"/packages?kind=subrepo", nil, nil, []*Package{testPackage}},
// Go repo
{"/commit", nil, tCommit("0001", "0000", ""), nil},
{"/commit", nil, tCommit("0002", "0001", ""), nil},
{"/commit", nil, tCommit("0003", "0002", ""), nil},
{"/commit", nil, tCommit("0001", "0000", "", true), nil},
{"/commit", nil, tCommit("0002", "0001", "", false), nil},
{"/commit", nil, tCommit("0003", "0002", "", true), nil},
{"/todo", url.Values{"kind": {"build-go-commit"}, "builder": {"linux-386"}}, nil, &Todo{Kind: "build-go-commit", Data: &Commit{Hash: "0003"}}},
{"/todo", url.Values{"kind": {"build-go-commit"}, "builder": {"linux-amd64"}}, nil, &Todo{Kind: "build-go-commit", Data: &Commit{Hash: "0003"}}},
{"/result", nil, &Result{Builder: "linux-386", Hash: "0001", OK: true}, nil},
@ -95,8 +101,8 @@ var testRequests = []struct {
{"/todo", url.Values{"kind": {"build-go-commit"}, "builder": {"linux-amd64"}}, nil, &Todo{Kind: "build-go-commit", Data: &Commit{Hash: "0002"}}},
// branches
{"/commit", nil, tCommit("0004", "0003", ""), nil},
{"/commit", nil, tCommit("0005", "0002", ""), nil},
{"/commit", nil, tCommit("0004", "0003", "", false), nil},
{"/commit", nil, tCommit("0005", "0002", "", false), nil},
{"/todo", url.Values{"kind": {"build-go-commit"}, "builder": {"linux-386"}}, nil, &Todo{Kind: "build-go-commit", Data: &Commit{Hash: "0005"}}},
{"/result", nil, &Result{Builder: "linux-386", Hash: "0005", OK: true}, nil},
{"/todo", url.Values{"kind": {"build-go-commit"}, "builder": {"linux-386"}}, nil, &Todo{Kind: "build-go-commit", Data: &Commit{Hash: "0004"}}},
@ -112,9 +118,9 @@ var testRequests = []struct {
{"/result", nil, &Result{Builder: "linux-386", Hash: "0003", OK: false, Log: "test"}, nil},
// non-Go repos
{"/commit", nil, tCommit("1001", "1000", testPkg), nil},
{"/commit", nil, tCommit("1002", "1001", testPkg), nil},
{"/commit", nil, tCommit("1003", "1002", testPkg), nil},
{"/commit", nil, tCommit("1001", "1000", testPkg, false), nil},
{"/commit", nil, tCommit("1002", "1001", testPkg, false), nil},
{"/commit", nil, tCommit("1003", "1002", testPkg, false), nil},
{"/todo", url.Values{"kind": {"build-package"}, "builder": {"linux-386"}, "packagePath": {testPkg}, "goHash": {"0001"}}, nil, &Todo{Kind: "build-package", Data: &Commit{Hash: "1003"}}},
{"/result", nil, &Result{PackagePath: testPkg, Builder: "linux-386", Hash: "1003", GoHash: "0001", OK: true}, nil},
{"/todo", url.Values{"kind": {"build-package"}, "builder": {"linux-386"}, "packagePath": {testPkg}, "goHash": {"0001"}}, nil, &Todo{Kind: "build-package", Data: &Commit{Hash: "1002"}}},
@ -128,6 +134,84 @@ var testRequests = []struct {
{"/todo", url.Values{"kind": {"build-go-commit"}, "builder": {"linux-386"}}, nil, &Todo{Kind: "build-go-commit", Data: &Commit{Hash: "0005"}}},
{"/result", nil, &Result{PackagePath: testPkg, Builder: "linux-386", Hash: "1001", GoHash: "0005", OK: false, Log: "boo"}, nil},
{"/todo", url.Values{"kind": {"build-go-commit"}, "builder": {"linux-386"}}, nil, nil},
// benchmarks
// build-go-commit must have precedence over benchmark-go-commit
{"/todo", url.Values{"kind": {"build-go-commit", "benchmark-go-commit"}, "builder": {"linux-amd64"}}, nil, &Todo{Kind: "build-go-commit", Data: &Commit{Hash: "0005"}}},
// drain build-go-commit todo
{"/result", nil, &Result{Builder: "linux-amd64", Hash: "0005", OK: true}, nil},
{"/todo", url.Values{"kind": {"build-go-commit", "benchmark-go-commit"}, "builder": {"linux-amd64"}}, nil, &Todo{Kind: "build-go-commit", Data: &Commit{Hash: "0004"}}},
{"/result", nil, &Result{Builder: "linux-amd64", Hash: "0004", OK: true}, nil},
{"/todo", url.Values{"kind": {"build-go-commit", "benchmark-go-commit"}, "builder": {"linux-amd64"}}, nil, &Todo{Kind: "build-go-commit", Data: &Commit{Hash: "0002"}}},
{"/result", nil, &Result{Builder: "linux-amd64", Hash: "0002", OK: true}, nil},
{"/todo", url.Values{"kind": {"build-go-commit", "benchmark-go-commit"}, "builder": {"linux-amd64"}}, nil, &Todo{Kind: "build-go-commit", Data: &Commit{Hash: "0001"}}},
{"/result", nil, &Result{Builder: "linux-amd64", Hash: "0001", OK: true}, nil},
{"/result", nil, &Result{PackagePath: testPkg, Builder: "linux-amd64", Hash: "1001", GoHash: "0005", OK: false}, nil},
{"/result", nil, &Result{PackagePath: testPkg, Builder: "linux-amd64", Hash: "1002", GoHash: "0005", OK: false}, nil},
{"/result", nil, &Result{PackagePath: testPkg, Builder: "linux-amd64", Hash: "1003", GoHash: "0005", OK: false}, nil},
// now we must get benchmark todo
{"/todo", url.Values{"kind": {"build-go-commit", "benchmark-go-commit"}, "builder": {"linux-amd64"}}, nil, &Todo{Kind: "benchmark-go-commit", Data: &Commit{Hash: "0003", PerfResults: []string{}}}},
{"/perf-result", nil, &PerfRequest{Builder: "linux-amd64", Benchmark: "http", Hash: "0003", OK: true}, nil},
{"/todo", url.Values{"kind": {"build-go-commit", "benchmark-go-commit"}, "builder": {"linux-amd64"}}, nil, &Todo{Kind: "benchmark-go-commit", Data: &Commit{Hash: "0003", PerfResults: []string{"http"}}}},
{"/perf-result", nil, &PerfRequest{Builder: "linux-amd64", Benchmark: "json", Hash: "0003", OK: true}, nil},
{"/todo", url.Values{"kind": {"build-go-commit", "benchmark-go-commit"}, "builder": {"linux-amd64"}}, nil, &Todo{Kind: "benchmark-go-commit", Data: &Commit{Hash: "0003", PerfResults: []string{"http", "json"}}}},
{"/perf-result", nil, &PerfRequest{Builder: "linux-amd64", Benchmark: "meta-done", Hash: "0003", OK: true}, nil},
{"/todo", url.Values{"kind": {"build-go-commit", "benchmark-go-commit"}, "builder": {"linux-amd64"}}, nil, &Todo{Kind: "benchmark-go-commit", Data: &Commit{Hash: "0001", PerfResults: []string{}}}},
{"/perf-result", nil, &PerfRequest{Builder: "linux-amd64", Benchmark: "http", Hash: "0001", OK: true}, nil},
{"/perf-result", nil, &PerfRequest{Builder: "linux-amd64", Benchmark: "meta-done", Hash: "0001", OK: true}, nil},
{"/todo", url.Values{"kind": {"build-go-commit", "benchmark-go-commit"}, "builder": {"linux-amd64"}}, nil, nil},
// create new commit, it must appear in todo
{"/commit", nil, tCommit("0006", "0005", "", true), nil},
// drain build-go-commit todo
{"/todo", url.Values{"kind": {"build-go-commit", "benchmark-go-commit"}, "builder": {"linux-amd64"}}, nil, &Todo{Kind: "build-go-commit", Data: &Commit{Hash: "0006"}}},
{"/result", nil, &Result{Builder: "linux-amd64", Hash: "0006", OK: true}, nil},
{"/result", nil, &Result{PackagePath: testPkg, Builder: "linux-amd64", Hash: "1003", GoHash: "0006", OK: false}, nil},
{"/result", nil, &Result{PackagePath: testPkg, Builder: "linux-amd64", Hash: "1002", GoHash: "0006", OK: false}, nil},
{"/result", nil, &Result{PackagePath: testPkg, Builder: "linux-amd64", Hash: "1001", GoHash: "0006", OK: false}, nil},
// now we must get benchmark todo
{"/todo", url.Values{"kind": {"build-go-commit", "benchmark-go-commit"}, "builder": {"linux-amd64"}}, nil, &Todo{Kind: "benchmark-go-commit", Data: &Commit{Hash: "0006", PerfResults: []string{}}}},
{"/perf-result", nil, &PerfRequest{Builder: "linux-amd64", Benchmark: "http", Hash: "0006", OK: true}, nil},
{"/perf-result", nil, &PerfRequest{Builder: "linux-amd64", Benchmark: "meta-done", Hash: "0006", OK: true}, nil},
{"/todo", url.Values{"kind": {"build-go-commit", "benchmark-go-commit"}, "builder": {"linux-amd64"}}, nil, nil},
// create new benchmark, all commits must re-appear in todo
{"/commit", nil, tCommit("0007", "0006", "", true), nil},
// drain build-go-commit todo
{"/todo", url.Values{"kind": {"build-go-commit", "benchmark-go-commit"}, "builder": {"linux-amd64"}}, nil, &Todo{Kind: "build-go-commit", Data: &Commit{Hash: "0007"}}},
{"/result", nil, &Result{Builder: "linux-amd64", Hash: "0007", OK: true}, nil},
{"/result", nil, &Result{PackagePath: testPkg, Builder: "linux-amd64", Hash: "1003", GoHash: "0007", OK: false}, nil},
{"/result", nil, &Result{PackagePath: testPkg, Builder: "linux-amd64", Hash: "1002", GoHash: "0007", OK: false}, nil},
{"/result", nil, &Result{PackagePath: testPkg, Builder: "linux-amd64", Hash: "1001", GoHash: "0007", OK: false}, nil},
// now we must get benchmark todo
{"/todo", url.Values{"kind": {"build-go-commit", "benchmark-go-commit"}, "builder": {"linux-amd64"}}, nil, &Todo{Kind: "benchmark-go-commit", Data: &Commit{Hash: "0007", PerfResults: []string{}}}},
{"/perf-result", nil, &PerfRequest{Builder: "linux-amd64", Benchmark: "bson", Hash: "0007", OK: true}, nil},
{"/perf-result", nil, &PerfRequest{Builder: "linux-amd64", Benchmark: "meta-done", Hash: "0007", OK: true}, nil},
{"/todo", url.Values{"kind": {"build-go-commit", "benchmark-go-commit"}, "builder": {"linux-amd64"}}, nil, &Todo{Kind: "benchmark-go-commit", Data: &Commit{Hash: "0007", PerfResults: []string{"bson"}}}},
{"/perf-result", nil, &PerfRequest{Builder: "linux-amd64", Benchmark: "meta-done", Hash: "0007", OK: true}, nil},
{"/todo", url.Values{"kind": {"build-go-commit", "benchmark-go-commit"}, "builder": {"linux-amd64"}}, nil, &Todo{Kind: "benchmark-go-commit", Data: &Commit{Hash: "0006", PerfResults: []string{"http"}}}},
{"/perf-result", nil, &PerfRequest{Builder: "linux-amd64", Benchmark: "meta-done", Hash: "0006", OK: true}, nil},
{"/todo", url.Values{"kind": {"build-go-commit", "benchmark-go-commit"}, "builder": {"linux-amd64"}}, nil, &Todo{Kind: "benchmark-go-commit", Data: &Commit{Hash: "0003", PerfResults: []string{"http", "json"}}}},
{"/perf-result", nil, &PerfRequest{Builder: "linux-amd64", Benchmark: "meta-done", Hash: "0003", OK: true}, nil},
{"/todo", url.Values{"kind": {"build-go-commit", "benchmark-go-commit"}, "builder": {"linux-amd64"}}, nil, &Todo{Kind: "benchmark-go-commit", Data: &Commit{Hash: "0001", PerfResults: []string{"http"}}}},
{"/perf-result", nil, &PerfRequest{Builder: "linux-amd64", Benchmark: "meta-done", Hash: "0001", OK: true}, nil},
{"/todo", url.Values{"kind": {"build-go-commit", "benchmark-go-commit"}, "builder": {"linux-amd64"}}, nil, nil},
// attach second builder
{"/todo", url.Values{"kind": {"build-go-commit", "benchmark-go-commit"}, "builder": {"linux-386"}}, nil, &Todo{Kind: "build-go-commit", Data: &Commit{Hash: "0007"}}},
// drain build-go-commit todo
{"/result", nil, &Result{Builder: "linux-386", Hash: "0007", OK: true}, nil},
{"/result", nil, &Result{Builder: "linux-386", Hash: "0006", OK: true}, nil},
{"/result", nil, &Result{PackagePath: testPkg, Builder: "linux-386", Hash: "1003", GoHash: "0007", OK: false}, nil},
{"/result", nil, &Result{PackagePath: testPkg, Builder: "linux-386", Hash: "1002", GoHash: "0007", OK: false}, nil},
{"/result", nil, &Result{PackagePath: testPkg, Builder: "linux-386", Hash: "1001", GoHash: "0007", OK: false}, nil},
// now we must get benchmark todo
{"/todo", url.Values{"kind": {"build-go-commit", "benchmark-go-commit"}, "builder": {"linux-386"}}, nil, &Todo{Kind: "benchmark-go-commit", Data: &Commit{Hash: "0007"}}},
{"/perf-result", nil, &PerfRequest{Builder: "linux-386", Benchmark: "meta-done", Hash: "0007", OK: true}, nil},
{"/todo", url.Values{"kind": {"build-go-commit", "benchmark-go-commit"}, "builder": {"linux-386"}}, nil, &Todo{Kind: "benchmark-go-commit", Data: &Commit{Hash: "0006"}}},
{"/perf-result", nil, &PerfRequest{Builder: "linux-386", Benchmark: "meta-done", Hash: "0006", OK: true}, nil},
{"/todo", url.Values{"kind": {"build-go-commit", "benchmark-go-commit"}, "builder": {"linux-386"}}, nil, &Todo{Kind: "benchmark-go-commit", Data: &Commit{Hash: "0003"}}},
{"/perf-result", nil, &PerfRequest{Builder: "linux-386", Benchmark: "meta-done", Hash: "0003", OK: true}, nil},
{"/todo", url.Values{"kind": {"build-go-commit", "benchmark-go-commit"}, "builder": {"linux-386"}}, nil, &Todo{Kind: "benchmark-go-commit", Data: &Commit{Hash: "0001"}}},
{"/perf-result", nil, &PerfRequest{Builder: "linux-386", Benchmark: "meta-done", Hash: "0001", OK: true}, nil},
{"/todo", url.Values{"kind": {"build-go-commit", "benchmark-go-commit"}, "builder": {"linux-386"}}, nil, nil},
}
func testHandler(w http.ResponseWriter, r *http.Request) {
@ -159,7 +243,7 @@ func testHandler(w http.ResponseWriter, r *http.Request) {
*r = origReq
}()
for i, t := range testRequests {
c.Infof("running test %d %s", i, t.path)
c.Infof("running test %d %s vals='%q' req='%q' res='%q'", i, t.path, t.vals, t.req, t.res)
errorf := func(format string, args ...interface{}) {
fmt.Fprintf(w, "%d %s: ", i, t.path)
fmt.Fprintf(w, format, args...)
@ -194,11 +278,12 @@ func testHandler(w http.ResponseWriter, r *http.Request) {
errorf(rec.Body.String())
return
}
c.Infof("response='%v'", rec.Body.String())
resp := new(dashResponse)
// If we're expecting a *Todo value,
// prime the Response field with a Todo and a Commit inside it.
if _, ok := t.res.(*Todo); ok {
if t.path == "/todo" {
resp.Response = &Todo{Data: &Commit{}}
}
@ -241,14 +326,28 @@ func testHandler(w http.ResponseWriter, r *http.Request) {
errorf("Response.Data not *Commit: %T", g.Data)
return
}
if eh := e.Data.(*Commit).Hash; eh != gd.Hash {
errorf("hashes don't match: got %q, want %q", gd.Hash, eh)
if g.Kind != e.Kind {
errorf("kind don't match: got %q, want %q", g.Kind, e.Kind)
return
}
ed := e.Data.(*Commit)
if ed.Hash != gd.Hash {
errorf("hashes don't match: got %q, want %q", gd.Hash, ed.Hash)
return
}
if len(gd.PerfResults) != len(ed.PerfResults) {
errorf("result data len don't match: got %v, want %v", len(gd.PerfResults), len(ed.PerfResults))
return
}
for i := range gd.PerfResults {
if gd.PerfResults[i] != ed.PerfResults[i] {
errorf("result data %v don't match: got %v, want %v", i, gd.PerfResults[i], ed.PerfResults[i])
return
}
}
}
if t.res == nil && resp.Response != nil {
errorf("response mismatch: got %q expected <nil>",
resp.Response)
errorf("response mismatch: got %q expected <nil>", resp.Response)
return
}
}

5
dashboard/app/cron.yaml Normal file
View File

@ -0,0 +1,5 @@
cron:
- description: updates noise level for benchmarking results
url: /perflearn
schedule: every 24 hours

26
dashboard/app/index.yaml
View File

@ -11,3 +11,29 @@ indexes:
properties:
- name: Time
direction: desc
- kind: Commit
ancestor: yes
properties:
- name: NeedsBenchmarking
- name: Num
direction: desc
- kind: CommitRun
ancestor: yes
properties:
- name: StartCommitNum
direction: desc
- kind: PerfResult
ancestor: yes
properties:
- name: CommitNum
direction: desc
- kind: PerfResult
ancestor: yes
properties:
- name: CommitNum
direction: asc