go/dashboard/app/build/perf.go

// Copyright 2014 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build appengine

package build

import (
	"fmt"
	"sort"
	"strconv"
	"strings"

	"appengine"
	"appengine/datastore"
)

var knownTags = map[string]string{
	"go1":   "0051c7442fed9c888de6617fa9239a913904d96e",
	"go1.1": "d29da2ced72ba2cf48ed6a8f1ec4abc01e4c5bf1",
	"go1.2": "b1edf8faa5d6cbc50c6515785df9df9c19296564",
	"go1.3": "f153208c0a0e306bfca14f71ef11f09859ccabc8",
}

var lastRelease = "go1.3"

func splitBench(benchProcs string) (string, int) {
	ss := strings.Split(benchProcs, "-")
	procs, _ := strconv.Atoi(ss[1])
	return ss[0], procs
}

func dashPerfCommits(c appengine.Context, page int) ([]*Commit, error) {
	q := datastore.NewQuery("Commit").
		Ancestor((&Package{}).Key(c)).
		Order("-Num").
		Filter("NeedsBenchmarking =", true).
		Limit(commitsPerPage).
		Offset(page * commitsPerPage)
	var commits []*Commit
	_, err := q.GetAll(c, &commits)
	if err == nil && len(commits) == 0 {
		err = fmt.Errorf("no commits")
	}
	return commits, err
}

func perfChangeStyle(pc *PerfConfig, v float64, builder, benchmark, metric string) string {
	noise := pc.NoiseLevel(builder, benchmark, metric)
	if isNoise(v, noise) {
		return "noise"
	}
	if v > 0 {
		return "bad"
	}
	return "good"
}

func isNoise(diff, noise float64) bool {
	rnoise := -100 * noise / (noise + 100)
	return diff < noise && diff > rnoise
}

func perfDiff(old, new uint64) float64 {
	return 100*float64(new)/float64(old) - 100
}

func isPerfFailed(res *PerfResult, builder string) bool {
	data := res.ParseData()[builder]
	return data != nil && data["meta-done"] != nil && !data["meta-done"].OK
}

// PerfResultCache caches a set of PerfResults so that it's easy to access them
// without lots of duplicate accesses to datastore.
// It allows to iterate over newer or older results for some base commit.
type PerfResultCache struct {
	c       appengine.Context
	newer   bool
	iter    *datastore.Iterator
	results map[int]*PerfResult
}

func MakePerfResultCache(c appengine.Context, com *Commit, newer bool) *PerfResultCache {
	p := &Package{}
	q := datastore.NewQuery("PerfResult").Ancestor(p.Key(c)).Limit(100)
	if newer {
		q = q.Filter("CommitNum >=", com.Num).Order("CommitNum")
	} else {
		q = q.Filter("CommitNum <=", com.Num).Order("-CommitNum")
	}
	rc := &PerfResultCache{c: c, newer: newer, iter: q.Run(c), results: make(map[int]*PerfResult)}
	return rc
}

func (rc *PerfResultCache) Get(commitNum int) *PerfResult {
	rc.Next(commitNum) // fetch the commit, if necessary
	return rc.results[commitNum]
}

// Next returns the next PerfResult for the commit commitNum.
// It does not care whether the result has any data, failed or whatever.
func (rc *PerfResultCache) Next(commitNum int) (*PerfResult, error) {
	// See if we have next result in the cache.
	next := -1
	for ci := range rc.results {
		if rc.newer {
			if ci > commitNum && (next == -1 || ci < next) {
				next = ci
			}
		} else {
			if ci < commitNum && (next == -1 || ci > next) {
				next = ci
			}
		}
	}
	if next != -1 {
		return rc.results[next], nil
	}
	// Fetch next result from datastore.
	res := new(PerfResult)
	_, err := rc.iter.Next(res)
	if err == datastore.Done {
		return nil, nil
	}
	if err != nil {
		return nil, fmt.Errorf("fetching perf results: %v", err)
	}
	if (rc.newer && res.CommitNum < commitNum) || (!rc.newer && res.CommitNum > commitNum) {
		rc.c.Errorf("PerfResultCache.Next: bad commit num")
	}
	rc.results[res.CommitNum] = res
	return res, nil
}

// NextForComparison returns PerfResult which we need to use for performance comprison.
// It skips failed results, but does not skip results with no data.
func (rc *PerfResultCache) NextForComparison(commitNum int, builder string) (*PerfResult, error) {
	for {
		res, err := rc.Next(commitNum)
		if err != nil {
			return nil, err
		}
		if res == nil {
			return nil, nil
		}
		if res.CommitNum == commitNum {
			continue
		}
		parsed := res.ParseData()
		if builder != "" {
			// Comparing for a particular builder.
			// This is used in perf_changes and in email notifications.
			b := parsed[builder]
			if b == nil || b["meta-done"] == nil {
				// No results yet, must not do the comparison.
				return nil, nil
			}
			if b["meta-done"].OK {
				// Have complete results, compare.
				return res, nil
			}
		} else {
			// Comparing for all builders, find a result with at least
			// one successful meta-done.
			// This is used in perf_detail.
			for _, benchs := range parsed {
				if data := benchs["meta-done"]; data != nil && data.OK {
					return res, nil
				}
			}
		}
		// Failed, try next result.
		commitNum = res.CommitNum
	}
}

type PerfChange struct {
	builder string
	bench   string
	metric  string
	old     uint64
	new     uint64
	diff    float64
}

func significantPerfChanges(pc *PerfConfig, builder string, prevRes, res *PerfResult) (changes []*PerfChange) {
	// First, collect all significant changes.
	for builder1, benchmarks1 := range res.ParseData() {
		if builder != "" && builder != builder1 {
			// This is not the builder you're looking for, Luke.
			continue
		}
		benchmarks0 := prevRes.ParseData()[builder1]
		if benchmarks0 == nil {
			continue
		}
		for benchmark, data1 := range benchmarks1 {
			data0 := benchmarks0[benchmark]
			if data0 == nil {
				continue
			}
			for metric, val := range data1.Metrics {
				val0 := data0.Metrics[metric]
				if val0 == 0 {
					continue
				}
				diff := perfDiff(val0, val)
				noise := pc.NoiseLevel(builder, benchmark, metric)
				if isNoise(diff, noise) {
					continue
				}
				ch := &PerfChange{builder: builder, bench: benchmark, metric: metric, old: val0, new: val, diff: diff}
				changes = append(changes, ch)
			}
		}
	}
	// Then, strip non-repeatable changes (flakes).
	// The hypothesis is that a real change must show up with the majority of GOMAXPROCS values.
	majority := len(pc.ProcList(builder))/2 + 1
	cnt := make(map[string]int)
	for _, ch := range changes {
		b, _ := splitBench(ch.bench)
		name := b + "|" + ch.metric
		if ch.diff < 0 {
			name += "--"
		}
		cnt[name] = cnt[name] + 1
	}
	for i := 0; i < len(changes); i++ {
		ch := changes[i]
		b, _ := splitBench(ch.bench)
		name := b + "|" + ch.metric
		if cnt[name] >= majority {
			continue
		}
		if cnt[name+"--"] >= majority {
			continue
		}
		// Remove flake.
		last := len(changes) - 1
		changes[i] = changes[last]
		changes = changes[:last]
		i--
	}
	return changes
}

// orderPrefTodo reorders commit nums for benchmarking todo.
// The resulting order is somewhat tricky. We want 2 things:
// 1. benchmark sequentially backwards (this provides information about most
// recent changes, and allows to estimate noise levels)
// 2. benchmark old commits in "scatter" order (this allows to quickly gather
// brief information about thousands of old commits)
// So this function interleaves the two orders.
func orderPrefTodo(nums []int) []int {
	sort.Ints(nums)
	n := len(nums)
	pow2 := uint32(0) // next power-of-two that is >= n
	npow2 := 0
	for npow2 <= n {
		pow2++
		npow2 = 1 << pow2
	}
	res := make([]int, n)
	resPos := n - 1            // result array is filled backwards
	present := make([]bool, n) // denotes values that already present in result array
	for i0, i1 := n-1, 0; i0 >= 0 || i1 < npow2; {
		// i0 represents "benchmark sequentially backwards" sequence
		// find the next commit that is not yet present and add it
		for cnt := 0; cnt < 2; cnt++ {
			for ; i0 >= 0; i0-- {
				if !present[i0] {
					present[i0] = true
					res[resPos] = nums[i0]
					resPos--
					i0--
					break
				}
			}
		}
		// i1 represents "scatter order" sequence
		// find the next commit that is not yet present and add it
		for ; i1 < npow2; i1++ {
			// do the "recursive split-ordering" trick
			idx := 0 // bitwise reverse of i1
			for j := uint32(0); j <= pow2; j++ {
				if (i1 & (1 << j)) != 0 {
					idx = idx | (1 << (pow2 - j - 1))
				}
			}
			if idx < n && !present[idx] {
				present[idx] = true
				res[resPos] = nums[idx]
				resPos--
				i1++
				break
			}
		}
	}
	// The above can't possibly be correct. Do dump check.
	res2 := make([]int, n)
	copy(res2, res)
	sort.Ints(res2)
	for i := range res2 {
		if res2[i] != nums[i] {
			panic(fmt.Sprintf("diff at %v: expect %v, want %v\nwas: %v\n become: %v",
				i, nums[i], res2[i], nums, res2))
		}
	}
	return res
}