dashboard: faster perf data fetching

Use single query to fetch commit runs and metric runs (instead of individual selects).
Hopefully this enables some kind of prefetching.
But more importantly it allows to work around "gaps" in commit nums,
as we only fetch data that is actually in the database
and don't try to query all commit runs in the "gap".

LGTM=adg
R=adg
CC=golang-codereviews, rsc
https://golang.org/cl/159910045

dashboard/app/build/build.go

@@ -315,28 +315,50 @@ func GetCommits(c appengine.Context, startCommitNum, n int) ([]*Commit, error) {
 	if startCommitNum < 0 || n <= 0 {
 		return nil, fmt.Errorf("GetCommits: invalid args (%v, %v)", startCommitNum, n)
 	}
-	var res []*Commit
-	for n > 0 {
-		cr, err := GetCommitRun(c, startCommitNum)
+
+	p := &Package{}
+	t := datastore.NewQuery("CommitRun").
+		Ancestor(p.Key(c)).
+		Filter("StartCommitNum >=", startCommitNum/PerfRunLength*PerfRunLength).
+		Order("StartCommitNum").
+		Limit(100).
+		Run(c)
+
+	res := make([]*Commit, n)
+	for {
+		cr := new(CommitRun)
+		_, err := t.Next(cr)
+		if err == datastore.Done {
+			break
+		}
 		if err != nil {
 			return nil, err
 		}
-		idx := startCommitNum - cr.StartCommitNum
-		cnt := PerfRunLength - idx
-		if cnt > n {
-			cnt = n
+		if cr.StartCommitNum >= startCommitNum+n {
+			break
 		}
-		for i := idx; i < idx+cnt; i++ {
+		// Calculate start index for copying.
+		i := 0
+		if cr.StartCommitNum < startCommitNum {
+			i = startCommitNum - cr.StartCommitNum
+		}
+		// Calculate end index for copying.
+		e := PerfRunLength
+		if cr.StartCommitNum+e > startCommitNum+n {
+			e = startCommitNum + n - cr.StartCommitNum
+		}
+		for ; i < e; i++ {
 			com := new(Commit)
 			com.Hash = cr.Hash[i]
 			com.User = cr.User[i]
 			com.Desc = cr.Desc[i]
 			com.Time = cr.Time[i]
 			com.NeedsBenchmarking = cr.NeedsBenchmarking[i]
-			res = append(res, com)
+			res[cr.StartCommitNum-startCommitNum+i] = com
+		}
+		if e != PerfRunLength {
+			break
 		}
-		startCommitNum += cnt
-		n -= cnt
 	}
 	return res, nil
 }
@@ -535,22 +557,47 @@ func GetPerfMetricsForCommits(c appengine.Context, builder, benchmark, metric st
 	if startCommitNum < 0 || n <= 0 {
 		return nil, fmt.Errorf("GetPerfMetricsForCommits: invalid args (%v, %v)", startCommitNum, n)
 	}
-	var res []uint64
-	for n > 0 {
-		metrics, err := GetPerfMetricRun(c, builder, benchmark, metric, startCommitNum)
+
+	p := &Package{}
+	t := datastore.NewQuery("PerfMetricRun").
+		Ancestor(p.Key(c)).
+		Filter("Builder =", builder).
+		Filter("Benchmark =", benchmark).
+		Filter("Metric =", metric).
+		Filter("StartCommitNum >=", startCommitNum/PerfRunLength*PerfRunLength).
+		Order("StartCommitNum").
+		Limit(100).
+		Run(c)
+
+	res := make([]uint64, n)
+	for {
+		metrics := new(PerfMetricRun)
+		_, err := t.Next(metrics)
+		if err == datastore.Done {
+			break
+		}
 		if err != nil {
 			return nil, err
 		}
-		idx := startCommitNum - metrics.StartCommitNum
-		cnt := PerfRunLength - idx
-		if cnt > n {
-			cnt = n
+		if metrics.StartCommitNum >= startCommitNum+n {
+			break
 		}
-		for _, v := range metrics.Vals[idx : idx+cnt] {
-			res = append(res, uint64(v))
+		// Calculate start index for copying.
+		i := 0
+		if metrics.StartCommitNum < startCommitNum {
+			i = startCommitNum - metrics.StartCommitNum
+		}
+		// Calculate end index for copying.
+		e := PerfRunLength
+		if metrics.StartCommitNum+e > startCommitNum+n {
+			e = startCommitNum + n - metrics.StartCommitNum
+		}
+		for ; i < e; i++ {
+			res[metrics.StartCommitNum-startCommitNum+i] = uint64(metrics.Vals[i])
+		}
+		if e != PerfRunLength {
+			break
 		}
-		startCommitNum += cnt
-		n -= cnt
 	}
 	return res, nil
 }

dashboard/app/build/perf_graph.go

@@ -231,7 +231,7 @@ func perfGraphHandler(w http.ResponseWriter, r *http.Request) {
 		idx := 0
 		for i := range vals[0] {
 			com := commits2[i]
-			if !com.NeedsBenchmarking {
+			if com == nil || !com.NeedsBenchmarking {
 				continue
 			}
 			c := perfGraphCommit{Id: idx, Name: fmt.Sprintf("%v (%v)", com.Desc, com.Time.Format("Jan 2, 2006 1:04"))}

dashboard/app/build/perf_learn.go

@@ -67,7 +67,7 @@ func perfLearnHandler(w http.ResponseWriter, r *http.Request) {
 					last := uint64(0)
 					for i, v := range values {
 						if v == 0 {
-							if commits[i].NeedsBenchmarking {
+							if com := commits[i]; com == nil || com.NeedsBenchmarking {
 								last = 0
 							}
 							continue

dashboard/app/index.yaml

@@ -37,3 +37,18 @@ indexes:
   - name: CommitNum
     direction: asc
 
+- kind: CommitRun
+  ancestor: yes
+  properties:
+  - name: StartCommitNum
+    direction: asc
+
+- kind: PerfMetricRun
+  ancestor: yes
+  properties:
+  - name: Builder
+  - name: Benchmark
+  - name: Metric
+  - name: StartCommitNum
+    direction: asc
+