test/bench/go1: first draft of Go 1 benchmark suite

I have included a few important microbenchmarks,
but the overall intent is to have mostly end-to-end
benchmarks timing real world operations.

The jsondata.go file is a summary of agl's
activity in various open source repositories.
It gets used as test data for many of the benchmarks.

Everything links into one binary (even the test data)
so that it is easy to run the benchmarks on many
computers: there is just one file to copy around.

R=golang-dev, r, bradfitz, adg, r
CC=golang-dev
https://golang.org/cl/5484071

src/clean.bash

@@ -24,7 +24,7 @@ for i in lib9 libbio libmach cmd pkg \
 	../misc/cgo/life ../misc/cgo/test \
 	../misc/dashboard/builder ../misc/goplay\
 	../doc/codelab/wiki\
-	../test/bench/shootout ../test/bench/garbage
+	../test/bench/shootout ../test/bench/garbage ../test/bench/go1
 do
 	# Do not use gomake here. It may not be available.
 	$MAKE -C "$GOROOT/src/$i" clean

src/run.bash

@@ -105,6 +105,10 @@ done
 ./timing.sh -test
 ) || exit $?
 
+(xcd ../test/bench/go1
+gomake test
+) || exit $?
+
 (xcd ../test
 ./run
 ) || exit $?

test/bench/go1/Makefile

@@ -0,0 +1,7 @@
+include $(GOROOT)/src/Make.inc
+
+TARG=go1
+GOFILES=\
+	dummy.go\
+
+include $(GOROOT)/src/Make.pkg

test/bench/go1/binarytree_test.go

@@ -0,0 +1,63 @@
+// Copyright 2011 The Go Authors.  All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// This benchmark, taken from the shootout, tests garbage collector
+// performance by generating and discarding large binary trees.
+
+package go1
+
+import "testing"
+
+type binaryNode struct {
+	item        int
+	left, right *binaryNode
+}
+
+func bottomUpTree(item, depth int) *binaryNode {
+	if depth <= 0 {
+		return &binaryNode{item: item}
+	}
+	return &binaryNode{item, bottomUpTree(2*item-1, depth-1), bottomUpTree(2*item, depth-1)}
+}
+
+func (n *binaryNode) itemCheck() int {
+	if n.left == nil {
+		return n.item
+	}
+	return n.item + n.left.itemCheck() - n.right.itemCheck()
+}
+
+const minDepth = 4
+
+func binarytree(n int) {
+	maxDepth := n
+	if minDepth+2 > n {
+		maxDepth = minDepth + 2
+	}
+	stretchDepth := maxDepth + 1
+
+	check := bottomUpTree(0, stretchDepth).itemCheck()
+	//fmt.Printf("stretch tree of depth %d\t check: %d\n", stretchDepth, check)
+
+	longLivedTree := bottomUpTree(0, maxDepth)
+
+	for depth := minDepth; depth <= maxDepth; depth += 2 {
+		iterations := 1 << uint(maxDepth-depth+minDepth)
+		check = 0
+
+		for i := 1; i <= iterations; i++ {
+			check += bottomUpTree(i, depth).itemCheck()
+			check += bottomUpTree(-i, depth).itemCheck()
+		}
+		//fmt.Printf("%d\t trees of depth %d\t check: %d\n", iterations*2, depth, check)
+	}
+	longLivedTree.itemCheck()
+	//fmt.Printf("long lived tree of depth %d\t check: %d\n", maxDepth, longLivedTree.itemCheck())
+}
+
+func BenchmarkBinaryTree17(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		binarytree(17)
+	}
+}

test/bench/go1/dummy.go

@@ -0,0 +1,3 @@
+package go1
+
+// Nothing to see here: everything is in the _test files.

test/bench/go1/fannkuch_test.go

@@ -0,0 +1,84 @@
+// Copyright 2011 The Go Authors.  All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// This benchmark, taken from the shootout, tests array indexing
+// and array bounds elimination performance.
+
+package go1
+
+import "testing"
+
+func fannkuch(n int) int {
+	if n < 1 {
+		return 0
+	}
+
+	n1 := n - 1
+	perm := make([]int, n)
+	perm1 := make([]int, n)
+	count := make([]int, n)
+
+	for i := 0; i < n; i++ {
+		perm1[i] = i // initial (trivial) permutation
+	}
+
+	r := n
+	didpr := 0
+	flipsMax := 0
+	for {
+		if didpr < 30 {
+			didpr++
+		}
+		for ; r != 1; r-- {
+			count[r-1] = r
+		}
+
+		if perm1[0] != 0 && perm1[n1] != n1 {
+			flips := 0
+			for i := 1; i < n; i++ { // perm = perm1
+				perm[i] = perm1[i]
+			}
+			k := perm1[0] // cache perm[0] in k
+			for {         // k!=0 ==> k>0
+				for i, j := 1, k-1; i < j; i, j = i+1, j-1 {
+					perm[i], perm[j] = perm[j], perm[i]
+				}
+				flips++
+				// Now exchange k (caching perm[0]) and perm[k]... with care!
+				j := perm[k]
+				perm[k] = k
+				k = j
+				if k == 0 {
+					break
+				}
+			}
+			if flipsMax < flips {
+				flipsMax = flips
+			}
+		}
+
+		for ; r < n; r++ {
+			// rotate down perm[0..r] by one
+			perm0 := perm1[0]
+			for i := 0; i < r; i++ {
+				perm1[i] = perm1[i+1]
+			}
+			perm1[r] = perm0
+			count[r]--
+			if count[r] > 0 {
+				break
+			}
+		}
+		if r == n {
+			return flipsMax
+		}
+	}
+	return 0
+}
+
+func BenchmarkFannkuch11(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		fannkuch(11)
+	}
+}

test/bench/go1/fasta_test.go

@@ -0,0 +1,164 @@
+// Copyright 2011 The Go Authors.  All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package go1
+
+// Not a benchmark; input for revcomp.
+
+var fasta25m = fasta(25e6)
+
+func fasta(n int) []byte {
+	out := make(fastaBuffer, 0, 11*n)
+
+	iub := []fastaAcid{
+		{prob: 0.27, sym: 'a'},
+		{prob: 0.12, sym: 'c'},
+		{prob: 0.12, sym: 'g'},
+		{prob: 0.27, sym: 't'},
+		{prob: 0.02, sym: 'B'},
+		{prob: 0.02, sym: 'D'},
+		{prob: 0.02, sym: 'H'},
+		{prob: 0.02, sym: 'K'},
+		{prob: 0.02, sym: 'M'},
+		{prob: 0.02, sym: 'N'},
+		{prob: 0.02, sym: 'R'},
+		{prob: 0.02, sym: 'S'},
+		{prob: 0.02, sym: 'V'},
+		{prob: 0.02, sym: 'W'},
+		{prob: 0.02, sym: 'Y'},
+	}
+
+	homosapiens := []fastaAcid{
+		{prob: 0.3029549426680, sym: 'a'},
+		{prob: 0.1979883004921, sym: 'c'},
+		{prob: 0.1975473066391, sym: 'g'},
+		{prob: 0.3015094502008, sym: 't'},
+	}
+
+	alu := []byte(
+		"GGCCGGGCGCGGTGGCTCACGCCTGTAATCCCAGCACTTTGG" +
+			"GAGGCCGAGGCGGGCGGATCACCTGAGGTCAGGAGTTCGAGA" +
+			"CCAGCCTGGCCAACATGGTGAAACCCCGTCTCTACTAAAAAT" +
+			"ACAAAAATTAGCCGGGCGTGGTGGCGCGCGCCTGTAATCCCA" +
+			"GCTACTCGGGAGGCTGAGGCAGGAGAATCGCTTGAACCCGGG" +
+			"AGGCGGAGGTTGCAGTGAGCCGAGATCGCGCCACTGCACTCC" +
+			"AGCCTGGGCGACAGAGCGAGACTCCGTCTCAAAAA")
+
+	out.WriteString(">ONE Homo sapiens alu\n")
+	fastaRepeat(&out, alu, 2*n)
+	out.WriteString(">TWO IUB ambiguity codes\n")
+	fastaRandom(&out, iub, 3*n)
+	out.WriteString(">THREE Homo sapiens frequency\n")
+	fastaRandom(&out, homosapiens, 5*n)
+	return out
+}
+
+type fastaBuffer []byte
+
+func (b *fastaBuffer) Flush() {
+	panic("flush")
+}
+
+func (b *fastaBuffer) WriteString(s string) {
+	p := b.NextWrite(len(s))
+	copy(p, s)
+}
+
+func (b *fastaBuffer) NextWrite(n int) []byte {
+	p := *b
+	if len(p)+n > cap(p) {
+		b.Flush()
+		p = *b
+	}
+	out := p[len(p) : len(p)+n]
+	*b = p[:len(p)+n]
+	return out
+}
+
+const fastaLine = 60
+
+func fastaRepeat(out *fastaBuffer, alu []byte, n int) {
+	buf := append(alu, alu...)
+	off := 0
+	for n > 0 {
+		m := n
+		if m > fastaLine {
+			m = fastaLine
+		}
+		buf1 := out.NextWrite(m + 1)
+		copy(buf1, buf[off:])
+		buf1[m] = '\n'
+		if off += m; off >= len(alu) {
+			off -= len(alu)
+		}
+		n -= m
+	}
+}
+
+const (
+	fastaLookupSize          = 4096
+	fastaLookupScale float64 = fastaLookupSize - 1
+)
+
+var fastaRand uint32 = 42
+
+type fastaAcid struct {
+	sym   byte
+	prob  float64
+	cprob float64
+	next  *fastaAcid
+}
+
+func fastaComputeLookup(acid []fastaAcid) *[fastaLookupSize]*fastaAcid {
+	var lookup [fastaLookupSize]*fastaAcid
+	var p float64
+	for i := range acid {
+		p += acid[i].prob
+		acid[i].cprob = p * fastaLookupScale
+		if i > 0 {
+			acid[i-1].next = &acid[i]
+		}
+	}
+	acid[len(acid)-1].cprob = 1.0 * fastaLookupScale
+
+	j := 0
+	for i := range lookup {
+		for acid[j].cprob < float64(i) {
+			j++
+		}
+		lookup[i] = &acid[j]
+	}
+
+	return &lookup
+}
+
+func fastaRandom(out *fastaBuffer, acid []fastaAcid, n int) {
+	const (
+		IM = 139968
+		IA = 3877
+		IC = 29573
+	)
+	lookup := fastaComputeLookup(acid)
+	for n > 0 {
+		m := n
+		if m > fastaLine {
+			m = fastaLine
+		}
+		buf := out.NextWrite(m + 1)
+		f := fastaLookupScale / IM
+		myrand := fastaRand
+		for i := 0; i < m; i++ {
+			myrand = (myrand*IA + IC) % IM
+			r := float64(int(myrand)) * f
+			a := lookup[int(r)]
+			for a.cprob < r {
+				a = a.next
+			}
+			buf[i] = a.sym
+		}
+		fastaRand = myrand
+		buf[m] = '\n'
+		n -= m
+	}
+}

test/bench/go1/gob_test.go

@@ -0,0 +1,95 @@
+// Copyright 2011 The Go Authors.  All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// This benchmark tests gob encoding and decoding performance.
+
+package go1
+
+import (
+	"bytes"
+	"encoding/gob"
+	"encoding/json"
+	"io/ioutil"
+	"log"
+	"reflect"
+	"testing"
+)
+
+var (
+	gobbytes []byte
+	gobdata  *JSONResponse
+)
+
+func gobinit() {
+	// gobinit is called after json's init,
+	// because it uses jsondata.
+	gobdata = gobResponse(&jsondata)
+
+	var buf bytes.Buffer
+	if err := gob.NewEncoder(&buf).Encode(gobdata); err != nil {
+		panic(err)
+	}
+	gobbytes = buf.Bytes()
+
+	var r JSONResponse
+	if err := gob.NewDecoder(bytes.NewBuffer(gobbytes)).Decode(&r); err != nil {
+		panic(err)
+	}
+	if !reflect.DeepEqual(gobdata, &r) {
+		log.Printf("%v\n%v", jsondata, r)
+		b, _ := json.Marshal(&jsondata)
+		br, _ := json.Marshal(&r)
+		log.Printf("%s\n%s\n", b, br)
+		panic("gob: encode+decode lost data")
+	}
+}
+
+// gob turns [] into null, so make a copy of the data structure like that
+func gobResponse(r *JSONResponse) *JSONResponse {
+	return &JSONResponse{gobNode(r.Tree), r.Username}
+}
+
+func gobNode(n *JSONNode) *JSONNode {
+	n1 := new(JSONNode)
+	*n1 = *n
+	if len(n1.Kids) == 0 {
+		n1.Kids = nil
+	} else {
+		for i, k := range n1.Kids {
+			n1.Kids[i] = gobNode(k)
+		}
+	}
+	return n1
+}
+
+func gobdec() {
+	if gobbytes == nil {
+		panic("gobdata not initialized")
+	}
+	var r JSONResponse
+	if err := gob.NewDecoder(bytes.NewBuffer(gobbytes)).Decode(&r); err != nil {
+		panic(err)
+	}
+	_ = r
+}
+
+func gobenc() {
+	if err := gob.NewEncoder(ioutil.Discard).Encode(&gobdata); err != nil {
+		panic(err)
+	}
+}
+
+func BenchmarkGobDecode(b *testing.B) {
+	b.SetBytes(int64(len(gobbytes)))
+	for i := 0; i < b.N; i++ {
+		gobdec()
+	}
+}
+
+func BenchmarkGobEncode(b *testing.B) {
+	b.SetBytes(int64(len(gobbytes)))
+	for i := 0; i < b.N; i++ {
+		gobenc()
+	}
+}

test/bench/go1/gzip_test.go

@@ -0,0 +1,69 @@
+// Copyright 2011 The Go Authors.  All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// This benchmark tests gzip and gunzip performance.
+
+package go1
+
+import (
+	"bytes"
+	gz "compress/gzip"
+	"io"
+	"io/ioutil"
+	"testing"
+)
+
+var (
+	jsongunz = bytes.Repeat(jsonbytes, 10)
+	jsongz   []byte
+)
+
+func init() {
+	var buf bytes.Buffer
+	c, err := gz.NewWriter(&buf)
+	if err != nil {
+		panic(err)
+	}
+	c.Write(jsongunz)
+	c.Close()
+	jsongz = buf.Bytes()
+}
+
+func gzip() {
+	c, err := gz.NewWriter(ioutil.Discard)
+	if err != nil {
+		panic(err)
+	}
+	if _, err := c.Write(jsongunz); err != nil {
+		panic(err)
+	}
+	if err := c.Close(); err != nil {
+		panic(err)
+	}
+}
+
+func gunzip() {
+	r, err := gz.NewReader(bytes.NewBuffer(jsongz))
+	if err != nil {
+		panic(err)
+	}
+	if _, err := io.Copy(ioutil.Discard, r); err != nil {
+		panic(err)
+	}
+	r.Close()
+}
+
+func BenchmarkGzip(b *testing.B) {
+	b.SetBytes(int64(len(jsongunz)))
+	for i := 0; i < b.N; i++ {
+		gzip()
+	}
+}
+
+func BenchmarkGunzip(b *testing.B) {
+	b.SetBytes(int64(len(jsongunz)))
+	for i := 0; i < b.N; i++ {
+		gunzip()
+	}
+}

test/bench/go1/json_test.go

@@ -0,0 +1,84 @@
+// Copyright 2011 The Go Authors.  All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// This benchmark tests JSON encoding and decoding performance.
+
+package go1
+
+import (
+	"compress/bzip2"
+	"encoding/base64"
+	"encoding/json"
+	"io"
+	"io/ioutil"
+	"strings"
+	"testing"
+)
+
+var (
+	jsonbytes []byte
+	jsondata  JSONResponse
+)
+
+func init() {
+	var r io.Reader
+	r = strings.NewReader(jsonbz2_base64)
+	r = base64.NewDecoder(base64.StdEncoding, r)
+	r = bzip2.NewReader(r)
+	b, err := ioutil.ReadAll(r)
+	if err != nil {
+		panic(err)
+	}
+	jsonbytes = b
+
+	if err := json.Unmarshal(jsonbytes, &jsondata); err != nil {
+		panic(err)
+	}
+	gobinit()
+}
+
+type JSONResponse struct {
+	Tree     *JSONNode `json:"tree"`
+	Username string    `json:"username"`
+}
+
+type JSONNode struct {
+	Name     string      `json:"name"`
+	Kids     []*JSONNode `json:"kids"`
+	CLWeight float64     `json:"cl_weight"`
+	Touches  int         `json:"touches"`
+	MinT     int64       `json:"min_t"`
+	MaxT     int64       `json:"max_t"`
+	MeanT    int64       `json:"mean_t"`
+}
+
+func jsondec() {
+	var r JSONResponse
+	if err := json.Unmarshal(jsonbytes, &r); err != nil {
+		panic(err)
+	}
+	_ = r
+}
+
+func jsonenc() {
+	buf, err := json.Marshal(&jsondata)
+	if err != nil {
+		panic(err)
+	}
+	_ = buf
+}
+
+func BenchmarkJSONEncode(b *testing.B) {
+	b.SetBytes(int64(len(jsonbytes)))
+	for i := 0; i < b.N; i++ {
+		jsonenc()
+	}
+}
+
+func BenchmarkJSONDecode(b *testing.B) {
+	b.SetBytes(int64(len(jsonbytes)))
+	for i := 0; i < b.N; i++ {
+		jsondec()
+	}
+}

test/bench/go1/revcomp_test.go

@@ -0,0 +1,85 @@
+// Copyright 2011 The Go Authors.  All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// This benchmark, taken from the shootout, tests array indexing
+// and array bounds elimination performance.
+
+package go1
+
+import (
+	"bufio"
+	"bytes"
+	"io/ioutil"
+	"testing"
+)
+
+var revCompTable = [256]uint8{
+	'A': 'T', 'a': 'T',
+	'C': 'G', 'c': 'G',
+	'G': 'C', 'g': 'C',
+	'T': 'A', 't': 'A',
+	'U': 'A', 'u': 'A',
+	'M': 'K', 'm': 'K',
+	'R': 'Y', 'r': 'Y',
+	'W': 'W', 'w': 'W',
+	'S': 'S', 's': 'S',
+	'Y': 'R', 'y': 'R',
+	'K': 'M', 'k': 'M',
+	'V': 'B', 'v': 'B',
+	'H': 'D', 'h': 'D',
+	'D': 'H', 'd': 'H',
+	'B': 'V', 'b': 'V',
+	'N': 'N', 'n': 'N',
+}
+
+func revcomp(data []byte) {
+	in := bufio.NewReader(bytes.NewBuffer(data))
+	out := ioutil.Discard
+	buf := make([]byte, 1024*1024)
+	line, err := in.ReadSlice('\n')
+	for err == nil {
+		out.Write(line)
+
+		// Accumulate reversed complement in buf[w:]
+		nchar := 0
+		w := len(buf)
+		for {
+			line, err = in.ReadSlice('\n')
+			if err != nil || line[0] == '>' {
+				break
+			}
+			line = line[0 : len(line)-1]
+			nchar += len(line)
+			if len(line)+nchar/60+128 >= w {
+				nbuf := make([]byte, len(buf)*5)
+				copy(nbuf[len(nbuf)-len(buf):], buf)
+				w += len(nbuf) - len(buf)
+				buf = nbuf
+			}
+
+			// This loop is the bottleneck.
+			for _, c := range line {
+				w--
+				buf[w] = revCompTable[c]
+			}
+		}
+
+		// Copy down to beginning of buffer, inserting newlines.
+		// The loop left room for the newlines and 128 bytes of padding.
+		i := 0
+		for j := w; j < len(buf); j += 60 {
+			n := copy(buf[i:i+60], buf[j:])
+			buf[i+n] = '\n'
+			i += n + 1
+		}
+		out.Write(buf[0:i])
+	}
+}
+
+func BenchmarkRevcomp25M(b *testing.B) {
+	b.SetBytes(int64(len(fasta25m)))
+	for i := 0; i < b.N; i++ {
+		revcomp(fasta25m)
+	}
+}

test/bench/go1/template_test.go

@@ -0,0 +1,76 @@
+// Copyright 2011 The Go Authors.  All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// This benchmark tests text/template throughput,
+// converting a large data structure with a simple template.
+
+package go1
+
+import (
+	"bytes"
+	"io/ioutil"
+	"strings"
+	"testing"
+	"text/template"
+)
+
+// After removing \t and \n this generates identical output to
+// json.Marshal, making it easy to test for correctness.
+const tmplText = `
+{
+	"tree":{{template "node" .Tree}},
+	"username":"{{.Username}}"
+}
+{{define "node"}}
+{
+	"name":"{{.Name}}",
+	"kids":[
+	{{range $i, $k := .Kids}}
+		{{if $i}}
+			,
+		{{end}}
+		{{template "node" $k}}
+	{{end}}
+	],
+	"cl_weight":{{.CLWeight}},
+	"touches":{{.Touches}},
+	"min_t":{{.MinT}},
+	"max_t":{{.MaxT}},
+	"mean_t":{{.MeanT}}
+}
+{{end}}
+`
+
+func stripTabNL(r rune) rune {
+	if r == '\t' || r == '\n' {
+		return -1
+	}
+	return r
+}
+
+var tmpl = template.Must(template.New("main").Parse(strings.Map(stripTabNL, tmplText)))
+
+func init() {
+	var buf bytes.Buffer
+	if err := tmpl.Execute(&buf, &jsondata); err != nil {
+		panic(err)
+	}
+	if !bytes.Equal(buf.Bytes(), jsonbytes) {
+		println(buf.Len(), len(jsonbytes))
+		panic("wrong output")
+	}
+}
+
+func tmplexec() {
+	if err := tmpl.Execute(ioutil.Discard, &jsondata); err != nil {
+		panic(err)
+	}
+}
+
+func BenchmarkTemplate(b *testing.B) {
+	b.SetBytes(int64(len(jsonbytes)))
+	for i := 0; i < b.N; i++ {
+		tmplexec()
+	}
+}