internal/stack: adding an internal stack dump parsing library

This can be used either to directly parse runtime.Stack output or
process text that includes stack dumps, like test timeouts or panics.
It includes a binary, gostacks that processes stdin to stdout replacing
stack dumps in place.

Change-Id: Id7b1cfd69b8aea36c66f12ec0bdf38b68cba5afb
Reviewed-on: https://go-review.googlesource.com/c/tools/+/232658
Run-TryBot: Ian Cottrell <iancottrell@google.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Robert Findley <rfindley@google.com>

internal/stack/gostacks/gostacks.go

@@ -0,0 +1,23 @@
+// Copyright 2020 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// The gostacks command processes stdin looking for things that look like
+// stack traces and simplifying them to make the log more readable.
+// It collates stack traces that have the same path as well as simplifying the
+// individual lines of the trace.
+// The processed log is printed to stdout.
+package main
+
+import (
+	"fmt"
+	"os"
+
+	"golang.org/x/tools/internal/stack"
+)
+
+func main() {
+	if err := stack.Process(os.Stdout, os.Stdin); err != nil {
+		fmt.Fprintln(os.Stderr, err)
+	}
+}

internal/stack/parse.go

@@ -0,0 +1,175 @@
+// Copyright 2020 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package stack
+
+import (
+	"bufio"
+	"errors"
+	"io"
+	"regexp"
+	"strconv"
+)
+
+var (
+	reBlank     = regexp.MustCompile(`^\s*$`)
+	reGoroutine = regexp.MustCompile(`^\s*goroutine (\d+) \[([^\]]*)\]:\s*$`)
+	reCall      = regexp.MustCompile(`^\s*` +
+		`(created by )?` + //marker
+		`(([\w/.]+/)?[\w]+)\.` + //package
+		`(\(([^:.)]*)\)\.)?` + //optional type
+		`([\w\.]+)` + //function
+		`(\(.*\))?` + // args
+		`\s*$`)
+	rePos = regexp.MustCompile(`^\s*(.*):(\d+)( .*)?$`)
+
+	errBreakParse = errors.New("break parse")
+)
+
+// Scanner splits an input stream into lines in a way that is consumable by
+// the parser.
+type Scanner struct {
+	lines *bufio.Scanner
+	done  bool
+}
+
+// NewScanner creates a scanner on top of a reader.
+func NewScanner(r io.Reader) *Scanner {
+	s := &Scanner{
+		lines: bufio.NewScanner(r),
+	}
+	s.Skip() // prefill
+	return s
+}
+
+// Peek returns the next line without consuming it.
+func (s *Scanner) Peek() string {
+	if s.done {
+		return ""
+	}
+	return s.lines.Text()
+}
+
+// Skip consumes the next line without looking at it.
+// Normally used after it has already been looked at using Peek.
+func (s *Scanner) Skip() {
+	if !s.lines.Scan() {
+		s.done = true
+	}
+}
+
+// Next consumes and returns the next line.
+func (s *Scanner) Next() string {
+	line := s.Peek()
+	s.Skip()
+	return line
+}
+
+// Done returns true if the scanner has reached the end of the underlying
+// stream.
+func (s *Scanner) Done() bool {
+	return s.done
+}
+
+// Err returns true if the scanner has reached the end of the underlying
+// stream.
+func (s *Scanner) Err() error {
+	return s.lines.Err()
+}
+
+// Match returns the submatchs of the regular expression against the next line.
+// If it matched the line is also consumed.
+func (s *Scanner) Match(re *regexp.Regexp) []string {
+	if s.done {
+		return nil
+	}
+	match := re.FindStringSubmatch(s.Peek())
+	if match != nil {
+		s.Skip()
+	}
+	return match
+}
+
+// SkipBlank skips any number of pure whitespace lines.
+func (s *Scanner) SkipBlank() {
+	for !s.done {
+		line := s.Peek()
+		if len(line) != 0 && !reBlank.MatchString(line) {
+			return
+		}
+		s.Skip()
+	}
+}
+
+// Parse the current contiguous block of goroutine stack traces until the
+// scanned content no longer matches.
+func Parse(scanner *Scanner) (Dump, error) {
+	dump := Dump{}
+	for {
+		gr, ok := parseGoroutine(scanner)
+		if !ok {
+			return dump, nil
+		}
+		dump = append(dump, gr)
+	}
+}
+
+func parseGoroutine(scanner *Scanner) (Goroutine, bool) {
+	match := scanner.Match(reGoroutine)
+	if match == nil {
+		return Goroutine{}, false
+	}
+	id, _ := strconv.ParseInt(match[1], 0, 32)
+	gr := Goroutine{
+		ID:    int(id),
+		State: match[2],
+	}
+	for {
+		frame, ok := parseFrame(scanner)
+		if !ok {
+			scanner.SkipBlank()
+			return gr, true
+		}
+		if frame.Position.Filename != "" {
+			gr.Stack = append(gr.Stack, frame)
+		}
+	}
+}
+
+func parseFrame(scanner *Scanner) (Frame, bool) {
+	fun, ok := parseFunction(scanner)
+	if !ok {
+		return Frame{}, false
+	}
+	frame := Frame{
+		Function: fun,
+	}
+	frame.Position, ok = parsePosition(scanner)
+	// if ok is false, then this is a broken state.
+	// we got the func but not the file that must follow
+	// the consumed line can be recovered from the frame
+	//TODO: push back the fun raw
+	return frame, ok
+}
+
+func parseFunction(scanner *Scanner) (Function, bool) {
+	match := scanner.Match(reCall)
+	if match == nil {
+		return Function{}, false
+	}
+	return Function{
+		Package: match[2],
+		Type:    match[5],
+		Name:    match[6],
+	}, true
+}
+
+func parsePosition(scanner *Scanner) (Position, bool) {
+	match := scanner.Match(rePos)
+	if match == nil {
+		return Position{}, false
+	}
+	line, _ := strconv.ParseInt(match[2], 0, 32)
+	return Position{Filename: match[1], Line: int(line)}, true
+}

internal/stack/process.go

@@ -0,0 +1,112 @@
+// Copyright 2020 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package stack
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"runtime"
+	"sort"
+)
+
+// Capture get the current stack traces from the runtime.
+func Capture() Dump {
+	buf := make([]byte, 2<<20)
+	buf = buf[:runtime.Stack(buf, true)]
+	scanner := NewScanner(bytes.NewReader(buf))
+	dump, _ := Parse(scanner)
+	return dump
+}
+
+// Summarize a dump for easier consumption.
+// This collates goroutines with equivalent stacks.
+func Summarize(dump Dump) Summary {
+	s := Summary{
+		Total: len(dump),
+	}
+	for _, gr := range dump {
+		s.addGoroutine(gr)
+	}
+	return s
+}
+
+// Process and input stream to an output stream, summarizing any stacks that
+// are detected in place.
+func Process(out io.Writer, in io.Reader) error {
+	scanner := NewScanner(in)
+	for {
+		dump, err := Parse(scanner)
+		summary := Summarize(dump)
+		switch {
+		case len(dump) > 0:
+			fmt.Fprintf(out, "%+v\n\n", summary)
+		case err != nil:
+			return err
+		case scanner.Done():
+			return scanner.Err()
+		default:
+			// must have been a line that is not part of a dump
+			fmt.Fprintln(out, scanner.Next())
+		}
+	}
+}
+
+// Diff calculates the delta between two dumps.
+func Diff(before, after Dump) Delta {
+	result := Delta{}
+	processed := make(map[int]bool)
+	for _, gr := range before {
+		processed[gr.ID] = false
+	}
+	for _, gr := range after {
+		if _, found := processed[gr.ID]; found {
+			result.Shared = append(result.Shared, gr)
+		} else {
+			result.After = append(result.After, gr)
+		}
+		processed[gr.ID] = true
+	}
+	for _, gr := range before {
+		if done := processed[gr.ID]; !done {
+			result.Before = append(result.Before, gr)
+		}
+	}
+	return result
+}
+
+// TODO: do we want to allow contraction of stacks before comparison?
+func (s *Summary) addGoroutine(gr Goroutine) {
+	index := sort.Search(len(s.Calls), func(i int) bool {
+		return !s.Calls[i].Stack.less(gr.Stack)
+	})
+	if index >= len(s.Calls) || !s.Calls[index].Stack.equal(gr.Stack) {
+		// insert new stack, first increase the length
+		s.Calls = append(s.Calls, Call{})
+		// move the top part upward to make space
+		copy(s.Calls[index+1:], s.Calls[index:])
+		// insert the new call
+		s.Calls[index] = Call{
+			Stack: gr.Stack,
+		}
+	}
+	// merge the goroutine into the matched call
+	s.Calls[index].merge(gr)
+}
+
+//TODO: do we want other grouping strategies?
+func (c *Call) merge(gr Goroutine) {
+	for i := range c.Groups {
+		canditate := &c.Groups[i]
+		if canditate.State == gr.State {
+			canditate.Goroutines = append(canditate.Goroutines, gr)
+			return
+		}
+	}
+	c.Groups = append(c.Groups, Group{
+		State:      gr.State,
+		Goroutines: []Goroutine{gr},
+	})
+}

internal/stack/stack.go

@@ -0,0 +1,170 @@
+// Copyright 2020 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package stack provides support for parsing standard goroutine stack traces.
+package stack
+
+import (
+	"fmt"
+	"text/tabwriter"
+)
+
+// Dump is a raw set of goroutines and their stacks.
+type Dump []Goroutine
+
+// Goroutine is a single parsed goroutine dump.
+type Goroutine struct {
+	State string // state that the goroutine is in.
+	ID    int    // id of the goroutine.
+	Stack Stack  // call frames that make up the stack
+}
+
+// Stack is a set of frames in a callstack.
+type Stack []Frame
+
+// Frame is a point in a call stack.
+type Frame struct {
+	Function Function
+	Position Position
+}
+
+// Function is the function called at a frame.
+type Function struct {
+	Package string // package name of function if known
+	Type    string // if set function is a method of this type
+	Name    string // function name of the frame
+}
+
+// Position is the file position for a frame.
+type Position struct {
+	Filename string // source filename
+	Line     int    // line number within file
+}
+
+// Summary is a set of stacks processed and collated into Calls.
+type Summary struct {
+	Total int    // the total count of goroutines in the summary
+	Calls []Call // the collated stack traces
+}
+
+// Call is set of goroutines that all share the same callstack.
+// They will be grouped by state.
+type Call struct {
+	Stack  Stack   // the shared callstack information
+	Groups []Group // the sets of goroutines with the same state
+}
+
+// Group is a set of goroutines with the same stack that are in the same state.
+type Group struct {
+	State      string      // the shared state of the goroutines
+	Goroutines []Goroutine // the set of goroutines in this group
+}
+
+// Delta represents the difference between two stack dumps.
+type Delta struct {
+	Before Dump // The goroutines that were only in the before set.
+	Shared Dump // The goroutines that were in both sets.
+	After  Dump // The goroutines that were only in the after set.
+}
+
+func (s Stack) equal(other Stack) bool {
+	if len(s) != len(other) {
+		return false
+	}
+	for i, frame := range s {
+		if !frame.equal(other[i]) {
+			return false
+		}
+	}
+	return true
+}
+
+func (s Stack) less(other Stack) bool {
+	for i, frame := range s {
+		if i >= len(other) {
+			return false
+		}
+		if frame.less(other[i]) {
+			return true
+		}
+		if !frame.equal(other[i]) {
+			return false
+		}
+	}
+	return len(s) < len(other)
+}
+
+func (f Frame) equal(other Frame) bool {
+	return f.Position.equal(other.Position)
+}
+
+func (f Frame) less(other Frame) bool {
+	return f.Position.less(other.Position)
+}
+
+func (p Position) equal(other Position) bool {
+	return p.Filename == other.Filename && p.Line == other.Line
+}
+
+func (p Position) less(other Position) bool {
+	if p.Filename < other.Filename {
+		return true
+	}
+	if p.Filename > other.Filename {
+		return false
+	}
+	return p.Line < other.Line
+}
+
+func (s Summary) Format(w fmt.State, r rune) {
+	tw := tabwriter.NewWriter(w, 0, 0, 1, ' ', 0)
+	for i, c := range s.Calls {
+		if i > 0 {
+			fmt.Fprintf(tw, "\n\n")
+			tw.Flush()
+		}
+		fmt.Fprint(tw, c)
+	}
+	tw.Flush()
+	if s.Total > 0 && w.Flag('+') {
+		fmt.Fprintf(w, "\n\n%d goroutines, %d unique", s.Total, len(s.Calls))
+	}
+}
+
+func (c Call) Format(w fmt.State, r rune) {
+	for i, g := range c.Groups {
+		if i > 0 {
+			fmt.Fprint(w, " ")
+		}
+		fmt.Fprint(w, g)
+	}
+	for _, f := range c.Stack {
+		fmt.Fprintf(w, "\n%v", f)
+	}
+}
+
+func (g Group) Format(w fmt.State, r rune) {
+	fmt.Fprintf(w, "[%v]: ", g.State)
+	for i, gr := range g.Goroutines {
+		if i > 0 {
+			fmt.Fprint(w, ", ")
+		}
+		fmt.Fprintf(w, "$%d", gr.ID)
+	}
+}
+
+func (f Frame) Format(w fmt.State, c rune) {
+	fmt.Fprintf(w, "%v:\t%v", f.Position, f.Function)
+}
+
+func (f Function) Format(w fmt.State, c rune) {
+	if f.Type != "" {
+		fmt.Fprintf(w, "(%v).", f.Type)
+	}
+	fmt.Fprintf(w, "%v", f.Name)
+}
+
+func (p Position) Format(w fmt.State, c rune) {
+	fmt.Fprintf(w, "%v:%v", p.Filename, p.Line)
+}

internal/stack/stack_test.go

@@ -0,0 +1,193 @@
+// Copyright 2020 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package stack_test
+
+import (
+	"bytes"
+	"strings"
+	"testing"
+
+	"golang.org/x/tools/internal/stack"
+)
+
+func TestProcess(t *testing.T) {
+	for _, test := range []struct{ name, input, expect string }{{
+		name:   `empty`,
+		input:  ``,
+		expect: ``,
+	}, {
+		name:  `no_frame`,
+		input: `goroutine 1 [running]:`,
+		expect: `
+[running]: $1
+
+1 goroutines, 1 unique
+`,
+	}, {
+		name: `one_frame`,
+		input: `
+goroutine 1 [running]:
+package.function(args)
+	file.go:10
+`,
+		expect: `
+[running]: $1
+file.go:10: function
+
+1 goroutines, 1 unique
+`,
+	}, {
+		name: `one_call`,
+		input: `
+goroutine 1 [running]:
+package1.functionA(args)
+	file1.go:10
+package2.functionB(args)
+	file2.go:20
+package3.functionC(args)
+	file3.go:30
+`,
+		expect: `
+[running]: $1
+file1.go:10: functionA
+file2.go:20: functionB
+file3.go:30: functionC
+
+1 goroutines, 1 unique
+`,
+	}, {
+		name: `two_call`,
+		input: `
+goroutine 1 [running]:
+package1.functionA(args)
+	file1.go:10
+goroutine 2 [running]:
+package2.functionB(args)
+	file2.go:20
+`,
+		expect: `
+[running]: $1
+file1.go:10: functionA
+
+[running]: $2
+file2.go:20: functionB
+
+2 goroutines, 2 unique
+`,
+	}, {
+		name: `merge_call`,
+		input: `
+goroutine 1 [running]:
+package1.functionA(args)
+	file1.go:10
+goroutine 2 [running]:
+package1.functionA(args)
+	file1.go:10
+`,
+		expect: `
+[running]: $1, $2
+file1.go:10: functionA
+
+2 goroutines, 1 unique
+`,
+	}, {
+		name: `alternating_call`,
+		input: `
+goroutine 1 [running]:
+package1.functionA(args)
+	file1.go:10
+goroutine 2 [running]:
+package2.functionB(args)
+	file2.go:20
+goroutine 3 [running]:
+package1.functionA(args)
+	file1.go:10
+goroutine 4 [running]:
+package2.functionB(args)
+	file2.go:20
+goroutine 5 [running]:
+package1.functionA(args)
+	file1.go:10
+goroutine 6 [running]:
+package2.functionB(args)
+	file2.go:20
+`,
+		expect: `
+[running]: $1, $3, $5
+file1.go:10: functionA
+
+[running]: $2, $4, $6
+file2.go:20: functionB
+
+6 goroutines, 2 unique
+`,
+	}, {
+		name: `sort_calls`,
+		input: `
+goroutine 1 [running]:
+package3.functionC(args)
+	file3.go:30
+goroutine 2 [running]:
+package2.functionB(args)
+	file2.go:20
+goroutine 3 [running]:
+package1.functionA(args)
+	file1.go:10
+`,
+		expect: `
+[running]: $3
+file1.go:10: functionA
+
+[running]: $2
+file2.go:20: functionB
+
+[running]: $1
+file3.go:30: functionC
+
+3 goroutines, 3 unique
+`,
+	}, {
+		name: `real_single`,
+		input: `
+panic: oops
+
+goroutine 53 [running]:
+golang.org/x/tools/internal/jsonrpc2_test.testHandler.func1(0x1240c20, 0xc000013350, 0xc0000133b0, 0x1240ca0, 0xc00002ab00, 0x3, 0x3)
+	/work/tools/internal/jsonrpc2/jsonrpc2_test.go:160 +0x74c
+golang.org/x/tools/internal/jsonrpc2.(*Conn).Run(0xc000204330, 0x1240c20, 0xc000204270, 0x1209570, 0xc000212120, 0x1242700)
+	/work/tools/internal/jsonrpc2/jsonrpc2.go:187 +0x777
+golang.org/x/tools/internal/jsonrpc2_test.run.func1(0x123ebe0, 0xc000206018, 0x123ec20, 0xc000206010, 0xc0002080a0, 0xc000204330, 0x1240c20, 0xc000204270, 0xc000212120)
+	/work/tools/internal/jsonrpc2/jsonrpc2_test.go:131 +0xe2
+created by golang.org/x/tools/internal/jsonrpc2_test.run
+	/work/tools/internal/jsonrpc2/jsonrpc2_test.go:121 +0x263
+FAIL    golang.org/x/tools/internal/jsonrpc2    0.252s
+FAIL
+`,
+		expect: `
+panic: oops
+
+[running]: $53
+/work/tools/internal/jsonrpc2/jsonrpc2_test.go:160: testHandler.func1
+/work/tools/internal/jsonrpc2/jsonrpc2.go:187:      (*Conn).Run
+/work/tools/internal/jsonrpc2/jsonrpc2_test.go:131: run.func1
+/work/tools/internal/jsonrpc2/jsonrpc2_test.go:121: run
+
+1 goroutines, 1 unique
+
+FAIL    golang.org/x/tools/internal/jsonrpc2    0.252s
+FAIL
+`,
+	}} {
+		t.Run(test.name, func(t *testing.T) {
+			buf := &bytes.Buffer{}
+			stack.Process(buf, strings.NewReader(test.input))
+			expect := strings.TrimSpace(test.expect)
+			got := strings.TrimSpace(buf.String())
+			if got != expect {
+				t.Errorf("got:\n%s\nexpect:\n%s", got, expect)
+			}
+		})
+	}
+}