diff --git a/internal/stack/gostacks/gostacks.go b/internal/stack/gostacks/gostacks.go new file mode 100644 index 0000000000..699471f202 --- /dev/null +++ b/internal/stack/gostacks/gostacks.go @@ -0,0 +1,23 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// The gostacks command processes stdin looking for things that look like +// stack traces and simplifying them to make the log more readable. +// It collates stack traces that have the same path as well as simplifying the +// individual lines of the trace. +// The processed log is printed to stdout. +package main + +import ( + "fmt" + "os" + + "golang.org/x/tools/internal/stack" +) + +func main() { + if err := stack.Process(os.Stdout, os.Stdin); err != nil { + fmt.Fprintln(os.Stderr, err) + } +} diff --git a/internal/stack/parse.go b/internal/stack/parse.go new file mode 100644 index 0000000000..e01da8f0ee --- /dev/null +++ b/internal/stack/parse.go @@ -0,0 +1,175 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package stack + +import ( + "bufio" + "errors" + "io" + "regexp" + "strconv" +) + +var ( + reBlank = regexp.MustCompile(`^\s*$`) + reGoroutine = regexp.MustCompile(`^\s*goroutine (\d+) \[([^\]]*)\]:\s*$`) + reCall = regexp.MustCompile(`^\s*` + + `(created by )?` + //marker + `(([\w/.]+/)?[\w]+)\.` + //package + `(\(([^:.)]*)\)\.)?` + //optional type + `([\w\.]+)` + //function + `(\(.*\))?` + // args + `\s*$`) + rePos = regexp.MustCompile(`^\s*(.*):(\d+)( .*)?$`) + + errBreakParse = errors.New("break parse") +) + +// Scanner splits an input stream into lines in a way that is consumable by +// the parser. +type Scanner struct { + lines *bufio.Scanner + done bool +} + +// NewScanner creates a scanner on top of a reader. +func NewScanner(r io.Reader) *Scanner { + s := &Scanner{ + lines: bufio.NewScanner(r), + } + s.Skip() // prefill + return s +} + +// Peek returns the next line without consuming it. +func (s *Scanner) Peek() string { + if s.done { + return "" + } + return s.lines.Text() +} + +// Skip consumes the next line without looking at it. +// Normally used after it has already been looked at using Peek. +func (s *Scanner) Skip() { + if !s.lines.Scan() { + s.done = true + } +} + +// Next consumes and returns the next line. +func (s *Scanner) Next() string { + line := s.Peek() + s.Skip() + return line +} + +// Done returns true if the scanner has reached the end of the underlying +// stream. +func (s *Scanner) Done() bool { + return s.done +} + +// Err returns true if the scanner has reached the end of the underlying +// stream. +func (s *Scanner) Err() error { + return s.lines.Err() +} + +// Match returns the submatchs of the regular expression against the next line. +// If it matched the line is also consumed. +func (s *Scanner) Match(re *regexp.Regexp) []string { + if s.done { + return nil + } + match := re.FindStringSubmatch(s.Peek()) + if match != nil { + s.Skip() + } + return match +} + +// SkipBlank skips any number of pure whitespace lines. +func (s *Scanner) SkipBlank() { + for !s.done { + line := s.Peek() + if len(line) != 0 && !reBlank.MatchString(line) { + return + } + s.Skip() + } +} + +// Parse the current contiguous block of goroutine stack traces until the +// scanned content no longer matches. +func Parse(scanner *Scanner) (Dump, error) { + dump := Dump{} + for { + gr, ok := parseGoroutine(scanner) + if !ok { + return dump, nil + } + dump = append(dump, gr) + } +} + +func parseGoroutine(scanner *Scanner) (Goroutine, bool) { + match := scanner.Match(reGoroutine) + if match == nil { + return Goroutine{}, false + } + id, _ := strconv.ParseInt(match[1], 0, 32) + gr := Goroutine{ + ID: int(id), + State: match[2], + } + for { + frame, ok := parseFrame(scanner) + if !ok { + scanner.SkipBlank() + return gr, true + } + if frame.Position.Filename != "" { + gr.Stack = append(gr.Stack, frame) + } + } +} + +func parseFrame(scanner *Scanner) (Frame, bool) { + fun, ok := parseFunction(scanner) + if !ok { + return Frame{}, false + } + frame := Frame{ + Function: fun, + } + frame.Position, ok = parsePosition(scanner) + // if ok is false, then this is a broken state. + // we got the func but not the file that must follow + // the consumed line can be recovered from the frame + //TODO: push back the fun raw + return frame, ok +} + +func parseFunction(scanner *Scanner) (Function, bool) { + match := scanner.Match(reCall) + if match == nil { + return Function{}, false + } + return Function{ + Package: match[2], + Type: match[5], + Name: match[6], + }, true +} + +func parsePosition(scanner *Scanner) (Position, bool) { + match := scanner.Match(rePos) + if match == nil { + return Position{}, false + } + line, _ := strconv.ParseInt(match[2], 0, 32) + return Position{Filename: match[1], Line: int(line)}, true +} diff --git a/internal/stack/process.go b/internal/stack/process.go new file mode 100644 index 0000000000..ac19366644 --- /dev/null +++ b/internal/stack/process.go @@ -0,0 +1,112 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package stack + +import ( + "bytes" + "fmt" + "io" + "runtime" + "sort" +) + +// Capture get the current stack traces from the runtime. +func Capture() Dump { + buf := make([]byte, 2<<20) + buf = buf[:runtime.Stack(buf, true)] + scanner := NewScanner(bytes.NewReader(buf)) + dump, _ := Parse(scanner) + return dump +} + +// Summarize a dump for easier consumption. +// This collates goroutines with equivalent stacks. +func Summarize(dump Dump) Summary { + s := Summary{ + Total: len(dump), + } + for _, gr := range dump { + s.addGoroutine(gr) + } + return s +} + +// Process and input stream to an output stream, summarizing any stacks that +// are detected in place. +func Process(out io.Writer, in io.Reader) error { + scanner := NewScanner(in) + for { + dump, err := Parse(scanner) + summary := Summarize(dump) + switch { + case len(dump) > 0: + fmt.Fprintf(out, "%+v\n\n", summary) + case err != nil: + return err + case scanner.Done(): + return scanner.Err() + default: + // must have been a line that is not part of a dump + fmt.Fprintln(out, scanner.Next()) + } + } +} + +// Diff calculates the delta between two dumps. +func Diff(before, after Dump) Delta { + result := Delta{} + processed := make(map[int]bool) + for _, gr := range before { + processed[gr.ID] = false + } + for _, gr := range after { + if _, found := processed[gr.ID]; found { + result.Shared = append(result.Shared, gr) + } else { + result.After = append(result.After, gr) + } + processed[gr.ID] = true + } + for _, gr := range before { + if done := processed[gr.ID]; !done { + result.Before = append(result.Before, gr) + } + } + return result +} + +// TODO: do we want to allow contraction of stacks before comparison? +func (s *Summary) addGoroutine(gr Goroutine) { + index := sort.Search(len(s.Calls), func(i int) bool { + return !s.Calls[i].Stack.less(gr.Stack) + }) + if index >= len(s.Calls) || !s.Calls[index].Stack.equal(gr.Stack) { + // insert new stack, first increase the length + s.Calls = append(s.Calls, Call{}) + // move the top part upward to make space + copy(s.Calls[index+1:], s.Calls[index:]) + // insert the new call + s.Calls[index] = Call{ + Stack: gr.Stack, + } + } + // merge the goroutine into the matched call + s.Calls[index].merge(gr) +} + +//TODO: do we want other grouping strategies? +func (c *Call) merge(gr Goroutine) { + for i := range c.Groups { + canditate := &c.Groups[i] + if canditate.State == gr.State { + canditate.Goroutines = append(canditate.Goroutines, gr) + return + } + } + c.Groups = append(c.Groups, Group{ + State: gr.State, + Goroutines: []Goroutine{gr}, + }) +} diff --git a/internal/stack/stack.go b/internal/stack/stack.go new file mode 100644 index 0000000000..479301a78d --- /dev/null +++ b/internal/stack/stack.go @@ -0,0 +1,170 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package stack provides support for parsing standard goroutine stack traces. +package stack + +import ( + "fmt" + "text/tabwriter" +) + +// Dump is a raw set of goroutines and their stacks. +type Dump []Goroutine + +// Goroutine is a single parsed goroutine dump. +type Goroutine struct { + State string // state that the goroutine is in. + ID int // id of the goroutine. + Stack Stack // call frames that make up the stack +} + +// Stack is a set of frames in a callstack. +type Stack []Frame + +// Frame is a point in a call stack. +type Frame struct { + Function Function + Position Position +} + +// Function is the function called at a frame. +type Function struct { + Package string // package name of function if known + Type string // if set function is a method of this type + Name string // function name of the frame +} + +// Position is the file position for a frame. +type Position struct { + Filename string // source filename + Line int // line number within file +} + +// Summary is a set of stacks processed and collated into Calls. +type Summary struct { + Total int // the total count of goroutines in the summary + Calls []Call // the collated stack traces +} + +// Call is set of goroutines that all share the same callstack. +// They will be grouped by state. +type Call struct { + Stack Stack // the shared callstack information + Groups []Group // the sets of goroutines with the same state +} + +// Group is a set of goroutines with the same stack that are in the same state. +type Group struct { + State string // the shared state of the goroutines + Goroutines []Goroutine // the set of goroutines in this group +} + +// Delta represents the difference between two stack dumps. +type Delta struct { + Before Dump // The goroutines that were only in the before set. + Shared Dump // The goroutines that were in both sets. + After Dump // The goroutines that were only in the after set. +} + +func (s Stack) equal(other Stack) bool { + if len(s) != len(other) { + return false + } + for i, frame := range s { + if !frame.equal(other[i]) { + return false + } + } + return true +} + +func (s Stack) less(other Stack) bool { + for i, frame := range s { + if i >= len(other) { + return false + } + if frame.less(other[i]) { + return true + } + if !frame.equal(other[i]) { + return false + } + } + return len(s) < len(other) +} + +func (f Frame) equal(other Frame) bool { + return f.Position.equal(other.Position) +} + +func (f Frame) less(other Frame) bool { + return f.Position.less(other.Position) +} + +func (p Position) equal(other Position) bool { + return p.Filename == other.Filename && p.Line == other.Line +} + +func (p Position) less(other Position) bool { + if p.Filename < other.Filename { + return true + } + if p.Filename > other.Filename { + return false + } + return p.Line < other.Line +} + +func (s Summary) Format(w fmt.State, r rune) { + tw := tabwriter.NewWriter(w, 0, 0, 1, ' ', 0) + for i, c := range s.Calls { + if i > 0 { + fmt.Fprintf(tw, "\n\n") + tw.Flush() + } + fmt.Fprint(tw, c) + } + tw.Flush() + if s.Total > 0 && w.Flag('+') { + fmt.Fprintf(w, "\n\n%d goroutines, %d unique", s.Total, len(s.Calls)) + } +} + +func (c Call) Format(w fmt.State, r rune) { + for i, g := range c.Groups { + if i > 0 { + fmt.Fprint(w, " ") + } + fmt.Fprint(w, g) + } + for _, f := range c.Stack { + fmt.Fprintf(w, "\n%v", f) + } +} + +func (g Group) Format(w fmt.State, r rune) { + fmt.Fprintf(w, "[%v]: ", g.State) + for i, gr := range g.Goroutines { + if i > 0 { + fmt.Fprint(w, ", ") + } + fmt.Fprintf(w, "$%d", gr.ID) + } +} + +func (f Frame) Format(w fmt.State, c rune) { + fmt.Fprintf(w, "%v:\t%v", f.Position, f.Function) +} + +func (f Function) Format(w fmt.State, c rune) { + if f.Type != "" { + fmt.Fprintf(w, "(%v).", f.Type) + } + fmt.Fprintf(w, "%v", f.Name) +} + +func (p Position) Format(w fmt.State, c rune) { + fmt.Fprintf(w, "%v:%v", p.Filename, p.Line) +} diff --git a/internal/stack/stack_test.go b/internal/stack/stack_test.go new file mode 100644 index 0000000000..371492aebe --- /dev/null +++ b/internal/stack/stack_test.go @@ -0,0 +1,193 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package stack_test + +import ( + "bytes" + "strings" + "testing" + + "golang.org/x/tools/internal/stack" +) + +func TestProcess(t *testing.T) { + for _, test := range []struct{ name, input, expect string }{{ + name: `empty`, + input: ``, + expect: ``, + }, { + name: `no_frame`, + input: `goroutine 1 [running]:`, + expect: ` +[running]: $1 + +1 goroutines, 1 unique +`, + }, { + name: `one_frame`, + input: ` +goroutine 1 [running]: +package.function(args) + file.go:10 +`, + expect: ` +[running]: $1 +file.go:10: function + +1 goroutines, 1 unique +`, + }, { + name: `one_call`, + input: ` +goroutine 1 [running]: +package1.functionA(args) + file1.go:10 +package2.functionB(args) + file2.go:20 +package3.functionC(args) + file3.go:30 +`, + expect: ` +[running]: $1 +file1.go:10: functionA +file2.go:20: functionB +file3.go:30: functionC + +1 goroutines, 1 unique +`, + }, { + name: `two_call`, + input: ` +goroutine 1 [running]: +package1.functionA(args) + file1.go:10 +goroutine 2 [running]: +package2.functionB(args) + file2.go:20 +`, + expect: ` +[running]: $1 +file1.go:10: functionA + +[running]: $2 +file2.go:20: functionB + +2 goroutines, 2 unique +`, + }, { + name: `merge_call`, + input: ` +goroutine 1 [running]: +package1.functionA(args) + file1.go:10 +goroutine 2 [running]: +package1.functionA(args) + file1.go:10 +`, + expect: ` +[running]: $1, $2 +file1.go:10: functionA + +2 goroutines, 1 unique +`, + }, { + name: `alternating_call`, + input: ` +goroutine 1 [running]: +package1.functionA(args) + file1.go:10 +goroutine 2 [running]: +package2.functionB(args) + file2.go:20 +goroutine 3 [running]: +package1.functionA(args) + file1.go:10 +goroutine 4 [running]: +package2.functionB(args) + file2.go:20 +goroutine 5 [running]: +package1.functionA(args) + file1.go:10 +goroutine 6 [running]: +package2.functionB(args) + file2.go:20 +`, + expect: ` +[running]: $1, $3, $5 +file1.go:10: functionA + +[running]: $2, $4, $6 +file2.go:20: functionB + +6 goroutines, 2 unique +`, + }, { + name: `sort_calls`, + input: ` +goroutine 1 [running]: +package3.functionC(args) + file3.go:30 +goroutine 2 [running]: +package2.functionB(args) + file2.go:20 +goroutine 3 [running]: +package1.functionA(args) + file1.go:10 +`, + expect: ` +[running]: $3 +file1.go:10: functionA + +[running]: $2 +file2.go:20: functionB + +[running]: $1 +file3.go:30: functionC + +3 goroutines, 3 unique +`, + }, { + name: `real_single`, + input: ` +panic: oops + +goroutine 53 [running]: +golang.org/x/tools/internal/jsonrpc2_test.testHandler.func1(0x1240c20, 0xc000013350, 0xc0000133b0, 0x1240ca0, 0xc00002ab00, 0x3, 0x3) + /work/tools/internal/jsonrpc2/jsonrpc2_test.go:160 +0x74c +golang.org/x/tools/internal/jsonrpc2.(*Conn).Run(0xc000204330, 0x1240c20, 0xc000204270, 0x1209570, 0xc000212120, 0x1242700) + /work/tools/internal/jsonrpc2/jsonrpc2.go:187 +0x777 +golang.org/x/tools/internal/jsonrpc2_test.run.func1(0x123ebe0, 0xc000206018, 0x123ec20, 0xc000206010, 0xc0002080a0, 0xc000204330, 0x1240c20, 0xc000204270, 0xc000212120) + /work/tools/internal/jsonrpc2/jsonrpc2_test.go:131 +0xe2 +created by golang.org/x/tools/internal/jsonrpc2_test.run + /work/tools/internal/jsonrpc2/jsonrpc2_test.go:121 +0x263 +FAIL golang.org/x/tools/internal/jsonrpc2 0.252s +FAIL +`, + expect: ` +panic: oops + +[running]: $53 +/work/tools/internal/jsonrpc2/jsonrpc2_test.go:160: testHandler.func1 +/work/tools/internal/jsonrpc2/jsonrpc2.go:187: (*Conn).Run +/work/tools/internal/jsonrpc2/jsonrpc2_test.go:131: run.func1 +/work/tools/internal/jsonrpc2/jsonrpc2_test.go:121: run + +1 goroutines, 1 unique + +FAIL golang.org/x/tools/internal/jsonrpc2 0.252s +FAIL +`, + }} { + t.Run(test.name, func(t *testing.T) { + buf := &bytes.Buffer{} + stack.Process(buf, strings.NewReader(test.input)) + expect := strings.TrimSpace(test.expect) + got := strings.TrimSpace(buf.String()) + if got != expect { + t.Errorf("got:\n%s\nexpect:\n%s", got, expect) + } + }) + } +}