// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package debug
import (
"bytes"
"context"
"fmt"
"go/token"
"html/template"
"io"
stdlog "log"
"net"
"net/http"
"net/http/pprof"
_ "net/http/pprof" // pull in the standard pprof handlers
"os"
"path"
"path/filepath"
"reflect"
"runtime"
rpprof "runtime/pprof"
"strconv"
"strings"
"sync"
"time"
"golang.org/x/tools/internal/span"
"golang.org/x/tools/internal/telemetry/export"
"golang.org/x/tools/internal/telemetry/export/ocagent"
"golang.org/x/tools/internal/telemetry/export/prometheus"
"golang.org/x/tools/internal/telemetry/log"
"golang.org/x/tools/internal/telemetry/tag"
)
type Instance struct {
Logfile string
StartTime time.Time
ServerAddress string
DebugAddress string
Workdir string
OCAgentConfig string
LogWriter io.Writer
ocagent export.Exporter
prometheus *prometheus.Exporter
rpcs *rpcs
traces *traces
}
type Cache interface {
ID() string
FileSet() *token.FileSet
MemStats() map[reflect.Type]int
}
type Session interface {
ID() string
Cache() Cache
Files() []*File
File(hash string) *File
}
type View interface {
ID() string
Name() string
Folder() span.URI
Session() Session
}
type File struct {
Session Session
URI span.URI
Data string
Error error
Hash string
}
var (
mu sync.Mutex
data = struct {
Caches []Cache
Sessions []Session
Views []View
}{}
)
// AddCache adds a cache to the set being served
func AddCache(cache Cache) {
mu.Lock()
defer mu.Unlock()
data.Caches = append(data.Caches, cache)
}
// DropCache drops a cache from the set being served
func DropCache(cache Cache) {
mu.Lock()
defer mu.Unlock()
//find and remove the cache
if i, _ := findCache(cache.ID()); i >= 0 {
copy(data.Caches[i:], data.Caches[i+1:])
data.Caches[len(data.Caches)-1] = nil
data.Caches = data.Caches[:len(data.Caches)-1]
}
}
func findCache(id string) (int, Cache) {
for i, c := range data.Caches {
if c.ID() == id {
return i, c
}
}
return -1, nil
}
func getCache(r *http.Request) interface{} {
mu.Lock()
defer mu.Unlock()
id := path.Base(r.URL.Path)
result := struct {
Cache
Sessions []Session
}{}
_, result.Cache = findCache(id)
// now find all the views that belong to this session
for _, v := range data.Sessions {
if v.Cache().ID() == id {
result.Sessions = append(result.Sessions, v)
}
}
return result
}
func findSession(id string) (int, Session) {
for i, c := range data.Sessions {
if c.ID() == id {
return i, c
}
}
return -1, nil
}
func getSession(r *http.Request) interface{} {
mu.Lock()
defer mu.Unlock()
id := path.Base(r.URL.Path)
_, session := findSession(id)
result := struct {
Session
Views []View
}{
Session: session,
}
// now find all the views that belong to this session
for _, v := range data.Views {
if v.Session().ID() == id {
result.Views = append(result.Views, v)
}
}
return result
}
func findView(id string) (int, View) {
for i, c := range data.Views {
if c.ID() == id {
return i, c
}
}
return -1, nil
}
func getView(r *http.Request) interface{} {
mu.Lock()
defer mu.Unlock()
id := path.Base(r.URL.Path)
_, v := findView(id)
return v
}
func getFile(r *http.Request) interface{} {
mu.Lock()
defer mu.Unlock()
hash := path.Base(r.URL.Path)
sid := path.Base(path.Dir(r.URL.Path))
_, session := findSession(sid)
return session.File(hash)
}
func (i *Instance) getInfo() dataFunc {
return func(r *http.Request) interface{} {
buf := &bytes.Buffer{}
i.PrintServerInfo(buf)
return template.HTML(buf.String())
}
}
func getMemory(r *http.Request) interface{} {
var m runtime.MemStats
runtime.ReadMemStats(&m)
return m
}
// AddSession adds a session to the set being served
func AddSession(session Session) {
mu.Lock()
defer mu.Unlock()
data.Sessions = append(data.Sessions, session)
}
// DropSession drops a session from the set being served
func DropSession(session Session) {
mu.Lock()
defer mu.Unlock()
if i, _ := findSession(session.ID()); i >= 0 {
copy(data.Sessions[i:], data.Sessions[i+1:])
data.Sessions[len(data.Sessions)-1] = nil
data.Sessions = data.Sessions[:len(data.Sessions)-1]
}
}
// AddView adds a view to the set being served
func AddView(view View) {
mu.Lock()
defer mu.Unlock()
data.Views = append(data.Views, view)
}
// DropView drops a view from the set being served
func DropView(view View) {
mu.Lock()
defer mu.Unlock()
//find and remove the view
if i, _ := findView(view.ID()); i >= 0 {
copy(data.Views[i:], data.Views[i+1:])
data.Views[len(data.Views)-1] = nil
data.Views = data.Views[:len(data.Views)-1]
}
}
// Prepare gets a debug instance ready for use using the supplied configuration.
func (i *Instance) Prepare(ctx context.Context) {
mu.Lock()
defer mu.Unlock()
i.LogWriter = os.Stderr
ocConfig := ocagent.Discover()
//TODO: we should not need to adjust the discovered configuration
ocConfig.Address = i.OCAgentConfig
i.ocagent = ocagent.Connect(ocConfig)
i.prometheus = prometheus.New()
i.rpcs = &rpcs{}
i.traces = &traces{}
export.AddExporters(i.ocagent, i.prometheus, i.rpcs, i.traces)
}
func (i *Instance) SetLogFile(logfile string) error {
if logfile != "" {
if logfile == "auto" {
logfile = filepath.Join(os.TempDir(), fmt.Sprintf("gopls-%d.log", os.Getpid()))
}
f, err := os.Create(logfile)
if err != nil {
return fmt.Errorf("Unable to create log file: %v", err)
}
defer f.Close()
stdlog.SetOutput(io.MultiWriter(os.Stderr, f))
i.LogWriter = f
}
i.Logfile = logfile
return nil
}
// Serve starts and runs a debug server in the background.
// It also logs the port the server starts on, to allow for :0 auto assigned
// ports.
func (i *Instance) Serve(ctx context.Context) error {
mu.Lock()
defer mu.Unlock()
if i.DebugAddress == "" {
return nil
}
listener, err := net.Listen("tcp", i.DebugAddress)
if err != nil {
return err
}
port := listener.Addr().(*net.TCPAddr).Port
if strings.HasSuffix(i.DebugAddress, ":0") {
stdlog.Printf("debug server listening on port %d", port)
}
log.Print(ctx, "Debug serving", tag.Of("Port", port))
go func() {
mux := http.NewServeMux()
mux.HandleFunc("/", render(mainTmpl, func(*http.Request) interface{} { return data }))
mux.HandleFunc("/debug/", render(debugTmpl, nil))
mux.HandleFunc("/debug/pprof/", pprof.Index)
mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline)
mux.HandleFunc("/debug/pprof/profile", pprof.Profile)
mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
mux.HandleFunc("/debug/pprof/trace", pprof.Trace)
if i.prometheus != nil {
mux.HandleFunc("/metrics/", i.prometheus.Serve)
}
if i.rpcs != nil {
mux.HandleFunc("/rpc/", render(rpcTmpl, i.rpcs.getData))
}
if i.traces != nil {
mux.HandleFunc("/trace/", render(traceTmpl, i.traces.getData))
}
mux.HandleFunc("/cache/", render(cacheTmpl, getCache))
mux.HandleFunc("/session/", render(sessionTmpl, getSession))
mux.HandleFunc("/view/", render(viewTmpl, getView))
mux.HandleFunc("/file/", render(fileTmpl, getFile))
mux.HandleFunc("/info", render(infoTmpl, i.getInfo()))
mux.HandleFunc("/memory", render(memoryTmpl, getMemory))
if err := http.Serve(listener, mux); err != nil {
log.Error(ctx, "Debug server failed", err)
return
}
log.Print(ctx, "Debug server finished")
}()
return nil
}
func (i *Instance) MonitorMemory(ctx context.Context) {
tick := time.NewTicker(time.Second)
nextThresholdGiB := uint64(5)
go func() {
for {
<-tick.C
var mem runtime.MemStats
runtime.ReadMemStats(&mem)
if mem.HeapAlloc < nextThresholdGiB*1<<30 {
continue
}
i.writeMemoryDebug(nextThresholdGiB)
log.Print(ctx, fmt.Sprintf("Wrote memory usage debug info to %v", os.TempDir()))
nextThresholdGiB++
}
}()
}
func (i *Instance) writeMemoryDebug(threshold uint64) error {
fname := func(t string) string {
return fmt.Sprintf("gopls.%d-%dGiB-%s", os.Getpid(), threshold, t)
}
f, err := os.Create(filepath.Join(os.TempDir(), fname("heap.pb.gz")))
if err != nil {
return err
}
defer f.Close()
if err := rpprof.Lookup("heap").WriteTo(f, 0); err != nil {
return err
}
f, err = os.Create(filepath.Join(os.TempDir(), fname("goroutines.txt")))
if err != nil {
return err
}
defer f.Close()
if err := rpprof.Lookup("goroutine").WriteTo(f, 1); err != nil {
return err
}
return nil
}
type dataFunc func(*http.Request) interface{}
func render(tmpl *template.Template, fun dataFunc) func(http.ResponseWriter, *http.Request) {
return func(w http.ResponseWriter, r *http.Request) {
var data interface{}
if fun != nil {
data = fun(r)
}
if err := tmpl.Execute(w, data); err != nil {
log.Error(context.Background(), "", err)
}
}
}
func commas(s string) string {
for i := len(s); i > 3; {
i -= 3
s = s[:i] + "," + s[i:]
}
return s
}
func fuint64(v uint64) string {
return commas(strconv.FormatUint(v, 10))
}
func fuint32(v uint32) string {
return commas(strconv.FormatUint(uint64(v), 10))
}
var baseTemplate = template.Must(template.New("").Parse(`
{{template "title" .}}
{{block "head" .}}{{end}}
Main
Info
Memory
Metrics
RPC
Trace
{{template "title" .}}
{{block "body" .}}
Unknown page
{{end}}
{{define "cachelink"}}Cache {{.}}{{end}}
{{define "sessionlink"}}Session {{.}}{{end}}
{{define "viewlink"}}View {{.}}{{end}}
{{define "filelink"}}{{.URI}}{{end}}
`)).Funcs(template.FuncMap{
"fuint64": fuint64,
"fuint32": fuint32,
})
var mainTmpl = template.Must(template.Must(baseTemplate.Clone()).Parse(`
{{define "title"}}GoPls server information{{end}}
{{define "body"}}
Caches
{{range .Caches}}- {{template "cachelink" .ID}}
{{end}}
Sessions
{{range .Sessions}}- {{template "sessionlink" .ID}} from {{template "cachelink" .Cache.ID}}
{{end}}
Views
{{range .Views}}- {{.Name}} is {{template "viewlink" .ID}} from {{template "sessionlink" .Session.ID}} in {{.Folder}}
{{end}}
{{end}}
`))
var infoTmpl = template.Must(template.Must(baseTemplate.Clone()).Parse(`
{{define "title"}}GoPls version information{{end}}
{{define "body"}}
{{.}}
{{end}}
`))
var memoryTmpl = template.Must(template.Must(baseTemplate.Clone()).Parse(`
{{define "title"}}GoPls memory usage{{end}}
{{define "head"}}{{end}}
{{define "body"}}
Stats
Allocated bytes | {{fuint64 .HeapAlloc}} |
Total allocated bytes | {{fuint64 .TotalAlloc}} |
System bytes | {{fuint64 .Sys}} |
Heap system bytes | {{fuint64 .HeapSys}} |
Malloc calls | {{fuint64 .Mallocs}} |
Frees | {{fuint64 .Frees}} |
Idle heap bytes | {{fuint64 .HeapIdle}} |
In use bytes | {{fuint64 .HeapInuse}} |
Released to system bytes | {{fuint64 .HeapReleased}} |
Heap object count | {{fuint64 .HeapObjects}} |
Stack in use bytes | {{fuint64 .StackInuse}} |
Stack from system bytes | {{fuint64 .StackSys}} |
Bucket hash bytes | {{fuint64 .BuckHashSys}} |
GC metadata bytes | {{fuint64 .GCSys}} |
Off heap bytes | {{fuint64 .OtherSys}} |
By size
Size | Mallocs | Frees |
{{range .BySize}}{{fuint32 .Size}} | {{fuint64 .Mallocs}} | {{fuint64 .Frees}} |
{{end}}
{{end}}
`))
var debugTmpl = template.Must(template.Must(baseTemplate.Clone()).Parse(`
{{define "title"}}GoPls Debug pages{{end}}
{{define "body"}}
Profiling
{{end}}
`))
var cacheTmpl = template.Must(template.Must(baseTemplate.Clone()).Parse(`
{{define "title"}}Cache {{.ID}}{{end}}
{{define "body"}}
Sessions
{{range .Sessions}}- {{template "sessionlink" .ID}}
{{end}}
memoize.Store entries
{{range $k,$v := .MemStats}}- {{$k}} - {{$v}}
{{end}}
{{end}}
`))
var sessionTmpl = template.Must(template.Must(baseTemplate.Clone()).Parse(`
{{define "title"}}Session {{.ID}}{{end}}
{{define "body"}}
From: {{template "cachelink" .Cache.ID}}
Views
{{range .Views}}- {{.Name}} is {{template "viewlink" .ID}} in {{.Folder}}
{{end}}
Files
{{range .Files}}- {{template "filelink" .}}
{{end}}
{{end}}
`))
var viewTmpl = template.Must(template.Must(baseTemplate.Clone()).Parse(`
{{define "title"}}View {{.ID}}{{end}}
{{define "body"}}
Name: {{.Name}}
Folder: {{.Folder}}
From: {{template "sessionlink" .Session.ID}}
Environment
{{range .Env}}- {{.}}
{{end}}
{{end}}
`))
var fileTmpl = template.Must(template.Must(baseTemplate.Clone()).Parse(`
{{define "title"}}File {{.Hash}}{{end}}
{{define "body"}}
From: {{template "sessionlink" .Session.ID}}
URI: {{.URI}}
Hash: {{.Hash}}
Error: {{.Error}}
Contents
{{.Data}}
{{end}}
`))