From 855bbc50ad86de9aa46f829ff040ed8edc5b4c37 Mon Sep 17 00:00:00 2001 From: Michael Matloob Date: Mon, 8 Aug 2016 14:19:54 -0400 Subject: [PATCH] cmd/heapview: add an internal core package for reading cores This is based on github.com/tombergan/goheapdump/heapdump. This CL mostly just copies over the 'raw' data structures based on the profiler records' data structures. Many of them may need to be changed, but I think it will be good to have these definitions here to provide a base. Change-Id: I609202b6b87d980b0835c8087b3d78e11bd6dfe3 Reviewed-on: https://go-review.googlesource.com/25584 Reviewed-by: Hyang-Ah Hana Kim --- cmd/heapview/internal/core/mmapfile.go | 143 ++++++++++++ cmd/heapview/internal/core/raw.go | 308 +++++++++++++++++++++++++ 2 files changed, 451 insertions(+) create mode 100644 cmd/heapview/internal/core/mmapfile.go create mode 100644 cmd/heapview/internal/core/raw.go diff --git a/cmd/heapview/internal/core/mmapfile.go b/cmd/heapview/internal/core/mmapfile.go new file mode 100644 index 0000000000..9886d357fe --- /dev/null +++ b/cmd/heapview/internal/core/mmapfile.go @@ -0,0 +1,143 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package core + +import ( + "errors" + "fmt" + "io" + "os" + "syscall" +) + +var errMmapClosed = errors.New("mmap: closed") + +// mmapFile wraps a memory-mapped file. +type mmapFile struct { + data []byte + pos uint64 + writable bool +} + +// mmapOpen opens the named file for reading. +// If writable is true, the file is also open for writing. +func mmapOpen(filename string, writable bool) (*mmapFile, error) { + f, err := os.Open(filename) + if err != nil { + return nil, err + } + defer f.Close() + st, err := f.Stat() + if err != nil { + return nil, err + } + + size := st.Size() + if size == 0 { + return &mmapFile{data: []byte{}}, nil + } + if size < 0 { + return nil, fmt.Errorf("mmap: file %q has negative size: %d", filename, size) + } + if size != int64(int(size)) { + return nil, fmt.Errorf("mmap: file %q is too large", filename) + } + + prot := syscall.PROT_READ + if writable { + prot |= syscall.PROT_WRITE + } + data, err := syscall.Mmap(int(f.Fd()), 0, int(size), prot, syscall.MAP_SHARED) + if err != nil { + return nil, err + } + return &mmapFile{data: data, writable: writable}, nil +} + +// Size returns the size of the mapped file. +func (f *mmapFile) Size() uint64 { + return uint64(len(f.data)) +} + +// Pos returns the current file pointer. +func (f *mmapFile) Pos() uint64 { + return f.pos +} + +// SeekTo sets the current file pointer relative to the start of the file. +func (f *mmapFile) SeekTo(offset uint64) { + f.pos = offset +} + +// Read implements io.Reader. +func (f *mmapFile) Read(p []byte) (int, error) { + if f.data == nil { + return 0, errMmapClosed + } + if f.pos >= f.Size() { + return 0, io.EOF + } + n := copy(p, f.data[f.pos:]) + f.pos += uint64(n) + if n < len(p) { + return n, io.EOF + } + return n, nil +} + +// ReadByte implements io.ByteReader. +func (f *mmapFile) ReadByte() (byte, error) { + if f.data == nil { + return 0, errMmapClosed + } + if f.pos >= f.Size() { + return 0, io.EOF + } + b := f.data[f.pos] + f.pos++ + return b, nil +} + +// ReadSlice returns a slice of size n that points directly at the +// underlying mapped file. There is no copying. Fails if it cannot +// read at least n bytes. +func (f *mmapFile) ReadSlice(n uint64) ([]byte, error) { + if f.data == nil { + return nil, errMmapClosed + } + if f.pos+n >= f.Size() { + return nil, io.EOF + } + first := f.pos + f.pos += n + return f.data[first:f.pos:f.pos], nil +} + +// ReadSliceAt is like ReadSlice, but reads from a specific offset. +// The file pointer is not used or advanced. +func (f *mmapFile) ReadSliceAt(offset, n uint64) ([]byte, error) { + if f.data == nil { + return nil, errMmapClosed + } + if f.Size() < offset { + return nil, fmt.Errorf("mmap: out-of-bounds ReadSliceAt offset %d, size is %d", offset, f.Size()) + } + if offset+n >= f.Size() { + return nil, io.EOF + } + end := offset + n + return f.data[offset:end:end], nil +} + +// Close closes the file. +func (f *mmapFile) Close() error { + if f.data == nil { + return nil + } + err := syscall.Munmap(f.data) + f.data = nil + f.pos = 0 + return err +} diff --git a/cmd/heapview/internal/core/raw.go b/cmd/heapview/internal/core/raw.go new file mode 100644 index 0000000000..febed0f4dc --- /dev/null +++ b/cmd/heapview/internal/core/raw.go @@ -0,0 +1,308 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package core provides functions for reading core dumps +// and examining their contained heaps. +package core + +import ( + "bytes" + "encoding/binary" + "fmt" + "runtime" + "sort" +) + +// RawDump provides raw access to the heap records in a core file. +// The raw records in this file are described by other structs named Raw{*}. +// All []byte slices are direct references to the underlying mmap'd file. +// These references will become invalid as soon as the RawDump is closed. +type RawDump struct { + Params *RawParams + MemStats *runtime.MemStats + + HeapObjects []RawSegment // heap objects sorted by Addr, low-to-high + GlobalSegments []RawSegment // data, bss, and noptrbss segments + + OSThreads []*RawOSThread + Goroutines []*RawGoroutine + StackFrames []*RawStackFrame + OtherRoots []*RawOtherRoot + Finalizers []*RawFinalizer + Defers []*RawDefer + Panics []*RawPanic + + TypeFromItab map[uint64]uint64 // map from itab address to the type address that itab represents + TypeFromAddr map[uint64]*RawType // map from RawType.Addr to RawType + + MemProfMap map[uint64]*RawMemProfEntry + AllocSamples []*RawAllocSample + + fmap *mmapFile +} + +// RawParams holds metadata about the program that generated the dump. +type RawParams struct { + // Info about the memory space + + ByteOrder binary.ByteOrder // byte order of all memory in this dump + PtrSize uint64 // in bytes + HeapStart uint64 // heap start address + HeapEnd uint64 // heap end address (this is the last byte in the heap + 1) + + // Info about the program that generated this heapdump + + GoArch string // GOARCH of the runtime library that generated this dump + GoExperiment string // GOEXPERIMENT of the toolchain that build the runtime library + NCPU uint64 // number of physical cpus available to the program +} + +// RawSegment represents a segment of memory. +type RawSegment struct { + Addr uint64 // base address of the segment + Data []byte // data for this segment + PtrFields RawPtrFields // offsets of ptr fields within this segment +} + +// RawPtrFields represents a pointer field. +type RawPtrFields struct { + encoded []byte // list of uvarint-encoded offsets, or nil if none + startOff, endOff uint64 // decoded offsets are translated and clipped to [startOff,endOff) +} + +// RawOSThread represents an OS thread. +type RawOSThread struct { + MAddr uint64 // address of the OS thread descriptor (M) + GoID uint64 // go's internal ID for the thread + ProcID uint64 // kernel's ID for the thread +} + +// RawGoroutine represents a goroutine structure. +type RawGoroutine struct { + GAddr uint64 // address of the goroutine descriptor + SP uint64 // current stack pointer (lowest address in the currently running frame) + GoID uint64 // goroutine ID + GoPC uint64 // PC of the go statement that created this goroutine + Status uint64 + IsSystem bool // true if started by the system + IsBackground bool // always false in go1.7 + WaitSince uint64 // time the goroutine started waiting, in nanoseconds since the Unix epoch + WaitReason string + CtxtAddr uint64 // address of the scheduling ctxt + MAddr uint64 // address of the OS thread descriptor (M) + TopDeferAddr uint64 // address of the top defer record + TopPanicAddr uint64 // address of the top panic record +} + +// RawStackFrame represents a stack frame. +type RawStackFrame struct { + Name string + Depth uint64 // 0 = bottom of stack (currently running frame) + CalleeSP uint64 // stack pointer of the child frame (or 0 for the bottom-most frame) + EntryPC uint64 // entry PC for the function + PC uint64 // current PC being executed + NextPC uint64 // for callers, where the function resumes (if anywhere) after the callee is done + Segment RawSegment // local vars (Segment.Addr is the stack pointer, i.e., lowest address in the frame) +} + +// RawOtherRoot represents the other roots not in RawDump's other fields. +type RawOtherRoot struct { + Description string + Addr uint64 // address pointed to by this root +} + +// RawFinalizer represents a finalizer. +type RawFinalizer struct { + IsQueued bool // if true, the object is unreachable and the finalizer is ready to run + ObjAddr uint64 // address of the object to finalize + ObjTypeAddr uint64 // address of the descriptor for typeof(obj) + FnAddr uint64 // function to be run (a FuncVal*) + FnArgTypeAddr uint64 // address of the descriptor for the type of the function argument + FnPC uint64 // PC of finalizer entry point +} + +// RawDefer represents a defer. +type RawDefer struct { + Addr uint64 // address of the defer record + GAddr uint64 // address of the containing goroutine's descriptor + ArgP uint64 // stack pointer giving the args for defer (TODO: is this right?) + PC uint64 // PC of the defer instruction + FnAddr uint64 // function to be run (a FuncVal*) + FnPC uint64 // PC of the defered function's entry point + LinkAddr uint64 // address of the next defer record in this chain +} + +// RawPanic represents a panic. +type RawPanic struct { + Addr uint64 // address of the panic record + GAddr uint64 // address of the containing goroutine's descriptor + ArgTypeAddr uint64 // type of the panic arg + ArgAddr uint64 // address of the panic arg + DeferAddr uint64 // address of the defer record that is currently running + LinkAddr uint64 // address of the next panic record in this chain +} + +// RawType repesents the Go runtime's representation of a type. +type RawType struct { + Addr uint64 // address of the type descriptor + Size uint64 // in bytes + Name string // not necessarily unique + // If true, this type is equivalent to a single pointer, so ifaces can store + // this type directly in the data field (without indirection). + DirectIFace bool +} + +// RawMemProfEntry represents a memory profiler entry. +type RawMemProfEntry struct { + Size uint64 // size of the allocated object + NumAllocs uint64 // number of allocations + NumFrees uint64 // number of frees + Stacks []RawMemProfFrame // call stacks +} + +// RawMemProfFrame represents a memory profiler frame. +type RawMemProfFrame struct { + Func []byte // string left as []byte reference to save memory + File []byte // string left as []byte reference to save memory + Line uint64 +} + +// RawAllocSample represents a memory profiler allocation sample. +type RawAllocSample struct { + Addr uint64 // address of object + Prof *RawMemProfEntry // record of allocation site +} + +// Close closes the file. +func (r *RawDump) Close() error { + return r.fmap.Close() +} + +// FindSegment returns the segment that contains the given address, or +// nil of no segment contains the address. +func (r *RawDump) FindSegment(addr uint64) *RawSegment { + // Binary search for an upper-bound heap object, then check + // if the previous object contains addr. + k := sort.Search(len(r.HeapObjects), func(k int) bool { + return addr < r.HeapObjects[k].Addr + }) + k-- + if k >= 0 && r.HeapObjects[k].Contains(addr) { + return &r.HeapObjects[k] + } + + // Check all global segments. + for k := range r.GlobalSegments { + if r.GlobalSegments[k].Contains(addr) { + return &r.GlobalSegments[k] + } + } + + // NB: Stack-local vars are technically allocated in the heap, since stack frames are + // allocated in the heap space, however, stack frames don't show up in r.HeapObjects. + for _, f := range r.StackFrames { + if f.Segment.Contains(addr) { + return &f.Segment + } + } + + return nil +} + +// Contains returns true if the segment contains the given address. +func (r RawSegment) Contains(addr uint64) bool { + return r.Addr <= addr && addr < r.Addr+r.Size() +} + +// ContainsRange returns true if the segment contains the range [addr, addr+size). +func (r RawSegment) ContainsRange(addr, size uint64) bool { + if !r.Contains(addr) { + return false + } + if size > 0 && !r.Contains(addr+size-1) { + return false + } + return true +} + +// Size returns the size of the segment in bytes. +func (r RawSegment) Size() uint64 { + return uint64(len(r.Data)) +} + +// Slice takes a slice of the given segment. Panics if [offset,offset+size) +// is out-of-bounds. The resulting RawSegment.PtrOffsets will clipped and +// translated into the new segment. +func (r RawSegment) Slice(offset, size uint64) *RawSegment { + if offset+size > uint64(len(r.Data)) { + panic(fmt.Errorf("slice(%d,%d) out-of-bounds of segment @%x sz=%d", offset, size, r.Addr, len(r.Data))) + } + return &RawSegment{ + Addr: r.Addr + offset, + Data: r.Data[offset : offset+size : offset+size], + PtrFields: RawPtrFields{ + encoded: r.PtrFields.encoded, + startOff: r.PtrFields.startOff + offset, + endOff: r.PtrFields.startOff + offset + size, + }, + } +} + +// Offsets decodes the list of ptr field offsets. +func (r RawPtrFields) Offsets() []uint64 { + if r.encoded == nil { + return nil + } + + // NB: This should never fail since we already decoded the varints once + // when parsing the file originally. Hence we panic on failure. + reader := bytes.NewReader(r.encoded) + readUint64 := func() uint64 { + x, err := binary.ReadUvarint(reader) + if err != nil { + panic(fmt.Errorf("unexpected failure decoding uvarint: %v", err)) + } + return x + } + + var out []uint64 + for { + k := readUint64() + switch k { + case 0: // end + return out + case 1: // ptr + x := readUint64() + if r.startOff <= x && x < r.endOff { + out = append(out, x-r.startOff) + } + default: + panic(fmt.Errorf("unexpected FieldKind %d", k)) + } + } +} + +// ReadPtr decodes a ptr from the given byte slice. +func (r *RawParams) ReadPtr(b []byte) uint64 { + switch r.PtrSize { + case 4: + return uint64(r.ByteOrder.Uint32(b)) + case 8: + return r.ByteOrder.Uint64(b) + default: + panic(fmt.Errorf("unsupported PtrSize=%d", r.PtrSize)) + } +} + +// WritePtr encodes a ptr into the given byte slice. +func (r *RawParams) WritePtr(b []byte, addr uint64) { + switch r.PtrSize { + case 4: + r.ByteOrder.PutUint32(b, uint32(addr)) + case 8: + r.ByteOrder.PutUint64(b, addr) + default: + panic(fmt.Errorf("unsupported PtrSize=%d", r.PtrSize)) + } +}