qbit/go
1
0
Fork 0
mirror of https://github.com/golang/go synced 2024-11-26 17:56:55 -07:00
Code Releases Activity

cmd/compile: Enables PGO in Go and performs profile-guided inlining

Browse Source 
For #55022

Change-Id: I51f1ba166d5a66dcaf4b280756be4a6bf9545c5e
Reviewed-on: https://go-review.googlesource.com/c/go/+/429863
TryBot-Result: Gopher Robot <gobot@golang.org>
Reviewed-by: Cherry Mui <cherryyz@google.com>
Reviewed-by: Michael Pratt <mpratt@google.com>
Run-TryBot: Cherry Mui <cherryyz@google.com>
This commit is contained in:
Raj Barik 2022-09-09 11:29:32 -07:00 committed by Michael Pratt
parent 537c4354cb
commit 99862cd57d
12 changed files with 2015 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

62
src/cmd/compile/internal/base/debug.go
View File

@ -16,35 +16,39 @@ var Debug DebugFlags
// The -d option takes a comma-separated list of settings.
// Each setting is name=value; for ints, name is short for name=1.
type DebugFlags struct {
Append int `help:"print information about append compilation"`
Checkptr int `help:"instrument unsafe pointer conversions\n0: instrumentation disabled\n1: conversions involving unsafe.Pointer are instrumented\n2: conversions to unsafe.Pointer force heap allocation"`
Closure int `help:"print information about closure compilation"`
DclStack int `help:"run internal dclstack check"`
Defer int `help:"print information about defer compilation"`
DisableNil int `help:"disable nil checks"`
DumpPtrs int `help:"show Node pointers values in dump output"`
DwarfInl int `help:"print information about DWARF inlined function creation"`
Export int `help:"print export data"`
GCProg int `help:"print dump of GC programs"`
InlFuncsWithClosures int `help:"allow functions with closures to be inlined"`
Libfuzzer int `help:"enable coverage instrumentation for libfuzzer"`
LocationLists int `help:"print information about DWARF location list creation"`
Nil int `help:"print information about nil checks"`
NoOpenDefer int `help:"disable open-coded defers"`
NoRefName int `help:"do not include referenced symbol names in object file"`
PCTab string `help:"print named pc-value table\nOne of: pctospadj, pctofile, pctoline, pctoinline, pctopcdata"`
Panic int `help:"show all compiler panics"`
Reshape int `help:"print information about expression reshaping"`
Shapify int `help:"print information about shaping recursive types"`
Slice int `help:"print information about slice compilation"`
SoftFloat int `help:"force compiler to emit soft-float code"`
SyncFrames int `help:"how many writer stack frames to include at sync points in unified export data"`
TypeAssert int `help:"print information about type assertion inlining"`
TypecheckInl int `help:"eager typechecking of inline function bodies"`
Unified int `help:"enable unified IR construction"`
WB int `help:"print information about write barriers"`
ABIWrap int `help:"print information about ABI wrapper generation"`
MayMoreStack string `help:"call named function before all stack growth checks"`
Append int `help:"print information about append compilation"`
Checkptr int `help:"instrument unsafe pointer conversions\n0: instrumentation disabled\n1: conversions involving unsafe.Pointer are instrumented\n2: conversions to unsafe.Pointer force heap allocation"`
Closure int `help:"print information about closure compilation"`
DclStack int `help:"run internal dclstack check"`
Defer int `help:"print information about defer compilation"`
DisableNil int `help:"disable nil checks"`
DumpPtrs int `help:"show Node pointers values in dump output"`
DwarfInl int `help:"print information about DWARF inlined function creation"`
Export int `help:"print export data"`
GCProg int `help:"print dump of GC programs"`
InlFuncsWithClosures int `help:"allow functions with closures to be inlined"`
Libfuzzer int `help:"enable coverage instrumentation for libfuzzer"`
LocationLists int `help:"print information about DWARF location list creation"`
Nil int `help:"print information about nil checks"`
NoOpenDefer int `help:"disable open-coded defers"`
NoRefName int `help:"do not include referenced symbol names in object file"`
PCTab string `help:"print named pc-value table\nOne of: pctospadj, pctofile, pctoline, pctoinline, pctopcdata"`
Panic int `help:"show all compiler panics"`
Reshape int `help:"print information about expression reshaping"`
Shapify int `help:"print information about shaping recursive types"`
Slice int `help:"print information about slice compilation"`
SoftFloat int `help:"force compiler to emit soft-float code"`
SyncFrames int `help:"how many writer stack frames to include at sync points in unified export data"`
TypeAssert int `help:"print information about type assertion inlining"`
TypecheckInl int `help:"eager typechecking of inline function bodies"`
Unified int `help:"enable unified IR construction"`
WB int `help:"print information about write barriers"`
ABIWrap int `help:"print information about ABI wrapper generation"`
MayMoreStack string `help:"call named function before all stack growth checks"`
InlineHotFuncThreshold string `help:"threshold percentage for determining functions as hot candidates for inlining"`
InlineHotCallSiteThreshold string `help:"threshold percentage for determining call sites as hot candidates for inlining"`
InlineHotBudget int `help:"inline budget for hot functions"`
PGOInline int `help:"debug profile-guided inlining"`
Any bool // set when any of the debug flags have been set
}

1
src/cmd/compile/internal/base/flag.go
View File

@ -124,6 +124,7 @@ type CmdFlags struct {
TrimPath string "help:\"remove `prefix` from recorded source file paths\""
WB bool "help:\"enable write barrier\"" // TODO: remove
AltComparable bool "help:\"enable alternative comparable semantics\"" // experiment - remove eventually
PgoProfile string "help:\"read profile from `file`\""
// Configuration derived from flags; not a flag itself.
Cfg struct {

17
src/cmd/compile/internal/gc/main.go
View File

@ -17,6 +17,7 @@ import (
"cmd/compile/internal/ir"
"cmd/compile/internal/logopt"
"cmd/compile/internal/noder"
"cmd/compile/internal/pgo"
"cmd/compile/internal/pkginit"
"cmd/compile/internal/reflectdata"
"cmd/compile/internal/ssa"
@ -249,10 +250,26 @@ func Main(archInit func(*ssagen.ArchInfo)) {
typecheck.AllImportedBodies()
}
// Read profile file and build profile-graph and weighted-call-graph.
base.Timer.Start("fe", "pgoprofile")
if base.Flag.PgoProfile != "" {
pgo.BuildProfileGraph(base.Flag.PgoProfile)
pgo.BuildWeightedCallGraph()
}
// Inlining
base.Timer.Start("fe", "inlining")
if base.Flag.LowerL != 0 {
if pgo.WeightedCG != nil {
inline.InlinePrologue()
}
inline.InlinePackage()
if pgo.WeightedCG != nil {
inline.InlineEpilogue()
// Delete the graphs as no other optimization uses this currently.
pgo.WeightedCG = nil
pgo.ProfileGraph = nil
}
}
noder.MakeWrappers(typecheck.Target) // must happen after inlining

169
src/cmd/compile/internal/inline/inl.go
View File

@ -29,11 +29,13 @@ package inline
import (
"fmt"
"go/constant"
"strconv"
"strings"
"cmd/compile/internal/base"
"cmd/compile/internal/ir"
"cmd/compile/internal/logopt"
"cmd/compile/internal/pgo"
"cmd/compile/internal/typecheck"
"cmd/compile/internal/types"
"cmd/internal/obj"
@ -53,6 +55,91 @@ const (
inlineBigFunctionMaxCost = 20 // Max cost of inlinee when inlining into a "big" function.
)
var (
// List of all hot ndes.
candHotNodeMap = make(map[*pgo.IRNode]struct{})
// List of all hot call sites.
candHotEdgeMap = make(map[pgo.CallSiteInfo]struct{})
// List of inlined call sites.
inlinedCallSites = make(map[pgo.CallSiteInfo]struct{})
// Threshold in percentage for hot function inlining.
inlineHotFuncThresholdPercent = float64(2)
// Threshold in percentage for hot callsite inlining.
inlineHotCallSiteThresholdPercent = float64(0.1)
// Budget increased due to hotness.
inlineHotMaxBudget int32 = 160
)
// InlinePrologue records the hot callsites from ir-graph.
func InlinePrologue() {
if s, err := strconv.ParseFloat(base.Debug.InlineHotFuncThreshold, 64); err == nil {
inlineHotFuncThresholdPercent = s
if base.Debug.PGOInline > 0 {
fmt.Printf("hot-node-thres=%v\n", inlineHotFuncThresholdPercent)
}
}
if s, err := strconv.ParseFloat(base.Debug.InlineHotCallSiteThreshold, 64); err == nil {
inlineHotCallSiteThresholdPercent = s
if base.Debug.PGOInline > 0 {
fmt.Printf("hot-callsite-thres=%v\n", inlineHotCallSiteThresholdPercent)
}
}
if base.Debug.InlineHotBudget != 0 {
inlineHotMaxBudget = int32(base.Debug.InlineHotBudget)
}
ir.VisitFuncsBottomUp(typecheck.Target.Decls, func(list []*ir.Func, recursive bool) {
for _, f := range list {
name := ir.PkgFuncName(f)
if n, ok := pgo.WeightedCG.IRNodes[name]; ok {
nodeweight := pgo.WeightInPercentage(n.Flat, pgo.GlobalTotalNodeWeight)
if nodeweight > inlineHotFuncThresholdPercent {
candHotNodeMap[n] = struct{}{}
}
for _, e := range pgo.WeightedCG.OutEdges[n] {
if e.Weight != 0 {
edgeweightpercent := pgo.WeightInPercentage(e.Weight, pgo.GlobalTotalEdgeWeight)
if edgeweightpercent > inlineHotCallSiteThresholdPercent {
csi := pgo.CallSiteInfo{Line: e.CallSite, Caller: n.AST, Callee: e.Dst.AST}
if _, ok := candHotEdgeMap[csi]; !ok {
candHotEdgeMap[csi] = struct{}{}
}
}
}
}
}
}
})
if base.Debug.PGOInline > 0 {
fmt.Printf("hot-cg before inline in dot format:")
pgo.PrintWeightedCallGraphDOT(inlineHotFuncThresholdPercent, inlineHotCallSiteThresholdPercent)
}
}
// InlineEpilogue updates IRGraph after inlining.
func InlineEpilogue() {
if base.Debug.PGOInline > 0 {
ir.VisitFuncsBottomUp(typecheck.Target.Decls, func(list []*ir.Func, recursive bool) {
for _, f := range list {
name := ir.PkgFuncName(f)
if n, ok := pgo.WeightedCG.IRNodes[name]; ok {
pgo.RedirectEdges(n, inlinedCallSites)
}
}
})
// Print the call-graph after inlining. This is a debugging feature.
fmt.Printf("hot-cg after inline in dot:")
pgo.PrintWeightedCallGraphDOT(inlineHotFuncThresholdPercent, inlineHotCallSiteThresholdPercent)
}
}
// InlinePackage finds functions that can be inlined and clones them before walk expands them.
func InlinePackage() {
ir.VisitFuncsBottomUp(typecheck.Target.Decls, func(list []*ir.Func, recursive bool) {
@ -81,6 +168,9 @@ func CanInline(fn *ir.Func) {
base.Fatalf("CanInline no nname %+v", fn)
}
// Initialize an empty list of hot callsites for this caller.
pgo.ListOfHotCallSites = make(map[pgo.CallSiteInfo]struct{})
var reason string // reason, if any, that the function was not inlined
if base.Flag.LowerM > 1 || logopt.Enabled() {
defer func() {
@ -168,6 +258,19 @@ func CanInline(fn *ir.Func) {
cc = 1 // this appears to yield better performance than 0.
}
// Update the budget for profile-guided inlining.
budget := int32(inlineMaxBudget)
if base.Flag.PgoProfile != "" && pgo.WeightedCG != nil {
if n, ok := pgo.WeightedCG.IRNodes[ir.PkgFuncName(fn)]; ok {
if _, ok := candHotNodeMap[n]; ok {
budget = int32(inlineHotMaxBudget)
if base.Debug.PGOInline > 0 {
fmt.Printf("hot-node enabled increased budget=%v for func=%v\n", budget, ir.PkgFuncName(fn))
}
}
}
}
// At this point in the game the function we're looking at may
// have "stale" autos, vars that still appear in the Dcl list, but
// which no longer have any uses in the function body (due to
@ -178,7 +281,9 @@ func CanInline(fn *ir.Func) {
// list. See issue 25249 for more context.
visitor := hairyVisitor{
budget: inlineMaxBudget,
curFunc: fn,
budget: budget,
maxBudget: budget,
extraCallCost: cc,
}
if visitor.tooHairy(fn) {
@ -187,7 +292,7 @@ func CanInline(fn *ir.Func) {
}
n.Func.Inl = &ir.Inline{
Cost: inlineMaxBudget - visitor.budget,
Cost: budget - visitor.budget,
Dcl: pruneUnusedAutos(n.Defn.(*ir.Func).Dcl, &visitor),
Body: inlcopylist(fn.Body),
@ -195,12 +300,12 @@ func CanInline(fn *ir.Func) {
}
if base.Flag.LowerM > 1 {
fmt.Printf("%v: can inline %v with cost %d as: %v { %v }\n", ir.Line(fn), n, inlineMaxBudget-visitor.budget, fn.Type(), ir.Nodes(n.Func.Inl.Body))
fmt.Printf("%v: can inline %v with cost %d as: %v { %v }\n", ir.Line(fn), n, budget-visitor.budget, fn.Type(), ir.Nodes(n.Func.Inl.Body))
} else if base.Flag.LowerM != 0 {
fmt.Printf("%v: can inline %v\n", ir.Line(fn), n)
}
if logopt.Enabled() {
logopt.LogOpt(fn.Pos(), "canInlineFunction", "inline", ir.FuncName(fn), fmt.Sprintf("cost: %d", inlineMaxBudget-visitor.budget))
logopt.LogOpt(fn.Pos(), "canInlineFunction", "inline", ir.FuncName(fn), fmt.Sprintf("cost: %d", budget-visitor.budget))
}
}
@ -239,7 +344,10 @@ func canDelayResults(fn *ir.Func) bool {
// hairyVisitor visits a function body to determine its inlining
// hairiness and whether or not it can be inlined.
type hairyVisitor struct {
// This is needed to access the current caller in the doNode function.
curFunc *ir.Func
budget int32
maxBudget int32
reason string
extraCallCost int32
usedLocals ir.NameSet
@ -252,7 +360,7 @@ func (v *hairyVisitor) tooHairy(fn *ir.Func) bool {
return true
}
if v.budget < 0 {
v.reason = fmt.Sprintf("function too complex: cost %d exceeds budget %d", inlineMaxBudget-v.budget, inlineMaxBudget)
v.reason = fmt.Sprintf("function too complex: cost %d exceeds budget %d", v.maxBudget-v.budget, v.maxBudget)
return true
}
return false
@ -330,6 +438,20 @@ func (v *hairyVisitor) doNode(n ir.Node) bool {
}
}
// Determine if the callee edge is a for hot callee or not.
if base.Flag.PgoProfile != "" && pgo.WeightedCG != nil && v.curFunc != nil {
if fn := inlCallee(n.X); fn != nil && typecheck.HaveInlineBody(fn) {
ln := pgo.ConvertLine2Int(ir.Line(n))
csi := pgo.CallSiteInfo{Line: ln, Caller: v.curFunc, Callee: fn}
if _, o := candHotEdgeMap[csi]; o {
pgo.ListOfHotCallSites[pgo.CallSiteInfo{Line: ln, Caller: v.curFunc}] = struct{}{}
if base.Debug.PGOInline > 0 {
fmt.Printf("hot-callsite identified at line=%v for func=%v\n", ir.Line(n), ir.PkgFuncName(v.curFunc))
}
}
}
}
if ir.IsIntrinsicCall(n) {
// Treat like any other node.
break
@ -750,13 +872,29 @@ func mkinlcall(n *ir.CallExpr, fn *ir.Func, maxCost int32, inlCalls *[]*ir.Inlin
return n
}
if fn.Inl.Cost > maxCost {
// The inlined function body is too big. Typically we use this check to restrict
// inlining into very big functions. See issue 26546 and 17566.
if logopt.Enabled() {
logopt.LogOpt(n.Pos(), "cannotInlineCall", "inline", ir.FuncName(ir.CurFunc),
fmt.Sprintf("cost %d of %s exceeds max large caller cost %d", fn.Inl.Cost, ir.PkgFuncName(fn), maxCost))
// If the callsite is hot and it is under the inlineHotMaxBudget budget, then try to inline it, or else bail.
ln := pgo.ConvertLine2Int(ir.Line(n))
csi := pgo.CallSiteInfo{Line: ln, Caller: ir.CurFunc}
if _, ok := pgo.ListOfHotCallSites[csi]; ok {
if fn.Inl.Cost > inlineHotMaxBudget {
if logopt.Enabled() {
logopt.LogOpt(n.Pos(), "cannotInlineCall", "inline", ir.FuncName(ir.CurFunc),
fmt.Sprintf("cost %d of %s exceeds max large caller cost %d", fn.Inl.Cost, ir.PkgFuncName(fn), inlineHotMaxBudget))
}
return n
}
if base.Debug.PGOInline > 0 {
fmt.Printf("hot-budget check allows inlining for callsite at %v\n", ir.Line(n))
}
} else {
// The inlined function body is too big. Typically we use this check to restrict
// inlining into very big functions. See issue 26546 and 17566.
if logopt.Enabled() {
logopt.LogOpt(n.Pos(), "cannotInlineCall", "inline", ir.FuncName(ir.CurFunc),
fmt.Sprintf("cost %d of %s exceeds max large caller cost %d", fn.Inl.Cost, ir.PkgFuncName(fn), maxCost))
}
return n
}
return n
}
if fn == ir.CurFunc {
@ -899,7 +1037,16 @@ func mkinlcall(n *ir.CallExpr, fn *ir.Func, maxCost int32, inlCalls *[]*ir.Inlin
fmt.Printf("%v: Before inlining: %+v\n", ir.Line(n), n)
}
if base.Debug.PGOInline > 0 {
ln := pgo.ConvertLine2Int(ir.Line(n))
csi := pgo.CallSiteInfo{Line: ln, Caller: ir.CurFunc}
if _, ok := inlinedCallSites[csi]; !ok {
inlinedCallSites[csi] = struct{}{}
}
}
res := InlineCall(n, fn, inlIndex)
if res == nil {
base.FatalfAt(n.Pos(), "inlining call to %v failed", fn)
}

1033
src/cmd/compile/internal/pgo/graph.go Normal file
View File

File diff suppressed because it is too large Load Diff

480
src/cmd/compile/internal/pgo/irgraph.go Normal file
View File

@ -0,0 +1,480 @@
// Copyright 2022 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// WORK IN PROGRESS
package pgo
import (
"cmd/compile/internal/ir"
"cmd/compile/internal/typecheck"
"cmd/compile/internal/types"
"fmt"
"internal/profile"
"log"
"os"
"strconv"
"strings"
)
// IRGraph is the key datastrcture that is built from profile. It is essentially a call graph with nodes pointing to IRs of functions and edges carrying weights and callsite information. The graph is bidirectional that helps in removing nodes efficiently.
type IRGraph struct {
// Nodes of the graph
IRNodes map[string]*IRNode
OutEdges IREdgeMap
InEdges IREdgeMap
}
// IRNode represents a node in the IRGraph.
type IRNode struct {
// Pointer to the IR of the Function represented by this node.
AST *ir.Func
// Flat weight of the IRNode, obtained from profile.
Flat int64
// Cumulative weight of the IRNode.
Cum int64
}
// IREdgeMap maps an IRNode to its successors.
type IREdgeMap map[*IRNode][]*IREdge
// IREdge represents a call edge in the IRGraph with source, destination, weight, callsite, and line number information.
type IREdge struct {
// Source and destination of the edge in IRNode.
Src, Dst *IRNode
Weight int64
CallSite int
}
// NodeMapKey represents a hash key to identify unique call-edges in profile and in IR. Used for deduplication of call edges found in profile.
type NodeMapKey struct {
CallerName string
CalleeName string
CallSite int
}
// Weights capture both node weight and edge weight.
type Weights struct {
NFlat int64
NCum int64
EWeight int64
}
// CallSiteInfo captures call-site information and its caller/callee.
type CallSiteInfo struct {
Line int
Caller *ir.Func
Callee *ir.Func
}
var (
// Aggregated NodeWeights and EdgeWeights across profiles. This helps us determine the percentage threshold for hot/cold partitioning.
GlobalTotalNodeWeight = int64(0)
GlobalTotalEdgeWeight = int64(0)
// Global node and their aggregated weight information.
GlobalNodeMap = make(map[NodeMapKey]*Weights)
// WeightedCG represents the IRGraph built from profile, which we will update as part of inlining.
WeightedCG *IRGraph
// Original profile-graph.
ProfileGraph *Graph
// Per-caller data structure to track the list of hot call sites. This gets rewritten every caller leaving it to GC for cleanup.
ListOfHotCallSites = make(map[CallSiteInfo]struct{})
)
// BuildProfileGraph generates a profile-graph from the profile.
func BuildProfileGraph(profileFile string) {
// if possible, we should cache the profile-graph.
if ProfileGraph != nil {
return
}
// open the profile file.
f, err := os.Open(profileFile)
if err != nil {
log.Fatal("failed to open file " + profileFile)
return
}
defer f.Close()
p, err := profile.Parse(f)
if err != nil {
log.Fatal("failed to Parse profile file.")
return
}
// Build the options.
opt := &Options{
CallTree: false,
SampleValue: func(v []int64) int64 { return v[1] },
}
// Build the graph using profile package.
ProfileGraph = New(p, opt)
// Build various global maps from profile.
preprocessProfileGraph()
}
// BuildWeightedCallGraph generates a weighted callgraph from the profile for the current package.
func BuildWeightedCallGraph() {
// Bail if there is no profile-graph available.
if ProfileGraph == nil {
return
}
// Create package-level call graph with weights from profile and IR.
WeightedCG = createIRGraph()
}
// ConvertLine2Int converts ir.Line string to integer.
func ConvertLine2Int(line string) int {
splits := strings.Split(line, ":")
cs, _ := strconv.ParseInt(splits[len(splits)-2], 0, 64)
return int(cs)
}
// preprocessProfileGraph builds various maps from the profile-graph. It builds GlobalNodeMap and other information based on the name and callsite to compute node and edge weights which will be used later on to create edges for WeightedCG.
func preprocessProfileGraph() {
nFlat := make(map[string]int64)
nCum := make(map[string]int64)
// Accummulate weights for the same node.
for _, n := range ProfileGraph.Nodes {
canonicalName := n.Info.Name
nFlat[canonicalName] += n.FlatValue()
nCum[canonicalName] += n.CumValue()
}
// Process ProfileGraph and build various node and edge maps which will be consumed by AST walk.
for _, n := range ProfileGraph.Nodes {
GlobalTotalNodeWeight += n.FlatValue()
canonicalName := n.Info.Name
// Create the key to the NodeMapKey.
nodeinfo := NodeMapKey{
CallerName: canonicalName,
CallSite: n.Info.Lineno,
}
for _, e := range n.Out {
GlobalTotalEdgeWeight += e.WeightValue()
nodeinfo.CalleeName = e.Dest.Info.Name
if w, ok := GlobalNodeMap[nodeinfo]; ok {
w.EWeight += e.WeightValue()
} else {
weights := new(Weights)
weights.NFlat = nFlat[canonicalName]
weights.NCum = nCum[canonicalName]
weights.EWeight = e.WeightValue()
GlobalNodeMap[nodeinfo] = weights
}
}
}
}
// createIRGraph builds the IRGraph by visting all the ir.Func in decl list of a package.
func createIRGraph() *IRGraph {
var g IRGraph
// Bottomup walk over the function to create IRGraph.
ir.VisitFuncsBottomUp(typecheck.Target.Decls, func(list []*ir.Func, recursive bool) {
for _, n := range list {
g.Visit(n, recursive)
}
})
return &g
}
// Visit traverses the body of each ir.Func and use GlobalNodeMap to determine if we need to add an edge from ir.Func and any node in the ir.Func body.
func (g *IRGraph) Visit(fn *ir.Func, recursive bool) {
if g.IRNodes == nil {
g.IRNodes = make(map[string]*IRNode)
}
if g.OutEdges == nil {
g.OutEdges = make(map[*IRNode][]*IREdge)
}
if g.InEdges == nil {
g.InEdges = make(map[*IRNode][]*IREdge)
}
name := ir.PkgFuncName(fn)
node := new(IRNode)
node.AST = fn
if g.IRNodes[name] == nil {
g.IRNodes[name] = node
}
// Create the key for the NodeMapKey.
nodeinfo := NodeMapKey{
CallerName: name,
CalleeName: "",
CallSite: -1,
}
// If the node exists, then update its node weight.
if weights, ok := GlobalNodeMap[nodeinfo]; ok {
g.IRNodes[name].Flat = weights.NFlat
g.IRNodes[name].Cum = weights.NCum
}
// Recursively walk over the body of the function to create IRGraph edges.
g.createIRGraphEdge(fn, g.IRNodes[name], name)
}
// addEdge adds an edge between caller and new node that points to `callee` based on the profile-graph and GlobalNodeMap.
func (g *IRGraph) addEdge(caller *IRNode, callee *ir.Func, n *ir.Node, callername string, line int) {
// Create an IRNode for the callee.
calleenode := new(IRNode)
calleenode.AST = callee
calleename := ir.PkgFuncName(callee)
// Create key for NodeMapKey.
nodeinfo := NodeMapKey{
CallerName: callername,
CalleeName: calleename,
CallSite: line,
}
// Create the callee node with node weight.
if g.IRNodes[calleename] == nil {
g.IRNodes[calleename] = calleenode
nodeinfo2 := NodeMapKey{
CallerName: calleename,
CalleeName: "",
CallSite: -1,
}
if weights, ok := GlobalNodeMap[nodeinfo2]; ok {
g.IRNodes[calleename].Flat = weights.NFlat
g.IRNodes[calleename].Cum = weights.NCum
}
}
if weights, ok := GlobalNodeMap[nodeinfo]; ok {
caller.Flat = weights.NFlat
caller.Cum = weights.NCum
// Add edge in the IRGraph from caller to callee.
info := &IREdge{Src: caller, Dst: g.IRNodes[calleename], Weight: weights.EWeight, CallSite: line}
g.OutEdges[caller] = append(g.OutEdges[caller], info)
g.InEdges[g.IRNodes[calleename]] = append(g.InEdges[g.IRNodes[calleename]], info)
} else {
nodeinfo.CalleeName = ""
nodeinfo.CallSite = -1
if weights, ok := GlobalNodeMap[nodeinfo]; ok {
caller.Flat = weights.NFlat
caller.Cum = weights.NCum
info := &IREdge{Src: caller, Dst: g.IRNodes[calleename], Weight: 0, CallSite: line}
g.OutEdges[caller] = append(g.OutEdges[caller], info)
g.InEdges[g.IRNodes[calleename]] = append(g.InEdges[g.IRNodes[calleename]], info)
} else {
info := &IREdge{Src: caller, Dst: g.IRNodes[calleename], Weight: 0, CallSite: line}
g.OutEdges[caller] = append(g.OutEdges[caller], info)
g.InEdges[g.IRNodes[calleename]] = append(g.InEdges[g.IRNodes[calleename]], info)
}
}
}
// createIRGraphEdge traverses the nodes in the body of ir.Func and add edges between callernode which points to the ir.Func and the nodes in the body.
func (g *IRGraph) createIRGraphEdge(fn *ir.Func, callernode *IRNode, name string) {
var doNode func(ir.Node) bool
doNode = func(n ir.Node) bool {
switch n.Op() {
default:
ir.DoChildren(n, doNode)
case ir.OCALLFUNC:
call := n.(*ir.CallExpr)
line := ConvertLine2Int(ir.Line(n))
// Find the callee function from the call site and add the edge.
f := inlCallee(call.X)
if f != nil {
g.addEdge(callernode, f, &n, name, line)
}
case ir.OCALLMETH:
call := n.(*ir.CallExpr)
// Find the callee method from the call site and add the edge.
fn2 := ir.MethodExprName(call.X).Func
line := ConvertLine2Int(ir.Line(n))
g.addEdge(callernode, fn2, &n, name, line)
}
return false
}
doNode(fn)
}
// WeightInPercentage converts profile weights to a percentage.
func WeightInPercentage(value int64, total int64) float64 {
var ratio float64
if total != 0 {
ratio = (float64(value) / float64(total)) * 100
}
return ratio
}
// PrintWeightedCallGraphDOT prints IRGraph in DOT format.
func PrintWeightedCallGraphDOT(nodeThreshold float64, edgeThreshold float64) {
fmt.Printf("\ndigraph G {\n")
fmt.Printf("forcelabels=true;\n")
// List of functions in this package.
funcs := make(map[string]struct{})
ir.VisitFuncsBottomUp(typecheck.Target.Decls, func(list []*ir.Func, recursive bool) {
for _, f := range list {
name := ir.PkgFuncName(f)
funcs[name] = struct{}{}
}
})
// Determine nodes of DOT.
nodes := make(map[string]*ir.Func)
for name, _ := range funcs {
if n, ok := WeightedCG.IRNodes[name]; ok {
for _, e := range WeightedCG.OutEdges[n] {
if _, ok := nodes[ir.PkgFuncName(e.Src.AST)]; !ok {
nodes[ir.PkgFuncName(e.Src.AST)] = e.Src.AST
}
if _, ok := nodes[ir.PkgFuncName(e.Dst.AST)]; !ok {
nodes[ir.PkgFuncName(e.Dst.AST)] = e.Dst.AST
}
}
if _, ok := nodes[ir.PkgFuncName(n.AST)]; !ok {
nodes[ir.PkgFuncName(n.AST)] = n.AST
}
}
}
// Print nodes.
for name, ast := range nodes {
if n, ok := WeightedCG.IRNodes[name]; ok {
nodeweight := WeightInPercentage(n.Flat, GlobalTotalNodeWeight)
color := "black"
if nodeweight > nodeThreshold {
color = "red"
}
if ast.Inl != nil {
fmt.Printf("\"%v\" [color=%v,label=\"%v,freq=%.2f,inl_cost=%d\"];\n", ir.PkgFuncName(ast), color, ir.PkgFuncName(ast), nodeweight, ast.Inl.Cost)
} else {
fmt.Printf("\"%v\" [color=%v, label=\"%v,freq=%.2f\"];\n", ir.PkgFuncName(ast), color, ir.PkgFuncName(ast), nodeweight)
}
}
}
// Print edges.
ir.VisitFuncsBottomUp(typecheck.Target.Decls, func(list []*ir.Func, recursive bool) {
for _, f := range list {
name := ir.PkgFuncName(f)
if n, ok := WeightedCG.IRNodes[name]; ok {
for _, e := range WeightedCG.OutEdges[n] {
edgepercent := WeightInPercentage(e.Weight, GlobalTotalEdgeWeight)
if edgepercent > edgeThreshold {
fmt.Printf("edge [color=red, style=solid];\n")
} else {
fmt.Printf("edge [color=black, style=solid];\n")
}
fmt.Printf("\"%v\" -> \"%v\" [label=\"%.2f\"];\n", ir.PkgFuncName(n.AST), ir.PkgFuncName(e.Dst.AST), edgepercent)
}
}
}
})
fmt.Printf("}\n")
}
// redirectEdges deletes the cur node out-edges and redirect them so now these edges are the parent node out-edges.
func redirectEdges(g *IRGraph, parent *IRNode, cur *IRNode) {
for _, outEdge := range g.OutEdges[cur] {
outEdge.Src = parent
g.OutEdges[parent] = append(g.OutEdges[parent], outEdge)
}
delete(g.OutEdges, cur)
}
// RedirectEdges deletes and redirects out-edges from node cur based on inlining information via inlinedCallSites.
func RedirectEdges(cur *IRNode, inlinedCallSites map[CallSiteInfo]struct{}) {
g := WeightedCG
for i, outEdge := range g.OutEdges[cur] {
if _, found := inlinedCallSites[CallSiteInfo{Line: outEdge.CallSite, Caller: cur.AST}]; !found {
for _, InEdge := range g.InEdges[cur] {
if _, ok := inlinedCallSites[CallSiteInfo{Line: InEdge.CallSite, Caller: InEdge.Src.AST}]; ok {
weight := calculateweight(g, InEdge.Src, cur)
redirectEdge(g, InEdge.Src, cur, outEdge, weight, i)
}
}
} else {
remove(g, cur, i, outEdge.Dst.AST.Nname)
}
}
removeall(g, cur)
}
// calculateweight calculates the weight of the new redirected edge.
func calculateweight(g *IRGraph, parent *IRNode, cur *IRNode) int64 {
sum := int64(0)
pw := int64(0)
for _, InEdge := range g.InEdges[cur] {
sum = sum + InEdge.Weight
if InEdge.Src == parent {
pw = InEdge.Weight
}
}
weight := int64(0)
if sum != 0 {
weight = pw / sum
} else {
weight = pw
}
return weight
}
// redirectEdge deletes the cur-node's out-edges and redirect them so now these edges are the parent node out-edges.
func redirectEdge(g *IRGraph, parent *IRNode, cur *IRNode, outEdge *IREdge, weight int64, idx int) {
outEdge.Src = parent
outEdge.Weight = weight * outEdge.Weight
g.OutEdges[parent] = append(g.OutEdges[parent], outEdge)
remove(g, cur, idx, outEdge.Dst.AST.Nname)
}
// remove deletes the cur-node's out-edges at index idx.
func remove(g *IRGraph, cur *IRNode, idx int, name *ir.Name) {
if len(g.OutEdges[cur]) >= 2 {
g.OutEdges[cur][idx] = &IREdge{CallSite: -1}
} else {
delete(g.OutEdges, cur)
}
}
// removeall deletes all cur-node's out-edges that marked to be removed .
func removeall(g *IRGraph, cur *IRNode) {
for i := len(g.OutEdges[cur]) - 1; i >= 0; i-- {
if g.OutEdges[cur][i].CallSite == -1 {
g.OutEdges[cur][i] = g.OutEdges[cur][len(g.OutEdges[cur])-1]
g.OutEdges[cur] = g.OutEdges[cur][:len(g.OutEdges[cur])-1]
}
}
}
// inlCallee is same as the implementation for inl.go with one change. The change is that we do not invoke CanInline on a closure.
func inlCallee(fn ir.Node) *ir.Func {
fn = ir.StaticValue(fn)
switch fn.Op() {
case ir.OMETHEXPR:
fn := fn.(*ir.SelectorExpr)
n := ir.MethodExprName(fn)
// Check that receiver type matches fn.X.
// TODO(mdempsky): Handle implicit dereference
// of pointer receiver argument?
if n == nil || !types.Identical(n.Type().Recv().Type, fn.X.Type()) {
return nil
}
return n.Func
case ir.ONAME:
fn := fn.(*ir.Name)
if fn.Class == ir.PFUNC {
return fn.Func
}
case ir.OCLOSURE:
fn := fn.(*ir.ClosureExpr)
c := fn.Func
return c
}
return nil
}

148
src/cmd/compile/internal/test/pgo_inl_test.go Normal file
View File

@ -0,0 +1,148 @@
// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package test
import (
"bufio"
"fmt"
"internal/testenv"
"io"
"io/ioutil"
"os"
"os/exec"
"regexp"
"strings"
"testing"
)
// TestPGOIntendedInlining tests that specific functions are inlined.
func TestPGOIntendedInlining(t *testing.T) {
testenv.MustHaveGoRun(t)
t.Parallel()
// Make a temporary directory to work in.
tmpdir, err := ioutil.TempDir("", "TestCode")
if err != nil {
t.Fatalf("Failed to create temporary directory: %v", err)
}
defer os.RemoveAll(tmpdir)
want := map[string][]string{
"cmd/compile/internal/test/testdata/pgo/inline": {
"(*BS).NS",
},
}
// The functions which are not expected to be inlined are as follows.
wantNot := map[string][]string{
"cmd/compile/internal/test/testdata/pgo/inline": {
// The calling edge main->A is hot and the cost of A is large than
// inlineHotCalleeMaxBudget.
"A",
// The calling edge BenchmarkA" -> benchmarkB is cold
// and the cost of A is large than inlineMaxBudget.
"benchmarkB",
},
}
must := map[string]bool{
"(*BS).NS": true,
}
notInlinedReason := make(map[string]string)
pkgs := make([]string, 0, len(want))
for pname, fnames := range want {
pkgs = append(pkgs, pname)
for _, fname := range fnames {
fullName := pname + "." + fname
if _, ok := notInlinedReason[fullName]; ok {
t.Errorf("duplicate func: %s", fullName)
}
notInlinedReason[fullName] = "unknown reason"
}
}
// If the compiler emit "cannot inline for function A", the entry A
// in expectedNotInlinedList will be removed.
expectedNotInlinedList := make(map[string]struct{})
for pname, fnames := range wantNot {
for _, fname := range fnames {
fullName := pname + "." + fname
expectedNotInlinedList[fullName] = struct{}{}
}
}
// go test -bench=. -cpuprofile testdata/pgo/inline/inline_hot.pprof cmd/compile/internal/test/testdata/pgo/inline
curdir, err1 := os.Getwd()
if err1 != nil {
t.Fatal(err1)
}
gcflag_option := "-gcflags=-m -m -pgoprofile %s/testdata/pgo/inline/inline_hot.pprof"
gcflag := fmt.Sprintf(gcflag_option, curdir)
args := append([]string{"test", "-run=nope", gcflag}, pkgs...)
cmd := testenv.CleanCmdEnv(exec.Command(testenv.GoToolPath(t), args...))
pr, pw := io.Pipe()
cmd.Stdout = pw
cmd.Stderr = pw
cmdErr := make(chan error, 1)
go func() {
cmdErr <- cmd.Run()
pw.Close()
}()
scanner := bufio.NewScanner(pr)
curPkg := ""
canInline := regexp.MustCompile(`: can inline ([^ ]*)`)
haveInlined := regexp.MustCompile(`: inlining call to ([^ ]*)`)
cannotInline := regexp.MustCompile(`: cannot inline ([^ ]*): (.*)`)
for scanner.Scan() {
line := scanner.Text()
if strings.HasPrefix(line, "# ") {
curPkg = line[2:]
splits := strings.Split(curPkg, " ")
curPkg = splits[0]
continue
}
if m := haveInlined.FindStringSubmatch(line); m != nil {
fname := m[1]
delete(notInlinedReason, curPkg+"."+fname)
continue
}
if m := canInline.FindStringSubmatch(line); m != nil {
fname := m[1]
fullname := curPkg + "." + fname
// If function must be inlined somewhere, being inlinable is not enough
if _, ok := must[fullname]; !ok {
delete(notInlinedReason, fullname)
continue
}
}
if m := cannotInline.FindStringSubmatch(line); m != nil {
fname, reason := m[1], m[2]
fullName := curPkg + "." + fname
if _, ok := notInlinedReason[fullName]; ok {
// cmd/compile gave us a reason why
notInlinedReason[fullName] = reason
}
delete(expectedNotInlinedList, fullName)
continue
}
}
if err := <-cmdErr; err != nil {
t.Fatal(err)
}
if err := scanner.Err(); err != nil {
t.Fatal(err)
}
for fullName, reason := range notInlinedReason {
t.Errorf("%s was not inlined: %s", fullName, reason)
}
// If the list expectedNotInlinedList is not empty, it indicates
// the functions in the expectedNotInlinedList are marked with caninline.
for fullName, _ := range expectedNotInlinedList {
t.Errorf("%s was expected not inlined", fullName)
}
}

86
src/cmd/compile/internal/test/testdata/pgo/inline/inline_hot.go vendored Normal file
View File

@ -0,0 +1,86 @@
// Copyright 2022 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// WARNING: Please avoid updating this file. If this file needs to be updated, then a new inline_hot.pprof file should be generated via "go test -bench=. -cpuprofile testdata/pgo/inline/inline_hot.pprof cmd/compile/internal/test/testdata/pgo/inline".
package main
import (
"time"
)
type BS struct {
length uint
s []uint64
}
const wSize = uint(64)
const lWSize = uint(6)
func D(i uint) int {
return int((i + (wSize - 1)) >> lWSize)
}
func N(length uint) (bs *BS) {
bs = &BS{
length,
make([]uint64, D(length)),
}
return bs
}
func (b *BS) S(i uint) *BS {
b.s[i>>lWSize] |= 1 << (i & (wSize - 1))
return b
}
var jn = [...]byte{
0, 1, 56, 2, 57, 49, 28, 3, 61, 58, 42, 50, 38, 29, 17, 4,
62, 47, 59, 36, 45, 43, 51, 22, 53, 39, 33, 30, 24, 18, 12, 5,
63, 55, 48, 27, 60, 41, 37, 16, 46, 35, 44, 21, 52, 32, 23, 11,
54, 26, 40, 15, 34, 20, 31, 10, 25, 14, 19, 9, 13, 8, 7, 6,
}
func T(v uint64) uint {
return uint(jn[((v&-v)*0x03f79d71b4ca8b09)>>58])
}
func (b *BS) NS(i uint) (uint, bool) {
x := int(i >> lWSize)
if x >= len(b.s) {
return 0, false
}
w := b.s[x]
w = w >> (i & (wSize - 1))
if w != 0 {
return i + T(w), true
}
x = x + 1
for x < len(b.s) {
if b.s[x] != 0 {
return uint(x)*wSize + T(b.s[x]), true
}
x = x + 1
}
return 0, false
}
func A() {
s := N(100000)
for i := 0; i < 1000; i += 30 {
s.S(uint(i))
}
for j := 0; j < 1000; j++ {
c := uint(0)
for i, e := s.NS(0); e; i, e = s.NS(i + 1) {
c++
}
}
}
func main() {
time.Sleep(time.Second)
A()
}

BIN
src/cmd/compile/internal/test/testdata/pgo/inline/inline_hot.pprof vendored Normal file
View File

Binary file not shown.

47
src/cmd/compile/internal/test/testdata/pgo/inline/inline_hot_test.go vendored Normal file
View File

@ -0,0 +1,47 @@
// Copyright 2022 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// WARNING: Please avoid updating this file. If this file needs to be updated, then a new inline_hot.pprof file should be generated via "go test -bench=. -cpuprofile testdata/pgo/inline/inline_hot.pprof cmd/compile/internal/test/testdata/pgo/inline".
package main
import "testing"
func BenchmarkA(b *testing.B) {
benchmarkB(b)
}
func benchmarkB(b *testing.B) {
for i := 0; true; {
A()
i = i + 1
if i >= b.N {
break
}
A()
i = i + 1
if i >= b.N {
break
}
A()
i = i + 1
if i >= b.N {
break
}
A()
i = i + 1
if i >= b.N {
break
}
A()
i = i + 1
if i >= b.N {
break
}
A()
i = i + 1
if i >= b.N {
break
}
}
}

1
src/cmd/dist/buildtool.go vendored
View File

@ -66,6 +66,7 @@ var bootstrapDirs = []string{
"internal/goroot",
"internal/goversion",
"internal/pkgbits",
"internal/profile",
"internal/race",
"internal/saferio",
"internal/platform",

16
src/internal/profile/legacy_profile.go
View File

@ -722,7 +722,7 @@ func parseCppContention(r *bytes.Buffer) (*Profile, error) {
var l string
var err error
// Parse text of the form "attribute = value" before the samples.
const delimiter = "="
const delimiter = '='
for {
l, err = r.ReadString('\n')
if err != nil {
@ -746,10 +746,13 @@ func parseCppContention(r *bytes.Buffer) (*Profile, error) {
break
}
key, val, ok := strings.Cut(l, delimiter)
if !ok {
index := strings.IndexByte(l, delimiter)
if index < 0 {
break
}
key := l[:index]
val := l[index+1:]
key, val = strings.TrimSpace(key), strings.TrimSpace(val)
var err error
switch key {
@ -1023,7 +1026,7 @@ func (p *Profile) ParseMemoryMap(rd io.Reader) error {
var attrs []string
var r *strings.Replacer
const delimiter = "="
const delimiter = '='
for {
l, err := b.ReadString('\n')
if err != nil {
@ -1046,7 +1049,10 @@ func (p *Profile) ParseMemoryMap(rd io.Reader) error {
if err == errUnrecognized {
// Recognize assignments of the form: attr=value, and replace
// $attr with value on subsequent mappings.
if attr, value, ok := strings.Cut(l, delimiter); ok {
idx := strings.IndexByte(l, delimiter)
if idx >= 0 {
attr := l[:idx]
value := l[idx+1:]
attrs = append(attrs, "$"+strings.TrimSpace(attr), strings.TrimSpace(value))
r = strings.NewReplacer(attrs...)
}