go.tools/ssa: add test of SSA construction on $GOROOT/src/pkg/...

stdlib_test runs the builder (in sanity-checking mode) over
the Go standard library.  It also prints some stats about
the time and memory usage.

Also:
- importer.LoadPackage too (not just doImport) must consult
  the cache to avoid creating duplicate Package instances for
  the same import path when called serially from a test.
- importer: skip empty directories without an error.
- importer: print all errors, not just the first.
- visit.go: added AllFunctions utility for enumerating all
  Functions in a Program.
- ssa.MethodSet is not safe to expose from the package since
  it must be accessed under an (inaccessible) lock.  (!!!)
  This CL makes it unexported and restricts its use to the
  single function Program.LookupMethod().
- Program.MethodSet() has gone.
  Clients should instead iterate over the types.MethodSet
  and call LookupMethod.
- Package.DumpTo(): improved efficiency of methodset printing
  (by not creating wrappers) and accuracy (by showing * on
  receiver type only when necessary).
- Program.CreatePackage: documented precondition and added
  assertion.

R=gri
CC=golang-dev
https://golang.org/cl/12058048

importer/importer.go

@@ -8,8 +8,10 @@
 package importer
 
 import (
+	"fmt"
 	"go/ast"
 	"go/token"
+	"os"
 
 	"code.google.com/p/go.tools/go/exact"
 	"code.google.com/p/go.tools/go/types"
@@ -28,8 +30,8 @@ type Importer struct {
 type Config struct {
 	// TypeChecker contains options relating to the type checker.
 	// The Importer will override any user-supplied values for its
-	// Expr, Ident, ImplicitObj and Import fields; other fields
-	// will be passed through to the type checker.
+	// Error and Import fields; other fields will be passed
+	// through to the type checker.
 	TypeChecker types.Config
 
 	// If Loader is non-nil, it is used to satisfy imports.
@@ -68,6 +70,7 @@ func New(config *Config) *Importer {
 		Packages: make(map[string]*PackageInfo),
 		errors:   make(map[string]error),
 	}
+	imp.config.TypeChecker.Error = func(e error) { fmt.Fprintln(os.Stderr, e) }
 	imp.config.TypeChecker.Import = imp.doImport
 	return imp
 }
@@ -81,25 +84,27 @@ func (imp *Importer) doImport(imports map[string]*types.Package, path string) (p
 		return types.Unsafe, nil
 	}
 
-	if info, ok := imp.Packages[path]; ok {
-		imports[path] = info.Pkg
-		pkg = info.Pkg
-		return // positive cache hit
-	}
-
-	if err = imp.errors[path]; err != nil {
-		return // negative cache hit
-	}
-
 	// Load the source/binary for 'path', type-check it, construct
 	// a PackageInfo and update our map (imp.Packages) and the
 	// type-checker's map (imports).
 	var info *PackageInfo
 	if imp.config.Loader != nil {
 		info, err = imp.LoadPackage(path)
-	} else if pkg, err = types.GcImport(imports, path); err == nil {
-		info = &PackageInfo{Pkg: pkg}
-		imp.Packages[path] = info
+	} else {
+		if info, ok := imp.Packages[path]; ok {
+			imports[path] = info.Pkg
+			pkg = info.Pkg
+			return // positive cache hit
+		}
+
+		if err = imp.errors[path]; err != nil {
+			return // negative cache hit
+		}
+
+		if pkg, err = types.GcImport(imports, path); err == nil {
+			info = &PackageInfo{Pkg: pkg}
+			imp.Packages[path] = info
+		}
 	}
 
 	if err == nil {
@@ -124,8 +129,15 @@ func (imp *Importer) doImport(imports map[string]*types.Package, path string) (p
 // Not thread-safe!
 // TODO(adonovan): rethink this API.
 //
-//
 func (imp *Importer) LoadPackage(importPath string) (*PackageInfo, error) {
+	if info, ok := imp.Packages[importPath]; ok {
+		return info, nil // positive cache hit
+	}
+
+	if err := imp.errors[importPath]; err != nil {
+		return nil, err // negative cache hit
+	}
+
 	if imp.config.Loader == nil {
 		panic("Importer.LoadPackage without a SourceLoader")
 	}

importer/util.go

@@ -73,6 +73,9 @@ func MakeGoBuildLoader(ctxt *build.Context) SourceLoader {
 		// TODO(adonovan): fix: Do we need cwd? Shouldn't
 		// ImportDir(path) / $GOROOT suffice?
 		bp, err := ctxt.Import(path, srcDir, 0)
+		if _, ok := err.(*build.NoGoError); ok {
+			return nil, nil // empty directory
+		}
 		if err != nil {
 			return // import failed
 		}

ssa/builder.go

@@ -2257,6 +2257,10 @@ func (prog *Program) BuildAll() {
 
 // Build builds SSA code for all functions and vars in package p.
 //
+// Precondition: CreatePackage must have been called for all of p's
+// direct imports (and hence its direct imports must have been
+// error-free).
+//
 // Build is idempotent and thread-safe.
 //
 func (p *Package) Build() {
@@ -2283,8 +2287,12 @@ func (p *Package) Build() {
 
 	// Call the init() function of each package we import.
 	for _, obj := range p.info.Imports() {
+		prereq := p.Prog.packages[obj]
+		if prereq == nil {
+			panic(fmt.Sprintf("Package(%q).Build(): unsatisified import: Program.CreatePackage(%q) was not called", p.Object.Path(), obj.Path()))
+		}
 		var v Call
-		v.Call.Value = p.Prog.packages[obj].init
+		v.Call.Value = prereq.init
 		v.Call.pos = init.pos
 		v.setType(types.NewTuple())
 		init.emit(&v)

ssa/builder_test.go

@@ -93,7 +93,14 @@ func main() {
 				if !isExt {
 					t.Fatalf("unexpected name type in main package: %s", mem)
 				}
-				for _, m := range prog.MethodSet(types.NewPointer(mem.Type())) {
+				if _, ok := mem.Type().Underlying().(*types.Interface); ok {
+					// TODO(adonovan): workaround bug in types.MethodSet
+					// whereby mset(*I) is nonempty if I is an interface.
+					continue
+				}
+				mset := types.NewPointer(mem.Type()).MethodSet()
+				for i, n := 0, mset.Len(); i < n; i++ {
+					m := prog.LookupMethod(mset.At(i))
 					// For external types, only synthetic wrappers have code.
 					expExt := !strings.Contains(m.Synthetic, "wrapper")
 					if expExt && !isEmpty(m) {

ssa/interp/interp.go

@@ -79,14 +79,16 @@ const (
 	EnableTracing                   // Print a trace of all instructions as they are interpreted.
 )
 
+type methodSet map[string]*ssa.Function
+
 // State shared between all interpreted goroutines.
 type interpreter struct {
 	prog           *ssa.Program         // the SSA program
 	globals        map[ssa.Value]*value // addresses of global variables (immutable)
 	mode           Mode                 // interpreter options
 	reflectPackage *ssa.Package         // the fake reflect package
-	errorMethods   ssa.MethodSet        // the method set of reflect.error, which implements the error interface.
-	rtypeMethods   ssa.MethodSet        // the method set of rtype, which implements the reflect.Type interface.
+	errorMethods   methodSet            // the method set of reflect.error, which implements the error interface.
+	rtypeMethods   methodSet            // the method set of rtype, which implements the reflect.Type interface.
 }
 
 type frame struct {

ssa/interp/reflect.go

@@ -408,7 +408,7 @@ func initReflect(i *interpreter) {
 		Members: make(map[string]ssa.Member),
 	}
 
-	i.rtypeMethods = ssa.MethodSet{
+	i.rtypeMethods = methodSet{
 		"Bits":   newMethod(i.reflectPackage, rtypeType, "Bits"),
 		"Elem":   newMethod(i.reflectPackage, rtypeType, "Elem"),
 		"Kind":   newMethod(i.reflectPackage, rtypeType, "Kind"),
@@ -416,7 +416,7 @@ func initReflect(i *interpreter) {
 		"Out":    newMethod(i.reflectPackage, rtypeType, "Out"),
 		"String": newMethod(i.reflectPackage, rtypeType, "String"),
 	}
-	i.errorMethods = ssa.MethodSet{
+	i.errorMethods = methodSet{
 		"Error": newMethod(i.reflectPackage, errorType, "Error"),
 	}
 }

ssa/print.go

@@ -379,31 +379,22 @@ func (p *Package) DumpTo(w io.Writer) {
 
 		case *Type:
 			fmt.Fprintf(w, "  type  %-*s %s\n", maxname, name, mem.Type().Underlying())
-			// We display only mset(*T) since its keys
-			// are a superset of mset(T)'s keys, though the
-			// methods themselves may differ,
-			// e.g. promotion wrappers.
-			// NB: if mem.Type() is a pointer, mset is empty.
-			//
-			// TODO(adonovan): opt: avoid constructing the
-			// entire ssa.MethodSet by using the
-			// types.MethodSet if possible.
-			mset := p.Prog.MethodSet(types.NewPointer(mem.Type()))
-			var keys []string
-			for id := range mset {
-				keys = append(keys, id)
-			}
-			sort.Strings(keys)
-			for _, id := range keys {
-				method := mset[id]
-				// TODO(adonovan): show pointerness of receiver of declared method, not the index
-
-				fmt.Fprintf(w, "    method %s %s\n", id, method.Signature)
+			// Iterate over the keys of mset(*T) since they
+			// are a superset of mset(T)'s keys.
+			// The keys of a types.MethodSet are sorted (by Id).
+			mset := methodSetOf(mem.Type())
+			pmset := methodSetOf(types.NewPointer(mem.Type()))
+			for i, n := 0, pmset.Len(); i < n; i++ {
+				meth := pmset.At(i)
+				// If the method also exists in mset(T), show that instead.
+				if m := mset.Lookup(meth.Obj().Pkg(), meth.Obj().Name()); m != nil {
+					meth = m
+				}
+				fmt.Fprintf(w, "    %s\n", meth)
 			}
 
 		case *Global:
 			fmt.Fprintf(w, "  var   %-*s %s\n", maxname, name, mem.Type())
-
 		}
 	}
 }

ssa/promote.go

@@ -22,81 +22,7 @@ func recvType(obj *types.Func) types.Type {
 	return obj.Type().(*types.Signature).Recv().Type()
 }
 
-// MethodSet returns the method set for type typ, building wrapper
-// methods as needed for embedded field promotion, and indirection for
-// *T receiver types, etc.
-// A nil result indicates an empty set.
-//
-// This function should only be called when you need to construct the
-// entire method set, synthesizing all wrappers, for example during
-// the processing of a MakeInterface instruction or when visiting all
-// reachable functions.
-//
-// If you only need to look up a single method (obj), avoid this
-// function and use LookupMethod instead:
-//
-//      meth := types.MethodSet(typ).Lookup(pkg, name)
-// 	m := prog.MethodSet(typ)[meth.Id()]   // don't do this
-//	m := prog.LookupMethod(meth)          // use this instead
-//
-// If you only need to enumerate the keys, use types.MethodSet
-// instead.
-//
-// EXCLUSIVE_LOCKS_ACQUIRED(prog.methodsMu)
-//
-// Thread-safe.
-//
-func (prog *Program) MethodSet(typ types.Type) MethodSet {
-	return prog.populateMethodSet(typ, nil)
-}
-
-// populateMethodSet returns the method set for typ, ensuring that it
-// contains at least the function for meth, if that is a key.
-// If meth is nil, the entire method set is populated.
-//
-// EXCLUSIVE_LOCKS_ACQUIRED(prog.methodsMu)
-//
-func (prog *Program) populateMethodSet(typ types.Type, meth *types.Selection) MethodSet {
-	tmset := methodSet(typ)
-	n := tmset.Len()
-	if n == 0 {
-		return nil
-	}
-
-	if prog.mode&LogSource != 0 {
-		defer logStack("populateMethodSet %s meth=%v", typ, meth)()
-	}
-
-	prog.methodsMu.Lock()
-	defer prog.methodsMu.Unlock()
-
-	mset, _ := prog.methodSets.At(typ).(MethodSet)
-	if mset == nil {
-		mset = make(MethodSet)
-		prog.methodSets.Set(typ, mset)
-	}
-
-	if len(mset) < n {
-		if meth != nil { // single method
-			id := meth.Obj().Id()
-			if mset[id] == nil {
-				mset[id] = findMethod(prog, meth)
-			}
-		} else {
-			// complete set
-			for i := 0; i < n; i++ {
-				meth := tmset.At(i)
-				if id := meth.Obj().Id(); mset[id] == nil {
-					mset[id] = findMethod(prog, meth)
-				}
-			}
-		}
-	}
-
-	return mset
-}
-
-func methodSet(typ types.Type) *types.MethodSet {
+func methodSetOf(typ types.Type) *types.MethodSet {
 	// TODO(adonovan): temporary workaround.  Inline it away when fixed.
 	if _, ok := deref(typ).Underlying().(*types.Interface); ok && isPointer(typ) {
 		// TODO(gri): fix: go/types bug: pointer-to-interface
@@ -115,7 +41,31 @@ func methodSet(typ types.Type) *types.MethodSet {
 // EXCLUSIVE_LOCKS_ACQUIRED(prog.methodsMu)
 //
 func (prog *Program) LookupMethod(meth *types.Selection) *Function {
-	return prog.populateMethodSet(meth.Recv(), meth)[meth.Obj().Id()]
+	if meth == nil {
+		panic("LookupMethod(nil)")
+	}
+	typ := meth.Recv()
+	if prog.mode&LogSource != 0 {
+		defer logStack("LookupMethod %s %v", typ, meth)()
+	}
+
+	prog.methodsMu.Lock()
+	defer prog.methodsMu.Unlock()
+
+	type methodSet map[string]*Function
+	mset, _ := prog.methodSets.At(typ).(methodSet)
+	if mset == nil {
+		mset = make(methodSet)
+		prog.methodSets.Set(typ, mset)
+	}
+
+	id := meth.Obj().Id()
+	fn := mset[id]
+	if fn == nil {
+		fn = findMethod(prog, meth)
+		mset[id] = fn
+	}
+	return fn
 }
 
 // declaredFunc returns the concrete function/method denoted by obj.

ssa/source.go

@@ -1,6 +1,7 @@
 package ssa
 
-// This file defines utilities for working with source positions.
+// This file defines utilities for working with source positions
+// or source-level named entities ("objects").
 
 import (
 	"go/ast"
@@ -100,14 +101,12 @@ func findNamedFunc(pkg *Package, pos token.Pos) *Function {
 				return mem
 			}
 		case *Type:
-			for _, meth := range pkg.Prog.MethodSet(mem.Type()) {
-				if meth.Synthetic == "" && meth.Pos() == pos {
-					return meth
-				}
-			}
-			for _, meth := range pkg.Prog.MethodSet(types.NewPointer(mem.Type())) {
-				if meth.Synthetic == "" && meth.Pos() == pos {
-					return meth
+			mset := methodSetOf(types.NewPointer(mem.Type()))
+			for i, n := 0, mset.Len(); i < n; i++ {
+				// Don't call LookupMethod: avoid creating wrappers.
+				obj := mset.At(i).Obj().(*types.Func)
+				if obj.Pos() == pos {
+					return pkg.values[obj].(*Function)
 				}
 			}
 		}

ssa/ssa.go

@@ -25,7 +25,7 @@ type Program struct {
 	mode           BuilderMode                 // set of mode bits for SSA construction
 
 	methodsMu           sync.Mutex                // guards the following maps:
-	methodSets          typemap.M                 // maps type to its concrete MethodSet
+	methodSets          typemap.M                 // maps type to its concrete methodSet
 	boundMethodWrappers map[*types.Func]*Function // wrappers for curried x.Method closures
 	ifaceMethodWrappers map[*types.Func]*Function // wrappers for curried I.Method functions
 }
@@ -61,22 +61,6 @@ type Member interface {
 	Token() token.Token   // token.{VAR,FUNC,CONST,TYPE}
 }
 
-// A MethodSet contains all the methods for a particular type T.
-// The method sets for T and *T are distinct entities.
-//
-// All methods in the method set for T have a receiver type of exactly
-// T.  The method set of *T may contain synthetic indirection methods
-// that wrap methods whose receiver type is T.
-//
-// The keys of a method set are strings returned by the types.Id()
-// function.
-//
-// TODO(adonovan): encapsulate the representation behind both Id-based
-// and types.Method-based accessors and enable lazy population.
-// Perhaps hide it entirely within the Program API.
-//
-type MethodSet map[string]*Function
-
 // A Type is a Member of a Package representing a package-level named type.
 //
 // Type() returns a *types.Named.
@@ -599,7 +583,8 @@ type ChangeInterface struct {
 // MakeInterface constructs an instance of an interface type from a
 // value of a concrete type.
 //
-// Use Program.MethodSet(X.Type()) to find the method-set of X.
+// Use X.Type().MethodSet() to find the method-set of X, and
+// Program.LookupMethod(m) to find the implementation of a method.
 //
 // To construct the zero value of an interface type T, use:
 // 	NewConst(exact.MakeNil(), T, pos)

ssa/stdlib_test.go

@@ -0,0 +1,118 @@
+package ssa_test
+
+// This file runs the SSA builder in sanity-checking mode on all
+// packages beneath $GOROOT and prints some summary information.
+//
+// Run test with GOMAXPROCS=8 and CGO_ENABLED=0.  The latter cannot be
+// set from the test because it's too late to stop go/build.init()
+// from picking up the value from the parent's environment.
+
+import (
+	"go/build"
+	"os"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"testing"
+	"time"
+
+	"code.google.com/p/go.tools/importer"
+	"code.google.com/p/go.tools/ssa"
+)
+
+func allPackages() []string {
+	var pkgs []string
+	root := filepath.Join(runtime.GOROOT(), "src/pkg") + "/"
+	filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
+		// Prune the search if we encounter any of these names:
+		switch filepath.Base(path) {
+		case "testdata", ".hg":
+			return filepath.SkipDir
+		}
+		if info.IsDir() {
+			pkg := strings.TrimPrefix(path, root)
+			switch pkg {
+			case "builtin", "pkg", "code.google.com":
+				return filepath.SkipDir // skip these subtrees
+			case "":
+				return nil // ignore root of tree
+			}
+			pkgs = append(pkgs, pkg)
+		}
+
+		return nil
+	})
+	return pkgs
+}
+
+func TestStdlib(t *testing.T) {
+	ctxt := build.Default
+	ctxt.CgoEnabled = false
+	impctx := importer.Config{Loader: importer.MakeGoBuildLoader(&ctxt)}
+
+	// Load, parse and type-check the program.
+	t0 := time.Now()
+
+	var hasErrors bool
+	imp := importer.New(&impctx)
+	for _, importPath := range allPackages() {
+		if _, err := imp.LoadPackage(importPath); err != nil {
+			t.Errorf("LoadPackage(%s): %s", importPath, err)
+			hasErrors = true
+		}
+	}
+
+	t1 := time.Now()
+
+	runtime.GC()
+	var memstats runtime.MemStats
+	runtime.ReadMemStats(&memstats)
+	alloc := memstats.Alloc
+
+	// Create SSA packages.
+	prog := ssa.NewProgram(imp.Fset, ssa.DebugInfo|ssa.SanityCheckFunctions)
+	for _, info := range imp.Packages {
+		if info.Err == nil {
+			prog.CreatePackage(info)
+		}
+	}
+
+	t2 := time.Now()
+
+	// Build SSA IR... if it's safe.
+	if !hasErrors {
+		prog.BuildAll()
+	}
+
+	t3 := time.Now()
+
+	runtime.GC()
+	runtime.ReadMemStats(&memstats)
+
+	numPkgs := len(prog.PackagesByPath)
+	if want := 140; numPkgs < want {
+		t.Errorf("Loaded only %d packages, want at least %d", numPkgs, want)
+	}
+
+	// Dump some statistics.
+	allFuncs := ssa.AllFunctions(prog)
+	var numInstrs int
+	for fn := range allFuncs {
+		for _, b := range fn.Blocks {
+			numInstrs += len(b.Instrs)
+		}
+	}
+
+	t.Log("GOMAXPROCS:           ", runtime.GOMAXPROCS(0))
+	t.Log("Load/parse/typecheck: ", t1.Sub(t0))
+	t.Log("SSA create:           ", t2.Sub(t1))
+	if !hasErrors {
+		t.Log("SSA build:            ", t3.Sub(t2))
+	}
+
+	// SSA stats:
+	t.Log("#Packages:            ", numPkgs)
+	t.Log("#Functions:           ", len(allFuncs))
+	t.Log("#Instructions:        ", numInstrs)
+	t.Log("#MB:                  ", (memstats.Alloc-alloc)/1000000)
+}

ssa/visit.go

@@ -0,0 +1,73 @@
+package ssa
+
+// This file defines utilities for visiting the SSA representation of
+// a Program.
+//
+// TODO(adonovan): improve the API:
+// - permit client to supply a callback for each function,
+//   instruction, type with methods, etc?
+// - return graph information about the traversal?
+// - test coverage.
+
+import "code.google.com/p/go.tools/go/types"
+
+// AllFunctions returns the set of all functions (including anonymous
+// functions and synthetic wrappers) in program prog.
+//
+// Precondition: all packages are built.
+//
+func AllFunctions(prog *Program) map[*Function]bool {
+	visit := visitor{
+		prog: prog,
+		seen: make(map[*Function]bool),
+	}
+	visit.program()
+	return visit.seen
+}
+
+type visitor struct {
+	prog *Program
+	seen map[*Function]bool
+}
+
+func (visit *visitor) program() {
+	for _, pkg := range visit.prog.PackagesByPath {
+		for _, mem := range pkg.Members {
+			switch mem := mem.(type) {
+			case *Function:
+				visit.function(mem)
+			case *Type:
+				visit.methodSet(mem.Type())
+				visit.methodSet(types.NewPointer(mem.Type()))
+			}
+		}
+	}
+}
+
+func (visit *visitor) methodSet(typ types.Type) {
+	mset := methodSetOf(typ)
+	for i, n := 0, mset.Len(); i < n; i++ {
+		// Side-effect: creates all wrapper methods.
+		visit.function(visit.prog.LookupMethod(mset.At(i)))
+	}
+}
+
+func (visit *visitor) function(fn *Function) {
+	if !visit.seen[fn] {
+		visit.seen[fn] = true
+		for _, b := range fn.Blocks {
+			for _, instr := range b.Instrs {
+				switch instr := instr.(type) {
+				case *MakeInterface:
+					visit.methodSet(instr.X.Type())
+				}
+				var buf [10]*Value // avoid alloc in common case
+				for _, op := range instr.Operands(buf[:0]) {
+					if fn, ok := (*op).(*Function); ok {
+						visit.function(fn)
+					}
+				}
+			}
+		}
+	}
+}