go.tools/cmd/godex: support for common path prefixes

All of these work now as expected:

        godex code.google.com/p/go.tools/go/types
        godex go.tools/go/types
        godex go/types
        godex types

Also improved logging/verbose mode.

LGTM=adonovan
R=adonovan
CC=golang-codereviews
https://golang.org/cl/80930043

cmd/godex/doc.go

@@ -9,18 +9,35 @@
 // object files. Hence the exported data is truly what a compiler will
 // see, at the cost of missing commentary.
 //
-// Usage: godex [flags] {path|qualifiedIdent}
+// Usage: godex [flags] {path[.name]}
 //
-// Each argument must be a package path, or a qualified identifier.
+// Each argument must be a (possibly partial) package path, optionally
+// followed by a dot and the name of a package object:
+//
+//	godex math
+//	godex math.Sin
+//	godex math.Sin fmt.Printf
+//      godex go/types
+//
+// All but the last path element may contain dots. godex automatically
+// tries all possible package path prefixes for non-standard library
+// packages if only a partial package path is given. For instance, for
+// the path "go/types", godex prepends "code.google.com/p/go.tools".
+//
+// The prefixes are computed by searching the directories specified by
+// the GOPATH environment variable (and by excluding the build os and
+// architecture specific directory names from the path). The search
+// order is depth-first and alphabetic; for a partial path "foo", a
+// package "a/foo" is found before "b/foo".
 //
 // The flags are:
 //
-//	-s=src
+//	-s=""
 //		only consider packages from src, where src is one of the supported compilers
-//	-v
+//	-v=false
 //		verbose mode
 //
-// The following sources are supported:
+// The following sources (-s arguments) are supported:
 //
 //	gc
 //		gc-generated object files
@@ -33,6 +50,6 @@
 //
 // If no -s argument is provided, godex will try to find a matching source.
 //
-// TODO(gri) expand this documentation
-//
 package main
+
+// BUG(gri) std-library packages should also benefit from auto-generated prefixes.

cmd/godex/gc.go

@@ -11,5 +11,5 @@ import (
 )
 
 func init() {
-	register("gc", protect(gcimporter.Import))
+	register("gc", gcimporter.Import)
 }

cmd/godex/gccgo.go

@@ -21,10 +21,10 @@ func init() {
 	// importer for default gccgo
 	var inst gccgoimporter.GccgoInstallation
 	inst.InitFromDriver("gccgo")
-	register("gccgo", protect(inst.GetImporter(nil)))
+	register("gccgo", inst.GetImporter(nil))
 
 	// importer for gccgo using condensed export format (experimental)
-	register("gccgo-new", protect(gccgoNewImporter))
+	register("gccgo-new", gccgoNewImporter)
 }
 
 func gccgoNewImporter(packages map[string]*types.Package, path string) (*types.Package, error) {

cmd/godex/godex.go

@@ -8,6 +8,8 @@ import (
 	"errors"
 	"flag"
 	"fmt"
+	"go/build"
+	"io/ioutil"
 	"os"
 	"path/filepath"
 	"strings"
@@ -16,17 +18,20 @@ import (
 )
 
 var (
-	source  = flag.String("s", "", "only consider packages from this source")
+	source  = flag.String("s", "", "only consider packages from src, where src is one of the supported compilers")
 	verbose = flag.Bool("v", false, "verbose mode")
 )
 
+// lists of registered sources and corresponding importers
 var (
-	sources      []string         // sources of export data corresponding to importers
+	sources      []string
-	importers    []types.Importer // importers for corresponding sources
+	importers    []types.Importer
 	importFailed = errors.New("import failed")
-	packages     = make(map[string]*types.Package)
 )
 
+// map of imported packages
+var packages = make(map[string]*types.Package)
+
 func usage() {
 	fmt.Fprintln(os.Stderr, "usage: godex [flags] {path|qualifiedIdent}")
 	flag.PrintDefaults()
@@ -46,7 +51,7 @@ func main() {
 		report("no package name, path, or file provided")
 	}
 
-	imp := tryImport
+	imp := tryImports
 	if *source != "" {
 		imp = lookup(*source)
 		if imp == nil {
@@ -56,14 +61,17 @@ func main() {
 
 	for _, arg := range flag.Args() {
 		path, name := splitPathIdent(arg)
-		if *verbose {
+		logf("\tprocessing %q: path = %q, name = %s\n", arg, path, name)
-			fmt.Fprintf(os.Stderr, "(processing %q: path = %q, name = %s)\n", arg, path, name)
+
-		}
+		// generate possible package path prefixes
+		// (at the moment we do this for each argument - should probably cache this)
+		prefixes := make(chan string)
+		go genPrefixes(prefixes)
 
 		// import package
-		pkg, err := imp(packages, path)
+		pkg, err := tryPrefixes(packages, prefixes, path, imp)
 		if err != nil {
-			fmt.Fprintf(os.Stderr, "ignoring %q: %s\n", path, err)
+			logf("\t=> ignoring %q: %s\n", path, err)
 			continue
 		}
 
@@ -82,6 +90,57 @@ func main() {
 	}
 }
 
+func logf(format string, args ...interface{}) {
+	if *verbose {
+		fmt.Fprintf(os.Stderr, format, args...)
+	}
+}
+
+// splitPathIdent splits a path.name argument into its components.
+// All but the last path element may contain dots.
+func splitPathIdent(arg string) (path, name string) {
+	if i := strings.LastIndex(arg, "."); i >= 0 {
+		if j := strings.LastIndex(arg, "/"); j < i {
+			// '.' is not part of path
+			path = arg[:i]
+			name = arg[i+1:]
+			return
+		}
+	}
+	path = arg
+	return
+}
+
+// tryPrefixes tries to import the package given by (the possibly partial) path using the given importer imp
+// by prepending all possible prefixes to path. It returns with the first package that it could import, or
+// with an error.
+func tryPrefixes(packages map[string]*types.Package, prefixes chan string, path string, imp types.Importer) (pkg *types.Package, err error) {
+	for prefix := range prefixes {
+		logf("\ttrying prefix %q\n", prefix)
+		prepath := filepath.Join(prefix, path)
+		pkg, err = imp(packages, prepath)
+		if err == nil {
+			break
+		}
+		logf("\t=> importing %q failed: %s\n", prepath, err)
+	}
+	return
+}
+
+// tryImports is an importer that tries all registered importers
+// successively until one of them succeeds or all of them failed.
+func tryImports(packages map[string]*types.Package, path string) (pkg *types.Package, err error) {
+	for i, imp := range importers {
+		logf("\t\ttrying %s import\n", sources[i])
+		pkg, err = imp(packages, path)
+		if err == nil {
+			break
+		}
+		logf("\t\t=> %s import failed: %s\n", sources[i], err)
+	}
+	return
+}
+
 // protect protects an importer imp from panics and returns the protected importer.
 func protect(imp types.Importer) types.Importer {
 	return func(packages map[string]*types.Package, path string) (pkg *types.Package, err error) {
@@ -95,44 +154,16 @@ func protect(imp types.Importer) types.Importer {
 	}
 }
 
-func tryImport(packages map[string]*types.Package, path string) (pkg *types.Package, err error) {
+// register registers an importer imp for a given source src.
-	for i, imp := range importers {
-		if *verbose {
-			fmt.Fprintf(os.Stderr, "(trying source: %s)\n", sources[i])
-		}
-		pkg, err = imp(packages, path)
-		if err == nil {
-			break
-		}
-	}
-	return
-}
-
-// splitPathIdent splits a path.name argument into its components.
-// All but the last path element may contain dots.
-// TODO(gri) document this also in doc.go.
-func splitPathIdent(arg string) (path, name string) {
-	const sep = string(filepath.Separator)
-	if i := strings.LastIndex(arg, "."); i >= 0 {
-		if j := strings.LastIndex(arg, sep); j < i {
-			// '.' is not part of path
-			path = arg[:i]
-			name = arg[i+1:]
-			return
-		}
-	}
-	path = arg
-	return
-}
-
 func register(src string, imp types.Importer) {
 	if lookup(src) != nil {
 		panic(src + " importer already registered")
 	}
 	sources = append(sources, src)
-	importers = append(importers, imp)
+	importers = append(importers, protect(imp))
 }
 
+// lookup returns the importer imp for a given source src.
 func lookup(src string) types.Importer {
 	for i, s := range sources {
 		if s == src {
@@ -141,3 +172,26 @@ func lookup(src string) types.Importer {
 	}
 	return nil
 }
+
+func genPrefixes(out chan string) {
+	out <- "" // try no prefix
+	platform := build.Default.GOOS + "_" + build.Default.GOARCH
+	for _, dirname := range filepath.SplitList(build.Default.GOPATH) {
+		walkDir(filepath.Join(dirname, "pkg", platform), "", out)
+	}
+	close(out)
+}
+
+func walkDir(dirname, prefix string, out chan string) {
+	fiList, err := ioutil.ReadDir(dirname)
+	if err != nil {
+		return
+	}
+	for _, fi := range fiList {
+		if fi.IsDir() && !strings.HasPrefix(fi.Name(), ".") {
+			prefix := filepath.Join(prefix, fi.Name())
+			out <- prefix
+			walkDir(filepath.Join(dirname, fi.Name()), prefix, out)
+		}
+	}
+}

cmd/godex/print.go

@@ -85,7 +85,7 @@ func (p *printer) printPackage(pkg *types.Package, filter func(types.Object) boo
 		}
 	}
 
-	p.printf("package %s\n\n", pkg.Name())
+	p.printf("package %s  // %q\n\n", pkg.Name(), pkg.Path())
 
 	if len(consts) > 0 {
 		p.print("const (\n")

cmd/godex/source.go

@@ -11,7 +11,7 @@ import (
 )
 
 func init() {
-	register("source", protect(sourceImporter))
+	register("source", sourceImporter)
 }
 
 func sourceImporter(packages map[string]*types.Package, path string) (*types.Package, error) {