go/gcimporter15: a copy of go/gcimporter using go1.5 std go/types

Because the std go/importer's Lookup functionality is incomplete,
clients have no way to create a Go 1.5 types.Package from an io.Reader
reading from gc export data.  This package provides a stopgap until
the standard library is complete.

The go/gcimporter package remains unchanged, and uses only the
golang.org/x/tools/go/types package.

Change-Id: I47a817f4b6a52ddab26c6b01de6e28099301faf5
Reviewed-on: https://go-review.googlesource.com/18382
Reviewed-by: Robert Griesemer <gri@golang.org>

go/gcimporter15/exportdata.go

@@ -0,0 +1,112 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build go1.5
+
+// This file is a copy of go/gcimporter/exportdata.go, tagged for go1.5.
+
+// This file implements FindExportData.
+
+package gcimporter
+
+import (
+	"bufio"
+	"errors"
+	"fmt"
+	"io"
+	"strconv"
+	"strings"
+)
+
+func readGopackHeader(r *bufio.Reader) (name string, size int, err error) {
+	// See $GOROOT/include/ar.h.
+	hdr := make([]byte, 16+12+6+6+8+10+2)
+	_, err = io.ReadFull(r, hdr)
+	if err != nil {
+		return
+	}
+	// leave for debugging
+	if false {
+		fmt.Printf("header: %s", hdr)
+	}
+	s := strings.TrimSpace(string(hdr[16+12+6+6+8:][:10]))
+	size, err = strconv.Atoi(s)
+	if err != nil || hdr[len(hdr)-2] != '`' || hdr[len(hdr)-1] != '\n' {
+		err = errors.New("invalid archive header")
+		return
+	}
+	name = strings.TrimSpace(string(hdr[:16]))
+	return
+}
+
+// FindExportData positions the reader r at the beginning of the
+// export data section of an underlying GC-created object/archive
+// file by reading from it. The reader must be positioned at the
+// start of the file before calling this function.
+//
+func FindExportData(r *bufio.Reader) (err error) {
+	// Read first line to make sure this is an object file.
+	line, err := r.ReadSlice('\n')
+	if err != nil {
+		return
+	}
+	if string(line) == "!<arch>\n" {
+		// Archive file. Scan to __.PKGDEF.
+		var name string
+		var size int
+		if name, size, err = readGopackHeader(r); err != nil {
+			return
+		}
+
+		// Optional leading __.GOSYMDEF or __.SYMDEF.
+		// Read and discard.
+		if name == "__.SYMDEF" || name == "__.GOSYMDEF" {
+			const block = 4096
+			tmp := make([]byte, block)
+			for size > 0 {
+				n := size
+				if n > block {
+					n = block
+				}
+				if _, err = io.ReadFull(r, tmp[:n]); err != nil {
+					return
+				}
+				size -= n
+			}
+
+			if name, size, err = readGopackHeader(r); err != nil {
+				return
+			}
+		}
+
+		// First real entry should be __.PKGDEF.
+		if name != "__.PKGDEF" {
+			err = errors.New("go archive is missing __.PKGDEF")
+			return
+		}
+
+		// Read first line of __.PKGDEF data, so that line
+		// is once again the first line of the input.
+		if line, err = r.ReadSlice('\n'); err != nil {
+			return
+		}
+	}
+
+	// Now at __.PKGDEF in archive or still at beginning of file.
+	// Either way, line should begin with "go object ".
+	if !strings.HasPrefix(string(line), "go object ") {
+		err = errors.New("not a go object file")
+		return
+	}
+
+	// Skip over object header to export data.
+	// Begins after first line with $$.
+	for line[0] != '$' {
+		if line, err = r.ReadSlice('\n'); err != nil {
+			return
+		}
+	}
+
+	return
+}

go/gcimporter15/gcimporter_test.go

@@ -0,0 +1,248 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build go1.5
+
+// This file is:
+// - a copy of go/gcimporter/gcimporter_test.go
+// - tagged for Go 1.5
+// - changed to use the standard go/types package.
+
+package gcimporter
+
+import (
+	"fmt"
+	"go/build"
+	"go/types"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"testing"
+	"time"
+)
+
+// skipSpecialPlatforms causes the test to be skipped for platforms where
+// builders (build.golang.org) don't have access to compiled packages for
+// import.
+func skipSpecialPlatforms(t *testing.T) {
+	switch platform := runtime.GOOS + "-" + runtime.GOARCH; platform {
+	case "nacl-amd64p32",
+		"nacl-386",
+		"nacl-arm",
+		"darwin-arm",
+		"darwin-arm64":
+		t.Skipf("no compiled packages available for import on %s", platform)
+	}
+}
+
+var gcPath string // Go compiler path
+
+func init() {
+	if char, err := build.ArchChar(runtime.GOARCH); err == nil {
+		gcPath = filepath.Join(build.ToolDir, char+"g")
+		return
+	}
+	gcPath = "unknown-GOARCH-compiler"
+}
+
+func compile(t *testing.T, dirname, filename string) string {
+	cmd := exec.Command(gcPath, filename)
+	cmd.Dir = dirname
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		t.Logf("%s", out)
+		t.Fatalf("%s %s failed: %s", gcPath, filename, err)
+	}
+	archCh, _ := build.ArchChar(runtime.GOARCH)
+	// filename should end with ".go"
+	return filepath.Join(dirname, filename[:len(filename)-2]+archCh)
+}
+
+// Use the same global imports map for all tests. The effect is
+// as if all tested packages were imported into a single package.
+var imports = make(map[string]*types.Package)
+
+func testPath(t *testing.T, path string) *types.Package {
+	t0 := time.Now()
+	pkg, err := Import(imports, path)
+	if err != nil {
+		t.Errorf("testPath(%s): %s", path, err)
+		return nil
+	}
+	t.Logf("testPath(%s): %v", path, time.Since(t0))
+	return pkg
+}
+
+const maxTime = 30 * time.Second
+
+func testDir(t *testing.T, dir string, endTime time.Time) (nimports int) {
+	dirname := filepath.Join(runtime.GOROOT(), "pkg", runtime.GOOS+"_"+runtime.GOARCH, dir)
+	list, err := ioutil.ReadDir(dirname)
+	if err != nil {
+		t.Fatalf("testDir(%s): %s", dirname, err)
+	}
+	for _, f := range list {
+		if time.Now().After(endTime) {
+			t.Log("testing time used up")
+			return
+		}
+		switch {
+		case !f.IsDir():
+			// try extensions
+			for _, ext := range pkgExts {
+				if strings.HasSuffix(f.Name(), ext) {
+					name := f.Name()[0 : len(f.Name())-len(ext)] // remove extension
+					if testPath(t, filepath.Join(dir, name)) != nil {
+						nimports++
+					}
+				}
+			}
+		case f.IsDir():
+			nimports += testDir(t, filepath.Join(dir, f.Name()), endTime)
+		}
+	}
+	return
+}
+
+func TestImport(t *testing.T) {
+	// This package only handles gc export data.
+	if runtime.Compiler != "gc" {
+		t.Skipf("gc-built packages not available (compiler = %s)", runtime.Compiler)
+		return
+	}
+
+	// On cross-compile builds, the path will not exist.
+	// Need to use GOHOSTOS, which is not available.
+	if _, err := os.Stat(gcPath); err != nil {
+		t.Skipf("skipping test: %v", err)
+	}
+
+	if outFn := compile(t, "testdata", "exports.go"); outFn != "" {
+		defer os.Remove(outFn)
+	}
+
+	nimports := 0
+	if pkg := testPath(t, "./testdata/exports"); pkg != nil {
+		nimports++
+		// The package's Imports should include all the types
+		// referenced by the exportdata, which may be more than
+		// the import statements in the package's source, but
+		// fewer than the transitive closure of dependencies.
+		want := `[package ast ("go/ast") package token ("go/token") package runtime ("runtime")]`
+		got := fmt.Sprint(pkg.Imports())
+		if got != want {
+			t.Errorf(`Package("exports").Imports() = %s, want %s`, got, want)
+		}
+	}
+	nimports += testDir(t, "", time.Now().Add(maxTime)) // installed packages
+	t.Logf("tested %d imports", nimports)
+}
+
+var importedObjectTests = []struct {
+	name string
+	want string
+}{
+	{"unsafe.Pointer", "type Pointer unsafe.Pointer"},
+	{"math.Pi", "const Pi untyped float"},
+	{"io.Reader", "type Reader interface{Read(p []byte) (n int, err error)}"},
+	{"io.ReadWriter", "type ReadWriter interface{Read(p []byte) (n int, err error); Write(p []byte) (n int, err error)}"},
+	{"math.Sin", "func Sin(x float64) float64"},
+	// TODO(gri) add more tests
+}
+
+func TestImportedTypes(t *testing.T) {
+	skipSpecialPlatforms(t)
+
+	// This package only handles gc export data.
+	if runtime.Compiler != "gc" {
+		t.Skipf("gc-built packages not available (compiler = %s)", runtime.Compiler)
+		return
+	}
+
+	for _, test := range importedObjectTests {
+		s := strings.Split(test.name, ".")
+		if len(s) != 2 {
+			t.Fatal("inconsistent test data")
+		}
+		importPath := s[0]
+		objName := s[1]
+
+		pkg, err := Import(imports, importPath)
+		if err != nil {
+			t.Error(err)
+			continue
+		}
+
+		obj := pkg.Scope().Lookup(objName)
+		if obj == nil {
+			t.Errorf("%s: object not found", test.name)
+			continue
+		}
+
+		got := types.ObjectString(obj, types.RelativeTo(pkg))
+		if got != test.want {
+			t.Errorf("%s: got %q; want %q", test.name, got, test.want)
+		}
+	}
+}
+
+func TestIssue5815(t *testing.T) {
+	skipSpecialPlatforms(t)
+
+	// This package only handles gc export data.
+	if runtime.Compiler != "gc" {
+		t.Skipf("gc-built packages not available (compiler = %s)", runtime.Compiler)
+		return
+	}
+
+	pkg, err := Import(make(map[string]*types.Package), "strings")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	scope := pkg.Scope()
+	for _, name := range scope.Names() {
+		obj := scope.Lookup(name)
+		if obj.Pkg() == nil {
+			t.Errorf("no pkg for %s", obj)
+		}
+		if tname, _ := obj.(*types.TypeName); tname != nil {
+			named := tname.Type().(*types.Named)
+			for i := 0; i < named.NumMethods(); i++ {
+				m := named.Method(i)
+				if m.Pkg() == nil {
+					t.Errorf("no pkg for %s", m)
+				}
+			}
+		}
+	}
+}
+
+// Smoke test to ensure that imported methods get the correct package.
+func TestCorrectMethodPackage(t *testing.T) {
+	skipSpecialPlatforms(t)
+
+	// This package only handles gc export data.
+	if runtime.Compiler != "gc" {
+		t.Skipf("gc-built packages not available (compiler = %s)", runtime.Compiler)
+		return
+	}
+
+	imports := make(map[string]*types.Package)
+	_, err := Import(imports, "net/http")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	mutex := imports["sync"].Scope().Lookup("Mutex").(*types.TypeName).Type()
+	mset := types.NewMethodSet(types.NewPointer(mutex)) // methods of *sync.Mutex
+	sel := mset.Lookup(nil, "Lock")
+	lock := sel.Obj().(*types.Func)
+	if got, want := lock.Pkg().Path(), "sync"; got != want {
+		t.Errorf("got package path %q; want %q", got, want)
+	}
+}

go/gcimporter15/testdata/exports.go

@@ -0,0 +1,89 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// This file is used to generate an object file which
+// serves as test file for gcimporter_test.go.
+
+package exports
+
+import (
+	"go/ast"
+)
+
+// Issue 3682: Correctly read dotted identifiers from export data.
+const init1 = 0
+
+func init() {}
+
+const (
+	C0 int = 0
+	C1     = 3.14159265
+	C2     = 2.718281828i
+	C3     = -123.456e-789
+	C4     = +123.456E+789
+	C5     = 1234i
+	C6     = "foo\n"
+	C7     = `bar\n`
+)
+
+type (
+	T1  int
+	T2  [10]int
+	T3  []int
+	T4  *int
+	T5  chan int
+	T6a chan<- int
+	T6b chan (<-chan int)
+	T6c chan<- (chan int)
+	T7  <-chan *ast.File
+	T8  struct{}
+	T9  struct {
+		a    int
+		b, c float32
+		d    []string `go:"tag"`
+	}
+	T10 struct {
+		T8
+		T9
+		_ *T10
+	}
+	T11 map[int]string
+	T12 interface{}
+	T13 interface {
+		m1()
+		m2(int) float32
+	}
+	T14 interface {
+		T12
+		T13
+		m3(x ...struct{}) []T9
+	}
+	T15 func()
+	T16 func(int)
+	T17 func(x int)
+	T18 func() float32
+	T19 func() (x float32)
+	T20 func(...interface{})
+	T21 struct{ next *T21 }
+	T22 struct{ link *T23 }
+	T23 struct{ link *T22 }
+	T24 *T24
+	T25 *T26
+	T26 *T27
+	T27 *T25
+	T28 func(T28) T28
+)
+
+var (
+	V0 int
+	V1 = -991.0
+)
+
+func F1()         {}
+func F2(x int)    {}
+func F3() int     { return 0 }
+func F4() float32 { return 0 }
+func F5(a, b, c int, u, v, w struct{ x, y T1 }, more ...interface{}) (p, q, r chan<- T10)
+
+func (p *T1) M1()