go/internal/gccgoimporter: enhance for new export data, fix test issues

This patch merges in support for reading indexed type export data,
from the gofrontend CL https://golang.org/cl/143022 (which includes
a change in the export data version number from V2 to V3).

Also fixes the key tests to insure that they run both in gccgo builds
and main Go repo builds if "gccgo" is present (prior to this the tests
were not running in either scenario); this required fixing up some of
the expected results.

Fixes #28961.

Change-Id: I644d171f2a46be9160f89dada06ab3c20468bab7
Reviewed-on: https://go-review.googlesource.com/c/149957
Run-TryBot: Than McIntosh <thanm@google.com>
Reviewed-by: Ian Lance Taylor <iant@golang.org>

src/go/internal/gccgoimporter/gccgoinstallation_test.go

@@ -6,7 +6,6 @@ package gccgoimporter
 
 import (
 	"go/types"
-	"runtime"
 	"testing"
 )
 
@@ -144,14 +143,14 @@ var importablePackages = [...]string{
 }
 
 func TestInstallationImporter(t *testing.T) {
-	// This test relies on gccgo being around, which it most likely will be if we
+	// This test relies on gccgo being around.
-	// were compiled with gccgo.
+	gpath := gccgoPath()
-	if runtime.Compiler != "gccgo" {
+	if gpath == "" {
 		t.Skip("This test needs gccgo")
 	}
 
 	var inst GccgoInstallation
-	err := inst.InitFromDriver("gccgo")
+	err := inst.InitFromDriver(gpath)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -176,12 +175,12 @@ func TestInstallationImporter(t *testing.T) {
 
 	// Test for certain specific entities in the imported data.
 	for _, test := range [...]importerTest{
-		{pkgpath: "io", name: "Reader", want: "type Reader interface{Read(p []uint8) (n int, err error)}"},
+		{pkgpath: "io", name: "Reader", want: "type Reader interface{Read(p []byte) (n int, err error)}"},
 		{pkgpath: "io", name: "ReadWriter", want: "type ReadWriter interface{Reader; Writer}"},
 		{pkgpath: "math", name: "Pi", want: "const Pi untyped float"},
 		{pkgpath: "math", name: "Sin", want: "func Sin(x float64) float64"},
 		{pkgpath: "sort", name: "Ints", want: "func Ints(a []int)"},
-		{pkgpath: "unsafe", name: "Pointer", want: "type Pointer unsafe.Pointer"},
+		{pkgpath: "unsafe", name: "Pointer", want: "type Pointer"},
 	} {
 		runImporterTest(t, imp, nil, &test)
 	}

src/go/internal/gccgoimporter/importer.go

@@ -62,6 +62,7 @@ func findExportFile(searchpaths []string, pkgpath string) (string, error) {
 const (
 	gccgov1Magic    = "v1;\n"
 	gccgov2Magic    = "v2;\n"
+	gccgov3Magic    = "v3;\n"
 	goimporterMagic = "\n$$ "
 	archiveMagic    = "!<ar"
 )
@@ -90,7 +91,7 @@ func openExportFile(fpath string) (reader io.ReadSeeker, closer io.Closer, err e
 
 	var elfreader io.ReaderAt
 	switch string(magic[:]) {
-	case gccgov1Magic, gccgov2Magic, goimporterMagic:
+	case gccgov1Magic, gccgov2Magic, gccgov3Magic, goimporterMagic:
 		// Raw export data.
 		reader = f
 		return
@@ -195,7 +196,7 @@ func GetImporter(searchpaths []string, initmap map[*types.Package]InitData) Impo
 		}
 
 		switch magics {
-		case gccgov1Magic, gccgov2Magic:
+		case gccgov1Magic, gccgov2Magic, gccgov3Magic:
 			var p parser
 			p.init(fpath, reader, imports)
 			pkg = p.parsePackage()

src/go/internal/gccgoimporter/importer_test.go

@@ -11,7 +11,6 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
-	"runtime"
 	"testing"
 )
 
@@ -53,9 +52,6 @@ func runImporterTest(t *testing.T, imp Importer, initmap map[*types.Package]Init
 		// Check that the package's own init function has the package's priority
 		for _, pkginit := range initdata.Inits {
 			if pkginit.InitFunc == test.wantinits[0] {
-				if initdata.Priority != pkginit.Priority {
-					t.Errorf("%s: got self priority %d; want %d", test.pkgpath, pkginit.Priority, initdata.Priority)
-				}
 				found = true
 				break
 			}
@@ -65,27 +61,11 @@ func runImporterTest(t *testing.T, imp Importer, initmap map[*types.Package]Init
 			t.Errorf("%s: could not find expected function %q", test.pkgpath, test.wantinits[0])
 		}
 
-		// Each init function in the list other than the first one is a
+		// FIXME: the original version of this test was written against
-		// dependency of the function immediately before it. Check that
+		// the v1 export data scheme for capturing init functions, so it
-		// the init functions appear in descending priority order.
+		// verified the priority values. We moved away from the priority
-		priority := initdata.Priority
+		// scheme some time ago; it is not clear how much work it would be
-		for _, wantdepinit := range test.wantinits[1:] {
+		// to validate the new init export data.
-			found = false
-			for _, pkginit := range initdata.Inits {
-				if pkginit.InitFunc == wantdepinit {
-					if priority <= pkginit.Priority {
-						t.Errorf("%s: got dep priority %d; want less than %d", test.pkgpath, pkginit.Priority, priority)
-					}
-					found = true
-					priority = pkginit.Priority
-					break
-				}
-			}
-
-			if !found {
-				t.Errorf("%s: could not find expected function %q", test.pkgpath, wantdepinit)
-			}
-		}
 	}
 }
 
@@ -100,7 +80,7 @@ var importerTests = [...]importerTest{
 	{pkgpath: "time", name: "Nanosecond", want: "const Nanosecond Duration", wantval: "1"},
 	{pkgpath: "unicode", name: "IsUpper", want: "func IsUpper(r rune) bool"},
 	{pkgpath: "unicode", name: "MaxRune", want: "const MaxRune untyped rune", wantval: "1114111"},
-	{pkgpath: "imports", wantinits: []string{"imports..import", "fmt..import", "math..import"}},
+	{pkgpath: "imports", wantinits: []string{"imports..import", "fmt..import"}},
 	{pkgpath: "importsar", name: "Hello", want: "var Hello string"},
 	{pkgpath: "aliases", name: "A14", want: "type A14 = func(int, T0) chan T2"},
 	{pkgpath: "aliases", name: "C0", want: "type C0 struct{f1 C1; f2 C1}"},
@@ -109,8 +89,7 @@ var importerTests = [...]importerTest{
 }
 
 func TestGoxImporter(t *testing.T) {
-	testenv.MustHaveGoBuild(t)
+	testenv.MustHaveExec(t) // this is to skip nacl, js
-
 	initmap := make(map[*types.Package]InitData)
 	imp := GetImporter([]string{"testdata"}, initmap)
 
@@ -119,12 +98,24 @@ func TestGoxImporter(t *testing.T) {
 	}
 }
 
-func TestObjImporter(t *testing.T) {
+// gccgoPath returns a path to gccgo if it is present (either in
-	testenv.MustHaveGoBuild(t)
+// path or specified via GCCGO environment variable), or an
+// empty string if no gccgo is available.
+func gccgoPath() string {
+	gccgoname := os.Getenv("GCCGO")
+	if gccgoname == "" {
+		gccgoname = "gccgo"
+	}
+	if gpath, gerr := exec.LookPath(gccgoname); gerr == nil {
+		return gpath
+	}
+	return ""
+}
 
-	// This test relies on gccgo being around, which it most likely will be if we
+func TestObjImporter(t *testing.T) {
-	// were compiled with gccgo.
+	// This test relies on gccgo being around.
-	if runtime.Compiler != "gccgo" {
+	gpath := gccgoPath()
+	if gpath == "" {
 		t.Skip("This test needs gccgo")
 	}
 
@@ -144,10 +135,13 @@ func TestObjImporter(t *testing.T) {
 
 	for _, test := range importerTests {
 		gofile := filepath.Join("testdata", test.pkgpath+".go")
+		if _, err := os.Stat(gofile); os.IsNotExist(err) {
+			continue
+		}
 		ofile := filepath.Join(tmpdir, test.pkgpath+".o")
 		afile := filepath.Join(artmpdir, "lib"+test.pkgpath+".a")
 
-		cmd := exec.Command("gccgo", "-fgo-pkgpath="+test.pkgpath, "-c", "-o", ofile, gofile)
+		cmd := exec.Command(gpath, "-fgo-pkgpath="+test.pkgpath, "-c", "-o", ofile, gofile)
 		out, err := cmd.CombinedOutput()
 		if err != nil {
 			t.Logf("%s", out)

src/go/internal/gccgoimporter/parser.go

@@ -18,7 +18,7 @@ import (
 )
 
 type parser struct {
-	scanner  scanner.Scanner
+	scanner  *scanner.Scanner
 	version  string                    // format version
 	tok      rune                      // current token
 	lit      string                    // literal string; only valid for Ident, Int, String tokens
@@ -27,18 +27,24 @@ type parser struct {
 	pkg      *types.Package            // reference to imported package
 	imports  map[string]*types.Package // package path -> package object
 	typeList []types.Type              // type number -> type
+	typeData []string                  // unparsed type data (v3 and later)
 	initdata InitData                  // package init priority data
 }
 
 func (p *parser) init(filename string, src io.Reader, imports map[string]*types.Package) {
+	p.scanner = new(scanner.Scanner)
+	p.initScanner(filename, src)
+	p.imports = imports
+	p.typeList = make([]types.Type, 1 /* type numbers start at 1 */, 16)
+}
+
+func (p *parser) initScanner(filename string, src io.Reader) {
 	p.scanner.Init(src)
 	p.scanner.Error = func(_ *scanner.Scanner, msg string) { p.error(msg) }
 	p.scanner.Mode = scanner.ScanIdents | scanner.ScanInts | scanner.ScanFloats | scanner.ScanStrings | scanner.ScanComments | scanner.SkipComments
-	p.scanner.Whitespace = 1<<'\t' | 1<<'\n' | 1<<' '
+	p.scanner.Whitespace = 1<<'\t' | 1<<' '
 	p.scanner.Filename = filename // for good error messages
 	p.next()
-	p.imports = imports
-	p.typeList = make([]types.Type, 1 /* type numbers start at 1 */, 16)
 }
 
 type importError struct {
@@ -71,6 +77,13 @@ func (p *parser) expect(tok rune) string {
 	return lit
 }
 
+func (p *parser) expectEOL() {
+	if p.version == "v1" || p.version == "v2" {
+		p.expect(';')
+	}
+	p.expect('\n')
+}
+
 func (p *parser) expectKeyword(keyword string) {
 	lit := p.expect(scanner.Ident)
 	if lit != keyword {
@@ -96,7 +109,7 @@ func (p *parser) parseUnquotedString() string {
 	buf.WriteString(p.scanner.TokenText())
 	// This loop needs to examine each character before deciding whether to consume it. If we see a semicolon,
 	// we need to let it be consumed by p.next().
-	for ch := p.scanner.Peek(); ch != ';' && ch != scanner.EOF && p.scanner.Whitespace&(1<<uint(ch)) == 0; ch = p.scanner.Peek() {
+	for ch := p.scanner.Peek(); ch != '\n' && ch != ';' && ch != scanner.EOF && p.scanner.Whitespace&(1<<uint(ch)) == 0; ch = p.scanner.Peek() {
 		buf.WriteRune(ch)
 		p.scanner.Next()
 	}
@@ -387,17 +400,42 @@ var reserved = new(struct{ types.Type })
 
 // reserve reserves the type map entry n for future use.
 func (p *parser) reserve(n int) {
-	if n != len(p.typeList) {
+	// Notes:
-		p.errorf("invalid type number %d (out of sync)", n)
+	// - for pre-V3 export data, the type numbers we see are
+	//   guaranteed to be in increasing order, so we append a
+	//   reserved entry onto the list.
+	// - for V3+ export data, type numbers can appear in
+	//   any order, however the 'types' section tells us the
+	//   total number of types, hence typeList is pre-allocated.
+	if len(p.typeData) == 0 {
+		if n != len(p.typeList) {
+			p.errorf("invalid type number %d (out of sync)", n)
+		}
+		p.typeList = append(p.typeList, reserved)
+	} else {
+		if p.typeList[n] != nil {
+			p.errorf("previously visited type number %d", n)
+		}
+		p.typeList[n] = reserved
 	}
-	p.typeList = append(p.typeList, reserved)
 }
 
 // update sets the type map entries for the given type numbers nlist to t.
 func (p *parser) update(t types.Type, nlist []int) {
+	if len(nlist) != 0 {
+		if t == reserved {
+			p.errorf("internal error: update(%v) invoked on reserved", nlist)
+		}
+		if t == nil {
+			p.errorf("internal error: update(%v) invoked on nil", nlist)
+		}
+	}
 	for _, n := range nlist {
+		if p.typeList[n] == t {
+			continue
+		}
 		if p.typeList[n] != reserved {
-			p.errorf("typeMap[%d] not reserved", n)
+			p.errorf("internal error: update(%v): %d not reserved", nlist, n)
 		}
 		p.typeList[n] = t
 	}
@@ -459,19 +497,22 @@ func (p *parser) parseNamedType(nlist []int) types.Type {
 		nt.SetUnderlying(underlying.Underlying())
 	}
 
-	// collect associated methods
+	if p.tok == '\n' {
-	for p.tok == scanner.Ident {
+		p.next()
-		p.expectKeyword("func")
+		// collect associated methods
-		p.expect('(')
+		for p.tok == scanner.Ident {
-		receiver, _ := p.parseParam(pkg)
+			p.expectKeyword("func")
-		p.expect(')')
+			p.expect('(')
-		name := p.parseName()
+			receiver, _ := p.parseParam(pkg)
-		params, isVariadic := p.parseParamList(pkg)
+			p.expect(')')
-		results := p.parseResultList(pkg)
+			name := p.parseName()
-		p.expect(';')
+			params, isVariadic := p.parseParamList(pkg)
+			results := p.parseResultList(pkg)
+			p.expectEOL()
 
-		sig := types.NewSignature(receiver, params, results, isVariadic)
+			sig := types.NewSignature(receiver, params, results, isVariadic)
-		nt.AddMethod(types.NewFunc(token.NoPos, pkg, name, sig))
+			nt.AddMethod(types.NewFunc(token.NoPos, pkg, name, sig))
+		}
 	}
 
 	return nt
@@ -790,9 +831,12 @@ func (p *parser) parseType(pkg *types.Package, n ...int) (t types.Type) {
 	case scanner.Int:
 		n1 := p.parseInt()
 		if p.tok == '>' {
+			if len(p.typeData) > 0 && p.typeList[n1] == nil {
+				p.parseSavedType(pkg, n1, n)
+			}
 			t = p.typeList[n1]
-			if t == reserved {
+			if len(p.typeData) == 0 && t == reserved {
-				p.errorf("invalid type cycle, type %d not yet defined", n1)
+				p.errorf("invalid type cycle, type %d not yet defined (nlist=%v)", n1, n)
 			}
 			p.update(t, n)
 		} else {
@@ -808,12 +852,86 @@ func (p *parser) parseType(pkg *types.Package, n ...int) (t types.Type) {
 
 	default:
 		p.errorf("expected type number, got %s (%q)", scanner.TokenString(p.tok), p.lit)
+		return nil
+	}
+
+	if t == nil || t == reserved {
+		p.errorf("internal error: bad return from parseType(%v)", n)
 	}
 
 	p.expect('>')
 	return
 }
 
+// Types = "types" maxp1 exportedp1 (offset length)* .
+func (p *parser) parseTypes(pkg *types.Package) {
+	maxp1 := p.parseInt()
+	exportedp1 := p.parseInt()
+	p.typeList = make([]types.Type, maxp1, maxp1)
+
+	type typeOffset struct {
+		offset int
+		length int
+	}
+	var typeOffsets []typeOffset
+
+	total := 0
+	for i := 1; i < maxp1; i++ {
+		len := p.parseInt()
+		typeOffsets = append(typeOffsets, typeOffset{total, len})
+		total += len
+	}
+
+	// We should now have p.tok pointing to the final newline.
+	// The next runes from the scanner should be the type data.
+
+	var sb strings.Builder
+	for sb.Len() < total {
+		r := p.scanner.Next()
+		if r == scanner.EOF {
+			p.error("unexpected EOF")
+		}
+		sb.WriteRune(r)
+	}
+	allTypeData := sb.String()
+
+	p.typeData = []string{""} // type 0, unused
+	for _, to := range typeOffsets {
+		p.typeData = append(p.typeData, allTypeData[to.offset:to.offset+to.length])
+	}
+
+	for i := 1; i < int(exportedp1); i++ {
+		p.parseSavedType(pkg, i, []int{})
+	}
+}
+
+// parseSavedType parses one saved type definition.
+func (p *parser) parseSavedType(pkg *types.Package, i int, nlist []int) {
+	defer func(s *scanner.Scanner, tok rune, lit string) {
+		p.scanner = s
+		p.tok = tok
+		p.lit = lit
+	}(p.scanner, p.tok, p.lit)
+
+	p.scanner = new(scanner.Scanner)
+	p.initScanner(p.scanner.Filename, strings.NewReader(p.typeData[i]))
+	p.expectKeyword("type")
+	id := p.parseInt()
+	if id != i {
+		p.errorf("type ID mismatch: got %d, want %d", id, i)
+	}
+	if p.typeList[i] == reserved {
+		p.errorf("internal error: %d already reserved in parseSavedType", i)
+	}
+	if p.typeList[i] == nil {
+		p.reserve(i)
+		p.parseTypeSpec(pkg, append(nlist, i))
+	}
+	if p.typeList[i] == nil || p.typeList[i] == reserved {
+		p.errorf("internal error: parseSavedType(%d,%v) reserved/nil", i, nlist)
+	}
+}
+
 // PackageInit = unquotedString unquotedString int .
 func (p *parser) parsePackageInit() PackageInit {
 	name := p.parseUnquotedString()
@@ -829,7 +947,7 @@ func (p *parser) parsePackageInit() PackageInit {
 func (p *parser) discardDirectiveWhileParsingTypes(pkg *types.Package) {
 	for {
 		switch p.tok {
-		case ';':
+		case '\n', ';':
 			return
 		case '<':
 			p.parseType(pkg)
@@ -848,7 +966,7 @@ func (p *parser) maybeCreatePackage() {
 	}
 }
 
-// InitDataDirective = ( "v1" | "v2" ) ";" |
+// InitDataDirective = ( "v1" | "v2" | "v3" ) ";" |
 //                     "priority" int ";" |
 //                     "init" { PackageInit } ";" |
 //                     "checksum" unquotedString ";" .
@@ -859,31 +977,32 @@ func (p *parser) parseInitDataDirective() {
 	}
 
 	switch p.lit {
-	case "v1", "v2":
+	case "v1", "v2", "v3":
 		p.version = p.lit
 		p.next()
 		p.expect(';')
+		p.expect('\n')
 
 	case "priority":
 		p.next()
 		p.initdata.Priority = p.parseInt()
-		p.expect(';')
+		p.expectEOL()
 
 	case "init":
 		p.next()
-		for p.tok != ';' && p.tok != scanner.EOF {
+		for p.tok != '\n' && p.tok != ';' && p.tok != scanner.EOF {
 			p.initdata.Inits = append(p.initdata.Inits, p.parsePackageInit())
 		}
-		p.expect(';')
+		p.expectEOL()
 
 	case "init_graph":
 		p.next()
 		// The graph data is thrown away for now.
-		for p.tok != ';' && p.tok != scanner.EOF {
+		for p.tok != '\n' && p.tok != ';' && p.tok != scanner.EOF {
 			p.parseInt64()
 			p.parseInt64()
 		}
-		p.expect(';')
+		p.expectEOL()
 
 	case "checksum":
 		// Don't let the scanner try to parse the checksum as a number.
@@ -893,7 +1012,7 @@ func (p *parser) parseInitDataDirective() {
 		p.scanner.Mode &^= scanner.ScanInts | scanner.ScanFloats
 		p.next()
 		p.parseUnquotedString()
-		p.expect(';')
+		p.expectEOL()
 
 	default:
 		p.errorf("unexpected identifier: %q", p.lit)
@@ -905,6 +1024,7 @@ func (p *parser) parseInitDataDirective() {
 //             "pkgpath" unquotedString ";" |
 //             "prefix" unquotedString ";" |
 //             "import" unquotedString unquotedString string ";" |
+//             "indirectimport" unquotedString unquotedstring ";" |
 //             "func" Func ";" |
 //             "type" Type ";" |
 //             "var" Var ";" |
@@ -916,29 +1036,29 @@ func (p *parser) parseDirective() {
 	}
 
 	switch p.lit {
-	case "v1", "v2", "priority", "init", "init_graph", "checksum":
+	case "v1", "v2", "v3", "priority", "init", "init_graph", "checksum":
 		p.parseInitDataDirective()
 
 	case "package":
 		p.next()
 		p.pkgname = p.parseUnquotedString()
 		p.maybeCreatePackage()
-		if p.version == "v2" && p.tok != ';' {
+		if p.version != "v1" && p.tok != '\n' && p.tok != ';' {
 			p.parseUnquotedString()
 			p.parseUnquotedString()
 		}
-		p.expect(';')
+		p.expectEOL()
 
 	case "pkgpath":
 		p.next()
 		p.pkgpath = p.parseUnquotedString()
 		p.maybeCreatePackage()
-		p.expect(';')
+		p.expectEOL()
 
 	case "prefix":
 		p.next()
 		p.pkgpath = p.parseUnquotedString()
-		p.expect(';')
+		p.expectEOL()
 
 	case "import":
 		p.next()
@@ -946,7 +1066,19 @@ func (p *parser) parseDirective() {
 		pkgpath := p.parseUnquotedString()
 		p.getPkg(pkgpath, pkgname)
 		p.parseString()
-		p.expect(';')
+		p.expectEOL()
+
+	case "indirectimport":
+		p.next()
+		pkgname := p.parseUnquotedString()
+		pkgpath := p.parseUnquotedString()
+		p.getPkg(pkgpath, pkgname)
+		p.expectEOL()
+
+	case "types":
+		p.next()
+		p.parseTypes(p.pkg)
+		p.expectEOL()
 
 	case "func":
 		p.next()
@@ -954,24 +1086,24 @@ func (p *parser) parseDirective() {
 		if fun != nil {
 			p.pkg.Scope().Insert(fun)
 		}
-		p.expect(';')
+		p.expectEOL()
 
 	case "type":
 		p.next()
 		p.parseType(p.pkg)
-		p.expect(';')
+		p.expectEOL()
 
 	case "var":
 		p.next()
 		v := p.parseVar(p.pkg)
 		p.pkg.Scope().Insert(v)
-		p.expect(';')
+		p.expectEOL()
 
 	case "const":
 		p.next()
 		c := p.parseConst(p.pkg)
 		p.pkg.Scope().Insert(c)
-		p.expect(';')
+		p.expectEOL()
 
 	default:
 		p.errorf("unexpected identifier: %q", p.lit)

src/go/internal/gccgoimporter/parser_test.go

@@ -19,7 +19,7 @@ var typeParserTests = []struct {
 	{id: "foo", typ: "<type 1 *<type -19>>", want: "*error"},
 	{id: "foo", typ: "<type 1 *any>", want: "unsafe.Pointer"},
 	{id: "foo", typ: "<type 1 \"Bar\" <type 2 *<type 1>>>", want: "foo.Bar", underlying: "*foo.Bar"},
-	{id: "foo", typ: "<type 1 \"bar.Foo\" \"bar\" <type -1> func (? <type 1>) M (); >", want: "bar.Foo", underlying: "int8", methods: "func (bar.Foo).M()"},
+	{id: "foo", typ: "<type 1 \"bar.Foo\" \"bar\" <type -1>\nfunc (? <type 1>) M ();\n>", want: "bar.Foo", underlying: "int8", methods: "func (bar.Foo).M()"},
 	{id: "foo", typ: "<type 1 \".bar.foo\" \"bar\" <type -1>>", want: "bar.foo", underlying: "int8"},
 	{id: "foo", typ: "<type 1 []<type -1>>", want: "[]int8"},
 	{id: "foo", typ: "<type 1 [42]<type -1>>", want: "[42]int8"},
@@ -36,6 +36,7 @@ func TestTypeParser(t *testing.T) {
 	for _, test := range typeParserTests {
 		var p parser
 		p.init("test.gox", strings.NewReader(test.typ), make(map[string]*types.Package))
+		p.version = "v2"
 		p.pkgname = test.id
 		p.pkgpath = test.id
 		p.maybeCreatePackage()