From 984be3b0f8388161e166039de8b7ac5383a76d10 Mon Sep 17 00:00:00 2001 From: Than McIntosh Date: Fri, 16 Nov 2018 10:11:47 -0500 Subject: [PATCH] go/internal/gccgoimporter: enhance for new export data, fix test issues This patch merges in support for reading indexed type export data, from the gofrontend CL https://golang.org/cl/143022 (which includes a change in the export data version number from V2 to V3). Also fixes the key tests to insure that they run both in gccgo builds and main Go repo builds if "gccgo" is present (prior to this the tests were not running in either scenario); this required fixing up some of the expected results. Fixes #28961. Change-Id: I644d171f2a46be9160f89dada06ab3c20468bab7 Reviewed-on: https://go-review.googlesource.com/c/149957 Run-TryBot: Than McIntosh Reviewed-by: Ian Lance Taylor --- .../gccgoimporter/gccgoinstallation_test.go | 13 +- src/go/internal/gccgoimporter/importer.go | 5 +- .../internal/gccgoimporter/importer_test.go | 62 +++-- src/go/internal/gccgoimporter/parser.go | 216 ++++++++++++++---- src/go/internal/gccgoimporter/parser_test.go | 3 +- 5 files changed, 213 insertions(+), 86 deletions(-) diff --git a/src/go/internal/gccgoimporter/gccgoinstallation_test.go b/src/go/internal/gccgoimporter/gccgoinstallation_test.go index da4931ef1e7..732159ca632 100644 --- a/src/go/internal/gccgoimporter/gccgoinstallation_test.go +++ b/src/go/internal/gccgoimporter/gccgoinstallation_test.go @@ -6,7 +6,6 @@ package gccgoimporter import ( "go/types" - "runtime" "testing" ) @@ -144,14 +143,14 @@ var importablePackages = [...]string{ } func TestInstallationImporter(t *testing.T) { - // This test relies on gccgo being around, which it most likely will be if we - // were compiled with gccgo. - if runtime.Compiler != "gccgo" { + // This test relies on gccgo being around. + gpath := gccgoPath() + if gpath == "" { t.Skip("This test needs gccgo") } var inst GccgoInstallation - err := inst.InitFromDriver("gccgo") + err := inst.InitFromDriver(gpath) if err != nil { t.Fatal(err) } @@ -176,12 +175,12 @@ func TestInstallationImporter(t *testing.T) { // Test for certain specific entities in the imported data. for _, test := range [...]importerTest{ - {pkgpath: "io", name: "Reader", want: "type Reader interface{Read(p []uint8) (n int, err error)}"}, + {pkgpath: "io", name: "Reader", want: "type Reader interface{Read(p []byte) (n int, err error)}"}, {pkgpath: "io", name: "ReadWriter", want: "type ReadWriter interface{Reader; Writer}"}, {pkgpath: "math", name: "Pi", want: "const Pi untyped float"}, {pkgpath: "math", name: "Sin", want: "func Sin(x float64) float64"}, {pkgpath: "sort", name: "Ints", want: "func Ints(a []int)"}, - {pkgpath: "unsafe", name: "Pointer", want: "type Pointer unsafe.Pointer"}, + {pkgpath: "unsafe", name: "Pointer", want: "type Pointer"}, } { runImporterTest(t, imp, nil, &test) } diff --git a/src/go/internal/gccgoimporter/importer.go b/src/go/internal/gccgoimporter/importer.go index 159cc50719f..ea111136cda 100644 --- a/src/go/internal/gccgoimporter/importer.go +++ b/src/go/internal/gccgoimporter/importer.go @@ -62,6 +62,7 @@ func findExportFile(searchpaths []string, pkgpath string) (string, error) { const ( gccgov1Magic = "v1;\n" gccgov2Magic = "v2;\n" + gccgov3Magic = "v3;\n" goimporterMagic = "\n$$ " archiveMagic = "! package object typeList []types.Type // type number -> type + typeData []string // unparsed type data (v3 and later) initdata InitData // package init priority data } func (p *parser) init(filename string, src io.Reader, imports map[string]*types.Package) { + p.scanner = new(scanner.Scanner) + p.initScanner(filename, src) + p.imports = imports + p.typeList = make([]types.Type, 1 /* type numbers start at 1 */, 16) +} + +func (p *parser) initScanner(filename string, src io.Reader) { p.scanner.Init(src) p.scanner.Error = func(_ *scanner.Scanner, msg string) { p.error(msg) } p.scanner.Mode = scanner.ScanIdents | scanner.ScanInts | scanner.ScanFloats | scanner.ScanStrings | scanner.ScanComments | scanner.SkipComments - p.scanner.Whitespace = 1<<'\t' | 1<<'\n' | 1<<' ' + p.scanner.Whitespace = 1<<'\t' | 1<<' ' p.scanner.Filename = filename // for good error messages p.next() - p.imports = imports - p.typeList = make([]types.Type, 1 /* type numbers start at 1 */, 16) } type importError struct { @@ -71,6 +77,13 @@ func (p *parser) expect(tok rune) string { return lit } +func (p *parser) expectEOL() { + if p.version == "v1" || p.version == "v2" { + p.expect(';') + } + p.expect('\n') +} + func (p *parser) expectKeyword(keyword string) { lit := p.expect(scanner.Ident) if lit != keyword { @@ -96,7 +109,7 @@ func (p *parser) parseUnquotedString() string { buf.WriteString(p.scanner.TokenText()) // This loop needs to examine each character before deciding whether to consume it. If we see a semicolon, // we need to let it be consumed by p.next(). - for ch := p.scanner.Peek(); ch != ';' && ch != scanner.EOF && p.scanner.Whitespace&(1<' { + if len(p.typeData) > 0 && p.typeList[n1] == nil { + p.parseSavedType(pkg, n1, n) + } t = p.typeList[n1] - if t == reserved { - p.errorf("invalid type cycle, type %d not yet defined", n1) + if len(p.typeData) == 0 && t == reserved { + p.errorf("invalid type cycle, type %d not yet defined (nlist=%v)", n1, n) } p.update(t, n) } else { @@ -808,12 +852,86 @@ func (p *parser) parseType(pkg *types.Package, n ...int) (t types.Type) { default: p.errorf("expected type number, got %s (%q)", scanner.TokenString(p.tok), p.lit) + return nil + } + + if t == nil || t == reserved { + p.errorf("internal error: bad return from parseType(%v)", n) } p.expect('>') return } +// Types = "types" maxp1 exportedp1 (offset length)* . +func (p *parser) parseTypes(pkg *types.Package) { + maxp1 := p.parseInt() + exportedp1 := p.parseInt() + p.typeList = make([]types.Type, maxp1, maxp1) + + type typeOffset struct { + offset int + length int + } + var typeOffsets []typeOffset + + total := 0 + for i := 1; i < maxp1; i++ { + len := p.parseInt() + typeOffsets = append(typeOffsets, typeOffset{total, len}) + total += len + } + + // We should now have p.tok pointing to the final newline. + // The next runes from the scanner should be the type data. + + var sb strings.Builder + for sb.Len() < total { + r := p.scanner.Next() + if r == scanner.EOF { + p.error("unexpected EOF") + } + sb.WriteRune(r) + } + allTypeData := sb.String() + + p.typeData = []string{""} // type 0, unused + for _, to := range typeOffsets { + p.typeData = append(p.typeData, allTypeData[to.offset:to.offset+to.length]) + } + + for i := 1; i < int(exportedp1); i++ { + p.parseSavedType(pkg, i, []int{}) + } +} + +// parseSavedType parses one saved type definition. +func (p *parser) parseSavedType(pkg *types.Package, i int, nlist []int) { + defer func(s *scanner.Scanner, tok rune, lit string) { + p.scanner = s + p.tok = tok + p.lit = lit + }(p.scanner, p.tok, p.lit) + + p.scanner = new(scanner.Scanner) + p.initScanner(p.scanner.Filename, strings.NewReader(p.typeData[i])) + p.expectKeyword("type") + id := p.parseInt() + if id != i { + p.errorf("type ID mismatch: got %d, want %d", id, i) + } + if p.typeList[i] == reserved { + p.errorf("internal error: %d already reserved in parseSavedType", i) + } + if p.typeList[i] == nil { + p.reserve(i) + p.parseTypeSpec(pkg, append(nlist, i)) + } + if p.typeList[i] == nil || p.typeList[i] == reserved { + p.errorf("internal error: parseSavedType(%d,%v) reserved/nil", i, nlist) + } +} + // PackageInit = unquotedString unquotedString int . func (p *parser) parsePackageInit() PackageInit { name := p.parseUnquotedString() @@ -829,7 +947,7 @@ func (p *parser) parsePackageInit() PackageInit { func (p *parser) discardDirectiveWhileParsingTypes(pkg *types.Package) { for { switch p.tok { - case ';': + case '\n', ';': return case '<': p.parseType(pkg) @@ -848,7 +966,7 @@ func (p *parser) maybeCreatePackage() { } } -// InitDataDirective = ( "v1" | "v2" ) ";" | +// InitDataDirective = ( "v1" | "v2" | "v3" ) ";" | // "priority" int ";" | // "init" { PackageInit } ";" | // "checksum" unquotedString ";" . @@ -859,31 +977,32 @@ func (p *parser) parseInitDataDirective() { } switch p.lit { - case "v1", "v2": + case "v1", "v2", "v3": p.version = p.lit p.next() p.expect(';') + p.expect('\n') case "priority": p.next() p.initdata.Priority = p.parseInt() - p.expect(';') + p.expectEOL() case "init": p.next() - for p.tok != ';' && p.tok != scanner.EOF { + for p.tok != '\n' && p.tok != ';' && p.tok != scanner.EOF { p.initdata.Inits = append(p.initdata.Inits, p.parsePackageInit()) } - p.expect(';') + p.expectEOL() case "init_graph": p.next() // The graph data is thrown away for now. - for p.tok != ';' && p.tok != scanner.EOF { + for p.tok != '\n' && p.tok != ';' && p.tok != scanner.EOF { p.parseInt64() p.parseInt64() } - p.expect(';') + p.expectEOL() case "checksum": // Don't let the scanner try to parse the checksum as a number. @@ -893,7 +1012,7 @@ func (p *parser) parseInitDataDirective() { p.scanner.Mode &^= scanner.ScanInts | scanner.ScanFloats p.next() p.parseUnquotedString() - p.expect(';') + p.expectEOL() default: p.errorf("unexpected identifier: %q", p.lit) @@ -905,6 +1024,7 @@ func (p *parser) parseInitDataDirective() { // "pkgpath" unquotedString ";" | // "prefix" unquotedString ";" | // "import" unquotedString unquotedString string ";" | +// "indirectimport" unquotedString unquotedstring ";" | // "func" Func ";" | // "type" Type ";" | // "var" Var ";" | @@ -916,29 +1036,29 @@ func (p *parser) parseDirective() { } switch p.lit { - case "v1", "v2", "priority", "init", "init_graph", "checksum": + case "v1", "v2", "v3", "priority", "init", "init_graph", "checksum": p.parseInitDataDirective() case "package": p.next() p.pkgname = p.parseUnquotedString() p.maybeCreatePackage() - if p.version == "v2" && p.tok != ';' { + if p.version != "v1" && p.tok != '\n' && p.tok != ';' { p.parseUnquotedString() p.parseUnquotedString() } - p.expect(';') + p.expectEOL() case "pkgpath": p.next() p.pkgpath = p.parseUnquotedString() p.maybeCreatePackage() - p.expect(';') + p.expectEOL() case "prefix": p.next() p.pkgpath = p.parseUnquotedString() - p.expect(';') + p.expectEOL() case "import": p.next() @@ -946,7 +1066,19 @@ func (p *parser) parseDirective() { pkgpath := p.parseUnquotedString() p.getPkg(pkgpath, pkgname) p.parseString() - p.expect(';') + p.expectEOL() + + case "indirectimport": + p.next() + pkgname := p.parseUnquotedString() + pkgpath := p.parseUnquotedString() + p.getPkg(pkgpath, pkgname) + p.expectEOL() + + case "types": + p.next() + p.parseTypes(p.pkg) + p.expectEOL() case "func": p.next() @@ -954,24 +1086,24 @@ func (p *parser) parseDirective() { if fun != nil { p.pkg.Scope().Insert(fun) } - p.expect(';') + p.expectEOL() case "type": p.next() p.parseType(p.pkg) - p.expect(';') + p.expectEOL() case "var": p.next() v := p.parseVar(p.pkg) p.pkg.Scope().Insert(v) - p.expect(';') + p.expectEOL() case "const": p.next() c := p.parseConst(p.pkg) p.pkg.Scope().Insert(c) - p.expect(';') + p.expectEOL() default: p.errorf("unexpected identifier: %q", p.lit) diff --git a/src/go/internal/gccgoimporter/parser_test.go b/src/go/internal/gccgoimporter/parser_test.go index 4a103dc462a..00128b44d26 100644 --- a/src/go/internal/gccgoimporter/parser_test.go +++ b/src/go/internal/gccgoimporter/parser_test.go @@ -19,7 +19,7 @@ var typeParserTests = []struct { {id: "foo", typ: ">", want: "*error"}, {id: "foo", typ: "", want: "unsafe.Pointer"}, {id: "foo", typ: ">>", want: "foo.Bar", underlying: "*foo.Bar"}, - {id: "foo", typ: " func (? ) M (); >", want: "bar.Foo", underlying: "int8", methods: "func (bar.Foo).M()"}, + {id: "foo", typ: "\nfunc (? ) M ();\n>", want: "bar.Foo", underlying: "int8", methods: "func (bar.Foo).M()"}, {id: "foo", typ: ">", want: "bar.foo", underlying: "int8"}, {id: "foo", typ: ">", want: "[]int8"}, {id: "foo", typ: ">", want: "[42]int8"}, @@ -36,6 +36,7 @@ func TestTypeParser(t *testing.T) { for _, test := range typeParserTests { var p parser p.init("test.gox", strings.NewReader(test.typ), make(map[string]*types.Package)) + p.version = "v2" p.pkgname = test.id p.pkgpath = test.id p.maybeCreatePackage()