qbit/go
1
0
Fork 0
mirror of https://github.com/golang/go synced 2024-09-25 09:10:14 -06:00
Code Releases Activity

go/internal/gccgoimporter: enhance for new export data, fix test issues

Browse Source 
This patch merges in support for reading indexed type export data,
from the gofrontend CL https://golang.org/cl/143022 (which includes
a change in the export data version number from V2 to V3).

Also fixes the key tests to insure that they run both in gccgo builds
and main Go repo builds if "gccgo" is present (prior to this the tests
were not running in either scenario); this required fixing up some of
the expected results.

Fixes #28961.

Change-Id: I644d171f2a46be9160f89dada06ab3c20468bab7
Reviewed-on: https://go-review.googlesource.com/c/149957
Run-TryBot: Than McIntosh <thanm@google.com>
Reviewed-by: Ian Lance Taylor <iant@golang.org>
This commit is contained in:
Than McIntosh 2018-11-16 10:11:47 -05:00
parent ca3749230b
commit 984be3b0f8
5 changed files with 213 additions and 86 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
src/go/internal/gccgoimporter/gccgoinstallation_test.go
View File

@ -6,7 +6,6 @@ package gccgoimporter
import (
"go/types"
"runtime"
"testing"
)
@ -144,14 +143,14 @@ var importablePackages = [...]string{
}
func TestInstallationImporter(t *testing.T) {
// This test relies on gccgo being around, which it most likely will be if we
// were compiled with gccgo.
if runtime.Compiler != "gccgo" {
// This test relies on gccgo being around.
gpath := gccgoPath()
if gpath == "" {
t.Skip("This test needs gccgo")
}
var inst GccgoInstallation
err := inst.InitFromDriver("gccgo")
err := inst.InitFromDriver(gpath)
if err != nil {
t.Fatal(err)
}
@ -176,12 +175,12 @@ func TestInstallationImporter(t *testing.T) {
// Test for certain specific entities in the imported data.
for _, test := range [...]importerTest{
{pkgpath: "io", name: "Reader", want: "type Reader interface{Read(p []uint8) (n int, err error)}"},
{pkgpath: "io", name: "Reader", want: "type Reader interface{Read(p []byte) (n int, err error)}"},
{pkgpath: "io", name: "ReadWriter", want: "type ReadWriter interface{Reader; Writer}"},
{pkgpath: "math", name: "Pi", want: "const Pi untyped float"},
{pkgpath: "math", name: "Sin", want: "func Sin(x float64) float64"},
{pkgpath: "sort", name: "Ints", want: "func Ints(a []int)"},
{pkgpath: "unsafe", name: "Pointer", want: "type Pointer unsafe.Pointer"},
{pkgpath: "unsafe", name: "Pointer", want: "type Pointer"},
} {
runImporterTest(t, imp, nil, &test)
}

5
src/go/internal/gccgoimporter/importer.go
View File

@ -62,6 +62,7 @@ func findExportFile(searchpaths []string, pkgpath string) (string, error) {
const (
gccgov1Magic = "v1;\n"
gccgov2Magic = "v2;\n"
gccgov3Magic = "v3;\n"
goimporterMagic = "\n$$ "
archiveMagic = "!<ar"
)
@ -90,7 +91,7 @@ func openExportFile(fpath string) (reader io.ReadSeeker, closer io.Closer, err e
var elfreader io.ReaderAt
switch string(magic[:]) {
case gccgov1Magic, gccgov2Magic, goimporterMagic:
case gccgov1Magic, gccgov2Magic, gccgov3Magic, goimporterMagic:
// Raw export data.
reader = f
return
@ -195,7 +196,7 @@ func GetImporter(searchpaths []string, initmap map[*types.Package]InitData) Impo
}
switch magics {
case gccgov1Magic, gccgov2Magic:
case gccgov1Magic, gccgov2Magic, gccgov3Magic:
var p parser
p.init(fpath, reader, imports)
pkg = p.parsePackage()

62
src/go/internal/gccgoimporter/importer_test.go
View File

@ -11,7 +11,6 @@ import (
"os"
"os/exec"
"path/filepath"
"runtime"
"testing"
)
@ -53,9 +52,6 @@ func runImporterTest(t *testing.T, imp Importer, initmap map[*types.Package]Init
// Check that the package's own init function has the package's priority
for _, pkginit := range initdata.Inits {
if pkginit.InitFunc == test.wantinits[0] {
if initdata.Priority != pkginit.Priority {
t.Errorf("%s: got self priority %d; want %d", test.pkgpath, pkginit.Priority, initdata.Priority)
}
found = true
break
}
@ -65,27 +61,11 @@ func runImporterTest(t *testing.T, imp Importer, initmap map[*types.Package]Init
t.Errorf("%s: could not find expected function %q", test.pkgpath, test.wantinits[0])
}
// Each init function in the list other than the first one is a
// dependency of the function immediately before it. Check that
// the init functions appear in descending priority order.
priority := initdata.Priority
for _, wantdepinit := range test.wantinits[1:] {
found = false
for _, pkginit := range initdata.Inits {
if pkginit.InitFunc == wantdepinit {
if priority <= pkginit.Priority {
t.Errorf("%s: got dep priority %d; want less than %d", test.pkgpath, pkginit.Priority, priority)
}
found = true
priority = pkginit.Priority
break
}
}
if !found {
t.Errorf("%s: could not find expected function %q", test.pkgpath, wantdepinit)
}
}
// FIXME: the original version of this test was written against
// the v1 export data scheme for capturing init functions, so it
// verified the priority values. We moved away from the priority
// scheme some time ago; it is not clear how much work it would be
// to validate the new init export data.
}
}
@ -100,7 +80,7 @@ var importerTests = [...]importerTest{
{pkgpath: "time", name: "Nanosecond", want: "const Nanosecond Duration", wantval: "1"},
{pkgpath: "unicode", name: "IsUpper", want: "func IsUpper(r rune) bool"},
{pkgpath: "unicode", name: "MaxRune", want: "const MaxRune untyped rune", wantval: "1114111"},
{pkgpath: "imports", wantinits: []string{"imports..import", "fmt..import", "math..import"}},
{pkgpath: "imports", wantinits: []string{"imports..import", "fmt..import"}},
{pkgpath: "importsar", name: "Hello", want: "var Hello string"},
{pkgpath: "aliases", name: "A14", want: "type A14 = func(int, T0) chan T2"},
{pkgpath: "aliases", name: "C0", want: "type C0 struct{f1 C1; f2 C1}"},
@ -109,8 +89,7 @@ var importerTests = [...]importerTest{
}
func TestGoxImporter(t *testing.T) {
testenv.MustHaveGoBuild(t)
testenv.MustHaveExec(t) // this is to skip nacl, js
initmap := make(map[*types.Package]InitData)
imp := GetImporter([]string{"testdata"}, initmap)
@ -119,12 +98,24 @@ func TestGoxImporter(t *testing.T) {
}
}
func TestObjImporter(t *testing.T) {
testenv.MustHaveGoBuild(t)
// gccgoPath returns a path to gccgo if it is present (either in
// path or specified via GCCGO environment variable), or an
// empty string if no gccgo is available.
func gccgoPath() string {
gccgoname := os.Getenv("GCCGO")
if gccgoname == "" {
gccgoname = "gccgo"
}
if gpath, gerr := exec.LookPath(gccgoname); gerr == nil {
return gpath
}
return ""
}
// This test relies on gccgo being around, which it most likely will be if we
// were compiled with gccgo.
if runtime.Compiler != "gccgo" {
func TestObjImporter(t *testing.T) {
// This test relies on gccgo being around.
gpath := gccgoPath()
if gpath == "" {
t.Skip("This test needs gccgo")
}
@ -144,10 +135,13 @@ func TestObjImporter(t *testing.T) {
for _, test := range importerTests {
gofile := filepath.Join("testdata", test.pkgpath+".go")
if _, err := os.Stat(gofile); os.IsNotExist(err) {
continue
}
ofile := filepath.Join(tmpdir, test.pkgpath+".o")
afile := filepath.Join(artmpdir, "lib"+test.pkgpath+".a")
cmd := exec.Command("gccgo", "-fgo-pkgpath="+test.pkgpath, "-c", "-o", ofile, gofile)
cmd := exec.Command(gpath, "-fgo-pkgpath="+test.pkgpath, "-c", "-o", ofile, gofile)
out, err := cmd.CombinedOutput()
if err != nil {
t.Logf("%s", out)

216
src/go/internal/gccgoimporter/parser.go
View File

@ -18,7 +18,7 @@ import (
)
type parser struct {
scanner scanner.Scanner
scanner *scanner.Scanner
version string // format version
tok rune // current token
lit string // literal string; only valid for Ident, Int, String tokens
@ -27,18 +27,24 @@ type parser struct {
pkg *types.Package // reference to imported package
imports map[string]*types.Package // package path -> package object
typeList []types.Type // type number -> type
typeData []string // unparsed type data (v3 and later)
initdata InitData // package init priority data
}
func (p *parser) init(filename string, src io.Reader, imports map[string]*types.Package) {
p.scanner = new(scanner.Scanner)
p.initScanner(filename, src)
p.imports = imports
p.typeList = make([]types.Type, 1 /* type numbers start at 1 */, 16)
}
func (p *parser) initScanner(filename string, src io.Reader) {
p.scanner.Init(src)
p.scanner.Error = func(_ *scanner.Scanner, msg string) { p.error(msg) }
p.scanner.Mode = scanner.ScanIdents | scanner.ScanInts | scanner.ScanFloats | scanner.ScanStrings | scanner.ScanComments | scanner.SkipComments
p.scanner.Whitespace = 1<<'\t' | 1<<'\n' | 1<<' '
p.scanner.Whitespace = 1<<'\t' | 1<<' '
p.scanner.Filename = filename // for good error messages
p.next()
p.imports = imports
p.typeList = make([]types.Type, 1 /* type numbers start at 1 */, 16)
}
type importError struct {
@ -71,6 +77,13 @@ func (p *parser) expect(tok rune) string {
return lit
}
func (p *parser) expectEOL() {
if p.version == "v1" || p.version == "v2" {
p.expect(';')
}
p.expect('\n')
}
func (p *parser) expectKeyword(keyword string) {
lit := p.expect(scanner.Ident)
if lit != keyword {
@ -96,7 +109,7 @@ func (p *parser) parseUnquotedString() string {
buf.WriteString(p.scanner.TokenText())
// This loop needs to examine each character before deciding whether to consume it. If we see a semicolon,
// we need to let it be consumed by p.next().
for ch := p.scanner.Peek(); ch != ';' && ch != scanner.EOF && p.scanner.Whitespace&(1<<uint(ch)) == 0; ch = p.scanner.Peek() {
for ch := p.scanner.Peek(); ch != '\n' && ch != ';' && ch != scanner.EOF && p.scanner.Whitespace&(1<<uint(ch)) == 0; ch = p.scanner.Peek() {
buf.WriteRune(ch)
p.scanner.Next()
}
@ -387,17 +400,42 @@ var reserved = new(struct{ types.Type })
// reserve reserves the type map entry n for future use.
func (p *parser) reserve(n int) {
if n != len(p.typeList) {
p.errorf("invalid type number %d (out of sync)", n)
// Notes:
// - for pre-V3 export data, the type numbers we see are
// guaranteed to be in increasing order, so we append a
// reserved entry onto the list.
// - for V3+ export data, type numbers can appear in
// any order, however the 'types' section tells us the
// total number of types, hence typeList is pre-allocated.
if len(p.typeData) == 0 {
if n != len(p.typeList) {
p.errorf("invalid type number %d (out of sync)", n)
}
p.typeList = append(p.typeList, reserved)
} else {
if p.typeList[n] != nil {
p.errorf("previously visited type number %d", n)
}
p.typeList[n] = reserved
}
p.typeList = append(p.typeList, reserved)
}
// update sets the type map entries for the given type numbers nlist to t.
func (p *parser) update(t types.Type, nlist []int) {
if len(nlist) != 0 {
if t == reserved {
p.errorf("internal error: update(%v) invoked on reserved", nlist)
}
if t == nil {
p.errorf("internal error: update(%v) invoked on nil", nlist)
}
}
for _, n := range nlist {
if p.typeList[n] == t {
continue
}
if p.typeList[n] != reserved {
p.errorf("typeMap[%d] not reserved", n)
p.errorf("internal error: update(%v): %d not reserved", nlist, n)
}
p.typeList[n] = t
}
@ -459,19 +497,22 @@ func (p *parser) parseNamedType(nlist []int) types.Type {
nt.SetUnderlying(underlying.Underlying())
}
// collect associated methods
for p.tok == scanner.Ident {
p.expectKeyword("func")
p.expect('(')
receiver, _ := p.parseParam(pkg)
p.expect(')')
name := p.parseName()
params, isVariadic := p.parseParamList(pkg)
results := p.parseResultList(pkg)
p.expect(';')
if p.tok == '\n' {
p.next()
// collect associated methods
for p.tok == scanner.Ident {
p.expectKeyword("func")
p.expect('(')
receiver, _ := p.parseParam(pkg)
p.expect(')')
name := p.parseName()
params, isVariadic := p.parseParamList(pkg)
results := p.parseResultList(pkg)
p.expectEOL()
sig := types.NewSignature(receiver, params, results, isVariadic)
nt.AddMethod(types.NewFunc(token.NoPos, pkg, name, sig))
sig := types.NewSignature(receiver, params, results, isVariadic)
nt.AddMethod(types.NewFunc(token.NoPos, pkg, name, sig))
}
}
return nt
@ -790,9 +831,12 @@ func (p *parser) parseType(pkg *types.Package, n ...int) (t types.Type) {
case scanner.Int:
n1 := p.parseInt()
if p.tok == '>' {
if len(p.typeData) > 0 && p.typeList[n1] == nil {
p.parseSavedType(pkg, n1, n)
}
t = p.typeList[n1]
if t == reserved {
p.errorf("invalid type cycle, type %d not yet defined", n1)
if len(p.typeData) == 0 && t == reserved {
p.errorf("invalid type cycle, type %d not yet defined (nlist=%v)", n1, n)
}
p.update(t, n)
} else {
@ -808,12 +852,86 @@ func (p *parser) parseType(pkg *types.Package, n ...int) (t types.Type) {
default:
p.errorf("expected type number, got %s (%q)", scanner.TokenString(p.tok), p.lit)
return nil
}
if t == nil || t == reserved {
p.errorf("internal error: bad return from parseType(%v)", n)
}
p.expect('>')
return
}
// Types = "types" maxp1 exportedp1 (offset length)* .
func (p *parser) parseTypes(pkg *types.Package) {
maxp1 := p.parseInt()
exportedp1 := p.parseInt()
p.typeList = make([]types.Type, maxp1, maxp1)
type typeOffset struct {
offset int
length int
}
var typeOffsets []typeOffset
total := 0
for i := 1; i < maxp1; i++ {
len := p.parseInt()
typeOffsets = append(typeOffsets, typeOffset{total, len})
total += len
}
// We should now have p.tok pointing to the final newline.
// The next runes from the scanner should be the type data.
var sb strings.Builder
for sb.Len() < total {
r := p.scanner.Next()
if r == scanner.EOF {
p.error("unexpected EOF")
}
sb.WriteRune(r)
}
allTypeData := sb.String()
p.typeData = []string{""} // type 0, unused
for _, to := range typeOffsets {
p.typeData = append(p.typeData, allTypeData[to.offset:to.offset+to.length])
}
for i := 1; i < int(exportedp1); i++ {
p.parseSavedType(pkg, i, []int{})
}
}
// parseSavedType parses one saved type definition.
func (p *parser) parseSavedType(pkg *types.Package, i int, nlist []int) {
defer func(s *scanner.Scanner, tok rune, lit string) {
p.scanner = s
p.tok = tok
p.lit = lit
}(p.scanner, p.tok, p.lit)
p.scanner = new(scanner.Scanner)
p.initScanner(p.scanner.Filename, strings.NewReader(p.typeData[i]))
p.expectKeyword("type")
id := p.parseInt()
if id != i {
p.errorf("type ID mismatch: got %d, want %d", id, i)
}
if p.typeList[i] == reserved {
p.errorf("internal error: %d already reserved in parseSavedType", i)
}
if p.typeList[i] == nil {
p.reserve(i)
p.parseTypeSpec(pkg, append(nlist, i))
}
if p.typeList[i] == nil || p.typeList[i] == reserved {
p.errorf("internal error: parseSavedType(%d,%v) reserved/nil", i, nlist)
}
}
// PackageInit = unquotedString unquotedString int .
func (p *parser) parsePackageInit() PackageInit {
name := p.parseUnquotedString()
@ -829,7 +947,7 @@ func (p *parser) parsePackageInit() PackageInit {
func (p *parser) discardDirectiveWhileParsingTypes(pkg *types.Package) {
for {
switch p.tok {
case ';':
case '\n', ';':
return
case '<':
p.parseType(pkg)
@ -848,7 +966,7 @@ func (p *parser) maybeCreatePackage() {
}
}
// InitDataDirective = ( "v1" | "v2" ) ";" |
// InitDataDirective = ( "v1" | "v2" | "v3" ) ";" |
// "priority" int ";" |
// "init" { PackageInit } ";" |
// "checksum" unquotedString ";" .
@ -859,31 +977,32 @@ func (p *parser) parseInitDataDirective() {
}
switch p.lit {
case "v1", "v2":
case "v1", "v2", "v3":
p.version = p.lit
p.next()
p.expect(';')
p.expect('\n')
case "priority":
p.next()
p.initdata.Priority = p.parseInt()
p.expect(';')
p.expectEOL()
case "init":
p.next()
for p.tok != ';' && p.tok != scanner.EOF {
for p.tok != '\n' && p.tok != ';' && p.tok != scanner.EOF {
p.initdata.Inits = append(p.initdata.Inits, p.parsePackageInit())
}
p.expect(';')
p.expectEOL()
case "init_graph":
p.next()
// The graph data is thrown away for now.
for p.tok != ';' && p.tok != scanner.EOF {
for p.tok != '\n' && p.tok != ';' && p.tok != scanner.EOF {
p.parseInt64()
p.parseInt64()
}
p.expect(';')
p.expectEOL()
case "checksum":
// Don't let the scanner try to parse the checksum as a number.
@ -893,7 +1012,7 @@ func (p *parser) parseInitDataDirective() {
p.scanner.Mode &^= scanner.ScanInts | scanner.ScanFloats
p.next()
p.parseUnquotedString()
p.expect(';')
p.expectEOL()
default:
p.errorf("unexpected identifier: %q", p.lit)
@ -905,6 +1024,7 @@ func (p *parser) parseInitDataDirective() {
// "pkgpath" unquotedString ";" |
// "prefix" unquotedString ";" |
// "import" unquotedString unquotedString string ";" |
// "indirectimport" unquotedString unquotedstring ";" |
// "func" Func ";" |
// "type" Type ";" |
// "var" Var ";" |
@ -916,29 +1036,29 @@ func (p *parser) parseDirective() {
}
switch p.lit {
case "v1", "v2", "priority", "init", "init_graph", "checksum":
case "v1", "v2", "v3", "priority", "init", "init_graph", "checksum":
p.parseInitDataDirective()
case "package":
p.next()
p.pkgname = p.parseUnquotedString()
p.maybeCreatePackage()
if p.version == "v2" && p.tok != ';' {
if p.version != "v1" && p.tok != '\n' && p.tok != ';' {
p.parseUnquotedString()
p.parseUnquotedString()
}
p.expect(';')
p.expectEOL()
case "pkgpath":
p.next()
p.pkgpath = p.parseUnquotedString()
p.maybeCreatePackage()
p.expect(';')
p.expectEOL()
case "prefix":
p.next()
p.pkgpath = p.parseUnquotedString()
p.expect(';')
p.expectEOL()
case "import":
p.next()
@ -946,7 +1066,19 @@ func (p *parser) parseDirective() {
pkgpath := p.parseUnquotedString()
p.getPkg(pkgpath, pkgname)
p.parseString()
p.expect(';')
p.expectEOL()
case "indirectimport":
p.next()
pkgname := p.parseUnquotedString()
pkgpath := p.parseUnquotedString()
p.getPkg(pkgpath, pkgname)
p.expectEOL()
case "types":
p.next()
p.parseTypes(p.pkg)
p.expectEOL()
case "func":
p.next()
@ -954,24 +1086,24 @@ func (p *parser) parseDirective() {
if fun != nil {
p.pkg.Scope().Insert(fun)
}
p.expect(';')
p.expectEOL()
case "type":
p.next()
p.parseType(p.pkg)
p.expect(';')
p.expectEOL()
case "var":
p.next()
v := p.parseVar(p.pkg)
p.pkg.Scope().Insert(v)
p.expect(';')
p.expectEOL()
case "const":
p.next()
c := p.parseConst(p.pkg)
p.pkg.Scope().Insert(c)
p.expect(';')
p.expectEOL()
default:
p.errorf("unexpected identifier: %q", p.lit)

3
src/go/internal/gccgoimporter/parser_test.go
View File

@ -19,7 +19,7 @@ var typeParserTests = []struct {
{id: "foo", typ: "<type 1 *<type -19>>", want: "*error"},
{id: "foo", typ: "<type 1 *any>", want: "unsafe.Pointer"},
{id: "foo", typ: "<type 1 \"Bar\" <type 2 *<type 1>>>", want: "foo.Bar", underlying: "*foo.Bar"},
{id: "foo", typ: "<type 1 \"bar.Foo\" \"bar\" <type -1> func (? <type 1>) M (); >", want: "bar.Foo", underlying: "int8", methods: "func (bar.Foo).M()"},
{id: "foo", typ: "<type 1 \"bar.Foo\" \"bar\" <type -1>\nfunc (? <type 1>) M ();\n>", want: "bar.Foo", underlying: "int8", methods: "func (bar.Foo).M()"},
{id: "foo", typ: "<type 1 \".bar.foo\" \"bar\" <type -1>>", want: "bar.foo", underlying: "int8"},
{id: "foo", typ: "<type 1 []<type -1>>", want: "[]int8"},
{id: "foo", typ: "<type 1 [42]<type -1>>", want: "[42]int8"},
@ -36,6 +36,7 @@ func TestTypeParser(t *testing.T) {
for _, test := range typeParserTests {
var p parser
p.init("test.gox", strings.NewReader(test.typ), make(map[string]*types.Package))
p.version = "v2"
p.pkgname = test.id
p.pkgpath = test.id
p.maybeCreatePackage()