1
0
mirror of https://github.com/golang/go synced 2024-11-18 18:14:43 -07:00

go/internal/gcimporter: support indexed export data

Support indexed export data in the x/tools gcimporter. Fixes tests with
tip.

This is just a copy of the bimport.go and iimport.go files in the
standard library gcimporter package, including some minor fixes to the
existing bimport.go. The iexport logic in x/tools still needs to be
updated.

Fixes golang/go#25052

Change-Id: I2858e5c0853735c904f32b7b27c1c288a9e62e88
Reviewed-on: https://go-review.googlesource.com/109595
Reviewed-by: Alan Donovan <adonovan@google.com>
Reviewed-by: Matthew Dempsky <mdempsky@google.com>
This commit is contained in:
Rebecca Stambler 2018-04-26 13:58:52 -04:00 committed by Alan Donovan
parent c1def519f0
commit 6e0e2181b9
2 changed files with 662 additions and 46 deletions

View File

@ -39,8 +39,7 @@ type importer struct {
posInfoFormat bool
prevFile string
prevLine int
fset *token.FileSet
files map[string]*token.File
fake fakeFileSet
// debugging support
debugFormat bool
@ -62,6 +61,10 @@ func BImportData(fset *token.FileSet, imports map[string]*types.Package, data []
}
}()
if len(data) > 0 && data[0] == 'i' {
return iImportData(fset, imports, data[1:], path)
}
p := importer{
imports: imports,
data: data,
@ -69,8 +72,10 @@ func BImportData(fset *token.FileSet, imports map[string]*types.Package, data []
version: -1, // unknown version
strList: []string{""}, // empty string is mapped to 0
pathList: []string{""}, // empty string is mapped to 0
fset: fset,
files: make(map[string]*token.File),
fake: fakeFileSet{
fset: fset,
files: make(map[string]*token.File),
},
}
// read version info
@ -125,7 +130,7 @@ func BImportData(fset *token.FileSet, imports map[string]*types.Package, data []
// read package data
pkg = p.pkg()
// read objects of phase 1 only (see cmd/compiler/internal/gc/bexport.go)
// read objects of phase 1 only (see cmd/compile/internal/gc/bexport.go)
objcount := 0
for {
tag := p.tagOrIndex()
@ -262,7 +267,7 @@ func (p *importer) obj(tag int) {
case constTag:
pos := p.pos()
pkg, name := p.qualifiedName()
typ := p.typ(nil)
typ := p.typ(nil, nil)
val := p.value()
p.declare(types.NewConst(pos, pkg, name, typ, val))
@ -270,16 +275,16 @@ func (p *importer) obj(tag int) {
// TODO(gri) verify type alias hookup is correct
pos := p.pos()
pkg, name := p.qualifiedName()
typ := p.typ(nil)
typ := p.typ(nil, nil)
p.declare(types.NewTypeName(pos, pkg, name, typ))
case typeTag:
p.typ(nil)
p.typ(nil, nil)
case varTag:
pos := p.pos()
pkg, name := p.qualifiedName()
typ := p.typ(nil)
typ := p.typ(nil, nil)
p.declare(types.NewVar(pos, pkg, name, typ))
case funcTag:
@ -326,15 +331,23 @@ func (p *importer) pos() token.Pos {
p.prevFile = file
p.prevLine = line
// Synthesize a token.Pos
return p.fake.pos(file, line)
}
// Synthesize a token.Pos
type fakeFileSet struct {
fset *token.FileSet
files map[string]*token.File
}
func (s *fakeFileSet) pos(file string, line int) token.Pos {
// Since we don't know the set of needed file positions, we
// reserve maxlines positions per file.
const maxlines = 64 * 1024
f := p.files[file]
f := s.files[file]
if f == nil {
f = p.fset.AddFile(file, -1, maxlines)
p.files[file] = f
f = s.fset.AddFile(file, -1, maxlines)
s.files[file] = f
// Allocate the fake linebreak indices on first use.
// TODO(adonovan): opt: save ~512KB using a more complex scheme?
fakeLinesOnce.Do(func() {
@ -384,7 +397,11 @@ func (t *dddSlice) String() string { return "..." + t.elem.String() }
// the package currently imported. The parent package is needed for
// exported struct fields and interface methods which don't contain
// explicit package information in the export data.
func (p *importer) typ(parent *types.Package) types.Type {
//
// A non-nil tname is used as the "owner" of the result type; i.e.,
// the result type is the underlying type of tname. tname is used
// to give interface methods a named receiver type where possible.
func (p *importer) typ(parent *types.Package, tname *types.Named) types.Type {
// if the type was seen before, i is its index (>= 0)
i := p.tagOrIndex()
if i >= 0 {
@ -414,15 +431,15 @@ func (p *importer) typ(parent *types.Package) types.Type {
t0 := types.NewNamed(obj.(*types.TypeName), nil, nil)
// but record the existing type, if any
t := obj.Type().(*types.Named)
p.record(t)
tname := obj.Type().(*types.Named) // tname is either t0 or the existing type
p.record(tname)
// read underlying type
t0.SetUnderlying(p.typ(parent))
t0.SetUnderlying(p.typ(parent, t0))
// interfaces don't have associated methods
if types.IsInterface(t0) {
return t
return tname
}
// read associated methods
@ -443,7 +460,7 @@ func (p *importer) typ(parent *types.Package) types.Type {
t0.AddMethod(types.NewFunc(pos, parent, name, sig))
}
return t
return tname
case arrayTag:
t := new(types.Array)
@ -452,7 +469,7 @@ func (p *importer) typ(parent *types.Package) types.Type {
}
n := p.int64()
*t = *types.NewArray(p.typ(parent), n)
*t = *types.NewArray(p.typ(parent, nil), n)
return t
case sliceTag:
@ -461,7 +478,7 @@ func (p *importer) typ(parent *types.Package) types.Type {
p.record(t)
}
*t = *types.NewSlice(p.typ(parent))
*t = *types.NewSlice(p.typ(parent, nil))
return t
case dddTag:
@ -470,7 +487,7 @@ func (p *importer) typ(parent *types.Package) types.Type {
p.record(t)
}
t.elem = p.typ(parent)
t.elem = p.typ(parent, nil)
return t
case structTag:
@ -488,7 +505,7 @@ func (p *importer) typ(parent *types.Package) types.Type {
p.record(t)
}
*t = *types.NewPointer(p.typ(parent))
*t = *types.NewPointer(p.typ(parent, nil))
return t
case signatureTag:
@ -507,6 +524,8 @@ func (p *importer) typ(parent *types.Package) types.Type {
// cannot expect the interface type to appear in a cycle, as any
// such cycle must contain a named type which would have been
// first defined earlier.
// TODO(gri) Is this still true now that we have type aliases?
// See issue #23225.
n := len(p.typList)
if p.trackAllTypes {
p.record(nil)
@ -515,10 +534,10 @@ func (p *importer) typ(parent *types.Package) types.Type {
var embeddeds []*types.Named
for n := p.int(); n > 0; n-- {
p.pos()
embeddeds = append(embeddeds, p.typ(parent).(*types.Named))
embeddeds = append(embeddeds, p.typ(parent, nil).(*types.Named))
}
t := types.NewInterface(p.methodList(parent), embeddeds)
t := types.NewInterface(p.methodList(parent, tname), embeddeds)
p.interfaceList = append(p.interfaceList, t)
if p.trackAllTypes {
p.typList[n] = t
@ -531,8 +550,8 @@ func (p *importer) typ(parent *types.Package) types.Type {
p.record(t)
}
key := p.typ(parent)
val := p.typ(parent)
key := p.typ(parent, nil)
val := p.typ(parent, nil)
*t = *types.NewMap(key, val)
return t
@ -542,19 +561,8 @@ func (p *importer) typ(parent *types.Package) types.Type {
p.record(t)
}
var dir types.ChanDir
// tag values must match the constants in cmd/compile/internal/gc/go.go
switch d := p.int(); d {
case 1 /* Crecv */ :
dir = types.RecvOnly
case 2 /* Csend */ :
dir = types.SendOnly
case 3 /* Cboth */ :
dir = types.SendRecv
default:
errorf("unexpected channel dir %d", d)
}
val := p.typ(parent)
dir := chanDir(p.int())
val := p.typ(parent, nil)
*t = *types.NewChan(dir, val)
return t
@ -564,6 +572,21 @@ func (p *importer) typ(parent *types.Package) types.Type {
}
}
func chanDir(d int) types.ChanDir {
// tag values must match the constants in cmd/compile/internal/gc/go.go
switch d {
case 1 /* Crecv */ :
return types.RecvOnly
case 2 /* Csend */ :
return types.SendOnly
case 3 /* Cboth */ :
return types.SendRecv
default:
errorf("unexpected channel dir %d", d)
return 0
}
}
func (p *importer) fieldList(parent *types.Package) (fields []*types.Var, tags []string) {
if n := p.int(); n > 0 {
fields = make([]*types.Var, n)
@ -578,7 +601,7 @@ func (p *importer) fieldList(parent *types.Package) (fields []*types.Var, tags [
func (p *importer) field(parent *types.Package) (*types.Var, string) {
pos := p.pos()
pkg, name, alias := p.fieldName(parent)
typ := p.typ(parent)
typ := p.typ(parent, nil)
tag := p.string()
anonymous := false
@ -602,22 +625,30 @@ func (p *importer) field(parent *types.Package) (*types.Var, string) {
return types.NewField(pos, pkg, name, typ, anonymous), tag
}
func (p *importer) methodList(parent *types.Package) (methods []*types.Func) {
func (p *importer) methodList(parent *types.Package, baseType *types.Named) (methods []*types.Func) {
if n := p.int(); n > 0 {
methods = make([]*types.Func, n)
for i := range methods {
methods[i] = p.method(parent)
methods[i] = p.method(parent, baseType)
}
}
return
}
func (p *importer) method(parent *types.Package) *types.Func {
func (p *importer) method(parent *types.Package, baseType *types.Named) *types.Func {
pos := p.pos()
pkg, name, _ := p.fieldName(parent)
// If we don't have a baseType, use a nil receiver.
// A receiver using the actual interface type (which
// we don't know yet) will be filled in when we call
// types.Interface.Complete.
var recv *types.Var
if baseType != nil {
recv = types.NewVar(token.NoPos, parent, "", baseType)
}
params, isddd := p.paramList()
result, _ := p.paramList()
sig := types.NewSignature(nil, params, result, isddd)
sig := types.NewSignature(recv, params, result, isddd)
return types.NewFunc(pos, pkg, name, sig)
}
@ -673,7 +704,7 @@ func (p *importer) paramList() (*types.Tuple, bool) {
}
func (p *importer) param(named bool) (*types.Var, bool) {
t := p.typ(nil)
t := p.typ(nil, nil)
td, isddd := t.(*dddSlice)
if isddd {
t = types.NewSlice(td.elem)

View File

@ -0,0 +1,585 @@
// Copyright 2018 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Indexed package import.
// See cmd/compile/internal/gc/iexport.go for the export data format.
// This file is a copy of $GOROOT/src/go/internal/gcimporter/iimport.go.
package gcimporter
import (
"bytes"
"encoding/binary"
"go/constant"
"go/token"
"go/types"
"io"
"sort"
)
type intReader struct {
*bytes.Reader
path string
}
func (r *intReader) int64() int64 {
i, err := binary.ReadVarint(r.Reader)
if err != nil {
errorf("import %q: read varint error: %v", r.path, err)
}
return i
}
func (r *intReader) uint64() uint64 {
i, err := binary.ReadUvarint(r.Reader)
if err != nil {
errorf("import %q: read varint error: %v", r.path, err)
}
return i
}
const predeclReserved = 32
type itag uint64
const (
// Types
definedType itag = iota
pointerType
sliceType
arrayType
chanType
mapType
signatureType
structType
interfaceType
)
// iImportData imports a package from the serialized package data
// and returns the number of bytes consumed and a reference to the package.
// If the export data version is not recognized or the format is otherwise
// compromised, an error is returned.
func iImportData(fset *token.FileSet, imports map[string]*types.Package, data []byte, path string) (_ int, pkg *types.Package, err error) {
r := &intReader{bytes.NewReader(data), path}
version := r.uint64()
switch version {
case 0:
default:
errorf("cannot import %q: unknown iexport format version %d", path, version)
}
sLen := int64(r.uint64())
dLen := int64(r.uint64())
whence, _ := r.Seek(0, io.SeekCurrent)
stringData := data[whence : whence+sLen]
declData := data[whence+sLen : whence+sLen+dLen]
r.Seek(sLen+dLen, io.SeekCurrent)
p := iimporter{
ipath: path,
stringData: stringData,
stringCache: make(map[uint64]string),
pkgCache: make(map[uint64]*types.Package),
declData: declData,
pkgIndex: make(map[*types.Package]map[string]uint64),
typCache: make(map[uint64]types.Type),
fake: fakeFileSet{
fset: fset,
files: make(map[string]*token.File),
},
}
for i, pt := range predeclared {
p.typCache[uint64(i)] = pt
}
pkgList := make([]*types.Package, r.uint64())
for i := range pkgList {
pkgPathOff := r.uint64()
pkgPath := p.stringAt(pkgPathOff)
pkgName := p.stringAt(r.uint64())
_ = r.uint64() // package height; unused by go/types
if pkgPath == "" {
pkgPath = path
}
pkg := imports[pkgPath]
if pkg == nil {
pkg = types.NewPackage(pkgPath, pkgName)
imports[pkgPath] = pkg
} else if pkg.Name() != pkgName {
errorf("conflicting names %s and %s for package %q", pkg.Name(), pkgName, path)
}
p.pkgCache[pkgPathOff] = pkg
nameIndex := make(map[string]uint64)
for nSyms := r.uint64(); nSyms > 0; nSyms-- {
name := p.stringAt(r.uint64())
nameIndex[name] = r.uint64()
}
p.pkgIndex[pkg] = nameIndex
pkgList[i] = pkg
}
localpkg := pkgList[0]
names := make([]string, 0, len(p.pkgIndex[localpkg]))
for name := range p.pkgIndex[localpkg] {
names = append(names, name)
}
sort.Strings(names)
for _, name := range names {
p.doDecl(localpkg, name)
}
for _, typ := range p.interfaceList {
typ.Complete()
}
// record all referenced packages as imports
list := append(([]*types.Package)(nil), pkgList[1:]...)
sort.Sort(byPath(list))
localpkg.SetImports(list)
// package was imported completely and without errors
localpkg.MarkComplete()
consumed, _ := r.Seek(0, io.SeekCurrent)
return int(consumed), localpkg, nil
}
type iimporter struct {
ipath string
stringData []byte
stringCache map[uint64]string
pkgCache map[uint64]*types.Package
declData []byte
pkgIndex map[*types.Package]map[string]uint64
typCache map[uint64]types.Type
fake fakeFileSet
interfaceList []*types.Interface
}
func (p *iimporter) doDecl(pkg *types.Package, name string) {
// See if we've already imported this declaration.
if obj := pkg.Scope().Lookup(name); obj != nil {
return
}
off, ok := p.pkgIndex[pkg][name]
if !ok {
errorf("%v.%v not in index", pkg, name)
}
r := &importReader{p: p, currPkg: pkg}
r.declReader.Reset(p.declData[off:])
r.obj(name)
}
func (p *iimporter) stringAt(off uint64) string {
if s, ok := p.stringCache[off]; ok {
return s
}
slen, n := binary.Uvarint(p.stringData[off:])
if n <= 0 {
errorf("varint failed")
}
spos := off + uint64(n)
s := string(p.stringData[spos : spos+slen])
p.stringCache[off] = s
return s
}
func (p *iimporter) pkgAt(off uint64) *types.Package {
if pkg, ok := p.pkgCache[off]; ok {
return pkg
}
path := p.stringAt(off)
errorf("missing package %q in %q", path, p.ipath)
return nil
}
func (p *iimporter) typAt(off uint64, base *types.Named) types.Type {
if t, ok := p.typCache[off]; ok && (base == nil || !isInterface(t)) {
return t
}
if off < predeclReserved {
errorf("predeclared type missing from cache: %v", off)
}
r := &importReader{p: p}
r.declReader.Reset(p.declData[off-predeclReserved:])
t := r.doType(base)
if base == nil || !isInterface(t) {
p.typCache[off] = t
}
return t
}
type importReader struct {
p *iimporter
declReader bytes.Reader
currPkg *types.Package
prevFile string
prevLine int64
}
func (r *importReader) obj(name string) {
tag := r.byte()
pos := r.pos()
switch tag {
case 'A':
typ := r.typ()
r.declare(types.NewTypeName(pos, r.currPkg, name, typ))
case 'C':
typ, val := r.value()
r.declare(types.NewConst(pos, r.currPkg, name, typ, val))
case 'F':
sig := r.signature(nil)
r.declare(types.NewFunc(pos, r.currPkg, name, sig))
case 'T':
// Types can be recursive. We need to setup a stub
// declaration before recursing.
obj := types.NewTypeName(pos, r.currPkg, name, nil)
named := types.NewNamed(obj, nil, nil)
r.declare(obj)
underlying := r.p.typAt(r.uint64(), named).Underlying()
named.SetUnderlying(underlying)
if !isInterface(underlying) {
for n := r.uint64(); n > 0; n-- {
mpos := r.pos()
mname := r.ident()
recv := r.param()
msig := r.signature(recv)
named.AddMethod(types.NewFunc(mpos, r.currPkg, mname, msig))
}
}
case 'V':
typ := r.typ()
r.declare(types.NewVar(pos, r.currPkg, name, typ))
default:
errorf("unexpected tag: %v", tag)
}
}
func (r *importReader) declare(obj types.Object) {
obj.Pkg().Scope().Insert(obj)
}
func (r *importReader) value() (typ types.Type, val constant.Value) {
typ = r.typ()
switch b := typ.Underlying().(*types.Basic); b.Info() & types.IsConstType {
case types.IsBoolean:
val = constant.MakeBool(r.bool())
case types.IsString:
val = constant.MakeString(r.string())
case types.IsInteger:
val = r.mpint(b)
case types.IsFloat:
val = r.mpfloat(b)
case types.IsComplex:
re := r.mpfloat(b)
im := r.mpfloat(b)
val = constant.BinaryOp(re, token.ADD, constant.MakeImag(im))
default:
errorf("unexpected type %v", typ) // panics
panic("unreachable")
}
return
}
func intSize(b *types.Basic) (signed bool, maxBytes uint) {
if (b.Info() & types.IsUntyped) != 0 {
return true, 64
}
switch b.Kind() {
case types.Float32, types.Complex64:
return true, 3
case types.Float64, types.Complex128:
return true, 7
}
signed = (b.Info() & types.IsUnsigned) == 0
switch b.Kind() {
case types.Int8, types.Uint8:
maxBytes = 1
case types.Int16, types.Uint16:
maxBytes = 2
case types.Int32, types.Uint32:
maxBytes = 4
default:
maxBytes = 8
}
return
}
func (r *importReader) mpint(b *types.Basic) constant.Value {
signed, maxBytes := intSize(b)
maxSmall := 256 - maxBytes
if signed {
maxSmall = 256 - 2*maxBytes
}
if maxBytes == 1 {
maxSmall = 256
}
n, _ := r.declReader.ReadByte()
if uint(n) < maxSmall {
v := int64(n)
if signed {
v >>= 1
if n&1 != 0 {
v = ^v
}
}
return constant.MakeInt64(v)
}
v := -n
if signed {
v = -(n &^ 1) >> 1
}
if v < 1 || uint(v) > maxBytes {
errorf("weird decoding: %v, %v => %v", n, signed, v)
}
buf := make([]byte, v)
io.ReadFull(&r.declReader, buf)
// convert to little endian
// TODO(gri) go/constant should have a more direct conversion function
// (e.g., once it supports a big.Float based implementation)
for i, j := 0, len(buf)-1; i < j; i, j = i+1, j-1 {
buf[i], buf[j] = buf[j], buf[i]
}
x := constant.MakeFromBytes(buf)
if signed && n&1 != 0 {
x = constant.UnaryOp(token.SUB, x, 0)
}
return x
}
func (r *importReader) mpfloat(b *types.Basic) constant.Value {
x := r.mpint(b)
if constant.Sign(x) == 0 {
return x
}
exp := r.int64()
switch {
case exp > 0:
x = constant.Shift(x, token.SHL, uint(exp))
case exp < 0:
d := constant.Shift(constant.MakeInt64(1), token.SHL, uint(-exp))
x = constant.BinaryOp(x, token.QUO, d)
}
return x
}
func (r *importReader) ident() string {
return r.string()
}
func (r *importReader) qualifiedIdent() (*types.Package, string) {
name := r.string()
pkg := r.pkg()
return pkg, name
}
func (r *importReader) pos() token.Pos {
delta := r.int64()
if delta != deltaNewFile {
r.prevLine += delta
} else if l := r.int64(); l == -1 {
r.prevLine += deltaNewFile
} else {
r.prevFile = r.string()
r.prevLine = l
}
if r.prevFile == "" && r.prevLine == 0 {
return token.NoPos
}
return r.p.fake.pos(r.prevFile, int(r.prevLine))
}
func (r *importReader) typ() types.Type {
return r.p.typAt(r.uint64(), nil)
}
func isInterface(t types.Type) bool {
_, ok := t.(*types.Interface)
return ok
}
func (r *importReader) pkg() *types.Package { return r.p.pkgAt(r.uint64()) }
func (r *importReader) string() string { return r.p.stringAt(r.uint64()) }
func (r *importReader) doType(base *types.Named) types.Type {
switch k := r.kind(); k {
default:
errorf("unexpected kind tag in %q: %v", r.p.ipath, k)
return nil
case definedType:
pkg, name := r.qualifiedIdent()
r.p.doDecl(pkg, name)
return pkg.Scope().Lookup(name).(*types.TypeName).Type()
case pointerType:
return types.NewPointer(r.typ())
case sliceType:
return types.NewSlice(r.typ())
case arrayType:
n := r.uint64()
return types.NewArray(r.typ(), int64(n))
case chanType:
dir := chanDir(int(r.uint64()))
return types.NewChan(dir, r.typ())
case mapType:
return types.NewMap(r.typ(), r.typ())
case signatureType:
r.currPkg = r.pkg()
return r.signature(nil)
case structType:
r.currPkg = r.pkg()
fields := make([]*types.Var, r.uint64())
tags := make([]string, len(fields))
for i := range fields {
fpos := r.pos()
fname := r.ident()
ftyp := r.typ()
emb := r.bool()
tag := r.string()
fields[i] = types.NewField(fpos, r.currPkg, fname, ftyp, emb)
tags[i] = tag
}
return types.NewStruct(fields, tags)
case interfaceType:
r.currPkg = r.pkg()
embeddeds := make([]*types.Named, r.uint64())
for i := range embeddeds {
_ = r.pos()
embeddeds[i] = r.typ().(*types.Named)
}
methods := make([]*types.Func, r.uint64())
for i := range methods {
mpos := r.pos()
mname := r.ident()
// TODO(mdempsky): Matches bimport.go, but I
// don't agree with this.
var recv *types.Var
if base != nil {
recv = types.NewVar(token.NoPos, r.currPkg, "", base)
}
msig := r.signature(recv)
methods[i] = types.NewFunc(mpos, r.currPkg, mname, msig)
}
typ := types.NewInterface(methods, embeddeds)
r.p.interfaceList = append(r.p.interfaceList, typ)
return typ
}
}
func (r *importReader) kind() itag {
return itag(r.uint64())
}
func (r *importReader) signature(recv *types.Var) *types.Signature {
params := r.paramList()
results := r.paramList()
variadic := params.Len() > 0 && r.bool()
return types.NewSignature(recv, params, results, variadic)
}
func (r *importReader) paramList() *types.Tuple {
xs := make([]*types.Var, r.uint64())
for i := range xs {
xs[i] = r.param()
}
return types.NewTuple(xs...)
}
func (r *importReader) param() *types.Var {
pos := r.pos()
name := r.ident()
typ := r.typ()
return types.NewParam(pos, r.currPkg, name, typ)
}
func (r *importReader) bool() bool {
return r.uint64() != 0
}
func (r *importReader) int64() int64 {
n, err := binary.ReadVarint(&r.declReader)
if err != nil {
errorf("readVarint: %v", err)
}
return n
}
func (r *importReader) uint64() uint64 {
n, err := binary.ReadUvarint(&r.declReader)
if err != nil {
errorf("readUvarint: %v", err)
}
return n
}
func (r *importReader) byte() byte {
x, err := r.declReader.ReadByte()
if err != nil {
errorf("declReader.ReadByte: %v", err)
}
return x
}