go/gcimporter15: set Pos attribute of decoded types.Objects

BImportData now requires a token.FileSet.
Column numbers are not exported, and are always reported as 1.
Line numbers greater than the limit (currently 64K) are reported as 1.

+ Test that file/line info is correctly preserved for the entire
  standard library.

Change-Id: I80cf370685320240dfb262d36fafd6cdf8569bfb
Reviewed-on: https://go-review.googlesource.com/22788
Reviewed-by: Robert Griesemer <gri@golang.org>

go/gcimporter15/bexport_test.go

@@ -11,10 +11,12 @@ import (
 	"go/ast"
 	"go/build"
 	"go/constant"
+	"go/parser"
 	"go/token"
 	"go/types"
 	"reflect"
 	"runtime"
+	"strings"
 	"testing"
 
 	"golang.org/x/tools/go/buildutil"
@@ -66,7 +68,8 @@ type UnknownType undefined
 		exportdata := gcimporter.BExportData(conf.Fset, pkg)
 
 		imports := make(map[string]*types.Package)
-		n, pkg2, err := gcimporter.BImportData(imports, exportdata, pkg.Path())
+		fset2 := token.NewFileSet()
+		n, pkg2, err := gcimporter.BImportData(fset2, imports, exportdata, pkg.Path())
 		if err != nil {
 			t.Errorf("BImportData(%s): %v", pkg.Path(), err)
 			continue
@@ -87,6 +90,14 @@ type UnknownType undefined
 				t.Errorf("%s.%s not found, want %s", pkg.Path(), name, obj1)
 				continue
 			}
+
+			fl1 := fileLine(conf.Fset, obj1)
+			fl2 := fileLine(fset2, obj2)
+			if fl1 != fl2 {
+				t.Errorf("%s.%s: got posn %s, want %s",
+					pkg.Path(), name, fl2, fl1)
+			}
+
 			if err := equalObj(obj1, obj2); err != nil {
 				t.Errorf("%s.%s: %s\ngot:  %s\nwant: %s",
 					pkg.Path(), name, err, obj2, obj1)
@@ -95,6 +106,11 @@ type UnknownType undefined
 	}
 }
 
+func fileLine(fset *token.FileSet, obj types.Object) string {
+	posn := fset.Position(obj.Pos())
+	return fmt.Sprintf("%s:%d", posn.Filename, posn.Line)
+}
+
 // equalObj reports how x and y differ.  They are assumed to belong to
 // different universes so cannot be compared directly.
 func equalObj(x, y types.Object) error {
@@ -273,3 +289,40 @@ func equalType(x, y types.Type) error {
 	}
 	return nil
 }
+
+// TestVeryLongFile tests the position of an import object declared in
+// a very long input file.  Line numbers greater than maxlines are
+// reported as line 1, not garbage or token.NoPos.
+func TestVeryLongFile(t *testing.T) {
+	// parse and typecheck
+	longFile := "package foo" + strings.Repeat("\n", 123456) + "var X int"
+	fset1 := token.NewFileSet()
+	f, err := parser.ParseFile(fset1, "foo.go", longFile, 0)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var conf types.Config
+	pkg, err := conf.Check("foo", fset1, []*ast.File{f}, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// export
+	exportdata := gcimporter.BExportData(fset1, pkg)
+
+	// import
+	imports := make(map[string]*types.Package)
+	fset2 := token.NewFileSet()
+	_, pkg2, err := gcimporter.BImportData(fset2, imports, exportdata, pkg.Path())
+	if err != nil {
+		t.Fatalf("BImportData(%s): %v", pkg.Path(), err)
+	}
+
+	// compare
+	posn1 := fset1.Position(pkg.Scope().Lookup("X").Pos())
+	posn2 := fset2.Position(pkg2.Scope().Lookup("X").Pos())
+	if want := "foo.go:1:1"; posn2.String() != want {
+		t.Errorf("X position = %s, want %s (orig was %s)",
+			posn2, want, posn1)
+	}
+}

go/gcimporter15/bimport.go

@@ -16,6 +16,7 @@ import (
 	"go/types"
 	"sort"
 	"strings"
+	"sync"
 	"unicode"
 	"unicode/utf8"
 )
@@ -35,6 +36,8 @@ type importer struct {
 	posInfoFormat bool
 	prevFile      string
 	prevLine      int
+	fset          *token.FileSet
+	files         map[string]*token.File
 
 	// debugging support
 	debugFormat bool
@@ -45,12 +48,14 @@ type importer struct {
 // and returns the number of bytes consumed and a reference to the package.
 // If data is obviously malformed, an error is returned but in
 // general it is not recommended to call BImportData on untrusted data.
-func BImportData(imports map[string]*types.Package, data []byte, path string) (int, *types.Package, error) {
+func BImportData(fset *token.FileSet, imports map[string]*types.Package, data []byte, path string) (int, *types.Package, error) {
 	p := importer{
 		imports: imports,
 		data:    data,
 		path:    path,
 		strList: []string{""}, // empty string is mapped to 0
+		fset:    fset,
+		files:   make(map[string]*token.File),
 	}
 
 	// read low-level encoding format
@@ -173,37 +178,37 @@ func (p *importer) declare(obj types.Object) {
 func (p *importer) obj(tag int) {
 	switch tag {
 	case constTag:
-		p.pos()
+		pos := p.pos()
 		pkg, name := p.qualifiedName()
 		typ := p.typ(nil)
 		val := p.value()
-		p.declare(types.NewConst(token.NoPos, pkg, name, typ, val))
+		p.declare(types.NewConst(pos, pkg, name, typ, val))
 
 	case typeTag:
 		_ = p.typ(nil)
 
 	case varTag:
-		p.pos()
+		pos := p.pos()
 		pkg, name := p.qualifiedName()
 		typ := p.typ(nil)
-		p.declare(types.NewVar(token.NoPos, pkg, name, typ))
+		p.declare(types.NewVar(pos, pkg, name, typ))
 
 	case funcTag:
-		p.pos()
+		pos := p.pos()
 		pkg, name := p.qualifiedName()
 		params, isddd := p.paramList()
 		result, _ := p.paramList()
 		sig := types.NewSignature(nil, params, result, isddd)
-		p.declare(types.NewFunc(token.NoPos, pkg, name, sig))
+		p.declare(types.NewFunc(pos, pkg, name, sig))
 
 	default:
 		panic(fmt.Sprintf("unexpected object tag %d", tag))
 	}
 }
 
-func (p *importer) pos() {
+func (p *importer) pos() token.Pos {
 	if !p.posInfoFormat {
-		return
+		return token.NoPos
 	}
 
 	file := p.prevFile
@@ -219,9 +224,40 @@ func (p *importer) pos() {
 	}
 	p.prevLine = line
 
-	// TODO(gri) register new position
+	// Synthesize a token.Pos
+
+	// Since we don't know the set of needed file positions, we
+	// reserve maxlines positions per file.
+	const maxlines = 64 * 1024
+	f := p.files[file]
+	if f == nil {
+		f = p.fset.AddFile(file, -1, maxlines)
+		p.files[file] = f
+		// Allocate the fake linebreak indices on first use.
+		// TODO(adonovan): opt: save ~512KB using a more complex scheme?
+		fakeLinesOnce.Do(func() {
+			fakeLines = make([]int, maxlines)
+			for i := range fakeLines {
+				fakeLines[i] = i
+			}
+		})
+		f.SetLines(fakeLines)
+	}
+
+	if line > maxlines {
+		line = 1
+	}
+
+	// Treat the file as if it contained only newlines
+	// and column=1: use the line number as the offset.
+	return f.Pos(line - 1)
 }
 
+var (
+	fakeLines     []int
+	fakeLinesOnce sync.Once
+)
+
 func (p *importer) qualifiedName() (pkg *types.Package, name string) {
 	name = p.string()
 	pkg = p.pkg()
@@ -257,14 +293,14 @@ func (p *importer) typ(parent *types.Package) types.Type {
 	switch i {
 	case namedTag:
 		// read type object
-		p.pos()
+		pos := p.pos()
 		parent, name := p.qualifiedName()
 		scope := parent.Scope()
 		obj := scope.Lookup(name)
 
 		// if the object doesn't exist yet, create and insert it
 		if obj == nil {
-			obj = types.NewTypeName(token.NoPos, parent, name, nil)
+			obj = types.NewTypeName(pos, parent, name, nil)
 			scope.Insert(obj)
 		}
 
@@ -290,7 +326,7 @@ func (p *importer) typ(parent *types.Package) types.Type {
 		// read associated methods
 		for i := p.int(); i > 0; i-- {
 			// TODO(gri) replace this with something closer to fieldName
-			p.pos()
+			pos := p.pos()
 			name := p.string()
 			if !exported(name) {
 				p.pkg()
@@ -301,7 +337,7 @@ func (p *importer) typ(parent *types.Package) types.Type {
 			result, _ := p.paramList()
 
 			sig := types.NewSignature(recv.At(0), params, result, isddd)
-			t0.AddMethod(types.NewFunc(token.NoPos, parent, name, sig))
+			t0.AddMethod(types.NewFunc(pos, parent, name, sig))
 		}
 
 		return t
@@ -415,7 +451,7 @@ func (p *importer) fieldList(parent *types.Package) (fields []*types.Var, tags [
 }
 
 func (p *importer) field(parent *types.Package) *types.Var {
-	p.pos()
+	pos := p.pos()
 	pkg, name := p.fieldName(parent)
 	typ := p.typ(parent)
 
@@ -434,7 +470,7 @@ func (p *importer) field(parent *types.Package) *types.Var {
 		anonymous = true
 	}
 
-	return types.NewField(token.NoPos, pkg, name, typ, anonymous)
+	return types.NewField(pos, pkg, name, typ, anonymous)
 }
 
 func (p *importer) methodList(parent *types.Package) (methods []*types.Func) {
@@ -448,12 +484,12 @@ func (p *importer) methodList(parent *types.Package) (methods []*types.Func) {
 }
 
 func (p *importer) method(parent *types.Package) *types.Func {
-	p.pos()
+	pos := p.pos()
 	pkg, name := p.fieldName(parent)
 	params, isddd := p.paramList()
 	result, _ := p.paramList()
 	sig := types.NewSignature(nil, params, result, isddd)
-	return types.NewFunc(token.NoPos, pkg, name, sig)
+	return types.NewFunc(pos, pkg, name, sig)
 }
 
 func (p *importer) fieldName(parent *types.Package) (*types.Package, string) {

go/gcimporter15/gcimporter.go

@@ -174,7 +174,8 @@ func Import(packages map[string]*types.Package, path, srcDir string) (pkg *types
 		var data []byte
 		data, err = ioutil.ReadAll(buf)
 		if err == nil {
-			_, pkg, err = BImportData(packages, data, path)
+			fset := token.NewFileSet()
+			_, pkg, err = BImportData(fset, packages, data, path)
 			return
 		}
 	default: