1
0
mirror of https://github.com/golang/go synced 2024-11-25 12:17:56 -07:00

internal/exportdata: introduce shared library for exportdata

Deduplicates FindPkg and FindExportData which were shared by
go/internal/gcimporter and cmd/compile/internal/importer into
a new package internal/exportdata.

This change only moves code.

Change-Id: I1daf24dd79fafbe9014b2b15671dcde46b54711e
Reviewed-on: https://go-review.googlesource.com/c/go/+/626700
Commit-Queue: Tim King <taking@google.com>
Reviewed-by: Robert Griesemer <gri@google.com>
LUCI-TryBot-Result: Go LUCI <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>
This commit is contained in:
Tim King 2024-11-08 15:53:42 -08:00 committed by Go LUCI
parent 9fe70bcd65
commit cd35323be9
10 changed files with 244 additions and 434 deletions

View File

@ -1,76 +0,0 @@
// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// This file implements FindExportData.
package importer
import (
"bufio"
"cmd/internal/archive"
"fmt"
"strings"
)
// FindExportData positions the reader r at the beginning of the
// export data section of an underlying GC-created object/archive
// file by reading from it. The reader must be positioned at the
// start of the file before calling this function. The hdr result
// is the string before the export data, either "$$" or "$$B".
//
// If size is non-negative, it's the number of bytes of export data
// still available to read from r.
//
// This function should only be used in tests.
func FindExportData(r *bufio.Reader) (hdr string, size int, err error) {
// TODO(taking): Move into a src/internal package then
// dedup with cmd/compile/internal/noder.findExportData and go/internal/gcimporter.FindExportData.
// Read first line to make sure this is an object file.
line, err := r.ReadSlice('\n')
if err != nil {
err = fmt.Errorf("can't find export data (%v)", err)
return
}
// Is the first line an archive file signature?
if string(line) != "!<arch>\n" {
err = fmt.Errorf("not the start of an archive file (%q)", line)
return
}
// package export block should be first
size = archive.ReadHeader(r, "__.PKGDEF")
if size <= 0 {
err = fmt.Errorf("not a package file")
return
}
// Read first line of __.PKGDEF data, so that line
// is once again the first line of the input.
if line, err = r.ReadSlice('\n'); err != nil {
err = fmt.Errorf("can't find export data (%v)", err)
return
}
// Now at __.PKGDEF in archive. line should begin with "go object ".
if !strings.HasPrefix(string(line), "go object ") {
err = fmt.Errorf("not a Go object file")
return
}
size -= len(line)
// Skip over object header to export data.
// Begins after first line starting with $$.
for line[0] != '$' {
if line, err = r.ReadSlice('\n'); err != nil {
err = fmt.Errorf("can't find export data (%v)", err)
return
}
size -= len(line)
}
hdr = string(line)
return
}

View File

@ -2,155 +2,23 @@
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// This file contains the FindPkg and Import functions for tests
// to use gc-generated object files.
// This file implements the Import function for tests to use gc-generated object files.
package importer
import (
"bufio"
"bytes"
"errors"
"fmt"
"go/build"
"internal/exportdata"
"internal/pkgbits"
"internal/saferio"
"io"
"os"
"os/exec"
"path/filepath"
"strings"
"sync"
"cmd/compile/internal/types2"
)
var exportMap sync.Map // package dir → func() (string, error)
// lookupGorootExport returns the location of the export data
// (normally found in the build cache, but located in GOROOT/pkg
// in prior Go releases) for the package located in pkgDir.
//
// (We use the package's directory instead of its import path
// mainly to simplify handling of the packages in src/vendor
// and cmd/vendor.)
func lookupGorootExport(pkgDir string) (string, error) {
f, ok := exportMap.Load(pkgDir)
if !ok {
var (
listOnce sync.Once
exportPath string
err error
)
f, _ = exportMap.LoadOrStore(pkgDir, func() (string, error) {
listOnce.Do(func() {
cmd := exec.Command(filepath.Join(build.Default.GOROOT, "bin", "go"), "list", "-export", "-f", "{{.Export}}", pkgDir)
cmd.Dir = build.Default.GOROOT
cmd.Env = append(os.Environ(), "PWD="+cmd.Dir, "GOROOT="+build.Default.GOROOT)
var output []byte
output, err = cmd.Output()
if err != nil {
if ee, ok := err.(*exec.ExitError); ok && len(ee.Stderr) > 0 {
err = errors.New(string(ee.Stderr))
}
return
}
exports := strings.Split(string(bytes.TrimSpace(output)), "\n")
if len(exports) != 1 {
err = fmt.Errorf("go list reported %d exports; expected 1", len(exports))
return
}
exportPath = exports[0]
})
return exportPath, err
})
}
return f.(func() (string, error))()
}
var pkgExts = [...]string{".a", ".o"} // a file from the build cache will have no extension
// FindPkg returns the filename and unique package id for an import
// path based on package information provided by build.Import (using
// the build.Default build.Context). A relative srcDir is interpreted
// relative to the current working directory.
//
// This function should only be used in tests.
func FindPkg(path, srcDir string) (filename, id string, err error) {
// TODO(taking): move FindPkg into src/internal and dedup src/go/internal/gcimporter.FindPkg
if path == "" {
return "", "", errors.New("path is empty")
}
var noext string
switch {
default:
// "x" -> "$GOPATH/pkg/$GOOS_$GOARCH/x.ext", "x"
// Don't require the source files to be present.
if abs, err := filepath.Abs(srcDir); err == nil { // see issue 14282
srcDir = abs
}
var bp *build.Package
bp, err = build.Import(path, srcDir, build.FindOnly|build.AllowBinary)
if bp.PkgObj == "" {
if bp.Goroot && bp.Dir != "" {
filename, err = lookupGorootExport(bp.Dir)
if err == nil {
_, err = os.Stat(filename)
}
if err == nil {
return filename, bp.ImportPath, nil
}
}
goto notfound
} else {
noext = strings.TrimSuffix(bp.PkgObj, ".a")
}
id = bp.ImportPath
case build.IsLocalImport(path):
// "./x" -> "/this/directory/x.ext", "/this/directory/x"
noext = filepath.Join(srcDir, path)
id = noext
case filepath.IsAbs(path):
// for completeness only - go/build.Import
// does not support absolute imports
// "/x" -> "/x.ext", "/x"
noext = path
id = path
}
if false { // for debugging
if path != id {
fmt.Printf("%s -> %s\n", path, id)
}
}
// try extensions
for _, ext := range pkgExts {
filename = noext + ext
f, statErr := os.Stat(filename)
if statErr == nil && !f.IsDir() {
return filename, id, nil
}
if err == nil {
err = statErr
}
}
notfound:
if err == nil {
return "", path, fmt.Errorf("can't find import: %q", path)
}
return "", path, fmt.Errorf("can't find import: %q: %w", path, err)
}
// Import imports a gc-generated package given its import path and srcDir, adds
// the corresponding package object to the packages map, and returns the object.
// The packages map must contain all packages already imported.
@ -178,7 +46,7 @@ func Import(packages map[string]*types2.Package, path, srcDir string, lookup fun
rc = f
} else {
var filename string
filename, id, err = FindPkg(path, srcDir)
filename, id, err = exportdata.FindPkg(path, srcDir)
if filename == "" {
if path == "unsafe" {
return types2.Unsafe, nil
@ -207,7 +75,7 @@ func Import(packages map[string]*types2.Package, path, srcDir string, lookup fun
defer rc.Close()
buf := bufio.NewReader(rc)
hdr, size, err := FindExportData(buf)
hdr, size, err := exportdata.FindExportData(buf)
if err != nil {
return
}

View File

@ -10,6 +10,7 @@ import (
"cmd/compile/internal/types2"
"fmt"
"go/build"
"internal/exportdata"
"internal/testenv"
"os"
"os/exec"
@ -101,7 +102,7 @@ func TestImportTestdata(t *testing.T) {
importMap := map[string]string{}
for _, pkg := range wantImports {
export, _, err := FindPkg(pkg, "testdata")
export, _, err := exportdata.FindPkg(pkg, "testdata")
if export == "" {
t.Fatalf("no export data found for %s: %v", pkg, err)
}
@ -278,7 +279,7 @@ var importedObjectTests = []struct {
{"math.Pi", "const Pi untyped float"},
{"math.Sin", "func Sin(x float64) float64"},
{"go/ast.NotNilFilter", "func NotNilFilter(_ string, v reflect.Value) bool"},
{"go/internal/gcimporter.FindPkg", "func FindPkg(path string, srcDir string) (filename string, id string, err error)"},
{"internal/exportdata.FindPkg", "func FindPkg(path string, srcDir string) (filename string, id string, err error)"},
// interfaces
{"context.Context", "type Context interface{Deadline() (deadline time.Time, ok bool); Done() <-chan struct{}; Err() error; Value(key any) any}"},
@ -440,7 +441,7 @@ func TestIssue13566(t *testing.T) {
t.Fatal(err)
}
jsonExport, _, err := FindPkg("encoding/json", "testdata")
jsonExport, _, err := exportdata.FindPkg("encoding/json", "testdata")
if jsonExport == "" {
t.Fatalf("no export data found for encoding/json: %v", err)
}

View File

@ -73,6 +73,7 @@ var bootstrapDirs = []string{
"cmd/internal/cov/covcmd",
"internal/bisect",
"internal/buildcfg",
"internal/exportdata",
"internal/goarch",
"internal/godebugs",
"internal/goexperiment",

View File

@ -549,7 +549,7 @@ var depsRules = `
# crypto-aware packages
DEBUG, go/build, go/types, text/scanner, crypto/md5
< internal/pkgbits
< internal/pkgbits, internal/exportdata
< go/internal/gcimporter, go/internal/gccgoimporter, go/internal/srcimporter
< go/importer;

View File

@ -1,87 +0,0 @@
// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// This file implements FindExportData.
package gcimporter
import (
"bufio"
"fmt"
"io"
"strconv"
"strings"
)
// Copy of cmd/internal/archive.ReadHeader.
func readArchiveHeader(b *bufio.Reader, name string) int {
// architecture-independent object file output
const HeaderSize = 60
var buf [HeaderSize]byte
if _, err := io.ReadFull(b, buf[:]); err != nil {
return -1
}
aname := strings.Trim(string(buf[0:16]), " ")
if !strings.HasPrefix(aname, name) {
return -1
}
asize := strings.Trim(string(buf[48:58]), " ")
i, _ := strconv.Atoi(asize)
return i
}
// FindExportData positions the reader r at the beginning of the
// export data section of an underlying GC-created object/archive
// file by reading from it. The reader must be positioned at the
// start of the file before calling this function. The hdr result
// is the string before the export data, either "$$" or "$$B".
func FindExportData(r *bufio.Reader) (hdr string, size int, err error) {
// Read first line to make sure this is an object file.
line, err := r.ReadSlice('\n')
if err != nil {
err = fmt.Errorf("can't find export data (%v)", err)
return
}
// Is the first line an archive file signature?
if string(line) != "!<arch>\n" {
err = fmt.Errorf("not the start of an archive file (%q)", line)
return
}
// package export block should be first
size = readArchiveHeader(r, "__.PKGDEF")
if size <= 0 {
err = fmt.Errorf("not a package file")
return
}
// Read first line of __.PKGDEF data, so that line
// is once again the first line of the input.
if line, err = r.ReadSlice('\n'); err != nil {
err = fmt.Errorf("can't find export data (%v)", err)
return
}
// Now at __.PKGDEF in archive. line should begin with "go object ".
if !strings.HasPrefix(string(line), "go object ") {
err = fmt.Errorf("not a Go object file")
return
}
size -= len(line)
// Skip over object header to export data.
// Begins after first line starting with $$.
for line[0] != '$' {
if line, err = r.ReadSlice('\n'); err != nil {
err = fmt.Errorf("can't find export data (%v)", err)
return
}
size -= len(line)
}
hdr = string(line)
return
}

View File

@ -7,144 +7,17 @@ package gcimporter // import "go/internal/gcimporter"
import (
"bufio"
"bytes"
"errors"
"fmt"
"go/build"
"go/token"
"go/types"
"internal/exportdata"
"internal/pkgbits"
"internal/saferio"
"io"
"os"
"os/exec"
"path/filepath"
"strings"
"sync"
)
var exportMap sync.Map // package dir → func() (string, error)
// lookupGorootExport returns the location of the export data
// (normally found in the build cache, but located in GOROOT/pkg
// in prior Go releases) for the package located in pkgDir.
//
// (We use the package's directory instead of its import path
// mainly to simplify handling of the packages in src/vendor
// and cmd/vendor.)
func lookupGorootExport(pkgDir string) (string, error) {
f, ok := exportMap.Load(pkgDir)
if !ok {
var (
listOnce sync.Once
exportPath string
err error
)
f, _ = exportMap.LoadOrStore(pkgDir, func() (string, error) {
listOnce.Do(func() {
cmd := exec.Command(filepath.Join(build.Default.GOROOT, "bin", "go"), "list", "-export", "-f", "{{.Export}}", pkgDir)
cmd.Dir = build.Default.GOROOT
cmd.Env = append(os.Environ(), "PWD="+cmd.Dir, "GOROOT="+build.Default.GOROOT)
var output []byte
output, err = cmd.Output()
if err != nil {
if ee, ok := err.(*exec.ExitError); ok && len(ee.Stderr) > 0 {
err = errors.New(string(ee.Stderr))
}
return
}
exports := strings.Split(string(bytes.TrimSpace(output)), "\n")
if len(exports) != 1 {
err = fmt.Errorf("go list reported %d exports; expected 1", len(exports))
return
}
exportPath = exports[0]
})
return exportPath, err
})
}
return f.(func() (string, error))()
}
var pkgExts = [...]string{".a", ".o"} // a file from the build cache will have no extension
// FindPkg returns the filename and unique package id for an import
// path based on package information provided by build.Import (using
// the build.Default build.Context). A relative srcDir is interpreted
// relative to the current working directory.
func FindPkg(path, srcDir string) (filename, id string, err error) {
if path == "" {
return "", "", errors.New("path is empty")
}
var noext string
switch {
default:
// "x" -> "$GOPATH/pkg/$GOOS_$GOARCH/x.ext", "x"
// Don't require the source files to be present.
if abs, err := filepath.Abs(srcDir); err == nil { // see issue 14282
srcDir = abs
}
var bp *build.Package
bp, err = build.Import(path, srcDir, build.FindOnly|build.AllowBinary)
if bp.PkgObj == "" {
if bp.Goroot && bp.Dir != "" {
filename, err = lookupGorootExport(bp.Dir)
if err == nil {
_, err = os.Stat(filename)
}
if err == nil {
return filename, bp.ImportPath, nil
}
}
goto notfound
} else {
noext = strings.TrimSuffix(bp.PkgObj, ".a")
}
id = bp.ImportPath
case build.IsLocalImport(path):
// "./x" -> "/this/directory/x.ext", "/this/directory/x"
noext = filepath.Join(srcDir, path)
id = noext
case filepath.IsAbs(path):
// for completeness only - go/build.Import
// does not support absolute imports
// "/x" -> "/x.ext", "/x"
noext = path
id = path
}
if false { // for debugging
if path != id {
fmt.Printf("%s -> %s\n", path, id)
}
}
// try extensions
for _, ext := range pkgExts {
filename = noext + ext
f, statErr := os.Stat(filename)
if statErr == nil && !f.IsDir() {
return filename, id, nil
}
if err == nil {
err = statErr
}
}
notfound:
if err == nil {
return "", path, fmt.Errorf("can't find import: %q", path)
}
return "", path, fmt.Errorf("can't find import: %q: %w", path, err)
}
// Import imports a gc-generated package given its import path and srcDir, adds
// the corresponding package object to the packages map, and returns the object.
// The packages map must contain all packages already imported.
@ -170,7 +43,7 @@ func Import(fset *token.FileSet, packages map[string]*types.Package, path, srcDi
rc = f
} else {
var filename string
filename, id, err = FindPkg(path, srcDir)
filename, id, err = exportdata.FindPkg(path, srcDir)
if filename == "" {
if path == "unsafe" {
return types.Unsafe, nil
@ -199,7 +72,7 @@ func Import(fset *token.FileSet, packages map[string]*types.Package, path, srcDi
defer rc.Close()
buf := bufio.NewReader(rc)
hdr, size, err := FindExportData(buf)
hdr, size, err := exportdata.FindExportData(buf)
if err != nil {
return
}

View File

@ -403,7 +403,7 @@ var importedObjectTests = []struct {
{"math.Pi", "const Pi untyped float"},
{"math.Sin", "func Sin(x float64) float64"},
{"go/ast.NotNilFilter", "func NotNilFilter(_ string, v reflect.Value) bool"},
{"go/internal/gcimporter.FindPkg", "func FindPkg(path string, srcDir string) (filename string, id string, err error)"},
{"internal/exportdata.FindPkg", "func FindPkg(path string, srcDir string) (filename string, id string, err error)"},
// interfaces
{"context.Context", "type Context interface{Deadline() (deadline time.Time, ok bool); Done() <-chan struct{}; Err() error; Value(key any) any}"},

View File

@ -0,0 +1,198 @@
// Copyright 2024 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Package exportdata implements common utilities for finding
// and reading gc-generated object files.
package exportdata
import (
"bufio"
"bytes"
"errors"
"fmt"
"go/build"
"os"
"os/exec"
"path/filepath"
"strings"
"sync"
)
// FindExportData positions the reader r at the beginning of the
// export data section of an underlying GC-created object/archive
// file by reading from it. The reader must be positioned at the
// start of the file before calling this function. The hdr result
// is the string before the export data, either "$$" or "$$B".
func FindExportData(r *bufio.Reader) (hdr string, size int, err error) {
// TODO(taking): Merge with cmd/compile/internal/noder.findExportData.
// Read first line to make sure this is an object file.
line, err := r.ReadSlice('\n')
if err != nil {
err = fmt.Errorf("can't find export data (%v)", err)
return
}
// Is the first line an archive file signature?
if string(line) != "!<arch>\n" {
err = fmt.Errorf("not the start of an archive file (%q)", line)
return
}
// package export block should be first
size = readArchiveHeader(r, "__.PKGDEF")
if size <= 0 {
err = fmt.Errorf("not a package file")
return
}
// Read first line of __.PKGDEF data, so that line
// is once again the first line of the input.
if line, err = r.ReadSlice('\n'); err != nil {
err = fmt.Errorf("can't find export data (%v)", err)
return
}
// Now at __.PKGDEF in archive. line should begin with "go object ".
if !strings.HasPrefix(string(line), "go object ") {
err = fmt.Errorf("not a Go object file")
return
}
size -= len(line)
// Skip over object header to export data.
// Begins after first line starting with $$.
for line[0] != '$' {
if line, err = r.ReadSlice('\n'); err != nil {
err = fmt.Errorf("can't find export data (%v)", err)
return
}
size -= len(line)
}
hdr = string(line)
return
}
// FindPkg returns the filename and unique package id for an import
// path based on package information provided by build.Import (using
// the build.Default build.Context). A relative srcDir is interpreted
// relative to the current working directory.
func FindPkg(path, srcDir string) (filename, id string, err error) {
if path == "" {
return "", "", errors.New("path is empty")
}
var noext string
switch {
default:
// "x" -> "$GOPATH/pkg/$GOOS_$GOARCH/x.ext", "x"
// Don't require the source files to be present.
if abs, err := filepath.Abs(srcDir); err == nil { // see issue 14282
srcDir = abs
}
var bp *build.Package
bp, err = build.Import(path, srcDir, build.FindOnly|build.AllowBinary)
if bp.PkgObj == "" {
if bp.Goroot && bp.Dir != "" {
filename, err = lookupGorootExport(bp.Dir)
if err == nil {
_, err = os.Stat(filename)
}
if err == nil {
return filename, bp.ImportPath, nil
}
}
goto notfound
} else {
noext = strings.TrimSuffix(bp.PkgObj, ".a")
}
id = bp.ImportPath
case build.IsLocalImport(path):
// "./x" -> "/this/directory/x.ext", "/this/directory/x"
noext = filepath.Join(srcDir, path)
id = noext
case filepath.IsAbs(path):
// for completeness only - go/build.Import
// does not support absolute imports
// "/x" -> "/x.ext", "/x"
noext = path
id = path
}
if false { // for debugging
if path != id {
fmt.Printf("%s -> %s\n", path, id)
}
}
// try extensions
for _, ext := range pkgExts {
filename = noext + ext
f, statErr := os.Stat(filename)
if statErr == nil && !f.IsDir() {
return filename, id, nil
}
if err == nil {
err = statErr
}
}
notfound:
if err == nil {
return "", path, fmt.Errorf("can't find import: %q", path)
}
return "", path, fmt.Errorf("can't find import: %q: %w", path, err)
}
var pkgExts = [...]string{".a", ".o"} // a file from the build cache will have no extension
var exportMap sync.Map // package dir → func() (string, error)
// lookupGorootExport returns the location of the export data
// (normally found in the build cache, but located in GOROOT/pkg
// in prior Go releases) for the package located in pkgDir.
//
// (We use the package's directory instead of its import path
// mainly to simplify handling of the packages in src/vendor
// and cmd/vendor.)
func lookupGorootExport(pkgDir string) (string, error) {
f, ok := exportMap.Load(pkgDir)
if !ok {
var (
listOnce sync.Once
exportPath string
err error
)
f, _ = exportMap.LoadOrStore(pkgDir, func() (string, error) {
listOnce.Do(func() {
cmd := exec.Command(filepath.Join(build.Default.GOROOT, "bin", "go"), "list", "-export", "-f", "{{.Export}}", pkgDir)
cmd.Dir = build.Default.GOROOT
cmd.Env = append(os.Environ(), "PWD="+cmd.Dir, "GOROOT="+build.Default.GOROOT)
var output []byte
output, err = cmd.Output()
if err != nil {
if ee, ok := err.(*exec.ExitError); ok && len(ee.Stderr) > 0 {
err = errors.New(string(ee.Stderr))
}
return
}
exports := strings.Split(string(bytes.TrimSpace(output)), "\n")
if len(exports) != 1 {
err = fmt.Errorf("go list reported %d exports; expected 1", len(exports))
return
}
exportPath = exports[0]
})
return exportPath, err
})
}
return f.(func() (string, error))()
}

View File

@ -0,0 +1,32 @@
// Copyright 2024 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// This file contains support functions for exportdata.
package exportdata
import (
"bufio"
"io"
"strconv"
"strings"
)
// Copy of cmd/internal/archive.ReadHeader.
func readArchiveHeader(b *bufio.Reader, name string) int {
// architecture-independent object file output
const HeaderSize = 60
var buf [HeaderSize]byte
if _, err := io.ReadFull(b, buf[:]); err != nil {
return -1
}
aname := strings.Trim(string(buf[0:16]), " ")
if !strings.HasPrefix(aname, name) {
return -1
}
asize := strings.Trim(string(buf[48:58]), " ")
i, _ := strconv.Atoi(asize)
return i
}