From 461c3e52633638ee6e385a6fed3d78dc0a02f210 Mon Sep 17 00:00:00 2001 From: Russ Cox Date: Fri, 13 Jan 2017 14:59:37 -0500 Subject: [PATCH] cmd/go: split out cmd/go/internal/buildid This is one CL in a long sequence of changes to break up the go command from one package into a plausible group of packages. This sequence is concerned only with moving code, not changing or cleaning up code. There will still be more cleanup after this sequence. The entire sequence will be submitted together: it is not a goal for the tree to build at every step. For #18653. Change-Id: I20dbc352c3df3c83a75811dd8e78c580a46b2202 Reviewed-on: https://go-review.googlesource.com/36195 Reviewed-by: David Crawshaw --- src/cmd/go/build.go | 5 +- src/cmd/go/internal/buildid/buildid.go | 201 ++++++++++++++++++++++ src/cmd/go/{ => internal/buildid}/note.go | 4 +- src/cmd/go/note_test.go | 13 +- src/cmd/go/pkg.go | 200 +-------------------- 5 files changed, 220 insertions(+), 203 deletions(-) create mode 100644 src/cmd/go/internal/buildid/buildid.go rename src/cmd/go/{ => internal/buildid}/note.go (98%) diff --git a/src/cmd/go/build.go b/src/cmd/go/build.go index caa876c4ae5..848b648d8d6 100644 --- a/src/cmd/go/build.go +++ b/src/cmd/go/build.go @@ -27,8 +27,9 @@ import ( "sync" "time" - "cmd/go/internal/cfg" "cmd/go/internal/base" + "cmd/go/internal/buildid" + "cmd/go/internal/cfg" "cmd/go/internal/str" ) @@ -889,7 +890,7 @@ func readpkglist(shlibpath string) (pkgs []*Package) { } } } else { - pkglistbytes, err := readELFNote(shlibpath, "Go\x00\x00", 1) + pkglistbytes, err := buildid.ReadELFNote(shlibpath, "Go\x00\x00", 1) if err != nil { base.Fatalf("readELFNote failed: %v", err) } diff --git a/src/cmd/go/internal/buildid/buildid.go b/src/cmd/go/internal/buildid/buildid.go new file mode 100644 index 00000000000..091c9090c86 --- /dev/null +++ b/src/cmd/go/internal/buildid/buildid.go @@ -0,0 +1,201 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package buildid + +import ( + "bytes" + "cmd/go/internal/cfg" + "fmt" + "io" + "os" + "strconv" + "strings" +) + +var ( + errBuildIDToolchain = fmt.Errorf("build ID only supported in gc toolchain") + errBuildIDMalformed = fmt.Errorf("malformed object file") + errBuildIDUnknown = fmt.Errorf("lost build ID") +) + +var ( + bangArch = []byte("!") + pkgdef = []byte("__.PKGDEF") + goobject = []byte("go object ") + buildid = []byte("build id ") +) + +// ReadBuildID reads the build ID from an archive or binary. +// It only supports the gc toolchain. +// Other toolchain maintainers should adjust this function. +func ReadBuildID(name, target string) (id string, err error) { + if cfg.BuildToolchainName != "gc" { + return "", errBuildIDToolchain + } + + // For commands, read build ID directly from binary. + if name == "main" { + return ReadBuildIDFromBinary(target) + } + + // Otherwise, we expect to have an archive (.a) file, + // and we can read the build ID from the Go export data. + if !strings.HasSuffix(target, ".a") { + return "", &os.PathError{Op: "parse", Path: target, Err: errBuildIDUnknown} + } + + // Read just enough of the target to fetch the build ID. + // The archive is expected to look like: + // + // ! + // __.PKGDEF 0 0 0 644 7955 ` + // go object darwin amd64 devel X:none + // build id "b41e5c45250e25c9fd5e9f9a1de7857ea0d41224" + // + // The variable-sized strings are GOOS, GOARCH, and the experiment list (X:none). + // Reading the first 1024 bytes should be plenty. + f, err := os.Open(target) + if err != nil { + return "", err + } + data := make([]byte, 1024) + n, err := io.ReadFull(f, data) + f.Close() + + if err != nil && n == 0 { + return "", err + } + + bad := func() (string, error) { + return "", &os.PathError{Op: "parse", Path: target, Err: errBuildIDMalformed} + } + + // Archive header. + for i := 0; ; i++ { // returns during i==3 + j := bytes.IndexByte(data, '\n') + if j < 0 { + return bad() + } + line := data[:j] + data = data[j+1:] + switch i { + case 0: + if !bytes.Equal(line, bangArch) { + return bad() + } + case 1: + if !bytes.HasPrefix(line, pkgdef) { + return bad() + } + case 2: + if !bytes.HasPrefix(line, goobject) { + return bad() + } + case 3: + if !bytes.HasPrefix(line, buildid) { + // Found the object header, just doesn't have a build id line. + // Treat as successful, with empty build id. + return "", nil + } + id, err := strconv.Unquote(string(line[len(buildid):])) + if err != nil { + return bad() + } + return id, nil + } + } +} + +var ( + goBuildPrefix = []byte("\xff Go build ID: \"") + goBuildEnd = []byte("\"\n \xff") + + elfPrefix = []byte("\x7fELF") + + machoPrefixes = [][]byte{ + {0xfe, 0xed, 0xfa, 0xce}, + {0xfe, 0xed, 0xfa, 0xcf}, + {0xce, 0xfa, 0xed, 0xfe}, + {0xcf, 0xfa, 0xed, 0xfe}, + } +) + +var BuildIDReadSize = 32 * 1024 // changed for testing + +// ReadBuildIDFromBinary reads the build ID from a binary. +// +// ELF binaries store the build ID in a proper PT_NOTE section. +// +// Other binary formats are not so flexible. For those, the linker +// stores the build ID as non-instruction bytes at the very beginning +// of the text segment, which should appear near the beginning +// of the file. This is clumsy but fairly portable. Custom locations +// can be added for other binary types as needed, like we did for ELF. +func ReadBuildIDFromBinary(filename string) (id string, err error) { + if filename == "" { + return "", &os.PathError{Op: "parse", Path: filename, Err: errBuildIDUnknown} + } + + // Read the first 32 kB of the binary file. + // That should be enough to find the build ID. + // In ELF files, the build ID is in the leading headers, + // which are typically less than 4 kB, not to mention 32 kB. + // In Mach-O files, there's no limit, so we have to parse the file. + // On other systems, we're trying to read enough that + // we get the beginning of the text segment in the read. + // The offset where the text segment begins in a hello + // world compiled for each different object format today: + // + // Plan 9: 0x20 + // Windows: 0x600 + // + f, err := os.Open(filename) + if err != nil { + return "", err + } + defer f.Close() + + data := make([]byte, BuildIDReadSize) + _, err = io.ReadFull(f, data) + if err == io.ErrUnexpectedEOF { + err = nil + } + if err != nil { + return "", err + } + + if bytes.HasPrefix(data, elfPrefix) { + return readELFGoBuildID(filename, f, data) + } + for _, m := range machoPrefixes { + if bytes.HasPrefix(data, m) { + return readMachoGoBuildID(filename, f, data) + } + } + + return readRawGoBuildID(filename, data) +} + +// readRawGoBuildID finds the raw build ID stored in text segment data. +func readRawGoBuildID(filename string, data []byte) (id string, err error) { + i := bytes.Index(data, goBuildPrefix) + if i < 0 { + // Missing. Treat as successful but build ID empty. + return "", nil + } + + j := bytes.Index(data[i+len(goBuildPrefix):], goBuildEnd) + if j < 0 { + return "", &os.PathError{Op: "parse", Path: filename, Err: errBuildIDMalformed} + } + + quoted := data[i+len(goBuildPrefix)-1 : i+len(goBuildPrefix)+j+1] + id, err = strconv.Unquote(string(quoted)) + if err != nil { + return "", &os.PathError{Op: "parse", Path: filename, Err: errBuildIDMalformed} + } + + return id, nil +} diff --git a/src/cmd/go/note.go b/src/cmd/go/internal/buildid/note.go similarity index 98% rename from src/cmd/go/note.go rename to src/cmd/go/internal/buildid/note.go index fae9536d13f..68c91e27047 100644 --- a/src/cmd/go/note.go +++ b/src/cmd/go/internal/buildid/note.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -package main +package buildid import ( "bytes" @@ -25,7 +25,7 @@ func readAligned4(r io.Reader, sz int32) ([]byte, error) { return data, nil } -func readELFNote(filename, name string, typ int32) ([]byte, error) { +func ReadELFNote(filename, name string, typ int32) ([]byte, error) { f, err := elf.Open(filename) if err != nil { return nil, err diff --git a/src/cmd/go/note_test.go b/src/cmd/go/note_test.go index 2ee013faf39..289af9908a4 100644 --- a/src/cmd/go/note_test.go +++ b/src/cmd/go/note_test.go @@ -5,10 +5,11 @@ package main_test import ( - main "cmd/go" "go/build" "runtime" "testing" + + "cmd/go/internal/buildid" ) func TestNoteReading(t *testing.T) { @@ -21,9 +22,9 @@ func TestNoteReading2K(t *testing.T) { } // Set BuildIDReadSize to 2kB to exercise Mach-O parsing more strictly. defer func(old int) { - main.BuildIDReadSize = old - }(main.BuildIDReadSize) - main.BuildIDReadSize = 2 * 1024 + buildid.BuildIDReadSize = old + }(buildid.BuildIDReadSize) + buildid.BuildIDReadSize = 2 * 1024 testNoteReading(t) } @@ -34,7 +35,7 @@ func testNoteReading(t *testing.T) { tg.tempFile("hello.go", `package main; func main() { print("hello, world\n") }`) const buildID = "TestNoteReading-Build-ID" tg.run("build", "-ldflags", "-buildid="+buildID, "-o", tg.path("hello.exe"), tg.path("hello.go")) - id, err := main.ReadBuildIDFromBinary(tg.path("hello.exe")) + id, err := buildid.ReadBuildIDFromBinary(tg.path("hello.exe")) if err != nil { t.Fatalf("reading build ID from hello binary: %v", err) } @@ -54,7 +55,7 @@ func testNoteReading(t *testing.T) { } tg.run("build", "-ldflags", "-buildid="+buildID+" -linkmode=external", "-o", tg.path("hello.exe"), tg.path("hello.go")) - id, err = main.ReadBuildIDFromBinary(tg.path("hello.exe")) + id, err = buildid.ReadBuildIDFromBinary(tg.path("hello.exe")) if err != nil { t.Fatalf("reading build ID from hello binary (linkmode=external): %v", err) } diff --git a/src/cmd/go/pkg.go b/src/cmd/go/pkg.go index 69acbd3230e..e9b73667a40 100644 --- a/src/cmd/go/pkg.go +++ b/src/cmd/go/pkg.go @@ -6,25 +6,25 @@ package main import ( "bytes" - "cmd/go/internal/cfg" - "cmd/go/internal/base" - "cmd/go/internal/str" "crypto/sha1" "errors" "fmt" "go/build" "go/scanner" "go/token" - "io" "io/ioutil" "os" pathpkg "path" "path/filepath" "runtime" "sort" - "strconv" "strings" "unicode" + + "cmd/go/internal/base" + "cmd/go/internal/buildid" + "cmd/go/internal/cfg" + "cmd/go/internal/str" ) var ignoreImports bool // control whether we ignore imports in packages @@ -1127,7 +1127,7 @@ func (p *Package) load(stk *importStack, bp *build.Package, err error) *Package if p.BinaryOnly { // For binary-only package, use build ID from supplied package binary. - buildID, err := readBuildID(p.Name, p.Target) + buildID, err := buildid.ReadBuildID(p.Name, p.Target) if err == nil { p.buildID = buildID } @@ -1498,7 +1498,7 @@ func isStale(p *Package) (bool, string) { // It also catches changes in toolchain, like when flipping between // two versions of Go compiling a single GOPATH. // See issue 8290 and issue 10702. - targetBuildID, err := readBuildID(p.Name, p.Target) + targetBuildID, err := buildid.ReadBuildID(p.Name, p.Target) if err == nil && targetBuildID != p.buildID { return true, "build ID mismatch" } @@ -1849,189 +1849,3 @@ func hasSubdir(root, dir string) (rel string, ok bool) { } return filepath.ToSlash(dir[len(root):]), true } - -var ( - errBuildIDToolchain = fmt.Errorf("build ID only supported in gc toolchain") - errBuildIDMalformed = fmt.Errorf("malformed object file") - errBuildIDUnknown = fmt.Errorf("lost build ID") -) - -var ( - bangArch = []byte("!") - pkgdef = []byte("__.PKGDEF") - goobject = []byte("go object ") - buildid = []byte("build id ") -) - -// readBuildID reads the build ID from an archive or binary. -// It only supports the gc toolchain. -// Other toolchain maintainers should adjust this function. -func readBuildID(name, target string) (id string, err error) { - if cfg.BuildToolchainName != "gc" { - return "", errBuildIDToolchain - } - - // For commands, read build ID directly from binary. - if name == "main" { - return ReadBuildIDFromBinary(target) - } - - // Otherwise, we expect to have an archive (.a) file, - // and we can read the build ID from the Go export data. - if !strings.HasSuffix(target, ".a") { - return "", &os.PathError{Op: "parse", Path: target, Err: errBuildIDUnknown} - } - - // Read just enough of the target to fetch the build ID. - // The archive is expected to look like: - // - // ! - // __.PKGDEF 0 0 0 644 7955 ` - // go object darwin amd64 devel X:none - // build id "b41e5c45250e25c9fd5e9f9a1de7857ea0d41224" - // - // The variable-sized strings are GOOS, GOARCH, and the experiment list (X:none). - // Reading the first 1024 bytes should be plenty. - f, err := os.Open(target) - if err != nil { - return "", err - } - data := make([]byte, 1024) - n, err := io.ReadFull(f, data) - f.Close() - - if err != nil && n == 0 { - return "", err - } - - bad := func() (string, error) { - return "", &os.PathError{Op: "parse", Path: target, Err: errBuildIDMalformed} - } - - // Archive header. - for i := 0; ; i++ { // returns during i==3 - j := bytes.IndexByte(data, '\n') - if j < 0 { - return bad() - } - line := data[:j] - data = data[j+1:] - switch i { - case 0: - if !bytes.Equal(line, bangArch) { - return bad() - } - case 1: - if !bytes.HasPrefix(line, pkgdef) { - return bad() - } - case 2: - if !bytes.HasPrefix(line, goobject) { - return bad() - } - case 3: - if !bytes.HasPrefix(line, buildid) { - // Found the object header, just doesn't have a build id line. - // Treat as successful, with empty build id. - return "", nil - } - id, err := strconv.Unquote(string(line[len(buildid):])) - if err != nil { - return bad() - } - return id, nil - } - } -} - -var ( - goBuildPrefix = []byte("\xff Go build ID: \"") - goBuildEnd = []byte("\"\n \xff") - - elfPrefix = []byte("\x7fELF") - - machoPrefixes = [][]byte{ - {0xfe, 0xed, 0xfa, 0xce}, - {0xfe, 0xed, 0xfa, 0xcf}, - {0xce, 0xfa, 0xed, 0xfe}, - {0xcf, 0xfa, 0xed, 0xfe}, - } -) - -var BuildIDReadSize = 32 * 1024 // changed for testing - -// ReadBuildIDFromBinary reads the build ID from a binary. -// -// ELF binaries store the build ID in a proper PT_NOTE section. -// -// Other binary formats are not so flexible. For those, the linker -// stores the build ID as non-instruction bytes at the very beginning -// of the text segment, which should appear near the beginning -// of the file. This is clumsy but fairly portable. Custom locations -// can be added for other binary types as needed, like we did for ELF. -func ReadBuildIDFromBinary(filename string) (id string, err error) { - if filename == "" { - return "", &os.PathError{Op: "parse", Path: filename, Err: errBuildIDUnknown} - } - - // Read the first 32 kB of the binary file. - // That should be enough to find the build ID. - // In ELF files, the build ID is in the leading headers, - // which are typically less than 4 kB, not to mention 32 kB. - // In Mach-O files, there's no limit, so we have to parse the file. - // On other systems, we're trying to read enough that - // we get the beginning of the text segment in the read. - // The offset where the text segment begins in a hello - // world compiled for each different object format today: - // - // Plan 9: 0x20 - // Windows: 0x600 - // - f, err := os.Open(filename) - if err != nil { - return "", err - } - defer f.Close() - - data := make([]byte, BuildIDReadSize) - _, err = io.ReadFull(f, data) - if err == io.ErrUnexpectedEOF { - err = nil - } - if err != nil { - return "", err - } - - if bytes.HasPrefix(data, elfPrefix) { - return readELFGoBuildID(filename, f, data) - } - for _, m := range machoPrefixes { - if bytes.HasPrefix(data, m) { - return readMachoGoBuildID(filename, f, data) - } - } - - return readRawGoBuildID(filename, data) -} - -// readRawGoBuildID finds the raw build ID stored in text segment data. -func readRawGoBuildID(filename string, data []byte) (id string, err error) { - i := bytes.Index(data, goBuildPrefix) - if i < 0 { - // Missing. Treat as successful but build ID empty. - return "", nil - } - - j := bytes.Index(data[i+len(goBuildPrefix):], goBuildEnd) - if j < 0 { - return "", &os.PathError{Op: "parse", Path: filename, Err: errBuildIDMalformed} - } - - quoted := data[i+len(goBuildPrefix)-1 : i+len(goBuildPrefix)+j+1] - id, err = strconv.Unquote(string(quoted)) - if err != nil { - return "", &os.PathError{Op: "parse", Path: filename, Err: errBuildIDMalformed} - } - - return id, nil -}