1
0
mirror of https://github.com/golang/go synced 2024-11-20 00:14:44 -07:00
go/misc/dist/bindist.go
Brad Fitzpatrick b8b48abe0f misc/dist: don't ship cmd/api
cmd/api is a tool to prevent the Go developers from breaking
the Go 1 API promise. It has no utility to end users and
doesn't run on arbitrary packages (it's always been full of
hacks for its bespoke type checker to work on the standard
library)

Robert's in-progress rewrite depends on the go.tools repo for
go/types, so we won't be able to ship this tool later
anyway. Just remove it from binary distributions.

A future change to run.bash can conditionally build & run
cmd/api, perhaps automatically fetching go/types if
necessary. I assume people don't want to vendor go/types into
a private gopath just for cmd/api.

I will need help with run.bat.

R=golang-dev, adg, dsymonds, rsc
CC=golang-dev
https://golang.org/cl/12316043
2013-08-02 10:19:52 -07:00

948 lines
21 KiB
Go

// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// This is a tool for packaging binary releases.
// It supports FreeBSD, Linux, NetBSD, OS X, and Windows.
package main
import (
"archive/tar"
"archive/zip"
"bufio"
"bytes"
"compress/gzip"
"encoding/base64"
"flag"
"fmt"
"io"
"io/ioutil"
"log"
"mime/multipart"
"net/http"
"os"
"os/exec"
"path/filepath"
"regexp"
"runtime"
"strings"
)
var (
tag = flag.String("tag", "release", "mercurial tag to check out")
repo = flag.String("repo", "https://code.google.com/p/go", "repo URL")
verbose = flag.Bool("v", false, "verbose output")
upload = flag.Bool("upload", true, "upload resulting files to Google Code")
wxsFile = flag.String("wxs", "", "path to custom installer.wxs")
addLabel = flag.String("label", "", "additional label to apply to file when uploading")
includeRace = flag.Bool("race", true, "build race detector packages")
versionOverride = flag.String("version", "", "override version name")
username, password string // for Google Code upload
)
const (
uploadURL = "https://go.googlecode.com/files"
godocPath = "code.google.com/p/go.tools/cmd/godoc"
tourPath = "code.google.com/p/go-tour"
)
var preBuildCleanFiles = []string{
"lib/codereview",
"misc/dashboard/godashboard",
"src/cmd/api",
"src/cmd/cov",
"src/cmd/prof",
"src/pkg/exp",
"src/pkg/old",
}
var cleanFiles = []string{
".hg",
".hgtags",
".hgignore",
"VERSION.cache",
}
var sourceCleanFiles = []string{
"bin",
"pkg",
}
var tourPackages = []string{
"pic",
"tree",
"wc",
}
var tourContent = []string{
"js",
"prog",
"solutions",
"static",
"template",
"tour.article",
}
// The os-arches that support the race toolchain.
var raceAvailable = []string{
"darwin-amd64",
"linux-amd64",
"windows-amd64",
}
var fileRe = regexp.MustCompile(`^(go[a-z0-9-.]+)\.(src|([a-z0-9]+)-([a-z0-9]+))\.`)
func main() {
flag.Usage = func() {
fmt.Fprintf(os.Stderr, "usage: %s [flags] targets...\n", os.Args[0])
flag.PrintDefaults()
os.Exit(2)
}
flag.Parse()
if flag.NArg() == 0 {
flag.Usage()
}
if runtime.GOOS == "windows" {
checkWindowsDeps()
}
if *upload {
if err := readCredentials(); err != nil {
log.Println("readCredentials:", err)
}
}
for _, targ := range flag.Args() {
var b Build
if m := fileRe.FindStringSubmatch(targ); m != nil {
// targ is a file name; upload it to googlecode.
version := m[1]
if m[2] == "src" {
b.Source = true
} else {
b.OS = m[3]
b.Arch = m[4]
}
if !*upload {
log.Printf("%s: -upload=false, skipping", targ)
continue
}
if err := b.Upload(version, targ); err != nil {
log.Printf("%s: %v", targ, err)
}
continue
}
if targ == "source" {
b.Source = true
} else {
p := strings.SplitN(targ, "-", 2)
if len(p) != 2 {
log.Println("Ignoring unrecognized target:", targ)
continue
}
b.OS = p[0]
b.Arch = p[1]
if *includeRace {
for _, t := range raceAvailable {
if t == targ {
b.Race = true
}
}
}
}
if err := b.Do(); err != nil {
log.Printf("%s: %v", targ, err)
}
}
}
type Build struct {
Source bool // if true, OS and Arch must be empty
Race bool // build race toolchain
OS string
Arch string
root string
gopath string
}
func (b *Build) Do() error {
work, err := ioutil.TempDir("", "bindist")
if err != nil {
return err
}
defer os.RemoveAll(work)
b.root = filepath.Join(work, "go")
b.gopath = work
// Clone Go distribution and update to tag.
_, err = b.hgCmd(work, "clone", *repo, b.root)
if err != nil {
return err
}
_, err = b.hgCmd(b.root, "update", *tag)
if err != nil {
return err
}
// Remove exp and old packages.
if err := b.clean(preBuildCleanFiles); err != nil {
return err
}
src := filepath.Join(b.root, "src")
if b.Source {
if runtime.GOOS == "windows" {
log.Print("Warning: running make.bash on Windows; source builds are intended to be run on a Unix machine")
}
// Build dist tool only.
_, err = b.run(src, "bash", "make.bash", "--dist-tool")
} else {
// Build.
if b.OS == "windows" {
_, err = b.run(src, "cmd", "/C", "make.bat")
} else {
_, err = b.run(src, "bash", "make.bash")
}
if b.Race {
if err != nil {
return err
}
goCmd := filepath.Join(b.root, "bin", "go")
if b.OS == "windows" {
goCmd += ".exe"
}
_, err = b.run(src, goCmd, "install", "-race", "std")
if err != nil {
return err
}
// Re-install std without -race, so that we're not left
// with a slower, race-enabled cmd/go, etc.
_, err = b.run(src, goCmd, "install", "-a", "std")
// Re-building go command leaves old versions of go.exe as go.exe~ on windows.
// See (*builder).copyFile in $GOROOT/src/cmd/go/build.go for details.
// Remove it manually.
if b.OS == "windows" {
os.Remove(goCmd + "~")
}
}
if err != nil {
return err
}
err = b.godoc()
if err != nil {
return err
}
err = b.tour()
}
if err != nil {
return err
}
// Get version strings.
var (
version string // "weekly.2012-03-04"
fullVersion []byte // "weekly.2012-03-04 9353aa1efdf3"
)
pat := filepath.Join(b.root, "pkg/tool/*/dist*") // trailing * for .exe
m, err := filepath.Glob(pat)
if err != nil {
return err
}
if len(m) == 0 {
return fmt.Errorf("couldn't find dist in %q", pat)
}
fullVersion, err = b.run("", m[0], "version")
if err != nil {
return err
}
fullVersion = bytes.TrimSpace(fullVersion)
v := bytes.SplitN(fullVersion, []byte(" "), 2)
version = string(v[0])
if *versionOverride != "" {
version = *versionOverride
}
// Write VERSION file.
err = ioutil.WriteFile(filepath.Join(b.root, "VERSION"), fullVersion, 0644)
if err != nil {
return err
}
// Clean goroot.
if err := b.clean(cleanFiles); err != nil {
return err
}
if b.Source {
if err := b.clean(sourceCleanFiles); err != nil {
return err
}
}
// Create packages.
base := fmt.Sprintf("%s.%s-%s", version, b.OS, b.Arch)
if !strings.HasPrefix(base, "go") {
base = "go." + base
}
var targs []string
switch b.OS {
case "linux", "freebsd", "netbsd", "":
// build tarball
targ := base
if b.Source {
targ = fmt.Sprintf("%s.src", version)
if !strings.HasPrefix(targ, "go") {
targ = "go." + targ
}
}
targ += ".tar.gz"
err = makeTar(targ, work)
targs = append(targs, targ)
case "darwin":
// build tarball
targ := base + ".tar.gz"
err = makeTar(targ, work)
targs = append(targs, targ)
// build pkg
// arrange work so it's laid out as the dest filesystem
etc := filepath.Join(b.root, "misc/dist/darwin/etc")
_, err = b.run(work, "cp", "-r", etc, ".")
if err != nil {
return err
}
localDir := filepath.Join(work, "usr/local")
err = os.MkdirAll(localDir, 0755)
if err != nil {
return err
}
_, err = b.run(work, "mv", "go", localDir)
if err != nil {
return err
}
// build package
pkgdest, err := ioutil.TempDir("", "pkgdest")
if err != nil {
return err
}
defer os.RemoveAll(pkgdest)
dist := filepath.Join(runtime.GOROOT(), "misc/dist")
_, err = b.run("", "pkgbuild",
"--identifier", "com.googlecode.go",
"--version", "1.0",
"--scripts", filepath.Join(dist, "darwin/scripts"),
"--root", work,
filepath.Join(pkgdest, "com.googlecode.go.pkg"))
if err != nil {
return err
}
targ = base + ".pkg"
_, err = b.run("", "productbuild",
"--distribution", filepath.Join(dist, "darwin/Distribution"),
"--resources", filepath.Join(dist, "darwin/Resources"),
"--package-path", pkgdest,
targ)
if err != nil {
return err
}
targs = append(targs, targ)
case "windows":
// Create ZIP file.
zip := filepath.Join(work, base+".zip")
err = makeZip(zip, work)
// Copy zip to target file.
targ := base + ".zip"
err = cp(targ, zip)
if err != nil {
return err
}
targs = append(targs, targ)
// Create MSI installer.
win := filepath.Join(b.root, "misc/dist/windows")
installer := filepath.Join(win, "installer.wxs")
if *wxsFile != "" {
installer = *wxsFile
}
appfiles := filepath.Join(work, "AppFiles.wxs")
msi := filepath.Join(work, "installer.msi")
// Gather files.
_, err = b.run(work, "heat", "dir", "go",
"-nologo",
"-gg", "-g1", "-srd", "-sfrag",
"-cg", "AppFiles",
"-template", "fragment",
"-dr", "INSTALLDIR",
"-var", "var.SourceDir",
"-out", appfiles)
if err != nil {
return err
}
// Build package.
_, err = b.run(work, "candle",
"-nologo",
"-dVersion="+version,
"-dArch="+b.Arch,
"-dSourceDir=go",
installer, appfiles)
if err != nil {
return err
}
appfiles = filepath.Join(work, "AppFiles.wixobj")
installer = filepath.Join(work, "installer.wixobj")
_, err = b.run(win, "light",
"-nologo",
"-ext", "WixUIExtension",
"-ext", "WixUtilExtension",
installer, appfiles,
"-o", msi)
if err != nil {
return err
}
// Copy installer to target file.
targ = base + ".msi"
err = cp(targ, msi)
targs = append(targs, targ)
}
if err == nil && *upload {
for _, targ := range targs {
err = b.Upload(version, targ)
if err != nil {
return err
}
}
}
return err
}
func (b *Build) godoc() error {
defer func() {
// Clean work files from GOPATH directory.
for _, d := range []string{"bin", "pkg", "src"} {
os.RemoveAll(filepath.Join(b.gopath, d))
}
}()
// go get the godoc package.
// The go tool knows to install to $GOROOT/bin.
_, err := b.run(b.gopath, filepath.Join(b.root, "bin", "go"), "get", godocPath)
if err != nil {
return err
}
// Copy templates from go.tools/cmd/godoc/template to GOROOT/lib/godoc.
return cpDir(
filepath.Join(b.root, "lib", "godoc"),
filepath.Join(b.gopath, "src", filepath.FromSlash(godocPath), "template"),
)
}
func (b *Build) tour() error {
defer func() {
// Clean work files from GOPATH directory.
for _, d := range []string{"bin", "pkg", "src"} {
os.RemoveAll(filepath.Join(b.gopath, d))
}
}()
// go get the gotour package.
_, err := b.run(b.gopath, filepath.Join(b.root, "bin", "go"), "get", tourPath+"/gotour")
if err != nil {
return err
}
// Copy all the tour content to $GOROOT/misc/tour.
importPath := filepath.FromSlash(tourPath)
tourSrc := filepath.Join(b.gopath, "src", importPath)
contentDir := filepath.Join(b.root, "misc", "tour")
if err = cpAllDir(contentDir, tourSrc, tourContent...); err != nil {
return err
}
// Copy the tour source code so it's accessible with $GOPATH pointing to $GOROOT/misc/tour.
if err = cpAllDir(filepath.Join(contentDir, "src", importPath), tourSrc, tourPackages...); err != nil {
return err
}
// Copy gotour binary to tool directory as "tour"; invoked as "go tool tour".
return cp(
filepath.Join(b.root, "pkg", "tool", b.OS+"_"+b.Arch, "tour"+ext()),
filepath.Join(b.gopath, "bin", "gotour"+ext()),
)
}
func ext() string {
if runtime.GOOS == "windows" {
return ".exe"
}
return ""
}
func (b *Build) hgCmd(dir string, args ...string) ([]byte, error) {
return b.run(dir, "hg", append([]string{"--config", "extensions.codereview=!"}, args...)...)
}
func (b *Build) run(dir, name string, args ...string) ([]byte, error) {
buf := new(bytes.Buffer)
absName, err := lookPath(name)
if err != nil {
return nil, err
}
cmd := exec.Command(absName, args...)
var output io.Writer = buf
if *verbose {
log.Printf("Running %q %q", absName, args)
output = io.MultiWriter(buf, os.Stdout)
}
cmd.Stdout = output
cmd.Stderr = output
cmd.Dir = dir
cmd.Env = b.env()
if err := cmd.Run(); err != nil {
fmt.Fprintf(os.Stderr, "%s", buf.Bytes())
return nil, fmt.Errorf("%s %s: %v", name, strings.Join(args, " "), err)
}
return buf.Bytes(), nil
}
var cleanEnv = []string{
"GOARCH",
"GOBIN",
"GOHOSTARCH",
"GOHOSTOS",
"GOOS",
"GOROOT",
"GOROOT_FINAL",
"GOPATH",
}
func (b *Build) env() []string {
env := os.Environ()
for i := 0; i < len(env); i++ {
for _, c := range cleanEnv {
if strings.HasPrefix(env[i], c+"=") {
env = append(env[:i], env[i+1:]...)
}
}
}
final := "/usr/local/go"
if b.OS == "windows" {
final = `c:\go`
}
env = append(env,
"GOARCH="+b.Arch,
"GOHOSTARCH="+b.Arch,
"GOHOSTOS="+b.OS,
"GOOS="+b.OS,
"GOROOT="+b.root,
"GOROOT_FINAL="+final,
"GOPATH="+b.gopath,
)
return env
}
func (b *Build) Upload(version string, filename string) error {
// Prepare upload metadata.
var labels []string
os_, arch := b.OS, b.Arch
switch b.Arch {
case "386":
arch = "x86 32-bit"
case "amd64":
arch = "x86 64-bit"
}
if arch != "" {
labels = append(labels, "Arch-"+b.Arch)
}
var opsys, ftype string // labels
switch b.OS {
case "linux":
os_ = "Linux"
opsys = "Linux"
case "freebsd":
os_ = "FreeBSD"
opsys = "FreeBSD"
case "darwin":
os_ = "Mac OS X"
opsys = "OSX"
case "netbsd":
os_ = "NetBSD"
opsys = "NetBSD"
case "windows":
os_ = "Windows"
opsys = "Windows"
}
summary := fmt.Sprintf("%s %s (%s)", version, os_, arch)
switch {
case strings.HasSuffix(filename, ".msi"):
ftype = "Installer"
summary += " MSI installer"
case strings.HasSuffix(filename, ".pkg"):
ftype = "Installer"
summary += " PKG installer"
case strings.HasSuffix(filename, ".zip"):
ftype = "Archive"
summary += " ZIP archive"
case strings.HasSuffix(filename, ".tar.gz"):
ftype = "Archive"
summary += " tarball"
}
if b.Source {
ftype = "Source"
summary = fmt.Sprintf("%s (source only)", version)
}
if opsys != "" {
labels = append(labels, "OpSys-"+opsys)
}
if ftype != "" {
labels = append(labels, "Type-"+ftype)
}
if *addLabel != "" {
labels = append(labels, *addLabel)
}
// Put "Go" prefix on summary when it doesn't already begin with "go".
if !strings.HasPrefix(strings.ToLower(summary), "go") {
summary = "Go " + summary
}
// Open file to upload.
f, err := os.Open(filename)
if err != nil {
return err
}
defer f.Close()
// Prepare multipart payload.
body := new(bytes.Buffer)
w := multipart.NewWriter(body)
if err := w.WriteField("summary", summary); err != nil {
return err
}
for _, l := range labels {
if err := w.WriteField("label", l); err != nil {
return err
}
}
fw, err := w.CreateFormFile("filename", filename)
if err != nil {
return err
}
if _, err = io.Copy(fw, f); err != nil {
return err
}
if err := w.Close(); err != nil {
return err
}
// Send the file to Google Code.
req, err := http.NewRequest("POST", uploadURL, body)
if err != nil {
return err
}
token := fmt.Sprintf("%s:%s", username, password)
token = base64.StdEncoding.EncodeToString([]byte(token))
req.Header.Set("Authorization", "Basic "+token)
req.Header.Set("Content-type", w.FormDataContentType())
resp, err := http.DefaultTransport.RoundTrip(req)
if err != nil {
return err
}
if resp.StatusCode/100 != 2 {
fmt.Fprintln(os.Stderr, "upload failed")
defer resp.Body.Close()
io.Copy(os.Stderr, resp.Body)
return fmt.Errorf("upload: %s", resp.Status)
}
return nil
}
func (b *Build) clean(files []string) error {
for _, name := range files {
err := os.RemoveAll(filepath.Join(b.root, name))
if err != nil {
return err
}
}
return nil
}
func exists(path string) bool {
_, err := os.Stat(path)
return err == nil
}
func readCredentials() error {
name := os.Getenv("HOME")
if runtime.GOOS == "windows" {
name = os.Getenv("HOMEDRIVE") + os.Getenv("HOMEPATH")
}
name = filepath.Join(name, ".gobuildkey")
f, err := os.Open(name)
if err != nil {
return err
}
defer f.Close()
r := bufio.NewReader(f)
for i := 0; i < 3; i++ {
b, _, err := r.ReadLine()
if err != nil {
return err
}
b = bytes.TrimSpace(b)
switch i {
case 1:
username = string(b)
case 2:
password = string(b)
}
}
return nil
}
func cp(dst, src string) error {
sf, err := os.Open(src)
if err != nil {
return err
}
defer sf.Close()
fi, err := sf.Stat()
if err != nil {
return err
}
df, err := os.Create(dst)
if err != nil {
return err
}
defer df.Close()
// Windows doesn't currently implement Fchmod
if runtime.GOOS != "windows" {
if err := df.Chmod(fi.Mode()); err != nil {
return err
}
}
_, err = io.Copy(df, sf)
return err
}
func cpDir(dst, src string) error {
walk := func(srcPath string, info os.FileInfo, err error) error {
if err != nil {
return err
}
dstPath := filepath.Join(dst, srcPath[len(src):])
if info.IsDir() {
return os.MkdirAll(dstPath, 0755)
}
return cp(dstPath, srcPath)
}
return filepath.Walk(src, walk)
}
func cpAllDir(dst, basePath string, dirs ...string) error {
for _, dir := range dirs {
if err := cpDir(filepath.Join(dst, dir), filepath.Join(basePath, dir)); err != nil {
return err
}
}
return nil
}
func makeTar(targ, workdir string) error {
f, err := os.Create(targ)
if err != nil {
return err
}
zout := gzip.NewWriter(f)
tw := tar.NewWriter(zout)
err = filepath.Walk(workdir, func(path string, fi os.FileInfo, err error) error {
if !strings.HasPrefix(path, workdir) {
log.Panicf("walked filename %q doesn't begin with workdir %q", path, workdir)
}
name := path[len(workdir):]
// Chop of any leading / from filename, leftover from removing workdir.
if strings.HasPrefix(name, "/") {
name = name[1:]
}
// Don't include things outside of the go subdirectory (for instance,
// the zip file that we're currently writing here.)
if !strings.HasPrefix(name, "go/") {
return nil
}
if *verbose {
log.Printf("adding to tar: %s", name)
}
target, _ := os.Readlink(path)
hdr, err := tar.FileInfoHeader(fi, target)
if err != nil {
return err
}
hdr.Name = name
hdr.Uname = "root"
hdr.Gname = "root"
hdr.Uid = 0
hdr.Gid = 0
// Force permissions to 0755 for executables, 0644 for everything else.
if fi.Mode().Perm()&0111 != 0 {
hdr.Mode = hdr.Mode&^0777 | 0755
} else {
hdr.Mode = hdr.Mode&^0777 | 0644
}
err = tw.WriteHeader(hdr)
if err != nil {
return fmt.Errorf("Error writing file %q: %v", name, err)
}
if fi.IsDir() {
return nil
}
r, err := os.Open(path)
if err != nil {
return err
}
defer r.Close()
_, err = io.Copy(tw, r)
return err
})
if err != nil {
return err
}
if err := tw.Close(); err != nil {
return err
}
if err := zout.Close(); err != nil {
return err
}
return f.Close()
}
func makeZip(targ, workdir string) error {
f, err := os.Create(targ)
if err != nil {
return err
}
zw := zip.NewWriter(f)
err = filepath.Walk(workdir, func(path string, fi os.FileInfo, err error) error {
if !strings.HasPrefix(path, workdir) {
log.Panicf("walked filename %q doesn't begin with workdir %q", path, workdir)
}
name := path[len(workdir):]
// Convert to Unix-style named paths, as that's the
// type of zip file that archive/zip creates.
name = strings.Replace(name, "\\", "/", -1)
// Chop of any leading / from filename, leftover from removing workdir.
if strings.HasPrefix(name, "/") {
name = name[1:]
}
// Don't include things outside of the go subdirectory (for instance,
// the zip file that we're currently writing here.)
if !strings.HasPrefix(name, "go/") {
return nil
}
if *verbose {
log.Printf("adding to zip: %s", name)
}
fh, err := zip.FileInfoHeader(fi)
if err != nil {
return err
}
fh.Name = name
fh.Method = zip.Deflate
if fi.IsDir() {
fh.Name += "/" // append trailing slash
fh.Method = zip.Store // no need to deflate 0 byte files
}
w, err := zw.CreateHeader(fh)
if err != nil {
return err
}
if fi.IsDir() {
return nil
}
r, err := os.Open(path)
if err != nil {
return err
}
defer r.Close()
_, err = io.Copy(w, r)
return err
})
if err != nil {
return err
}
if err := zw.Close(); err != nil {
return err
}
return f.Close()
}
type tool struct {
name string
commonDirs []string
}
var wixTool = tool{
"http://wix.sourceforge.net/, version 3.5",
[]string{`C:\Program Files\Windows Installer XML v3.5\bin`,
`C:\Program Files (x86)\Windows Installer XML v3.5\bin`},
}
var hgTool = tool{
"http://mercurial.selenic.com/wiki/WindowsInstall",
[]string{`C:\Program Files\Mercurial`,
`C:\Program Files (x86)\Mercurial`,
},
}
var gccTool = tool{
"Mingw gcc; http://sourceforge.net/projects/mingw/files/Installer/mingw-get-inst/",
[]string{`C:\Mingw\bin`},
}
var windowsDeps = map[string]tool{
"gcc": gccTool,
"heat": wixTool,
"candle": wixTool,
"light": wixTool,
"cmd": {"Windows cmd.exe", nil},
"hg": hgTool,
}
func checkWindowsDeps() {
for prog, help := range windowsDeps {
absPath, err := lookPath(prog)
if err != nil {
log.Fatalf("Failed to find necessary binary %q in path or common locations; %s", prog, help)
}
if *verbose {
log.Printf("found windows dep %s at %s", prog, absPath)
}
}
}
func lookPath(prog string) (absPath string, err error) {
absPath, err = exec.LookPath(prog)
if err == nil {
return
}
t, ok := windowsDeps[prog]
if !ok {
return
}
for _, dir := range t.commonDirs {
for _, ext := range []string{"exe", "bat"} {
absPath = filepath.Join(dir, prog+"."+ext)
if _, err1 := os.Stat(absPath); err1 == nil {
err = nil
os.Setenv("PATH", os.Getenv("PATH")+";"+dir)
return
}
}
}
return
}