1
0
mirror of https://github.com/golang/go synced 2024-11-14 14:40:23 -07:00
go/misc/dist/bindist.go
Brad Fitzpatrick d5e97ea2f5 build: change how cmd/api is run in run.bash and run.bat
In prep for Robert's forthcoming cmd/api rewrite which
depends on the go.tools subrepo, we'll need to be more
careful about how and when we run cmd/api.

Rather than implement this policy in both run.bash and
run.bat, this change moves the policy and mechanism into
cmd/api/run.go, which will then evolve.

The plan is in a TODO in run.go.

R=golang-dev, gri
CC=golang-dev
https://golang.org/cl/12482044
2013-08-07 13:49:37 -07:00

947 lines
21 KiB
Go

// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// This is a tool for packaging binary releases.
// It supports FreeBSD, Linux, NetBSD, OS X, and Windows.
package main
import (
"archive/tar"
"archive/zip"
"bufio"
"bytes"
"compress/gzip"
"encoding/base64"
"flag"
"fmt"
"io"
"io/ioutil"
"log"
"mime/multipart"
"net/http"
"os"
"os/exec"
"path/filepath"
"regexp"
"runtime"
"strings"
)
var (
tag = flag.String("tag", "release", "mercurial tag to check out")
repo = flag.String("repo", "https://code.google.com/p/go", "repo URL")
verbose = flag.Bool("v", false, "verbose output")
upload = flag.Bool("upload", true, "upload resulting files to Google Code")
wxsFile = flag.String("wxs", "", "path to custom installer.wxs")
addLabel = flag.String("label", "", "additional label to apply to file when uploading")
includeRace = flag.Bool("race", true, "build race detector packages")
versionOverride = flag.String("version", "", "override version name")
username, password string // for Google Code upload
)
const (
uploadURL = "https://go.googlecode.com/files"
godocPath = "code.google.com/p/go.tools/cmd/godoc"
tourPath = "code.google.com/p/go-tour"
)
var preBuildCleanFiles = []string{
"lib/codereview",
"misc/dashboard/godashboard",
"src/cmd/cov",
"src/cmd/prof",
"src/pkg/exp",
"src/pkg/old",
}
var cleanFiles = []string{
".hg",
".hgtags",
".hgignore",
"VERSION.cache",
}
var sourceCleanFiles = []string{
"bin",
"pkg",
}
var tourPackages = []string{
"pic",
"tree",
"wc",
}
var tourContent = []string{
"js",
"prog",
"solutions",
"static",
"template",
"tour.article",
}
// The os-arches that support the race toolchain.
var raceAvailable = []string{
"darwin-amd64",
"linux-amd64",
"windows-amd64",
}
var fileRe = regexp.MustCompile(`^(go[a-z0-9-.]+)\.(src|([a-z0-9]+)-([a-z0-9]+))\.`)
func main() {
flag.Usage = func() {
fmt.Fprintf(os.Stderr, "usage: %s [flags] targets...\n", os.Args[0])
flag.PrintDefaults()
os.Exit(2)
}
flag.Parse()
if flag.NArg() == 0 {
flag.Usage()
}
if runtime.GOOS == "windows" {
checkWindowsDeps()
}
if *upload {
if err := readCredentials(); err != nil {
log.Println("readCredentials:", err)
}
}
for _, targ := range flag.Args() {
var b Build
if m := fileRe.FindStringSubmatch(targ); m != nil {
// targ is a file name; upload it to googlecode.
version := m[1]
if m[2] == "src" {
b.Source = true
} else {
b.OS = m[3]
b.Arch = m[4]
}
if !*upload {
log.Printf("%s: -upload=false, skipping", targ)
continue
}
if err := b.Upload(version, targ); err != nil {
log.Printf("%s: %v", targ, err)
}
continue
}
if targ == "source" {
b.Source = true
} else {
p := strings.SplitN(targ, "-", 2)
if len(p) != 2 {
log.Println("Ignoring unrecognized target:", targ)
continue
}
b.OS = p[0]
b.Arch = p[1]
if *includeRace {
for _, t := range raceAvailable {
if t == targ {
b.Race = true
}
}
}
}
if err := b.Do(); err != nil {
log.Printf("%s: %v", targ, err)
}
}
}
type Build struct {
Source bool // if true, OS and Arch must be empty
Race bool // build race toolchain
OS string
Arch string
root string
gopath string
}
func (b *Build) Do() error {
work, err := ioutil.TempDir("", "bindist")
if err != nil {
return err
}
defer os.RemoveAll(work)
b.root = filepath.Join(work, "go")
b.gopath = work
// Clone Go distribution and update to tag.
_, err = b.hgCmd(work, "clone", *repo, b.root)
if err != nil {
return err
}
_, err = b.hgCmd(b.root, "update", *tag)
if err != nil {
return err
}
// Remove exp and old packages.
if err := b.clean(preBuildCleanFiles); err != nil {
return err
}
src := filepath.Join(b.root, "src")
if b.Source {
if runtime.GOOS == "windows" {
log.Print("Warning: running make.bash on Windows; source builds are intended to be run on a Unix machine")
}
// Build dist tool only.
_, err = b.run(src, "bash", "make.bash", "--dist-tool")
} else {
// Build.
if b.OS == "windows" {
_, err = b.run(src, "cmd", "/C", "make.bat")
} else {
_, err = b.run(src, "bash", "make.bash")
}
if b.Race {
if err != nil {
return err
}
goCmd := filepath.Join(b.root, "bin", "go")
if b.OS == "windows" {
goCmd += ".exe"
}
_, err = b.run(src, goCmd, "install", "-race", "std")
if err != nil {
return err
}
// Re-install std without -race, so that we're not left
// with a slower, race-enabled cmd/go, etc.
_, err = b.run(src, goCmd, "install", "-a", "std")
// Re-building go command leaves old versions of go.exe as go.exe~ on windows.
// See (*builder).copyFile in $GOROOT/src/cmd/go/build.go for details.
// Remove it manually.
if b.OS == "windows" {
os.Remove(goCmd + "~")
}
}
if err != nil {
return err
}
err = b.godoc()
if err != nil {
return err
}
err = b.tour()
}
if err != nil {
return err
}
// Get version strings.
var (
version string // "weekly.2012-03-04"
fullVersion []byte // "weekly.2012-03-04 9353aa1efdf3"
)
pat := filepath.Join(b.root, "pkg/tool/*/dist*") // trailing * for .exe
m, err := filepath.Glob(pat)
if err != nil {
return err
}
if len(m) == 0 {
return fmt.Errorf("couldn't find dist in %q", pat)
}
fullVersion, err = b.run("", m[0], "version")
if err != nil {
return err
}
fullVersion = bytes.TrimSpace(fullVersion)
v := bytes.SplitN(fullVersion, []byte(" "), 2)
version = string(v[0])
if *versionOverride != "" {
version = *versionOverride
}
// Write VERSION file.
err = ioutil.WriteFile(filepath.Join(b.root, "VERSION"), fullVersion, 0644)
if err != nil {
return err
}
// Clean goroot.
if err := b.clean(cleanFiles); err != nil {
return err
}
if b.Source {
if err := b.clean(sourceCleanFiles); err != nil {
return err
}
}
// Create packages.
base := fmt.Sprintf("%s.%s-%s", version, b.OS, b.Arch)
if !strings.HasPrefix(base, "go") {
base = "go." + base
}
var targs []string
switch b.OS {
case "linux", "freebsd", "netbsd", "":
// build tarball
targ := base
if b.Source {
targ = fmt.Sprintf("%s.src", version)
if !strings.HasPrefix(targ, "go") {
targ = "go." + targ
}
}
targ += ".tar.gz"
err = makeTar(targ, work)
targs = append(targs, targ)
case "darwin":
// build tarball
targ := base + ".tar.gz"
err = makeTar(targ, work)
targs = append(targs, targ)
// build pkg
// arrange work so it's laid out as the dest filesystem
etc := filepath.Join(b.root, "misc/dist/darwin/etc")
_, err = b.run(work, "cp", "-r", etc, ".")
if err != nil {
return err
}
localDir := filepath.Join(work, "usr/local")
err = os.MkdirAll(localDir, 0755)
if err != nil {
return err
}
_, err = b.run(work, "mv", "go", localDir)
if err != nil {
return err
}
// build package
pkgdest, err := ioutil.TempDir("", "pkgdest")
if err != nil {
return err
}
defer os.RemoveAll(pkgdest)
dist := filepath.Join(runtime.GOROOT(), "misc/dist")
_, err = b.run("", "pkgbuild",
"--identifier", "com.googlecode.go",
"--version", "1.0",
"--scripts", filepath.Join(dist, "darwin/scripts"),
"--root", work,
filepath.Join(pkgdest, "com.googlecode.go.pkg"))
if err != nil {
return err
}
targ = base + ".pkg"
_, err = b.run("", "productbuild",
"--distribution", filepath.Join(dist, "darwin/Distribution"),
"--resources", filepath.Join(dist, "darwin/Resources"),
"--package-path", pkgdest,
targ)
if err != nil {
return err
}
targs = append(targs, targ)
case "windows":
// Create ZIP file.
zip := filepath.Join(work, base+".zip")
err = makeZip(zip, work)
// Copy zip to target file.
targ := base + ".zip"
err = cp(targ, zip)
if err != nil {
return err
}
targs = append(targs, targ)
// Create MSI installer.
win := filepath.Join(b.root, "misc/dist/windows")
installer := filepath.Join(win, "installer.wxs")
if *wxsFile != "" {
installer = *wxsFile
}
appfiles := filepath.Join(work, "AppFiles.wxs")
msi := filepath.Join(work, "installer.msi")
// Gather files.
_, err = b.run(work, "heat", "dir", "go",
"-nologo",
"-gg", "-g1", "-srd", "-sfrag",
"-cg", "AppFiles",
"-template", "fragment",
"-dr", "INSTALLDIR",
"-var", "var.SourceDir",
"-out", appfiles)
if err != nil {
return err
}
// Build package.
_, err = b.run(work, "candle",
"-nologo",
"-dVersion="+version,
"-dArch="+b.Arch,
"-dSourceDir=go",
installer, appfiles)
if err != nil {
return err
}
appfiles = filepath.Join(work, "AppFiles.wixobj")
installer = filepath.Join(work, "installer.wixobj")
_, err = b.run(win, "light",
"-nologo",
"-ext", "WixUIExtension",
"-ext", "WixUtilExtension",
installer, appfiles,
"-o", msi)
if err != nil {
return err
}
// Copy installer to target file.
targ = base + ".msi"
err = cp(targ, msi)
targs = append(targs, targ)
}
if err == nil && *upload {
for _, targ := range targs {
err = b.Upload(version, targ)
if err != nil {
return err
}
}
}
return err
}
func (b *Build) godoc() error {
defer func() {
// Clean work files from GOPATH directory.
for _, d := range []string{"bin", "pkg", "src"} {
os.RemoveAll(filepath.Join(b.gopath, d))
}
}()
// go get the godoc package.
// The go tool knows to install to $GOROOT/bin.
_, err := b.run(b.gopath, filepath.Join(b.root, "bin", "go"), "get", godocPath)
if err != nil {
return err
}
// Copy templates from go.tools/cmd/godoc/template to GOROOT/lib/godoc.
return cpDir(
filepath.Join(b.root, "lib", "godoc"),
filepath.Join(b.gopath, "src", filepath.FromSlash(godocPath), "template"),
)
}
func (b *Build) tour() error {
defer func() {
// Clean work files from GOPATH directory.
for _, d := range []string{"bin", "pkg", "src"} {
os.RemoveAll(filepath.Join(b.gopath, d))
}
}()
// go get the gotour package.
_, err := b.run(b.gopath, filepath.Join(b.root, "bin", "go"), "get", tourPath+"/gotour")
if err != nil {
return err
}
// Copy all the tour content to $GOROOT/misc/tour.
importPath := filepath.FromSlash(tourPath)
tourSrc := filepath.Join(b.gopath, "src", importPath)
contentDir := filepath.Join(b.root, "misc", "tour")
if err = cpAllDir(contentDir, tourSrc, tourContent...); err != nil {
return err
}
// Copy the tour source code so it's accessible with $GOPATH pointing to $GOROOT/misc/tour.
if err = cpAllDir(filepath.Join(contentDir, "src", importPath), tourSrc, tourPackages...); err != nil {
return err
}
// Copy gotour binary to tool directory as "tour"; invoked as "go tool tour".
return cp(
filepath.Join(b.root, "pkg", "tool", b.OS+"_"+b.Arch, "tour"+ext()),
filepath.Join(b.gopath, "bin", "gotour"+ext()),
)
}
func ext() string {
if runtime.GOOS == "windows" {
return ".exe"
}
return ""
}
func (b *Build) hgCmd(dir string, args ...string) ([]byte, error) {
return b.run(dir, "hg", append([]string{"--config", "extensions.codereview=!"}, args...)...)
}
func (b *Build) run(dir, name string, args ...string) ([]byte, error) {
buf := new(bytes.Buffer)
absName, err := lookPath(name)
if err != nil {
return nil, err
}
cmd := exec.Command(absName, args...)
var output io.Writer = buf
if *verbose {
log.Printf("Running %q %q", absName, args)
output = io.MultiWriter(buf, os.Stdout)
}
cmd.Stdout = output
cmd.Stderr = output
cmd.Dir = dir
cmd.Env = b.env()
if err := cmd.Run(); err != nil {
fmt.Fprintf(os.Stderr, "%s", buf.Bytes())
return nil, fmt.Errorf("%s %s: %v", name, strings.Join(args, " "), err)
}
return buf.Bytes(), nil
}
var cleanEnv = []string{
"GOARCH",
"GOBIN",
"GOHOSTARCH",
"GOHOSTOS",
"GOOS",
"GOROOT",
"GOROOT_FINAL",
"GOPATH",
}
func (b *Build) env() []string {
env := os.Environ()
for i := 0; i < len(env); i++ {
for _, c := range cleanEnv {
if strings.HasPrefix(env[i], c+"=") {
env = append(env[:i], env[i+1:]...)
}
}
}
final := "/usr/local/go"
if b.OS == "windows" {
final = `c:\go`
}
env = append(env,
"GOARCH="+b.Arch,
"GOHOSTARCH="+b.Arch,
"GOHOSTOS="+b.OS,
"GOOS="+b.OS,
"GOROOT="+b.root,
"GOROOT_FINAL="+final,
"GOPATH="+b.gopath,
)
return env
}
func (b *Build) Upload(version string, filename string) error {
// Prepare upload metadata.
var labels []string
os_, arch := b.OS, b.Arch
switch b.Arch {
case "386":
arch = "x86 32-bit"
case "amd64":
arch = "x86 64-bit"
}
if arch != "" {
labels = append(labels, "Arch-"+b.Arch)
}
var opsys, ftype string // labels
switch b.OS {
case "linux":
os_ = "Linux"
opsys = "Linux"
case "freebsd":
os_ = "FreeBSD"
opsys = "FreeBSD"
case "darwin":
os_ = "Mac OS X"
opsys = "OSX"
case "netbsd":
os_ = "NetBSD"
opsys = "NetBSD"
case "windows":
os_ = "Windows"
opsys = "Windows"
}
summary := fmt.Sprintf("%s %s (%s)", version, os_, arch)
switch {
case strings.HasSuffix(filename, ".msi"):
ftype = "Installer"
summary += " MSI installer"
case strings.HasSuffix(filename, ".pkg"):
ftype = "Installer"
summary += " PKG installer"
case strings.HasSuffix(filename, ".zip"):
ftype = "Archive"
summary += " ZIP archive"
case strings.HasSuffix(filename, ".tar.gz"):
ftype = "Archive"
summary += " tarball"
}
if b.Source {
ftype = "Source"
summary = fmt.Sprintf("%s (source only)", version)
}
if opsys != "" {
labels = append(labels, "OpSys-"+opsys)
}
if ftype != "" {
labels = append(labels, "Type-"+ftype)
}
if *addLabel != "" {
labels = append(labels, *addLabel)
}
// Put "Go" prefix on summary when it doesn't already begin with "go".
if !strings.HasPrefix(strings.ToLower(summary), "go") {
summary = "Go " + summary
}
// Open file to upload.
f, err := os.Open(filename)
if err != nil {
return err
}
defer f.Close()
// Prepare multipart payload.
body := new(bytes.Buffer)
w := multipart.NewWriter(body)
if err := w.WriteField("summary", summary); err != nil {
return err
}
for _, l := range labels {
if err := w.WriteField("label", l); err != nil {
return err
}
}
fw, err := w.CreateFormFile("filename", filename)
if err != nil {
return err
}
if _, err = io.Copy(fw, f); err != nil {
return err
}
if err := w.Close(); err != nil {
return err
}
// Send the file to Google Code.
req, err := http.NewRequest("POST", uploadURL, body)
if err != nil {
return err
}
token := fmt.Sprintf("%s:%s", username, password)
token = base64.StdEncoding.EncodeToString([]byte(token))
req.Header.Set("Authorization", "Basic "+token)
req.Header.Set("Content-type", w.FormDataContentType())
resp, err := http.DefaultTransport.RoundTrip(req)
if err != nil {
return err
}
if resp.StatusCode/100 != 2 {
fmt.Fprintln(os.Stderr, "upload failed")
defer resp.Body.Close()
io.Copy(os.Stderr, resp.Body)
return fmt.Errorf("upload: %s", resp.Status)
}
return nil
}
func (b *Build) clean(files []string) error {
for _, name := range files {
err := os.RemoveAll(filepath.Join(b.root, name))
if err != nil {
return err
}
}
return nil
}
func exists(path string) bool {
_, err := os.Stat(path)
return err == nil
}
func readCredentials() error {
name := os.Getenv("HOME")
if runtime.GOOS == "windows" {
name = os.Getenv("HOMEDRIVE") + os.Getenv("HOMEPATH")
}
name = filepath.Join(name, ".gobuildkey")
f, err := os.Open(name)
if err != nil {
return err
}
defer f.Close()
r := bufio.NewReader(f)
for i := 0; i < 3; i++ {
b, _, err := r.ReadLine()
if err != nil {
return err
}
b = bytes.TrimSpace(b)
switch i {
case 1:
username = string(b)
case 2:
password = string(b)
}
}
return nil
}
func cp(dst, src string) error {
sf, err := os.Open(src)
if err != nil {
return err
}
defer sf.Close()
fi, err := sf.Stat()
if err != nil {
return err
}
df, err := os.Create(dst)
if err != nil {
return err
}
defer df.Close()
// Windows doesn't currently implement Fchmod
if runtime.GOOS != "windows" {
if err := df.Chmod(fi.Mode()); err != nil {
return err
}
}
_, err = io.Copy(df, sf)
return err
}
func cpDir(dst, src string) error {
walk := func(srcPath string, info os.FileInfo, err error) error {
if err != nil {
return err
}
dstPath := filepath.Join(dst, srcPath[len(src):])
if info.IsDir() {
return os.MkdirAll(dstPath, 0755)
}
return cp(dstPath, srcPath)
}
return filepath.Walk(src, walk)
}
func cpAllDir(dst, basePath string, dirs ...string) error {
for _, dir := range dirs {
if err := cpDir(filepath.Join(dst, dir), filepath.Join(basePath, dir)); err != nil {
return err
}
}
return nil
}
func makeTar(targ, workdir string) error {
f, err := os.Create(targ)
if err != nil {
return err
}
zout := gzip.NewWriter(f)
tw := tar.NewWriter(zout)
err = filepath.Walk(workdir, func(path string, fi os.FileInfo, err error) error {
if !strings.HasPrefix(path, workdir) {
log.Panicf("walked filename %q doesn't begin with workdir %q", path, workdir)
}
name := path[len(workdir):]
// Chop of any leading / from filename, leftover from removing workdir.
if strings.HasPrefix(name, "/") {
name = name[1:]
}
// Don't include things outside of the go subdirectory (for instance,
// the zip file that we're currently writing here.)
if !strings.HasPrefix(name, "go/") {
return nil
}
if *verbose {
log.Printf("adding to tar: %s", name)
}
target, _ := os.Readlink(path)
hdr, err := tar.FileInfoHeader(fi, target)
if err != nil {
return err
}
hdr.Name = name
hdr.Uname = "root"
hdr.Gname = "root"
hdr.Uid = 0
hdr.Gid = 0
// Force permissions to 0755 for executables, 0644 for everything else.
if fi.Mode().Perm()&0111 != 0 {
hdr.Mode = hdr.Mode&^0777 | 0755
} else {
hdr.Mode = hdr.Mode&^0777 | 0644
}
err = tw.WriteHeader(hdr)
if err != nil {
return fmt.Errorf("Error writing file %q: %v", name, err)
}
if fi.IsDir() {
return nil
}
r, err := os.Open(path)
if err != nil {
return err
}
defer r.Close()
_, err = io.Copy(tw, r)
return err
})
if err != nil {
return err
}
if err := tw.Close(); err != nil {
return err
}
if err := zout.Close(); err != nil {
return err
}
return f.Close()
}
func makeZip(targ, workdir string) error {
f, err := os.Create(targ)
if err != nil {
return err
}
zw := zip.NewWriter(f)
err = filepath.Walk(workdir, func(path string, fi os.FileInfo, err error) error {
if !strings.HasPrefix(path, workdir) {
log.Panicf("walked filename %q doesn't begin with workdir %q", path, workdir)
}
name := path[len(workdir):]
// Convert to Unix-style named paths, as that's the
// type of zip file that archive/zip creates.
name = strings.Replace(name, "\\", "/", -1)
// Chop of any leading / from filename, leftover from removing workdir.
if strings.HasPrefix(name, "/") {
name = name[1:]
}
// Don't include things outside of the go subdirectory (for instance,
// the zip file that we're currently writing here.)
if !strings.HasPrefix(name, "go/") {
return nil
}
if *verbose {
log.Printf("adding to zip: %s", name)
}
fh, err := zip.FileInfoHeader(fi)
if err != nil {
return err
}
fh.Name = name
fh.Method = zip.Deflate
if fi.IsDir() {
fh.Name += "/" // append trailing slash
fh.Method = zip.Store // no need to deflate 0 byte files
}
w, err := zw.CreateHeader(fh)
if err != nil {
return err
}
if fi.IsDir() {
return nil
}
r, err := os.Open(path)
if err != nil {
return err
}
defer r.Close()
_, err = io.Copy(w, r)
return err
})
if err != nil {
return err
}
if err := zw.Close(); err != nil {
return err
}
return f.Close()
}
type tool struct {
name string
commonDirs []string
}
var wixTool = tool{
"http://wix.sourceforge.net/, version 3.5",
[]string{`C:\Program Files\Windows Installer XML v3.5\bin`,
`C:\Program Files (x86)\Windows Installer XML v3.5\bin`},
}
var hgTool = tool{
"http://mercurial.selenic.com/wiki/WindowsInstall",
[]string{`C:\Program Files\Mercurial`,
`C:\Program Files (x86)\Mercurial`,
},
}
var gccTool = tool{
"Mingw gcc; http://sourceforge.net/projects/mingw/files/Installer/mingw-get-inst/",
[]string{`C:\Mingw\bin`},
}
var windowsDeps = map[string]tool{
"gcc": gccTool,
"heat": wixTool,
"candle": wixTool,
"light": wixTool,
"cmd": {"Windows cmd.exe", nil},
"hg": hgTool,
}
func checkWindowsDeps() {
for prog, help := range windowsDeps {
absPath, err := lookPath(prog)
if err != nil {
log.Fatalf("Failed to find necessary binary %q in path or common locations; %s", prog, help)
}
if *verbose {
log.Printf("found windows dep %s at %s", prog, absPath)
}
}
}
func lookPath(prog string) (absPath string, err error) {
absPath, err = exec.LookPath(prog)
if err == nil {
return
}
t, ok := windowsDeps[prog]
if !ok {
return
}
for _, dir := range t.commonDirs {
for _, ext := range []string{"exe", "bat"} {
absPath = filepath.Join(dir, prog+"."+ext)
if _, err1 := os.Stat(absPath); err1 == nil {
err = nil
os.Setenv("PATH", os.Getenv("PATH")+";"+dir)
return
}
}
}
return
}