From 7b848c69647c52d69127ccef79cc7d01c0ec02c6 Mon Sep 17 00:00:00 2001 From: Russ Cox Date: Mon, 13 Feb 2012 22:31:51 -0500 Subject: [PATCH] cmd/dist: cross-compiling fixes This CL makes it possible to run make.bash with GOOS and GOARCH set to something other than the native host GOOS and GOARCH. As part of the CL, the tool directory moves from bin/tool/ to pkg/tool/goos_goarch where goos and goarch are the values for the host system (running the build), not the target. pkg/ is not technically appropriate, but C objects are there now tool (pkg/obj/) so this puts all the generated binaries in one place (rm -rf $GOROOT/pkg cleans everything). Including goos_goarch in the name allows different systems to share a single $GOROOT on a shared file system. Fixes #2920. R=golang-dev, r CC=golang-dev https://golang.org/cl/5645093 --- src/all.bash | 4 +- src/cmd/dist/a.h | 2 + src/cmd/dist/build.c | 194 +++++++++++++++++++-------- src/cmd/dist/buildruntime.c | 4 +- src/cmd/dist/goc2c.c | 22 ++- src/cmd/dist/unix.c | 7 + src/cmd/go/build.go | 27 ++-- src/cmd/go/pkg.go | 11 +- src/cmd/go/tool.go | 3 +- src/make.bash | 28 ++-- src/make.bat | 36 ++++- src/pkg/exp/types/gcimporter_test.go | 11 +- src/pkg/go/build/path.go | 3 + src/run.bash | 2 +- test/fixedbugs/bug302.go | 2 +- test/run | 4 +- 16 files changed, 256 insertions(+), 104 deletions(-) diff --git a/src/all.bash b/src/all.bash index d80a09d1162..932b65dc070 100755 --- a/src/all.bash +++ b/src/all.bash @@ -9,5 +9,5 @@ if [ ! -f make.bash ]; then exit 1 fi . ./make.bash --no-banner -bash run.bash --no-rebuild -../bin/tool/dist banner # print build info +bash run.bash --no-rebuild --banner +$GOTOOLDIR/dist banner # print build info diff --git a/src/cmd/dist/a.h b/src/cmd/dist/a.h index e90f7fe65d8..3fbace3f6d2 100644 --- a/src/cmd/dist/a.h +++ b/src/cmd/dist/a.h @@ -72,7 +72,9 @@ extern char *goroot; extern char *goroot_final; extern char *goversion; extern char *workdir; +extern char *tooldir; extern char *slash; +extern bool rebuildall; int find(char*, char**, int); void init(void); diff --git a/src/cmd/dist/build.c b/src/cmd/dist/build.c index c8984300b09..72097230a11 100644 --- a/src/cmd/dist/build.c +++ b/src/cmd/dist/build.c @@ -13,15 +13,19 @@ char *goarch; char *gobin; char *gohostarch; +char *gohostchar; char *gohostos; char *goos; char *goroot = GOROOT_FINAL; char *goroot_final = GOROOT_FINAL; char *workdir; +char *tooldir; char *gochar; char *goversion; char *slash; // / for unix, \ for windows +bool rebuildall = 0; + static bool shouldbuild(char*, char*); static void copy(char*, char*); static char *findgoversion(void); @@ -99,14 +103,18 @@ init(void) if(b.len > 0) gohostarch = btake(&b); - if(find(gohostarch, okgoarch, nelem(okgoarch)) < 0) + i = find(gohostarch, okgoarch, nelem(okgoarch)); + if(i < 0) fatal("unknown $GOHOSTARCH %s", gohostarch); + bprintf(&b, "%c", gochars[i]); + gohostchar = btake(&b); xgetenv(&b, "GOARCH"); if(b.len == 0) bwritestr(&b, gohostarch); goarch = btake(&b); - if((i=find(goarch, okgoarch, nelem(okgoarch))) < 0) + i = find(goarch, okgoarch, nelem(okgoarch)); + if(i < 0) fatal("unknown $GOARCH %s", goarch); bprintf(&b, "%c", gochars[i]); gochar = btake(&b); @@ -124,6 +132,9 @@ init(void) workdir = xworkdir(); xatexit(rmworkdir); + bpathf(&b, "%s/pkg/tool/%s_%s", goroot, gohostos, gohostarch); + tooldir = btake(&b); + bfree(&b); } @@ -256,6 +267,7 @@ static char *oldtool[] = { "8a", "8c", "8g", "8l", "6cov", "6nm", + "6prof", "cgo", "ebnflint", "goapi", @@ -281,11 +293,8 @@ setup(void) binit(&b); - // Create tool directory. + // Create bin directory. p = bpathf(&b, "%s/bin", goroot); - if(!isdir(p)) - xmkdir(p); - p = bpathf(&b, "%s/bin/tool", goroot); if(!isdir(p)) xmkdir(p); @@ -293,21 +302,42 @@ setup(void) p = bpathf(&b, "%s/pkg", goroot); if(!isdir(p)) xmkdir(p); - p = bpathf(&b, "%s/pkg/%s_%s", goroot, goos, goarch); - xremoveall(p); + p = bpathf(&b, "%s/pkg/%s_%s", goroot, gohostos, gohostarch); + if(rebuildall) + xremoveall(p); xmkdir(p); - + if(!streq(goos, gohostos) || !streq(goarch, gohostarch)) { + p = bpathf(&b, "%s/pkg/%s_%s", goroot, goos, goarch); + if(rebuildall) + xremoveall(p); + xmkdir(p); + } + // Create object directory. // We keep it in pkg/ so that all the generated binaries - // are in one tree. - p = bpathf(&b, "%s/pkg/obj", goroot); - xremoveall(p); - xmkdir(p); + // are in one tree. If pkg/obj/libgc.a exists, it is a dreg from + // before we used subdirectories of obj. Delete all of obj + // to clean up. + bpathf(&b, "%s/pkg/obj/libgc.a", goroot); + if(isfile(bstr(&b))) + xremoveall(bpathf(&b, "%s/pkg/obj", goroot)); + p = bpathf(&b, "%s/pkg/obj/%s_%s", goroot, gohostos, gohostarch); + if(rebuildall) + xremoveall(p); + xmkdirall(p); + + // Create tool directory. + // We keep it in pkg/, just like the object directory above. + xremoveall(tooldir); + xmkdirall(tooldir); + + // Remove tool binaries from before the tool/gohostos_gohostarch + xremoveall(bpathf(&b, "%s/bin/tool", goroot)); // Remove old pre-tool binaries. for(i=0; i 1) + xprintf("skip build for cross-compile %s\n", dir); + goto nobuild; + } // Compile the files. for(i=0; i 0 { - var stk importStack - p1 := loadPackage("cmd/cgo", &stk) - if p1.Error != nil { - fatalf("load cmd/cgo: %v", p1.Error) + // If we are not doing a cross-build, then record the binary we'll + // generate for cgo as a dependency of the build of any package + // using cgo, to make sure we do not overwrite the binary while + // a package is using it. If this is a cross-build, then the cgo we + // are writing is not the cgo we need to use. + if b.goos == runtime.GOOS && b.goarch == runtime.GOARCH { + if len(p.CgoFiles) > 0 || p.Standard && p.ImportPath == "runtime/cgo" { + var stk importStack + p1 := loadPackage("cmd/cgo", &stk) + if p1.Error != nil { + fatalf("load cmd/cgo: %v", p1.Error) + } + a.cgo = b.action(depMode, depMode, p1) + a.deps = append(a.deps, a.cgo) } - a.cgo = b.action(depMode, depMode, p1) - a.deps = append(a.deps, a.cgo) } if p.Standard { @@ -567,7 +574,11 @@ func (b *builder) build(a *action) error { sfiles = nil } - outGo, outObj, err := b.cgo(a.p, a.cgo.target, obj, gccfiles) + cgoExe := tool("cgo") + if a.cgo != nil { + cgoExe = a.cgo.target + } + outGo, outObj, err := b.cgo(a.p, cgoExe, obj, gccfiles) if err != nil { return err } diff --git a/src/cmd/go/pkg.go b/src/cmd/go/pkg.go index e9fe845c56d..00c5d30b50d 100644 --- a/src/cmd/go/pkg.go +++ b/src/cmd/go/pkg.go @@ -295,13 +295,14 @@ func scanPackage(ctxt *build.Context, t *build.Tree, arg, importPath, dir string if info.Package == "main" { _, elem := filepath.Split(importPath) - if ctxt.GOOS != toolGOOS || ctxt.GOARCH != toolGOARCH { - // Install cross-compiled binaries to subdirectories of bin. - elem = ctxt.GOOS + "_" + ctxt.GOARCH + "/" + elem - } + full := ctxt.GOOS + "_" + ctxt.GOARCH + "/" + elem if t.Goroot && isGoTool[p.ImportPath] { - p.target = filepath.Join(t.Path, "bin/tool", elem) + p.target = filepath.Join(t.Path, "pkg/tool", full) } else { + if ctxt.GOOS != toolGOOS || ctxt.GOARCH != toolGOARCH { + // Install cross-compiled binaries to subdirectories of bin. + elem = full + } p.target = filepath.Join(t.BinDir(), elem) } if ctxt.GOOS == "windows" { diff --git a/src/cmd/go/tool.go b/src/cmd/go/tool.go index 0ec5cf6a747..19b0d37dc00 100644 --- a/src/cmd/go/tool.go +++ b/src/cmd/go/tool.go @@ -31,7 +31,7 @@ var ( toolGOOS = runtime.GOOS toolGOARCH = runtime.GOARCH toolIsWindows = toolGOOS == "windows" - toolDir = filepath.Join(build.Path[0].Path, "bin", "tool") + toolDir = build.ToolDir ) const toolWindowsExtension = ".exe" @@ -97,6 +97,7 @@ func listTools() { setExitStatus(2) return } + sort.Strings(names) for _, name := range names { // Unify presentation by going to lower case. diff --git a/src/make.bash b/src/make.bash index 81ceeb7298d..e30743b6831 100755 --- a/src/make.bash +++ b/src/make.bash @@ -61,24 +61,36 @@ mkdir -p ../bin/tool export GOROOT="$(cd .. && pwd)" GOROOT_FINAL="${GOROOT_FINAL:-$GOROOT}" DEFGOROOT='-DGOROOT_FINAL="'"$GOROOT_FINAL"'"' -gcc -O2 -Wall -Werror -o ../bin/tool/dist -Icmd/dist "$DEFGOROOT" cmd/dist/*.c +gcc -O2 -Wall -Werror -ggdb -o cmd/dist/dist -Icmd/dist "$DEFGOROOT" cmd/dist/*.c +eval $(./cmd/dist/dist env) echo if [ "$1" = "--dist-tool" ]; then # Stop after building dist tool. + mv cmd/dist/dist $GOTOOLDIR/dist exit 0 fi -echo '# Building compilers and Go bootstrap tool.' -../bin/tool/dist bootstrap -v # builds go_bootstrap +echo "# Building compilers and Go bootstrap tool for host, $GOHOSTOS/$GOHOSTARCH." +./cmd/dist/dist bootstrap -a -v # builds go_bootstrap +# Delay move of dist tool to now, because bootstrap cleared tool directory. +mv cmd/dist/dist $GOTOOLDIR/dist +$GOTOOLDIR/go_bootstrap clean -i std echo -echo '# Building packages and commands.' -../bin/tool/go_bootstrap clean std -../bin/tool/go_bootstrap install -a -v std -rm -f ../bin/tool/go_bootstrap +if [ "$GOHOSTARCH" != "$GOARCH" -o "$GOHOSTOS" != "$GOOS" ]; then + echo "# Building packages and commands for host, $GOHOSTOS/$GOHOSTARCH." + GOOS=$GOHOSTOS GOARCH=$GOHOSTARCH \ + $GOTOOLDIR/go_bootstrap install -v std + echo +fi + +echo "# Building packages and commands for $GOOS/$GOARCH." +$GOTOOLDIR/go_bootstrap install -v std echo +rm -f $GOTOOLDIR/go_bootstrap + if [ "$1" != "--no-banner" ]; then - ../bin/tool/dist banner + $GOTOOLDIR/dist banner fi diff --git a/src/make.bat b/src/make.bat index 010e418c015..6618bc4ed98 100644 --- a/src/make.bat +++ b/src/make.bat @@ -25,26 +25,48 @@ echo # Building C bootstrap tool. echo cmd/dist if not exist ..\bin\tool mkdir ..\bin\tool :: Windows has no glob expansion, so spell out cmd/dist/*.c. -gcc -O2 -Wall -Werror -o ../bin/tool/dist.exe -Icmd/dist %DEFGOROOT% cmd/dist/buf.c cmd/dist/build.c cmd/dist/buildgc.c cmd/dist/buildruntime.c cmd/dist/goc2c.c cmd/dist/main.c cmd/dist/windows.c +gcc -O2 -Wall -Werror -o cmd/dist/dist.exe -Icmd/dist %DEFGOROOT% cmd/dist/buf.c cmd/dist/build.c cmd/dist/buildgc.c cmd/dist/buildruntime.c cmd/dist/goc2c.c cmd/dist/main.c cmd/dist/windows.c if errorlevel 1 goto fail +.\cmd\dist\dist env -wp >env.bat +if errorlevel 1 goto fail +call env.bat +del env.bat :: Echo with no arguments prints whether echo is turned on, so echo dot. echo . echo # Building compilers and Go bootstrap tool. -..\bin\tool\dist bootstrap -v +.\cmd\dist\dist bootstrap -a -v +if errorlevel 1 goto fail +:: Delay move of dist tool to now, because bootstrap cleared tool directory. +move .\cmd\dist\dist.exe %GOTOOLDIR%\dist.exe +%GOTOOLDIR%\go_bootstrap clean -i std +echo . + +if not %GOHOSTARCH% == %GOARCH% goto localbuild +if not %GOHOSTOS% == %GOOS% goto localbuild +goto mainbuild + +:localbuild +echo # Building tools for local system. %GOHOSTOS%/%GOHOSTARCH% +set oldGOOS=%GOOS% +set oldGOARCH=%GOARCH% +set GOOS=%GOHOSTOS% +set GOARCH=%GOHOSTARCH% +%GOTOOLDIR%\go_bootstrap install -v std +set GOOS=%oldGOOS% +set GOARCH=%oldGOARCH% if errorlevel 1 goto fail echo . +:mainbuild echo # Building packages and commands. -..\bin\tool\go_bootstrap clean std +%GOTOOLDIR%\go_bootstrap install -a -v std if errorlevel 1 goto fail -..\bin\tool\go_bootstrap install -a -v std -if errorlevel 1 goto fail -del ..\bin\tool\go_bootstrap.exe +del %GOTOOLDIR%\go_bootstrap.exe echo . if "x%1"=="x--no-banner" goto nobanner -..\bin\tool\dist banner +%GOTOOLDIR%\dist banner :nobanner goto end diff --git a/src/pkg/exp/types/gcimporter_test.go b/src/pkg/exp/types/gcimporter_test.go index 5411f3bccef..c229b50113d 100644 --- a/src/pkg/exp/types/gcimporter_test.go +++ b/src/pkg/exp/types/gcimporter_test.go @@ -6,7 +6,9 @@ package types import ( "go/ast" + "go/build" "io/ioutil" + "os" "os/exec" "path/filepath" "runtime" @@ -31,7 +33,7 @@ func init() { gcPath = gcName return } - gcPath = filepath.Join(runtime.GOROOT(), "/bin/tool/", gcName) + gcPath = filepath.Join(build.ToolDir, gcName) } func compile(t *testing.T, dirname, filename string) { @@ -90,6 +92,13 @@ func testDir(t *testing.T, dir string, endTime time.Time) (nimports int) { } func TestGcImport(t *testing.T) { + // On cross-compile builds, the path will not exist. + // Need to use GOHOSTOS, which is not available. + if _, err := os.Stat(gcPath); err != nil { + t.Logf("skipping test: %v", err) + return + } + compile(t, "testdata", "exports.go") nimports := 0 diff --git a/src/pkg/go/build/path.go b/src/pkg/go/build/path.go index 7e931faff19..e160ac3b280 100644 --- a/src/pkg/go/build/path.go +++ b/src/pkg/go/build/path.go @@ -12,6 +12,9 @@ import ( "runtime" ) +// ToolDir is the directory containing build tools. +var ToolDir = filepath.Join(runtime.GOROOT(), "pkg/tool/"+runtime.GOOS+"_"+runtime.GOARCH) + // Path is a validated list of Trees derived from $GOROOT and $GOPATH at init. var Path []*Tree diff --git a/src/run.bash b/src/run.bash index 8f282249fb8..d818751b2c6 100755 --- a/src/run.bash +++ b/src/run.bash @@ -5,7 +5,7 @@ set -e -eval $(../bin/tool/dist env -p) +eval $(go tool dist env) unset CDPATH # in case user has it set diff --git a/test/fixedbugs/bug302.go b/test/fixedbugs/bug302.go index b42db7f72be..1088b2f3c29 100644 --- a/test/fixedbugs/bug302.go +++ b/test/fixedbugs/bug302.go @@ -1,4 +1,4 @@ -// $G $D/bug302.dir/p.go && "$GOROOT"/bin/tool/pack grc pp.a p.$A && $G $D/bug302.dir/main.go +// $G $D/bug302.dir/p.go && pack grc pp.a p.$A && $G $D/bug302.dir/main.go // Copyright 2010 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style diff --git a/test/run b/test/run index 4b0481caa8b..7f4c350fa32 100755 --- a/test/run +++ b/test/run @@ -3,7 +3,7 @@ # Use of this source code is governed by a BSD-style # license that can be found in the LICENSE file. -eval $(../bin/tool/dist env) +eval $(go tool dist env) export GOARCH GOOS GOROOT export E= @@ -34,7 +34,7 @@ failed=0 PATH=${GOBIN:-$GOROOT/bin}:`pwd`:/bin:/usr/bin:/usr/local/bin # TODO: We add the tool directory to the PATH to avoid thinking about a better way. -PATH="$GOROOT/bin/tool":$PATH +PATH="$GOTOOLDIR:$PATH" RUNFILE="/tmp/gorun-$$-$USER" TMP1FILE="/tmp/gotest1-$$-$USER"