go: implement build, install, run

clean is gone; all the intermediate files are created
in a temporary tree that is wiped when the command ends.

Not using go/build's Script because it is not well aligned
with this API.  The various builder methods are copied from
go/build and adapted.  Probably once we delete goinstall
we can delete the Script API too.

R=rogpeppe, adg, adg
CC=golang-dev
https://golang.org/cl/5483069

src/cmd/cgo/gcc.go

@@ -729,7 +729,9 @@ func (p *Package) gccMachine() []string {
 	return nil
 }
 
-var gccTmp = objDir + "_cgo_.o"
+func gccTmp() string {
+	return *objDir + "_cgo_.o"
+}
 
 // gccCmd returns the gcc command line to use for compiling
 // the input.
@@ -738,7 +740,7 @@ func (p *Package) gccCmd() []string {
 		p.gccName(),
 		"-Wall",                             // many warnings
 		"-Werror",                           // warnings are errors
-		"-o" + gccTmp,                       // write object to tmp
+		"-o" + gccTmp(),                     // write object to tmp
 		"-gdwarf-2",                         // generate DWARF v2 debugging symbols
 		"-fno-eliminate-unused-debug-types", // gets rid of e.g. untyped enum otherwise
 		"-c",                                // do not link
@@ -755,10 +757,10 @@ func (p *Package) gccCmd() []string {
 func (p *Package) gccDebug(stdin []byte) (*dwarf.Data, binary.ByteOrder, []byte) {
 	runGcc(stdin, p.gccCmd())
 
-	if f, err := macho.Open(gccTmp); err == nil {
+	if f, err := macho.Open(gccTmp()); err == nil {
 		d, err := f.DWARF()
 		if err != nil {
-			fatalf("cannot load DWARF output from %s: %v", gccTmp, err)
+			fatalf("cannot load DWARF output from %s: %v", gccTmp(), err)
 		}
 		var data []byte
 		if f.Symtab != nil {
@@ -784,23 +786,23 @@ func (p *Package) gccDebug(stdin []byte) (*dwarf.Data, binary.ByteOrder, []byte)
 	// Can skip debug data block in ELF and PE for now.
 	// The DWARF information is complete.
 
-	if f, err := elf.Open(gccTmp); err == nil {
+	if f, err := elf.Open(gccTmp()); err == nil {
 		d, err := f.DWARF()
 		if err != nil {
-			fatalf("cannot load DWARF output from %s: %v", gccTmp, err)
+			fatalf("cannot load DWARF output from %s: %v", gccTmp(), err)
 		}
 		return d, f.ByteOrder, nil
 	}
 
-	if f, err := pe.Open(gccTmp); err == nil {
+	if f, err := pe.Open(gccTmp()); err == nil {
 		d, err := f.DWARF()
 		if err != nil {
-			fatalf("cannot load DWARF output from %s: %v", gccTmp, err)
+			fatalf("cannot load DWARF output from %s: %v", gccTmp(), err)
 		}
 		return d, binary.LittleEndian, nil
 	}
 
-	fatalf("cannot parse gcc output %s as ELF, Mach-O, PE object", gccTmp)
+	fatalf("cannot parse gcc output %s as ELF, Mach-O, PE object", gccTmp())
 	panic("not reached")
 }
 

src/cmd/cgo/main.go

@@ -123,12 +123,14 @@ var cPrefix string
 var fset = token.NewFileSet()
 
 var dynobj = flag.String("dynimport", "", "if non-empty, print dynamic import data for that file")
+var dynout = flag.String("dynout", "", "write -dynobj output to this file")
 
 // These flags are for bootstrapping a new Go implementation,
 // to generate Go and C headers that match the data layout and
 // constant values used in the host's C libraries and system calls.
 var godefs = flag.Bool("godefs", false, "for bootstrap: write Go definitions for C file to standard output")
 var cdefs = flag.Bool("cdefs", false, "for bootstrap: write C definitions for C file to standard output")
+var objDir = flag.String("objdir", "", "object directory")
 
 var gccgo = flag.Bool("gccgo", false, "generate files for use with gccgo")
 
@@ -202,9 +204,13 @@ func main() {
 		fs[i] = f
 	}
 
+	if *objDir == "" {
 		// make sure that _obj directory exists, so that we can write
 		// all the output files there.
 		os.Mkdir("_obj", 0777)
+		*objDir = "_obj"
+	}
+	*objDir += string(filepath.Separator)
 
 	for i, input := range goFiles {
 		f := fs[i]

src/cmd/cgo/out.go

@@ -14,20 +14,17 @@ import (
 	"go/printer"
 	"go/token"
 	"os"
-	"path/filepath"
 	"strings"
 )
 
-var objDir = "_obj" + string(filepath.Separator)
-
 // writeDefs creates output files to be compiled by 6g, 6c, and gcc.
 // (The comments here say 6g and 6c but the code applies to the 8 and 5 tools too.)
 func (p *Package) writeDefs() {
-	fgo2 := creat(objDir + "_cgo_gotypes.go")
+	fgo2 := creat(*objDir + "_cgo_gotypes.go")
-	fc := creat(objDir + "_cgo_defun.c")
+	fc := creat(*objDir + "_cgo_defun.c")
-	fm := creat(objDir + "_cgo_main.c")
+	fm := creat(*objDir + "_cgo_main.c")
 
-	fflg := creat(objDir + "_cgo_flags")
+	fflg := creat(*objDir + "_cgo_flags")
 	for k, v := range p.CgoFlags {
 		fmt.Fprintf(fflg, "_CGO_%s=%s\n", k, v)
 	}
@@ -109,6 +106,15 @@ func (p *Package) writeDefs() {
 }
 
 func dynimport(obj string) {
+	stdout := os.Stdout
+	if *dynout != "" {
+		f, err := os.Create(*dynout)
+		if err != nil {
+			fatalf("%s", err)
+		}
+		stdout = f
+	}
+
 	if f, err := elf.Open(obj); err == nil {
 		sym, err := f.ImportedSymbols()
 		if err != nil {
@@ -119,14 +125,14 @@ func dynimport(obj string) {
 			if s.Version != "" {
 				targ += "@" + s.Version
 			}
-			fmt.Printf("#pragma dynimport %s %s %q\n", s.Name, targ, s.Library)
+			fmt.Fprintf(stdout, "#pragma dynimport %s %s %q\n", s.Name, targ, s.Library)
 		}
 		lib, err := f.ImportedLibraries()
 		if err != nil {
 			fatalf("cannot load imported libraries from ELF file %s: %v", obj, err)
 		}
 		for _, l := range lib {
-			fmt.Printf("#pragma dynimport _ _ %q\n", l)
+			fmt.Fprintf(stdout, "#pragma dynimport _ _ %q\n", l)
 		}
 		return
 	}
@@ -140,14 +146,14 @@ func dynimport(obj string) {
 			if len(s) > 0 && s[0] == '_' {
 				s = s[1:]
 			}
-			fmt.Printf("#pragma dynimport %s %s %q\n", s, s, "")
+			fmt.Fprintf(stdout, "#pragma dynimport %s %s %q\n", s, s, "")
 		}
 		lib, err := f.ImportedLibraries()
 		if err != nil {
 			fatalf("cannot load imported libraries from Mach-O file %s: %v", obj, err)
 		}
 		for _, l := range lib {
-			fmt.Printf("#pragma dynimport _ _ %q\n", l)
+			fmt.Fprintf(stdout, "#pragma dynimport _ _ %q\n", l)
 		}
 		return
 	}
@@ -159,7 +165,7 @@ func dynimport(obj string) {
 		}
 		for _, s := range sym {
 			ss := strings.Split(s, ":")
-			fmt.Printf("#pragma dynimport %s %s %q\n", ss[0], ss[0], strings.ToLower(ss[1]))
+			fmt.Fprintf(stdout, "#pragma dynimport %s %s %q\n", ss[0], ss[0], strings.ToLower(ss[1]))
 		}
 		return
 	}
@@ -307,8 +313,8 @@ func (p *Package) writeOutput(f *File, srcfile string) {
 		base = base[0 : len(base)-3]
 	}
 	base = strings.Map(slashToUnderscore, base)
-	fgo1 := creat(objDir + base + ".cgo1.go")
+	fgo1 := creat(*objDir + base + ".cgo1.go")
-	fgcc := creat(objDir + base + ".cgo2.c")
+	fgcc := creat(*objDir + base + ".cgo2.c")
 
 	p.GoFiles = append(p.GoFiles, base+".cgo1.go")
 	p.GccFiles = append(p.GccFiles, base+".cgo2.c")
@@ -383,7 +389,7 @@ func (p *Package) writeOutputFunc(fgcc *os.File, n *Name) {
 // Write out the various stubs we need to support functions exported
 // from Go so that they are callable from C.
 func (p *Package) writeExports(fgo2, fc, fm *os.File) {
-	fgcc := creat(objDir + "_cgo_export.c")
+	fgcc := creat(*objDir + "_cgo_export.c")
 	fgcch := creat("_cgo_export.h")
 
 	fmt.Fprintf(fgcch, "/* Created by cgo - DO NOT EDIT. */\n")

src/cmd/go/Makefile

@@ -7,7 +7,6 @@ include ../../Make.inc
 TARG=go
 GOFILES=\
 	build.go\
-	clean.go\
 	fix.go\
 	get.go\
 	fmt.go\
@@ -15,6 +14,7 @@ GOFILES=\
 	list.go\
 	main.go\
 	pkg.go\
+	run.go\
 	test.go\
 	version.go\
 	vet.go\

src/cmd/go/build.go

@@ -4,14 +4,38 @@
 
 package main
 
+import (
+	"bytes"
+	"fmt"
+	"go/build"
+	"io"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"regexp"
+	"runtime"
+	"strings"
+)
+
+// Break init cycles
+func init() {
+	cmdBuild.Run = runBuild
+	cmdInstall.Run = runInstall
+}
+
 var cmdBuild = &Command{
-	Run:       runBuild,
+	UsageLine: "build [-a] [-n] [-v] [importpath... | gofiles...]",
-	UsageLine: "build [-n] [-v] [importpath...]",
+	Short:     "compile packages and dependencies",
-	Short:     "compile and install packages and dependencies",
 	Long: `
 Build compiles the packages named by the import paths,
 along with their dependencies, but it does not install the results.
 
+If the arguments are a list of .go files, build compiles them into
+a package object or command executable named for the first
+source file.
+
+The -a flag forces rebuilding of packages that are already up-to-date.
 The -n flag prints the commands but does not run them.
 The -v flag prints the commands.
 
@@ -21,23 +45,34 @@ See also: go install, go get, go clean.
 	`,
 }
 
+var buildA = cmdBuild.Flag.Bool("a", false, "")
 var buildN = cmdBuild.Flag.Bool("n", false, "")
 var buildV = cmdBuild.Flag.Bool("v", false, "")
 
 func runBuild(cmd *Command, args []string) {
-	args = importPaths(args)
+	var b builder
-	_ = args
+	b.init(*buildA, *buildN, *buildV)
-	panic("build not implemented")
+
+	if len(args) > 0 && strings.HasSuffix(args[0], ".go") {
+		b.do(b.action(modeInstall, modeBuild, goFilesPackage(args, "")))
+		return
+	}
+
+	a := &action{f: (*builder).nop}
+	for _, p := range packages(args) {
+		a.deps = append(a.deps, b.action(modeBuild, modeBuild, p))
+	}
+	b.do(a)
 }
 
 var cmdInstall = &Command{
-	Run:       runInstall,
+	UsageLine: "install [-a] [-n] [-v] [importpath...]",
-	UsageLine: "install [-n] [-v] [importpath...]",
+	Short:     "compile and install packages and dependencies",
-	Short:     "install packages and dependencies",
 	Long: `
 Install compiles and installs the packages named by the import paths,
 along with their dependencies.
 
+The -a flag forces reinstallation of packages that are already up-to-date.
 The -n flag prints the commands but does not run them.
 The -v flag prints the commands.
 
@@ -47,11 +82,541 @@ See also: go build, go get, go clean.
 	`,
 }
 
+var installA = cmdInstall.Flag.Bool("a", false, "")
 var installN = cmdInstall.Flag.Bool("n", false, "")
 var installV = cmdInstall.Flag.Bool("v", false, "")
 
 func runInstall(cmd *Command, args []string) {
-	args = importPaths(args)
+	var b builder
-	_ = args
+	b.init(*installA, *installN, *installV)
-	panic("install not implemented")
+	a := &action{f: (*builder).nop}
+	for _, p := range packages(args) {
+		a.deps = append(a.deps, b.action(modeInstall, modeInstall, p))
+	}
+	b.do(a)
+}
+
+// A builder holds global state about a build.
+// It does not hold per-package state, because eventually we will
+// build packages in parallel, and the builder will be shared.
+type builder struct {
+	work        string               // the temporary work directory (ends in filepath.Separator)
+	aflag       bool                 // the -a flag
+	nflag       bool                 // the -n flag
+	vflag       bool                 // the -v flag
+	arch        string               // e.g., "6"
+	actionCache map[cacheKey]*action // a cache of already-constructed actions
+}
+
+// An action represents a single action in the action graph.
+type action struct {
+	f func(*builder, *action) error // the action itself
+
+	p      *Package  // the package this action works on
+	deps   []*action // actions that must happen before this one
+	done   bool      // whether the action is done (might have failed)
+	failed bool      // whether the action failed
+
+	// Results left for communication with other code.
+	pkgobj string // the built .a file
+	pkgbin string // the built a.out file, if one exists
+}
+
+// cacheKey is the key for the action cache.
+type cacheKey struct {
+	mode buildMode
+	p    *Package
+}
+
+// buildMode specifies the build mode:
+// are we just building things or also installing the results?
+type buildMode int
+
+const (
+	modeBuild buildMode = iota
+	modeInstall
+)
+
+func (b *builder) init(aflag, nflag, vflag bool) {
+	var err error
+	b.aflag = aflag
+	b.nflag = nflag
+	b.vflag = vflag
+	b.actionCache = make(map[cacheKey]*action)
+
+	b.arch, err = build.ArchChar(build.DefaultContext.GOARCH)
+	if err != nil {
+		fatalf("%s", err)
+	}
+
+	if nflag {
+		b.work = "$WORK"
+	} else {
+		b.work, err = ioutil.TempDir("", "go-build")
+		if err != nil {
+			fatalf("%s", err)
+		}
+		if vflag {
+			fmt.Printf("WORK=%s\n", b.work)
+		}
+		atexit(func() { os.RemoveAll(b.work) })
+	}
+}
+
+// goFilesPackage creates a package for building a collection of Go files
+// (typically named on the command line).  If target is given, the package
+// target is target.  Otherwise, the target is named p.a for
+// package p or named after the first Go file for package main.
+func goFilesPackage(gofiles []string, target string) *Package {
+	// Synthesize fake "directory" that only shows those two files,
+	// to make it look like this is a standard package or
+	// command directory.
+	var dir []os.FileInfo
+	for _, file := range gofiles {
+		fi, err := os.Stat(file)
+		if err != nil {
+			fatalf("%s", err)
+		}
+		if fi.IsDir() {
+			fatalf("%s is a directory, should be a Go file", file)
+		}
+		dir = append(dir, fi)
+	}
+	ctxt := build.DefaultContext
+	ctxt.ReadDir = func(string) ([]os.FileInfo, error) { return dir, nil }
+	pwd, _ := os.Getwd()
+	pkg, err := scanPackage(&ctxt, &build.Tree{Path: "."}, "<command line>", "<command line>", pwd)
+	if err != nil {
+		fatalf("%s", err)
+	}
+	if target != "" {
+		pkg.targ = target
+	} else if pkg.Name == "main" {
+		pkg.targ = gofiles[0][:len(gofiles[0])-len(".go")]
+	} else {
+		pkg.targ = pkg.Name + ".a"
+	}
+	pkg.ImportPath = "_/" + pkg.targ
+	return pkg
+}
+
+// action returns the action for applying the given operation (mode) to the package.
+// depMode is the action to use when building dependencies.
+func (b *builder) action(mode buildMode, depMode buildMode, p *Package) *action {
+	key := cacheKey{mode, p}
+	a := b.actionCache[key]
+	if a != nil {
+		return a
+	}
+
+	a = &action{p: p}
+	b.actionCache[key] = a
+
+	switch mode {
+	case modeBuild, modeInstall:
+		if !needInstall(p) && !b.aflag {
+			a.f = (*builder).nop
+			return a
+		}
+		if p.Standard {
+			switch p.ImportPath {
+			case "runtime", "runtime/cgo":
+				// Too complex - can't build.
+				a.f = (*builder).nop
+				return a
+			case "builtin", "unsafe":
+				// Fake packages - nothing to build.
+				a.f = (*builder).nop
+				return a
+			}
+		}
+
+		if mode == modeInstall {
+			a.f = (*builder).install
+			a.deps = []*action{b.action(modeBuild, depMode, p)}
+			return a
+		}
+
+		a.f = (*builder).build
+		for _, p1 := range p.imports {
+			a.deps = append(a.deps, b.action(depMode, depMode, p1))
+		}
+	}
+
+	return a
+}
+
+// needInstall reports whether p needs to be built and installed.
+// That is only true if some source file is newer than the installed package binary.
+func needInstall(p *Package) bool {
+	if p.targ == "" {
+		return true
+	}
+	fi, err := os.Stat(p.targ)
+	if err != nil {
+		return true
+	}
+	t := fi.ModTime()
+
+	srcss := [][]string{
+		p.GoFiles,
+		p.CFiles,
+		p.SFiles,
+		p.CgoFiles,
+	}
+	for _, srcs := range srcss {
+		for _, src := range srcs {
+			fi, err := os.Stat(filepath.Join(p.Dir, src))
+			if err != nil {
+				return true
+			}
+			if fi.ModTime().After(t) {
+				return true
+			}
+		}
+	}
+
+	return false
+}
+
+// do runs the action graph rooted at a.
+func (b *builder) do(a *action) {
+	if a.done {
+		return
+	}
+	for _, a1 := range a.deps {
+		b.do(a1)
+		if a1.failed {
+			a.failed = true
+			a.done = true
+			return
+		}
+	}
+	if err := a.f(b, a); err != nil {
+		errorf("%s", err)
+		a.failed = true
+	}
+	a.done = true
+}
+
+func (b *builder) nop(a *action) error {
+	return nil
+}
+
+// build is the action for building a single package.
+func (b *builder) build(a *action) error {
+	obj := filepath.Join(b.work, filepath.FromSlash(a.p.ImportPath+"/_obj")) + string(filepath.Separator)
+	a.pkgobj = filepath.Join(b.work, filepath.FromSlash(a.p.ImportPath+".a"))
+
+	// make build directory
+	if err := b.mkdir(obj); err != nil {
+		return err
+	}
+
+	var objects []string
+	var gofiles []string
+	gofiles = append(gofiles, a.p.GoFiles...)
+
+	// run cgo
+	if len(a.p.CgoFiles) > 0 {
+		outGo, outObj, err := b.cgo(a.p.Dir, obj, a.p.info)
+		if err != nil {
+			return err
+		}
+		objects = append(objects, outObj...)
+		gofiles = append(gofiles, outGo...)
+	}
+
+	// prepare Go import path list
+	var inc []string
+	inc = append(inc, "-I", b.work)
+	incMap := map[string]bool{}
+	for _, a1 := range a.deps {
+		p1 := a1.p
+		if p1.t.Goroot {
+			continue
+		}
+		pkgdir := p1.t.PkgDir()
+		if !incMap[pkgdir] {
+			incMap[pkgdir] = true
+			inc = append(inc, "-I", pkgdir)
+		}
+	}
+
+	// compile Go
+	if len(gofiles) > 0 {
+		out := "_go_.6"
+		if err := b.gc(a.p.Dir, obj+out, a.p.ImportPath, inc, gofiles); err != nil {
+			return err
+		}
+		objects = append(objects, out)
+	}
+
+	// assemble .s files
+	if len(a.p.SFiles) > 0 {
+		for _, sfile := range a.p.SFiles {
+			out := sfile[:len(sfile)-len(".s")] + "." + b.arch
+			if err := b.asm(a.p.Dir, obj+out, sfile); err != nil {
+				return err
+			}
+			objects = append(objects, out)
+		}
+	}
+
+	// pack into archive
+	if err := b.gopack(obj, a.pkgobj, objects); err != nil {
+		return err
+	}
+
+	if a.p.Name == "main" {
+		// command.
+		// import paths for compiler are introduced by -I.
+		// for linker, they are introduced by -L.
+		for i := 0; i < len(inc); i += 2 {
+			inc[i] = "-L"
+		}
+		a.pkgbin = obj + "a.out"
+		if err := b.ld(a.p.Dir, a.pkgbin, inc, a.pkgobj); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// install is the action for installing a single package.
+func (b *builder) install(a *action) error {
+	if err := b.build(a); err != nil {
+		return err
+	}
+
+	var src string
+	var perm uint32
+	if a.pkgbin != "" {
+		src = a.pkgbin
+		perm = 0777
+	} else {
+		src = a.pkgobj
+		perm = 0666
+	}
+
+	// make target directory
+	dst := a.p.targ
+	dir, _ := filepath.Split(dst)
+	if dir != "" {
+		if err := b.mkdir(dir); err != nil {
+			return err
+		}
+	}
+
+	return b.copyFile(dst, src, perm)
+}
+
+// copyFile is like 'cp src dst'.
+func (b *builder) copyFile(dst, src string, perm uint32) error {
+	if b.nflag || b.vflag {
+		b.showcmd("cp %s %s", src, dst)
+		if b.nflag {
+			return nil
+		}
+	}
+
+	sf, err := os.Open(src)
+	if err != nil {
+		return err
+	}
+	defer sf.Close()
+	os.Remove(dst)
+	df, err := os.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, perm)
+	if err != nil {
+		return err
+	}
+	_, err = io.Copy(df, sf)
+	df.Close()
+	if err != nil {
+		os.Remove(dst)
+		return err
+	}
+	return nil
+}
+
+// fmtcmd is like fmt.Sprintf but replaces references to the
+// work directory (a temporary directory with a clumsy name)
+// with $WORK.
+func (b *builder) fmtcmd(format string, args ...interface{}) string {
+	s := fmt.Sprintf(format, args...)
+	s = strings.Replace(s, b.work, "$WORK", -1)
+	return s
+}
+
+// showcmd prints the given command to standard output
+// for the implementation of -n or -v.
+func (b *builder) showcmd(format string, args ...interface{}) {
+	fmt.Println(b.fmtcmd(format, args...))
+}
+
+// run runs the command given by cmdline in the directory dir.
+// If the commnd fails, run prints information about the failure
+// and returns a non-nil error.
+func (b *builder) run(dir string, cmdline ...string) error {
+	if b.nflag || b.vflag {
+		b.showcmd("cd %s; %s", dir, strings.Join(cmdline, " "))
+		if b.nflag {
+			return nil
+		}
+	}
+
+	var buf bytes.Buffer
+	cmd := exec.Command(cmdline[0], cmdline[1:]...)
+	cmd.Stdout = &buf
+	cmd.Stderr = &buf
+	cmd.Dir = dir
+	// TODO: cmd.Env
+	err := cmd.Run()
+	if buf.Len() > 0 {
+		fmt.Fprintf(os.Stderr, "# cd %s; %s\n", dir, strings.Join(cmdline, " "))
+		fmt.Fprintf(os.Stderr, "%s\n", buf.Bytes())
+	}
+	return err
+}
+
+// mkdir makes the named directory.
+func (b *builder) mkdir(dir string) error {
+	if b.nflag || b.vflag {
+		b.showcmd("mkdir -p %s", dir)
+		if b.nflag {
+			return nil
+		}
+	}
+
+	if err := os.MkdirAll(dir, 0777); err != nil {
+		return err
+	}
+	return nil
+}
+
+// gc runs the Go compiler in a specific directory on a set of files
+// to generate the named output file. 
+func (b *builder) gc(dir, ofile, importPath string, importArgs []string, gofiles []string) error {
+	args := append([]string{b.arch + "g", "-o", ofile, "-p", importPath}, importArgs...)
+	args = append(args, gofiles...)
+	return b.run(dir, args...)
+}
+
+// asm runs the assembler in a specific directory on a specific file
+// to generate the named output file. 
+func (b *builder) asm(dir, ofile, sfile string) error {
+	return b.run(dir, b.arch+"a", "-o", ofile, sfile)
+}
+
+// gopack runs the assembler in a specific directory to create
+// an archive from a set of object files.
+// typically it is run in the object directory.
+func (b *builder) gopack(objDir, afile string, ofiles []string) error {
+	return b.run(objDir, append([]string{"gopack", "grc", afile}, ofiles...)...)
+}
+
+// ld runs the linker to create a package starting at mainpkg.
+func (b *builder) ld(dir, out string, importArgs []string, mainpkg string) error {
+	return b.run(dir, append(append([]string{b.arch + "l", "-o", out}, importArgs...), mainpkg)...)
+}
+
+// cc runs the gc-toolchain C compiler in a directory on a C file
+// to produce an output file.
+func (b *builder) cc(dir, ofile, cfile string) error {
+	inc := filepath.Join(runtime.GOROOT(), "pkg",
+		fmt.Sprintf("%s_%s", build.DefaultContext.GOOS, build.DefaultContext.GOARCH))
+	return b.run(dir, b.arch+"c", "-FVW", "-I", inc, "-o", ofile, cfile)
+}
+
+// gcc runs the gcc C compiler to create an object from a single C file.
+func (b *builder) gcc(dir, out string, flags []string, cfile string) error {
+	return b.run(dir, b.gccCmd(dir, flags, "-o", out, "-c", cfile)...)
+}
+
+// gccld runs the gcc linker to create an executable from a set of object files
+func (b *builder) gccld(dir, out string, flags []string, obj []string) error {
+	return b.run(dir, append(b.gccCmd(dir, flags, "-o", out), obj...)...)
+}
+
+// gccCmd returns a gcc command line ending with args
+func (b *builder) gccCmd(objdir string, flags []string, args ...string) []string {
+	// TODO: HOST_CC?
+	a := []string{"gcc", "-I", objdir, "-g", "-fPIC", "-O2"}
+	switch b.arch {
+	case "8":
+		a = append(a, "-m32")
+	case "6":
+		a = append(a, "-m64")
+	}
+	a = append(a, flags...)
+	return append(a, args...)
+}
+
+var cgoRe = regexp.MustCompile(`[/\\:]`)
+
+func (b *builder) cgo(dir, obj string, info *build.DirInfo) (outGo, outObj []string, err error) {
+	// cgo
+	// TODO: CGOPKGPATH, CGO_FLAGS?
+	gofiles := []string{obj + "_cgo_gotypes.go"}
+	cfiles := []string{"_cgo_main.c", "_cgo_export.c"}
+	for _, fn := range info.CgoFiles {
+		f := cgoRe.ReplaceAllString(fn[:len(fn)-2], "_")
+		gofiles = append(gofiles, obj+f+"cgo1.go")
+		cfiles = append(cfiles, f+"cgo2.c")
+	}
+	defunC := obj + "_cgo_defun.c"
+	// TODO: make cgo not depend on $GOARCH?
+	// TODO: make cgo write to obj
+	if err := b.run(dir, append([]string{"cgo", "-objdir", obj, "--"}, info.CgoFiles...)...); err != nil {
+		return nil, nil, err
+	}
+	outGo = append(outGo, gofiles...)
+
+	// cc _cgo_defun.c
+	defunObj := obj + "_cgo_defun." + b.arch
+	if err := b.cc(dir, defunObj, defunC); err != nil {
+		return nil, nil, err
+	}
+	outObj = append(outObj, defunObj)
+
+	// gcc
+	var linkobj []string
+	for _, cfile := range cfiles {
+		ofile := obj + cfile[:len(cfile)-1] + "o"
+		if err := b.gcc(dir, ofile, info.CgoCFLAGS, obj+cfile); err != nil {
+			return nil, nil, err
+		}
+		linkobj = append(linkobj, ofile)
+		if !strings.HasSuffix(ofile, "_cgo_main.o") {
+			outObj = append(outObj, ofile)
+		}
+	}
+	for _, cfile := range info.CFiles {
+		ofile := obj + cgoRe.ReplaceAllString(cfile[:len(cfile)-1], "_") + "o"
+		if err := b.gcc(dir, ofile, info.CgoCFLAGS, cfile); err != nil {
+			return nil, nil, err
+		}
+		linkobj = append(linkobj, ofile)
+		outObj = append(outObj, ofile)
+	}
+	dynobj := obj + "_cgo_.o"
+	if err := b.gccld(dir, dynobj, info.CgoLDFLAGS, linkobj); err != nil {
+		return nil, nil, err
+	}
+
+	// cgo -dynimport
+	importC := obj + "_cgo_import.c"
+	if err := b.run(dir, "cgo", "-objdir", obj, "-dynimport", dynobj, "-dynout", importC); err != nil {
+		return nil, nil, err
+	}
+
+	// cc _cgo_import.ARCH
+	importObj := obj + "_cgo_import." + b.arch
+	if err := b.cc(dir, importObj, importC); err != nil {
+		return nil, nil, err
+	}
+	outObj = append(outObj, importObj)
+
+	return outGo, outObj, nil
 }

src/cmd/go/clean.go

@@ -1,31 +0,0 @@
-// Copyright 2011 The Go Authors.  All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package main
-
-var cmdClean = &Command{
-	Run:       runClean,
-	UsageLine: "clean [-nuke] [importpath...]",
-	Short:     "remove intermediate objects",
-	Long: `
-Clean removes intermediate object files generated during
-the compilation of the packages named by the import paths,
-but by default it does not remove the installed package binaries.
-
-The -nuke flag causes clean to remove the installed package binaries too.
-
-TODO: Clean does not clean dependencies of the packages.
-TODO: Rename -nuke.
-
-For more about import paths, see 'go help importpath'.
-	`,
-}
-
-var cleanNuke = cmdClean.Flag.Bool("nuke", false, "")
-
-func runClean(cmd *Command, args []string) {
-	args = importPaths(args)
-	_ = args
-	panic("nuke not implemented")
-}

src/cmd/go/list.go

@@ -58,12 +58,19 @@ func init() {
 
 var listFmt = cmdList.Flag.String("f", "{{.Name}} {{.Dir}}", "")
 var listJson = cmdList.Flag.Bool("json", false, "")
+var nl = []byte{'\n'}
 
 func runList(cmd *Command, args []string) {
 	var do func(*Package)
 	if *listJson {
-		enc := json.NewEncoder(os.Stdout)
+		do = func(p *Package) {
-		do = func(p *Package) { enc.Encode(p) }
+			b, err := json.MarshalIndent(p, "", "\t")
+			if err != nil {
+				fatalf("%s", err)
+			}
+			os.Stdout.Write(b)
+			os.Stdout.Write(nl)
+		}
 	} else {
 		tmpl, err := template.New("main").Parse(*listFmt + "\n")
 		if err != nil {

src/cmd/go/main.go

@@ -7,10 +7,12 @@ package main
 import (
 	"flag"
 	"fmt"
+	"go/build"
 	"io"
 	"log"
 	"os"
 	"os/exec"
+	"path/filepath"
 	"strings"
 	"text/template"
 )
@@ -56,13 +58,13 @@ func (c *Command) Usage() {
 // The order here is the order in which they are printed by 'go help'.
 var commands = []*Command{
 	cmdBuild,
-	cmdClean,
 	cmdDoc,
 	cmdFix,
 	cmdFmt,
 	cmdGet,
 	cmdInstall,
 	cmdList,
+	cmdRun,
 	cmdTest,
 	cmdVersion,
 	cmdVet,
@@ -95,7 +97,7 @@ func main() {
 			cmd.Flag.Parse(args[1:])
 			args = cmd.Flag.Args()
 			cmd.Run(cmd, args)
-			os.Exit(exitStatus)
+			exit()
 			return
 		}
 	}
@@ -173,16 +175,31 @@ func help(args []string) {
 
 // importPaths returns the import paths to use for the given command line.
 func importPaths(args []string) []string {
-	// TODO: "all"
+	if len(args) == 1 && args[0] == "all" {
+		return allPackages()
+	}
 	if len(args) == 0 {
 		return []string{"."}
 	}
 	return args
 }
 
+var atexitFuncs []func()
+
+func atexit(f func()) {
+	atexitFuncs = append(atexitFuncs, f)
+}
+
+func exit() {
+	for _, f := range atexitFuncs {
+		f()
+	}
+	os.Exit(exitStatus)
+}
+
 func fatalf(format string, args ...interface{}) {
-	log.Printf(format, args...)
+	errorf(format, args...)
-	os.Exit(1)
+	exit()
 }
 
 func errorf(format string, args ...interface{}) {
@@ -192,7 +209,7 @@ func errorf(format string, args ...interface{}) {
 
 func exitIfErrors() {
 	if exitStatus != 0 {
-		os.Exit(exitStatus)
+		exit()
 	}
 }
 
@@ -204,3 +221,52 @@ func run(cmdline ...string) {
 		errorf("%v", err)
 	}
 }
+
+// allPackages returns all the packages that can be found
+// under the $GOPATH directories and $GOROOT.
+func allPackages() []string {
+	have := make(map[string]bool)
+	var pkgs []string
+	runtime := filepath.Join(build.Path[0].SrcDir(), "runtime") + string(filepath.Separator)
+	for _, t := range build.Path {
+		src := t.SrcDir() + string(filepath.Separator)
+		filepath.Walk(src, func(path string, fi os.FileInfo, err error) error {
+			if err != nil || !fi.IsDir() {
+				return nil
+			}
+
+			// Avoid testdata directory trees.
+			if strings.HasSuffix(path, string(filepath.Separator)+"testdata") {
+				return filepath.SkipDir
+			}
+			// Avoid runtime subdirectories.
+			if strings.HasPrefix(path, runtime) {
+				switch path {
+				case runtime + "darwin", runtime + "freebsd", runtime + "linux", runtime + "netbsd", runtime + "openbsd", runtime + "windows":
+					return filepath.SkipDir
+				}
+			}
+
+			_, err = build.ScanDir(path)
+			if err != nil {
+				return nil
+			}
+			name := path[len(src):]
+			if have[name] {
+				return nil
+			}
+			pkgs = append(pkgs, name)
+			have[name] = true
+
+			// Avoid go/build test data.
+			if path == filepath.Join(build.Path[0].SrcDir(), "go/build") {
+				return filepath.SkipDir
+			}
+
+			return nil
+		})
+
+		// TODO: Commands.
+	}
+	return pkgs
+}

src/cmd/go/pkg.go

@@ -8,6 +8,7 @@ import (
 	"fmt"
 	"go/build"
 	"go/doc"
+	"os"
 	"path/filepath"
 	"sort"
 	"strings"
@@ -40,6 +41,7 @@ type Package struct {
 	info    *build.DirInfo
 	imports []*Package
 	gofiles []string // GoFiles+CgoFiles
+	targ    string
 }
 
 // packageCache is a lookup cache for loadPackage,
@@ -66,10 +68,21 @@ func loadPackage(arg string) (*Package, error) {
 
 	// Find basic information about package path.
 	t, importPath, err := build.FindTree(arg)
+	// Maybe it is a standard command.
+	if err != nil && !filepath.IsAbs(arg) && !strings.HasPrefix(arg, ".") {
+		goroot := build.Path[0]
+		p := filepath.Join(goroot.Path, "src/cmd", arg)
+		if st, err1 := os.Stat(p); err1 == nil && st.IsDir() {
+			t = goroot
+			importPath = "../cmd/" + arg
+			err = nil
+		}
+	}
 	if err != nil {
 		return nil, err
 	}
-	dir := filepath.Join(t.SrcDir(), importPath)
+
+	dir := filepath.Join(t.SrcDir(), filepath.FromSlash(importPath))
 
 	// Maybe we know the package by its directory.
 	if p := packageCache[dir]; p != nil {
@@ -79,13 +92,25 @@ func loadPackage(arg string) (*Package, error) {
 		return p, nil
 	}
 
+	return scanPackage(&build.DefaultContext, t, arg, importPath, dir)
+}
+
+func scanPackage(ctxt *build.Context, t *build.Tree, arg, importPath, dir string) (*Package, error) {
 	// Read the files in the directory to learn the structure
 	// of the package.
-	info, err := build.ScanDir(dir)
+	info, err := ctxt.ScanDir(dir)
 	if err != nil {
 		return nil, err
 	}
 
+	var targ string
+	if info.Package == "main" {
+		_, elem := filepath.Split(importPath)
+		targ = filepath.Join(t.BinDir(), elem)
+	} else {
+		targ = filepath.Join(t.PkgDir(), filepath.FromSlash(importPath)+".a")
+	}
+
 	p := &Package{
 		Name:       info.Package,
 		Doc:        doc.CommentText(info.PackageComment),
@@ -97,6 +122,9 @@ func loadPackage(arg string) (*Package, error) {
 		SFiles:     info.SFiles,
 		CgoFiles:   info.CgoFiles,
 		Standard:   t.Goroot && !strings.Contains(importPath, "."),
+		targ:       targ,
+		t:          t,
+		info:       info,
 	}
 
 	// Build list of full paths to all Go files in the package,
@@ -123,7 +151,7 @@ func loadPackage(arg string) (*Package, error) {
 		}
 		p1, err := loadPackage(path)
 		if err != nil {
-			delete(packageCache, arg)
+			delete(packageCache, dir)
 			delete(packageCache, importPath)
 			// Add extra error detail to show full import chain.
 			// Always useful, but especially useful in import loops.

src/cmd/go/run.go

@@ -0,0 +1,47 @@
+// Copyright 2011 The Go Authors.  All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import ()
+
+// Break init loop.
+func init() {
+	cmdRun.Run = runRun
+}
+
+var cmdRun = &Command{
+	UsageLine: "run [-a] [-n] [-v] gofiles...",
+	Short:     "compile and run Go program",
+	Long: `
+Run compiles and runs the main package comprising the named Go source files.
+
+The -a flag forces reinstallation of packages that are already up-to-date.
+The -n flag prints the commands but does not run them.
+The -v flag prints the commands.
+
+See also: go build.
+	`,
+}
+
+var runA = cmdRun.Flag.Bool("a", false, "")
+var runN = cmdRun.Flag.Bool("n", false, "")
+var runV = cmdRun.Flag.Bool("v", false, "")
+
+func runRun(cmd *Command, args []string) {
+	var b builder
+	b.init(*runA, *runN, *runV)
+	p := goFilesPackage(args, "")
+	p.targ = "" // force rebuild - no up-to-date copy anywhere
+	a1 := b.action(modeBuild, modeBuild, p)
+	a := &action{f: (*builder).runProgram, deps: []*action{a1}}
+	b.do(a)
+}
+
+// runProgram is the action for running a binary that has already
+// been compiled.  We ignore exit status.
+func (b *builder) runProgram(a *action) error {
+	run(a.deps[0].pkgbin)
+	return nil
+}