go/dashboard/builder/env.go

// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bytes"
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"runtime"
	"strings"

	"golang.org/x/tools/go/vcs"
)

// builderEnv represents the environment that a Builder will run tests in.
type builderEnv interface {
	// setup sets up the builder environment and returns the directory to run the buildCmd in.
	setup(repo *Repo, workpath, hash string, envv []string) (string, error)
}

// goEnv represents the builderEnv for the main Go repo.
type goEnv struct {
	goos, goarch string
}

func (b *Builder) envv() []string {
	if runtime.GOOS == "windows" {
		return b.envvWindows()
	}

	var e []string
	if *buildTool == "go" {
		e = []string{
			"GOOS=" + b.goos,
			"GOARCH=" + b.goarch,
			"GOROOT_FINAL=/usr/local/go",
		}
		switch b.goos {
		case "android", "nacl":
			// Cross compile.
		default:
			// If we are building, for example, linux/386 on a linux/amd64 machine we want to
			// make sure that the whole build is done as a if this were compiled on a real
			// linux/386 machine. In other words, we want to not do a cross compilation build.
			// To do this we set GOHOSTOS and GOHOSTARCH to override the detection in make.bash.
			//
			// The exception to this rule is when we are doing nacl/android builds. These are by
			// definition always cross compilation, and we have support built into cmd/go to be
			// able to handle this case.
			e = append(e, "GOHOSTOS="+b.goos, "GOHOSTARCH="+b.goarch)
		}
	}

	for _, k := range extraEnv() {
		if s, ok := getenvOk(k); ok {
			e = append(e, k+"="+s)
		}
	}
	return e
}

func (b *Builder) envvWindows() []string {
	var start map[string]string
	if *buildTool == "go" {
		start = map[string]string{
			"GOOS":         b.goos,
			"GOHOSTOS":     b.goos,
			"GOARCH":       b.goarch,
			"GOHOSTARCH":   b.goarch,
			"GOROOT_FINAL": `c:\go`,
			"GOBUILDEXIT":  "1", // exit all.bat with completion status.
		}
	}

	for _, name := range extraEnv() {
		if s, ok := getenvOk(name); ok {
			start[name] = s
		}
	}
	if b.goos == "windows" {
		switch b.goarch {
		case "amd64":
			start["PATH"] = `c:\TDM-GCC-64\bin;` + start["PATH"]
		case "386":
			start["PATH"] = `c:\TDM-GCC-32\bin;` + start["PATH"]
		}
	}
	skip := map[string]bool{
		"GOBIN":   true,
		"GOPATH":  true,
		"GOROOT":  true,
		"INCLUDE": true,
		"LIB":     true,
	}
	var e []string
	for name, v := range start {
		e = append(e, name+"="+v)
		skip[name] = true
	}
	for _, kv := range os.Environ() {
		s := strings.SplitN(kv, "=", 2)
		name := strings.ToUpper(s[0])
		switch {
		case name == "":
			// variables, like "=C:=C:\", just copy them
			e = append(e, kv)
		case !skip[name]:
			e = append(e, kv)
			skip[name] = true
		}
	}
	return e
}

// setup for a goEnv clones the main go repo to workpath/go at the provided hash
// and returns the path workpath/go/src, the location of all go build scripts.
func (env *goEnv) setup(repo *Repo, workpath, hash string, envv []string) (string, error) {
	goworkpath := filepath.Join(workpath, "go")
	if err := repo.Export(goworkpath, hash); err != nil {
		return "", fmt.Errorf("error exporting repository: %s", err)
	}
	return filepath.Join(goworkpath, "src"), nil
}

// gccgoEnv represents the builderEnv for the gccgo compiler.
type gccgoEnv struct{}

// setup for a gccgoEnv clones the gofrontend repo to workpath/go at the hash
// and clones the latest GCC branch to repo.Path/gcc. The gccgo sources are
// replaced with the updated sources in the gofrontend repo and gcc gets
// gets configured and built in workpath/gcc-objdir. The path to
// workpath/gcc-objdir is returned.
func (env *gccgoEnv) setup(repo *Repo, workpath, hash string, envv []string) (string, error) {
	gccpath := filepath.Join(repo.Path, "gcc")

	// get a handle to Git vcs.Cmd for pulling down GCC from the mirror.
	git := vcs.ByCmd("git")

	// only pull down gcc if we don't have a local copy.
	if _, err := os.Stat(gccpath); err != nil {
		if err := timeout(*cmdTimeout, func() error {
			// pull down a working copy of GCC.

			cloneCmd := []string{
				"clone",
				// This is just a guess since there are ~6000 commits to
				// GCC per year. It's likely there will be enough history
				// to cross-reference the Gofrontend commit against GCC.
				// The disadvantage would be if the commit being built is more than
				// a year old; in this case, the user should make a clone that has
				// the full history.
				"--depth", "6000",
				// We only care about the master branch.
				"--branch", "master", "--single-branch",
				*gccPath,
			}

			// Clone Kind			Clone Time(Dry run)	Clone Size
			// ---------------------------------------------------------------
			// Full Clone			10 - 15 min		2.2 GiB
			// Master Branch		2 - 3 min		1.5 GiB
			// Full Clone(shallow)		1 min			900 MiB
			// Master Branch(shallow)	40 sec			900 MiB
			//
			// The shallow clones have the same size, which is expected,
			// but the full shallow clone will only have 6000 commits
			// spread across all branches.  There are ~50 branches.
			return run(exec.Command("git", cloneCmd...), runEnv(envv), allOutput(os.Stdout), runDir(repo.Path))
		}); err != nil {
			return "", err
		}
	}

	if err := git.Download(gccpath); err != nil {
		return "", err
	}

	// get the modified files for this commit.

	var buf bytes.Buffer
	if err := run(exec.Command("hg", "status", "--no-status", "--change", hash),
		allOutput(&buf), runDir(repo.Path), runEnv(envv)); err != nil {
		return "", fmt.Errorf("Failed to find the modified files for %s: %s", hash, err)
	}
	modifiedFiles := strings.Split(buf.String(), "\n")
	var isMirrored bool
	for _, f := range modifiedFiles {
		if strings.HasPrefix(f, "go/") || strings.HasPrefix(f, "libgo/") {
			isMirrored = true
			break
		}
	}

	// use git log to find the corresponding commit to sync to in the gcc mirror.
	// If the files modified in the gofrontend are mirrored to gcc, we expect a
	// commit with a similar description in the gcc mirror. If the files modified are
	// not mirrored, e.g. in support/, we can sync to the most recent gcc commit that
	// occurred before those files were modified to verify gccgo's status at that point.
	logCmd := []string{
		"log",
		"-1",
		"--format=%H",
	}
	var errMsg string
	if isMirrored {
		commitDesc, err := repo.Master.VCS.LogAtRev(repo.Path, hash, "{desc|firstline|escape}")
		if err != nil {
			return "", err
		}

		quotedDesc := regexp.QuoteMeta(string(commitDesc))
		logCmd = append(logCmd, "--grep", quotedDesc, "--regexp-ignore-case", "--extended-regexp")
		errMsg = fmt.Sprintf("Failed to find a commit with a similar description to '%s'", string(commitDesc))
	} else {
		commitDate, err := repo.Master.VCS.LogAtRev(repo.Path, hash, "{date|rfc3339date}")
		if err != nil {
			return "", err
		}

		logCmd = append(logCmd, "--before", string(commitDate))
		errMsg = fmt.Sprintf("Failed to find a commit before '%s'", string(commitDate))
	}

	buf.Reset()
	if err := run(exec.Command("git", logCmd...), runEnv(envv), allOutput(&buf), runDir(gccpath)); err != nil {
		return "", fmt.Errorf("%s: %s", errMsg, err)
	}
	gccRev := buf.String()
	if gccRev == "" {
		return "", fmt.Errorf(errMsg)
	}

	// checkout gccRev
	// TODO(cmang): Fix this to work in parallel mode.
	if err := run(exec.Command("git", "reset", "--hard", strings.TrimSpace(gccRev)), runEnv(envv), runDir(gccpath)); err != nil {
		return "", fmt.Errorf("Failed to checkout commit at revision %s: %s", gccRev, err)
	}

	// make objdir to work in
	gccobjdir := filepath.Join(workpath, "gcc-objdir")
	if err := os.Mkdir(gccobjdir, mkdirPerm); err != nil {
		return "", err
	}

	// configure GCC with substituted gofrontend and libgo
	if err := run(exec.Command(filepath.Join(gccpath, "configure"),
		"--enable-languages=c,c++,go",
		"--disable-bootstrap",
		"--disable-multilib",
	), runEnv(envv), runDir(gccobjdir)); err != nil {
		return "", fmt.Errorf("Failed to configure GCC: %v", err)
	}

	// build gcc
	if err := run(exec.Command("make", *gccOpts), runTimeout(*buildTimeout), runEnv(envv), runDir(gccobjdir)); err != nil {
		return "", fmt.Errorf("Failed to build GCC: %s", err)
	}

	return gccobjdir, nil
}

func getenvOk(k string) (v string, ok bool) {
	v = os.Getenv(k)
	if v != "" {
		return v, true
	}
	keq := k + "="
	for _, kv := range os.Environ() {
		if kv == keq {
			return "", true
		}
	}
	return "", false
}

// extraEnv returns environment variables that need to be copied from
// the gobuilder's environment to the envv of its subprocesses.
func extraEnv() []string {
	extra := []string{
		"GOARM",
		"GO386",
		"CGO_ENABLED",
		"CC",
		"CC_FOR_TARGET",
		"PATH",
		"TMPDIR",
		"USER",
	}
	if runtime.GOOS == "plan9" {
		extra = append(extra, "objtype", "cputype", "path")
	}
	return extra
}