[dev.cmdgo] cmd/go: add the workspace mode

This change adds the outline of the implementation of the workspace mode.
The go command will now locate go.work files, and read them to determine
which modules are in the workspace. It will then put those modules in
the root of the workspace when building the build list. It supports
building, running, testing, and listing in workspaces. There are still
many TODOs for undone work and other changes to fix certain cases. Some
of these undone parts include: replaces and go.work.sum files, as well
as go mod {test,why,verify}, excludes in workspaces, updating work files
to include module names in comments and setting the GOWORK variable.

For #45713

Change-Id: I72716af7a300a2896087fc8a79c04e951d248278
Reviewed-on: https://go-review.googlesource.com/c/go/+/334934
Trust: Michael Matloob <matloob@golang.org>
Run-TryBot: Michael Matloob <matloob@golang.org>
TryBot-Result: Go Bot <gobot@golang.org>
Reviewed-by: Jay Conrod <jayconrod@google.com>

src/cmd/go/alldocs.go

@@ -167,6 +167,14 @@
 // 		directory, but it is not accessed. When -modfile is specified, an
 // 		alternate go.sum file is also used: its path is derived from the
 // 		-modfile flag by trimming the ".mod" extension and appending ".sum".
+//   -workfile file
+//     in module aware mode, use the given go.work file as a workspace file.
+// 		By default or when -workfile is "auto", the go command searches for a
+// 		file named go.work in the current directory and then containing directories
+// 		until one is found. If a valid go.work file is found, the modules
+// 		specified will collectively be used as the main modules. If -workfile
+// 		is "off", or a go.work file is not found in "auto" mode, workspace
+// 		mode is disabled.
 // 	-overlay file
 // 		read a JSON config file that provides an overlay for build operations.
 // 		The file is a JSON struct with a single field, named 'Replace', that

src/cmd/go/internal/base/flag.go

@@ -62,6 +62,13 @@ func AddModFlag(flags *flag.FlagSet) {
 	flags.Var(explicitStringFlag{value: &cfg.BuildMod, explicit: &cfg.BuildModExplicit}, "mod", "")
 }
 
+// AddWorkfileFlag adds the workfile flag to the flag set. It enables workspace
+// mode for commands that support it by resetting the cfg.WorkFile variable
+// to "" (equivalent to auto) rather than off.
+func AddWorkfileFlag(flags *flag.FlagSet) {
+	flags.Var(explicitStringFlag{value: &cfg.WorkFile, explicit: &cfg.WorkFileExplicit}, "workfile", "")
+}
+
 // AddModCommonFlags adds the module-related flags common to build commands
 // and 'go mod' subcommands.
 func AddModCommonFlags(flags *flag.FlagSet) {

src/cmd/go/internal/cfg/cfg.go

@@ -47,8 +47,10 @@ var (
 	BuildWork              bool // -work flag
 	BuildX                 bool // -x flag
 
-	ModCacheRW bool   // -modcacherw flag
-	ModFile    string // -modfile flag
+	ModCacheRW       bool   // -modcacherw flag
+	ModFile          string // -modfile flag
+	WorkFile         string // -workfile flag
+	WorkFileExplicit bool   // whether -workfile was set explicitly
 
 	CmdName string // "build", "install", "list", "mod tidy", etc.
 

src/cmd/go/internal/envcmd/env.go

@@ -145,6 +145,7 @@ func findEnv(env []cfg.EnvVar, name string) string {
 // ExtraEnvVars returns environment variables that should not leak into child processes.
 func ExtraEnvVars() []cfg.EnvVar {
 	gomod := ""
+	modload.Init()
 	if modload.HasModRoot() {
 		gomod = filepath.Join(modload.ModRoot(), "go.mod")
 	} else if modload.Enabled() {

src/cmd/go/internal/list/list.go

@@ -316,6 +316,7 @@ For more about modules, see https://golang.org/ref/mod.
 func init() {
 	CmdList.Run = runList // break init cycle
 	work.AddBuildFlags(CmdList, work.DefaultBuildFlags)
+	base.AddWorkfileFlag(&CmdList.Flag)
 }
 
 var (
@@ -336,6 +337,8 @@ var (
 var nl = []byte{'\n'}
 
 func runList(ctx context.Context, cmd *base.Command, args []string) {
+	modload.InitWorkfile()
+
 	if *listFmt != "" && *listJson == true {
 		base.Fatalf("go list -f cannot be used with -json")
 	}

src/cmd/go/internal/modcmd/download.go

@@ -66,6 +66,7 @@ func init() {
 	// TODO(jayconrod): https://golang.org/issue/35849 Apply -x to other 'go mod' commands.
 	cmdDownload.Flag.BoolVar(&cfg.BuildX, "x", false, "")
 	base.AddModCommonFlags(&cmdDownload.Flag)
+	base.AddWorkfileFlag(&cmdDownload.Flag)
 }
 
 type moduleJSON struct {
@@ -81,6 +82,8 @@ type moduleJSON struct {
 }
 
 func runDownload(ctx context.Context, cmd *base.Command, args []string) {
+	modload.InitWorkfile()
+
 	// Check whether modules are enabled and whether we're in a module.
 	modload.ForceUseModules = true
 	if !modload.HasModRoot() && len(args) == 0 {

src/cmd/go/internal/modcmd/graph.go

@@ -42,9 +42,12 @@ var (
 func init() {
 	cmdGraph.Flag.Var(&graphGo, "go", "")
 	base.AddModCommonFlags(&cmdGraph.Flag)
+	base.AddWorkfileFlag(&cmdGraph.Flag)
 }
 
 func runGraph(ctx context.Context, cmd *base.Command, args []string) {
+	modload.InitWorkfile()
+
 	if len(args) > 0 {
 		base.Fatalf("go mod graph: graph takes no arguments")
 	}

src/cmd/go/internal/modcmd/verify.go

@@ -39,9 +39,12 @@ See https://golang.org/ref/mod#go-mod-verify for more about 'go mod verify'.
 
 func init() {
 	base.AddModCommonFlags(&cmdVerify.Flag)
+	base.AddWorkfileFlag(&cmdVerify.Flag)
 }
 
 func runVerify(ctx context.Context, cmd *base.Command, args []string) {
+	modload.InitWorkfile()
+
 	if len(args) != 0 {
 		// NOTE(rsc): Could take a module pattern.
 		base.Fatalf("go mod verify: verify takes no arguments")

src/cmd/go/internal/modcmd/why.go

@@ -61,9 +61,11 @@ var (
 func init() {
 	cmdWhy.Run = runWhy // break init cycle
 	base.AddModCommonFlags(&cmdWhy.Flag)
+	base.AddWorkfileFlag(&cmdWhy.Flag)
 }
 
 func runWhy(ctx context.Context, cmd *base.Command, args []string) {
+	modload.InitWorkfile()
 	modload.ForceUseModules = true
 	modload.RootMode = modload.NeedRoot
 

src/cmd/go/internal/modload/init.go

@@ -53,10 +53,6 @@ func TODOWorkspaces(s string) error {
 var (
 	initialized bool
 
-	// The directory containing go.work file. Set if in a go.work file is found
-	// and the go command is operating in workspace mode.
-	workRoot string
-
 	// These are primarily used to initialize the MainModules, and should be
 	// eventually superceded by them but are still used in cases where the module
 	// roots are required but MainModules hasn't been initialized yet. Set to
@@ -66,6 +62,12 @@ var (
 	gopath   string
 )
 
+// Variable set in InitWorkfile
+var (
+	// Set to the path to the go.work file, or "" if workspace mode is disabled.
+	workFilePath string
+)
+
 type MainModuleSet struct {
 	// versions are the module.Version values of each of the main modules.
 	// For each of them, the Path fields are ordinary module paths and the Version
@@ -187,6 +189,20 @@ func BinDir() string {
 	return filepath.Join(gopath, "bin")
 }
 
+// InitWorkfile initializes the workFilePath variable for commands that
+// operate in workspace mode. It should not be called by other commands,
+// for example 'go mod tidy', that don't operate in workspace mode.
+func InitWorkfile() {
+	switch cfg.WorkFile {
+	case "off":
+		workFilePath = ""
+	case "", "auto":
+		workFilePath = findWorkspaceFile(base.Cwd())
+	default:
+		workFilePath = cfg.WorkFile
+	}
+}
+
 // Init determines whether module mode is enabled, locates the root of the
 // current module (if any), sets environment variables for Git subprocesses, and
 // configures the cfg, codehost, load, modfetch, and search packages for use
@@ -259,6 +275,8 @@ func Init() {
 			base.Fatalf("go: -modfile cannot be used with commands that ignore the current module")
 		}
 		modRoots = nil
+	} else if inWorkspaceMode() {
+		// We're in workspace mode.
 	} else {
 		modRoots = findModuleRoots(base.Cwd())
 		if modRoots == nil {
@@ -293,6 +311,7 @@ func Init() {
 	// We're in module mode. Set any global variables that need to be set.
 	cfg.ModulesEnabled = true
 	setDefaultBuildMod()
+	_ = TODOWorkspaces("ensure that buildmod is readonly")
 	list := filepath.SplitList(cfg.BuildContext.GOPATH)
 	if len(list) == 0 || list[0] == "" {
 		base.Fatalf("missing $GOPATH")
@@ -302,7 +321,17 @@ func Init() {
 		base.Fatalf("$GOPATH/go.mod exists but should not")
 	}
 
-	if modRoots == nil {
+	if inWorkspaceMode() {
+
+		_ = TODOWorkspaces("go.work.sum, and also allow modfetch to fall back to individual go.sums")
+		_ = TODOWorkspaces("replaces")
+		var err error
+		modRoots, err = loadWorkFile(workFilePath)
+		if err != nil {
+			base.Fatalf("reading go.work: %v", err)
+		}
+		// TODO(matloob) should workRoot just be workFile?
+	} else if modRoots == nil {
 		// We're in module mode, but not inside a module.
 		//
 		// Commands like 'go build', 'go run', 'go list' have no go.mod file to
@@ -388,12 +417,24 @@ func ModRoot() string {
 	if !HasModRoot() {
 		die()
 	}
+	if inWorkspaceMode() {
+		panic("ModRoot called in workspace mode")
+	}
+	// This is similar to MustGetSingleMainModule but we can't call that
+	// because MainModules may not yet exist when ModRoot is called.
 	if len(modRoots) != 1 {
-		panic(TODOWorkspaces("need to handle multiple modroots here"))
+		panic("not in workspace mode but there are multiple ModRoots")
 	}
 	return modRoots[0]
 }
 
+func inWorkspaceMode() bool {
+	if !initialized {
+		panic("inWorkspaceMode called before modload.Init called")
+	}
+	return workFilePath != ""
+}
+
 // HasModRoot reports whether a main module is present.
 // HasModRoot may return false even if Enabled returns true: for example, 'get'
 // does not require a main module.
@@ -451,6 +492,31 @@ func (goModDirtyError) Error() string {
 
 var errGoModDirty error = goModDirtyError{}
 
+func loadWorkFile(path string) (modRoots []string, err error) {
+	workDir := filepath.Dir(path)
+	workData, err := lockedfile.Read(path)
+	if err != nil {
+		return nil, err
+	}
+	wf, err := modfile.ParseWork(path, workData, nil)
+	if err != nil {
+		return nil, err
+	}
+	seen := map[string]bool{}
+	for _, d := range wf.Directory {
+		modRoot := d.Path
+		if !filepath.IsAbs(modRoot) {
+			modRoot = filepath.Join(workDir, modRoot)
+		}
+		if seen[modRoot] {
+			return nil, fmt.Errorf("path %s appears multiple times in workspace", modRoot)
+		}
+		seen[modRoot] = true
+		modRoots = append(modRoots, modRoot)
+	}
+	return modRoots, nil
+}
+
 // LoadModFile sets Target and, if there is a main module, parses the initial
 // build list from its go.mod file.
 //
@@ -498,40 +564,62 @@ func loadModFile(ctx context.Context) (rs *Requirements, needCommit bool) {
 		return requirements, false
 	}
 
-	gomod := ModFilePath()
-	data, err := lockedfile.Read(gomod)
-	if err != nil {
-		base.Fatalf("go: %v", err)
-	}
-
-	var fixed bool
-	f, err := modfile.Parse(gomod, data, fixVersion(ctx, &fixed))
-	if err != nil {
-		// Errors returned by modfile.Parse begin with file:line.
-		base.Fatalf("go: errors parsing go.mod:\n%s\n", err)
-	}
-	if f.Module == nil {
-		// No module declaration. Must add module path.
-		base.Fatalf("go: no module declaration in go.mod. To specify the module path:\n\tgo mod edit -module=example.com/mod")
-	}
-
-	// For now, this code assumes there's a single main module, because there's
-	// no way to specify multiple main modules yet. TODO(#45713): update this
-	// in a later CL.
-	modFile = f
-	mainModule := f.Module.Mod
-	MainModules = makeMainModules([]module.Version{mainModule}, modRoots)
-	index = indexModFile(data, f, mainModule, fixed)
-
-	if err := module.CheckImportPath(f.Module.Mod.Path); err != nil {
-		if pathErr, ok := err.(*module.InvalidPathError); ok {
-			pathErr.Kind = "module"
+	var modFiles []*modfile.File
+	var mainModules []module.Version
+	for _, modroot := range modRoots {
+		gomod := modFilePath(modroot)
+		var data []byte
+		var err error
+		if gomodActual, ok := fsys.OverlayPath(gomod); ok {
+			// Don't lock go.mod if it's part of the overlay.
+			// On Plan 9, locking requires chmod, and we don't want to modify any file
+			// in the overlay. See #44700.
+			data, err = os.ReadFile(gomodActual)
+		} else {
+			data, err = lockedfile.Read(gomodActual)
+		}
+		if err != nil {
+			base.Fatalf("go: %v", err)
+		}
+
+		var fixed bool
+		f, err := modfile.Parse(gomod, data, fixVersion(ctx, &fixed))
+		if err != nil {
+			// Errors returned by modfile.Parse begin with file:line.
+			base.Fatalf("go: errors parsing go.mod:\n%s\n", err)
+		}
+		if f.Module == nil {
+			// No module declaration. Must add module path.
+			base.Fatalf("go: no module declaration in go.mod. To specify the module path:\n\tgo mod edit -module=example.com/mod")
+		}
+
+		modFile = f // TODO(golang.org/cl/327329): remove the global modFile variable and replace it with multiple modfiles
+		modFiles = append(modFiles, f)
+		mainModule := f.Module.Mod
+		mainModules = append(mainModules, mainModule)
+		index = indexModFile(data, f, mainModule, fixed)
+
+		if err := module.CheckImportPath(f.Module.Mod.Path); err != nil {
+			if pathErr, ok := err.(*module.InvalidPathError); ok {
+				pathErr.Kind = "module"
+			}
+			base.Fatalf("go: %v", err)
 		}
-		base.Fatalf("go: %v", err)
 	}
 
+	MainModules = makeMainModules(mainModules, modRoots)
 	setDefaultBuildMod() // possibly enable automatic vendoring
-	rs = requirementsFromModFile(ctx)
+	rs = requirementsFromModFiles(ctx, modFiles)
+
+	if inWorkspaceMode() {
+		// We don't need to do anything for vendor or update the mod file so
+		// return early.
+
+		_ = TODOWorkspaces("don't worry about commits for now, but eventually will want to update go.work files")
+		return rs, false
+	}
+
+	mainModule := MainModules.mustGetSingleMainModule()
 
 	if cfg.BuildMod == "vendor" {
 		readVendorList()
@@ -549,6 +637,7 @@ func loadModFile(ctx context.Context) (rs *Requirements, needCommit bool) {
 				// Go 1.11 through 1.16 have eager requirements, but the latest Go
 				// version uses lazy requirements instead — so we need to cnvert the
 				// requirements to be lazy.
+				var err error
 				rs, err = convertDepth(ctx, rs, lazy)
 				if err != nil {
 					base.Fatalf("go: %v", err)
@@ -613,7 +702,7 @@ func CreateModFile(ctx context.Context, modPath string) {
 		base.Fatalf("go: %v", err)
 	}
 
-	commitRequirements(ctx, modFileGoVersion(), requirementsFromModFile(ctx))
+	commitRequirements(ctx, modFileGoVersion(), requirementsFromModFiles(ctx, []*modfile.File{modFile}))
 
 	// Suggest running 'go mod tidy' unless the project is empty. Even if we
 	// imported all the correct requirements above, we're probably missing
@@ -737,29 +826,36 @@ func makeMainModules(ms []module.Version, rootDirs []string) *MainModuleSet {
 	return mainModules
 }
 
-// requirementsFromModFile returns the set of non-excluded requirements from
+// requirementsFromModFiles returns the set of non-excluded requirements from
 // the global modFile.
-func requirementsFromModFile(ctx context.Context) *Requirements {
-	roots := make([]module.Version, 0, len(modFile.Require))
+func requirementsFromModFiles(ctx context.Context, modFiles []*modfile.File) *Requirements {
+	rootCap := 0
+	for i := range modFiles {
+		rootCap += len(modFiles[i].Require)
+	}
+	roots := make([]module.Version, 0, rootCap)
 	mPathCount := make(map[string]int)
 	for _, m := range MainModules.Versions() {
 		mPathCount[m.Path] = 1
 	}
 	direct := map[string]bool{}
-	for _, r := range modFile.Require {
-		if index != nil && index.exclude[r.Mod] {
-			if cfg.BuildMod == "mod" {
-				fmt.Fprintf(os.Stderr, "go: dropping requirement on excluded version %s %s\n", r.Mod.Path, r.Mod.Version)
-			} else {
-				fmt.Fprintf(os.Stderr, "go: ignoring requirement on excluded version %s %s\n", r.Mod.Path, r.Mod.Version)
+	for _, modFile := range modFiles {
+		// TODO(golang.org/cl/327329): Use the correct index here.
+		for _, r := range modFile.Require {
+			if index != nil && index.exclude[r.Mod] {
+				if cfg.BuildMod == "mod" {
+					fmt.Fprintf(os.Stderr, "go: dropping requirement on excluded version %s %s\n", r.Mod.Path, r.Mod.Version)
+				} else {
+					fmt.Fprintf(os.Stderr, "go: ignoring requirement on excluded version %s %s\n", r.Mod.Path, r.Mod.Version)
+				}
+				continue
 			}
-			continue
-		}
 
-		roots = append(roots, r.Mod)
-		mPathCount[r.Mod.Path]++
-		if !r.Indirect {
-			direct[r.Mod.Path] = true
+			roots = append(roots, r.Mod)
+			mPathCount[r.Mod.Path]++
+			if !r.Indirect {
+				direct[r.Mod.Path] = true
+			}
 		}
 	}
 	module.Sort(roots)
@@ -786,6 +882,11 @@ func requirementsFromModFile(ctx context.Context) *Requirements {
 // wasn't provided. setDefaultBuildMod may be called multiple times.
 func setDefaultBuildMod() {
 	if cfg.BuildModExplicit {
+		if inWorkspaceMode() {
+			base.Fatalf("go: -mod can't be set explicitly when in workspace mode." +
+				"\n\tRemove the -mod flag to use the default readonly value," +
+				"\n\tor set -workfile=off to disable workspace mode.")
+		}
 		// Don't override an explicit '-mod=' argument.
 		return
 	}
@@ -944,6 +1045,31 @@ func findModuleRoots(dir string) (roots []string) {
 	return nil
 }
 
+func findWorkspaceFile(dir string) (root string) {
+	if dir == "" {
+		panic("dir not set")
+	}
+	dir = filepath.Clean(dir)
+
+	// Look for enclosing go.mod.
+	for {
+		f := filepath.Join(dir, "go.work")
+		if fi, err := fsys.Stat(f); err == nil && !fi.IsDir() {
+			return f
+		}
+		d := filepath.Dir(dir)
+		if d == dir {
+			break
+		}
+		if d == cfg.GOROOT {
+			_ = TODOWorkspaces("Address how go.work files interact with GOROOT")
+			return "" // As a special case, don't cross GOROOT to find a go.work file.
+		}
+		dir = d
+	}
+	return ""
+}
+
 func findAltConfig(dir string) (root, name string) {
 	if dir == "" {
 		panic("dir not set")

src/cmd/go/internal/run/run.go

@@ -65,6 +65,7 @@ func init() {
 	CmdRun.Run = runRun // break init loop
 
 	work.AddBuildFlags(CmdRun, work.DefaultBuildFlags)
+	base.AddWorkfileFlag(&CmdRun.Flag)
 	CmdRun.Flag.Var((*base.StringsFlag)(&work.ExecCmd), "exec", "")
 }
 
@@ -73,6 +74,8 @@ func printStderr(args ...interface{}) (int, error) {
 }
 
 func runRun(ctx context.Context, cmd *base.Command, args []string) {
+	modload.InitWorkfile()
+
 	if shouldUseOutsideModuleMode(args) {
 		// Set global module flags for 'go run cmd@version'.
 		// This must be done before modload.Init, but we need to call work.BuildInit

src/cmd/go/internal/test/test.go

@@ -29,6 +29,7 @@ import (
 	"cmd/go/internal/cfg"
 	"cmd/go/internal/load"
 	"cmd/go/internal/lockedfile"
+	"cmd/go/internal/modload"
 	"cmd/go/internal/search"
 	"cmd/go/internal/str"
 	"cmd/go/internal/trace"
@@ -577,6 +578,7 @@ var defaultVetFlags = []string{
 }
 
 func runTest(ctx context.Context, cmd *base.Command, args []string) {
+	modload.InitWorkfile()
 	pkgArgs, testArgs = testFlags(args)
 
 	if cfg.DebugTrace != "" {

src/cmd/go/internal/test/testflag.go

@@ -29,6 +29,7 @@ import (
 
 func init() {
 	work.AddBuildFlags(CmdTest, work.OmitVFlag)
+	base.AddWorkfileFlag(&CmdTest.Flag)
 
 	cf := CmdTest.Flag
 	cf.BoolVar(&testC, "c", false, "")

src/cmd/go/internal/work/build.go

@@ -121,6 +121,14 @@ and test commands:
 		directory, but it is not accessed. When -modfile is specified, an
 		alternate go.sum file is also used: its path is derived from the
 		-modfile flag by trimming the ".mod" extension and appending ".sum".
+  -workfile file
+    in module aware mode, use the given go.work file as a workspace file.
+		By default or when -workfile is "auto", the go command searches for a
+		file named go.work in the current directory and then containing directories
+		until one is found. If a valid go.work file is found, the modules
+		specified will collectively be used as the main modules. If -workfile
+		is "off", or a go.work file is not found in "auto" mode, workspace
+		mode is disabled.
 	-overlay file
 		read a JSON config file that provides an overlay for build operations.
 		The file is a JSON struct with a single field, named 'Replace', that
@@ -201,6 +209,7 @@ func init() {
 
 	AddBuildFlags(CmdBuild, DefaultBuildFlags)
 	AddBuildFlags(CmdInstall, DefaultBuildFlags)
+	base.AddWorkfileFlag(&CmdBuild.Flag)
 }
 
 // Note that flags consulted by other parts of the code
@@ -364,6 +373,7 @@ var pkgsFilter = func(pkgs []*load.Package) []*load.Package { return pkgs }
 var runtimeVersion = runtime.Version()
 
 func runBuild(ctx context.Context, cmd *base.Command, args []string) {
+	modload.InitWorkfile()
 	BuildInit()
 	var b Builder
 	b.Init()

src/cmd/go/testdata/script/work.txt

@@ -0,0 +1,71 @@
+go run example.com/b
+stdout 'Hello from module A'
+
+# And try from a different directory
+cd c
+go run example.com/b
+stdout 'Hello from module A'
+cd $GOPATH/src
+
+go list all # all includes both modules
+stdout 'example.com/a'
+stdout 'example.com/b'
+
+# -mod can't be set in workspace mode, even to readonly
+! go list -mod=readonly all
+stderr '^go: -mod can''t be set explicitly'
+go list -mod=readonly -workfile=off all
+
+# Test that duplicates in the directory list return an error
+cp go.work go.work.backup
+cp go.work.dup go.work
+! go run example.com/b
+stderr 'reading go.work: path .* appears multiple times in workspace'
+cp go.work.backup go.work
+
+-- go.work.dup --
+go 1.17
+
+directory (
+  a
+  b
+  ../src/a
+)
+-- go.work --
+go 1.17
+
+directory (
+  ./a
+  ./b
+)
+
+-- a/go.mod --
+
+module example.com/a
+
+-- a/a.go --
+package a
+
+import "fmt"
+
+func HelloFromA() {
+  fmt.Println("Hello from module A")
+}
+
+-- b/go.mod --
+
+module example.com/b
+
+-- b/main.go --
+package main
+
+import "example.com/a"
+
+func main() {
+  a.HelloFromA()
+}
+
+-- c/README --
+Create this directory so we can cd to
+it and make sure paths are interpreted
+relative to the go.work, not the cwd.