go/packages: enable an external source of package information

This allows an external binary (not go list) to be the source of package
information.
It uses a binary called gopackagesraw if present in the PATH.
The binary can be overriden by specifying the GOPACKAGESRAW
environment variable.
The command must accept the -test -deps -export and -flags, and take a
list of package patterns to match. It then returns a raw.Results followed by
the matching raw.Package structs in json format on stdout.

Change-Id: I836014d837a284999ded0c1157b8e6dac058ba69
Reviewed-on: https://go-review.googlesource.com/125938
Run-TryBot: Ian Cottrell <iancottrell@google.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Michael Matloob <matloob@golang.org>

go/packages/external.go

@@ -0,0 +1,79 @@
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// This file enables an external tool to intercept package requests.
+// If the tool is present then its results are used in preference to
+// the go list command.
+
+package packages
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"os/exec"
+	"strings"
+
+	"golang.org/x/tools/go/packages/raw"
+)
+
+// externalPackages uses an external command to interpret the words and produce
+// raw packages.
+// dir may be "" and env may be nil, as per os/exec.Command.
+func findRawTool(ctx context.Context, cfg *raw.Config) string {
+	const toolPrefix = "GOPACKAGESRAW="
+	for _, env := range cfg.Env {
+		if val := strings.TrimPrefix(env, toolPrefix); val != env {
+			return val
+		}
+	}
+	if found, err := exec.LookPath("gopackagesraw"); err == nil {
+		return found
+	}
+	return ""
+}
+
+// externalPackages uses an external command to interpret the words and produce
+// raw packages.
+// cfg.Dir may be "" and cfg.Env may be nil, as per os/exec.Command.
+func externalPackages(ctx context.Context, cfg *raw.Config, tool string, words ...string) ([]string, []*raw.Package, error) {
+	buf := new(bytes.Buffer)
+	fullargs := []string{
+		fmt.Sprintf("-test=%t", cfg.Tests),
+		fmt.Sprintf("-export=%t", cfg.Export),
+		fmt.Sprintf("-deps=%t", cfg.Deps),
+	}
+	for _, f := range cfg.Flags {
+		fullargs = append(fullargs, fmt.Sprintf("-flags=%v", f))
+	}
+	fullargs = append(fullargs, "--")
+	fullargs = append(fullargs, words...)
+	cmd := exec.CommandContext(ctx, tool, fullargs...)
+	cmd.Env = cfg.Env
+	cmd.Dir = cfg.Dir
+	cmd.Stdout = buf
+	cmd.Stderr = new(bytes.Buffer)
+	if err := cmd.Run(); err != nil {
+		return nil, nil, fmt.Errorf("%v: %v: %s", tool, err, cmd.Stderr)
+	}
+	var results raw.Results
+	var pkgs []*raw.Package
+	dec := json.NewDecoder(buf)
+	if err := dec.Decode(&results); err != nil {
+		return nil, nil, fmt.Errorf("JSON decoding raw.Results failed: %v", err)
+	}
+	if results.Error != "" {
+		return nil, nil, errors.New(results.Error)
+	}
+	for dec.More() {
+		p := new(raw.Package)
+		if err := dec.Decode(p); err != nil {
+			return nil, nil, fmt.Errorf("JSON decoding raw.Package failed: %v", err)
+		}
+		pkgs = append(pkgs, p)
+	}
+	return results.Roots, pkgs, nil
+}

go/packages/packages.go

@@ -162,6 +162,9 @@ func Load(cfg *Config, patterns ...string) ([]*Package, error) {
 // an error if the operation failed.
 func loadRaw(ctx context.Context, cfg *raw.Config, patterns ...string) ([]string, []*raw.Package, error) {
 	//TODO: this is the seam at which we enable alternate build systems
+	if tool := findRawTool(ctx, cfg); tool != "" {
+		return externalPackages(ctx, cfg, tool, patterns...)
+	}
 	return golist.LoadRaw(ctx, cfg, patterns...)
 }
 

go/packages/raw/raw.go

@@ -17,6 +17,18 @@ data for the packages API, all tools should interact only with the packages API.
 */
 package raw
 
+import (
+	"flag"
+)
+
+// Results describes the results of a load operation.
+type Results struct {
+	// Roots is the set of package identifiers that directly matched the patterns.
+	Roots []string
+	// Error is an error message if the query failed for some reason.
+	Error string `json:",omitempty"`
+}
+
 // Package is the raw serialized form of a packages.Package
 type Package struct {
 	// ID is a unique identifier for a package,
@@ -102,3 +114,29 @@ type Config struct {
 	// set on them.
 	Deps bool
 }
+
+// AddFlags adds the standard flags used to set a Config to the supplied flag set.
+// This is used by implementations of the external raw package binary to correctly
+// interpret the flags passed from the config.
+func (cfg *Config) AddFlags(flags *flag.FlagSet) {
+	flags.BoolVar(&cfg.Deps, "deps", false, "include all dependencies")
+	flags.BoolVar(&cfg.Tests, "test", false, "include all test packages")
+	flags.BoolVar(&cfg.Export, "export", false, "include export data files")
+	flags.Var(extraFlags{cfg}, "flags", "extra flags to pass to the underlying command")
+}
+
+// extraFlags collects all occurrences of --flags into a single array
+// We do this because it's much easier than escaping joining and splitting
+// the extra flags that must be passed across the boundary unmodified
+type extraFlags struct {
+	cfg *Config
+}
+
+func (e extraFlags) String() string {
+	return ""
+}
+
+func (e extraFlags) Set(value string) error {
+	e.cfg.Flags = append(e.cfg.Flags, value)
+	return nil
+}