internal/imports: return non-stdlib candidates from GetAllCandidates

Scan most sources, including GOPATH, the module cache, the main module,
and replace targets as appropriate. Use the cached stdlib instead of
scanning GOROOT.

We heavily cache the contents of the module cache, so performance is
decent. But we have to look at all the modules not in the module cache
too to get the right versions of modules (see
(*ModuleResolver).canonicalize), which currently isn't cached at all,
even just for a single run. That ends up being pretty expensive.

The implementation changes are relatively small; add package name
loading to scan(), cache that result, and allow callers to control what
directories are scanned so that it can skip GOROOT.

I also cleared out most of the stdlib from the unimported completion
test and added a simple external completion to it for safety's sake.

Change-Id: Id50fd4703b1126be35a000fe90719e19c3ab84bf
Reviewed-on: https://go-review.googlesource.com/c/tools/+/199178
Run-TryBot: Heschi Kreinick <heschi@google.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Rebecca Stambler <rstambler@golang.org>

internal/imports/fix.go

@@ -586,18 +586,20 @@ func getFixes(fset *token.FileSet, f *ast.File, filename string, env *ProcessEnv
 
 // getAllCandidates gets all of the candidates to be imported, regardless of if they are needed.
 func getAllCandidates(filename string, env *ProcessEnv) ([]ImportFix, error) {
-	// TODO(suzmue): scan for additional candidates and filter out
-	// current package.
+	// TODO(heschi): filter out current package. (Don't forget x_test can import x.)
 
-	// Get the stdlib candidates and sort by import path.
-	var paths []string
-	for importPath := range stdlib {
-		paths = append(paths, importPath)
+	// Exclude goroot results -- getting them is relatively expensive, not cached,
+	// and generally redundant with the in-memory version.
+	exclude := []gopathwalk.RootType{gopathwalk.RootGOROOT}
+	// Only the go/packages resolver uses the first argument, and nobody uses that resolver.
+	pkgs, err := env.GetResolver().scan(nil, true, exclude)
+	if err != nil {
+		return nil, err
 	}
-	sort.Strings(paths)
 
+	// Start off with the standard library.
 	var imports []ImportFix
-	for _, importPath := range paths {
+	for importPath := range stdlib {
 		imports = append(imports, ImportFix{
 			StmtInfo: ImportInfo{
 				ImportPath: importPath,
@@ -606,6 +608,27 @@ func getAllCandidates(filename string, env *ProcessEnv) ([]ImportFix, error) {
 			FixType:   AddImport,
 		})
 	}
+
+	dupCheck := map[string]struct{}{}
+	for _, pkg := range pkgs {
+		if !canUse(filename, pkg.dir) {
+			continue
+		}
+		if _, ok := dupCheck[pkg.importPathShort]; ok {
+			continue
+		}
+		dupCheck[pkg.importPathShort] = struct{}{}
+		imports = append(imports, ImportFix{
+			StmtInfo: ImportInfo{
+				ImportPath: pkg.importPathShort,
+			},
+			IdentName: pkg.packageName,
+			FixType:   AddImport,
+		})
+	}
+	sort.Slice(imports, func(i int, j int) bool {
+		return imports[i].StmtInfo.ImportPath < imports[j].StmtInfo.ImportPath
+	})
 	return imports, nil
 }
 
@@ -736,8 +759,10 @@ func addStdlibCandidates(pass *pass, refs references) {
 type Resolver interface {
 	// loadPackageNames loads the package names in importPaths.
 	loadPackageNames(importPaths []string, srcDir string) (map[string]string, error)
-	// scan finds (at least) the packages satisfying refs. The returned slice is unordered.
-	scan(refs references) ([]*pkg, error)
+	// scan finds (at least) the packages satisfying refs. If loadNames is true,
+	// package names will be set on the results, and dirs whose package name
+	// could not be determined will be excluded.
+	scan(refs references, loadNames bool, exclude []gopathwalk.RootType) ([]*pkg, error)
 	// loadExports returns the set of exported symbols in the package at dir.
 	// It returns an error if the package name in dir does not match expectPackage.
 	// loadExports may be called concurrently.
@@ -773,7 +798,7 @@ func (r *goPackagesResolver) loadPackageNames(importPaths []string, srcDir strin
 
 }
 
-func (r *goPackagesResolver) scan(refs references) ([]*pkg, error) {
+func (r *goPackagesResolver) scan(refs references, _ bool, _ []gopathwalk.RootType) ([]*pkg, error) {
 	var loadQueries []string
 	for pkgName := range refs {
 		loadQueries = append(loadQueries, "iamashamedtousethedisabledqueryname="+pkgName)
@@ -791,6 +816,7 @@ func (r *goPackagesResolver) scan(refs references) ([]*pkg, error) {
 			dir:             filepath.Dir(goPackage.CompiledGoFiles[0]),
 			importPathShort: VendorlessPath(goPackage.PkgPath),
 			goPackage:       goPackage,
+			packageName:     goPackage.Name,
 		})
 	}
 	return scan, nil
@@ -817,7 +843,7 @@ func (r *goPackagesResolver) loadExports(ctx context.Context, expectPackage stri
 }
 
 func addExternalCandidates(pass *pass, refs references, filename string) error {
-	dirScan, err := pass.env.GetResolver().scan(refs)
+	dirScan, err := pass.env.GetResolver().scan(refs, false, nil)
 	if err != nil {
 		return err
 	}
@@ -1001,6 +1027,7 @@ type pkg struct {
 	goPackage       *packages.Package
 	dir             string // absolute file path to pkg directory ("/usr/lib/go/src/net/http")
 	importPathShort string // vendorless import path ("net/http", "a/b")
+	packageName     string // package name loaded from source if requested
 }
 
 type pkgDistance struct {
@@ -1044,7 +1071,7 @@ func distance(basepath, targetpath string) int {
 	return strings.Count(p, string(filepath.Separator)) + 1
 }
 
-func (r *gopathResolver) scan(_ references) ([]*pkg, error) {
+func (r *gopathResolver) scan(_ references, loadNames bool, exclude []gopathwalk.RootType) ([]*pkg, error) {
 	dupCheck := make(map[string]bool)
 	var result []*pkg
 
@@ -1059,15 +1086,38 @@ func (r *gopathResolver) scan(_ references) ([]*pkg, error) {
 		}
 		dupCheck[dir] = true
 		importpath := filepath.ToSlash(dir[len(root.Path)+len("/"):])
-		result = append(result, &pkg{
+		p := &pkg{
 			importPathShort: VendorlessPath(importpath),
 			dir:             dir,
-		})
+		}
+		if loadNames {
+			var err error
+			p.packageName, err = packageDirToName(dir)
+			if err != nil {
+				return // Typically an unimportable package like main.
+			}
+		}
+		result = append(result, p)
 	}
-	gopathwalk.Walk(gopathwalk.SrcDirsRoots(r.env.buildContext()), add, gopathwalk.Options{Debug: r.env.Debug, ModulesEnabled: false})
+	roots := filterRoots(gopathwalk.SrcDirsRoots(r.env.buildContext()), exclude)
+	gopathwalk.Walk(roots, add, gopathwalk.Options{Debug: r.env.Debug, ModulesEnabled: false})
 	return result, nil
 }
 
+func filterRoots(roots []gopathwalk.Root, exclude []gopathwalk.RootType) []gopathwalk.Root {
+	var result []gopathwalk.Root
+outer:
+	for _, root := range roots {
+		for _, i := range exclude {
+			if i == root.Type {
+				continue outer
+			}
+		}
+		result = append(result, root)
+	}
+	return result
+}
+
 func (r *gopathResolver) loadExports(ctx context.Context, expectPackage string, pkg *pkg) (map[string]bool, error) {
 	return loadExportsFromFiles(ctx, r.env, expectPackage, pkg.dir)
 }

internal/imports/fix_test.go

@@ -9,6 +9,7 @@ import (
 	"fmt"
 	"go/build"
 	"path/filepath"
+	"reflect"
 	"runtime"
 	"strings"
 	"sync"
@@ -2513,33 +2514,48 @@ var _ = bytes.Buffer{}
 
 // TestStdLibGetCandidates tests that get packages finds std library packages
 // with correct priorities.
-func TestStdLibGetCandidates(t *testing.T) {
-	want := []struct {
-		wantName string
-		wantPkg  string
-	}{
+func TestGetCandidates(t *testing.T) {
+	type res struct {
+		name, path string
+	}
+	want := []res{
+		{"bar", "bar.com/bar"},
 		{"bytes", "bytes"},
 		{"rand", "crypto/rand"},
+		{"foo", "foo.com/foo"},
 		{"rand", "math/rand"},
 		{"http", "net/http"},
 	}
 
-	got, err := GetAllCandidates("", nil)
-	if err != nil {
-		t.Fatalf("Process() = %v", err)
-	}
-	wantIdx := 0
-	for _, fix := range got {
-		if wantIdx >= len(want) {
-			break
+	testConfig{
+		modules: []packagestest.Module{
+			{
+				Name:  "foo.com",
+				Files: fm{"foo/foo.go": "package foo\n"},
+			},
+			{
+				Name:  "bar.com",
+				Files: fm{"bar/bar.go": "package bar\n"},
+			},
+		},
+		goPackagesIncompatible: true, // getAllCandidates doesn't support the go/packages resolver.
+	}.test(t, func(t *goimportTest) {
+		candidates, err := getAllCandidates("x.go", t.env)
+		if err != nil {
+			t.Fatalf("GetAllCandidates() = %v", err)
 		}
-		if want[wantIdx].wantName == fix.IdentName && want[wantIdx].wantPkg == fix.StmtInfo.ImportPath && "" == fix.StmtInfo.Name {
-			wantIdx++
+		var got []res
+		for _, c := range candidates {
+			for _, w := range want {
+				if c.StmtInfo.ImportPath == w.path {
+					got = append(got, res{c.IdentName, c.StmtInfo.ImportPath})
+				}
+			}
 		}
-	}
-	if wantIdx < len(want) {
-		t.Errorf("expected to find candidate with path: %q, name: %q next in ordered scan of results`", want[wantIdx].wantPkg, want[wantIdx].wantName)
-	}
+		if !reflect.DeepEqual(want, got) {
+			t.Errorf("wanted stdlib results in order %v, got %v", want, got)
+		}
+	})
 }
 
 // Tests #34895: process should not panic on concurrent calls.

internal/imports/mod.go

@@ -119,14 +119,19 @@ func (r *ModuleResolver) findPackage(importPath string) (*ModuleJSON, string) {
 		}
 
 		if info, ok := r.moduleCacheInfo.Load(pkgDir); ok {
-			if packageScanned, err := info.reachedStatus(directoryScanned); packageScanned {
+			// This is slightly wrong: a directory doesn't have to have an
+			// importable package to count as a package for package-to-module
+			// resolution. package main or _test files should count but
+			// don't.
+			if loaded, err := info.reachedStatus(nameLoaded); loaded {
 				if err != nil {
-					// There was some error with scanning this directory.
-					// It does not contain a valid package.
-					continue
+					continue // No package in this dir.
 				}
 				return m, pkgDir
 			}
+			if scanned, err := info.reachedStatus(directoryScanned); scanned && err != nil {
+				continue // Dir is unreadable, etc.
+			}
 		}
 
 		pkgFiles, err := ioutil.ReadDir(pkgDir)
@@ -237,7 +242,7 @@ func (r *ModuleResolver) loadPackageNames(importPaths []string, srcDir string) (
 	return names, nil
 }
 
-func (r *ModuleResolver) scan(_ references) ([]*pkg, error) {
+func (r *ModuleResolver) scan(_ references, loadNames bool, exclude []gopathwalk.RootType) ([]*pkg, error) {
 	if err := r.init(); err != nil {
 		return nil, err
 	}
@@ -261,6 +266,8 @@ func (r *ModuleResolver) scan(_ references) ([]*pkg, error) {
 		}
 	}
 
+	roots = filterRoots(roots, exclude)
+
 	var result []*pkg
 	dupCheck := make(map[string]bool)
 	var mu sync.Mutex
@@ -311,9 +318,16 @@ func (r *ModuleResolver) scan(_ references) ([]*pkg, error) {
 			return
 		}
 
+		if loadNames {
+			info.packageName, info.err = packageDirToName(dir)
+			if info.err != nil {
+				return
+			}
+		}
+
 		// The rest of this function canonicalizes the packages using the results
 		// of initializing the resolver from 'go list -m'.
-		res, err := r.canonicalize(root.Type, info.nonCanonicalImportPath, info.dir, info.needsReplace)
+		res, err := r.canonicalize(root.Type, info)
 		if err != nil {
 			return
 		}
@@ -335,7 +349,23 @@ func (r *ModuleResolver) scan(_ references) ([]*pkg, error) {
 			continue
 		}
 
-		res, err := r.canonicalize(gopathwalk.RootModuleCache, info.nonCanonicalImportPath, info.dir, info.needsReplace)
+		// If we want package names, make sure the cache has them.
+		if loadNames {
+			loaded, err := info.reachedStatus(nameLoaded)
+			if loaded && err != nil {
+				continue
+			}
+			if !loaded {
+				info.packageName, info.err = packageDirToName(info.dir)
+				info.status = nameLoaded
+				r.moduleCacheInfo.Store(info.dir, info)
+				if info.err != nil {
+					continue
+				}
+			}
+		}
+
+		res, err := r.canonicalize(gopathwalk.RootModuleCache, info)
 		if err != nil {
 			continue
 		}
@@ -347,38 +377,42 @@ func (r *ModuleResolver) scan(_ references) ([]*pkg, error) {
 
 // canonicalize gets the result of canonicalizing the packages using the results
 // of initializing the resolver from 'go list -m'.
-func (r *ModuleResolver) canonicalize(rootType gopathwalk.RootType, importPath, dir string, needsReplace bool) (res *pkg, err error) {
+func (r *ModuleResolver) canonicalize(rootType gopathwalk.RootType, info directoryPackageInfo) (*pkg, error) {
 	// Packages in GOROOT are already canonical, regardless of the std/cmd modules.
 	if rootType == gopathwalk.RootGOROOT {
 		return &pkg{
-			importPathShort: importPath,
-			dir:             dir,
+			importPathShort: info.nonCanonicalImportPath,
+			dir:             info.dir,
+			packageName:     path.Base(info.nonCanonicalImportPath),
 		}, nil
 	}
 
+	importPath := info.nonCanonicalImportPath
 	// Check if the directory is underneath a module that's in scope.
-	if mod := r.findModuleByDir(dir); mod != nil {
+	if mod := r.findModuleByDir(info.dir); mod != nil {
 		// It is. If dir is the target of a replace directive,
 		// our guessed import path is wrong. Use the real one.
-		if mod.Dir == dir {
+		if mod.Dir == info.dir {
 			importPath = mod.Path
 		} else {
-			dirInMod := dir[len(mod.Dir)+len("/"):]
+			dirInMod := info.dir[len(mod.Dir)+len("/"):]
 			importPath = path.Join(mod.Path, filepath.ToSlash(dirInMod))
 		}
-	} else if needsReplace {
-		return nil, fmt.Errorf("needed this package to be in scope: %s", dir)
+	} else if info.needsReplace {
+		return nil, fmt.Errorf("package in %q is not valid without a replace statement", info.dir)
 	}
 
+	res := &pkg{
+		importPathShort: VendorlessPath(importPath),
+		dir:             info.dir,
+		packageName:     info.packageName, // may not be populated if the caller didn't ask for it
+	}
 	// We may have discovered a package that has a different version
 	// in scope already. Canonicalize to that one if possible.
 	if _, canonicalDir := r.findPackage(importPath); canonicalDir != "" {
-		dir = canonicalDir
+		res.dir = canonicalDir
 	}
-	return &pkg{
-		importPathShort: VendorlessPath(importPath),
-		dir:             dir,
-	}, nil
+	return res, nil
 }
 
 func (r *ModuleResolver) loadExports(ctx context.Context, expectPackage string, pkg *pkg) (map[string]bool, error) {

internal/imports/mod_cache.go

@@ -30,6 +30,7 @@ type directoryPackageStatus int
 const (
 	_ directoryPackageStatus = iota
 	directoryScanned
+	nameLoaded
 )
 
 type directoryPackageInfo struct {
@@ -38,7 +39,7 @@ type directoryPackageInfo struct {
 	// err is non-nil when there was an error trying to reach status.
 	err error
 
-	// Set when status > directoryScanned.
+	// Set when status >= directoryScanned.
 
 	// dir is the absolute directory of this package.
 	dir string
@@ -49,6 +50,10 @@ type directoryPackageInfo struct {
 	// the modules declared path, making it impossible to import without a
 	// replace directive.
 	needsReplace bool
+
+	// Set when status >= nameLoaded.
+
+	packageName string // the package name, as declared in the source.
 }
 
 // reachedStatus returns true when info has a status at least target and any error associated with
@@ -90,13 +95,8 @@ type moduleCacheInfo struct {
 func (d *moduleCacheInfo) Store(dir string, info directoryPackageInfo) {
 	d.mu.Lock()
 	defer d.mu.Unlock()
-	d.modCacheDirInfo[dir] = &directoryPackageInfo{
-		status:                 info.status,
-		err:                    info.err,
-		dir:                    info.dir,
-		nonCanonicalImportPath: info.nonCanonicalImportPath,
-		needsReplace:           info.needsReplace,
-	}
+	stored := info // defensive copy
+	d.modCacheDirInfo[dir] = &stored
 }
 
 // Load returns a copy of the directoryPackageInfo for absolute directory dir.

internal/imports/mod_test.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"go/build"
 	"io/ioutil"
+	"log"
 	"os"
 	"path/filepath"
 	"regexp"
@@ -14,6 +15,7 @@ import (
 	"sync"
 	"testing"
 
+	"golang.org/x/tools/internal/gopathwalk"
 	"golang.org/x/tools/internal/module"
 	"golang.org/x/tools/internal/testenv"
 	"golang.org/x/tools/internal/txtar"
@@ -86,7 +88,7 @@ package z
 
 	mt.assertFound("y", "y")
 
-	scan, err := mt.resolver.scan(nil)
+	scan, err := mt.resolver.scan(nil, false, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -164,8 +166,8 @@ import _ "rsc.io/quote"
 `, "")
 	defer mt.cleanup()
 
-	mt.assertScanFinds("rsc.io/quote/buggy", "buggy")
-	mt.assertScanFinds("rsc.io/quote/buggy", "buggy")
+	mt.assertScanFinds("rsc.io/quote", "quote")
+	mt.assertScanFinds("rsc.io/quote", "quote")
 }
 
 // Tests that scanning the module cache > 1 after changing a package in module cache to make it unimportable
@@ -215,7 +217,7 @@ import _ "rsc.io/quote"
 }
 
 // Tests that -mod=vendor sort of works. Adapted from mod_getmode_vendor.txt.
-func TestModeGetmodeVendor(t *testing.T) {
+func TestModGetmodeVendor(t *testing.T) {
 	t.Skip("'go list -m -mod=vendor' currently not allowed: see golang.org/issue/34826")
 	mt := setup(t, `
 -- go.mod --
@@ -540,7 +542,7 @@ func (t *modTest) assertFound(importPath, pkgName string) (string, *pkg) {
 
 func (t *modTest) assertScanFinds(importPath, pkgName string) *pkg {
 	t.Helper()
-	scan, err := t.resolver.scan(nil)
+	scan, err := t.resolver.scan(nil, true, nil)
 	if err != nil {
 		t.Errorf("scan failed: %v", err)
 	}
@@ -795,5 +797,20 @@ func TestInvalidModCache(t *testing.T) {
 		WorkingDir:  dir,
 	}
 	resolver := &ModuleResolver{env: env}
-	resolver.scan(nil)
+	resolver.scan(nil, true, nil)
+}
+
+func BenchmarkScanModCache(b *testing.B) {
+	env := &ProcessEnv{
+		Debug:  true,
+		GOPATH: build.Default.GOPATH,
+		GOROOT: build.Default.GOROOT,
+		Logf:   log.Printf,
+	}
+	exclude := []gopathwalk.RootType{gopathwalk.RootCurrentModule, gopathwalk.RootGOROOT, gopathwalk.RootOther}
+	env.GetResolver().scan(nil, true, exclude)
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		env.GetResolver().scan(nil, true, exclude)
+	}
 }

internal/lsp/cache/view.go

@@ -173,6 +173,7 @@ func (v *view) buildProcessEnv(ctx context.Context) (*imports.ProcessEnv, error)
 		Logf: func(format string, args ...interface{}) {
 			log.Print(ctx, fmt.Sprintf(format, args...))
 		},
+		Debug: true,
 	}
 	for _, kv := range cfg.Env {
 		split := strings.Split(kv, "=")

internal/lsp/completion_test.go

@@ -51,7 +51,7 @@ func (r *runner) UnimportedCompletion(t *testing.T, src span.Span, test tests.Co
 		got = tests.FilterBuiltins(got)
 	}
 	want := expected(t, test, items)
-	if diff := tests.DiffCompletionItems(want, got); diff != "" {
+	if diff := tests.CheckCompletionOrder(want, got); diff != "" {
 		t.Errorf("%s: %s", src, diff)
 	}
 }

internal/lsp/source/source_test.go

@@ -136,7 +136,7 @@ func (r *runner) UnimportedCompletion(t *testing.T, src span.Span, test tests.Co
 	if !strings.Contains(string(src.URI()), "builtins") {
 		got = tests.FilterBuiltins(got)
 	}
-	if diff := tests.DiffCompletionItems(want, got); diff != "" {
+	if diff := tests.CheckCompletionOrder(want, got); diff != "" {
 		t.Errorf("%s: %s", src, diff)
 	}
 }

internal/lsp/testdata/unimported/mkunimported.go

@@ -1,112 +0,0 @@
-// +build ignore
-
-// mkunimported generates the unimported.go file, containing the Go standard
-// library packages.
-// The completion items from the std library are computed in the same way as in the
-// golang.org/x/tools/internal/imports.
-package main
-
-import (
-	"bufio"
-	"bytes"
-	"fmt"
-	"go/format"
-	"io"
-	"io/ioutil"
-	"log"
-	"os"
-	pkgpath "path"
-	"path/filepath"
-	"regexp"
-	"runtime"
-	"sort"
-	"strings"
-)
-
-func mustOpen(name string) io.Reader {
-	f, err := os.Open(name)
-	if err != nil {
-		log.Fatal(err)
-	}
-	return f
-}
-
-func api(base string) string {
-	return filepath.Join(runtime.GOROOT(), "api", base)
-}
-
-var sym = regexp.MustCompile(`^pkg (\S+).*?, (?:var|func|type|const) ([A-Z]\w*)`)
-
-var unsafeSyms = map[string]bool{"Alignof": true, "ArbitraryType": true, "Offsetof": true, "Pointer": true, "Sizeof": true}
-
-func main() {
-	var buf bytes.Buffer
-	outf := func(format string, args ...interface{}) {
-		fmt.Fprintf(&buf, format, args...)
-	}
-	outf("// Code generated by mkstdlib.go. DO NOT EDIT.\n\n")
-	outf("package unimported\n")
-	outf("func _() {\n")
-	f := io.MultiReader(
-		mustOpen(api("go1.txt")),
-		mustOpen(api("go1.1.txt")),
-		mustOpen(api("go1.2.txt")),
-		mustOpen(api("go1.3.txt")),
-		mustOpen(api("go1.4.txt")),
-		mustOpen(api("go1.5.txt")),
-		mustOpen(api("go1.6.txt")),
-		mustOpen(api("go1.7.txt")),
-		mustOpen(api("go1.8.txt")),
-		mustOpen(api("go1.9.txt")),
-		mustOpen(api("go1.10.txt")),
-		mustOpen(api("go1.11.txt")),
-		mustOpen(api("go1.12.txt")),
-		mustOpen(api("go1.13.txt")),
-	)
-	sc := bufio.NewScanner(f)
-
-	pkgs := map[string]bool{
-		"unsafe":     true,
-		"syscall/js": true,
-	}
-	paths := []string{"unsafe", "syscall/js"}
-	for sc.Scan() {
-		l := sc.Text()
-		if m := sym.FindStringSubmatch(l); m != nil {
-			path, _ := m[1], m[2]
-
-			if _, ok := pkgs[path]; !ok {
-				pkgs[path] = true
-				paths = append(paths, path)
-			}
-		}
-	}
-	if err := sc.Err(); err != nil {
-		log.Fatal(err)
-	}
-	sort.Strings(paths)
-
-	var markers []string
-	for _, path := range paths {
-		marker := strings.ReplaceAll(path, "/", "slash")
-		markers = append(markers, marker)
-	}
-	outf("	//@unimported(\"\", %s)\n", strings.Join(markers, ", "))
-	outf("}\n")
-	outf("// Create markers for unimported std lib packages. Only for use by this test.\n")
-
-	for i, path := range paths {
-		name := pkgpath.Base(path)
-		marker := markers[i]
-		outf("/* %s *///@item(%s, \"%s\", \"\\\"%s\\\"\", \"package\")\n", name, marker, name, path)
-	}
-
-	fmtbuf, err := format.Source(buf.Bytes())
-	if err != nil {
-		log.Fatal(err)
-	}
-	err = ioutil.WriteFile("unimported.go", fmtbuf, 0666)
-	if err != nil {
-		log.Fatal(err)
-	}
-}

internal/lsp/testdata/unimported/unimported.go

@@ -1,152 +1,13 @@
-// Code generated by mkstdlib.go. DO NOT EDIT.
-
 package unimported
 
 func _() {
-	//@unimported("", archiveslashtar, archiveslashzip, bufio, bytes, compressslashbzip2, compressslashflate, compressslashgzip, compressslashlzw, compressslashzlib, containerslashheap, containerslashlist, containerslashring, context, crypto, cryptoslashaes, cryptoslashcipher, cryptoslashdes, cryptoslashdsa, cryptoslashecdsa, cryptoslashed25519, cryptoslashelliptic, cryptoslashhmac, cryptoslashmd5, cryptoslashrand, cryptoslashrc4, cryptoslashrsa, cryptoslashsha1, cryptoslashsha256, cryptoslashsha512, cryptoslashsubtle, cryptoslashtls, cryptoslashx509, cryptoslashx509slashpkix, databaseslashsql, databaseslashsqlslashdriver, debugslashdwarf, debugslashelf, debugslashgosym, debugslashmacho, debugslashpe, debugslashplan9obj, encoding, encodingslashascii85, encodingslashasn1, encodingslashbase32, encodingslashbase64, encodingslashbinary, encodingslashcsv, encodingslashgob, encodingslashhex, encodingslashjson, encodingslashpem, encodingslashxml, errors, expvar, flag, fmt, goslashast, goslashbuild, goslashconstant, goslashdoc, goslashformat, goslashimporter, goslashparser, goslashprinter, goslashscanner, goslashtoken, goslashtypes, hash, hashslashadler32, hashslashcrc32, hashslashcrc64, hashslashfnv, html, htmlslashtemplate, image, imageslashcolor, imageslashcolorslashpalette, imageslashdraw, imageslashgif, imageslashjpeg, imageslashpng, indexslashsuffixarray, io, ioslashioutil, log, logslashsyslog, math, mathslashbig, mathslashbits, mathslashcmplx, mathslashrand, mime, mimeslashmultipart, mimeslashquotedprintable, net, netslashhttp, netslashhttpslashcgi, netslashhttpslashcookiejar, netslashhttpslashfcgi, netslashhttpslashhttptest, netslashhttpslashhttptrace, netslashhttpslashhttputil, netslashhttpslashpprof, netslashmail, netslashrpc, netslashrpcslashjsonrpc, netslashsmtp, netslashtextproto, netslashurl, os, osslashexec, osslashsignal, osslashuser, path, pathslashfilepath, plugin, reflect, regexp, regexpslashsyntax, runtime, runtimeslashdebug, runtimeslashpprof, runtimeslashtrace, sort, strconv, strings, sync, syncslashatomic, syscall, syscallslashjs, testing, testingslashiotest, testingslashquick, textslashscanner, textslashtabwriter, textslashtemplate, textslashtemplateslashparse, time, unicode, unicodeslashutf16, unicodeslashutf8, unsafe)
+	//@unimported("", bytes, context, cryptoslashrand, externalpackage, time, unsafe)
 }
 
 // Create markers for unimported std lib packages. Only for use by this test.
-/* tar */ //@item(archiveslashtar, "tar", "\"archive/tar\"", "package")
-/* zip */ //@item(archiveslashzip, "zip", "\"archive/zip\"", "package")
-/* bufio */ //@item(bufio, "bufio", "\"bufio\"", "package")
 /* bytes */ //@item(bytes, "bytes", "\"bytes\"", "package")
-/* bzip2 */ //@item(compressslashbzip2, "bzip2", "\"compress/bzip2\"", "package")
-/* flate */ //@item(compressslashflate, "flate", "\"compress/flate\"", "package")
-/* gzip */ //@item(compressslashgzip, "gzip", "\"compress/gzip\"", "package")
-/* lzw */ //@item(compressslashlzw, "lzw", "\"compress/lzw\"", "package")
-/* zlib */ //@item(compressslashzlib, "zlib", "\"compress/zlib\"", "package")
-/* heap */ //@item(containerslashheap, "heap", "\"container/heap\"", "package")
-/* list */ //@item(containerslashlist, "list", "\"container/list\"", "package")
-/* ring */ //@item(containerslashring, "ring", "\"container/ring\"", "package")
 /* context */ //@item(context, "context", "\"context\"", "package")
-/* crypto */ //@item(crypto, "crypto", "\"crypto\"", "package")
-/* aes */ //@item(cryptoslashaes, "aes", "\"crypto/aes\"", "package")
-/* cipher */ //@item(cryptoslashcipher, "cipher", "\"crypto/cipher\"", "package")
-/* des */ //@item(cryptoslashdes, "des", "\"crypto/des\"", "package")
-/* dsa */ //@item(cryptoslashdsa, "dsa", "\"crypto/dsa\"", "package")
-/* ecdsa */ //@item(cryptoslashecdsa, "ecdsa", "\"crypto/ecdsa\"", "package")
-/* ed25519 */ //@item(cryptoslashed25519, "ed25519", "\"crypto/ed25519\"", "package")
-/* elliptic */ //@item(cryptoslashelliptic, "elliptic", "\"crypto/elliptic\"", "package")
-/* hmac */ //@item(cryptoslashhmac, "hmac", "\"crypto/hmac\"", "package")
-/* md5 */ //@item(cryptoslashmd5, "md5", "\"crypto/md5\"", "package")
 /* rand */ //@item(cryptoslashrand, "rand", "\"crypto/rand\"", "package")
-/* rc4 */ //@item(cryptoslashrc4, "rc4", "\"crypto/rc4\"", "package")
-/* rsa */ //@item(cryptoslashrsa, "rsa", "\"crypto/rsa\"", "package")
-/* sha1 */ //@item(cryptoslashsha1, "sha1", "\"crypto/sha1\"", "package")
-/* sha256 */ //@item(cryptoslashsha256, "sha256", "\"crypto/sha256\"", "package")
-/* sha512 */ //@item(cryptoslashsha512, "sha512", "\"crypto/sha512\"", "package")
-/* subtle */ //@item(cryptoslashsubtle, "subtle", "\"crypto/subtle\"", "package")
-/* tls */ //@item(cryptoslashtls, "tls", "\"crypto/tls\"", "package")
-/* x509 */ //@item(cryptoslashx509, "x509", "\"crypto/x509\"", "package")
-/* pkix */ //@item(cryptoslashx509slashpkix, "pkix", "\"crypto/x509/pkix\"", "package")
-/* sql */ //@item(databaseslashsql, "sql", "\"database/sql\"", "package")
-/* driver */ //@item(databaseslashsqlslashdriver, "driver", "\"database/sql/driver\"", "package")
-/* dwarf */ //@item(debugslashdwarf, "dwarf", "\"debug/dwarf\"", "package")
-/* elf */ //@item(debugslashelf, "elf", "\"debug/elf\"", "package")
-/* gosym */ //@item(debugslashgosym, "gosym", "\"debug/gosym\"", "package")
-/* macho */ //@item(debugslashmacho, "macho", "\"debug/macho\"", "package")
-/* pe */ //@item(debugslashpe, "pe", "\"debug/pe\"", "package")
-/* plan9obj */ //@item(debugslashplan9obj, "plan9obj", "\"debug/plan9obj\"", "package")
-/* encoding */ //@item(encoding, "encoding", "\"encoding\"", "package")
-/* ascii85 */ //@item(encodingslashascii85, "ascii85", "\"encoding/ascii85\"", "package")
-/* asn1 */ //@item(encodingslashasn1, "asn1", "\"encoding/asn1\"", "package")
-/* base32 */ //@item(encodingslashbase32, "base32", "\"encoding/base32\"", "package")
-/* base64 */ //@item(encodingslashbase64, "base64", "\"encoding/base64\"", "package")
-/* binary */ //@item(encodingslashbinary, "binary", "\"encoding/binary\"", "package")
-/* csv */ //@item(encodingslashcsv, "csv", "\"encoding/csv\"", "package")
-/* gob */ //@item(encodingslashgob, "gob", "\"encoding/gob\"", "package")
-/* hex */ //@item(encodingslashhex, "hex", "\"encoding/hex\"", "package")
-/* json */ //@item(encodingslashjson, "json", "\"encoding/json\"", "package")
-/* pem */ //@item(encodingslashpem, "pem", "\"encoding/pem\"", "package")
-/* xml */ //@item(encodingslashxml, "xml", "\"encoding/xml\"", "package")
-/* errors */ //@item(errors, "errors", "\"errors\"", "package")
-/* expvar */ //@item(expvar, "expvar", "\"expvar\"", "package")
-/* flag */ //@item(flag, "flag", "\"flag\"", "package")
-/* fmt */ //@item(fmt, "fmt", "\"fmt\"", "package")
-/* ast */ //@item(goslashast, "ast", "\"go/ast\"", "package")
-/* build */ //@item(goslashbuild, "build", "\"go/build\"", "package")
-/* constant */ //@item(goslashconstant, "constant", "\"go/constant\"", "package")
-/* doc */ //@item(goslashdoc, "doc", "\"go/doc\"", "package")
-/* format */ //@item(goslashformat, "format", "\"go/format\"", "package")
-/* importer */ //@item(goslashimporter, "importer", "\"go/importer\"", "package")
-/* parser */ //@item(goslashparser, "parser", "\"go/parser\"", "package")
-/* printer */ //@item(goslashprinter, "printer", "\"go/printer\"", "package")
-/* scanner */ //@item(goslashscanner, "scanner", "\"go/scanner\"", "package")
-/* token */ //@item(goslashtoken, "token", "\"go/token\"", "package")
-/* types */ //@item(goslashtypes, "types", "\"go/types\"", "package")
-/* hash */ //@item(hash, "hash", "\"hash\"", "package")
-/* adler32 */ //@item(hashslashadler32, "adler32", "\"hash/adler32\"", "package")
-/* crc32 */ //@item(hashslashcrc32, "crc32", "\"hash/crc32\"", "package")
-/* crc64 */ //@item(hashslashcrc64, "crc64", "\"hash/crc64\"", "package")
-/* fnv */ //@item(hashslashfnv, "fnv", "\"hash/fnv\"", "package")
-/* html */ //@item(html, "html", "\"html\"", "package")
-/* template */ //@item(htmlslashtemplate, "template", "\"html/template\"", "package")
-/* image */ //@item(image, "image", "\"image\"", "package")
-/* color */ //@item(imageslashcolor, "color", "\"image/color\"", "package")
-/* palette */ //@item(imageslashcolorslashpalette, "palette", "\"image/color/palette\"", "package")
-/* draw */ //@item(imageslashdraw, "draw", "\"image/draw\"", "package")
-/* gif */ //@item(imageslashgif, "gif", "\"image/gif\"", "package")
-/* jpeg */ //@item(imageslashjpeg, "jpeg", "\"image/jpeg\"", "package")
-/* png */ //@item(imageslashpng, "png", "\"image/png\"", "package")
-/* suffixarray */ //@item(indexslashsuffixarray, "suffixarray", "\"index/suffixarray\"", "package")
-/* io */ //@item(io, "io", "\"io\"", "package")
-/* ioutil */ //@item(ioslashioutil, "ioutil", "\"io/ioutil\"", "package")
-/* log */ //@item(log, "log", "\"log\"", "package")
-/* syslog */ //@item(logslashsyslog, "syslog", "\"log/syslog\"", "package")
-/* math */ //@item(math, "math", "\"math\"", "package")
-/* big */ //@item(mathslashbig, "big", "\"math/big\"", "package")
-/* bits */ //@item(mathslashbits, "bits", "\"math/bits\"", "package")
-/* cmplx */ //@item(mathslashcmplx, "cmplx", "\"math/cmplx\"", "package")
-/* rand */ //@item(mathslashrand, "rand", "\"math/rand\"", "package")
-/* mime */ //@item(mime, "mime", "\"mime\"", "package")
-/* multipart */ //@item(mimeslashmultipart, "multipart", "\"mime/multipart\"", "package")
-/* quotedprintable */ //@item(mimeslashquotedprintable, "quotedprintable", "\"mime/quotedprintable\"", "package")
-/* net */ //@item(net, "net", "\"net\"", "package")
-/* http */ //@item(netslashhttp, "http", "\"net/http\"", "package")
-/* cgi */ //@item(netslashhttpslashcgi, "cgi", "\"net/http/cgi\"", "package")
-/* cookiejar */ //@item(netslashhttpslashcookiejar, "cookiejar", "\"net/http/cookiejar\"", "package")
-/* fcgi */ //@item(netslashhttpslashfcgi, "fcgi", "\"net/http/fcgi\"", "package")
-/* httptest */ //@item(netslashhttpslashhttptest, "httptest", "\"net/http/httptest\"", "package")
-/* httptrace */ //@item(netslashhttpslashhttptrace, "httptrace", "\"net/http/httptrace\"", "package")
-/* httputil */ //@item(netslashhttpslashhttputil, "httputil", "\"net/http/httputil\"", "package")
-/* pprof */ //@item(netslashhttpslashpprof, "pprof", "\"net/http/pprof\"", "package")
-/* mail */ //@item(netslashmail, "mail", "\"net/mail\"", "package")
-/* rpc */ //@item(netslashrpc, "rpc", "\"net/rpc\"", "package")
-/* jsonrpc */ //@item(netslashrpcslashjsonrpc, "jsonrpc", "\"net/rpc/jsonrpc\"", "package")
-/* smtp */ //@item(netslashsmtp, "smtp", "\"net/smtp\"", "package")
-/* textproto */ //@item(netslashtextproto, "textproto", "\"net/textproto\"", "package")
-/* url */ //@item(netslashurl, "url", "\"net/url\"", "package")
-/* os */ //@item(os, "os", "\"os\"", "package")
-/* exec */ //@item(osslashexec, "exec", "\"os/exec\"", "package")
-/* signal */ //@item(osslashsignal, "signal", "\"os/signal\"", "package")
-/* user */ //@item(osslashuser, "user", "\"os/user\"", "package")
-/* path */ //@item(path, "path", "\"path\"", "package")
-/* filepath */ //@item(pathslashfilepath, "filepath", "\"path/filepath\"", "package")
-/* plugin */ //@item(plugin, "plugin", "\"plugin\"", "package")
-/* reflect */ //@item(reflect, "reflect", "\"reflect\"", "package")
-/* regexp */ //@item(regexp, "regexp", "\"regexp\"", "package")
-/* syntax */ //@item(regexpslashsyntax, "syntax", "\"regexp/syntax\"", "package")
-/* runtime */ //@item(runtime, "runtime", "\"runtime\"", "package")
-/* debug */ //@item(runtimeslashdebug, "debug", "\"runtime/debug\"", "package")
-/* pprof */ //@item(runtimeslashpprof, "pprof", "\"runtime/pprof\"", "package")
-/* trace */ //@item(runtimeslashtrace, "trace", "\"runtime/trace\"", "package")
-/* sort */ //@item(sort, "sort", "\"sort\"", "package")
-/* strconv */ //@item(strconv, "strconv", "\"strconv\"", "package")
-/* strings */ //@item(strings, "strings", "\"strings\"", "package")
-/* sync */ //@item(sync, "sync", "\"sync\"", "package")
-/* atomic */ //@item(syncslashatomic, "atomic", "\"sync/atomic\"", "package")
-/* syscall */ //@item(syscall, "syscall", "\"syscall\"", "package")
-/* js */ //@item(syscallslashjs, "js", "\"syscall/js\"", "package")
-/* testing */ //@item(testing, "testing", "\"testing\"", "package")
-/* iotest */ //@item(testingslashiotest, "iotest", "\"testing/iotest\"", "package")
-/* quick */ //@item(testingslashquick, "quick", "\"testing/quick\"", "package")
-/* scanner */ //@item(textslashscanner, "scanner", "\"text/scanner\"", "package")
-/* tabwriter */ //@item(textslashtabwriter, "tabwriter", "\"text/tabwriter\"", "package")
-/* template */ //@item(textslashtemplate, "template", "\"text/template\"", "package")
-/* parse */ //@item(textslashtemplateslashparse, "parse", "\"text/template/parse\"", "package")
-/* time */ //@item(time, "time", "\"time\"", "package")
-/* unicode */ //@item(unicode, "unicode", "\"unicode\"", "package")
-/* utf16 */ //@item(unicodeslashutf16, "utf16", "\"unicode/utf16\"", "package")
-/* utf8 */ //@item(unicodeslashutf8, "utf8", "\"unicode/utf8\"", "package")
+/* pkg */ //@item(externalpackage, "pkg", "\"example.com/extramodule/pkg\"", "package" )
 /* unsafe */ //@item(unsafe, "unsafe", "\"unsafe\"", "package")
+/* time */ //@item(time, "time", "\"time\"", "package")

internal/lsp/tests/tests.go

@@ -251,6 +251,12 @@ func Load(t testing.TB, exporter packagestest.Exporter, dir string) *Data {
 			Files:   files,
 			Overlay: overlays,
 		},
+		{
+			Name: "example.com/extramodule",
+			Files: map[string]interface{}{
+				"pkg/x.go": "package pkg\n",
+			},
+		},
 	}
 	data.Exported = packagestest.Export(t, exporter, modules)
 	for fragment := range files {