internal/lsp: fix deadlocks loading lots of files at once

The first deadlock involved differing mutex acquisition order in two
code paths:

1. loadParseTypecheck() holds the "mcache" mutex then eventually
   acquires the "handleMu" file mutex.
2. (*goFile).invalidateContent() acquires the "handleMu" mutex first and
   then the "mcache" mutex.

Fix by changing the acquisition order in invalidateContent().

The second deadlock involved the file watcher. The two code paths
involved were:

1. (*goFile).GetPackages() holds the view mutex and eventually calls
   (*WatchMap).Watch, which acquires the watcher mutex.
2. (*session).openOverlay acquires the watcher mutex as it triggers a
   file's callbacks, and then the callback
   "(*goFile).invalidateContent" acquires the view mutex.

Fix by not holding the watcher mutex as we invoke the callbacks.

Fixes golang/go#32910

Change-Id: I9d060e0d80fd86a317a1d6c7aaa736a8ce10bd07
GitHub-Last-Rev: 04944fa0249c0e6f1022a415787e23abce21bc2e
GitHub-Pull-Request: golang/tools#129
Reviewed-on: https://go-review.googlesource.com/c/tools/+/184880
Reviewed-by: Rebecca Stambler <rstambler@golang.org>
Run-TryBot: Rebecca Stambler <rstambler@golang.org>
TryBot-Result: Gobot Gobot <gobot@golang.org>

internal/lsp/cache/view.go

@@ -225,15 +225,17 @@ func (v *view) SetContent(ctx context.Context, uri span.URI, content []byte) err
 // invalidateContent invalidates the content of a Go file,
 // including any position and type information that depends on it.
 func (f *goFile) invalidateContent(ctx context.Context) {
-	f.handleMu.Lock()
-	defer f.handleMu.Unlock()
-
+	// Mutex acquisition order here is important. It must match the order
+	// in loadParseTypecheck to avoid deadlocks.
 	f.view.mcache.mu.Lock()
 	defer f.view.mcache.mu.Unlock()
 
 	f.view.pcache.mu.Lock()
 	defer f.view.pcache.mu.Unlock()
 
+	f.handleMu.Lock()
+	defer f.handleMu.Unlock()
+
 	f.invalidateAST(ctx)
 	f.handle = nil
 }

internal/lsp/cache/watcher.go

@@ -48,9 +48,15 @@ func (w *WatchMap) Watch(key interface{}, callback func()) func() {
 }
 
 func (w *WatchMap) Notify(key interface{}) {
+	// Make a copy of the watcher callbacks so we don't need to hold
+	// the mutex during the callbacks (to avoid deadlocks).
 	w.mu.Lock()
-	defer w.mu.Unlock()
-	for _, entry := range w.watchers[key] {
+	entries := w.watchers[key]
+	entriesCopy := make([]watcher, len(entries))
+	copy(entriesCopy, entries)
+	w.mu.Unlock()
+
+	for _, entry := range entriesCopy {
 		entry.callback()
 	}
 }