runtime: lift systemstack part of gcAssistAlloc

This lifts the part of gcAssistAlloc that runs on the system stack to
its own function in preparation for letting assists perform root jobs
(notably stack scanning). This makes it easy to see that there are no
references to the user stack once we've entered gcAssistAlloc1, which
means it's safe to shrink the stack while in gcAssistAlloc1.

This does not yet make assists perform root jobs, so it's not actually
possible for the stack to shrink yet. That will happen in the next
commit.

The code in gcAssistAlloc1 is identical to the code that's currently
passed in a closure to systemstack with one exception. Currently, we
set the "completed" variable in the enclosing scope to indicate that
the assist completed the mark phase. This is exactly the sort of
cross-stack reference lifting this function is meant to prevent. We
replace this variable with setting gp.param to nil or non-nil to
indicate the completion status.

Updates #15361.

Change-Id: Iba7cfb758c781070a441aea86c0117b399a24dbd
Reviewed-on: https://go-review.googlesource.com/32431
TryBot-Result: Gobot Gobot <gobot@golang.org>
Run-TryBot: Austin Clements <austin@google.com>
Reviewed-by: Rick Hudson <rlh@golang.org>

src/runtime/mgcmark.go

@@ -391,7 +391,6 @@ func markrootSpans(gcw *gcWork, shard int) {
 // gp must be the calling user gorountine.
 //
 // This must be called with preemption enabled.
-//go:nowritebarrier
 func gcAssistAlloc(gp *g) {
 	// Don't assist in non-preemptible contexts. These are
 	// generally fragile and won't allow the assist to block.
@@ -442,8 +441,64 @@ retry:
 	}
 
 	// Perform assist work
-	completed := false
 	systemstack(func() {
+		gcAssistAlloc1(gp, scanWork)
+		// The user stack may have moved, so this can't touch
+		// anything on it until it returns from systemstack.
+	})
+
+	completed := gp.param != nil
+	gp.param = nil
+	if completed {
+		gcMarkDone()
+	}
+
+	if gp.gcAssistBytes < 0 {
+		// We were unable steal enough credit or perform
+		// enough work to pay off the assist debt. We need to
+		// do one of these before letting the mutator allocate
+		// more to prevent over-allocation.
+		//
+		// If this is because we were preempted, reschedule
+		// and try some more.
+		if gp.preempt {
+			Gosched()
+			goto retry
+		}
+
+		// Add this G to an assist queue and park. When the GC
+		// has more background credit, it will satisfy queued
+		// assists before flushing to the global credit pool.
+		//
+		// Note that this does *not* get woken up when more
+		// work is added to the work list. The theory is that
+		// there wasn't enough work to do anyway, so we might
+		// as well let background marking take care of the
+		// work that is available.
+		if !gcParkAssist() {
+			goto retry
+		}
+
+		// At this point either background GC has satisfied
+		// this G's assist debt, or the GC cycle is over.
+	}
+}
+
+// gcAssistAlloc1 is the part of gcAssistAlloc that runs on the system
+// stack. This is a separate function to make it easier to see that
+// we're not capturing anything from the user stack, since the user
+// stack may move while we're in this function.
+//
+// gcAssistAlloc1 indicates whether this assist completed the mark
+// phase by setting gp.param to non-nil. This can't be communicated on
+// the stack since it may move.
+//
+//go:systemstack
+func gcAssistAlloc1(gp *g, scanWork int64) {
+	// Clear the flag indicating that this assist completed the
+	// mark phase.
+	gp.param = nil
+
 	if atomic.Load(&gcBlackenEnabled) == 0 {
 		// The gcBlackenEnabled check in malloc races with the
 		// store that clears it but an atomic check in every malloc
@@ -495,9 +550,11 @@ retry:
 	}
 
 	if incnwait == work.nproc && !gcMarkWorkAvailable(nil) {
-			// This has reached a background completion
-			// point.
-			completed = true
+		// This has reached a background completion point. Set
+		// gp.param to a non-nil value to indicate this. It
+		// doesn't matter what we set it to (it just has to be
+		// a valid pointer).
+		gp.param = unsafe.Pointer(gp)
 	}
 	duration := nanotime() - startTime
 	_p_ := gp.m.p.ptr()
@@ -506,41 +563,6 @@ retry:
 		atomic.Xaddint64(&gcController.assistTime, _p_.gcAssistTime)
 		_p_.gcAssistTime = 0
 	}
-	})
-
-	if completed {
-		gcMarkDone()
-	}
-
-	if gp.gcAssistBytes < 0 {
-		// We were unable steal enough credit or perform
-		// enough work to pay off the assist debt. We need to
-		// do one of these before letting the mutator allocate
-		// more to prevent over-allocation.
-		//
-		// If this is because we were preempted, reschedule
-		// and try some more.
-		if gp.preempt {
-			Gosched()
-			goto retry
-		}
-
-		// Add this G to an assist queue and park. When the GC
-		// has more background credit, it will satisfy queued
-		// assists before flushing to the global credit pool.
-		//
-		// Note that this does *not* get woken up when more
-		// work is added to the work list. The theory is that
-		// there wasn't enough work to do anyway, so we might
-		// as well let background marking take care of the
-		// work that is available.
-		if !gcParkAssist() {
-			goto retry
-		}
-
-		// At this point either background GC has satisfied
-		// this G's assist debt, or the GC cycle is over.
-	}
 }
 
 // gcWakeAllAssists wakes all currently blocked assists. This is used