mirror of
https://github.com/golang/go
synced 2024-09-28 23:14:38 -06:00
runtime: fix timers.wakeTime inaccuracy race
timers.wakeTime, which is called concurrently by P's trying to decide how long they should sleep, can return inaccurate values while timers.adjust is running. (Before the refactoring, this was still true but the code did not have good names and was spread across more files, making the race harder to see.) The runtime thread sleeping code is complex enough that I am not confident that the inaccuracy can cause delayed timer wakeups, but I am also not confident that it can't, nor that it won't in the future. There are two parts to the fix: 1. A simple logic change in timers.adjust. 2. The introduction of t.maybeAdd to avoid having a t that is marked as belonging to a specific timers ts but not present in ts.heap. That was okay before when everything was racy but needs to be eliminated to make timers.adjust fully consistent. The cost of the change is an extra CAS-lock operation on a timer add (close to free since the CAS-lock was just unlocked) and a change in the static lock ranking to allow malloc while holding a timer lock. Change-Id: I1249e6e24ae9ef74a69837f453e15b513f0d75c0 Reviewed-on: https://go-review.googlesource.com/c/go/+/564977 Reviewed-by: Austin Clements <austin@google.com> LUCI-TryBot-Result: Go LUCI <golang-scoped@luci-project-accounts.iam.gserviceaccount.com> Auto-Submit: Russ Cox <rsc@golang.org>
This commit is contained in:
Russ Cox
Gopher Robot
parent
3ad5cd4741
commit
85bbb12141
@ -32,8 +32,8 @@ const (
|
||||
lockRankAllg
|
||||
lockRankAllp
|
||||
lockRankTimers
|
||||
lockRankNetpollInit
|
||||
lockRankTimer
|
||||
lockRankNetpollInit
|
||||
lockRankHchan
|
||||
lockRankNotifyList
|
||||
lockRankSudog
|
||||
@ -101,8 +101,8 @@ var lockNames = []string{
|
||||
lockRankAllg: "allg",
|
||||
lockRankAllp: "allp",
|
||||
lockRankTimers: "timers",
|
||||
lockRankNetpollInit: "netpollInit",
|
||||
lockRankTimer: "timer",
|
||||
lockRankNetpollInit: "netpollInit",
|
||||
lockRankHchan: "hchan",
|
||||
lockRankNotifyList: "notifyList",
|
||||
lockRankSudog: "sudog",
|
||||
@ -177,8 +177,8 @@ var lockPartialOrder [][]lockRank = [][]lockRank{
|
||||
lockRankAllg: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankTestR, lockRankCpuprof, lockRankPollDesc, lockRankWakeableSleep, lockRankAllocmR, lockRankExecR, lockRankSched},
|
||||
lockRankAllp: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankTestR, lockRankCpuprof, lockRankPollDesc, lockRankWakeableSleep, lockRankAllocmR, lockRankExecR, lockRankSched},
|
||||
lockRankTimers: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankTestR, lockRankCpuprof, lockRankPollDesc, lockRankWakeableSleep, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllp, lockRankTimers},
|
||||
lockRankNetpollInit: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankTestR, lockRankCpuprof, lockRankPollDesc, lockRankWakeableSleep, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllp, lockRankTimers},
|
||||
lockRankTimer: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankTestR, lockRankCpuprof, lockRankPollDesc, lockRankWakeableSleep, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllp, lockRankTimers},
|
||||
lockRankNetpollInit: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankTestR, lockRankCpuprof, lockRankPollDesc, lockRankWakeableSleep, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllp, lockRankTimers, lockRankTimer},
|
||||
lockRankHchan: {lockRankSysmon, lockRankScavenge, lockRankSweep, lockRankTestR, lockRankWakeableSleep, lockRankHchan},
|
||||
lockRankNotifyList: {},
|
||||
lockRankSudog: {lockRankSysmon, lockRankScavenge, lockRankSweep, lockRankTestR, lockRankWakeableSleep, lockRankHchan, lockRankNotifyList},
|
||||
@ -188,24 +188,24 @@ var lockPartialOrder [][]lockRank = [][]lockRank{
|
||||
lockRankUserArenaState: {},
|
||||
lockRankTraceBuf: {lockRankSysmon, lockRankScavenge},
|
||||
lockRankTraceStrings: {lockRankSysmon, lockRankScavenge, lockRankTraceBuf},
|
||||
lockRankFin: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankTestR, lockRankExecW, lockRankCpuprof, lockRankPollDesc, lockRankWakeableSleep, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankTimers, lockRankHchan, lockRankNotifyList, lockRankItab, lockRankReflectOffs, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings},
|
||||
lockRankSpanSetSpine: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankTestR, lockRankExecW, lockRankCpuprof, lockRankPollDesc, lockRankWakeableSleep, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankTimers, lockRankHchan, lockRankNotifyList, lockRankItab, lockRankReflectOffs, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings},
|
||||
lockRankMspanSpecial: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankTestR, lockRankExecW, lockRankCpuprof, lockRankPollDesc, lockRankWakeableSleep, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankTimers, lockRankHchan, lockRankNotifyList, lockRankItab, lockRankReflectOffs, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings},
|
||||
lockRankGcBitsArenas: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankTestR, lockRankExecW, lockRankCpuprof, lockRankPollDesc, lockRankWakeableSleep, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankTimers, lockRankHchan, lockRankNotifyList, lockRankItab, lockRankReflectOffs, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings, lockRankMspanSpecial},
|
||||
lockRankProfInsert: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankTestR, lockRankExecW, lockRankCpuprof, lockRankPollDesc, lockRankWakeableSleep, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankTimers, lockRankHchan, lockRankNotifyList, lockRankItab, lockRankReflectOffs, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings},
|
||||
lockRankProfBlock: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankTestR, lockRankExecW, lockRankCpuprof, lockRankPollDesc, lockRankWakeableSleep, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankTimers, lockRankHchan, lockRankNotifyList, lockRankItab, lockRankReflectOffs, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings},
|
||||
lockRankProfMemActive: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankTestR, lockRankExecW, lockRankCpuprof, lockRankPollDesc, lockRankWakeableSleep, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankTimers, lockRankHchan, lockRankNotifyList, lockRankItab, lockRankReflectOffs, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings},
|
||||
lockRankProfMemFuture: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankTestR, lockRankExecW, lockRankCpuprof, lockRankPollDesc, lockRankWakeableSleep, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankTimers, lockRankHchan, lockRankNotifyList, lockRankItab, lockRankReflectOffs, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings, lockRankProfMemActive},
|
||||
lockRankGscan: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankTestR, lockRankExecW, lockRankCpuprof, lockRankPollDesc, lockRankWakeableSleep, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankTimers, lockRankNetpollInit, lockRankHchan, lockRankNotifyList, lockRankRoot, lockRankItab, lockRankReflectOffs, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings, lockRankFin, lockRankSpanSetSpine, lockRankMspanSpecial, lockRankGcBitsArenas, lockRankProfInsert, lockRankProfBlock, lockRankProfMemActive, lockRankProfMemFuture},
|
||||
lockRankStackpool: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankTestR, lockRankExecW, lockRankCpuprof, lockRankPollDesc, lockRankWakeableSleep, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankTimers, lockRankNetpollInit, lockRankHchan, lockRankNotifyList, lockRankRoot, lockRankItab, lockRankReflectOffs, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings, lockRankFin, lockRankSpanSetSpine, lockRankMspanSpecial, lockRankGcBitsArenas, lockRankProfInsert, lockRankProfBlock, lockRankProfMemActive, lockRankProfMemFuture, lockRankGscan},
|
||||
lockRankStackLarge: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankTestR, lockRankExecW, lockRankCpuprof, lockRankPollDesc, lockRankWakeableSleep, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankTimers, lockRankNetpollInit, lockRankHchan, lockRankNotifyList, lockRankRoot, lockRankItab, lockRankReflectOffs, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings, lockRankFin, lockRankSpanSetSpine, lockRankMspanSpecial, lockRankGcBitsArenas, lockRankProfInsert, lockRankProfBlock, lockRankProfMemActive, lockRankProfMemFuture, lockRankGscan},
|
||||
lockRankHchanLeaf: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankTestR, lockRankExecW, lockRankCpuprof, lockRankPollDesc, lockRankWakeableSleep, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankTimers, lockRankNetpollInit, lockRankHchan, lockRankNotifyList, lockRankRoot, lockRankItab, lockRankReflectOffs, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings, lockRankFin, lockRankSpanSetSpine, lockRankMspanSpecial, lockRankGcBitsArenas, lockRankProfInsert, lockRankProfBlock, lockRankProfMemActive, lockRankProfMemFuture, lockRankGscan, lockRankHchanLeaf},
|
||||
lockRankWbufSpans: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankDefer, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankTestR, lockRankExecW, lockRankCpuprof, lockRankPollDesc, lockRankWakeableSleep, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankTimers, lockRankNetpollInit, lockRankTimer, lockRankHchan, lockRankNotifyList, lockRankSudog, lockRankRoot, lockRankItab, lockRankReflectOffs, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings, lockRankFin, lockRankSpanSetSpine, lockRankMspanSpecial, lockRankGcBitsArenas, lockRankProfInsert, lockRankProfBlock, lockRankProfMemActive, lockRankProfMemFuture, lockRankGscan},
|
||||
lockRankMheap: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankDefer, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankTestR, lockRankExecW, lockRankCpuprof, lockRankPollDesc, lockRankWakeableSleep, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankTimers, lockRankNetpollInit, lockRankTimer, lockRankHchan, lockRankNotifyList, lockRankSudog, lockRankRoot, lockRankItab, lockRankReflectOffs, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings, lockRankFin, lockRankSpanSetSpine, lockRankMspanSpecial, lockRankGcBitsArenas, lockRankProfInsert, lockRankProfBlock, lockRankProfMemActive, lockRankProfMemFuture, lockRankGscan, lockRankStackpool, lockRankStackLarge, lockRankWbufSpans},
|
||||
lockRankMheapSpecial: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankDefer, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankTestR, lockRankExecW, lockRankCpuprof, lockRankPollDesc, lockRankWakeableSleep, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankTimers, lockRankNetpollInit, lockRankTimer, lockRankHchan, lockRankNotifyList, lockRankSudog, lockRankRoot, lockRankItab, lockRankReflectOffs, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings, lockRankFin, lockRankSpanSetSpine, lockRankMspanSpecial, lockRankGcBitsArenas, lockRankProfInsert, lockRankProfBlock, lockRankProfMemActive, lockRankProfMemFuture, lockRankGscan, lockRankStackpool, lockRankStackLarge, lockRankWbufSpans, lockRankMheap},
|
||||
lockRankGlobalAlloc: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankDefer, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankTestR, lockRankExecW, lockRankCpuprof, lockRankPollDesc, lockRankWakeableSleep, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankTimers, lockRankNetpollInit, lockRankTimer, lockRankHchan, lockRankNotifyList, lockRankSudog, lockRankRoot, lockRankItab, lockRankReflectOffs, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings, lockRankFin, lockRankSpanSetSpine, lockRankMspanSpecial, lockRankGcBitsArenas, lockRankProfInsert, lockRankProfBlock, lockRankProfMemActive, lockRankProfMemFuture, lockRankGscan, lockRankStackpool, lockRankStackLarge, lockRankWbufSpans, lockRankMheap, lockRankMheapSpecial},
|
||||
lockRankTrace: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankDefer, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankTestR, lockRankExecW, lockRankCpuprof, lockRankPollDesc, lockRankWakeableSleep, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankTimers, lockRankNetpollInit, lockRankTimer, lockRankHchan, lockRankNotifyList, lockRankSudog, lockRankRoot, lockRankItab, lockRankReflectOffs, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings, lockRankFin, lockRankSpanSetSpine, lockRankMspanSpecial, lockRankGcBitsArenas, lockRankProfInsert, lockRankProfBlock, lockRankProfMemActive, lockRankProfMemFuture, lockRankGscan, lockRankStackpool, lockRankStackLarge, lockRankWbufSpans, lockRankMheap},
|
||||
lockRankTraceStackTab: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankDefer, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankTestR, lockRankExecW, lockRankCpuprof, lockRankPollDesc, lockRankWakeableSleep, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankTimers, lockRankNetpollInit, lockRankTimer, lockRankHchan, lockRankNotifyList, lockRankSudog, lockRankRoot, lockRankItab, lockRankReflectOffs, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings, lockRankFin, lockRankSpanSetSpine, lockRankMspanSpecial, lockRankGcBitsArenas, lockRankProfInsert, lockRankProfBlock, lockRankProfMemActive, lockRankProfMemFuture, lockRankGscan, lockRankStackpool, lockRankStackLarge, lockRankWbufSpans, lockRankMheap, lockRankTrace},
|
||||
lockRankFin: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankTestR, lockRankExecW, lockRankCpuprof, lockRankPollDesc, lockRankWakeableSleep, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankTimers, lockRankTimer, lockRankHchan, lockRankNotifyList, lockRankItab, lockRankReflectOffs, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings},
|
||||
lockRankSpanSetSpine: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankTestR, lockRankExecW, lockRankCpuprof, lockRankPollDesc, lockRankWakeableSleep, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankTimers, lockRankTimer, lockRankHchan, lockRankNotifyList, lockRankItab, lockRankReflectOffs, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings},
|
||||
lockRankMspanSpecial: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankTestR, lockRankExecW, lockRankCpuprof, lockRankPollDesc, lockRankWakeableSleep, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankTimers, lockRankTimer, lockRankHchan, lockRankNotifyList, lockRankItab, lockRankReflectOffs, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings},
|
||||
lockRankGcBitsArenas: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankTestR, lockRankExecW, lockRankCpuprof, lockRankPollDesc, lockRankWakeableSleep, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankTimers, lockRankTimer, lockRankHchan, lockRankNotifyList, lockRankItab, lockRankReflectOffs, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings, lockRankMspanSpecial},
|
||||
lockRankProfInsert: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankTestR, lockRankExecW, lockRankCpuprof, lockRankPollDesc, lockRankWakeableSleep, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankTimers, lockRankTimer, lockRankHchan, lockRankNotifyList, lockRankItab, lockRankReflectOffs, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings},
|
||||
lockRankProfBlock: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankTestR, lockRankExecW, lockRankCpuprof, lockRankPollDesc, lockRankWakeableSleep, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankTimers, lockRankTimer, lockRankHchan, lockRankNotifyList, lockRankItab, lockRankReflectOffs, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings},
|
||||
lockRankProfMemActive: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankTestR, lockRankExecW, lockRankCpuprof, lockRankPollDesc, lockRankWakeableSleep, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankTimers, lockRankTimer, lockRankHchan, lockRankNotifyList, lockRankItab, lockRankReflectOffs, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings},
|
||||
lockRankProfMemFuture: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankTestR, lockRankExecW, lockRankCpuprof, lockRankPollDesc, lockRankWakeableSleep, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankTimers, lockRankTimer, lockRankHchan, lockRankNotifyList, lockRankItab, lockRankReflectOffs, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings, lockRankProfMemActive},
|
||||
lockRankGscan: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankTestR, lockRankExecW, lockRankCpuprof, lockRankPollDesc, lockRankWakeableSleep, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankTimers, lockRankTimer, lockRankNetpollInit, lockRankHchan, lockRankNotifyList, lockRankRoot, lockRankItab, lockRankReflectOffs, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings, lockRankFin, lockRankSpanSetSpine, lockRankMspanSpecial, lockRankGcBitsArenas, lockRankProfInsert, lockRankProfBlock, lockRankProfMemActive, lockRankProfMemFuture},
|
||||
lockRankStackpool: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankTestR, lockRankExecW, lockRankCpuprof, lockRankPollDesc, lockRankWakeableSleep, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankTimers, lockRankTimer, lockRankNetpollInit, lockRankHchan, lockRankNotifyList, lockRankRoot, lockRankItab, lockRankReflectOffs, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings, lockRankFin, lockRankSpanSetSpine, lockRankMspanSpecial, lockRankGcBitsArenas, lockRankProfInsert, lockRankProfBlock, lockRankProfMemActive, lockRankProfMemFuture, lockRankGscan},
|
||||
lockRankStackLarge: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankTestR, lockRankExecW, lockRankCpuprof, lockRankPollDesc, lockRankWakeableSleep, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankTimers, lockRankTimer, lockRankNetpollInit, lockRankHchan, lockRankNotifyList, lockRankRoot, lockRankItab, lockRankReflectOffs, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings, lockRankFin, lockRankSpanSetSpine, lockRankMspanSpecial, lockRankGcBitsArenas, lockRankProfInsert, lockRankProfBlock, lockRankProfMemActive, lockRankProfMemFuture, lockRankGscan},
|
||||
lockRankHchanLeaf: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankTestR, lockRankExecW, lockRankCpuprof, lockRankPollDesc, lockRankWakeableSleep, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankTimers, lockRankTimer, lockRankNetpollInit, lockRankHchan, lockRankNotifyList, lockRankRoot, lockRankItab, lockRankReflectOffs, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings, lockRankFin, lockRankSpanSetSpine, lockRankMspanSpecial, lockRankGcBitsArenas, lockRankProfInsert, lockRankProfBlock, lockRankProfMemActive, lockRankProfMemFuture, lockRankGscan, lockRankHchanLeaf},
|
||||
lockRankWbufSpans: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankDefer, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankTestR, lockRankExecW, lockRankCpuprof, lockRankPollDesc, lockRankWakeableSleep, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankTimers, lockRankTimer, lockRankNetpollInit, lockRankHchan, lockRankNotifyList, lockRankSudog, lockRankRoot, lockRankItab, lockRankReflectOffs, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings, lockRankFin, lockRankSpanSetSpine, lockRankMspanSpecial, lockRankGcBitsArenas, lockRankProfInsert, lockRankProfBlock, lockRankProfMemActive, lockRankProfMemFuture, lockRankGscan},
|
||||
lockRankMheap: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankDefer, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankTestR, lockRankExecW, lockRankCpuprof, lockRankPollDesc, lockRankWakeableSleep, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankTimers, lockRankTimer, lockRankNetpollInit, lockRankHchan, lockRankNotifyList, lockRankSudog, lockRankRoot, lockRankItab, lockRankReflectOffs, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings, lockRankFin, lockRankSpanSetSpine, lockRankMspanSpecial, lockRankGcBitsArenas, lockRankProfInsert, lockRankProfBlock, lockRankProfMemActive, lockRankProfMemFuture, lockRankGscan, lockRankStackpool, lockRankStackLarge, lockRankWbufSpans},
|
||||
lockRankMheapSpecial: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankDefer, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankTestR, lockRankExecW, lockRankCpuprof, lockRankPollDesc, lockRankWakeableSleep, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankTimers, lockRankTimer, lockRankNetpollInit, lockRankHchan, lockRankNotifyList, lockRankSudog, lockRankRoot, lockRankItab, lockRankReflectOffs, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings, lockRankFin, lockRankSpanSetSpine, lockRankMspanSpecial, lockRankGcBitsArenas, lockRankProfInsert, lockRankProfBlock, lockRankProfMemActive, lockRankProfMemFuture, lockRankGscan, lockRankStackpool, lockRankStackLarge, lockRankWbufSpans, lockRankMheap},
|
||||
lockRankGlobalAlloc: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankDefer, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankTestR, lockRankExecW, lockRankCpuprof, lockRankPollDesc, lockRankWakeableSleep, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankTimers, lockRankTimer, lockRankNetpollInit, lockRankHchan, lockRankNotifyList, lockRankSudog, lockRankRoot, lockRankItab, lockRankReflectOffs, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings, lockRankFin, lockRankSpanSetSpine, lockRankMspanSpecial, lockRankGcBitsArenas, lockRankProfInsert, lockRankProfBlock, lockRankProfMemActive, lockRankProfMemFuture, lockRankGscan, lockRankStackpool, lockRankStackLarge, lockRankWbufSpans, lockRankMheap, lockRankMheapSpecial},
|
||||
lockRankTrace: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankDefer, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankTestR, lockRankExecW, lockRankCpuprof, lockRankPollDesc, lockRankWakeableSleep, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankTimers, lockRankTimer, lockRankNetpollInit, lockRankHchan, lockRankNotifyList, lockRankSudog, lockRankRoot, lockRankItab, lockRankReflectOffs, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings, lockRankFin, lockRankSpanSetSpine, lockRankMspanSpecial, lockRankGcBitsArenas, lockRankProfInsert, lockRankProfBlock, lockRankProfMemActive, lockRankProfMemFuture, lockRankGscan, lockRankStackpool, lockRankStackLarge, lockRankWbufSpans, lockRankMheap},
|
||||
lockRankTraceStackTab: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankDefer, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankTestR, lockRankExecW, lockRankCpuprof, lockRankPollDesc, lockRankWakeableSleep, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankTimers, lockRankTimer, lockRankNetpollInit, lockRankHchan, lockRankNotifyList, lockRankSudog, lockRankRoot, lockRankItab, lockRankReflectOffs, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings, lockRankFin, lockRankSpanSetSpine, lockRankMspanSpecial, lockRankGcBitsArenas, lockRankProfInsert, lockRankProfBlock, lockRankProfMemActive, lockRankProfMemFuture, lockRankGscan, lockRankStackpool, lockRankStackLarge, lockRankWbufSpans, lockRankMheap, lockRankTrace},
|
||||
lockRankPanic: {},
|
||||
lockRankDeadlock: {lockRankPanic, lockRankDeadlock},
|
||||
lockRankRaceFini: {lockRankPanic},
|
||||
|
||||
@ -79,8 +79,7 @@ assistQueue,
|
||||
< sched;
|
||||
sched < allg, allp;
|
||||
allp, wakeableSleep < timers;
|
||||
timers < netpollInit;
|
||||
timers < timer;
|
||||
timers < timer < netpollInit;
|
||||
|
||||
# Channels
|
||||
scavenge, sweep, testR, wakeableSleep < hchan;
|
||||
@ -117,7 +116,7 @@ allg,
|
||||
hchan,
|
||||
notifyList,
|
||||
reflectOffs,
|
||||
timers,
|
||||
timer,
|
||||
traceStrings,
|
||||
userArenaState
|
||||
# Above MALLOC are things that can allocate memory.
|
||||
|
||||
@ -269,10 +269,12 @@ func goroutineReady(arg any, seq uintptr) {
|
||||
goready(arg.(*g), 0)
|
||||
}
|
||||
|
||||
// add adds t to the timers.
|
||||
// The caller must have set t.ts = t, unlocked t,
|
||||
// and then locked ts.lock.
|
||||
func (ts *timers) add(t *timer) {
|
||||
// addHeap adds t to the timers heap.
|
||||
// The caller must hold ts.lock.
|
||||
// The caller must also have checked that t belongs in the heap.
|
||||
// Callers that are not sure can call t.maybeAdd instead,
|
||||
// but note that maybeAdd has different locking requirements.
|
||||
func (ts *timers) addHeap(t *timer) {
|
||||
assertLockHeld(&ts.lock)
|
||||
// Timers rely on the network poller, so make sure the poller
|
||||
// has started.
|
||||
@ -280,13 +282,14 @@ func (ts *timers) add(t *timer) {
|
||||
netpollGenericInit()
|
||||
}
|
||||
|
||||
if t.ts != ts {
|
||||
throw("ts not set in timer")
|
||||
if t.ts != nil {
|
||||
throw("ts set in timer")
|
||||
}
|
||||
t.ts = ts
|
||||
ts.heap = append(ts.heap, t)
|
||||
ts.siftUp(len(ts.heap) - 1)
|
||||
if t == ts.heap[0] {
|
||||
ts.minWhen.Store(t.when)
|
||||
ts.updateMinWhen()
|
||||
}
|
||||
ts.len.Store(uint32(len(ts.heap)))
|
||||
}
|
||||
@ -332,7 +335,7 @@ func (ts *timers) deleteMin() {
|
||||
ts.updateMinWhen()
|
||||
ts.len.Store(uint32(last))
|
||||
if last == 0 {
|
||||
// If there are no timers, then clearly none are modified.
|
||||
// If there are no timers, then clearly there are no timerNextWhen timers.
|
||||
ts.minNextWhen.Store(0)
|
||||
}
|
||||
}
|
||||
@ -359,22 +362,9 @@ func (t *timer) modify(when, period int64, f func(any, uintptr), arg any, seq ui
|
||||
// to avoid lock inversion with timers lock.
|
||||
// Since t is not in a heap yet, nothing will
|
||||
// find and modify it until after the ts.add.
|
||||
state |= timerHeaped
|
||||
t.when = when
|
||||
|
||||
ts := &getg().m.p.ptr().timers
|
||||
t.ts = ts
|
||||
// pass mp=nil to t.unlock to avoid preemption
|
||||
// between t.unlock and lock of timersLock.
|
||||
// releasem done manually below
|
||||
t.unlock(state, nil)
|
||||
|
||||
lock(&ts.lock)
|
||||
ts.add(t)
|
||||
unlock(&ts.lock)
|
||||
releasem(mp)
|
||||
|
||||
wakeNetPoller(when)
|
||||
t.unlock(state, mp)
|
||||
t.maybeAdd()
|
||||
return false
|
||||
}
|
||||
|
||||
@ -405,6 +395,48 @@ func (t *timer) modify(when, period int64, f func(any, uintptr), arg any, seq ui
|
||||
return pending
|
||||
}
|
||||
|
||||
// needsAdd reports whether t needs to be added to a timers heap.
|
||||
func (t *timer) needsAdd(state uint32) bool {
|
||||
return state&timerHeaped == 0 &&
|
||||
t.when > 0
|
||||
}
|
||||
|
||||
// maybeAdd adds t to the local timers heap if it needs to be in a heap.
|
||||
// The caller must not hold t's lock nor any timers heap lock.
|
||||
// The caller probably just unlocked t, but that lock must be dropped
|
||||
// in order to acquire a ts.lock, to avoid lock inversions.
|
||||
// (timers.adjust holds ts.lock while acquiring each t's lock,
|
||||
// so we cannot hold any t's lock while acquiring ts.lock).
|
||||
//
|
||||
// Strictly speaking it *might* be okay to hold t.lock and
|
||||
// acquire ts.lock at the same time, because we know that
|
||||
// t is not in any ts.heap, so nothing holding a ts.lock would
|
||||
// be acquiring the t.lock at the same time, meaning there
|
||||
// isn't a possible deadlock. But it is easier and safer not to be
|
||||
// too clever and respect the static ordering.
|
||||
// (If we don't, we have to change the static lock checking of t and ts.)
|
||||
//
|
||||
// Because we are not holding t while acquiring ts,
|
||||
// concurrent calls to enqueueTimerChan may result in
|
||||
// concurrent calls to t.maybeAdd, so we cannot assume that
|
||||
// t is not in a heap on entry to t.maybeAdd.
|
||||
func (t *timer) maybeAdd() {
|
||||
ts := &getg().m.p.ptr().timers
|
||||
lock(&ts.lock)
|
||||
state, mp := t.lock()
|
||||
when := int64(0)
|
||||
if t.needsAdd(state) {
|
||||
state |= timerHeaped
|
||||
ts.addHeap(t)
|
||||
when = t.when
|
||||
}
|
||||
t.unlock(state, mp)
|
||||
unlock(&ts.lock)
|
||||
if when > 0 {
|
||||
wakeNetPoller(when)
|
||||
}
|
||||
}
|
||||
|
||||
// reset resets the time when a timer should fire.
|
||||
// If used for an inactive timer, the timer will become active.
|
||||
// Reports whether the timer was active and was stopped.
|
||||
@ -487,16 +519,10 @@ func (ts *timers) move(timers []*timer) {
|
||||
state, mp := t.lock()
|
||||
t.ts = nil
|
||||
state, _ = t.updateWhen(state, nil)
|
||||
// Unlock before add, to avoid append (allocation)
|
||||
// while holding lock. This would be correct even if the world wasn't
|
||||
// stopped (but it is), and it makes staticlockranking happy.
|
||||
if state&timerHeaped != 0 {
|
||||
t.ts = ts
|
||||
ts.addHeap(t)
|
||||
}
|
||||
t.unlock(state, mp)
|
||||
if state&timerHeaped != 0 {
|
||||
ts.add(t)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -507,7 +533,7 @@ func (ts *timers) move(timers []*timer) {
|
||||
// and removes deleted timers. The caller must have locked ts.
|
||||
func (ts *timers) adjust(now int64, force bool) {
|
||||
assertLockHeld(&ts.lock)
|
||||
// If we haven't yet reached the time of the earliest timerModified
|
||||
// If we haven't yet reached the time of the earliest modified
|
||||
// timer, don't do anything. This speeds up programs that adjust
|
||||
// a lot of timers back and forth if the timers rarely expire.
|
||||
// We'll postpone looking through all the adjusted timers until
|
||||
@ -522,7 +548,49 @@ func (ts *timers) adjust(now int64, force bool) {
|
||||
}
|
||||
}
|
||||
|
||||
// We are going to clear all timerModified timers.
|
||||
// minNextWhen is a lower bound on the earliest t.nextWhen
|
||||
// among the timerNextWhen timers. We want to make it more precise:
|
||||
// we are going to scan the heap and clean out all the timerNextWhen bits,
|
||||
// at which point minNextWhen can be set to 0 (indicating none at all).
|
||||
//
|
||||
// Other P's can be calling ts.wakeTime concurrently, and we'd like to
|
||||
// keep ts.wakeTime returning an accurate value throughout this entire process.
|
||||
//
|
||||
// Setting minNextWhen = 0 *before* the scan could make wakeTime
|
||||
// return an incorrect value: if minNextWhen < nextWhen, then clearing
|
||||
// it to 0 will make wakeTime return nextWhen (too late) until the scan finishes.
|
||||
// To avoid that, we want to set minNextWhen to 0 *after* the scan.
|
||||
//
|
||||
// Setting minNextWhen = 0 *after* the scan could result in missing
|
||||
// concurrent timer modifications in other goroutines; those will lock
|
||||
// the specific timer, set the timerNextWhen bit, and set t.nextWhen.
|
||||
// To avoid that, we want to set minNextWhen to 0 *before* the scan.
|
||||
//
|
||||
// The way out of this dilemma is to preserve wakeTime a different way.
|
||||
// wakeTime is min(minWhen, minNextWhen), and minWhen is protected by
|
||||
// ts.lock, which we hold, so we can modify it however we like in service
|
||||
// of keeping wakeTime accurate.
|
||||
//
|
||||
// So we can:
|
||||
//
|
||||
// 1. Set minWhen = min(minWhen, minNextWhen)
|
||||
// 2. Set minNextWhen = 0
|
||||
// (Other goroutines may modify timers and update minNextWhen now.)
|
||||
// 3. Scan timers
|
||||
// 4. Set minWhen = heap[0].when
|
||||
//
|
||||
// That order preserves a correct value of wakeTime throughout the entire
|
||||
// operation:
|
||||
// Step 1 “locks in” an accurate wakeTime even with minNextWhen cleared.
|
||||
// Step 2 makes sure concurrent nextWhen updates are not lost during the scan.
|
||||
// Step 3 clears all the old nextWhen values, justifying minNextWhen = 0.
|
||||
// Step 4 corrects minWhen to a precise value.
|
||||
//
|
||||
// The wakeTime method implementation reads minNextWhen *before* minWhen,
|
||||
// so that if the minNextWhen observes a 0, that means the minWhen
|
||||
// will include the information about what was zeroed.
|
||||
|
||||
ts.minWhen.Store(ts.wakeTime())
|
||||
ts.minNextWhen.Store(0)
|
||||
|
||||
changed := false
|
||||
@ -554,8 +622,8 @@ func (ts *timers) adjust(now int64, force bool) {
|
||||
|
||||
if changed {
|
||||
ts.initHeap()
|
||||
ts.updateMinWhen()
|
||||
}
|
||||
ts.updateMinWhen()
|
||||
|
||||
if verifyTimers {
|
||||
ts.verify()
|
||||
@ -569,12 +637,17 @@ func (ts *timers) adjust(now int64, force bool) {
|
||||
//
|
||||
//go:nowritebarrierrec
|
||||
func (ts *timers) wakeTime() int64 {
|
||||
next := ts.minWhen.Load()
|
||||
nextAdj := ts.minNextWhen.Load()
|
||||
if next == 0 || (nextAdj != 0 && nextAdj < next) {
|
||||
next = nextAdj
|
||||
// Note that the order of these two loads matters:
|
||||
// adjust updates minWhen to make it safe to clear minNextWhen.
|
||||
// We read minWhen after reading minNextWhen so that
|
||||
// if we see a cleared minNextWhen, we are guaranteed to see
|
||||
// the updated minWhen.
|
||||
nextWhen := ts.minNextWhen.Load()
|
||||
when := ts.minWhen.Load()
|
||||
if when == 0 || (nextWhen != 0 && nextWhen < when) {
|
||||
when = nextWhen
|
||||
}
|
||||
return next
|
||||
return when
|
||||
}
|
||||
|
||||
// check runs any timers in ts that are ready.
|
||||
@ -590,12 +663,7 @@ func (ts *timers) wakeTime() int64 {
|
||||
func (ts *timers) check(now int64) (rnow, pollUntil int64, ran bool) {
|
||||
// If it's not yet time for the first timer, or the first adjusted
|
||||
// timer, then there is nothing to do.
|
||||
next := ts.minWhen.Load()
|
||||
nextAdj := ts.minNextWhen.Load()
|
||||
if next == 0 || (nextAdj != 0 && nextAdj < next) {
|
||||
next = nextAdj
|
||||
}
|
||||
|
||||
next := ts.wakeTime()
|
||||
if next == 0 {
|
||||
// No timers to run or adjust.
|
||||
return now, 0, false
|
||||
@ -838,14 +906,8 @@ func timeSleepUntil() int64 {
|
||||
continue
|
||||
}
|
||||
|
||||
w := pp.timers.minWhen.Load()
|
||||
if w != 0 && w < next {
|
||||
next = w
|
||||
}
|
||||
|
||||
w = pp.timers.minNextWhen.Load()
|
||||
if w != 0 && w < next {
|
||||
next = w
|
||||
if w := pp.timers.wakeTime(); w != 0 {
|
||||
next = min(next, w)
|
||||
}
|
||||
}
|
||||
unlock(&allpLock)
|
||||
|
||||
Loading…
Reference in New Issue
Block a user