From 4c383951b9601b488486add020ad5b7f10fb3d39 Mon Sep 17 00:00:00 2001 From: Michael Anthony Knyszek Date: Tue, 20 Sep 2022 18:33:03 +0000 Subject: [PATCH] runtime: make (*mheap).sysAlloc more general This change makes (*mheap).sysAlloc take an explicit list of hints and a boolean as to whether or not any newly-created heapArenas should be registered in the full arena list. This is a refactoring in preparation for arenas. For #51317. Change-Id: I0584a033fce3fcb60c5d0bc033d5fb8bd23b2378 Reviewed-on: https://go-review.googlesource.com/c/go/+/432078 TryBot-Result: Gopher Robot Reviewed-by: Cherry Mui Run-TryBot: Michael Knyszek --- src/runtime/malloc.go | 69 +++++++++++++++++++++++++------------------ src/runtime/mheap.go | 2 +- 2 files changed, 42 insertions(+), 29 deletions(-) diff --git a/src/runtime/malloc.go b/src/runtime/malloc.go index c18ed07d49..d651cbc14e 100644 --- a/src/runtime/malloc.go +++ b/src/runtime/malloc.go @@ -556,26 +556,37 @@ func mallocinit() { // heapArenaBytes. sysAlloc returns nil on failure. // There is no corresponding free function. // +// hintList is a list of hint addresses for where to allocate new +// heap arenas. It must be non-nil. +// +// register indicates whether the heap arena should be registered +// in allArenas. +// // sysAlloc returns a memory region in the Reserved state. This region must // be transitioned to Prepared and then Ready before use. // // h must be locked. -func (h *mheap) sysAlloc(n uintptr) (v unsafe.Pointer, size uintptr) { +func (h *mheap) sysAlloc(n uintptr, hintList **arenaHint, register bool) (v unsafe.Pointer, size uintptr) { assertLockHeld(&h.lock) n = alignUp(n, heapArenaBytes) - // First, try the arena pre-reservation. - // Newly-used mappings are considered released. - v = h.arena.alloc(n, heapArenaBytes, &gcController.heapReleased) - if v != nil { - size = n - goto mapped + if hintList == &h.arenaHints { + // First, try the arena pre-reservation. + // Newly-used mappings are considered released. + // + // Only do this if we're using the regular heap arena hints. + // This behavior is only for the heap. + v = h.arena.alloc(n, heapArenaBytes, &gcController.heapReleased) + if v != nil { + size = n + goto mapped + } } // Try to grow the heap at a hint address. - for h.arenaHints != nil { - hint := h.arenaHints + for *hintList != nil { + hint := *hintList p := hint.addr if hint.down { p -= n @@ -607,7 +618,7 @@ func (h *mheap) sysAlloc(n uintptr) (v unsafe.Pointer, size uintptr) { if v != nil { sysFreeOS(v, n) } - h.arenaHints = hint.next + *hintList = hint.next h.arenaHintAlloc.free(unsafe.Pointer(hint)) } @@ -692,26 +703,28 @@ mapped: } } - // Add the arena to the arenas list. - if len(h.allArenas) == cap(h.allArenas) { - size := 2 * uintptr(cap(h.allArenas)) * goarch.PtrSize - if size == 0 { - size = physPageSize + // Register the arena in allArenas if requested. + if register { + if len(h.allArenas) == cap(h.allArenas) { + size := 2 * uintptr(cap(h.allArenas)) * goarch.PtrSize + if size == 0 { + size = physPageSize + } + newArray := (*notInHeap)(persistentalloc(size, goarch.PtrSize, &memstats.gcMiscSys)) + if newArray == nil { + throw("out of memory allocating allArenas") + } + oldSlice := h.allArenas + *(*notInHeapSlice)(unsafe.Pointer(&h.allArenas)) = notInHeapSlice{newArray, len(h.allArenas), int(size / goarch.PtrSize)} + copy(h.allArenas, oldSlice) + // Do not free the old backing array because + // there may be concurrent readers. Since we + // double the array each time, this can lead + // to at most 2x waste. } - newArray := (*notInHeap)(persistentalloc(size, goarch.PtrSize, &memstats.gcMiscSys)) - if newArray == nil { - throw("out of memory allocating allArenas") - } - oldSlice := h.allArenas - *(*notInHeapSlice)(unsafe.Pointer(&h.allArenas)) = notInHeapSlice{newArray, len(h.allArenas), int(size / goarch.PtrSize)} - copy(h.allArenas, oldSlice) - // Do not free the old backing array because - // there may be concurrent readers. Since we - // double the array each time, this can lead - // to at most 2x waste. + h.allArenas = h.allArenas[:len(h.allArenas)+1] + h.allArenas[len(h.allArenas)-1] = ri } - h.allArenas = h.allArenas[:len(h.allArenas)+1] - h.allArenas[len(h.allArenas)-1] = ri // Store atomically just in case an object from the // new heap arena becomes visible before the heap lock diff --git a/src/runtime/mheap.go b/src/runtime/mheap.go index 913d812767..8616d6e06e 100644 --- a/src/runtime/mheap.go +++ b/src/runtime/mheap.go @@ -1437,7 +1437,7 @@ func (h *mheap) grow(npage uintptr) (uintptr, bool) { // Not enough room in the current arena. Allocate more // arena space. This may not be contiguous with the // current arena, so we have to request the full ask. - av, asize := h.sysAlloc(ask) + av, asize := h.sysAlloc(ask, &h.arenaHints, true) if av == nil { inUse := gcController.heapFree.load() + gcController.heapReleased.load() + gcController.heapInUse.load() print("runtime: out of memory: cannot allocate ", ask, "-byte block (", inUse, " in use)\n")