From f407ca9288c8556c466e316f390ee7e7e99647ae Mon Sep 17 00:00:00 2001 From: Austin Clements Date: Mon, 18 Jul 2016 16:01:22 -0400 Subject: [PATCH] runtime: support smaller physical pages than PhysPageSize Most operations need an upper bound on the physical page size, which is what sys.PhysPageSize is for (this is checked at runtime init on Linux). However, a few operations need a *lower* bound on the physical page size. Introduce a "minPhysPageSize" constant to act as this lower bound and use it where it makes sense: 1) In addrspace_free, we have to query each page in the given range. Currently we increment by the upper bound on the physical page size, which means we may skip over pages if the true size is smaller. Worse, we currently pass a result buffer that only has enough room for one page. If there are actually multiple pages in the range passed to mincore, the kernel will overflow this buffer. Fix these problems by incrementing by the lower-bound on the physical page size and by passing "1" for the length, which the kernel will round up to the true physical page size. 2) In the write barrier, the bad pointer check tests for pointers to the first physical page, which are presumably small integers masquerading as pointers. However, if physical pages are smaller than we think, we may have legitimate pointers below sys.PhysPageSize. Hence, use minPhysPageSize for this test since pointers should never fall below that. In particular, this applies to ARM64 and MIPS. The runtime is configured to use 64kB pages on ARM64, but by default Linux uses 4kB pages. Similarly, the runtime assumes 16kB pages on MIPS, but both 4kB and 16kB kernel configurations are common. This also applies to ARM on systems where the runtime is recompiled to deal with a larger page size. It is also a step toward making the runtime use only a dynamically-queried page size. Change-Id: I1fdfd18f6e7cbca170cc100354b9faa22fde8a69 Reviewed-on: https://go-review.googlesource.com/25020 Reviewed-by: Ian Lance Taylor Reviewed-by: Cherry Zhang Run-TryBot: Austin Clements --- src/runtime/mbarrier.go | 4 ++-- src/runtime/mem_linux.go | 22 ++++++++++++++-------- src/runtime/mheap.go | 5 +++++ 3 files changed, 21 insertions(+), 10 deletions(-) diff --git a/src/runtime/mbarrier.go b/src/runtime/mbarrier.go index bf75934ed6e..4a8f501dfec 100644 --- a/src/runtime/mbarrier.go +++ b/src/runtime/mbarrier.go @@ -145,7 +145,7 @@ func writebarrierptr(dst *uintptr, src uintptr) { if !writeBarrier.needed { return } - if src != 0 && src < sys.PhysPageSize { + if src != 0 && src < minPhysPageSize { systemstack(func() { print("runtime: writebarrierptr *", dst, " = ", hex(src), "\n") throw("bad pointer in write barrier") @@ -164,7 +164,7 @@ func writebarrierptr_nostore(dst *uintptr, src uintptr) { if !writeBarrier.needed { return } - if src != 0 && src < sys.PhysPageSize { + if src != 0 && src < minPhysPageSize { systemstack(func() { throw("bad pointer in write barrier") }) } writebarrierptr_nostore1(dst, src) diff --git a/src/runtime/mem_linux.go b/src/runtime/mem_linux.go index 61fdcee5430..cd0bf263285 100644 --- a/src/runtime/mem_linux.go +++ b/src/runtime/mem_linux.go @@ -10,8 +10,8 @@ import ( ) const ( - _PAGE_SIZE = sys.PhysPageSize - _EACCES = 13 + _EACCES = 13 + _EINVAL = 22 ) // NOTE: vec must be just 1 byte long here. @@ -22,13 +22,19 @@ const ( var addrspace_vec [1]byte func addrspace_free(v unsafe.Pointer, n uintptr) bool { - var chunk uintptr - for off := uintptr(0); off < n; off += chunk { - chunk = _PAGE_SIZE * uintptr(len(addrspace_vec)) - if chunk > (n - off) { - chunk = n - off + // Step by the minimum possible physical page size. This is + // safe even if we have the wrong physical page size; mincore + // will just return EINVAL for unaligned addresses. + for off := uintptr(0); off < n; off += minPhysPageSize { + // Use a length of 1 byte, which the kernel will round + // up to one physical page regardless of the true + // physical page size. + errval := mincore(unsafe.Pointer(uintptr(v)+off), 1, &addrspace_vec[0]) + if errval == -_EINVAL { + // Address is not a multiple of the physical + // page size. That's fine. + continue } - errval := mincore(unsafe.Pointer(uintptr(v)+off), chunk, &addrspace_vec[0]) // ENOMEM means unmapped, which is what we want. // Anything else we assume means the pages are mapped. if errval != -_ENOMEM { diff --git a/src/runtime/mheap.go b/src/runtime/mheap.go index 4093288a7cb..db60f7a8727 100644 --- a/src/runtime/mheap.go +++ b/src/runtime/mheap.go @@ -14,6 +14,11 @@ import ( "unsafe" ) +// minPhysPageSize is a lower-bound on the physical page size. The +// true physical page size may be larger than this. In contrast, +// sys.PhysPageSize is an upper-bound on the physical page size. +const minPhysPageSize = 4096 + // Main malloc heap. // The heap itself is the "free[]" and "large" arrays, // but all the other global data is here too.