qbit/go
1
0
Fork 0
mirror of https://github.com/golang/go synced 2024-11-06 22:46:14 -07:00
Code Releases Activity

runtime: count scavenged bits for new allocation for new page allocator

Browse Source 
This change makes it so that the new page allocator returns the number
of pages that are scavenged in a new allocation so that mheap can update
memstats appropriately.

The accounting could be embedded into pageAlloc, but that would make
the new allocator more difficult to test.

Updates #35112.

Change-Id: I0f94f563d7af2458e6d534f589d2e7dd6af26d12
Reviewed-on: https://go-review.googlesource.com/c/go/+/195698
Reviewed-by: Austin Clements <austin@google.com>
This commit is contained in:
Michael Anthony Knyszek 2019-09-10 18:53:51 +00:00 committed by Michael Knyszek
parent 73317080e1
commit e1ddf0507c
5 changed files with 281 additions and 76 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
src/runtime/export_test.go
View File

@ -751,9 +751,10 @@ type PallocBits pallocBits
func (b *PallocBits) Find(npages uintptr, searchIdx uint) (uint, uint) { func (b *PallocBits) Find(npages uintptr, searchIdx uint) (uint, uint) {
return (*pallocBits)(b).find(npages, searchIdx) return (*pallocBits)(b).find(npages, searchIdx)
} }
func (b *PallocBits) AllocRange(i, n uint) { (*pallocBits)(b).allocRange(i, n) } func (b *PallocBits) AllocRange(i, n uint) { (*pallocBits)(b).allocRange(i, n) }
func (b *PallocBits) Free(i, n uint) { (*pallocBits)(b).free(i, n) } func (b *PallocBits) Free(i, n uint) { (*pallocBits)(b).free(i, n) }
func (b *PallocBits) Summarize() PallocSum { return PallocSum((*pallocBits)(b).summarize()) } func (b *PallocBits) Summarize() PallocSum { return PallocSum((*pallocBits)(b).summarize()) }
func (b *PallocBits) PopcntRange(i, n uint) uint { return (*pageBits)(b).popcntRange(i, n) }
// SummarizeSlow is a slow but more obviously correct implementation // SummarizeSlow is a slow but more obviously correct implementation
// of (*pallocBits).summarize. Used for testing. // of (*pallocBits).summarize. Used for testing.
@ -853,8 +854,12 @@ type ChunkIdx chunkIdx
// not in the heap, so is PageAlloc. // not in the heap, so is PageAlloc.
type PageAlloc pageAlloc type PageAlloc pageAlloc
func (p *PageAlloc) Alloc(npages uintptr) uintptr { return (*pageAlloc)(p).alloc(npages) } func (p *PageAlloc) Alloc(npages uintptr) (uintptr, uintptr) {
func (p *PageAlloc) Free(base, npages uintptr) { (*pageAlloc)(p).free(base, npages) } return (*pageAlloc)(p).alloc(npages)
}
func (p *PageAlloc) Free(base, npages uintptr) {
(*pageAlloc)(p).free(base, npages)
}
func (p *PageAlloc) Bounds() (ChunkIdx, ChunkIdx) { func (p *PageAlloc) Bounds() (ChunkIdx, ChunkIdx) {
return ChunkIdx((*pageAlloc)(p).start), ChunkIdx((*pageAlloc)(p).end) return ChunkIdx((*pageAlloc)(p).start), ChunkIdx((*pageAlloc)(p).end)
} }

29
src/runtime/mpagealloc.go
View File

@ -467,24 +467,33 @@ func (s *pageAlloc) update(base, npages uintptr, contig, alloc bool) {
// allocated. It also updates the summaries to reflect the newly-updated // allocated. It also updates the summaries to reflect the newly-updated
// bitmap. // bitmap.
// //
// Returns the amount of scavenged memory in bytes present in the
// allocated range.
//
// s.mheapLock must be held. // s.mheapLock must be held.
func (s *pageAlloc) allocRange(base, npages uintptr) { func (s *pageAlloc) allocRange(base, npages uintptr) uintptr {
limit := base + npages*pageSize - 1 limit := base + npages*pageSize - 1
sc, ec := chunkIndex(base), chunkIndex(limit) sc, ec := chunkIndex(base), chunkIndex(limit)
si, ei := chunkPageIndex(base), chunkPageIndex(limit) si, ei := chunkPageIndex(base), chunkPageIndex(limit)
scav := uint(0)
if sc == ec { if sc == ec {
// The range doesn't cross any chunk boundaries. // The range doesn't cross any chunk boundaries.
scav += s.chunks[sc].scavenged.popcntRange(si, ei+1-si)
s.chunks[sc].allocRange(si, ei+1-si) s.chunks[sc].allocRange(si, ei+1-si)
} else { } else {
// The range crosses at least one chunk boundary. // The range crosses at least one chunk boundary.
scav += s.chunks[sc].scavenged.popcntRange(si, pallocChunkPages-si)
s.chunks[sc].allocRange(si, pallocChunkPages-si) s.chunks[sc].allocRange(si, pallocChunkPages-si)
for c := sc + 1; c < ec; c++ { for c := sc + 1; c < ec; c++ {
scav += s.chunks[c].scavenged.popcntRange(0, pallocChunkPages)
s.chunks[c].allocAll() s.chunks[c].allocAll()
} }
scav += s.chunks[ec].scavenged.popcntRange(0, ei+1)
s.chunks[ec].allocRange(0, ei+1) s.chunks[ec].allocRange(0, ei+1)
} }
s.update(base, npages, true, true) s.update(base, npages, true, true)
return uintptr(scav) * pageSize
} }
// find searches for the first (address-ordered) contiguous free region of // find searches for the first (address-ordered) contiguous free region of
@ -714,21 +723,23 @@ nextLevel:
} }
// alloc allocates npages worth of memory from the page heap, returning the base // alloc allocates npages worth of memory from the page heap, returning the base
// address for the allocation. // address for the allocation and the amount of scavenged memory in bytes
// contained in the region [base address, base address + npages*pageSize).
// //
// Returns 0 on failure. // Returns a 0 base address on failure, in which case other returned values
// should be ignored.
// //
// s.mheapLock must be held. // s.mheapLock must be held.
func (s *pageAlloc) alloc(npages uintptr) uintptr { func (s *pageAlloc) alloc(npages uintptr) (addr uintptr, scav uintptr) {
// If the searchAddr refers to a region which has a higher address than // If the searchAddr refers to a region which has a higher address than
// any known chunk, then we know we're out of memory. // any known chunk, then we know we're out of memory.
if chunkIndex(s.searchAddr) >= s.end { if chunkIndex(s.searchAddr) >= s.end {
return 0 return 0, 0
} }
// If npages has a chance of fitting in the chunk where the searchAddr is, // If npages has a chance of fitting in the chunk where the searchAddr is,
// search it directly. // search it directly.
var addr, searchAddr uintptr searchAddr := uintptr(0)
if pallocChunkPages-chunkPageIndex(s.searchAddr) >= uint(npages) { if pallocChunkPages-chunkPageIndex(s.searchAddr) >= uint(npages) {
// npages is guaranteed to be no greater than pallocChunkPages here. // npages is guaranteed to be no greater than pallocChunkPages here.
i := chunkIndex(s.searchAddr) i := chunkIndex(s.searchAddr)
@ -756,11 +767,11 @@ func (s *pageAlloc) alloc(npages uintptr) uintptr {
// accommodate npages. // accommodate npages.
s.searchAddr = maxSearchAddr s.searchAddr = maxSearchAddr
} }
return 0 return 0, 0
} }
Found: Found:
// Go ahead and actually mark the bits now that we have an address. // Go ahead and actually mark the bits now that we have an address.
s.allocRange(addr, npages) scav = s.allocRange(addr, npages)
// If we found a higher (linearized) searchAddr, we know that all the // If we found a higher (linearized) searchAddr, we know that all the
// heap memory before that searchAddr in a linear address space is // heap memory before that searchAddr in a linear address space is
@ -768,7 +779,7 @@ Found:
if s.compareSearchAddrTo(searchAddr) > 0 { if s.compareSearchAddrTo(searchAddr) > 0 {
s.searchAddr = searchAddr s.searchAddr = searchAddr
} }
return addr return addr, scav
} }
// free returns npages worth of memory starting at base back to the page heap. // free returns npages worth of memory starting at base back to the page heap.

201
src/runtime/mpagealloc_test.go
View File

@ -36,9 +36,10 @@ func checkPageAlloc(t *testing.T, want, got *PageAlloc) {
func TestPageAllocAlloc(t *testing.T) { func TestPageAllocAlloc(t *testing.T) {
type hit struct { type hit struct {
npages, base uintptr npages, base, scav uintptr
} }
tests := map[string]struct { tests := map[string]struct {
scav map[ChunkIdx][]BitRange
before map[ChunkIdx][]BitRange before map[ChunkIdx][]BitRange
after map[ChunkIdx][]BitRange after map[ChunkIdx][]BitRange
hits []hit hits []hit
@ -47,12 +48,15 @@ func TestPageAllocAlloc(t *testing.T) {
before: map[ChunkIdx][]BitRange{ before: map[ChunkIdx][]BitRange{
BaseChunkIdx: {}, BaseChunkIdx: {},
}, },
scav: map[ChunkIdx][]BitRange{
BaseChunkIdx: {{0, 1}, {2, 2}},
},
hits: []hit{ hits: []hit{
{1, PageBase(BaseChunkIdx, 0)}, {1, PageBase(BaseChunkIdx, 0), PageSize},
{1, PageBase(BaseChunkIdx, 1)}, {1, PageBase(BaseChunkIdx, 1), 0},
{1, PageBase(BaseChunkIdx, 2)}, {1, PageBase(BaseChunkIdx, 2), PageSize},
{1, PageBase(BaseChunkIdx, 3)}, {1, PageBase(BaseChunkIdx, 3), PageSize},
{1, PageBase(BaseChunkIdx, 4)}, {1, PageBase(BaseChunkIdx, 4), 0},
}, },
after: map[ChunkIdx][]BitRange{ after: map[ChunkIdx][]BitRange{
BaseChunkIdx: {{0, 5}}, BaseChunkIdx: {{0, 5}},
@ -64,8 +68,13 @@ func TestPageAllocAlloc(t *testing.T) {
BaseChunkIdx + 1: {{0, PallocChunkPages}}, BaseChunkIdx + 1: {{0, PallocChunkPages}},
BaseChunkIdx + 2: {{0, PallocChunkPages - 1}}, BaseChunkIdx + 2: {{0, PallocChunkPages - 1}},
}, },
scav: map[ChunkIdx][]BitRange{
BaseChunkIdx: {{0, PallocChunkPages}},
BaseChunkIdx + 1: {{0, PallocChunkPages}},
BaseChunkIdx + 2: {{0, PallocChunkPages}},
},
hits: []hit{ hits: []hit{
{1, PageBase(BaseChunkIdx+2, PallocChunkPages-1)}, {1, PageBase(BaseChunkIdx+2, PallocChunkPages-1), PageSize},
}, },
after: map[ChunkIdx][]BitRange{ after: map[ChunkIdx][]BitRange{
BaseChunkIdx: {{0, PallocChunkPages}}, BaseChunkIdx: {{0, PallocChunkPages}},
@ -78,8 +87,12 @@ func TestPageAllocAlloc(t *testing.T) {
BaseChunkIdx: {{0, PallocChunkPages}}, BaseChunkIdx: {{0, PallocChunkPages}},
BaseChunkIdx + 0xff: {{0, 0}}, BaseChunkIdx + 0xff: {{0, 0}},
}, },
scav: map[ChunkIdx][]BitRange{
BaseChunkIdx: {{0, PallocChunkPages}},
BaseChunkIdx + 0xff: {{0, PallocChunkPages}},
},
hits: []hit{ hits: []hit{
{1, PageBase(BaseChunkIdx+0xff, 0)}, {1, PageBase(BaseChunkIdx+0xff, 0), PageSize},
}, },
after: map[ChunkIdx][]BitRange{ after: map[ChunkIdx][]BitRange{
BaseChunkIdx: {{0, PallocChunkPages}}, BaseChunkIdx: {{0, PallocChunkPages}},
@ -90,12 +103,15 @@ func TestPageAllocAlloc(t *testing.T) {
before: map[ChunkIdx][]BitRange{ before: map[ChunkIdx][]BitRange{
BaseChunkIdx: {}, BaseChunkIdx: {},
}, },
scav: map[ChunkIdx][]BitRange{
BaseChunkIdx: {{0, 3}, {7, 1}},
},
hits: []hit{ hits: []hit{
{2, PageBase(BaseChunkIdx, 0)}, {2, PageBase(BaseChunkIdx, 0), 2 * PageSize},
{2, PageBase(BaseChunkIdx, 2)}, {2, PageBase(BaseChunkIdx, 2), PageSize},
{2, PageBase(BaseChunkIdx, 4)}, {2, PageBase(BaseChunkIdx, 4), 0},
{2, PageBase(BaseChunkIdx, 6)}, {2, PageBase(BaseChunkIdx, 6), PageSize},
{2, PageBase(BaseChunkIdx, 8)}, {2, PageBase(BaseChunkIdx, 8), 0},
}, },
after: map[ChunkIdx][]BitRange{ after: map[ChunkIdx][]BitRange{
BaseChunkIdx: {{0, 10}}, BaseChunkIdx: {{0, 10}},
@ -106,8 +122,12 @@ func TestPageAllocAlloc(t *testing.T) {
BaseChunkIdx: {{0, PallocChunkPages - 1}}, BaseChunkIdx: {{0, PallocChunkPages - 1}},
BaseChunkIdx + 1: {{1, PallocChunkPages - 1}}, BaseChunkIdx + 1: {{1, PallocChunkPages - 1}},
}, },
scav: map[ChunkIdx][]BitRange{
BaseChunkIdx: {{PallocChunkPages - 1, 1}},
BaseChunkIdx + 1: {},
},
hits: []hit{ hits: []hit{
{2, PageBase(BaseChunkIdx, PallocChunkPages-1)}, {2, PageBase(BaseChunkIdx, PallocChunkPages-1), PageSize},
}, },
after: map[ChunkIdx][]BitRange{ after: map[ChunkIdx][]BitRange{
BaseChunkIdx: {{0, PallocChunkPages}}, BaseChunkIdx: {{0, PallocChunkPages}},
@ -118,12 +138,15 @@ func TestPageAllocAlloc(t *testing.T) {
before: map[ChunkIdx][]BitRange{ before: map[ChunkIdx][]BitRange{
BaseChunkIdx: {}, BaseChunkIdx: {},
}, },
scav: map[ChunkIdx][]BitRange{
BaseChunkIdx: {{0, 8}, {9, 1}, {17, 5}},
},
hits: []hit{ hits: []hit{
{5, PageBase(BaseChunkIdx, 0)}, {5, PageBase(BaseChunkIdx, 0), 5 * PageSize},
{5, PageBase(BaseChunkIdx, 5)}, {5, PageBase(BaseChunkIdx, 5), 4 * PageSize},
{5, PageBase(BaseChunkIdx, 10)}, {5, PageBase(BaseChunkIdx, 10), 0},
{5, PageBase(BaseChunkIdx, 15)}, {5, PageBase(BaseChunkIdx, 15), 3 * PageSize},
{5, PageBase(BaseChunkIdx, 20)}, {5, PageBase(BaseChunkIdx, 20), 2 * PageSize},
}, },
after: map[ChunkIdx][]BitRange{ after: map[ChunkIdx][]BitRange{
BaseChunkIdx: {{0, 25}}, BaseChunkIdx: {{0, 25}},
@ -133,10 +156,13 @@ func TestPageAllocAlloc(t *testing.T) {
before: map[ChunkIdx][]BitRange{ before: map[ChunkIdx][]BitRange{
BaseChunkIdx: {}, BaseChunkIdx: {},
}, },
scav: map[ChunkIdx][]BitRange{
BaseChunkIdx: {{21, 1}, {63, 65}},
},
hits: []hit{ hits: []hit{
{64, PageBase(BaseChunkIdx, 0)}, {64, PageBase(BaseChunkIdx, 0), 2 * PageSize},
{64, PageBase(BaseChunkIdx, 64)}, {64, PageBase(BaseChunkIdx, 64), 64 * PageSize},
{64, PageBase(BaseChunkIdx, 128)}, {64, PageBase(BaseChunkIdx, 128), 0},
}, },
after: map[ChunkIdx][]BitRange{ after: map[ChunkIdx][]BitRange{
BaseChunkIdx: {{0, 192}}, BaseChunkIdx: {{0, 192}},
@ -146,10 +172,13 @@ func TestPageAllocAlloc(t *testing.T) {
before: map[ChunkIdx][]BitRange{ before: map[ChunkIdx][]BitRange{
BaseChunkIdx: {}, BaseChunkIdx: {},
}, },
scav: map[ChunkIdx][]BitRange{
BaseChunkIdx: {{129, 1}},
},
hits: []hit{ hits: []hit{
{65, PageBase(BaseChunkIdx, 0)}, {65, PageBase(BaseChunkIdx, 0), 0},
{65, PageBase(BaseChunkIdx, 65)}, {65, PageBase(BaseChunkIdx, 65), PageSize},
{65, PageBase(BaseChunkIdx, 130)}, {65, PageBase(BaseChunkIdx, 130), 0},
}, },
after: map[ChunkIdx][]BitRange{ after: map[ChunkIdx][]BitRange{
BaseChunkIdx: {{0, 195}}, BaseChunkIdx: {{0, 195}},
@ -160,13 +189,16 @@ func TestPageAllocAlloc(t *testing.T) {
before: map[ChunkIdx][]BitRange{ before: map[ChunkIdx][]BitRange{
BaseChunkIdx: {}, BaseChunkIdx: {},
}, },
scav: map[ChunkIdx][]BitRange{
BaseChunkIdx: {{10, 1}},
},
hits: []hit{ hits: []hit{
{PallocChunkPages - 3, PageBase(BaseChunkIdx, 0)}, {PallocChunkPages - 3, PageBase(BaseChunkIdx, 0), PageSize},
{PallocChunkPages - 3, 0}, {PallocChunkPages - 3, 0, 0},
{1, PageBase(BaseChunkIdx, PallocChunkPages-3)}, {1, PageBase(BaseChunkIdx, PallocChunkPages-3), 0},
{2, PageBase(BaseChunkIdx, PallocChunkPages-2)}, {2, PageBase(BaseChunkIdx, PallocChunkPages-2), 0},
{1, 0}, {1, 0, 0},
{PallocChunkPages - 3, 0}, {PallocChunkPages - 3, 0, 0},
}, },
after: map[ChunkIdx][]BitRange{ after: map[ChunkIdx][]BitRange{
BaseChunkIdx: {{0, PallocChunkPages}}, BaseChunkIdx: {{0, PallocChunkPages}},
@ -176,10 +208,13 @@ func TestPageAllocAlloc(t *testing.T) {
before: map[ChunkIdx][]BitRange{ before: map[ChunkIdx][]BitRange{
BaseChunkIdx: {}, BaseChunkIdx: {},
}, },
scav: map[ChunkIdx][]BitRange{
BaseChunkIdx: {{0, 1}, {PallocChunkPages - 1, 1}},
},
hits: []hit{ hits: []hit{
{PallocChunkPages, PageBase(BaseChunkIdx, 0)}, {PallocChunkPages, PageBase(BaseChunkIdx, 0), 2 * PageSize},
{PallocChunkPages, 0}, {PallocChunkPages, 0, 0},
{1, 0}, {1, 0, 0},
}, },
after: map[ChunkIdx][]BitRange{ after: map[ChunkIdx][]BitRange{
BaseChunkIdx: {{0, PallocChunkPages}}, BaseChunkIdx: {{0, PallocChunkPages}},
@ -190,10 +225,14 @@ func TestPageAllocAlloc(t *testing.T) {
BaseChunkIdx: {{0, PallocChunkPages / 2}}, BaseChunkIdx: {{0, PallocChunkPages / 2}},
BaseChunkIdx + 1: {{PallocChunkPages / 2, PallocChunkPages / 2}}, BaseChunkIdx + 1: {{PallocChunkPages / 2, PallocChunkPages / 2}},
}, },
scav: map[ChunkIdx][]BitRange{
BaseChunkIdx: {},
BaseChunkIdx + 1: {{3, 100}},
},
hits: []hit{ hits: []hit{
{PallocChunkPages, PageBase(BaseChunkIdx, PallocChunkPages/2)}, {PallocChunkPages, PageBase(BaseChunkIdx, PallocChunkPages/2), 100 * PageSize},
{PallocChunkPages, 0}, {PallocChunkPages, 0, 0},
{1, 0}, {1, 0, 0},
}, },
after: map[ChunkIdx][]BitRange{ after: map[ChunkIdx][]BitRange{
BaseChunkIdx: {{0, PallocChunkPages}}, BaseChunkIdx: {{0, PallocChunkPages}},
@ -205,10 +244,14 @@ func TestPageAllocAlloc(t *testing.T) {
BaseChunkIdx: {{0, PallocChunkPages / 2}}, BaseChunkIdx: {{0, PallocChunkPages / 2}},
BaseChunkIdx + 1: {}, BaseChunkIdx + 1: {},
}, },
scav: map[ChunkIdx][]BitRange{
BaseChunkIdx: {{0, PallocChunkPages}},
BaseChunkIdx + 1: {{0, PallocChunkPages}},
},
hits: []hit{ hits: []hit{
{PallocChunkPages + 1, PageBase(BaseChunkIdx, PallocChunkPages/2)}, {PallocChunkPages + 1, PageBase(BaseChunkIdx, PallocChunkPages/2), (PallocChunkPages + 1) * PageSize},
{PallocChunkPages, 0}, {PallocChunkPages, 0, 0},
{1, PageBase(BaseChunkIdx+1, PallocChunkPages/2+1)}, {1, PageBase(BaseChunkIdx+1, PallocChunkPages/2+1), PageSize},
}, },
after: map[ChunkIdx][]BitRange{ after: map[ChunkIdx][]BitRange{
BaseChunkIdx: {{0, PallocChunkPages}}, BaseChunkIdx: {{0, PallocChunkPages}},
@ -220,10 +263,14 @@ func TestPageAllocAlloc(t *testing.T) {
BaseChunkIdx: {}, BaseChunkIdx: {},
BaseChunkIdx + 1: {}, BaseChunkIdx + 1: {},
}, },
scav: map[ChunkIdx][]BitRange{
BaseChunkIdx: {},
BaseChunkIdx + 1: {},
},
hits: []hit{ hits: []hit{
{PallocChunkPages * 2, PageBase(BaseChunkIdx, 0)}, {PallocChunkPages * 2, PageBase(BaseChunkIdx, 0), 0},
{PallocChunkPages * 2, 0}, {PallocChunkPages * 2, 0, 0},
{1, 0}, {1, 0, 0},
}, },
after: map[ChunkIdx][]BitRange{ after: map[ChunkIdx][]BitRange{
BaseChunkIdx: {{0, PallocChunkPages}}, BaseChunkIdx: {{0, PallocChunkPages}},
@ -236,10 +283,15 @@ func TestPageAllocAlloc(t *testing.T) {
BaseChunkIdx + 0x100: {}, BaseChunkIdx + 0x100: {},
BaseChunkIdx + 0x101: {}, BaseChunkIdx + 0x101: {},
}, },
scav: map[ChunkIdx][]BitRange{
BaseChunkIdx: {{0, PallocChunkPages}},
BaseChunkIdx + 0x100: {},
BaseChunkIdx + 0x101: {},
},
hits: []hit{ hits: []hit{
{PallocChunkPages * 2, PageBase(BaseChunkIdx+0x100, 0)}, {PallocChunkPages * 2, PageBase(BaseChunkIdx+0x100, 0), 0},
{21, PageBase(BaseChunkIdx, 0)}, {21, PageBase(BaseChunkIdx, 0), 21 * PageSize},
{1, PageBase(BaseChunkIdx, 21)}, {1, PageBase(BaseChunkIdx, 21), PageSize},
}, },
after: map[ChunkIdx][]BitRange{ after: map[ChunkIdx][]BitRange{
BaseChunkIdx: {{0, 22}}, BaseChunkIdx: {{0, 22}},
@ -253,10 +305,15 @@ func TestPageAllocAlloc(t *testing.T) {
BaseChunkIdx + 1: {}, BaseChunkIdx + 1: {},
BaseChunkIdx + 2: {{PallocChunkPages / 2, PallocChunkPages / 2}}, BaseChunkIdx + 2: {{PallocChunkPages / 2, PallocChunkPages / 2}},
}, },
scav: map[ChunkIdx][]BitRange{
BaseChunkIdx: {{0, 7}},
BaseChunkIdx + 1: {{3, 5}, {121, 10}},
BaseChunkIdx + 2: {{PallocChunkPages/2 + 12, 2}},
},
hits: []hit{ hits: []hit{
{PallocChunkPages * 2, PageBase(BaseChunkIdx, PallocChunkPages/2)}, {PallocChunkPages * 2, PageBase(BaseChunkIdx, PallocChunkPages/2), 15 * PageSize},
{PallocChunkPages * 2, 0}, {PallocChunkPages * 2, 0, 0},
{1, 0}, {1, 0, 0},
}, },
after: map[ChunkIdx][]BitRange{ after: map[ChunkIdx][]BitRange{
BaseChunkIdx: {{0, PallocChunkPages}}, BaseChunkIdx: {{0, PallocChunkPages}},
@ -271,10 +328,16 @@ func TestPageAllocAlloc(t *testing.T) {
BaseChunkIdx + 2: {{0, PallocChunkPages * 3 / 4}}, BaseChunkIdx + 2: {{0, PallocChunkPages * 3 / 4}},
BaseChunkIdx + 3: {{0, 0}}, BaseChunkIdx + 3: {{0, 0}},
}, },
scav: map[ChunkIdx][]BitRange{
BaseChunkIdx: {{0, PallocChunkPages}},
BaseChunkIdx + 1: {{PallocChunkPages / 2, PallocChunkPages/4 + 1}},
BaseChunkIdx + 2: {{PallocChunkPages / 3, 1}},
BaseChunkIdx + 3: {{PallocChunkPages * 2 / 3, 1}},
},
hits: []hit{ hits: []hit{
{PallocChunkPages * 5 / 4, PageBase(BaseChunkIdx+2, PallocChunkPages*3/4)}, {PallocChunkPages * 5 / 4, PageBase(BaseChunkIdx+2, PallocChunkPages*3/4), PageSize},
{PallocChunkPages * 5 / 4, 0}, {PallocChunkPages * 5 / 4, 0, 0},
{1, PageBase(BaseChunkIdx+1, PallocChunkPages*3/4)}, {1, PageBase(BaseChunkIdx+1, PallocChunkPages*3/4), PageSize},
}, },
after: map[ChunkIdx][]BitRange{ after: map[ChunkIdx][]BitRange{
BaseChunkIdx: {{0, PallocChunkPages}}, BaseChunkIdx: {{0, PallocChunkPages}},
@ -294,10 +357,20 @@ func TestPageAllocAlloc(t *testing.T) {
BaseChunkIdx + 6: {}, BaseChunkIdx + 6: {},
BaseChunkIdx + 7: {}, BaseChunkIdx + 7: {},
}, },
scav: map[ChunkIdx][]BitRange{
BaseChunkIdx: {{50, 1}},
BaseChunkIdx + 1: {{31, 1}},
BaseChunkIdx + 2: {{7, 1}},
BaseChunkIdx + 3: {{200, 1}},
BaseChunkIdx + 4: {{3, 1}},
BaseChunkIdx + 5: {{51, 1}},
BaseChunkIdx + 6: {{20, 1}},
BaseChunkIdx + 7: {{1, 1}},
},
hits: []hit{ hits: []hit{
{PallocChunkPages*7 + 5, PageBase(BaseChunkIdx, 0)}, {PallocChunkPages*7 + 5, PageBase(BaseChunkIdx, 0), 8 * PageSize},
{PallocChunkPages*7 + 5, 0}, {PallocChunkPages*7 + 5, 0, 0},
{1, PageBase(BaseChunkIdx+7, 5)}, {1, PageBase(BaseChunkIdx+7, 5), 0},
}, },
after: map[ChunkIdx][]BitRange{ after: map[ChunkIdx][]BitRange{
BaseChunkIdx: {{0, PallocChunkPages}}, BaseChunkIdx: {{0, PallocChunkPages}},
@ -314,15 +387,19 @@ func TestPageAllocAlloc(t *testing.T) {
for name, v := range tests { for name, v := range tests {
v := v v := v
t.Run(name, func(t *testing.T) { t.Run(name, func(t *testing.T) {
b := NewPageAlloc(v.before, nil) b := NewPageAlloc(v.before, v.scav)
defer FreePageAlloc(b) defer FreePageAlloc(b)
for iter, i := range v.hits { for iter, i := range v.hits {
if a := b.Alloc(i.npages); a != i.base { a, s := b.Alloc(i.npages)
t.Fatalf("bad alloc #%d: want 0x%x, got 0x%x", iter+1, i.base, a) if a != i.base {
t.Fatalf("bad alloc #%d: want base 0x%x, got 0x%x", iter+1, i.base, a)
}
if s != i.scav {
t.Fatalf("bad alloc #%d: want scav %d, got %d", iter+1, i.scav, s)
} }
} }
want := NewPageAlloc(v.after, nil) want := NewPageAlloc(v.after, v.scav)
defer FreePageAlloc(want) defer FreePageAlloc(want)
checkPageAlloc(t, want, b) checkPageAlloc(t, want, b)
@ -346,13 +423,13 @@ func TestPageAllocExhaust(t *testing.T) {
nAlloc := (PallocChunkPages * 4) / int(npages) nAlloc := (PallocChunkPages * 4) / int(npages)
for i := 0; i < nAlloc; i++ { for i := 0; i < nAlloc; i++ {
addr := PageBase(BaseChunkIdx, uint(i)*uint(npages)) addr := PageBase(BaseChunkIdx, uint(i)*uint(npages))
if a := b.Alloc(npages); a != addr { if a, _ := b.Alloc(npages); a != addr {
t.Fatalf("bad alloc #%d: want 0x%x, got 0x%x", i+1, addr, a) t.Fatalf("bad alloc #%d: want 0x%x, got 0x%x", i+1, addr, a)
} }
} }
// Check to make sure the next allocation fails. // Check to make sure the next allocation fails.
if a := b.Alloc(npages); a != 0 { if a, _ := b.Alloc(npages); a != 0 {
t.Fatalf("bad alloc #%d: want 0, got 0x%x", nAlloc, a) t.Fatalf("bad alloc #%d: want 0, got 0x%x", nAlloc, a)
} }
@ -651,7 +728,7 @@ func TestPageAllocAllocAndFree(t *testing.T) {
for iter, i := range v.hits { for iter, i := range v.hits {
if i.alloc { if i.alloc {
if a := b.Alloc(i.npages); a != i.base { if a, _ := b.Alloc(i.npages); a != i.base {
t.Fatalf("bad alloc #%d: want 0x%x, got 0x%x", iter+1, i.base, a) t.Fatalf("bad alloc #%d: want 0x%x, got 0x%x", iter+1, i.base, a)
} }
} else { } else {

20
src/runtime/mpallocbits.go
View File

@ -88,6 +88,26 @@ func (b *pageBits) clearAll() {
} }
} }
// popcntRange counts the number of set bits in the
// range [i, i+n).
func (b *pageBits) popcntRange(i, n uint) (s uint) {
if n == 1 {
return uint((b[i/64] >> (i % 64)) & 1)
}
_ = b[i/64]
j := i + n - 1
if i/64 == j/64 {
return uint(bits.OnesCount64((b[i/64] >> (i % 64)) & ((1 << n) - 1)))
}
_ = b[j/64]
s += uint(bits.OnesCount64(b[i/64] >> (i % 64)))
for k := i/64 + 1; k < j/64; k++ {
s += uint(bits.OnesCount64(b[k]))
}
s += uint(bits.OnesCount64(b[j/64] & ((1 << (j%64 + 1)) - 1)))
return
}
// pallocBits is a bitmap that tracks page allocations for at most one // pallocBits is a bitmap that tracks page allocations for at most one
// palloc chunk. // palloc chunk.
// //

92
src/runtime/mpallocbits_test.go
View File

@ -113,6 +113,98 @@ func checkPallocSum(t *testing.T, got, want PallocSum) {
} }
} }
func TestMallocBitsPopcntRange(t *testing.T) {
type test struct {
i, n uint // bit range to popcnt over.
want uint // expected popcnt result on that range.
}
tests := map[string]struct {
init []BitRange // bit ranges to set to 1 in the bitmap.
tests []test // a set of popcnt tests to run over the bitmap.
}{
"None": {
tests: []test{
{0, 1, 0},
{5, 3, 0},
{2, 11, 0},
{PallocChunkPages/4 + 1, PallocChunkPages / 2, 0},
{0, PallocChunkPages, 0},
},
},
"All": {
init: []BitRange{{0, PallocChunkPages}},
tests: []test{
{0, 1, 1},
{5, 3, 3},
{2, 11, 11},
{PallocChunkPages/4 + 1, PallocChunkPages / 2, PallocChunkPages / 2},
{0, PallocChunkPages, PallocChunkPages},
},
},
"Half": {
init: []BitRange{{PallocChunkPages / 2, PallocChunkPages / 2}},
tests: []test{
{0, 1, 0},
{5, 3, 0},
{2, 11, 0},
{PallocChunkPages/2 - 1, 1, 0},
{PallocChunkPages / 2, 1, 1},
{PallocChunkPages/2 + 10, 1, 1},
{PallocChunkPages/2 - 1, 2, 1},
{PallocChunkPages / 4, PallocChunkPages / 4, 0},
{PallocChunkPages / 4, PallocChunkPages/4 + 1, 1},
{PallocChunkPages/4 + 1, PallocChunkPages / 2, PallocChunkPages/4 + 1},
{0, PallocChunkPages, PallocChunkPages / 2},
},
},
"OddBound": {
init: []BitRange{{0, 111}},
tests: []test{
{0, 1, 1},
{5, 3, 3},
{2, 11, 11},
{110, 2, 1},
{99, 50, 12},
{110, 1, 1},
{111, 1, 0},
{99, 1, 1},
{120, 1, 0},
{PallocChunkPages / 2, PallocChunkPages / 2, 0},
{0, PallocChunkPages, 111},
},
},
"Scattered": {
init: []BitRange{
{1, 3}, {5, 1}, {7, 1}, {10, 2}, {13, 1}, {15, 4},
{21, 1}, {23, 1}, {26, 2}, {30, 5}, {36, 2}, {40, 3},
{44, 6}, {51, 1}, {53, 2}, {58, 3}, {63, 1}, {67, 2},
{71, 10}, {84, 1}, {89, 7}, {99, 2}, {103, 1}, {107, 2},
{111, 1}, {113, 1}, {115, 1}, {118, 1}, {120, 2}, {125, 5},
},
tests: []test{
{0, 11, 6},
{0, 64, 39},
{13, 64, 40},
{64, 64, 34},
{0, 128, 73},
{1, 128, 74},
{0, PallocChunkPages, 75},
},
},
}
for name, v := range tests {
v := v
t.Run(name, func(t *testing.T) {
b := makePallocBits(v.init)
for _, h := range v.tests {
if got := b.PopcntRange(h.i, h.n); got != h.want {
t.Errorf("bad popcnt (i=%d, n=%d): got %d, want %d", h.i, h.n, got, h.want)
}
}
})
}
}
// Ensures computing bit summaries works as expected by generating random // Ensures computing bit summaries works as expected by generating random
// bitmaps and checking against a reference implementation. // bitmaps and checking against a reference implementation.
func TestPallocBitsSummarizeRandom(t *testing.T) { func TestPallocBitsSummarizeRandom(t *testing.T) {