diff --git a/src/go/build/deps_test.go b/src/go/build/deps_test.go index 2ef90a977e1..eca6c6efba9 100644 --- a/src/go/build/deps_test.go +++ b/src/go/build/deps_test.go @@ -36,7 +36,8 @@ var pkgDeps = map[string][]string{ // L0 is the lowest level, core, nearly unavoidable packages. "errors": {"runtime", "internal/reflectlite"}, "io": {"errors", "sync", "sync/atomic"}, - "runtime": {"unsafe", "runtime/internal/atomic", "runtime/internal/sys", "runtime/internal/math", "internal/cpu", "internal/bytealg"}, + "math/bits": {"unsafe"}, + "runtime": {"math/bits", "unsafe", "runtime/internal/atomic", "runtime/internal/sys", "runtime/internal/math", "internal/cpu", "internal/bytealg"}, "runtime/internal/sys": {}, "runtime/internal/atomic": {"unsafe", "internal/cpu"}, "runtime/internal/math": {"runtime/internal/sys"}, @@ -64,7 +65,6 @@ var pkgDeps = map[string][]string{ // L1 adds simple functions and strings processing, // but not Unicode tables. "math": {"internal/cpu", "unsafe", "math/bits"}, - "math/bits": {"unsafe"}, "math/cmplx": {"math"}, "math/rand": {"L0", "math"}, "strconv": {"L0", "unicode/utf8", "math", "math/bits"}, diff --git a/src/runtime/export_test.go b/src/runtime/export_test.go index 3c1b4db7504..27692791107 100644 --- a/src/runtime/export_test.go +++ b/src/runtime/export_test.go @@ -730,3 +730,66 @@ func RunGetgThreadSwitchTest() { panic("g1 != g3") } } + +const ( + PallocChunkPages = pallocChunkPages +) + +// Expose pallocBits for testing. +type PallocBits pallocBits + +func (b *PallocBits) Find(npages uintptr, searchIdx uint) (uint, uint) { + return (*pallocBits)(b).find(npages, searchIdx) +} +func (b *PallocBits) AllocRange(i, n uint) { (*pallocBits)(b).allocRange(i, n) } +func (b *PallocBits) Free(i, n uint) { (*pallocBits)(b).free(i, n) } + +// Expose non-trivial helpers for testing. +func FindBitRange64(c uint64, n uint) uint { return findBitRange64(c, n) } + +// Given two PallocBits, returns a set of bit ranges where +// they differ. +func DiffPallocBits(a, b *PallocBits) []BitRange { + ba := (*pageBits)(a) + bb := (*pageBits)(b) + + var d []BitRange + base, size := uint(0), uint(0) + for i := uint(0); i < uint(len(ba))*64; i++ { + if ba.get(i) != bb.get(i) { + if size == 0 { + base = i + } + size++ + } else { + if size != 0 { + d = append(d, BitRange{base, size}) + } + size = 0 + } + } + if size != 0 { + d = append(d, BitRange{base, size}) + } + return d +} + +// StringifyPallocBits gets the bits in the bit range r from b, +// and returns a string containing the bits as ASCII 0 and 1 +// characters. +func StringifyPallocBits(b *PallocBits, r BitRange) string { + str := "" + for j := r.I; j < r.I+r.N; j++ { + if (*pageBits)(b).get(j) != 0 { + str += "1" + } else { + str += "0" + } + } + return str +} + +// BitRange represents a range over a bitmap. +type BitRange struct { + I, N uint // bit index and length in bits +} diff --git a/src/runtime/mpallocbits.go b/src/runtime/mpallocbits.go new file mode 100644 index 00000000000..fe8cde92256 --- /dev/null +++ b/src/runtime/mpallocbits.go @@ -0,0 +1,260 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +import ( + "math/bits" +) + +// pageBits is a bitmap representing one bit per page in a palloc chunk. +type pageBits [pallocChunkPages / 64]uint64 + +// get returns the value of the i'th bit in the bitmap. +func (b *pageBits) get(i uint) uint { + return uint((b[i/64] >> (i % 64)) & 1) +} + +// set sets bit i of pageBits. +func (b *pageBits) set(i uint) { + b[i/64] |= 1 << (i % 64) +} + +// setRange sets bits in the range [i, i+n). +func (b *pageBits) setRange(i, n uint) { + _ = b[i/64] + if n == 1 { + // Fast path for the n == 1 case. + b.set(i) + return + } + // Set bits [i, j]. + j := i + n - 1 + if i/64 == j/64 { + b[i/64] |= ((uint64(1) << n) - 1) << (i % 64) + return + } + _ = b[j/64] + // Set leading bits. + b[i/64] |= ^uint64(0) << (i % 64) + for k := i/64 + 1; k < j/64; k++ { + b[k] = ^uint64(0) + } + // Set trailing bits. + b[j/64] |= (uint64(1) << (j%64 + 1)) - 1 +} + +// setAll sets all the bits of b. +func (b *pageBits) setAll() { + for i := range b { + b[i] = ^uint64(0) + } +} + +// clear clears bit i of pageBits. +func (b *pageBits) clear(i uint) { + b[i/64] &^= 1 << (i % 64) +} + +// clearRange clears bits in the range [i, i+n). +func (b *pageBits) clearRange(i, n uint) { + _ = b[i/64] + if n == 1 { + // Fast path for the n == 1 case. + b.clear(i) + return + } + // Clear bits [i, j]. + j := i + n - 1 + if i/64 == j/64 { + b[i/64] &^= ((uint64(1) << n) - 1) << (i % 64) + return + } + _ = b[j/64] + // Clear leading bits. + b[i/64] &^= ^uint64(0) << (i % 64) + for k := i/64 + 1; k < j/64; k++ { + b[k] = 0 + } + // Clear trailing bits. + b[j/64] &^= (uint64(1) << (j%64 + 1)) - 1 +} + +// clearAll frees all the bits of b. +func (b *pageBits) clearAll() { + for i := range b { + b[i] = 0 + } +} + +// pallocBits is a bitmap that tracks page allocations for at most one +// palloc chunk. +// +// The precise representation is an implementation detail, but for the +// sake of documentation, 0s are free pages and 1s are allocated pages. +type pallocBits pageBits + +// find searches for npages contiguous free pages in pallocBits and returns +// the index where that run starts, as well as the index of the first free page +// it found in the search. searchIdx represents the first known free page and +// where to begin the search from. +// +// If find fails to find any free space, it returns an index of ^uint(0) and +// the new searchIdx should be ignored. +// +// The returned searchIdx is always the index of the first free page found +// in this bitmap during the search, except if npages == 1, in which +// case it will be the index just after the first free page, because the +// index returned as the first result is assumed to be allocated and so +// represents a minor optimization for that case. +func (b *pallocBits) find(npages uintptr, searchIdx uint) (uint, uint) { + if npages == 1 { + addr := b.find1(searchIdx) + // Return a searchIdx of addr + 1 since we assume addr will be + // allocated. + return addr, addr + 1 + } else if npages <= 64 { + return b.findSmallN(npages, searchIdx) + } + return b.findLargeN(npages, searchIdx) +} + +// find1 is a helper for find which searches for a single free page +// in the pallocBits and returns the index. +// +// See find for an explanation of the searchIdx parameter. +func (b *pallocBits) find1(searchIdx uint) uint { + for i := searchIdx / 64; i < uint(len(b)); i++ { + x := b[i] + if x == ^uint64(0) { + continue + } + return i*64 + uint(bits.TrailingZeros64(^x)) + } + return ^uint(0) +} + +// findSmallN is a helper for find which searches for npages contiguous free pages +// in this pallocBits and returns the index where that run of contiguous pages +// starts as well as the index of the first free page it finds in its search. +// +// See find for an explanation of the searchIdx parameter. +// +// Returns a ^uint(0) index on failure and the new searchIdx should be ignored. +// +// findSmallN assumes npages <= 64, where any such contiguous run of pages +// crosses at most one aligned 64-bit boundary in the bits. +func (b *pallocBits) findSmallN(npages uintptr, searchIdx uint) (uint, uint) { + end, newSearchIdx := uint(0), ^uint(0) + for i := searchIdx / 64; i < uint(len(b)); i++ { + bi := b[i] + if bi == ^uint64(0) { + end = 0 + continue + } + // First see if we can pack our allocation in the trailing + // zeros plus the end of the last 64 bits. + start := uint(bits.TrailingZeros64(bi)) + if newSearchIdx == ^uint(0) { + // The new searchIdx is going to be at these 64 bits after any + // 1s we file, so count trailing 1s. + newSearchIdx = i*64 + uint(bits.TrailingZeros64(^bi)) + } + if end+start >= uint(npages) { + return i*64 - end, newSearchIdx + } + // Next, check the interior of the 64-bit chunk. + j := findBitRange64(^bi, uint(npages)) + if j < 64 { + return i*64 + j, newSearchIdx + } + end = uint(bits.LeadingZeros64(bi)) + } + return ^uint(0), newSearchIdx +} + +// findLargeN is a helper for find which searches for npages contiguous free pages +// in this pallocBits and returns the index where that run starts, as well as the +// index of the first free page it found it its search. +// +// See alloc for an explanation of the searchIdx parameter. +// +// Returns a ^uint(0) index on failure and the new searchIdx should be ignored. +// +// findLargeN assumes npages > 64, where any such run of free pages +// crosses at least one aligned 64-bit boundary in the bits. +func (b *pallocBits) findLargeN(npages uintptr, searchIdx uint) (uint, uint) { + start, size, newSearchIdx := ^uint(0), uint(0), ^uint(0) + for i := searchIdx / 64; i < uint(len(b)); i++ { + x := b[i] + if x == ^uint64(0) { + size = 0 + continue + } + if newSearchIdx == ^uint(0) { + // The new searchIdx is going to be at these 64 bits after any + // 1s we file, so count trailing 1s. + newSearchIdx = i*64 + uint(bits.TrailingZeros64(^x)) + } + if size == 0 { + size = uint(bits.LeadingZeros64(x)) + start = i*64 + 64 - size + continue + } + s := uint(bits.TrailingZeros64(x)) + if s+size >= uint(npages) { + size += s + return start, newSearchIdx + } + if s < 64 { + size = uint(bits.LeadingZeros64(x)) + start = i*64 + 64 - size + continue + } + size += 64 + } + if size < uint(npages) { + return ^uint(0), newSearchIdx + } + return start, newSearchIdx +} + +// allocRange allocates the range [i, i+n). +func (b *pallocBits) allocRange(i, n uint) { + (*pageBits)(b).setRange(i, n) +} + +// allocAll allocates all the bits of b. +func (b *pallocBits) allocAll() { + (*pageBits)(b).setAll() +} + +// free1 frees a single page in the pallocBits at i. +func (b *pallocBits) free1(i uint) { + (*pageBits)(b).clear(i) +} + +// free frees the range [i, i+n) of pages in the pallocBits. +func (b *pallocBits) free(i, n uint) { + (*pageBits)(b).clearRange(i, n) +} + +// freeAll frees all the bits of b. +func (b *pallocBits) freeAll() { + (*pageBits)(b).clearAll() +} + +// findBitRange64 returns the bit index of the first set of +// n consecutive 1 bits. If no consecutive set of 1 bits of +// size n may be found in c, then it returns an integer >= 64. +func findBitRange64(c uint64, n uint) uint { + i := uint(0) + cont := uint(bits.TrailingZeros64(^c)) + for cont < n && i < 64 { + i += cont + i += uint(bits.TrailingZeros64(c >> i)) + cont = uint(bits.TrailingZeros64(^(c >> i))) + } + return i +} diff --git a/src/runtime/mpallocbits_test.go b/src/runtime/mpallocbits_test.go new file mode 100644 index 00000000000..2ac7899c36b --- /dev/null +++ b/src/runtime/mpallocbits_test.go @@ -0,0 +1,290 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime_test + +import ( + . "runtime" + "testing" +) + +// Ensures that got and want are the same, and if not, reports +// detailed diff information. +func checkPallocBits(t *testing.T, got, want *PallocBits) { + d := DiffPallocBits(got, want) + if len(d) != 0 { + t.Errorf("%d range(s) different", len(d)) + for _, bits := range d { + t.Logf("\t@ bit index %d", bits.I) + t.Logf("\t| got: %s", StringifyPallocBits(got, bits)) + t.Logf("\t| want: %s", StringifyPallocBits(want, bits)) + } + } +} + +// makePallocBits produces an initialized PallocBits by setting +// the ranges in s to 1 and the rest to zero. +func makePallocBits(s []BitRange) *PallocBits { + b := new(PallocBits) + for _, v := range s { + b.AllocRange(v.I, v.N) + } + return b +} + +// Ensures that PallocBits.AllocRange works, which is a fundamental +// method used for testing and initialization since it's used by +// makePallocBits. +func TestPallocBitsAllocRange(t *testing.T) { + test := func(t *testing.T, i, n uint, want *PallocBits) { + checkPallocBits(t, makePallocBits([]BitRange{{i, n}}), want) + } + t.Run("OneLow", func(t *testing.T) { + want := new(PallocBits) + want[0] = 0x1 + test(t, 0, 1, want) + }) + t.Run("OneHigh", func(t *testing.T) { + want := new(PallocBits) + want[PallocChunkPages/64-1] = 1 << 63 + test(t, PallocChunkPages-1, 1, want) + }) + t.Run("Inner", func(t *testing.T) { + want := new(PallocBits) + want[2] = 0x3e + test(t, 129, 5, want) + }) + t.Run("Aligned", func(t *testing.T) { + want := new(PallocBits) + want[2] = ^uint64(0) + want[3] = ^uint64(0) + test(t, 128, 128, want) + }) + t.Run("Begin", func(t *testing.T) { + want := new(PallocBits) + want[0] = ^uint64(0) + want[1] = ^uint64(0) + want[2] = ^uint64(0) + want[3] = ^uint64(0) + want[4] = ^uint64(0) + want[5] = 0x1 + test(t, 0, 321, want) + }) + t.Run("End", func(t *testing.T) { + want := new(PallocBits) + want[PallocChunkPages/64-1] = ^uint64(0) + want[PallocChunkPages/64-2] = ^uint64(0) + want[PallocChunkPages/64-3] = ^uint64(0) + want[PallocChunkPages/64-4] = 1 << 63 + test(t, PallocChunkPages-(64*3+1), 64*3+1, want) + }) + t.Run("All", func(t *testing.T) { + want := new(PallocBits) + for i := range want { + want[i] = ^uint64(0) + } + test(t, 0, PallocChunkPages, want) + }) +} + +// Inverts every bit in the PallocBits. +func invertPallocBits(b *PallocBits) { + for i := range b { + b[i] = ^b[i] + } +} + +// Ensures page allocation works. +func TestPallocBitsAlloc(t *testing.T) { + tests := map[string]struct { + before []BitRange + after []BitRange + npages uintptr + hits []uint + }{ + "AllFree1": { + npages: 1, + hits: []uint{0, 1, 2, 3, 4, 5}, + after: []BitRange{{0, 6}}, + }, + "AllFree2": { + npages: 2, + hits: []uint{0, 2, 4, 6, 8, 10}, + after: []BitRange{{0, 12}}, + }, + "AllFree5": { + npages: 5, + hits: []uint{0, 5, 10, 15, 20}, + after: []BitRange{{0, 25}}, + }, + "AllFree64": { + npages: 64, + hits: []uint{0, 64, 128}, + after: []BitRange{{0, 192}}, + }, + "AllFree65": { + npages: 65, + hits: []uint{0, 65, 130}, + after: []BitRange{{0, 195}}, + }, + "SomeFree64": { + before: []BitRange{{0, 32}, {64, 32}, {100, PallocChunkPages - 100}}, + npages: 64, + hits: []uint{^uint(0)}, + after: []BitRange{{0, 32}, {64, 32}, {100, PallocChunkPages - 100}}, + }, + "NoneFree1": { + before: []BitRange{{0, PallocChunkPages}}, + npages: 1, + hits: []uint{^uint(0), ^uint(0)}, + after: []BitRange{{0, PallocChunkPages}}, + }, + "NoneFree2": { + before: []BitRange{{0, PallocChunkPages}}, + npages: 2, + hits: []uint{^uint(0), ^uint(0)}, + after: []BitRange{{0, PallocChunkPages}}, + }, + "NoneFree5": { + before: []BitRange{{0, PallocChunkPages}}, + npages: 5, + hits: []uint{^uint(0), ^uint(0)}, + after: []BitRange{{0, PallocChunkPages}}, + }, + "NoneFree65": { + before: []BitRange{{0, PallocChunkPages}}, + npages: 65, + hits: []uint{^uint(0), ^uint(0)}, + after: []BitRange{{0, PallocChunkPages}}, + }, + "ExactFit1": { + before: []BitRange{{0, PallocChunkPages/2 - 3}, {PallocChunkPages/2 - 2, PallocChunkPages/2 + 2}}, + npages: 1, + hits: []uint{PallocChunkPages/2 - 3, ^uint(0)}, + after: []BitRange{{0, PallocChunkPages}}, + }, + "ExactFit2": { + before: []BitRange{{0, PallocChunkPages/2 - 3}, {PallocChunkPages/2 - 1, PallocChunkPages/2 + 1}}, + npages: 2, + hits: []uint{PallocChunkPages/2 - 3, ^uint(0)}, + after: []BitRange{{0, PallocChunkPages}}, + }, + "ExactFit5": { + before: []BitRange{{0, PallocChunkPages/2 - 3}, {PallocChunkPages/2 + 2, PallocChunkPages/2 - 2}}, + npages: 5, + hits: []uint{PallocChunkPages/2 - 3, ^uint(0)}, + after: []BitRange{{0, PallocChunkPages}}, + }, + "ExactFit65": { + before: []BitRange{{0, PallocChunkPages/2 - 31}, {PallocChunkPages/2 + 34, PallocChunkPages/2 - 34}}, + npages: 65, + hits: []uint{PallocChunkPages/2 - 31, ^uint(0)}, + after: []BitRange{{0, PallocChunkPages}}, + }, + "SomeFree161": { + before: []BitRange{{0, 185}, {331, 1}}, + npages: 161, + hits: []uint{332}, + after: []BitRange{{0, 185}, {331, 162}}, + }, + } + for name, v := range tests { + v := v + t.Run(name, func(t *testing.T) { + b := makePallocBits(v.before) + for iter, i := range v.hits { + a, _ := b.Find(v.npages, 0) + if i != a { + t.Fatalf("find #%d picked wrong index: want %d, got %d", iter+1, i, a) + } + if i != ^uint(0) { + b.AllocRange(a, uint(v.npages)) + } + } + want := makePallocBits(v.after) + checkPallocBits(t, b, want) + }) + } +} + +// Ensures page freeing works. +func TestPallocBitsFree(t *testing.T) { + tests := map[string]struct { + beforeInv []BitRange + afterInv []BitRange + frees []uint + npages uintptr + }{ + "SomeFree": { + npages: 1, + beforeInv: []BitRange{{0, 32}, {64, 32}, {100, 1}}, + frees: []uint{32}, + afterInv: []BitRange{{0, 33}, {64, 32}, {100, 1}}, + }, + "NoneFree1": { + npages: 1, + frees: []uint{0, 1, 2, 3, 4, 5}, + afterInv: []BitRange{{0, 6}}, + }, + "NoneFree2": { + npages: 2, + frees: []uint{0, 2, 4, 6, 8, 10}, + afterInv: []BitRange{{0, 12}}, + }, + "NoneFree5": { + npages: 5, + frees: []uint{0, 5, 10, 15, 20}, + afterInv: []BitRange{{0, 25}}, + }, + "NoneFree64": { + npages: 64, + frees: []uint{0, 64, 128}, + afterInv: []BitRange{{0, 192}}, + }, + "NoneFree65": { + npages: 65, + frees: []uint{0, 65, 130}, + afterInv: []BitRange{{0, 195}}, + }, + } + for name, v := range tests { + v := v + t.Run(name, func(t *testing.T) { + b := makePallocBits(v.beforeInv) + invertPallocBits(b) + for _, i := range v.frees { + b.Free(i, uint(v.npages)) + } + want := makePallocBits(v.afterInv) + invertPallocBits(want) + checkPallocBits(t, b, want) + }) + } +} + +func TestFindBitRange64(t *testing.T) { + check := func(x uint64, n uint, result uint) { + i := FindBitRange64(x, n) + if result == ^uint(0) && i < 64 { + t.Errorf("case (%016x, %d): got %d, want failure", x, n, i) + } else if result != ^uint(0) && i != result { + t.Errorf("case (%016x, %d): got %d, want %d", x, n, i, result) + } + } + for i := uint(0); i <= 64; i++ { + check(^uint64(0), i, 0) + } + check(0, 0, 0) + for i := uint(1); i <= 64; i++ { + check(0, i, ^uint(0)) + } + check(0x8000000000000000, 1, 63) + check(0xc000010001010000, 2, 62) + check(0xc000010001030000, 2, 16) + check(0xe000030001030000, 3, 61) + check(0xe000030001070000, 3, 16) + check(0xffff03ff01070000, 16, 48) + check(0xffff03ff0107ffff, 16, 0) + check(0x0fff03ff01079fff, 16, ^uint(0)) +}