runtime: FreeBSD fast clock_gettime HPET timecounter support

This is a followup for CL 93156.

Fixes #22942.

Change-Id: Ic6e2de44011d041b91454353a6f2e3b0cf590060
Reviewed-on: https://go-review.googlesource.com/108095
Run-TryBot: Ian Lance Taylor <iant@golang.org>
Reviewed-by: Ian Lance Taylor <iant@golang.org>

src/runtime/defs_freebsd.go

@@ -53,6 +53,7 @@ const (
 	PROT_EXEC  = C.PROT_EXEC
 
 	MAP_ANON    = C.MAP_ANON
+	MAP_SHARED  = C.MAP_SHARED
 	MAP_PRIVATE = C.MAP_PRIVATE
 	MAP_FIXED   = C.MAP_FIXED
 

src/runtime/defs_freebsd_386.go

@@ -22,6 +22,7 @@ const (
 	_PROT_EXEC  = 0x4
 
 	_MAP_ANON    = 0x1000
+	_MAP_SHARED  = 0x1
 	_MAP_PRIVATE = 0x2
 	_MAP_FIXED   = 0x10
 

src/runtime/defs_freebsd_amd64.go

@@ -22,6 +22,7 @@ const (
 	_PROT_EXEC  = 0x4
 
 	_MAP_ANON    = 0x1000
+	_MAP_SHARED  = 0x1
 	_MAP_PRIVATE = 0x2
 	_MAP_FIXED   = 0x10
 

src/runtime/defs_freebsd_arm.go

@@ -22,6 +22,7 @@ const (
 	_PROT_EXEC  = 0x4
 
 	_MAP_ANON    = 0x1000
+	_MAP_SHARED  = 0x1
 	_MAP_PRIVATE = 0x2
 	_MAP_FIXED   = 0x10
 

src/runtime/vdso_freebsd.go

@@ -16,37 +16,41 @@ const _VDSO_TH_NUM = 4 // defined in <sys/vdso.h> #ifdef _KERNEL
 var timekeepSharedPage *vdsoTimekeep
 
 //go:nosplit
-func (bt bintime) Add(bt2 bintime) bintime {
+func (bt *bintime) Add(bt2 *bintime) {
 	u := bt.frac
 	bt.frac += bt2.frac
 	if u > bt.frac {
 		bt.sec++
 	}
 	bt.sec += bt2.sec
-	return bt
 }
 
 //go:nosplit
-func (bt bintime) AddX(x uint64) bintime {
+func (bt *bintime) AddX(x uint64) {
 	u := bt.frac
 	bt.frac += x
 	if u > bt.frac {
 		bt.sec++
 	}
-	return bt
 }
 
-var binuptimeDummy uint32
+var (
+	// binuptimeDummy is used in binuptime as the address of an atomic.Load, to simulate
+	// an atomic_thread_fence_acq() call which behaves as an instruction reordering and
+	// memory barrier.
+	binuptimeDummy uint32
+
+	zeroBintime bintime
+)
 
 // based on /usr/src/lib/libc/sys/__vdso_gettimeofday.c
 //
 //go:nosplit
-func binuptime(abs bool) (bintime, bool) {
+func binuptime(abs bool) (bt bintime) {
-	var bt bintime
 	timehands := (*[_VDSO_TH_NUM]vdsoTimehands)(add(unsafe.Pointer(timekeepSharedPage), vdsoTimekeepSize))
 	for {
 		if timekeepSharedPage.enabled == 0 {
-			return bt, false
+			return zeroBintime
 		}
 
 		curr := atomic.Load(&timekeepSharedPage.current) // atomic_load_acq_32
@@ -55,13 +59,13 @@ func binuptime(abs bool) (bintime, bool) {
 		bt = th.offset
 
 		if tc, ok := th.getTimecounter(); !ok {
-			return bt, false
+			return zeroBintime
 		} else {
 			delta := (tc - th.offset_count) & th.counter_mask
-			bt = bt.AddX(th.scale * uint64(delta))
+			bt.AddX(th.scale * uint64(delta))
 		}
 		if abs {
-			bt = bt.Add(th.boottime)
+			bt.Add(&th.boottime)
 		}
 
 		atomic.Load(&binuptimeDummy) // atomic_thread_fence_acq()
@@ -69,13 +73,13 @@ func binuptime(abs bool) (bintime, bool) {
 			break
 		}
 	}
-	return bt, true
+	return bt
 }
 
 //go:nosplit
-func vdsoClockGettime(clockID int32) (bintime, bool) {
+func vdsoClockGettime(clockID int32) bintime {
 	if timekeepSharedPage == nil || timekeepSharedPage.ver != _VDSO_TK_VER_CURR {
-		return bintime{}, false
+		return zeroBintime
 	}
 	abs := false
 	switch clockID {
@@ -84,9 +88,8 @@ func vdsoClockGettime(clockID int32) (bintime, bool) {
 	case _CLOCK_REALTIME:
 		abs = true
 	default:
-		return bintime{}, false
+		return zeroBintime
 	}
-
 	return binuptime(abs)
 }
 
@@ -95,16 +98,16 @@ func fallback_walltime() (sec int64, nsec int32)
 
 //go:nosplit
 func nanotime() int64 {
-	bt, ok := vdsoClockGettime(_CLOCK_MONOTONIC)
+	bt := vdsoClockGettime(_CLOCK_MONOTONIC)
-	if !ok {
+	if bt == zeroBintime {
 		return fallback_nanotime()
 	}
 	return int64((1e9 * uint64(bt.sec)) + ((1e9 * uint64(bt.frac>>32)) >> 32))
 }
 
 func walltime() (sec int64, nsec int32) {
-	bt, ok := vdsoClockGettime(_CLOCK_REALTIME)
+	bt := vdsoClockGettime(_CLOCK_REALTIME)
-	if !ok {
+	if bt == zeroBintime {
 		return fallback_walltime()
 	}
 	return int64(bt.sec), int32((1e9 * uint64(bt.frac>>32)) >> 32)

src/runtime/vdso_freebsd_x86.go

@@ -7,8 +7,35 @@
 
 package runtime
 
+import (
+	"runtime/internal/atomic"
+	"unsafe"
+)
+
 const (
 	_VDSO_TH_ALGO_X86_TSC  = 1
+	_VDSO_TH_ALGO_X86_HPET = 2
+)
+
+const (
+	_HPET_DEV_MAP_MAX  = 10
+	_HPET_MAIN_COUNTER = 0xf0 /* Main counter register */
+)
+
+var (
+	hpetDevMap  [_HPET_DEV_MAP_MAX]uintptr
+	hpetDevPath = [_HPET_DEV_MAP_MAX][11]byte{
+		{'/', 'd', 'e', 'v', '/', 'h', 'p', 'e', 't', '0', 0},
+		{'/', 'd', 'e', 'v', '/', 'h', 'p', 'e', 't', '1', 0},
+		{'/', 'd', 'e', 'v', '/', 'h', 'p', 'e', 't', '2', 0},
+		{'/', 'd', 'e', 'v', '/', 'h', 'p', 'e', 't', '3', 0},
+		{'/', 'd', 'e', 'v', '/', 'h', 'p', 'e', 't', '4', 0},
+		{'/', 'd', 'e', 'v', '/', 'h', 'p', 'e', 't', '5', 0},
+		{'/', 'd', 'e', 'v', '/', 'h', 'p', 'e', 't', '6', 0},
+		{'/', 'd', 'e', 'v', '/', 'h', 'p', 'e', 't', '7', 0},
+		{'/', 'd', 'e', 'v', '/', 'h', 'p', 'e', 't', '8', 0},
+		{'/', 'd', 'e', 'v', '/', 'h', 'p', 'e', 't', '9', 0},
+	}
 )
 
 //go:nosplit
@@ -20,11 +47,45 @@ func (th *vdsoTimehands) getTSCTimecounter() uint32 {
 	return uint32(tsc)
 }
 
+//go:nosplit
+func (th *vdsoTimehands) getHPETTimecounter() (uint32, bool) {
+	idx := int(th.x86_hpet_idx)
+	if idx >= len(hpetDevMap) {
+		return 0, false
+	}
+
+	p := atomic.Loaduintptr(&hpetDevMap[idx])
+	if p == 0 {
+		fd := open(&hpetDevPath[idx][0], 0 /* O_RDONLY */, 0)
+		if fd < 0 {
+			atomic.Casuintptr(&hpetDevMap[idx], 0, ^uintptr(0))
+			return 0, false
+		}
+
+		addr, mmapErr := mmap(nil, physPageSize, _PROT_READ, _MAP_SHARED, fd, 0)
+		closefd(fd)
+		newP := uintptr(addr)
+		if mmapErr != 0 {
+			newP = ^uintptr(0)
+		}
+		if !atomic.Casuintptr(&hpetDevMap[idx], 0, newP) && mmapErr == 0 {
+			munmap(addr, physPageSize)
+		}
+		p = atomic.Loaduintptr(&hpetDevMap[idx])
+	}
+	if p == ^uintptr(0) {
+		return 0, false
+	}
+	return *(*uint32)(unsafe.Pointer(p + _HPET_MAIN_COUNTER)), true
+}
+
 //go:nosplit
 func (th *vdsoTimehands) getTimecounter() (uint32, bool) {
 	switch th.algo {
 	case _VDSO_TH_ALGO_X86_TSC:
 		return th.getTSCTimecounter(), true
+	case _VDSO_TH_ALGO_X86_HPET:
+		return th.getHPETTimecounter()
 	default:
 		return 0, false
 	}