svn commit: r311376 - in stable/11: include lib/libc/x86/sys sys/dev/hyperv/include sys/dev/hyperv/vmbus sys/dev/hyperv/vmbus/amd64 sys/sys sys/x86/include
Sepherosa Ziehau
sephe at FreeBSD.org
Thu Jan 5 07:42:10 UTC 2017
Author: sephe
Date: Thu Jan 5 07:42:08 2017
New Revision: 311376
URL: https://svnweb.freebsd.org/changeset/base/311376
Log:
MFC 310048,310101,310239
310048
hyperv: Implement "enlightened" time counter, which is rdtsc based.
Reviewed by: kib
Sponsored by: Microsoft
Differential Revision: https://reviews.freebsd.org/D8763
310101
hyperv: Allow userland to ro-mmap reference TSC page
This paves way to implement VDSO for the enlightened time counter.
Reviewed by: kib
Sponsored by: Microsoft
Differential Revision: https://reviews.freebsd.org/D8768
310239
hyperv: Implement userspace gettimeofday(2) with Hyper-V reference TSC
This 6 times gettimeofday performance, as measured by
tools/tools/syscall_timing
Reviewed by: kib
Sponsored by: Microsoft
Differential Revision: https://reviews.freebsd.org/D8789
Modified:
stable/11/include/Makefile
stable/11/lib/libc/x86/sys/__vdso_gettc.c
stable/11/sys/dev/hyperv/include/hyperv.h
stable/11/sys/dev/hyperv/vmbus/amd64/hyperv_machdep.c
stable/11/sys/dev/hyperv/vmbus/hyperv_reg.h
stable/11/sys/sys/vdso.h
stable/11/sys/x86/include/vdso.h
Directory Properties:
stable/11/ (props changed)
Modified: stable/11/include/Makefile
==============================================================================
--- stable/11/include/Makefile Thu Jan 5 06:25:16 2017 (r311375)
+++ stable/11/include/Makefile Thu Jan 5 07:42:08 2017 (r311376)
@@ -184,6 +184,9 @@ copies: .PHONY .META
${DESTDIR}${INCLUDEDIR}/dev/evdev; \
${INSTALL} -C -o ${BINOWN} -g ${BINGRP} -m 444 uinput.h \
${DESTDIR}${INCLUDEDIR}/dev/evdev
+ cd ${.CURDIR}/../sys/dev/hyperv/include; \
+ ${INSTALL} -C ${TAG_ARGS} -o ${BINOWN} -g ${BINGRP} -m 444 hyperv.h \
+ ${DESTDIR}${INCLUDEDIR}/dev/hyperv
cd ${.CURDIR}/../sys/dev/hyperv/utilities; \
${INSTALL} -C ${TAG_ARGS} -o ${BINOWN} -g ${BINGRP} -m 444 hv_snapshot.h \
${DESTDIR}${INCLUDEDIR}/dev/hyperv
@@ -292,6 +295,11 @@ symlinks: .PHONY .META
ln -fs ../../../../sys/dev/evdev/$$h \
${DESTDIR}${INCLUDEDIR}/dev/evdev; \
done
+ cd ${.CURDIR}/../sys/dev/hyperv/include; \
+ for h in hyperv.h; do \
+ ${INSTALL_SYMLINK} ${TAG_ARGS} ../../../../sys/dev/hyperv/include/$$h \
+ ${DESTDIR}${INCLUDEDIR}/dev/hyperv; \
+ done
cd ${.CURDIR}/../sys/dev/hyperv/utilities; \
for h in hv_snapshot.h; do \
${INSTALL_SYMLINK} ${TAG_ARGS} ../../../../sys/dev/hyperv/utilities/$$h \
Modified: stable/11/lib/libc/x86/sys/__vdso_gettc.c
==============================================================================
--- stable/11/lib/libc/x86/sys/__vdso_gettc.c Thu Jan 5 06:25:16 2017 (r311375)
+++ stable/11/lib/libc/x86/sys/__vdso_gettc.c Thu Jan 5 07:42:08 2017 (r311376)
@@ -45,6 +45,10 @@ __FBSDID("$FreeBSD$");
#include <machine/cpufunc.h>
#include <machine/specialreg.h>
#include <dev/acpica/acpi_hpet.h>
+#ifdef __amd64__
+#include <machine/atomic.h>
+#include <dev/hyperv/hyperv.h>
+#endif
#include "libc_private.h"
static void
@@ -144,6 +148,67 @@ __vdso_init_hpet(uint32_t u)
_close(fd);
}
+#ifdef __amd64__
+
+#define HYPERV_REFTSC_DEVPATH "/dev/" HYPERV_REFTSC_DEVNAME
+
+/*
+ * NOTE:
+ * We use 'NULL' for this variable to indicate that initialization
+ * is required. And if this variable is 'MAP_FAILED', then Hyper-V
+ * reference TSC can not be used, e.g. in misconfigured jail.
+ */
+static struct hyperv_reftsc *hyperv_ref_tsc;
+
+static void
+__vdso_init_hyperv_tsc(void)
+{
+ int fd;
+
+ fd = _open(HYPERV_REFTSC_DEVPATH, O_RDONLY);
+ if (fd < 0) {
+ /* Prevent the caller from re-entering. */
+ hyperv_ref_tsc = MAP_FAILED;
+ return;
+ }
+ hyperv_ref_tsc = mmap(NULL, sizeof(*hyperv_ref_tsc), PROT_READ,
+ MAP_SHARED, fd, 0);
+ _close(fd);
+}
+
+static int
+__vdso_hyperv_tsc(struct hyperv_reftsc *tsc_ref, u_int *tc)
+{
+ uint64_t disc, ret, tsc, scale;
+ uint32_t seq;
+ int64_t ofs;
+
+ while ((seq = atomic_load_acq_int(&tsc_ref->tsc_seq)) != 0) {
+ scale = tsc_ref->tsc_scale;
+ ofs = tsc_ref->tsc_ofs;
+
+ lfence_mb();
+ tsc = rdtsc();
+
+ /* ret = ((tsc * scale) >> 64) + ofs */
+ __asm__ __volatile__ ("mulq %3" :
+ "=d" (ret), "=a" (disc) :
+ "a" (tsc), "r" (scale));
+ ret += ofs;
+
+ atomic_thread_fence_acq();
+ if (tsc_ref->tsc_seq == seq) {
+ *tc = ret;
+ return (0);
+ }
+
+ /* Sequence changed; re-sync. */
+ }
+ return (ENOSYS);
+}
+
+#endif /* __amd64__ */
+
#pragma weak __vdso_gettc
int
__vdso_gettc(const struct vdso_timehands *th, u_int *tc)
@@ -165,6 +230,14 @@ __vdso_gettc(const struct vdso_timehands
return (ENOSYS);
*tc = *(volatile uint32_t *)(hpet_dev_map + HPET_MAIN_COUNTER);
return (0);
+#ifdef __amd64__
+ case VDSO_TH_ALGO_X86_HVTSC:
+ if (hyperv_ref_tsc == NULL)
+ __vdso_init_hyperv_tsc();
+ if (hyperv_ref_tsc == MAP_FAILED)
+ return (ENOSYS);
+ return (__vdso_hyperv_tsc(hyperv_ref_tsc, tc));
+#endif
default:
return (ENOSYS);
}
Modified: stable/11/sys/dev/hyperv/include/hyperv.h
==============================================================================
--- stable/11/sys/dev/hyperv/include/hyperv.h Thu Jan 5 06:25:16 2017 (r311375)
+++ stable/11/sys/dev/hyperv/include/hyperv.h Thu Jan 5 07:42:08 2017 (r311376)
@@ -31,10 +31,10 @@
#ifndef _HYPERV_H_
#define _HYPERV_H_
-#include <sys/param.h>
+#ifdef _KERNEL
-#include <vm/vm.h>
-#include <vm/pmap.h>
+#include <sys/param.h>
+#include <sys/systm.h>
#define MSR_HV_TIME_REF_COUNT 0x40000020
@@ -45,6 +45,7 @@
#define CPUID_HV_MSR_HYPERCALL 0x0020 /* MSR_HV_GUEST_OS_ID
* MSR_HV_HYPERCALL */
#define CPUID_HV_MSR_VP_INDEX 0x0040 /* MSR_HV_VP_INDEX */
+#define CPUID_HV_MSR_REFERENCE_TSC 0x0200 /* MSR_HV_REFERENCE_TSC */
#define CPUID_HV_MSR_GUEST_IDLE 0x0400 /* MSR_HV_GUEST_IDLE */
#ifndef NANOSEC
@@ -53,14 +54,35 @@
#define HYPERV_TIMER_NS_FACTOR 100ULL
#define HYPERV_TIMER_FREQ (NANOSEC / HYPERV_TIMER_NS_FACTOR)
+#endif /* _KERNEL */
+
+#define HYPERV_REFTSC_DEVNAME "hv_tsc"
+
+/*
+ * Hyper-V Reference TSC
+ */
+struct hyperv_reftsc {
+ volatile uint32_t tsc_seq;
+ volatile uint32_t tsc_rsvd1;
+ volatile uint64_t tsc_scale;
+ volatile int64_t tsc_ofs;
+} __packed __aligned(PAGE_SIZE);
+#ifdef CTASSERT
+CTASSERT(sizeof(struct hyperv_reftsc) == PAGE_SIZE);
+#endif
+
+#ifdef _KERNEL
+
struct hyperv_guid {
- uint8_t hv_guid[16];
+ uint8_t hv_guid[16];
} __packed;
-#define HYPERV_GUID_STRLEN 40
+#define HYPERV_GUID_STRLEN 40
int hyperv_guid2str(const struct hyperv_guid *, char *, size_t);
extern u_int hyperv_features; /* CPUID_HV_MSR_ */
+#endif /* _KERNEL */
+
#endif /* _HYPERV_H_ */
Modified: stable/11/sys/dev/hyperv/vmbus/amd64/hyperv_machdep.c
==============================================================================
--- stable/11/sys/dev/hyperv/vmbus/amd64/hyperv_machdep.c Thu Jan 5 06:25:16 2017 (r311375)
+++ stable/11/sys/dev/hyperv/vmbus/amd64/hyperv_machdep.c Thu Jan 5 07:42:08 2017 (r311376)
@@ -28,7 +28,55 @@
__FBSDID("$FreeBSD$");
#include <sys/param.h>
+#include <sys/conf.h>
+#include <sys/fcntl.h>
+#include <sys/kernel.h>
+#include <sys/systm.h>
+#include <sys/timetc.h>
+#include <sys/vdso.h>
+
+#include <machine/cpufunc.h>
+#include <machine/cputypes.h>
+#include <machine/md_var.h>
+#include <machine/specialreg.h>
+
+#include <vm/vm.h>
+
+#include <dev/hyperv/include/hyperv.h>
+#include <dev/hyperv/include/hyperv_busdma.h>
#include <dev/hyperv/vmbus/hyperv_machdep.h>
+#include <dev/hyperv/vmbus/hyperv_reg.h>
+#include <dev/hyperv/vmbus/hyperv_var.h>
+
+struct hyperv_reftsc_ctx {
+ struct hyperv_reftsc *tsc_ref;
+ struct hyperv_dma tsc_ref_dma;
+};
+
+static uint32_t hyperv_tsc_vdso_timehands(
+ struct vdso_timehands *,
+ struct timecounter *);
+
+static d_open_t hyperv_tsc_open;
+static d_mmap_t hyperv_tsc_mmap;
+
+static struct timecounter hyperv_tsc_timecounter = {
+ .tc_get_timecount = NULL, /* based on CPU vendor. */
+ .tc_counter_mask = 0xffffffff,
+ .tc_frequency = HYPERV_TIMER_FREQ,
+ .tc_name = "Hyper-V-TSC",
+ .tc_quality = 3000,
+ .tc_fill_vdso_timehands = hyperv_tsc_vdso_timehands,
+};
+
+static struct cdevsw hyperv_tsc_cdevsw = {
+ .d_version = D_VERSION,
+ .d_open = hyperv_tsc_open,
+ .d_mmap = hyperv_tsc_mmap,
+ .d_name = HYPERV_REFTSC_DEVNAME
+};
+
+static struct hyperv_reftsc_ctx hyperv_ref_tsc;
uint64_t
hypercall_md(volatile void *hc_addr, uint64_t in_val,
@@ -41,3 +89,131 @@ hypercall_md(volatile void *hc_addr, uin
"c" (in_val), "d" (in_paddr), "m" (hc_addr));
return (status);
}
+
+static int
+hyperv_tsc_open(struct cdev *dev __unused, int oflags, int devtype __unused,
+ struct thread *td __unused)
+{
+
+ if (oflags & FWRITE)
+ return (EPERM);
+ return (0);
+}
+
+static int
+hyperv_tsc_mmap(struct cdev *dev __unused, vm_ooffset_t offset,
+ vm_paddr_t *paddr, int nprot __unused, vm_memattr_t *memattr __unused)
+{
+
+ KASSERT(hyperv_ref_tsc.tsc_ref != NULL, ("reftsc has not been setup"));
+
+ /*
+ * NOTE:
+ * 'nprot' does not contain information interested to us;
+ * WR-open is blocked by d_open.
+ */
+
+ if (offset != 0)
+ return (EOPNOTSUPP);
+
+ *paddr = hyperv_ref_tsc.tsc_ref_dma.hv_paddr;
+ return (0);
+}
+
+static uint32_t
+hyperv_tsc_vdso_timehands(struct vdso_timehands *vdso_th,
+ struct timecounter *tc __unused)
+{
+
+ vdso_th->th_algo = VDSO_TH_ALGO_X86_HVTSC;
+ vdso_th->th_x86_shift = 0;
+ vdso_th->th_x86_hpet_idx = 0;
+ bzero(vdso_th->th_res, sizeof(vdso_th->th_res));
+ return (1);
+}
+
+#define HYPERV_TSC_TIMECOUNT(fence) \
+static u_int \
+hyperv_tsc_timecount_##fence(struct timecounter *tc) \
+{ \
+ struct hyperv_reftsc *tsc_ref = hyperv_ref_tsc.tsc_ref; \
+ uint32_t seq; \
+ \
+ while ((seq = atomic_load_acq_int(&tsc_ref->tsc_seq)) != 0) { \
+ uint64_t disc, ret, tsc; \
+ uint64_t scale = tsc_ref->tsc_scale; \
+ int64_t ofs = tsc_ref->tsc_ofs; \
+ \
+ fence(); \
+ tsc = rdtsc(); \
+ \
+ /* ret = ((tsc * scale) >> 64) + ofs */ \
+ __asm__ __volatile__ ("mulq %3" : \
+ "=d" (ret), "=a" (disc) : \
+ "a" (tsc), "r" (scale)); \
+ ret += ofs; \
+ \
+ atomic_thread_fence_acq(); \
+ if (tsc_ref->tsc_seq == seq) \
+ return (ret); \
+ \
+ /* Sequence changed; re-sync. */ \
+ } \
+ /* Fallback to the generic timecounter, i.e. rdmsr. */ \
+ return (rdmsr(MSR_HV_TIME_REF_COUNT)); \
+} \
+struct __hack
+
+HYPERV_TSC_TIMECOUNT(lfence);
+HYPERV_TSC_TIMECOUNT(mfence);
+
+static void
+hyperv_tsc_tcinit(void *dummy __unused)
+{
+ uint64_t val, orig;
+
+ if ((hyperv_features &
+ (CPUID_HV_MSR_TIME_REFCNT | CPUID_HV_MSR_REFERENCE_TSC)) !=
+ (CPUID_HV_MSR_TIME_REFCNT | CPUID_HV_MSR_REFERENCE_TSC) ||
+ (cpu_feature & CPUID_SSE2) == 0) /* SSE2 for mfence/lfence */
+ return;
+
+ switch (cpu_vendor_id) {
+ case CPU_VENDOR_AMD:
+ hyperv_tsc_timecounter.tc_get_timecount =
+ hyperv_tsc_timecount_mfence;
+ break;
+
+ case CPU_VENDOR_INTEL:
+ hyperv_tsc_timecounter.tc_get_timecount =
+ hyperv_tsc_timecount_lfence;
+ break;
+
+ default:
+ /* Unsupport CPU vendors. */
+ return;
+ }
+
+ hyperv_ref_tsc.tsc_ref = hyperv_dmamem_alloc(NULL, PAGE_SIZE, 0,
+ sizeof(struct hyperv_reftsc), &hyperv_ref_tsc.tsc_ref_dma,
+ BUS_DMA_WAITOK | BUS_DMA_ZERO);
+ if (hyperv_ref_tsc.tsc_ref == NULL) {
+ printf("hyperv: reftsc page allocation failed\n");
+ return;
+ }
+
+ orig = rdmsr(MSR_HV_REFERENCE_TSC);
+ val = MSR_HV_REFTSC_ENABLE | (orig & MSR_HV_REFTSC_RSVD_MASK) |
+ ((hyperv_ref_tsc.tsc_ref_dma.hv_paddr >> PAGE_SHIFT) <<
+ MSR_HV_REFTSC_PGSHIFT);
+ wrmsr(MSR_HV_REFERENCE_TSC, val);
+
+ /* Register "enlightened" timecounter. */
+ tc_init(&hyperv_tsc_timecounter);
+
+ /* Add device for mmap(2). */
+ make_dev(&hyperv_tsc_cdevsw, 0, UID_ROOT, GID_WHEEL, 0444,
+ HYPERV_REFTSC_DEVNAME);
+}
+SYSINIT(hyperv_tsc_init, SI_SUB_DRIVERS, SI_ORDER_FIRST, hyperv_tsc_tcinit,
+ NULL);
Modified: stable/11/sys/dev/hyperv/vmbus/hyperv_reg.h
==============================================================================
--- stable/11/sys/dev/hyperv/vmbus/hyperv_reg.h Thu Jan 5 06:25:16 2017 (r311375)
+++ stable/11/sys/dev/hyperv/vmbus/hyperv_reg.h Thu Jan 5 07:42:08 2017 (r311376)
@@ -57,6 +57,11 @@
#define MSR_HV_VP_INDEX 0x40000002
+#define MSR_HV_REFERENCE_TSC 0x40000021
+#define MSR_HV_REFTSC_ENABLE 0x0001ULL
+#define MSR_HV_REFTSC_RSVD_MASK 0x0ffeULL
+#define MSR_HV_REFTSC_PGSHIFT 12
+
#define MSR_HV_SCONTROL 0x40000080
#define MSR_HV_SCTRL_ENABLE 0x0001ULL
#define MSR_HV_SCTRL_RSVD_MASK 0xfffffffffffffffeULL
Modified: stable/11/sys/sys/vdso.h
==============================================================================
--- stable/11/sys/sys/vdso.h Thu Jan 5 06:25:16 2017 (r311375)
+++ stable/11/sys/sys/vdso.h Thu Jan 5 07:42:08 2017 (r311376)
@@ -54,6 +54,8 @@ struct vdso_timekeep {
#define VDSO_TK_VER_CURR VDSO_TK_VER_1
#define VDSO_TH_ALGO_1 0x1
#define VDSO_TH_ALGO_2 0x2
+#define VDSO_TH_ALGO_3 0x3
+#define VDSO_TH_ALGO_4 0x4
#ifndef _KERNEL
Modified: stable/11/sys/x86/include/vdso.h
==============================================================================
--- stable/11/sys/x86/include/vdso.h Thu Jan 5 06:25:16 2017 (r311375)
+++ stable/11/sys/x86/include/vdso.h Thu Jan 5 07:42:08 2017 (r311376)
@@ -39,6 +39,7 @@
#define VDSO_TH_ALGO_X86_TSC VDSO_TH_ALGO_1
#define VDSO_TH_ALGO_X86_HPET VDSO_TH_ALGO_2
+#define VDSO_TH_ALGO_X86_HVTSC VDSO_TH_ALGO_3 /* Hyper-V ref. TSC */
#ifdef _KERNEL
#ifdef COMPAT_FREEBSD32
More information about the svn-src-all
mailing list