git: 4ab4f06952ea - stable/12 - tsc: add RDTSCP or faster variants of get_timecount()

[Konstantin Belousov]

The branch stable/12 has been updated by kib:

URL: https://cgit.FreeBSD.org/src/commit/?id=4ab4f06952eaee865326e0da7512f49c580d8a32

commit 4ab4f06952eaee865326e0da7512f49c580d8a32
Author:     Konstantin Belousov <kib at FreeBSD.org>
AuthorDate: 2021-01-05 21:00:14 +0000
Commit:     Konstantin Belousov <kib at FreeBSD.org>
CommitDate: 2021-02-09 08:36:08 +0000

    tsc: add RDTSCP or faster variants of get_timecount()
    
    Tested by:      pho
    
    (cherry picked from commit 9e680e4005b77e3028d28377ee3722a5260f4422)
    (cherry picked from commit a013e285dfd6b89b1908ca13febb0fdb0a7f3b1f)
    (cherry picked from commit 9f47eeffa3cfdcb512e2011fb00fc23c7c1a7d75)
---
 sys/x86/x86/tsc.c | 32 +++++++++++++++++++++++++++++++-
 1 file changed, 31 insertions(+), 1 deletion(-)

diff --git a/sys/x86/x86/tsc.c b/sys/x86/x86/tsc.c
index cb1ba0758db9..f14e0707651c 100644
--- a/sys/x86/x86/tsc.c
+++ b/sys/x86/x86/tsc.c
@@ -96,6 +96,8 @@ static u_int tsc_get_timecount_lfence(struct timecounter *tc);
 static u_int tsc_get_timecount_low_lfence(struct timecounter *tc);
 static u_int tsc_get_timecount_mfence(struct timecounter *tc);
 static u_int tsc_get_timecount_low_mfence(struct timecounter *tc);
+static u_int tscp_get_timecount(struct timecounter *tc);
+static u_int tscp_get_timecount_low(struct timecounter *tc);
 static void tsc_levels_changed(void *arg, int unit);
 static uint32_t x86_tsc_vdso_timehands(struct vdso_timehands *vdso_th,
     struct timecounter *tc);
@@ -632,7 +634,18 @@ init_TSC_tc(void)
 init:
 	for (shift = 0; shift <= 31 && (tsc_freq >> shift) > max_freq; shift++)
 		;
-	if ((cpu_feature & CPUID_SSE2) != 0 && mp_ncpus > 1) {
+
+	/*
+	 * Timecounter implementation selection, top to bottom:
+	 * - If RDTSCP is available, use RDTSCP.
+	 * - If fence instructions are provided (SSE2), use LFENCE;RDTSC
+	 *   on Intel, and MFENCE;RDTSC on AMD.
+	 * - For really old CPUs, just use RDTSC.
+	 */
+	if ((amd_feature & AMDID_RDTSCP) != 0) {
+		tsc_timecounter.tc_get_timecount = shift > 0 ?
+		    tscp_get_timecount_low : tscp_get_timecount;
+	} else if ((cpu_feature & CPUID_SSE2) != 0 && mp_ncpus > 1) {
 		if (cpu_vendor_id == CPU_VENDOR_AMD ||
 		    cpu_vendor_id == CPU_VENDOR_HYGON) {
 			tsc_timecounter.tc_get_timecount = shift > 0 ?
@@ -785,6 +798,13 @@ tsc_get_timecount(struct timecounter *tc __unused)
 	return (rdtsc32());
 }
 
+static u_int
+tscp_get_timecount(struct timecounter *tc __unused)
+{
+
+	return (rdtscp32());
+}
+
 static inline u_int
 tsc_get_timecount_low(struct timecounter *tc)
 {
@@ -795,6 +815,16 @@ tsc_get_timecount_low(struct timecounter *tc)
 	return (rv);
 }
 
+static u_int
+tscp_get_timecount_low(struct timecounter *tc)
+{
+	uint32_t rv;
+
+	__asm __volatile("rdtscp; movl %1, %%ecx; shrd %%cl, %%edx, %0"
+	    : "=&a" (rv) : "m" (tc->tc_priv) : "ecx", "edx");
+	return (rv);
+}
+
 static u_int
 tsc_get_timecount_lfence(struct timecounter *tc __unused)
 {