git: 13dbf9da686c - stable/13 - x86: Perform late TSC calibration before LAPIC timer calibration

The branch stable/13 has been updated by markj:

URL: https://cgit.FreeBSD.org/src/commit/?id=13dbf9da686c1c3ea1a1e197b932ea3abdf27a43

commit 13dbf9da686c1c3ea1a1e197b932ea3abdf27a43
Author:     Mark Johnston <markj@FreeBSD.org>
AuthorDate: 2021-12-06 15:42:19 +0000
Commit:     Mark Johnston <markj@FreeBSD.org>
CommitDate: 2021-12-29 15:39:22 +0000

    x86: Perform late TSC calibration before LAPIC timer calibration
    
    This ensures that LAPIC calibration is done using the correct tsc_freq
    value, i.e., the one associated with the TSC timecounter.  It does mean
    though that TSC calibration cannot use sbinuptime() to read the
    reference timecounter, as timehands are not yet set up.
    
    Reviewed by:    kib, jhb
    Sponsored by:   The FreeBSD Foundation
    
    (cherry picked from commit 553af8f1ec71d397b5b4fd5876622b9269936e63)
---
 sys/x86/include/clock.h |  1 +
 sys/x86/isa/clock.c     |  2 ++
 sys/x86/x86/tsc.c       | 26 +++++++++++++++++---------
 3 files changed, 20 insertions(+), 9 deletions(-)

diff --git a/sys/x86/include/clock.h b/sys/x86/include/clock.h
index d492196bac85..83c8351ed31c 100644
--- a/sys/x86/include/clock.h
+++ b/sys/x86/include/clock.h
@@ -28,6 +28,7 @@ void	i8254_init(void);
 void	i8254_delay(int);
 void	clock_init(void);
 void	lapic_calibrate(void);
+void	tsc_calibrate(void);
 
 /*
  * Driver to clock driver interface.
diff --git a/sys/x86/isa/clock.c b/sys/x86/isa/clock.c
index 2eb1c343f4db..513174626892 100644
--- a/sys/x86/isa/clock.c
+++ b/sys/x86/isa/clock.c
@@ -413,6 +413,7 @@ cpu_initclocks(void)
 
 	td = curthread;
 
+	tsc_calibrate();
 	lapic_calibrate_timer();
 	cpu_initclocks_bsp();
 	CPU_FOREACH(i) {
@@ -428,6 +429,7 @@ cpu_initclocks(void)
 		sched_unbind(td);
 	thread_unlock(td);
 #else
+	tsc_calibrate();
 	lapic_calibrate_timer();
 	cpu_initclocks_bsp();
 #endif
diff --git a/sys/x86/x86/tsc.c b/sys/x86/x86/tsc.c
index 53e2c7dcfe42..6debe3ecca6d 100644
--- a/sys/x86/x86/tsc.c
+++ b/sys/x86/x86/tsc.c
@@ -693,23 +693,27 @@ tsc_update_freq(uint64_t new_freq)
 
 /*
  * Perform late calibration of the TSC frequency once ACPI-based timecounters
- * are available.
+ * are available.  At this point timehands are not set up, so we read the
+ * highest-quality timecounter directly rather than using (s)binuptime().
  */
-static void
-tsc_calib(void *arg __unused)
+void
+tsc_calibrate(void)
 {
-	sbintime_t t_start, t_end;
+	struct timecounter *tc;
 	uint64_t freq_khz, tsc_start, tsc_end;
+	u_int t_start, t_end;
 	register_t flags;
 	int cpu;
 
 	if (tsc_disabled)
 		return;
 
+	tc = atomic_load_ptr(&timecounter);
+
 	flags = intr_disable();
 	cpu = curcpu;
 	tsc_start = rdtsc_ordered();
-	t_start = sbinuptime();
+	t_start = tc->tc_get_timecount(tc) & tc->tc_counter_mask;
 	intr_restore(flags);
 
 	DELAY(1000000);
@@ -719,19 +723,23 @@ tsc_calib(void *arg __unused)
 
 	flags = intr_disable();
 	tsc_end = rdtsc_ordered();
-	t_end = sbinuptime();
+	t_end = tc->tc_get_timecount(tc) & tc->tc_counter_mask;
 	intr_restore(flags);
 
 	sched_unbind(curthread);
 	thread_unlock(curthread);
 
-	freq_khz = (SBT_1S / 1024) * (tsc_end - tsc_start) / (t_end - t_start);
+	if (t_end <= t_start) {
+		/* Assume that the counter has wrapped around at most once. */
+		t_end += (uint64_t)tc->tc_counter_mask + 1;
+	}
+
+	freq_khz = tc->tc_frequency * (tsc_end - tsc_start) / (t_end - t_start);
 
-	tsc_update_freq(freq_khz * 1024);
+	tsc_update_freq(freq_khz);
 	tc_init(&tsc_timecounter);
 	set_cputicker(rdtsc, tsc_freq, !tsc_is_invariant);
 }
-SYSINIT(tsc_calib, SI_SUB_CLOCKS + 1, SI_ORDER_ANY, tsc_calib, NULL);
 
 void
 resume_TSC(void)