svn commit: r250772 - stable/9/sys/x86/x86

[Alexander Motin]

Author: mav
Date: Sat May 18 13:19:31 2013
New Revision: 250772
URL: http://svnweb.freebsd.org/changeset/base/250772

Log:
  MFC r249625:
  Introduce kern.timecounter.smp_tsc_adjust tunable (disabled by default) and
  respective functionality, allowing to synchronize TSC on APs to match BSP's
  during boot.  It may be unsafe in general case due to theoretical chance of
  later drift if CPUs are using different clock rate or source, but it allows
  to use TSC in some cases when difference caused by some initialization bug,
  while TSCs are known to increment synchronously.

Modified:
  stable/9/sys/x86/x86/tsc.c
Directory Properties:
  stable/9/sys/   (props changed)

Modified: stable/9/sys/x86/x86/tsc.c
==============================================================================
--- stable/9/sys/x86/x86/tsc.c	Sat May 18 13:15:19 2013	(r250771)
+++ stable/9/sys/x86/x86/tsc.c	Sat May 18 13:19:31 2013	(r250772)
@@ -65,6 +65,11 @@ static int	smp_tsc;
 SYSCTL_INT(_kern_timecounter, OID_AUTO, smp_tsc, CTLFLAG_RDTUN, &smp_tsc, 0,
     "Indicates whether the TSC is safe to use in SMP mode");
 TUNABLE_INT("kern.timecounter.smp_tsc", &smp_tsc);
+
+int	smp_tsc_adjust = 0;
+SYSCTL_INT(_kern_timecounter, OID_AUTO, smp_tsc_adjust, CTLFLAG_RDTUN,
+    &smp_tsc_adjust, 0, "Try to adjust TSC on APs to match BSP");
+TUNABLE_INT("kern.timecounter.smp_tsc_adjust", &smp_tsc_adjust);
 #endif
 
 static int	tsc_shift = 1;
@@ -403,25 +408,77 @@ comp_smp_tsc(void *arg)
 		}
 }
 
+static void
+adj_smp_tsc(void *arg)
+{
+	uint64_t *tsc;
+	int64_t d, min, max;
+	u_int cpu = PCPU_GET(cpuid);
+	u_int first, i, size;
+
+	first = CPU_FIRST();
+	if (cpu == first)
+		return;
+	min = INT64_MIN;
+	max = INT64_MAX;
+	size = (mp_maxid + 1) * 3;
+	for (i = 0, tsc = arg; i < N; i++, tsc += size) {
+		d = tsc[first * 3] - tsc[cpu * 3 + 1];
+		if (d > min)
+			min = d;
+		d = tsc[first * 3 + 1] - tsc[cpu * 3 + 2];
+		if (d > min)
+			min = d;
+		d = tsc[first * 3 + 1] - tsc[cpu * 3];
+		if (d < max)
+			max = d;
+		d = tsc[first * 3 + 2] - tsc[cpu * 3 + 1];
+		if (d < max)
+			max = d;
+	}
+	if (min > max)
+		return;
+	d = min / 2 + max / 2;
+	__asm __volatile (
+		"movl $0x10, %%ecx\n\t"
+		"rdmsr\n\t"
+		"addl %%edi, %%eax\n\t"
+		"adcl %%esi, %%edx\n\t"
+		"wrmsr\n"
+		: /* No output */
+		: "D" ((uint32_t)d), "S" ((uint32_t)(d >> 32))
+		: "ax", "cx", "dx", "cc"
+	);
+}
+
 static int
 test_tsc(void)
 {
 	uint32_t *data, *tsc;
-	u_int i, size;
+	u_int i, size, adj;
 
 	if ((!smp_tsc && !tsc_is_invariant) || vm_guest)
 		return (-100);
 	size = (mp_maxid + 1) * 3;
 	data = malloc(sizeof(*data) * size * N, M_TEMP, M_WAITOK);
+	adj = 0;
+retry:
 	for (i = 0, tsc = data; i < N; i++, tsc += size)
 		smp_rendezvous(tsc_read_0, tsc_read_1, tsc_read_2, tsc);
 	smp_tsc = 1;	/* XXX */
 	smp_rendezvous(smp_no_rendevous_barrier, comp_smp_tsc,
 	    smp_no_rendevous_barrier, data);
+	if (!smp_tsc && adj < smp_tsc_adjust) {
+		adj++;
+		smp_rendezvous(smp_no_rendevous_barrier, adj_smp_tsc,
+		    smp_no_rendevous_barrier, data);
+		goto retry;
+	}
 	free(data, M_TEMP);
 	if (bootverbose)
-		printf("SMP: %sed TSC synchronization test\n",
-		    smp_tsc ? "pass" : "fail");
+		printf("SMP: %sed TSC synchronization test%s\n",
+		    smp_tsc ? "pass" : "fail", 
+		    adj > 0 ? " after adjustment" : "");
 	if (smp_tsc && tsc_is_invariant) {
 		switch (cpu_vendor_id) {
 		case CPU_VENDOR_AMD: