PERFORCE change 138320 for review

[John Birrell]

http://perforce.freebsd.org/chv.cgi?CH=138320

Change 138320 by jb at jb_freebsd1 on 2008/03/23 03:04:49

	Get the skew between per-cpu tsc values. Ignore the possibility of
	drift like Solaris does.
	
	This makes the 'timestamp' DIF variable accurate across CPUs now
	so that timing function entry -> return makes sense.

Affected files ...

.. //depot/projects/dtrace/src/sys/cddl/dev/dtrace/amd64/dtrace_subr.c#10 edit
.. //depot/projects/dtrace/src/sys/cddl/dev/dtrace/i386/dtrace_subr.c#13 edit

Differences ...

==== //depot/projects/dtrace/src/sys/cddl/dev/dtrace/amd64/dtrace_subr.c#10 (text+ko) ====

@@ -36,6 +36,7 @@
 #include <sys/smp.h>
 #include <sys/dtrace_impl.h>
 #include <sys/dtrace_bsd.h>
+#include <machine/clock.h>
 #include <machine/frame.h>
 #include <vm/pmap.h>
 
@@ -377,6 +378,72 @@
 }
 #endif
 
+static int64_t	tgt_cpu_tsc;
+static int64_t	hst_cpu_tsc;
+static int64_t	tsc_skew[MAXCPU];
+
+static void
+dtrace_gethrtime_init_sync(void *arg)
+{
+#ifdef CHECK_SYNC
+	/*
+	 * Delay this function from returning on one
+	 * of the CPUs to check that the synchronisation
+	 * works.
+	 */
+	uintptr_t cpu = (uintptr_t) arg;
+
+	if (cpu == curcpu) {
+		int i;
+		for (i = 0; i < 1000000000; i++)
+			tgt_cpu_tsc = rdtsc();
+		tgt_cpu_tsc = 0;
+	}
+#endif
+}
+
+static void
+dtrace_gethrtime_init_cpu(void *arg)
+{
+	uintptr_t cpu = (uintptr_t) arg;
+
+	if (cpu == curcpu)
+		tgt_cpu_tsc = rdtsc();
+	else
+		hst_cpu_tsc = rdtsc();
+}
+
+static void
+dtrace_gethrtime_init(void *arg)
+{
+	cpumask_t map;
+	int i;
+	struct pcpu *cp;
+
+	/* The current CPU is the reference one. */
+	tsc_skew[curcpu] = 0;
+
+	for (i = 0; i <= mp_maxid; i++) {
+		if (i == curcpu)
+			continue;
+
+		if ((cp = pcpu_find(i)) == NULL)
+			continue;
+
+		map = 0;
+		map |= (1 << curcpu);
+		map |= (1 << i);
+
+		smp_rendezvous_cpus(map, dtrace_gethrtime_init_sync,
+		    dtrace_gethrtime_init_cpu,
+		    smp_no_rendevous_barrier, (void *)(uintptr_t) i);
+
+		tsc_skew[i] = tgt_cpu_tsc - hst_cpu_tsc;
+	}
+}
+
+SYSINIT(dtrace_gethrtime_init, SI_SUB_SMP, SI_ORDER_ANY, dtrace_gethrtime_init, NULL);
+
 /*
  * DTrace needs a high resolution time function which can
  * be called from a probe context and guaranteed not to have
@@ -387,7 +454,7 @@
 uint64_t
 dtrace_gethrtime()
 {
-	return (rdtsc() * (uint64_t) 1000000000 / tsc_freq);
+	return ((rdtsc() + tsc_skew[curcpu]) * (int64_t) 1000000000 / tsc_freq);
 }
 
 uint64_t

==== //depot/projects/dtrace/src/sys/cddl/dev/dtrace/i386/dtrace_subr.c#13 (text+ko) ====

@@ -376,6 +376,72 @@
 }
 #endif
 
+static int64_t	tgt_cpu_tsc;
+static int64_t	hst_cpu_tsc;
+static int64_t	tsc_skew[MAXCPU];
+
+static void
+dtrace_gethrtime_init_sync(void *arg)
+{
+#ifdef CHECK_SYNC
+	/*
+	 * Delay this function from returning on one
+	 * of the CPUs to check that the synchronisation
+	 * works.
+	 */
+	uintptr_t cpu = (uintptr_t) arg;
+
+	if (cpu == curcpu) {
+		int i;
+		for (i = 0; i < 1000000000; i++)
+			tgt_cpu_tsc = rdtsc();
+		tgt_cpu_tsc = 0;
+	}
+#endif
+}
+
+static void
+dtrace_gethrtime_init_cpu(void *arg)
+{
+	uintptr_t cpu = (uintptr_t) arg;
+
+	if (cpu == curcpu)
+		tgt_cpu_tsc = rdtsc();
+	else
+		hst_cpu_tsc = rdtsc();
+}
+
+static void
+dtrace_gethrtime_init(void *arg)
+{
+	cpumask_t map;
+	int i;
+	struct pcpu *cp;
+
+	/* The current CPU is the reference one. */
+	tsc_skew[curcpu] = 0;
+
+	for (i = 0; i <= mp_maxid; i++) {
+		if (i == curcpu)
+			continue;
+
+		if ((cp = pcpu_find(i)) == NULL)
+			continue;
+
+		map = 0;
+		map |= (1 << curcpu);
+		map |= (1 << i);
+
+		smp_rendezvous_cpus(map, dtrace_gethrtime_init_sync,
+		    dtrace_gethrtime_init_cpu,
+		    smp_no_rendevous_barrier, (void *)(uintptr_t) i);
+
+		tsc_skew[i] = tgt_cpu_tsc - hst_cpu_tsc;
+	}
+}
+
+SYSINIT(dtrace_gethrtime_init, SI_SUB_SMP, SI_ORDER_ANY, dtrace_gethrtime_init, NULL);
+
 /*
  * DTrace needs a high resolution time function which can
  * be called from a probe context and guaranteed not to have
@@ -386,7 +452,7 @@
 uint64_t
 dtrace_gethrtime()
 {
-	return (rdtsc() * (uint64_t) 1000000000 / tsc_freq);
+	return ((rdtsc() + tsc_skew[curcpu]) * (int64_t) 1000000000 / tsc_freq);
 }
 
 uint64_t