svn commit: r286284 - in head: lib/libc/amd64/sys lib/libc/i386/sys lib/libc/sys sys/kern

Konstantin Belousov kib at FreeBSD.org
Tue Aug 4 12:33:54 UTC 2015


Author: kib
Date: Tue Aug  4 12:33:51 2015
New Revision: 286284
URL: https://svnweb.freebsd.org/changeset/base/286284

Log:
  Copy the fencing of the algorithm to do lock-less update and reading
  of the timehands, from the kern_tc.c implementation to vdso.  Add
  comments giving hints where to look for the algorithm explanation.
  
  To compensate the removal of rmb() in userspace binuptime(), add
  explicit lfence instruction before rdtsc.  On i386, add usual
  complications to detect SSE2 presence; assume that old CPUs which do
  not implement SSE2 also execute rdtsc almost in order.
  
  Reviewed by:	alc, bde (previous version)
  Sponsored by:	The FreeBSD Foundation
  MFC after:	2 weeks

Modified:
  head/lib/libc/amd64/sys/__vdso_gettc.c
  head/lib/libc/i386/sys/__vdso_gettc.c
  head/lib/libc/sys/__vdso_gettimeofday.c
  head/sys/kern/kern_sharedpage.c

Modified: head/lib/libc/amd64/sys/__vdso_gettc.c
==============================================================================
--- head/lib/libc/amd64/sys/__vdso_gettc.c	Tue Aug  4 10:40:08 2015	(r286283)
+++ head/lib/libc/amd64/sys/__vdso_gettc.c	Tue Aug  4 12:33:51 2015	(r286284)
@@ -36,19 +36,29 @@ __FBSDID("$FreeBSD$");
 static u_int
 __vdso_gettc_low(const struct vdso_timehands *th)
 {
-	uint32_t rv;
+	u_int rv;
 
-	__asm __volatile("rdtsc; shrd %%cl, %%edx, %0"
+	__asm __volatile("lfence; rdtsc; shrd %%cl, %%edx, %0"
 	    : "=a" (rv) : "c" (th->th_x86_shift) : "edx");
 	return (rv);
 }
 
+static u_int
+__vdso_rdtsc32(void)
+{
+	u_int rv;
+
+	__asm __volatile("lfence;rdtsc" : "=a" (rv) : : "edx");
+	return (rv);
+}
+
 #pragma weak __vdso_gettc
 u_int
 __vdso_gettc(const struct vdso_timehands *th)
 {
 
-	return (th->th_x86_shift > 0 ? __vdso_gettc_low(th) : rdtsc32());
+	return (th->th_x86_shift > 0 ? __vdso_gettc_low(th) :
+	    __vdso_rdtsc32());
 }
 
 #pragma weak __vdso_gettimekeep

Modified: head/lib/libc/i386/sys/__vdso_gettc.c
==============================================================================
--- head/lib/libc/i386/sys/__vdso_gettc.c	Tue Aug  4 10:40:08 2015	(r286283)
+++ head/lib/libc/i386/sys/__vdso_gettc.c	Tue Aug  4 12:33:51 2015	(r286284)
@@ -31,24 +31,78 @@ __FBSDID("$FreeBSD$");
 #include <sys/time.h>
 #include <sys/vdso.h>
 #include <machine/cpufunc.h>
+#include <machine/specialreg.h>
 #include "libc_private.h"
 
+static int lfence_works = -1;
+
+static int
+get_lfence_usage(void)
+{
+	u_int cpuid_supported, p[4];
+
+	if (lfence_works == -1) {
+		__asm __volatile(
+		    "	pushfl\n"
+		    "	popl	%%eax\n"
+		    "	movl    %%eax,%%ecx\n"
+		    "	xorl    $0x200000,%%eax\n"
+		    "	pushl	%%eax\n"
+		    "	popfl\n"
+		    "	pushfl\n"
+		    "	popl    %%eax\n"
+		    "	xorl    %%eax,%%ecx\n"
+		    "	je	1f\n"
+		    "	movl	$1,%0\n"
+		    "	jmp	2f\n"
+		    "1:	movl	$0,%0\n"
+		    "2:\n"
+		    : "=r" (cpuid_supported) : : "eax", "ecx");
+		if (cpuid_supported) {
+			__asm __volatile(
+			    "	pushl	%%ebx\n"
+			    "	cpuid\n"
+			    "	movl	%%ebx,%1\n"
+			    "	popl	%%ebx\n"
+			    : "=a" (p[0]), "=r" (p[1]), "=c" (p[2]), "=d" (p[3])
+			    :  "0" (0x1));
+			lfence_works = (p[3] & CPUID_SSE2) != 0;
+		} else
+			lfence_works = 0;
+	}
+	return (lfence_works);
+}
+
 static u_int
 __vdso_gettc_low(const struct vdso_timehands *th)
 {
-	uint32_t rv;
+	u_int rv;
 
+	if (get_lfence_usage() == 1)
+		lfence();
 	__asm __volatile("rdtsc; shrd %%cl, %%edx, %0"
 	    : "=a" (rv) : "c" (th->th_x86_shift) : "edx");
 	return (rv);
 }
 
+static u_int
+__vdso_rdtsc32(void)
+{
+	u_int rv;
+
+	if (get_lfence_usage() == 1)
+		lfence();
+	rv = rdtsc32();
+	return (rv);
+}
+
 #pragma weak __vdso_gettc
 u_int
 __vdso_gettc(const struct vdso_timehands *th)
 {
 
-	return (th->th_x86_shift > 0 ? __vdso_gettc_low(th) : rdtsc32());
+	return (th->th_x86_shift > 0 ? __vdso_gettc_low(th) :
+	    __vdso_rdtsc32());
 }
 
 #pragma weak __vdso_gettimekeep

Modified: head/lib/libc/sys/__vdso_gettimeofday.c
==============================================================================
--- head/lib/libc/sys/__vdso_gettimeofday.c	Tue Aug  4 10:40:08 2015	(r286283)
+++ head/lib/libc/sys/__vdso_gettimeofday.c	Tue Aug  4 12:33:51 2015	(r286284)
@@ -42,6 +42,15 @@ tc_delta(const struct vdso_timehands *th
 	    th->th_counter_mask);
 }
 
+/*
+ * Calculate the absolute or boot-relative time from the
+ * machine-specific fast timecounter and the published timehands
+ * structure read from the shared page.
+ *
+ * The lockless reading scheme is similar to the one used to read the
+ * in-kernel timehands, see sys/kern/kern_tc.c:binuptime().  This code
+ * is based on the kernel implementation.
+ */
 static int
 binuptime(struct bintime *bt, struct vdso_timekeep *tk, int abs)
 {
@@ -52,27 +61,21 @@ binuptime(struct bintime *bt, struct vds
 		if (!tk->tk_enabled)
 			return (ENOSYS);
 
-		/*
-		 * XXXKIB. The load of tk->tk_current should use
-		 * atomic_load_acq_32 to provide load barrier. But
-		 * since tk points to r/o mapped page, x86
-		 * implementation of atomic_load_acq faults.
-		 */
-		curr = tk->tk_current;
-		rmb();
+		curr = atomic_load_acq_32(&tk->tk_current);
 		th = &tk->tk_th[curr];
 		if (th->th_algo != VDSO_TH_ALGO_1)
 			return (ENOSYS);
-		gen = th->th_gen;
+		gen = atomic_load_acq_32(&th->th_gen);
 		*bt = th->th_offset;
 		bintime_addx(bt, th->th_scale * tc_delta(th));
 		if (abs)
 			bintime_add(bt, &th->th_boottime);
 
 		/*
-		 * Barrier for load of both tk->tk_current and th->th_gen.
+		 * Ensure that the load of th_offset is completed
+		 * before the load of th_gen.
 		 */
-		rmb();
+		atomic_thread_fence_acq();
 	} while (curr != tk->tk_current || gen == 0 || gen != th->th_gen);
 	return (0);
 }

Modified: head/sys/kern/kern_sharedpage.c
==============================================================================
--- head/sys/kern/kern_sharedpage.c	Tue Aug  4 10:40:08 2015	(r286283)
+++ head/sys/kern/kern_sharedpage.c	Tue Aug  4 12:33:51 2015	(r286284)
@@ -119,6 +119,13 @@ shared_page_init(void *dummy __unused)
 SYSINIT(shp, SI_SUB_EXEC, SI_ORDER_FIRST, (sysinit_cfunc_t)shared_page_init,
     NULL);
 
+/*
+ * Push the timehands update to the shared page.
+ *
+ * The lockless update scheme is similar to the one used to update the
+ * in-kernel timehands, see sys/kern/kern_tc.c:tc_windup() (which
+ * calls us after the timehands are updated).
+ */
 static void
 timehands_update(struct sysentvec *sv)
 {
@@ -127,47 +134,56 @@ timehands_update(struct sysentvec *sv)
 	uint32_t enabled, idx;
 
 	enabled = tc_fill_vdso_timehands(&th);
-	tk = (struct vdso_timekeep *)(shared_page_mapping +
-	    sv->sv_timekeep_off);
+	th.th_gen = 0;
 	idx = sv->sv_timekeep_curr;
-	atomic_store_rel_32(&tk->tk_th[idx].th_gen, 0);
 	if (++idx >= VDSO_TH_NUM)
 		idx = 0;
 	sv->sv_timekeep_curr = idx;
 	if (++sv->sv_timekeep_gen == 0)
 		sv->sv_timekeep_gen = 1;
-	th.th_gen = 0;
+
+	tk = (struct vdso_timekeep *)(shared_page_mapping +
+	    sv->sv_timekeep_off);
+	tk->tk_th[idx].th_gen = 0;
+	atomic_thread_fence_rel();
 	if (enabled)
 		tk->tk_th[idx] = th;
-	tk->tk_enabled = enabled;
 	atomic_store_rel_32(&tk->tk_th[idx].th_gen, sv->sv_timekeep_gen);
-	tk->tk_current = idx;
+	atomic_store_rel_32(&tk->tk_current, idx);
+
+	/*
+	 * The ordering of the assignment to tk_enabled relative to
+	 * the update of the vdso_timehands is not important.
+	 */
+	tk->tk_enabled = enabled;
 }
 
 #ifdef COMPAT_FREEBSD32
 static void
 timehands_update32(struct sysentvec *sv)
 {
-	struct vdso_timekeep32 *tk;
 	struct vdso_timehands32 th;
+	struct vdso_timekeep32 *tk;
 	uint32_t enabled, idx;
 
 	enabled = tc_fill_vdso_timehands32(&th);
-	tk = (struct vdso_timekeep32 *)(shared_page_mapping +
-	    sv->sv_timekeep_off);
+	th.th_gen = 0;
 	idx = sv->sv_timekeep_curr;
-	atomic_store_rel_32(&tk->tk_th[idx].th_gen, 0);
 	if (++idx >= VDSO_TH_NUM)
 		idx = 0;
 	sv->sv_timekeep_curr = idx;
 	if (++sv->sv_timekeep_gen == 0)
 		sv->sv_timekeep_gen = 1;
-	th.th_gen = 0;
+
+	tk = (struct vdso_timekeep32 *)(shared_page_mapping +
+	    sv->sv_timekeep_off);
+	tk->tk_th[idx].th_gen = 0;
+	atomic_thread_fence_rel();
 	if (enabled)
 		tk->tk_th[idx] = th;
-	tk->tk_enabled = enabled;
 	atomic_store_rel_32(&tk->tk_th[idx].th_gen, sv->sv_timekeep_gen);
-	tk->tk_current = idx;
+	atomic_store_rel_32(&tk->tk_current, idx);
+	tk->tk_enabled = enabled;
 }
 #endif
 


More information about the svn-src-all mailing list