svn commit: r313875 - in head/sys: kern sys

[Mateusz Guzik]

Author: mjg
Date: Fri Feb 17 14:55:59 2017
New Revision: 313875
URL: https://svnweb.freebsd.org/changeset/base/313875

Log:
  mtx: microoptimize lockstat handling in __mtx_lock_sleep
  
  This saves a function call and multiple branches after the lock is acquired.

Modified:
  head/sys/kern/kern_lockstat.c
  head/sys/kern/kern_mutex.c
  head/sys/sys/lockstat.h

Modified: head/sys/kern/kern_lockstat.c
==============================================================================
--- head/sys/kern/kern_lockstat.c	Fri Feb 17 14:05:57 2017	(r313874)
+++ head/sys/kern/kern_lockstat.c	Fri Feb 17 14:55:59 2017	(r313875)
@@ -62,7 +62,7 @@ SDT_PROBE_DEFINE1(lockstat, , , sx__down
 
 SDT_PROBE_DEFINE2(lockstat, , , thread__spin, "struct mtx *", "uint64_t");
 
-int __read_mostly lockstat_enabled;
+volatile int __read_mostly lockstat_enabled;
 
 uint64_t 
 lockstat_nsecs(struct lock_object *lo)

Modified: head/sys/kern/kern_mutex.c
==============================================================================
--- head/sys/kern/kern_mutex.c	Fri Feb 17 14:05:57 2017	(r313874)
+++ head/sys/kern/kern_mutex.c	Fri Feb 17 14:55:59 2017	(r313875)
@@ -446,6 +446,7 @@ __mtx_lock_sleep(volatile uintptr_t *c, 
 	u_int sleep_cnt = 0;
 	int64_t sleep_time = 0;
 	int64_t all_time = 0;
+	int doing_lockstat;
 #endif
 
 	if (SCHEDULER_STOPPED())
@@ -484,7 +485,9 @@ __mtx_lock_sleep(volatile uintptr_t *c, 
 		    "_mtx_lock_sleep: %s contested (lock=%p) at %s:%d",
 		    m->lock_object.lo_name, (void *)m->mtx_lock, file, line);
 #ifdef KDTRACE_HOOKS
-	all_time -= lockstat_nsecs(&m->lock_object);
+	doing_lockstat = lockstat_enabled;
+	if (__predict_false(doing_lockstat))
+		all_time -= lockstat_nsecs(&m->lock_object);
 #endif
 
 	for (;;) {
@@ -591,9 +594,6 @@ __mtx_lock_sleep(volatile uintptr_t *c, 
 #endif
 		v = MTX_READ_VALUE(m);
 	}
-#ifdef KDTRACE_HOOKS
-	all_time += lockstat_nsecs(&m->lock_object);
-#endif
 #ifdef KTR
 	if (cont_logged) {
 		CTR4(KTR_CONTENTION,
@@ -601,6 +601,11 @@ __mtx_lock_sleep(volatile uintptr_t *c, 
 		    m->lock_object.lo_name, (void *)tid, file, line);
 	}
 #endif
+#ifdef KDTRACE_HOOKS
+	if (__predict_true(!doing_lockstat))
+		return;
+	all_time += lockstat_nsecs(&m->lock_object);
+#endif
 	LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(adaptive__acquire, m, contested,
 	    waittime, file, line);
 #ifdef KDTRACE_HOOKS

Modified: head/sys/sys/lockstat.h
==============================================================================
--- head/sys/sys/lockstat.h	Fri Feb 17 14:05:57 2017	(r313874)
+++ head/sys/sys/lockstat.h	Fri Feb 17 14:55:59 2017	(r313875)
@@ -68,7 +68,7 @@ SDT_PROBE_DECLARE(lockstat, , , thread__
 #define	LOCKSTAT_WRITER		0
 #define	LOCKSTAT_READER		1
 
-extern int lockstat_enabled;
+extern volatile int lockstat_enabled;
 
 #ifdef KDTRACE_HOOKS