svn commit: r313282 - in head/sys: kern sys

[Mateusz Guzik]

Author: mjg
Date: Sun Feb  5 13:37:23 2017
New Revision: 313282
URL: https://svnweb.freebsd.org/changeset/base/313282

Log:
  rwlock: move lockstat handling out of inline primitives
  
  See r313275 for details.
  
  One difference here is that recursion handling was removed from the fallback
  routine. As it is it was never supposed to see a recursed lock in the first
  place. Future changes will move it out of inline variants, but right now
  there is no easy to way to test if the lock is recursed without reading
  additional words.

Modified:
  head/sys/kern/kern_rwlock.c
  head/sys/sys/rwlock.h

Modified: head/sys/kern/kern_rwlock.c
==============================================================================
--- head/sys/kern/kern_rwlock.c	Sun Feb  5 13:24:54 2017	(r313281)
+++ head/sys/kern/kern_rwlock.c	Sun Feb  5 13:37:23 2017	(r313282)
@@ -283,6 +283,7 @@ void
 _rw_wlock_cookie(volatile uintptr_t *c, const char *file, int line)
 {
 	struct rwlock *rw;
+	uintptr_t tid, v;
 
 	if (SCHEDULER_STOPPED())
 		return;
@@ -296,7 +297,14 @@ _rw_wlock_cookie(volatile uintptr_t *c, 
 	    ("rw_wlock() of destroyed rwlock @ %s:%d", file, line));
 	WITNESS_CHECKORDER(&rw->lock_object, LOP_NEWORDER | LOP_EXCLUSIVE, file,
 	    line, NULL);
-	__rw_wlock(rw, curthread, file, line);
+	tid = (uintptr_t)curthread;
+	v = RW_UNLOCKED;
+	if (!_rw_write_lock_fetch(rw, &v, tid))
+		_rw_wlock_hard(rw, v, tid, file, line);
+	else
+		LOCKSTAT_PROFILE_OBTAIN_RWLOCK_SUCCESS(rw__acquire, rw,
+		    0, 0, file, line, LOCKSTAT_WRITER);
+
 	LOCK_LOG_LOCK("WLOCK", &rw->lock_object, 0, rw->rw_recurse, file, line);
 	WITNESS_LOCK(&rw->lock_object, LOP_EXCLUSIVE, file, line);
 	TD_LOCKS_INC(curthread);
@@ -355,7 +363,11 @@ _rw_wunlock_cookie(volatile uintptr_t *c
 	WITNESS_UNLOCK(&rw->lock_object, LOP_EXCLUSIVE, file, line);
 	LOCK_LOG_LOCK("WUNLOCK", &rw->lock_object, 0, rw->rw_recurse, file,
 	    line);
-	__rw_wunlock(rw, curthread, file, line);
+	if (rw->rw_recurse)
+		rw->rw_recurse--;
+	else
+		_rw_wunlock_hard(rw, (uintptr_t)curthread, file, line);
+
 	TD_LOCKS_DEC(curthread);
 }
 
@@ -998,13 +1010,12 @@ __rw_wunlock_hard(volatile uintptr_t *c,
 		return;
 
 	rw = rwlock2rw(c);
+	MPASS(!rw_recursed(rw));
 
-	if (rw_wlocked(rw) && rw_recursed(rw)) {
-		rw->rw_recurse--;
-		if (LOCK_LOG_TEST(&rw->lock_object, 0))
-			CTR2(KTR_LOCK, "%s: %p unrecursing", __func__, rw);
+	LOCKSTAT_PROFILE_RELEASE_RWLOCK(rw__release, rw,
+	    LOCKSTAT_WRITER);
+	if (_rw_write_unlock(rw, tid))
 		return;
-	}
 
 	KASSERT(rw->rw_lock & (RW_LOCK_READ_WAITERS | RW_LOCK_WRITE_WAITERS),
 	    ("%s: neither of the waiter flags are set", __func__));

Modified: head/sys/sys/rwlock.h
==============================================================================
--- head/sys/sys/rwlock.h	Sun Feb  5 13:24:54 2017	(r313281)
+++ head/sys/sys/rwlock.h	Sun Feb  5 13:37:23 2017	(r313282)
@@ -84,10 +84,8 @@
 #define	_rw_write_lock(rw, tid)						\
 	atomic_cmpset_acq_ptr(&(rw)->rw_lock, RW_UNLOCKED, (tid))
 
-#define	_rw_write_lock_fetch(rw, vp, tid) ({				\
-	*vp = RW_UNLOCKED;						\
-	atomic_fcmpset_acq_ptr(&(rw)->rw_lock, vp, (tid));		\
-})
+#define	_rw_write_lock_fetch(rw, vp, tid)				\
+	atomic_fcmpset_acq_ptr(&(rw)->rw_lock, vp, (tid))
 
 /* Release a write lock quickly if there are no waiters. */
 #define	_rw_write_unlock(rw, tid)					\
@@ -102,13 +100,11 @@
 /* Acquire a write lock. */
 #define	__rw_wlock(rw, tid, file, line) do {				\
 	uintptr_t _tid = (uintptr_t)(tid);				\
-	uintptr_t _v;							\
+	uintptr_t _v = RW_UNLOCKED;					\
 									\
-	if (!_rw_write_lock_fetch((rw), &_v, _tid))			\
+	if (__predict_false(LOCKSTAT_PROFILE_ENABLED(rw__acquire) ||	\
+	    !_rw_write_lock_fetch((rw), &_v, _tid)))			\
 		_rw_wlock_hard((rw), _v, _tid, (file), (line));		\
-	else 								\
-		LOCKSTAT_PROFILE_OBTAIN_RWLOCK_SUCCESS(rw__acquire, rw,	\
-		    0, 0, file, line, LOCKSTAT_WRITER);			\
 } while (0)
 
 /* Release a write lock. */
@@ -118,9 +114,8 @@
 	if ((rw)->rw_recurse)						\
 		(rw)->rw_recurse--;					\
 	else {								\
-		LOCKSTAT_PROFILE_RELEASE_RWLOCK(rw__release, rw,	\
-		    LOCKSTAT_WRITER);					\
-		if (!_rw_write_unlock((rw), _tid))			\
+		if (__predict_false(LOCKSTAT_PROFILE_ENABLED(rw__release) ||\
+		    !_rw_write_unlock((rw), _tid)))			\
 			_rw_wunlock_hard((rw), _tid, (file), (line));	\
 	}								\
 } while (0)