svn commit: r313392 - in head/sys: kern sys

[Mateusz Guzik]

Author: mjg
Date: Tue Feb  7 17:04:31 2017
New Revision: 313392
URL: https://svnweb.freebsd.org/changeset/base/313392

Log:
  rwlock: implement RW_LOCK_WRITER_RECURSED bit
  
  This moves recursion handling out of the inlined wunlock path and in
  particular saves a read and a branch.
  
  Discussed with:

Modified:
  head/sys/kern/kern_rwlock.c
  head/sys/sys/rwlock.h

Modified: head/sys/kern/kern_rwlock.c
==============================================================================
--- head/sys/kern/kern_rwlock.c	Tue Feb  7 17:03:22 2017	(r313391)
+++ head/sys/kern/kern_rwlock.c	Tue Feb  7 17:04:31 2017	(r313392)
@@ -312,6 +312,7 @@ __rw_try_wlock(volatile uintptr_t *c, co
 	if (rw_wlocked(rw) &&
 	    (rw->lock_object.lo_flags & LO_RECURSABLE) != 0) {
 		rw->rw_recurse++;
+		atomic_set_ptr(&rw->rw_lock, RW_LOCK_WRITER_RECURSED);
 		rval = 1;
 	} else
 		rval = atomic_cmpset_acq_ptr(&rw->rw_lock, RW_UNLOCKED,
@@ -345,10 +346,8 @@ _rw_wunlock_cookie(volatile uintptr_t *c
 	WITNESS_UNLOCK(&rw->lock_object, LOP_EXCLUSIVE, file, line);
 	LOCK_LOG_LOCK("WUNLOCK", &rw->lock_object, 0, rw->rw_recurse, file,
 	    line);
-	if (rw->rw_recurse)
-		rw->rw_recurse--;
-	else
-		_rw_wunlock_hard(rw, (uintptr_t)curthread, file, line);
+
+	_rw_wunlock_hard(rw, (uintptr_t)curthread, file, line);
 
 	TD_LOCKS_DEC(curthread);
 }
@@ -802,6 +801,7 @@ __rw_wlock_hard(volatile uintptr_t *c, u
 		    ("%s: recursing but non-recursive rw %s @ %s:%d\n",
 		    __func__, rw->lock_object.lo_name, file, line));
 		rw->rw_recurse++;
+		atomic_set_ptr(&rw->rw_lock, RW_LOCK_WRITER_RECURSED);
 		if (LOCK_LOG_TEST(&rw->lock_object, 0))
 			CTR2(KTR_LOCK, "%s: %p recursing", __func__, rw);
 		return;
@@ -994,12 +994,17 @@ __rw_wunlock_hard(volatile uintptr_t *c,
 		return;
 
 	rw = rwlock2rw(c);
-	MPASS(!rw_recursed(rw));
 
-	LOCKSTAT_PROFILE_RELEASE_RWLOCK(rw__release, rw,
-	    LOCKSTAT_WRITER);
-	if (_rw_write_unlock(rw, tid))
+	if (!rw_recursed(rw)) {
+		LOCKSTAT_PROFILE_RELEASE_RWLOCK(rw__release, rw,
+		    LOCKSTAT_WRITER);
+		if (_rw_write_unlock(rw, tid))
+			return;
+	} else {
+		if (--(rw->rw_recurse) == 0)
+			atomic_clear_ptr(&rw->rw_lock, RW_LOCK_WRITER_RECURSED);
 		return;
+	}
 
 	KASSERT(rw->rw_lock & (RW_LOCK_READ_WAITERS | RW_LOCK_WRITE_WAITERS),
 	    ("%s: neither of the waiter flags are set", __func__));

Modified: head/sys/sys/rwlock.h
==============================================================================
--- head/sys/sys/rwlock.h	Tue Feb  7 17:03:22 2017	(r313391)
+++ head/sys/sys/rwlock.h	Tue Feb  7 17:04:31 2017	(r313392)
@@ -58,9 +58,10 @@
 #define	RW_LOCK_READ_WAITERS	0x02
 #define	RW_LOCK_WRITE_WAITERS	0x04
 #define	RW_LOCK_WRITE_SPINNER	0x08
+#define	RW_LOCK_WRITER_RECURSED	0x10
 #define	RW_LOCK_FLAGMASK						\
 	(RW_LOCK_READ | RW_LOCK_READ_WAITERS | RW_LOCK_WRITE_WAITERS |	\
-	RW_LOCK_WRITE_SPINNER)
+	RW_LOCK_WRITE_SPINNER | RW_LOCK_WRITER_RECURSED)
 #define	RW_LOCK_WAITERS		(RW_LOCK_READ_WAITERS | RW_LOCK_WRITE_WAITERS)
 
 #define	RW_OWNER(x)		((x) & ~RW_LOCK_FLAGMASK)
@@ -111,13 +112,9 @@
 #define	__rw_wunlock(rw, tid, file, line) do {				\
 	uintptr_t _tid = (uintptr_t)(tid);				\
 									\
-	if ((rw)->rw_recurse)						\
-		(rw)->rw_recurse--;					\
-	else {								\
-		if (__predict_false(LOCKSTAT_PROFILE_ENABLED(rw__release) ||\
-		    !_rw_write_unlock((rw), _tid)))			\
-			_rw_wunlock_hard((rw), _tid, (file), (line));	\
-	}								\
+	if (__predict_false(LOCKSTAT_PROFILE_ENABLED(rw__release) ||	\
+	    !_rw_write_unlock((rw), _tid)))				\
+		_rw_wunlock_hard((rw), _tid, (file), (line));		\
 } while (0)
 
 /*