svn commit: r326110 - head/sys/kern

[Mateusz Guzik]

Author: mjg
Date: Wed Nov 22 23:52:05 2017
New Revision: 326110
URL: https://svnweb.freebsd.org/changeset/base/326110

Log:
  rwlock: don't check for curthread's read lock count in the fast path

Modified:
  head/sys/kern/kern_rwlock.c

Modified: head/sys/kern/kern_rwlock.c
==============================================================================
--- head/sys/kern/kern_rwlock.c	Wed Nov 22 23:10:20 2017	(r326109)
+++ head/sys/kern/kern_rwlock.c	Wed Nov 22 23:52:05 2017	(r326110)
@@ -364,12 +364,20 @@ _rw_wunlock_cookie(volatile uintptr_t *c, const char *
  * is unlocked and has no writer waiters or spinners.  Failing otherwise
  * prioritizes writers before readers.
  */
-#define	RW_CAN_READ(td, _rw)						\
-    (((_rw) & (RW_LOCK_READ | RW_LOCK_WRITE_WAITERS | RW_LOCK_WRITE_SPINNER)) ==\
-    RW_LOCK_READ || ((td)->td_rw_rlocks && (_rw) & RW_LOCK_READ))
+static bool __always_inline
+__rw_can_read(struct thread *td, uintptr_t v, bool fp)
+{
 
+	if ((v & (RW_LOCK_READ | RW_LOCK_WRITE_WAITERS | RW_LOCK_WRITE_SPINNER))
+	    == RW_LOCK_READ)
+		return (true);
+	if (!fp && td->td_rw_rlocks && (v & RW_LOCK_READ))
+		return (true);
+	return (false);
+}
+
 static bool __always_inline
-__rw_rlock_try(struct rwlock *rw, struct thread *td, uintptr_t *vp
+__rw_rlock_try(struct rwlock *rw, struct thread *td, uintptr_t *vp, bool fp
     LOCK_FILE_LINE_ARG_DEF)
 {
 
@@ -383,7 +391,7 @@ __rw_rlock_try(struct rwlock *rw, struct thread *td, u
 	 * completely unlocked rwlock since such a lock is encoded
 	 * as a read lock with no waiters.
 	 */
-	while (RW_CAN_READ(td, *vp)) {
+	while (__rw_can_read(td, *vp, fp)) {
 		if (atomic_fcmpset_acq_ptr(&rw->rw_lock, vp,
 			*vp + RW_ONE_READER)) {
 			if (LOCK_LOG_TEST(&rw->lock_object, 0))
@@ -452,7 +460,7 @@ __rw_rlock_hard(struct rwlock *rw, struct thread *td, 
 #endif
 
 	for (;;) {
-		if (__rw_rlock_try(rw, td, &v LOCK_FILE_LINE_ARG))
+		if (__rw_rlock_try(rw, td, &v, false LOCK_FILE_LINE_ARG))
 			break;
 #ifdef KDTRACE_HOOKS
 		lda.spin_cnt++;
@@ -492,7 +500,7 @@ __rw_rlock_hard(struct rwlock *rw, struct thread *td, 
 				n = RW_READERS(v);
 				lock_delay_spin(n);
 				v = RW_READ_VALUE(rw);
-				if ((v & RW_LOCK_READ) == 0 || RW_CAN_READ(td, v))
+				if ((v & RW_LOCK_READ) == 0 || __rw_can_read(td, v, false))
 					break;
 			}
 #ifdef KDTRACE_HOOKS
@@ -518,7 +526,7 @@ __rw_rlock_hard(struct rwlock *rw, struct thread *td, 
 		 * recheck its state and restart the loop if needed.
 		 */
 		v = RW_READ_VALUE(rw);
-		if (RW_CAN_READ(td, v)) {
+		if (__rw_can_read(td, v, false)) {
 			turnstile_cancel(ts);
 			continue;
 		}
@@ -630,7 +638,7 @@ __rw_rlock_int(struct rwlock *rw LOCK_FILE_LINE_ARG_DE
 
 	v = RW_READ_VALUE(rw);
 	if (__predict_false(LOCKSTAT_OOL_PROFILE_ENABLED(rw__acquire) ||
-	    !__rw_rlock_try(rw, td, &v LOCK_FILE_LINE_ARG)))
+	    !__rw_rlock_try(rw, td, &v, true LOCK_FILE_LINE_ARG)))
 		__rw_rlock_hard(rw, td, v LOCK_FILE_LINE_ARG);
 
 	LOCK_LOG_LOCK("RLOCK", &rw->lock_object, 0, 0, file, line);