git: 87ee63bac69d - main - locks: add a runtime check for missing turnstile

The branch main has been updated by mjg:

URL: https://cgit.FreeBSD.org/src/commit/?id=87ee63bac69dc49291f55590b8baa57cad6c7d85

commit 87ee63bac69dc49291f55590b8baa57cad6c7d85
Author:     Mateusz Guzik <mjg@FreeBSD.org>
AuthorDate: 2024-07-11 00:17:27 +0000
Commit:     Mateusz Guzik <mjg@FreeBSD.org>
CommitDate: 2024-07-11 11:06:52 +0000

    locks: add a runtime check for missing turnstile
    
    There are sometimes bugs which result in the unlock fast path failing,
    which in turns causes a not-helpful crash report when dereferencing a
    NULL turnstile. Help debugging such cases by pointing out what happened
    along with some debug.
    
    Sponsored by:   Rubicon Communications, LLC ("Netgate")
---
 sys/kern/kern_mutex.c  |  4 +++-
 sys/kern/kern_rwlock.c | 16 ++++++++++++----
 2 files changed, 15 insertions(+), 5 deletions(-)

diff --git a/sys/kern/kern_mutex.c b/sys/kern/kern_mutex.c
index 90361b23c09a..0fa624cc4bb1 100644
--- a/sys/kern/kern_mutex.c
+++ b/sys/kern/kern_mutex.c
@@ -1053,7 +1053,9 @@ __mtx_unlock_sleep(volatile uintptr_t *c, uintptr_t v)
 	turnstile_chain_lock(&m->lock_object);
 	_mtx_release_lock_quick(m);
 	ts = turnstile_lookup(&m->lock_object);
-	MPASS(ts != NULL);
+	if (__predict_false(ts == NULL)) {
+		panic("got NULL turnstile on mutex %p v %zx", m, v);
+	}
 	if (LOCK_LOG_TEST(&m->lock_object, opts))
 		CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p contested", m);
 	turnstile_broadcast(ts, TS_EXCLUSIVE_QUEUE);
diff --git a/sys/kern/kern_rwlock.c b/sys/kern/kern_rwlock.c
index f53c69b5e6ec..31ff8a7213fd 100644
--- a/sys/kern/kern_rwlock.c
+++ b/sys/kern/kern_rwlock.c
@@ -770,11 +770,12 @@ __rw_runlock_hard(struct rwlock *rw, struct thread *td, uintptr_t v
     LOCK_FILE_LINE_ARG_DEF)
 {
 	struct turnstile *ts;
-	uintptr_t setv, queue;
+	uintptr_t setv, passedv, queue;
 
 	if (SCHEDULER_STOPPED())
 		return;
 
+	passedv = v;
 	if (__rw_runlock_try(rw, td, &v))
 		goto out_lockstat;
 
@@ -827,7 +828,10 @@ __rw_runlock_hard(struct rwlock *rw, struct thread *td, uintptr_t v
 		 * release the lock.
 		 */
 		ts = turnstile_lookup(&rw->lock_object);
-		MPASS(ts != NULL);
+		if (__predict_false(ts == NULL)) {
+			panic("got NULL turnstile on rwlock %p passedv %zx v %zx",
+			    rw, passedv, v);
+		}
 		turnstile_broadcast(ts, queue);
 		turnstile_unpend(ts);
 		td->td_rw_rlocks--;
@@ -1206,7 +1210,7 @@ __rw_wunlock_hard(volatile uintptr_t *c, uintptr_t v LOCK_FILE_LINE_ARG_DEF)
 {
 	struct rwlock *rw;
 	struct turnstile *ts;
-	uintptr_t tid, setv;
+	uintptr_t tid, setv, passedv;
 	int queue;
 
 	tid = (uintptr_t)curthread;
@@ -1254,6 +1258,7 @@ __rw_wunlock_hard(volatile uintptr_t *c, uintptr_t v LOCK_FILE_LINE_ARG_DEF)
 	 * of waiters or doing some complicated lock handoff gymnastics.
 	 */
 	setv = RW_UNLOCKED;
+	passedv = v;
 	v = RW_READ_VALUE(rw);
 	queue = TS_SHARED_QUEUE;
 	if (v & RW_LOCK_WRITE_WAITERS) {
@@ -1268,7 +1273,10 @@ __rw_wunlock_hard(volatile uintptr_t *c, uintptr_t v LOCK_FILE_LINE_ARG_DEF)
 		    queue == TS_SHARED_QUEUE ? "read" : "write");
 
 	ts = turnstile_lookup(&rw->lock_object);
-	MPASS(ts != NULL);
+	if (__predict_false(ts == NULL)) {
+		panic("got NULL turnstile on rwlock %p passedv %zx v %zx", rw,
+		    passedv, v);
+	}
 	turnstile_broadcast(ts, queue);
 	turnstile_unpend(ts);
 	turnstile_chain_unlock(&rw->lock_object);