svn commit: r355782 - head/sys/kern
Jeff Roberson
jeff at FreeBSD.org
Sun Dec 15 21:19:42 UTC 2019
Author: jeff
Date: Sun Dec 15 21:19:41 2019
New Revision: 355782
URL: https://svnweb.freebsd.org/changeset/base/355782
Log:
schedlock 3/4
Eliminate lock recursion from turnstiles. This was simply used to avoid
tracking the top-level turnstile lock. explicitly check for it before
picking up and dropping locks.
Reviewed by: kib
Tested by: pho
Differential Revision: https://reviews.freebsd.org/D22746
Modified:
head/sys/kern/subr_turnstile.c
Modified: head/sys/kern/subr_turnstile.c
==============================================================================
--- head/sys/kern/subr_turnstile.c Sun Dec 15 21:18:07 2019 (r355781)
+++ head/sys/kern/subr_turnstile.c Sun Dec 15 21:19:41 2019 (r355782)
@@ -175,6 +175,22 @@ SDT_PROBE_DEFINE(sched, , , sleep);
SDT_PROBE_DEFINE2(sched, , , wakeup, "struct thread *",
"struct proc *");
+static inline void
+propagate_unlock_ts(struct turnstile *top, struct turnstile *ts)
+{
+
+ if (ts != top)
+ mtx_unlock_spin(&ts->ts_lock);
+}
+
+static inline void
+propagate_unlock_td(struct turnstile *top, struct thread *td)
+{
+
+ if (td->td_lock != &top->ts_lock)
+ thread_unlock(td);
+}
+
/*
* Walks the chain of turnstiles and their owners to propagate the priority
* of the thread being blocked to all the threads holding locks that have to
@@ -183,20 +199,19 @@ SDT_PROBE_DEFINE2(sched, , , wakeup, "struct thread *"
static void
propagate_priority(struct thread *td)
{
- struct turnstile *ts;
+ struct turnstile *ts, *top;
int pri;
THREAD_LOCK_ASSERT(td, MA_OWNED);
pri = td->td_priority;
- ts = td->td_blocked;
+ top = ts = td->td_blocked;
THREAD_LOCKPTR_ASSERT(td, &ts->ts_lock);
+
/*
- * Grab a recursive lock on this turnstile chain so it stays locked
- * for the whole operation. The caller expects us to return with
- * the original lock held. We only ever lock down the chain so
- * the lock order is constant.
+ * The original turnstile lock is held across the entire
+ * operation. We only ever lock down the chain so the lock
+ * order is constant.
*/
- mtx_lock_spin(&ts->ts_lock);
for (;;) {
td = ts->ts_owner;
@@ -205,12 +220,19 @@ propagate_priority(struct thread *td)
* This might be a read lock with no owner. There's
* not much we can do, so just bail.
*/
- mtx_unlock_spin(&ts->ts_lock);
+ propagate_unlock_ts(top, ts);
return;
}
- thread_lock_flags(td, MTX_DUPOK);
- mtx_unlock_spin(&ts->ts_lock);
+ /*
+ * Wait for the thread lock to be stable and then only
+ * acquire if it is not the turnstile lock.
+ */
+ thread_lock_block_wait(td);
+ if (td->td_lock != &ts->ts_lock) {
+ thread_lock_flags(td, MTX_DUPOK);
+ propagate_unlock_ts(top, ts);
+ }
MPASS(td->td_proc != NULL);
MPASS(td->td_proc->p_magic == P_MAGIC);
@@ -233,7 +255,7 @@ propagate_priority(struct thread *td)
* thread that is being blocked, we are finished.
*/
if (td->td_priority <= pri) {
- thread_unlock(td);
+ propagate_unlock_td(top, td);
return;
}
@@ -248,7 +270,7 @@ propagate_priority(struct thread *td)
*/
if (TD_IS_RUNNING(td) || TD_ON_RUNQ(td)) {
MPASS(td->td_blocked == NULL);
- thread_unlock(td);
+ propagate_unlock_td(top, td);
return;
}
@@ -276,7 +298,7 @@ propagate_priority(struct thread *td)
THREAD_LOCKPTR_ASSERT(td, &ts->ts_lock);
/* Resort td on the list if needed. */
if (!turnstile_adjust_thread(ts, td)) {
- mtx_unlock_spin(&ts->ts_lock);
+ propagate_unlock_ts(top, ts);
return;
}
/* The thread lock is released as ts lock above. */
@@ -498,7 +520,7 @@ turnstile_init(void *mem, int size, int flags)
TAILQ_INIT(&ts->ts_blocked[TS_SHARED_QUEUE]);
TAILQ_INIT(&ts->ts_pending);
LIST_INIT(&ts->ts_free);
- mtx_init(&ts->ts_lock, "turnstile lock", NULL, MTX_SPIN | MTX_RECURSE);
+ mtx_init(&ts->ts_lock, "turnstile lock", NULL, MTX_SPIN);
return (0);
}
More information about the svn-src-all
mailing list