git: a09a84f84f34 - stable/13 - umtx: Split do_unlock_pi on two counterparts.

The branch stable/13 has been updated by dchagin:

URL: https://cgit.FreeBSD.org/src/commit/?id=a09a84f84f34af4a887af21c48d18bdd2ff05136

commit a09a84f84f34af4a887af21c48d18bdd2ff05136
Author:     Dmitry Chagin <dchagin@FreeBSD.org>
AuthorDate: 2021-07-29 09:47:39 +0000
Commit:     Dmitry Chagin <dchagin@FreeBSD.org>
CommitDate: 2022-06-17 19:33:18 +0000

    umtx: Split do_unlock_pi on two counterparts.
    
    The umtx_pi_frop() will be used by Linux emulation layer.
    
    Reviewed by:            kib
    Differential Revision:  https://reviews.freebsd.org/D31238
    MFC after:              2 weeks
    
    (cherry picked from commit 9e32efa79bf2dc95dab86d801c08cffcd6e171db)
---
 sys/kern/kern_umtx.c | 144 ++++++++++++++++++++++++++++++---------------------
 sys/sys/umtxvar.h    |   1 +
 2 files changed, 87 insertions(+), 58 deletions(-)

diff --git a/sys/kern/kern_umtx.c b/sys/kern/kern_umtx.c
index f7d9d2ee4949..442f275b658c 100644
--- a/sys/kern/kern_umtx.c
+++ b/sys/kern/kern_umtx.c
@@ -90,6 +90,17 @@ __FBSDID("$FreeBSD$");
 #endif
 
 #define	UMTXQ_LOCKED_ASSERT(uc)		mtx_assert(&(uc)->uc_lock, MA_OWNED)
+#ifdef INVARIANTS
+#define	UMTXQ_ASSERT_LOCKED_BUSY(key) do {				\
+	struct umtxq_chain *uc;						\
+									\
+	uc = umtxq_getchain(key);					\
+	mtx_assert(&uc->uc_lock, MA_OWNED);				\
+	KASSERT(uc->uc_busy != 0, ("umtx chain is not busy"));		\
+} while (0)
+#else
+#define	UMTXQ_ASSERT_LOCKED_BUSY(key) do {} while (0)
+#endif
 
 /*
  * Don't propagate time-sharing priority, there is a security reason,
@@ -2077,6 +2088,73 @@ umtx_pi_insert(struct umtx_pi *pi)
 	TAILQ_INSERT_TAIL(&uc->uc_pi_list, pi, pi_hashlink);
 }
 
+/*
+ * Drop a PI mutex and wakeup a top waiter.
+ */
+int
+umtx_pi_drop(struct thread *td, struct umtx_key *key, bool rb, int *count)
+{
+	struct umtx_q *uq_first, *uq_first2, *uq_me;
+	struct umtx_pi *pi, *pi2;
+	int pri;
+
+	UMTXQ_ASSERT_LOCKED_BUSY(key);
+	*count = umtxq_count_pi(key, &uq_first);
+	if (uq_first != NULL) {
+		mtx_lock(&umtx_lock);
+		pi = uq_first->uq_pi_blocked;
+		KASSERT(pi != NULL, ("pi == NULL?"));
+		if (pi->pi_owner != td && !(rb && pi->pi_owner == NULL)) {
+			mtx_unlock(&umtx_lock);
+			/* userland messed the mutex */
+			return (EPERM);
+		}
+		uq_me = td->td_umtxq;
+		if (pi->pi_owner == td)
+			umtx_pi_disown(pi);
+		/* get highest priority thread which is still sleeping. */
+		uq_first = TAILQ_FIRST(&pi->pi_blocked);
+		while (uq_first != NULL &&
+		    (uq_first->uq_flags & UQF_UMTXQ) == 0) {
+			uq_first = TAILQ_NEXT(uq_first, uq_lockq);
+		}
+		pri = PRI_MAX;
+		TAILQ_FOREACH(pi2, &uq_me->uq_pi_contested, pi_link) {
+			uq_first2 = TAILQ_FIRST(&pi2->pi_blocked);
+			if (uq_first2 != NULL) {
+				if (pri > UPRI(uq_first2->uq_thread))
+					pri = UPRI(uq_first2->uq_thread);
+			}
+		}
+		thread_lock(td);
+		sched_lend_user_prio(td, pri);
+		thread_unlock(td);
+		mtx_unlock(&umtx_lock);
+		if (uq_first)
+			umtxq_signal_thread(uq_first);
+	} else {
+		pi = umtx_pi_lookup(key);
+		/*
+		 * A umtx_pi can exist if a signal or timeout removed the
+		 * last waiter from the umtxq, but there is still
+		 * a thread in do_lock_pi() holding the umtx_pi.
+		 */
+		if (pi != NULL) {
+			/*
+			 * The umtx_pi can be unowned, such as when a thread
+			 * has just entered do_lock_pi(), allocated the
+			 * umtx_pi, and unlocked the umtxq.
+			 * If the current thread owns it, it must disown it.
+			 */
+			mtx_lock(&umtx_lock);
+			if (pi->pi_owner == td)
+				umtx_pi_disown(pi);
+			mtx_unlock(&umtx_lock);
+		}
+	}
+	return (0);
+}
+
 /*
  * Lock a PI mutex.
  */
@@ -2298,10 +2376,8 @@ static int
 do_unlock_pi(struct thread *td, struct umutex *m, uint32_t flags, bool rb)
 {
 	struct umtx_key key;
-	struct umtx_q *uq_first, *uq_first2, *uq_me;
-	struct umtx_pi *pi, *pi2;
 	uint32_t id, new_owner, old, owner;
-	int count, error, pri;
+	int count, error;
 
 	id = td->td_tid;
 
@@ -2342,61 +2418,13 @@ usrloop:
 
 	umtxq_lock(&key);
 	umtxq_busy(&key);
-	count = umtxq_count_pi(&key, &uq_first);
-	if (uq_first != NULL) {
-		mtx_lock(&umtx_lock);
-		pi = uq_first->uq_pi_blocked;
-		KASSERT(pi != NULL, ("pi == NULL?"));
-		if (pi->pi_owner != td && !(rb && pi->pi_owner == NULL)) {
-			mtx_unlock(&umtx_lock);
-			umtxq_unbusy(&key);
-			umtxq_unlock(&key);
-			umtx_key_release(&key);
-			/* userland messed the mutex */
-			return (EPERM);
-		}
-		uq_me = td->td_umtxq;
-		if (pi->pi_owner == td)
-			umtx_pi_disown(pi);
-		/* get highest priority thread which is still sleeping. */
-		uq_first = TAILQ_FIRST(&pi->pi_blocked);
-		while (uq_first != NULL &&
-		    (uq_first->uq_flags & UQF_UMTXQ) == 0) {
-			uq_first = TAILQ_NEXT(uq_first, uq_lockq);
-		}
-		pri = PRI_MAX;
-		TAILQ_FOREACH(pi2, &uq_me->uq_pi_contested, pi_link) {
-			uq_first2 = TAILQ_FIRST(&pi2->pi_blocked);
-			if (uq_first2 != NULL) {
-				if (pri > UPRI(uq_first2->uq_thread))
-					pri = UPRI(uq_first2->uq_thread);
-			}
-		}
-		thread_lock(td);
-		sched_lend_user_prio(td, pri);
-		thread_unlock(td);
-		mtx_unlock(&umtx_lock);
-		if (uq_first)
-			umtxq_signal_thread(uq_first);
-	} else {
-		pi = umtx_pi_lookup(&key);
-		/*
-		 * A umtx_pi can exist if a signal or timeout removed the
-		 * last waiter from the umtxq, but there is still
-		 * a thread in do_lock_pi() holding the umtx_pi.
-		 */
-		if (pi != NULL) {
-			/*
-			 * The umtx_pi can be unowned, such as when a thread
-			 * has just entered do_lock_pi(), allocated the
-			 * umtx_pi, and unlocked the umtxq.
-			 * If the current thread owns it, it must disown it.
-			 */
-			mtx_lock(&umtx_lock);
-			if (pi->pi_owner == td)
-				umtx_pi_disown(pi);
-			mtx_unlock(&umtx_lock);
-		}
+	error = umtx_pi_drop(td, &key, rb, &count);
+	if (error != 0) {
+		umtxq_unbusy(&key);
+		umtxq_unlock(&key);
+		umtx_key_release(&key);
+		/* userland messed the mutex */
+		return (error);
 	}
 	umtxq_unlock(&key);
 
diff --git a/sys/sys/umtxvar.h b/sys/sys/umtxvar.h
index fd193de5818b..a0ef7931a88d 100644
--- a/sys/sys/umtxvar.h
+++ b/sys/sys/umtxvar.h
@@ -222,6 +222,7 @@ int kern_umtx_wake(struct thread *, void *, int, int);
 void umtx_pi_adjust(struct thread *, u_char);
 struct umtx_pi *umtx_pi_alloc(int);
 int umtx_pi_claim(struct umtx_pi *, struct thread *);
+int umtx_pi_drop(struct thread *, struct umtx_key *, bool, int *);
 void umtx_pi_free(struct umtx_pi *);
 void umtx_pi_insert(struct umtx_pi *);
 struct umtx_pi *umtx_pi_lookup(struct umtx_key *);