svn commit: r334062 - head/sys/kern

[Mateusz Guzik]

Author: mjg
Date: Tue May 22 19:24:57 2018
New Revision: 334062
URL: https://svnweb.freebsd.org/changeset/base/334062

Log:
  Move preemption handling out of critical_exit.
  
  In preperataion for making the enter/exit pair inline.
  
  Reviewed by:	kib

Modified:
  head/sys/kern/kern_switch.c

Modified: head/sys/kern/kern_switch.c
==============================================================================
--- head/sys/kern/kern_switch.c	Tue May 22 19:11:06 2018	(r334061)
+++ head/sys/kern/kern_switch.c	Tue May 22 19:24:57 2018	(r334062)
@@ -209,48 +209,49 @@ critical_enter(void)
 	    (long)td->td_proc->p_pid, td->td_name, td->td_critnest);
 }
 
-void
-critical_exit(void)
+static void __noinline
+critical_exit_preempt(void)
 {
 	struct thread *td;
 	int flags;
 
 	td = curthread;
-	KASSERT(td->td_critnest != 0,
-	    ("critical_exit: td_critnest == 0"));
+	KASSERT(td->td_owepreempt != 0,
+	    ("critical_exit: td_owepreempt == 0"));
+	if (td->td_critnest != 0)
+		return;
+	if (kdb_active)
+		return;
 
-	if (td->td_critnest == 1) {
-		td->td_critnest = 0;
+	/*
+	 * Microoptimization: we committed to switch,
+	 * disable preemption in interrupt handlers
+	 * while spinning for the thread lock.
+	 */
+	td->td_critnest = 1;
+	thread_lock(td);
+	td->td_critnest--;
+	flags = SW_INVOL | SW_PREEMPT;
+	if (TD_IS_IDLETHREAD(td))
+		flags |= SWT_IDLE;
+	else
+		flags |= SWT_OWEPREEMPT;
+	mi_switch(flags, NULL);
+	thread_unlock(td);
+}
 
-		/*
-		 * Interrupt handlers execute critical_exit() on
-		 * leave, and td_owepreempt may be left set by an
-		 * interrupt handler only when td_critnest > 0.  If we
-		 * are decrementing td_critnest from 1 to 0, read
-		 * td_owepreempt after decrementing, to not miss the
-		 * preempt.  Disallow compiler to reorder operations.
-		 */
-		__compiler_membar();
-		if (td->td_owepreempt && !kdb_active) {
-			/*
-			 * Microoptimization: we committed to switch,
-			 * disable preemption in interrupt handlers
-			 * while spinning for the thread lock.
-			 */
-			td->td_critnest = 1;
-			thread_lock(td);
-			td->td_critnest--;
-			flags = SW_INVOL | SW_PREEMPT;
-			if (TD_IS_IDLETHREAD(td))
-				flags |= SWT_IDLE;
-			else
-				flags |= SWT_OWEPREEMPT;
-			mi_switch(flags, NULL);
-			thread_unlock(td);
-		}
-	} else
-		td->td_critnest--;
+void
+critical_exit(void)
+{
+	struct thread *td;
 
+	td = curthread;
+	KASSERT(td->td_critnest != 0,
+	    ("critical_exit: td_critnest == 0"));
+	td->td_critnest--;
+	__compiler_membar();
+	if (__predict_false(td->td_owepreempt))
+		critical_exit_preempt();
 	CTR4(KTR_CRITICAL, "critical_exit by thread %p (%ld, %s) to %d", td,
 	    (long)td->td_proc->p_pid, td->td_name, td->td_critnest);
 }