svn commit: r333697 - in head/sys: kern sys

[Matt Macy]

Author: mmacy
Date: Thu May 17 01:13:40 2018
New Revision: 333697
URL: https://svnweb.freebsd.org/changeset/base/333697

Log:
  epoch(9): make recursion lighter weight
  
  There isn't any real work to do except bump td_epochnest when recursing.
  Skip the additional work in this case.
  
  Approved by:	sbruno

Modified:
  head/sys/kern/subr_epoch.c
  head/sys/sys/epoch.h

Modified: head/sys/kern/subr_epoch.c
==============================================================================
--- head/sys/kern/subr_epoch.c	Thu May 17 00:52:48 2018	(r333696)
+++ head/sys/kern/subr_epoch.c	Thu May 17 01:13:40 2018	(r333697)
@@ -284,21 +284,15 @@ epoch_free(epoch_t epoch)
 	} while (0)
 
 void
-epoch_enter(epoch_t epoch)
+epoch_enter_internal(epoch_t epoch, struct thread *td)
 {
 	struct epoch_pcpu_state *eps;
-	struct thread *td;
 
 	INIT_CHECK(epoch);
-
-	td = curthread;
 	critical_enter();
 	eps = epoch->e_pcpu[curcpu];
-	td->td_epochnest++;
-	MPASS(td->td_epochnest < UCHAR_MAX - 2);
-	if (td->td_epochnest == 1)
-		TAILQ_INSERT_TAIL(&eps->eps_record.er_tdlist, td, td_epochq);
 #ifdef INVARIANTS
+	MPASS(td->td_epochnest < UCHAR_MAX - 2);
 	if (td->td_epochnest > 1) {
 		struct thread *curtd;
 		int found = 0;
@@ -307,38 +301,31 @@ epoch_enter(epoch_t epoch)
 			if (curtd == td)
 				found = 1;
 		KASSERT(found, ("recursing on a second epoch"));
-	}
-#endif
-	if (td->td_epochnest > 1) {
 		critical_exit();
 		return;
 	}
+#endif
+	TAILQ_INSERT_TAIL(&eps->eps_record.er_tdlist, td, td_epochq);
 	sched_pin();
 	ck_epoch_begin(&eps->eps_record.er_record, (ck_epoch_section_t*)&td->td_epoch_section);
 	critical_exit();
 }
 
 void
-epoch_exit(epoch_t epoch)
+epoch_exit_internal(epoch_t epoch, struct thread *td)
 {
 	struct epoch_pcpu_state *eps;
-	struct thread *td;
 
 	td = curthread;
+	MPASS(td->td_epochnest == 0);
 	INIT_CHECK(epoch);
-	MPASS(td->td_epochnest);
 	critical_enter();
 	eps = epoch->e_pcpu[curcpu];
-	td->td_epochnest--;
-	if (td->td_epochnest == 0)
-		TAILQ_REMOVE(&eps->eps_record.er_tdlist, td, td_epochq);
-	else {
-		critical_exit();
-		return;
-	}
-	sched_unpin();
+
 	ck_epoch_end(&eps->eps_record.er_record, (ck_epoch_section_t*)&td->td_epoch_section);
+	TAILQ_REMOVE(&eps->eps_record.er_tdlist, td, td_epochq);
 	eps->eps_record.er_gen++;
+	sched_unpin();
 	critical_exit();
 }
 

Modified: head/sys/sys/epoch.h
==============================================================================
--- head/sys/sys/epoch.h	Thu May 17 00:52:48 2018	(r333696)
+++ head/sys/sys/epoch.h	Thu May 17 01:13:40 2018	(r333697)
@@ -29,6 +29,8 @@
 
 #ifndef _SYS_EPOCH_H_
 #define _SYS_EPOCH_H_
+#include <sys/lock.h>
+#include <sys/proc.h>
 
 struct epoch;
 typedef struct epoch *epoch_t;
@@ -43,10 +45,35 @@ typedef struct epoch_context *epoch_context_t;
 
 epoch_t epoch_alloc(void);
 void epoch_free(epoch_t epoch);
-void epoch_enter(epoch_t epoch);
-void epoch_exit(epoch_t epoch);
+void epoch_enter_internal(epoch_t epoch, struct thread *td);
+void epoch_exit_internal(epoch_t epoch, struct thread *td);
 void epoch_wait(epoch_t epoch);
 void epoch_call(epoch_t epoch, epoch_context_t ctx, void (*callback) (epoch_context_t));
 int in_epoch(void);
+
+static __inline void
+epoch_enter(epoch_t epoch)
+{
+	struct thread *td;
+	int nesting;
+
+	td = curthread;
+	nesting = td->td_epochnest++;
+#ifndef INVARIANTS
+	if (nesting == 0)
+#endif
+		epoch_enter_internal(epoch, td);
+}
+
+static __inline void
+epoch_exit(epoch_t epoch)
+{
+	struct thread *td;
+
+	td = curthread;
+	MPASS(td->td_epochnest);
+	if (td->td_epochnest-- == 1)
+		epoch_exit_internal(epoch, td);
+}
 
 #endif