svn commit: r333339 - in head/sys: kern sys

[Mateusz Guzik]

Author: mjg
Date: Mon May  7 22:29:32 2018
New Revision: 333339
URL: https://svnweb.freebsd.org/changeset/base/333339

Log:
  Avoid calls to syscall_thread_enter/exit for statically defined syscalls
  
  The entire mechanism is rarely used and is quite not performant due to
  atomci ops on the syscall table. It also has added overhead for completely
  unrelated syscalls.
  
  Reduce it by avoiding the func calls if possible (which consistutes vast
  majority of cases).
  
  Provides about 3% syscall rate speed up for getuid on Broadwell.

Modified:
  head/sys/kern/kern_syscalls.c
  head/sys/sys/sysent.h

Modified: head/sys/kern/kern_syscalls.c
==============================================================================
--- head/sys/kern/kern_syscalls.c	Mon May  7 21:42:22 2018	(r333338)
+++ head/sys/kern/kern_syscalls.c	Mon May  7 22:29:32 2018	(r333339)
@@ -80,14 +80,12 @@ syscall_thread_drain(struct sysent *se)
 }
 
 int
-syscall_thread_enter(struct thread *td, struct sysent *se)
+_syscall_thread_enter(struct thread *td, struct sysent *se)
 {
 	u_int32_t cnt, oldcnt;
 
 	do {
 		oldcnt = se->sy_thrcnt;
-		if ((oldcnt & SY_THR_STATIC) != 0)
-			return (0);
 		if ((oldcnt & (SY_THR_DRAINING | SY_THR_ABSENT)) != 0)
 			return (ENOSYS);
 		cnt = oldcnt + SY_THR_INCR;
@@ -96,14 +94,12 @@ syscall_thread_enter(struct thread *td, struct sysent 
 }
 
 void
-syscall_thread_exit(struct thread *td, struct sysent *se)
+_syscall_thread_exit(struct thread *td, struct sysent *se)
 {
 	u_int32_t cnt, oldcnt;
 
 	do {
 		oldcnt = se->sy_thrcnt;
-		if ((oldcnt & SY_THR_STATIC) != 0)
-			return;
 		cnt = oldcnt - SY_THR_INCR;
 	} while (atomic_cmpset_rel_32(&se->sy_thrcnt, oldcnt, cnt) == 0);
 }

Modified: head/sys/sys/sysent.h
==============================================================================
--- head/sys/sys/sysent.h	Mon May  7 21:42:22 2018	(r333338)
+++ head/sys/sys/sysent.h	Mon May  7 22:29:32 2018	(r333339)
@@ -289,8 +289,26 @@ struct nosys_args;
 int	lkmnosys(struct thread *, struct nosys_args *);
 int	lkmressys(struct thread *, struct nosys_args *);
 
-int	syscall_thread_enter(struct thread *td, struct sysent *se);
-void	syscall_thread_exit(struct thread *td, struct sysent *se);
+int	_syscall_thread_enter(struct thread *td, struct sysent *se);
+void	_syscall_thread_exit(struct thread *td, struct sysent *se);
+
+static inline int
+syscall_thread_enter(struct thread *td, struct sysent *se)
+{
+
+	if (__predict_true((se->sy_thrcnt & SY_THR_STATIC) != 0))
+		return (0);
+	return (_syscall_thread_enter(td, se));
+}
+
+static inline void
+syscall_thread_exit(struct thread *td, struct sysent *se)
+{
+
+	if (__predict_true((se->sy_thrcnt & SY_THR_STATIC) != 0))
+		return;
+	_syscall_thread_exit(td, se);
+}
 
 int shared_page_alloc(int size, int align);
 int shared_page_fill(int size, int align, const void *data);