git: 233225ef7d56 - stable/13 - select: employ fget_only_user

[Mateusz Guzik]

The branch stable/13 has been updated by mjg:

URL: https://cgit.FreeBSD.org/src/commit/?id=233225ef7d5604162b6bda47e35fa7a8075bd559

commit 233225ef7d5604162b6bda47e35fa7a8075bd559
Author:     Mateusz Guzik <mjg at FreeBSD.org>
AuthorDate: 2021-01-28 23:33:46 +0000
Commit:     Mateusz Guzik <mjg at FreeBSD.org>
CommitDate: 2021-02-01 12:39:18 +0000

    select: employ fget_only_user
    
    Since most select users are single-threaded this avoid a lot of work
    in the common case.
    
    For example select of 16 fds (ops/s):
    before: 2114536
    after:  2991010
    
    (cherry picked from commit 6affe1b71238df7bbbca6e2059e5494d91e68d2d)
---
 sys/kern/sys_generic.c | 35 ++++++++++++++++++++++-------------
 1 file changed, 22 insertions(+), 13 deletions(-)

diff --git a/sys/kern/sys_generic.c b/sys/kern/sys_generic.c
index a055f4a9b597..dc2cd56f5077 100644
--- a/sys/kern/sys_generic.c
+++ b/sys/kern/sys_generic.c
@@ -1304,13 +1304,6 @@ selsetbits(fd_mask **ibits, fd_mask **obits, int idx, fd_mask bit, int events)
 	return (n);
 }
 
-static __inline int
-getselfd_cap(struct filedesc *fdp, int fd, struct file **fpp)
-{
-
-	return (fget_unlocked(fdp, fd, &cap_event_rights, fpp));
-}
-
 /*
  * Traverse the list of fds attached to this thread's seltd and check for
  * completion.
@@ -1327,10 +1320,12 @@ selrescan(struct thread *td, fd_mask **ibits, fd_mask **obits)
 	fd_mask bit;
 	int fd, ev, n, idx;
 	int error;
+	bool only_user;
 
 	fdp = td->td_proc->p_fd;
 	stp = td->td_sel;
 	n = 0;
+	only_user = FILEDESC_IS_ONLY_USER(fdp);
 	STAILQ_FOREACH_SAFE(sfp, &stp->st_selq, sf_link, sfn) {
 		fd = (int)(uintptr_t)sfp->sf_cookie;
 		si = sfp->sf_si;
@@ -1338,13 +1333,19 @@ selrescan(struct thread *td, fd_mask **ibits, fd_mask **obits)
 		/* If the selinfo wasn't cleared the event didn't fire. */
 		if (si != NULL)
 			continue;
-		error = getselfd_cap(fdp, fd, &fp);
-		if (error)
+		if (only_user)
+			error = fget_only_user(fdp, fd, &cap_event_rights, &fp);
+		else
+			error = fget_unlocked(fdp, fd, &cap_event_rights, &fp);
+		if (__predict_false(error != 0))
 			return (error);
 		idx = fd / NFDBITS;
 		bit = (fd_mask)1 << (fd % NFDBITS);
 		ev = fo_poll(fp, selflags(ibits, idx, bit), td->td_ucred, td);
-		fdrop(fp, td);
+		if (only_user)
+			fput_only_user(fdp, fp);
+		else
+			fdrop(fp, td);
 		if (ev != 0)
 			n += selsetbits(ibits, obits, idx, bit, ev);
 	}
@@ -1366,9 +1367,11 @@ selscan(struct thread *td, fd_mask **ibits, fd_mask **obits, int nfd)
 	int ev, flags, end, fd;
 	int n, idx;
 	int error;
+	bool only_user;
 
 	fdp = td->td_proc->p_fd;
 	n = 0;
+	only_user = FILEDESC_IS_ONLY_USER(fdp);
 	for (idx = 0, fd = 0; fd < nfd; idx++) {
 		end = imin(fd + NFDBITS, nfd);
 		for (bit = 1; fd < end; bit <<= 1, fd++) {
@@ -1376,12 +1379,18 @@ selscan(struct thread *td, fd_mask **ibits, fd_mask **obits, int nfd)
 			flags = selflags(ibits, idx, bit);
 			if (flags == 0)
 				continue;
-			error = getselfd_cap(fdp, fd, &fp);
-			if (error)
+			if (only_user)
+				error = fget_only_user(fdp, fd, &cap_event_rights, &fp);
+			else
+				error = fget_unlocked(fdp, fd, &cap_event_rights, &fp);
+			if (__predict_false(error != 0))
 				return (error);
 			selfdalloc(td, (void *)(uintptr_t)fd);
 			ev = fo_poll(fp, flags, td->td_ucred, td);
-			fdrop(fp, td);
+			if (only_user)
+				fput_only_user(fdp, fp);
+			else
+				fdrop(fp, td);
 			if (ev != 0)
 				n += selsetbits(ibits, obits, idx, bit, ev);
 		}