git: 6c5dae287ee7 - stable/13 - nfscl: Add a "has acquired a delegation" flag for delegations

[Rick Macklem]

The branch stable/13 has been updated by rmacklem:

URL: https://cgit.FreeBSD.org/src/commit/?id=6c5dae287ee722036240bde5bc0b3a8d5a000836

commit 6c5dae287ee722036240bde5bc0b3a8d5a000836
Author:     Rick Macklem <rmacklem at FreeBSD.org>
AuthorDate: 2021-06-09 15:00:43 +0000
Commit:     Rick Macklem <rmacklem at FreeBSD.org>
CommitDate: 2021-06-26 23:01:09 +0000

    nfscl: Add a "has acquired a delegation" flag for delegations
    
    A problem was reported via email, where a large (130000+) accumulation
    of NFSv4 opens on an NFSv4 mount caused significant lock contention
    on the mutex used to protect the client mount's open/lock state.
    Although the root cause for the accumulation of opens was not
    resolved, it is obvious that the NFSv4 client is not designed to
    handle 100000+ opens efficiently.
    
    For a common case where delegations are not being issued by the
    NFSv4 server, the code acquires the mutex lock for open/lock state,
    finds the delegation list empty and just unlocks the mutex and returns.
    This patch adds an NFS mount point flag that is set when a delegation
    is issued for the mount.  Then the patched code checks for this flag
    before acquiring the open/lock mutex, avoiding the need to acquire
    the lock for the case where delegations are not being issued by the
    NFSv4 server.
    This change appears to be performance neutral for a small number
    of opens, but should reduce lock contention for a large number of opens
    for the common case where server is not issuing delegations.
    
    This commit should not affect the high level semantics of delegation
    handling.
    
    (cherry picked from commit 5e5ca4c8fc53d31bf71e182e08c4ccd8290428c7)
---
 sys/fs/nfsclient/nfs_clstate.c | 44 +++++++++++++++++++++++++++++++++++++++---
 sys/fs/nfsclient/nfsmount.h    |  1 +
 2 files changed, 42 insertions(+), 3 deletions(-)

diff --git a/sys/fs/nfsclient/nfs_clstate.c b/sys/fs/nfsclient/nfs_clstate.c
index 5e4ac2ae9d4d..0659e77289e9 100644
--- a/sys/fs/nfsclient/nfs_clstate.c
+++ b/sys/fs/nfsclient/nfs_clstate.c
@@ -438,20 +438,32 @@ nfscl_deleg(mount_t mp, struct nfsclclient *clp, u_int8_t *nfhp,
     int fhlen, struct ucred *cred, NFSPROC_T *p, struct nfscldeleg **dpp)
 {
 	struct nfscldeleg *dp = *dpp, *tdp;
+	struct nfsmount *nmp;
 
+	KASSERT(mp != NULL, ("nfscl_deleg: mp NULL"));
+	nmp = VFSTONFS(mp);
 	/*
 	 * First, if we have received a Read delegation for a file on a
 	 * read/write file system, just return it, because they aren't
 	 * useful, imho.
 	 */
-	if (mp != NULL && dp != NULL && !NFSMNT_RDONLY(mp) &&
+	if (dp != NULL && !NFSMNT_RDONLY(mp) &&
 	    (dp->nfsdl_flags & NFSCLDL_READ)) {
-		(void) nfscl_trydelegreturn(dp, cred, VFSTONFS(mp), p);
+		nfscl_trydelegreturn(dp, cred, nmp, p);
 		free(dp, M_NFSCLDELEG);
 		*dpp = NULL;
 		return (0);
 	}
 
+	/*
+	 * Since a delegation might be added to the mount,
+	 * set NFSMNTP_DELEGISSUED now.  If a delegation already
+	 * exagain ists, setting this flag is harmless.
+	 */
+	NFSLOCKMNT(nmp);
+	nmp->nm_privflag |= NFSMNTP_DELEGISSUED;
+	NFSUNLOCKMNT(nmp);
+
 	/* Look for the correct deleg, based upon FH */
 	NFSLOCKCLSTATE();
 	tdp = nfscl_finddeleg(clp, nfhp, fhlen);
@@ -3356,12 +3368,20 @@ nfscl_delegreturnvp(vnode_t vp, NFSPROC_T *p)
 	struct nfscldeleg *dp;
 	struct ucred *cred;
 	struct nfsnode *np;
+	struct nfsmount *nmp;
 
+	nmp = VFSTONFS(vp->v_mount);
+	NFSLOCKMNT(nmp);
+	if ((nmp->nm_privflag & NFSMNTP_DELEGISSUED) == 0) {
+		NFSUNLOCKMNT(nmp);
+		return;
+	}
+	NFSUNLOCKMNT(nmp);
 	np = VTONFS(vp);
 	cred = newnfs_getcred();
 	dp = NULL;
 	NFSLOCKCLSTATE();
-	clp = VFSTONFS(vp->v_mount)->nm_clp;
+	clp = nmp->nm_clp;
 	if (clp != NULL)
 		dp = nfscl_finddeleg(clp, np->n_fhp->nfh_fh,
 		    np->n_fhp->nfh_len);
@@ -4500,6 +4520,12 @@ nfscl_nodeleg(vnode_t vp, int writedeleg)
 	nmp = VFSTONFS(vp->v_mount);
 	if (!NFSHASNFSV4(nmp))
 		return (1);
+	NFSLOCKMNT(nmp);
+	if ((nmp->nm_privflag & NFSMNTP_DELEGISSUED) == 0) {
+		NFSUNLOCKMNT(nmp);
+		return (1);
+	}
+	NFSUNLOCKMNT(nmp);
 	NFSLOCKCLSTATE();
 	clp = nfscl_findcl(nmp);
 	if (clp == NULL) {
@@ -4856,6 +4882,12 @@ nfscl_delegmodtime(vnode_t vp)
 	nmp = VFSTONFS(vp->v_mount);
 	if (!NFSHASNFSV4(nmp))
 		return;
+	NFSLOCKMNT(nmp);
+	if ((nmp->nm_privflag & NFSMNTP_DELEGISSUED) == 0) {
+		NFSUNLOCKMNT(nmp);
+		return;
+	}
+	NFSUNLOCKMNT(nmp);
 	NFSLOCKCLSTATE();
 	clp = nfscl_findcl(nmp);
 	if (clp == NULL) {
@@ -4885,6 +4917,12 @@ nfscl_deleggetmodtime(vnode_t vp, struct timespec *mtime)
 	nmp = VFSTONFS(vp->v_mount);
 	if (!NFSHASNFSV4(nmp))
 		return;
+	NFSLOCKMNT(nmp);
+	if ((nmp->nm_privflag & NFSMNTP_DELEGISSUED) == 0) {
+		NFSUNLOCKMNT(nmp);
+		return;
+	}
+	NFSUNLOCKMNT(nmp);
 	NFSLOCKCLSTATE();
 	clp = nfscl_findcl(nmp);
 	if (clp == NULL) {
diff --git a/sys/fs/nfsclient/nfsmount.h b/sys/fs/nfsclient/nfsmount.h
index 57adcd8f2fca..f8ea8c9a1418 100644
--- a/sys/fs/nfsclient/nfsmount.h
+++ b/sys/fs/nfsclient/nfsmount.h
@@ -115,6 +115,7 @@ struct	nfsmount {
 #define	NFSMNTP_NOXATTR		0x00000080
 #define	NFSMNTP_NOADVISE	0x00000100
 #define	NFSMNTP_NOALLOCATE	0x00000200
+#define	NFSMNTP_DELEGISSUED	0x00000400
 
 /* New mount flags only used by the kernel via nmount(2). */
 #define	NFSMNT_TLS		0x00000001