svn commit: r336229 - in head/sys/fs: nfs nfsclient

[Rick Macklem]

Author: rmacklem
Date: Thu Jul 12 20:46:22 2018
New Revision: 336229
URL: https://svnweb.freebsd.org/changeset/base/336229

Log:
  Modify the NFSv4.1 pNFS client to use separate TCP connections for DSs.
  
  Without this patch, the NFSv4.1 pNFS client shared a single TCP connection
  for all DSs that resided on the same machine. This made disabling one of
  the DSs impossible. Although unlikely, it is possible that the storage
  subsystem has failed in such a way that the storage for one DS on a machine
  is no longer functioning correctly, but the storage used by another DS on
  the same machine is still ok. For this case, it would be nice if a system
  can fail one of the DSs without failing them all.
  This patch changes the default behaviour to use separate TCP connections
  for each DS even if they reside on the same machine.
  I do not believe that this will be a problem for extant pNFS servers, but
  a sysctl can be set to restore the old behaviour if this change causes a
  problem for an extant pNFS server.
  This patch only affects the NFSv4.1 pNFS client.
  
  MFC after:	2 weeks

Modified:
  head/sys/fs/nfs/nfsclstate.h
  head/sys/fs/nfsclient/nfs_clrpcops.c

Modified: head/sys/fs/nfs/nfsclstate.h
==============================================================================
--- head/sys/fs/nfs/nfsclstate.h	Thu Jul 12 20:28:52 2018	(r336228)
+++ head/sys/fs/nfs/nfsclstate.h	Thu Jul 12 20:46:22 2018	(r336229)
@@ -94,6 +94,7 @@ struct nfsclds {
 #define	NFSCLDS_MDS		0x0002
 #define	NFSCLDS_DS		0x0004
 #define	NFSCLDS_CLOSED		0x0008
+#define	NFSCLDS_SAMECONN	0x0010
 
 struct nfsclclient {
 	LIST_ENTRY(nfsclclient) nfsc_list;

Modified: head/sys/fs/nfsclient/nfs_clrpcops.c
==============================================================================
--- head/sys/fs/nfsclient/nfs_clrpcops.c	Thu Jul 12 20:28:52 2018	(r336228)
+++ head/sys/fs/nfsclient/nfs_clrpcops.c	Thu Jul 12 20:46:22 2018	(r336229)
@@ -57,6 +57,10 @@ static int	nfsignore_eexist = 0;
 SYSCTL_INT(_vfs_nfs, OID_AUTO, ignore_eexist, CTLFLAG_RW,
     &nfsignore_eexist, 0, "NFS ignore EEXIST replies for mkdir/symlink");
 
+static int	nfscl_dssameconn = 0;
+SYSCTL_INT(_vfs_nfs, OID_AUTO, dssameconn, CTLFLAG_RW,
+    &nfscl_dssameconn, 0, "Use same TCP connection to multiple DSs");
+
 /*
  * Global variables
  */
@@ -162,7 +166,7 @@ static int nfsrpc_writedsmir(vnode_t, int *, int *, nf
     struct nfsclds *, uint64_t, int, struct nfsfh *, struct mbuf *, int, int,
     struct ucred *, NFSPROC_T *);
 static enum nfsclds_state nfscl_getsameserver(struct nfsmount *,
-    struct nfsclds *, struct nfsclds **);
+    struct nfsclds *, struct nfsclds **, uint32_t *);
 static int nfsio_commitds(vnode_t, uint64_t, int, struct nfsclds *,
     struct nfsfh *, int, int, struct nfsclwritedsdorpc *, struct ucred *,
     NFSPROC_T *);
@@ -5498,9 +5502,14 @@ nfsrpc_fillsa(struct nfsmount *nmp, struct sockaddr_in
 		dsp->nfsclds_sockp = nrp;
 		if (vers == NFS_VER4) {
 			NFSLOCKMNT(nmp);
-			retv = nfscl_getsameserver(nmp, dsp, &tdsp);
+			retv = nfscl_getsameserver(nmp, dsp, &tdsp,
+			    &sequenceid);
 			NFSCL_DEBUG(3, "getsame ret=%d\n", retv);
-			if (retv == NFSDSP_USETHISSESSION) {
+			if (retv == NFSDSP_USETHISSESSION &&
+			    nfscl_dssameconn != 0) {
+				NFSLOCKDS(tdsp);
+				tdsp->nfsclds_flags |= NFSCLDS_SAMECONN;
+				NFSUNLOCKDS(tdsp);
 				NFSUNLOCKMNT(nmp);
 				/*
 				 * If there is already a session for this
@@ -5511,11 +5520,8 @@ nfsrpc_fillsa(struct nfsmount *nmp, struct sockaddr_in
 				*dspp = tdsp;
 				return (0);
 			}
-			if (retv == NFSDSP_SEQTHISSESSION)
+			if (retv == NFSDSP_NOTFOUND)
 				sequenceid =
-				    tdsp->nfsclds_sess.nfsess_sequenceid;
-			else
-				sequenceid =
 				    dsp->nfsclds_sess.nfsess_sequenceid;
 			NFSUNLOCKMNT(nmp);
 			error = nfsrpc_createsession(nmp, &dsp->nfsclds_sess,
@@ -6516,15 +6522,16 @@ nfscl_freenfsclds(struct nfsclds *dsp)
 
 static enum nfsclds_state
 nfscl_getsameserver(struct nfsmount *nmp, struct nfsclds *newdsp,
-    struct nfsclds **retdspp)
+    struct nfsclds **retdspp, uint32_t *sequencep)
 {
-	struct nfsclds *dsp, *cur_dsp;
+	struct nfsclds *dsp;
+	int fndseq;
 
 	/*
 	 * Search the list of nfsclds structures for one with the same
 	 * server.
 	 */
-	cur_dsp = NULL;
+	fndseq = 0;
 	TAILQ_FOREACH(dsp, &nmp->nm_sess, nfsclds_list) {
 		if (dsp->nfsclds_servownlen == newdsp->nfsclds_servownlen &&
 		    dsp->nfsclds_servownlen != 0 &&
@@ -6534,24 +6541,22 @@ nfscl_getsameserver(struct nfsmount *nmp, struct nfscl
 			NFSCL_DEBUG(4, "fnd same fdsp=%p dsp=%p flg=0x%x\n",
 			    TAILQ_FIRST(&nmp->nm_sess), dsp,
 			    dsp->nfsclds_flags);
+			if (fndseq == 0) {
+				/* Get sequenceid# from first entry. */
+				*sequencep =
+				    dsp->nfsclds_sess.nfsess_sequenceid;
+				fndseq = 1;
+			}
 			/* Server major id matches. */
 			if ((dsp->nfsclds_flags & NFSCLDS_DS) != 0) {
 				*retdspp = dsp;
 				return (NFSDSP_USETHISSESSION);
 			}
 
-			/*
-			 * Note the first match, so it can be used for
-			 * sequence'ing new sessions.
-			 */
-			if (cur_dsp == NULL)
-				cur_dsp = dsp;
 		}
 	}
-	if (cur_dsp != NULL) {
-		*retdspp = cur_dsp;
+	if (fndseq != 0)
 		return (NFSDSP_SEQTHISSESSION);
-	}
 	return (NFSDSP_NOTFOUND);
 }