svn commit: r336018 - in projects/pnfs-planb-server/sys/fs: nfs nfsclient
Rick Macklem
rmacklem at FreeBSD.org
Fri Jul 6 00:58:53 UTC 2018
Author: rmacklem
Date: Fri Jul 6 00:58:51 2018
New Revision: 336018
URL: https://svnweb.freebsd.org/changeset/base/336018
Log:
Change the pNFS client so that it uses separate TCP connections for same server.
This patch changes the pNFS client so that it will use separate TCP connections
to the same machine by default. This makes it possible to disable a connection
to a DS when multiple DSs reside on the same server.
This may be overridden by a sysctl, in case some extant pNFS servers require
the same connection to be used for the same server machine.
The patch also adds the nfscl_cancelreqs() function to shut down the DS
connection if it isn't shared with other DSs.
Modified:
projects/pnfs-planb-server/sys/fs/nfs/nfsclstate.h
projects/pnfs-planb-server/sys/fs/nfsclient/nfs_clrpcops.c
projects/pnfs-planb-server/sys/fs/nfsclient/nfs_clstate.c
Modified: projects/pnfs-planb-server/sys/fs/nfs/nfsclstate.h
==============================================================================
--- projects/pnfs-planb-server/sys/fs/nfs/nfsclstate.h Thu Jul 5 22:56:13 2018 (r336017)
+++ projects/pnfs-planb-server/sys/fs/nfs/nfsclstate.h Fri Jul 6 00:58:51 2018 (r336018)
@@ -94,6 +94,7 @@ struct nfsclds {
#define NFSCLDS_MDS 0x0002
#define NFSCLDS_DS 0x0004
#define NFSCLDS_CLOSED 0x0008
+#define NFSCLDS_SAMECONN 0x0010
struct nfsclclient {
LIST_ENTRY(nfsclclient) nfsc_list;
Modified: projects/pnfs-planb-server/sys/fs/nfsclient/nfs_clrpcops.c
==============================================================================
--- projects/pnfs-planb-server/sys/fs/nfsclient/nfs_clrpcops.c Thu Jul 5 22:56:13 2018 (r336017)
+++ projects/pnfs-planb-server/sys/fs/nfsclient/nfs_clrpcops.c Fri Jul 6 00:58:51 2018 (r336018)
@@ -57,6 +57,10 @@ static int nfsignore_eexist = 0;
SYSCTL_INT(_vfs_nfs, OID_AUTO, ignore_eexist, CTLFLAG_RW,
&nfsignore_eexist, 0, "NFS ignore EEXIST replies for mkdir/symlink");
+static int nfscl_dssameconn = 0;
+SYSCTL_INT(_vfs_nfs, OID_AUTO, dssameconn, CTLFLAG_RW,
+ &nfscl_dssameconn, 0, "Use same TCP connection to multiple DSs");
+
/*
* Global variables
*/
@@ -162,7 +166,7 @@ static int nfsrpc_writedsmir(vnode_t, int *, int *, nf
struct nfsclds *, uint64_t, int, struct nfsfh *, struct mbuf *, int, int,
struct ucred *, NFSPROC_T *);
static enum nfsclds_state nfscl_getsameserver(struct nfsmount *,
- struct nfsclds *, struct nfsclds **);
+ struct nfsclds *, struct nfsclds **, uint32_t *);
static int nfsio_commitds(vnode_t, uint64_t, int, struct nfsclds *,
struct nfsfh *, int, int, struct nfsclwritedsdorpc *, struct ucred *,
NFSPROC_T *);
@@ -5498,9 +5502,14 @@ nfsrpc_fillsa(struct nfsmount *nmp, struct sockaddr_in
dsp->nfsclds_sockp = nrp;
if (vers == NFS_VER4) {
NFSLOCKMNT(nmp);
- retv = nfscl_getsameserver(nmp, dsp, &tdsp);
+ retv = nfscl_getsameserver(nmp, dsp, &tdsp,
+ &sequenceid);
NFSCL_DEBUG(3, "getsame ret=%d\n", retv);
- if (retv == NFSDSP_USETHISSESSION) {
+ if (retv == NFSDSP_USETHISSESSION &&
+ nfscl_dssameconn != 0) {
+ NFSLOCKDS(tdsp);
+ tdsp->nfsclds_flags |= NFSCLDS_SAMECONN;
+ NFSUNLOCKDS(tdsp);
NFSUNLOCKMNT(nmp);
/*
* If there is already a session for this
@@ -5511,11 +5520,8 @@ nfsrpc_fillsa(struct nfsmount *nmp, struct sockaddr_in
*dspp = tdsp;
return (0);
}
- if (retv == NFSDSP_SEQTHISSESSION)
+ if (retv == NFSDSP_NOTFOUND)
sequenceid =
- tdsp->nfsclds_sess.nfsess_sequenceid;
- else
- sequenceid =
dsp->nfsclds_sess.nfsess_sequenceid;
NFSUNLOCKMNT(nmp);
error = nfsrpc_createsession(nmp, &dsp->nfsclds_sess,
@@ -6518,15 +6524,16 @@ nfscl_freenfsclds(struct nfsclds *dsp)
static enum nfsclds_state
nfscl_getsameserver(struct nfsmount *nmp, struct nfsclds *newdsp,
- struct nfsclds **retdspp)
+ struct nfsclds **retdspp, uint32_t *sequencep)
{
- struct nfsclds *dsp, *cur_dsp;
+ struct nfsclds *dsp;
+ int fndseq;
/*
* Search the list of nfsclds structures for one with the same
* server.
*/
- cur_dsp = NULL;
+ fndseq = 0;
TAILQ_FOREACH(dsp, &nmp->nm_sess, nfsclds_list) {
if (dsp->nfsclds_servownlen == newdsp->nfsclds_servownlen &&
dsp->nfsclds_servownlen != 0 &&
@@ -6536,24 +6543,22 @@ nfscl_getsameserver(struct nfsmount *nmp, struct nfscl
NFSCL_DEBUG(4, "fnd same fdsp=%p dsp=%p flg=0x%x\n",
TAILQ_FIRST(&nmp->nm_sess), dsp,
dsp->nfsclds_flags);
+ if (fndseq == 0) {
+ /* Get sequenceid# from first entry. */
+ *sequencep =
+ dsp->nfsclds_sess.nfsess_sequenceid;
+ fndseq = 1;
+ }
/* Server major id matches. */
if ((dsp->nfsclds_flags & NFSCLDS_DS) != 0) {
*retdspp = dsp;
return (NFSDSP_USETHISSESSION);
}
- /*
- * Note the first match, so it can be used for
- * sequence'ing new sessions.
- */
- if (cur_dsp == NULL)
- cur_dsp = dsp;
}
}
- if (cur_dsp != NULL) {
- *retdspp = cur_dsp;
+ if (fndseq != 0)
return (NFSDSP_SEQTHISSESSION);
- }
return (NFSDSP_NOTFOUND);
}
Modified: projects/pnfs-planb-server/sys/fs/nfsclient/nfs_clstate.c
==============================================================================
--- projects/pnfs-planb-server/sys/fs/nfsclient/nfs_clstate.c Thu Jul 5 22:56:13 2018 (r336017)
+++ projects/pnfs-planb-server/sys/fs/nfsclient/nfs_clstate.c Fri Jul 6 00:58:51 2018 (r336018)
@@ -125,6 +125,7 @@ static struct nfscldeleg *nfscl_finddeleg(struct nfscl
static void nfscl_retoncloselayout(vnode_t, struct nfsclclient *, uint8_t *,
int, struct nfsclrecalllayout **);
static void nfscl_reldevinfo_locked(struct nfscldevinfo *);
+static void nfscl_cancelreqs(struct nfsclds *);
static struct nfscllayout *nfscl_findlayout(struct nfsclclient *, u_int8_t *,
int);
static struct nfscldevinfo *nfscl_finddevinfo(struct nfsclclient *, uint8_t *);
@@ -4975,11 +4976,12 @@ nfscl_retoncloselayout(vnode_t vp, struct nfsclclient
*/
void
nfscl_dserr(uint32_t op, uint32_t stat, struct nfscldevinfo *dp,
- struct nfscllayout *lyp, __unused struct nfsclds *dsp)
+ struct nfscllayout *lyp, struct nfsclds *dsp)
{
struct nfsclrecalllayout *recallp;
uint32_t iomode;
+ printf("DS being disabled, error=%d\n", stat);
/* Set up the return of the layout. */
recallp = malloc(sizeof(*recallp), M_NFSLAYRECALL, M_WAITOK);
iomode = 0;
@@ -4998,6 +5000,39 @@ nfscl_dserr(uint32_t op, uint32_t stat, struct nfsclde
NFSUNLOCKCLSTATE();
free(recallp, M_NFSLAYRECALL);
}
+
+ /* If the connection isn't used for other DSs, we can shut it down. */
+ if ((dsp->nfsclds_flags & NFSCLDS_SAMECONN) == 0)
+ nfscl_cancelreqs(dsp);
+}
+
+/*
+ * Cancel all RPCs for this "dsp" by closing the connection.
+ * Also, mark the session as defunct.
+ */
+static void
+nfscl_cancelreqs(struct nfsclds *dsp)
+{
+ struct __rpc_client *cl;
+ static int non_event;
+
+ NFSLOCKDS(dsp);
+ if ((dsp->nfsclds_flags & (NFSCLDS_CLOSED | NFSCLDS_SAMECONN)) == 0 &&
+ dsp->nfsclds_sockp != NULL &&
+ dsp->nfsclds_sockp->nr_client != NULL) {
+ dsp->nfsclds_flags |= NFSCLDS_CLOSED;
+ cl = dsp->nfsclds_sockp->nr_client;
+ dsp->nfsclds_sess.nfsess_defunct = 1;
+ NFSUNLOCKDS(dsp);
+ CLNT_CLOSE(cl);
+ /*
+ * This 1sec sleep is done to give the MDS time to recall
+ * layouts before this client attempts further I/O.
+ */
+ tsleep(&non_event, PVFS, "ndscls", hz);
+ return;
+ }
+ NFSUNLOCKDS(dsp);
}
/*
More information about the svn-src-projects
mailing list