svn commit: r334036 - in projects/pnfs-planb-server/sys/fs: nfs nfsserver
Rick Macklem
rmacklem at FreeBSD.org
Tue May 22 11:57:12 UTC 2018
Author: rmacklem
Date: Tue May 22 11:57:10 2018
New Revision: 334036
URL: https://svnweb.freebsd.org/changeset/base/334036
Log:
Add code that does CBLAYOUTRECALLs when the total # of layouts exceeds
the vfs.nfsd.layouthighwater limit.
Unfortunately this required changes to the arguments for nfsrv_recalllayout()
and nfsrv_docallback() so the patch got bigger than it would have been.
Modified:
projects/pnfs-planb-server/sys/fs/nfs/nfs_var.h
projects/pnfs-planb-server/sys/fs/nfs/nfsrvstate.h
projects/pnfs-planb-server/sys/fs/nfsserver/nfs_nfsdsocket.c
projects/pnfs-planb-server/sys/fs/nfsserver/nfs_nfsdstate.c
Modified: projects/pnfs-planb-server/sys/fs/nfs/nfs_var.h
==============================================================================
--- projects/pnfs-planb-server/sys/fs/nfs/nfs_var.h Tue May 22 11:26:41 2018 (r334035)
+++ projects/pnfs-planb-server/sys/fs/nfs/nfs_var.h Tue May 22 11:57:10 2018 (r334036)
@@ -149,6 +149,7 @@ int nfsrv_layoutget(struct nfsrv_descript *, vnode_t,
int, int *, uint64_t *, uint64_t *, uint64_t, nfsv4stateid_t *, int, int *,
int *, char *, struct ucred *, NFSPROC_T *);
void nfsrv_flexmirrordel(char *, NFSPROC_T *);
+void nfsrv_recalloldlayout(NFSPROC_T *);
int nfsrv_layoutreturn(struct nfsrv_descript *, vnode_t, int, int, uint64_t,
uint64_t, int, int, nfsv4stateid_t *, int, uint32_t *, int *,
struct ucred *, NFSPROC_T *);
Modified: projects/pnfs-planb-server/sys/fs/nfs/nfsrvstate.h
==============================================================================
--- projects/pnfs-planb-server/sys/fs/nfs/nfsrvstate.h Tue May 22 11:26:41 2018 (r334035)
+++ projects/pnfs-planb-server/sys/fs/nfs/nfsrvstate.h Tue May 22 11:57:10 2018 (r334036)
@@ -145,6 +145,7 @@ struct nfslayout {
#define NFSLAY_RW 0x0002
#define NFSLAY_RECALL 0x0004
#define NFSLAY_RETURNED 0x0008
+#define NFSLAY_CALLB 0x0010
/*
* Structure for an NFSv4.1 session.
@@ -290,7 +291,6 @@ struct nfsrollback {
struct nfslockfile {
LIST_HEAD(, nfsstate) lf_open; /* Open list */
LIST_HEAD(, nfsstate) lf_deleg; /* Delegation list */
- LIST_HEAD(, nfsstate) lf_layout; /* Layout list */
LIST_HEAD(, nfslock) lf_lock; /* Lock list */
LIST_HEAD(, nfslock) lf_locallock; /* Local lock list */
LIST_HEAD(, nfsrollback) lf_rollback; /* Local lock rollback list */
Modified: projects/pnfs-planb-server/sys/fs/nfsserver/nfs_nfsdsocket.c
==============================================================================
--- projects/pnfs-planb-server/sys/fs/nfsserver/nfs_nfsdsocket.c Tue May 22 11:26:41 2018 (r334035)
+++ projects/pnfs-planb-server/sys/fs/nfsserver/nfs_nfsdsocket.c Tue May 22 11:57:10 2018 (r334036)
@@ -52,6 +52,8 @@ extern struct nfsclienthashhead *nfsclienthash;
extern int nfsrv_clienthashsize;
extern int nfsrc_floodlevel, nfsrc_tcpsavedreplies;
extern int nfsd_debuglevel;
+extern int nfsrv_layouthighwater;
+extern volatile int nfsrv_layoutcnt;
NFSV4ROOTLOCKMUTEX;
NFSSTATESPINLOCK;
@@ -726,6 +728,10 @@ nfsrvd_compound(struct nfsrv_descript *nd, int isdgram
if (nfsrv_stablefirst.nsf_flags & NFSNSF_NOOPENS) {
nfsrv_throwawayopens(p);
}
+
+ /* Do a CBLAYOUTRECALL callback if over the high water mark. */
+ if (nfsrv_layoutcnt > nfsrv_layouthighwater)
+ nfsrv_recalloldlayout(p);
savevp = vp = NULL;
save_fsid.val[0] = save_fsid.val[1] = 0;
Modified: projects/pnfs-planb-server/sys/fs/nfsserver/nfs_nfsdstate.c
==============================================================================
--- projects/pnfs-planb-server/sys/fs/nfsserver/nfs_nfsdstate.c Tue May 22 11:26:41 2018 (r334035)
+++ projects/pnfs-planb-server/sys/fs/nfsserver/nfs_nfsdstate.c Tue May 22 11:57:10 2018 (r334036)
@@ -40,6 +40,7 @@ int nfsrv_dolocallocks = 0;
struct nfsv4lock nfsv4rootfs_lock;
time_t nfsdev_time = 0;
int nfsrv_layouthashsize;
+volatile int nfsrv_layoutcnt = 0;
extern int newnfs_numnfsd;
extern struct nfsstatsv1 nfsstatsv1;
@@ -161,8 +162,7 @@ static int nfsrv_checkgrace(struct nfsrv_descript *nd,
u_int32_t flags);
static int nfsrv_docallback(struct nfsclient *clp, int procnum,
nfsv4stateid_t *stateidp, int trunc, fhandle_t *fhp,
- struct nfsvattr *nap, nfsattrbit_t *attrbitp, struct nfslayout *,
- NFSPROC_T *p);
+ struct nfsvattr *nap, nfsattrbit_t *attrbitp, int laytype, NFSPROC_T *p);
static int nfsrv_cbcallargs(struct nfsrv_descript *nd, struct nfsclient *clp,
uint32_t callback, int op, const char *optag, struct nfsdsession **sepp);
static u_int32_t nfsrv_nextclientindex(void);
@@ -216,9 +216,10 @@ static void nfsrv_allocdevid(struct nfsdevice *ds, cha
static void nfsrv_freealldevids(void);
static void nfsrv_flexlayouterr(struct nfsrv_descript *nd, uint32_t *layp,
int maxcnt, NFSPROC_T *p);
-static void nfsrv_recalllayout(struct nfslayout *lyp, NFSPROC_T *p);
-static int nfsrv_findlayout(struct nfsrv_descript *nd, fhandle_t *fhp,
- int laytype, NFSPROC_T *, struct nfslayout **lypp);
+static int nfsrv_recalllayout(nfsquad_t clid, nfsv4stateid_t *stateidp,
+ fhandle_t *fhp, struct nfslayout *lyp, int laytype, NFSPROC_T *p);
+static int nfsrv_findlayout(nfsquad_t *clientidp, fhandle_t *fhp, int laytype,
+ NFSPROC_T *, struct nfslayout **lypp);
static int nfsrv_fndclid(nfsquad_t *clidvec, nfsquad_t clid, int clidcnt);
static struct nfslayout *nfsrv_filelayout(struct nfsrv_descript *nd, int iomode,
fhandle_t *fhp, fhandle_t *dsfhp, char *devid, fsid_t fs);
@@ -1288,7 +1289,7 @@ nfsrv_zapclient(struct nfsclient *clp, NFSPROC_T *p)
clp->lc_hand.nfsh_flag &= ~NFSG_COMPLETE;
clp->lc_hand.nfsh_flag |= NFSG_DESTROYED;
(void) nfsrv_docallback(clp, NFSV4PROC_CBNULL,
- NULL, 0, NULL, NULL, NULL, NULL, p);
+ NULL, 0, NULL, NULL, NULL, 0, p);
}
#endif
newnfs_disconnect(&clp->lc_req);
@@ -2641,7 +2642,7 @@ tryagain:
* harmless.
*/
cbret = nfsrv_docallback(clp, NFSV4PROC_CBNULL,
- NULL, 0, NULL, NULL, NULL, NULL, p);
+ NULL, 0, NULL, NULL, NULL, 0, p);
NFSLOCKSTATE();
clp->lc_flags &= ~LCL_NEEDSCBNULL;
if (!cbret)
@@ -4196,10 +4197,9 @@ out:
* Do a server callback.
*/
static int
-nfsrv_docallback(struct nfsclient *clp, int procnum,
- nfsv4stateid_t *stateidp, int trunc, fhandle_t *fhp,
- struct nfsvattr *nap, nfsattrbit_t *attrbitp, struct nfslayout *lyp,
- NFSPROC_T *p)
+nfsrv_docallback(struct nfsclient *clp, int procnum, nfsv4stateid_t *stateidp,
+ int trunc, fhandle_t *fhp, struct nfsvattr *nap, nfsattrbit_t *attrbitp,
+ int laytype, NFSPROC_T *p)
{
mbuf_t m;
u_int32_t *tl;
@@ -4295,18 +4295,18 @@ nfsrv_docallback(struct nfsclient *clp, int procnum,
goto errout;
}
NFSM_BUILD(tl, u_int32_t *, 4 * NFSX_UNSIGNED);
- *tl++ = txdr_unsigned(NFSLAYOUT_FLEXFILE);
+ *tl++ = txdr_unsigned(laytype);
*tl++ = txdr_unsigned(NFSLAYOUTIOMODE_ANY);
*tl++ = newnfs_true;
*tl = txdr_unsigned(NFSV4LAYOUTRET_FILE);
- nfsm_fhtom(nd, (uint8_t *)&lyp->lay_fh, NFSX_MYFH, 0);
+ nfsm_fhtom(nd, (uint8_t *)fhp, NFSX_MYFH, 0);
NFSM_BUILD(tl, u_int32_t *, 2 * NFSX_HYPER + NFSX_STATEID);
tval = 0;
txdr_hyper(tval, tl); tl += 2;
tval = UINT64_MAX;
txdr_hyper(tval, tl); tl += 2;
- *tl++ = txdr_unsigned(lyp->lay_stateid.seqid);
- NFSBCOPY(lyp->lay_stateid.other, tl, NFSX_STATEIDOTHER);
+ *tl++ = txdr_unsigned(stateidp->seqid);
+ NFSBCOPY(stateidp->other, tl, NFSX_STATEIDOTHER);
tl += (NFSX_STATEIDOTHER / NFSX_UNSIGNED);
NFSD_DEBUG(4, "aft args\n");
} else if (procnum == NFSV4PROC_CBNULL) {
@@ -5043,7 +5043,7 @@ nfsrv_delegconflict(struct nfsstate *stp, int *haslock
retrycnt = 0;
do {
error = nfsrv_docallback(clp, NFSV4OP_CBRECALL,
- &tstateid, 0, &tfh, NULL, NULL, NULL, p);
+ &tstateid, 0, &tfh, NULL, NULL, 0, p);
retrycnt++;
} while ((error == NFSERR_BADSTATEID ||
error == NFSERR_BADHANDLE) && retrycnt < NFSV4_CBRETRYCNT);
@@ -5479,7 +5479,7 @@ nfsrv_checkgetattr(struct nfsrv_descript *nd, vnode_t
NFSVNO_ATTRINIT(&nva);
nva.na_filerev = NFS64BITSSET;
error = nfsrv_docallback(clp, NFSV4OP_CBGETATTR, NULL,
- 0, &nfh, &nva, &cbbits, NULL, p);
+ 0, &nfh, &nva, &cbbits, 0, p);
if (!error) {
if ((nva.na_filerev != NFS64BITSSET &&
nva.na_filerev > delegfilerev) ||
@@ -6326,19 +6326,36 @@ nfsrv_layoutget(struct nfsrv_descript *nd, vnode_t vp,
/* First, see if a layout already exists and return if found. */
lhyp = NFSLAYOUTHASH(&fh);
NFSLOCKLAYOUT(lhyp);
- error = nfsrv_findlayout(nd, &fh, layouttype, p, &lyp);
+ error = nfsrv_findlayout(&nd->nd_clientid, &fh, layouttype, p, &lyp);
NFSD_DEBUG(4, "layoutget findlay=%d\n", error);
- if (error == 0) {
- /*
- * Not sure if the seqid must be the same, so I won't check it.
- */
- if (stateidp->other[0] != lyp->lay_stateid.other[0] ||
- stateidp->other[1] != lyp->lay_stateid.other[1] ||
- stateidp->other[2] != lyp->lay_stateid.other[2]) {
+ /*
+ * Not sure if the seqid must be the same, so I won't check it.
+ */
+ if (error == 0 && (stateidp->other[0] != lyp->lay_stateid.other[0] ||
+ stateidp->other[1] != lyp->lay_stateid.other[1] ||
+ stateidp->other[2] != lyp->lay_stateid.other[2])) {
+ if ((lyp->lay_flags & NFSLAY_CALLB) == 0) {
NFSUNLOCKLAYOUT(lhyp);
NFSD_DEBUG(1, "ret bad stateid\n");
return (NFSERR_BADSTATEID);
}
+ /*
+ * I believe we get here because there is a race between
+ * the client processing the CBLAYOUTRECALL and the layout
+ * being deleted here on the server.
+ * The client has now done a LayoutGet with a non-layout
+ * stateid, as it would when there is no layout.
+ * As such, free this layout and set error == NFSERR_BADSTATEID
+ * so the code below will create a new layout structure as
+ * would happen if no layout was found.
+ * "lyp" will be set before being used below, but set it NULL
+ * as a safety belt.
+ */
+ nfsrv_freelayout(&lhyp->list, lyp);
+ lyp = NULL;
+ error = NFSERR_BADSTATEID;
+ }
+ if (error == 0) {
if (lyp->lay_layoutlen > maxcnt) {
NFSUNLOCKLAYOUT(lhyp);
NFSD_DEBUG(1, "ret layout too small\n");
@@ -6607,7 +6624,8 @@ nfsrv_flexmirrordel(char *devid, NFSPROC_T *p)
*/
if (++lyp->lay_stateid.seqid == 0)
lyp->lay_stateid.seqid = 1;
- nfsrv_recalllayout(lyp, p);
+ nfsrv_recalllayout(lyp->lay_clientid, &lyp->lay_stateid,
+ &lyp->lay_fh, lyp, lyp->lay_type, p);
nfsrv_freelayout(&loclyp, lyp);
}
}
@@ -6615,22 +6633,23 @@ nfsrv_flexmirrordel(char *devid, NFSPROC_T *p)
/*
* Do a recall callback to the client for this layout.
*/
-static void
-nfsrv_recalllayout(struct nfslayout *lyp, NFSPROC_T *p)
+static int
+nfsrv_recalllayout(nfsquad_t clid, nfsv4stateid_t *stateidp, fhandle_t *fhp,
+ struct nfslayout *lyp, int laytype, NFSPROC_T *p)
{
struct nfsclient *clp;
int error;
NFSD_DEBUG(4, "nfsrv_recalllayout\n");
- error = nfsrv_getclient(lyp->lay_clientid, 0, &clp, NULL,
- (nfsquad_t)((u_quad_t)0), 0, NULL, p);
+ error = nfsrv_getclient(clid, 0, &clp, NULL, (nfsquad_t)((u_quad_t)0),
+ 0, NULL, p);
NFSD_DEBUG(4, "aft nfsrv_getclient=%d\n", error);
if (error != 0)
- return;
+ return (error);
if ((clp->lc_flags & LCL_NFSV41) != 0) {
- error = nfsrv_docallback(clp, NFSV4OP_CBLAYOUTRECALL, NULL, 0,
- NULL, NULL, NULL, lyp, p);
- if (error == NFSERR_NOMATCHLAYOUT) {
+ error = nfsrv_docallback(clp, NFSV4OP_CBLAYOUTRECALL,
+ stateidp, 0, fhp, NULL, NULL, laytype, p);
+ if (lyp != NULL && error == NFSERR_NOMATCHLAYOUT) {
NFSDRECALLLOCK();
if ((lyp->lay_flags & NFSLAY_RECALL) != 0) {
lyp->lay_flags |= NFSLAY_RETURNED;
@@ -6640,9 +6659,68 @@ nfsrv_recalllayout(struct nfslayout *lyp, NFSPROC_T *p
}
} else
printf("nfsrv_recalllayout: clp not NFSv4.1\n");
+ return (error);
}
/*
+ * Find a layout to recall when we exceed our high water mark.
+ */
+void
+nfsrv_recalloldlayout(NFSPROC_T *p)
+{
+ struct nfslayouthash *lhyp;
+ struct nfslayout *lyp;
+ nfsquad_t clientid;
+ nfsv4stateid_t stateid;
+ fhandle_t fh;
+ int error, laytype;
+
+ lhyp = &nfslayouthash[arc4random() % nfsrv_layouthashsize];
+ NFSLOCKLAYOUT(lhyp);
+ TAILQ_FOREACH_REVERSE(lyp, &lhyp->list, nfslayouthead, lay_list) {
+ if ((lyp->lay_flags & NFSLAY_CALLB) == 0) {
+ lyp->lay_flags |= NFSLAY_CALLB;
+ /*
+ * The layout stateid.seqid needs to be incremented
+ * before doing a LAYOUT_RECALL callback.
+ */
+ if (++lyp->lay_stateid.seqid == 0)
+ lyp->lay_stateid.seqid = 1;
+ clientid = lyp->lay_clientid;
+ stateid = lyp->lay_stateid;
+ fh = lyp->lay_fh;
+ laytype = lyp->lay_type;
+ break;
+ }
+ }
+ NFSUNLOCKLAYOUT(lhyp);
+ if (lyp != NULL) {
+ error = nfsrv_recalllayout(clientid, &stateid, &fh, NULL,
+ laytype, p);
+ if (error != 0 && error != NFSERR_NOMATCHLAYOUT)
+ printf("recallold=%d\n", error);
+ if (error == NFSERR_NOMATCHLAYOUT) {
+ /*
+ * The client no longer knows this layout, so it can
+ * be free'd now.
+ * Since the hash list was unlocked, we need to find
+ * it again.
+ */
+ NFSLOCKLAYOUT(lhyp);
+ error = nfsrv_findlayout(&clientid, &fh, laytype, p,
+ &lyp);
+ if (error == 0 &&
+ (lyp->lay_flags & NFSLAY_CALLB) != 0 &&
+ lyp->lay_stateid.other[0] == stateid.other[0] &&
+ lyp->lay_stateid.other[1] == stateid.other[1] &&
+ lyp->lay_stateid.other[2] == stateid.other[2])
+ nfsrv_freelayout(&lhyp->list, lyp);
+ NFSUNLOCKLAYOUT(lhyp);
+ }
+ }
+}
+
+/*
* Try and return layout(s).
*/
int
@@ -6677,7 +6755,8 @@ nfsrv_layoutreturn(struct nfsrv_descript *nd, vnode_t
lhyp = NFSLAYOUTHASH(&fh);
NFSDRECALLLOCK();
NFSLOCKLAYOUT(lhyp);
- error = nfsrv_findlayout(nd, &fh, layouttype, p, &lyp);
+ error = nfsrv_findlayout(&nd->nd_clientid, &fh,
+ layouttype, p, &lyp);
NFSD_DEBUG(4, "layoutret findlay=%d\n", error);
if (error == 0 &&
stateidp->other[0] == lyp->lay_stateid.other[0] &&
@@ -6750,21 +6829,19 @@ nfsrv_layoutreturn(struct nfsrv_descript *nd, vnode_t
* Look for an existing layout.
*/
static int
-nfsrv_findlayout(struct nfsrv_descript *nd, fhandle_t *fhp, int laytype,
+nfsrv_findlayout(nfsquad_t *clientidp, fhandle_t *fhp, int laytype,
NFSPROC_T *p, struct nfslayout **lypp)
{
struct nfslayouthash *lhyp;
struct nfslayout *lyp;
int ret;
- KASSERT((nd->nd_flag & ND_IMPLIEDCLID) != 0,
- ("nfsrv_layoutget: no nd_clientid\n"));
*lypp = NULL;
ret = 0;
lhyp = NFSLAYOUTHASH(fhp);
TAILQ_FOREACH(lyp, &lhyp->list, lay_list) {
if (NFSBCMP(&lyp->lay_fh, fhp, sizeof(*fhp)) == 0 &&
- lyp->lay_clientid.qval == nd->nd_clientid.qval &&
+ lyp->lay_clientid.qval == clientidp->qval &&
lyp->lay_type == laytype)
break;
}
@@ -6832,6 +6909,7 @@ nfsrv_addlayout(struct nfsrv_descript *nd, struct nfsl
/* Insert the new layout in the lists. */
*lypp = NULL;
+ atomic_add_int(&nfsrv_layoutcnt, 1);
NFSBCOPY(lyp->lay_xdr, layp, lyp->lay_layoutlen);
*layoutlenp = lyp->lay_layoutlen;
TAILQ_INSERT_HEAD(&lhyp->list, lyp, lay_list);
@@ -6907,8 +6985,6 @@ nfsrv_getdevinfo(char *devid, int layouttype, uint32_t
/*
* Free a list of layout state structures.
- * (This function will also free all nfslockfile structures that no
- * longer have associated state.)
*/
static void
nfsrv_freelayoutlist(nfsquad_t clientid)
@@ -6936,6 +7012,7 @@ nfsrv_freelayout(struct nfslayouthead *lhp, struct nfs
{
NFSD_DEBUG(4, "Freelayout=%p\n", lyp);
+ atomic_add_int(&nfsrv_layoutcnt, -1);
TAILQ_REMOVE(lhp, lyp, lay_list);
free(lyp, M_NFSDSTATE);
}
@@ -7778,7 +7855,8 @@ tryagain:
if (++lyp->lay_stateid.seqid == 0)
lyp->lay_stateid.seqid = 1;
NFSDRECALLUNLOCK();
- nfsrv_recalllayout(lyp, p);
+ nfsrv_recalllayout(lyp->lay_clientid, &lyp->lay_stateid,
+ &lyp->lay_fh, lyp, lyp->lay_type, p);
NFSD_DEBUG(4, "nfsrv_copymr: recalled layout\n");
goto tryagain;
}
More information about the svn-src-projects
mailing list