svn commit: r212506 - head/sys/nfsclient

[Konstantin Belousov]

Author: kib
Date: Sun Sep 12 19:06:08 2010
New Revision: 212506
URL: http://svn.freebsd.org/changeset/base/212506

Log:
  Do not fork nfsiod directly from the vop methods. This causes LORs between
  vnode lock and several locks needed during fork, like fd lock.
  
  Instead, schedule the task to be executed in the taskqueue context. We
  still waiting for the fork to finish, but the context of the thread
  executing the task does not make real LORs with our vnode lock.
  
  Submitted by:	pluknet at gmail com
  Reviewed by:	jhb
  Tested by:	pho
  MFC after:	3 weeks

Modified:
  head/sys/nfsclient/nfs.h
  head/sys/nfsclient/nfs_nfsiod.c
  head/sys/nfsclient/nfs_subs.c

Modified: head/sys/nfsclient/nfs.h
==============================================================================
--- head/sys/nfsclient/nfs.h	Sun Sep 12 18:53:44 2010	(r212505)
+++ head/sys/nfsclient/nfs.h	Sun Sep 12 19:06:08 2010	(r212506)
@@ -125,6 +125,7 @@ extern struct uma_zone *nfsmount_zone;
 
 extern struct nfsstats nfsstats;
 extern struct mtx nfs_iod_mtx;
+extern struct task nfs_nfsiodnew_task;
 
 extern int nfs_numasync;
 extern unsigned int nfs_iodmax;
@@ -253,6 +254,7 @@ int	nfs_commit(struct vnode *vp, u_quad_
 	    struct ucred *cred, struct thread *td);
 int	nfs_readdirrpc(struct vnode *, struct uio *, struct ucred *);
 int	nfs_nfsiodnew(int);
+void	nfs_nfsiodnew_tq(__unused void *, int);
 int	nfs_asyncio(struct nfsmount *, struct buf *, struct ucred *, struct thread *);
 int	nfs_doio(struct vnode *, struct buf *, struct ucred *, struct thread *);
 void	nfs_doio_directwrite (struct buf *);

Modified: head/sys/nfsclient/nfs_nfsiod.c
==============================================================================
--- head/sys/nfsclient/nfs_nfsiod.c	Sun Sep 12 18:53:44 2010	(r212505)
+++ head/sys/nfsclient/nfs_nfsiod.c	Sun Sep 12 19:06:08 2010	(r212506)
@@ -59,6 +59,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/fcntl.h>
 #include <sys/lockf.h>
 #include <sys/mutex.h>
+#include <sys/taskqueue.h>
 
 #include <netinet/in.h>
 #include <netinet/tcp.h>
@@ -75,6 +76,16 @@ static MALLOC_DEFINE(M_NFSSVC, "nfsclien
 
 static void	nfssvc_iod(void *);
 
+struct nfsiod_str {
+	STAILQ_ENTRY(nfsiod_str) ni_links;
+	int *ni_inst;
+	int ni_iod;
+	int ni_error;
+	int ni_done;
+};
+static STAILQ_HEAD(, nfsiod_str) nfsiodhead =
+    STAILQ_HEAD_INITIALIZER(nfsiodhead);
+
 static int nfs_asyncdaemon[NFS_MAXASYNCDAEMON];
 
 SYSCTL_DECL(_vfs_nfs);
@@ -159,11 +170,30 @@ SYSCTL_PROC(_vfs_nfs, OID_AUTO, iodmax, 
     sizeof (nfs_iodmax), sysctl_iodmax, "IU",
     "Max number of nfsiod kthreads");
 
+void
+nfs_nfsiodnew_tq(__unused void *arg, int pending)
+{
+	struct nfsiod_str *nip;
+
+	mtx_lock(&nfs_iod_mtx);
+	while ((nip = STAILQ_FIRST(&nfsiodhead)) != NULL) {
+		STAILQ_REMOVE_HEAD(&nfsiodhead, ni_links);
+		mtx_unlock(&nfs_iod_mtx);
+		nip->ni_error = kproc_create(nfssvc_iod, nip->ni_inst, NULL,
+		    RFHIGHPID, 0, "nfsiod %d", nip->ni_iod);
+		nip->ni_done = 1;
+		mtx_lock(&nfs_iod_mtx);
+		wakeup(nip);
+	}
+	mtx_unlock(&nfs_iod_mtx);
+}
+
 int
 nfs_nfsiodnew(int set_iodwant)
 {
 	int error, i;
 	int newiod;
+	struct nfsiod_str *nip;
 
 	if (nfs_numasync >= nfs_iodmax)
 		return (-1);
@@ -179,9 +209,16 @@ nfs_nfsiodnew(int set_iodwant)
 	if (set_iodwant > 0)
 		nfs_iodwant[i] = NFSIOD_CREATED_FOR_NFS_ASYNCIO;
 	mtx_unlock(&nfs_iod_mtx);
-	error = kproc_create(nfssvc_iod, nfs_asyncdaemon + i, NULL, RFHIGHPID,
-	    0, "nfsiod %d", newiod);
+	nip = malloc(sizeof(*nip), M_TEMP, M_WAITOK | M_ZERO);
+	nip->ni_inst = nfs_asyncdaemon + i;
+	nip->ni_iod = newiod;
 	mtx_lock(&nfs_iod_mtx);
+	STAILQ_INSERT_TAIL(&nfsiodhead, nip, ni_links);
+	taskqueue_enqueue(taskqueue_thread, &nfs_nfsiodnew_task);
+	while (!nip->ni_done)
+		mtx_sleep(nip, &nfs_iod_mtx, 0, "niwt", 0);
+	error = nip->ni_error;
+	free(nip, M_TEMP);
 	if (error) {
 		if (set_iodwant > 0)
 			nfs_iodwant[i] = NFSIOD_NOT_AVAILABLE;

Modified: head/sys/nfsclient/nfs_subs.c
==============================================================================
--- head/sys/nfsclient/nfs_subs.c	Sun Sep 12 18:53:44 2010	(r212505)
+++ head/sys/nfsclient/nfs_subs.c	Sun Sep 12 19:06:08 2010	(r212506)
@@ -59,6 +59,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/sysent.h>
 #include <sys/syscall.h>
 #include <sys/sysproto.h>
+#include <sys/taskqueue.h>
 
 #include <vm/vm.h>
 #include <vm/vm_object.h>
@@ -117,6 +118,7 @@ int		nfs_pbuf_freecnt = -1;	/* start out
 
 struct nfs_bufq	nfs_bufq;
 static struct mtx nfs_xid_mtx;
+struct task	nfs_nfsiodnew_task;
 
 /*
  * and the reverse mapping from generic to Version 2 procedure numbers
@@ -354,6 +356,7 @@ nfs_init(struct vfsconf *vfsp)
 	 */
 	mtx_init(&nfs_iod_mtx, "NFS iod lock", NULL, MTX_DEF);
 	mtx_init(&nfs_xid_mtx, "NFS xid lock", NULL, MTX_DEF);
+	TASK_INIT(&nfs_nfsiodnew_task, 0, nfs_nfsiodnew_tq, NULL);
 
 	nfs_pbuf_freecnt = nswbuf / 2 + 1;
 
@@ -368,9 +371,13 @@ nfs_uninit(struct vfsconf *vfsp)
 	/*
 	 * Tell all nfsiod processes to exit. Clear nfs_iodmax, and wakeup
 	 * any sleeping nfsiods so they check nfs_iodmax and exit.
+	 * Drain nfsiodnew task before we wait for them to finish.
 	 */
 	mtx_lock(&nfs_iod_mtx);
 	nfs_iodmax = 0;
+	mtx_unlock(&nfs_iod_mtx);
+	taskqueue_drain(taskqueue_thread, &nfs_nfsiodnew_task);
+	mtx_lock(&nfs_iod_mtx);
 	for (i = 0; i < nfs_numasync; i++)
 		if (nfs_iodwant[i] == NFSIOD_AVAILABLE)
 			wakeup(&nfs_iodwant[i]);