svn commit: r190175 - in stable/7/sys: . contrib/pf dev/ath/ath_hal dev/cxgb netgraph

[Alexander Motin]

Author: mav
Date: Fri Mar 20 20:55:57 2009
New Revision: 190175
URL: http://svn.freebsd.org/changeset/base/190175

Log:
  MFC rev. 186093
  
  To avoid one doubtless netgraph SMP scalability limitation point, switch
  node queues processing from single swi:net thread to several specialized
  threads.

Modified:
  stable/7/sys/   (props changed)
  stable/7/sys/contrib/pf/   (props changed)
  stable/7/sys/dev/ath/ath_hal/   (props changed)
  stable/7/sys/dev/cxgb/   (props changed)
  stable/7/sys/netgraph/ng_base.c

Modified: stable/7/sys/netgraph/ng_base.c
==============================================================================
--- stable/7/sys/netgraph/ng_base.c	Fri Mar 20 19:04:31 2009	(r190174)
+++ stable/7/sys/netgraph/ng_base.c	Fri Mar 20 20:55:57 2009	(r190175)
@@ -61,10 +61,11 @@
 #include <sys/syslog.h>
 #include <sys/refcount.h>
 #include <sys/proc.h>
+#include <sys/unistd.h>
+#include <sys/kthread.h>
+#include <sys/smp.h>
 #include <machine/cpu.h>
 
-#include <net/netisr.h>
-
 #include <netgraph/ng_message.h>
 #include <netgraph/netgraph.h>
 #include <netgraph/ng_parse.h>
@@ -203,7 +204,7 @@ static int	ng_generic_msg(node_p here, i
 static ng_ID_t	ng_decodeidname(const char *name);
 static int	ngb_mod_event(module_t mod, int event, void *data);
 static void	ng_worklist_add(node_p node);
-static void	ngintr(void);
+static void	ngthread(void *);
 static int	ng_apply_item(node_p node, item_p item, int rw);
 static void	ng_flush_input_queue(struct ng_queue * ngq);
 static node_p	ng_ID2noderef(ng_ID_t ID);
@@ -251,6 +252,10 @@ MALLOC_DEFINE(M_NETGRAPH_MSG, "netgraph_
 	mtx_lock(&ng_worklist_mtx)
 #define	NG_WORKLIST_UNLOCK()			\
 	mtx_unlock(&ng_worklist_mtx)
+#define	NG_WORKLIST_SLEEP()			\
+	mtx_sleep(&ng_worklist, &ng_worklist_mtx, PI_NET, "sleep", 0)
+#define	NG_WORKLIST_WAKEUP()			\
+	wakeup_one(&ng_worklist)
 
 #ifdef NETGRAPH_DEBUG /*----------------------------------------------*/
 /*
@@ -2865,9 +2870,13 @@ out:
 
 uma_zone_t			ng_qzone;
 uma_zone_t			ng_qdzone;
+static int			numthreads = 0; /* number of queue threads */
 static int			maxalloc = 4096;/* limit the damage of a leak */
 static int			maxdata = 512;	/* limit the damage of a DoS */
 
+TUNABLE_INT("net.graph.threads", &numthreads);
+SYSCTL_INT(_net_graph, OID_AUTO, threads, CTLFLAG_RDTUN, &numthreads,
+    0, "Number of queue processing threads to create");
 TUNABLE_INT("net.graph.maxalloc", &maxalloc);
 SYSCTL_INT(_net_graph, OID_AUTO, maxalloc, CTLFLAG_RDTUN, &maxalloc,
     0, "Maximum number of non-data queue items to allocate");
@@ -3061,7 +3070,7 @@ ng_mod_event(module_t mod, int event, vo
 static int
 ngb_mod_event(module_t mod, int event, void *data)
 {
-	int error = 0;
+	int i, error = 0;
 
 	switch (event) {
 	case MOD_LOAD:
@@ -3087,8 +3096,17 @@ ngb_mod_event(module_t mod, int event, v
 		ng_qdzone = uma_zcreate("NetGraph data items", sizeof(struct ng_item),
 		    NULL, NULL, NULL, NULL, UMA_ALIGN_CACHE, 0);
 		uma_zone_set_max(ng_qdzone, maxdata);
-		netisr_register(NETISR_NETGRAPH, (netisr_t *)ngintr, NULL,
-		    NETISR_MPSAFE);
+		/* Autoconfigure number of threads. */
+		if (numthreads <= 0)
+			numthreads = mp_ncpus;
+		/* Create threads. */
+		for (i = 0; i < numthreads; i++) {
+			if (kthread_create(ngthread, NULL, NULL,
+			    0, 0, "ng_queue%d", i)) {
+				numthreads = i - 1;
+				break;
+			}
+		}
 		break;
 	case MOD_UNLOAD:
 		/* You can't unload it because an interface may be using it. */
@@ -3243,25 +3261,20 @@ SYSCTL_PROC(_debug, OID_AUTO, ng_dump_it
 /***********************************************************************
 * Worklist routines
 **********************************************************************/
-/* NETISR thread enters here */
 /*
  * Pick a node off the list of nodes with work,
- * try get an item to process off it.
- * If there are no more, remove the node from the list.
+ * try get an item to process off it. Remove the node from the list.
  */
 static void
-ngintr(void)
+ngthread(void *arg)
 {
 	for (;;) {
 		node_p  node;
 
 		/* Get node from the worklist. */
 		NG_WORKLIST_LOCK();
-		node = TAILQ_FIRST(&ng_worklist);
-		if (!node) {
-			NG_WORKLIST_UNLOCK();
-			break;
-		}
+		while ((node = TAILQ_FIRST(&ng_worklist)) == NULL)
+			NG_WORKLIST_SLEEP();
 		TAILQ_REMOVE(&ng_worklist, node, nd_work);
 		NG_WORKLIST_UNLOCK();
 		CTR3(KTR_NET, "%20s: node [%x] (%p) taken off worklist",
@@ -3317,9 +3330,9 @@ ng_worklist_add(node_p node)
 		NG_WORKLIST_LOCK();
 		TAILQ_INSERT_TAIL(&ng_worklist, node, nd_work);
 		NG_WORKLIST_UNLOCK();
-		schednetisr(NETISR_NETGRAPH);
 		CTR3(KTR_NET, "%20s: node [%x] (%p) put on worklist", __func__,
 		    node->nd_ID, node);
+		NG_WORKLIST_WAKEUP();
 	} else {
 		CTR3(KTR_NET, "%20s: node [%x] (%p) already on worklist",
 		    __func__, node->nd_ID, node);