patch to let witness monitor the mtx pool

[Don Lewis]

I've been running with the patch below for a little while now.  It
helped me find a situation where a thread attemped to grab a "pool
mutex" while it already held one, which I suspect could have caused a
deadlock in certain circumstances.  In any case, this was illegal
because these mutexes are only supposed be used as leaf mutexes.

I had to patch kern_sx.c to quiet warnings about lock order reversal
that look bogus to me.  I believe the way that kern_sx.c uses the pool
mutexes is safe.

Index: sys/kern/kern_mtxpool.c
===================================================================
RCS file: /home/ncvs/src/sys/kern/kern_mtxpool.c,v
retrieving revision 1.7
diff -u -r1.7 kern_mtxpool.c
--- sys/kern/kern_mtxpool.c	13 Jun 2003 19:39:21 -0000	1.7
+++ sys/kern/kern_mtxpool.c	20 Jun 2003 06:36:55 -0000
@@ -33,6 +33,7 @@
 #include <sys/lock.h>
 #include <sys/malloc.h>
 #include <sys/mutex.h>
+#include <sys/sysctl.h>
 #include <sys/systm.h>
 
 #ifndef MTX_POOL_SIZE
@@ -44,6 +45,12 @@
 
 int mtx_pool_valid = 0;
 
+int     witness_skip_mtx_pool = 0;
+
+TUNABLE_INT("debug.witness_skip_mtx_pool", &witness_skip_mtx_pool);
+SYSCTL_INT(_debug, OID_AUTO, witness_skip_mtx_pool, CTLFLAG_RD,
+    &witness_skip_mtx_pool, 0, "");
+
 /*
  * Inline version of mtx_pool_find(), used to streamline our main API
  * function calls.
@@ -60,11 +67,13 @@
 static void
 mtx_pool_setup(void *dummy __unused)
 {
-    int i;
+    int i, mtx_opts;
 
+    mtx_opts = MTX_DEF | MTX_QUIET;
+    if (witness_skip_mtx_pool)
+	mtx_opts |= MTX_NOWITNESS;
     for (i = 0; i < MTX_POOL_SIZE; ++i)
-	mtx_init(&mtx_pool_ary[i], "pool mutex", NULL,
-	    MTX_DEF | MTX_NOWITNESS | MTX_QUIET);
+	mtx_init(&mtx_pool_ary[i], "pool mutex", NULL, mtx_opts);
     mtx_pool_valid = 1;
 }
 
Index: sys/kern/kern_sx.c
===================================================================
RCS file: /home/ncvs/src/sys/kern/kern_sx.c,v
retrieving revision 1.19
diff -u -r1.19 kern_sx.c
--- sys/kern/kern_sx.c	11 Jun 2003 00:56:56 -0000	1.19
+++ sys/kern/kern_sx.c	19 Jun 2003 20:28:19 -0000
@@ -126,9 +126,8 @@
 	sx->sx_cnt++;
 
 	LOCK_LOG_LOCK("SLOCK", &sx->sx_object, 0, 0, file, line);
-	WITNESS_LOCK(&sx->sx_object, 0, file, line);
-
 	mtx_unlock(sx->sx_lock);
+	WITNESS_LOCK(&sx->sx_object, 0, file, line);
 }
 
 int
@@ -139,8 +138,8 @@
 	if (sx->sx_cnt >= 0) {
 		sx->sx_cnt++;
 		LOCK_LOG_TRY("SLOCK", &sx->sx_object, 0, 1, file, line);
-		WITNESS_LOCK(&sx->sx_object, LOP_TRYLOCK, file, line);
 		mtx_unlock(sx->sx_lock);
+		WITNESS_LOCK(&sx->sx_object, LOP_TRYLOCK, file, line);
 		return (1);
 	} else {
 		LOCK_LOG_TRY("SLOCK", &sx->sx_object, 0, 0, file, line);
@@ -180,9 +179,8 @@
 	sx->sx_xholder = curthread;
 
 	LOCK_LOG_LOCK("XLOCK", &sx->sx_object, 0, 0, file, line);
-	WITNESS_LOCK(&sx->sx_object, LOP_EXCLUSIVE, file, line);
-
 	mtx_unlock(sx->sx_lock);
+	WITNESS_LOCK(&sx->sx_object, LOP_EXCLUSIVE, file, line);
 }
 
 int
@@ -194,9 +192,9 @@
 		sx->sx_cnt--;
 		sx->sx_xholder = curthread;
 		LOCK_LOG_TRY("XLOCK", &sx->sx_object, 0, 1, file, line);
+		mtx_unlock(sx->sx_lock);
 		WITNESS_LOCK(&sx->sx_object, LOP_EXCLUSIVE | LOP_TRYLOCK, file,
 		    line);
-		mtx_unlock(sx->sx_lock);
 		return (1);
 	} else {
 		LOCK_LOG_TRY("XLOCK", &sx->sx_object, 0, 0, file, line);