kern/144003: [lor] kqueue / system map

[Matthew Fleming]

>Number:         144003
>Category:       kern
>Synopsis:       [lor] kqueue / system map
>Confidential:   no
>Severity:       non-critical
>Priority:       low
>Responsible:    freebsd-bugs
>State:          open
>Quarter:        
>Keywords:       
>Date-Required:
>Class:          sw-bug
>Submitter-Id:   current-users
>Arrival-Date:   Tue Feb 16 16:30:05 UTC 2010
>Closed-Date:
>Last-Modified:
>Originator:     Matthew Fleming
>Release:        releng/7.2
>Organization:
Isilon Systems
>Environment:
Isilon OneFS mdf-head-1 vHEAD Isilon OneFS vHEAD B_6211(DEBUG): 0x600000E00184300:Mon Feb 15 04:47:17 PST 2010    root at fastbuild-02.west.isilon.com:/build/mnt/obj.DEBUG/build/mnt/src/sys/IQ.amd64.debug amd64

>Description:
We've been seeing LOR #185 on http://sources.zabbadoz.net/freebsd/lor.html since October 2007 on and off.

lock order reversal:
 1st 0xffffff010a371600 kqueue (kqueue) @ /build/mnt/src/sys/kern/kern_event.c:1056
 2nd 0xffffff009541b3d0 system map (system map) @ /build/mnt/src/sys/vm/vm_map.c:2424
Stack: -------------------------------------------------

0xffffffff802117f9 -> 0xffffffff802117b0 (fp=0xffffffffde8f95c0): _mtx_lock_flags
0xffffffff80366047 -> 0xffffffff80366020 (fp=0xffffffffde8f95e0): _vm_map_lock
0xffffffff80368d8d -> 0xffffffff80368d60 (fp=0xffffffffde8f9610): vm_map_remove
0xffffffff8035f0a4 -> 0xffffffff8035f050 (fp=0xffffffffde8f9630): uma_large_free
0xffffffff8020ea20 -> 0xffffffff8020e9b0 (fp=0xffffffffde8f9660): free
0xffffffff801f82ed -> 0xffffffff801f8120 (fp=0xffffffffde8f96a0): kqueue_expand
0xffffffff801f9675 -> 0xffffffff801f9110 (fp=0xffffffffde8f9720): kqueue_register
0xffffffff801f9e60 -> 0xffffffff801f9d70 (fp=0xffffffffde8f98e0): kern_kevent
0xffffffff803ba626 -> 0xffffffff803ba5e0 (fp=0xffffffffde8f9940): freebsd32_kevent
0xffffffff803b97aa -> 0xffffffff803b94c0 (fp=0xffffffffde8f9c20): ia32_syscall

>How-To-Repeat:
unknown; kqueue_expand will call free(9) regularly but I think most of the time it doesn't cause the LOR.
>Fix:


Patch attached with submission follows:

Index: kern_event.c
===================================================================
--- kern_event.c	(revision 140805)
+++ kern_event.c	(working copy)
@@ -1094,82 +1094,86 @@ kqueue_schedtask(struct kqueue *kq)
 
 /*
  * Expand the kq to make sure we have storage for fops/ident pair.
  *
  * Return 0 on success (or no work necessary), return errno on failure.
  *
  * Not calling hashinit w/ waitok (proper malloc flag) should be safe.
  * If kqueue_register is called from a non-fd context, there usually/should
  * be no locks held.
  */
 static int
 kqueue_expand(struct kqueue *kq, struct filterops *fops, uintptr_t ident,
 	int waitok)
 {
 	struct klist *list, *tmp_knhash;
+	struct klist *to_free = NULL;
 	u_long tmp_knhashmask;
 	int size;
 	int fd;
 	int mflag = waitok ? M_WAITOK : M_NOWAIT;
 
 	KQ_NOTOWNED(kq);
 
 	if (fops->f_isfd) {
 		fd = ident;
 		if (kq->kq_knlistsize <= fd) {
 			size = kq->kq_knlistsize;
 			while (size <= fd)
 				size += KQEXTENT;
 			list = malloc(size * sizeof(*list), M_KQUEUE, mflag);
 			if (list == NULL)
 				return ENOMEM;
 			KQ_LOCK(kq);
 			if (kq->kq_knlistsize > fd) {
-				free(list, M_KQUEUE);
+				to_free = list;
 				list = NULL;
 			} else {
 				if (kq->kq_knlist != NULL) {
 					bcopy(kq->kq_knlist, list,
 					    kq->kq_knlistsize * sizeof list);
-					free(kq->kq_knlist, M_KQUEUE);
+					to_free = kq->kq_knlist;
 					kq->kq_knlist = NULL;
 				}
 				bzero((caddr_t)list +
 				    kq->kq_knlistsize * sizeof list,
 				    (size - kq->kq_knlistsize) * sizeof list);
 				kq->kq_knlistsize = size;
 				kq->kq_knlist = list;
 			}
 			KQ_UNLOCK(kq);
 		}
 	} else {
 		if (kq->kq_knhashmask == 0) {
 			tmp_knhash = hashinit(KN_HASHSIZE, M_KQUEUE,
 			    &tmp_knhashmask);
 			if (tmp_knhash == NULL)
 				return ENOMEM;
 			KQ_LOCK(kq);
 			if (kq->kq_knhashmask == 0) {
 				kq->kq_knhash = tmp_knhash;
 				kq->kq_knhashmask = tmp_knhashmask;
 			} else {
-				free(tmp_knhash, M_KQUEUE);
+				to_free = tmp_knhash;
 			}
 			KQ_UNLOCK(kq);
 		}
 	}
 
+	if (to_free != NULL)
+		free(to_free, M_KQUEUE);
+
 	KQ_NOTOWNED(kq);
 	return 0;
 }
 
 static void
 kqueue_task(void *arg, int pending)
 {
 	struct kqueue *kq;
 	int haskqglobal;
 
 	haskqglobal = 0;
 	kq = arg;
 
 	KQ_GLOBAL_LOCK(&kq_global, haskqglobal);
 	KQ_LOCK(kq);


>Release-Note:
>Audit-Trail:
>Unformatted: