kern/104406: [ufs] Processes get stuck in "ufs" state under persistent CPU load

[Oleg Derevenetz]

The following reply was made to PR kern/104406; it has been noted by GNATS.

From: "Oleg Derevenetz" <oleg at vsi.ru>
To: <bug-followup at FreeBSD.org>, <doublef-ctm at yandex.ru>
Cc:  
Subject: Re: kern/104406: [ufs] Processes get stuck in "ufs" state under persistent CPU load
Date: Mon, 22 Oct 2007 16:24:53 +0400

 There are all required information from "Debugging Deadlocks" chapter of FreeBSD Developer's Handbook:
 
 db> show pcpu
 cpuid        = 1
 curthread    = 0xc7ce3900: pid 12 "idle: cpu1"
 curpcb       = 0xe68e7d90
 fpcurthread  = none
 idlethread   = 0xc7ce3900: pid 12 "idle: cpu1"
 APIC ID      = 1
 currentldt   = 0x50
 spin locks held:
 exclusive spin mutex sio r = 0 (0xc07f61a0) locked @ /usr/src/sys/dev/sio/sio.c:1390
 
 db> show allpcpu
 Current CPU: 1
 
 cpuid        = 0
 curthread    = 0xc7ce3a80: pid 13 "idle: cpu0"
 curpcb       = 0xe68ead90
 fpcurthread  = none
 idlethread   = 0xc7ce3a80: pid 13 "idle: cpu0"
 APIC ID      = 0
 currentldt   = 0x50
 spin locks held:
 
 cpuid        = 1
 curthread    = 0xc7ce3900: pid 12 "idle: cpu1"
 curpcb       = 0xe68e7d90
 fpcurthread  = none
 idlethread   = 0xc7ce3900: pid 12 "idle: cpu1"
 APIC ID      = 1
 currentldt   = 0x50
 spin locks held:
 exclusive spin mutex sio r = 0 (0xc07f61a0) locked @ /usr/src/sys/dev/sio/sio.c:1390
 
 cpuid        = 2
 curthread    = 0xc7ce3780: pid 11 "idle: cpu2"
 curpcb       = 0xe68e4d90
 fpcurthread  = none
 idlethread   = 0xc7ce3780: pid 11 "idle: cpu2"
 APIC ID      = 2
 currentldt   = 0x50
 spin locks held:
 
 cpuid        = 3
 curthread    = 0xc7ce3600: pid 10 "idle: cpu3"
 curpcb       = 0xe68e1d90
 fpcurthread  = none
 idlethread   = 0xc7ce3600: pid 10 "idle: cpu3"
 APIC ID      = 3
 currentldt   = 0x50
 spin locks held:
 
 db> show locks
 exclusive spin mutex sio r = 0 (0xc07f61a0) locked @ /usr/src/sys/dev/sio/sio.c:1390
 
 db> show alllocks
 Process 938 (sendmail) thread 0xc82e8300 (100115)
 shared sx acct_sx r = 0 (0xc0795f00) locked @ /usr/src/sys/kern/kern_acct.c:308
 Process 937 (sendmail) thread 0xc83c4780 (100154)
 shared sx acct_sx r = 0 (0xc0795f00) locked @ /usr/src/sys/kern/kern_acct.c:308
 
 db> show lockedvnods
 Locked vnodes
 
 0xc7eb1ae0: tag ufs, type VREG
     usecount 3, writecount 0, refcount 4 mountedhere 0
     flags (VV_TEXT)
     v_object 0xc106d318 ref 1 pages 25
      lock type ufs: EXCL (count 1) by thread 0xc7dbdc00 (pid 41)#0 0xc0559c15 at lockmgr+0x4ed
 #1 0xc0673b06 at ffs_lock+0x76
 #2 0xc06e9797 at VOP_LOCK_APV+0x87
 #3 0xc05c9444 at vn_lock+0xac
 #4 0xc05bd2ea at vget+0xbe
 #5 0xc06725b1 at ffs_sync+0x1c1
 #6 0xc05bee5c at sync_fsync+0x164
 #7 0xc06e8b27 at VOP_FSYNC_APV+0x9b
 #8 0xc05bcace at sync_vnode+0x106
 #9 0xc05bcd49 at sched_sync+0x1ed
 #10 0xc0550cc4 at fork_exit+0xa0
 #11 0xc06c23bc at fork_trampoline+0x8
 
         ino 70723, on dev da0s1a
 
 0xc817a570: tag syncer, type VNON
     usecount 1, writecount 0, refcount 2 mountedhere 0
     flags ()
      lock type syncer: EXCL (count 1) by thread 0xc7dbdc00 (pid 41)#0 0xc0559c15 at lockmgr+0x4ed
 #1 0xc05b4189 at vop_stdlock+0x21
 #2 0xc06e9797 at VOP_LOCK_APV+0x87
 #3 0xc05c9444 at vn_lock+0xac
 #4 0xc05bcaab at sync_vnode+0xe3
 #5 0xc05bcd49 at sched_sync+0x1ed
 #6 0xc0550cc4 at fork_exit+0xa0
 #7 0xc06c23bc at fork_trampoline+0x8
 
 
 0xc8228000: tag ufs, type VDIR
     usecount 1, writecount 0, refcount 2 mountedhere 0
     flags ()
     v_object 0xc814e4a4 ref 0 pages 0
      lock type ufs: EXCL (count 1) by thread 0xc8105d80 (pid 939)#0 0xc0559c15 at lockmgr+0x4ed
 #1 0xc0673b06 at ffs_lock+0x76
 #2 0xc06e9797 at VOP_LOCK_APV+0x87
 #3 0xc05c9444 at vn_lock+0xac
 #4 0xc05bd2ea at vget+0xbe
 #5 0xc05b1a66 at cache_lookup+0x34a
 #6 0xc05b1f9a at vfs_cache_lookup+0x92
 #7 0xc06e7857 at VOP_LOOKUP_APV+0x87
 #8 0xc05b61d0 at lookup+0x4a4
 #9 0xc05b5a8a at namei+0x39a
 #10 0xc05c8443 at vn_open_cred+0x26f
 #11 0xc05c81d2 at vn_open+0x1e
 #12 0xc05c1879 at kern_open+0x231
 #13 0xc05c1612 at open+0x1a
 #14 0xc06d6d4b at syscall+0x25b
 #15 0xc06c23af at Xint0x80_syscall+0x1f
 
         ino 282624, on dev da0s1e
 
 0xc8302ae0: tag ufs, type VREG
     usecount 1, writecount 1, refcount 3 mountedhere 0
     flags ()
     v_object 0xc8233840 ref 0 pages 10
      lock type ufs: EXCL (count 1) by thread 0xc8087a80 (pid 548)#0 0xc0559c15 at lockmgr+0x4ed
 #1 0xc0673b06 at ffs_lock+0x76
 #2 0xc06e9797 at VOP_LOCK_APV+0x87
 #3 0xc05c9444 at vn_lock+0xac
 #4 0xc05c8de0 at vn_write+0x138
 #5 0xc058ba28 at dofilewrite+0x7c
 #6 0xc058b8c7 at kern_writev+0x3b
 #7 0xc058b870 at writev+0x30
 #8 0xc06d6d4b at syscall+0x25b
 #9 0xc06c23af at Xint0x80_syscall+0x1f
 
         ino 259099, on dev da0s1e
 
 0xc830bc3c: tag ufs, type VREG
     usecount 1, writecount 1, refcount 4 mountedhere 0
     flags ()
     v_object 0xc81578c4 ref 0 pages 12
      lock type ufs: EXCL (count 1) by thread 0xc83c4780 (pid 937) with 1 pending#0 0xc0559c15 at lockmgr+0x4ed
 #1 0xc0673b06 at ffs_lock+0x76
 #2 0xc06e9797 at VOP_LOCK_APV+0x87
 #3 0xc05c9444 at vn_lock+0xac
 #4 0xc05c87dd at vn_rdwr+0x85
 #5 0xc053d3bf at acct_process+0x31f
 #6 0xc054eb82 at exit1+0x716
 #7 0xc054e46c at exit1+0
 #8 0xc06d6d4b at syscall+0x25b
 #9 0xc06c23af at Xint0x80_syscall+0x1f
 
         ino 23555, on dev da0s1e
 
 0xc837915c: tag ufs, type VREG
     usecount 0, writecount 0, refcount 4 mountedhere 0
     flags ()
     v_object 0xc8282210 ref 0 pages 1
      lock type ufs: EXCL (count 1) by thread 0xc83c4480 (pid 936) with 1 pending#0 0xc0559c15 at lockmgr+0x4ed
 #1 0xc0673b06 at ffs_lock+0x76
 #2 0xc06e9797 at VOP_LOCK_APV+0x87
 #3 0xc05c9444 at vn_lock+0xac
 #4 0xc05c8708 at vn_close+0x5c
 #5 0xc05c9582 at vn_closefile+0xca
 #6 0xc05488ac at fdrop_locked+0xb0
 #7 0xc05487f4 at fdrop+0x24
 #8 0xc05472e3 at closef+0x367
 #9 0xc0544dd2 at kern_close+0x1aa
 #10 0xc0544c24 at close+0x10
 #11 0xc06d6d4b at syscall+0x25b
 #12 0xc06c23af at Xint0x80_syscall+0x1f
 
         ino 282642, on dev da0s1e
 
 0xc9feb984: tag ufs, type VREG
     usecount 1, writecount 1, refcount 19 mountedhere 0
     flags ()
     v_object 0xc9fee210 ref 0 pages 68
      lock type ufs: EXCL (count 1) by thread 0xc8193c00 (pid 844)#0 0xc0559c15 at lockmgr+0x4ed
 #1 0xc0673b06 at ffs_lock+0x76
 #2 0xc06e9797 at VOP_LOCK_APV+0x87
 #3 0xc05c9444 at vn_lock+0xac
 #4 0xc05c8de0 at vn_write+0x138
 #5 0xc058ba28 at dofilewrite+0x7c
 #6 0xc058b8c7 at kern_writev+0x3b
 #7 0xc058b7ed at write+0x45
 #8 0xc06d6d4b at syscall+0x25b
 #9 0xc06c23af at Xint0x80_syscall+0x1f
 
         ino 4098799, on dev da1s1d
 
 db> alltrace
 
 Tracing command sshd pid 939 tid 100071 td 0xc8105d80
 sched_switch(c8105d80,0,1) at sched_switch+0x177
 mi_switch(1,0) at mi_switch+0x270
 sleepq_switch(c83791b4,c0797aa0,0,c0726d95,220,...) at sleepq_switch+0xc1
 sleepq_wait(c83791b4,0,c8379224,b7,c072d198,...) at sleepq_wait+0x46
 msleep(c83791b4,c07969b0,50,c072b884,0,...) at msleep+0x27d
 acquire(ea87e820,40,60000,c8105d80,0,...) at acquire+0x76
 lockmgr(c83791b4,2002,c8379224,c8105d80) at lockmgr+0x44e
 ffs_lock(ea87e878) at ffs_lock+0x76
 VOP_LOCK_APV(c077a640,ea87e878) at VOP_LOCK_APV+0x87
 vn_lock(c837915c,2002,c8105d80,c837915c) at vn_lock+0xac
 vget(c837915c,2002,c8105d80) at vget+0xbe
 cache_lookup(c8228000,ea87ebe0,ea87ebf4) at cache_lookup+0x34a
 vfs_cache_lookup(ea87e9bc) at vfs_cache_lookup+0x92
 VOP_LOOKUP_APV(c077a640,ea87e9bc) at VOP_LOOKUP_APV+0x87
 lookup(ea87ebcc) at lookup+0x4a4
 namei(ea87ebcc) at namei+0x39a
 vn_open_cred(ea87ebcc,ea87eccc,0,c82fe380,3,...) at vn_open_cred+0x26f
 vn_open(ea87ebcc,ea87eccc,0,3,c0797aa0,...) at vn_open+0x1e
 kern_open(c8105d80,280915c0,0,1,0,...) at kern_open+0x231
 open(c8105d80,ea87ed04) at open+0x1a
 syscall(3b,3b,3b,2809d000,804aa41,...) at syscall+0x25b
 Xint0x80_syscall() at Xint0x80_syscall+0x1f
 --- syscall (5, FreeBSD ELF32, open), eip = 0x2807ca7f, esp = 0xbfbfec1c, ebp = 0xbfbfecc8 ---
 
 Tracing command sendmail pid 938 tid 100115 td 0xc82e8300
 sched_switch(c82e8300,0,1) at sched_switch+0x177
 mi_switch(1,0) at mi_switch+0x270
 sleepq_switch(c830bc94,c0797aa0,0,c0726d95,220,...) at sleepq_switch+0xc1
 sleepq_wait(c830bc94,0,c830bd04,b7,c071f3eb,...) at sleepq_wait+0x46
 msleep(c830bc94,c07975ec,50,c072b884,0,...) at msleep+0x27d
 acquire(ea951ac0,40,60000,c82e8300,0,...) at acquire+0x76
 lockmgr(c830bc94,3002,c830bd04,c82e8300) at lockmgr+0x44e
 ffs_lock(ea951b18) at ffs_lock+0x76
 VOP_LOCK_APV(c077a640,ea951b18) at VOP_LOCK_APV+0x87
 vn_lock(c830bc3c,1002,c82e8300) at vn_lock+0xac
 vn_rdwr(1,c830bc3c,ea951c34,30,0,0,1,3,c7ce0780,0,0,c82e8300) at vn_rdwr+0x85
 acct_process(c82e8300,c0797c20,c07209a0,163,c8364648,...) at acct_process+0x31f
 exit1(c82e8300,0,ea951d30,c06d6d4b,c82e8300,...) at exit1+0x716
 exit1(c82e8300,ea951d04) at exit1
 syscall(3b,3b,3b,0,0,...) at syscall+0x25b
 Xint0x80_syscall() at Xint0x80_syscall+0x1f
 --- syscall (1, FreeBSD ELF32, sys_exit), eip = 0x282fc0fb, esp = 0xbfbfcb1c, ebp = 0xbfbfcb38 ---
 
 Tracing command sendmail pid 937 tid 100154 td 0xc83c4780
 sched_switch(c83c4780,0,1) at sched_switch+0x177
 mi_switch(1,0) at mi_switch+0x270
 sleepq_switch(dc011d14,c0797aa0,0,c0726d95,220,...) at sleepq_switch+0xc1
 sleepq_wait(dc011d14,0,c830bd04,b7,c071f3eb,...) at sleepq_wait+0x46
 msleep(dc011d14,c0796dc4,50,c072c4b4,0,...) at msleep+0x27d
 acquire(ea9db860,120,60000,c83c4780,0,...) at acquire+0x76
 lockmgr(dc011d14,202122,c830bd04,c83c4780) at lockmgr+0x44e
 getblk(c830bc3c,10,0,4000,0,...) at getblk+0x12f
 breadn(c830bc3c,10,0,4000,0,...) at breadn+0x31
 bread(c830bc3c,10,0,4000,0,ea9db970) at bread+0x20
 ffs_balloc_ufs2(c830bc3c,41a60,0,30,c7ce0780,...) at ffs_balloc_ufs2+0x1849
 ffs_write(ea9dbb54) at ffs_write+0x2c0
 VOP_WRITE_APV(c077a640,ea9dbb54) at VOP_WRITE_APV+0x132
 vn_rdwr(1,c830bc3c,ea9dbc34,30,0,0,1,3,c7ce0780,0,0,c83c4780) at vn_rdwr+0x142
 acct_process(c83c4780,c0797c20,c07209a0,163,c865b860,...) at acct_process+0x31f
 exit1(c83c4780,0,ea9dbd30,c06d6d4b,c83c4780,...) at exit1+0x716
 exit1(c83c4780,ea9dbd04) at exit1
 syscall(3b,3b,3b,0,0,...) at syscall+0x25b
 Xint0x80_syscall() at Xint0x80_syscall+0x1f
 --- syscall (1, FreeBSD ELF32, sys_exit), eip = 0x282fc0fb, esp = 0xbfbfbf4c, ebp = 0xbfbfbf68 ---
 
 Tracing command sh pid 936 tid 100156 td 0xc83c4480
 sched_switch(c83c4480,0,1) at sched_switch+0x177
 mi_switch(1,0) at mi_switch+0x270
 sleepq_switch(dbfd0b84,c0797aa0,0,c0726d95,220,...) at sleepq_switch+0xc1
 sleepq_wait(dbfd0b84,0,c817a8f0,b7,c072e794,...) at sleepq_wait+0x46
 msleep(dbfd0b84,c0796af4,50,c072c4b4,0,...) at msleep+0x27d
 acquire(ea9d59bc,120,60000,c83c4480,0,...) at acquire+0x76
 lockmgr(dbfd0b84,202122,c817a8f0,c83c4480) at lockmgr+0x44e
 getblk(c817a828,44ea60,0,4000,0,...) at getblk+0x12f
 breadn(c817a828,44ea60,0,4000,0,...) at breadn+0x31
 bread(c817a828,44ea60,0,4000,0,ea9d5aa8) at bread+0x20
 ffs_update(c837915c,0) at ffs_update+0x22c
 ufs_inactive(ea9d5b04) at ufs_inactive+0x2a6
 VOP_INACTIVE_APV(c077a640,ea9d5b04) at VOP_INACTIVE_APV+0x9b
 vinactive(c837915c,c83c4480) at vinactive+0x8d
 vput(c837915c,c8227000,c078b740,c837915c,1,...) at vput+0x1d3
 vn_close(c837915c,1,c8593780,c83c4480) at vn_close+0x97
 vn_closefile(c812e2d0,c83c4480) at vn_closefile+0xca
 fdrop_locked(c812e2d0,c83c4480,c7ccba0c,0,c072016e,...) at fdrop_locked+0xb0
 fdrop(c812e2d0,c83c4480,6ac,c079efa0,0,...) at fdrop+0x24
 closef(c812e2d0,c83c4480,0,c83c4480,0,...) at closef+0x367
 kern_close(c83c4480,3,ea9d5d30,c06d6d4b,c83c4480,...) at kern_close+0x1aa
 close(c83c4480,ea9d5d04) at close+0x10
 syscall(3b,3b,3b,2808e000,3,...) at syscall+0x25b
 Xint0x80_syscall() at Xint0x80_syscall+0x1f
 --- syscall (6, FreeBSD ELF32, close), eip = 0x2806ba9f, esp = 0xbfbfebcc, ebp = 0xbfbfec78 ---
 
 Tracing command cron pid 935 tid 100160 td 0xc8366d80
 sched_switch(c8366d80,0,1) at sched_switch+0x177
 mi_switch(1,0) at mi_switch+0x270
 sleepq_switch(c8121330) at sleepq_switch+0xc1
 sleepq_wait_sig(c8121330) at sleepq_wait_sig+0x1d
 msleep(c8121330,c81214a0,14c,c0728d39,0) at msleep+0x26e
 pipe_read(c8750ca8,ea9c9cbc,c810e400,0,c8366d80) at pipe_read+0x377
 dofileread(c8366d80,5,c8750ca8,ea9c9cbc,ffffffff,...) at dofileread+0x85
 kern_readv(c8366d80,5,ea9c9cbc,805a000,1000,...) at kern_readv+0x36
 read(c8366d80,ea9c9d04) at read+0x45
 syscall(3b,3b,3b,28171040,28171040,...) at syscall+0x25b
 Xint0x80_syscall() at Xint0x80_syscall+0x1f
 --- syscall (3, FreeBSD ELF32, read), eip = 0x2814b953, esp = 0xbfbfe65c, ebp = 0xbfbfe678 ---
 
 Tracing command mc pid 844 tid 100148 td 0xc8193c00
 sched_switch(c8193c00,0,1) at sched_switch+0x177
 mi_switch(1,0) at mi_switch+0x270
 sleepq_switch(c07e5d04,c0797aa0,0,c0726d95,220,...) at sleepq_switch+0xc1
 sleepq_wait(c07e5d04,0,c072be5a,180,c072e794,...) at sleepq_wait+0x46
 msleep(c07e5d04,c07e5d20,44,c072be76,0) at msleep+0x27d
 waitrunningbufspace(dbd8a778,10,0,ea90d9a4,c05acc09,...) at waitrunningbufspace+0x4a
 bufwrite(dbd502cc) at bufwrite+0x191
 bawrite(dbd502cc,c9feba4c,c1e75650,c9feba4c,c9feba4c,...) at bawrite+0x51
 cluster_wbuild(c9feb984,4000,8,0,8) at cluster_wbuild+0x61c
 cluster_write(c9feb984,dbd8a778,40000,0,50) at cluster_write+0x503
 ffs_write(ea90dbf4) at ffs_write+0x523
 VOP_WRITE_APV(c077a640,ea90dbf4) at VOP_WRITE_APV+0x132
 vn_write(c8738af8,ea90dcbc,c810b200,0,c8193c00) at vn_write+0x1ea
 dofilewrite(c8193c00,7,c8738af8,ea90dcbc,ffffffff,...) at dofilewrite+0x7c
 kern_writev(c8193c00,7,ea90dcbc,86c4130,570,...) at kern_writev+0x3b
 write(c8193c00,ea90dd04) at write+0x45
 syscall(3b,3b,3b,16a0,7,...) at syscall+0x25b
 Xint0x80_syscall() at Xint0x80_syscall+0x1f
 --- syscall (4, FreeBSD ELF32, write), eip = 0x2836d933, esp = 0xbfbfde6c, ebp = 0xbfbfde98 ---
 
 Tracing command csh pid 841 tid 100143 td 0xc83c5a80
 sched_switch(c83c5a80,0,1) at sched_switch+0x177
 mi_switch(1,0) at mi_switch+0x270
 sleepq_switch(c852f034) at sleepq_switch+0xc1
 sleepq_wait_sig(c852f034) at sleepq_wait_sig+0x1d
 msleep(c852f034,c852f068,168,c0724047,0) at msleep+0x26e
 kern_sigsuspend(c83c5a80,2,0,0,0,...) at kern_sigsuspend+0xa3
 sigsuspend(c83c5a80,ea9ffd04) at sigsuspend+0x33
 syscall(3b,3b,3b,8299008,bfbf1ad0,...) at syscall+0x25b
 Xint0x80_syscall() at Xint0x80_syscall+0x1f
 --- syscall (341, FreeBSD ELF32, sigsuspend), eip = 0x281573fb, esp = 0xbfbf1abc, ebp = 0xbfbf1ae8 ---
 
 Tracing command su pid 840 tid 100146 td 0xc8193d80
 sched_switch(c8193d80,0,1) at sched_switch+0x177
 mi_switch(1,0) at mi_switch+0x270
 sleepq_switch(c852f648) at sleepq_switch+0xc1
 sleepq_wait_sig(c852f648) at sleepq_wait_sig+0x1d
 msleep(c852f648,c852f6b0,15c,c0729071,0) at msleep+0x26e
 kern_wait(c8193d80,349,ea910c84,2,0) at kern_wait+0x6bb
 wait4(c8193d80,ea910d04) at wait4+0x2a
 syscall(3b,3b,3b,bfbfe794,bfbfe810,...) at syscall+0x25b
 Xint0x80_syscall() at Xint0x80_syscall+0x1f
 --- syscall (7, FreeBSD ELF32, wait4), eip = 0x280f451f, esp = 0xbfbfe70c, ebp = 0xbfbfe728 ---
 
 Tracing command csh pid 835 tid 100098 td 0xc8102480
 sched_switch(c8102480,0,1) at sched_switch+0x177
 mi_switch(1,0) at mi_switch+0x270
 sleepq_switch(c82e6034) at sleepq_switch+0xc1
 sleepq_wait_sig(c82e6034) at sleepq_wait_sig+0x1d
 msleep(c82e6034,c82e6068,168,c0724047,0) at msleep+0x26e
 kern_sigsuspend(c8102480,2,0,0,0,...) at kern_sigsuspend+0xa3
 sigsuspend(c8102480,ea84ed04) at sigsuspend+0x33
 syscall(3b,3b,3b,81c1308,bfbf1bd0,...) at syscall+0x25b
 Xint0x80_syscall() at Xint0x80_syscall+0x1f
 --- syscall (341, FreeBSD ELF32, sigsuspend), eip = 0x281573fb, esp = 0xbfbf1bbc, ebp = 0xbfbf1be8 ---
 
 Tracing command sshd pid 834 tid 100042 td 0xc7e05480
 sched_switch(c7e05480,0,1) at sched_switch+0x177
 mi_switch(1,0) at mi_switch+0x270
 sleepq_switch(c07e5744) at sleepq_switch+0xc1
 sleepq_wait_sig(c07e5744,246,c0797aa0,c0728ac4,303,...) at sleepq_wait_sig+0x1d
 cv_wait_sig(c07e5744,c07e5720) at cv_wait_sig+0x18b
 kern_select(c7e05480,9,807bb60,807bb70,0,...) at kern_select+0x4bf
 select(c7e05480,e8386d04) at select+0x44
 syscall(3b,3b,3b,0,bfbfe11c,...) at syscall+0x25b
 Xint0x80_syscall() at Xint0x80_syscall+0x1f
 --- syscall (93, FreeBSD ELF32, select), eip = 0x2834d8d3, esp = 0xbfbfe0ac, ebp = 0xbfbfe0f8 ---
 
 Tracing command sshd pid 831 tid 100074 td 0xc8105900
 sched_switch(c8105900,0,1) at sched_switch+0x177
 mi_switch(1,0) at mi_switch+0x270
 sleepq_switch(c8275370) at sleepq_switch+0xc1
 sleepq_wait_sig(c8275370) at sleepq_wait_sig+0x1d
 msleep(c8275370,c827533c,158,c072b38c,0,...) at msleep+0x26e
 sbwait(c8275318,ea875c0c,c058ae7c,4,0,...) at sbwait+0x4b
 soreceive(c82752c8,0,ea875cbc,0,0,0) at soreceive+0x2ee
 soo_read(c8111c18,ea875cbc,c82e1380,0,c8105900) at soo_read+0x41
 dofileread(c8105900,5,c8111c18,ea875cbc,ffffffff,...) at dofileread+0x85
 kern_readv(c8105900,5,ea875cbc,bfbfe108,4,...) at kern_readv+0x36
 read(c8105900,ea875d04) at read+0x45
 syscall(3b,3b,3b,4,0,...) at syscall+0x25b
 Xint0x80_syscall() at Xint0x80_syscall+0x1f
 --- syscall (3, FreeBSD ELF32, read), eip = 0x2834d953, esp = 0xbfbfe0ac, ebp = 0xbfbfe0d8 ---
 
 Tracing command httpd pid 830 tid 100087 td 0xc818a780
 sched_switch(c818a780,0,1) at sched_switch+0x177
 mi_switch(1,0) at mi_switch+0x270
 sleepq_switch(c80b0900) at sleepq_switch+0xc1
 sleepq_wait_sig(c80b0900) at sleepq_wait_sig+0x1d
 msleep(c80b0900,0,168,c0765c84,0) at msleep+0x26e
 lf_setlock(c80b0900,16,ffffffff,ffffffff,ea8c8cb0,...) at lf_setlock+0x181
 lf_advlock(ea8c8c90,c84cfd9c,0,0,ea8c8c7c,...) at lf_advlock+0x203
 ufs_advlock(ea8c8c90) at ufs_advlock+0x1c
 VOP_ADVLOCK_APV(c077a640,ea8c8c90) at VOP_ADVLOCK_APV+0xa9
 flock(c818a780,ea8c8d04) at flock+0x1a1
 syscall(3b,3b,3b,8635018,862e050,...) at syscall+0x25b
 Xint0x80_syscall() at Xint0x80_syscall+0x1f
 --- syscall (131, FreeBSD ELF32, flock), eip = 0x282ab5cf, esp = 0xbfbfebec, ebp = 0xbfbfec08 ---
 
 Tracing command httpd pid 829 tid 100084 td 0xc818ac00
 sched_switch(c818ac00,0,1) at sched_switch+0x177
 mi_switch(1,0) at mi_switch+0x270
 sleepq_switch(c80b0440) at sleepq_switch+0xc1
 sleepq_wait_sig(c80b0440) at sleepq_wait_sig+0x1d
 msleep(c80b0440,0,168,c0765c84,0) at msleep+0x26e
 lf_setlock(c80b0440,16,ffffffff,ffffffff,ea8d1cb0,...) at lf_setlock+0x181
 lf_advlock(ea8d1c90,c84cfd9c,0,0,ea8d1c7c,...) at lf_advlock+0x203
 ufs_advlock(ea8d1c90) at ufs_advlock+0x1c
 VOP_ADVLOCK_APV(c077a640,ea8d1c90) at VOP_ADVLOCK_APV+0xa9
 flock(c818ac00,ea8d1d04) at flock+0x1a1
 syscall(3b,3b,3b,8635018,862e050,...) at syscall+0x25b
 Xint0x80_syscall() at Xint0x80_syscall+0x1f
 --- syscall (131, FreeBSD ELF32, flock), eip = 0x282ab5cf, esp = 0xbfbfebec, ebp = 0xbfbfec08 ---
 
 Tracing command httpd pid 828 tid 100063 td 0xc815ea80
 sched_switch(c815ea80,0,1) at sched_switch+0x177
 mi_switch(1,0) at mi_switch+0x270
 sleepq_switch(c815b3c0) at sleepq_switch+0xc1
 sleepq_wait_sig(c815b3c0) at sleepq_wait_sig+0x1d
 msleep(c815b3c0,0,168,c0765c84,0) at msleep+0x26e
 lf_setlock(c815b3c0,16,ffffffff,ffffffff,ea8a3cb0,...) at lf_setlock+0x181
 lf_advlock(ea8a3c90,c84cfd9c,0,0,ea8a3c7c,...) at lf_advlock+0x203
 ufs_advlock(ea8a3c90) at ufs_advlock+0x1c
 VOP_ADVLOCK_APV(c077a640,ea8a3c90) at VOP_ADVLOCK_APV+0xa9
 flock(c815ea80,ea8a3d04) at flock+0x1a1
 syscall(3b,3b,3b,8635018,862e050,...) at syscall+0x25b
 Xint0x80_syscall() at Xint0x80_syscall+0x1f
 --- syscall (131, FreeBSD ELF32, flock), eip = 0x282ab5cf, esp = 0xbfbfebec, ebp = 0xbfbfec08 ---
 
 Tracing command httpd pid 827 tid 100080 td 0xc818e300
 sched_switch(c818e300,0,1) at sched_switch+0x177
 mi_switch(1,0) at mi_switch+0x270
 sleepq_switch(c8067540) at sleepq_switch+0xc1
 sleepq_wait_sig(c8067540) at sleepq_wait_sig+0x1d
 msleep(c8067540,0,168,c0765c84,0) at msleep+0x26e
 lf_setlock(c8067540,16,ffffffff,ffffffff,ea8ddcb0,...) at lf_setlock+0x181
 lf_advlock(ea8ddc90,c84cfd9c,0,0,ea8ddc7c,...) at lf_advlock+0x203
 ufs_advlock(ea8ddc90) at ufs_advlock+0x1c
 VOP_ADVLOCK_APV(c077a640,ea8ddc90) at VOP_ADVLOCK_APV+0xa9
 flock(c818e300,ea8ddd04) at flock+0x1a1
 syscall(3b,3b,3b,8635018,862e050,...) at syscall+0x25b
 Xint0x80_syscall() at Xint0x80_syscall+0x1f
 --- syscall (131, FreeBSD ELF32, flock), eip = 0x282ab5cf, esp = 0xbfbfebec, ebp = 0xbfbfec08 ---
 
 Tracing command httpd pid 826 tid 100043 td 0xc7e05300
 sched_switch(c7e05300,0,1) at sched_switch+0x177
 mi_switch(1,0) at mi_switch+0x270
 sleepq_switch(c8067c80) at sleepq_switch+0xc1
 sleepq_wait_sig(c8067c80) at sleepq_wait_sig+0x1d
 msleep(c8067c80,0,168,c0765c84,0) at msleep+0x26e
 lf_setlock(c8067c80,16,ffffffff,ffffffff,e8383cb0,...) at lf_setlock+0x181
 lf_advlock(e8383c90,c84cfd9c,0,0,e8383c7c,...) at lf_advlock+0x203
 ufs_advlock(e8383c90) at ufs_advlock+0x1c
 VOP_ADVLOCK_APV(c077a640,e8383c90) at VOP_ADVLOCK_APV+0xa9
 flock(c7e05300,e8383d04) at flock+0x1a1
 syscall(3b,3b,3b,8635018,862e050,...) at syscall+0x25b
 Xint0x80_syscall() at Xint0x80_syscall+0x1f
 --- syscall (131, FreeBSD ELF32, flock), eip = 0x282ab5cf, esp = 0xbfbfebec, ebp = 0xbfbfec08 ---
 
 Tracing command httpd pid 825 tid 100076 td 0xc8105600
 sched_switch(c8105600,0,1) at sched_switch+0x177
 mi_switch(1,0) at mi_switch+0x270
 sleepq_switch(c80b0940) at sleepq_switch+0xc1
 sleepq_wait_sig(c80b0940) at sleepq_wait_sig+0x1d
 msleep(c80b0940,0,168,c0765c84,0) at msleep+0x26e
 lf_setlock(c80b0940,16,ffffffff,ffffffff,ea86fcb0,...) at lf_setlock+0x181
 lf_advlock(ea86fc90,c84cfd9c,0,0,ea86fc7c,...) at lf_advlock+0x203
 ufs_advlock(ea86fc90) at ufs_advlock+0x1c
 VOP_ADVLOCK_APV(c077a640,ea86fc90) at VOP_ADVLOCK_APV+0xa9
 flock(c8105600,ea86fd04) at flock+0x1a1
 syscall(3b,3b,3b,8635018,862e050,...) at syscall+0x25b
 Xint0x80_syscall() at Xint0x80_syscall+0x1f
 --- syscall (131, FreeBSD ELF32, flock), eip = 0x282ab5cf, esp = 0xbfbfebec, ebp = 0xbfbfec08 ---
 
 Tracing command httpd pid 824 tid 100064 td 0xc815e900
 sched_switch(c815e900,0,1) at sched_switch+0x177
 mi_switch(1,0) at mi_switch+0x270
 sleepq_switch(c80b0980) at sleepq_switch+0xc1
 sleepq_wait_sig(c80b0980) at sleepq_wait_sig+0x1d
 msleep(c80b0980,0,168,c0765c84,0) at msleep+0x26e
 lf_setlock(c80b0980,16,ffffffff,ffffffff,ea8a0cb0,...) at lf_setlock+0x181
 lf_advlock(ea8a0c90,c84cfd9c,0,0,ea8a0c7c,...) at lf_advlock+0x203
 ufs_advlock(ea8a0c90) at ufs_advlock+0x1c
 VOP_ADVLOCK_APV(c077a640,ea8a0c90) at VOP_ADVLOCK_APV+0xa9
 flock(c815e900,ea8a0d04) at flock+0x1a1
 syscall(3b,3b,3b,8635018,862e050,...) at syscall+0x25b
 Xint0x80_syscall() at Xint0x80_syscall+0x1f
 --- syscall (131, FreeBSD ELF32, flock), eip = 0x282ab5cf, esp = 0xbfbfebec, ebp = 0xbfbfec08 ---
 
 Tracing command httpd pid 823 tid 100110 td 0xc82e8a80
 sched_switch(c82e8a80,0,1) at sched_switch+0x177
 mi_switch(1,0) at mi_switch+0x270
 sleepq_switch(c812a540) at sleepq_switch+0xc1
 sleepq_wait_sig(c812a540) at sleepq_wait_sig+0x1d
 msleep(c812a540,0,168,c0765c84,0) at msleep+0x26e
 lf_setlock(c812a540,16,ffffffff,ffffffff,ea960cb0,...) at lf_setlock+0x181
 lf_advlock(ea960c90,c84cfd9c,0,0,ea960c7c,...) at lf_advlock+0x203
 ufs_advlock(ea960c90) at ufs_advlock+0x1c
 VOP_ADVLOCK_APV(c077a640,ea960c90) at VOP_ADVLOCK_APV+0xa9
 flock(c82e8a80,ea960d04) at flock+0x1a1
 syscall(3b,3b,3b,8635018,862e050,...) at syscall+0x25b
 Xint0x80_syscall() at Xint0x80_syscall+0x1f
 --- syscall (131, FreeBSD ELF32, flock), eip = 0x282ab5cf, esp = 0xbfbfebec, ebp = 0xbfbfec08 ---
 
 Tracing command httpd pid 822 tid 100105 td 0xc8087900
 sched_switch(c8087900,0,1) at sched_switch+0x177
 mi_switch(1,0) at mi_switch+0x270
 sleepq_switch(c812bd40) at sleepq_switch+0xc1
 sleepq_wait_sig(c812bd40) at sleepq_wait_sig+0x1d
 msleep(c812bd40,0,168,c0765c84,0) at msleep+0x26e
 lf_setlock(c812bd40,16,ffffffff,ffffffff,e864dcb0,...) at lf_setlock+0x181
 lf_advlock(e864dc90,c84cfd9c,0,0,e864dc7c,...) at lf_advlock+0x203
 ufs_advlock(e864dc90) at ufs_advlock+0x1c
 VOP_ADVLOCK_APV(c077a640,e864dc90) at VOP_ADVLOCK_APV+0xa9
 flock(c8087900,e864dd04) at flock+0x1a1
 syscall(3b,3b,3b,8635018,862e050,...) at syscall+0x25b
 Xint0x80_syscall() at Xint0x80_syscall+0x1f
 --- syscall (131, FreeBSD ELF32, flock), eip = 0x282ab5cf, esp = 0xbfbfebec, ebp = 0xbfbfec08 ---
 
 Tracing command httpd pid 821 tid 100039 td 0xc7e05900
 sched_switch(c7e05900,0,1) at sched_switch+0x177
 mi_switch(1,0) at mi_switch+0x270
 sleepq_switch(c80b09c0) at sleepq_switch+0xc1
 sleepq_wait_sig(c80b09c0) at sleepq_wait_sig+0x1d
 msleep(c80b09c0,0,168,c0765c84,0) at msleep+0x26e
 lf_setlock(c80b09c0,16,ffffffff,ffffffff,e838fcb0,...) at lf_setlock+0x181
 lf_advlock(e838fc90,c84cfd9c,0,0,e838fc7c,...) at lf_advlock+0x203
 ufs_advlock(e838fc90) at ufs_advlock+0x1c
 VOP_ADVLOCK_APV(c077a640,e838fc90) at VOP_ADVLOCK_APV+0xa9
 flock(c7e05900,e838fd04) at flock+0x1a1
 syscall(3b,3b,3b,8635018,862e050,...) at syscall+0x25b
 Xint0x80_syscall() at Xint0x80_syscall+0x1f
 --- syscall (131, FreeBSD ELF32, flock), eip = 0x282ab5cf, esp = 0xbfbfebec, ebp = 0xbfbfec08 ---
 
 Tracing command httpd pid 820 tid 100089 td 0xc818a480
 sched_switch(c818a480,0,1) at sched_switch+0x177
 mi_switch(1,0) at mi_switch+0x270
 sleepq_switch(c80af540) at sleepq_switch+0xc1
 sleepq_wait_sig(c80af540) at sleepq_wait_sig+0x1d
 msleep(c80af540,0,168,c0765c84,0) at msleep+0x26e
 lf_setlock(c80af540,16,ffffffff,ffffffff,ea8c2cb0,...) at lf_setlock+0x181
 lf_advlock(ea8c2c90,c84cfd9c,0,0,ea8c2c7c,...) at lf_advlock+0x203
 ufs_advlock(ea8c2c90) at ufs_advlock+0x1c
 VOP_ADVLOCK_APV(c077a640,ea8c2c90) at VOP_ADVLOCK_APV+0xa9
 flock(c818a480,ea8c2d04) at flock+0x1a1
 syscall(3b,3b,3b,8635018,862e050,...) at syscall+0x25b
 Xint0x80_syscall() at Xint0x80_syscall+0x1f
 --- syscall (131, FreeBSD ELF32, flock), eip = 0x282ab5cf, esp = 0xbfbfebec, ebp = 0xbfbfec08 ---
 
 Tracing command httpd pid 819 tid 100092 td 0xc818a000
 sched_switch(c818a000,0,1) at sched_switch+0x177
 mi_switch(1,0) at mi_switch+0x270
 sleepq_switch(c8067480) at sleepq_switch+0xc1
 sleepq_wait_sig(c8067480) at sleepq_wait_sig+0x1d
 msleep(c8067480,0,168,c0765c84,0) at msleep+0x26e
 lf_setlock(c8067480,16,ffffffff,ffffffff,ea8b9cb0,...) at lf_setlock+0x181
 lf_advlock(ea8b9c90,c84cfd9c,0,0,ea8b9c7c,...) at lf_advlock+0x203
 ufs_advlock(ea8b9c90) at ufs_advlock+0x1c
 VOP_ADVLOCK_APV(c077a640,ea8b9c90) at VOP_ADVLOCK_APV+0xa9
 flock(c818a000,ea8b9d04) at flock+0x1a1
 syscall(3b,3b,3b,8635018,862e050,...) at syscall+0x25b
 Xint0x80_syscall() at Xint0x80_syscall+0x1f
 --- syscall (131, FreeBSD ELF32, flock), eip = 0x282ab5cf, esp = 0xbfbfebec, ebp = 0xbfbfec08 ---
 
 Tracing command httpd pid 818 tid 100106 td 0xc8087780
 sched_switch(c8087780,0,1) at sched_switch+0x177
 mi_switch(1,0) at mi_switch+0x270
 sleepq_switch(c815b4c0) at sleepq_switch+0xc1
 sleepq_wait_sig(c815b4c0) at sleepq_wait_sig+0x1d
 msleep(c815b4c0,0,168,c0765c84,0) at msleep+0x26e
 lf_setlock(c815b4c0,16,ffffffff,ffffffff,e864acb0,...) at lf_setlock+0x181
 lf_advlock(e864ac90,c84cfd9c,0,0,e864ac7c,...) at lf_advlock+0x203
 ufs_advlock(e864ac90) at ufs_advlock+0x1c
 VOP_ADVLOCK_APV(c077a640,e864ac90) at VOP_ADVLOCK_APV+0xa9
 flock(c8087780,e864ad04) at flock+0x1a1
 syscall(3b,3b,3b,8635018,862e050,...) at syscall+0x25b
 Xint0x80_syscall() at Xint0x80_syscall+0x1f
 --- syscall (131, FreeBSD ELF32, flock), eip = 0x282ab5cf, esp = 0xbfbfebec, ebp = 0xbfbfec08 ---
 
 Tracing command httpd pid 817 tid 100079 td 0xc818e480
 sched_switch(c818e480,0,1) at sched_switch+0x177
 mi_switch(1,0) at mi_switch+0x270
 sleepq_switch(c8067240) at sleepq_switch+0xc1
 sleepq_wait_sig(c8067240) at sleepq_wait_sig+0x1d
 msleep(c8067240,0,168,c0765c84,0) at msleep+0x26e
 lf_setlock(c8067240,16,ffffffff,ffffffff,ea8e0cb0,...) at lf_setlock+0x181
 lf_advlock(ea8e0c90,c84cfd9c,0,0,ea8e0c7c,...) at lf_advlock+0x203
 ufs_advlock(ea8e0c90) at ufs_advlock+0x1c
 VOP_ADVLOCK_APV(c077a640,ea8e0c90) at VOP_ADVLOCK_APV+0xa9
 flock(c818e480,ea8e0d04) at flock+0x1a1
 syscall(3b,3b,3b,8635018,862e050,...) at syscall+0x25b
 Xint0x80_syscall() at Xint0x80_syscall+0x1f
 --- syscall (131, FreeBSD ELF32, flock), eip = 0x282ab5cf, esp = 0xbfbfebec, ebp = 0xbfbfec08 ---
 
 Tracing command httpd pid 816 tid 100041 td 0xc7e05600
 sched_switch(c7e05600,0,1) at sched_switch+0x177
 mi_switch(1,0) at mi_switch+0x270
 sleepq_switch(c80b0a00) at sleepq_switch+0xc1
 sleepq_wait_sig(c80b0a00) at sleepq_wait_sig+0x1d
 msleep(c80b0a00,0,168,c0765c84,0) at msleep+0x26e
 lf_setlock(c80b0a00,16,ffffffff,ffffffff,e8389cb0,...) at lf_setlock+0x181
 lf_advlock(e8389c90,c84cfd9c,0,0,e8389c7c,...) at lf_advlock+0x203
 ufs_advlock(e8389c90) at ufs_advlock+0x1c
 VOP_ADVLOCK_APV(c077a640,e8389c90) at VOP_ADVLOCK_APV+0xa9
 flock(c7e05600,e8389d04) at flock+0x1a1
 syscall(3b,3b,3b,8635018,862e050,...) at syscall+0x25b
 Xint0x80_syscall() at Xint0x80_syscall+0x1f
 --- syscall (131, FreeBSD ELF32, flock), eip = 0x282ab5cf, esp = 0xbfbfebec, ebp = 0xbfbfec08 ---
 
 Tracing command httpd pid 815 tid 100075 td 0xc8105780
 sched_switch(c8105780,0,1) at sched_switch+0x177
 mi_switch(1,0) at mi_switch+0x270
 sleepq_switch(c815b580) at sleepq_switch+0xc1
 sleepq_wait_sig(c815b580) at sleepq_wait_sig+0x1d
 msleep(c815b580,0,168,c0765c84,0) at msleep+0x26e
 lf_setlock(c815b580,16,ffffffff,ffffffff,ea872cb0,...) at lf_setlock+0x181
 lf_advlock(ea872c90,c84cfd9c,0,0,ea872c7c,...) at lf_advlock+0x203
 ufs_advlock(ea872c90) at ufs_advlock+0x1c
 VOP_ADVLOCK_APV(c077a640,ea872c90) at VOP_ADVLOCK_APV+0xa9
 flock(c8105780,ea872d04) at flock+0x1a1
 syscall(3b,3b,3b,8635018,862e050,...) at syscall+0x25b
 Xint0x80_syscall() at Xint0x80_syscall+0x1f
 --- syscall (131, FreeBSD ELF32, flock), eip = 0x282ab5cf, esp = 0xbfbfebec, ebp = 0xbfbfec08 ---
 
 Tracing command httpd pid 814 tid 100078 td 0xc818e600
 sched_switch(c818e600,0,1) at sched_switch+0x177
 mi_switch(1,0) at mi_switch+0x270
 sleepq_switch(c8067180) at sleepq_switch+0xc1
 sleepq_wait_sig(c8067180) at sleepq_wait_sig+0x1d
 msleep(c8067180,0,168,c0765c84,0) at msleep+0x26e
 lf_setlock(c8067180,16,ffffffff,ffffffff,ea8e3cb0,...) at lf_setlock+0x181
 lf_advlock(ea8e3c90,c84cfd9c,0,0,ea8e3c7c,...) at lf_advlock+0x203
 ufs_advlock(ea8e3c90) at ufs_advlock+0x1c
 VOP_ADVLOCK_APV(c077a640,ea8e3c90) at VOP_ADVLOCK_APV+0xa9
 flock(c818e600,ea8e3d04) at flock+0x1a1
 syscall(3b,3b,3b,8635018,862e050,...) at syscall+0x25b
 Xint0x80_syscall() at Xint0x80_syscall+0x1f
 --- syscall (131, FreeBSD ELF32, flock), eip = 0x282ab5cf, esp = 0xbfbfebec, ebp = 0xbfbfec08 ---
 
 Tracing command httpd pid 813 tid 100095 td 0xc8195600
 sched_switch(c8195600,0,1) at sched_switch+0x177
 mi_switch(1,0) at mi_switch+0x270
 sleepq_switch(c80b0a40) at sleepq_switch+0xc1
 sleepq_wait_sig(c80b0a40) at sleepq_wait_sig+0x1d
 msleep(c80b0a40,0,168,c0765c84,0) at msleep+0x26e
 lf_setlock(c80b0a40,16,ffffffff,ffffffff,ea91fcb0,...) at lf_setlock+0x181
 lf_advlock(ea91fc90,c84cfd9c,0,0,ea91fc7c,...) at lf_advlock+0x203
 ufs_advlock(ea91fc90) at ufs_advlock+0x1c
 VOP_ADVLOCK_APV(c077a640,ea91fc90) at VOP_ADVLOCK_APV+0xa9
 flock(c8195600,ea91fd04) at flock+0x1a1
 syscall(3b,3b,3b,8635018,862e050,...) at syscall+0x25b
 Xint0x80_syscall() at Xint0x80_syscall+0x1f
 --- syscall (131, FreeBSD ELF32, flock), eip = 0x282ab5cf, esp = 0xbfbfebec, ebp = 0xbfbfec08 ---
 
 Tracing command httpd pid 812 tid 100120 td 0xc8195a80
 sched_switch(c8195a80,0,1) at sched_switch+0x177
 mi_switch(1,0) at mi_switch+0x270
 sleepq_switch(c815b6c0) at sleepq_switch+0xc1
 sleepq_wait_sig(c815b6c0) at sleepq_wait_sig+0x1d
 msleep(c815b6c0,0,168,c0765c84,0) at msleep+0x26e
 lf_setlock(c815b6c0,16,ffffffff,ffffffff,ea928cb0,...) at lf_setlock+0x181
 lf_advlock(ea928c90,c84cfd9c,0,0,ea928c7c,...) at lf_advlock+0x203
 ufs_advlock(ea928c90) at ufs_advlock+0x1c
 VOP_ADVLOCK_APV(c077a640,ea928c90) at VOP_ADVLOCK_APV+0xa9
 flock(c8195a80,ea928d04) at flock+0x1a1
 syscall(3b,3b,3b,8635018,862e050,...) at syscall+0x25b
 Xint0x80_syscall() at Xint0x80_syscall+0x1f
 --- syscall (131, FreeBSD ELF32, flock), eip = 0x282ab5cf, esp = 0xbfbfebec, ebp = 0xbfbfec08 ---
 
 Tracing command httpd pid 811 tid 100051 td 0xc8087480
 sched_switch(c8087480,0,1) at sched_switch+0x177
 mi_switch(1,0) at mi_switch+0x270
 sleepq_switch(c8066e00) at sleepq_switch+0xc1
 sleepq_wait_sig(c8066e00) at sleepq_wait_sig+0x1d
 msleep(c8066e00,0,168,c0765c84,0) at msleep+0x26e
 lf_setlock(c8066e00,16,ffffffff,ffffffff,e8644cb0,...) at lf_setlock+0x181
 lf_advlock(e8644c90,c84cfd9c,0,0,e8644c7c,...) at lf_advlock+0x203
 ufs_advlock(e8644c90) at ufs_advlock+0x1c
 VOP_ADVLOCK_APV(c077a640,e8644c90) at VOP_ADVLOCK_APV+0xa9
 flock(c8087480,e8644d04) at flock+0x1a1
 syscall(3b,3b,3b,8635018,862e050,...) at syscall+0x25b
 Xint0x80_syscall() at Xint0x80_syscall+0x1f
 --- syscall (131, FreeBSD ELF32, flock), eip = 0x282ab5cf, esp = 0xbfbfebec, ebp = 0xbfbfec08 ---
 
 Tracing command httpd pid 810 tid 100117 td 0xc82e8000
 sched_switch(c82e8000,0,1) at sched_switch+0x177
 mi_switch(1,0) at mi_switch+0x270
 sleepq_switch(c80b0a80) at sleepq_switch+0xc1
 sleepq_wait_sig(c80b0a80) at sleepq_wait_sig+0x1d
 msleep(c80b0a80,0,168,c0765c84,0) at msleep+0x26e
 lf_setlock(c80b0a80,16,ffffffff,ffffffff,ea94bcb0,...) at lf_setlock+0x181
 lf_advlock(ea94bc90,c84cfd9c,0,0,ea94bc7c,...) at lf_advlock+0x203
 ufs_advlock(ea94bc90) at ufs_advlock+0x1c
 VOP_ADVLOCK_APV(c077a640,ea94bc90) at VOP_ADVLOCK_APV+0xa9
 flock(c82e8000,ea94bd04) at flock+0x1a1
 syscall(3b,3b,3b,8635018,862e050,...) at syscall+0x25b
 Xint0x80_syscall() at Xint0x80_syscall+0x1f
 --- syscall (131, FreeBSD ELF32, flock), eip = 0x282ab5cf, esp = 0xbfbfebec, ebp = 0xbfbfec08 ---
 
 Tracing command httpd pid 809 tid 100052 td 0xc8087300
 sched_switch(c8087300,0,1) at sched_switch+0x177
 mi_switch(1,0) at mi_switch+0x270
 sleepq_switch(c815b8c0) at sleepq_switch+0xc1
 sleepq_wait_sig(c815b8c0) at sleepq_wait_sig+0x1d
 msleep(c815b8c0,0,168,c0765c84,0) at msleep+0x26e
 lf_setlock(c815b8c0,16,ffffffff,ffffffff,e8641cb0,...) at lf_setlock+0x181
 lf_advlock(e8641c90,c84cfd9c,0,0,e8641c7c,...) at lf_advlock+0x203
 ufs_advlock(e8641c90) at ufs_advlock+0x1c
 VOP_ADVLOCK_APV(c077a640,e8641c90) at VOP_ADVLOCK_APV+0xa9
 flock(c8087300,e8641d04) at flock+0x1a1
 syscall(3b,3b,3b,8635018,862e050,...) at syscall+0x25b
 Xint0x80_syscall() at Xint0x80_syscall+0x1f
 --- syscall (131, FreeBSD ELF32, flock), eip = 0x282ab5cf, esp = 0xbfbfebec, ebp = 0xbfbfec08 ---
 
 Tracing command httpd pid 808 tid 100038 td 0xc7e05a80
 sched_switch(c7e05a80,0,1) at sched_switch+0x177
 mi_switch(1,0) at mi_switch+0x270
 sleepq_switch(c8066c80) at sleepq_switch+0xc1
 sleepq_wait_sig(c8066c80) at sleepq_wait_sig+0x1d
 msleep(c8066c80,0,168,c0765c84,0) at msleep+0x26e
 lf_setlock(c8066c80,16,ffffffff,ffffffff,e8392cb0,...) at lf_setlock+0x181
 lf_advlock(e8392c90,c84cfd9c,0,0,e8392c7c,...) at lf_advlock+0x203
 ufs_advlock(e8392c90) at ufs_advlock+0x1c
 VOP_ADVLOCK_APV(c077a640,e8392c90) at VOP_ADVLOCK_APV+0xa9
 flock(c7e05a80,e8392d04) at flock+0x1a1
 syscall(3b,3b,3b,8635018,862e050,...) at syscall+0x25b
 Xint0x80_syscall() at Xint0x80_syscall+0x1f
 --- syscall (131, FreeBSD ELF32, flock), eip = 0x282ab5cf, esp = 0xbfbfebec, ebp = 0xbfbfec08 ---
 
 Tracing command httpd pid 807 tid 100070 td 0xc815e000
 sched_switch(c815e000,0,1) at sched_switch+0x177
 mi_switch(1,0) at mi_switch+0x270
 sleepq_switch(c80b0ac0) at sleepq_switch+0xc1
 sleepq_wait_sig(c80b0ac0) at sleepq_wait_sig+0x1d
 msleep(c80b0ac0,0,168,c0765c84,0) at msleep+0x26e
 lf_setlock(c80b0ac0,16,ffffffff,ffffffff,ea88ecb0,...) at lf_setlock+0x181
 lf_advlock(ea88ec90,c84cfd9c,0,0,ea88ec7c,...) at lf_advlock+0x203
 ufs_advlock(ea88ec90) at ufs_advlock+0x1c
 VOP_ADVLOCK_APV(c077a640,ea88ec90) at VOP_ADVLOCK_APV+0xa9
 flock(c815e000,ea88ed04) at flock+0x1a1
 syscall(3b,3b,3b,8635018,862e050,...) at syscall+0x25b
 Xint0x80_syscall() at Xint0x80_syscall+0x1f
 --- syscall (131, FreeBSD ELF32, flock), eip = 0x282ab5cf, esp = 0xbfbfebec, ebp = 0xbfbfec08 ---
 
 Tracing command httpd pid 806 tid 100077 td 0xc818e780
 sched_switch(c818e780,0,1) at sched_switch+0x177
 mi_switch(1,0) at mi_switch+0x270
 sleepq_switch(c815a440) at sleepq_switch+0xc1
 sleepq_wait_sig(c815a440) at sleepq_wait_sig+0x1d
 msleep(c815a440,0,168,c0765c84,0) at msleep+0x26e
 lf_setlock(c815a440,16,ffffffff,ffffffff,ea8e6cb0,...) at lf_setlock+0x181
 lf_advlock(ea8e6c90,c84cfd9c,0,0,ea8e6c7c,...) at lf_advlock+0x203
 ufs_advlock(ea8e6c90) at ufs_advlock+0x1c
 VOP_ADVLOCK_APV(c077a640,ea8e6c90) at VOP_ADVLOCK_APV+0xa9
 flock(c818e780,ea8e6d04) at flock+0x1a1
 syscall(3b,3b,3b,8635018,862e050,...) at syscall+0x25b
 Xint0x80_syscall() at Xint0x80_syscall+0x1f
 --- syscall (131, FreeBSD ELF32, flock), eip = 0x282ab5cf, esp = 0xbfbfebec, ebp = 0xbfbfec08 ---
 
 Tracing command httpd pid 805 tid 100083 td 0xc818ad80
 sched_switch(c818ad80,0,1) at sched_switch+0x177
 mi_switch(1,0) at mi_switch+0x270
 sleepq_switch(c815a000) at sleepq_switch+0xc1
 sleepq_wait_sig(c815a000) at sleepq_wait_sig+0x1d
 msleep(c815a000,0,168,c0765c84,0) at msleep+0x26e
 lf_setlock(c815a000,16,ffffffff,ffffffff,ea8d4cb0,...) at lf_setlock+0x181
 lf_advlock(ea8d4c90,c84cfd9c,0,0,ea8d4c7c,...) at lf_advlock+0x203
 ufs_advlock(ea8d4c90) at ufs_advlock+0x1c
 VOP_ADVLOCK_APV(c077a640,ea8d4c90) at VOP_ADVLOCK_APV+0xa9
 flock(c818ad80,ea8d4d04) at flock+0x1a1
 syscall(3b,3b,3b,8635018,862e050,...) at syscall+0x25b
 Xint0x80_syscall() at Xint0x80_syscall+0x1f
 --- syscall (131, FreeBSD ELF32, flock), eip = 0x282ab5cf, esp = 0xbfbfebec, ebp = 0xbfbfec08 ---
 
 Tracing command httpd pid 804 tid 100067 td 0xc815e480
 sched_switch(c815e480,0,1) at sched_switch+0x177
 mi_switch(1,0) at mi_switch+0x270
 sleepq_switch(c80b0b00) at sleepq_switch+0xc1
 sleepq_wait_sig(c80b0b00) at sleepq_wait_sig+0x1d
 msleep(c80b0b00,0,168,c0765c84,0) at msleep+0x26e
 lf_setlock(c80b0b00,16,ffffffff,ffffffff,ea897cb0,...) at lf_setlock+0x181
 lf_advlock(ea897c90,c84cfd9c,0,0,ea897c7c,...) at lf_advlock+0x203
 ufs_advlock(ea897c90) at ufs_advlock+0x1c
 VOP_ADVLOCK_APV(c077a640,ea897c90) at VOP_ADVLOCK_APV+0xa9
 flock(c815e480,ea897d04) at flock+0x1a1
 syscall(3b,3b,3b,8635018,862e050,...) at syscall+0x25b
 Xint0x80_syscall() at Xint0x80_syscall+0x1f
 --- syscall (131, FreeBSD ELF32, flock), eip = 0x282ab5cf, esp = 0xbfbfebec, ebp = 0xbfbfec08 ---
 
 Tracing command httpd pid 803 tid 100112 td 0xc82e8780
 sched_switch(c82e8780,0,1) at sched_switch+0x177
 mi_switch(1,0) at mi_switch+0x270
 sleepq_switch(c8066c40) at sleepq_switch+0xc1
 sleepq_wait_sig(c8066c40) at sleepq_wait_sig+0x1d
 msleep(c8066c40,0,168,c0765c84,0) at msleep+0x26e
 lf_setlock(c8066c40,16,ffffffff,ffffffff,ea95acb0,...) at lf_setlock+0x181
 lf_advlock(ea95ac90,c84cfd9c,0,0,ea95ac7c,...) at lf_advlock+0x203
 ufs_advlock(ea95ac90) at ufs_advlock+0x1c
 VOP_ADVLOCK_APV(c077a640,ea95ac90) at VOP_ADVLOCK_APV+0xa9
 flock(c82e8780,ea95ad04) at flock+0x1a1
 syscall(3b,3b,3b,8635018,862e050,...) at syscall+0x25b
 Xint0x80_syscall() at Xint0x80_syscall+0x1f
 --- syscall (131, FreeBSD ELF32, flock), eip = 0x282ab5cf, esp = 0xbfbfebec, ebp = 0xbfbfec08 ---
 
 Tracing command httpd pid 802 tid 100088 td 0xc818a600
 sched_switch(c818a600,0,1) at sched_switch+0x177
 mi_switch(1,0) at mi_switch+0x270
 sleepq_switch(c815bbc0) at sleepq_switch+0xc1
 sleepq_wait_sig(c815bbc0) at sleepq_wait_sig+0x1d
 msleep(c815bbc0,0,168,c0765c84,0) at msleep+0x26e
 lf_setlock(c815bbc0,16,ffffffff,ffffffff,ea8c5cb0,...) at lf_setlock+0x181
 lf_advlock(ea8c5c90,c84cfd9c,0,0,ea8c5c7c,...) at lf_advlock+0x203
 ufs_advlock(ea8c5c90) at ufs_advlock+0x1c
 VOP_ADVLOCK_APV(c077a640,ea8c5c90) at VOP_ADVLOCK_APV+0xa9
 flock(c818a600,ea8c5d04) at flock+0x1a1
 syscall(3b,3b,3b,8635018,862e050,...) at syscall+0x25b
 Xint0x80_syscall() at Xint0x80_syscall+0x1f
 --- syscall (131, FreeBSD ELF32, flock), eip = 0x282ab5cf, esp = 0xbfbfebec, ebp = 0xbfbfec08 ---
 
 Tracing command httpd pid 801 tid 100082 td 0xc818e000
 sched_switch(c818e000,0,1) at sched_switch+0x177
 mi_switch(1,0) at mi_switch+0x270
 sleepq_switch(c07e5744) at sleepq_switch+0xc1
 sleepq_wait_sig(c07e5744,246,c0797aa0,c0728ac4,3b7,...) at sleepq_wait_sig+0x1d
 cv_wait_sig(c07e5744,c07e5720) at cv_wait_sig+0x18b
 poll(c818e000,ea8d7d04) at poll+0x3b9
 syscall(3b,3b,3b,2,28,...) at syscall+0x25b
 Xint0x80_syscall() at Xint0x80_syscall+0x1f
 --- syscall (209, FreeBSD ELF32, poll), eip = 0x282c3a87, esp = 0xbfbfebcc, ebp = 0xbfbfec18 ---
 
 Tracing command getty pid 800 tid 100093 td 0xc815ed80
 sched_switch(c815ed80,0,1) at sched_switch+0x177
 mi_switch(1,0) at mi_switch+0x270
 sleepq_switch(c7e95410) at sleepq_switch+0xc1
 sleepq_wait_sig(c7e95410) at sleepq_wait_sig+0x1d
 msleep(c7e95410,0,159,c072a5af,0) at msleep+0x26e
 ttysleep(c7e95400,c7e95410,159,c072a5af,0,...) at ttysleep+0x21
 ttread(c7e95400,ea8a9cbc,0) at ttread+0x48f
 ttyread(c7e8b800,ea8a9cbc,0,c0797ae0,0,...) at ttyread+0x2f
 giant_read(c7e8b800,ea8a9cbc,0) at giant_read+0x2d
 devfs_read_f(c83d6048,ea8a9cbc,c7ce0780,0,c815ed80) at devfs_read_f+0x62
 dofileread(c815ed80,0,c83d6048,ea8a9cbc,ffffffff,...) at dofileread+0x85
 kern_readv(c815ed80,0,ea8a9cbc,bfbfee1f,1,...) at kern_readv+0x36
 read(c815ed80,ea8a9d04) at read+0x45
 syscall(3b,3b,3b,0,804e580,...) at syscall+0x25b
 Xint0x80_syscall() at Xint0x80_syscall+0x1f
 --- syscall (3, FreeBSD ELF32, read), eip = 0x28148953, esp = 0xbfbfedfc, ebp = 0xbfbfee28 ---
 
 Tracing command getty pid 799 tid 100123 td 0xc8366900
 sched_switch(c8366900,0,1) at sched_switch+0x177
 mi_switch(1,0) at mi_switch+0x270
 sleepq_switch(c7ea7c10) at sleepq_switch+0xc1
 sleepq_wait_sig(c7ea7c10) at sleepq_wait_sig+0x1d
 msleep(c7ea7c10,0,159,c072a5af,0) at msleep+0x26e
 ttysleep(c7ea7c00,c7ea7c10,159,c072a5af,0,...) at ttysleep+0x21
 ttread(c7ea7c00,ea9c0cbc,0) at ttread+0x48f
 ttyread(c7e8c800,ea9c0cbc,0,ea9c0c14,c054002d,...) at ttyread+0x2f
 scread(c7e8c800,ea9c0cbc,0,c0797ae0,0,...) at scread+0x22
 giant_read(c7e8c800,ea9c0cbc,0) at giant_read+0x2d
 devfs_read_f(c811c2d0,ea9c0cbc,c7ce0780,0,c8366900) at devfs_read_f+0x62
 dofileread(c8366900,0,c811c2d0,ea9c0cbc,ffffffff,...) at dofileread+0x85
 kern_readv(c8366900,0,ea9c0cbc,bfbfee1f,1,...) at kern_readv+0x36
 read(c8366900,ea9c0d04) at read+0x45
 syscall(3b,3b,3b,0,804e580,...) at syscall+0x25b
 Xint0x80_syscall() at Xint0x80_syscall+0x1f
 --- syscall (3, FreeBSD ELF32, read), eip = 0x28148953, esp = 0xbfbfedfc, ebp = 0xbfbfee28 ---
 
 Tracing command getty pid 798 tid 100103 td 0xc8087c00
 sched_switch(c8087c00,0,1) at sched_switch+0x177
 mi_switch(1,0) at mi_switch+0x270
 sleepq_switch(c7ea6c10) at sleepq_switch+0xc1
 sleepq_wait_sig(c7ea6c10) at sleepq_wait_sig+0x1d
 msleep(c7ea6c10,0,159,c072a5af,0) at msleep+0x26e
 ttysleep(c7ea6c00,c7ea6c10,159,c072a5af,0,...) at ttysleep+0x21
 ttread(c7ea6c00,e8653cbc,0) at ttread+0x48f
 ttyread(c7e8c700,e8653cbc,0,e8653c14,c054002d,...) at ttyread+0x2f
 scread(c7e8c700,e8653cbc,0,c0797ae0,0,...) at scread+0x22
 giant_read(c7e8c700,e8653cbc,0) at giant_read+0x2d
 devfs_read_f(c8111e10,e8653cbc,c7ce0780,0,c8087c00) at devfs_read_f+0x62
 dofileread(c8087c00,0,c8111e10,e8653cbc,ffffffff,...) at dofileread+0x85
 kern_readv(c8087c00,0,e8653cbc,bfbfee1f,1,...) at kern_readv+0x36
 read(c8087c00,e8653d04) at read+0x45
 syscall(3b,3b,3b,0,804e580,...) at syscall+0x25b
 Xint0x80_syscall() at Xint0x80_syscall+0x1f
 --- syscall (3, FreeBSD ELF32, read), eip = 0x28148953, esp = 0xbfbfedfc, ebp = 0xbfbfee28 ---
 
 Tracing command getty pid 797 tid 100113 td 0xc82e8600
 sched_switch(c82e8600,0,1) at sched_switch+0x177
 mi_switch(1,0) at mi_switch+0x270
 sleepq_switch(c7ea5810) at sleepq_switch+0xc1
 sleepq_wait_sig(c7ea5810) at sleepq_wait_sig+0x1d
 msleep(c7ea5810,0,159,c072a5af,0) at msleep+0x26e
 ttysleep(c7ea5800,c7ea5810,159,c072a5af,0,...) at ttysleep+0x21
 ttread(c7ea5800,ea957cbc,0) at ttread+0x48f
 ttyread(c7e8c600,ea957cbc,0,ea957c14,c054002d,...) at ttyread+0x2f
 scread(c7e8c600,ea957cbc,0,c0797ae0,0,...) at scread+0x22
 giant_read(c7e8c600,ea957cbc,0) at giant_read+0x2d
 devfs_read_f(c811c558,ea957cbc,c7ce0780,0,c82e8600) at devfs_read_f+0x62
 dofileread(c82e8600,0,c811c558,ea957cbc,ffffffff,...) at dofileread+0x85
 kern_readv(c82e8600,0,ea957cbc,bfbfee1f,1,...) at kern_readv+0x36
 read(c82e8600,ea957d04) at read+0x45
 syscall(3b,3b,3b,0,804e580,...) at syscall+0x25b
 Xint0x80_syscall() at Xint0x80_syscall+0x1f
 --- syscall (3, FreeBSD ELF32, read), eip = 0x28148953, esp = 0xbfbfedfc, ebp = 0xbfbfee28 ---
 
 Tracing command getty pid 796 tid 100081 td 0xc818e180
 sched_switch(c818e180,0,1) at sched_switch+0x177
 mi_switch(1,0) at mi_switch+0x270
 sleepq_switch(c7ea5c10) at sleepq_switch+0xc1
 sleepq_wait_sig(c7ea5c10) at sleepq_wait_sig+0x1d
 msleep(c7ea5c10,0,159,c072a5af,0) at msleep+0x26e
 ttysleep(c7ea5c00,c7ea5c10,159,c072a5af,0,...) at ttysleep+0x21
 ttread(c7ea5c00,ea8dacbc,0) at ttread+0x48f
 ttyread(c7e8c500,ea8dacbc,0,ea8dac14,c054002d,...) at ttyread+0x2f
 scread(c7e8c500,ea8dacbc,0,c0797ae0,0,...) at scread+0x22
 giant_read(c7e8c500,ea8dacbc,0) at giant_read+0x2d
 devfs_read_f(c8108ca8,ea8dacbc,c7ce0780,0,c818e180) at devfs_read_f+0x62
 dofileread(c818e180,0,c8108ca8,ea8dacbc,ffffffff,...) at dofileread+0x85
 kern_readv(c818e180,0,ea8dacbc,bfbfee1f,1,...) at kern_readv+0x36
 read(c818e180,ea8dad04) at read+0x45
 syscall(3b,3b,3b,0,804e580,...) at syscall+0x25b
 Xint0x80_syscall() at Xint0x80_syscall+0x1f
 --- syscall (3, FreeBSD ELF32, read), eip = 0x28148953, esp = 0xbfbfedfc, ebp = 0xbfbfee28 ---
 
 Tracing command getty pid 795 tid 100116 td 0xc82e8180
 sched_switch(c82e8180,0,1) at sched_switch+0x177
 mi_switch(1,0) at mi_switch+0x270
 sleepq_switch(c7ea8410) at sleepq_switch+0xc1
 sleepq_wait_sig(c7ea8410) at sleepq_wait_sig+0x1d
 msleep(c7ea8410,0,159,c072a5af,0) at msleep+0x26e
 ttysleep(c7ea8400,c7ea8410,159,c072a5af,0,...) at ttysleep+0x21
 ttread(c7ea8400,ea94ecbc,0) at ttread+0x48f
 ttyread(c7e8c400,ea94ecbc,0,ea94ec14,c054002d,...) at ttyread+0x2f
 scread(c7e8c400,ea94ecbc,0,c0797ae0,0,...) at scread+0x22
 giant_read(c7e8c400,ea94ecbc,0) at giant_read+0x2d
 devfs_read_f(c811c480,ea94ecbc,c7ce0780,0,c82e8180) at devfs_read_f+0x62
 dofileread(c82e8180,0,c811c480,ea94ecbc,ffffffff,...) at dofileread+0x85
 kern_readv(c82e8180,0,ea94ecbc,bfbfee1f,1,...) at kern_readv+0x36
 read(c82e8180,ea94ed04) at read+0x45
 syscall(3b,3b,3b,0,804e580,...) at syscall+0x25b
 Xint0x80_syscall() at Xint0x80_syscall+0x1f
 --- syscall (3, FreeBSD ELF32, read), eip = 0x28148953, esp = 0xbfbfedfc, ebp = 0xbfbfee28 ---
 
 Tracing command getty pid 794 tid 100072 td 0xc8105c00
 sched_switch(c8105c00,0,1) at sched_switch+0x177
 mi_switch(1,0) at mi_switch+0x270
 sleepq_switch(c7e9f410) at sleepq_switch+0xc1
 sleepq_wait_sig(c7e9f410) at sleepq_wait_sig+0x1d
 msleep(c7e9f410,0,159,c072a5af,0) at msleep+0x26e
 ttysleep(c7e9f400,c7e9f410,159,c072a5af,0,...) at ttysleep+0x21
 ttread(c7e9f400,ea87bcbc,0) at ttread+0x48f
 ttyread(c7e8c300,ea87bcbc,0,ea87bc14,c054002d,...) at ttyread+0x2f
 scread(c7e8c300,ea87bcbc,0,c0797ae0,0,...) at scread+0x22
 giant_read(c7e8c300,ea87bcbc,0) at giant_read+0x2d
 devfs_read_f(c8111c60,ea87bcbc,c7ce0780,0,c8105c00) at devfs_read_f+0x62
 dofileread(c8105c00,0,c8111c60,ea87bcbc,ffffffff,...) at dofileread+0x85
 kern_readv(c8105c00,0,ea87bcbc,bfbfee1f,1,...) at kern_readv+0x36
 read(c8105c00,ea87bd04) at read+0x45
 syscall(3b,3b,3b,0,804e580,...) at syscall+0x25b
 Xint0x80_syscall() at Xint0x80_syscall+0x1f
 --- syscall (3, FreeBSD ELF32, read), eip = 0x28148953, esp = 0xbfbfedfc, ebp = 0xbfbfee28 ---
 
 Tracing command getty pid 793 tid 100122 td 0xc8195780
 sched_switch(c8195780,0,1) at sched_switch+0x177
 mi_switch(1,0) at mi_switch+0x270
 sleepq_switch(c7ea7010) at sleepq_switch+0xc1
 sleepq_wait_sig(c7ea7010) at sleepq_wait_sig+0x1d
 msleep(c7ea7010,0,159,c072a5af,0) at msleep+0x26e
 ttysleep(c7ea7000,c7ea7010,159,c072a5af,0,...) at ttysleep+0x21
 ttread(c7ea7000,ea922cbc,0) at ttread+0x48f
 ttyread(c7e8c200,ea922cbc,0,ea922c14,c054002d,...) at ttyread+0x2f
 scread(c7e8c200,ea922cbc,0,c0797ae0,0,...) at scread+0x22
 giant_read(c7e8c200,ea922cbc,0) at giant_read+0x2d
 devfs_read_f(c811cd80,ea922cbc,c7ce0780,0,c8195780) at devfs_read_f+0x62
 dofileread(c8195780,0,c811cd80,ea922cbc,ffffffff,...) at dofileread+0x85
 kern_readv(c8195780,0,ea922cbc,bfbfee1f,1,...) at kern_readv+0x36
 read(c8195780,ea922d04) at read+0x45
 syscall(3b,3b,3b,0,804e580,...) at syscall+0x25b
 Xint0x80_syscall() at Xint0x80_syscall+0x1f
 --- syscall (3, FreeBSD ELF32, read), eip = 0x28148953, esp = 0xbfbfedfc, ebp = 0xbfbfee28 ---
 
 Tracing command getty pid 792 tid 100066 td 0xc815e600
 sched_switch(c815e600,0,1) at sched_switch+0x177
 mi_switch(1,0) at mi_switch+0x270
 sleepq_switch(c7e9bc10) at sleepq_switch+0xc1
 sleepq_wait_sig(c7e9bc10) at sleepq_wait_sig+0x1d
 msleep(c7e9bc10,0,159,c072a5af,0) at msleep+0x26e
 ttysleep(c7e9bc00,c7e9bc10,159,c072a5af,0,...) at ttysleep+0x21
 ttread(c7e9bc00,ea89acbc,0) at ttread+0x48f
 ttyread(c7e8c100,ea89acbc,0,ea89ac14,c054002d,...) at ttyread+0x2f
 scread(c7e8c100,ea89acbc,0,c0797ae0,0,...) at scread+0x22
 giant_read(c7e8c100,ea89acbc,0) at giant_read+0x2d
 devfs_read_f(c8108240,ea89acbc,c7ce0780,0,c815e600) at devfs_read_f+0x62
 dofileread(c815e600,0,c8108240,ea89acbc,ffffffff,...) at dofileread+0x85
 kern_readv(c815e600,0,ea89acbc,bfbfee1f,1,...) at kern_readv+0x36
 read(c815e600,ea89ad04) at read+0x45
 syscall(3b,3b,3b,0,804e580,...) at syscall+0x25b
 Xint0x80_syscall() at Xint0x80_syscall+0x1f
 --- syscall (3, FreeBSD ELF32, read), eip = 0x28148953, esp = 0xbfbfedfc, ebp = 0xbfbfee28 ---
 
 Tracing command inetd pid 778 tid 100057 td 0xc8102d80
 sched_switch(c8102d80,0,1) at sched_switch+0x177
 mi_switch(1,0) at mi_switch+0x270
 sleepq_switch(c07e5744) at sleepq_switch+0xc1
 sleepq_wait_sig(c07e5744,246,c0797aa0,c0728ac4,303,...) at sleepq_wait_sig+0x1d
 cv_wait_sig(c07e5744,c07e5720) at cv_wait_sig+0x18b
 kern_select(c8102d80,7,bfbfe8a0,0,0,...) at kern_select+0x4bf
 select(c8102d80,ea860d04) at select+0x44
 syscall(3b,3b,3b,bfbfe8a0,bfbfee30,...) at syscall+0x25b
 Xint0x80_syscall() at Xint0x80_syscall+0x1f
 --- syscall (93, FreeBSD ELF32, select), eip = 0x2815a8d3, esp = 0xbfbfdb7c, ebp = 0xbfbfee68 ---
 
 Tracing command httpd pid 758 tid 100054 td 0xc8087000
 sched_switch(c8087000,0,1) at sched_switch+0x177
 mi_switch(1,0) at mi_switch+0x270
 sleepq_switch(c07e5744) at sleepq_switch+0xc1
 sleepq_timedwait_sig(c07e5744,c07e5744,3e9,e863bb40,246,...) at sleepq_timedwait_sig+0x1e
 cv_timedwait_sig(c07e5744,c07e5720,3e9) at cv_timedwait_sig+0x194
 kern_select(c8087000,0,0,0,0,...) at kern_select+0x4a9
 select(c8087000,e863bd04) at select+0x44
 syscall(3b,3b,3b,0,f4240,...) at syscall+0x25b
 Xint0x80_syscall() at Xint0x80_syscall+0x1f
 --- syscall (93, FreeBSD ELF32, select), eip = 0x2830c8d3, esp = 0xbfbfec3c, ebp = 0xbfbfec78 ---
 
 Tracing command mysqld pid 757 tid 100128 td 0xc82e9480
 sched_switch(c82e9480,0,1) at sched_switch+0x177
 mi_switch(1,0) at mi_switch+0x270
 sleepq_switch(c7e066f4) at sleepq_switch+0xc1
 sleepq_timedwait_sig(c7e066f4) at sleepq_timedwait_sig+0x1e
 msleep(c7e066f4,c8364cf8,168,c0721153,290,...) at msleep+0x244
 kse_release(c82e9480,ea972d04) at kse_release+0x1df
 syscall(3b,3b,3b,8530900,0,...) at syscall+0x25b
 Xint0x80_syscall() at Xint0x80_syscall+0x1f
 --- syscall (383, FreeBSD ELF32, kse_release), eip = 0x284734f7, esp = 0x13c76f28, ebp = 0x13c76f64 ---
 
 Tracing command mysqld pid 757 tid 100129 td 0xc818ed80
 sched_switch(c818ed80,0,1) at sched_switch+0x177
 mi_switch(1,0) at mi_switch+0x270
 sleepq_switch(c7e066f4) at sleepq_switch+0xc1
 sleepq_timedwait_sig(c7e066f4) at sleepq_timedwait_sig+0x1e
 msleep(c7e066f4,c8364cf8,168,c0721153,291,...) at msleep+0x244
 kse_release(c818ed80,ea8f2d04) at kse_release+0x1df
 syscall(3b,3b,3b,8530a00,0,...) at syscall+0x25b
 Xint0x80_syscall() at Xint0x80_syscall+0x1f
 --- syscall (383, FreeBSD ELF32, kse_release), eip = 0x284734f7, esp = 0x13c7af28, ebp = 0x13c7af64 ---
 
 Tracing command mysqld pid 757 tid 100132 td 0xc82e9a80
 sched_switch(c82e9a80,0,1) at sched_switch+0x177
 mi_switch(1,0) at mi_switch+0x270
 sleepq_switch(c7e066f4) at sleepq_switch+0xc1
 sleepq_timedwait_sig(c7e066f4) at sleepq_timedwait_sig+0x1e
 msleep(c7e066f4,c8364cf8,168,c0721153,290,...) at msleep+0x244
 kse_release(c82e9a80,ea97ed04) at kse_release+0x1df
 syscall(3b,3b,3b,8530b00,0,...) at syscall+0x25b
 Xint0x80_syscall() at Xint0x80_syscall+0x1f
 --- syscall (383, FreeBSD ELF32, kse_release), eip = 0x284734f7, esp = 0x13c7ef28, ebp = 0x13c7ef64 ---
 
 Tracing command mysqld pid 757 tid 100131 td 0xc8193a80
 sched_switch(c8193a80,0,1) at sched_switch+0x177
 mi_switch(1,0) at mi_switch+0x270
 sleepq_switch(c7e066f4) at sleepq_switch+0xc1
 sleepq_timedwait_sig(c7e066f4) at sleepq_timedwait_sig+0x1e
 msleep(c7e066f4,c8364cf8,168,c0721153,290,...) at msleep+0x244
 kse_release(c8193a80,ea90ad04) at kse_release+0x1df
 syscall(3b,3b,3b,8530000,0,...) at syscall+0x25b
 Xint0x80_syscall() at Xint0x80_syscall+0x1f
 --- syscall (383, FreeBSD ELF32, kse_release), eip = 0x284734f7, esp = 0x8534f28, ebp = 0x8534f64 ---
 
 Tracing command mysqld pid 757 tid 100127 td 0xc83c5180
 sched_switch(c83c5180,c82e9480,1) at sched_switch+0x177
 mi_switch(1,c82e9480) at mi_switch+0x270
 sleepq_switch(c07e5744) at sleepq_switch+0xc1
 sleepq_wait_sig(c07e5744,246,c0797aa0,c0728ac4,303,...) at sleepq_wait_sig+0x1d
 cv_wait_sig(c07e5744,c07e5720) at cv_wait_sig+0x18b
 kern_select(c83c5180,f,bfbfebf0,0,0,...) at kern_select+0x4bf
 select(c83c5180,ea9edd04) at select+0x44
 syscall(3b,3b,3b,8536000,0,...) at syscall+0x25b
 Xint0x80_syscall() at Xint0x80_syscall+0x1f
 --- syscall (93, FreeBSD ELF32, select), eip = 0x2861b8d3, esp = 0xbfbfeacc, ebp = 0xbfbfeb08 ---
 
 Tracing command mysqld pid 757 tid 100134 td 0xc83c5780
 sched_switch(c83c5780,0,1) at sched_switch+0x177
 mi_switch(1,0) at mi_switch+0x270
 sleepq_switch(ea9f9c14) at sleepq_switch+0xc1
 sleepq_wait_sig(ea9f9c14) at sleepq_wait_sig+0x1d
 msleep(ea9f9c14,c8364cf8,168,c072114b,0,...) at msleep+0x26e
 kern_sigtimedwait(c83c5780,26005,0,0,0,ea9f9c80,0) at kern_sigtimedwait+0x30b
 sigtimedwait(c83c5780,ea9f9d04) at sigtimedwait+0x6b
 syscall(3b,3b,3b,13c83e00,bf0f5f37,...) at syscall+0x25b
 Xint0x80_syscall() at Xint0x80_syscall+0x1f
 --- syscall (345, FreeBSD ELF32, sigtimedwait), eip = 0x284735b7, esp = 0xbf0f5e2c, ebp = 0xbf0f5ee8 ---
 
 Tracing command mysqld pid 757 tid 100126 td 0xc8105480
 sched_switch(c8105480,0,1) at sched_switch+0x177
 mi_switch(1,0) at mi_switch+0x270
 sleepq_switch(c8364dc8) at sleepq_switch+0xc1
 sleepq_timedwait_sig(c8364dc8) at sleepq_timedwait_sig+0x1e
 msleep(c8364dc8,c8364cf8,168,c0721148,7531,...) at msleep+0x244
 kse_release(c8105480,ea86cd04) at kse_release+0x121
 syscall(3b,3b,3b,8530800,81,...) at syscall+0x25b
 Xint0x80_syscall() at Xint0x80_syscall+0x1f
 --- syscall (383, FreeBSD ELF32, kse_release), eip = 0x284734f7, esp = 0xbf9fef2c, ebp = 0xbf9fef88 ---
 
 Tracing command sh pid 727 tid 100111 td 0xc82e8900
 sched_switch(c82e8900,0,1) at sched_switch+0x177
 mi_switch(1,0) at mi_switch+0x270
 sleepq_switch(c82e7c90) at sleepq_switch+0xc1
 sleepq_wait_sig(c82e7c90) at sleepq_wait_sig+0x1d
 msleep(c82e7c90,c82e7cf8,15c,c0729071,0) at msleep+0x26e
 kern_wait(c82e8900,ffffffff,ea95dc84,2,0) at kern_wait+0x6bb
 wait4(c82e8900,ea95dd04) at wait4+0x2a
 syscall(3b,3b,3b,bfbfe678,2d,...) at syscall+0x25b
 Xint0x80_syscall() at Xint0x80_syscall+0x1f
 --- syscall (7, FreeBSD ELF32, wait4), eip = 0x2813d51f, esp = 0xbfbfe62c, ebp = 0xbfbfe648 ---
 
 Tracing command cron pid 724 tid 100108 td 0xc82e8d80
 sched_switch(c82e8d80,0,1) at sched_switch+0x177
 mi_switch(1,0) at mi_switch+0x270
 sleepq_switch(c07983cc) at sleepq_switch+0xc1
 sleepq_timedwait_sig(c07983cc) at sleepq_timedwait_sig+0x1e
 msleep(c07983cc,0,15c,c0724ee0,ea61,...) at msleep+0x244
 kern_nanosleep(c82e8d80,ea966ccc,ea966cc4) at kern_nanosleep+0xab
 nanosleep(c82e8d80,ea966d04) at nanosleep+0x4f
 syscall(3b,3b,3b,8050e6c,3c,...) at syscall+0x25b
 Xint0x80_syscall() at Xint0x80_syscall+0x1f
 --- syscall (240, FreeBSD ELF32, nanosleep), eip = 0x28132513, esp = 0xbfbfecdc, ebp = 0xbfbfed08 ---
 
 Tracing command snmpd pid 716 tid 100101 td 0xc8102000
 sched_switch(c8102000,0,1) at sched_switch+0x177
 mi_switch(1,0) at mi_switch+0x270
 sleepq_switch(c07e5744) at sleepq_switch+0xc1
 sleepq_timedwait_sig(c07e5744,c07e5744,1389,ea845b40,246,...) at sleepq_timedwait_sig+0x1e
 cv_timedwait_sig(c07e5744,c07e5720,1389) at cv_timedwait_sig+0x194
 kern_select(c8102000,9,bfbfed50,bfbfecd0,bfbfec50,...) at kern_select+0x4a9
 select(c8102000,ea845d04) at select+0x44
 syscall(3b,3b,3b,bfbfecd0,bfbfec38,...) at syscall+0x25b
 Xint0x80_syscall() at Xint0x80_syscall+0x1f
 --- syscall (93, FreeBSD ELF32, select), eip = 0x284598d3, esp = 0xbfbfea4c, ebp = 0xbfbfee68 ---
 
 Tracing command cron pid 705 tid 100119 td 0xc8195c00
 sched_switch(c8195c00,0,1) at sched_switch+0x177
 mi_switch(1,0) at mi_switch+0x270
 sleepq_switch(c07983cc) at sleepq_switch+0xc1
 sleepq_timedwait_sig(c07983cc) at sleepq_timedwait_sig+0x1e
 msleep(c07983cc,0,15c,c0724ee0,ea61,...) at msleep+0x244
 kern_nanosleep(c8195c00,ea92bccc,ea92bcc4) at kern_nanosleep+0xab
 nanosleep(c8195c00,ea92bd04) at nanosleep+0x4f
 syscall(3b,3b,3b,805008c,3c,...) at syscall+0x25b
 Xint0x80_syscall() at Xint0x80_syscall+0x1f
 --- syscall (240, FreeBSD ELF32, nanosleep), eip = 0x28132513, esp = 0xbfbfecdc, ebp = 0xbfbfed08 ---
 
 Tracing command sendmail pid 699 tid 100121 td 0xc8195900
 sched_switch(c8195900,0,1) at sched_switch+0x177
 mi_switch(1,0) at mi_switch+0x270
 sleepq_switch(c8365464) at sleepq_switch+0xc1
 sleepq_wait_sig(c8365464) at sleepq_wait_sig+0x1d
 msleep(c8365464,c8365498,168,c0724047,0) at msleep+0x26e
 kern_sigsuspend(c8195900,0,0,0,0,...) at kern_sigsuspend+0xa3
 sigsuspend(c8195900,ea925d04) at sigsuspend+0x33
 syscall(3b,3b,3b,0,bfbfd000,...) at syscall+0x25b
 Xint0x80_syscall() at Xint0x80_syscall+0x1f
 --- syscall (341, FreeBSD ELF32, sigsuspend), eip = 0x2827e3fb, esp = 0xbfbfcfec, ebp = 0xbfbfd018 ---
 
 Tracing command sendmail pid 693 tid 100056 td 0xc8105000
 sched_switch(c8105000,0,1) at sched_switch+0x177
 mi_switch(1,0) at mi_switch+0x270
 sleepq_switch(c07e5744) at sleepq_switch+0xc1
 sleepq_timedwait_sig(c07e5744,c07e5744,1389,ea863b40,246,...) at sleepq_timedwait_sig+0x1e
 cv_timedwait_sig(c07e5744,c07e5720,1389) at cv_timedwait_sig+0x194
 kern_select(c8105000,6,bfbfc530,0,0,...) at kern_select+0x4a9
 select(c8105000,ea863d04) at select+0x44
 syscall(3b,3b,3b,2,5,...) at syscall+0x25b
 Xint0x80_syscall() at Xint0x80_syscall+0x1f
 --- syscall (93, FreeBSD ELF32, select), eip = 0x282fc8d3, esp = 0xbfbfc44c, ebp = 0xbfbfd038 ---
 
 Tracing command sshd pid 688 tid 100059 td 0xc8102a80
 sched_switch(c8102a80,0,1) at sched_switch+0x177
 mi_switch(1,0) at mi_switch+0x270
 sleepq_switch(c07e5744) at sleepq_switch+0xc1
 sleepq_wait_sig(c07e5744,246,c0797aa0,c0728ac4,303,...) at sleepq_wait_sig+0x1d
 cv_wait_sig(c07e5744,c07e5720) at cv_wait_sig+0x18b
 kern_select(c8102a80,7,807b920,0,0,...) at kern_select+0x4bf
 select(c8102a80,ea85ad04) at select+0x44
 syscall(3b,3b,3b,1,a,...) at syscall+0x25b
 Xint0x80_syscall() at Xint0x80_syscall+0x1f
 --- syscall (93, FreeBSD ELF32, select), eip = 0x2834d8d3, esp = 0xbfbfe16c, ebp = 0xbfbfee78 ---
 
 Tracing command ntpd pid 663 tid 100069 td 0xc815e180
 sched_switch(c815e180,0,1) at sched_switch+0x177
 mi_switch(1,0) at mi_switch+0x270
 sleepq_switch(c07e5744) at sleepq_switch+0xc1
 sleepq_wait_sig(c07e5744,246,c0797aa0,c0728ac4,303,...) at sleepq_wait_sig+0x1d
 cv_wait_sig(c07e5744,c07e5720) at cv_wait_sig+0x18b
 kern_select(c815e180,a,bfbfed60,0,0,...) at kern_select+0x4bf
 select(c815e180,ea891d04) at select+0x44
 syscall(3b,3b,3b,bfbfed60,0,...) at syscall+0x25b
 Xint0x80_syscall() at Xint0x80_syscall+0x1f
 --- syscall (93, FreeBSD ELF32, select), eip = 0x2828a8d3, esp = 0xbfbfed1c, ebp = 0xbfbfedf8 ---
 
 Tracing command usbd pid 632 tid 100085 td 0xc818aa80
 sched_switch(c818aa80,0,1) at sched_switch+0x177
 mi_switch(1,0) at mi_switch+0x270
 sleepq_switch(c07e5744) at sleepq_switch+0xc1
 sleepq_timedwait_sig(c07e5744,c07e5744,7531,ea8ceb40,246,...) at sleepq_timedwait_sig+0x1e
 cv_timedwait_sig(c07e5744,c07e5720,7531) at cv_timedwait_sig+0x194
 kern_select(c818aa80,6,bfbfeda0,bfbfed20,0,...) at kern_select+0x4a9
 select(c818aa80,ea8ced04) at select+0x44
 syscall(3b,3b,3b,bfbfed20,1e,...) at syscall+0x25b
 Xint0x80_syscall() at Xint0x80_syscall+0x1f
 --- syscall (93, FreeBSD ELF32, select), eip = 0x2813c8d3, esp = 0xbfbfeccc, ebp = 0xbfbfee78 ---
 
 Tracing command accounting pid 585 tid 100053 td 0xc8087180
 sched_switch(c8087180,0,1) at sched_switch+0x177
 mi_switch(1,0) at mi_switch+0x270
 sleepq_switch(c0795f00,c0797aa0,0,c0726d95,220,...) at sleepq_switch+0xc1
 sleepq_wait(c0795f00) at sleepq_wait+0x46
 _sx_xlock_hard(c0795f00,c8087180,0,c071f3eb,215) at _sx_xlock_hard+0x195
 _sx_xlock(c0795f00,0,c071f3eb,215) at _sx_xlock+0x85
 acct_thread(0,e863ed38,0,c053d6c4,0,...) at acct_thread+0xee
 fork_exit(c053d6c4,0,e863ed38) at fork_exit+0xa0
 fork_trampoline() at fork_trampoline+0x8
 --- trap 0x1, eip = 0, esp = 0xe863ed6c, ebp = 0 ---
 
 Tracing command named pid 568 tid 100086 td 0xc818a900
 sched_switch(c818a900,0,1) at sched_switch+0x177
 mi_switch(1,0) at mi_switch+0x270
 sleepq_switch(c07e5744) at sleepq_switch+0xc1
 sleepq_timedwait_sig(c07e5744,c07e5744,7531,ea8cbb40,246,...) at sleepq_timedwait_sig+0x1e
 cv_timedwait_sig(c07e5744,c07e5720,7531) at cv_timedwait_sig+0x194
 kern_select(c818a900,1a,bfbfed60,bfbfece0,0,...) at kern_select+0x4a9
 select(c818a900,ea8cbd04) at select+0x44
 syscall(3b,3b,3b,0,0,...) at syscall+0x25b
 Xint0x80_syscall() at Xint0x80_syscall+0x1f
 --- syscall (93, FreeBSD ELF32, select), eip = 0x2835e8d3, esp = 0xbfbfec9c, ebp = 0xbfbfedf8 ---
 
 Tracing command syslogd pid 548 tid 100104 td 0xc8087a80
 sched_switch(c8087a80,0,1) at sched_switch+0x177
 mi_switch(1,0) at mi_switch+0x270
 sleepq_switch(dbfc2388,c0797aa0,0,c0726d95,220,...) at sleepq_switch+0xc1
 sleepq_wait(dbfc2388,0,c8302ba8,b7,c072e794,...) at sleepq_wait+0x46
 msleep(dbfc2388,c0797a48,50,c072c4b4,0,...) at msleep+0x27d
 acquire(e865090c,120,60000,c8087a80,0,...) at acquire+0x76
 lockmgr(dbfc2388,202122,c8302ba8,c8087a80) at lockmgr+0x44e
 getblk(c8302ae0,4,0,1800,0,...) at getblk+0x12f
 breadn(c8302ae0,4,0,1800,0,...) at breadn+0x31
 bread(c8302ae0,4,0,1800,0,e8650a2c) at bread+0x20
 ffs_balloc_ufs2(c8302ae0,1173b,0,7d,c7ce0780,...) at ffs_balloc_ufs2+0xcd3
 ffs_write(e8650c10) at ffs_write+0x2c0
 VOP_WRITE_APV(c077a640,e8650c10) at VOP_WRITE_APV+0x132
 vn_write(c8111ab0,c8593980,c7ce0780,0,c8087a80) at vn_write+0x1ea
 dofilewrite(c8087a80,c,c8111ab0,c8593980,ffffffff,...) at dofilewrite+0x7c
 kern_writev(c8087a80,c,c8593980,c8593980,0,...) at kern_writev+0x3b
 writev(c8087a80,e8650d04) at writev+0x30
 syscall(3b,3b,3b,0,bfbfd050,...) at syscall+0x25b
 Xint0x80_syscall() at Xint0x80_syscall+0x1f
 --- syscall (121, FreeBSD ELF32, writev), eip = 0x28137f1f, esp = 0xbfbfc96c, ebp = 0xbfbfd078 ---
 
 Tracing command ipmon pid 531 tid 100050 td 0xc8087600
 sched_switch(c8087600,0,1) at sched_switch+0x177
 mi_switch(1,0) at mi_switch+0x270
 sleepq_switch(c07983cc) at sleepq_switch+0xc1
 sleepq_timedwait_sig(c07983cc) at sleepq_timedwait_sig+0x1e
 msleep(c07983cc,0,15c,c0724ee0,3e9,...) at msleep+0x244
 kern_nanosleep(c8087600,e8647ccc,e8647cc4) at kern_nanosleep+0xab
 nanosleep(c8087600,e8647d04) at nanosleep+0x4f
 syscall(3b,3b,3b,2,1,...) at syscall+0x25b
 Xint0x80_syscall() at Xint0x80_syscall+0x1f
 --- syscall (240, FreeBSD ELF32, nanosleep), eip = 0x2812f513, esp = 0xbfbfcd3c, ebp = 0xbfbfcd68 ---
 
 Tracing command devd pid 435 tid 100107 td 0xc82e9000
 sched_switch(c82e9000,0,1) at sched_switch+0x177
 mi_switch(1,0) at mi_switch+0x270
 sleepq_switch(c07e5744) at sleepq_switch+0xc1
 sleepq_wait_sig(c07e5744,246,c0797aa0,c0728ac4,303,...) at sleepq_wait_sig+0x1d
 cv_wait_sig(c07e5744,c07e5720) at cv_wait_sig+0x18b
 kern_select(c82e9000,5,bfbfe9d0,0,0,...) at kern_select+0x4bf
 select(c82e9000,ea969d04) at select+0x44
 syscall(3b,3b,3b,0,8,...) at syscall+0x25b
 Xint0x80_syscall() at Xint0x80_syscall+0x1f
 --- syscall (93, FreeBSD ELF32, select), eip = 0x8075917, esp = 0xbfbfe96c, ebp = 0xbfbfee78 ---
 
 Tracing command adjkerntz pid 164 tid 100060 td 0xc8102900
 sched_switch(c8102900,0,1) at sched_switch+0x177
 mi_switch(1,0) at mi_switch+0x270
 sleepq_switch(c8101894) at sleepq_switch+0xc1
 sleepq_wait_sig(c8101894) at sleepq_wait_sig+0x1d
 msleep(c8101894,c81018c8,168,c0724047,0) at msleep+0x26e
 kern_sigsuspend(c8102900,0,0,0,0,...) at kern_sigsuspend+0xa3
 sigsuspend(c8102900,ea857d04) at sigsuspend+0x33
 syscall(3b,3b,3b,0,ffffc7c0,...) at syscall+0x25b
 Xint0x80_syscall() at Xint0x80_syscall+0x1f
 --- syscall (341, FreeBSD ELF32, sigsuspend), eip = 0x280bc3fb, esp = 0xbfbfed9c, ebp = 0xbfbfee78 ---
 
 Tracing command schedcpu pid 44 tid 100044 td 0xc7e05180
 sched_switch(c7e05180,0,1) at sched_switch+0x177
 mi_switch(1,0) at mi_switch+0x270
 sleepq_switch(e8380d04,c0797aa0,0,c0726d95,245,...) at sleepq_switch+0xc1
 sleepq_timedwait(e8380d04) at sleepq_timedwait+0x4a
 msleep(e8380d04,0,0,c071cc2c,3e8) at msleep+0x259
 schedcpu_thread(0,e8380d38,0,c05768b4,0,...) at schedcpu_thread+0x26
 fork_exit(c05768b4,0,e8380d38) at fork_exit+0xa0
 fork_trampoline() at fork_trampoline+0x8
 --- trap 0x1, eip = 0, esp = 0xe8380d6c, ebp = 0 ---
 
 Tracing command softdepflush pid 43 tid 100045 td 0xc7e05000
 sched_switch(c7e05000,0,1) at sched_switch+0x177
 mi_switch(1,0) at mi_switch+0x270
 sleepq_switch(c07f3254,c0797aa0,0,c0726d95,245,...) at sleepq_switch+0xc1
 sleepq_timedwait(c07f3254) at sleepq_timedwait+0x4a
 msleep(c07f3254,c07f3220,44,c0738a08,3e8) at msleep+0x259
 softdep_flush(0,e837dd38,0,c06669b0,0,...) at softdep_flush+0x227
 fork_exit(c06669b0,0,e837dd38) at fork_exit+0xa0
 fork_trampoline() at fork_trampoline+0x8
 --- trap 0x1, eip = 0, esp = 0xe837dd6c, ebp = 0 ---
 
 Tracing command vnlru pid 42 tid 100046 td 0xc7dbdd80
 sched_switch(c7dbdd80,0,1) at sched_switch+0x177
 mi_switch(1,0) at mi_switch+0x270
 sleepq_switch(c8084860,c0797aa0,0,c0726d95,245,...) at sleepq_switch+0xc1
 sleepq_timedwait(c8084860) at sleepq_timedwait+0x4a
 msleep(c8084860,c07e60e0,250,c072dc00,3e8,c07e6160) at msleep+0x259
 vnlru_proc(0,e836dd38,0,c05bb048,0,...) at vnlru_proc+0xdf
 fork_exit(c05bb048,0,e836dd38) at fork_exit+0xa0
 fork_trampoline() at fork_trampoline+0x8
 --- trap 0x1, eip = 0, esp = 0xe836dd6c, ebp = 0 ---
 
 Tracing command syncer pid 41 tid 100047 td 0xc7dbdc00
 sched_switch(c7dbdc00,0,1) at sched_switch+0x177
 mi_switch(1,0) at mi_switch+0x270
 sleepq_switch(dbf1b09c,c0797aa0,0,c0726d95,220,...) at sleepq_switch+0xc1
 sleepq_wait(dbf1b09c,0,c7eb0ba8,b7,c072db32,...) at sleepq_wait+0x46
 msleep(dbf1b09c,c07975ec,50,c072c4b4,0,...) at msleep+0x27d
 acquire(e836aacc,120,60000,c7dbdc00,0,...) at acquire+0x76
 lockmgr(dbf1b09c,202122,c7eb0ba8,c7dbdc00) at lockmgr+0x44e
 getblk(c7eb0ae0,113b60,0,4000,0,...) at getblk+0x12f
 breadn(c7eb0ae0,113b60,0,4000,0,...) at breadn+0x31
 bread(c7eb0ae0,113b60,0,4000,0,e836abb8) at bread+0x20
 ffs_update(c7eb1ae0,0,7,0,0,...) at ffs_update+0x22c
 ffs_syncvnode(c7eb1ae0,3) at ffs_syncvnode+0x3be
 ffs_sync(c80cbb90,3,c7dbdc00,c80cbb90,2,...) at ffs_sync+0x209
 sync_fsync(e836acbc) at sync_fsync+0x164
 VOP_FSYNC_APV(c0770380,e836acbc) at VOP_FSYNC_APV+0x9b
 sync_vnode(c817a67c,c7dbdc00) at sync_vnode+0x106
 sched_sync(0,e836ad38,0,c05bcb5c,0,...) at sched_sync+0x1ed
 fork_exit(c05bcb5c,0,e836ad38) at fork_exit+0xa0
 fork_trampoline() at fork_trampoline+0x8
 --- trap 0x1, eip = 0, esp = 0xe836ad6c, ebp = 0 ---
 
 Tracing command bufdaemon pid 40 tid 100048 td 0xc7dbda80
 sched_switch(c7dbda80,0,1) at sched_switch+0x177
 mi_switch(1,0) at mi_switch+0x270
 sleepq_switch(c07e5cc0,c0797aa0,0,c0726d95,245,...) at sleepq_switch+0xc1
 sleepq_timedwait(c07e5cc0) at sleepq_timedwait+0x4a
 msleep(c07e5cc0,c07e5ce0,44,c072c3ea,3e8) at msleep+0x259
 buf_daemon(0,e8367d38,0,c05ae534,0,...) at buf_daemon+0x108
 fork_exit(c05ae534,0,e8367d38) at fork_exit+0xa0
 fork_trampoline() at fork_trampoline+0x8
 --- trap 0x1, eip = 0, esp = 0xe8367d6c, ebp = 0 ---
 
 Tracing command pagezero pid 39 tid 100049 td 0xc7dbd900
 sched_switch(c7dbd900,0,1) at sched_switch+0x177
 mi_switch(1,0) at mi_switch+0x270
 sleepq_switch(c07f4224,c0797aa0,0,c0726d95,245,...) at sleepq_switch+0xc1
 sleepq_timedwait(c07f4224) at sleepq_timedwait+0x4a
 msleep(c07f4224,c07f3ca0,200,c073dabc,493e0,c07f3ca0,0,c073da9e,9d) at msleep+0x259
 vm_pagezero(0,e8364d38,0,c0697cbc,0,...) at vm_pagezero+0x6c
 fork_exit(c0697cbc,0,e8364d38) at fork_exit+0xa0
 fork_trampoline() at fork_trampoline+0x8
 --- trap 0x1, eip = 0, esp = 0xe8364d6c, ebp = 0 ---
 
 Tracing command vmdaemon pid 38 tid 100027 td 0xc7dbd180
 sched_switch(c7dbd180,0,1) at sched_switch+0x177
 mi_switch(1,0) at mi_switch+0x270
 sleepq_switch(c07f3d2c,c0797aa0,0,c0726d95,220,...) at sleepq_switch+0xc1
 sleepq_wait(c07f3d2c,0,c073d73f,63e,0,...) at sleepq_wait+0x46
 msleep(c07f3d2c,c07f3d40,68,c072c3ea,0,...) at msleep+0x27d
 vm_daemon(0,e8355d38,0,c0696cd0,0,...) at vm_daemon+0x36
 fork_exit(c0696cd0,0,e8355d38) at fork_exit+0xa0
 fork_trampoline() at fork_trampoline+0x8
 --- trap 0x1, eip = 0, esp = 0xe8355d6c, ebp = 0 ---
 
 Tracing command pagedaemon pid 37 tid 100028 td 0xc7dbd000
 sched_switch(c7dbd000,0,1) at sched_switch+0x177
 mi_switch(1,0) at mi_switch+0x270
 sleepq_switch(c07f3ce8,c0797aa0,0,c0726d95,245,...) at sleepq_switch+0xc1
 sleepq_timedwait(c07f3ce8) at sleepq_timedwait+0x4a
 msleep(c07f3ce8,c07f3ca0,44,c072c3ea,1388) at msleep+0x259
 vm_pageout(0,e8352d38,0,c0696958,0,...) at vm_pageout+0x260
 fork_exit(c0696958,0,e8352d38) at fork_exit+0xa0
 fork_trampoline() at fork_trampoline+0x8
 --- trap 0x1, eip = 0, esp = 0xe8352d6c, ebp = 0 ---
 
 Tracing command irq7: ppc0 pid 36 tid 100029 td 0xc7ce9d80
 fork_trampoline() at fork_trampoline
 
 Tracing command fdc0 pid 35 tid 100030 td 0xc7ce9c00
 sched_switch(c7ce9c00,0,1) at sched_switch+0x177
 mi_switch(1,0) at mi_switch+0x270
 sleepq_switch(c7e1f03c,c0797aa0,0,c0726d95,245,...) at sleepq_switch+0xc1
 sleepq_timedwait(c7e1f03c) at sleepq_timedwait+0x4a
 msleep(c7e1f03c,c7e1f0f0,4c,c071cc2c,3e8) at msleep+0x259
 fdc_worker(c7e1f000,c7e1f0f0,0,c073f3ff,4a1) at fdc_worker+0x1fe
 fdc_thread(c7e1f000,e6929d38,c7e1f000,c06a57f4,0,...) at fdc_thread+0x54
 fork_exit(c06a57f4,c7e1f000,e6929d38) at fork_exit+0xa0
 fork_trampoline() at fork_trampoline+0x8
 --- trap 0x1, eip = 0, esp = 0xe6929d6c, ebp = 0 ---
 
 Tracing command swi0: sio pid 34 tid 100031 td 0xc7ce9a80
 sched_switch(c7ce9a80,0,1) at sched_switch+0x177
 mi_switch(1,0) at mi_switch+0x270
 ithread_loop(c7e89540,e6926d38,c7e89540,c0551b84,0,...) at ithread_loop+0xda
 fork_exit(c0551b84,c7e89540,e6926d38) at fork_exit+0xa0
 fork_trampoline() at fork_trampoline+0x8
 --- trap 0x1, eip = 0, esp = 0xe6926d6c, ebp = 0 ---
 
 Tracing command irq1: atkbd0 pid 33 tid 100032 td 0xc7ce9900
 sched_switch(c7ce9900,0,1) at sched_switch+0x177
 mi_switch(1,0) at mi_switch+0x270
 ithread_loop(c7e846c0,e6923d38,c7e846c0,c0551b84,0,...) at ithread_loop+0xda
 fork_exit(c0551b84,c7e846c0,e6923d38) at fork_exit+0xa0
 fork_trampoline() at fork_trampoline+0x8
 --- trap 0x1, eip = 0, esp = 0xe6923d6c, ebp = 0 ---
 
 Tracing command irq22: fxp0 pid 32 tid 100033 td 0xc7ce9780
 sched_switch(c7ce9780,0,1) at sched_switch+0x177
 mi_switch(1,0) at mi_switch+0x270
 ithread_loop(c7e25010,e6920d38,c7e25010,c0551b84,0,...) at ithread_loop+0xda
 fork_exit(c0551b84,c7e25010,e6920d38) at fork_exit+0xa0
 fork_trampoline() at fork_trampoline+0x8
 --- trap 0x1, eip = 0, esp = 0xe6920d6c, ebp = 0 ---
 
 Tracing command irq20: mly0 pid 31 tid 100034 td 0xc7ce9600
 sched_switch(c7ce9600,0,1) at sched_switch+0x177
 mi_switch(1,0) at mi_switch+0x270
 ithread_loop(c7dfea40,e691dd38,c7dfea40,c0551b84,0,...) at ithread_loop+0xda
 fork_exit(c0551b84,c7dfea40,e691dd38) at fork_exit+0xa0
 fork_trampoline() at fork_trampoline+0x8
 --- trap 0x1, eip = 0, esp = 0xe691dd6c, ebp = 0 ---
 
 Tracing command irq15: ata1 pid 30 tid 100035 td 0xc7ce9480
 fork_trampoline() at fork_trampoline
 
 Tracing command irq14: ata0 pid 29 tid 100036 td 0xc7e05d80
 sched_switch(c7e05d80,0,1) at sched_switch+0x177
 mi_switch(1,0) at mi_switch+0x270
 ithread_loop(c7e072e0,e8398d38,c7e072e0,c0551b84,0,...) at ithread_loop+0xda
 fork_exit(c0551b84,c7e072e0,e8398d38) at fork_exit+0xa0
 fork_trampoline() at fork_trampoline+0x8
 --- trap 0x1, eip = 0, esp = 0xe8398d6c, ebp = 0 ---
 
 Tracing command usb1 pid 28 tid 100037 td 0xc7e05c00
 sched_switch(c7e05c00,0,1) at sched_switch+0x177
 mi_switch(1,0) at mi_switch+0x270
 sleepq_switch(c7dada10,c0797aa0,0,c0726d95,245,...) at sleepq_switch+0xc1
 sleepq_timedwait(c7dada10) at sleepq_timedwait+0x4a
 msleep(c7dada10,0,5c,c071b821,ea60,c7debd00) at msleep+0x259
 usb_event_thread(c7debd00,e8395d38,c7debd00,c051db30,0,...) at usb_event_thread+0x70
 fork_exit(c051db30,c7debd00,e8395d38) at fork_exit+0xa0
 fork_trampoline() at fork_trampoline+0x8
 --- trap 0x1, eip = 0, esp = 0xe8395d6c, ebp = 0 ---
 
 Tracing command irq17: ehci0 pid 27 tid 100017 td 0xc7ce5900
 fork_trampoline() at fork_trampoline
 
 Tracing command usbtask pid 26 tid 100018 td 0xc7ce5780
 sched_switch(c7ce5780,0,1) at sched_switch+0x177
 mi_switch(1,0) at mi_switch+0x270
 sleepq_switch(c07953e4,c0797aa0,0,c0726d95,220,...) at sleepq_switch+0xc1
 sleepq_wait(c07953e4,0,c055d0cc,c0797ae0,c071b806,...) at sleepq_wait+0x46
 msleep(c07953e4,0,5c,c071b828,0,...) at msleep+0x27d
 usb_task_thread(0,e6902d38,0,c051dbc0,0,...) at usb_task_thread+0x47
 fork_exit(c051dbc0,0,e6902d38) at fork_exit+0xa0
 fork_trampoline() at fork_trampoline+0x8
 --- trap 0x1, eip = 0, esp = 0xe6902d6c, ebp = 0 ---
 
 Tracing command usb0 pid 25 tid 100019 td 0xc7ce5600
 sched_switch(c7ce5600,0,1) at sched_switch+0x177
 mi_switch(1,0) at mi_switch+0x270
 sleepq_switch(c7db4210,c0797aa0,0,c0726d95,245,...) at sleepq_switch+0xc1
 sleepq_timedwait(c7db4210) at sleepq_timedwait+0x4a
 msleep(c7db4210,0,5c,c071b821,ea60,c7dea640) at msleep+0x259
 usb_event_thread(c7dea640,e68ffd38,c7dea640,c051db30,0,...) at usb_event_thread+0x70
 fork_exit(c051db30,c7dea640,e68ffd38) at fork_exit+0xa0
 fork_trampoline() at fork_trampoline+0x8
 --- trap 0x1, eip = 0, esp = 0xe68ffd6c, ebp = 0 ---
 
 Tracing command irq16: ohci0 pid 24 tid 100020 td 0xc7ce5480
 fork_trampoline() at fork_trampoline
 
 Tracing command irq9: acpi0 pid 23 tid 100021 td 0xc7ce5300
 fork_trampoline() at fork_trampoline
 
 Tracing command acpi_task_2 pid 22 tid 100022 td 0xc7ce5180
 sched_switch(c7ce5180,0,1) at sched_switch+0x177
 mi_switch(1,0) at mi_switch+0x270
 sleepq_switch(c7d43c00,c0797aa0,0,c0726d95,220,...) at sleepq_switch+0xc1
 sleepq_wait(c7d43c00,0,c07273e4,49,8,...) at sleepq_wait+0x46
 msleep(c7d43c00,c7d43c1c,0,c071cc2c,0) at msleep+0x27d
 taskqueue_thread_loop(c0791fc8,e68f6d38,c0791fc8,c0585fd4,0,...) at taskqueue_thread_loop+0x78
 fork_exit(c0585fd4,c0791fc8,e68f6d38) at fork_exit+0xa0
 fork_trampoline() at fork_trampoline+0x8
 --- trap 0x1, eip = 0, esp = 0xe68f6d6c, ebp = 0 ---
 
 Tracing command acpi_task_1 pid 9 tid 100023 td 0xc7dbd780
 sched_switch(c7dbd780,0,1) at sched_switch+0x177
 mi_switch(1,0) at mi_switch+0x270
 sleepq_switch(c7d43c00,c0797aa0,0,c0726d95,220,...) at sleepq_switch+0xc1
 sleepq_wait(c7d43c00,0,c07273e4,49,8,...) at sleepq_wait+0x46
 msleep(c7d43c00,c7d43c1c,0,c071cc2c,0) at msleep+0x27d
 taskqueue_thread_loop(c0791fc8,e8361d38,c0791fc8,c0585fd4,0,...) at taskqueue_thread_loop+0x78
 fork_exit(c0585fd4,c0791fc8,e8361d38) at fork_exit+0xa0
 fork_trampoline() at fork_trampoline+0x8
 --- trap 0x1, eip = 0, esp = 0xe8361d6c, ebp = 0 ---
 
 Tracing command acpi_task_0 pid 8 tid 100024 td 0xc7dbd600
 sched_switch(c7dbd600,0,1) at sched_switch+0x177
 mi_switch(1,0) at mi_switch+0x270
 sleepq_switch(c7d43c00,c0797aa0,0,c0726d95,220,...) at sleepq_switch+0xc1
 sleepq_wait(c7d43c00,0,c07273e4,49,c7ddf260,...) at sleepq_wait+0x46
 msleep(c7d43c00,c7d43c1c,0,c071cc2c,0) at msleep+0x27d
 taskqueue_thread_loop(c0791fc8,e835ed38,c0791fc8,c0585fd4,0,...) at taskqueue_thread_loop+0x78
 fork_exit(c0585fd4,c0791fc8,e835ed38) at fork_exit+0xa0
 fork_trampoline() at fork_trampoline+0x8
 --- trap 0x1, eip = 0, esp = 0xe835ed6c, ebp = 0 ---
 
 Tracing command swi5: + pid 21 tid 100025 td 0xc7dbd480
 fork_trampoline() at fork_trampoline
 
 Tracing command thread taskq pid 7 tid 100026 td 0xc7dbd300
 sched_switch(c7dbd300,0,1) at sched_switch+0x177
 mi_switch(1,0) at mi_switch+0x270
 sleepq_switch(c7d43d80,c0797aa0,0,c0726d95,220,...) at sleepq_switch+0xc1
 sleepq_wait(c7d43d80,0,c07273e4,49,c806aeb0,...) at sleepq_wait+0x46
 msleep(c7d43d80,c7d43d9c,0,c071cc2c,0) at msleep+0x27d
 taskqueue_thread_loop(c079da88,e8358d38,c079da88,c0585fd4,0,...) at taskqueue_thread_loop+0x78
 fork_exit(c0585fd4,c079da88,e8358d38) at fork_exit+0xa0
 fork_trampoline() at fork_trampoline+0x8
 --- trap 0x1, eip = 0, esp = 0xe8358d6c, ebp = 0 ---
 
 Tracing command swi6: Giant taskq pid 20 tid 100008 td 0xc7ce3300
 sched_switch(c7ce3300,0,1) at sched_switch+0x177
 mi_switch(1,0) at mi_switch+0x270
 ithread_loop(c7cc8b70,e68dbd38,c7cc8b70,c0551b84,0,...) at ithread_loop+0xda
 fork_exit(c0551b84,c7cc8b70,e68dbd38) at fork_exit+0xa0
 fork_trampoline() at fork_trampoline+0x8
 --- trap 0x1, eip = 0, esp = 0xe68dbd6c, ebp = 0 ---
 
 Tracing command swi6: task queue pid 19 tid 100009 td 0xc7ce3180
 sched_switch(c7ce3180,0,1) at sched_switch+0x177
 mi_switch(1,0) at mi_switch+0x270
 ithread_loop(c7cc8b80,e68d8d38,c7cc8b80,c0551b84,0,...) at ithread_loop+0xda
 fork_exit(c0551b84,c7cc8b80,e68d8d38) at fork_exit+0xa0
 fork_trampoline() at fork_trampoline+0x8
 --- trap 0x1, eip = 0, esp = 0xe68d8d6c, ebp = 0 ---
 
 Tracing command kqueue taskq pid 6 tid 100010 td 0xc7ce3000
 sched_switch(c7ce3000,0,1) at sched_switch+0x177
 mi_switch(1,0) at mi_switch+0x270
 sleepq_switch(c7d72100,c0797aa0,0,c0726d95,220,...) at sleepq_switch+0xc1
 sleepq_wait(c7d72100,0,c07273e4,49,8,...) at sleepq_wait+0x46
 msleep(c7d72100,c7d7211c,0,c071cc2c,0) at msleep+0x27d
 taskqueue_thread_loop(c0796120,e68d5d38,c0796120,c0585fd4,0,...) at taskqueue_thread_loop+0x78
 fork_exit(c0585fd4,c0796120,e68d5d38) at fork_exit+0xa0
 fork_trampoline() at fork_trampoline+0x8
 --- trap 0x1, eip = 0, esp = 0xe68d5d6c, ebp = 0 ---
 
 Tracing command swi2: cambio pid 18 tid 100011 td 0xc7ce9300
 sched_switch(c7ce9300,0,1) at sched_switch+0x177
 mi_switch(1,0) at mi_switch+0x270
 ithread_loop(c7cc8ba0,e6917d38,c7cc8ba0,c0551b84,0,...) at ithread_loop+0xda
 fork_exit(c0551b84,c7cc8ba0,e6917d38) at fork_exit+0xa0
 fork_trampoline() at fork_trampoline+0x8
 --- trap 0x1, eip = 0, esp = 0xe6917d6c, ebp = 0 ---
 
 Tracing command xpt_thrd pid 5 tid 100012 td 0xc7ce9180
 sched_switch(c7ce9180,0,1) at sched_switch+0x177
 mi_switch(1,0) at mi_switch+0x270
 sleepq_switch(c078c4c4,c0797aa0,0,c0726d95,220,...) at sleepq_switch+0xc1
 sleepq_wait(c078c4c4,0,c055d0cc,c0797ae0,c06fdf9f,...) at sleepq_wait+0x46
 msleep(c078c4c4,0,4c,c06fdfba,0) at msleep+0x27d
 xpt_scanner_thread(0,e6914d38,0,c04441f4,0,...) at xpt_scanner_thread+0x35
 fork_exit(c04441f4,0,e6914d38) at fork_exit+0xa0
 fork_trampoline() at fork_trampoline+0x8
 --- trap 0x1, eip = 0, esp = 0xe6914d6c, ebp = 0 ---
 
 Tracing command yarrow pid 17 tid 100013 td 0xc7ce9000
 sched_switch(c7ce9000,0,1) at sched_switch+0x177
 mi_switch(1,0) at mi_switch+0x270
 sleepq_switch(c07933e0,c0797aa0,0,c0726d95,245,...) at sleepq_switch+0xc1
 sleepq_timedwait(c07933e0) at sleepq_timedwait+0x4a
 msleep(c07933e0,0,0,c071cc2c,64) at msleep+0x259
 random_kthread(0,e6911d38,0,c0506b28,0,...) at random_kthread+0x180
 fork_exit(c0506b28,0,e6911d38) at fork_exit+0xa0
 fork_trampoline() at fork_trampoline+0x8
 --- trap 0x1, eip = 0, esp = 0xe6911d6c, ebp = 0 ---
 
 Tracing command g_down pid 4 tid 100014 td 0xc7ce5d80
 sched_switch(c7ce5d80,0,1) at sched_switch+0x177
 mi_switch(1,0) at mi_switch+0x270
 sleepq_switch(c0795888,c0797aa0,0,c0726d95,245,...) at sleepq_switch+0xc1
 sleepq_timedwait(c0795888) at sleepq_timedwait+0x4a
 msleep(c0795888,c0795768,24c,c071cc2c,64) at msleep+0x259
 g_io_schedule_down(c7ce5d80) at g_io_schedule_down+0x56
 g_down_procbody(0,e690ed38,0,c052fab0,0,...) at g_down_procbody+0x5a
 fork_exit(c052fab0,0,e690ed38) at fork_exit+0xa0
 fork_trampoline() at fork_trampoline+0x8
 --- trap 0x1, eip = 0, esp = 0xe690ed6c, ebp = 0 ---
 
 Tracing command g_up pid 3 tid 100015 td 0xc7ce5c00
 sched_switch(c7ce5c00,0,1) at sched_switch+0x177
 mi_switch(1,0) at mi_switch+0x270
 sleepq_switch(c0795884,c0797aa0,0,c0726d95,245,...) at sleepq_switch+0xc1
 sleepq_timedwait(c0795884) at sleepq_timedwait+0x4a
 msleep(c0795884,c07957a8,24c,c071cc2c,64) at msleep+0x259
 g_io_schedule_up(c7ce5c00) at g_io_schedule_up+0x127
 g_up_procbody(0,e690bd38,0,c052fa50,0,...) at g_up_procbody+0x5a
 fork_exit(c052fa50,0,e690bd38) at fork_exit+0xa0
 fork_trampoline() at fork_trampoline+0x8
 --- trap 0x1, eip = 0, esp = 0xe690bd6c, ebp = 0 ---
 
 Tracing command g_event pid 2 tid 100016 td 0xc7ce5a80
 sched_switch(c7ce5a80,0,1) at sched_switch+0x177
 mi_switch(1,0) at mi_switch+0x270
 sleepq_switch(c079587c,c0797aa0,0,c0726d95,245,...) at sleepq_switch+0xc1
 sleepq_timedwait(c079587c) at sleepq_timedwait+0x4a
 msleep(c079587c,0,4c,c071cc2c,64) at msleep+0x259
 g_event_procbody(0,e6908d38,0,c052fb10,0,...) at g_event_procbody+0x9e
 fork_exit(c052fb10,0,e6908d38) at fork_exit+0xa0
 fork_trampoline() at fork_trampoline+0x8
 --- trap 0x1, eip = 0, esp = 0xe6908d6c, ebp = 0 ---
 
 Tracing command swi3: vm pid 16 tid 100000 td 0xc7ce5000
 fork_trampoline() at fork_trampoline
 
 Tracing command swi4: clock sio pid 15 tid 100001 td 0xc7ce3d80
 sched_switch(c7ce3d80,0,1) at sched_switch+0x177
 mi_switch(1,0) at mi_switch+0x270
 ithread_loop(c7cc8870,e68f0d38,c7cc8870,c0551b84,0,...) at ithread_loop+0xda
 fork_exit(c0551b84,c7cc8870,e68f0d38) at fork_exit+0xa0
 fork_trampoline() at fork_trampoline+0x8
 --- trap 0x1, eip = 0, esp = 0xe68f0d6c, ebp = 0 ---
 
 Tracing command swi1: net pid 14 tid 100002 td 0xc7ce3c00
 sched_switch(c7ce3c00,0,1) at sched_switch+0x177
 mi_switch(1,0) at mi_switch+0x270
 ithread_loop(c7cc8880,e68edd38,c7cc8880,c0551b84,0,...) at ithread_loop+0xda
 fork_exit(c0551b84,c7cc8880,e68edd38) at fork_exit+0xa0
 fork_trampoline() at fork_trampoline+0x8
 --- trap 0x1, eip = 0, esp = 0xe68edd6c, ebp = 0 ---
 
 Tracing command idle: cpu0 pid 13 tid 100003 td 0xc7ce3a80
 sched_switch(c072811b,6a9,c07e5594,e68eac40,c058a4e6,...) at sched_switch+0x177
 (null)(c072813c,50000,c7ccb010,c07981b0,0,...) at 0
 __func__.0(c055d930,0,2001,1,94378,...) at 0xc072813c
 (null)(2f007865,2f727375,2f637273,2f737973,6e72656b,...) at 0xa
 
 Tracing command idle: cpu1 pid 12 tid 100004 td 0xc7ce3900
 kdb_enter(c074002a) at kdb_enter+0x2b
 siointr1(c7e95800,c07f61a0,0,c073fe30,56e,...) at siointr1+0xce
 siointr(c7e95800) at siointr+0x21
 intr_execute_handlers(c7cd74c8,e68e7c94,4,e68e7cd8,c06c2713,...) at intr_execute_handlers+0xe1
 lapic_handle_intr(38) at lapic_handle_intr+0x2e
 Xapic_isr1() at Xapic_isr1+0x33
 --- interrupt, eip = 0xc06bb645, esp = 0xe68e7cd8, ebp = 0xe68e7cd8 ---
 acpi_cpu_c1(c0797aa0,2,e68e7cf8,2,c7ce2860,...) at acpi_cpu_c1+0x5
 acpi_cpu_idle(e68e7d0c,c0550ee1,c0550e84,e68e7d24,c0550cc4,...) at acpi_cpu_idle+0x15d
 cpu_idle(c0550e84,e68e7d24,c0550cc4,0,e68e7d38,...) at cpu_idle+0x28
 idle_proc(0,e68e7d38,0,c0550e84,0,...) at idle_proc+0x5d
 fork_exit(c0550e84,0,e68e7d38) at fork_exit+0xa0
 fork_trampoline() at fork_trampoline+0x8
 --- trap 0x1, eip = 0, esp = 0xe68e7d6c, ebp = 0 ---
 
 Tracing command idle: cpu2 pid 11 tid 100005 td 0xc7ce3780
 sched_switch(c079efa0,0,ffc06034) at sched_switch+0x177
 mi_switch(c055d321,c07e5590,e68e4c1c,c058ae7c,c079efa0,...) at mi_switch+0x270
 end(c072813c,50000,c7ccb010,c07981b0,0,...) at 0xe68e4bf0
 __func__.0(c055d930,0,2001,1,94378,...) at 0xc072813c
 (null)(2f007865,2f727375,2f637273,2f737973,6e72656b,...) at 0xa
 
 Tracing command idle: cpu3 pid 10 tid 100006 td 0xc7ce3600
 sched_switch(28,12,c06c2aa1,c080fd90,8,...) at sched_switch+0x177
 
 Tracing command init pid 1 tid 100007 td 0xc7ce3480
 sched_switch(c7ce3480,0,1) at sched_switch+0x177
 mi_switch(1,0) at mi_switch+0x270
 sleepq_switch(c7ce8000) at sleepq_switch+0xc1
 sleepq_wait_sig(c7ce8000) at sleepq_wait_sig+0x1d
 msleep(c7ce8000,c7ce8068,15c,c0729071,0) at msleep+0x26e
 kern_wait(c7ce3480,ffffffff,e68dec84,0,0) at kern_wait+0x6bb
 wait4(c7ce3480,e68ded04) at wait4+0x2a
 syscall(3b,3b,3b,0,0,...) at syscall+0x25b
 Xint0x80_syscall() at Xint0x80_syscall+0x1f
 --- syscall (7, FreeBSD ELF32, wait4), eip = 0x8052f47, esp = 0xbfbfed7c, ebp = 0xbfbfed98 ---
 
 Tracing command swapper pid 0 tid 0 td 0xc0795ba0
 sched_switch(c0795ba0,0,1) at sched_switch+0x177
 mi_switch(1,0) at mi_switch+0x270
 scheduler(0,c1ec00,c1e000,0,c0440d25,...) at scheduler+0x199
 mi_startup() at mi_startup+0x96
 begin() at begin+0x2c
 
 db> call doadump
 Dumping 3070 MB (2 chunks)
 Aborting dump due to I/O error.
 status == 0xb, scsi status == 0x0
 
 ** DUMP FAILED (ERROR 5) **
 = 0x1d
 
 --
 Oleg Derevenetz <oleg at vsi.ru> OOD3-RIPE
 Phone: +7 4732 539880
 Fax:   +7 4732 531415         http://www.vsi.ru
 CenterTelecom Voronezh ISP    http://isp.vsi.ru