system locks up in vmmaps
pluknet
pluknet at gmail.com
Thu May 28 06:57:06 UTC 2009
System 6.2-R locks in vmmaps state on relatively small workload.
This happens the first time. I don't know if I could reproduce this lockup.
The picture at whole looks like on
http://lists.freebsd.org/pipermail/freebsd-current/2005-September/055982.html
(though that problem was fixed before 6).
I'd be great to get any ideas on that.
db> ps
pid ppid pgrp uid state wmesg wchan cmd
10877 1025 10877 0 Ss vmmaps 0xc1068478 sshd
10876 10875 10876 0 SVs vmmaps 0xc1068478 cron
10875 1091 1091 0 S ppwait 0xcfc82a78 cron
10874 10872 10874 0 SVs vmmaps 0xc1068478 cron
10873 10871 10873 0 SVs vmmaps 0xc1068478 cron
10872 1091 1091 0 S ppwait 0xcfc83000 cron
10871 1091 1091 0 S ppwait 0xcfc83218 cron
10870 10869 895 0 S vmmaps 0xc1068478 guard
10869 896 895 0 S select 0xc0a13084 guard
10868 914 914 0 S ufs 0xc82e49e8 proftpd
10867 10863 10867 0 SVs vmmaps 0xc1068478 cron
10866 10860 10866 0 SVs vmmaps 0xc1068478 cron
10865 10862 10865 0 SVs vmmaps 0xc1068478 cron
10864 10861 10864 0 SVs vmmaps 0xc1068478 cron
10863 1091 1091 0 S ppwait 0xcfc83648 cron
10862 1091 1091 0 S ppwait 0xcfc83860 cron
10861 1091 1091 0 S ppwait 0xcfbd1860 cron
10860 1091 1091 0 S ppwait 0xcfbd1a78 cron
10859 1025 10859 0 Ss vmmaps 0xc1068478 sshd
[..some more..]
db> bt 10868
Tracing pid 10868 tid 100134 td 0xc85f44b0
sched_switch(c85f44b0,0,1) at sched_switch+0x143
mi_switch(1,0,c85f44b0,ee89c8cc,c06a484c,...) at mi_switch+0x1ba
sleepq_switch(c82e49e8) at sleepq_switch+0x87
sleepq_wait(c82e49e8,0,c85f44b0,c82e49e8,4,...) at sleepq_wait+0x5c
msleep(c82e49e8,c0a070c8,50,c0928d0d,0,...) at msleep+0x269
acquire(ee89c94c,40,60000,c85f44b0,0,...) at acquire+0x7b
lockmgr(c82e49e8,2002,c82e4a0c,c85f44b0) at lockmgr+0x3fe
ffs_lock(ee89c9a8) at ffs_lock+0x88
VOP_LOCK_APV(c09d5e60,ee89c9a8) at VOP_LOCK_APV+0x43
vn_lock(c82e4990,2002,c85f44b0,c82e4990) at vn_lock+0xf4
vget(c82e4990,2002,c85f44b0) at vget+0xbe
vfs_hash_get(c820b7c8,2,2,c85f44b0,ee89cac0,0,0) at vfs_hash_get+0xf5
ffs_vget(c820b7c8,2,2,ee89cac0) at ffs_vget+0x27
ufs_root(c820b7c8,2,ee89cb04,c85f44b0,1,...) at ufs_root+0x19
lookup(ee89cc00) at lookup+0x7f4
namei(ee89cc00) at namei+0x39a
kern_stat(c85f44b0,bfbfd770,0,ee89cc74) at kern_stat+0x35
stat(c85f44b0,ee89cd04) at stat+0x1b
syscall(3b,bfbf003b,bfbf003b,0,bfbfc710,...) at syscall+0x2bf
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (188, FreeBSD ELF32, stat), eip = 0x28315453, esp = 0xbfbfc63c, ebp
= 0xbfbfc668 ---
db> bt 10864
Tracing pid 10864 tid 100241 td 0xc8a457d0
sched_switch(c8a457d0,0,1) at sched_switch+0x143
mi_switch(1,0,c8a457d0,ee8fcc10,c06a484c,...) at mi_switch+0x1ba
sleepq_switch(c1068478) at sleepq_switch+0x87
sleepq_wait(c1068478,0,c8a457d0,c10683c0,42000,...) at sleepq_wait+0x5c
msleep(c1068478,c0a21460,244,c093ab48,0,...) at msleep+0x269
vm_map_unlock_and_wait(c10683c0,0) at vm_map_unlock_and_wait+0x63
kmem_alloc_wait(c10683c0,41400,1,c8e9ea78,ee8fccb4,...) at kmem_alloc_wait+0x6b
exec_copyin_args(ee8fccb4,80551c6,0,bfbfe6b0,8051140) at exec_copyin_args+0x3a
execve(c8a457d0,ee8fcd04) at execve+0x1e
syscall(805003b,3b,bfbf003b,bfbfe6f8,bfbfe6b0,...) at syscall+0x2bf
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (59, FreeBSD ELF32, execve), eip = 0x280e168f, esp = 0xbfbfe69c, ebp
= 0xbfbfe6e8 ---
db> bt 10875
Tracing pid 10875 tid 100580 td 0xc8eb1640
sched_switch(c8eb1640,0,1) at sched_switch+0x143
mi_switch(1,0,c8eb1640,eebe4c54,c06a484c,...) at mi_switch+0x1ba
sleepq_switch(cfc82a78) at sleepq_switch+0x87
sleepq_wait(cfc82a78,0,c8eb1640,cfc828c8,cfc82860,...) at sleepq_wait+0x5c
msleep(cfc82a78,cfc828c8,5c,c09232c2,0) at msleep+0x269
fork1(c8eb1640,80000034,0,eebe4cd4) at fork1+0x14d7
vfork(c8eb1640,eebe4d04) at vfork+0x1b
syscall(805003b,3b,bfbf003b,80512f8,804e444,...) at syscall+0x2bf
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (66, FreeBSD ELF32, vfork), eip = 0x280cde68, esp =
0xbfbfe710, ebp = 0xbfbfecc8 ---
db> bt 47
Tracing pid 47 tid 100041 td 0xc7f3d640
sched_switch(c7f3d640,0,1) at sched_switch+0x143
mi_switch(1,0,c7f3d640,e8506aa4,c06a484c,...) at mi_switch+0x1ba
sleepq_switch(c8a58e28) at sleepq_switch+0x87
sleepq_wait(c8a58e28,0,c7f3d640,c8a58e28,4,...) at sleepq_wait+0x5c
msleep(c8a58e28,c0a06f84,50,c0928d0d,0,...) at msleep+0x269
acquire(e8506b24,40,60000,c7f3d640,0,...) at acquire+0x7b
lockmgr(c8a58e28,2002,c8a58e4c,c7f3d640) at lockmgr+0x3fe
ffs_lock(e8506b80) at ffs_lock+0x88
VOP_LOCK_APV(c09d5e60,e8506b80) at VOP_LOCK_APV+0x43
vn_lock(c8a58dd0,2002,c7f3d640,c8a58dd0) at vn_lock+0xf4
vget(c8a58dd0,2002,c7f3d640,c820b574,0,...) at vget+0xbe
qsync(c820b530,2c1cf,0,0,0,...) at qsync+0x166
ffs_sync(c820b530,3,c7f3d640,c820b530,2) at ffs_sync+0x3d0
sync_fsync(e8506cc0) at sync_fsync+0x1e6
VOP_FSYNC_APV(c09a38e0,e8506cc0) at VOP_FSYNC_APV+0x38
sync_vnode(c82e4720,c7f3d640) at sync_vnode+0x153
sched_sync(0,e8506d38) at sched_sync+0x259
fork_exit(c06dea18,0,e8506d38) at fork_exit+0x71
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe8506d6c, ebp = 0 ---
db> show lockedvnods
Locked vnodes
0xc8603aa0: tag ufs, type VREG
usecount 1, writecount 1, refcount 3 mountedhere 0
flags ()
v_object 0xc860818c ref 0 pages 10
lock type ufs: EXCL (count 1) by thread 0xc7f99e10 (pid 767)
ino 329736, on dev aacdu0s1e
0xc82e4990: tag ufs, type VDIR
usecount 1, writecount 0, refcount 34 mountedhere 0
flags (VV_ROOT)
v_object 0xc8a9f108 ref 0 pages 1
lock type ufs: EXCL (count 1) by thread 0xcf68e190 (pid 10691)
with 31 pending
ino 2, on dev aacdu0s1f
0xc833b110: tag ufs, type VDIR
usecount 1, writecount 0, refcount 4 mountedhere 0
flags ()
v_object 0xca5d44a4 ref 0 pages 1
lock type ufs: EXCL (count 1) by thread 0xc8adae10 (pid 10689)
with 1 pending
ino 2190336, on dev aacdu0s1f
0xc85f8440: tag ufs, type VDIR
usecount 1, writecount 0, refcount 4 mountedhere 0
flags ()
v_object 0xca5d4084 ref 0 pages 1
lock type ufs: EXCL (count 1) by thread 0xcf701190 (pid 10283)
with 1 pending
ino 2190344, on dev aacdu0s1f
0xc8d9d880: tag ufs, type VDIR
usecount 1, writecount 0, refcount 4 mountedhere 0
flags ()
v_object 0xca5d3d68 ref 0 pages 1
lock type ufs: EXCL (count 1) by thread 0xcf68c960 (pid 9049)
with 1 pending
ino 2190347, on dev aacdu0s1f
0xc8d9d770: tag ufs, type VREG
usecount 1, writecount 0, refcount 28 mountedhere 0
flags ()
v_object 0xc8cc439c ref 0 pages 100
lock type ufs: SHARED (count 1) with 1 pending
ino 2190376, on dev aacdu0s1f
0xc82e4660: tag syncer, type VNON
usecount 1, writecount 0, refcount 2 mountedhere 0
flags ()
lock type syncer: EXCL (count 1) by thread 0xc7f3d640 (pid 47)
0xc8a58dd0: tag ufs, type VREG
usecount 1, writecount 1, refcount 11 mountedhere 0
flags ()
v_object 0xc86977bc ref 0 pages 120
lock type ufs: EXCL (count 1) by thread 0xc917baf0 (pid 989) with 1 pending
ino 57442995, on dev aacdu0s1g
0xc8db2000: tag ufs, type VREG
usecount 3, writecount 3, refcount 22 mountedhere 0
flags ()
v_object 0xc8d64000 ref 0 pages 72
lock type ufs: EXCL (count 1) by thread 0xc90fb7d0 (pid 989)
ino 57443061, on dev aacdu0s1g
db> bt 989
Tracing pid 989 tid 100183 td 0xc90feaf0
sched_switch(c90feaf0,0,1) at sched_switch+0x143
mi_switch(1,0,c7f3e874,eed4dc70,c06a499e,...) at mi_switch+0x1ba
sleepq_switch(c7f3e874) at sleepq_switch+0x87
sleepq_timedwait_sig(c7f3e874) at sleepq_timedwait_sig+0x1e
msleep(c7f3e874,c8a42498,168,c0923768,3e9,...) at msleep+0x230
kse_release(c90feaf0,eed4dd04) at kse_release+0x237
syscall(842003b,3b,a89003b,8423f00,0,...) at syscall+0x2bf
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (383, FreeBSD ELF32, kse_release), eip = 0x285384f7, esp =
0xa899f28, ebp = 0xa899f64 ---
db> bt 10691
Tracing pid 10691 tid 100448 td 0xcf68e190
sched_switch(cf68e190,0,1) at sched_switch+0x143
mi_switch(1,0,cf68e190,ef0f4884,c06a484c,...) at mi_switch+0x1ba
sleepq_switch(c833b168) at sleepq_switch+0x87
sleepq_wait(c833b168,0,cf68e190,c833b168,4,...) at sleepq_wait+0x5c
msleep(c833b168,c0a06ff0,50,c0928d0d,0,...) at msleep+0x269
acquire(ef0f4904,40,60000,cf68e190,0,...) at acquire+0x7b
lockmgr(c833b168,2002,c833b18c,cf68e190) at lockmgr+0x3fe
ffs_lock(ef0f4960) at ffs_lock+0x88
VOP_LOCK_APV(c09d5e60,ef0f4960) at VOP_LOCK_APV+0x43
vn_lock(c833b110,2002,cf68e190,c833b110) at vn_lock+0xf4
vget(c833b110,2002,cf68e190) at vget+0xbe
cache_lookup(c82e4990,ef0f4c3c,ef0f4c50) at cache_lookup+0x458
vfs_cache_lookup(ef0f4aa0) at vfs_cache_lookup+0x8f
VOP_LOOKUP_APV(c09d5e60,ef0f4aa0) at VOP_LOOKUP_APV+0x43
lookup(ef0f4c28) at lookup+0x4c1
namei(ef0f4c28) at namei+0x39a
do_execve(cf68e190,ef0f4cb4,0) at do_execve+0x17d
kern_execve(cf68e190,ef0f4cb4,0) at kern_execve+0xd4
execve(cf68e190,ef0f4d04) at execve+0x32
syscall(819003b,bfbf003b,bfbf003b,819ffe0,819ffe0,...) at syscall+0x2bf
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (59, FreeBSD ELF32, execve), eip = 0x284d368f, esp =
0xbfbf92dc, ebp = 0xbfbf92f8 ---
db> bt 767
Tracing pid 767 tid 100116 td 0xc7f99e10
sched_switch(c7f99e10,0,1) at sched_switch+0x143
mi_switch(1,0,c7f99e10,ec58dbf0,c06a484c,...) at mi_switch+0x1ba
sleepq_switch(c8603b84) at sleepq_switch+0x87
sleepq_wait(c8603b84,0,c7f99e10,c8603b60,0,...) at sleepq_wait+0x5c
msleep(c8603b84,c8603b1c,4f,c0929762,0) at msleep+0x269
bufobj_wwait(c8603b60,3,0) at bufobj_wwait+0x37
ffs_syncvnode(c8603aa0,1) at ffs_syncvnode+0x371
ffs_fsync(ec58dcc0) at ffs_fsync+0x12
VOP_FSYNC_APV(c09d5e60,ec58dcc0) at VOP_FSYNC_APV+0x38
fsync(c7f99e10,ec58dd04) at fsync+0x170
syscall(805003b,805003b,bfbf003b,8056064,8056060,...) at syscall+0x2bf
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (95, FreeBSD ELF32, fsync), eip = 0x28147ab7, esp =
0xbfbfda3c, ebp = 0xbfbfee78 ---
db> bt 10283
Tracing pid 10283 tid 100447 td 0xcf701190
sched_switch(cf701190,0,1) at sched_switch+0x143
mi_switch(1,0,cf701190,ef1687ac,c06a484c,...) at mi_switch+0x1ba
sleepq_switch(c8d9d8d8) at sleepq_switch+0x87
sleepq_wait(c8d9d8d8,0,cf701190,c8d9d8d8,4,...) at sleepq_wait+0x5c
msleep(c8d9d8d8,c0a06a08,50,c0928d0d,0,...) at msleep+0x269
acquire(ef16882c,40,60000,cf701190,0,...) at acquire+0x7b
lockmgr(c8d9d8d8,2002,c8d9d8fc,cf701190) at lockmgr+0x3fe
ffs_lock(ef168888) at ffs_lock+0x88
VOP_LOCK_APV(c09d5e60,ef168888) at VOP_LOCK_APV+0x43
vn_lock(c8d9d880,2002,cf701190,c8d9d880) at vn_lock+0xf4
vget(c8d9d880,2002,cf701190) at vget+0xbe
cache_lookup(c85f8440,ef168be0,ef168bf4) at cache_lookup+0x458
vfs_cache_lookup(ef1689c8) at vfs_cache_lookup+0x8f
VOP_LOOKUP_APV(c09d5e60,ef1689c8) at VOP_LOOKUP_APV+0x43
lookup(ef168bcc) at lookup+0x4c1
namei(ef168bcc) at namei+0x39a
vn_open_cred(ef168bcc,ef168ccc,1a4,ce4db200,4,...) at vn_open_cred+0x2ad
vn_open(ef168bcc,ef168ccc,1a4,4) at vn_open+0x1e
kern_open(cf701190,805120c,0,1,1b6,...) at kern_open+0xb6
open(cf701190,ef168d04) at open+0x1a
syscall(3b,3b,3b,4,28242df8,...) at syscall+0x2bf
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (5, FreeBSD ELF32, open), eip = 0x282114b3, esp =
0xbfbfe8fc, ebp = 0xbfbfe928 ---
db> bt
Tracing pid 16 tid 100000 td 0xc7cfe000
kdb_enter(c09408b4) at kdb_enter+0x2b
siointr1(c7f93000) at siointr1+0xce
siointr(c7f93000) at siointr+0x5e
intr_execute_handlers(c7cf24c8,e687ac94,4,e687acd8,c0899743,...) at
intr_execute_handlers+0xe1
lapic_handle_intr(37) at lapic_handle_intr+0x2e
Xapic_isr1() at Xapic_isr1+0x33
--- interrupt, eip = 0xc0b96165, esp = 0xe687acd8, ebp = 0xe687acd8 ---
acpi_cpu_c1(7b404985,35f2a28c,c7cfe000,c7cfe000,2,...) at acpi_cpu_c1+0x5
acpi_cpu_idle(e687ad10,c066c435,c7cfc000,c066c3a0,e687ad24,...) at
acpi_cpu_idle+0x152
cpu_idle(c7cfc000,c066c3a0,e687ad24,c066c121,0,...) at cpu_idle+0x28
idle_proc(0,e687ad38) at idle_proc+0x95
fork_exit(c066c3a0,0,e687ad38) at fork_exit+0x71
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe687ad6c, ebp = 0 ---
All 8 CPUs are in idle state.
--
wbr,
pluknet
More information about the freebsd-stable
mailing list