machine wedged -> KDB: enter: lock violation

Brad Pitney pitney.brad at googlemail.com
Wed Mar 19 20:39:09 PDT 2008 

Not sure why it keeps wedging, at first I thought it was something to
do with the  LORs, now after adding some more debugging options I
think I might have found the answer!

KDB: stack backtrace:
db_trace_self_wrapper(c074b5ee,e70599ac,c05b6853,c4a9e000,e70599ac,...)
at db_trace_self_wrapper+0x26
kdb_backtrace(c4a9e000,e70599ac,c07025c5,e70599bc,c4c44d98,...) at
kdb_backtrace+0x29
vfs_badlock(c4a37900,e70599bc,c07b00a0,c4c44d98,c4a9e000) at vfs_badlock+0x23
assert_vop_elocked(c4c44d98,c0752ee7,c4a9e000,1b9,0,...) at
assert_vop_elocked+0x53
cache_lookup(c4c4815c,e7059bc0,e7059bd4,e7059bc0,c4aa4400,...) at
cache_lookup+0x53c
vfs_cache_lookup(e7059aa8,c07545ba,c4c4815c,2,c4c4815c,...) at
vfs_cache_lookup+0xaa
VOP_LOOKUP_APV(c4a37900,e7059aa8,c4a9e000,c075356a,19b,...) at
VOP_LOOKUP_APV+0xe5
lookup(e7059bac,e7059ae8,c6,bf,c4aa542c,...) at lookup+0x53e
namei(e7059bac,2,c0754d92,c0577808,c0811ae0,...) at namei+0x28e
kern_stat(c4a9e000,2820258c,0,e7059c1c,c074d152,...) at kern_stat+0x3d
stat(c4a9e000,e7059cfc,8,c074e1dc,c0785e00,...) at stat+0x2f
syscall(e7059d38) at syscall+0x273
Xint0x80_syscall() at Xint0x80_syscall+0x20
--- syscall (188, FreeBSD ELF32, stat), eip = 0x281aa48f, esp =
0xbfbfea4c, ebp = 0xbfbfeae8 ---
cache_lookup: 0xc4c44d98 is not exclusive locked but should be
KDB: enter: lock violation

Locked vnodes

0xc4a10828: tag ufs, type VDIR
    usecount 1, writecount 0, refcount 3 mountedhere 0
    flags ()
     lock type ufs: EXCL (count 1) by thread 0xc4a9e000 (pid 85094)#0
0xc052fe50 at _lockmgr+0x560
#1 0xc0668df1 at ffs_lock+0x91
#2 0xc0705775 at VOP_LOCK1_APV+0xa5
#3 0xc4a345aa at unionfs_lock+0x25a
#4 0xc0705775 at VOP_LOCK1_APV+0xa5
#5 0xc05c53b8 at _vn_lock+0xf8
#6 0xc05ba130 at vget+0x110
#7 0xc05a9657 at cache_lookup+0x4e7
#8 0xc05a978a at vfs_cache_lookup+0xaa
#9 0xc0706915 at VOP_LOOKUP_APV+0xe5
#10 0xc05af3ee at lookup+0x53e
#11 0xc05afe6e at namei+0x28e
#12 0xc05bd5ad at kern_stat+0x3d
#13 0xc05bd73f at stat+0x2f
#14 0xc06ec7e3 at syscall+0x273
#15 0xc06d5f30 at Xint0x80_syscall+0x20

        ino 499318, on dev ad0s3a

0xc49cc414: tag ufs, type VDIR
    usecount 1, writecount 0, refcount 3 mountedhere 0
    flags ()
     lock type ufs: EXCL (count 1) by thread 0xc4a9e000 (pid 85094)#0
0xc052fe50 at _lockmgr+0x560
#1 0xc0668df1 at ffs_lock+0x91
#2 0xc0705775 at VOP_LOCK1_APV+0xa5
#3 0xc4a344c3 at unionfs_lock+0x173
#4 0xc0705775 at VOP_LOCK1_APV+0xa5
#5 0xc05c53b8 at _vn_lock+0xf8
#6 0xc05ba130 at vget+0x110
#7 0xc05a9657 at cache_lookup+0x4e7
#8 0xc05a978a at vfs_cache_lookup+0xaa
#9 0xc0706915 at VOP_LOOKUP_APV+0xe5
#10 0xc05af3ee at lookup+0x53e
#11 0xc05afe6e at namei+0x28e
#12 0xc05bd5ad at kern_stat+0x3d
#13 0xc05bd73f at stat+0x2f
#14 0xc06ec7e3 at syscall+0x273
#15 0xc06d5f30 at Xint0x80_syscall+0x20

        ino 141664, on dev ad0s3a

0xc48f62b8: tag ufs, type VREG
    usecount 6, writecount 0, refcount 9 mountedhere 0
    flags (VV_TEXT)
    v_object 0xc4c5f364 ref 3 pages 63
     lock type ufs: SHARED (count 1)#0 0xc052fab5 at _lockmgr+0x1c5
#1 0xc0668df1 at ffs_lock+0x91
#2 0xc0705775 at VOP_LOCK1_APV+0xa5
#3 0xc05c53b8 at _vn_lock+0xf8
#4 0xc05ba130 at vget+0x110
#5 0xc0699c20 at vnode_pager_lock+0x1b0
#6 0xc06825df at vm_fault+0x1df
#7 0xc06ec478 at trap_pfault+0x118
#8 0xc06ecd07 at trap+0x2b7
#9 0xc06d5ecb at calltrap+0x6

        ino 144832, on dev ad0s3a

0xc4c4815c: tag unionfs, type VDIR
    usecount 2, writecount 0, refcount 3 mountedhere 0
    flags ()
     lock type ufs: EXCL (count 1) by thread 0xc4a9e000 (pid 85094)#0
0xc052fe50 at _lockmgr+0x560
#1 0xc0668df1 at ffs_lock+0x91
#2 0xc0705775 at VOP_LOCK1_APV+0xa5
#3 0xc4a345aa at unionfs_lock+0x25a
#4 0xc0705775 at VOP_LOCK1_APV+0xa5
#5 0xc05c53b8 at _vn_lock+0xf8
#6 0xc05ba130 at vget+0x110
#7 0xc05a9657 at cache_lookup+0x4e7
#8 0xc05a978a at vfs_cache_lookup+0xaa
#9 0xc0706915 at VOP_LOOKUP_APV+0xe5
#10 0xc05af3ee at lookup+0x53e
#11 0xc05afe6e at namei+0x28e
#12 0xc05bd5ad at kern_stat+0x3d
#13 0xc05bd73f at stat+0x2f
#14 0xc06ec7e3 at syscall+0x273
#15 0xc06d5f30 at Xint0x80_syscall+0x20

unionfs_vp=0xc4c4815c, uppervp=0xc4a10828, lowervp=0xc49cc414
unionfs: upper
0xc4a10828: tag ufs, type VDIR
    usecount 1, writecount 0, refcount 3 mountedhere 0
    flags ()
     lock type ufs: EXCL (count 1) by thread 0xc4a9e000 (pid 85094)#0
0xc052fe50 at _lockmgr+0x560
#1 0xc0668df1 at ffs_lock+0x91
#2 0xc0705775 at VOP_LOCK1_APV+0xa5
#3 0xc4a345aa at unionfs_lock+0x25a
#4 0xc0705775 at VOP_LOCK1_APV+0xa5
#5 0xc05c53b8 at _vn_lock+0xf8
#6 0xc05ba130 at vget+0x110
#7 0xc05a9657 at cache_lookup+0x4e7
#8 0xc05a978a at vfs_cache_lookup+0xaa
#9 0xc0706915 at VOP_LOOKUP_APV+0xe5
#10 0xc05af3ee at lookup+0x53e
#11 0xc05afe6e at namei+0x28e
#12 0xc05bd5ad at kern_stat+0x3d
#13 0xc05bd73f at stat+0x2f
#14 0xc06ec7e3 at syscall+0x273
#15 0xc06d5f30 at Xint0x80_syscall+0x20

        ino 499318, on dev ad0s3a
unionfs: lower
0xc49cc414: tag ufs, type VDIR
    usecount 1, writecount 0, refcount 3 mountedhere 0
    flags ()
     lock type ufs: EXCL (count 1) by thread 0xc4a9e000 (pid 85094)#0
0xc052fe50 at _lockmgr+0x560
#1 0xc0668df1 at ffs_lock+0x91
#2 0xc0705775 at VOP_LOCK1_APV+0xa5
#3 0xc4a344c3 at unionfs_lock+0x173
#4 0xc0705775 at VOP_LOCK1_APV+0xa5
#5 0xc05c53b8 at _vn_lock+0xf8
#6 0xc05ba130 at vget+0x110
#7 0xc05a9657 at cache_lookup+0x4e7
#8 0xc05a978a at vfs_cache_lookup+0xaa
#9 0xc0706915 at VOP_LOOKUP_APV+0xe5
#10 0xc05af3ee at lookup+0x53e
#11 0xc05afe6e at namei+0x28e
#12 0xc05bd5ad at kern_stat+0x3d
#13 0xc05bd73f at stat+0x2f
#14 0xc06ec7e3 at syscall+0x273
#15 0xc06d5f30 at Xint0x80_syscall+0x20

        ino 141664, on dev ad0s3a
Locked vnodes

0xc4a10828: tag ufs, type VDIR
    usecount 1, writecount 0, refcount 3 mountedhere 0
    flags ()
     lock type ufs: EXCL (count 1) by thread 0xc4a9e000 (pid 85094)#0
0xc052fe50 at _lockmgr+0x560
#1 0xc0668df1 at ffs_lock+0x91
#2 0xc0705775 at VOP_LOCK1_APV+0xa5
#3 0xc4a345aa at unionfs_lock+0x25a
#4 0xc0705775 at VOP_LOCK1_APV+0xa5
#5 0xc05c53b8 at _vn_lock+0xf8
#6 0xc05ba130 at vget+0x110
#7 0xc05a9657 at cache_lookup+0x4e7
#8 0xc05a978a at vfs_cache_lookup+0xaa
#9 0xc0706915 at VOP_LOOKUP_APV+0xe5
#10 0xc05af3ee at lookup+0x53e
#11 0xc05afe6e at namei+0x28e
#12 0xc05bd5ad at kern_stat+0x3d
#13 0xc05bd73f at stat+0x2f
#14 0xc06ec7e3 at syscall+0x273
#15 0xc06d5f30 at Xint0x80_syscall+0x20

        ino 499318, on dev ad0s3a

0xc49cc414: tag ufs, type VDIR
    usecount 1, writecount 0, refcount 3 mountedhere 0
    flags ()
     lock type ufs: EXCL (count 1) by thread 0xc4a9e000 (pid 85094)#0
0xc052fe50 at _lockmgr+0x560
#1 0xc0668df1 at ffs_lock+0x91
#2 0xc0705775 at VOP_LOCK1_APV+0xa5
#3 0xc4a344c3 at unionfs_lock+0x173
#4 0xc0705775 at VOP_LOCK1_APV+0xa5
#5 0xc05c53b8 at _vn_lock+0xf8
#6 0xc05ba130 at vget+0x110
#7 0xc05a9657 at cache_lookup+0x4e7
#8 0xc05a978a at vfs_cache_lookup+0xaa
#9 0xc0706915 at VOP_LOOKUP_APV+0xe5
#10 0xc05af3ee at lookup+0x53e
#11 0xc05afe6e at namei+0x28e
#12 0xc05bd5ad at kern_stat+0x3d
#13 0xc05bd73f at stat+0x2f
#14 0xc06ec7e3 at syscall+0x273
#15 0xc06d5f30 at Xint0x80_syscall+0x20

        ino 141664, on dev ad0s3a

0xc48f62b8: tag ufs, type VREG
    usecount 6, writecount 0, refcount 9 mountedhere 0
    flags (VV_TEXT)
    v_object 0xc4c5f364 ref 3 pages 63
     lock type ufs: SHARED (count 1)#0 0xc052fab5 at _lockmgr+0x1c5
#1 0xc0668df1 at ffs_lock+0x91
#2 0xc0705775 at VOP_LOCK1_APV+0xa5
#3 0xc05c53b8 at _vn_lock+0xf8
#4 0xc05ba130 at vget+0x110
#5 0xc0699c20 at vnode_pager_lock+0x1b0
#6 0xc06825df at vm_fault+0x1df
#7 0xc06ec478 at trap_pfault+0x118
#8 0xc06ecd07 at trap+0x2b7
#9 0xc06d5ecb at calltrap+0x6

        ino 144832, on dev ad0s3a

0xc4c4815c: tag unionfs, type VDIR
    usecount 2, writecount 0, refcount 3 mountedhere 0
    flags ()
     lock type ufs: EXCL (count 1) by thread 0xc4a9e000 (pid 85094)#0
0xc052fe50 at _lockmgr+0x560
#1 0xc0668df1 at ffs_lock+0x91
#2 0xc0705775 at VOP_LOCK1_APV+0xa5
#3 0xc4a345aa at unionfs_lock+0x25a
#4 0xc0705775 at VOP_LOCK1_APV+0xa5
#5 0xc05c53b8 at _vn_lock+0xf8
#6 0xc05ba130 at vget+0x110
#7 0xc05a9657 at cache_lookup+0x4e7
#8 0xc05a978a at vfs_cache_lookup+0xaa
#9 0xc0706915 at VOP_LOOKUP_APV+0xe5
#10 0xc05af3ee at lookup+0x53e
#11 0xc05afe6e at namei+0x28e
#12 0xc05bd5ad at kern_stat+0x3d
#13 0xc05bd73f at stat+0x2f
#14 0xc06ec7e3 at syscall+0x273
#15 0xc06d5f30 at Xint0x80_syscall+0x20

unionfs_vp=0xc4c4815c, uppervp=0xc4a10828, lowervp=0xc49cc414
unionfs: upper
0xc4a10828: tag ufs, type VDIR
    usecount 1, writecount 0, refcount 3 mountedhere 0
    flags ()
     lock type ufs: EXCL (count 1) by thread 0xc4a9e000 (pid 85094)#0
0xc052fe50 at _lockmgr+0x560
#1 0xc0668df1 at ffs_lock+0x91
#2 0xc0705775 at VOP_LOCK1_APV+0xa5
#3 0xc4a345aa at unionfs_lock+0x25a
#4 0xc0705775 at VOP_LOCK1_APV+0xa5
#5 0xc05c53b8 at _vn_lock+0xf8
#6 0xc05ba130 at vget+0x110
#7 0xc05a9657 at cache_lookup+0x4e7
#8 0xc05a978a at vfs_cache_lookup+0xaa
#9 0xc0706915 at VOP_LOOKUP_APV+0xe5
#10 0xc05af3ee at lookup+0x53e
#11 0xc05afe6e at namei+0x28e
#12 0xc05bd5ad at kern_stat+0x3d
#13 0xc05bd73f at stat+0x2f
#14 0xc06ec7e3 at syscall+0x273
#15 0xc06d5f30 at Xint0x80_syscall+0x20

        ino 499318, on dev ad0s3a
unionfs: lower
0xc49cc414: tag ufs, type VDIR
    usecount 1, writecount 0, refcount 3 mountedhere 0
    flags ()
     lock type ufs: EXCL (count 1) by thread 0xc4a9e000 (pid 85094)#0
0xc052fe50 at _lockmgr+0x560
#1 0xc0668df1 at ffs_lock+0x91
#2 0xc0705775 at VOP_LOCK1_APV+0xa5
#3 0xc4a344c3 at unionfs_lock+0x173
#4 0xc0705775 at VOP_LOCK1_APV+0xa5
#5 0xc05c53b8 at _vn_lock+0xf8
#6 0xc05ba130 at vget+0x110
#7 0xc05a9657 at cache_lookup+0x4e7
#8 0xc05a978a at vfs_cache_lookup+0xaa
#9 0xc0706915 at VOP_LOOKUP_APV+0xe5
#10 0xc05af3ee at lookup+0x53e
#11 0xc05afe6e at namei+0x28e
#12 0xc05bd5ad at kern_stat+0x3d
#13 0xc05bd73f at stat+0x2f
#14 0xc06ec7e3 at syscall+0x273
#15 0xc06d5f30 at Xint0x80_syscall+0x20

        ino 141664, on dev ad0s3a


the box was running CURRENT until it was branched for RELENG_7 which
was running code from September 2007 fine until I updated to "todays"
code I get my problems.

I don't know, I've been running FreeBSD for a few years, never had
anything this serious

-- 
Best regards,
 Brad

More information about the freebsd-stable mailing list