Lock order reversals

[Kai Lockwood]

hp rx2600, 4GB RAM, FreeBSD 8.0-CURRENT-200906 #0: Fri Jun 12 22:56:41 
UTC 2009 /usr/obj/usr/src/sys/GENERIC  ia64


These are displayed on the console:


lock order reversal:
 1st 0xa00000003d48c7b8 bufwait (bufwait) @ /usr/src/sys/kern/vfs_bio.c:2558
 2nd 0xe000000010969a00 dirhash (dirhash) @ 
/usr/src/sys/ufs/ufs/ufs_dirhash.c:2
85
KDB: stack backtrace:
db_trace_self(0xe000000004115b50) at db_trace_self+0x20
db_trace_self_wrapper(0xe000000004654560) at db_trace_self_wrapper+0x70
kdb_backtrace(0xe000000004cf9c38, 0xe00000000467bf60) at kdb_backtrace+0xc0
_witness_debugger(0x1, 0xe000000004b4bac0, 0xe00000000467d800, 0x999, 
0xe0000000
04b73fd0) at _witness_debugger+0x60
witness_checkorder(0xe000000010969a00, 0x9, 0xffffffffffffffff, 0x11d, 
0x0) at w
itness_checkorder+0x12c0
_sx_xlock(0xe000000010969a00, 0x0, 0xe000000004b73fd0, 0x11d) at 
_sx_xlock+0xc0
ufsdirhash_acquire(0xe000000010cee738, 0xe000000010969a00, 
0xe000000004a0ef40, 0
x38b) at ufsdirhash_acquire+0x50
ufsdirhash_remove(0xe000000010cee738, 0xa00000003f9ba068, 0x68, 
0xa000000062caf3
08) at ufsdirhash_remove+0x20
ufs_dirremove(0xe000000010cfc938, 0xe000000010db8150, 0x0, 0x0) at 
ufs_dirremove
+0x240
ufs_remove(0xa000000062caf3d8, 0xe000000010db8150, 0xe00000000471c7f0) 
at ufs_re
move+0xe0
VOP_REMOVE_APV(0xe000000004cc21c0, 0xa000000062caf3d8, 
0xe000000010dbcce8, 0xe00
000000471c860) at VOP_REMOVE_APV+0x1c0
kern_unlinkat(0xe0000000109fb270, 0xffffffffffffff9c, 
0x20000000408280c0, 0x0) a
t kern_unlinkat+0x2f0
kern_unlink(0xe0000000109fb270, 0x20000000408280c0, 0x0) at kern_unlink+0x30
unlink(0xe0000000109fb270, 0xa000000062caf4e8, 0xe000000004ac03e0, 
0x58f) at unl
ink+0x30
syscall(0xa000000062caf400, 0xa, 0x8, 0xe0000000109fb270, 
0xe0000000109f6448, 0x
e000000004c95528, 0xa, 0xa000000062caf4e8) at syscall+0x3e0
epc_syscall_return() at epc_syscall_return
lock order reversal:
 1st 0xe000000010c996b8 ufs (ufs) @ /usr/src/sys/kern/vfs_mount.c:1197
 2nd 0xe000000010ba3308 devfs (devfs) @ /usr/src/sys/kern/vfs_subr.c:2189
KDB: stack backtrace:
db_trace_self(0xe000000004115b50) at db_trace_self+0x20
db_trace_self_wrapper(0xe000000004654560) at db_trace_self_wrapper+0x70
kdb_backtrace(0xe000000004cf9c38, 0xe00000000467bf60) at kdb_backtrace+0xc0
_witness_debugger(0x1, 0xe000000004b4bac0, 0xe00000000467d800, 0x999, 
0xe0000000
04b54550) at _witness_debugger+0x60
witness_checkorder(0xe000000010ba3308, 0x9, 0xffffffffffffffff, 0x88d, 
0x0) at w
itness_checkorder+0x12c0
__lockmgr_args(0xe000000010ba3308, 0x80100, 0xe000000010ba3330, 
0xe000000004b34f
60, 0x50, 0x33, 0xe000000004b54550, 0x88d) at __lockmgr_args+0xe10
vop_stdlock(0xa000000062c6f380, 0xe000000004acdc90, 0x309) at 
vop_stdlock+0x80
VOP_LOCK1_APV(0xe000000004cae908, 0xa000000062c6f360, 0x4) at 
VOP_LOCK1_APV+0x1d
0
_vn_lock(0xe000000010ba3270, 0x80100, 0xe000000004b54550, 0x88d) at 
_vn_lock+0xf
0
vrele(0xe000000010ba3270, 0xe000000010ba3330, 0xe000000010a00b10, 
0xe000000010ba
3360) at vrele+0x240
cd9660_unmount(0xe000000010a90000, 0x1, 0x1, 0xe000000010c99508, 
0xe000000004d1a
0c0) at cd9660_unmount+0x410
dounmount(0xe000000010c99620, 0x80000, 0x1001, 0xe000000004cfa268) at 
dounmount+
0x990
unmount(0xe000000010a00b10, 0x0, 0xe000000010a90000, 0xe000000011916a80) 
at unmo
unt+0x5c0
syscall(0xa000000062c6f400, 0x16, 0x58f, 0xe000000010a00b10, 
0xe0000000109f8000,
 0xe000000004c95768, 0x16, 0xa000000062c6f4e8) at syscall+0x3e0
epc_syscall_return() at epc_syscall_return
lock order reversal:
 1st 0xe000000014c174e0 ufs (ufs) @ /usr/src/sys/kern/vfs_lookup.c:495
 2nd 0xa00000003daa2e98 bufwait (bufwait) @ 
/usr/src/sys/ufs/ffs/ffs_softdep.c:6
150
 3rd 0xe00000001549b6b8 ufs (ufs) @ /usr/src/sys/kern/vfs_subr.c:2084
KDB: stack backtrace:
db_trace_self(0xe000000004115b50) at db_trace_self+0x20
db_trace_self_wrapper(0xe000000004654560) at db_trace_self_wrapper+0x70
kdb_backtrace(0xe000000004cf9c38, 0xe00000000467bf60) at kdb_backtrace+0xc0
_witness_debugger(0x1, 0xe000000004b4bac0, 0xe00000000467d800, 0x999, 
0xe0000000
04b54550) at _witness_debugger+0x60
witness_checkorder(0xe00000001549b6b8, 0x9, 0x0, 0x824, 0x0) at 
witness_checkord
er+0x12c0
__lockmgr_args(0xe00000001549b6b8, 0x80100, 0xe00000001549b6e0, 
0xe000000004b3a1
58, 0x50, 0x33, 0xe000000004b54550, 0x824) at __lockmgr_args+0xe10
ffs_lock(0xa000000062e3ef90, 0xe00000001549b6b8, 0x80100) at ffs_lock+0x130
VOP_LOCK1_APV(0xe000000004cc19f0, 0xa000000062e3ef70, 
0xe000000004b53828) at VOP
_LOCK1_APV+0x1d0
_vn_lock(0xe00000001549b620, 0x80100, 0xe000000004b54550, 0x824) at 
_vn_lock+0xf
0
vget(0xe00000001549b620, 0x80100, 0xe00000001703b270, 0x50) at vget+0x160
vfs_hash_get(0xe000000010a902f0, 0xc7836e, 0xe00000001549b620, 
0xe00000001703b27
0, 0xa000000062e3efc8, 0x0, 0x0, 0xe000000004b53828) at vfs_hash_get+0x180
ffs_vgetf(0xe000000010a902f0, 0xc7836e, 0x80000, 0xa000000062e3efc8, 
0x1, 0xea5,
 0xea5, 0xe000000004ed4250) at ffs_vgetf+0x50
softdep_sync_metadata(0xe000000014c17448, 0xe000000010d66a9a, 
0xe000000004b71d50
, 0x131, 0xe000000004a07b50, 0xe000000010d66a80, 0xc7836e, 
0xe000000004b71d50) a
t softdep_sync_metadata+0xb10
ffs_syncvnode(0xe000000014c17448, 0x0, 0x0, 0xa00000003daa2e98, 0xea3, 
0xe000000
004b73ac8) at ffs_syncvnode+0x760
ffs_truncate(0xe000000014c17448, 0x800, 0x880, 0xe00000001099e600, 
0xe0000000170
3b270, 0x0) at ffs_truncate+0x990
ufs_direnter(0xe000000014c17448, 0x0, 0xa000000062e3f150, 
0xa000000062e3f380, 0x
0) at ufs_direnter+0x14d0
ufs_rename(0xa000000062e3f3c8, 0xa000000062e3f150, 0xa000000062e3f148) 
at ufs_re
name+0xdc0
VOP_RENAME_APV(0xe000000004cc21c0, 0xa000000062e3f3c8, 0x0, 
0xe000000004721550,
0x815) at VOP_RENAME_APV+0x1d0
kern_renameat(0xa000000062e3f380, 0xa000000062e3f3c8, 0x0, 
0xffffffffffffff9c, 0
x9fffffffffffeae6, 0x0) at kern_renameat+0x5b0
kern_rename(0xe00000001703b270, 0x9fffffffffffeadb, 0x9fffffffffffeae6, 
0x0) at
kern_rename+0x40
rename(0xe00000001703b270, 0xa000000062e3f4e8, 0xe000000004ac03e0, 
0x58f) at ren
ame+0x30
syscall(0xa000000062e3f400, 0x80, 0x2, 0xe00000001703b270, 
0xe000000019c42890, 0
xe000000004c96b48, 0x80, 0xa000000062e3f4e8) at syscall+0x3e0
epc_syscall_return() at epc_syscall_return
Jun 27 05:02:08 tamar sshd[44866]: error: PAM: authentication error for 
illegal
user kick5 from 192.168.1.78
Jun 27 05:02:18 tamar last message repeated 2 times
panic: mtx_lock_spin() of destroyed mutex @ 
/usr/src/sys/ia64/ia64/trap.c:680
cpuid = 1
KDB: enter: panic
[thread pid 49714 tid 100137 ]
Stopped at      kdb_enter+0x92: [I2]    addl r14=0xffffffffffe2a8e8,gp ;;
db>