kern/120529: LOR opensolaris_vfs.c:206 and kern_resource.c:1273 - 7.0-RC2

Mon Feb 11 10:50:01 UTC 2008

>Number:         120529
>Category:       kern
>Synopsis:       LOR opensolaris_vfs.c:206 and kern_resource.c:1273 - 7.0-RC2
>Confidential:   no
>Severity:       non-critical
>Priority:       medium
>Responsible:    freebsd-bugs
>State:          open
>Quarter:        
>Keywords:       
>Date-Required:
>Class:          sw-bug
>Submitter-Id:   current-users
>Arrival-Date:   Mon Feb 11 10:50:01 UTC 2008
>Closed-Date:
>Last-Modified:
>Originator:     Henri Hennebert
>Release:        FreeBSD 7.0-RC2
>Organization:
>Environment:
FreeBSD morzine.restart.bel 7.0-RC2 FreeBSD 7.0-RC2 #0: Thu Feb  7 19:39:23 CET 2008     root at morzine.restart.bel:/usr/obj/usr/src/sys/MORZINE  i386

>Description:
During heavy io activity between 4 zpools -- one on a usb disk -- I get in log/messages:

At Feb 10 12:23:26

lock order reversal:
1st 0xa8157804 struct mount mtx (struct mount mtx) @ /usr/src/sys/modules/zfs/../../compat/opensolaris/kern/opensolaris_vfs.c:206
2nd 0xa54f0010 sleep mtxpool (sleep mtxpool) @ /usr/src/sys/kern/kern_resource.c:1273
KDB: stack backtrace:
db_trace_self_wrapper(a07d3fd1,f9b3773c,a05ca405,a07d5389,a54f0010,...) at db_trace_self_wrapper+0x26
kdb_backtrace(a07d5389,a54f0010,a07d135c,a07d135c,a07d1914,...) at kdb_backtrace+0x29
witness_checkorder(a54f0010,9,a07d1914,4f9,38,...) at witness_checkorder+0x5e5
_mtx_lock_flags(a54f0010,0,a07d1914,4f9,a8232a00,...) at _mtx_lock_flags+0x34
uifree(a5522020,adfbdbb0,a81577d4,f9b377d8,a098cebf,...) at uifree+0x2d
crfree(a8232a00,0,a0a049bc,ce,c2,...) at crfree+0x27
domount(a822e880,adfbdbb0,a0a09ae1,ac7a5b20,f9b37814,...) at domount+0x1ff
zfsctl_snapdir_lookup(f9b37aa0,f9b37a58,2,a8aeb770,f9b37ac0,...) at zfsctl_snapdir_lookup+0x362
VOP_LOOKUP_APV(a0a0daa0,f9b37aa0,a822e880,a07d9b3e,19b,...) at VOP_LOOKUP_APV+0x46
lookup(f9b37ba8,a07d9b3e,c6,bf,a691602c,...) at lookup+0x57e
namei(f9b37ba8,2,281d5000,a058de02,a085d434,...) at namei+0x2bb
kern_stat(a822e880,282226c4,0,f9b37c18,a07e7092,...) at kern_stat+0x3d
stat(a822e880,f9b37cfc,8,a822e880,a080c060,...) at stat+0x2f
syscall(f9b37d38) at syscall+0x2b3
Xint0x80_syscall() at Xint0x80_syscall+0x20
--- syscall (188, FreeBSD ELF32, stat), eip = 0x281b632b, esp = 0x9fbfe8ec, ebp = 0x9fbfe9f8 ---
uma_zalloc_arg: zone "256" with the following non-sleepable locks held:
exclusive sleep mutex struct mount mtx r = 0 (0xa8157804) locked @ /usr/src/sys/modules/zfs/../../compat/opensolaris/kern/opensolaris_vfs.c:206
KDB: stack backtrace:
db_trace_self_wrapper(a07d3fd1,f9b376ec,a05ca76d,a07d4342,f9b37700,...) at db_trace_self_wrapper+0x26
kdb_backtrace(a07d4342,f9b37700,4,1,0,...) at kdb_backtrace+0x29
witness_warn(5,0,a07ea73d,a07e6167,a822e880,...) at witness_warn+0x1cd
uma_zalloc_arg(a18671e0,0,102,a81577d4,adfbdbb0,...) at uma_zalloc_arg+0x34
malloc(a0,a0811ba0,102,a81577d4,f9b37794,...) at malloc+0x51
crget(a8232a00,a0811ba0,a81577d4,f9b377d8,a098cecc,...) at crget+0x23
crdup(a5520700,0,a0a049bc,ce,c2,...) at crdup+0xc
domount(a822e880,adfbdbb0,a0a09ae1,ac7a5b20,f9b37814,...) at domount+0x20c
zfsctl_snapdir_lookup(f9b37aa0,f9b37a58,2,a8aeb770,f9b37ac0,...) at zfsctl_snapdir_lookup+0x362
VOP_LOOKUP_APV(a0a0daa0,f9b37aa0,a822e880,a07d9b3e,19b,...) at VOP_LOOKUP_APV+0x46
lookup(f9b37ba8,a07d9b3e,c6,bf,a691602c,...) at lookup+0x57e
namei(f9b37ba8,2,281d5000,a058de02,a085d434,...) at namei+0x2bb
kern_stat(a822e880,282226c4,0,f9b37c18,a07e7092,...) at kern_stat+0x3d
stat(a822e880,f9b37cfc,8,a822e880,a080c060,...) at stat+0x2f
syscall(f9b37d38) at syscall+0x2b3
Xint0x80_syscall() at Xint0x80_syscall+0x20
--- syscall (188, FreeBSD ELF32, stat), eip = 0x281b632b, esp = 0x9fbfe8ec, ebp = 0x9fbfe9f8 ---

At Feb 10 12:24:25

uma_zalloc_arg: zone "256" with the following non-sleepable locks held:
exclusive sleep mutex struct mount mtx r = 0 (0xa815e804) locked @ /usr/src/sys/modules/zfs/../../compat/opensolaris/kern/opensolaris_vfs.c:206
KDB: stack backtrace:
db_trace_self_wrapper(a07d3fd1,f9b376ec,a05ca76d,a07d4342,f9b37700,...) at db_trace_self_wrapper+0x26
kdb_backtrace(a07d4342,f9b37700,4,1,0,...) at kdb_backtrace+0x29
witness_warn(5,0,a07ea73d,a07e6167,a822e880,...) at witness_warn+0x1cd
uma_zalloc_arg(a18671e0,0,102,a815e7d4,b39fe330,...) at uma_zalloc_arg+0x34
malloc(a0,a0811ba0,102,a815e7d4,f9b37794,...) at malloc+0x51
crget(a8021700,a0811ba0,a815e7d4,f9b377d8,a098cecc,...) at crget+0x23
crdup(a5520700,0,a0a049bc,ce,c2,...) at crdup+0xc
domount(a822e880,b39fe330,a0a09ae1,ab5f08a0,f9b37814,...) at domount+0x20c
zfsctl_snapdir_lookup(f9b37aa0,f9b37a58,2,b0a11aa0,f9b37ac0,...) at zfsctl_snapdir_lookup+0x362
VOP_LOOKUP_APV(a0a0daa0,f9b37aa0,a822e880,a07d9b3e,2a9,...) at VOP_LOOKUP_APV+0x46
lookup(f9b37ba8,a07d9b3e,c6,bf,a691602c,...) at lookup+0x57e
namei(f9b37ba8,2,281d5000,a058de02,a085d434,...) at namei+0x2bb
kern_stat(a822e880,282226c8,0,f9b37c18,a07e7092,...) at kern_stat+0x3d
stat(a822e880,f9b37cfc,8,a822e880,a080c060,...) at stat+0x2f
syscall(f9b37d38) at syscall+0x2b3
Xint0x80_syscall() at Xint0x80_syscall+0x20
--- syscall (188, FreeBSD ELF32, stat), eip = 0x281b632b, esp = 0x9fbfe8ec, ebp = 0x9fbfe9f8 ---

At Feb 10 12:44:33

uma_zalloc_arg: zone "256" with the following non-sleepable locks held:
exclusive sleep mutex struct mount mtx r = 0 (0xa75a22cc) locked @ /usr/src/sys/modules/zfs/../../compat/opensolaris/kern/opensolaris_vfs.c:206
KDB: stack backtrace:
db_trace_self_wrapper(a07d3fd1,f9b376ec,a05ca76d,a07d4342,f9b37700,...) at db_trace_self_wrapper+0x26
kdb_backtrace(a07d4342,f9b37700,4,1,0,...) at kdb_backtrace+0x29
witness_warn(5,0,a07ea73d,a07e6167,a822e880,...) at witness_warn+0x1cd
uma_zalloc_arg(a18671e0,0,102,a75a229c,ab89f440,...) at uma_zalloc_arg+0x34
malloc(a0,a0811ba0,102,a75a229c,f9b37794,...) at malloc+0x51
crget(a7a57b00,a0811ba0,a75a229c,f9b377d8,a098cecc,...) at crget+0x23
crdup(a5520700,0,a0a049bc,ce,c2,...) at crdup+0xc
domount(a822e880,ab89f440,a0a09ae1,ac7a54e0,f9b37814,...) at domount+0x20c
zfsctl_snapdir_lookup(f9b37aa0,f9b37a58,2,a81e7440,f9b37ac0,...) at zfsctl_snapdir_lookup+0x362
VOP_LOOKUP_APV(a0a0daa0,f9b37aa0,a822e880,a07d9b3e,2a9,...) at VOP_LOOKUP_APV+0x46
lookup(f9b37ba8,a07d9b3e,c6,bf,a691602c,...) at lookup+0x57e
namei(f9b37ba8,2,281d5000,a058de02,a085d434,...) at namei+0x2bb
kern_stat(a822e880,282226c8,0,f9b37c18,a07e7092,...) at kern_stat+0x3d
stat(a822e880,f9b37cfc,8,a822e880,a080c060,...) at stat+0x2f
syscall(f9b37d38) at syscall+0x2b3
Xint0x80_syscall() at Xint0x80_syscall+0x20
--- syscall (188, FreeBSD ELF32, stat), eip = 0x281b632b, esp = 0x9fbfe8ec, ebp = 0x9fbfe9f8 ---

At Feb 10 12:45:59

uma_zalloc_arg: zone "256" with the following non-sleepable locks held:
exclusive sleep mutex struct mount mtx r = 0 (0xa815e2cc) locked @ /usr/src/sys/modules/zfs/../../compat/opensolaris/kern/opensolaris_vfs.c:206
KDB: stack backtrace:
db_trace_self_wrapper(a07d3fd1,f9b376ec,a05ca76d,a07d4342,f9b37700,...) at db_trace_self_wrapper+0x26
kdb_backtrace(a07d4342,f9b37700,4,1,0,...) at kdb_backtrace+0x29
witness_warn(5,0,a07ea73d,a07e6167,a822e880,...) at witness_warn+0x1cd
uma_zalloc_arg(a18671e0,0,102,a815e29c,af419220,...) at uma_zalloc_arg+0x34
malloc(a0,a0811ba0,102,a815e29c,f9b37794,...) at malloc+0x51
crget(a7e4d400,a0811ba0,a815e29c,f9b377d8,a098cecc,...) at crget+0x23
crdup(a5520700,0,a0a049bc,ce,c2,...) at crdup+0xc
domount(a822e880,af419220,a0a09ae1,ad8b6de0,f9b37814,...) at domount+0x20c
zfsctl_snapdir_lookup(f9b37aa0,f9b37a58,2,aa178330,f9b37ac0,...) at zfsctl_snapdir_lookup+0x362
VOP_LOOKUP_APV(a0a0daa0,f9b37aa0,a822e880,a07d9b3e,2a9,...) at VOP_LOOKUP_APV+0x46
lookup(f9b37ba8,a07d9b3e,c6,bf,a691602c,...) at lookup+0x57e
namei(f9b37ba8,2,281d5000,a058de02,a085d434,...) at namei+0x2bb
kern_stat(a822e880,282226d0,0,f9b37c18,a07e7092,...) at kern_stat+0x3d
stat(a822e880,f9b37cfc,8,a822e880,a080c060,...) at stat+0x2f
syscall(f9b37d38) at syscall+0x2b3
Xint0x80_syscall() at Xint0x80_syscall+0x20
--- syscall (188, FreeBSD ELF32, stat), eip = 0x281b632b, esp = 0x9fbfe8ec, ebp = 0x9fbfe9f8 ---

At Feb 10 13:10:03

uma_zalloc_arg: zone "256" with the following non-sleepable locks held:
exclusive sleep mutex struct mount mtx r = 0 (0xa8157aa0) locked @ /usr/src/sys/modules/zfs/../../compat/opensolaris/kern/opensolaris_vfs.c:206
KDB: stack backtrace:
db_trace_self_wrapper(a07d3fd1,f9b376ec,a05ca76d,a07d4342,f9b37700,...) at db_trace_self_wrapper+0x26
kdb_backtrace(a07d4342,f9b37700,4,1,0,...) at kdb_backtrace+0x29
witness_warn(5,0,a07ea73d,a07e6167,a822e880,...) at witness_warn+0x1cd
uma_zalloc_arg(a18671e0,0,102,a8157a70,afc33dd0,...) at uma_zalloc_arg+0x34
malloc(a0,a0811ba0,102,a8157a70,f9b37794,...) at malloc+0x51
crget(a7a39700,a0811ba0,a8157a70,f9b377d8,a098cecc,...) at crget+0x23
crdup(a5520700,0,a0a049bc,ce,c2,...) at crdup+0xc
domount(a822e880,afc33dd0,a0a09ae1,afb38060,f9b37814,...) at domount+0x20c
zfsctl_snapdir_lookup(f9b37aa0,f9b37a58,2,ab2ee000,f9b37ac0,...) at zfsctl_snapdir_lookup+0x362
VOP_LOOKUP_APV(a0a0daa0,f9b37aa0,a822e880,a07d9b3e,2a9,...) at VOP_LOOKUP_APV+0x46
lookup(f9b37ba8,a07d9b3e,c6,bf,a691602c,...) at lookup+0x57e
namei(f9b37ba8,2,281d5000,a058de02,a085d434,...) at namei+0x2bb
kern_stat(a822e880,282226c8,0,f9b37c18,a07e7092,...) at kern_stat+0x3d
stat(a822e880,f9b37cfc,8,a822e880,a080c060,...) at stat+0x2f
syscall(f9b37d38) at syscall+0x2b3
Xint0x80_syscall() at Xint0x80_syscall+0x20
--- syscall (188, FreeBSD ELF32, stat), eip = 0x281b632b, esp = 0x9fbfe8ec, ebp = 0x9fbfe9f8 ---

>How-To-Repeat:
I believe with heavy io activities...
>Fix:


>Release-Note:
>Audit-Trail:
>Unformatted: