zfs i/o hangs on 9-PRERELEASE

[Mark Felder]

13:14:32 nas:~ > uname -a
FreeBSD nas.feld.me 9.0-PRERELEASE FreeBSD 9.0-PRERELEASE #3 r227971M: 
Fri Nov 25 10:07:48 CST 2011     
root at nas.feld.me:/usr/obj/tank/svn/sys/GENERIC  amd64

This seemed to start happening sometime after RC1. I tried 8-STABLE and 
it's happening there too right now. I think whatever caused this was 
MFC'd. I've also reproduced this on completely different hardware 
running a single disk ZFS pool.


I'm getting this output in dmesg after these hangs I keep seeing.


uma_zalloc_arg: zone "pfrktable" with the following non-sleepable locks 
held:
exclusive sleep mutex pf task mtx (pf task mtx) r = 0 
(0xffffffff8199af20) locked @ 
/tank/svn/sys/modules/pf/../../contrib/pf/net/pf_ioctl.c:1589
KDB: stack backtrace:
db_trace_self_wrapper() at db_trace_self_wrapper+0x2a
kdb_backtrace() at kdb_backtrace+0x37
_witness_debugger() at _witness_debugger+0x2e
witness_warn() at witness_warn+0x2c4
uma_zalloc_arg() at uma_zalloc_arg+0x335
pfr_create_ktable() at pfr_create_ktable+0xd8
pfr_ina_define() at pfr_ina_define+0x12b
pfioctl() at pfioctl+0x1c5a
devfs_ioctl_f() at devfs_ioctl_f+0x7a
kern_ioctl() at kern_ioctl+0xcd
sys_ioctl() at sys_ioctl+0xfd
amd64_syscall() at amd64_syscall+0x3ac
Xfast_syscall() at Xfast_syscall+0xf7
--- syscall (54, FreeBSD ELF64, sys_ioctl), rip = 0x800da711c, rsp = 
0x7fffffff9d28, rbp = 0x7fffffffa1f0 ---
uma_zalloc_arg: zone "pfrktable" with the following non-sleepable locks 
held:
exclusive sleep mutex pf task mtx (pf task mtx) r = 0 
(0xffffffff8199af20) locked @ 
/tank/svn/sys/modules/pf/../../contrib/pf/net/pf_ioctl.c:1589
KDB: stack backtrace:
db_trace_self_wrapper() at db_trace_self_wrapper+0x2a
kdb_backtrace() at kdb_backtrace+0x37
_witness_debugger() at _witness_debugger+0x2e
witness_warn() at witness_warn+0x2c4
uma_zalloc_arg() at uma_zalloc_arg+0x335
pfr_create_ktable() at pfr_create_ktable+0xd8
pfr_ina_define() at pfr_ina_define+0x179
pfioctl() at pfioctl+0x1c5a
devfs_ioctl_f() at devfs_ioctl_f+0x7a
kern_ioctl() at kern_ioctl+0xcd
sys_ioctl() at sys_ioctl+0xfd
amd64_syscall() at amd64_syscall+0x3ac
Xfast_syscall() at Xfast_syscall+0xf7
--- syscall (54, FreeBSD ELF64, sys_ioctl), rip = 0x800da711c, rsp = 
0x7fffffff9d28, rbp = 0x7fffffffa1f0 ---
uma_zalloc_arg: zone "pfrkentry" with the following non-sleepable locks 
held:
exclusive sleep mutex pf task mtx (pf task mtx) r = 0 
(0xffffffff8199af20) locked @ 
/tank/svn/sys/modules/pf/../../contrib/pf/net/pf_table.c:75
KDB: stack backtrace:
db_trace_self_wrapper() at db_trace_self_wrapper+0x2a
kdb_backtrace() at kdb_backtrace+0x37
_witness_debugger() at _witness_debugger+0x2e
witness_warn() at witness_warn+0x2c4
uma_zalloc_arg() at uma_zalloc_arg+0x335
pfr_create_kentry() at pfr_create_kentry+0x73
pfr_ina_define() at pfr_ina_define+0x2ef
pfioctl() at pfioctl+0x1c5a
devfs_ioctl_f() at devfs_ioctl_f+0x7a
kern_ioctl() at kern_ioctl+0xcd
sys_ioctl() at sys_ioctl+0xfd
amd64_syscall() at amd64_syscall+0x3ac
Xfast_syscall() at Xfast_syscall+0xf7
--- syscall (54, FreeBSD ELF64, sys_ioctl), rip = 0x800da711c, rsp = 
0x7fffffff9d28, rbp = 0x7fffffffa1f0 ---
uma_zalloc_arg: zone "pfrkentry" with the following non-sleepable locks 
held:
exclusive sleep mutex pf task mtx (pf task mtx) r = 0 
(0xffffffff8199af20) locked @ 
/tank/svn/sys/modules/pf/../../contrib/pf/net/pf_table.c:75
KDB: stack backtrace:
db_trace_self_wrapper() at db_trace_self_wrapper+0x2a
kdb_backtrace() at kdb_backtrace+0x37
_witness_debugger() at _witness_debugger+0x2e
witness_warn() at witness_warn+0x2c4
uma_zalloc_arg() at uma_zalloc_arg+0x335
pfr_create_kentry() at pfr_create_kentry+0x73
pfr_ina_define() at pfr_ina_define+0x2ef
pfioctl() at pfioctl+0x1c5a
devfs_ioctl_f() at devfs_ioctl_f+0x7a
kern_ioctl() at kern_ioctl+0xcd
sys_ioctl() at sys_ioctl+0xfd
amd64_syscall() at amd64_syscall+0x3ac
Xfast_syscall() at Xfast_syscall+0xf7
--- syscall (54, FreeBSD ELF64, sys_ioctl), rip = 0x800da711c, rsp = 
0x7fffffff9d28, rbp = 0x7fffffffa1f0 ---
uma_zalloc_arg: zone "pfrkentry" with the following non-sleepable locks 
held:
exclusive sleep mutex pf task mtx (pf task mtx) r = 0 
(0xffffffff8199af20) locked @ 
/tank/svn/sys/modules/pf/../../contrib/pf/net/pf_table.c:75
KDB: stack backtrace:
db_trace_self_wrapper() at db_trace_self_wrapper+0x2a
kdb_backtrace() at kdb_backtrace+0x37
_witness_debugger() at _witness_debugger+0x2e
witness_warn() at witness_warn+0x2c4
uma_zalloc_arg() at uma_zalloc_arg+0x335
pfr_create_kentry() at pfr_create_kentry+0x73
pfr_ina_define() at pfr_ina_define+0x2ef
pfioctl() at pfioctl+0x1c5a
devfs_ioctl_f() at devfs_ioctl_f+0x7a
kern_ioctl() at kern_ioctl+0xcd
sys_ioctl() at sys_ioctl+0xfd
amd64_syscall() at amd64_syscall+0x3ac
Xfast_syscall() at Xfast_syscall+0xf7
--- syscall (54, FreeBSD ELF64, sys_ioctl), rip = 0x800da711c, rsp = 
0x7fffffff9d28, rbp = 0x7fffffffa1f0 ---
uma_zalloc_arg: zone "pfrkentry" with the following non-sleepable locks 
held:
exclusive sleep mutex pf task mtx (pf task mtx) r = 0 
(0xffffffff8199af20) locked @ 
/tank/svn/sys/modules/pf/../../contrib/pf/net/pf_table.c:75
KDB: stack backtrace:
db_trace_self_wrapper() at db_trace_self_wrapper+0x2a
kdb_backtrace() at kdb_backtrace+0x37
_witness_debugger() at _witness_debugger+0x2e
witness_warn() at witness_warn+0x2c4
uma_zalloc_arg() at uma_zalloc_arg+0x335
pfr_create_kentry() at pfr_create_kentry+0x73
pfr_ina_define() at pfr_ina_define+0x2ef
pfioctl() at pfioctl+0x1c5a
devfs_ioctl_f() at devfs_ioctl_f+0x7a
kern_ioctl() at kern_ioctl+0xcd
sys_ioctl() at sys_ioctl+0xfd
amd64_syscall() at amd64_syscall+0x3ac
Xfast_syscall() at Xfast_syscall+0xf7
--- syscall (54, FreeBSD ELF64, sys_ioctl), rip = 0x800da711c, rsp = 
0x7fffffff9d28, rbp = 0x7fffffffa1f0 ---
uma_zalloc_arg: zone "pfrkentry" with the following non-sleepable locks 
held:
exclusive sleep mutex pf task mtx (pf task mtx) r = 0 
(0xffffffff8199af20) locked @ 
/tank/svn/sys/modules/pf/../../contrib/pf/net/pf_table.c:75
KDB: stack backtrace:
db_trace_self_wrapper() at db_trace_self_wrapper+0x2a
kdb_backtrace() at kdb_backtrace+0x37
_witness_debugger() at _witness_debugger+0x2e
witness_warn() at witness_warn+0x2c4
uma_zalloc_arg() at uma_zalloc_arg+0x335
pfr_create_kentry() at pfr_create_kentry+0x73
pfr_ina_define() at pfr_ina_define+0x2ef
pfioctl() at pfioctl+0x1c5a
devfs_ioctl_f() at devfs_ioctl_f+0x7a
kern_ioctl() at kern_ioctl+0xcd
sys_ioctl() at sys_ioctl+0xfd
amd64_syscall() at amd64_syscall+0x3ac
Xfast_syscall() at Xfast_syscall+0xf7
--- syscall (54, FreeBSD ELF64, sys_ioctl), rip = 0x800da711c, rsp = 
0x7fffffff9d28, rbp = 0x7fffffffa1f0 ---
uma_zalloc_arg: zone "pfrktable" with the following non-sleepable locks 
held:
exclusive sleep mutex pf task mtx (pf task mtx) r = 0 
(0xffffffff8199af20) locked @ 
/tank/svn/sys/modules/pf/../../contrib/pf/net/pf_ioctl.c:1589
KDB: stack backtrace:
db_trace_self_wrapper() at db_trace_self_wrapper+0x2a
kdb_backtrace() at kdb_backtrace+0x37
_witness_debugger() at _witness_debugger+0x2e
witness_warn() at witness_warn+0x2c4
uma_zalloc_arg() at uma_zalloc_arg+0x335
pfr_create_ktable() at pfr_create_ktable+0xd8
pfr_ina_define() at pfr_ina_define+0x12b
pfioctl() at pfioctl+0x1c5a
devfs_ioctl_f() at devfs_ioctl_f+0x7a
kern_ioctl() at kern_ioctl+0xcd
sys_ioctl() at sys_ioctl+0xfd
amd64_syscall() at amd64_syscall+0x3ac
Xfast_syscall() at Xfast_syscall+0xf7
--- syscall (54, FreeBSD ELF64, sys_ioctl), rip = 0x800da711c, rsp = 
0x7fffffff9d28, rbp = 0x7fffffffa1f0 ---
uma_zalloc_arg: zone "pfrktable" with the following non-sleepable locks 
held:
exclusive sleep mutex pf task mtx (pf task mtx) r = 0 
(0xffffffff8199af20) locked @ 
/tank/svn/sys/modules/pf/../../contrib/pf/net/pf_ioctl.c:1589
KDB: stack backtrace:
db_trace_self_wrapper() at db_trace_self_wrapper+0x2a
kdb_backtrace() at kdb_backtrace+0x37
_witness_debugger() at _witness_debugger+0x2e
witness_warn() at witness_warn+0x2c4
uma_zalloc_arg() at uma_zalloc_arg+0x335
pfr_create_ktable() at pfr_create_ktable+0xd8
pfr_ina_define() at pfr_ina_define+0x179
pfioctl() at pfioctl+0x1c5a
devfs_ioctl_f() at devfs_ioctl_f+0x7a
kern_ioctl() at kern_ioctl+0xcd
sys_ioctl() at sys_ioctl+0xfd
amd64_syscall() at amd64_syscall+0x3ac
Xfast_syscall() at Xfast_syscall+0xf7
--- syscall (54, FreeBSD ELF64, sys_ioctl), rip = 0x800da711c, rsp = 
0x7fffffff9d28, rbp = 0x7fffffffa1f0 ---