panic: uma_zalloc_arg: Returning an empty bucket.

[Andriy Gapon]

I've been doing some stress-testing for VM+ZFS on a kernel with
WITNESS+INVARIANTS and I have already stumbled into some curious panics.

Here's another one:
Unread portion of the kernel message buffer:
panic: uma_zalloc_arg: Returning an empty bucket.
cpuid = 0
KDB: stack backtrace:
db_trace_self_wrapper() at 0xffffffff801b8b3a = db_trace_self_wrapper+0x2a
kdb_backtrace() at 0xffffffff803a7a8a = kdb_backtrace+0x3a
panic() at 0xffffffff80370632 = panic+0x1d2
uma_zalloc_arg() at 0xffffffff80507850 = uma_zalloc_arg+0x300
kmem_cache_alloc() at 0xffffffff80dac385 = kmem_cache_alloc+0x15
arc_buf_alloc() at 0xffffffff80c016e4 = arc_buf_alloc+0x34
arc_read_nolock() at 0xffffffff80c0389d = arc_read_nolock+0x2bd
arc_read() at 0xffffffff80c03dff = arc_read+0x7f
dbuf_prefetch() at 0xffffffff80c0826a = dbuf_prefetch+0x15a
dmu_zfetch_dofetch() at 0xffffffff80c21444 = dmu_zfetch_dofetch+0x104
dmu_zfetch() at 0xffffffff80c21c89 = dmu_zfetch+0x5e9
dbuf_read() at 0xffffffff80c06c04 = dbuf_read+0x454
dmu_buf_hold_array_by_dnode() at 0xffffffff80c0a58d =
dmu_buf_hold_array_by_dnode+0x27d
dmu_read() at 0xffffffff80c0a80f = dmu_read+0xbf
zfs_freebsd_getpages() at 0xffffffff80c6c9b5 = zfs_freebsd_getpages+0x8c5
VOP_GETPAGES_APV() at 0xffffffff805a6618 = VOP_GETPAGES_APV+0xe8
vnode_pager_getpages() at 0xffffffff80523c09 = vnode_pager_getpages+0xb9
vm_fault() at 0xffffffff8050a67a = vm_fault+0xf5a
trap_pfault() at 0xffffffff80545d6f = trap_pfault+0x11f
trap() at 0xffffffff80546448 = trap+0x4a8
calltrap() at 0xffffffff80530628 = calltrap+0x8

(kgdb) bt
#0  doadump () at pcpu.h:224
#1  0xffffffff8036fc9a in kern_reboot (howto=260) at
/usr/src/sys/kern/kern_shutdown.c:306
#2  0xffffffff80370668 in panic (fmt=Variable "fmt" is not available.
) at /usr/src/sys/kern/kern_shutdown.c:609
#3  0xffffffff80507850 in uma_zalloc_arg (zone=0xffffff00071c6e00,
udata=0xffffff00071c8380, flags=2) at /usr/src/sys/vm/uma_core.c:2132
#4  0xffffffff80dac385 in kmem_cache_alloc (cache=Variable "cache" is not available.
) at
/usr/src/sys/modules/opensolaris/../../cddl/compat/opensolaris/kern/opensolaris_kmem.c:196
#5  0xffffffff80c016e4 in arc_buf_alloc (spa=0xffffff00071e5000, size=131072,
tag=Variable "tag" is not available.
)
    at
/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c:1344
#6  0xffffffff80c0389d in arc_read_nolock (pio=0x0, spa=0xffffff00071e5000,
bp=0xffffff8002c71e80, done=0, private=0x0, priority=6, zio_flags=3,
    arc_flags=0xffffff812673b2bc, zb=0xffffff812673b280) at
/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c:2931
#7  0xffffffff80c03dff in arc_read (pio=0x0, spa=0xffffff00071e5000,
bp=0xffffff8002c71e80, pbuf=0xffffff00abaa31b0, done=0, private=0x0, priority=6,
    zio_flags=3, arc_flags=0xffffff812673b2bc, zb=0xffffff812673b280)
    at
/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c:2832
#8  0xffffffff80c0826a in dbuf_prefetch (dn=0xffffff00b440f000, blkid=221)
    at
/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/dbuf.c:1641
#9  0xffffffff80c21444 in dmu_zfetch_dofetch (zf=0xffffff00b440f2b8,
zs=0xffffff0126bb6480)
    at
/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_zfetch.c:308
#10 0xffffffff80c21c89 in dmu_zfetch (zf=0xffffff00b440f2b8, offset=Variable
"offset" is not available.
)
    at
/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_zfetch.c:526
#11 0xffffffff80c06c04 in dbuf_read (db=0xffffff0031970b60,
zio=0xffffff0078bb2328, flags=54)
    at
/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/dbuf.c:568
#12 0xffffffff80c0a58d in dmu_buf_hold_array_by_dnode (dn=0xffffff00b440f000,
offset=Variable "offset" is not available.
)
    at
/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/dmu.c:235
#13 0xffffffff80c0a80f in dmu_read (os=Variable "os" is not available.
) at
/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/dmu.c:572
#14 0xffffffff80c6c9b5 in zfs_freebsd_getpages (ap=Variable "ap" is not available.
) at
/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c:4348
#15 0xffffffff805a6618 in VOP_GETPAGES_APV (vop=0xffffffff80cd8680,
a=0xffffff812673b740) at vnode_if.c:2666
#16 0xffffffff80523c09 in vnode_pager_getpages (object=Variable "object" is not
available.
) at vnode_if.h:1154
#17 0xffffffff8050a67a in vm_fault (map=0xffffff00ab3d5c40, vaddr=35585523712,
fault_type=1 '\001', fault_flags=Variable "fault_flags" is not available.
) at vm_pager.h:130
#18 0xffffffff80545d6f in trap_pfault (frame=0xffffff812673bc40, usermode=1) at
/usr/src/sys/amd64/amd64/trap.c:729
#19 0xffffffff80546448 in trap (frame=0xffffff812673bc40) at
/usr/src/sys/amd64/amd64/trap.c:403
#20 0xffffffff80530628 in calltrap () at /usr/src/sys/amd64/amd64/exception.S:228
#21 0x0000000800858fe6 in ?? ()
-- 
Andriy Gapon