stuck reboot, zfs, paging in progress

[Andriy Gapon]

Has anyone else seen anything like this?
Seems to be related to mmap + zfs.

This is on stable/13, f165641df4da17 from a couple of weeks ago.

#0  sched_switch (td=td at entry=0xfffffe00357f8ac0, flags=<optimized out>,
flags at entry=260) at /usr/devel/git/trant/sys/kern/sched_ule.c:2147
#1  0xffffffff80847246 in mi_switch (flags=flags at entry=260) at
/usr/devel/git/trant/sys/kern/kern_synch.c:542
#2  0xffffffff8088b9e6 in sleepq_switch (wchan=wchan at entry=0xfffff80681a9ccf0,
pri=pri at entry=84) at /usr/devel/git/trant/sys/kern/subr_sleepqueue.c:608
#3  0xffffffff8088b88c in sleepq_wait (wchan=0xfffff80681a9ccf0,
pri=pri at entry=84) at /usr/devel/git/trant/sys/kern/subr_sleepqueue.c:659
#4  0xffffffff80846f1f in _blockcount_sleep (bc=bc at entry=0xfffff80681a9ccf0,
lock=lock at entry=0x0, wmesg=wmesg at entry=0xffffffff80c05803 "bovlbx",
prio=prio at entry=84) at /usr/devel/git/trant/sys/kern/kern_synch.c:461
#5  0xffffffff80a9ac53 in _blockcount_wait (bc=bc at entry=0xfffff80681a9ccf0,
lo=lo at entry=0x0, wmesg=wmesg at entry=0xffffffff80c05803 "bovlbx", prio=84) at
/usr/devel/git/trant/sys/sys/blockcount.h:85
#6  0xffffffff80a9aca8 in vm_object_pip_wait_unlocked (object=<optimized out>,
waitid=<unavailable>) at /usr/devel/git/trant/sys/vm/vm_object.c:405
#7  0xffffffff80902d9a in bufobj_invalbuf (bo=bo at entry=0xfffff806a60c6a70,
flags=flags at entry=1, slpflag=slpflag at entry=0, slptimeo=slptimeo at entry=0) at
/usr/devel/git/trant/sys/kern/vfs_subr.c:1978
#8  0xffffffff809031ea in vinvalbuf (vp=<optimized out>,
vp at entry=0xfffff806a60c6988, flags=flags at entry=1, slpflag=slpflag at entry=0,
slptimeo=slptimeo at entry=0) at /usr/devel/git/trant/sys/kern/vfs_subr.c:2021
#9  0xffffffff80ab4dc1 in vnode_destroy_vobject (vp=vp at entry=0xfffff806a60c6988)
at /usr/devel/git/trant/sys/vm/vnode_pager.c:213
#10 0xffffffff80906421 in vgonel (vp=vp at entry=0xfffff806a60c6988) at
/usr/devel/git/trant/sys/kern/vfs_subr.c:3978
#11 0xffffffff80905c4a in vflush (mp=mp at entry=0xfffffe01bc462ac0,
rootrefs=rootrefs at entry=0, flags=flags at entry=2, td=<optimized out>,
td at entry=0xfffffe00357f8ac0) at /usr/devel/git/trant/sys/kern/vfs_subr.c:3722
#12 0xffffffff80378136 in zfs_umount (vfsp=0xfffffe01bc462ac0, vfsp at entry=<error
reading variable: value is not available>, fflag=<optimized out>,
fflag at entry=<error reading variable: value is not available>)
    at
/usr/devel/git/trant/sys/contrib/openzfs/module/os/freebsd/zfs/zfs_vfsops.c:1662
#13 0xffffffff808fbacf in dounmount (mp=<unavailable>,
mp at entry=0xfffffe01bc462ac0, flags=flags at entry=524288, td=0xfffffe00357f8ac0) at
/usr/devel/git/trant/sys/kern/vfs_mount.c:1843
#14 0xffffffff80906c5f in unmount_or_warn (mp=mp at entry=0xfffffe01bc462ac0) at
/usr/devel/git/trant/sys/kern/vfs_subr.c:4640
#15 0xffffffff80906c3c in vfs_unmountall () at
/usr/devel/git/trant/sys/kern/vfs_subr.c:4674
#16 0xffffffff808df142 in bufshutdown (show_busybufs=0) at
/usr/devel/git/trant/sys/kern/vfs_bio.c:1443
#17 0xffffffff80839c75 in kern_reboot (howto=0) at
/usr/devel/git/trant/sys/kern/kern_shutdown.c:472
#18 0xffffffff808396f2 in sys_reboot (td=<optimized out>,
uap=0xfffffe00357f8ea8) at /usr/devel/git/trant/sys/kern/kern_shutdown.c:313
#19 0xffffffff80b22409 in syscallenter (td=0xfffffe00357f8ac0) at
/usr/devel/git/trant/sys/amd64/amd64/../../kern/subr_syscall.c:189
#20 0xffffffff80b220e5 in amd64_syscall (td=0xfffffe00357f8ac0, traced=0) at
/usr/devel/git/trant/sys/amd64/amd64/trap.c:1156

(kgdb) p *vp
$13 = {v_type = VREG, v_irflag = 1, v_seqc = 1, v_nchash = 3207342140, v_op =
0xffffffff80e592c0 <zfs_vnodeops>, v_data = 0xfffff805410f6ce8, v_mount =
0xfffffe01bc462ac0, v_nmntvnodes = {tqe_next = 0xfffff8057f14dc00,
    tqe_prev = 0xfffffe01bc462b48}, {v_mountedhere = 0x0, v_unpcb = 0x0, v_rdev
= 0x0, v_fifoinfo = 0x0}, v_hashlist = {le_next = 0x0, le_prev = 0x0},
v_cache_src = {lh_first = 0x0}, v_cache_dst = {tqh_first = 0x0,
    tqh_last = 0xfffff806a60c69e0}, v_cache_dd = 0x0, v_lock = {lock_object =
{lo_name = 0xffffffff80c4edb4 "zfs", lo_flags = 117112832, lo_data = 0,
lo_witness = 0x0}, lk_lock = 18446741875583847104, lk_exslpfail = 0,
    lk_timo = 51, lk_pri = 96}, v_interlock = {lock_object = {lo_name =
0xffffffff80cf0f55 "vnode interlock", lo_flags = 16973824, lo_data = 0,
lo_witness = 0x0}, mtx_lock = 0}, v_vnlock = 0xfffff806a60c69f8, v_vnodelist = {
    tqe_next = 0xfffff80732d603d0, tqe_prev = 0xfffff801cde180c8}, v_lazylist =
{tqe_next = 0x0, tqe_prev = 0x0}, v_bufobj = {bo_lock = {lock_object = {lo_name
= 0xffffffff80cb36ff "bufobj interlock", lo_flags = 86179840,
        lo_data = 0, lo_witness = 0x0}, rw_lock = 1}, bo_ops =
0xffffffff80ee28a0 <buf_ops_bio>, bo_object = 0xfffff80681a9cc60, bo_synclist =
{le_next = 0x0, le_prev = 0x0}, bo_private = 0xfffff806a60c6988, bo_clean = {
      bv_hd = {tqh_first = 0x0, tqh_last = 0xfffff806a60c6ab8}, bv_root =
{pt_root = 0}, bv_cnt = 0}, bo_dirty = {bv_hd = {tqh_first = 0x0, tqh_last =
0xfffff806a60c6ad8}, bv_root = {pt_root = 0}, bv_cnt = 0}, bo_numoutput = 0,
    bo_flag = 0, bo_domain = 3, bo_bsize = 1048576}, v_pollinfo =
0xfffff804d3647700, v_label = 0x0, v_lockf = 0x0, v_rl = {rl_waiters =
{tqh_first = 0x0, tqh_last = 0xfffff806a60c6b20}, rl_currdep = 0x0}, v_cstart = 0,
  v_lasta = 0, v_lastw = 0, v_clen = 0, v_holdcnt = 2, v_usecount = 0, v_iflag =
0, v_vflag = 0, v_mflag = 0, v_dbatchcpu = -1, v_writecount = 0, v_seqc_users =
1, v_hash = 111545449}

(kgdb) p vp->v_bufobj
$14 = {bo_lock = {lock_object = {lo_name = 0xffffffff80cb36ff "bufobj
interlock", lo_flags = 86179840, lo_data = 0, lo_witness = 0x0}, rw_lock = 1},
bo_ops = 0xffffffff80ee28a0 <buf_ops_bio>, bo_object = 0xfffff80681a9cc60,
  bo_synclist = {le_next = 0x0, le_prev = 0x0}, bo_private = 0xfffff806a60c6988,
bo_clean = {bv_hd = {tqh_first = 0x0, tqh_last = 0xfffff806a60c6ab8}, bv_root =
{pt_root = 0}, bv_cnt = 0}, bo_dirty = {bv_hd = {tqh_first = 0x0,
      tqh_last = 0xfffff806a60c6ad8}, bv_root = {pt_root = 0}, bv_cnt = 0},
bo_numoutput = 0, bo_flag = 0, bo_domain = 3, bo_bsize = 1048576}

(kgdb) p *vp->v_bufobj->bo_object
$15 = {lock = {lock_object = {lo_name = 0xffffffff80c0f804 "vm object", lo_flags
= 627245056, lo_data = 0, lo_witness = 0x0}, rw_lock = 1}, object_list =
{tqe_next = 0xfffff80681a9cd68, tqe_prev = 0xfffff80681a9cb78},
  shadow_head = {lh_first = 0x0}, shadow_list = {le_next = 0x0, le_prev =
0xfffff80019a69030}, memq = {tqh_first = 0xfffffe0012d02360, tqh_last =
0xfffffe001c379ed0}, rtree = {rt_root = 18446735311032427072}, size = 798661,
  domain = {dr_policy = 0x0, dr_iter = 0}, generation = 997761, cleangeneration
= 1, ref_count = 0, shadow_count = 0, memattr = 6 '\006', type = 2 '\002', flags
= 4104, pg_color = 0, paging_in_progress = {__count = 2147483777},
  busy = {__count = 0}, resident_page_count = 518525, backing_object = 0x0,
backing_object_offset = 0, pager_object_list = {tqe_next = 0x0, tqe_prev = 0x0},
rvq = {lh_first = 0xfffff807f9d6c9e0}, handle = 0xfffff806a60c6988,
  un_pager = {vnp = {vnp_size = 3271314029, writemappings = 0}, devp =
{devp_pglist = {tqh_first = 0xc2fc4a6d, tqh_last = 0x0}, ops = 0x0, dev = 0x0},
sgp = {sgp_pglist = {tqh_first = 0xc2fc4a6d, tqh_last = 0x0}}, swp = {
      swp_tmpfs = 0xc2fc4a6d, swp_blks = {pt_root = 0}, writemappings = 0}, phys
= {ops = 0xc2fc4a6d, {data_ptr = 0x0, data_val = 0}}}, cred = 0x0, charge = 0,
umtx_data = 0x0}

(kgdb) p/x $15.paging_in_progress
$16 = {__count = 0x80000081}

-- 
Andriy Gapon