Re: panic in ZFS: VERIFY3U(BP_GET_BIRTH(bp), ==, BP_GET_BIRTH(&found->le_bp)) failed (15427394 == 15427393)
Date: Wed, 13 Aug 2025 10:37:45 UTC
On 13 Aug 2025, at 11:46, Alexander Leidinger <Alexander@Leidinger.net> wrote:
>
> I get this panic with -current as of 2025-08-11-154054 (CEST):
> ---snip---
> (kgdb) #0 __curthread () at /space/system/usr_src/sys/amd64/include/pcpu_aux.h:57
> td = <optimized out>
> #1 doadump (textdump=textdump@entry=1)
> at /space/system/usr_src/sys/kern/kern_shutdown.c:399
> error = 0
> coredump = <optimized out>
> #2 0xffffffff8053b305 in kern_reboot (howto=260)
> at /space/system/usr_src/sys/kern/kern_shutdown.c:519
> once = 0
> __pc = 0x0
> #3 0xffffffff8053b837 in vpanic (
> fmt=0xffffffff816d11f0 "VERIFY3U(BP_GET_BIRTH(bp), ==, BP_GET_BIRTH(&found->le_bp)) failed (%llu == %llu)\n", ap=ap@entry=0xfffffe0420634b30)
> at /space/system/usr_src/sys/kern/kern_shutdown.c:974
> buf = "VERIFY3U(BP_GET_BIRTH(bp), ==, BP_GET_BIRTH(&found->le_bp)) failed (15427394 == 15427393)\n", '\000' <repeats 165 times>
> __pc = 0x0
> __pc = 0x0
> __pc = 0x0
> other_cpus = {__bits = {16777213, 0 <repeats 15 times>}}
> td = 0xfffff801228ee780
> bootopt = <unavailable>
> newpanic = <optimized out>
> #4 0xffffffff8140650a in spl_panic (file=<optimized out>,
> func=<optimized out>, line=<unavailable>, fmt=<unavailable>)
> at /space/system/usr_src/sys/contrib/openzfs/module/os/freebsd/spl/spl_misc.c:100
> ap = {{gp_offset = 48, fp_offset = 48,
> overflow_arg_area = 0xfffffe0420634b60,
> reg_save_area = 0xfffffe0420634b00}}
> #5 0xffffffff814d4615 in dsl_livelist_iterate (arg=<optimized out>,
> bp=<optimized out>, bp_freed=0, tx=<optimized out>)
> at /space/system/usr_src/sys/contrib/openzfs/module/zfs/dsl_deadlist.c:1052
> _verify3_left = <unavailable>
> _verify3_right = <unavailable>
> node = {le_bp = {blk_dva = {{dva_word = {24, 5140063200}}, {
> dva_word = {0, 0}}, {dva_word = {0, 0}}},
> blk_prop = 9228727766583607296, blk_prop2 = 0, blk_pad = 0,
> blk_birth_word = {15427393, 15427394}, blk_fill = 0, blk_cksum = {
> zc_word = {0, 0, 0, 0}}}, le_refcnt = 9, le_node = {avl_child = {
> 0xfffffe0420634c30, 0xffffffff81492e2a <dmu_buf_hold+58>},
> avl_pcb = 18446735324294374176}}
> lia = <optimized out>
> avl = 0xfffffe0420634d88
> to_free = <optimized out>
> t = <optimized out>
> found = <optimized out>
> #6 0xffffffff8146a402 in bpobj_iterate_blkptrs (
> bpi=bpi@entry=0xfffff80eca95e3c0,
> func=func@entry=0xffffffff814d4320 <dsl_livelist_iterate>,
> arg=arg@entry=0xfffffe0420634da8, start=start@entry=0, tx=tx@entry=0x0,
> free=free@entry=0)
> at /space/system/usr_src/sys/contrib/openzfs/module/zfs/bpobj.c:325
> bp = 0xfffffe07b50a4880
> blkoff = <optimized out>
> bparray = <unavailable>
> offset = 6272
> bp_freed = 0
> dbuf = 0xfffff806386e5bc0
> err = <optimized out>
> freed = 0
> comp_freed = 0
> uncomp_freed = 0
> bpo = 0xfffff80ade376320
> i = 49
> pe = <optimized out>
> ps = 0
> pb = 0
> #7 0xffffffff814699c1 in bpobj_iterate_impl (
> initial_bpo=initial_bpo@entry=0xfffff80ade376320,
> func=0xffffffff814d4320 <dsl_livelist_iterate>,
> arg=arg@entry=0xfffffe0420634da8, tx=tx@entry=0x0, free=free@entry=0,
> bpobj_size=bpobj_size@entry=0x0)
> at /space/system/usr_src/sys/contrib/openzfs/module/zfs/bpobj.c:401
> bpo = 0xfffff80ade376320
> stack = {list_offset = 40, list_head = {
> list_next = 0xfffff80eca95e3e8, list_prev = 0xfffff80eca95e3e8}}
> err = <optimized out>
> bpi = 0xfffff80eca95e3c0
> #8 0xffffffff8146a0d2 in bpobj_iterate_nofree (bpo=<unavailable>,
> bpo@entry=0xfffff80ade376320, func=<unavailable>, arg=<unavailable>,
> arg@entry=0xfffffe0420634da8, bpobj_size=<unavailable>,
> bpobj_size@entry=0x0)
> at /space/system/usr_src/sys/contrib/openzfs/module/zfs/bpobj.c:567
> No locals.
> #9 0xffffffff814d422c in dsl_process_sub_livelist (
> bpobj=bpobj@entry=0xfffff80ade376320,
> to_free=to_free@entry=0xfffffe0420634e08, t=t@entry=0xfffff803e479e300,
> size=size@entry=0x0)
> at /space/system/usr_src/sys/contrib/openzfs/module/zfs/dsl_deadlist.c:1106
> avl = {avl_root = 0xfffff811914e9488,
> avl_compar = 0xffffffff814d42e0 <livelist_compare>,
> avl_offset = 136, avl_numnodes = 25}
> arg = {avl = 0xfffffe0420634d88, to_free = 0xfffffe0420634e08,
> t = 0xfffff803e479e300}
> cookie = 0xfffff80ade376300
> err = <optimized out>
> le = <optimized out>
> #10 0xffffffff81536a52 in spa_livelist_delete_cb (arg=0xfffffe022835d000,
> z=0xfffff803e479e300)
> at /space/system/usr_src/sys/contrib/openzfs/module/zfs/spa.c:3223
> to_free = {bpl_lock = {lock_object = {
> lo_name = 0xffffffff816f57e9 <.L.str+1> "bpl->bpl_lock",
> lo_flags = 577830912, lo_data = 0, lo_witness = 0x0},
> sx_lock = 1}, bpl_list = {list_offset = 128, list_head = {
> list_next = 0xfffff80a5a71e580, list_prev = 0xfffff8101a903680}}}
> ll = 0xfffff8053f3e1000
> dle = 0xfffff80ade376300
> err = <optimized out>
> count = 1
> spa = 0xfffffe022835d000
> ll_obj = <optimized out>
> mos = <optimized out>
> zap_obj = 14665
> #11 0xffffffff8164c16f in zthr_procedure (arg=0xfffff803e479e300)
> at /space/system/usr_src/sys/contrib/openzfs/module/zfs/zthr.c:246
> t = 0xfffff803e479e300
> #12 0xffffffff804ee0f7 in fork_exit (
> callout=0xffffffff8164c0c0 <zthr_procedure>, arg=0xfffff803e479e300,
> frame=0xfffffe0420634f40)
> at /space/system/usr_src/sys/kern/kern_fork.c:1153
> __pc = 0x0
> __pc = 0x0
> td = 0xfffff801228ee780
> p = 0xfffffe021d808558
> dtd = <optimized out>
>
> Tracing command "zfskern", '\000' <repeats 12 times> pid 6 tid 101414 (CPU 1)
> #0 __curthread () at /space/system/usr_src/sys/amd64/include/pcpu_aux.h:57
> #1 doadump (textdump=textdump@entry=1)
> at /space/system/usr_src/sys/kern/kern_shutdown.c:399
> #2 0xffffffff8053b305 in kern_reboot (howto=260)
> at /space/system/usr_src/sys/kern/kern_shutdown.c:519
> #3 0xffffffff8053b837 in vpanic (
> fmt=0xffffffff816d11f0 "VERIFY3U(BP_GET_BIRTH(bp), ==, BP_GET_BIRTH(&found->le_bp)) failed (%llu == %llu)\n", ap=ap@entry=0xfffffe0420634b30)
> at /space/system/usr_src/sys/kern/kern_shutdown.c:974
> #4 0xffffffff8140650a in spl_panic (file=<optimized out>,
> func=<optimized out>, line=<unavailable>, fmt=<unavailable>)
> at /space/system/usr_src/sys/contrib/openzfs/module/os/freebsd/spl/spl_misc.c:100
> #5 0xffffffff814d4615 in dsl_livelist_iterate (arg=<optimized out>,
> bp=<optimized out>, bp_freed=0, tx=<optimized out>)
> at /space/system/usr_src/sys/contrib/openzfs/module/zfs/dsl_deadlist.c:1052
> #6 0xffffffff8146a402 in bpobj_iterate_blkptrs (
> bpi=bpi@entry=0xfffff80eca95e3c0,
> func=func@entry=0xffffffff814d4320 <dsl_livelist_iterate>,
> arg=arg@entry=0xfffffe0420634da8, start=start@entry=0, tx=tx@entry=0x0,
> free=free@entry=0)
> at /space/system/usr_src/sys/contrib/openzfs/module/zfs/bpobj.c:325
> #7 0xffffffff814699c1 in bpobj_iterate_impl (
> initial_bpo=initial_bpo@entry=0xfffff80ade376320,
> func=0xffffffff814d4320 <dsl_livelist_iterate>,
> arg=arg@entry=0xfffffe0420634da8, tx=tx@entry=0x0, free=free@entry=0,
> bpobj_size=bpobj_size@entry=0x0)
> at /space/system/usr_src/sys/contrib/openzfs/module/zfs/bpobj.c:401
> #8 0xffffffff8146a0d2 in bpobj_iterate_nofree (bpo=<unavailable>,
> bpo@entry=0xfffff80ade376320, func=<unavailable>, arg=<unavailable>,
> arg@entry=0xfffffe0420634da8, bpobj_size=<unavailable>,
> bpobj_size@entry=0x0)
> at /space/system/usr_src/sys/contrib/openzfs/module/zfs/bpobj.c:567
> #9 0xffffffff814d422c in dsl_process_sub_livelist (
> bpobj=bpobj@entry=0xfffff80ade376320,
> to_free=to_free@entry=0xfffffe0420634e08, t=t@entry=0xfffff803e479e300,
> size=size@entry=0x0)
> at /space/system/usr_src/sys/contrib/openzfs/module/zfs/dsl_deadlist.c:1106
> #10 0xffffffff81536a52 in spa_livelist_delete_cb (arg=0xfffffe022835d000,
> z=0xfffff803e479e300)
> at /space/system/usr_src/sys/contrib/openzfs/module/zfs/spa.c:3223
> #11 0xffffffff8164c16f in zthr_procedure (arg=0xfffff803e479e300)
> at /space/system/usr_src/sys/contrib/openzfs/module/zfs/zthr.c:246
> #12 0xffffffff804ee0f7 in fork_exit (
> callout=0xffffffff8164c0c0 <zthr_procedure>, arg=0xfffff803e479e300,
> frame=0xfffffe0420634f40)
> at /space/system/usr_src/sys/kern/kern_fork.c:1153
> #13 <signal handler called>
> ---snip---
This is due to https://cgit.freebsd.org/src/commit/?id=df58e8b1506f241670be86a560fb6e8432043aee, see https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=288722 .
It's currently unknown what the root cause is. We need some ZFS wizards. :)
-Dimitry