panic: solaris assert: rt->rt_space == 0 (0xe000 == 0x0), file: /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/range_tree.c, line: 153
Fabian Keil
freebsd-listen at fabiankeil.de
Tue Jan 27 13:20:33 UTC 2015
Using an 11.0-CURRENT based on r276255 I just got a panic
after trying to export a certain ZFS pool:
(kgdb) where
#0 doadump (textdump=0) at pcpu.h:219
#1 0xffffffff80313e8e in db_dump (dummy=<value optimized out>, dummy2=0, dummy3=0, dummy4=0x0) at /usr/src/sys/ddb/db_command.c:533
#2 0xffffffff8031396d in db_command (cmd_table=0x0) at /usr/src/sys/ddb/db_command.c:440
#3 0xffffffff803136e4 in db_command_loop () at /usr/src/sys/ddb/db_command.c:493
#4 0xffffffff803161f0 in db_trap (type=<value optimized out>, code=0) at /usr/src/sys/ddb/db_main.c:251
#5 0xffffffff805f63c1 in kdb_trap (type=3, code=0, tf=<value optimized out>) at /usr/src/sys/kern/subr_kdb.c:654
#6 0xffffffff80878717 in trap (frame=0xfffffe0094a62540) at /usr/src/sys/amd64/amd64/trap.c:542
#7 0xffffffff8085c472 in calltrap () at /usr/src/sys/amd64/amd64/exception.S:235
#8 0xffffffff805f5abe in kdb_enter (why=0xffffffff80995a6d "panic", msg=<value optimized out>) at cpufunc.h:63
#9 0xffffffff805b3b81 in panic (fmt=<value optimized out>) at /usr/src/sys/kern/kern_shutdown.c:739
#10 0xffffffff81bdd22f in assfail3 (a=<value optimized out>, lv=<value optimized out>, op=<value optimized out>, rv=<value optimized out>, f=<value optimized out>, l=<value optimized out>)
at /usr/src/sys/cddl/compat/opensolaris/kern/opensolaris_cmn_err.c:91
#11 0xffffffff8194afc4 in range_tree_destroy (rt=0xfffff80011586000) at /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/range_tree.c:153
#12 0xffffffff819488bc in metaslab_fini (msp=0xfffff800611a9800) at /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/metaslab.c:1398
#13 0xffffffff81965841 in vdev_free (vd=0xfffff8000696d800) at /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/vdev.c:994
#14 0xffffffff819657e1 in vdev_free (vd=0xfffff80040532000) at /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/vdev.c:683
#15 0xffffffff81953948 in spa_unload (spa=0xfffff800106af000) at /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/spa.c:1314
#16 0xffffffff81957a58 in spa_export_common (pool=<value optimized out>, new_state=1, oldconfig=0x0, force=<value optimized out>, hardforce=0)
at /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/spa.c:4540
#17 0xffffffff81957b08 in spa_export (pool=0x0, oldconfig=0xfffffe0094a624f0, force=128, hardforce=50) at /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/spa.c:4574
#18 0xffffffff8199ed50 in zfs_ioc_pool_export (zc=0xfffffe0006fbf000) at /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_ioctl.c:1618
#19 0xffffffff8199c2aa in zfsdev_ioctl (dev=<value optimized out>, zcmd=<value optimized out>, arg=<value optimized out>, flag=<value optimized out>, td=<value optimized out>)
at /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_ioctl.c:6198
#20 0xffffffff8047d32b in devfs_ioctl_f (fp=0xfffff8002adcb280, com=3222821379, data=0xfffffe0094a62a20, cred=<value optimized out>, td=0xfffff80056f15000) at /usr/src/sys/fs/devfs/devfs_vnops.c:775
#21 0xffffffff8061210e in kern_ioctl (td=0xfffff80056f15000, fd=<value optimized out>, com=50) at file.h:318
#22 0xffffffff80611c90 in sys_ioctl (td=0xfffff80056f15000, uap=0xfffffe0094a62b80) at /usr/src/sys/kern/sys_generic.c:718
#23 0xffffffff8087981d in amd64_syscall (td=0xfffff80056f15000, traced=0) at subr_syscall.c:133
---Type <return> to continue, or q <return> to quit---
#24 0xffffffff8085c75b in Xfast_syscall () at /usr/src/sys/amd64/amd64/exception.S:395
#25 0x00000008019ffb2a in ?? ()
Previous frame inner to this frame (corrupt stack?)
Current language: auto; currently minimal
(kgdb) f 11
#11 0xffffffff8194afc4 in range_tree_destroy (rt=0xfffff80011586000) at /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/range_tree.c:153
153 VERIFY0(rt->rt_space);
(kgdb) p *rt
$1 = {rt_root = {avl_root = 0xfffff80014b47dc0, avl_compar = 0xffffffff8194af50 <range_tree_seg_compare>, avl_offset = 0, avl_numnodes = 8, avl_size = 64}, rt_space = 57344, rt_ops = 0x0,
rt_arg = 0xfffff800611a9800, rt_histogram = {0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 2, 2, 1, 0 <repeats 49 times>}, rt_lock = 0xfffff800611a9800}
(kgdb) f 15
#15 0xffffffff81953948 in spa_unload (spa=0xfffff800106af000) at /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/spa.c:1314
warning: Source file is more recent than executable.
1314 vdev_free(spa->spa_root_vdev);
(kgdb) p *spa
$2 = {spa_name = "wde4", '\0' <repeats 251 times>, spa_comment = 0x0, spa_avl = {avl_child = {0x0, 0x0}, avl_pcb = 18446735277722407181}, spa_config = 0xfffff8005a4b55e0, spa_config_syncing = 0x0,
spa_config_splitting = 0x0, spa_load_info = 0xfffff80019613e60, spa_config_txg = 1200533, spa_sync_pass = 0, spa_state = POOL_STATE_EXPORTED, spa_inject_ref = 0, spa_sync_on = 0 '\0',
spa_load_state = SPA_LOAD_NONE, spa_import_flags = 0, spa_zio_taskq = {{{stqs_count = 1, stqs_taskq = 0xfffff80056f34b70}, {stqs_count = 0, stqs_taskq = 0x0}, {stqs_count = 1,
stqs_taskq = 0xfffff80056f34b90}, {stqs_count = 0, stqs_taskq = 0x0}}, {{stqs_count = 1, stqs_taskq = 0xfffff80056f34b80}, {stqs_count = 0, stqs_taskq = 0x0}, {stqs_count = 8,
stqs_taskq = 0xfffff8005513cb40}, {stqs_count = 0, stqs_taskq = 0x0}}, {{stqs_count = 1, stqs_taskq = 0xfffff80056f34b30}, {stqs_count = 1, stqs_taskq = 0xfffff80056f34b10}, {stqs_count = 1,
stqs_taskq = 0xfffff80061202b40}, {stqs_count = 1, stqs_taskq = 0xfffff80061202bf0}}, {{stqs_count = 8, stqs_taskq = 0xfffff8005512f200}, {stqs_count = 0, stqs_taskq = 0x0}, {stqs_count = 1,
stqs_taskq = 0xfffff80061202be0}, {stqs_count = 0, stqs_taskq = 0x0}}, {{stqs_count = 1, stqs_taskq = 0xfffff80061202c30}, {stqs_count = 0, stqs_taskq = 0x0}, {stqs_count = 1,
stqs_taskq = 0xfffff80061202c50}, {stqs_count = 0, stqs_taskq = 0x0}}, {{stqs_count = 1, stqs_taskq = 0xfffff80056f34bd0}, {stqs_count = 0, stqs_taskq = 0x0}, {stqs_count = 1,
stqs_taskq = 0xfffff80061202ab0}, {stqs_count = 0, stqs_taskq = 0x0}}}, spa_dsl_pool = 0xfffff80002573000, spa_is_initializing = 0, spa_normal_class = 0xfffff80014134400,
spa_log_class = 0xfffff80014134000, spa_first_txg = 1200009, spa_final_txg = 1200535, spa_freeze_txg = 18446744073709551615, spa_load_max_txg = 18446744073709551615, spa_claim_max_txg = 1200009,
spa_loaded_ts = {tv_sec = 1422360075, tv_nsec = 210277415}, spa_meta_objset = 0xfffff8003792b000, spa_vdev_txg_list = {tl_lock = {lock_object = {lo_name = 0xffffffff81a19abd "tl->tl_lock",
lo_flags = 40960000, lo_data = 0, lo_witness = 0x0}, sx_lock = 1}, tl_offset = 624, tl_head = {0xfffff8000696da70, 0x0, 0x0, 0x0}}, spa_root_vdev = 0xfffff80040532000,
spa_config_guid = 14402430966328721211, spa_load_guid = 1228122528, spa_last_synced_guid = 14402430966328721211, spa_config_dirty_list = {list_size = 1624, list_offset = 672, list_head = {
list_next = 0xfffff800106af3c0, list_prev = 0xfffff800106af3c0}}, spa_state_dirty_list = {list_size = 1624, list_offset = 688, list_head = {list_next = 0xfffff800106af3e0,
list_prev = 0xfffff800106af3e0}}, spa_spares = {sav_object = 0, sav_config = 0x0, sav_vdevs = 0x0, sav_count = 0, sav_sync = 0, sav_pending = 0x0, sav_npending = 0}, spa_l2cache = {sav_object = 0,
sav_config = 0x0, sav_vdevs = 0x0, sav_count = 0, sav_sync = 0, sav_pending = 0x0, sav_npending = 0}, spa_label_features = 0xfffff800224d58c0, spa_config_object = 27, spa_config_generation = 0,
spa_syncing_txg = 1200535, spa_deferred_bpobj = {bpo_lock = {lock_object = {lo_name = 0xffffffff81a0c9bf "bpo->bpo_lock", lo_flags = 40894464, lo_data = 0, lo_witness = 0x0}, sx_lock = 6},
bpo_os = 0xfffff8003792b000, bpo_object = 0, bpo_epb = 128, bpo_havecomp = 1 '\001', bpo_havesubobj = 1 '\001', bpo_phys = 0x0, bpo_dbuf = 0x0, bpo_cached_dbuf = 0x0}, spa_free_bplist = {{bpl_lock = {
lock_object = {lo_name = 0xffffffff81a0c72f "bpl->bpl_lock", lo_flags = 40960000, lo_data = 0, lo_witness = 0x0}, sx_lock = 1}, bpl_list = {list_size = 144, list_offset = 128, list_head = {
list_next = 0xfffff800106af4f0, list_prev = 0xfffff800106af4f0}}}, {bpl_lock = {lock_object = {lo_name = 0xffffffff81a0c72f "bpl->bpl_lock", lo_flags = 40960000, lo_data = 0, lo_witness = 0x0},
sx_lock = 1}, bpl_list = {list_size = 144, list_offset = 128, list_head = {list_next = 0xfffff800106af530, list_prev = 0xfffff800106af530}}}, {bpl_lock = {lock_object = {
lo_name = 0xffffffff81a0c72f "bpl->bpl_lock", lo_flags = 40960000, lo_data = 0, lo_witness = 0x0}, sx_lock = 1}, bpl_list = {list_size = 144, list_offset = 128, list_head = {
list_next = 0xfffff800106af570, list_prev = 0xfffff800106af570}}}, {bpl_lock = {lock_object = {lo_name = 0xffffffff81a0c72f "bpl->bpl_lock", lo_flags = 40960000, lo_data = 0, lo_witness = 0x0},
sx_lock = 1}, bpl_list = {list_size = 144, list_offset = 128, list_head = {list_next = 0xfffff800106af5b0, list_prev = 0xfffff800106af5b0}}}}, spa_ubsync = {ub_magic = 12235020, ub_version = 5000,
ub_txg = 1200535, ub_guid_sum = 15325664383032364435, ub_timestamp = 1422361173, ub_rootbp = {blk_dva = {{dva_word = {1, 2518699004}}, {dva_word = {1, 3102650012}}, {dva_word = {1, 292073561}}},
blk_prop = 9226476022604496899, blk_pad = {0, 0}, blk_phys_birth = 0, blk_birth = 1200535, blk_fill = 294153, blk_cksum = {zc_word = {64317136616, 6262344505063, 315253367504245, 10929851884474535}}},
ub_software_version = 5000}, spa_uberblock = {ub_magic = 12235020, ub_version = 5000, ub_txg = 1200535, ub_guid_sum = 15325664383032364435, ub_timestamp = 1422361173, ub_rootbp = {blk_dva = {{dva_word = {
---Type <return> to continue, or q <return> to quit---
1, 2518699004}}, {dva_word = {1, 3102650012}}, {dva_word = {1, 292073561}}}, blk_prop = 9226476022604496899, blk_pad = {0, 0}, blk_phys_birth = 0, blk_birth = 1200535, blk_fill = 294153,
blk_cksum = {zc_word = {64317136616, 6262344505063, 315253367504245, 10929851884474535}}}, ub_software_version = 5000}, spa_extreme_rewind = 0, spa_last_io = 5583377, spa_scrub_lock = {lock_object = {
lo_name = 0xffffffff81a19122 "spa->spa_scrub_lock", lo_flags = 40960000, lo_data = 0, lo_witness = 0x0}, sx_lock = 1}, spa_scrub_inflight = 0, spa_scrub_io_cv = {
cv_description = 0xffffffff81a1918b "spa->spa_scrub_io_cv", cv_waiters = 0}, spa_scrub_active = 1 '\001', spa_scrub_type = 0 '\0', spa_scrub_finished = 0 '\0', spa_scrub_started = 0 '\0',
spa_scrub_reopen = 0 '\0', spa_scan_pass_start = 1422360075, spa_scan_pass_exam = 0, spa_async_lock = {lock_object = {lo_name = 0xffffffff81a190a0 "spa->spa_async_lock", lo_flags = 40960000, lo_data = 0,
lo_witness = 0x0}, sx_lock = 1}, spa_async_thread = 0x0, spa_async_thread_vd = 0x0, spa_async_suspended = 2, spa_async_cv = {cv_description = 0xffffffff81a19166 "spa->spa_async_cv", cv_waiters = 0},
spa_async_tasks = 0, spa_root = 0x0, spa_ena = 0, spa_last_open_failed = 0, spa_last_ubsync_txg = 0, spa_last_ubsync_txg_ts = 0, spa_load_txg = 0, spa_load_txg_ts = 1421709971, spa_load_meta_errors = 0,
spa_load_data_errors = 0, spa_verify_min_txg = 1200005, spa_errlog_lock = {lock_object = {lo_name = 0xffffffff81a190cc "spa->spa_errlog_lock", lo_flags = 40960000, lo_data = 0, lo_witness = 0x0},
sx_lock = 1}, spa_errlog_last = 716876, spa_errlog_scrub = 717131, spa_errlist_lock = {lock_object = {lo_name = 0xffffffff81a190b5 "spa->spa_errlist_lock", lo_flags = 40960000, lo_data = 0,
lo_witness = 0x0}, sx_lock = 1}, spa_errlist_last = {avl_root = 0x0, avl_compar = 0xffffffff81951970 <spa_error_entry_compare>, avl_offset = 40, avl_numnodes = 0, avl_size = 64}, spa_errlist_scrub = {
avl_root = 0x0, avl_compar = 0xffffffff81951970 <spa_error_entry_compare>, avl_offset = 40, avl_numnodes = 0, avl_size = 64}, spa_deflate = 1, spa_history = 29, spa_history_lock = {lock_object = {
lo_name = 0xffffffff81a190e2 "spa->spa_history_lock", lo_flags = 40960000, lo_data = 0, lo_witness = 0x0}, sx_lock = 1}, spa_pending_vdev = 0x0, spa_props_lock = {lock_object = {
lo_name = 0xffffffff81a1910d "spa->spa_props_lock", lo_flags = 40960000, lo_data = 0, lo_witness = 0x0}, sx_lock = 1}, spa_pool_props_object = 347723, spa_bootfs = 0, spa_failmode = 0,
spa_delegation = 1, spa_config_list = {list_size = 24, list_offset = 0, list_head = {list_next = 0xfffff800433e0600, list_prev = 0xfffff800433e0600}}, spa_async_zio_root = 0x0, spa_suspend_zio_root = 0x0,
spa_suspend_lock = {lock_object = {lo_name = 0xffffffff81a19137 "spa->spa_suspend_lock", lo_flags = 40960000, lo_data = 0, lo_witness = 0x0}, sx_lock = 1}, spa_suspend_cv = {
cv_description = 0xffffffff81a191a1 "spa->spa_suspend_cv", cv_waiters = 0}, spa_suspended = 0 '\0', spa_claiming = 0 '\0', spa_debug = 0, spa_is_root = 0, spa_minref = 11, spa_mode = 3,
spa_log_state = SPA_LOG_GOOD, spa_autoexpand = 0, spa_ddt = {0xfffffe0030698000, 0xfffffe002ebb1000, 0xfffffe002f8e6000, 0xfffffe002af0f000, 0xfffffe001a3e0000, 0xfffffe001a3e8000, 0xfffffe0030985000,
0xfffffe003098d000, 0xfffffe003096d000, 0xfffffe0030975000, 0xfffffe0030690000}, spa_ddt_stat_object = 0, spa_dedup_ditto = 0, spa_dedup_checksum = 8, spa_dspace = 1992864825344, spa_vdev_top_lock = {
lock_object = {lo_name = 0xffffffff81a1914e "spa->spa_vdev_top_lock", lo_flags = 40960000, lo_data = 0, lo_witness = 0x0}, sx_lock = 1}, spa_proc_lock = {lock_object = {
lo_name = 0xffffffff81a190f9 "spa->spa_proc_lock", lo_flags = 40960000, lo_data = 0, lo_witness = 0x0}, sx_lock = 1}, spa_proc_cv = {cv_description = 0xffffffff81a19179 "spa->spa_proc_cv",
cv_waiters = 0}, spa_proc_state = SPA_PROC_NONE, spa_proc = 0xffffffff81669578, spa_did = 0, spa_trim_thread = 0x0, spa_trim_lock = {lock_object = {lo_name = 0x0, lo_flags = 0, lo_data = 0,
lo_witness = 0x0}, sx_lock = 0}, spa_trim_cv = {cv_description = 0x0, cv_waiters = 0}, spa_autoreplace = 1, spa_vdev_locks = 0, spa_creation_version = 28, spa_prev_software_version = 5000,
spa_feat_for_write_obj = 496823, spa_feat_for_read_obj = 496822, spa_feat_desc_obj = 496824, spa_feat_enabled_txg_obj = 672382, spa_feat_refcount_cache = {1, 6005264, 1, 0, 117, 7, 1, 0, 1, 0, 0, 0},
spa_deadman_cycid = {c_links = {le = {le_next = 0x0, le_prev = 0xfffffe0000b70b00}, sle = {sle_next = 0x0}, tqe = {tqe_next = 0x0, tqe_prev = 0xfffffe0000b70b00}}, c_time = 28275380581533,
c_precision = 268435437500, c_arg = 0xfffff800106af000, c_func = 0xffffffff8195f1e0 <spa_deadman>, c_lock = 0x0, c_flags = 16, c_cpu = 0}, spa_deadman_calls = 0, spa_sync_starttime = 5583375137907,
spa_deadman_synctime = 1000000000000, spa_ccw_fail_time = 0, spa_config_lock = {{scl_lock = {lock_object = {lo_name = 0xffffffff81a1944b "scl->scl_lock", lo_flags = 40960000, lo_data = 0, lo_witness = 0x0},
sx_lock = 1}, scl_writer = 0xfffff80056f15000, scl_write_wanted = 0, scl_cv = {cv_description = 0xffffffff81a1945a "scl->scl_cv", cv_waiters = 0}, scl_count = {rc_count = 1}}, {scl_lock = {
---Type <return> to continue, or q <return> to quit---
lock_object = {lo_name = 0xffffffff81a1944b "scl->scl_lock", lo_flags = 40960000, lo_data = 0, lo_witness = 0x0}, sx_lock = 1}, scl_writer = 0xfffff80056f15000, scl_write_wanted = 0, scl_cv = {
cv_description = 0xffffffff81a1945a "scl->scl_cv", cv_waiters = 0}, scl_count = {rc_count = 1}}, {scl_lock = {lock_object = {lo_name = 0xffffffff81a1944b "scl->scl_lock", lo_flags = 40960000,
lo_data = 0, lo_witness = 0x0}, sx_lock = 1}, scl_writer = 0xfffff80056f15000, scl_write_wanted = 0, scl_cv = {cv_description = 0xffffffff81a1945a "scl->scl_cv", cv_waiters = 0}, scl_count = {
rc_count = 1}}, {scl_lock = {lock_object = {lo_name = 0xffffffff81a1944b "scl->scl_lock", lo_flags = 40960000, lo_data = 0, lo_witness = 0x0}, sx_lock = 1}, scl_writer = 0xfffff80056f15000,
scl_write_wanted = 0, scl_cv = {cv_description = 0xffffffff81a1945a "scl->scl_cv", cv_waiters = 0}, scl_count = {rc_count = 1}}, {scl_lock = {lock_object = {lo_name = 0xffffffff81a1944b "scl->scl_lock",
lo_flags = 40960000, lo_data = 0, lo_witness = 0x0}, sx_lock = 1}, scl_writer = 0xfffff80056f15000, scl_write_wanted = 0, scl_cv = {cv_description = 0xffffffff81a1945a "scl->scl_cv",
cv_waiters = 0}, scl_count = {rc_count = 1}}, {scl_lock = {lock_object = {lo_name = 0xffffffff81a1944b "scl->scl_lock", lo_flags = 40960000, lo_data = 0, lo_witness = 0x0}, sx_lock = 1},
scl_writer = 0xfffff80056f15000, scl_write_wanted = 0, scl_cv = {cv_description = 0xffffffff81a1945a "scl->scl_cv", cv_waiters = 0}, scl_count = {rc_count = 1}}, {scl_lock = {lock_object = {
lo_name = 0xffffffff81a1944b "scl->scl_lock", lo_flags = 40960000, lo_data = 0, lo_witness = 0x0}, sx_lock = 1}, scl_writer = 0xfffff80056f15000, scl_write_wanted = 0, scl_cv = {
cv_description = 0xffffffff81a1945a "scl->scl_cv", cv_waiters = 0}, scl_count = {rc_count = 1}}}, spa_refcount = {rc_count = 11}, spa_splitting_newspa = 0}
(kgdb)
After rebooting and reimporting the pool it looked like this:
fk at r500 ~ $sudo zpool status -v wde4
pool: wde4
state: ONLINE
status: One or more devices has experienced an error resulting in data
corruption. Applications may be affected.
action: Restore the file in question if possible. Otherwise restore the
entire pool from backup.
see: http://illumos.org/msg/ZFS-8000-8A
scan: scrub canceled on Tue Jan 20 00:22:26 2015
config:
NAME STATE READ WRITE CKSUM
wde4 ONLINE 0 0 19
label/wde4.eli ONLINE 0 0 76
errors: Permanent errors have been detected in the following files:
<0xaf11f>:<0x0>
wde4/backup/r500/tank/home/fk:<0x0>
<0xffffffffffffffff>:<0x0>
The export triggered the same panic again, but with a different rt->rt_space value:
panic: solaris assert: rt->rt_space == 0 (0x22800 == 0x0), file: /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/range_tree.c, line: 153
I probably won't have time to scrub the pool and investigate this further
until next week.
Fabian
-------------- next part --------------
A non-text attachment was scrubbed...
Name: not available
Type: application/pgp-signature
Size: 181 bytes
Desc: OpenPGP digital signature
URL: <http://lists.freebsd.org/pipermail/freebsd-fs/attachments/20150127/afa0fbaa/attachment.sig>
More information about the freebsd-fs
mailing list