panic while zfs scrubbing

[Roger Hammerstein]

Thank you for the reply.


> Not sure what triggers this problem but it looks like zio is issued for a
> block-pointer with no valid DVA.  It's either a result of some logical bug in ZFS
> code or some severe on-disk corruption.
> 
> > #7  0xffffffff816818cf in vdev_mirror_io_start (zio=0xfffffe0037e5e000)
> >     at /usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/vdev_mirror.c:89
> 
> Could you please print *zio and *zio->io_bp in this frame?
> It might also be good idea to report this issue to zfs-discuss at opensolaris.org.





/var/crash # kgdb /boot/kernel/kernel ./vmcore.10

GNU gdb 6.1.1 [FreeBSD]

Copyright 2004 Free Software Foundation, Inc.

GDB is free software, covered by the GNU General Public License, and you are

welcome to change it and/or distribute copies of it under certain conditions.

Type "show copying" to see the conditions.

There is absolutely no warranty for GDB.  Type "show warranty" for details.

This GDB was configured as "amd64-marcel-freebsd"...



Unread portion of the kernel message buffer:





Fatal trap 18: integer divide fault while in kernel mode

cpuid = 5; apic id = 05

instruction pointer     = 0x20:0xffffffff81674a14

stack pointer           = 0x28:0xffffff810c3d4520

frame pointer           = 0x28:0xffffff810c3d4540

code segment            = base 0x0, limit 0xfffff, type 0x1b

                        = DPL 0, pres 1, long 1, def32 0, gran 1

processor eflags        = interrupt enabled, resume, IOPL = 0

current process         = 9480 (txg_thread_enter)

trap number             = 18

panic: integer divide fault

cpuid = 5

KDB: stack backtrace:

#0 0xffffffff80920346 at kdb_backtrace+0x66

#1 0xffffffff808ea35e at panic+0x1ce

#2 0xffffffff80bd7a30 at trap_fatal+0x290

#3 0xffffffff80bd80c5 at trap+0x105

#4 0xffffffff80bc295f at calltrap+0x8

#5 0xffffffff816818cf at vdev_mirror_io_start+0x2bf

#6 0xffffffff81699542 at zio_vdev_io_start+0x232

#7 0xffffffff81698fe3 at zio_execute+0xc3

#8 0xffffffff8165ea1c at dsl_scan_scrub_cb+0x3ec

#9 0xffffffff8165fe14 at dsl_scan_visitbp+0x534

#10 0xffffffff8165fd99 at dsl_scan_visitbp+0x4b9

#11 0xffffffff81660c84 at dsl_scan_visitdnode+0x84

#12 0xffffffff81660070 at dsl_scan_visitbp+0x790

#13 0xffffffff8165fd99 at dsl_scan_visitbp+0x4b9

#14 0xffffffff8165fd99 at dsl_scan_visitbp+0x4b9

#15 0xffffffff8165fd99 at dsl_scan_visitbp+0x4b9

#16 0xffffffff8165fd99 at dsl_scan_visitbp+0x4b9

#17 0xffffffff8165fd99 at dsl_scan_visitbp+0x4b9

Uptime: 1h51m55s

Dumping 614 out of 3818 MB:..3%..11%..21%..32%..42%..53%..63%..71%..81%..92%



Reading symbols from /boot/kernel/zfs.ko...Reading symbols from /boot/kernel/zfs.ko.symbols...done.

done.

Loaded symbols for /boot/kernel/zfs.ko

Reading symbols from /boot/kernel/opensolaris.ko...Reading symbols from /boot/kernel/opensolaris.ko.symbols...done.

done.

Loaded symbols for /boot/kernel/opensolaris.ko

#0  doadump (textdump=Variable "textdump" is not available.

) at pcpu.h:224

224     pcpu.h: No such file or directory.

        in pcpu.h

(kgdb) frame 7

#7  0xffffffff816818cf in vdev_mirror_io_start (zio=0xfffffe0037e5e000)

    at /usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/vdev_mirror.c:89

89                      mm->mm_preferred = spa_get_random(c);

(kgdb) p *zio

$1 = {io_bookmark = {zb_objset = 21, zb_object = 291417, zb_level = 0, zb_blkid = 41}, io_prop = {

    zp_checksum = ZIO_CHECKSUM_INHERIT, zp_compress = ZIO_COMPRESS_INHERIT, zp_type = DMU_OT_NONE,

    zp_level = 0 '\0', zp_copies = 0 '\0', zp_dedup = 0 '\0', zp_dedup_verify = 0 '\0'},

  io_type = ZIO_TYPE_READ, io_child_type = ZIO_CHILD_LOGICAL, io_cmd = 0, io_priority = 20 '\024',

  io_reexecute = 0 '\0', io_state = "\001", io_txg = 223, io_spa = 0xfffffe0008161000,

  io_bp = 0xfffffe0037e5e060, io_bp_override = 0x0, io_bp_copy = {blk_dva = {{dva_word = {0, 0}}, {

        dva_word = {0, 0}}, {dva_word = {0, 0}}}, blk_prop = 0, blk_pad = {0, 223}, blk_phys_birth = 223,

    blk_birth = 250, blk_fill = 105, blk_cksum = {zc_word = {63, 125, 202, 251}}}, io_parent_list = {

    list_size = 48, list_offset = 16, list_head = {list_next = 0xfffffe0037ea28b0,

      list_prev = 0xfffffe0037ea28b0}}, io_child_list = {list_size = 48, list_offset = 32, list_head = {

      list_next = 0xfffffe0037e5e110, list_prev = 0xfffffe0037e5e110}}, io_walk_link = 0x0,

  io_logical = 0xfffffe0037e5e000, io_transform_stack = 0x0, io_ready = 0,

  io_done = 0xffffffff8165e130 <dsl_scan_scrub_done>, io_private = 0x0, io_prev_space_delta = 0,

  io_bp_orig = {blk_dva = {{dva_word = {0, 0}}, {dva_word = {0, 0}}, {dva_word = {0, 0}}}, blk_prop = 0,

    blk_pad = {0, 223}, blk_phys_birth = 223, blk_birth = 250, blk_fill = 105, blk_cksum = {zc_word = {63,

        125, 202, 251}}}, io_data = 0xfffffe003704e200, io_orig_data = 0xfffffe003704e200, io_size = 512,

  io_orig_size = 512, io_vd = 0x0, io_vsd = 0x0, io_vsd_ops = 0x0, io_offset = 0, io_deadline = 0,

  io_offset_node = {avl_child = {0x0, 0x0}, avl_pcb = 0}, io_deadline_node = {avl_child = {0x0, 0x0},

    avl_pcb = 0}, io_vdev_tree = 0x0, io_flags = 2097264, io_stage = ZIO_STAGE_VDEV_IO_START,

  io_pipeline = 2064386, io_orig_flags = 2097264, io_orig_stage = ZIO_STAGE_OPEN,

  io_orig_pipeline = 2064386, io_error = 0, io_child_error = {0, 0, 0, 0}, io_children = {{0, 0}, {0, 0}, {

      0, 0}, {0, 0}}, io_child_count = 0, io_parent_count = 1, io_stall = 0x0, io_gang_leader = 0x0,

  io_gang_tree = 0x0, io_executor = 0xfffffe0015e9d470, io_waiter = 0x0, io_lock = {lock_object = {

      lo_name = 0xffffffff81725cee "zio->io_lock", lo_flags = 40960000, lo_data = 0, lo_witness = 0x0},

    sx_lock = 1}, io_cv = {cv_description = 0xffffffff81725cfd "zio->io_cv)", cv_waiters = 0},

  io_cksum_report = 0x0, io_ena = 0, io_task = {ost_task = {ta_link = {stqe_next = 0x0}, ta_pending = 0,

      ta_priority = 0, ta_func = 0, ta_context = 0x0}, ost_func = 0, ost_arg = 0x0}}

(kgdb)





(kgdb) p *zio->io_bp

$2 = {blk_dva = {{dva_word = {0, 0}}, {dva_word = {0, 0}}, {dva_word = {0, 0}}}, blk_prop = 0, blk_pad = {

    0, 223}, blk_phys_birth = 223, blk_birth = 250, blk_fill = 105, blk_cksum = {zc_word = {63, 125, 202,

      251}}}

(kgdb)













> > #8  0xffffffff81699542 in zio_vdev_io_start (zio=0xfffffe0037e5e000)
> >     at /usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c:2305
> > #9  0xffffffff81698fe3 in zio_execute (zio=0xfffffe0037e5e000)
> >     at /usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c:1196
> > #10 0xffffffff8165ea1c in dsl_scan_scrub_cb (dp=0xffffff810c3d4538, 
> >     bp=0xffffff8003c53480, zb=0xffffff810c3d4970)
> >     at /usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_scan.c:1737
> 
> And *bp and *scn here too.






(kgdb) frame 10

#10 0xffffffff8165ea1c in dsl_scan_scrub_cb (dp=0xffffff810c3d4538, bp=0xffffff8003c53480,

    zb=0xffffff810c3d4970)

    at /usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_scan.c:1737

1737                    zio_nowait(zio_read(NULL, spa, bp, data, size,

(kgdb) p *bp

$3 = {blk_dva = {{dva_word = {0, 0}}, {dva_word = {0, 0}}, {dva_word = {0, 0}}}, blk_prop = 0, blk_pad = {

    0, 223}, blk_phys_birth = 223, blk_birth = 250, blk_fill = 105, blk_cksum = {zc_word = {63, 125, 202,

      251}}}

(kgdb) p *scn

Cannot access memory at address 0x20

(kgdb)