Many processes stuck in zfs
Borja Marcos
borjam at sarenet.es
Wed Mar 10 14:37:51 UTC 2010
On Mar 10, 2010, at 12:02 PM, Pawel Jakub Dawidek wrote:
> Recompile your kernel and modules and reboot.
>
> Once the deadlock occur, enter DDB and send me the output of:
>
> ps
> show alllocks
> show lockedvnods
> show allchains
> alltrace
(copying to freebsd-fs, as this is filesystem-related)
There we go. Thanks for the textdump hint :)
Instead of a deadlock I had a crash. Maybe some of the additional debugging options triggered it. It is a 8-stable cvsupped on 2nd March, and with this patch applied:
http://people.freebsd.org/~pjd/patches/zfs_3way_deadlock.patch
This is ddb.txt:
#
#
# cat ddb.txt
db> set $terminal=0
Unknown variable
db> set $lines=0
db> ps
pid ppid pgrp uid state wmesg wchan cmd
1425 1423 1423 0 S+ zfsvfs-> 0xffffff000256d310 bsdtar
1424 1423 1423 0 S+ pipdwt 0xffffff00025d6000 bsdtar
1423 1384 1423 0 S+ wait 0xffffff0005106000 sh
1422 1420 1420 0 R+ bsdtar
1421 1420 1420 0 S+ pipdwt 0xffffff00025d65b0 bsdtar
1420 1384 1420 0 S+ wait 0xffffff0005107000 sh
1419 1417 1417 0 S+ zfs 0xffffff00057ac578 bsdtar
1418 1417 1417 0 S+ pipdwt 0xffffff00025d4000 bsdtar
1417 1384 1417 0 S+ wait 0xffffff0002e79000 sh
1416 1414 1414 0 S+ zio->io_ 0xffffff00026c5b28 bsdtar
1415 1414 1414 0 S+ pipdwt 0xffffff00025d6b60 bsdtar
1414 1384 1414 0 S+ wait 0xffffff0002e788c0 sh
1413 1409 1413 0 S+ select 0xffffff00029d56c0 top
1409 1408 1409 0 S+ pause 0xffffff0002d83500 csh
1408 1405 1408 1001 S+ wait 0xffffff0002e7a000 su
1405 1404 1405 1001 Ss+ pause 0xffffff0002e78500 csh
1404 1401 1401 1001 S select 0xffffff0002cce1c0 sshd
1401 1280 1401 0 Ss sbwait 0xffffff0002f63694 sshd
1384 1383 1384 0 S+ ttyin 0xffffff00024b54a8 csh
1383 1377 1383 1001 S+ wait 0xffffff0002e77460 su
1377 1376 1377 1001 Ss+ pause 0xffffff0002e79500 csh
1376 1373 1373 1001 S select 0xffffff0002916440 sshd
1373 1280 1373 0 Ss sbwait 0xffffff0002cff3ec sshd
1372 1 1372 0 Ss+ ttyin 0xffffff00024daca8 getty
1371 1 1371 0 Ss+ ttyin 0xffffff00024dc0a8 getty
1370 1 1370 0 Ss+ ttyin 0xffffff00024cdca8 getty
1369 1 1369 0 Ss+ ttyin 0xffffff00024c94a8 getty
1368 1 1368 0 Ss+ ttyin 0xffffff00024dd4a8 getty
1367 1 1367 0 Ss+ ttyin 0xffffff00024c28a8 getty
1366 1 1366 0 Ss+ ttyin 0xffffff00024db4a8 getty
1365 1 1365 0 Ss+ ttyin 0xffffff00024dc8a8 getty
1301 1 1301 0 Ss nanslp 0xffffffff80c4d168 cron
1294 1 1294 25 Ss pause 0xffffff0002623960 sendmail
1288 1 1288 0 Ss select 0xffffff0002917cc0 sendmail
1280 1 1280 0 Ss select 0xffffff00029d5340 sshd
1005 1 1005 0 Rs syslogd
827 1 827 0 Ss select 0xffffff0002ccf2c0 devd
20 0 0 0 SL flowclea 0xffffffff80e0e950 [flowcleaner]
19 0 0 0 SL sdflush 0xffffffff80e1d8b8 [softdepflush]
18 0 0 0 RL [syncer]
17 0 0 0 SL vlruwt 0xffffff000256a8c0 [vnlru]
16 0 0 0 SL psleep 0xffffffff80e0e1c8 [bufdaemon]
15 0 0 0 SL pgzero 0xffffffff80e1f34c [pagezero]
14 0 0 0 SL psleep 0xffffffff80e1e6e8 [vmdaemon]
9 0 0 0 RL [pagedaemon]
8 0 0 0 SL ccb_scan 0xffffffff80c16060 [xpt_thrd]
7 0 0 0 RL (threaded) zfskern
100085 Run CPU 0 [txg_thread_enter]
100084 RunQ [txg_thread_enter]
100082 D vgeom:io 0xffffff00026acd90 [vdev da2]
100036 D l2arc_fe 0xffffffff81150620 [l2arc_feed_thread]
100035 D arc_recl 0xffffffff81147fe0 [arc_reclaim_thread]
6 0 0 0 SL - 0xffffff000244e648 [fdc0]
5 0 0 0 SL idle 0xffffff800023c000 [mpt_recovery0]
13 0 0 0 SL - 0xffffffff80c4ce44 [yarrow]
4 0 0 0 RL [g_down]
3 0 0 0 RL [g_up]
2 0 0 0 RL [g_event]
12 0 0 0 WL (threaded) intr
100033 I [swi0: uart uart]
100032 I [irq7: ppc0]
100031 I [irq12: psm0]
100030 I [irq1: atkbd0]
100024 I [irq17: em2 mpt0]
100023 I [irq15: ata1]
100022 I [irq14: ata0]
100021 I [irq9: acpi0]
100019 I [swi5: +]
100018 I [swi2: cambio]
100013 I [swi6: task queue]
100012 I [swi6: Giant taskq]
100006 I [swi1: netisr 0]
100005 I [swi4: clock]
100004 I [swi3: vm]
11 0 0 0 RL [idle: cpu0]
1 0 1 0 SLs wait 0xffffff00021e18c0 [init]
10 0 0 0 SL audit_wo 0xffffffff80e1cc10 [audit]
0 0 0 0 SLs (threaded) kernel
100088 D - 0xffffff0002914880 [zil_clean]
100087 D - 0xffffff0002916780 [zil_clean]
100086 D - 0xffffff0002913900 [zil_clean]
100083 D - 0xffffff0002628480 [zfs_vn_rele_taskq]
100081 D - 0xffffff00026acc00 [spa_zio]
100080 D - 0xffffff00026acb80 [spa_zio]
100079 D - 0xffffff00026acb00 [spa_zio]
100078 D - 0xffffff00026aca80 [spa_zio]
100077 D - 0xffffff00026aca00 [spa_zio]
100076 D - 0xffffff00026ac980 [spa_zio]
100075 D - 0xffffff00026ac900 [spa_zio]
100074 D - 0xffffff00026ac880 [spa_zio_7]
100073 D - 0xffffff00026ac880 [spa_zio_6]
100072 D - 0xffffff00026ac880 [spa_zio_5]
100071 D - 0xffffff00026ac880 [spa_zio_4]
100070 D - 0xffffff00026ac880 [spa_zio_3]
100069 D - 0xffffff00026ac880 [spa_zio_2]
100068 D - 0xffffff00026ac880 [spa_zio_1]
100067 D - 0xffffff00026ac880 [spa_zio_0]
100066 D - 0xffffff00026ac800 [spa_zio_7]
100065 D - 0xffffff00026ac800 [spa_zio_6]
100064 D - 0xffffff00026ac800 [spa_zio_5]
100063 D - 0xffffff00026ac800 [spa_zio_4]
100062 D - 0xffffff00026ac800 [spa_zio_3]
100061 D - 0xffffff00026ac800 [spa_zio_2]
100060 D - 0xffffff00026ac800 [spa_zio_1]
100059 D - 0xffffff00026ac800 [spa_zio_0]
100058 D - 0xffffff00026ac780 [spa_zio]
100057 D - 0xffffff00026ac700 [spa_zio]
100056 D - 0xffffff00026ac680 [spa_zio]
100029 D - 0xffffff0002441980 [em3 taskq]
100028 D - 0xffffff0002440d80 [em2 taskq]
100027 D - 0xffffff0002436200 [em1 taskq]
100026 D - 0xffffff000242fb80 [em0 taskq]
100020 D - 0xffffff0002358600 [thread taskq]
100017 D - 0xffffff0002316b80 [acpi_task_2]
100016 D - 0xffffff0002316b80 [acpi_task_1]
100015 D - 0xffffff0002316b80 [acpi_task_0]
100014 D - 0xffffff0002316c00 [kqueue taskq]
100010 D - 0xffffff00021e0180 [firmware taskq]
100000 D sched 0xffffffff80c49740 [swapper]
db> show alllocks
Process 1425 (bsdtar) thread 0xffffff000528b3a0 (100115)
Process 1422 (bsdtar) thread 0xffffff000528c000 (100112)
Process 1419 (bsdtar) thread 0xffffff0002d81000 (100109)
Process 1416 (bsdtar) thread 0xffffff000262a3a0 (100091)
Process 1401 (sshd) thread 0xffffff0002510740 (100055)
Process 1373 (sshd) thread 0xffffff000262a740 (100090)
Process 7 (zfskern) thread 0xffffff000262bae0 (100085)
db> show lockedvnods
Locked vnodes
db> show allchains
db> alltrace
Tracing command bsdtar pid 1425 tid 100115 td 0xffffff000528b3a0
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_wait() at sleepq_wait+0x4d
_sx_xlock_hard() at _sx_xlock_hard+0x267
_sx_xlock() at _sx_xlock+0xc1
zfs_zinactive() at zfs_zinactive+0x95
zfs_inactive() at zfs_inactive+0x7e
zfs_freebsd_inactive() at zfs_freebsd_inactive+0x1a
VOP_INACTIVE_APV() at VOP_INACTIVE_APV+0xb5
vinactive() at vinactive+0x90
vputx() at vputx+0x2fc
kern_statat_vnhook() at kern_statat_vnhook+0xfa
kern_statat() at kern_statat+0x15
lstat() at lstat+0x2a
syscall() at syscall+0x118
Xfast_syscall() at Xfast_syscall+0xe1
--- syscall (190, FreeBSD ELF64, lstat), rip = 0x800e35acc, rsp = 0x7fffffffe768, rbp = 0x801025380 ---
Tracing command bsdtar pid 1424 tid 100114 td 0xffffff000528b740
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_catch_signals() at sleepq_catch_signals+0x2af
sleepq_wait_sig() at sleepq_wait_sig+0x16
_sleep() at _sleep+0x309
pipe_write() at pipe_write+0x13fc
dofilewrite() at dofilewrite+0x85
kern_writev() at kern_writev+0x60
write() at write+0x55
syscall() at syscall+0x118
Xfast_syscall() at Xfast_syscall+0xe1
--- syscall (4, FreeBSD ELF64, write), rip = 0x800e44d5c, rsp = 0x7fffffffe0d8, rbp = 0x801004140 ---
Tracing command sh pid 1423 tid 100113 td 0xffffff000528bae0
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_catch_signals() at sleepq_catch_signals+0x2af
sleepq_wait_sig() at sleepq_wait_sig+0x16
_sleep() at _sleep+0x309
kern_wait() at kern_wait+0x3f7
wait4() at wait4+0x35
syscall() at syscall+0x118
Xfast_syscall() at Xfast_syscall+0xe1
--- syscall (7, FreeBSD ELF64, wait4), rip = 0x80093554c, rsp = 0x7fffffffe8b8, rbp = 0x58f ---
Tracing command bsdtar pid 1422 tid 100112 td 0xffffff000528c000
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_wait() at sleepq_wait+0x4d
_cv_wait() at _cv_wait+0x17a
txg_wait_open() at txg_wait_open+0x95
zfs_freebsd_setattr() at zfs_freebsd_setattr+0x862
VOP_SETATTR_APV() at VOP_SETATTR_APV+0xaf
setutimes() at setutimes+0x214
kern_lutimes() at kern_lutimes+0xf2
syscall() at syscall+0x118
Xfast_syscall() at Xfast_syscall+0xe1
--- syscall (276, FreeBSD ELF64, lutimes), rip = 0x800da66dc, rsp = 0x7fffffffe7e8, rbp = 0x801025380 ---
Tracing command bsdtar pid 1421 tid 100111 td 0xffffff000528c3a0
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_catch_signals() at sleepq_catch_signals+0x2af
sleepq_wait_sig() at sleepq_wait_sig+0x16
_sleep() at _sleep+0x309
pipe_write() at pipe_write+0x13fc
dofilewrite() at dofilewrite+0x85
kern_writev() at kern_writev+0x60
write() at write+0x55
syscall() at syscall+0x118
Xfast_syscall() at Xfast_syscall+0xe1
--- syscall (4, FreeBSD ELF64, write), rip = 0x800e44d5c, rsp = 0x7fffffffe0d8, rbp = 0x801004140 ---
Tracing command sh pid 1420 tid 100110 td 0xffffff000262dae0
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_catch_signals() at sleepq_catch_signals+0x2af
sleepq_wait_sig() at sleepq_wait_sig+0x16
_sleep() at _sleep+0x309
kern_wait() at kern_wait+0x3f7
wait4() at wait4+0x35
syscall() at syscall+0x118
Xfast_syscall() at Xfast_syscall+0xe1
--- syscall (7, FreeBSD ELF64, wait4), rip = 0x80093554c, rsp = 0x7fffffffe8b8, rbp = 0x58c ---
Tracing command bsdtar pid 1419 tid 100109 td 0xffffff0002d81000
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_wait() at sleepq_wait+0x4d
__lockmgr_args() at __lockmgr_args+0x7a3
vop_stdlock() at vop_stdlock+0x39
VOP_LOCK1_APV() at VOP_LOCK1_APV+0x9b
_vn_lock() at _vn_lock+0x57
vget() at vget+0x7b
cache_lookup() at cache_lookup+0x4e0
vfs_cache_lookup() at vfs_cache_lookup+0xc0
VOP_LOOKUP_APV() at VOP_LOOKUP_APV+0xb7
lookup() at lookup+0x3d3
namei() at namei+0x4a9
kern_statat_vnhook() at kern_statat_vnhook+0x8f
kern_statat() at kern_statat+0x15
lstat() at lstat+0x2a
syscall() at syscall+0x118
Xfast_syscall() at Xfast_syscall+0xe1
--- syscall (190, FreeBSD ELF64, lstat), rip = 0x800e35acc, rsp = 0x7fffffffe768, rbp = 0x801025380 ---
Tracing command bsdtar pid 1418 tid 100108 td 0xffffff0002d813a0
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_catch_signals() at sleepq_catch_signals+0x2af
sleepq_wait_sig() at sleepq_wait_sig+0x16
_sleep() at _sleep+0x309
pipe_write() at pipe_write+0x13fc
dofilewrite() at dofilewrite+0x85
kern_writev() at kern_writev+0x60
write() at write+0x55
syscall() at syscall+0x118
Xfast_syscall() at Xfast_syscall+0xe1
--- syscall (4, FreeBSD ELF64, write), rip = 0x800e44d5c, rsp = 0x7fffffffe0d8, rbp = 0x801004140 ---
Tracing command sh pid 1417 tid 100101 td 0xffffff0002cfb000
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_catch_signals() at sleepq_catch_signals+0x2af
sleepq_wait_sig() at sleepq_wait_sig+0x16
_sleep() at _sleep+0x309
kern_wait() at kern_wait+0x3f7
wait4() at wait4+0x35
syscall() at syscall+0x118
Xfast_syscall() at Xfast_syscall+0xe1
--- syscall (7, FreeBSD ELF64, wait4), rip = 0x80093554c, rsp = 0x7fffffffe8b8, rbp = 0x589 ---
Tracing command bsdtar pid 1416 tid 100091 td 0xffffff000262a3a0
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_wait() at sleepq_wait+0x4d
_cv_wait() at _cv_wait+0x17a
zio_wait() at zio_wait+0x7d
dbuf_read() at dbuf_read+0x2bb
dnode_hold_impl() at dnode_hold_impl+0x114
dmu_bonus_hold() at dmu_bonus_hold+0x31
zfs_zget() at zfs_zget+0x259
zfs_dirent_lock() at zfs_dirent_lock+0x4a0
zfs_dirlook() at zfs_dirlook+0x90
zfs_lookup() at zfs_lookup+0x256
zfs_freebsd_lookup() at zfs_freebsd_lookup+0x8d
VOP_CACHEDLOOKUP_APV() at VOP_CACHEDLOOKUP_APV+0xaf
vfs_cache_lookup() at vfs_cache_lookup+0xf0
VOP_LOOKUP_APV() at VOP_LOOKUP_APV+0xb7
lookup() at lookup+0x3d3
namei() at namei+0x4a9
kern_mkdirat() at kern_mkdirat+0xc3
syscall() at syscall+0x118
Xfast_syscall() at Xfast_syscall+0xe1
--- syscall (136, FreeBSD ELF64, mkdir), rip = 0x800e35b0c, rsp = 0x7fffffffe748, rbp = 0x1ed ---
Tracing command bsdtar pid 1415 tid 100105 td 0xffffff0002cfa000
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_catch_signals() at sleepq_catch_signals+0x2af
sleepq_wait_sig() at sleepq_wait_sig+0x16
_sleep() at _sleep+0x309
pipe_write() at pipe_write+0x13fc
dofilewrite() at dofilewrite+0x85
kern_writev() at kern_writev+0x60
write() at write+0x55
syscall() at syscall+0x118
Xfast_syscall() at Xfast_syscall+0xe1
--- syscall (4, FreeBSD ELF64, write), rip = 0x800e44d5c, rsp = 0x7fffffffe0d8, rbp = 0x801004140 ---
Tracing command sh pid 1414 tid 100102 td 0xffffff0002cfaae0
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_catch_signals() at sleepq_catch_signals+0x2af
sleepq_wait_sig() at sleepq_wait_sig+0x16
_sleep() at _sleep+0x309
kern_wait() at kern_wait+0x3f7
wait4() at wait4+0x35
syscall() at syscall+0x118
Xfast_syscall() at Xfast_syscall+0xe1
--- syscall (7, FreeBSD ELF64, wait4), rip = 0x80093554c, rsp = 0x7fffffffe8b8, rbp = 0x586 ---
Tracing command top pid 1413 tid 100054 td 0xffffff0002510ae0
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_catch_signals() at sleepq_catch_signals+0x2af
sleepq_timedwait_sig() at sleepq_timedwait_sig+0x19
_cv_timedwait_sig() at _cv_timedwait_sig+0x18c
seltdwait() at seltdwait+0x56
kern_select() at kern_select+0x620
select() at select+0x5d
syscall() at syscall+0x118
Xfast_syscall() at Xfast_syscall+0xe1
--- syscall (93, FreeBSD ELF64, select), rip = 0x800ab7cfc, rsp = 0x7fffffffea08, rbp = 0x11 ---
Tracing command csh pid 1409 tid 100093 td 0xffffff0002645ae0
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_catch_signals() at sleepq_catch_signals+0x2af
sleepq_wait_sig() at sleepq_wait_sig+0x16
_sleep() at _sleep+0x309
kern_sigsuspend() at kern_sigsuspend+0x9a
sigsuspend() at sigsuspend+0x34
syscall() at syscall+0x118
Xfast_syscall() at Xfast_syscall+0xe1
--- syscall (4, FreeBSD ELF64, write), rip = 0x80094d19c, rsp = 0x7fffffffe638, rbp = 0x800c5a400 ---
Tracing command su pid 1408 tid 100098 td 0xffffff0002cfbae0
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_catch_signals() at sleepq_catch_signals+0x2af
sleepq_wait_sig() at sleepq_wait_sig+0x16
_sleep() at _sleep+0x309
kern_wait() at kern_wait+0x3f7
wait4() at wait4+0x35
syscall() at syscall+0x118
Xfast_syscall() at Xfast_syscall+0xe1
--- syscall (7, FreeBSD ELF64, wait4), rip = 0x8009e854c, rsp = 0x7fffffffe598, rbp = 0x581 ---
Tracing command csh pid 1405 tid 100103 td 0xffffff0002cfa740
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_catch_signals() at sleepq_catch_signals+0x2af
sleepq_wait_sig() at sleepq_wait_sig+0x16
_sleep() at _sleep+0x309
kern_sigsuspend() at kern_sigsuspend+0x9a
sigsuspend() at sigsuspend+0x34
syscall() at syscall+0x118
Xfast_syscall() at Xfast_syscall+0xe1
--- syscall (4, FreeBSD ELF64, write), rip = 0x80094d19c, rsp = 0x7fffffffe558, rbp = 0x800c0f900 ---
Tracing command sshd pid 1404 tid 100104 td 0xffffff0002cfa3a0
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_catch_signals() at sleepq_catch_signals+0x2af
sleepq_wait_sig() at sleepq_wait_sig+0x16
_cv_wait_sig() at _cv_wait_sig+0x17e
seltdwait() at seltdwait+0xac
kern_select() at kern_select+0x620
select() at select+0x5d
syscall() at syscall+0x118
Xfast_syscall() at Xfast_syscall+0xe1
--- syscall (93, FreeBSD ELF64, select), rip = 0x8013d7cfc, rsp = 0x7fffffffdcb8, rbp = 0x7fffffffdd40 ---
Tracing command sshd pid 1401 tid 100055 td 0xffffff0002510740
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_catch_signals() at sleepq_catch_signals+0x2af
sleepq_wait_sig() at sleepq_wait_sig+0x16
_sleep() at _sleep+0x309
soreceive_generic() at soreceive_generic+0xf99
dofileread() at dofileread+0xa1
kern_readv() at kern_readv+0x60
read() at read+0x55
syscall() at syscall+0x118
Xfast_syscall() at Xfast_syscall+0xe1
--- syscall (3, FreeBSD ELF64, read), rip = 0x8013d7d7c, rsp = 0x7fffffffdcd8, rbp = 0 ---
Tracing command csh pid 1384 tid 100107 td 0xffffff0002d81740
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_catch_signals() at sleepq_catch_signals+0x2af
sleepq_wait_sig() at sleepq_wait_sig+0x16
_cv_wait_sig() at _cv_wait_sig+0x17e
tty_wait() at tty_wait+0x48
ttydisc_read() at ttydisc_read+0x2f1
ttydev_read() at ttydev_read+0xab
devfs_read_f() at devfs_read_f+0x86
dofileread() at dofileread+0xa1
kern_readv() at kern_readv+0x60
read() at read+0x55
syscall() at syscall+0x118
Xfast_syscall() at Xfast_syscall+0xe1
--- syscall (3, FreeBSD ELF64, read), rip = 0x8009ebd7c, rsp = 0x7fffffffe738, rbp = 0x1 ---
Tracing command su pid 1383 tid 100106 td 0xffffff0002d81ae0
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_catch_signals() at sleepq_catch_signals+0x2af
sleepq_wait_sig() at sleepq_wait_sig+0x16
_sleep() at _sleep+0x309
kern_wait() at kern_wait+0x3f7
wait4() at wait4+0x35
syscall() at syscall+0x118
Xfast_syscall() at Xfast_syscall+0xe1
--- syscall (7, FreeBSD ELF64, wait4), rip = 0x8009e854c, rsp = 0x7fffffffe598, rbp = 0x568 ---
Tracing command csh pid 1377 tid 100100 td 0xffffff0002cfb3a0
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_catch_signals() at sleepq_catch_signals+0x2af
sleepq_wait_sig() at sleepq_wait_sig+0x16
_sleep() at _sleep+0x309
kern_sigsuspend() at kern_sigsuspend+0x9a
sigsuspend() at sigsuspend+0x34
syscall() at syscall+0x118
Xfast_syscall() at Xfast_syscall+0xe1
--- syscall (4, FreeBSD ELF64, write), rip = 0x80094d19c, rsp = 0x7fffffffe558, rbp = 0x800c0f600 ---
Tracing command sshd pid 1376 tid 100099 td 0xffffff0002cfb740
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_catch_signals() at sleepq_catch_signals+0x2af
sleepq_wait_sig() at sleepq_wait_sig+0x16
_cv_wait_sig() at _cv_wait_sig+0x17e
seltdwait() at seltdwait+0xac
kern_select() at kern_select+0x620
select() at select+0x5d
syscall() at syscall+0x118
Xfast_syscall() at Xfast_syscall+0xe1
--- syscall (93, FreeBSD ELF64, select), rip = 0x8013d7cfc, rsp = 0x7fffffffdcb8, rbp = 0x7fffffffdd40 ---
Tracing command sshd pid 1373 tid 100090 td 0xffffff000262a740
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_catch_signals() at sleepq_catch_signals+0x2af
sleepq_wait_sig() at sleepq_wait_sig+0x16
_sleep() at _sleep+0x309
soreceive_generic() at soreceive_generic+0xf99
dofileread() at dofileread+0xa1
kern_readv() at kern_readv+0x60
read() at read+0x55
syscall() at syscall+0x118
Xfast_syscall() at Xfast_syscall+0xe1
--- syscall (3, FreeBSD ELF64, read), rip = 0x8013d7d7c, rsp = 0x7fffffffdcd8, rbp = 0 ---
Tracing command getty pid 1372 tid 100046 td 0xffffff00025b1ae0
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_catch_signals() at sleepq_catch_signals+0x2af
sleepq_wait_sig() at sleepq_wait_sig+0x16
_cv_wait_sig() at _cv_wait_sig+0x17e
tty_wait() at tty_wait+0x48
ttydisc_read() at ttydisc_read+0x2f1
ttydev_read() at ttydev_read+0xab
devfs_read_f() at devfs_read_f+0x86
dofileread() at dofileread+0xa1
kern_readv() at kern_readv+0x60
read() at read+0x55
syscall() at syscall+0x118
Xfast_syscall() at Xfast_syscall+0xe1
--- syscall (3, FreeBSD ELF64, read), rip = 0x80084ed7c, rsp = 0x7fffffffecc8, rbp = 0 ---
Tracing command getty pid 1371 tid 100095 td 0xffffff00026453a0
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_catch_signals() at sleepq_catch_signals+0x2af
sleepq_wait_sig() at sleepq_wait_sig+0x16
_cv_wait_sig() at _cv_wait_sig+0x17e
tty_wait() at tty_wait+0x48
ttydisc_read() at ttydisc_read+0x2f1
ttydev_read() at ttydev_read+0xab
devfs_read_f() at devfs_read_f+0x86
dofileread() at dofileread+0xa1
kern_readv() at kern_readv+0x60
read() at read+0x55
syscall() at syscall+0x118
Xfast_syscall() at Xfast_syscall+0xe1
--- syscall (3, FreeBSD ELF64, read), rip = 0x80084ed7c, rsp = 0x7fffffffecc8, rbp = 0 ---
Tracing command getty pid 1370 tid 100096 td 0xffffff0002da63a0
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_catch_signals() at sleepq_catch_signals+0x2af
sleepq_wait_sig() at sleepq_wait_sig+0x16
_cv_wait_sig() at _cv_wait_sig+0x17e
tty_wait() at tty_wait+0x48
ttydisc_read() at ttydisc_read+0x2f1
ttydev_read() at ttydev_read+0xab
devfs_read_f() at devfs_read_f+0x86
dofileread() at dofileread+0xa1
kern_readv() at kern_readv+0x60
read() at read+0x55
syscall() at syscall+0x118
Xfast_syscall() at Xfast_syscall+0xe1
--- syscall (3, FreeBSD ELF64, read), rip = 0x80084ed7c, rsp = 0x7fffffffecc8, rbp = 0 ---
Tracing command getty pid 1369 tid 100097 td 0xffffff0002da6000
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_catch_signals() at sleepq_catch_signals+0x2af
sleepq_wait_sig() at sleepq_wait_sig+0x16
_cv_wait_sig() at _cv_wait_sig+0x17e
tty_wait() at tty_wait+0x48
ttydisc_read() at ttydisc_read+0x2f1
ttydev_read() at ttydev_read+0xab
devfs_read_f() at devfs_read_f+0x86
dofileread() at dofileread+0xa1
kern_readv() at kern_readv+0x60
read() at read+0x55
syscall() at syscall+0x118
Xfast_syscall() at Xfast_syscall+0xe1
--- syscall (3, FreeBSD ELF64, read), rip = 0x80084ed7c, rsp = 0x7fffffffecc8, rbp = 0 ---
Tracing command getty pid 1368 tid 100092 td 0xffffff000262a000
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_catch_signals() at sleepq_catch_signals+0x2af
sleepq_wait_sig() at sleepq_wait_sig+0x16
_cv_wait_sig() at _cv_wait_sig+0x17e
tty_wait() at tty_wait+0x48
ttydisc_read() at ttydisc_read+0x2f1
ttydev_read() at ttydev_read+0xab
devfs_read_f() at devfs_read_f+0x86
dofileread() at dofileread+0xa1
kern_readv() at kern_readv+0x60
read() at read+0x55
syscall() at syscall+0x118
Xfast_syscall() at Xfast_syscall+0xe1
--- syscall (3, FreeBSD ELF64, read), rip = 0x80084ed7c, rsp = 0x7fffffffecc8, rbp = 0 ---
Tracing command getty pid 1367 tid 100094 td 0xffffff0002645740
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_catch_signals() at sleepq_catch_signals+0x2af
sleepq_wait_sig() at sleepq_wait_sig+0x16
_cv_wait_sig() at _cv_wait_sig+0x17e
tty_wait() at tty_wait+0x48
ttydisc_read() at ttydisc_read+0x2f1
ttydev_read() at ttydev_read+0xab
devfs_read_f() at devfs_read_f+0x86
dofileread() at dofileread+0xa1
kern_readv() at kern_readv+0x60
read() at read+0x55
syscall() at syscall+0x118
Xfast_syscall() at Xfast_syscall+0xe1
--- syscall (3, FreeBSD ELF64, read), rip = 0x80084ed7c, rsp = 0x7fffffffecc8, rbp = 0 ---
Tracing command getty pid 1366 tid 100089 td 0xffffff000262aae0
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_catch_signals() at sleepq_catch_signals+0x2af
sleepq_wait_sig() at sleepq_wait_sig+0x16
_cv_wait_sig() at _cv_wait_sig+0x17e
tty_wait() at tty_wait+0x48
ttydisc_read() at ttydisc_read+0x2f1
ttydev_read() at ttydev_read+0xab
devfs_read_f() at devfs_read_f+0x86
dofileread() at dofileread+0xa1
kern_readv() at kern_readv+0x60
read() at read+0x55
syscall() at syscall+0x118
Xfast_syscall() at Xfast_syscall+0xe1
--- syscall (3, FreeBSD ELF64, read), rip = 0x80084ed7c, rsp = 0x7fffffffecc8, rbp = 0 ---
Tracing command getty pid 1365 tid 100052 td 0xffffff00025b03a0
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_catch_signals() at sleepq_catch_signals+0x2af
sleepq_wait_sig() at sleepq_wait_sig+0x16
_cv_wait_sig() at _cv_wait_sig+0x17e
tty_wait() at tty_wait+0x48
ttydisc_read() at ttydisc_read+0x2f1
ttydev_read() at ttydev_read+0xab
devfs_read_f() at devfs_read_f+0x86
dofileread() at dofileread+0xa1
kern_readv() at kern_readv+0x60
read() at read+0x55
syscall() at syscall+0x118
Xfast_syscall() at Xfast_syscall+0xe1
--- syscall (3, FreeBSD ELF64, read), rip = 0x80084ed7c, rsp = 0x7fffffffecc8, rbp = 0 ---
Tracing command cron pid 1301 tid 100053 td 0xffffff00025b0000
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_catch_signals() at sleepq_catch_signals+0x2af
sleepq_timedwait_sig() at sleepq_timedwait_sig+0x19
_sleep() at _sleep+0x23f
kern_nanosleep() at kern_nanosleep+0x118
nanosleep() at nanosleep+0x6e
syscall() at syscall+0x118
Xfast_syscall() at Xfast_syscall+0xe1
--- syscall (240, FreeBSD ELF64, nanosleep), rip = 0x80093efac, rsp = 0x7fffffffeb28, rbp = 0x3b ---
Tracing command sendmail pid 1294 tid 100048 td 0xffffff00025b13a0
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_catch_signals() at sleepq_catch_signals+0x2af
sleepq_wait_sig() at sleepq_wait_sig+0x16
_sleep() at _sleep+0x309
kern_sigsuspend() at kern_sigsuspend+0x9a
sigsuspend() at sigsuspend+0x34
syscall() at syscall+0x118
Xfast_syscall() at Xfast_syscall+0xe1
--- syscall (4, FreeBSD ELF64, write), rip = 0x800d3a19c, rsp = 0x7fffffffcd68, rbp = 0x1 ---
Tracing command sendmail pid 1288 tid 100051 td 0xffffff00025b0740
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_catch_signals() at sleepq_catch_signals+0x2af
sleepq_timedwait_sig() at sleepq_timedwait_sig+0x19
_cv_timedwait_sig() at _cv_timedwait_sig+0x18c
seltdwait() at seltdwait+0x56
kern_select() at kern_select+0x620
select() at select+0x5d
syscall() at syscall+0x118
Xfast_syscall() at Xfast_syscall+0xe1
--- syscall (93, FreeBSD ELF64, select), rip = 0x800dd8cfc, rsp = 0x7fffffffc1e8, rbp = 0x7fffffffc280 ---
Tracing command sshd pid 1280 tid 100047 td 0xffffff00025b1740
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_catch_signals() at sleepq_catch_signals+0x2af
sleepq_wait_sig() at sleepq_wait_sig+0x16
_cv_wait_sig() at _cv_wait_sig+0x17e
seltdwait() at seltdwait+0xac
kern_select() at kern_select+0x620
select() at select+0x5d
syscall() at syscall+0x118
Xfast_syscall() at Xfast_syscall+0xe1
--- syscall (93, FreeBSD ELF64, select), rip = 0x8013d7cfc, rsp = 0x7fffffffddd8, rbp = 0x2 ---
Tracing command syslogd pid 1005 tid 100050 td 0xffffff00025b0ae0
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_catch_signals() at sleepq_catch_signals+0x2af
sleepq_wait_sig() at sleepq_wait_sig+0x16
_cv_wait_sig() at _cv_wait_sig+0x17e
seltdwait() at seltdwait+0xac
kern_select() at kern_select+0x620
select() at select+0x5d
syscall() at syscall+0x118
Xfast_syscall() at Xfast_syscall+0xe1
--- syscall (93, FreeBSD ELF64, select), rip = 0x800851cfc, rsp = 0x7fffffffe118, rbp = 0x800a310c8 ---
Tracing command devd pid 827 tid 100049 td 0xffffff00025b1000
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_catch_signals() at sleepq_catch_signals+0x2af
sleepq_wait_sig() at sleepq_wait_sig+0x16
_cv_wait_sig() at _cv_wait_sig+0x17e
seltdwait() at seltdwait+0xac
kern_select() at kern_select+0x620
select() at select+0x5d
syscall() at syscall+0x118
Xfast_syscall() at Xfast_syscall+0xe1
--- syscall (93, FreeBSD ELF64, select), rip = 0x440ecc, rsp = 0x7fffffffe898, rbp = 0x7fffffffe8b0 ---
Tracing command flowcleaner pid 20 tid 100045 td 0xffffff0002485ae0
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_timedwait() at sleepq_timedwait+0x4d
_cv_timedwait() at _cv_timedwait+0x18c
flowtable_cleaner() at flowtable_cleaner+0xde
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip = 0, rsp = 0xffffff800018cd30, rbp = 0 ---
Tracing command softdepflush pid 19 tid 100044 td 0xffffff000250d000
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_timedwait() at sleepq_timedwait+0x4d
_sleep() at _sleep+0x341
softdep_flush() at softdep_flush+0x259
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip = 0, rsp = 0xffffff8000187d30, rbp = 0 ---
Tracing command syncer pid 18 tid 100043 td 0xffffff000250d3a0
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_timedwait() at sleepq_timedwait+0x4d
_cv_timedwait() at _cv_timedwait+0x18c
sched_sync() at sched_sync+0x4db
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip = 0, rsp = 0xffffff8000182d30, rbp = 0 ---
Tracing command vnlru pid 17 tid 100042 td 0xffffff000250d740
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_timedwait() at sleepq_timedwait+0x4d
_sleep() at _sleep+0x341
vnlru_proc() at vnlru_proc+0x5cf
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip = 0, rsp = 0xffffff800017dd30, rbp = 0 ---
Tracing command bufdaemon pid 16 tid 100041 td 0xffffff000250dae0
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_timedwait() at sleepq_timedwait+0x4d
_sleep() at _sleep+0x341
buf_daemon() at buf_daemon+0x14a
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip = 0, rsp = 0xffffff8000178d30, rbp = 0 ---
Tracing command pagezero pid 15 tid 100040 td 0xffffff000250e000
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_timedwait() at sleepq_timedwait+0x4d
_sleep() at _sleep+0x341
vm_pagezero() at vm_pagezero+0x73
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip = 0, rsp = 0xffffff8000173d30, rbp = 0 ---
Tracing command vmdaemon pid 14 tid 100039 td 0xffffff000250e3a0
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_wait() at sleepq_wait+0x4d
_sleep() at _sleep+0x357
vm_daemon() at vm_daemon+0x4d
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip = 0, rsp = 0xffffff800016ed30, rbp = 0 ---
Tracing command pagedaemon pid 9 tid 100038 td 0xffffff000250e740
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_timedwait() at sleepq_timedwait+0x4d
_sleep() at _sleep+0x341
vm_pageout() at vm_pageout+0x855
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip = 0, rsp = 0xffffff8000169d30, rbp = 0 ---
Tracing command xpt_thrd pid 8 tid 100037 td 0xffffff000250eae0
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_wait() at sleepq_wait+0x4d
_sleep() at _sleep+0x357
xpt_scanner_thread() at xpt_scanner_thread+0xd5
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip = 0, rsp = 0xffffff8000164d30, rbp = 0 ---
Tracing command zfskern pid 7 tid 100085 td 0xffffff000262bae0
kdb_enter() at kdb_enter+0x3d
panic() at panic+0x17b
dblfault_handler() at dblfault_handler+0x96
Xdblfault() at Xdblfault+0xac
--- trap 0x17, rip = 0xffffffff805d322d, rsp = 0xffffffff80e5ecd0, rbp = 0xffffff8012415090 ---
witness_checkorder() at witness_checkorder+0x2d
_mtx_lock_spin_flags() at _mtx_lock_spin_flags+0x94
callout_tick() at callout_tick+0x41
hardclock_cpu() at hardclock_cpu+0xc8
hardclock() at hardclock+0x23
lapic_handle_timer() at lapic_handle_timer+0xd2
Xtimerint() at Xtimerint+0x8c
--- interrupt, rip = 0xffffffff80866093, rsp = 0xffffff8012415270, rbp = 0xffffff8012415290 ---
spinlock_exit() at spinlock_exit+0x33
putcons() at putcons+0x40
putchar() at putchar+0x6a
kvprintf() at kvprintf+0x81
vprintf() at vprintf+0x85
printf() at printf+0x67
db_putc() at db_putc+0x81
kvprintf() at kvprintf+0x81
db_printf() at db_printf+0x86
db_backtrace() at db_backtrace+0x162
db_trace_self_wrapper() at db_trace_self_wrapper+0x2a
_witness_debugger() at _witness_debugger+0x2e
witness_checkorder() at witness_checkorder+0x81e
_sx_slock() at _sx_slock+0x55
dnode_verify() at dnode_verify+0x70
dnode_hold_impl() at dnode_hold_impl+0x73
dmu_bonus_hold() at dmu_bonus_hold+0x31
bplist_hold() at bplist_hold+0x48
bplist_enqueue() at bplist_enqueue+0x4c
dsl_dataset_block_kill() at dsl_dataset_block_kill+0x119
dbuf_write() at dbuf_write+0x24c
dbuf_sync_list() at dbuf_sync_list+0x3eb
dbuf_sync_list() at dbuf_sync_list+0x17f
dbuf_sync_list() at dbuf_sync_list+0x17f
dbuf_sync_list() at dbuf_sync_list+0x17f
dbuf_sync_list() at dbuf_sync_list+0x17f
dbuf_sync_list() at dbuf_sync_list+0x17f
dbuf_sync_list() at dbuf_sync_list+0x17f
dnode_sync() at dnode_sync+0xc12
dmu_objset_sync() at dmu_objset_sync+0x134
dsl_pool_sync() at dsl_pool_sync+0x88
spa_sync() at spa_sync+0x35e
txg_sync_thread() at txg_sync_thread+0x2d7
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip = 0, rsp = 0xffffff8012417d30, rbp = 0 ---
Tracing command zfskern pid 7 tid 100084 td 0xffffff000262d000
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_wait() at sleepq_wait+0x4d
_cv_wait() at _cv_wait+0x17a
txg_thread_wait() at txg_thread_wait+0x5b
txg_quiesce_thread() at txg_quiesce_thread+0xb4
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip = 0, rsp = 0xffffff8012413d30, rbp = 0 ---
Tracing command zfskern pid 7 tid 100082 td 0xffffff0002640000
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_wait() at sleepq_wait+0x4d
_sleep() at _sleep+0x357
vdev_geom_worker() at vdev_geom_worker+0xf3
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip = 0, rsp = 0xffffff80123ffd30, rbp = 0 ---
Tracing command zfskern pid 7 tid 100036 td 0xffffff0002510000
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_timedwait() at sleepq_timedwait+0x4d
_cv_timedwait() at _cv_timedwait+0x18c
l2arc_feed_thread() at l2arc_feed_thread+0x162
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip = 0, rsp = 0xffffff800015fd30, rbp = 0 ---
Tracing command zfskern pid 7 tid 100035 td 0xffffff0002356740
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_timedwait() at sleepq_timedwait+0x4d
_cv_timedwait() at _cv_timedwait+0x18c
arc_reclaim_thread() at arc_reclaim_thread+0x2b7
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip = 0, rsp = 0xffffff800015ad30, rbp = 0 ---
Tracing command fdc0 pid 6 tid 100034 td 0xffffff0002356ae0
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_timedwait() at sleepq_timedwait+0x4d
_sleep() at _sleep+0x341
fdc_thread() at fdc_thread+0x774
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip = 0, rsp = 0xffffff8000155d30, rbp = 0 ---
Tracing command mpt_recovery0 pid 5 tid 100025 td 0xffffff0002353ae0
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_wait() at sleepq_wait+0x4d
_sleep() at _sleep+0x357
mpt_recovery_thread() at mpt_recovery_thread+0x251
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip = 0, rsp = 0xffffff800011ed30, rbp = 0 ---
Tracing command yarrow pid 13 tid 100011 td 0xffffff00021e53a0
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_timedwait() at sleepq_timedwait+0x4d
_sleep() at _sleep+0x341
random_kthread() at random_kthread+0x1ad
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip = 0, rsp = 0xffffff800004cd30, rbp = 0 ---
Tracing command g_down pid 4 tid 100009 td 0xffffff00021e5ae0
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_wait() at sleepq_wait+0x4d
_sleep() at _sleep+0x357
g_io_schedule_down() at g_io_schedule_down+0x236
g_down_procbody() at g_down_procbody+0x6f
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip = 0, rsp = 0xffffff8000042d30, rbp = 0 ---
Tracing command g_up pid 3 tid 100008 td 0xffffff00021f3000
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_wait() at sleepq_wait+0x4d
_sleep() at _sleep+0x357
g_io_schedule_up() at g_io_schedule_up+0x133
g_up_procbody() at g_up_procbody+0x6f
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip = 0, rsp = 0xffffff800003dd30, rbp = 0 ---
Tracing command g_event pid 2 tid 100007 td 0xffffff00021f33a0
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_timedwait() at sleepq_timedwait+0x4d
_sleep() at _sleep+0x341
g_event_procbody() at g_event_procbody+0xa1
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip = 0, rsp = 0xffffff8000038d30, rbp = 0 ---
Tracing command intr pid 12 tid 100033 td 0xffffff0002483000
fork_trampoline() at fork_trampoline
Tracing command intr pid 12 tid 100032 td 0xffffff00024833a0
fork_trampoline() at fork_trampoline
Tracing command intr pid 12 tid 100031 td 0xffffff0002483740
fork_trampoline() at fork_trampoline
Tracing command intr pid 12 tid 100030 td 0xffffff0002483ae0
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
ithread_loop() at ithread_loop+0x246
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip = 0, rsp = 0xffffff8000137d30, rbp = 0 ---
Tracing command intr pid 12 tid 100024 td 0xffffff0002354000
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
ithread_loop() at ithread_loop+0x246
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip = 0, rsp = 0xffffff80000d7d30, rbp = 0 ---
Tracing command intr pid 12 tid 100023 td 0xffffff00023543a0
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
ithread_loop() at ithread_loop+0x246
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip = 0, rsp = 0xffffff80000d2d30, rbp = 0 ---
Tracing command intr pid 12 tid 100022 td 0xffffff0002354740
fork_trampoline() at fork_trampoline
Tracing command intr pid 12 tid 100021 td 0xffffff0002354ae0
fork_trampoline() at fork_trampoline
Tracing command intr pid 12 tid 100019 td 0xffffff00023563a0
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
ithread_loop() at ithread_loop+0x246
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip = 0, rsp = 0xffffff8000078d30, rbp = 0 ---
Tracing command intr pid 12 tid 100018 td 0xffffff00021f3ae0
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
ithread_loop() at ithread_loop+0x246
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip = 0, rsp = 0xffffff8000073d30, rbp = 0 ---
Tracing command intr pid 12 tid 100013 td 0xffffff0002353000
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
ithread_loop() at ithread_loop+0x246
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip = 0, rsp = 0xffffff800005ad30, rbp = 0 ---
Tracing command intr pid 12 tid 100012 td 0xffffff00023533a0
fork_trampoline() at fork_trampoline
Tracing command intr pid 12 tid 100006 td 0xffffff00021f3740
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
ithread_loop() at ithread_loop+0x246
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip = 0, rsp = 0xffffff8000033d30, rbp = 0 ---
Tracing command intr pid 12 tid 100005 td 0xffffff00021e4000
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
ithread_loop() at ithread_loop+0x246
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip = 0, rsp = 0xffffff800002ed30, rbp = 0 ---
Tracing command intr pid 12 tid 100004 td 0xffffff00021e43a0
fork_trampoline() at fork_trampoline
Tracing command idle pid 11 tid 100003 td 0xffffff00021e4740
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
critical_exit() at critical_exit+0xa3
intr_event_handle() at intr_event_handle+0xd7
intr_execute_handlers() at intr_execute_handlers+0x5f
lapic_handle_intr() at lapic_handle_intr+0x37
Xapic_isr1() at Xapic_isr1+0xa4
--- interrupt, rip = 0xffffffff808545f6, rsp = 0xffffff8000024b50, rbp = 0xffffff8000024b60 ---
acpi_cpu_c1() at acpi_cpu_c1+0x6
acpi_cpu_idle() at acpi_cpu_idle+0x1f6
sched_idletd() at sched_idletd+0x123
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip = 0, rsp = 0xffffff8000024d30, rbp = 0 ---
Tracing command init pid 1 tid 100002 td 0xffffff00021e4ae0
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_catch_signals() at sleepq_catch_signals+0x2af
sleepq_wait_sig() at sleepq_wait_sig+0x16
_sleep() at _sleep+0x309
kern_wait() at kern_wait+0x3f7
wait4() at wait4+0x35
syscall() at syscall+0x118
Xfast_syscall() at Xfast_syscall+0xe1
--- syscall (7, FreeBSD ELF64, wait4), rip = 0x40c86c, rsp = 0x7fffffffe808, rbp = 0x401d40 ---
Tracing command audit pid 10 tid 100001 td 0xffffff00021e5000
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_wait() at sleepq_wait+0x4d
_cv_wait() at _cv_wait+0x17a
audit_worker() at audit_worker+0x77
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip = 0, rsp = 0xffffff800001ad30, rbp = 0 ---
Tracing command kernel pid 0 tid 100088 td 0xffffff000262b000
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_wait() at sleepq_wait+0x4d
_sleep() at _sleep+0x357
taskqueue_thread_loop() at taskqueue_thread_loop+0xb7
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip = 0, rsp = 0xffffff8012426d30, rbp = 0 ---
Tracing command kernel pid 0 tid 100087 td 0xffffff000262b3a0
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_wait() at sleepq_wait+0x4d
_sleep() at _sleep+0x357
taskqueue_thread_loop() at taskqueue_thread_loop+0xb7
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip = 0, rsp = 0xffffff8012421d30, rbp = 0 ---
Tracing command kernel pid 0 tid 100086 td 0xffffff000262b740
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_wait() at sleepq_wait+0x4d
_sleep() at _sleep+0x357
taskqueue_thread_loop() at taskqueue_thread_loop+0xb7
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip = 0, rsp = 0xffffff801241cd30, rbp = 0 ---
Tracing command kernel pid 0 tid 100083 td 0xffffff000263eae0
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_wait() at sleepq_wait+0x4d
_sleep() at _sleep+0x357
taskqueue_thread_loop() at taskqueue_thread_loop+0xb7
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip = 0, rsp = 0xffffff8012404d30, rbp = 0 ---
Tracing command kernel pid 0 tid 100081 td 0xffffff00026403a0
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_wait() at sleepq_wait+0x4d
_sleep() at _sleep+0x357
taskqueue_thread_loop() at taskqueue_thread_loop+0xb7
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip = 0, rsp = 0xffffff80123fad30, rbp = 0 ---
Tracing command kernel pid 0 tid 100080 td 0xffffff000263d740
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_wait() at sleepq_wait+0x4d
_sleep() at _sleep+0x357
taskqueue_thread_loop() at taskqueue_thread_loop+0xb7
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip = 0, rsp = 0xffffff80123f5d30, rbp = 0 ---
Tracing command kernel pid 0 tid 100079 td 0xffffff000263dae0
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_wait() at sleepq_wait+0x4d
_sleep() at _sleep+0x357
taskqueue_thread_loop() at taskqueue_thread_loop+0xb7
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip = 0, rsp = 0xffffff80123f0d30, rbp = 0 ---
Tracing command kernel pid 0 tid 100078 td 0xffffff000263e000
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleep#
And this is msgbuf.txt:
# cat msgbuf.txt
Copyright (c) 1992-2010 The FreeBSD Project.
Copyright (c) 1979, 1980, 1983, 1986, 1988, 1989, 1991, 1992, 1993, 1994
The Regents of the University of California. All rights reserved.
FreeBSD is a registered trademark of The FreeBSD Foundation.
FreeBSD 8.0-STABLE #3: Wed Mar 10 14:41:54 CET 2010
root at pruebazfs1.sarenet.es:/pool/newsrc/obj/pool/newsrc/src/sys/DEBUG amd64
WARNING: WITNESS option enabled, expect reduced performance.
WARNING: DIAGNOSTIC option enabled, expect reduced performance.
Timecounter "i8254" frequency 1193182 Hz quality 0
CPU: Intel(R) Core(TM)2 Duo CPU T8100 @ 2.10GHz (2116.63-MHz K8-class CPU)
Origin = "GenuineIntel" Id = 0x10676 Stepping = 6
Features=0xfebfbff<FPU,VME,DE,PSE,TSC,MSR,PAE,MCE,CX8,APIC,SEP,MTRR,PGE,MCA,CMOV,PAT,PSE36,CLFLUSH,DTS,ACPI,MMX,FXSR,SSE,SSE2,SS>
Features2=0x80082201<SSE3,SSSE3,CX16,SSE4.1,<b31>>
AMD Features=0x20100800<SYSCALL,NX,LM>
AMD Features2=0x1<LAHF>
TSC: P-state invariant
real memory = 536870912 (512 MB)
avail memory = 495988736 (473 MB)
ACPI APIC Table: <PTLTD APIC >
MADT: Forcing active-low polarity and level trigger for SCI
ioapic0 <Version 1.1> irqs 0-23 on motherboard
kbd1 at kbdmux0
acpi0: <INTEL 440BX> on motherboard
acpi0: [ITHREAD]
acpi0: Power Button (fixed)
Timecounter "ACPI-safe" frequency 3579545 Hz quality 850
acpi_timer0: <24-bit timer at 3.579545MHz> port 0x1008-0x100b on acpi0
pcib0: <ACPI Host-PCI bridge> port 0xcf8-0xcff on acpi0
pci0: <ACPI PCI bus> on pcib0
pcib1: <ACPI PCI-PCI bridge> at device 1.0 on pci0
pci1: <ACPI PCI bus> on pcib1
isab0: <PCI-ISA bridge> at device 7.0 on pci0
isa0: <ISA bus> on isab0
atapci0: <Intel PIIX4 UDMA33 controller> port 0x1f0-0x1f7,0x3f6,0x170-0x177,0x376,0x10c0-0x10cf at device 7.1 on pci0
ata0: <ATA channel 0> on atapci0
ata0: [ITHREAD]
ata1: <ATA channel 1> on atapci0
ata1: [ITHREAD]
pci0: <bridge> at device 7.3 (no driver attached)
pci0: <base peripheral> at device 7.7 (no driver attached)
vgapci0: <VGA-compatible display> port 0x10d0-0x10df mem 0xd0000000-0xd7ffffff,0xd8000000-0xd87fffff irq 16 at device 15.0 on pci0
mpt0: <LSILogic 1030 Ultra4 Adapter> port 0x1400-0x14ff mem 0xd8820000-0xd883ffff,0xd8800000-0xd881ffff irq 17 at device 16.0 on pci0
mpt0: [ITHREAD]
mpt0: MPI Version=1.2.0.0
pcib2: <ACPI PCI-PCI bridge> at device 17.0 on pci0
pci2: <ACPI PCI bus> on pcib2
em0: <Intel(R) PRO/1000 Network Connection 6.9.14> port 0x2000-0x203f mem 0xd8940000-0xd895ffff,0xd8900000-0xd890ffff irq 18 at device 0.0 on pci2
em0: Memory Access and/or Bus Master bits were not set!
em0: [FILTER]
em0: Ethernet address: 00:0c:29:11:4f:d5
em1: <Intel(R) PRO/1000 Network Connection 6.9.14> port 0x2040-0x207f mem 0xd8960000-0xd897ffff,0xd8910000-0xd891ffff irq 19 at device 1.0 on pci2
em1: Memory Access and/or Bus Master bits were not set!
em1: [FILTER]
em1: Ethernet address: 00:0c:29:11:4f:cb
pci2: <multimedia, audio> at device 2.0 (no driver attached)
em2: <Intel(R) PRO/1000 Network Connection 6.9.14> port 0x20c0-0x20ff mem 0xd8980000-0xd899ffff,0xd8920000-0xd892ffff irq 17 at device 3.0 on pci2
em2: Memory Access and/or Bus Master bits were not set!
em2: [FILTER]
em2: Ethernet address: 00:0c:29:11:4f:df
em3: <Intel(R) PRO/1000 Network Connection 6.9.14> port 0x2400-0x243f mem 0xd89a0000-0xd89bffff,0xd8930000-0xd893ffff irq 19 at device 5.0 on pci2
em3: Memory Access and/or Bus Master bits were not set!
em3: [FILTER]
em3: Ethernet address: 00:0c:29:11:4f:e9
pcib3: <ACPI PCI-PCI bridge> at device 21.0 on pci0
pci3: <ACPI PCI bus> on pcib3
pcib4: <ACPI PCI-PCI bridge> at device 21.1 on pci0
pci4: <ACPI PCI bus> on pcib4
pcib5: <ACPI PCI-PCI bridge> at device 21.2 on pci0
pci5: <ACPI PCI bus> on pcib5
pcib6: <ACPI PCI-PCI bridge> at device 21.3 on pci0
pci6: <ACPI PCI bus> on pcib6
pcib7: <ACPI PCI-PCI bridge> at device 21.4 on pci0
pci7: <ACPI PCI bus> on pcib7
pcib8: <ACPI PCI-PCI bridge> at device 21.5 on pci0
pci8: <ACPI PCI bus> on pcib8
pcib9: <ACPI PCI-PCI bridge> at device 21.6 on pci0
pci9: <ACPI PCI bus> on pcib9
pcib10: <ACPI PCI-PCI bridge> at device 21.7 on pci0
pci10: <ACPI PCI bus> on pcib10
pcib11: <ACPI PCI-PCI bridge> at device 22.0 on pci0
pci11: <ACPI PCI bus> on pcib11
pcib12: <ACPI PCI-PCI bridge> at device 22.1 on pci0
pci12: <ACPI PCI bus> on pcib12
pcib13: <ACPI PCI-PCI bridge> at device 22.2 on pci0
pci13: <ACPI PCI bus> on pcib13
pcib14: <ACPI PCI-PCI bridge> at device 22.3 on pci0
pci14: <ACPI PCI bus> on pcib14
pcib15: <ACPI PCI-PCI bridge> at device 22.4 on pci0
pci15: <ACPI PCI bus> on pcib15
pcib16: <ACPI PCI-PCI bridge> at device 22.5 on pci0
pci16: <ACPI PCI bus> on pcib16
pcib17: <ACPI PCI-PCI bridge> at device 22.6 on pci0
pci17: <ACPI PCI bus> on pcib17
pcib18: <ACPI PCI-PCI bridge> at device 22.7 on pci0
pci18: <ACPI PCI bus> on pcib18
pcib19: <ACPI PCI-PCI bridge> at device 23.0 on pci0
pci19: <ACPI PCI bus> on pcib19
pcib20: <ACPI PCI-PCI bridge> at device 23.1 on pci0
pci20: <ACPI PCI bus> on pcib20
pcib21: <ACPI PCI-PCI bridge> at device 23.2 on pci0
pci21: <ACPI PCI bus> on pcib21
pcib22: <ACPI PCI-PCI bridge> at device 23.3 on pci0
pci22: <ACPI PCI bus> on pcib22
pcib23: <ACPI PCI-PCI bridge> at device 23.4 on pci0
pci23: <ACPI PCI bus> on pcib23
pcib24: <ACPI PCI-PCI bridge> at device 23.5 on pci0
pci24: <ACPI PCI bus> on pcib24
pcib25: <ACPI PCI-PCI bridge> at device 23.6 on pci0
pci25: <ACPI PCI bus> on pcib25
pcib26: <ACPI PCI-PCI bridge> at device 23.7 on pci0
pci26: <ACPI PCI bus> on pcib26
pcib27: <ACPI PCI-PCI bridge> at device 24.0 on pci0
pci27: <ACPI PCI bus> on pcib27
pcib28: <ACPI PCI-PCI bridge> at device 24.1 on pci0
pci28: <ACPI PCI bus> on pcib28
pcib29: <ACPI PCI-PCI bridge> at device 24.2 on pci0
pci29: <ACPI PCI bus> on pcib29
pcib30: <ACPI PCI-PCI bridge> at device 24.3 on pci0
pci30: <ACPI PCI bus> on pcib30
pcib31: <ACPI PCI-PCI bridge> at device 24.4 on pci0
pci31: <ACPI PCI bus> on pcib31
pcib32: <ACPI PCI-PCI bridge> at device 24.5 on pci0
pci32: <ACPI PCI bus> on pcib32
pcib33: <ACPI PCI-PCI bridge> at device 24.6 on pci0
pci33: <ACPI PCI bus> on pcib33
pcib34: <ACPI PCI-PCI bridge> at device 24.7 on pci0
pci34: <ACPI PCI bus> on pcib34
acpi_acad0: <AC Adapter> on acpi0
atrtc0: <AT realtime clock> port 0x70-0x71 irq 8 on acpi0
atkbdc0: <Keyboard controller (i8042)> port 0x60,0x64 irq 1 on acpi0
atkbd0: <AT Keyboard> irq 1 on atkbdc0
kbd0 at atkbd0
atkbd0: [GIANT-LOCKED]
atkbd0: [ITHREAD]
psm0: <PS/2 Mouse> irq 12 on atkbdc0
psm0: [GIANT-LOCKED]
psm0: [ITHREAD]
psm0: model IntelliMouse, device ID 3
ppc0: <Parallel port> port 0x378-0x37f irq 7 on acpi0
ppc0: Generic chipset (NIBBLE-only) in COMPATIBLE mode
ppc0: [ITHREAD]
ppbus0: <Parallel port bus> on ppc0
plip0: <PLIP network interface> on ppbus0
plip0: [ITHREAD]
lpt0: <Printer> on ppbus0
lpt0: [ITHREAD]
lpt0: Interrupt-driven port
ppi0: <Parallel I/O> on ppbus0
uart0: <16550 or compatible> port 0x3f8-0x3ff irq 4 flags 0x10 on acpi0
uart0: [FILTER]
uart1: <16550 or compatible> port 0x2f8-0x2ff irq 3 on acpi0
uart1: [FILTER]
fdc0: <floppy drive controller> port 0x3f0-0x3f5,0x3f7 irq 6 drq 2 on acpi0
fdc0: [FILTER]
fd0: <1440-KB 3.5" drive> on fdc0 drive 0
cpu0: <ACPI CPU> on acpi0
acpi_throttle0: <ACPI CPU Throttling> on cpu0
orm0: <ISA Option ROMs> at iomem 0xc0000-0xc7fff,0xca000-0xcafff,0xcb000-0xcbfff,0xcc000-0xccfff,0xcd000-0xcdfff,0xdc000-0xdffff,0xe0000-0xe3fff on isa0
sc0: <System console> at flags 0x100 on isa0
sc0: VGA <16 virtual consoles, flags=0x300>
vga0: <Generic ISA VGA> at port 0x3c0-0x3df iomem 0xa0000-0xbffff on isa0
ZFS NOTICE: Prefetch is disabled by default if less than 4GB of RAM is present;
to enable, add "vfs.zfs.prefetch_disable=0" to /boot/loader.conf.
ZFS WARNING: Recommended minimum kmem_size is 512MB; expect unstable behavior.
Consider tuning vm.kmem_size and vm.kmem_size_max
in /boot/loader.conf.
ZFS filesystem version 3
ZFS storage pool version 14
Timecounter "TSC" frequency 2116630847 Hz quality 800
Timecounters tick every 10.000 msec
Expensive timeout(9) function: 0xffffffff805ed840(0) 0.002412596 s
acd0: DVDR <VMware Virtual IDE CDROM Drive/00000001> at ata1-master UDMA33
da0 at mpt0 bus 0 scbus0 target 0 lun 0
da0: <VMware, VMware Virtual S 1.0> Fixed Direct Access SCSI-2 device
da0: 320.000MB/s transfers (160.000MHz, offset 127, 16bit)
da0: Command Queueing enabled
da0: 8192MB (16777216 512 byte sectors: 255H 63S/T 1044C)
da1 at mpt0 bus 0 scbus0 target 1 lun 0
da1: <VMware, VMware Virtual S 1.0> Fixed Direct Access SCSI-2 device
da1: 320.000MB/s transfers (160.000MHz, offset 127, 16bit)
da1: Command Queueing enabled
da1: 4096MB (8388608 512 byte sectors: 255H 63S/T 522C)
da2 at mpt0 bus 0 scbus0 target 2 lun 0
da2: <VMware, VMware Virtual S 1.0> Fixed Direct Access SCSI-2 device
da2: 320.000MB/s transfers (160.000MHz, offset 127, 16bit)
da2: Command Queueing enabled
da2: 4096MB (8388608 512 byte sectors: 255H 63S/T 522C)
da3 at mpt0 bus 0 scbus0 target 3 lun 0
da3: <VMware, VMware Virtual S 1.0> Fixed Direct Access SCSI-2 device
da3: 320.000MB/s transfers (160.000MHz, offset 127, 16bit)
da3: Command Queueing enabled
da3: 12288MB (25165824 512 byte sectors: 255H 63S/T 1566C)
WARNING: WITNESS option enabled, expect reduced performance.
WARNING: DIAGNOSTIC option enabled, expect reduced performance.
Trying to mount root from ufs:/dev/da0s1a
WARNING: / was not properly dismounted
/: mount pending error: blocks 4 files 1
<118>Setting hostuuid: 564d8c47-d7e5-fea4-92ef-f17760cfe4d2.
<118>Setting hostid: 0x4d345a2c.
lock order reversal:
1st 0xffffff000286d2e8 db->db_mtx (db->db_mtx) @ /pool/newsrc/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/dbuf.c:549
2nd 0xffffff000286c0d8 dn->dn_mtx (dn->dn_mtx) @ /pool/newsrc/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/dnode.c:1173
KDB: stack backtrace:
db_trace_self_wrapper() at db_trace_self_wrapper+0x2a
_witness_debugger() at _witness_debugger+0x2e
witness_checkorder() at witness_checkorder+0x81e
_sx_xlock() at _sx_xlock+0x55
dnode_block_freed() at dnode_block_freed+0x8e
dbuf_read() at dbuf_read+0x155
dmu_buf_hold_array_by_dnode() at dmu_buf_hold_array_by_dnode+0x12a
dmu_read() at dmu_read+0x80
load_nvlist() at load_nvlist+0x85
spa_load() at spa_load+0x49a
spa_open_common() at spa_open_common+0x12d
spa_get_stats() at spa_get_stats+0x42
zfs_ioc_pool_stats() at zfs_ioc_pool_stats+0x2c
zfsdev_ioctl() at zfsdev_ioctl+0x8d
devfs_ioctl_f() at devfs_ioctl_f+0x76
kern_ioctl() at kern_ioctl+0xc5
ioctl() at ioctl+0xfd
syscall() at syscall+0x118
Xfast_syscall() at Xfast_syscall+0xe1
--- syscall (54, FreeBSD ELF64, ioctl), rip = 0x800fe7d1c, rsp = 0x7fffffffd808, rbp = 0x801224140 ---
Expensive timeout(9) function: 0xffffffff80892b50(0xffffff000244fe00) 0.005049803 s
lock order reversal:
1st 0xffffff0002868e70 db->db_mtx (db->db_mtx) @ /pool/newsrc/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/dnode_sync.c:381
2nd 0xffffff00026e7940 osi->os_lock (osi->os_lock) @ /pool/newsrc/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/dnode.c:323
KDB: stack backtrace:
db_trace_self_wrapper() at db_trace_self_wrapper+0x2a
_witness_debugger() at _witness_debugger+0x2e
witness_checkorder() at witness_checkorder+0x81e
_sx_xlock() at _sx_xlock+0x55
dnode_destroy() at dnode_destroy+0xa6
dnode_buf_pageout() at dnode_buf_pageout+0xb2
dbuf_evict_user() at dbuf_evict_user+0x55
dbuf_clear() at dbuf_clear+0x5e
dnode_evict_dbufs() at dnode_evict_dbufs+0x98
dmu_objset_evict_dbufs() at dmu_objset_evict_dbufs+0x11c
dmu_objset_evict() at dmu_objset_evict+0xbf
dsl_pool_close() at dsl_pool_close+0x52
spa_unload() at spa_unload+0xb2
spa_load() at spa_load+0x4da
spa_open_common() at spa_open_common+0x12d
spa_get_stats() at spa_get_stats+0x42
zfs_ioc_pool_stats() at zfs_ioc_pool_stats+0x2c
zfsdev_ioctl() at zfsdev_ioctl+0x8d
devfs_ioctl_f() at devfs_ioctl_f+0x76
kern_ioctl() at kern_ioctl+0xc5
ioctl() at ioctl+0xfd
syscall() at syscall+0x118
Xfast_syscall() at Xfast_syscall+0xe1
--- syscall (54, FreeBSD ELF64, ioctl), rip = 0x800fe7d1c, rsp = 0x7fffffffd808, rbp = 0x801224140 ---
lock order reversal:
1st 0xffffff0002864910 buf->b_lock (buf->b_lock) @ /pool/newsrc/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c:2509
2nd 0xffffff00028681a0 db->db_mtx (db->db_mtx) @ /pool/newsrc/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/dbuf.c:421
KDB: stack backtrace:
db_trace_self_wrapper() at db_trace_self_wrapper+0x2a
_witness_debugger() at _witness_debugger+0x2e
witness_checkorder() at witness_checkorder+0x81e
_sx_xlock() at _sx_xlock+0x55
dbuf_read_done() at dbuf_read_done+0x3b
arc_read_done() at arc_read_done+0x1d2
zio_done() at zio_done+0x308
zio_execute() at zio_execute+0xb1
arc_read_nolock() at arc_read_nolock+0x3d0
arc_read() at arc_read+0xaf
dbuf_read() at dbuf_read+0x62b
dmu_buf_hold() at dmu_buf_hold+0xcc
zap_lockdir() at zap_lockdir+0x6e
zap_lookup_norm() at zap_lookup_norm+0x45
zap_lookup() at zap_lookup+0x2e
dsl_prop_get_dd() at dsl_prop_get_dd+0xfc
dsl_dataset_get_ref() at dsl_dataset_get_ref+0x4a7
dsl_dataset_hold() at dsl_dataset_hold+0xf2
dmu_objset_open() at dmu_objset_open+0xcf
zil_check_log_chain() at zil_check_log_chain+0x2a
dmu_objset_find_spa() at dmu_objset_find_spa+0x23f
dmu_objset_find_spa() at dmu_objset_find_spa+0x324
dmu_objset_find_spa() at dmu_objset_find_spa+0x324
dmu_objset_find() at dmu_objset_find+0x2a
spa_check_logs() at spa_check_logs+0x2a
spa_load() at spa_load+0x6ca
spa_load() at spa_load+0x4fe
spa_open_common() at spa_open_common+0x12d
spa_get_stats() at spa_get_stats+0x42
zfs_ioc_pool_stats() at zfs_ioc_pool_stats+0x2c
zfsdev_ioctl() at zfsdev_ioctl+0x8d
devfs_ioctl_f() at devfs_ioctl_f+0x76
kern_ioctl() at kern_ioctl+0xc5
ioctl() at ioctl+0xfd
syscall() at syscall+0x118
Xfast_syscall() at Xfast_syscall+0xe1
--- syscall (54, FreeBSD ELF64, ioctl), rip = 0x800fe7d1c, rsp = 0x7fffffffd808, rbp = 0x801224140 ---
lock order reversal:
1st 0xffffff000286d058 db->db_mtx (db->db_mtx) @ /pool/newsrc/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/dbuf.c:1116
2nd 0xffffff000259ac38 dr->dt.di.dr_mtx (dr->dt.di.dr_mtx) @ /pool/newsrc/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/dbuf.c:1120
KDB: stack backtrace:
db_trace_self_wrapper() at db_trace_self_wrapper+0x2a
_witness_debugger() at _witness_debugger+0x2e
witness_checkorder() at witness_checkorder+0x81e
_sx_xlock() at _sx_xlock+0x55
dbuf_dirty() at dbuf_dirty+0x892
dnode_setdirty() at dnode_setdirty+0x1a9
dbuf_dirty() at dbuf_dirty+0xa53
bplist_vacate() at bplist_vacate+0x4d
spa_sync() at spa_sync+0x297
txg_sync_thread() at txg_sync_thread+0x2d7
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip = 0, rsp = 0xffffff8012417d30, rbp = 0 ---
lock order reversal:
1st 0xffffff000292d538 dr->dt.di.dr_mtx (dr->dt.di.dr_mtx) @ /pool/newsrc/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/dbuf.c:1905
2nd 0xffffff000250c2f0 spa->spa_sync_bplist.bpl_lock (spa->spa_sync_bplist.bpl_lock) @ /pool/newsrc/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/bplist.c:235
KDB: stack backtrace:
db_trace_self_wrapper() at db_trace_self_wrapper+0x2a
_witness_debugger() at _witness_debugger+0x2e
witness_checkorder() at witness_checkorder+0x81e
_sx_xlock() at _sx_xlock+0x55
bplist_enqueue_deferred() at bplist_enqueue_deferred+0x47
zio_free() at zio_free+0x105
arc_free() at arc_free+0x11c
dsl_dataset_block_kill() at dsl_dataset_block_kill+0x483
dbuf_write() at dbuf_write+0x24c
dbuf_sync_list() at dbuf_sync_list+0x3eb
dbuf_sync_list() at dbuf_sync_list+0x17f
dnode_sync() at dnode_sync+0xc12
dmu_objset_sync() at dmu_objset_sync+0x134
dsl_pool_sync() at dsl_pool_sync+0x200
spa_sync() at spa_sync+0x35e
txg_sync_thread() at txg_sync_thread+0x2d7
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip = 0, rsp = 0xffffff8012417d30, rbp = 0 ---
lock order reversal:
1st 0xffffff0002948c38 dr->dt.di.dr_mtx (dr->dt.di.dr_mtx) @ /pool/newsrc/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/dbuf.c:1905
2nd 0xffffff00028a7000 dn->dn_struct_rwlock (dn->dn_struct_rwlock) @ /pool/newsrc/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/dbuf.c:1859
KDB: stack backtrace:
db_trace_self_wrapper() at db_trace_self_wrapper+0x2a
_witness_debugger() at _witness_debugger+0x2e
witness_checkorder() at witness_checkorder+0x81e
_sx_slock() at _sx_slock+0x55
dbuf_check_blkptr() at dbuf_check_blkptr+0x11e
dbuf_sync_list() at dbuf_sync_list+0x34a
dbuf_sync_list() at dbuf_sync_list+0x17f
dnode_sync() at dnode_sync+0xc12
dmu_objset_sync_dnodes() at dmu_objset_sync_dnodes+0x60
dmu_objset_sync() at dmu_objset_sync+0x162
dsl_pool_sync() at dsl_pool_sync+0x200
spa_sync() at spa_sync+0x35e
txg_sync_thread() at txg_sync_thread+0x2d7
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip = 0, rsp = 0xffffff8012417d30, rbp = 0 ---
<118>Entropy harvesting:
<118> interrupts
<118> ethernet
<118> point_to_point
<118> kickstart
<118>.
<118>Starting file system checks:
<118>/dev/da0s1a: FREE BLK COUNT(S) WRONG IN SUPERBLK (SALVAGED)
<118>/dev/da0s1a: SUMMARY INFORMATION BAD (SALVAGED)
<118>/dev/da0s1a: BLK(S) MISSING IN BIT MAPS (SALVAGED)
<118>/dev/da0s1a: 28961 files, 517324 used, 1004493 free (7877 frags, 124577 blocks, 0.5% fragmentation)
<118>Mounting local file systems:
<118>.
lock order reversal:
1st 0xffffff000263acc8 zfs (zfs) @ /pool/newsrc/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/gfs.c:437
2nd 0xffffff000256e310 zfsvfs->z_hold_mtx[i] (zfsvfs->z_hold_mtx[i]) @ /pool/newsrc/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_znode.c:866
KDB: stack backtrace:
db_trace_self_wrapper() at db_trace_self_wrapper+0x2a
_witness_debugger() at _witness_debugger+0x2e
witness_checkorder() at witness_checkorder+0x81e
_sx_xlock() at _sx_xlock+0x55
zfs_zget() at zfs_zget+0x23c
zfs_root() at zfs_root+0x50
zfsctl_create() at zfsctl_create+0x82
zfs_mount() at zfs_mount+0x7ef
vfs_donmount() at vfs_donmount+0xcde
nmount() at nmount+0x63
syscall() at syscall+0x118
Xfast_syscall() at Xfast_syscall+0xe1
--- syscall (378, FreeBSD ELF64, nmount), rip = 0x800f48f1c, rsp = 0x7fffffffced8, rbp = 0x7fffffffcef8 ---
lock order reversal:
1st 0xffffff0002996078 zp->z_name_lock (zp->z_name_lock) @ /pool/newsrc/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_dir.c:212
2nd 0xffffff000256e330 zfsvfs->z_hold_mtx[i] (zfsvfs->z_hold_mtx[i]) @ /pool/newsrc/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_znode.c:866
KDB: stack backtrace:
db_trace_self_wrapper() at db_trace_self_wrapper+0x2a
_witness_debugger() at _witness_debugger+0x2e
witness_checkorder() at witness_checkorder+0x81e
_sx_xlock() at _sx_xlock+0x55
zfs_zget() at zfs_zget+0x23c
zfs_dirent_lock() at zfs_dirent_lock+0x4a0
zfs_dirlook() at zfs_dirlook+0x90
zfs_lookup() at zfs_lookup+0x256
zfs_freebsd_lookup() at zfs_freebsd_lookup+0x8d
VOP_CACHEDLOOKUP_APV() at VOP_CACHEDLOOKUP_APV+0xaf
vfs_cache_lookup() at vfs_cache_lookup+0xf0
VOP_LOOKUP_APV() at VOP_LOOKUP_APV+0xb7
lookup() at lookup+0x3d3
namei() at namei+0x4a9
kern_statat_vnhook() at kern_statat_vnhook+0x8f
kern_statat() at kern_statat+0x15
lstat() at lstat+0x2a
syscall() at syscall+0x118
Xfast_syscall() at Xfast_syscall+0xe1
--- syscall (190, FreeBSD ELF64, lstat), rip = 0x800fd8acc, rsp = 0x7fffffffcf38, rbp = 0x7fffffffd3d0 ---
lock order reversal:
1st 0xffffff000256e210 zfsvfs->z_teardown_inactive_lock (zfsvfs->z_teardown_inactive_lock) @ /pool/newsrc/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c:3724
2nd 0xffffff000256e330 zfsvfs->z_hold_mtx[i] (zfsvfs->z_hold_mtx[i]) @ /pool/newsrc/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_znode.c:1027
KDB: stack backtrace:
db_trace_self_wrapper() at db_trace_self_wrapper+0x2a
_witness_debugger() at _witness_debugger+0x2e
witness_checkorder() at witness_checkorder+0x81e
_sx_xlock() at _sx_xlock+0x55
zfs_zinactive() at zfs_zinactive+0x95
zfs_inactive() at zfs_inactive+0x7e
zfs_freebsd_inactive() at zfs_freebsd_inactive+0x1a
VOP_INACTIVE_APV() at VOP_INACTIVE_APV+0xb5
vinactive() at vinactive+0x90
vputx() at vputx+0x2fc
kern_statat_vnhook() at kern_statat_vnhook+0xfa
kern_statat() at kern_statat+0x15
lstat() at lstat+0x2a
syscall() at syscall+0x118
Xfast_syscall() at Xfast_syscall+0xe1
--- syscall (190, FreeBSD ELF64, lstat), rip = 0x800fd8acc, rsp = 0x7fffffffcf38, rbp = 0x7fffffffd3d0 ---
<118>/etc/rc: WARNING: $hostname is not set -- see rc.conf(5).
<118>Starting Network: lo0 em0 em1 em2 em3.
<118>lo0: flags=8049<UP,LOOPBACK,RUNNING,MULTICAST> metric 0 mtu 16384
<118> options=3<RXCSUM,TXCSUM>
<118> inet6 fe80::1%lo0 prefixlen 64 scopeid 0x6
<118> inet6 ::1 prefixlen 128
<118> inet 127.0.0.1 netmask 0xff000000
<118> nd6 options=3<PERFORMNUD,ACCEPT_RTADV>
<118>em0: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0 mtu 1500
<118> options=9b<RXCSUM,TXCSUM,VLAN_MTU,VLAN_HWTAGGING,VLAN_HWCSUM>
<118> ether 00:0c:29:11:4f:d5
<118> inet 10.0.0.2 netmask 0xffffff00 broadcast 10.0.0.255
<118> media: Ethernet autoselect (1000baseT <full-duplex>)
<118> status: active
<118>em1: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0 mtu 1500
<118> options=9b<RXCSUM,TXCSUM,VLAN_MTU,VLAN_HWTAGGING,VLAN_HWCSUM>
<118> ether 00:0c:29:11:4f:cb
<118> inet 10.0.1.2 netmask 0xffffff00 broadcast 10.0.1.255
<118> media: Ethernet autoselect (1000baseT <full-duplex>)
<118> status: active
<118>em2: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0 mtu 1500
<118> options=9b<RXCSUM,TXCSUM,VLAN_MTU,VLAN_HWTAGGING,VLAN_HWCSUM>
<118> ether 00:0c:29:11:4f:df
<118> inet 10.0.2.2 netmask 0xffffff00 broadcast 10.0.2.255
<118> media: Ethernet autoselect (1000baseT <full-duplex>)
<118> status: active
<118>em3: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0 mtu 1500
<118> options=9b<RXCSUM,TXCSUM,VLAN_MTU,VLAN_HWTAGGING,VLAN_HWCSUM>
<118> ether 00:0c:29:11:4f:e9
<118> inet 172.16.2.118 netmask 0xffff0000 broadcast 172.16.255.255
<118> media: Ethernet autoselect (1000baseT <full-duplex>)
<118> status: active
<118>add net default: gateway 172.16.0.2
Expensive timeout(9) function: 0xffffffff80329580(0xffffff8000272000) 0.006700013 s
<118>Starting devd.
ipfw2 (+ipv6) initialized, divert loadable, nat loadable, rule-based forwarding disabled, default to deny, logging disabled
<118>Flushed all rules.
<118>00100 allow ip from any to any via lo0
<118>00200 deny ip from any to 127.0.0.0/8
<118>00300 deny ip from 127.0.0.0/8 to any
<118>65000 allow ip from any to any
<118>Firewall rules loaded.
<118>Creating and/or trimming log files
<118>.
<118>Starting syslogd.
<118>No core dumps found.
<118>ELF ldconfig path: /lib /usr/lib /usr/lib/compat /usr/local/lib
<118>32-bit compatibility ldconfig path: /usr/lib32
lock order reversal:
1st 0xffffff800a2f62e8 bufwait (bufwait) @ /pool/newsrc/src/sys/kern/vfs_bio.c:2559
2nd 0xffffff0002c71400 dirhash (dirhash) @ /pool/newsrc/src/sys/ufs/ufs/ufs_dirhash.c:285
KDB: stack backtrace:
db_trace_self_wrapper() at db_trace_self_wrapper+0x2a
_witness_debugger() at _witness_debugger+0x2e
witness_checkorder() at witness_checkorder+0x81e
_sx_xlock() at _sx_xlock+0x55
ufsdirhash_acquire() at ufsdirhash_acquire+0x44
ufsdirhash_add() at ufsdirhash_add+0x19
ufs_direnter() at ufs_direnter+0x88b
ufs_mkdir() at ufs_mkdir+0x623
VOP_MKDIR_APV() at VOP_MKDIR_APV+0xb9
kern_mkdirat() at kern_mkdirat+0x264
syscall() at syscall+0x118
Xfast_syscall() at Xfast_syscall+0xe1
--- syscall (136, FreeBSD ELF64, mkdir), rip = 0x80072bb0c, rsp = 0x7fffffffec88, rbp = 0x7fffffffef66 ---
<118>Clearing /tmp (X related).
<118>Updating motd:
<118>.
<118>Configuring syscons:
<118> blanktime
<118>.
<118>Starting sshd.
<118>Starting cron.
<118>Starting background file system checks in 60 seconds.
<118>
<118>Wed Mar 10 17:29:09 CET 2010
<118>Mar 10 17:29:21 su: BAD SU borjam to root on /dev/pts/0
<118>Mar 10 17:29:24 su: BAD SU borjam to root on /dev/pts/0
<118>Mar 10 17:29:26 su: borjam to root on /dev/pts/0
<118>Mar 10 17:30:36 su: borjam to root on /dev/pts/1
lock order reversal:
1st 0xffffff00027e2438 ds->ds_deadlist.bpl_lock (ds->ds_deadlist.bpl_lock) @ /pool/newsrc/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/bplist.c:189
2nd 0xffffff000286bb88 dn->dn_struct_rwlock (dn->dn_struct_rwlock) @ /pool/newsrc/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/dnode.c:130
KDB: stack backtrace:
db_trace_self_wrapper() at db_trace_self_wrapper+0x2a
_witness_debugger() at _witness_debugger+0x2e
witness_checkorder() at witness_checkorder+0x81e
_sx_slock() at _sx_slock+0x55
dnode_verify() at dnode_verify+0x70
dnode_hold_impl() at dnode_hold_impl+0x73
dmu_bonus_hold() at dmu_bonus_hold+0x31
bplist_hold() at bplist_hold+0x48
bplist_enqueue() at bplist_enqueue+0x4c
dsl_dataset_block_kill() at dsl_dataset_block_kill+0x119
dbuf_write() at dbuf_write+0x24c
dbuf_sync_list() at dbuf_sync_list+0x3eb
dbuf_sync_list() at dbuf_sync_list+0x17f
Fatal double fault
rip = 0xffffffff805d322d
rsp = 0xffffff8012414fe0
rbp = 0xffffff8012415090
cpuid = 0; apic id = 00
panic: double fault
cpuid = 0
KDB: enter: panic
shared sx zfsvfs->z_teardown_inactive_lock (zfsvfs->z_teardown_inactive_lock) r = 0 (0xffffff000256d210) locked @ /pool/newsrc/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c:3724
exclusive lockmgr zfs (zfs) r = 0 (0xffffff00055b3a58) locked @ /pool/newsrc/src/sys/kern/vfs_subr.c:2091
exclusive lockmgr zfs (zfs) r = 0 (0xffffff0005618308) locked @ /pool/newsrc/src/sys/kern/vfs_syscalls.c:3174
shared lockmgr zfs (zfs) r = 0 (0xffffff00057bda58) locked @ /pool/newsrc/src/sys/kern/vfs_subr.c:2091
exclusive sx zfsvfs->z_hold_mtx[i] (zfsvfs->z_hold_mtx[i]) r = 0 (0xffffff000256d310) locked @ /pool/newsrc/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_znode.c:866
shared sx zp->z_name_lock (zp->z_name_lock) r = 0 (0xffffff00057c7c38) locked @ /pool/newsrc/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_dir.c:212
exclusive lockmgr zfs (zfs) r = 0 (0xffffff00057ac578) locked @ /pool/newsrc/src/sys/kern/vfs_subr.c:2091
exclusive sx so_rcv_sx (so_rcv_sx) r = 0 (0xffffff0002f63648) locked @ /pool/newsrc/src/sys/kern/uipc_sockbuf.c:148
exclusive sx so_rcv_sx (so_rcv_sx) r = 0 (0xffffff0002cff3a0) locked @ /pool/newsrc/src/sys/kern/uipc_sockbuf.c:148
exclusive sx ds->ds_deadlist.bpl_lock (ds->ds_deadlist.bpl_lock) r = 0 (0xffffff00027e2438) locked @ /pool/newsrc/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/bplist.c:189
exclusive sx dr->dt.di.dr_mtx (dr->dt.di.dr_mtx) r = 0 (0xffffff0002948238) locked @ /pool/newsrc/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/dbuf.c:1905
exclusive sx dr->dt.di.dr_mtx (dr->dt.di.dr_mtx) r = 0 (0xffffff0002963538) locked @ /pool/newsrc/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/dbuf.c:1905
exclusive sx dr->dt.di.dr_mtx (dr->dt.di.dr_mtx) r = 0 (0xffffff0002949438) locked @ /pool/newsrc/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/dbuf.c:1905
exclusive sx dr->dt.di.dr_mtx (dr->dt.di.dr_mtx) r = 0 (0xffffff0002948b38) locked @ /pool/newsrc/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/dbuf.c:1905
exclusive sx dr->dt.di.dr_mtx (dr->dt.di.dr_mtx) r = 0 (0xffffff0002966238) locked @ /pool/newsrc/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/dbuf.c:1905
exclusive sx dr->dt.di.dr_mtx (dr->dt.di.dr_mtx) r = 0 (0xffffff0002963438) locked @ /pool/newsrc/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/dbuf.c:1905
0xffffff00055b39c0: tag zfs, type VDIR
usecount 0, writecount 0, refcount 1 mountedhere 0
flags (VI_DOINGINACT)
lock type zfs: EXCL by thread 0xffffff000528b3a0 (pid 1425)
#0 0xffffffff80578cef at __lockmgr_args+0x4ef
#1 0xffffffff80610a59 at vop_stdlock+0x39
#2 0xffffffff808caedb at VOP_LOCK1_APV+0x9b
#3 0xffffffff8062d957 at _vn_lock+0x57
#4 0xffffffff806210bb at vget+0x7b
#5 0xffffffff8060e570 at cache_lookup+0x4e0
#6 0xffffffff8060e890 at vfs_cache_lookup+0xc0
#7 0xffffffff808cbdb7 at VOP_LOOKUP_APV+0xb7
#8 0xffffffff80615513 at lookup+0x3d3
#9 0xffffffff806164e9 at namei+0x4a9
#10 0xffffffff806255cf at kern_statat_vnhook+0x8f
#11 0xffffffff80625775 at kern_statat+0x15
#12 0xffffffff8062583a at lstat+0x2a
#13 0xffffffff80876b48 at syscall+0x118
#14 0xffffffff8085cf91 at Xfast_syscall+0xe1
0xffffff0005618270: tag zfs, type VDIR
usecount 1, writecount 0, refcount 1 mountedhere 0
flags ()
lock type zfs: EXCL by thread 0xffffff000528c000 (pid 1422)
#0 0xffffffff80578f77 at __lockmgr_args+0x777
#1 0xffffffff80610a59 at vop_stdlock+0x39
#2 0xffffffff808caedb at VOP_LOCK1_APV+0x9b
#3 0xffffffff8062d957 at _vn_lock+0x57
#4 0xffffffff806299ba at setutimes+0x7a
#5 0xffffffff80629cb2 at kern_lutimes+0xf2
#6 0xffffffff80876b48 at syscall+0x118
#7 0xffffffff8085cf91 at Xfast_syscall+0xe1
0xffffff00057bd9c0: tag zfs, type VDIR
usecount 1, writecount 0, refcount 2 mountedhere 0
flags ()
lock type zfs: SHARED (count 1)
#0 0xffffffff80578cef at __lockmgr_args+0x4ef
#1 0xffffffff80610a59 at vop_stdlock+0x39
#2 0xffffffff808caedb at VOP_LOCK1_APV+0x9b
#3 0xffffffff8062d957 at _vn_lock+0x57
#4 0xffffffff806210bb at vget+0x7b
#5 0xffffffff8060e570 at cache_lookup+0x4e0
#6 0xffffffff8060e890 at vfs_cache_lookup+0xc0
#7 0xffffffff808cbdb7 at VOP_LOOKUP_APV+0xb7
#8 0xffffffff80615513 at lookup+0x3d3
#9 0xffffffff806164e9 at namei+0x4a9
#10 0xffffffff806255cf at kern_statat_vnhook+0x8f
#11 0xffffffff80625775 at kern_statat+0x15
#12 0xffffffff8062583a at lstat+0x2a
#13 0xffffffff80876b48 at syscall+0x118
#14 0xffffffff8085cf91 at Xfast_syscall+0xe1
0xffffff00057ac4e0: tag zfs, type VDIR
usecount 1, writecount 0, refcount 2 mountedhere 0
flags ()
lock type zfs: EXCL by thread 0xffffff000262a3a0 (pid 1416)
with shared waiters pending
#0 0xffffffff80578cef at __lockmgr_args+0x4ef
#1 0xffffffff80610a59 at vop_stdlock+0x39
#2 0xffffffff808caedb at VOP_LOCK1_APV+0x9b
#3 0xffffffff8062d957 at _vn_lock+0x57
#4 0xffffffff806210bb at vget+0x7b
#5 0xffffffff8060e570 at cache_lookup+0x4e0
#6 0xffffffff8060e890 at vfs_cache_lookup+0xc0
#7 0xffffffff808cbdb7 at VOP_LOOKUP_APV+0xb7
#8 0xffffffff80615513 at lookup+0x3d3
#9 0xffffffff806164e9 at namei+0x4a9
#10 0xffffffff80624df3 at kern_mkdirat+0xc3
#11 0xffffffff80876b48 at syscall+0x118
#12 0xffffffff8085cf91 at Xfast_syscall+0xe1
Uptime: 2m16s
#
Kernel configuration follows:
# cat config.txt
options CONFIG_AUTOGENERATED
ident DEBUG
machine amd64
cpu HAMMER
makeoptions DEBUG=-g
options AH_SUPPORT_AR5416
options IEEE80211_SUPPORT_MESH
options IEEE80211_AMPDU_AGE
options IEEE80211_DEBUG
options AHD_REG_PRETTY_PRINT
options AHC_REG_PRETTY_PRINT
options ATA_STATIC_ID
options SMP
options DEBUG_LOCKS
options DEBUG_VFS_LOCKS
options DIAGNOSTIC
options INVARIANT_SUPPORT
options INVARIANTS
options DDB
options KDB
options WITNESS_SKIPSPIN
options WITNESS
options INCLUDE_CONFIG_FILE
options FLOWTABLE
options MAC
options AUDIT
options HWPMC_HOOKS
options KBD_INSTALL_CDEV
options PRINTF_BUFR_SIZE=128
options _KPOSIX_PRIORITY_SCHEDULING
options P1003_1B_SEMAPHORES
options SYSVSEM
options SYSVMSG
options SYSVSHM
options STACK
options KTRACE
options SCSI_DELAY=5000
options COMPAT_FREEBSD7
options COMPAT_FREEBSD6
options COMPAT_FREEBSD5
options COMPAT_FREEBSD4
options COMPAT_IA32
options COMPAT_43TTY
options GEOM_LABEL
options GEOM_PART_GPT
options PSEUDOFS
options PROCFS
options CD9660
options MSDOSFS
options NFS_ROOT
options NFSLOCKD
options NFSSERVER
options NFSCLIENT
options MD_ROOT
options UFS_GJOURNAL
options UFS_DIRHASH
options UFS_ACL
options SOFTUPDATES
options FFS
options SCTP
options INET6
options INET
options PREEMPTION
options SCHED_ULE
options GEOM_PART_MBR
options GEOM_PART_EBR_COMPAT
options GEOM_PART_EBR
options GEOM_PART_BSD
device isa
device mem
device io
device uart_ns8250
device cpufreq
device acpi
device pci
device fdc
device ata
device atadisk
device ataraid
device atapicd
device atapifd
device atapist
device ahc
device ahd
device amd
device hptiop
device isp
device mpt
device sym
device trm
device adv
device adw
device aic
device bt
device scbus
device ch
device da
device sa
device cd
device pass
device ses
device amr
device arcmsr
device ciss
device dpt
device hptmv
device hptrr
device iir
device ips
device mly
device twa
device aac
device aacp
device ida
device mfi
device mlx
device twe
device atkbdc
device atkbd
device psm
device kbdmux
device vga
device splash
device sc
device agp
device cbb
device pccard
device cardbus
device uart
device ppc
device ppbus
device lpt
device plip
device ppi
device de
device em
device igb
device ixgbe
device le
device ti
device txp
device vx
device miibus
device ae
device age
device alc
device ale
device bce
device bfe
device bge
device dc
device et
device fxp
device jme
device lge
device msk
device nfe
device nge
device pcn
device re
device rl
device sf
device sis
device sk
device ste
device stge
device tl
device tx
device vge
device vr
device wb
device xl
device cs
device ed
device ex
device ep
device fe
device sn
device xe
device wlan
device wlan_wep
device wlan_ccmp
device wlan_tkip
device wlan_amrr
device an
device ath
device ath_hal
device ath_rate_sample
device ral
device wi
device loop
device random
device ether
device vlan
device tun
device pty
device md
device gif
device faith
device firmware
device bpf
device uhci
device ohci
device ehci
device usb
device uhid
device ukbd
device ulpt
device umass
device ums
device urio
device uark
device ubsa
device uftdi
device uipaq
device uplcom
device uslcom
device uvisor
device uvscom
device aue
device axe
device cdce
device cue
device kue
device rue
device udav
device rum
device uath
device ural
device zyd
device firewire
device fwe
device fwip
device dcons
device dcons_crom
#
More information about the freebsd-fs
mailing list