Giant deadlock related to twe
Doug White
dwhite at gumbysoft.com
Sun Aug 22 15:32:35 PDT 2004
Hey folks,
Just got this on my amd64 box. A disk flaked out in my machine, which has
a 3ware 8006-2LP with 2 80GB drives in a RAID0. My X session locked up
and was able to break to ddb. Some ddb twiddling follows. It looks like,
at first glance, some sort of deadlock against softupdates.
twed0: drive timeouttwed0: drive timeouttwed0: drive timeouttwed0: drive
timeout
twed0: drive timeouttwed0: drive timeouttwed0: drive timeouttwe0: AEN:
<twe0: drive timeout for unknown unit 1>
twed0: controller error - device failure (flags = 0x80)
twed0: controller error - device failure (flags = 0x80)
twed0: controller error - device failure (flags = 0x80)
twed0: controller error - device failure (flags = 0x80)
KDB: enter: Line break on console
[thread 100082]
Stopped at kdb_enter+0x31: leave
db> tr # NOTE: this is the syslog process on CPU0, pid 304
kdb_enter() at kdb_enter+0x31
siointr1() at siointr1+0xe3
siointr() at siointr+0x78
intr_execute_handlers() at intr_execute_handlers+0xb2
lapic_handle_intr() at lapic_handle_intr+0x38
Xapic_isr1() at Xapic_isr1+0x7d
--- interrupt, rip = 0xffffffff8034dd9a, rsp = 0xffffffffb23f3960, rbp =
0xffffffffb23f39b0 ---
inodedep_lookup() at inodedep_lookup+0x7a
flush_inodedep_deps() at flush_inodedep_deps+0x5c
softdep_sync_metadata() at softdep_sync_metadata+0x9f
ffs_fsync() at ffs_fsync+0x44c
fsync() at fsync+0x182
syscall() at syscall+0x374
Xfast_syscall() at Xfast_syscall+0xa8
--- syscall (95, FreeBSD ELF64, fsync), rip = 0x20079c0c0, rsp =
0x7fffffffe808, rbp = 0x50f080 ---
db> ps
pid proc uarea uid ppid pgrp flag stat wmesg wchan cmd
811 ffffff00530dab80 ffffffffb48e9000 0 809 800 0004002 [SLPQ
ttyin 0xffffff00009b8410][SLP] less
810 ffffff005313c2e0 ffffffffb48eb000 0 809 800 0004002 [SLPQ
pipdwt 0xffffff00527cc5a0][SLP] zcat
809 ffffff005c24a2e0 ffffffffb4814000 0 800 800 0004002 [SLPQ
wait 0xffffff005c24a2e0][SLP] sh
800 ffffff00626082e0 ffffffffb4765000 0 645 800 0004002 [SLPQ
wait 0xffffff00626082e0][SLP] man
783 ffffff0056d152e0 ffffffffb4823000 1001 653 783 0004002 [SLPQ
ttyin 0xffffff007ace8c10][SLP] less
744 ffffff00572fd000 ffffffffb4886000 1001 632 744 0004002 [SLPQ
ttyin 0xffffff0000eaf010][SLP] tcsh
740 ffffff0056d155c0 ffffffffb4824000 1001 1 594 0004001 [SLPQ
select 0xffffffff80583550][SLP] gconfd-2
738 ffffff0056d15b80 ffffffffb4826000 1001 733 594 000c000
(threaded) mozilla-bin
thread 0xffffff0051e40000 ksegrp 0xffffff005fc80270 [SLPQ kserel
0xffffff005fc80310][SLP]
thread 0xffffff0053001940 ksegrp 0xffffff005fc80270 [SLPQ kserel
0xffffff005fc80310][SLP]
thread 0xffffff00530016f0 ksegrp 0xffffff005fc80270 [SLPQ select
0xffffffff80583550][SLP]
thread 0xffffff0053001250 ksegrp 0xffffff005fc80270 [SLPQ select
0xffffffff80583550][SLP]
thread 0xffffff005ff05250 ksegrp 0xffffff005c46e1a0 [SLPQ ksesigwait
0xffffff0056d15d30][SLP]
733 ffffff005c30f5c0 ffffffffb481f000 1001 594 594 0004000 [SLPQ
wait 0xffffff005c30f5c0][SLP] sh
700 ffffff0056fd85c0 ffffffffb4883000 1001 651 700 0004002 [SLPQ
ttyin 0xffffff0000eae810][SLP] tcsh
653 ffffff005c24ab80 ffffffffb4817000 1001 651 653 0004002 [SLPQ
pause 0xffffff005c24abf0][SLP] tcsh
651 ffffff005fe1f2e0 ffffffffb476a000 1001 594 594 0000001 [SLPQ
select 0xffffffff80583550][SLP] kdeinit
645 ffffff005c24a5c0 ffffffffb4815000 0 644 645 0004002 [SLPQ
pause 0xffffff005c24a630][SLP] tcsh
644 ffffff005c24a8a0 ffffffffb4816000 1001 634 644 0004102 [SLPQ
wait 0xffffff005c24a8a0][SLP] su
634 ffffff0060029000 ffffffffb47cd000 1001 632 634 0004002 [SLPQ
pause 0xffffff0060029070][SLP] tcsh
632 ffffff005c54d2e0 ffffffffb4819000 1001 594 594 0000001 [SLPQ
select 0xffffffff80583550][SLP] kdeinit
630 ffffff005fdc88a0 ffffffffb47cb000 1001 1 594 0000001 [SLPQ
select 0xffffffff80583550][SLP] kdeinit
627 ffffff0060029b80 ffffffffb47d1000 1001 594 594 0000000 [SLPQ
select 0xffffffff80583550][SLP] kdeinit
626 ffffff005bfc92e0 ffffffffb47d3000 1001 1 594 0000001 [SLPQ
select 0xffffffff80583550][SLP] kdeinit
624 ffffff005bfc9000 ffffffffb47d2000 1001 1 594 0000001 [SLPQ
select 0xffffffff80583550][SLP] kdeinit
622 ffffff007b4b18a0 ffffffffb2400000 1001 1 594 0000001 [SLPQ
select 0xffffffff80583550][SLP] kdeinit
619 ffffff005bfc95c0 ffffffffb47d4000 1001 594 594 0000001 [SLPQ
select 0xffffffff80583550][SLP] kdeinit
617 ffffff005c54db80 ffffffffb481c000 1001 1 594 0000001 [SLPQ
select 0xffffffff80583550][SLP] kdeinit
615 ffffff005c24a000 ffffffffb47d7000 1001 575 575 0004000 [SLPQ
nanslp 0xffffffff8057ae80][SLP] kwrapper
603 ffffff005c54d000 ffffffffb4818000 1001 1 594 0000001 [LOCK
Giant ffffff00581b6400] kdeinit
600 ffffff005c54d8a0 ffffffffb481b000 1001 594 594 0000001 [SLPQ
select 0xffffffff80583550][SLP] kdeinit
597 ffffff005c30f000 ffffffffb481d000 1001 1 594 0000001 [SLPQ
select 0xffffffff80583550][SLP] kdeinit
594 ffffff007b4cd000 ffffffffb23b7000 1001 1 594 0000000 [SLPQ
select 0xffffffff80583550][SLP] kdeinit
575 ffffff007b7288a0 ffffffffb1ac7000 1001 559 575 0004000 [SLPQ
wait 0xffffff007b7288a0][SLP] sh
559 ffffff00627425c0 ffffffffb4761000 0 556 556 0000000 [SLPQ
wait 0xffffff00627425c0][SLP] kdm
558 ffffff005fe1fb80 ffffffffb476d000 0 556 558 0004000 [SLPQ
select 0xffffffff80583550][SLP] XFree86
556 ffffff0062742b80 ffffffffb4763000 0 1 556 0000001 [SLPQ
select 0xffffffff80583550][SLP] kdm
554 ffffff005fdc8000 ffffffffb47c8000 0 1 554 0004002 [SLPQ
ttyin 0xffffff0000ead410][SLP] getty
553 ffffff007b4b1b80 ffffffffb2401000 0 1 553 0004002 [SLPQ
ttyin 0xffffff0000a9e010][SLP] getty
552 ffffff005fdc82e0 ffffffffb47c9000 0 1 552 0004002 [SLPQ
ttyin 0xffffff0000a5cc10][SLP] getty
551 ffffff005fdc85c0 ffffffffb47ca000 0 1 551 0004002 [SLPQ
ttyin 0xffffff0000a9d410][SLP] getty
550 ffffff0062608000 ffffffffb4764000 0 1 550 0004002 [SLPQ
ttyin 0xffffff0000a5c010][SLP] getty
549 ffffff0062742000 ffffffffb4705000 0 1 549 0004002 [SLPQ
ttyin 0xffffff0000a5c810][SLP] getty
548 ffffff00627422e0 ffffffffb4760000 0 1 548 0004002 [SLPQ
ttyin 0xffffff0000a71410][SLP] getty
547 ffffff007b4ce000 ffffffffb23bc000 0 1 547 0004002 [SLPQ
ttyin 0xffffff0000a71810][SLP] getty
546 ffffff00600298a0 ffffffffb47d0000 0 1 546 0004002 [SLPQ
ttyin 0xffffff0000a71c10][SLP] getty
484 ffffff005fe1f5c0 ffffffffb476b000 0 1 484 0000000 [SLPQ
nanslp 0xffffffff8057ae80][SLP] cron
470 ffffff005fe1f8a0 ffffffffb476c000 25 1 470 0000100 [SLPQ
pause 0xffffff005fe1f910][SLP] sendmail
467 ffffff005fe1f000 ffffffffb4769000 0 1 467 0000100 [SLPQ
select 0xffffffff80583550][SLP] sendmail
461 ffffff007b4cd8a0 ffffffffb23ba000 0 1 461 0000100 [SLPQ
select 0xffffffff80583550][SLP] sshd
454 ffffff007b4cd5c0 ffffffffb23b9000 0 440 440 0000000 [SLPQ
pause 0xffffff007b4cd630][SLP] ntpd
440 ffffff00626085c0 ffffffffb4766000 0 1 440 0000000 [SLPQ
select 0xffffffff80583550][SLP] ntpd
408 ffffff007b4cd2e0 ffffffffb23b8000 0 1 408 0000000 [SLPQ
select 0xffffffff80583550][SLP] usbd
406 ffffff007b4b12e0 ffffffffb23fe000 0 1 406 0000000 [SLPQ
select 0xffffffff80583550][SLP] moused
319 ffffff007b4b15c0 ffffffffb23ff000 0 1 319 0000000 [SLPQ
select 0xffffffff80583550][SLP] rpcbind
304 ffffff007b4ce2e0 ffffffffb23f9000 0 1 304 0000000 [CPU 1]
syslogd
243 ffffff00626088a0 ffffffffb4767000 0 1 243 0000000 [SLPQ
select 0xffffffff80583550][SLP] dhclient
189 ffffff007b728b80 ffffffffb1ac8000 0 1 189 0000000 [SLPQ
pause 0xffffff007b728bf0][SLP] adjkerntz
67 ffffff007b4ce5c0 ffffffffb23fa000 0 0 0 0000204 [SLPQ -
0xffffffffb23eec0c][SLP] schedcpu
66 ffffff007b4ce8a0 ffffffffb23fb000 0 0 0 0000204 [SLPQ -
0xffffffff8058ce38][SLP] nfsiod 3
65 ffffff007b4ceb80 ffffffffb23fc000 0 0 0 0000204 [SLPQ -
0xffffffff8058ce30][SLP] nfsiod 2
64 ffffff007b4b1000 ffffffffb23fd000 0 0 0 0000204 [SLPQ -
0xffffffff8058ce28][SLP] nfsiod 1
63 ffffff007b7552e0 ffffffffb1a7f000 0 0 0 0000204 [SLPQ -
0xffffffff8058ce20][SLP] nfsiod 0
62 ffffff007b7555c0 ffffffffb1a80000 0 0 0 0000204 [SLPQ
vlruwt 0xffffff007b7555c0][SLP] vnlru
61 ffffff007b7558a0 ffffffffb1a81000 0 0 0 0000204 [SLPQ
syncer 0xffffffff8057ab80][SLP] syncer
60 ffffff007b755b80 ffffffffb1a82000 0 0 0 0000204 [SLPQ
psleep 0xffffffff80583e7c][SLP] bufdaemon
59 ffffff007b6e5000 ffffffffb1a83000 0 0 0 000020c [SLPQ
pgzero 0xffffffff805957f0][SLP] pagezero
58 ffffff007b6e52e0 ffffffffb1a84000 0 0 0 0000204 [SLPQ
psleep 0xffffffff8059585c][SLP] vmdaemon
57 ffffff007b6e55c0 ffffffffb1a85000 0 0 0 0000204 [SLPQ
psleep 0xffffffff8059580c][SLP] pagedaemon
56 ffffff007b6e58a0 ffffffffb1ac2000 0 0 0 0000204 [IWAIT]
swi0: sio
55 ffffff007b6e5b80 ffffffffb1ac3000 0 0 0 0000204 [SLPQ -
0xffffff00009ce248][SLP] fdc0
54 ffffff007b728000 ffffffffb1ac4000 0 0 0 0000204 [SLPQ
usbevt 0xffffffff80823420][SLP] usb1
53 ffffff007b7282e0 ffffffffb1ac5000 0 0 0 0000204 [SLPQ
usbtsk 0xffffffff8056e4f0][SLP] usbtask
52 ffffff007b7285c0 ffffffffb1ac6000 0 0 0 0000204 [SLPQ
usbevt 0xffffffff80821420][SLP] usb0
9 ffffff007b751000 ffffffffb1a38000 0 0 0 0000204 [SLPQ
actask 0xffffffff8056bd00][SLP] acpi_task2
8 ffffff007b7512e0 ffffffffb1a39000 0 0 0 0000204 [SLPQ
actask 0xffffffff8056bd00][SLP] acpi_task1
7 ffffff007b7515c0 ffffffffb1a3a000 0 0 0 0000204 [SLPQ
actask 0xffffffff8056bd00][SLP] acpi_task0
51 ffffff007b7518a0 ffffffffb1a3b000 0 0 0 0000204 [IWAIT]
swi6: task queue
6 ffffff007b751b80 ffffffffb1a3c000 0 0 0 0000204 [SLPQ -
0xffffff000099ae00][SLP] kqueue taskq
50 ffffff007b752000 ffffffffb1a3d000 0 0 0 0000204 [IWAIT]
swi6: acpitaskq
49 ffffff007b7522e0 ffffffffb1a7a000 0 0 0 0000204 [IWAIT]
swi6:+
5 ffffff007b7525c0 ffffffffb1a7b000 0 0 0 0000204 [SLPQ -
0xffffff00008c3a80][SLP] thread taskq
48 ffffff007b7528a0 ffffffffb1a7c000 0 0 0 0000204 [IWAIT]
swi6:+
47 ffffff007b752b80 ffffffffb1a7d000 0 0 0 0000204 [SLPQ -
0xffffffff8056c1c0][SLP] yarrow
4 ffffff007b755000 ffffffffb1a7e000 0 0 0 0000204 [SLPQ -
0xffffffff80571ea8][SLP] g_down
3 ffffff007b700000 ffffffffb19f2000 0 0 0 0000204 [SLPQ -
0xffffffff80571ea0][SLP] g_up
2 ffffff007b7002e0 ffffffffb19f3000 0 0 0 0000204 [SLPQ -
0xffffffff80571e90][SLP] g_event
46 ffffff007b7005c0 ffffffffb19f4000 0 0 0 0000204 [IWAIT]
swi1: net
45 ffffff007b7008a0 ffffffffb19f5000 0 0 0 0000204 [IWAIT]
swi4: vm
44 ffffff007b700b80 ffffffffb19f6000 0 0 0 000020c [LOCK
Giant ffffff00581b6400] swi5: clock sio
43 ffffff007b704000 ffffffffb1a33000 0 0 0 0000204 [IWAIT]
irq31:
42 ffffff007b7042e0 ffffffffb1a34000 0 0 0 0000204 [IWAIT]
irq30:
41 ffffff007b7045c0 ffffffffb1a35000 0 0 0 0000204 [LOCK
Giant ffffff00581b6400] irq29: twe0
40 ffffff007b7048a0 ffffffffb1a36000 0 0 0 0000204 [IWAIT]
irq28:
39 ffffff007b704b80 ffffffffb1a37000 0 0 0 0000204 [IWAIT]
irq27: bge0 bge1
38 ffffff007b7262e0 ffffffffb19ad000 0 0 0 0000204 [IWAIT]
irq26:
37 ffffff007b7265c0 ffffffffb19ae000 0 0 0 0000204 [IWAIT]
irq25:
36 ffffff007b7268a0 ffffffffb19af000 0 0 0 0000204 [IWAIT]
irq24:
35 ffffff007b726b80 ffffffffb19b0000 0 0 0 0000204 [IWAIT]
irq23:
34 ffffff007b754000 ffffffffb19ed000 0 0 0 0000204 [IWAIT]
irq22:
33 ffffff007b7542e0 ffffffffb19ee000 0 0 0 0000204 [IWAIT]
irq21:
32 ffffff007b7545c0 ffffffffb19ef000 0 0 0 0000204 [IWAIT]
irq20:
31 ffffff007b7548a0 ffffffffb19f0000 0 0 0 0000204 [LOCK
Giant ffffff00581b6400] irq19: ohci0 ohci1
30 ffffff007b754b80 ffffffffb19f1000 0 0 0 0000204 [IWAIT]
irq18:
29 ffffff007b77c8a0 ffffffffb1987000 0 0 0 0000204 [IWAIT]
irq17:
28 ffffff007b77cb80 ffffffffb1988000 0 0 0 0000204 [IWAIT]
irq16:
27 ffffff007b743000 ffffffffb1989000 0 0 0 0000204 [IWAIT]
irq15: ata1
26 ffffff007b7432e0 ffffffffb19a8000 0 0 0 0000204 [IWAIT]
irq14: ata0
25 ffffff007b7435c0 ffffffffb19a9000 0 0 0 0000204 [IWAIT]
irq13:
24 ffffff007b7438a0 ffffffffb19aa000 0 0 0 0000204 [IWAIT]
irq12:
23 ffffff007b743b80 ffffffffb19ab000 0 0 0 0000204 [IWAIT]
irq11:
22 ffffff007b726000 ffffffffb19ac000 0 0 0 0000204 [IWAIT]
irq10:
21 ffffff007b77a2e0 ffffffffb1962000 0 0 0 0000204 [IWAIT]
irq9: acpi0
20 ffffff007b77a5c0 ffffffffb1963000 0 0 0 0000204 [IWAIT]
irq8: rtc
19 ffffff007b77a8a0 ffffffffb1982000 0 0 0 0000204 [IWAIT]
irq7:
18 ffffff007b77ab80 ffffffffb1983000 0 0 0 0000204 [IWAIT]
irq6: fdc0
17 ffffff007b77c000 ffffffffb1984000 0 0 0 0000204 [IWAIT]
irq5:
16 ffffff007b77c2e0 ffffffffb1985000 0 0 0 0000204 [IWAIT]
irq4: sio0
15 ffffff007b77c5c0 ffffffffb1986000 0 0 0 0000204 [IWAIT]
irq3: sio1
14 ffffff007b771000 ffffffffb1920000 0 0 0 0000204 [IWAIT]
irq0: clk
13 ffffff007b7712e0 ffffffffb195d000 0 0 0 0000204 [LOCK
Giant ffffff00581b6400] irq1: atkbd0
12 ffffff007b7715c0 ffffffffb195e000 0 0 0 000020c [CPU 0]
idle: cpu0
11 ffffff007b7718a0 ffffffffb195f000 0 0 0 000020c [Can run]
idle: cpu1
1 ffffff007b771b80 ffffffffb1960000 0 0 1 0004200 [SLPQ
wait 0xffffff007b771b80][SLP] init
10 ffffff007b77a000 ffffffffb1961000 0 0 0 0000204 [SLPQ
ktrace 0xffffffff805777d0][SLP] ktrace
0 ffffffff80572020 ffffffff8065e000 0 0 0 0000200 [SLPQ
sched 0xffffffff80572020][SLP] swapper
db> tr 41
sched_switch() at sched_switch+0xc8
mi_switch() at mi_switch+0x1b0
turnstile_wait() at turnstile_wait+0x37b
_mtx_lock_sleep() at _mtx_lock_sleep+0x149
ithread_loop() at ithread_loop+0x1b2
fork_exit() at fork_exit+0xab
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip = 0, rsp = 0xffffffffb1a14d00, rbp = 0 ---
--
Doug White | FreeBSD: The Power to Serve
dwhite at gumbysoft.com | www.FreeBSD.org
More information about the freebsd-current
mailing list