kern/73321: Reproducible Panic (LOR: I4B / INET6)
Stefan Esser
se at FreeBSD.org
Sat Oct 30 12:10:23 PDT 2004
>Number: 73321
>Category: kern
>Synopsis: Reproducible Panic (LOR: I4B / INET6)
>Confidential: no
>Severity: serious
>Priority: medium
>Responsible: freebsd-bugs
>State: open
>Quarter:
>Keywords:
>Date-Required:
>Class: sw-bug
>Submitter-Id: current-users
>Arrival-Date: Sat Oct 30 19:10:22 GMT 2004
>Closed-Date:
>Last-Modified:
>Originator: Stefan Esser
>Release: FreeBSD 5.3-STABLE i386
>Organization:
>Environment:
System: FreeBSD gw.athome FreeBSD 5.3-STABLE #0: Sat Oct 30 13:00:11 CEST 2004
gw.athome:/usr/src/sys/i386/compile/GW i386
kernel configured with I4B (ISDN) and INET6
>Description:
There appears to be a reproducible LOR in a kernel built with both
IPv6 and ISDN support. This combination leads to a panic after 1 hour
of apparently correct operation:
Fatal trap 12: page fault while in kernel mode
fault virtual address = 0x8
fault code = supervisor read, page not present
instruction pointer = 0x8:0xc0589be6
stack pointer = 0x10:0xc748ccb8
frame pointer = 0x10:0xc748ccc4
code segment = base 0x0, limit 0xfffff, type 0x1b
= DPL 0, pres 1, def32 1, gran 1
processor eflags = interrupt enabled, resume, IOPL = 0
current process = 26 (swi5: clock sio)
Uptime: 1h0m2s
(kgdb) bt
#7 0xc04d01ba in kdb_trap (type=12, code=0, tf=0xc748cc78)
at ../../../kern/subr_kdb.c:418
#8 0xc0615c99 in trap_fatal (frame=0xc748cc78, eva=8)
at ../../../i386/i386/trap.c:804
#9 0xc06159bb in trap_pfault (frame=0xc748cc78, usermode=0, eva=8)
at ../../../i386/i386/trap.c:727
#10 0xc0615581 in trap (frame=
{tf_fs = 24, tf_es = 16, tf_ds = 16, tf_edi = 14848, tf_esi = -1066833824, tf_ebp = -951530300, tf_isp = -951530332, tf_ebx = -1057769088, tf_edx = 2326,tf_ecx = -1066665168, tf_eax = 0, tf_trapno = 12, tf_err = 0, tf_eip = -1067934746, tf_cs = 8, tf_eflags = 66182, tf_esp = 6, tf_ss = 4})
at ../../../i386/i386/trap.c:417
#11 0xc0589be6 in nd6_slowtimo (ignored_arg=0x0)
at ../../../netinet6/nd6.c:1801
#12 0xc04c3ae7 in softclock (dummy=0x0) at ../../../kern/kern_timeout.c:259
#13 0xc04a1775 in ithread_loop (arg=0xc0e86500)
at ../../../kern/kern_intr.c:547
#14 0xc04a06a9 in fork_exit (callout=0xc04a1600 <ithread_loop>,
arg=0xc0e86500, frame=0xc748cd48) at ../../../kern/kern_fork.c:811
(kgdb) frame 11
#11 0xc0589be6 in nd6_slowtimo (ignored_arg=0x0)
at ../../../netinet6/nd6.c:1801
1801 nd6if = ND_IFINFO(ifp);
(kgdb) list
1796
1797 callout_reset(&nd6_slowtimo_ch, ND6_SLOWTIMER_INTERVAL * hz,
1798 nd6_slowtimo, NULL);
1799 IFNET_RLOCK();
1800 for (ifp = TAILQ_FIRST(&ifnet); ifp; ifp = TAILQ_NEXT(ifp, if_list)) {
1801 ======> nd6if = ND_IFINFO(ifp);
1802 if (nd6if->basereachable && /* already initialized */
1803 (nd6if->recalctm -= ND6_SLOWTIMER_INTERVAL) <= 0) {
1804 /*
1805 * Since reachable time rarely changes by router
(kgdb) up
#12 0xc04c3ae7 in softclock (dummy=0x0) at ../../../kern/kern_timeout.c:259
259 c_func(c_arg);
(kgdb) list
254 }
255 #ifdef DIAGNOSTIC
256 binuptime(&bt1);
257 mtx_lock(&dont_sleep_in_callout);
258 #endif
259 ======> c_func(c_arg);
260 #ifdef DIAGNOSTIC
261 mtx_unlock(&dont_sleep_in_callout);
262 binuptime(&bt2);
263 bintime_sub(&bt2, &bt1);
#13 0xc04a1775 in ithread_loop (arg=0xc0e86500)
at ../../../kern/kern_intr.c:547
547 ih->ih_handler(ih->ih_argument);
(kgdb) list
542 mtx_unlock(&ithd->it_lock);
543 goto restart;
544 }
545 if ((ih->ih_flags & IH_MPSAFE) == 0)
546 mtx_lock(&Giant);
547 ======> ih->ih_handler(ih->ih_argument);
548 if ((ih->ih_flags & IH_MPSAFE) == 0)
549 mtx_unlock(&Giant);
550 }
551 if (ithd->it_enable != NULL) {
Surprisingly, the panic always occurs after exactly 1h0m2s, with
a literally identical panic message each time ...
The ifp argument to ND_IFINFO always corresponds to some I4N device,
e.g. "isp0" or "ipr0".
A kernel built with DIAGNOSTICS and WITNESS halts after 1h0m8s (yes,
I was warned about reduced performance, but I didn't expect it to
affect the time required to enter the kernel debugger ;-)
lock order reversal
1st 0xc06d7980 ifnet (ifnet) @ netinet6/nd6.c:1799
2nd 0xc06a9d24 user map (user map) @ vm/vm_map.c:2997
KDB: stack backtrace:
kdb_backtrace(0,ffffffff,c06b37e8,c06b40a8,c0662dec) at kdb_backtrace+0x29
witness_checkorder(c06a9d24,9,c0644508,bb5) at witness_checkorder+0x570
_sx_xlock(c06a9d24,c06444ff,bb5) at _sx_xlock+0x5c
_vm_map_lock_read(c06a9ce0,c06444ff,bb5,1000046,c0e9677c) at _vm_map_lock_read+0x3b
vm_map_lookup(c748cb6c,0,1,c748cb70,c748cb60) at vm_map_lookup+0x30
vm_fault(c06a9ce0,0,1,0,c0e97000) at vm_fault+0x69
trap_pfault(c748cc34,0,8) at trap_pfault+0xdc
trap(18,10,10,3a00,c0684bc0) at trap+0x331
calltrap() at calltrap+0x5
--- trap 0xc, eip = 0xc0579aa6, esp = 0xc748cc74, ebp = 0xc748cc80 ---
nd6_slowtimo(0,c06af2e0,0,c0630943,101) at nd6_slowtimo+0x66
softclock(0) at softclock+0x1d7
ithread_loop(c0e86500,c748cd48,c0e86500,c049e610,0) at ithread_loop+0x144
fork_exit(c049e610,c0e86500,c748cd48) at fork_exit+0xa8
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xc748cd7c, ebp = 0 ---
This is exactly the same situation that lead to the panic without
WITNESS (frame #17 here corresponds to frame #11 without WITNESS):
#9 0xc04c8beb in kdb_enter (msg=0x0) at cpufunc.h:56
#10 0xc04d2543 in witness_checkorder (lock=0xc06a9d24, flags=9,
file=0xc0644508 "vm/vm_map.c", line=2997)
at ../../../kern/subr_witness.c:952
#11 0xc04b63bc in _sx_xlock (sx=0xc06a9d24,
file=0xc06444ff "../../../vm/vm_map.c", line=2997)
at ../../../kern/kern_sx.c:169
#12 0xc05ba03b in _vm_map_lock_read (map=0x0, file=0x0, line=0)
at ../../../vm/vm_map.c:380
#13 0xc05bd850 in vm_map_lookup (var_map=0xc748cb6c, vaddr=0,
fault_typea=1 '\001', out_entry=0xc748cb70, object=0x0, pindex=0x0,
out_prot=0x0, wired=0xc748cb48) at ../../../vm/vm_map.c:2997
#14 0xc05b5cb9 in vm_fault (map=0xc06a9ce0, vaddr=0, fault_type=1 '\001',
fault_flags=0) at ../../../vm/vm_fault.c:234
#15 0xc05fee8c in trap_pfault (frame=0xc748cc34, usermode=0, eva=8)
at ../../../i386/i386/trap.c:704
#16 0xc05feb61 in trap (frame=
{tf_fs = 24, tf_es = 16, tf_ds = 16, tf_edi = 14848, tf_esi = -1066906688, tf_ebp = -951530368, tf_isp = -951530400, tf_ebx = -1057768640, tf_edx = 950, tf_ecx = 1, tf_eax = 0, tf_trapno = 12, tf_err = 0, tf_eip = -1068000602, tf_cs = 8, tf_eflags = 66182, tf_esp = 0, tf_ss = -1058511616})
at ../../../i386/i386/trap.c:417
#17 0xc0579aa6 in nd6_slowtimo (ignored_arg=0x0)
at ../../../netinet6/nd6.c:1801
#18 0xc04bcde7 in softclock (dummy=0x0) at ../../../kern/kern_timeout.c:259
#19 0xc049e754 in ithread_loop (arg=0xc0e86500)
at ../../../kern/kern_intr.c:547
#20 0xc049da78 in fork_exit (callout=0xc049e610 <ithread_loop>,
arg=0xc0e86500, frame=0xc748cd48) at ../../../kern/kern_fork.c:811
Seems that this is a "real" LOR leading to a panic.
A kernel without INET6 but with IB4 works flawlessly.
Boot message log:
Copyright (c) 1992-2004 The FreeBSD Project.
Copyright (c) 1979, 1980, 1983, 1986, 1988, 1989, 1991, 1992, 1993, 1994
The Regents of the University of California. All rights reserved.
FreeBSD 5.3-STABLE #1: Mon Oct 25 23:39:43 CEST 2004
se at gw.athome:/usr/src/sys/i386/compile/GW
WARNING: debug.mpsafenet forced to 0 as i4b_ipr requires Giant
WARNING: MPSAFE network stack disabled, expect reduced performance.
Timecounter "i8254" frequency 1193182 Hz quality 0
CPU: Pentium/P54C (99.72-MHz 586-class CPU)
Origin = "GenuineIntel" Id = 0x526 Stepping = 6
Features=0x1bf<FPU,VME,DE,PSE,TSC,MSR,MCE,CX8>
real memory = 125829120 (120 MB)
avail memory = 117673984 (112 MB)
npx0: [FAST]
npx0: <math processor> on motherboard
npx0: INT 16 interface
pcib0: <Host to PCI bridge> pcibus 0 on motherboard
pci0: <PCI bus> on pcib0
isab0: <PCI-ISA bridge> at device 1.0 on pci0
isa0: <ISA bus> on isab0
atapci0: <Intel PIIX3 WDMA2 controller> port 0xffa0-0xffaf,0x376,0x170-0x177,0x3f6,0x1f0-0x1f7 at device 1.1 on pci0
ata0: channel #0 on atapci0
ata1: channel #1 on atapci0
pci0: <serial bus, USB> at device 1.2 (no driver attached)
xl0: <3Com 3c905B-TX Fast Etherlink XL> port 0x5000-0x507f mem 0x80000000-0x8000007f irq 10 at device 6.0 on pci0
miibus0: <MII bus> on xl0
xlphy0: <3Com internal media interface> on miibus0
xlphy0: 10baseT, 10baseT-FDX, 100baseTX, 100baseTX-FDX, auto
xl0: Ethernet address: 00:50:da:48:eb:64
xl0: [GIANT-LOCKED]
vx0: <3COM 3C590 Etherlink III PCI> port 0x5080-0x509f irq 9 at device 7.0 on pci0
vx0: [GIANT-LOCKED]
utp/aui/bnc[*utp*]: disable 'auto select' with DOS util!vx0: Ethernet address: 00:a0:24:9e:cc:54
pci0: <display, VGA> at device 8.0 (no driver attached)
cpu0 on motherboard
orm0: <ISA Option ROM> at iomem 0xc0000-0xc7fff on isa0
atkbdc0: <Keyboard controller (i8042)> at port 0x64,0x60 on isa0
atkbd0: <AT Keyboard> flags 0x1 irq 1 on atkbdc0
atkbd0: [GIANT-LOCKED]
fdc0: <Enhanced floppy controller> at port 0x3f0-0x3f5 irq 6 drq 2 on isa0
fdc0: [FAST]
fd0: <1440-KB 3.5" drive> on fdc0 drive 0
isic0: [GIANT-LOCKED]
isic0 at port 0x580-0x59f,0x180-0x19f,0x980-0x99f,0xd80-0xd9f irq 5 flags 0x3 on isa0
isic0: passive stack unit 0
isic0: Teles S0/16.3
ppc0: <Parallel port> at port 0x3bc-0x3c3 irq 7 flags 0xc on isa0
ppc0: PC87306 chipset (ECP/EPP) in ECP+EPP mode (EPP 1.9)
ppc0: FIFO with 16/16/8 bytes threshold
ppbus0: <Parallel port bus> on ppc0
lpt0: <Printer> on ppbus0
lpt0: Interrupt-driven port
ppi0: <Parallel I/O> on ppbus0
sc0: <System console> at flags 0x100 on isa0
sc0: VGA <16 virtual consoles, flags=0x300>
sio0 at port 0x3f8-0x3ff irq 4 flags 0x10 on isa0
sio0: type 16550A
sio1: configured irq 3 not in bitmap of probed irqs 0
sio1: port may not be enabled
vga0: <Generic ISA VGA> at port 0x3c0-0x3df iomem 0xa0000-0xbffff on isa0
unknown: <PNP0303> can't assign resources (port)
unknown: <PNP0700> can't assign resources (port)
unknown: <PNP0501> can't assign resources (port)
unknown: <PNP0401> can't assign resources (port)
Timecounter "TSC" frequency 99716832 Hz quality 800
Timecounters tick every 10.000 msec
i4bing: 4 i4b NetGraph ISDN B-channel device(s) attached
i4brbch: 4 raw B channel access device(s) attached
i4btel: 4 ISDN telephony interface device(s) attached
i4bisppp: 4 ISDN SyncPPP device(s) attached
i4b: ISDN call control device attached
i4btrc: 4 ISDN trace device(s) attached
i4bipr: 4 IP over raw HDLC ISDN device(s) attached (VJ header compression)
i4bctl: ISDN system control port attached
ad0: 4028MB <IBM-DHEA-34330/HE4IA43B> [8184/16/63] at ata0-master WDMA2
acd0: CDROM <GCD-R540/1.16> at ata1-master PIO3
Mounting root from ufs:/dev/ad0s1a
WARNING: / was not properly dismounted
Kernel config file (simplified for debugging, with DIAGNOSTICS/WITNESS):
#
# ISDN586 -- Pentium with i4b ISDN driver
#
machine i386
cpu I586_CPU
ident "ISDN586"
makeoptions DEBUG="-g"
options INVARIANTS
options INVARIANT_SUPPORT
options DIAGNOSTIC
options MUTEX_DEBUG
options WITNESS
options WITNESS_KDB
options WITNESS_SKIPSPIN
options DDB
options KDB
options KDB_UNATTENDED
options KDB_TRACE
options SCHED_4BSD
options INET
options INET6
options TCP_DROP_SYNFIN
options NETGRAPH
options NETGRAPH_BPF
options NETGRAPH_ETHER
options NETGRAPH_IFACE
options NETGRAPH_PPP
options NETGRAPH_PPPOE
options NETGRAPH_SOCKET
options NETGRAPH_VJC
options SOFTUPDATES
options FFS
options UFS_DIRHASH
options COMPAT_43
options COMPAT_FREEBSD4
options SYSVSHM
options SYSVMSG
options SYSVSEM
options NO_F00F_HACK
options _KPOSIX_PRIORITY_SCHEDULING
device isa
device pci
device fdc
device ata
device atadisk
device atapicd
options ATA_STATIC_ID
device miibus
device vx
device xl
device atkbdc
device atkbd
device psm
device vga
device sc
device npx
device sio
device ppc
device ppbus
device lpt
device ppi
options PPC_PROBE_CHIPSET
device loop
device bpf
device ether
device tun
device pty
device md
device random
device io
device mem
device isic
device "i4b"
device "i4bctl"
device "i4bq921"
device "i4bq931"
device "i4btrc" 4
device "i4brbch" 4
device "i4btel" 4
device "i4bipr" 4
device "i4bisppp" 4
device "i4bing" 4
options ELSA_QS1PCI
options TEL_S0_16_3
options IPR_VJ
options IPR_LOG=32
>How-To-Repeat:
Build a kernel with IB4 and INET6, boot and wait.
I've got kernel images and crash dumps with and without DIAGNOSTICS and
WITNESS in case that some information is missing in this PR.
>Fix:
Work around: Remove either I4B or INET6 support from your kernel.
>Release-Note:
>Audit-Trail:
>Unformatted:
More information about the freebsd-bugs
mailing list