kern/153255: 8.2-PRERELEASE repeatable kernel panic under heavy
network load
Eugene Grosbein
egrosbein at rdtc.ru
Mon Dec 27 11:20:11 UTC 2010
The following reply was made to PR kern/153255; it has been noted by GNATS.
From: Eugene Grosbein <egrosbein at rdtc.ru>
To: bug-followup at FreeBSD.ORG
Cc:
Subject: Re: kern/153255: 8.2-PRERELEASE repeatable kernel panic under heavy
network load
Date: Mon, 27 Dec 2010 17:18:13 +0600
Hi!
I've rebuilt kernel of one of my NAS'es with options INVARIANTS
and it paniced again after 3 days. I've full crashdump.
It has paniced in netisr code trying to process mbuf which recvif field
points to uninitialized area of memory (0xdeadc0dedeadc0de).
Note: netgraph interfaces ngXXX are created/destroyed at high rate
within these PPPoE network access servers.
I will supply web link to crashdump, kernel and symbols on request.
Script started on Mon Dec 27 14:58:33 2010
kgdb kernel.debug /var/crash/vmcore.0
GNU gdb 6.1.1 [FreeBSD]
Copyright 2004 Free Software Foundation, Inc.
GDB is free software, covered by the GNU General Public License, and you are
welcome to change it and/or distribute copies of it under certain conditions.
Type "show copying" to see the conditions.
There is absolutely no warranty for GDB. Type "show warranty" for details.
This GDB was configured as "amd64-marcel-freebsd"...
Unread portion of the kernel message buffer:
Fatal trap 9: general protection fault while in kernel mode
cpuid = 0; apic id = 00
instruction pointer = 0x20:0xffffffff80383258
stack pointer = 0x28:0xffffff8000042810
frame pointer = 0x28:0xffffff8000042820
code segment = base 0x0, limit 0xfffff, type 0x1b
= DPL 0, pres 1, long 1, def32 0, gran 1
processor eflags = interrupt enabled, resume, IOPL = 0
current process = 12 (swi1: netisr 0)
trap number = 9
panic: general protection fault
cpuid = 0
Uptime: 3d23h48m28s
Dumping 3893 MB (6 chunks)
chunk 0: 1MB (151 pages) ... ok
chunk 1: 3251MB (832247 pages)
Fatal trap 12: page fault while in kernel mode
cpuid = 1; apic id = 04
fault virtual address = 0x1000422d0
fault code = supervisor read instruction, page not present
instruction pointer = 0x20:0x1000422d0
stack pointer = 0x28:0xffffff80e75b3ac0
frame pointer = 0x28:0xffffff80e75b3af0
code segment = base 0x0, limit 0xfffff, type 0x1b
= DPL 0, pres 1, long 1, def32 0, gran 1
processor eflags = interrupt enabled, resume, IOPL = 0
current process = 12 (irq19: uhci2 uhci3+)
trap number = 12
3236 3220 3204 3188 3172 3156 3140 3124 3108 3092 3076 3060 3044 3028 3012 2996 2980 2964 2948 2932 2916 2900 2884 2868 2852 2836 2820 2804 2788 2772 2756 2740 2724 2708 2692 2676 2660 2644 2628 2612 2596 2580 2564 2548 2532 2516 2500 2484 2468 2452 2436 2420 2404 2388 2372 2356 2340 2324 2308 2292 2276 2260 2244 2228 2212 2196 2180 2164 2148 2132 2116 2100 2084 2068 2052 2036 2020 2004 1988 1972 1956 1940 1924 1908 1892 1876 1860 1844 1828 1812 1796 1780 1764 1748 1732 1716 1700 1684 1668 1652 1636 1620 1604 1588 1572 1556 1540 1524 1508 1492 1476 1460 1444 1428 1412 1396 1380 1364 1348 1332 1316 1300 1284 1268 1252 1236 1220 1204 1188 1172 1156 1140 1124 1108 1092 1076 1060 1044 1028 1012 996 980 964 948 932 916 900 884 868 852 836 820 804 788 772 756 740 724 708 692 676 660 644 628 612 596 580 564 548 532 516 500 484 468 452 436 420 404 388 372 356 340 324 308 292 276 260 244 228 212 196 180 164 148 132 116 100 84 68 52 36 20 4 ... ok
chunk 2: 1MB (2 pages) ... ok
chunk 3: 1MB (1 pages) ... ok
chunk 4: 2MB (381 pages) ... ok
chunk 5: 640MB (163840 pages) 625 609 593 577 561 545 529 513 497 481 465 449 433 417 401 385 369 353 337 321 305 289 273 257 241 225 209 193 177 161 145 129 113 97 81 65 49 33 17 1
Reading symbols from /boot/modules/if_lagg.ko...done.
Loaded symbols for /boot/modules/if_lagg.ko
#0 doadump () at pcpu.h:224
224 __asm("movq %%gs:0,%0" : "=r" (td));
(kgdb) bt
#0 doadump () at pcpu.h:224
#1 0xffffffff802e5526 in boot (howto=260) at /home/src/sys/kern/kern_shutdown.c:419
#2 0xffffffff802e599f in panic (fmt=Variable "fmt" is not available.
) at /home/src/sys/kern/kern_shutdown.c:592
#3 0xffffffff80478b72 in trap_fatal (frame=0x9, eva=Variable "eva" is not available.
) at /home/src/sys/amd64/amd64/trap.c:783
#4 0xffffffff8047910f in trap (frame=0xffffff8000042760) at /home/src/sys/amd64/amd64/trap.c:592
#5 0xffffffff80461a14 in calltrap () at /home/src/sys/amd64/amd64/exception.S:224
#6 0xffffffff80383258 in strlen (str=0xdeadc0dedeadc0de <Address 0xdeadc0dedeadc0de out of bounds>)
at /home/src/sys/libkern/strlen.c:38
#7 0xffffffff80318148 in kvprintf (fmt=0xffffffff80501b6b " @ %s:%d", func=0xffffffff80316da0 <snprintf_func>,
arg=0xffffff8000042970, radix=10, ap=Variable "ap" is not available.
) at /home/src/sys/kern/subr_prf.c:728
#8 0xffffffff803183d0 in vsnprintf (str=Variable "str" is not available.
) at /home/src/sys/kern/subr_prf.c:461
#9 0xffffffff802e5a06 in panic (fmt=0xffffffff80501b50 "mtx_lock() of spin mutex %s @ %s:%d")
at /home/src/sys/kern/kern_shutdown.c:558
#10 0xffffffff802d7f6b in _mtx_lock_flags (m=Variable "m" is not available.
) at /home/src/sys/kern/kern_mutex.c:197
#11 0xffffffff803d3c96 in ip_input (m=0xffffff000dfaf700) at /home/src/sys/netinet/ip_input.c:636
#12 0xffffffff80395f54 in swi_net (arg=Variable "arg" is not available.
) at /home/src/sys/net/netisr.c:716
#13 0xffffffff802c2716 in intr_event_execute_handlers (p=Variable "p" is not available.
) at /home/src/sys/kern/kern_intr.c:1220
#14 0xffffffff802c3436 in ithread_loop (arg=0xffffff0001834840) at /home/src/sys/kern/kern_intr.c:1233
#15 0xffffffff802c06aa in fork_exit (callout=0xffffffff802c3389 <ithread_loop>, arg=0xffffff0001834840,
frame=0xffffff8000042c40) at /home/src/sys/kern/kern_fork.c:845
#16 0xffffffff80461eee in fork_trampoline () at /home/src/sys/amd64/amd64/exception.S:566
#17 0x0000000000000000 in ?? ()
#18 0x0000000000000000 in ?? ()
#19 0x0000000000000001 in ?? ()
#20 0x0000000000000000 in ?? ()
#21 0x0000000000000000 in ?? ()
#22 0x0000000000000000 in ?? ()
#23 0x0000000000000000 in ?? ()
#24 0x0000000000000000 in ?? ()
#25 0x0000000000000000 in ?? ()
#26 0x0000000000000000 in ?? ()
#27 0x0000000000000000 in ?? ()
#28 0x0000000000000000 in ?? ()
#29 0x0000000000000000 in ?? ()
#30 0x0000000000000000 in ?? ()
#31 0x0000000000000000 in ?? ()
#32 0x0000000000000000 in ?? ()
#33 0x0000000000000000 in ?? ()
#34 0x0000000000000000 in ?? ()
#35 0x0000000000000000 in ?? ()
#36 0x0000000000000000 in ?? ()
#37 0x0000000000000000 in ?? ()
#38 0x0000000000000000 in ?? ()
#39 0x0000000000000000 in ?? ()
#40 0x0000000000000000 in ?? ()
#41 0x0000000000000000 in ?? ()
#42 0xffffff0001849888 in ?? ()
---Type <return> to continue, or q <return> to quit---
#43 0xffffffff806ec1c0 in affinity ()
#44 0xffffff0005107000 in ?? ()
#45 0xffffff8000042b30 in ?? ()
#46 0xffffff8000042ad8 in ?? ()
#47 0xffffff0001849460 in ?? ()
#48 0xffffffff80308cf8 in sched_switch (td=0xffffff0001834840, newtd=0xffffffff802c3389, flags=Variable "flags" is not available.
)
at /home/src/sys/kern/sched_ule.c:1852
Previous frame inner to this frame (corrupt stack?)
(kgdb) frame 11
#11 0xffffffff803d3c96 in ip_input (m=0xffffff000dfaf700) at /home/src/sys/netinet/ip_input.c:636
636 IF_ADDR_LOCK(ifp);
(kgdb) p *ifp
$1 = {if_softc = 0xdeadc0dedeadc0de, if_l2com = 0xdeadc0dedeadc0de, if_vnet = 0xdeadc0dedeadc0de, if_link = {
tqe_next = 0xdeadc0dedeadc0de, tqe_prev = 0xdeadc0dedeadc0de}, if_xname = "оРÂооРÂооРÂооРÂо",
if_dname = 0xdeadc0dedeadc0de <Address 0xdeadc0dedeadc0de out of bounds>, if_dunit = -559038242, if_refcount = 3735929054,
if_addrhead = {tqh_first = 0xdeadc0dedeadc0de, tqh_last = 0xdeadc0dedeadc0de}, if_pcount = -559038242,
if_carp = 0xdeadc0dedeadc0de, if_bpf = 0xdeadc0dedeadc0de, if_index = 49374, if_timer = -8531,
if_vlantrunk = 0xdeadc0dedeadc0de, if_flags = -559038242, if_capabilities = -559038242, if_capenable = -559038242,
if_linkmib = 0xdeadc0dedeadc0de, if_linkmiblen = 16045693110842147038, if_data = {ifi_type = 222 'о',
ifi_physical = 192 'Ð ', ifi_addrlen = 173 'Â', ifi_hdrlen = 222 'о', ifi_link_state = 222 'о', ifi_spare_char1 = 192 'Ð ',
ifi_spare_char2 = 173 'Â', ifi_datalen = 222 'о', ifi_mtu = 16045693110842147038, ifi_metric = 16045693110842147038,
ifi_baudrate = 16045693110842147038, ifi_ipackets = 16045693110842147038, ifi_ierrors = 16045693110842147038,
ifi_opackets = 16045693110842147038, ifi_oerrors = 16045693110842147038, ifi_collisions = 16045693110842147038,
ifi_ibytes = 16045693110842147038, ifi_obytes = 16045693110842147038, ifi_imcasts = 16045693110842147038,
ifi_omcasts = 16045693110842147038, ifi_iqdrops = 16045693110842147038, ifi_noproto = 16045693110842147038,
ifi_hwassist = 16045693110842147038, ifi_epoch = -2401050962867404578, ifi_lastchange = {tv_sec = -2401050962867404578,
tv_usec = -2401050962867404578}}, if_multiaddrs = {tqh_first = 0xdeadc0dedeadc0de, tqh_last = 0xdeadc0dedeadc0de},
if_amcount = -559038242, if_output = 0xdeadc0dedeadc0de, if_input = 0xdeadc0dedeadc0de, if_start = 0xdeadc0dedeadc0de,
if_ioctl = 0xdeadc0dedeadc0de, if_watchdog = 0xdeadc0dedeadc0de, if_init = 0xdeadc0dedeadc0de,
if_resolvemulti = 0xdeadc0dedeadc0de, if_qflush = 0xdeadc0dedeadc0de, if_transmit = 0xdeadc0dedeadc0de,
if_reassign = 0xdeadc0dedeadc0de, if_home_vnet = 0xdeadc0dedeadc0de, if_addr = 0xdeadc0dedeadc0de,
if_llsoftc = 0xdeadc0dedeadc0de, if_drv_flags = -559038242, if_snd = {ifq_head = 0xdeadc0dedeadc0de,
ifq_tail = 0xdeadc0dedeadc0de, ifq_len = -559038242, ifq_maxlen = -559038242, ifq_drops = -559038242, ifq_mtx = {
lock_object = {lo_name = 0xdeadc0dedeadc0de <Address 0xdeadc0dedeadc0de out of bounds>, lo_flags = 3735929054,
lo_data = 3735929054, lo_witness = 0xdeadc0dedeadc0de}, mtx_lock = 16045693110842147038},
ifq_drv_head = 0xdeadc0dedeadc0de, ifq_drv_tail = 0xdeadc0dedeadc0de, ifq_drv_len = -559038242,
ifq_drv_maxlen = -559038242, altq_type = -559038242, altq_flags = -559038242, altq_disc = 0xdeadc0dedeadc0de,
altq_ifp = 0xdeadc0dedeadc0de, altq_enqueue = 0xdeadc0dedeadc0de, altq_dequeue = 0xdeadc0dedeadc0de,
altq_request = 0xdeadc0dedeadc0de, altq_clfier = 0xdeadc0dedeadc0de, altq_classify = 0xdeadc0dedeadc0de,
altq_tbr = 0xdeadc0dedeadc0de, altq_cdnr = 0xdeadc0dedeadc0de},
if_broadcastaddr = 0xdeadc0dedeadc0de <Address 0xdeadc0dedeadc0de out of bounds>, if_bridge = 0xdeadc0dedeadc0de,
if_label = 0xdeadc0dedeadc0de, if_prefixhead = {tqh_first = 0xdeadc0dedeadc0de, tqh_last = 0xdeadc0dedeadc0de},
if_afdata = {0xdeadc0dedeadc0de <repeats 38 times>}, if_afdata_initialized = -559038242, if_afdata_lock = {lock_object = {
lo_name = 0xdeadc0dedeadc0de <Address 0xdeadc0dedeadc0de out of bounds>, lo_flags = 3735929054, lo_data = 3735929054,
lo_witness = 0xdeadc0dedeadc0de}, rw_lock = 16045693110842147038}, if_linktask = {ta_link = {
stqe_next = 0xdeadc0dedeadc0de}, ta_pending = 49374, ta_priority = 57005, ta_func = 0xdeadc0dedeadc0de,
ta_context = 0xdeadc0dedeadc0de}, if_addr_mtx = {lock_object = {
lo_name = 0xdeadc0dedeadc0de <Address 0xdeadc0dedeadc0de out of bounds>, lo_flags = 3735929054, lo_data = 3735929054,
lo_witness = 0xdeadc0dedeadc0de}, mtx_lock = 16045693110842147038}, if_clones = {le_next = 0xdeadc0dedeadc0de,
le_prev = 0xdeadc0dedeadc0de}, if_groups = {tqh_first = 0xdeadc0dedeadc0de, tqh_last = 0xdeadc0dedeadc0de},
if_pf_kif = 0xdeadc0dedeadc0de, if_lagg = 0xdeadc0dedeadc0de, if_alloctype = 222 'о', if_cspare = "Ð Âо",
if_description = 0xdeadc0dedeadc0de <Address 0xdeadc0dedeadc0de out of bounds>, if_pspare = {0xdeadc0dedeadc0de,
0xdeadc0dedeadc0de, 0xdeadc0dedeadc0de, 0xdeadc0dedeadc0de, 0xdeadc0dedeadc0de, 0xdeadc0dedeadc0de, 0xdeadc0dedeadc0de},
if_ispare = {-559038242, -559038242, -559038242, -559038242}}
(kgdb) quit
Script done on Mon Dec 27 14:58:45 2010
More information about the freebsd-net
mailing list