100% repeatable crashes on 6.2-RELEASE-p3 (bt full)

Fri Mar 23 13:13:39 UTC 2007

Gregory Edigarov wrote:
> Hello,
>
> I've got these repeatable crashes with:
>
> klon# uname -a
> FreeBSD klon.klsp.kharkov.ua 6.2-RELEASE-p3 FreeBSD 6.2-RELEASE-p3 #7: 
> Fri Mar 23 11:26:01 EET 2007 
> root at klon.klsp.kharkov.ua:/usr/obj/usr/src/sys/KLON i386
>
> the system is running quagga and l2tpd built from the yesterday's ports.
> I noticed that this panics are usually happen when third ppp interface 
> going up.
> what can I do?
> Below is the complete back trace.
>
> klon# cd /usr/obj/usr/src/sys/KLON/
> klon# kgdb kernel.debug /var/crash/vmcore.0
> kgdb: kvm_nlist(_stopped_cpus):
> kgdb: kvm_nlist(_stoppcbs):
> [GDB will not be able to debug user-mode threads: 
> /usr/lib/libthread_db.so: Undefined symbol "ps_pglobal_lookup"]
> GNU gdb 6.1.1 [FreeBSD]
> Copyright 2004 Free Software Foundation, Inc.
> GDB is free software, covered by the GNU General Public License, and 
> you are
> welcome to change it and/or distribute copies of it under certain 
> conditions.
> Type "show copying" to see the conditions.
> There is absolutely no warranty for GDB. Type "show warranty" for 
> details.
> This GDB was configured as "i386-marcel-freebsd".
> Ready to go. Enter 'tr' to connect to the remote target
> with /dev/cuad0, 'tr /dev/cuad1' to connect to a different port
> or 'trf portno' to connect to the remote target with the firewire
> interface. portno defaults to 5556.
>
> Type 'getsyms' after connection to load kld symbols.
>
> If you're debugging a local system, you can use 'kldsyms' instead
> to load the kld symbols. That's a less obnoxious interface.
>
> Unread portion of the kernel message buffer:
>
> Fatal trap 12: page fault while in kernel mode
> fault virtual address = 0xffffff80
> fault code = supervisor write, page not present
> instruction pointer = 0x20:0xc050d011
> stack pointer = 0x28:0xcc76fa6c
> frame pointer = 0x28:0xcc76fa78
> code segment = base 0x0, limit 0xfffff, type 0x1b
> = DPL 0, pres 1, def32 1, gran 1
> processor eflags = interrupt enabled, resume, IOPL = 0
> current process = 302 (ripd)
> trap number = 12
> panic: page fault
> Uptime: 1h18m47s
> Dumping 254 MB (2 chunks)
> chunk 0: 1MB (159 pages) ... ok
> chunk 1: 254MB (64960 pages) 238 222 206 190 174 158 142 126 110 94 78 
> 62 46 30 14
>
> #0 doadump () at pcpu.h:165
> 165 __asm __volatile("movl %%fs:0,%0" : "=r" (td));
> (kgdb) bktr
> Undefined command: "bktr". Try "help".
> (kgdb) backtrace
> #0 doadump () at pcpu.h:165
> During symbol reading, Incomplete CFI data; unspecified registers at 
> 0xc04d87b5.
> #1 0xc04d8c96 in boot (howto=0x104) at 
> /usr/src/sys/kern/kern_shutdown.c:409
> #2 0xc04d8f2c in panic (fmt=0xc06496b4 "%s") at 
> /usr/src/sys/kern/kern_shutdown.c:565
> #3 0xc062a874 in trap_fatal (frame=0xcc76fa2c, eva=0xffffff80) at 
> /usr/src/sys/i386/i386/trap.c:837
> #4 0xc062a5db in trap_pfault (frame=0xcc76fa2c, usermode=0x0, 
> eva=0xffffff80) at /usr/src/sys/i386/i386/trap.c:745
> #5 0xc062a219 in trap (frame=
> {tf_fs = 0xc04e0008, tf_es = 0xc1da0028, tf_ds = 0xc2420028, tf_edi = 
> 0xc1e7296c, tf_esi = 0xc1d9c438, tf_ebp = 0xcc76fa78, tf_isp = 
> 0xcc76fa58, tf_ebx = 0xc22ec900, tf_edx = 0xc22ec900, tf_ecx = 
> 0xffffff80, tf_eax = 0xc239c800, tf_trapno = 0xc, tf_err = 0x2, tf_eip 
> = 0xc050d011, tf_cs = 0x20, tf_eflags = 0x10202, tf_esp = 0xc1d9c438, 
> tf_ss = 0xc1e728f6}) at /usr/src/sys/i386/i386/trap.c:435
> #6 0xc06188ea in calltrap () at /usr/src/sys/i386/i386/exception.s:139
> #7 0xc050d011 in putc (chr=0x20, clistp=0xc1d9c438) at 
> /usr/src/sys/kern/tty_subr.c:399
> #8 0xc055233b in pppasyncstart (sc=0xc24e5200) at 
> /usr/src/sys/net/ppp_tty.c:601
> #9 0xc054bf2e in pppoutput (ifp=0xc1ed0000, m0=0xc245d600, 
> dst=0xcc76fb18, rtp=0x0) at /usr/src/sys/net/if_ppp.c:961
> #10 0xc0564494 in ip_output (m=0xc245d600, opt=0xc1ed0000, 
> ro=0xcc76fb14, flags=0x20, imo=0xc239d680, inp=0xc1fef924)
> at /usr/src/sys/netinet/ip_output.c:777
> #11 0xc0574e07 in udp_output (inp=0xc1fef924, m=0xc245d600, 
> addr=0xc23a43c0, control=0x20, td=0xc1e36d80)
> at /usr/src/sys/netinet/udp_usrreq.c:913
> #12 0xc05757ae in udp_send (so=0xc239c800, flags=0x0, m=0xc2425b00, 
> addr=0xc23a43c0, control=0x0, td=0xc1e36d80)
> at /usr/src/sys/netinet/udp_usrreq.c:1090
> #13 0xc0511d8b in sosend (so=0xc23b29bc, addr=0xc23a43c0, 
> uio=0xcc76fc40, top=0xc2425b00, control=0x0, flags=0x0,
> td=0xc1e36d80) at /usr/src/sys/kern/uipc_socket.c:836
> #14 0xc0517729 in kern_sendit (td=0xc1e36d80, s=0x9, mp=0xcc76fcbc, 
> flags=0x0, control=0x0, segflg=3258566656)
> at /usr/src/sys/kern/uipc_syscalls.c:772
> #15 0xc05175e3 in sendit (td=0xc1e36d80, s=0x9, mp=0xcc76fcbc, 
> flags=0x0) at /usr/src/sys/kern/uipc_syscalls.c:712
> #16 0xc05178d1 in sendto (td=0xc1e36d80, uap=0xc22ec900) at 
> /usr/src/sys/kern/uipc_syscalls.c:830
> #17 0xc062ab8b in syscall (frame=
> {tf_fs = 0x3b, tf_es = 0x3b, tf_ds = 0xbfbf003b, tf_edi = 0x9, tf_esi 
> = 0xbfbfeb60, tf_ebp = 0xbfbfeb88, tf_isp = 0xcc76fd64, tf_ebx = 
> 0x80a9a20, tf_edx = 0xc000000, tf_ecx = 0xc, tf_eax = 0x85, tf_trapno 
> = 0x0, tf_err = 0x2, tf_eip = 0x281a8f43, tf_cs = 0x33, tf_eflags = 
> 0x296, tf_esp = 0xbfbfeafc, tf_ss = 0x3b}) at 
> /usr/src/sys/i386/i386/trap.c:983
> #18 0xc061893f in Xint0x80_syscall () at 
> /usr/src/sys/i386/i386/exception.s:200
> #19 0x00000033 in ?? ()
> Previous frame inner to this frame (corrupt stack?)
> (kgdb)
> #0 doadump () at pcpu.h:165
> #1 0xc04d8c96 in boot (howto=0x104) at 
> /usr/src/sys/kern/kern_shutdown.c:409
> #2 0xc04d8f2c in panic (fmt=0xc06496b4 "%s") at 
> /usr/src/sys/kern/kern_shutdown.c:565
> #3 0xc062a874 in trap_fatal (frame=0xcc76fa2c, eva=0xffffff80) at 
> /usr/src/sys/i386/i386/trap.c:837
> #4 0xc062a5db in trap_pfault (frame=0xcc76fa2c, usermode=0x0, 
> eva=0xffffff80) at /usr/src/sys/i386/i386/trap.c:745
> #5 0xc062a219 in trap (frame=
> {tf_fs = 0xc04e0008, tf_es = 0xc1da0028, tf_ds = 0xc2420028, tf_edi = 
> 0xc1e7296c, tf_esi = 0xc1d9c438, tf_ebp = 0xcc76fa78, tf_isp = 
> 0xcc76fa58, tf_ebx = 0xc22ec900, tf_edx = 0xc22ec900, tf_ecx = 
> 0xffffff80, tf_eax = 0xc239c800, tf_trapno = 0xc, tf_err = 0x2, tf_eip 
> = 0xc050d011, tf_cs = 0x20, tf_eflags = 0x10202, tf_esp = 0xc1d9c438, 
> tf_ss = 0xc1e728f6}) at /usr/src/sys/i386/i386/trap.c:435
> #6 0xc06188ea in calltrap () at /usr/src/sys/i386/i386/exception.s:139
> #7 0xc050d011 in putc (chr=0x20, clistp=0xc1d9c438) at 
> /usr/src/sys/kern/tty_subr.c:399
> #8 0xc055233b in pppasyncstart (sc=0xc24e5200) at 
> /usr/src/sys/net/ppp_tty.c:601
> #9 0xc054bf2e in pppoutput (ifp=0xc1ed0000, m0=0xc245d600, 
> dst=0xcc76fb18, rtp=0x0) at /usr/src/sys/net/if_ppp.c:961
> #10 0xc0564494 in ip_output (m=0xc245d600, opt=0xc1ed0000, 
> ro=0xcc76fb14, flags=0x20, imo=0xc239d680, inp=0xc1fef924)
> at /usr/src/sys/netinet/ip_output.c:777
> #11 0xc0574e07 in udp_output (inp=0xc1fef924, m=0xc245d600, 
> addr=0xc23a43c0, control=0x20, td=0xc1e36d80)
> at /usr/src/sys/netinet/udp_usrreq.c:913
> #12 0xc05757ae in udp_send (so=0xc239c800, flags=0x0, m=0xc2425b00, 
> addr=0xc23a43c0, control=0x0, td=0xc1e36d80)
> at /usr/src/sys/netinet/udp_usrreq.c:1090
> #13 0xc0511d8b in sosend (so=0xc23b29bc, addr=0xc23a43c0, 
> uio=0xcc76fc40, top=0xc2425b00, control=0x0, flags=0x0,
> td=0xc1e36d80) at /usr/src/sys/kern/uipc_socket.c:836
> #14 0xc0517729 in kern_sendit (td=0xc1e36d80, s=0x9, mp=0xcc76fcbc, 
> flags=0x0, control=0x0, segflg=3258566656)
> at /usr/src/sys/kern/uipc_syscalls.c:772
> #15 0xc05175e3 in sendit (td=0xc1e36d80, s=0x9, mp=0xcc76fcbc, 
> flags=0x0) at /usr/src/sys/kern/uipc_syscalls.c:712
> #16 0xc05178d1 in sendto (td=0xc1e36d80, uap=0xc22ec900) at 
> /usr/src/sys/kern/uipc_syscalls.c:830
> #17 0xc062ab8b in syscall (frame=
> {tf_fs = 0x3b, tf_es = 0x3b, tf_ds = 0xbfbf003b, tf_edi = 0x9, tf_esi 
> = 0xbfbfeb60, tf_ebp = 0xbfbfeb88, tf_isp = 0xcc76fd64, tf_ebx = 
> 0x80a9a20, tf_edx = 0xc000000, tf_ecx = 0xc, tf_eax = 0x85, tf_trapno 
> = 0x0, tf_err = 0x2, tf_eip = 0x281a8f43, tf_cs = 0x33, tf_eflags = 
> 0x296, tf_esp = 0xbfbfeafc, tf_ss = 0x3b}) at 
> /usr/src/sys/i386/i386/trap.c:983
> #18 0xc061893f in Xint0x80_syscall () at 
> /usr/src/sys/i386/i386/exception.s:200
> #19 0x00000033 in ?? ()
> (kgdb)
And here is bt full:
Unread portion of the kernel message buffer:


Fatal trap 12: page fault while in kernel mode
fault virtual address = 0xffffff80
fault code = supervisor write, page not present
instruction pointer = 0x20:0xc050d011
stack pointer = 0x28:0xcc76fa6c
frame pointer = 0x28:0xcc76fa78
code segment = base 0x0, limit 0xfffff, type 0x1b
= DPL 0, pres 1, def32 1, gran 1
processor eflags = interrupt enabled, resume, IOPL = 0
current process = 302 (ripd)
trap number = 12
panic: page fault
Uptime: 1h18m47s
Dumping 254 MB (2 chunks)
chunk 0: 1MB (159 pages) ... ok
chunk 1: 254MB (64960 pages) 238 222 206 190 174 158 142 126 110 94 78 
62 46 30 14

#0 doadump () at pcpu.h:165
165 __asm __volatile("movl %%fs:0,%0" : "=r" (td));
(kgdb)
(kgdb)
(kgdb) bt full
#0 doadump () at pcpu.h:165
No locals.
During symbol reading, Incomplete CFI data; unspecified registers at 
0xc04d87b5.
#1 0xc04d8c96 in boot (howto=0x104) at /usr/src/sys/kern/kern_shutdown.c:409
first_buf_printf = 0x1
#2 0xc04d8f2c in panic (fmt=0xc06496b4 "%s") at 
/usr/src/sys/kern/kern_shutdown.c:565
td = (struct thread *) 0xc1e36d80
bootopt = 0x104
newpanic = 0x0
ap = 0xc1e36d80 "`h\002??M??"
buf = "page fault", '\0' <repeats 245 times>
#3 0xc062a874 in trap_fatal (frame=0xcc76fa2c, eva=0xffffff80) at 
/usr/src/sys/i386/i386/trap.c:837
code = 0x28
type = 0xc
ss = 0x28
esp = 0x0
softseg = {
ssd_base = 0x0,
ssd_limit = 0xfffff,
ssd_type = 0x1b,
ssd_dpl = 0x0,
ssd_p = 0x1,
ssd_xx = 0x8,
ssd_xx1 = 0x2,
ssd_def32 = 0x1,
ssd_gran = 0x1
}
msg = 0x0
#4 0xc062a5db in trap_pfault (frame=0xcc76fa2c, usermode=0x0, 
eva=0xffffff80) at /usr/src/sys/i386/i386/trap.c:745
va = 0xfffff000
vm = (struct vmspace *) 0x0
map = 0xc0c4b000
rv = 0x1
ftype = 0x1
td = (struct thread *) 0xc1e36d80
p = (struct proc *) 0xc2026860
#5 0xc062a219 in trap (frame=
{tf_fs = 0xc04e0008, tf_es = 0xc1da0028, tf_ds = 0xc2420028, tf_edi = 
0xc1e7296c, tf_esi = 0xc1d9c438, tf_ebp = 0xcc76fa78, tf_isp = 
0xcc76fa58, tf_ebx = 0xc22ec900, tf_edx = 0xc22ec900, tf_ecx = 
0xffffff80, tf_eax = 0xc239c800, tf_trapno = 0xc, tf_err = 0x2, tf_eip = 
0xc050d011, tf_cs = 0x20, tf_eflags = 0x10202, tf_esp = 0xc1d9c438, 
tf_ss = 0xc1e728f6}) at /usr/src/sys/i386/i386/trap.c:435
td = (struct thread *) 0xc1e36d80
p = (struct proc *) 0xc2026860
sticks = 0xcc76fa28
i = 0x0
ucode = 0x0
type = 0xc
code = 0x2
eva = 0xffffff80
#6 0xc06188ea in calltrap () at /usr/src/sys/i386/i386/exception.s:139
No locals.
#7 0xc050d011 in putc (chr=0x20, clistp=0xc1d9c438) at 
/usr/src/sys/kern/tty_subr.c:399
prev = (struct cblock *) 0xffffff80
cblockp = (struct cblock *) 0xc22ec900
#8 0xc055233b in pppasyncstart (sc=0xc24e5200) at 
/usr/src/sys/net/ppp_tty.c:601
tp = (struct tty *) 0xc1d9c400
m = (struct mbuf *) 0xc2425b00
len = 0x76
start = (u_char *) 0xc1e728f6 ""
stop = (
u_char *) 0xc1e7296c "LD1AAAAAAI", 'A' <repeats 50 times>, 
"ICIAABnjAAAAEAAAAAARQGwAcA; 
Mpop=1174646000:4f446b065e786a5519050219091d011b030d0b4f6a5d5e465e000d011b03757b1f5c5e4d5053455f5c56145a54585819"...
cp = (u_char *) 0xc22ec900 ""
n = 0xc1d9c438
ndone = 0xc239c800
done = 0x1
idle = 0x0
#9 0xc054bf2e in pppoutput (ifp=0xc1ed0000, m0=0xc245d600, 
dst=0xcc76fb18, rtp=0x0) at /usr/src/sys/net/if_ppp.c:961
sc = (struct ppp_softc *) 0xc24e5200
protocol = 0x21
---Type <return> to continue, or q <return> to quit---
address = 0xff
control = 0x3
cp = (u_char *) 0xc239c800 ""
error = 0xc1ed00f8
ip = (struct ip *) 0xc239c800
ifq = (struct ifqueue *) 0xc1ed00f8
mode = NPMODE_PASS
len = 0x18c
#10 0xc0564494 in ip_output (m=0xc245d600, opt=0xc1ed0000, 
ro=0xcc76fb14, flags=0x20, imo=0xc239d680, inp=0xc1fef924)
at /usr/src/sys/netinet/ip_output.c:777
ip = (struct ip *) 0xc245d6e4
ifp = (struct ifnet *) 0xc1ed0000
m0 = (struct mbuf *) 0xc245d6e4
hlen = 0x14
len = 0x2c
error = 0x0
dst = (struct sockaddr_in *) 0xcc76fb18
ia = (struct in_ifaddr *) 0xc23a7200
isbroadcast = 0xffffff80
sw_csum = 0x1
iproute = {
ro_rt = 0x0,
ro_dst = {
sa_len = 0x10,
sa_family = 0x2,
sa_data = "\000\000?\000\000\t\000\000\000\000\000\000\000"
}
}
odst = {
s_addr = 0x1
}
#11 0xc0574e07 in udp_output (inp=0xc1fef924, m=0xc245d600, 
addr=0xc23a43c0, control=0x20, td=0xc1e36d80)
at /usr/src/sys/netinet/udp_usrreq.c:913
ui = (struct udpiphdr *) 0xc245d6e4
len = 0x16c
faddr = {
s_addr = 0x90000e0
}
laddr = {
s_addr = 0x81c8a8c0
}
cm = (struct cmsghdr *) 0xc245d6e4
src = {
sin_len = 0x40,
sin_family = 0x6b,
sin_port = 0xc0c5,
sin_addr = {
s_addr = 0x0
},
sin_zero = "$???$???"
}
error = 0x37
ipflags = 0x20
fport = 0x802
lport = 0x802
unlock_udbinfo = 0x1
#12 0xc05757ae in udp_send (so=0xc239c800, flags=0x0, m=0xc2425b00, 
addr=0xc23a43c0, control=0x0, td=0xc1e36d80)
at /usr/src/sys/netinet/udp_usrreq.c:1090
No locals.
#13 0xc0511d8b in sosend (so=0xc23b29bc, addr=0xc23a43c0, 
uio=0xcc76fc40, top=0xc2425b00, control=0x0, flags=0x0,
td=0xc1e36d80) at /usr/src/sys/kern/uipc_socket.c:836
mp = (struct mbuf **) 0xc2425b00
m = (struct mbuf *) 0xc2425b00
space = 0x2294
len = 0x16c
resid = 0x0
clen = 0x16c
error = 0x0
dontroute = 0x0
---Type <return> to continue, or q <return> to quit---
atomic = 0x1
#14 0xc0517729 in kern_sendit (td=0xc1e36d80, s=0x9, mp=0xcc76fcbc, 
flags=0x0, control=0x0, segflg=3258566656)
at /usr/src/sys/kern/uipc_syscalls.c:772
fp = (struct file *) 0xc21ad1f8
auio = {
uio_iov = 0xcc76fcb4,
uio_iovcnt = 0x1,
uio_offset = 0x16c,
uio_resid = 0x0,
uio_segflg = UIO_USERSPACE,
uio_rw = UIO_WRITE,
uio_td = 0xc1e36d80
}
iov = (struct iovec *) 0xc22ec900
so = (struct socket *) 0xc23b29bc
i = 0xffffff80
len = 0x16c
error = 0x0
ktruio = (struct uio *) 0x0
#15 0xc05175e3 in sendit (td=0xc1e36d80, s=0x9, mp=0xcc76fcbc, 
flags=0x0) at /usr/src/sys/kern/uipc_syscalls.c:712
control = (struct mbuf *) 0x0
to = (struct sockaddr *) 0xc23a43c0
error = 0x0
#16 0xc05178d1 in sendto (td=0xc1e36d80, uap=0xc22ec900) at 
/usr/src/sys/kern/uipc_syscalls.c:830
msg = {
msg_name = 0xc23a43c0,
msg_namelen = 0x10,
msg_iov = 0xcc76fcb4,
msg_iovlen = 0x1,
msg_control = 0x0,
msg_controllen = 0x0,
msg_flags = 0x0
}
aiov = {
iov_base = 0x806596c,
iov_len = 0x0
}
error = 0xc239c800
#17 0xc062ab8b in syscall (frame=
{tf_fs = 0x3b, tf_es = 0x3b, tf_ds = 0xbfbf003b, tf_edi = 0x9, tf_esi = 
0xbfbfeb60, tf_ebp = 0xbfbfeb88, tf_isp = 0xcc76fd64, tf_ebx = 
0x80a9a20, tf_edx = 0xc000000, tf_ecx = 0xc, tf_eax = 0x85, tf_trapno = 
0x0, tf_err = 0x2, tf_eip = 0x281a8f43, tf_cs = 0x33, tf_eflags = 0x296, 
tf_esp = 0xbfbfeafc, tf_ss = 0x3b}) at /usr/src/sys/i386/i386/trap.c:983
params = 0xbfbfeb00 <Address 0xbfbfeb00 out of bounds>
callp = (struct sysent *) 0xc067409c
td = (struct thread *) 0xc1e36d80
p = (struct proc *) 0xc2026860
orig_tf_eflags = 0x296
sticks = 0x16
error = 0x0
narg = 0x6
args = {0x9, 0x8065800, 0x16c, 0x0, 0xbfbfeb60, 0x10, 0xcc76fd34, 
0x280d43b4}
code = 0x85
#18 0xc061893f in Xint0x80_syscall () at 
/usr/src/sys/i386/i386/exception.s:200
No locals.
#19 0x00000033 in ?? ()
No symbol table info available.
Previous frame inner to this frame (corrupt stack?)
(kgdb)


--
With best regards,
    Gregory Edigarov