crash at in_pcb.c

Wed Oct 29 15:56:44 PDT 2008

2008/10/29 Jerry Toung <jrytoung at gmail.com>:
> Hello List,
> I can realiably reproduce this crash. We have a deamon that accept several
> connections
> per sec. We use iperf and Microsoft Web application stress 1.0 to push
> traffic to the FreeBSD box.
> Without further delay, the crash dump is below. I've been troubleshooting,
> but I am no longer sure
> if this is a race condition or a stack corruption. The socket pointer
> between frame 12 and 11 is different.
> This is on 6.2, but the code for 7.0 is identical, so I think it still
> applies.
>
> Any hint, patching or troubleshooting this is appreciated.
>
> Unread portion of the kernel message buffer:
>
>
> Fatal trap 12: page fault while in kernel mode
> cpuid = 0; apic id = 00
> fault virtual address   = 0x2aef0210
> fault code              = supervisor read, page not present
> instruction pointer     = 0x20:0xc0769098
> stack pointer           = 0x28:0xef781bc0
> frame pointer           = 0x28:0xef781bd0
> code segment            = base 0x0, limit 0xfffff, type 0x1b
>                        = DPL 0, pres 1, def32 1, gran 1
> processor eflags        = interrupt enabled, resume, IOPL = 0
> current process         = 1166 (ndaemon)
> trap number             = 12
> panic: page fault
> cpuid = 0
> Uptime: 8h32m25s
> Dumping 3325 MB (3 chunks)
> #0  doadump () at pcpu.h:165
> 165     pcpu.h: No such file or directory.
>        in pcpu.h
> (kgdb) l *0xc0769098
> 0xc0769098 is in in_pcblookup_local (/usr/src/sys/netinet/in_pcb.c:923).
> 918     /usr/src/sys/netinet/in_pcb.c: No such file or directory.
>        in /usr/src/sys/netinet/in_pcb.c
> (kgdb) bt
> #0  doadump () at pcpu.h:165
> #1  0xc06c2812 in boot (howto=260) at /usr/src/sys/kern/kern_shutdown.c:412
> #2  0xc06c2bbd in panic (fmt=0xc0940872 "%s") at
> /usr/src/sys/kern/kern_shutdown.c:573
> #3  0xc08f3e4e in trap_fatal (frame=0xef781b80, eva=720306704) at
> /usr/src/sys/i386/i386/trap.c:838
> #4  0xc08f3b57 in trap_pfault (frame=0xef781b80, usermode=0, eva=720306704)
> at /usr/src/sys/i386/i386/trap.c:745
> #5  0xc08f3745 in trap (frame=
>      {tf_fs = -277348344, tf_es = 40, tf_ds = -913309656, tf_edi = 6,
> tf_esi = 0, tf_ebp = -277341232, tf_isp = -277341268, tf_ebx = -1062683820,
> tf_edx = 720306704, tf_ecx = 14063, tf_eax = 720306704, tf_trapno = 12,
> tf_err = 0, tf_eip = -1065971560, tf_cs = 32, tf_eflags = 66050, tf_esp = 0,
> tf_ss = -1062683820}) at /usr/src/sys/i386/i386/trap.c:435
> #6  0xc08dddba in calltrap () at /usr/src/sys/i386/i386/exception.s:139
> #7  0xc0769098 in in_pcblookup_local (pcbinfo=0x2aef0210, laddr={s_addr =
> 0}, lport_arg=720306704, wild_okay=1)
>    at /usr/src/sys/netinet/in_pcb.c:923
> #8  0xc0768452 in in_pcbbind_setup (inp=0xc97150b4, nam=0x36ef,
> laddrp=0xc97150ec, lportp=0xc97150ce, cred=0xc8726780)
>    at /usr/src/sys/netinet/in_pcb.c:464
> #9  0xc0767f56 in in_pcbbind (inp=0xc97150b4, nam=0x2aef0210,
> cred=0xc8726780) at /usr/src/sys/netinet/in_pcb.c:240
> #10 0xc077f272 in tcp_connect (tp=0xc9897000, nam=0xc98a1ba0, td=0xc990e180)
> at /usr/src/sys/netinet/tcp_usrreq.c:864
> #11 0xc077e141 in tcp_usr_connect (so=0xc9897000, nam=0xc98a1ba0,
> td=0xc990e180)
>    at /usr/src/sys/netinet/tcp_usrreq.c:369
> #12 0xc06fec4e in soconnect (so=0xc97b39bc, nam=0xc98a1ba0, td=0xc990e180)
> at /usr/src/sys/kern/uipc_socket.c:558
> #13 0xc07046a8 in kern_connect (td=0xc990e180, fd=89, sa=0xc98a1ba0) at
> /usr/src/sys/kern/uipc_syscalls.c:536
> #14 0xc070460f in connect (td=0xc990e180, uap=0xef781d04) at
> /usr/src/sys/kern/uipc_syscalls.c:505
> #15 0xc08f4193 in syscall (frame=
>      {tf_fs = 135725115, tf_es = 59, tf_ds = -1088487365, tf_edi =
> 135745024, tf_esi = -1089511444, tf_ebp = -1089514536, tf_isp = -277340828,
> tf_ebx = 671753396, tf_edx = 0, tf_ecx = 135524256, tf_eax = 98, tf_trapno =
> 0, tf_err = 2, tf_eip = 674451435, tf_cs = 51, tf_eflags = 642, tf_esp =
> -1089514580, tf_ss = 59}) at /usr/src/sys/i386/i386/trap.c:984
> #16 0xc08dde0f in Xint0x80_syscall () at
> /usr/src/sys/i386/i386/exception.s:200
> #17 0x00000033 in ?? ()
> Previous frame inner to this frame (corrupt stack?)
> (kgdb) f 7
> #7  0xc0769098 in in_pcblookup_local (pcbinfo=0x2aef0210, laddr={s_addr =
> 0}, lport_arg=720306704, wild_okay=1)
>    at /usr/src/sys/netinet/in_pcb.c:923
> 923     in /usr/src/sys/netinet/in_pcb.c
> (kgdb) i loc
> phd = (struct inpcbport *) 0x2aef0210
> tmphd = (struct inpcbport *) 0x2aef0210
> match = (struct inpcb *) 0x0
> inp = (struct inpcb *) 0x2aef0210
> tmpinp = (struct inpcb *) 0x2aef0210
> matchwild = 6
> wildcard = -1062683820
> lport = 14063
> (kgdb) p phd
> $1 = (struct inpcbport *) 0x2aef0210
> (kgdb) p phd->phd_port
> Cannot access memory at address 0x2aef021c
>
> (kgdb) f 12
> #12 0xc06fec4e in soconnect (so=0xc97b39bc, nam=0xc98a1ba0, td=0xc990e180)
> at /usr/src/sys/kern/uipc_socket.c:558
> 558     /usr/src/sys/kern/uipc_socket.c: No such file or directory.
>        in /usr/src/sys/kern/uipc_socket.c
> (kgdb) p so
> $2 = (struct socket *) 0xc97b39bc
> (kgdb) p nam
> $3 = (struct sockaddr *) 0xc98a1ba0
> (kgdb) p td
> $4 = (struct thread *) 0xc990e180
> (kgdb) l
> 553     in /usr/src/sys/kern/uipc_socket.c
> (kgdb) f 11
> #11 0xc077e141 in tcp_usr_connect (so=0xc9897000, nam=0xc98a1ba0,
> td=0xc990e180)
>    at /usr/src/sys/netinet/tcp_usrreq.c:369
> 369     /usr/src/sys/netinet/tcp_usrreq.c: No such file or directory.
>        in /usr/src/sys/netinet/tcp_usrreq.c
> (kgdb)

Could you please get the following from kgdb?
f 7
p *inp
p *inp->inp_laddr

P.S. It's definitely 7.0 backtrace (or close to).. 6.2 has different
line numbers.

-- 
wbr,
pluknet