FreeBSD 7.2-BETA1 tcp retransmit crash

Blapp, Martin Martin.Blapp at t-systems.ch
Sun Apr 5 16:28:33 UTC 2009 

Hi all,

Looks like the same problem as PR 129197 (FreeBSD 7 panic)

http://www.freebsd.org/cgi/query-pr.cgi?pr=129197

OS:     FreeBSD 7.2 BETA1
PF:      Enabled
SACK: net.inet.tcp.sack.enable: 1

Happens after some/many soabort calls ...  I can reproduce it
after 3-4 hours running time. Currently I'm testing a workaround
but I guess the underlying problem should be fixed.

--
Martin

Fatal trap 12: page fault while in kernel mode
cpuid = 1; apic id = 01
fault virtual address   = 0xc
fault code              = supervisor read, page not present
instruction pointer     = 0x20:0xc07c6cb0
stack pointer           = 0x28:0xc2f9c97c
frame pointer           = 0x28:0xc2f9c984
code segment            = base 0x0, limit 0xfffff, type 0x1b
                        = DPL 0, pres 1, def32 1, gran 1
processor eflags        = interrupt enabled, resume, IOPL = 0
current process         = 25 (em0 taskq)
trap number             = 12
panic: page fault
cpuid = 1
Uptime: 4h12m47s
Physical memory: 499 MB
Dumping 104 MB: 89 73 57 41 25 9

Reading symbols from /boot/kernel/acpi.ko...Reading symbols from /boot/kernel/acpi.ko.symbols...done.
done.
Loaded symbols for /boot/kernel/acpi.ko
Reading symbols from /usr/local/lib/vmware-tools/modules/drivers/vmmemctl.ko...done.
Loaded symbols for /usr/local/lib/vmware-tools/modules/drivers/vmmemctl.ko
Reading symbols from /usr/local/lib/vmware-tools/modules/drivers/vmxnet.ko...done.
Loaded symbols for /usr/local/lib/vmware-tools/modules/drivers/vmxnet.ko
Reading symbols from /usr/local/lib/vmware-tools/modules/drivers/vmblock.ko...done.
Loaded symbols for /usr/local/lib/vmware-tools/modules/drivers/vmblock.ko
Reading symbols from /usr/local/lib/vmware-tools/modules/drivers/vmhgfs.ko...done.
Loaded symbols for /usr/local/lib/vmware-tools/modules/drivers/vmhgfs.ko
Reading symbols from /boot/kernel/pf.ko...Reading symbols from /boot/kernel/pf.ko.symbols...done.
done.
Loaded symbols for /boot/kernel/pf.ko
Reading symbols from /boot/kernel/linux.ko...Reading symbols from /boot/kernel/linux.ko.symbols...done.
done.
Loaded symbols for /boot/kernel/linux.ko
Reading symbols from /boot/modules/accf_smtp.ko...done.
Loaded symbols for /boot/modules/accf_smtp.ko

#0  doadump () at pcpu.h:196
#1  0xc0772d87 in boot (howto=260) at /usr/src/sys/kern/kern_shutdown.c:418
#2  0xc0773059 in panic (fmt=Variable "fmt" is not available.
) at /usr/src/sys/kern/kern_shutdown.c:574
#3  0xc0a5062c in trap_fatal (frame=0xc2f9c93c, eva=12) at /usr/src/sys/i386/i386/trap.c:939
#4  0xc0a508b0 in trap_pfault (frame=0xc2f9c93c, usermode=0, eva=12) at /usr/src/sys/i386/i386/trap.c:852
#5  0xc0a5125c in trap (frame=0xc2f9c93c) at /usr/src/sys/i386/i386/trap.c:530
#6  0xc0a3593b in calltrap () at /usr/src/sys/i386/i386/exception.s:159
#7  0xc07c6cb0 in sbsndptr (sb=0xc342ede4, off=112, len=113, moff=0xc2f9ca04) at /usr/src/sys/kern/uipc_sockbuf.c:939
#8  0xc089cd64 in tcp_output (tp=0xc43311d0) at /usr/src/sys/netinet/tcp_output.c:798
#9  0xc089974a in tcp_do_segment (m=0xc34a6600, th=0xc34c8024, so=0xc342ed00, tp=0xc43311d0, drop_hdrlen=52, tlen=0)
    at /usr/src/sys/netinet/tcp_input.c:1835
#10 0xc089b2ee in tcp_input (m=0xc34a6600, off0=20) at /usr/src/sys/netinet/tcp_input.c:846
#11 0xc08340a0 in ip_input (m=0xc34a6600) at /usr/src/sys/netinet/ip_input.c:664
#12 0xc081ae15 in netisr_dispatch (num=2, m=0xc34a6600) at /usr/src/sys/net/netisr.c:185
#13 0xc0810d81 in ether_demux (ifp=0xc31bb400, m=0xc34a6600) at /usr/src/sys/net/if_ethersubr.c:834
#14 0xc0811173 in ether_input (ifp=0xc31bb400, m=0xc34a6600) at /usr/src/sys/net/if_ethersubr.c:692
#15 0xc0561f2a in em_rxeof (adapter=0xc31bc000, count=99) at /usr/src/sys/dev/e1000/if_em.c:4539
#16 0xc0562a57 in em_handle_rxtx (context=0xc31bc000, pending=1) at /usr/src/sys/dev/e1000/if_em.c:1702
#17 0xc07a8015 in taskqueue_run (queue=0xc3181780) at /usr/src/sys/kern/subr_taskqueue.c:282
#18 0xc07a8228 in taskqueue_thread_loop (arg=0xc31c035c) at /usr/src/sys/kern/subr_taskqueue.c:401
#19 0xc074d839 in fork_exit (callout=0xc07a8160 <taskqueue_thread_loop>, arg=0xc31c035c, frame=0xc2f9cd38) at /usr/src/sys/kern/kern_fork.c:810
#20 0xc0a359b0 in fork_trampoline () at /usr/src/sys/i386/i386/exception.s:264

(kgdb) frame 7
#7  0xc07c6cb0 in sbsndptr (sb=0xc342ede4, off=112, len=113, moff=0xc2f9ca04) at /usr/src/sys/kern/uipc_sockbuf.c:939
939                  off > 0 && off >= m->m_len;
(kgdb) list
934             *moff = off - sb->sb_sndptroff;
935             m = ret = sb->sb_sndptr ? sb->sb_sndptr : sb->sb_mb;
936
937             /* Advance by len to be as close as possible for the next transmit. */
938             for (off = off - sb->sb_sndptroff + len - 1;
939                  off > 0 && off >= m->m_len;
940                  m = m->m_next) {
941                     sb->sb_sndptroff += m->m_len;
942                     off -= m->m_len;
943             }

(kgdb) p sb->sb_sndptr
$1 = (struct mbuf *) 0x0

(kgdb) p sb->sb_mb
$2 = (struct mbuf *) 0x0

Kein Wunder gibts da nen Crash ...

(kgdb) p *sb
$8 = {sb_sel = {si_thrlist = {tqe_next = 0x0, tqe_prev = 0x0}, si_thread = 0x0, si_note = {kl_list = {slh_first = 0x0},
      kl_lock = 0xc0747700 <knlist_mtx_lock>, kl_unlock = 0xc07470e0 <knlist_mtx_unlock>, kl_locked = 0xc07470c0 <knlist_mtx_locked>,
      kl_lockarg = 0xc342ee08}, si_flags = 0}, sb_mtx = {lock_object = {lo_name = 0xc0ad4ad8 "so_snd", lo_type = 0xc0ad4ad8 "so_snd", lo_flags = 16973824,
      lo_witness_data = {lod_list = {stqe_next = 0x0}, lod_witness = 0x0}}, mtx_lock = 3272403696, mtx_recurse = 0}, sb_sx = {lock_object = {
      lo_name = 0xc0ad4ae6 "so_snd_sx", lo_type = 0xc0ad4ae6 "so_snd_sx", lo_flags = 37421056, lo_witness_data = {lod_list = {stqe_next = 0x0},
        lod_witness = 0x0}}, sx_lock = 1, sx_recurse = 0}, sb_state = 16, sb_mb = 0x0, sb_mbtail = 0x0, sb_lastrecord = 0x0, sb_sndptr = 0x0,
  sb_sndptroff = 0, sb_cc = 0, sb_hiwat = 33580, sb_mbcnt = 0, sb_mcnt = 0, sb_ccnt = 0, sb_mbmax = 262144, sb_ctl = 0, sb_lowat = 2048, sb_timeo = 0,
  sb_flags = 2048}

(kgdb) f 8
#8  0xc089cd64 in tcp_output (tp=0xc43311d0) at /usr/src/sys/netinet/tcp_output.c:798
798                     mb = sbsndptr(&so->so_snd, off, len, &moff);

p *so
$9 = {so_count = 0, so_type = 1, so_options = 12, so_linger = 0, so_state = 24633, so_qstate = 2048, so_pcb = 0xc40e0708, so_proto = 0xc0b994a8,
  so_head = 0xc4056b60, so_incomp = {tqh_first = 0x0, tqh_last = 0x0}, so_comp = {tqh_first = 0x0, tqh_last = 0x0}, so_list = {tqe_next = 0x0,
    tqe_prev = 0xc445b02c}, so_qlen = 0, so_incqlen = 0, so_qlimit = 0, so_timeo = 0, so_error = 0, so_sigio = 0x0, so_oobmark = 0, so_aiojobq = {
    tqh_first = 0x0, tqh_last = 0xc342ed48}, so_rcv = {sb_sel = {si_thrlist = {tqe_next = 0x0, tqe_prev = 0x0}, si_thread = 0x0, si_note = {kl_list = {
          slh_first = 0x0}, kl_lock = 0xc0747700 <knlist_mtx_lock>, kl_unlock = 0xc07470e0 <knlist_mtx_unlock>, kl_locked = 0xc07470c0 <knlist_mtx_locked>,
        kl_lockarg = 0xc342ed74}, si_flags = 0}, sb_mtx = {lock_object = {lo_name = 0xc0ad4adf "so_rcv", lo_type = 0xc0ad4adf "so_rcv",
        lo_flags = 16973824, lo_witness_data = {lod_list = {stqe_next = 0x0}, lod_witness = 0x0}}, mtx_lock = 4, mtx_recurse = 0}, sb_sx = {lock_object = {
        lo_name = 0xc0ad4af0 "so_rcv_sx", lo_type = 0xc0ad4af0 "so_rcv_sx", lo_flags = 37421056, lo_witness_data = {lod_list = {stqe_next = 0x0},
          lod_witness = 0x0}}, sx_lock = 1, sx_recurse = 0}, sb_state = 32, sb_mb = 0x0, sb_mbtail = 0x0, sb_lastrecord = 0x0, sb_sndptr = 0x0,
    sb_sndptroff = 0, sb_cc = 0, sb_hiwat = 65700, sb_mbcnt = 0, sb_mcnt = 0, sb_ccnt = 0, sb_mbmax = 262144, sb_ctl = 0, sb_lowat = 1, sb_timeo = 0,
    sb_flags = 2048}, so_snd = {sb_sel = {si_thrlist = {tqe_next = 0x0, tqe_prev = 0x0}, si_thread = 0x0, si_note = {kl_list = {slh_first = 0x0},
        kl_lock = 0xc0747700 <knlist_mtx_lock>, kl_unlock = 0xc07470e0 <knlist_mtx_unlock>, kl_locked = 0xc07470c0 <knlist_mtx_locked>,
        kl_lockarg = 0xc342ee08}, si_flags = 0}, sb_mtx = {lock_object = {lo_name = 0xc0ad4ad8 "so_snd", lo_type = 0xc0ad4ad8 "so_snd",
        lo_flags = 16973824, lo_witness_data = {lod_list = {stqe_next = 0x0}, lod_witness = 0x0}}, mtx_lock = 3272403696, mtx_recurse = 0}, sb_sx = {
      lock_object = {lo_name = 0xc0ad4ae6 "so_snd_sx", lo_type = 0xc0ad4ae6 "so_snd_sx", lo_flags = 37421056, lo_witness_data = {lod_list = {
            stqe_next = 0x0}, lod_witness = 0x0}}, sx_lock = 1, sx_recurse = 0}, sb_state = 16, sb_mb = 0x0, sb_mbtail = 0x0, sb_lastrecord = 0x0,
    sb_sndptr = 0x0, sb_sndptroff = 0, sb_cc = 0, sb_hiwat = 33580, sb_mbcnt = 0, sb_mcnt = 0, sb_ccnt = 0, sb_mbmax = 262144, sb_ctl = 0, sb_lowat = 2048,
    sb_timeo = 0, sb_flags = 2048}, so_upcall = 0, so_upcallarg = 0x5dc0, so_cred = 0xc4260900, so_label = 0x0, so_peerlabel = 0x0, so_gencnt = 118111,
  so_emuldata = 0x0, so_accf = 0x0, so_fibnum = 0}

(kgdb) p so->so_snd
$10 = {sb_sel = {si_thrlist = {tqe_next = 0x0, tqe_prev = 0x0}, si_thread = 0x0, si_note = {kl_list = {slh_first = 0x0},
      kl_lock = 0xc0747700 <knlist_mtx_lock>, kl_unlock = 0xc07470e0 <knlist_mtx_unlock>, kl_locked = 0xc07470c0 <knlist_mtx_locked>,
      kl_lockarg = 0xc342ee08}, si_flags = 0}, sb_mtx = {lock_object = {lo_name = 0xc0ad4ad8 "so_snd", lo_type = 0xc0ad4ad8 "so_snd", lo_flags = 16973824,
      lo_witness_data = {lod_list = {stqe_next = 0x0}, lod_witness = 0x0}}, mtx_lock = 3272403696, mtx_recurse = 0}, sb_sx = {lock_object = {
      lo_name = 0xc0ad4ae6 "so_snd_sx", lo_type = 0xc0ad4ae6 "so_snd_sx", lo_flags = 37421056, lo_witness_data = {lod_list = {stqe_next = 0x0},
        lod_witness = 0x0}}, sx_lock = 1, sx_recurse = 0}, sb_state = 16, sb_mb = 0x0, sb_mbtail = 0x0, sb_lastrecord = 0x0, sb_sndptr = 0x0,
  sb_sndptroff = 0, sb_cc = 0, sb_hiwat = 33580, sb_mbcnt = 0, sb_mcnt = 0, sb_ccnt = 0, sb_mbmax = 262144, sb_ctl = 0, sb_lowat = 2048, sb_timeo = 0,
  sb_flags = 2048}

(kgdb) f 10
#10 0xc089b2ee in tcp_input (m=0xc34a6600, off0=20) at /usr/src/sys/netinet/tcp_input.c:846
846             tcp_do_segment(m, th, so, tp, drop_hdrlen, tlen);
(kgdb) p m
$13 = (struct mbuf *) 0xc34a6600
(kgdb) p *m
$14 = {m_hdr = {mh_next = 0x0, mh_nextpkt = 0x0, mh_data = 0xc34c8010 "E", mh_len = 52, mh_flags = 3, mh_type = 1, pad = "\000"}, M_dat = {MH = {
      MH_pkthdr = {rcvif = 0xc31bb400, header = 0x0, len = 52, csum_flags = 3840, csum_data = 65535, tso_segsz = 0, ether_vtag = 0, tags = {
          slh_first = 0xc4c06b80}}, MH_dat = {MH_ext = {ext_buf = 0xc34c8000 "\005", ext_free = 0, ext_args = 0x0, ext_size = 2048, ref_cnt = 0xc34af9dc,
          ext_type = 6},
        MH_databuf = "\000\200LÃ\000\000\000\000\000\000\000\000\000\b\000\000ÜùJÃ\006\000\000\000e\224ì\"\230\0058>\aÜ6\217FäîÍ?´ÊÆÃL©\tc\021Ûkëí¾s\177Ò\211­y\214\020\rXr»&yPI\v^N\210¡ß[\005½ª¹@Èd/\003\215ü®\2205­¹RE$\003\020Îf\035O0GÏÞ\216U\"ëå³õ¸\215`\002âÉÂ\n\212¾\207ïr\036ëæj°äÛ¨HU\234\034Ƥª.ÚbÚ\031\220Û¯íAe©\0333\207ÿzó½ \025v¥<\a¯ZÎ\205W<²\233'\205\002)\nRkÊä]\024>\214õ\217\217p]\230Ôw>ºsÄ"...}},
    M_databuf = "\000´\033Ã\000\000\000\0004\000\000\000\000\017\000\000ÿÿ\000\000\000\000\000\000\200kÀÄ\000\200LÃ\000\000\000\000\000\000\000\000\000\b\000\000ÜùJÃ\006\000\000\000e\224ì\"\230\0058>\aÜ6\217FäîÍ?´ÊÆÃL©\tc\021Ûkëí¾s\177Ò\211­y\214\020\rXr»&yPI\v^N\210¡ß[\005½ª¹@Èd/\003\215ü®\2205­¹RE$\003\020Îf\035O0GÏÞ\216U\"ëå³õ¸\215`\002âÉÂ\n\212¾\207ïr\036ëæj°äÛ¨HU\234\034Ƥª.ÚbÚ\031\220Û¯íAe©\0333\207ÿzó½ \025v¥<\a¯ZÎ\205W"...}}

More information about the freebsd-stable mailing list