[Bug 148807] [panic] 8.1-RELEASE "panic: sbdrop" and "panic: sbsndptr: sockbuf _ and mbuf _ clashing" under heavy load

bugzilla-noreply at freebsd.org bugzilla-noreply at freebsd.org
Mon Feb 2 19:57:44 UTC 2015 

https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=148807

--- Comment #14 from Andrey V. Elsukov <ae at FreeBSD.org> ---
Second panic:

panic: sbsndptr: sockbuf 0xfffffe03e62b5c20 and mbuf 0xfffffe01d8fd3900
clashing
cpuid = 31
KDB: stack backtrace:
db_trace_self_wrapper() at db_trace_self_wrapper+0x2a/frame 0xffffff90d4fca430
kdb_backtrace() at kdb_backtrace+0x37/frame 0xffffff90d4fca4f0
panic() at panic+0x1ce/frame 0xffffff90d4fca5f0
sbsndptr() at sbsndptr+0xe4/frame 0xffffff90d4fca610
tcp_output() at tcp_output+0x16cd/frame 0xffffff90d4fca7c0
tcp_usr_send() at tcp_usr_send+0x325/frame 0xffffff90d4fca820
sosend_generic() at sosend_generic+0x3f6/frame 0xffffff90d4fca8c0
soo_write() at soo_write+0x5e/frame 0xffffff90d4fca8f0
dofilewrite() at dofilewrite+0x85/frame 0xffffff90d4fca940
kern_writev() at kern_writev+0x6c/frame 0xffffff90d4fca980
sys_write() at sys_write+0x64/frame 0xffffff90d4fca9d0
amd64_syscall() at amd64_syscall+0x5ea/frame 0xffffff90d4fcaaf0
Xfast_syscall() at Xfast_syscall+0xf7/frame 0xffffff90d4fcaaf0
--- syscall (4, FreeBSD ELF64, sys_write), rip = 0x802da3bec, rsp =
0x7fffffffdae8, rbp = 0x7fffffffdbf0 ---
Uptime: 1m48s
Dumping 3468 out of 65475 MB:..1%..11%..21%..31%..41%..51%..61%..71%..81%..91%

Reading symbols from /boot/kernel/zfs.ko...Reading symbols from
/boot/kernel/zfs.ko.symbols...done.
done.
Loaded symbols for /boot/kernel/zfs.ko
Reading symbols from /boot/kernel/opensolaris.ko...Reading symbols from
/boot/kernel/opensolaris.ko.symbols...done.
done.
Loaded symbols for /boot/kernel/opensolaris.ko
Reading symbols from /boot/kernel/if_igb.ko...Reading symbols from
/boot/kernel/if_igb.ko.symbols...done.
done.
Loaded symbols for /boot/kernel/if_igb.ko
Reading symbols from /boot/kernel/aac.ko...Reading symbols from
/boot/kernel/aac.ko.symbols...done.
done.
Loaded symbols for /boot/kernel/aac.ko
Reading symbols from /boot/kernel/ipdivert.ko...Reading symbols from
/boot/kernel/ipdivert.ko.symbols...done.
done.
Loaded symbols for /boot/kernel/ipdivert.ko
Reading symbols from /boot/kernel/ipfw.ko...Reading symbols from
/boot/kernel/ipfw.ko.symbols...done.
done.
Loaded symbols for /boot/kernel/ipfw.ko
Reading symbols from /boot/kernel/t5fw_cfg.ko...Reading symbols from
/boot/kernel/t5fw_cfg.ko.symbols...done.
done.
Loaded symbols for /boot/kernel/t5fw_cfg.ko
Reading symbols from /boot/kernel/if_cxgbe.ko...Reading symbols from
/boot/kernel/if_cxgbe.ko.symbols...done.
done.
Loaded symbols for /boot/kernel/if_cxgbe.ko
Reading symbols from /boot/kernel/ipmi.ko...Reading symbols from
/boot/kernel/ipmi.ko.symbols...done.
done.
Loaded symbols for /boot/kernel/ipmi.ko
Reading symbols from /boot/kernel/smbus.ko...Reading symbols from
/boot/kernel/smbus.ko.symbols...done.
done.
Loaded symbols for /boot/kernel/smbus.ko
#0  doadump (textdump=1) at /usr/src/sys/kern/kern_shutdown.c:271
271        if (textdump && textdump_pending) {
(kgdb) bt
#0  doadump (textdump=1) at /usr/src/sys/kern/kern_shutdown.c:271
#1  0xffffffff80907eb4 in kern_reboot (howto=260) at
/usr/src/sys/kern/kern_shutdown.c:454
#2  0xffffffff809083a7 in panic (fmt=0x1 <Address 0x1 out of bounds>) at
/usr/src/sys/kern/kern_shutdown.c:642
#3  0xffffffff809766e4 in sbsndptr (sb=<value optimized out>, off=<value
optimized out>, len=<value optimized out>, moff=<value optimized out>)
    at /usr/src/sys/kern/uipc_sockbuf.c:985
#4  0xffffffff80aaedbd in tcp_output (tp=0xfffffe03e675a3d0) at
/usr/src/sys/netinet/tcp_output.c:954
#5  0xffffffff80abc555 in tcp_usr_send (so=0xfffffe03e62b5aa0, flags=0,
m=0xfffffe01d8fd2200, nam=0x0, control=<value optimized out>,
td=0xfffffe0021e90000)
    at /usr/src/sys/netinet/tcp_usrreq.c:874
#6  0xffffffff8097c1f6 in sosend_generic (so=0xfffffe03e62b5aa0, addr=0x0,
uio=0xffffff90d4fca990, top=0xfffffe01d8fd2200, control=0x0, flags=<value
optimized out>, 
    td=0xfffffe0021e90000) at /usr/src/sys/kern/uipc_socket.c:1376
#7  0xffffffff8095ea6e in soo_write (fp=<value optimized out>,
uio=0xffffff90d4fca990, active_cred=<value optimized out>, flags=<value
optimized out>, 
    td=<value optimized out>) at /usr/src/sys/kern/sys_socket.c:102
#8  0xffffffff80957195 in dofilewrite (td=0xfffffe0021e90000, fd=3,
fp=0xfffffe0021cf3820, auio=0xffffff90d4fca990, offset=<value optimized out>,
flags=0) at file.h:295
#9  0xffffffff809574cc in kern_writev (td=0xfffffe0021e90000, fd=3,
auio=0xffffff90d4fca990) at /usr/src/sys/kern/sys_generic.c:477
#10 0xffffffff80957554 in sys_write (td=<value optimized out>, uap=<value
optimized out>) at /usr/src/sys/kern/sys_generic.c:393
#11 0xffffffff80cfea4a in amd64_syscall (td=0xfffffe0021e90000, traced=0) at
subr_syscall.c:135
#12 0xffffffff80ce8ac7 in Xfast_syscall () at
/usr/src/sys/amd64/amd64/exception.S:391
#13 0x0000000802da3bec in ?? ()
Previous frame inner to this frame (corrupt stack?)

(kgdb) p *(struct sockbuf *)0xfffffe03e62b5c20
$1 = {sb_sel = {si_tdlist = {tqh_first = 0x0, tqh_last = 0x0}, si_note =
{kl_list = {slh_first = 0x0}, kl_lock = 0xffffffff808cd0c0 <knlist_mtx_lock>, 
      kl_unlock = 0xffffffff808cd090 <knlist_mtx_unlock>, kl_assert_locked =
0xffffffff808c9a10 <knlist_mtx_assert_locked>, 
      kl_assert_unlocked = 0xffffffff808c9a20 <knlist_mtx_assert_unlocked>,
kl_lockarg = 0xfffffe03e62b5c68}, si_mtx = 0x0}, sb_mtx = {lock_object = {
      lo_name = 0xffffffff80f3e7fd "so_snd", lo_flags = 16973824, lo_data = 0,
lo_witness = 0x0}, mtx_lock = 18446741875255214080}, sb_sx = {lock_object = {
      lo_name = 0xffffffff80f3ed6b "so_snd_sx", lo_flags = 36896768, lo_data =
0, lo_witness = 0x0}, sx_lock = 18446741875255214080}, sb_state = 0, 
  sb_mb = 0xfffffe01f4069900, sb_mbtail = 0xfffffe01d8fd3900, sb_lastrecord =
0xfffffe01f4069900, sb_sndptr = 0xfffffe01d8fd3900, sb_sndptroff = 1632, sb_cc
= 1716, 
  sb_hiwat = 131376, sb_mbcnt = 4864, sb_mcnt = 11, sb_ccnt = 1, sb_mbmax =
1051008, sb_ctl = 0, sb_lowat = 2048, sb_timeo = 0, sb_flags = 2048, sb_upcall
= 0, 
  sb_upcallarg = 0x0}

(kgdb) p *(struct mbuf *)0xfffffe01d8fd3900
$2 = {m_hdr = {mh_next = 0x0, mh_nextpkt = 0x0, mh_data = 0xfffffe01d8fd3928
"", mh_len = 68, mh_flags = 0, mh_type = 1, pad = "\000\000\000\000\000"},
M_dat = {MH = {
      MH_pkthdr = {rcvif = 0xb1dee9e530000000, header = 0xf10fc01307aab916, len
= -337628730, flowid = 2682375970, csum_flags = -966380398, csum_data =
-1624117065, 
        tso_segsz = 11596, PH_vt = {vt_vtag = 31606, vt_nrecs = 31606}, tags =
{slh_first = 0xa2b0a659a4311f25}}, MH_dat = {MH_ext = {
          ext_buf = 0x43772562c99aa431 <Address 0x43772562c99aa431 out of
bounds>, ext_free = 0x7e1cffd9b6b13fc6, ext_arg1 = 0x731c9ab425536605, 
          ext_arg2 = 0xebc6cac44b21a941, ext_size = 520953289, ref_cnt =
0x5165381046dcad94, ext_type = 1308134978}, 
        MH_databuf =
"1�\232�b%wC�?����\034~\005fS%�\232\034sA�!K�����\035\r\037Iܡq\224��F\0208eQB\216�M�P�/\000\026OS^Lq%�MY\212\200\030\b\004\021\000\000\000\001\001\b\n2��
\v��O\000\000\000
��n�ٻ�Er\032S\201\220\220��I�\"\210\233\v\0223?=�*a|\231\001\022�6}�G�\026�\036z\n\023�<���B8�\200\000\000\000\000\000\000\002%\220���B8\001\003Ip\000\000\000"}}, 
    M_databuf =
"\000\000\0000��ޱ\026��\a\023�\017��1��\"��\237\2224fƷ�1\237L-v{X�\235\214%\0371�Y���1�\232�b%wC�?����\034~\005fS%�\232\034sA�!K�����\035\r\037Iܡq\224��F\0208eQB\216�M�P�/\000\026OS^Lq%�MY\212\200\030\b\004\021\000\000\000\001\001\b\n2��
\v��O\000\000\000
��n�ٻ�Er\032S\201\220\220��I�\"\210\233\v\0223?=�*a|\231\001\022�6}�G�\026�\036z\n\023�<���B8�\200\000\000\000\000\000\000"...}}

(kgdb) f 6
#6  0xffffffff8097c1f6 in sosend_generic (so=0xfffffe03e62b5aa0, addr=0x0,
uio=0xffffff90d4fca990, top=0xfffffe01d8fd2200, control=0x0, flags=<value
optimized out>, 
    td=0xfffffe0021e90000) at /usr/src/sys/kern/uipc_socket.c:1376
1376                error = (*so->so_proto->pr_usrreqs->pru_send)(so,
(kgdb) p *so
$3 = {so_count = 1, so_type = 1, so_options = 12, so_linger = 0, so_state =
258, so_qstate = 0, so_pcb = 0xfffffe03e678a640, so_vnet = 0x0, 
  so_proto = 0xffffffff8143c3f0, so_head = 0x0, so_incomp = {tqh_first = 0x0,
tqh_last = 0x0}, so_comp = {tqh_first = 0x0, tqh_last = 0x0}, so_list =
{tqe_next = 0x0, 
    tqe_prev = 0xfffffe01d8f96040}, so_qlen = 0, so_incqlen = 0, so_qlimit = 0,
so_timeo = 0, so_error = 0, so_sigio = 0x0, so_oobmark = 0, so_aiojobq = {
    tqh_first = 0x0, tqh_last = 0xfffffe03e62b5b20}, so_rcv = {sb_sel =
{si_tdlist = {tqh_first = 0x0, tqh_last = 0xfffffe03e62b5b30}, si_note =
{kl_list = {
          slh_first = 0x0}, kl_lock = 0xffffffff808cd0c0 <knlist_mtx_lock>,
kl_unlock = 0xffffffff808cd090 <knlist_mtx_unlock>, 
        kl_assert_locked = 0xffffffff808c9a10 <knlist_mtx_assert_locked>,
kl_assert_unlocked = 0xffffffff808c9a20 <knlist_mtx_assert_unlocked>, 
        kl_lockarg = 0xfffffe03e62b5b78}, si_mtx = 0xffffff800e02f670}, sb_mtx
= {lock_object = {lo_name = 0xffffffff80f3e7f6 "so_rcv", lo_flags = 16973824, 
        lo_data = 0, lo_witness = 0x0}, mtx_lock = 4}, sb_sx = {lock_object =
{lo_name = 0xffffffff80f3ed75 "so_rcv_sx", lo_flags = 36896768, lo_data = 0, 
        lo_witness = 0x0}, sx_lock = 1}, sb_state = 0, sb_mb = 0x0, sb_mbtail =
0x0, sb_lastrecord = 0x0, sb_sndptr = 0x0, sb_sndptroff = 0, sb_cc = 0, 
    sb_hiwat = 131376, sb_mbcnt = 0, sb_mcnt = 0, sb_ccnt = 0, sb_mbmax =
1051008, sb_ctl = 0, sb_lowat = 1, sb_timeo = 0, sb_flags = 2056, sb_upcall =
0, 
    sb_upcallarg = 0x0}, so_snd = {sb_sel = {si_tdlist = {tqh_first = 0x0,
tqh_last = 0x0}, si_note = {kl_list = {slh_first = 0x0}, 
        kl_lock = 0xffffffff808cd0c0 <knlist_mtx_lock>, kl_unlock =
0xffffffff808cd090 <knlist_mtx_unlock>, 
        kl_assert_locked = 0xffffffff808c9a10 <knlist_mtx_assert_locked>,
kl_assert_unlocked = 0xffffffff808c9a20 <knlist_mtx_assert_unlocked>, 
        kl_lockarg = 0xfffffe03e62b5c68}, si_mtx = 0x0}, sb_mtx = {lock_object
= {lo_name = 0xffffffff80f3e7fd "so_snd", lo_flags = 16973824, lo_data = 0, 
        lo_witness = 0x0}, mtx_lock = 18446741875255214080}, sb_sx =
{lock_object = {lo_name = 0xffffffff80f3ed6b "so_snd_sx", lo_flags = 36896768,
lo_data = 0, 
        lo_witness = 0x0}, sx_lock = 18446741875255214080}, sb_state = 0, sb_mb
= 0xfffffe01f4069900, sb_mbtail = 0xfffffe01d8fd3900, 
    sb_lastrecord = 0xfffffe01f4069900, sb_sndptr = 0xfffffe01d8fd3900,
sb_sndptroff = 1632, sb_cc = 1716, sb_hiwat = 131376, sb_mbcnt = 4864, sb_mcnt
= 11, 
    sb_ccnt = 1, sb_mbmax = 1051008, sb_ctl = 0, sb_lowat = 2048, sb_timeo = 0,
sb_flags = 2048, sb_upcall = 0, sb_upcallarg = 0x0}, so_cred =
0xfffffe01f48ce900, 
  so_label = 0x0, so_peerlabel = 0x0, so_gencnt = 13244, so_emuldata = 0x0,
so_accf = 0x0, so_fibnum = 0, so_user_cookie = 0}

(kgdb) set $inp=(struct inpcb *)so->so_pcb
(kgdb) p *$inp
$4 = {inp_hash = {le_next = 0x0, le_prev = 0xfffffe0012f573b0},
inp_pcbgrouphash = {le_next = 0x0, le_prev = 0x0}, inp_list = {le_next =
0xfffffe03e679bc80, 
    le_prev = 0xfffffe03e6743020}, inp_ppcb = 0xfffffe03e675a3d0, inp_pcbinfo =
0xffffffff81531060, inp_pcbgroup = 0x0, inp_pcbgroup_wild = {le_next = 0x0, 
    le_prev = 0x0}, inp_socket = 0xfffffe03e62b5aa0, inp_cred =
0xfffffe01f48ce900, inp_flow = 3457486592, inp_flags = 545300480, inp_flags2 =
0, inp_vflag = 6 '\006', 
  inp_ip_ttl = 64 '@', inp_ip_p = 0 '\0', inp_ip_minttl = 0 '\0', inp_flowid =
1779132015, inp_refcount = 1, inp_pspare = {0x0, 0x0, 0x0, 0x0, 0x0},
inp_ispare = {0, 0, 
    0, 0, 0, 0}, inp_inc = {inc_flags = 1 '\001', inc_len = 0 '\0', inc_fibnum
= 0, inc_ie = {ie_fport = 21327, ie_lport = 5632, ie_dependfaddr =
{ie46_foreign = {
          ia46_pad32 = {3087401514, 17039360, 4283245058}, ia46_addr4 = {s_addr
= 801984766}}, ie6_foreign = {__u6_addr = {
            __u6_addr8 = "*\002\006�\000\000\004\001\002\"M��P�/", __u6_addr16
= {554, 47110, 0, 260, 8706, 65357, 20734, 12237}, __u6_addr32 = {3087401514,
17039360, 
              4283245058, 801984766}}}}, ie_dependladdr = {ie46_local =
{ia46_pad32 = {3087401514, 917504, 0}, ia46_addr4 = {s_addr = 1375797248}},
ie6_local = {
          __u6_addr = {__u6_addr8 =
"*\002\006�\000\000\016\000\000\000\000\000\000\000\001R", __u6_addr16 = {554,
47110, 0, 14, 0, 0, 0, 20993}, __u6_addr32 = {
              3087401514, 917504, 0, 1375797248}}}}, ie6_zoneid = 0}},
inp_label = 0x0, inp_sp = 0x0, inp_depend4 = {inp4_ip_tos = 0 '\0',
inp4_options = 0x0, 
    inp4_moptions = 0x0}, inp_depend6 = {inp6_options = 0x0, inp6_outputopts =
0xfffffe0013424500, inp6_moptions = 0x0, inp6_icmp6filt = 0x0, inp6_cksum = 0, 
    inp6_hops = -1}, inp_portlist = {le_next = 0xfffffe03e6d8f640, le_prev =
0xfffffe03e6743140}, inp_phd = 0xfffffe03e6dfa540, inp_gencnt = 1509, inp_lle =
0x0, 
  inp_rt = 0x0, inp_lock = {lock_object = {lo_name = 0xffffffff80f59235
"tcpinp", lo_flags = 90898432, lo_data = 0, lo_witness = 0x0}, rw_lock =
18446741875255214080}}

-- 
You are receiving this mail because:
You are the assignee for the bug.

More information about the freebsd-net mailing list