igb / buf_ring panic: buf already enqueue at

[Andriy Gapon]

Unread portion of the kernel message buffer:
panic: buf=0xfffffe0cfe9a0600 already enqueue at 1259 prod=1260 cons=1259

(kgdb) bt
#0  doadump (textdump=1) at pcpu.h:234
#1  0xffffffff808ea32a in kern_reboot (howto=260) at
/usr/src/sys/kern/kern_shutdown.c:449
#2  0xffffffff808e9b43 in panic (fmt=<value optimized out>) at
/usr/src/sys/kern/kern_shutdown.c:637
#3  0xffffffff804c5268 in igb_mq_start (ifp=0xfffffe0021566000,
m=0xfffffe0cfe9a0600) at buf_ring.h:86
#4  0xffffffff809b6dcd in ether_output_frame (ifp=0xfffffe0021566000,
m=0xfffffe0cfe9a0600) at /usr/src/sys/net/if_ethersubr.c:447
#5  0xffffffff809b73ae in ether_output (ifp=0xfffffe0021566000,
m=0xfffffe0cfe9a0600, dst=<value optimized out>, ro=<value optimized out>) at
/usr/src/sys/net/if_ethersubr.c:418
#6  0xffffffff80a2726f in ip_output (m=0xfffffe0cfe9a0600, opt=<value optimized
out>, ro=0xffffff9de0aca700, flags=<value optimized out>, imo=0x0,
inp=0xfffffe006cb85498) at /usr/src/sys/netinet/ip_output.c:631
#7  0xffffffff80aa4603 in udp_send (so=<value optimized out>, flags=<value
optimized out>, m=<value optimized out>, addr=<value optimized out>,
control=<value optimized out>, td=0xfffffe00322b1490)
    at /usr/src/sys/netinet/udp_usrreq.c:1243
#8  0xffffffff80963575 in sosend_dgram (so=0xfffffe064c9a2000, addr=0x0,
uio=0xffffff9de0aca990, top=0xfffffe0cfe9a0600, control=0x0, flags=0,
td=0xfffffe00322b1490) at /usr/src/sys/kern/uipc_socket.c:1175
#9  0xffffffff80960c2f in sosend (so=<value optimized out>, addr=<value
optimized out>, uio=<value optimized out>, top=<value optimized out>,
control=<value optimized out>, flags=<value optimized out>, td=0xfffffe00322b1490)
    at /usr/src/sys/kern/uipc_socket.c:1408
#10 0xffffffff809468f8 in soo_write (fp=<value optimized out>,
uio=0xffffff9de0aca990, active_cred=<value optimized out>, flags=<value
optimized out>, td=<value optimized out>) at /usr/src/sys/kern/sys_socket.c:102
#11 0xffffffff8093eefc in dofilewrite (td=0xfffffe00322b1490, fd=3,
fp=0xfffffe03bec525a0, auio=0xffffff9de0aca990, offset=<value optimized out>,
flags=0) at file.h:295
#12 0xffffffff809407e4 in kern_writev (td=0xfffffe00322b1490, fd=3,
auio=0xffffff9de0aca990) at /usr/src/sys/kern/sys_generic.c:459
#13 0xffffffff80940915 in sys_write (td=<value optimized out>, uap=<value
optimized out>) at /usr/src/sys/kern/sys_generic.c:375
#14 0xffffffff80ce4d2a in amd64_syscall (td=0xfffffe00322b1490, traced=0) at
subr_syscall.c:135
#15 0xffffffff80cceb67 in Xfast_syscall () at
/usr/src/sys/amd64/amd64/exception.S:391
#16 0x0000000800b39bec in ?? ()
Previous frame inner to this frame (corrupt stack?)
(kgdb) fr 3
#3  0xffffffff804c5268 in igb_mq_start (ifp=0xfffffe0021566000,
m=0xfffffe0cfe9a0600) at buf_ring.h:86
86                              panic("buf=%p already enqueue at %d prod=%d
cons=%d",
(kgdb) list
81      #ifdef DEBUG_BUFRING
82              int i;
83              for (i = br->br_cons_head; i != br->br_prod_head;
84                   i = ((i + 1) & br->br_cons_mask))
85                      if(br->br_ring[i] == buf)
86                              panic("buf=%p already enqueue at %d prod=%d
cons=%d",
87                                  buf, i, br->br_prod_tail, br->br_cons_tail);
88      #endif
89              critical_enter();
90              do {
(kgdb) p *buf
$1 = {b_bufobj = 0x0, b_bcount = 4096, b_caller1 = 0x0, b_data =
0xffffff9c5c600000 <Address 0xffffff9c5c600000 out of bounds>, b_error = 0,
b_iocmd = 2 '\002', b_ioflags = 2 '\002', b_iooffset = 196608, b_resid = 0,
  b_iodone = 0, b_blkno = 384, b_offset = 196608, b_bobufs = {tqe_next = 0x0,
tqe_prev = 0xfffffe02e5976738}, b_left = 0x0, b_right = 0x0, b_vflags =
2147483648, b_freelist = {tqe_next = 0xffffff9c4d247b70,
    tqe_prev = 0xffffffff8178a580}, b_qindex = 4, b_flags = 8704, b_xflags = 0
'\0', b_lock = {lock_object = {lo_name = 0xffffffff80fa6026 "bufwait", lo_flags
= 108199936, lo_data = 0, lo_witness = 0xffffff8ccd37a500},
    lk_lock = 1, lk_exslpfail = 0, lk_timo = 0, lk_pri = 96}, b_bufsize = 0,
b_runningbufspace = 0, b_kvabase = 0xffffff9c5c600000 <Address
0xffffff9c5c600000 out of bounds>, b_kvaalloc = 0x0, b_kvasize = 16384, b_lblkno
= 384,
  b_vp = 0x0, b_dirtyoff = 0, b_dirtyend = 0, b_rcred = 0xfffffe02b232b700,
b_wcred = 0x0, b_saveaddr = 0xffffff9c5c600000, b_pager = {pg_reqpage = 0},
b_cluster = {cluster_head = {tqh_first = 0x0, tqh_last = 0x0},
    cluster_entry = {tqe_next = 0x0, tqe_prev = 0x0}}, b_pages = {0x0 <repeats
256 times>}, b_npages = 0, b_dep = {lh_first = 0x0}, b_fsprivate1 = 0x0,
b_fsprivate2 = 0x0, b_fsprivate3 = 0x0, b_pin_count = 0}
(kgdb) p i
$2 = 0
(kgdb) p *txr->br
$3 = {br_prod_head = 1260, br_prod_tail = 1260, br_prod_size = 4096,
br_prod_mask = 4095, br_drops = 1888684, br_prod_bufs = 3286252, _pad0 = {0
<repeats 11 times>}, br_cons_head = 1259, br_cons_tail = 1259, br_cons_size = 4096,
  br_cons_mask = 4095, _pad1 = {0 <repeats 14 times>}, br_lock =
0xfffffe0011ca6810, br_ring = 0xffffff8ccd8cd100}

This is stable/9 with INVARIANTS.
Is this something known?
If not, is anybody interested?  I have the kernel and the vmcore for further
investigation.

-- 
Andriy Gapon