page fault while in kernel mode - after upgrade from 12.2 to 13.0

Michael Schmiedgen schmiedgen at gmx.net
Wed May 5 16:35:35 UTC 2021 

On 04.05.2021 21:02, Mark Johnston wrote:
> This looks like fairly random kernel memory corruption.  Are you able to
> build an INVARIANTS kernel and test that?  Assuming you're using 13.0,
> you'd grab the 13.0 sources, add "options INVARIANT_SUPPORT" and
> "options INVARIANTS" to the GENERIC kernel configuration in
> sys/amd64/conf, and do a "make buildkernel installkernel".

Below some info with an INVARIANTS kernel. Please let me know if I can provide
further information. Thank you!


--- kgdb backtrace


(kgdb) backtrace
#0  __curthread () at /usr/src/sys/amd64/include/pcpu_aux.h:55
#1  doadump (textdump=<optimized out>) at /usr/src/sys/kern/kern_shutdown.c:399
#2  0xffffffff80bf580b in kern_reboot (howto=260) at /usr/src/sys/kern/kern_shutdown.c:486
#3  0xffffffff80bf5c50 in vpanic (fmt=<optimized out>, ap=<optimized out>) at /usr/src/sys/kern/kern_shutdown.c:919
#4  0xffffffff80bf59b3 in panic (fmt=<unavailable>) at /usr/src/sys/kern/kern_shutdown.c:843
#5  0xffffffff80f1ae71 in uma_dbg_free (zone=0xfffffe006e3e3c00, slab=0xfffff8053b159fd8, item=0xfffff8053b159300) at /usr/src/sys/vm/uma_core.c:5437
#6  0xffffffff80f13a64 in item_dtor (zone=0xfffffe006e3e3c00, item=0xfffff8053b159300, size=256, udata=0x0, skip=SKIP_NONE) at
/usr/src/sys/vm/uma_core.c:3220
#7  uma_zfree_arg (zone=0xfffffe006e3e3c00, item=item at entry=0xfffff8053b159300, udata=udata at entry=0x0) at /usr/src/sys/vm/uma_core.c:4165
#8  0xffffffff80bcefcf in mb_free_ext (m=m at entry=0xfffff8053b159300) at /usr/src/sys/kern/kern_mbuf.c:1200
#9  0xffffffff80bcda68 in m_free (m=m at entry=0xfffff8053b159300) at /usr/src/sys/sys/mbuf.h:1441
#10 0xffffffff80bceda8 in m_freem (mb=mb at entry=0xfffff8053b159300) at /usr/src/sys/kern/kern_mbuf.c:1525
#11 0xffffffff82c4d79a in div_output (so=<optimized out>, m=0xfffff8053b159300, sin=<optimized out>, control=<optimized out>) at
/usr/src/sys/netinet/ip_divert.c:396
#12 div_send (so=<optimized out>, so at entry=<error reading variable: value is not available>, flags=<optimized out>, flags at entry=<error reading
variable: value is not available>, m=0xfffff8053b159300, m at entry=<error reading variable: value is not available>, nam=<optimized out>,
     nam at entry=<error reading variable: value is not available>, control=<optimized out>, control at entry=<error reading variable: value is not
available>, td=<optimized out>, td at entry=<error reading variable: value is not available>) at /usr/src/sys/netinet/ip_divert.c:659
#13 0xffffffff80c92f97 in sosend_generic (so=0xfffff800468d5760, so at entry=<error reading variable: value is not available>, addr=0xfffff800120c72e0,
addr at entry=<error reading variable: value is not available>, uio=<optimized out>, uio at entry=<error reading variable: value is not available>,
top=0xfffff8053b159300,
     top at entry=<error reading variable: value is not available>, control=<optimized out>, control at entry=<error reading variable: value is not
available>, flags=0, flags at entry=<error reading variable: value is not available>, td=0xfffffe019cdc2300, td at entry=<error reading variable: value is
not available>)
     at /usr/src/sys/kern/uipc_socket.c:1755
#14 0xffffffff80c93286 in sosend (so=<unavailable>, so at entry=0xfffff800468d5760, addr=<unavailable>, uio=<unavailable>, uio at entry=0xfffffe0199b338a8,
top=<unavailable>, top at entry=0x0, control=control at entry=0x0, flags=<unavailable>, flags at entry=0, td=0xfffffe019cdc2300) at
/usr/src/sys/kern/uipc_socket.c:1810
#15 0xffffffff80c99ffc in kern_sendit (td=<optimized out>, td at entry=0xfffffe019cdc2300, s=3, mp=<optimized out>, mp at entry=0xfffffe0199b33980, flags=0,
control=0x0, segflg=segflg at entry=UIO_USERSPACE) at /usr/src/sys/kern/uipc_syscalls.c:798
#16 0xffffffff80c9a39b in sendit (td=0xfffffe019cdc2300, td at entry=<unavailable>, s=<unavailable>, mp=mp at entry=0xfffffe0199b33980, flags=<unavailable>)
at /usr/src/sys/kern/uipc_syscalls.c:723
#17 0xffffffff80c9a1ad in sys_sendto (td=<unavailable>, td at entry=<error reading variable: value is not available>, uap=<unavailable>, uap at entry=<error
reading variable: value is not available>) at /usr/src/sys/kern/uipc_syscalls.c:841
#18 0xffffffff8108824e in syscallenter (td=<optimized out>) at /usr/src/sys/amd64/amd64/../../kern/subr_syscall.c:189
#19 amd64_syscall (td=0xfffffe019cdc2300, traced=0) at /usr/src/sys/amd64/amd64/trap.c:1156
#20 <signal handler called>


--- core.txt


panic: Duplicate free of 0xfffff8053b159300 from zone 0xfffffe006e3e3c00(mbuf_packet) slab 0xfffff8053b159fd8(3)

Unread portion of the kernel message buffer:
<110>ipfw: 4500 Deny UDP 192.168.10.100:137 192.168.10.255:137 out via bge0
panic: Duplicate free of 0xfffff8053b159300 from zone 0xfffffe006e3e3c00(mbuf_packet) slab 0xfffff8053b159fd8(3)
cpuid = 6
time = 1620231385
KDB: stack backtrace:
#0 0xffffffff80c400e5 at kdb_backtrace+0x65
#1 0xffffffff80bf5be1 at vpanic+0x181
#2 0xffffffff80bf59b3 at panic+0x43
#3 0xffffffff80f1ae71 at uma_dbg_free+0x1e1
#4 0xffffffff80f13a64 at uma_zfree_arg+0x144
#5 0xffffffff80bcefcf at mb_free_ext+0x11f
#6 0xffffffff80bcda68 at m_free+0xd8
#7 0xffffffff80bceda8 at m_freem+0x28
#8 0xffffffff82c4d79a at div_send+0x43a
#9 0xffffffff80c92f97 at sosend_generic+0x5f7
#10 0xffffffff80c93286 at sosend+0x66
#11 0xffffffff80c99ffc at kern_sendit+0x1ec
#12 0xffffffff80c9a39b at sendit+0x1db
#13 0xffffffff80c9a1ad at sys_sendto+0x4d
#14 0xffffffff8108824e at amd64_syscall+0x12e
#15 0xffffffff8105bf4e at fast_syscall_common+0xf8
Uptime: 5m17s
Dumping 2609 out of 65454 MB:..1%..11%..21%..31%..41%..51%..61%..71%..81%..91%

__curthread () at /usr/src/sys/amd64/include/pcpu_aux.h:55
55              __asm("movq %%gs:%P1,%0" : "=r" (td) : "n" (offsetof(struct pcpu,
(kgdb) #0  __curthread () at /usr/src/sys/amd64/include/pcpu_aux.h:55
#1  doadump (textdump=<optimized out>)
     at /usr/src/sys/kern/kern_shutdown.c:399
#2  0xffffffff80bf580b in kern_reboot (howto=260)
     at /usr/src/sys/kern/kern_shutdown.c:486
#3  0xffffffff80bf5c50 in vpanic (fmt=<optimized out>, ap=<optimized out>)
     at /usr/src/sys/kern/kern_shutdown.c:919
#4  0xffffffff80bf59b3 in panic (fmt=<unavailable>)
     at /usr/src/sys/kern/kern_shutdown.c:843
#5  0xffffffff80f1ae71 in uma_dbg_free (zone=0xfffffe006e3e3c00,
     slab=0xfffff8053b159fd8, item=0xfffff8053b159300)
     at /usr/src/sys/vm/uma_core.c:5437
#6  0xffffffff80f13a64 in item_dtor (zone=0xfffffe006e3e3c00,
     item=0xfffff8053b159300, size=256, udata=0x0, skip=SKIP_NONE)
     at /usr/src/sys/vm/uma_core.c:3220
#7  uma_zfree_arg (zone=0xfffffe006e3e3c00,
     item=item at entry=0xfffff8053b159300, udata=udata at entry=0x0)
     at /usr/src/sys/vm/uma_core.c:4165
#8  0xffffffff80bcefcf in mb_free_ext (m=m at entry=0xfffff8053b159300)
     at /usr/src/sys/kern/kern_mbuf.c:1200
#9  0xffffffff80bcda68 in m_free (m=m at entry=0xfffff8053b159300)
     at /usr/src/sys/sys/mbuf.h:1441
#10 0xffffffff80bceda8 in m_freem (mb=mb at entry=0xfffff8053b159300)
     at /usr/src/sys/kern/kern_mbuf.c:1525
#11 0xffffffff82c4d79a in div_output (so=<optimized out>,
     m=0xfffff8053b159300, sin=<optimized out>, control=<optimized out>)
     at /usr/src/sys/netinet/ip_divert.c:396
#12 div_send (so=<optimized out>,
     so at entry=<error reading variable: value is not available>,
     flags=<optimized out>,
     flags at entry=<error reading variable: value is not available>,
     m=0xfffff8053b159300,
     m at entry=<error reading variable: value is not available>,
     nam=<optimized out>,
     nam at entry=<error reading variable: value is not available>,
     control=<optimized out>,
     control at entry=<error reading variable: value is not available>,
     td=<optimized out>,
     td at entry=<error reading variable: value is not available>)
     at /usr/src/sys/netinet/ip_divert.c:659
#13 0xffffffff80c92f97 in sosend_generic (so=0xfffff800468d5760,
     so at entry=<error reading variable: value is not available>,
     addr=0xfffff800120c72e0,
     addr at entry=<error reading variable: value is not available>,
     uio=<optimized out>,
     uio at entry=<error reading variable: value is not available>,
     top=0xfffff8053b159300,
     top at entry=<error reading variable: value is not available>,
     control=<optimized out>,
     control at entry=<error reading variable: value is not available>, flags=0,
     flags at entry=<error reading variable: value is not available>,
     td=0xfffffe019cdc2300,
     td at entry=<error reading variable: value is not available>)
     at /usr/src/sys/kern/uipc_socket.c:1755
#14 0xffffffff80c93286 in sosend (so=<unavailable>,
     so at entry=0xfffff800468d5760, addr=<unavailable>, uio=<unavailable>,
     uio at entry=0xfffffe0199b338a8, top=<unavailable>, top at entry=0x0,
     control=control at entry=0x0, flags=<unavailable>, flags at entry=0,
     td=0xfffffe019cdc2300) at /usr/src/sys/kern/uipc_socket.c:1810
#15 0xffffffff80c99ffc in kern_sendit (td=<optimized out>,
     td at entry=0xfffffe019cdc2300, s=3, mp=<optimized out>,
     mp at entry=0xfffffe0199b33980, flags=0, control=0x0,
     segflg=segflg at entry=UIO_USERSPACE)
     at /usr/src/sys/kern/uipc_syscalls.c:798
#16 0xffffffff80c9a39b in sendit (td=0xfffffe019cdc2300,
     td at entry=<unavailable>, s=<unavailable>, mp=mp at entry=0xfffffe0199b33980,
     flags=<unavailable>) at /usr/src/sys/kern/uipc_syscalls.c:723
#17 0xffffffff80c9a1ad in sys_sendto (td=<unavailable>,
     td at entry=<error reading variable: value is not available>,
     uap=<unavailable>,
     uap at entry=<error reading variable: value is not available>)
     at /usr/src/sys/kern/uipc_syscalls.c:841
#18 0xffffffff8108824e in syscallenter (td=<optimized out>)
     at /usr/src/sys/amd64/amd64/../../kern/subr_syscall.c:189
#19 amd64_syscall (td=0xfffffe019cdc2300, traced=0)
     at /usr/src/sys/amd64/amd64/trap.c:1156
#20 <signal handler called>

More information about the freebsd-net mailing list