page fault while in kernel mode - after upgrade from 12.2 to 13.0

Michael Schmiedgen schmiedgen at gmx.net
Wed May 5 09:08:54 UTC 2021 

On 04.05.2021 21:02, Mark Johnston wrote:
> On Tue, May 04, 2021 at 08:38:39PM +0200, Michael Schmiedgen wrote:
>> Hi Mark,
>>
>> sorry for the delay, I only can test after work. I triggered another 2 panics, this time
>> with a different result (see below). Can I provide some more information?
>
> This looks like fairly random kernel memory corruption.  Are you able to
> build an INVARIANTS kernel and test that?  Assuming you're using 13.0,
> you'd grab the 13.0 sources, add "options INVARIANT_SUPPORT" and
> "options INVARIANTS" to the GENERIC kernel configuration in
> sys/amd64/conf, and do a "make buildkernel installkernel".

I will try INVARIANTS after work, but in the meantime I got 2 more panics
from tonight.



--- #1


Fatal trap 12: page fault while in kernel mode
cpuid = 7; apic id = 07
fault virtual address   = 0x8
fault code              = supervisor read data, page not present
instruction pointer     = 0x20:0xffffffff80ca599c
stack pointer           = 0x28:0xfffffe0115bc46c0
frame pointer           = 0x28:0xfffffe0115bc4700
code segment            = base 0x0, limit 0xfffff, type 0x1b
                         = DPL 0, pres 1, long 1, def32 0, gran 1
processor eflags        = interrupt enabled, resume, IOPL = 0
current process         = 12 (swi1: netisr 0)
trap number             = 12
panic: page fault
cpuid = 7
time = 1620172732
KDB: stack backtrace:
#0 0xffffffff80c57345 at kdb_backtrace+0x65
#1 0xffffffff80c09d21 at vpanic+0x181
#2 0xffffffff80c09b93 at panic+0x43
#3 0xffffffff8108b187 at trap_fatal+0x387
#4 0xffffffff8108b1df at trap_pfault+0x4f
#5 0xffffffff8108a83d at trap+0x27d
#6 0xffffffff810617a8 at calltrap+0x8
#7 0xffffffff80dbf0ae at tcp_do_segment+0x10ce
#8 0xffffffff80dbd21e at tcp_input+0xabe
#9 0xffffffff80dafc15 at ip_input+0x125
#10 0xffffffff80d3fa7b at swi_net+0x12b
#11 0xffffffff80bcae5d at ithread_loop+0x24d
#12 0xffffffff80bc7c5e at fork_exit+0x7e
#13 0xffffffff8106282e at fork_trampoline+0xe
Uptime: 5h36m39s
Dumping 7281 out of 65454 MB:..1%..11%..21%..31%..41%..51%..61%..71%..81%..91%

__curthread () at /usr/src/sys/amd64/include/pcpu_aux.h:55
55              __asm("movq %%gs:%P1,%0" : "=r" (td) : "n" (offsetof(struct pcpu,
(kgdb) #0  __curthread () at /usr/src/sys/amd64/include/pcpu_aux.h:55
#1  doadump (textdump=<optimized out>)
     at /usr/src/sys/kern/kern_shutdown.c:399
#2  0xffffffff80c09916 in kern_reboot (howto=260)
     at /usr/src/sys/kern/kern_shutdown.c:486
#3  0xffffffff80c09d90 in vpanic (fmt=<optimized out>, ap=<optimized out>)
     at /usr/src/sys/kern/kern_shutdown.c:919
#4  0xffffffff80c09b93 in panic (fmt=<unavailable>)
     at /usr/src/sys/kern/kern_shutdown.c:843
#5  0xffffffff8108b187 in trap_fatal (frame=0xfffffe0115bc4600, eva=8)
     at /usr/src/sys/amd64/amd64/trap.c:915
#6  0xffffffff8108b1df in trap_pfault (frame=frame at entry=0xfffffe0115bc4600,
     usermode=false, signo=<optimized out>, signo at entry=0x0,
     ucode=<optimized out>, ucode at entry=0x0)
     at /usr/src/sys/amd64/amd64/trap.c:732
#7  0xffffffff8108a83d in trap (frame=0xfffffe0115bc4600)
     at /usr/src/sys/amd64/amd64/trap.c:398
#8  <signal handler called>
#9  sbcut_internal (sb=0xfffff8043bc00610, len=57, len at entry=304)
     at /usr/src/sys/kern/uipc_sockbuf.c:1491
#10 0xffffffff80ca5b8a in sbcut_locked (sb=0xfffff8043bc00610,
     len=-1796951296, len at entry=304) at /usr/src/sys/kern/uipc_sockbuf.c:1591
#11 0xffffffff80dbf0ae in tcp_do_segment (m=0xfffff8024b9a6900,
     th=<optimized out>, so=<optimized out>, tp=<optimized out>,
     drop_hdrlen=52, tlen=<optimized out>, iptos=0 '\000')
     at /usr/src/sys/netinet/tcp_input.c:2918
#12 0xffffffff80dbd21e in tcp_input (mp=<optimized out>,
     offp=<optimized out>, proto=<optimized out>)
     at /usr/src/sys/netinet/tcp_input.c:1382
#13 0xffffffff80dafc15 in ip_input (m=0x0)
     at /usr/src/sys/netinet/ip_input.c:829
#14 0xffffffff80d3fa7b in netisr_process_workstream_proto (
     nwsp=<optimized out>, proto=1) at /usr/src/sys/net/netisr.c:919
#15 swi_net (arg=<optimized out>) at /usr/src/sys/net/netisr.c:966
#16 0xffffffff80bcae5d in intr_event_execute_handlers (p=<optimized out>,
     ie=0xfffff80003b88c00) at /usr/src/sys/kern/kern_intr.c:1168
#17 ithread_execute_handlers (p=<optimized out>, ie=0xfffff80003b88c00)
     at /usr/src/sys/kern/kern_intr.c:1181
#18 ithread_loop (arg=arg at entry=0xfffff80003b95d20)
     at /usr/src/sys/kern/kern_intr.c:1269
#19 0xffffffff80bc7c5e in fork_exit (
     callout=0xffffffff80bcac10 <ithread_loop>, arg=0xfffff80003b95d20,
     frame=0xfffffe0115bc4b00) at /usr/src/sys/kern/kern_fork.c:1069


--- #2


Unread portion of the kernel message buffer:
panic: sbappendaddr_locked
cpuid = 2
time = 1620181490
KDB: stack backtrace:
#0 0xffffffff80c57345 at kdb_backtrace+0x65
#1 0xffffffff80c09d21 at vpanic+0x181
#2 0xffffffff80c09b93 at panic+0x43
#3 0xffffffff80ca51e0 at sbappendaddr_locked_internal+0
#4 0xffffffff82c4efd0 at divert_packet+0x1a0
#5 0xffffffff82c2bc81 at ipfw_check_packet+0x2c1
#6 0xffffffff80d41f87 at pfil_run_hooks+0x97
#7 0xffffffff80dafeb5 at ip_input+0x3c5
#8 0xffffffff80d3f2da at netisr_dispatch_src+0xca
#9 0xffffffff80d23a68 at ether_demux+0x148
#10 0xffffffff80d24dec at ether_nh_input+0x34c
#11 0xffffffff80d3f2da at netisr_dispatch_src+0xca
#12 0xffffffff80d23eb9 at ether_input+0x69
#13 0xffffffff80d2074a at if_input+0xa
#14 0xffffffff8060a98e at bge_rxeof+0x49e
#15 0xffffffff80607f27 at bge_intr_task+0x1a7
#16 0xffffffff80c6afe1 at taskqueue_run_locked+0x181
#17 0xffffffff80c6c2fc at taskqueue_thread_loop+0xac
Uptime: 2h21m11s
Dumping 8148 out of 65454 MB:..1%..11%..21%..31%..41%..51%..61%..71%..81%..91%

__curthread () at /usr/src/sys/amd64/include/pcpu_aux.h:55
55              __asm("movq %%gs:%P1,%0" : "=r" (td) : "n" (offsetof(struct pcpu,
(kgdb) #0  __curthread () at /usr/src/sys/amd64/include/pcpu_aux.h:55
#1  doadump (textdump=<optimized out>)
     at /usr/src/sys/kern/kern_shutdown.c:399
#2  0xffffffff80c09916 in kern_reboot (howto=260)
     at /usr/src/sys/kern/kern_shutdown.c:486
#3  0xffffffff80c09d90 in vpanic (fmt=<optimized out>, ap=<optimized out>)
     at /usr/src/sys/kern/kern_shutdown.c:919
#4  0xffffffff80c09b93 in panic (fmt=<unavailable>)
     at /usr/src/sys/kern/kern_shutdown.c:843
#5  0xffffffff80ca51e0 in sbappendaddr_locked (sb=0xfffff8002829a8a8,
     asa=0xfffffe0115ebc5a0, m0=0xfffff804a977b700, control=0x0)
     at /usr/src/sys/kern/uipc_sockbuf.c:1198
#6  0xffffffff82c4efd0 in divert_packet (m=0xfffff804a977b700,
     incoming=<optimized out>) at /usr/src/sys/netinet/ip_divert.c:285
#7  0xffffffff82c2bc81 in ipfw_divert (m0=0xfffffe0115ebc760,
     args=0xfffffe0115ebc610, tee=<optimized out>)
     at /usr/src/sys/netpfil/ipfw/ip_fw_pfil.c:525
#8  ipfw_check_packet (m0=0xfffffe0115ebc760, ifp=0xfffff8000506f000,
     flags=65536, ruleset=<optimized out>, inp=0x0)
     at /usr/src/sys/netpfil/ipfw/ip_fw_pfil.c:283
#9  0xffffffff80d41f87 in pfil_run_hooks (head=<optimized out>, p=...,
     ifp=ifp at entry=0xfffff8000506f000, flags=flags at entry=65536,
     inp=inp at entry=0x0) at /usr/src/sys/net/pfil.c:187
#10 0xffffffff80dafeb5 in ip_input (m=0x0)
     at /usr/src/sys/netinet/ip_input.c:610
#11 0xffffffff80d3f2da in netisr_dispatch_src (proto=1,
     source=<optimized out>, source at entry=0, m=<unavailable>)
     at /usr/src/sys/net/netisr.c:1143
#12 0xffffffff80d3f5cf in netisr_dispatch (proto=<unavailable>,
     m=<unavailable>) at /usr/src/sys/net/netisr.c:1234
#13 0xffffffff80d23a68 in ether_demux (ifp=ifp at entry=0xfffff8000506f000,
     m=<unavailable>) at /usr/src/sys/net/if_ethersubr.c:923
#14 0xffffffff80d24dec in ether_input_internal (ifp=0xfffff8000506f000,
     m=<unavailable>) at /usr/src/sys/net/if_ethersubr.c:709
#15 ether_nh_input (m=<optimized out>) at /usr/src/sys/net/if_ethersubr.c:739
#16 0xffffffff80d3f2da in netisr_dispatch_src (proto=proto at entry=5,
     source=<optimized out>, source at entry=0, m=<unavailable>,
     m at entry=0xfffff804a977b700) at /usr/src/sys/net/netisr.c:1143
#17 0xffffffff80d3f5cf in netisr_dispatch (proto=<unavailable>,
     proto at entry=5, m=<unavailable>, m at entry=0xfffff804a977b700)
     at /usr/src/sys/net/netisr.c:1234
#18 0xffffffff80d23eb9 in ether_input (ifp=<optimized out>,
     ifp at entry=<error reading variable: value is not available>,
     m=0xfffff804a977b700,
     m at entry=<error reading variable: value is not available>)
     at /usr/src/sys/net/if_ethersubr.c:830
#19 0xffffffff80d2074a in if_input (ifp=<unavailable>,
     ifp at entry=0xfffff8000506f000, sendmp=<unavailable>,
     sendmp at entry=0xfffff804a977b700) at /usr/src/sys/net/if.c:4391
#20 0xffffffff8060a98e in bge_rxeof (sc=sc at entry=0xfffffe0115cd4000,
     rx_prod=rx_prod at entry=448, holdlck=holdlck at entry=0)
     at /usr/src/sys/dev/bge/if_bge.c:4412
#21 0xffffffff80607f27 in bge_intr_task (arg=0xfffffe0115cd4000,
     pending=<optimized out>) at /usr/src/sys/dev/bge/if_bge.c:4642
#22 0xffffffff80c6afe1 in taskqueue_run_locked (
     queue=queue at entry=0xfffff80005051d00)
     at /usr/src/sys/kern/subr_taskqueue.c:476
#23 0xffffffff80c6c2fc in taskqueue_thread_loop (arg=<optimized out>,
     arg at entry=0xfffffe0115cdb568) at /usr/src/sys/kern/subr_taskqueue.c:793
#24 0xffffffff80bc7c5e in fork_exit (
     callout=0xffffffff80c6c250 <taskqueue_thread_loop>,
     arg=0xfffffe0115cdb568, frame=0xfffffe0115ebcb00)
     at /usr/src/sys/kern/kern_fork.c:1069

More information about the freebsd-net mailing list