page fault while in kernel mode - after upgrade from 12.2 to 13.0

[Michael Schmiedgen]

Hi List,

if I start a Samba jail, after a few seconds the system crashes. Very reproducible.

System has ~10 jails and 3 bhyve VMs. Dell server, Xeon E3-1240, 64GB RAM, 3 way mirror ZFS.

It also occurs a few seconds after I start a phone call using the SIP VM of that machine,
very strange.

I got some log messages suggesting raising somaxconn, so I did

kern.ipc.somaxconn=4096

in sysctl.conf


Below some debug information, please let me know if I should provide further information.

Should I open a bug or something?

Thank you very much!
   Michael



Fatal trap 12: page fault while in kernel mode
cpuid = 0; apic id = 00
fault virtual address   = 0x0
fault code              = supervisor read data, page not present
instruction pointer     = 0x20:0xffffffff80ca52c0
stack pointer           = 0x28:0xfffffe019d039650
frame pointer           = 0x28:0xfffffe019d039690
code segment            = base 0x0, limit 0xfffff, type 0x1b
                         = DPL 0, pres 1, long 1, def32 0, gran 1
processor eflags        = interrupt enabled, resume, IOPL = 0
current process         = 649 (devd)
trap number             = 12
panic: page fault
cpuid = 0
time = 1620061253
KDB: stack backtrace:
#0 0xffffffff80c57345 at kdb_backtrace+0x65
#1 0xffffffff80c09d21 at vpanic+0x181
#2 0xffffffff80c09b93 at panic+0x43
#3 0xffffffff8108b187 at trap_fatal+0x387
#4 0xffffffff8108b1df at trap_pfault+0x4f
#5 0xffffffff8108a83d at trap+0x27d
#6 0xffffffff810617a8 at calltrap+0x8
#7 0xffffffff80ca51c3 at sbappendaddr_locked+0x93
#8 0xffffffff80cb437a at uipc_send+0x73a
#9 0xffffffff80ca9053 at sosend_generic+0x633
#10 0xffffffff80ca94e0 at sosend+0x50
#11 0xffffffff80caff2e at kern_sendit+0x20e
#12 0xffffffff80cb032b at sendit+0x1db
#13 0xffffffff80cb013d at sys_sendto+0x4d
#14 0xffffffff8108ba8c at amd64_syscall+0x10c
#15 0xffffffff810620ce at fast_syscall_common+0xf8
Uptime: 2m2s
Dumping 2373 out of 65454 MB:..1%..11%..21%..31%..41%..51%..61%..71%..81%..91%

__curthread () at /usr/src/sys/amd64/include/pcpu_aux.h:55
55              __asm("movq %%gs:%P1,%0" : "=r" (td) : "n" (offsetof(struct pcpu,
(kgdb) list *0xffffffff80ca52c0
0xffffffff80ca52c0 is in sbappendaddr_locked_internal (/usr/src/sys/kern/uipc_sockbuf.c:1169).
1164            if (ctrl_last)
1165                    ctrl_last->m_next = m0; /* concatenate data to control */
1166            else
1167                    control = m0;
1168            m->m_next = control;
1169            for (n = m; n->m_next != NULL; n = n->m_next)
1170                    sballoc(sb, n);
1171            sballoc(sb, n);
1172            nlast = n;
1173            SBLINKRECORD(sb, m);
(kgdb) backtrace
#0  __curthread () at /usr/src/sys/amd64/include/pcpu_aux.h:55
#1  doadump (textdump=<optimized out>) at /usr/src/sys/kern/kern_shutdown.c:399
#2  0xffffffff80c09916 in kern_reboot (howto=260) at /usr/src/sys/kern/kern_shutdown.c:486
#3  0xffffffff80c09d90 in vpanic (fmt=<optimized out>, ap=<optimized out>) at /usr/src/sys/kern/kern_shutdown.c:919
#4  0xffffffff80c09b93 in panic (fmt=<unavailable>) at /usr/src/sys/kern/kern_shutdown.c:843
#5  0xffffffff8108b187 in trap_fatal (frame=0xfffffe019d039590, eva=0) at /usr/src/sys/amd64/amd64/trap.c:915
#6  0xffffffff8108b1df in trap_pfault (frame=frame at entry=0xfffffe019d039590, usermode=false, signo=<optimized out>, signo at entry=0x0, ucode=<optimized
out>, ucode at entry=0x0)
     at /usr/src/sys/amd64/amd64/trap.c:732
#7  0xffffffff8108a83d in trap (frame=0xfffffe019d039590) at /usr/src/sys/amd64/amd64/trap.c:398
#8  <signal handler called>
#9  sbappendaddr_locked_internal (sb=sb at entry=0xfffff800447ef4f8, asa=asa at entry=0xffffffff815cde60 <sun_noname>, m0=<optimized out>,
m0 at entry=0xfffff8008b186500, control=0xfffff8008b186500,
     control at entry=0x0, ctrl_last=<optimized out>) at /usr/src/sys/kern/uipc_sockbuf.c:1169
#10 0xffffffff80ca51c3 in sbappendaddr_locked (sb=sb at entry=0xfffff800447ef4f8, asa=asa at entry=0xffffffff815cde60 <sun_noname>,
m0=m0 at entry=0xfffff8008b186500, control=0x0)
     at /usr/src/sys/kern/uipc_sockbuf.c:1205
#11 0xffffffff80cb437a in uipc_send (so=<optimized out>, flags=0, m=0xfffff8008b186500, nam=<optimized out>, control=0x10, td=<optimized out>) at
/usr/src/sys/kern/uipc_usrreq.c:1056
#12 0xffffffff80ca9053 in sosend_generic (so=0xfffff800444abb10, addr=0x0, uio=<optimized out>, top=0xfffff8008b186500, control=0x0, flags=0,
td=0xfffffe0165ddc500)
     at /usr/src/sys/kern/uipc_socket.c:1755
#13 0xffffffff80ca94e0 in sosend (so=0x100, so at entry=0xfffff800444abb10, addr=0xb5ea5000, uio=0xfffff8008b186500, uio at entry=0xfffffe019d039898,
top=0x10, top at entry=0x0,
     control=control at entry=0x0, flags=272, flags at entry=0, td=0xfffffe0165ddc500) at /usr/src/sys/kern/uipc_socket.c:1810
#14 0xffffffff80caff2e in kern_sendit (td=<optimized out>, td at entry=0xfffffe0165ddc500, s=8, mp=<optimized out>, mp at entry=0xfffffe019d039980, flags=0,
control=0x0,
     segflg=segflg at entry=UIO_USERSPACE) at /usr/src/sys/kern/uipc_syscalls.c:798
#15 0xffffffff80cb032b in sendit (td=0xfffffe0165ddc500, s=-1242935296, mp=mp at entry=0xfffffe019d039980, flags=16) at /usr/src/sys/kern/uipc_syscalls.c:723
#16 0xffffffff80cb013d in sys_sendto (td=0x100, uap=<optimized out>) at /usr/src/sys/kern/uipc_syscalls.c:841
#17 0xffffffff8108ba8c in syscallenter (td=0xfffffe0165ddc500) at /usr/src/sys/amd64/amd64/../../kern/subr_syscall.c:189
#18 amd64_syscall (td=0xfffffe0165ddc500, traced=0) at /usr/src/sys/amd64/amd64/trap.c:1156
#19 <signal handler called>
#20 0x00000000002858ea in ?? ()