Repeated similar panics on -STABLE

Wed Apr 2 05:44:32 PST 2003

Hello!

We have three machines under relatively high load.  They are running -STABLE
on the same hardware with 2 processors (and SMP kernel).
Periodically (approximately once a week) they panic with similar symptoms:

# gdb -k kernel.debug vmcore.2
GNU gdb 4.18 (FreeBSD)
Copyright 1998 Free Software Foundation, Inc.
GDB is free software, covered by the GNU General Public License, and you are
welcome to change it and/or distribute copies of it under certain conditions.
Type "show copying" to see the conditions.
There is absolutely no warranty for GDB.  Type "show warranty" for details.
This GDB was configured as "i386-unknown-freebsd"...Deprecated bfd_read called a
t /mnt/se3/releng_4/src/gnu/usr.bin/binutils/gdb/../../../../contrib/gdb/gdb/dbx
read.c line 2627 in elfstab_build_psymtabs
Deprecated bfd_read called at /mnt/se3/releng_4/src/gnu/usr.bin/binutils/gdb/../
../../../contrib/gdb/gdb/dbxread.c line 933 in fill_symbuf
SMP 2 cpus
IdlePTD at phsyical address 0x0034f000
initial pcb at physical address 0x002bd6a0
panicstr: page fault
panic messages:
---
Fatal trap 12: page fault while in kernel mode
mp_lock = 01000002; cpuid = 1; lapic.id = 00000000
fault virtual address   = 0x5cdd8000
fault code              = supervisor read, page not present
instruction pointer     = 0x8:0xc015daff
stack pointer           = 0x10:0xeb278e44
frame pointer           = 0x10:0xeb278e68
code segment            = base 0x0, limit 0xfffff, type 0x1b
                        = DPL 0, pres 1, def32 1, gran 1
processor eflags        = interrupt enabled, resume, IOPL = 0
current process         = 65648 (cronolog)
interrupt mask          = net tty bio cam  <- SMP: XXX
trap number             = 12
panic: page fault
mp_lock = 01000002; cpuid = 1; lapic.id = 00000000
boot() called on cpu#1
syncing disks...
Fatal trap 12: page fault while in kernel mode
mp_lock = 01000003; cpuid = 1; lapic.id = 00000000
fault virtual address   = 0x5cdd8000
fault code              = supervisor read, page not present
instruction pointer     = 0x8:0xc015daff
stack pointer           = 0x10:0xeb278b68
frame pointer           = 0x10:0xeb278b8c
code segment            = base 0x0, limit 0xfffff, type 0x1b
                        = DPL 0, pres 1, def32 1, gran 1
processor eflags        = interrupt enabled, resume, IOPL = 0
current process         = 65648 (cronolog)
interrupt mask          = net tty bio cam  <- SMP: XXX
trap number             = 12
panic: page fault
mp_lock = 01000003; cpuid = 1; lapic.id = 00000000
boot() called on cpu#1
Uptime: 5d0h48m54s
dumping to dev #da/0x20001, offset 2097280
dump 1023 1022 1021 1020 1019 1018 1017 1016 1015 1014 1013 1012 1011 1010 1009
<snip>
---
#0  dumpsys () at /mnt/se3/releng_4/src/sys/kern/kern_shutdown.c:487
487             if (dumping++) {
(kgdb) bt
#0  dumpsys () at /mnt/se3/releng_4/src/sys/kern/kern_shutdown.c:487
#1  0xc01620c6 in boot (howto=260)
    at /mnt/se3/releng_4/src/sys/kern/kern_shutdown.c:316
#2  0xc0162549 in panic (fmt=0xc028e3b9 "%s")
    at /mnt/se3/releng_4/src/sys/kern/kern_shutdown.c:595
#3  0xc0251b1a in trap_fatal (frame=0xeb278b28, eva=1558020096)
    at /mnt/se3/releng_4/src/sys/i386/i386/trap.c:974
#4  0xc0251775 in trap_pfault (frame=0xeb278b28, usermode=0, eva=1558020096)
    at /mnt/se3/releng_4/src/sys/i386/i386/trap.c:867
#5  0xc02512b7 in trap (frame={tf_fs = -65512, tf_es = -941031408,
      tf_ds = -942997488, tf_edi = -1070937504, tf_esi = -730301488,
      tf_ebp = -349729908, tf_isp = -349729964, tf_ebx = -1070870564,
      tf_edx = 1558020096, tf_ecx = 7, tf_eax = 128, tf_trapno = 12,
      tf_err = 0, tf_eip = -1072309505, tf_cs = 8, tf_eflags = 66054,
      tf_esp = 33281, tf_ss = -730301488})
    at /mnt/se3/releng_4/src/sys/i386/i386/trap.c:466
#6  0xc015daff in malloc (size=128, type=0xc02aca60, flags=2)
    at /mnt/se3/releng_4/src/sys/kern/kern_malloc.c:243
#7  0xc02085e1 in initiate_write_inodeblock (inodedep=0xc8e69400,
    bp=0xd4787bd0) at /mnt/se3/releng_4/src/sys/ufs/ffs/ffs_softdep.c:3091
#8  0xc02083b3 in softdep_disk_io_initiation (bp=0xd4787bd0)
    at /mnt/se3/releng_4/src/sys/ufs/ffs/ffs_softdep.c:2965
#9  0xc019d51a in spec_strategy (ap=0xeb278c0c)
    at /mnt/se3/releng_4/src/sys/miscfs/specfs/spec_vnops.c:453
#10 0xc0188cab in bwrite (bp=0xd4787bd0) at vnode_if.h:944
#11 0xc018e98f in vop_stdbwrite (ap=0xeb278c6c)
    at /mnt/se3/releng_4/src/sys/kern/vfs_default.c:344
#12 0xc018e791 in vop_defaultop (ap=0xeb278c6c)
    at /mnt/se3/releng_4/src/sys/kern/vfs_default.c:152
#13 0xc0189ce5 in vfs_bio_awrite (bp=0xd4787bd0) at vnode_if.h:1193
#14 0xc019d33f in spec_fsync (ap=0xeb278cd4)
    at /mnt/se3/releng_4/src/sys/miscfs/specfs/spec_vnops.c:391
#15 0xc020ca4d in ffs_sync (mp=0xc7ea1a00, waitfor=2, cred=0xc1c6e900,
    p=0xc02d25e0) at vnode_if.h:558
#16 0xc01941b7 in sync (p=0xc02d25e0, uap=0x0)
    at /mnt/se3/releng_4/src/sys/kern/vfs_syscalls.c:576
#17 0xc0161e7c in boot (howto=256)
    at /mnt/se3/releng_4/src/sys/kern/kern_shutdown.c:235
#18 0xc0162549 in panic (fmt=0xc028e3b9 "%s")
    at /mnt/se3/releng_4/src/sys/kern/kern_shutdown.c:595
#19 0xc0251b1a in trap_fatal (frame=0xeb278e04, eva=1558020096)
    at /mnt/se3/releng_4/src/sys/i386/i386/trap.c:974
#20 0xc0251775 in trap_pfault (frame=0xeb278e04, usermode=0, eva=1558020096)
    at /mnt/se3/releng_4/src/sys/i386/i386/trap.c:867
#21 0xc02512b7 in trap (frame={tf_fs = -1072300008, tf_es = -361627632,
      tf_ds = 16, tf_edi = -1070989600, tf_esi = -349729108,
      tf_ebp = -349729176, tf_isp = -349729232, tf_ebx = -1070870564,
      tf_edx = 1558020096, tf_ecx = 7, tf_eax = 128, tf_trapno = 12,
      tf_err = 0, tf_eip = -1072309505, tf_cs = 8, tf_eflags = 66054,
      tf_esp = 0, tf_ss = -349729108})
    at /mnt/se3/releng_4/src/sys/i386/i386/trap.c:466
#22 0xc015daff in malloc (size=72, type=0xc029fee0, flags=0)
    at /mnt/se3/releng_4/src/sys/kern/kern_malloc.c:243
#23 0xc015a3fe in exit1 (p=0xea726820, rv=15)
    at /mnt/se3/releng_4/src/sys/kern/kern_exit.c:166
#24 0xc0164011 in sigexit (p=0xea726820, sig=15)
    at /mnt/se3/releng_4/src/sys/kern/kern_sig.c:1503
#25 0xc0163d9c in postsig (sig=15)
    at /mnt/se3/releng_4/src/sys/kern/kern_sig.c:1406
#26 0xc0251fc5 in syscall2 (frame={tf_fs = 47, tf_es = 47, tf_ds = 47,
      tf_edi = 174, tf_esi = 1049187701, tf_ebp = -1077936960,
      tf_isp = -349728812, tf_ebx = 1, tf_edx = 3, tf_ecx = -1078002496,
      tf_eax = 3, tf_trapno = 7, tf_err = 2, tf_eip = 672039098, tf_cs = 31,
      tf_eflags = 659, tf_esp = -1078069180, tf_ss = 47})
    at /mnt/se3/releng_4/src/sys/i386/i386/trap.c:174
#27 0xc023e69b in Xint0x80_syscall ()
#28 0x8048b7f in ?? ()
(kgdb) 

Backtraces of all crashes look very similar.
I can provide crash dump(s) if someone interested.