kern/149797: panic: mutex sleepq chain not owned at
/usr/src/sys/kern/kern_mutex.c:265
sysadmin
freebsd-bugs at desert.net
Thu Aug 19 07:00:14 UTC 2010
>Number: 149797
>Category: kern
>Synopsis: panic: mutex sleepq chain not owned at /usr/src/sys/kern/kern_mutex.c:265
>Confidential: no
>Severity: serious
>Priority: high
>Responsible: freebsd-bugs
>State: open
>Quarter:
>Keywords:
>Date-Required:
>Class: sw-bug
>Submitter-Id: current-users
>Arrival-Date: Thu Aug 19 07:00:13 UTC 2010
>Closed-Date:
>Last-Modified:
>Originator: sysadmin
>Release: 8.0-RELEASE
>Organization:
DesertNet
>Environment:
FreeBSD cobalt.desert.net 8.0-RELEASE-p3 FreeBSD 8.0-RELEASE-p3 #3: Tue Aug 10 18:11:50 MST 2010 nick at cobalt.desert.net:/usr/obj/usr/src/sys/DELL_64BIT_80 amd64
>Description:
Been having panics with 8.0 and 8.1 on certain kinds of workloads. Workloads are CPU- and IO-heavy. Lots of kqueue, in perl. Machines have filesystems with lots of small files in relatively large directories -- panics occur with these files on both ufs and zfs.
7.x does not exhibit these panics no matter what we try.
Text output from "show pcpu; show allpcpu; bt; ps; show locks; show alllocks; show lockedvnods; alltrace" available here:
http://foundation.gyrobase.com/freebsd/cobalt-crashdump-20100818.txt
Excerpts:
panic: mutex sleepq chain not owned at /usr/src/sys/kern/kern_mutex.c:265^M
cpuid = 12^M
KDB: enter: panic^M
[thread pid 82678 tid 102172 ]
..
db:0:kdb.enter.panic> bt
Tracing pid 82678 tid 102172 td 0xffffff03bc0d0390
kdb_enter() at kdb_enter+0x3d
panic() at panic+0x17b
_mtx_assert() at _mtx_assert+0xdc
_mtx_unlock_spin_flags() at _mtx_unlock_spin_flags+0x7a
_callout_stop_safe() at _callout_stop_safe+0x2f2
filt_timerdetach() at filt_timerdetach+0x25
kqueue_register() at kqueue_register+0x797
kern_kevent() at kern_kevent+0x102
kevent() at kevent+0x90
syscall() at syscall+0x1d0
Xfast_syscall() at Xfast_syscall+0xe1
--- syscall (363, FreeBSD ELF64, kevent), rip = 0x800d179dc, rsp = 0x7fffffffe848, rbp = 0x8008f7c80 ---
..
db:0:kdb.enter.panic> show lockedvnods
Locked vnodes
0xffffff0bce6bd4e0: tag zfs, type VDIR
usecount 1, writecount 0, refcount 2 mountedhere 0
flags ()
lock type zfs: SHARED (count 1)
#0 0xffffffff8033ea3e at __lockmgr_args+0x4fe
#1 0xffffffff803d4829 at vop_stdlock+0x39
#2 0xffffffff8054c32b at VOP_LOCK1_APV+0x9b
#3 0xffffffff803f17f7 at _vn_lock+0x57
#4 0xffffffff803e5b8b at vget+0x7b
#5 0xffffffff803d2340 at cache_lookup+0x4e0
#6 0xffffffff803d2660 at vfs_cache_lookup+0xc0
#7 0xffffffff8054d207 at VOP_LOOKUP_APV+0xb7
#8 0xffffffff803d92e3 at lookup+0x3d3
#9 0xffffffff803da2b9 at namei+0x4a9
#10 0xffffffff803f1157 at vn_open_cred+0x1d7
#11 0xffffffff803ef419 at kern_openat+0x179
#12 0xffffffff8051f360 at syscall+0x1d0
#13 0xffffffff80505881 at Xfast_syscall+0xe1
/etc/make.conf:
KERNCONF= DELL_64BIT_80
BOOT_COMCONSOLE_SPEED= 57600
PERL_PORT= perl5.8
WITHOUT_X11= yes
FORCE_PACKAGE= yes
BUILD_OPTIMIZED= yes
# added by use.perl 2010-07-06 11:11:36
PERL_VERSION=5.8.9
kernel config:
cpu HAMMER
ident DELL_64BIT_80
makeoptions DEBUG=-g # Build kernel with gdb(1) debug symbols
options SCHED_ULE # ULE scheduler
options PREEMPTION # Enable kernel thread preemption
options INET # InterNETworking
options FFS # Berkeley Fast Filesystem
options SOFTUPDATES # Enable FFS soft updates support
options UFS_ACL # Support for access control lists
options UFS_DIRHASH # Improve performance on big directories
options UFS_GJOURNAL # Enable gjournal-based UFS journaling
options MD_ROOT # MD is a potential root device
options NFSCLIENT # Network Filesystem Client
options NFSSERVER # Network Filesystem Server
options NFSLOCKD # Network Lock Manager
options NFS_ROOT # NFS usable as /, requires NFSCLIENT
options MSDOSFS # MSDOS Filesystem
options CD9660 # ISO 9660 Filesystem
options PROCFS # Process filesystem (requires PSEUDOFS)
options PSEUDOFS # Pseudo-filesystem framework
options GEOM_PART_GPT # GUID Partition Tables.
options GEOM_LABEL # Provides labelization
options COMPAT_43TTY # BSD 4.3 TTY compat (sgtty)
options COMPAT_IA32 # Compatible with i386 binaries
options COMPAT_FREEBSD4 # Compatible with FreeBSD4
options COMPAT_FREEBSD5 # Compatible with FreeBSD5
options COMPAT_FREEBSD6 # Compatible with FreeBSD6
options COMPAT_FREEBSD7 # Compatible with FreeBSD7
options SCSI_DELAY=5000 # Delay (in ms) before probing SCSI
options KTRACE # ktrace(1) support
options STACK # stack(9) support
options SYSVSHM # SYSV-style shared memory
options SYSVMSG # SYSV-style message queues
options SYSVSEM # SYSV-style semaphores
options P1003_1B_SEMAPHORES # POSIX-style semaphores
options _KPOSIX_PRIORITY_SCHEDULING # POSIX P1003_1B real-time extensions
options PRINTF_BUFR_SIZE=128 # Prevent printf output being interspersed.
options KBD_INSTALL_CDEV # install a CDEV entry in /dev
options HWPMC_HOOKS # Necessary kernel hooks for hwpmc(4)
options AUDIT # Security event auditing
options MAC # TrustedBSD MAC Framework
options FLOWTABLE # per-cpu routing cache
options KDB # Enable kernel debugger support.
options DDB # Support DDB.
options GDB # Support remote GDB.
options SMP # Symmetric MultiProcessor Kernel
options INVARIANTS
options INVARIANT_SUPPORT
options WITNESS
options DEBUG_LOCKS
options DEBUG_VFS_LOCKS
options DIAGNOSTIC
options NO_ADAPTIVE_MUTEXES
options NO_ADAPTIVE_RWLOCKS
options NO_ADAPTIVE_SX
options CONSPEED=57600
device vlan
device pf
device pflog
device carp
device crypto
device cryptodev
options ALTQ
options ALTQ_CBQ
options ALTQ_HFSC
options ALTQ_RED
options ALTQ_RIO
device cpufreq
device acpi
device pci
device fdc
device ata
device atadisk # ATA disk drives
device ataraid # ATA RAID drives
device atapicd # ATAPI CDROM drives
device atapifd # ATAPI floppy drives
device atapist # ATAPI tape drives
options ATA_STATIC_ID # Static device numbering
device scbus # SCSI bus (required for SCSI)
device ch # SCSI media changers
device da # Direct Access (disks)
device sa # Sequential Access (tape etc)
device cd # CD
device pass # Passthrough device (direct SCSI access)
device ses # SCSI Environmental Services (and SAF-TE)
device amr # AMI MegaRAID
device mfi # LSI MegaRAID SAS
device atkbdc # AT keyboard controller
device atkbd # AT keyboard
device kbdmux # keyboard multiplexer
device vga # VGA video card driver
device splash # Splash screen and screen saver support
device sc
device agp # support several AGP chipsets
device uart # Generic UART driver
device ppc
device ppbus # Parallel port bus (required)
device lpt # Printer
device ppi # Parallel port interface device
device em # Intel PRO/1000 Gigabit Ethernet Family
device miibus # MII bus support
device bce # Broadcom BCM5706/BCM5708 Gigabit Ethernet
device loop # Network loopback
device random # Entropy device
device ether # Ethernet support
device tun # Packet tunnel.
device pty # BSD-style compatibility pseudo ttys
device md # Memory "disks"
device firmware # firmware assist module
device bpf # Berkeley packet filter
device uhci # UHCI PCI->USB interface
device ohci # OHCI PCI->USB interface
device ehci # EHCI PCI->USB interface (USB 2.0)
device usb # USB Bus (required)
device uhid # "Human Interface Devices"
device ukbd # Keyboard
>How-To-Repeat:
It's likely a substantial portion of our workload would be required. If a FreeBSD developer would like serial console access to a crashed system, or to core dumps, that can be arranged. Contact freebsd-bugs at desert.net
>Fix:
>Release-Note:
>Audit-Trail:
>Unformatted:
More information about the freebsd-bugs
mailing list