Fwd: kern/147890: [ufs] [regression] ufs-related lock problem in
RELENG_8 (18.04.2010 -> 20.04.2010)
Dmitry Pryanishnikov
lynx.ripe at gmail.com
Thu Jun 17 16:01:40 UTC 2010
I'm sorry for not replying to all recipients...
---------- Forwarded message ----------
From: Dmitry Pryanishnikov <lynx.ripe at gmail.com>
Date: 2010/6/17
Subject: Re: kern/147890: [ufs] [regression] ufs-related lock problem
in RELENG_8 (18.04.2010 -> 20.04.2010)
To: Kostik Belousov <kostikbel at gmail.com>
Hello!
2010/6/17 Kostik Belousov <kostikbel at gmail.com>:
> On Wed, Jun 16, 2010 at 10:20:03PM +0000, Dmitry Pryanishnikov wrote:
>> The following reply was made to PR kern/147890; it has been noted by GNATS.
>>
>> From: Dmitry Pryanishnikov <lynx.ripe at gmail.com>
>> To: Bruce Cran <bruce at cran.org.uk>
>> Cc: bug-followup at freebsd.org
>> Subject: Re: kern/147890: [ufs] [regression] ufs-related lock problem in
>> RELENG_8 (18.04.2010 -> 20.04.2010)
>> Date: Thu, 17 Jun 2010 01:12:17 +0300
>>
>> Hello!
>>
>> 2010/6/16 Bruce Cran <bruce at cran.org.uk>:
>> > Could you run the commands shown in
>> > http://www.freebsd.org/doc/en_US.ISO8859-1/books/developers-handbook/kerneldebug-deadlocks.html
>> > to get more information to help debug the problem please?
>>
>> Done - I've rebuilt the kernel from the sources as of 20.04.2010
>> with the missing debugging options, repeated the problem, turned
> I suspect you do not. In particular, DEBUG_VFS_LOCKS, DEBUG_LOCKS
> and WITNESS seems to be missed.
Actually they are present - here is the complete config of this kernel:
#
# DP, BSF8, 2010-06-16
ident LYNX
makeoptions DEBUG=-g #Build kernel with gdb(1) debug symbols
# This allows you to actually store this configuration file into
# the kernel binary itself. See config(8) for more details.
#
options INCLUDE_CONFIG_FILE # Include this file in kernel
#####################################################################
# CPU OPTIONS
cpu I686_CPU # aka Pentium Pro(tm)
#####################################################################
# Scheduler options:
options SCHED_ULE
#####################################################################
# SMP OPTIONS:
options SMP # Symmetric MultiProcessor Kernel
# SMP Debugging Options:
options PREEMPTION # Enable kernel thread preemption
options MUTEX_DEBUG
options WITNESS
device apic # I/O apic
#####################################################################
# COMPATIBILITY OPTIONS
options COMPAT_FREEBSD4
options COMPAT_FREEBSD5
options COMPAT_FREEBSD6
options COMPAT_FREEBSD7
options SYSVSHM
options SYSVSEM
options SYSVMSG
#####################################################################
# DEBUGGING OPTIONS
options KDB
options KDB_TRACE
options DDB
options DDB_NUMSYM
options BREAK_TO_DEBUGGER # BREAK on a serial console goes to DDB
options SYSCTL_DEBUG
options KTRACE #kernel tracing
options INVARIANTS
options INVARIANT_SUPPORT
options DEBUG_LOCKS
options DEBUG_VFS_LOCKS
options DIAGNOSTIC
options DEADLKRES #software deadlock resolver thread
#####################################################################
# NETWORKING OPTIONS
options INET #Internet communications protocols
options INET6 #IPv6 communications protocols
options IPSEC #IP security (requires device crypto)
options IPSEC_NAT_T #NAT-T support, UDP encap of ESP
device ether #Generic Ethernet
device loop #Network loopback device
device bpf #Berkeley packet filter
device enc #IPsec interface
options IPFIREWALL #firewall
options IPFIREWALL_VERBOSE #enable logging to syslogd(8)
options IPFIREWALL_VERBOSE_LIMIT=100 #limit verbosity
options IPFIREWALL_FORWARD #packet destination changes
options IPFIREWALL_NAT #ipfw kernel nat support
options LIBALIAS # libalias library, performing NAT
options BPF_JITTER #support for BPF just-in-time compiler
#####################################################################
# FILESYSTEM OPTIONS
options FFS #Fast filesystem
options SOFTUPDATES
options UFS_EXTATTR
options UFS_ACL
options UFS_DIRHASH
options UFS_GJOURNAL # Gjournal-based UFS journaling support.
options QUOTA #enable disk quotas
options SUIDDIR #The suiddir hack
#####################################################################
# POSIX P1003.1B
options _KPOSIX_PRIORITY_SCHEDULING
options P1003_1B_SEMAPHORES
#####################################################################
# MISCELLANEOUS DEVICES AND OPTIONS
device pty #Pseudo ttys
# Size of the kernel message buffer. Should be N * pagesize.
options MSGBUF_SIZE=131072
#####################################################################
# crypto subsystem
device crypto # core crypto support
#####################################################################
# HARDWARE DEVICE CONFIGURATION
# Bus support.
device pci
# The keyboard controller; it controls the keyboard and the PS/2 mouse.
device atkbdc # AT keyboard controller
device atkbd # AT keyboard
device psm # PS/2 mouse
options KBD_INSTALL_CDEV # install a CDEV entry in /dev
device vga # VGA video card driver
device splash # Splash screen and screen saver support
options X86BIOS # x86 real mode BIOS emulator,
required by atkbdc/dpms/vesa
options PRINTF_BUFR_SIZE=128 # Prevent printf output being
interspersed.
# The syscons console driver (SCO color console compatible).
device sc
options SC_HISTORY_SIZE=20000
options SC_TWOBUTTON_MOUSE
#options TEKEN_UTF8 # UTF-8 output handling
#options TEKEN_XTERM # xterm-style terminal emulation
device uart # Generic UART driver
# ATA and ATAPI devices
device ata
device atadisk # ATA disk drives
options ATA_STATIC_ID # Static device numbering
# SCSI peripherals
device scbus # SCSI bus (required for SCSI)
device da # Direct Access (disks)
device cd # CD
device pass # Passthrough device (direct SCSI access)
# Sound
options SND_PCM_64 # Process 32bit samples through 64bit integer
Also I've commented out 'debug.witness.watch=0' in /boot/loader.conf
before booting this kernel.
>
>> ddb(4) capture buffer on, issued recommended ddb commands, then called
>> 'doadump'; then rebooted and extracted capture buffer contents from
>> the crashdump using ddb(8).
> Due to missed options, we cannot trace the lock chains and see why the
> issue happens.
Please specify what options are still missing in my config.
>>
>> db> ps
>>
>> pid ppid pgrp uid state wmesg wchan cmd
>> 2143 1846 2143 0 S+ ufs 0xcd466dc8 sync
>> 2131 1 1911 1000 S ufs 0xcd466dc8 initial thread
> ...
>> 18 0 0 0 SL ufs 0xcd466dc8 [syncer]
> ....
>> Tracing command syncer pid 18 tid 100067 td 0xc7383000
>> sched_switch(c7383000,0,104,191,e660f504,...) at 0xc05292c0 = sched_switch+0x330
>> mi_switch(104,0,c0745097,1eb,50,...) at 0xc0510680 = mi_switch+0x200
>> sleepq_switch(c7383000,0,c0745097,260,50,...) at 0xc054028f =
>> sleepq_switch+0x15f
>> sleepq_wait(cd466dc8,50,c07393ba,4,0,...) at 0xc0540f63 = sleepq_wait+0x63
>> __lockmgr_args(cd466dc8,80100,cd466e30,0,0,...) at 0xc04f087f =
>> __lockmgr_args+0xb1f
>> ffs_lock(e732cb34,c054944b,c0763314,80100,cd466d70,...) at 0xc067e9f1
>> = ffs_lock+0xa1
>> VOP_LOCK1_APV(c07a47c0,e732cb34,c73830a4,c07b7a20,cd466d70,...) at
>> 0xc070c945 = VOP_LOCK1_APV+0xb5
>> _vn_lock(cd466d70,80100,c074e6ce,82b,4,...) at 0xc05a2fd8 = _vn_lock+0x78
>> vget(cd466d70,80100,c7383000,39a,c763b870,...) at 0xc05969db = vget+0xbb
>> qsync(c763b870,0,c0761f41,552,4,...) at 0xc0687db7 = qsync+0x197
>> ffs_sync(c763b870,3,c074e6ce,d6b,c763b870,...) at 0xc067a139 = ffs_sync+0x349
>> sync_fsync(e732cc7c,c0772c36,c76b8778,e732cc7c,c76b86b8,...) at
>> 0xc059812f = sync_fsync+0x18f
>> VOP_FSYNC_APV(c0799bc0,e732cc7c,c074e6ce,6a5,c7383000,...) at
>> 0xc070a545 = VOP_FSYNC_APV+0xc5
>> sync_vnode(c093b618,c093b604,3e8,6d4,4e20,...) at 0xc059840b = sync_vnode+0x16b
>> sched_sync(0,e732cd38,c073bc0b,343,c759f7f8,...) at 0xc0598753 =
>> sched_sync+0x273
>> fork_exit(c05984e0,0,e732cd38) at 0xc04dddf8 = fork_exit+0xb8
>> fork_trampoline() at 0xc06d6b80 = fork_trampoline+0x8
>> --- trap 0, eip = 0, esp = 0xe732cd70, ebp = 0 ---
> Do you have custoom kernel build, in particular, are quotas compiled in ?
I've posted my config during the initial PR submission, and
re-posting updated version now. Yes, QUOTA compiled in.
It is turned on for /usr.
> What are the drives ?
ad0: 152626MB <SAMSUNG SP1614N TM100-30> at ata0-master UDMA100
ad14: 953869MB <Seagate ST31000520AS CC32> at ata7-master UDMA100 SATA 3Gb/s
dmitry at lynx$ mount -t ufs
/dev/ad0s3a on / (ufs, local, synchronous)
/dev/ad0s3d on /usr (ufs, local, with quotas, soft-updates)
/dev/ad14s4a on /databig (ufs, local, soft-updates)
> Having to pull the generic information about the problematic system
> does not make debugging faster.
I'm ready to provide any additional information - just not sure what
else is significant for this problem.
--
Sincerely, Dmitry
nic-hdl: LYNX-RIPE
More information about the freebsd-fs
mailing list