FreeBSD 6.3 deadlock (interrupted pmap_remove_pages) with DDB
output
Stef Walter
stef-list at memberwebs.com
Mon Jun 16 11:10:35 UTC 2008
Stef Walter wrote:
> I've been trying to track down a deadlock on some newish production
> servers running FreeBSD 6.3-RELEASE-p2. The deadlock occurs on a
> specific (although mundane) hardware configuration, and each of several
> servers running this hardware deadlock about once per week.
>
> Although I suspect that this is not hardware related, from a (naive)
> perusal of the attached stack traces.
Here's another similar deadlock that occurred on an similar machine.
Last time pmap_enter was preempted by an interrupt, this time
pmap_remove_pages was preempted by a timer.
I tried changing machdep.cpu_idle_hlt to zero, based on a hint [1], but
that didn't seem to make a difference.
Again in this case the processes are waiting on the page queue mutex:
python (in trap > trap_pfault > vm_fault)
rsync (in ... cluster_read > getblk > getnewbuf > vfs_vmio_release)
And another rsync process is holding the vm page queue mutex:
rsync (in exit1 > vmspace_exit > pmap_remove_pages)
The rsync process in pmap_remove_pages was interrupted by a timer while
holding the vm page queue lock and then was not scheduled again.
Kernel config and full details attched.
I may try building the kernel without PREEMPTION to see if it fixes this
problem.
Again, any and all advice or hints are welcome. I really appreciate any
time spent on this.
Stef Walter
[1]
http://unix.derkeiler.com/Mailing-Lists/FreeBSD/current/2005-02/0508.html
-------------- next part --------------
#
# RACK1
#
machine i386
cpu I686_CPU
ident RACK1
# To statically compile in device wiring instead of /boot/device.hints
#hints "GENERIC.hints" # Default places to look for devices.
makeoptions DEBUG=-g # Build kernel with gdb(1) debug symbols
options SCHED_4BSD # 4BSD scheduler
options PREEMPTION # Enable kernel thread preemption
options INET # InterNETworking
options INET6 # IPv6 communications protocols
options FFS # Berkeley Fast Filesystem
options SOFTUPDATES # Enable FFS soft updates support
options UFS_ACL # Support for access control lists
options UFS_DIRHASH # Improve performance on big directories
options MD_ROOT # MD is a potential root device
options NFSCLIENT # Network Filesystem Client
options NFSSERVER # Network Filesystem Server
options NFS_ROOT # NFS usable as /, requires NFSCLIENT
options MSDOSFS # MSDOS Filesystem
options CD9660 # ISO 9660 Filesystem
options PROCFS # Process filesystem (requires PSEUDOFS)
options PSEUDOFS # Pseudo-filesystem framework
options GEOM_GPT # GUID Partition Tables.
options COMPAT_43 # Compatible with BSD 4.3 [KEEP THIS!]
options COMPAT_FREEBSD4 # Compatible with FreeBSD4
options COMPAT_FREEBSD5 # Compatible with FreeBSD5
options SCSI_DELAY=5000 # Delay (in ms) before probing SCSI
options KTRACE # ktrace(1) support
options SYSVSHM # SYSV-style shared memory
options SYSVMSG # SYSV-style message queues
options SYSVSEM # SYSV-style semaphores
options _KPOSIX_PRIORITY_SCHEDULING # POSIX P1003_1B real-time extensions
options KBD_INSTALL_CDEV # install a CDEV entry in /dev
options ADAPTIVE_GIANT # Giant mutex is adaptive.
device apic # I/O APIC
options SMP
options QUOTA
options FAST_IPSEC
options IPFIREWALL
options IPFIREWALL_DEFAULT_TO_ACCEPT
options IPFIREWALL_VERBOSE
options IPFIREWALL_FORWARD
options IPDIVERT
options DUMMYNET
options ALTQ
options ALTQ_CBQ
options ALTQ_RED
options ALTQ_RIO
options ALTQ_HFSC
options ALTQ_CDNR
options ALTQ_PRIQ
options KDB
options KDB_TRACE
options KDB_UNATTENDED
options DDB
options INVARIANTS
options INVARIANT_SUPPORT
options WITNESS
options WITNESS_SKIPSPIN
options DIAGNOSTIC
options BREAK_TO_DEBUGGER
options DEBUG_LOCKS
options DEBUG_VFS_LOCKS
device crypto
# Bus support.
device eisa
device pci
# Floppy drives
device fdc
# ATA and ATAPI devices
device ata
device atadisk # ATA disk drives
device ataraid # ATA RAID drives
device atapicd # ATAPI CDROM drives
#device atapifd # ATAPI floppy drives
#device atapist # ATAPI tape drives
options ATA_STATIC_ID # Static device numbering
# SCSI Controllers
#device ahb # EISA AHA1742 family
#device ahc # AHA2940 and onboard AIC7xxx devices
#options AHC_REG_PRETTY_PRINT # Print register bitfields in debug
# output. Adds ~128k to driver.
#device ahd # AHA39320/29320 and onboard AIC79xx devices
#options AHD_REG_PRETTY_PRINT # Print register bitfields in debug
# output. Adds ~215k to driver.
#device amd # AMD 53C974 (Tekram DC-390(T))
#device isp # Qlogic family
#device ispfw # Firmware for QLogic HBAs- normally a module
#device mpt # LSI-Logic MPT-Fusion
#device ncr # NCR/Symbios Logic
#device sym # NCR/Symbios Logic (newer chipsets + those of `ncr')
#device trm # Tekram DC395U/UW/F DC315U adapters
#device adv # Advansys SCSI adapters
#device adw # Advansys wide SCSI adapters
#device aha # Adaptec 154x SCSI adapters
#device aic # Adaptec 15[012]x SCSI adapters, AIC-6[23]60.
#device bt # Buslogic/Mylex MultiMaster SCSI adapters
#device ncv # NCR 53C500
#device nsp # Workbit Ninja SCSI-3
#device stg # TMC 18C30/18C50
# SCSI peripherals
device scbus # SCSI bus (required for SCSI)
#device ch # SCSI media changers
device da # Direct Access (disks)
#device sa # Sequential Access (tape etc)
#device cd # CD
#device pass # Passthrough device (direct SCSI access)
#device ses # SCSI Environmental Services (and SAF-TE)
# RAID controllers interfaced to the SCSI subsystem
#device amr # AMI MegaRAID
#device arcmsr # Areca SATA II RAID
#device asr # DPT SmartRAID V, VI and Adaptec SCSI RAID
#device ciss # Compaq Smart RAID 5*
#device dpt # DPT Smartcache III, IV - See NOTES for options
#device hptmv # Highpoint RocketRAID 182x
#device hptrr # Highpoint RocketRAID 17xx, 22xx, 23xx, 25xx
#device rr232x # Highpoint RocketRAID 232x
device iir # Intel Integrated RAID
#device ips # IBM (Adaptec) ServeRAID
#device mly # Mylex AcceleRAID/eXtremeRAID
#device twa # 3ware 9000 series PATA/SATA RAID
# RAID controllers
#device aac # Adaptec FSA RAID
#device aacp # SCSI passthrough for aac (requires CAM)
#device ida # Compaq Smart RAID
#device mfi # LSI MegaRAID SAS
#device mlx # Mylex DAC960 family
#device pst # Promise Supertrak SX6000
#device twe # 3ware ATA RAID
# atkbdc0 controls both the keyboard and the PS/2 mouse
device atkbdc # AT keyboard controller
device atkbd # AT keyboard
device psm # PS/2 mouse
device kbdmux # keyboard multiplexer
device vga # VGA video card driver
device splash # Splash screen and screen saver support
# syscons is the default console driver, resembling an SCO console
device sc
# Enable this for the pcvt (VT220 compatible) console driver
#device vt
#options XSERVER # support for X server on a vt console
#options FAT_CURSOR # start with block cursor
device agp # support several AGP chipsets
# Power management support (see NOTES for more options)
#device apm
# Add suspend/resume support for the i8254.
device pmtimer
# PCCARD (PCMCIA) support
# PCMCIA and cardbus bridge support
#device cbb # cardbus (yenta) bridge
#device pccard # PC Card (16-bit) bus
#device cardbus # CardBus (32-bit) bus
# Serial (COM) ports
device sio # 8250, 16[45]50 based serial ports
# Parallel port
#device ppc
#device ppbus # Parallel port bus (required)
#device lpt # Printer
#device plip # TCP/IP over parallel
#device ppi # Parallel port interface device
#device vpo # Requires scbus and da
# If you've got a "dumb" serial or parallel PCI card that is
# supported by the puc(4) glue driver, uncomment the following
# line to enable it (connects to the sio and/or ppc drivers):
#device puc
# PCI Ethernet NICs.
device de # DEC/Intel DC21x4x (``Tulip'')
device em # Intel PRO/1000 adapter Gigabit Ethernet Card
#device ixgb # Intel PRO/10GbE Ethernet Card
#device txp # 3Com 3cR990 (``Typhoon'')
#device vx # 3Com 3c590, 3c595 (``Vortex'')
# PCI Ethernet NICs that use the common MII bus controller code.
# NOTE: Be sure to keep the 'device miibus' line in order to use these NICs!
device miibus # MII bus support
#device bce # Broadcom BCM5706/BCM5708 Gigabit Ethernet
#device bfe # Broadcom BCM440x 10/100 Ethernet
#device bge # Broadcom BCM570xx Gigabit Ethernet
device dc # DEC/Intel 21143 and various workalikes
device fxp # Intel EtherExpress PRO/100B (82557, 82558)
#device lge # Level 1 LXT1001 gigabit Ethernet
#device msk # Marvell/SysKonnect Yukon II Gigabit Ethernet
#device nge # NatSemi DP83820 gigabit Ethernet
#device nve # nVidia nForce MCP on-board Ethernet Networking
#device pcn # AMD Am79C97x PCI 10/100(precedence over 'lnc')
#device re # RealTek 8139C+/8169/8169S/8110S
#device rl # RealTek 8129/8139
#device sf # Adaptec AIC-6915 (``Starfire'')
#device sis # Silicon Integrated Systems SiS 900/SiS 7016
#device sk # SysKonnect SK-984x & SK-982x gigabit Ethernet
#device ste # Sundance ST201 (D-Link DFE-550TX)
#device stge # Sundance/Tamarack TC9021 gigabit Ethernet
#device ti # Alteon Networks Tigon I/II gigabit Ethernet
#device tl # Texas Instruments ThunderLAN
#device tx # SMC EtherPower II (83c170 ``EPIC'')
#device vge # VIA VT612x gigabit Ethernet
#device vr # VIA Rhine, Rhine II
#device wb # Winbond W89C840F
#device xl # 3Com 3c90x (``Boomerang'', ``Cyclone'')
# ISA Ethernet NICs. pccard NICs included.
#device cs # Crystal Semiconductor CS89x0 NIC
# 'device ed' requires 'device miibus'
#device ed # NE[12]000, SMC Ultra, 3c503, DS8390 cards
#device ex # Intel EtherExpress Pro/10 and Pro/10+
#device ep # Etherlink III based cards
#device fe # Fujitsu MB8696x based cards
#device ie # EtherExpress 8/16, 3C507, StarLAN 10 etc.
#device lnc # NE2100, NE32-VL Lance Ethernet cards
#device sn # SMC's 9000 series of Ethernet chips
#device xe # Xircom pccard Ethernet
# Wireless NIC cards
#device wlan # 802.11 support
#device wlan_wep # 802.11 WEP support
#device wlan_ccmp # 802.11 CCMP support
#device wlan_tkip # 802.11 TKIP support
#device an # Aironet 4500/4800 802.11 wireless NICs.
#device ath # Atheros pci/cardbus NIC's
#device ath_hal # Atheros HAL (Hardware Access Layer)
#device ath_rate_sample # SampleRate tx rate control for ath
#device awi # BayStack 660 and others
#device ral # Ralink Technology RT2500 wireless NICs.
#device wi # WaveLAN/Intersil/Symbol 802.11 wireless NICs.
#device wl # Older non 802.11 Wavelan wireless NIC.
# Pseudo devices.
device loop # Network loopback
device random # Entropy device
device ether # Ethernet support
#device sl # Kernel SLIP
device ppp # Kernel PPP
device tun # Packet tunnel.
device pty # Pseudo-ttys (telnet etc)
device md # Memory "disks"
device gif # IPv6 and IPv4 tunneling
device faith # IPv6-to-IPv4 relaying (translation)
# The `bpf' device enables the Berkeley Packet Filter.
# Be aware of the administrative consequences of enabling this!
# Note that 'bpf' is required for DHCP.
device bpf # Berkeley packet filter
# USB support
#device uhci # UHCI PCI->USB interface
#device ohci # OHCI PCI->USB interface
#device ehci # EHCI PCI->USB interface (USB 2.0)
#device usb # USB Bus (required)
#device udbp # USB Double Bulk Pipe devices
#device ugen # Generic
#device uhid # "Human Interface Devices"
#device ukbd # Keyboard
#device ulpt # Printer
#device umass # Disks/Mass storage - Requires scbus and da
#device ums # Mouse
#device ural # Ralink Technology RT2500USB wireless NICs
#device urio # Diamond Rio 500 MP3 player
#device uscanner # Scanners
# USB Ethernet, requires miibus
#device aue # ADMtek USB Ethernet
#device axe # ASIX Electronics USB Ethernet
#device cdce # Generic USB over Ethernet
#device cue # CATC USB Ethernet
#device kue # Kawasaki LSI USB Ethernet
#device rue # RealTek RTL8150 USB Ethernet
# FireWire support
#device firewire # FireWire bus code
#device sbp # SCSI over FireWire (Requires scbus and da)
#device fwe # Ethernet over FireWire (non-standard!)
-------------- next part --------------
# --------------------------------------------------------------------
# FREEBSD BUILD
FreeBSD m1.npubs.com 6.3-RELEASE-p2 FreeBSD 6.3-RELEASE-p2 #0: Tue Jun 10 02:07:06 UTC 2008 nate at m1.npubs.com:/usr/obj/usr/src/sys/RACK1DBG i386
# --------------------------------------------------------------------
# PROCESS LIST
db> ps
pid ppid pgrp uid state wmesg wchan cmd
69677 69671 4145 65534 LLJ *vm page 0xcf2cc140 python
69674 69654 69486 0 RE rsync
69671 69670 4145 65534 SJ piperd 0xca44a660 python
69670 4145 4145 65534 SJ wait 0xcf673430 sh
69669 4684 4684 125 SJ select 0xc07b7ce4 smtpd
69668 69664 69664 0 S select 0xc07b7ce4 rsync
69664 5663 69664 0 Ls *vm page 0xcf2cc140 rsync
69654 69511 69486 0 S wait 0xca033c90 sh
69561 45988 45982 70 SJ sbwait 0xcabb4bc8 postgres
69557 4141 4141 125 SJ select 0xc07b7ce4 smtp
69541 4141 4141 125 SJ select 0xc07b7ce4 cleanup
69540 4141 4141 125 SJ select 0xc07b7ce4 smtpd
69535 4141 4141 125 SJ select 0xc07b7ce4 trivial-rewrite
69533 4141 4141 125 SJ select 0xc07b7ce4 anvil
69523 4141 4141 125 SJ select 0xc07b7ce4 proxymap
69511 69498 69486 0 S wait 0xc92e7430 sh
69510 4141 4141 125 SJ select 0xc07b7ce4 smtpd
69504 69495 69495 0 SJ sbwait 0xcf2b1638 python
69498 69486 69486 0 S wait 0xca67b648 sh
69495 69492 69495 0 SsJ wait 0xcf6aa218 sh
69492 1374 1374 0 SJ piperd 0xc91c8198 cron
69486 69484 69486 0 Ss wait 0xc92e4c90 sh
69484 691 691 0 S piperd 0xc901b990 cron
69455 1 69455 0 Ls *Giant 0xcf52b6c0 jail-measure
69434 1940 1940 125 SJ kqread 0xc9987a00 trivial-rewrite
69415 1940 1940 0 SJ kqread 0xc9b21700 local
69414 1940 1940 125 SJ kqread 0xca77a600 smtpd
69413 1940 1940 125 SJ kqread 0xc988ee00 smtp
69412 1940 1940 125 SJ kqread 0xcf87b900 cleanup
69400 4684 4684 0 SJ select 0xc07b7ce4 virtual
69379 45988 45982 70 SJ sbwait 0xd0b004d4 postgres
69374 45988 45982 70 SJ sbwait 0xc8f7579c postgres
69372 45988 45982 70 SJ sbwait 0xd08a2d2c postgres
69361 2260 2260 80 SJ accept 0xc97fe5ca httpd
69352 1940 1940 125 SJ lockf 0xc94529c0 smtpd
69335 45988 45982 70 SJ sbwait 0xcf2d5900 postgres
69327 1940 1940 125 SJ lockf 0xc9452740 bounce
69326 1940 1940 125 SJ kqread 0xd0080100 bounce
69276 1940 1940 125 SJ lockf 0xcf5158c0 smtp
69274 1940 1940 125 SJ kqread 0xca333800 smtp
69256 45988 45982 70 SJ sbwait 0xc8ce5638 postgres
69245 45988 45982 70 SJ sbwait 0xc9e23bc8 postgres
69240 45988 45982 70 SJ sbwait 0xd0ad379c postgres
69208 4684 4684 125 SJ select 0xc07b7ce4 smtp
69184 4684 4684 125 SJ select 0xc07b7ce4 smtpd
69183 4684 4684 125 SJ select 0xc07b7ce4 smtp
69182 4684 4684 125 SJ select 0xc07b7ce4 cleanup
69175 1940 1940 125 SJ kqread 0xc993ba00 smtpd
69174 4684 4684 125 SJ select 0xc07b7ce4 trivial-rewrite
69141 45988 45982 70 SJ sbwait 0xd0b030a8 postgres
68996 45988 45982 70 SJ sbwait 0xcf2b1e90 postgres
68790 2260 2260 80 SJ accept 0xc97fe5ca httpd
68693 1940 1940 125 SJ kqread 0xcbf11a00 anvil
68338 4557 4557 125 SJ select 0xc07b7ce4 imapd
68262 45988 45982 70 SJ sbwait 0xc9f54900 postgres
68199 4557 4557 125 SJ select 0xc07b7ce4 imapd
68189 45988 45982 70 SJ sbwait 0xd0afd0a8 postgres
67971 4684 4684 125 SJ select 0xc07b7ce4 proxymap
67135 4684 4684 125 SJ select 0xc07b7ce4 pickup
67085 4557 4557 125 SJ select 0xc07b7ce4 imapd
66421 45988 45982 70 SJ sbwait 0xd0ac9900 postgres
64620 1940 1940 125 SJ kqread 0xcbfa9400 pickup
63268 45988 45982 70 SJ sbwait 0xd0af9d2c postgres
62647 45988 45982 70 SJ sbwait 0xc9f23370 postgres
62453 45988 45982 70 SJ sbwait 0xc8d6379c postgres
62324 4141 4141 125 SJ select 0xc07b7ce4 pickup
59910 2260 2260 80 SJ accept 0xc97fe5ca httpd
58656 3815 3815 80 SJ accept 0xc986d892 httpd
58655 3815 3815 80 SJ accept 0xc986d892 httpd
58641 3815 3815 80 SJ accept 0xc986d892 httpd
58523 2260 2260 80 SJ accept 0xc97fe5ca httpd
58520 2260 2260 80 SJ accept 0xc97fe5ca httpd
58398 2260 2260 80 SJ accept 0xc97fe5ca httpd
57936 2260 2260 80 SJ accept 0xc97fe5ca httpd
55518 2260 2260 80 SJ accept 0xc97fe5ca httpd
49914 2260 2260 80 SJ accept 0xc97fe5ca httpd
46007 1 46007 0 SsJ nanslp 0xc076a96c cron
45999 1 45999 0 SsJ select 0xc07b7ce4 inetd
45993 45992 45982 70 S+J select 0xc07b7ce4 postgres
45992 45988 45982 70 S+J select 0xc07b7ce4 postgres
45991 45988 45982 70 S+J select 0xc07b7ce4 postgres
45989 45988 45982 70 S+J select 0xc07b7ce4 postgres
45988 1 45982 70 S+J select 0xc07b7ce4 postgres
45983 45958 45957 88 S+J (threaded) mysqld
100467 S kserel 0xcf638694 mysqld
100473 S kserel 0xcf638694 mysqld
100358 S kserel 0xcf638694 mysqld
100362 S kserel 0xcf638694 mysqld
100355 S select 0xc07b7ce4 mysqld
100948 S kserel 0xd092cc94 mysqld
100949 S sigwait 0xec446c14 mysqld
100938 S ksesigwa 0xcf6aabb0 mysqld
45958 1 45957 88 S+J wait 0xcf644000 sh
45942 1 45942 0 SsJ select 0xc07b7ce4 sshd
45876 1 45876 0 SsJ select 0xc07b7ce4 syslogd
33626 2260 2260 80 SJ accept 0xc97fe5ca httpd
23070 1336 1336 80 SJ lockf 0xd0401e40 httpd
23069 1336 1336 80 SJ kqread 0xca0be100 httpd
23057 1336 1336 80 SJ lockf 0xcf515680 httpd
22000 1336 1336 80 SJ lockf 0xc9c80380 httpd
21992 1336 1336 80 SJ lockf 0xcf515dc0 httpd
20343 1336 1336 80 SJ lockf 0xc91dd200 httpd
20342 1336 1336 80 SJ lockf 0xd0458ac0 httpd
20341 3815 3815 80 SJ accept 0xc986d892 httpd
20340 1336 1336 80 SJ lockf 0xd0401140 httpd
20339 3815 3815 80 SJ accept 0xc986d892 httpd
20337 1336 1336 80 SJ lockf 0xd0458800 httpd
20336 3815 3815 80 SJ accept 0xc986d892 httpd
20335 3815 3815 80 SJ accept 0xc986d892 httpd
20334 1336 1336 80 SJ lockf 0xcf4eb280 httpd
20333 3815 3815 80 SJ accept 0xc986d892 httpd
20324 20323 20323 0 SJ piperd 0xd02164c8 sockin
20323 4456 20323 0 SsJ wait 0xd092e430 sh
78897 0 0 0 SL ipmireq 0xcf93bec0 [ipmi0: kcs]
65042 64222 64222 70 SJ sbwait 0xcf0f14d4 postgres
64780 64222 64222 70 SJ sbwait 0xcabb4900 postgres
64778 64222 64222 70 SJ sbwait 0xd0af10a8 postgres
64777 64222 64222 70 SJ sbwait 0xc8f96370 postgres
64769 64222 64222 70 SJ sbwait 0xca43e20c postgres
64768 64222 64222 70 SJ sbwait 0xc9a26900 postgres
64767 64222 64222 70 SJ sbwait 0xd0afd4d4 postgres
64742 64222 64222 70 SJ sbwait 0xd0afb638 postgres
64666 64222 64222 70 SJ sbwait 0xccb3579c postgres
64665 64222 64222 70 SJ sbwait 0xd0afc20c postgres
64664 64222 64222 70 SJ sbwait 0xc9e4979c postgres
64663 64222 64222 70 SJ sbwait 0xd05c0e90 postgres
64662 64222 64222 70 SJ sbwait 0xd0ad3d2c postgres
64661 64222 64222 70 SJ sbwait 0xca39a79c postgres
64484 64222 64222 70 SJ sbwait 0xc8d6a370 postgres
64414 64222 64222 70 SJ sbwait 0xc9f2379c postgres
64361 64222 64222 70 SJ sbwait 0xd08a1370 postgres
64349 64222 64222 70 SJ sbwait 0xd05c8370 postgres
64348 64222 64222 70 SJ sbwait 0xcc9b70a8 postgres
64332 64222 64222 70 SJ sbwait 0xd0509a64 postgres
64331 64222 64222 70 SJ sbwait 0xc9a2679c postgres
64227 64226 64222 70 SJ select 0xc07b7ce4 postgres
64226 64222 64222 70 SJ select 0xc07b7ce4 postgres
64225 64222 64222 70 SJ select 0xc07b7ce4 postgres
64223 64222 64222 70 SJ select 0xc07b7ce4 postgres
64222 1 64222 70 SsJ select 0xc07b7ce4 postgres
43040 1114 43040 70 SsJ sbwait 0xca50b370 postgres
75907 4167 4167 1003 RJ CPU 0 stunnel
46911 1 46911 0 SsJ nanslp 0xc076a96c cron
46904 1 46904 0 SsJ select 0xc07b7ce4 sshd
46880 1 46880 0 SsJ (threaded) pdns_server
100752 S kserel 0xcf45e0f4 pdns_server
100748 S kserel 0xcf45e0f4 pdns_server
100695 S kserel 0xcf45e0f4 pdns_server
100171 S kserel 0xcf45e0f4 pdns_server
100464 S sbwait 0xcf7be638 pdns_server
100572 S sbwait 0xca872370 pdns_server
100836 S accept 0xcac13e22 pdns_server
100834 S select 0xc07b7ce4 pdns_server
100784 S ksesigwa 0xcf564bb0 pdns_server
46845 1 46845 389 SsJ (threaded) slapd
100729 S kserel 0xcf638454 slapd
100750 S kserel 0xcf638454 slapd
100747 S select 0xc07b7ce4 slapd
100630 S kserel 0xcf638454 slapd
100512 S kserel 0xcf638454 slapd
100733 S ksesigwa 0xcf6ab998 slapd
46825 1 46825 0 SsJ select 0xc07b7ce4 syslogd
68366 5772 5772 70 SJ sbwait 0xcabb379c postgres
57454 5772 5772 70 SJ sbwait 0xca4124d4 postgres
94638 5772 5772 70 SJ sbwait 0xc9f22bc8 postgres
94626 5772 5772 70 SJ sbwait 0xd052ee90 postgres
85402 5800 5800 80 SJ accept 0xc9cf4892 httpd
85398 5772 5772 70 SJ sbwait 0xceb45e90 postgres
85396 5772 5772 70 SJ sbwait 0xca4b4638 postgres
71130 1114 71130 70 SsJ sbwait 0xc9eeaa64 postgres
71129 1114 71129 70 SsJ sbwait 0xcf2b0bc8 postgres
71128 1114 71128 70 SsJ sbwait 0xc9e484d4 postgres
71127 1114 71127 70 SsJ sbwait 0xcf7bd20c postgres
71126 1114 71126 70 SsJ sbwait 0xc8f76900 postgres
71115 1114 71115 70 SsJ sbwait 0xca50ce90 postgres
71081 71080 71080 89 SJ (threaded) java
100360 S kserel 0xcf5dbe74 java
100255 S kserel 0xcf5dbe74 java
100722 S select 0xc07b7ce4 java
100746 S kserel 0xcf5dbe74 java
100505 S kserel 0xcf5dbe74 java
100409 S sbwait 0xceb88bc8 java
100718 S sbwait 0xca609900 java
100163 S sbwait 0xcf7be79c java
100518 S sbwait 0xca6094d4 java
100778 S sbwait 0xca609370 java
100258 S sbwait 0xc93454d4 java
100506 S ksesigwa 0xcf6ad780 java
71080 1 71080 0 SsJ select 0xc07b7ce4 udb-server
18181 2388 2388 80 SJ accept 0xc986d19e httpd
16803 2388 2388 80 SJ accept 0xc986d19e httpd
16802 2388 2388 80 SJ accept 0xc986d19e httpd
16801 2388 2388 80 SJ accept 0xc986d19e httpd
16795 2388 2388 80 SJ accept 0xc986d19e httpd
7788 5772 5772 70 SJ sbwait 0xc9e240a8 postgres
7723 5772 5772 70 SJ sbwait 0xcabb620c postgres
7705 5772 5772 70 SJ sbwait 0xca8dc370 postgres
7703 5800 5800 80 SJ accept 0xc9cf4892 httpd
7700 5800 5800 80 SJ accept 0xc9cf4892 httpd
7699 5772 5772 70 SJ sbwait 0xca39aa64 postgres
7696 5772 5772 70 SJ sbwait 0xc9cf3638 postgres
7694 5800 5800 80 SJ accept 0xc9cf4892 httpd
6376 4141 4141 125 SJ select 0xc07b7ce4 tlsmgr
5969 3438 3438 80 SJ lockf 0xce4036c0 httpd
5968 3438 3438 80 SJ lockf 0xcaf5dcc0 httpd
5966 3438 3438 80 SJ select 0xc07b7ce4 httpd
5962 3438 3438 80 SJ lockf 0xc94524c0 httpd
5854 5772 5772 70 SJ sbwait 0xca314638 postgres
5852 5800 5800 80 SJ accept 0xc9cf4892 httpd
5851 5800 5800 80 SJ accept 0xc9cf4892 httpd
5850 5800 5800 80 SJ accept 0xc9cf4892 httpd
5849 5800 5800 80 SJ accept 0xc9cf4892 httpd
5848 5800 5800 80 SJ accept 0xc9cf4892 httpd
5827 5826 5826 1020 SJ (threaded) java
100390 S kserel 0xc8e0b034 java
100475 S kserel 0xc8e0b034 java
100418 S kserel 0xc8e0b034 java
100389 S select 0xc07b7ce4 java
100180 S kserel 0xc8e0b034 java
100400 S sbwait 0xc9e49a64 java
100812 S sbwait 0xc9f5420c java
100382 S sbwait 0xc9f54e90 java
100394 S sbwait 0xc9cf3900 java
100388 S sbwait 0xca314370 java
100398 S sbwait 0xca02f370 java
100386 S ksesigwa 0xc8e0d138 java
5826 1 5826 0 SsJ select 0xc07b7ce4 udb-server
5819 1 5819 0 SsJ nanslp 0xc076a96c cron
5812 1 5812 0 SsJ select 0xc07b7ce4 sshd
5809 5805 5800 0 SJ piperd 0xc9cfb7f8 cronolog
5808 5803 5800 0 SJ piperd 0xc9e44b28 cronolog
5805 5800 5800 0 SJ wait 0xc969aa78 sh
5803 5800 5800 0 SJ wait 0xca033648 sh
5800 1 5800 0 SsJ nanslp 0xc076a96c httpd
5788 5787 5772 70 SJ select 0xc07b7ce4 postgres
5787 5772 5772 70 SJ select 0xc07b7ce4 postgres
5786 5772 5772 70 SJ select 0xc07b7ce4 postgres
5772 1 5772 70 SsJ select 0xc07b7ce4 postgres
5713 1 5713 0 SsJ select 0xc07b7ce4 syslogd
5704 1 5704 0 Ss+ ttyin 0xc8a61410 getty
5703 1 5703 0 Ss+ ttyin 0xc8a69c10 getty
5702 1 5702 0 Ss+ ttyin 0xc8a6c010 getty
5701 1 5701 0 Ss+ ttyin 0xc8a6b010 getty
5700 1 5700 0 Ss+ ttyin 0xc8a6a410 getty
5699 1 5699 0 Ss+ ttyin 0xc8a6a810 getty
5698 1 5698 0 Ss+ ttyin 0xc8a60810 getty
5697 1 5697 0 Ss+ ttyin 0xc8a6c410 getty
5696 1 5696 0 Ss+ ttyin 0xc8a6b410 getty
5675 1 5675 0 Ss select 0xc07b7ce4 bsnmpd
5663 1 5663 0 Ss select 0xc07b7ce4 inetd
5443 5442 5442 1020 SJ (threaded) java
100223 S kserel 0xc998b874 java
100095 S kserel 0xc998b874 java
100385 S sbwait 0xd0509978 java
100414 S sbwait 0xd089e814 java
100699 S kserel 0xc998b874 java
100187 S kserel 0xc998b874 java
100229 S select 0xc07b7ce4 java
100736 S sbwait 0xd025379c java
100157 S sbwait 0xc9644f08 java
100338 S ksesigwa 0xc998abb0 java
5442 1 5442 0 SsJ select 0xc07b7ce4 udb-server
5435 1 5435 0 SsJ nanslp 0xc076a96c cron
5419 1 5419 0 SsJ select 0xc07b7ce4 sshd
5265 1 5265 0 SsJ select 0xc07b7ce4 syslogd
4960 1 4960 0 SsJ nanslp 0xc076a96c cron
4953 1 4953 0 SsJ select 0xc07b7ce4 sshd
4906 1 4906 53 SsJ select 0xc07b7ce4 named
4891 1 4891 0 SsJ select 0xc07b7ce4 syslogd
4834 4684 4684 125 SJ select 0xc07b7ce4 anvil
4789 4684 4684 125 SJ select 0xc07b7ce4 tlsmgr
4709 1 4709 0 SsJ nanslp 0xc076a96c cron
4699 1 4699 0 SsJ accept 0xc9dfae22 saslauthd1
4691 4684 4684 125 SJ select 0xc07b7ce4 qmgr
4684 1 4684 0 SsJ select 0xc07b7ce4 master
4598 1 4598 0 SsJ select 0xc07b7ce4 rpc.dracd
4585 4584 4585 0 S+J select 0xc07b7ce4 couriertcpd
4584 1 4584 0 S+J piperd 0xc9cb6b28 courierlogger
4581 4532 4531 0 S+J select 0xc07b7ce4 authdaemond
4579 4532 4531 0 S+J select 0xc07b7ce4 authdaemond
4578 4532 4531 0 S+J select 0xc07b7ce4 authdaemond
4577 4532 4531 0 S+J select 0xc07b7ce4 authdaemond
4576 4532 4531 0 S+J select 0xc07b7ce4 authdaemond
4569 4568 4569 0 S+J select 0xc07b7ce4 couriertcpd
4568 1 4568 0 S+J piperd 0xc8e04330 courierlogger
4557 4556 4557 0 S+J select 0xc07b7ce4 couriertcpd
4556 1 4556 0 S+J piperd 0xc8f6e000 courierlogger
4544 4543 4544 0 S+J select 0xc07b7ce4 couriertcpd
4543 1 4543 0 S+J piperd 0xc9cb64c8 courierlogger
4532 4531 4531 0 S+J select 0xc07b7ce4 authdaemond
4531 1 4531 0 S+J piperd 0xc94917f8 courierlogger
4517 1 4517 0 SsJ select 0xc07b7ce4 sshd
4500 1 4500 0 SsJ select 0xc07b7ce4 bsnmpd
4467 1 4467 0 SsJ select 0xc07b7ce4 rpcbind
4456 1 4456 0 SsJ select 0xc07b7ce4 syslogd
4180 1 4180 0 SsJ nanslp 0xc076a96c cron
4172 1 4172 0 SsJ select 0xc07b7ce4 inetd
4167 1 4167 1003 SsJ (threaded) stunnel
100735 S kserel 0xc8e0b454 stunnel
100532 S kserel 0xc8e0b454 stunnel
100189 S kserel 0xc8e0b454 stunnel
100749 S kserel 0xc8e0b454 stunnel
100412 S select 0xc07b7ce4 stunnel
100080 S ksesigwa 0xc8e05780 stunnel
4160 1 4160 0 SsJ accept 0xc986d302 saslauthd1
4149 4141 4141 125 SJ select 0xc07b7ce4 qmgr
4145 1 4145 65534 SsJ (threaded) proxsmtpd
100762 S kserel 0xc8e0b9f4 proxsmtpd
100383 S kserel 0xc8e0b9f4 proxsmtpd
100761 S kserel 0xc8e0b9f4 proxsmtpd
100765 S kserel 0xc8e0b9f4 proxsmtpd
100740 S select 0xc07b7ce4 proxsmtpd
100783 S accept 0xc9a97892 proxsmtpd
100516 S ksesigwa 0xc9b99dc8 proxsmtpd
4141 1 4141 0 SsJ select 0xc07b7ce4 master
4077 1 4077 0 SsJ select 0xc07b7ce4 sshd
4021 1 4021 0 SsJ select 0xc07b7ce4 syslogd
3844 1 3844 0 SsJ select 0xc07b7ce4 inetd
3830 1 3830 0 SsJ nanslp 0xc076a96c cron
3823 1 3823 0 SsJ select 0xc07b7ce4 sshd
3815 1 3815 0 SsJ nanslp 0xc076a96c httpd
3680 1 3680 0 SsJ select 0xc07b7ce4 syslogd
3538 3534 3438 0 SJ piperd 0xc8f69660 cronolog
3537 3531 3438 0 SJ piperd 0xc9311cc0 cronolog
3536 3532 3438 0 SJ piperd 0xc9491b28 cronolog
3535 3438 3438 80 SJ lockf 0xcdedd2c0 httpd
3534 3438 3438 0 SJ wait 0xc8d95a78 sh
3532 3438 3438 0 SJ wait 0xc969a648 sh
3531 3438 3438 0 SJ wait 0xc9828430 sh
3486 3450 3449 88 S+J (threaded) mysqld
100165 S kserel 0xc894d0f4 mysqld
100256 S kserel 0xc894d0f4 mysqld
100176 S kserel 0xc894d0f4 mysqld
100181 S select 0xc07b7ce4 mysqld
100166 S kserel 0xc894d0f4 mysqld
100252 S kserel 0xc894ea54 mysqld
100253 S sigwait 0xec476c14 mysqld
100172 S ksesigwa 0xc8a4d780 mysqld
3479 1 3479 0 SsJ nanslp 0xc076a96c cron
3460 1 3460 0 SsJ select 0xc07b7ce4 inetd
3450 1 3449 88 S+J wait 0xc92e4000 sh
3448 3438 3438 80 SJ select 0xc07b7ce4 httpd
3438 1 3438 0 SsJ select 0xc07b7ce4 httpd
3326 1 3326 0 SsJ select 0xc07b7ce4 sshd
3265 1 3265 0 SsJ select 0xc07b7ce4 syslogd
2983 1 2983 0 SsJ nanslp 0xc076a96c cron
2976 1 2976 0 SsJ select 0xc07b7ce4 sshd
2968 1 2968 0 SsJ (threaded) httpauthd
100179 S kserel 0xc93a0454 httpauthd
100741 S kserel 0xc93a0454 httpauthd
100760 S kserel 0xc93a0454 httpauthd
100730 S kserel 0xc93a0454 httpauthd
100185 S sbwait 0xca02fd2c httpauthd
100593 S sbwait 0xceb5c79c httpauthd
100404 S sbwait 0xd05c8a64 httpauthd
100191 S sbwait 0xca43fd2c httpauthd
100393 S sbwait 0xca8dc20c httpauthd
100417 S sbwait 0xca41c638 httpauthd
100359 S sbwait 0xd05c04d4 httpauthd
100468 S sbwait 0xc9f224d4 httpauthd
100273 S sbwait 0xca50b638 httpauthd
100408 S sbwait 0xca50b0a8 httpauthd
100357 S sbwait 0xcabb7a64 httpauthd
100254 S sbwait 0xcf2b3900 httpauthd
100173 S sbwait 0xcf0f2900 httpauthd
100251 S sbwait 0xca50cbc8 httpauthd
100226 S sbwait 0xd0af00a8 httpauthd
100221 S sbwait 0xc9eeabc8 httpauthd
100391 S sbwait 0xca41c900 httpauthd
100162 S sbwait 0xcca28a64 httpauthd
100352 S sbwait 0xd0b01900 httpauthd
100421 S sbwait 0xc97a64d4 httpauthd
100411 S sbwait 0xd0b0120c httpauthd
100479 S accept 0xc9800892 httpauthd
100169 S sbwait 0xca41c370 httpauthd
100396 S sbwait 0xca4110a8 httpauthd
100265 S sbwait 0xcf2d5d2c httpauthd
100402 S sbwait 0xca411e90 httpauthd
100781 S sbwait 0xcf0f2bc8 httpauthd
100183 S sbwait 0xc95b7bc8 httpauthd
100356 S sbwait 0xd0afc638 httpauthd
100502 S sbwait 0xc9df8370 httpauthd
100751 S sbwait 0xc97ff0a8 httpauthd
100500 S sbwait 0xc900b0a8 httpauthd
100513 S sbwait 0xc9a794d4 httpauthd
100170 S sbwait 0xcabaf20c httpauthd
100476 S sbwait 0xc8d6a900 httpauthd
100851 S sbwait 0xca39b79c httpauthd
100274 S sbwait 0xceb44900 httpauthd
100510 S sbwait 0xca8dc79c httpauthd
100936 S sbwait 0xc986d4d4 httpauthd
100419 S sbwait 0xcabaf900 httpauthd
100711 S sbwait 0xd05aca64 httpauthd
100533 S sbwait 0xcefa7a64 httpauthd
100250 S sbwait 0xd0aef0a8 httpauthd
100422 S sbwait 0xcabb3a64 httpauthd
100405 S sbwait 0xc91dfd2c httpauthd
100514 S sbwait 0xd0ad34d4 httpauthd
100482 S sbwait 0xc9f23bc8 httpauthd
100470 S sbwait 0xc9e4879c httpauthd
100410 S sbwait 0xc96af4d4 httpauthd
100614 S sbwait 0xca50c20c httpauthd
100801 S sbwait 0xc9eea370 httpauthd
100175 S sbwait 0xd0ac9638 httpauthd
100737 S sbwait 0xd089ee90 httpauthd
100403 S ksesigwa 0xc93a1bb0 httpauthd
2920 1 2920 389 SsJ (threaded) slapd
100186 S kserel 0xc8ddedb4 slapd
100497 S kserel 0xc8ddedb4 slapd
100763 S kserel 0xc8ddedb4 slapd
100469 S kserel 0xc8ddedb4 slapd
100471 S select 0xc07b7ce4 slapd
100136 S ksesigwa 0xc8e0d998 slapd
2900 1 2900 0 SsJ select 0xc07b7ce4 syslogd
2668 1 2668 0 SsJ select 0xc07b7ce4 slapd
2661 1 2661 0 SsJ nanslp 0xc076a96c cron
2654 1 2654 0 SsJ select 0xc07b7ce4 sshd
2599 1 2599 0 SsJ select 0xc07b7ce4 syslogd
2547 2388 2388 80 SJ accept 0xc986d19e httpd
2546 2388 2388 80 SJ accept 0xc986d19e httpd
2545 2388 2388 80 SJ accept 0xc986d19e httpd
2544 2388 2388 80 SJ accept 0xc986d19e httpd
2543 2388 2388 80 SJ accept 0xc986d19e httpd
2403 1 2403 0 SsJ nanslp 0xc076a96c cron
2396 1 2396 0 SsJ select 0xc07b7ce4 sshd
2388 1 2388 0 SsJ nanslp 0xc076a96c httpd
2289 1 2289 0 SsJ select 0xc07b7ce4 inetd
2275 1 2275 0 SsJ nanslp 0xc076a96c cron
2268 1 2268 0 SsJ select 0xc07b7ce4 sshd
2260 1 2260 0 SsJ nanslp 0xc076a96c httpd
2206 1 2206 0 SsJ select 0xc07b7ce4 syslogd
2006 1951 1950 88 S+J (threaded) mysqld
100161 S kserel 0xc93a0514 mysqld
100038 S kserel 0xc93a0514 mysqld
100167 S kserel 0xc93a0514 mysqld
100225 S select 0xc07b7ce4 mysqld
100091 S kserel 0xc93a0514 mysqld
100222 S sigwait 0xec3e1c14 mysqld
100219 S ksesigwa 0xc93a1780 mysqld
1951 1 1950 88 S+J wait 0xc939f430 sh
1949 1940 1940 125 SJ kqread 0xc97a9b00 qmgr
1940 1 1940 0 SsJ kqread 0xc969eb00 master
1800 1795 1800 70 SsJ select 0xc07b7ce4 postgres
1799 1795 1799 70 SsJ nanslp 0xc076a96c postgres
1798 1795 1798 70 SsJ nanslp 0xc076a96c postgres
1797 1795 1797 70 SsJ nanslp 0xc076a96c postgres
1795 1 1791 70 S+J select 0xc07b7ce4 postgres
1765 1760 1760 0 SJ lockf 0xcca4ed00 saslauthd
1763 1760 1760 0 SJ lockf 0xca55b240 saslauthd
1762 1760 1760 0 SJ lockf 0xcf50de40 saslauthd
1761 1760 1760 0 SJ accept 0xc8f76cbe saslauthd
1760 1 1760 0 SsJ lockf 0xcff97c00 saslauthd
1731 1 1731 0 SsJ select 0xc07b7ce4 syslogd
1374 1 1374 0 SsJ nanslp 0xc076a96c cron
1364 1 1364 0 SsJ select 0xc07b7ce4 sshd
1336 1 1336 0 SsJ nanslp 0xc076a96c httpd
1117 1114 1117 70 SsJ select 0xc07b7ce4 postgres
1116 1114 1116 70 SsJ select 0xc07b7ce4 postgres
1114 1 1114 70 SsJ select 0xc07b7ce4 postgres
1057 1 1057 0 SsJ select 0xc07b7ce4 syslogd
691 1 691 0 Ss nanslp 0xc076a96c cron
685 1 685 25 Ss pause 0xc8cf0894 sendmail
681 1 681 0 Ss select 0xc07b7ce4 sendmail
675 1 675 0 Ss select 0xc07b7ce4 sshd
655 1 654 0 S nanslp 0xc076a96c python
615 1 615 0 Ss select 0xc07b7ce4 ntpd
525 1 525 0 Ss select 0xc07b7ce4 syslogd
470 1 470 0 Ss select 0xc07b7ce4 devd
41 0 0 0 SL - 0xe9c77d04 [schedcpu]
40 0 0 0 SL sdflush 0xc07c9294 [softdepflush]
39 0 0 0 SL syncer 0xc076a6dc [syncer]
38 0 0 0 SL vlruwt 0xc8a07c90 [vnlru]
37 0 0 0 SL psleep 0xc07b8260 [bufdaemon]
36 0 0 0 SL pgzero 0xc07ca264 [pagezero]
35 0 0 0 SL psleep 0xc07c9d6c [vmdaemon]
34 0 0 0 SL psleep 0xc07c9d28 [pagedaemon]
33 0 0 0 SL - 0xc8bee580 [dummynet]
32 0 0 0 WL [swi0: sio]
31 0 0 0 WL [irq1: atkbd0]
30 0 0 0 WL [irq19: atapci1]
29 0 0 0 WL [irq15: ata1]
28 0 0 0 WL [irq14: ata0]
27 0 0 0 LL *Giant 0xcf52b6c0 [irq17: em1]
26 0 0 0 LL *Giant 0xcf52b6c0 [irq16: em0]
25 0 0 0 WL [irq9: acpi0]
24 0 0 0 SL - 0xc89a2c00 [thread taskq]
23 0 0 0 SL - 0xc89a2c80 [acpi_task_2]
22 0 0 0 SL - 0xc89a2c80 [acpi_task_1]
9 0 0 0 SL - 0xc89a2c80 [acpi_task_0]
21 0 0 0 WL [swi6: Giant taskq]
20 0 0 0 WL [swi6: task queue]
19 0 0 0 WL [swi2: cambio]
8 0 0 0 SL ccb_scan 0xc0752ae4 [xpt_thrd]
7 0 0 0 SL - 0xc89ef180 [kqueue taskq]
18 0 0 0 WL [swi5: +]
17 0 0 0 SL - 0xc0767320 [yarrow]
6 0 0 0 SL crypto_r 0xc07c8ec4 [crypto returns]
5 0 0 0 SL crypto_w 0xc07c8e84 [crypto]
4 0 0 0 SL - 0xc0767e28 [g_down]
3 0 0 0 SL - 0xc0767e24 [g_up]
2 0 0 0 SL - 0xc0767e1c [g_event]
16 0 0 0 WL [swi3: vm]
15 0 0 0 LL *Giant 0xcf52b6c0 [swi4: clock sio]
14 0 0 0 WL [swi1: net]
13 0 0 0 RL [idle: cpu0]
12 0 0 0 RL CPU 1 [idle: cpu1]
11 0 0 0 RL CPU 2 [idle: cpu2]
10 0 0 0 RL CPU 3 [idle: cpu3]
1 0 1 0 SLs wait 0xc894f000 [init]
0 0 0 0 WLs [swapper]
# --------------------------------------------------------------------
# CPU INFO
db> show pcpu
cpuid = 1
curthread = 0xc894a900: pid 12 "idle: cpu1"
curpcb = 0xe81f2d90
fpcurthread = none
idlethread = 0xc894a900: pid 12 "idle: cpu1"
APIC ID = 1
currentldt = 0x50
spin locks held:
db> show allpcpu
Current CPU: 1
cpuid = 0
curthread = 0xcf2ba780: pid 75907 "stunnel"
curpcb = 0xed1b1d90
fpcurthread = none
idlethread = 0xc894aa80: pid 13 "idle: cpu0"
APIC ID = 0
currentldt = 0x58
spin locks held:
cpuid = 1
curthread = 0xc894a900: pid 12 "idle: cpu1"
curpcb = 0xe81f2d90
fpcurthread = none
idlethread = 0xc894a900: pid 12 "idle: cpu1"
APIC ID = 1
currentldt = 0x50
spin locks held:
cpuid = 2
curthread = 0xc894a780: pid 11 "idle: cpu2"
curpcb = 0xe81efd90
fpcurthread = none
idlethread = 0xc894a780: pid 11 "idle: cpu2"
APIC ID = 2
currentldt = 0x50
spin locks held:
cpuid = 3
curthread = 0xc894a600: pid 10 "idle: cpu3"
curpcb = 0xe81ecd90
fpcurthread = none
idlethread = 0xc894a600: pid 10 "idle: cpu3"
APIC ID = 3
currentldt = 0x50
spin locks held:
# --------------------------------------------------------------------
# LOCK INFO
db> show locks
db> show alllocks
Process 69677 (python) thread 0xca239900 (100432)
exclusive sleep mutex vm object (standard object) r = 0 (0xce1f3420) locked @ /usr/src/sys/vm/vm_fault.c:688
exclusive sx user map r = 0 (0xcf67f4e4) locked @ /usr/src/sys/vm/vm_map.c:3105
Process 69674 (rsync) thread 0xc8cf5a80 (100132)
exclusive sleep mutex pmap r = 0 (0xcf5b87b0) locked @ /usr/src/sys/i386/i386/pmap.c:2749
exclusive sleep mutex vm page queue mutex r = 0 (0xc07c9ce0) locked @ /usr/src/sys/i386/i386/pmap.c:2748
Process 69664 (rsync) thread 0xca239480 (100485)
exclusive sleep mutex vm object (standard object) r = 0 (0xcc3ad528) locked @ /usr/src/sys/kern/vfs_bio.c:1480
exclusive sleep mutex Giant r = 0 (0xc076a080) locked @ /usr/src/sys/kern/vfs_vnops.c:492
db> show lockedvnods
Locked vnodes
0xccc2e414: tag ufs, type VREG
usecount 1, writecount 0, refcount 5965 mountedhere 0
flags ()
v_object 0xcc3ad528 ref 0 pages 81936
lock type ufs: SHARED (count 1)#0 0xc04fa690 at lockmgr+0x160
#1 0xc0649d8a at ffs_lock+0x76
#2 0xc06bf8bb at VOP_LOCK_APV+0x87
#3 0xc056a59c at vn_lock+0xac
#4 0xc0569c72 at vn_read+0x132
#5 0xc052c741 at dofileread+0x85
#6 0xc052c5da at kern_readv+0x36
#7 0xc052c505 at read+0x45
#8 0xc06acec3 at syscall+0x25b
#9 0xc06984df at Xint0x80_syscall+0x1f
ino 13262597, on dev ad8s1e
# --------------------------------------------------------------------
# PROCESS STACK TRACES
# PYTHON
db> trace 69677
Tracing pid 69677 tid 100432 td 0xca239900
sched_switch(ca239900,0,1) at sched_switch+0x177
mi_switch(1,0) at mi_switch+0x270
turnstile_wait(c07c9ce0,c8cf5a80,0,c07c9ce0,2,c06e8e2b,233) at turnstile_wait+0x35a
_mtx_lock_sleep(c07c9ce0,ca239900,0,c07075e5,15a) at _mtx_lock_sleep+0x14c
_mtx_lock_flags(c07c9ce0,0,c07075e5,15a,c06eba6d,...) at _mtx_lock_flags+0xa2
vm_fault(cf67f4a0,8147000,2,8,ca239900,...) at vm_fault+0x311
trap_pfault(ec755d38,1,8147f20,8147f20,0,...) at trap_pfault+0xce
trap(812003b,811003b,bfbf003b,812511c,8147f20,...) at trap+0x1eb
calltrap() at calltrap+0x5
--- trap 0xc, eip = 0x8073f3c, esp = 0xbfbfda00, ebp = 0xbfbfda28 ---
# RSYNC
db> trace 69664
Tracing pid 69664 tid 100485 td 0xca239480
sched_switch(ca239480,0,1) at sched_switch+0x177
mi_switch(1,0) at mi_switch+0x270
turnstile_wait(c07c9ce0,c8cf5a80,0,c07c9ce0,2,c06e8e2b,233) at turnstile_wait+0x35a
_mtx_lock_sleep(c07c9ce0,ca239480,0,c06f2142,5c9) at _mtx_lock_sleep+0x14c
_mtx_lock_flags(c07c9ce0,0,c06f2142,5c9,cc3ad528,...) at _mtx_lock_flags+0xa2
vfs_vmio_release(dcc09838) at vfs_vmio_release+0x35
getnewbuf(0,0,4000,4000) at getnewbuf+0x276
getblk(ccc2e414,236e,0,4000,0,...) at getblk+0x3b3
cluster_read(ccc2e414,14002000,0,236e,0,...) at cluster_read+0xde
ffs_read(ec74cbf4) at ffs_read+0x2d6
VOP_READ_APV(c0741260,ec74cbf4) at VOP_READ_APV+0x9b
vn_read(cb7676c0,ec74ccbc,c94d6a00,0,ca239480) at vn_read+0x1fb
dofileread(ca239480,5,cb7676c0,ec74ccbc,ffffffff,...) at dofileread+0x85
kern_readv(ca239480,5,ec74ccbc,85063e8,42d10,...) at kern_readv+0x36
read(ca239480,ec74cd04) at read+0x45
syscall(3b,3b,3b,0,430f8,...) at syscall+0x25b
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (3, FreeBSD ELF32, read), eip = 0x28185a63, esp = 0xbfbf987c, ebp = 0xbfbf98c8 ---
# RSYNC
db> trace 69674
Tracing pid 69674 tid 100132 td 0xc8cf5a80
sched_switch(c8cf5a80,0,2) at sched_switch+0x177
mi_switch(2,0,c076a040,0,c06eba6d,...) at mi_switch+0x270
critical_exit(c5e1bd38,e9f25c30,c0698a70,0,e9f20008,...) at critical_exit+0x8b
lapic_handle_timer(0) at lapic_handle_timer+0xe1
Xtimerint(c5e1bd38,cf5b87b0,28120000,e9f25c48) at Xtimerint+0x30
pmap_remove_pages(cf5b87b0,0,bfc00000) at pmap_remove_pages+0x1c2
vmspace_exit(c8cf5a80,c076a240,0,c06e6c7d,125,...) at vmspace_exit+0x97
exit1(c8cf5a80,0,e9f25d30,c06acec3,c8cf5a80,...) at exit1+0x496
exit1(c8cf5a80,e9f25d04) at exit1
syscall(3b,3b,3b,8088eda,407,...) at syscall+0x25b
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (1, FreeBSD ELF32, sys_exit), eip = 0x2818520b, esp = 0xbfbfd82c, ebp = 0xbfbfd848 ---
# JAIL-MEASURE
db> trace 69455
Tracing pid 69455 tid 100839 td 0xc9a5d780
sched_switch(c9a5d780,0,1) at sched_switch+0x177
mi_switch(1,0) at mi_switch+0x270
turnstile_wait(c076a080,ca239480,0,c076a080,2,c06e8e2b,233) at turnstile_wait+0x35a
_mtx_lock_sleep(c076a080,c9a5d780,0,c06f3435,c1) at _mtx_lock_sleep+0x14c
_mtx_lock_flags(c076a080,0,c06f3435,c1) at _mtx_lock_flags+0xa2
namei(ece75ba0) at namei+0x227
kern_lstat(c9a5d780,80637a8,0,ece75c74) at kern_lstat+0x47
lstat(c9a5d780,ece75d04) at lstat+0x1b
syscall(805003b,805003b,bfbf003b,8063748,8063700,...) at syscall+0x25b
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (190, FreeBSD ELF32, lstat), eip = 0x28159b47, esp = 0xbfbfeafc, ebp = 0xbfbfeb98 ---
# EM1
db> trace 27
Tracing pid 27 tid 100017 td 0xc894c900
sched_switch(c894c900,0,1) at sched_switch+0x177
mi_switch(1,0) at mi_switch+0x270
turnstile_wait(c076a080,ca239480,0,c076a080,2,c06e8e2b,233) at turnstile_wait+0x35a
_mtx_lock_sleep(c076a080,c894c900,0,c06e6f85,2a9) at _mtx_lock_sleep+0x14c
_mtx_lock_flags(c076a080,0,c06e6f85,2a9) at _mtx_lock_flags+0xa2
ithread_execute_handlers(c89a3648,c89a0480) at ithread_execute_handlers+0xde
ithread_loop(c8a2cae0,e8210d38,c8a2cae0,c04f290c,0,...) at ithread_loop+0x67
fork_exit(c04f290c,c8a2cae0,e8210d38) at fork_exit+0xa0
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe8210d6c, ebp = 0 ---
# EM0
db> trace 26
Tracing pid 26 tid 100018 td 0xc894c780
sched_switch(c894c780,0,1) at sched_switch+0x177
mi_switch(1,0) at mi_switch+0x270
turnstile_wait(c076a080,ca239480,0,c076a080,2,c06e8e2b,233) at turnstile_wait+0x35a
_mtx_lock_sleep(c076a080,c894c780,0,c06e6f85,2a9) at _mtx_lock_sleep+0x14c
_mtx_lock_flags(c076a080,0,c06e6f85,2a9) at _mtx_lock_flags+0xa2
ithread_execute_handlers(c89a3860,c89a0500) at ithread_execute_handlers+0xde
ithread_loop(c8a2cc40,e820dd38,c8a2cc40,c04f290c,0,...) at ithread_loop+0x67
fork_exit(c04f290c,c8a2cc40,e820dd38) at fork_exit+0xa0
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe820dd6c, ebp = 0 ---
# --------------------------------------------------------------------
# OTHER INFO
db> x cpu_idle_hlt
cpu_idle_hlt: 0
More information about the freebsd-hackers
mailing list