4.9-STABLE panic in vlrureclaim()

Peter Pentchev roam at ringlet.net
Tue Nov 4 23:14:59 PST 2003


Hi,

A machine dedicated to network services - firewall, NAT, VPN, IPsec -
crashes from time to time.  Most of the crashes had a wildly corrupted
stack, but this morning I was able to capture one red-handed.  The OS is
-STABLE (RELENG_4) soon after the RELENG_4_9_RELEASE was tagged on the
other branch:

FreeBSD portal.nanolink.com 4.9-STABLE FreeBSD 4.9-STABLE #4: Tue Oct 28 08:59:40 EET 2003     root at portal.nanolink.com:/usr/obj/usr/src-bsd/4.0S/src/sys/PORTAL  i386

The panic backtrace and the kernel config are attached (after the sig);
there is a suspicious-looking null vp in frame #9, vlrureclaim().

G'luck,
Peter

-- 
Peter Pentchev	roam at ringlet.net    roam at sbnd.net    roam at FreeBSD.org
PGP key:	http://people.FreeBSD.org/~roam/roam.key.asc
Key fingerprint	FDBA FD79 C26F 3C51 C95E  DF9E ED18 B68D 1619 4553
This sentence claims to be an Epimenides paradox, but it is lying.

Script started on Wed Nov  5 08:58:11 2003
Setting up interactive shell params..
[root at portal:p2 /var/crash]# gdb -k /usr/obj/usr/src-bsd/4.0S/src/sys/PORTAL/kernel.debug vmcore.22
GNU gdb 4.18 (FreeBSD)
Copyright 1998 Free Software Foundation, Inc.
GDB is free software, covered by the GNU General Public License, and you are
welcome to change it and/or distribute copies of it under certain conditions.
Type "show copying" to see the conditions.
There is absolutely no warranty for GDB.  Type "show warranty" for details.
This GDB was configured as "i386-unknown-freebsd"...Deprecated bfd_read called at /usr/src-bsd/4.0S/src/gnu/usr.bin/binutils/gdb/../../../../contrib/gdb/gdb/dbxread.c line 2627 in elfstab_build_psymtabs
Deprecated bfd_read called at /usr/src-bsd/4.0S/src/gnu/usr.bin/binutils/gdb/../../../../contrib/gdb/gdb/dbxread.c line 933 in fill_symbuf

IdlePTD at phsyical address 0x003ad000
initial pcb at physical address 0x00306d40
panicstr: vm_page_remove(): page not found in hash
panic messages:
---
panic: vm_page_remove(): page not found in hash

syncing disks... 128 3 2 2 2 2 2 2 2 19 1 1 1 1 1 1 1 7 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 
giving up on 1 buffers
Uptime: 4d17h28m34s

dumping to dev #da/0x20001, offset 128
dump 511 510 509 508 507 506 505 504 503 502 501 500 499 498 497 496 495 494 493 492 491 490 489 488 487 486 485 484 483 482 481 480 479 478 477 476 475 474 473 472 471 470 469 468 467 466 465 464 463 462 461 460 459 458 457 456 455 454 453 452 451 450 449 448 447 446 445 444 443 442 441 440 439 438 437 436 435 434 433 432 431 430 429 428 427 426 425 424 423 422 421 420 419 418 417 416 415 414 413 412 411 410 409 408 407 406 405 404 403 402 401 400 399 398 397 396 395 394 393 392 391 390 389 388 387 386 385 384 383 382 381 380 379 378 377 376 375 374 373 372 371 370 369 368 367 366 365 364 363 362 361 360 359 358 357 356 355 354 353 352 351 350 349 348 347 346 345 344 343 342 341 340 339 338 337 336 335 334 333 332 331 330 329 328 327 326 325 324 323 322 321 320 319 318 317 316 315 314 313 312 311 310 309 308 307 306 305 304 303 302 301 300 299 298 297 296 295 294 293 292 291 290 289 288 287 286 285 284 283 282 281 280 279 278 277 276 275 274 273 272 271 270 269 268 267 266 265 264 263 262 261 260 259 258 257 256 255 254 253 252 251 250 249 248 247 246 245 244 243 242 241 240 239 238 237 236 235 234 233 232 231 230 229 228 227 226 225 224 223 222 221 220 219 218 217 216 215 214 213 212 211 210 209 208 207 206 205 204 203 202 201 200 199 198 197 196 195 194 193 192 191 190 189 188 187 186 185 184 183 182 181 180 179 178 177 176 175 174 173 172 171 170 169 168 167 166 165 164 163 162 161 160 159 158 157 156 155 154 153 152 151 150 149 148 147 146 145 144 143 142 141 140 139 138 137 136 135 134 133 132 131 130 129 128 127 126 125 124 123 122 121 120 119 118 117 116 115 114 113 112 111 110 109 108 107 106 105 104 103 102 101 100 99 98 97 96 95 94 93 92 91 90 89 88 87 86 85 84 83 82 81 80 79 78 77 76 75 74 73 72 71 70 69 68 67 66 65 64 63 62 61 60 59 58 57 56 55 54 53 52 51 50 49 48 47 46 45 44 43 42 41 40 39 38 37 36 35 34 33 32 31 30 29 28 27 26 25 24 23 22 21 20 19 18 17 16 15 14 13 12 11 10 9 8 7 6 5 4 3 2 1 0 
---
#0  dumpsys () at /usr/src-bsd/4.0S/src/sys/kern/kern_shutdown.c:487
487		if (dumping++) {
(kgdb) bt
#0  dumpsys () at /usr/src-bsd/4.0S/src/sys/kern/kern_shutdown.c:487
#1  0xc0180154 in boot (howto=256)
    at /usr/src-bsd/4.0S/src/sys/kern/kern_shutdown.c:316
#2  0xc01805a1 in panic (
    fmt=0xc02ca380 "vm_page_remove(): page not found in hash")
    at /usr/src-bsd/4.0S/src/sys/kern/kern_shutdown.c:595
#3  0xc025379b in vm_page_remove (m=0xc080b13c)
    at /usr/src-bsd/4.0S/src/sys/vm/vm_page.c:461
#4  0xc0253e34 in vm_page_free_toq (m=0xc080b13c)
    at /usr/src-bsd/4.0S/src/sys/vm/vm_page.c:1104
#5  0xc0252c0b in vm_object_page_remove (object=0xe4def844, start=0, end=0, 
    clean_only=1) at /usr/src-bsd/4.0S/src/sys/vm/vm_page.h:514
#6  0xc01aef8a in vinvalbuf (vp=0xe5275900, flags=1, cred=0x0, p=0xde78e440, 
    slpflag=0, slptimeo=0) at /usr/src-bsd/4.0S/src/sys/kern/vfs_subr.c:903
#7  0xc01b0017 in vclean (vp=0xe5275900, flags=8, p=0xde78e440)
    at /usr/src-bsd/4.0S/src/sys/kern/vfs_subr.c:1898
#8  0xc01b023c in vgonel (vp=0xe5275900, p=0xde78e440)
    at /usr/src-bsd/4.0S/src/sys/kern/vfs_subr.c:2062
#9  0xc01ae79c in vlrureclaim (mp=0xcd2c5000)
    at /usr/src-bsd/4.0S/src/sys/kern/vfs_subr.c:513
#10 0xc01ae882 in vnlru_proc ()
    at /usr/src-bsd/4.0S/src/sys/kern/vfs_subr.c:561
(kgdb) fr 10
#10 0xc01ae882 in vnlru_proc ()
    at /usr/src-bsd/4.0S/src/sys/kern/vfs_subr.c:561
561				done += vlrureclaim(mp);
(kgdb) info loc
mp = (struct mount *) 0xcd2c5000
nmp = (struct mount *) 0xcd2c5000
done = 31
p = (struct proc *) 0xde78e440
(kgdb) up
Initial frame selected; you cannot go up.
(kgdb) down
#9  0xc01ae79c in vlrureclaim (mp=0xcd2c5000)
    at /usr/src-bsd/4.0S/src/sys/kern/vfs_subr.c:513
513					vgonel(vp, curproc);
(kgdb) info loc
mp = (struct mount *) 0xcd2c5000
vp = (struct vnode *) 0x0
done = 630
trigger = 7
usevnodes = 0
count = 1652
(kgdb) down
#8  0xc01b023c in vgonel (vp=0xe5275900, p=0xde78e440)
    at /usr/src-bsd/4.0S/src/sys/kern/vfs_subr.c:2062
2062		vclean(vp, DOCLOSE, p);
(kgdb) info loc
vp = (struct vnode *) 0xe5275900
p = (struct proc *) 0x0
s = 0
(kgdb) down
#7  0xc01b0017 in vclean (vp=0xe5275900, flags=8, p=0xde78e440)
    at /usr/src-bsd/4.0S/src/sys/kern/vfs_subr.c:1898
1898		vinvalbuf(vp, V_SAVE, NOCRED, p, 0, 0);
(kgdb) info loc
vp = (struct vnode *) 0xe5275900
flags = 0
p = (struct proc *) 0xde78e440
active = 0
(kgdb) down
#6  0xc01aef8a in vinvalbuf (vp=0xe5275900, flags=1, cred=0x0, p=0xde78e440, 
    slpflag=0, slptimeo=0) at /usr/src-bsd/4.0S/src/sys/kern/vfs_subr.c:903
903			vm_object_page_remove(object, 0, 0,
(kgdb) info loc
cred = (struct ucred *) 0x0
p = (struct proc *) 0x0
bp = (struct buf *) 0xe5275900
nbp = (struct buf *) 0xde78e440
blist = (struct buf *) 0x0
s = 6815808
error = -450406144
object = 0xe4def844
(kgdb) down
#5  0xc0252c0b in vm_object_page_remove (object=0xe4def844, start=0, end=0, 
    clean_only=1) at /usr/src-bsd/4.0S/src/sys/vm/vm_page.h:514
514		vm_page_free_toq(m);
(kgdb) info loc
m = 0xc080b13c
p = 0xc080b13c
next = 0x0
size = 0
all = 1
(kgdb) down
#4  0xc0253e34 in vm_page_free_toq (m=0xc080b13c)
    at /usr/src-bsd/4.0S/src/sys/vm/vm_page.c:1104
1104		vm_page_remove(m);
(kgdb) info loc
m = 0xc080b13c
s = 6815808
pq = (struct vpgqueues *) 0x0
object = 0xe4def844
(kgdb) down
#3  0xc025379b in vm_page_remove (m=0xc080b13c)
    at /usr/src-bsd/4.0S/src/sys/vm/vm_page.c:461
461					panic("vm_page_remove(): page not found in hash");
(kgdb) info loc
bucket = (struct vm_page **) 0x0
m = 0xc080b13c
object = 0x0
(kgdb) down
#2  0xc01805a1 in panic (
    fmt=0xc02ca380 "vm_page_remove(): page not found in hash")
    at /usr/src-bsd/4.0S/src/sys/kern/kern_shutdown.c:595
595		boot(bootopt);
(kgdb) info loc
fmt = 0xc02ca380 "vm_page_remove(): page not found in hash"
bootopt = 256
buf = "vm_page_remove(): page not found in hash", '\000' <repeats 215 times>
(kgdb) down
#1  0xc0180154 in boot (howto=256)
    at /usr/src-bsd/4.0S/src/sys/kern/kern_shutdown.c:316
316			dumpsys();
(kgdb) info loc
howto = 256
(kgdb) down
#0  dumpsys () at /usr/src-bsd/4.0S/src/sys/kern/kern_shutdown.c:487
487		if (dumping++) {
(kgdb) info loc
error = 0
(kgdb) down
Bottom (i.e., innermost) frame selected; you cannot go down.
(kgdb) quit
[root at portal:p2 /var/crash]# exit
exit

Script done on Wed Nov  5 08:59:51 2003

# portal kernel config
#
# For more information on this file, please read the handbook section on
# Kernel Configuration Files:
#
#    http://www.freebsd.org/handbook/kernelconfig-config.html
#
# The handbook is also available locally in /usr/share/doc/handbook
# if you've installed the doc distribution, otherwise always see the
# FreeBSD World Wide Web server (http://www.FreeBSD.ORG/) for the
# latest information.
#
# An exhaustive list of options and more detailed explanations of the
# device lines is also present in the ./LINT configuration file. If you are
# in doubt as to the purpose or necessity of a line, check first in LINT.
#
# $FreeBSD: src/sys/i386/conf/GENERIC,v 1.246.2.34 2001/08/12 13:13:46 joerg Exp $

machine		i386
cpu		I686_CPU
ident		PORTAL
maxusers	32

options		MAXDSIZ="(512*1024*1024)"
options		DFLDSIZ="(256*1024*1024)"

makeoptions	DEBUG="-ggdb -g3"	#Build kernel with gdb(1) debug symbols
options		DDB
options 	DDB_UNATTENDED

options 	FFS			#Berkeley Fast Filesystem
options		SOFTUPDATES
options 	FFS_ROOT		#FFS usable as root device [keep this!]
options 	CD9660			#ISO 9660 Filesystem
options 	PROCFS			#Process filesystem
options 	COMPAT_43		#Compatible with BSD 4.3 [KEEP THIS!]
options 	SCSI_DELAY=15000	#Delay (in ms) before probing SCSI
options 	KTRACE			#ktrace(1) support
options 	SYSVSHM			#SYSV-style shared memory
options		SHMMNI=2048
options		SHMSEG=128
options 	SYSVMSG			#SYSV-style message queues
options		MSGMNI=2048
options		MSGTQL=2048
options 	SYSVSEM			#SYSV-style semaphores
options		SEMMNI=128
options		SEMMNS=2048
options		SEMMNU=1024
options		SEMUME=128
options 	P1003_1B		#Posix P1003_1B real-time extentions
options 	_KPOSIX_PRIORITY_SCHEDULING
options		ICMP_BANDLIM		#Rate limit bad replies

options 	INET			#InterNETworking
options		INET6
options		DUMMYNET
options		RANDOM_IP_ID

options		IPFIREWALL
options		IPFIREWALL_VERBOSE
options		IPFIREWALL_FORWARD
options		IPFIREWALL_VERBOSE_LIMIT=100
options		IPFIREWALL_DEFAULT_TO_ACCEPT

options		IPV6FIREWALL
options		IPV6FIREWALL_VERBOSE
options		IPV6FIREWALL_DEFAULT_TO_ACCEPT

options 	IPFILTER		#ipfilter support
options 	IPFILTER_LOG		#ipfilter logging

options 	NETGRAPH		#netgraph(4) system
options 	NETGRAPH_BPF
options 	NETGRAPH_IFACE
options 	NETGRAPH_KSOCKET
options 	NETGRAPH_MPPC_ENCRYPTION
options 	NETGRAPH_PPP
options 	NETGRAPH_PPTPGRE
options 	NETGRAPH_SOCKET
options 	NETGRAPH_VJC

pseudo-device	gif
pseudo-device	faith 1
pseudo-device	stf

options		IPSEC
options		IPSEC_ESP

device		isa
device		pci

# Floppy drives
device		fdc0	at isa? port IO_FD1 irq 6 drq 2
device		fd0	at fdc0 drive 0

# SCSI Controllers
device		ahc		# AHA2940 and onboard AIC7xxx devices
options		AHC_ALLOW_MEMIO

# SCSI peripherals
device		scbus		# SCSI bus (required)
device		da		# Direct Access (disks)
device		cd		# CD

# atkbdc0 controls both the keyboard and the PS/2 mouse
device		atkbdc0	at isa? port IO_KBD
device		atkbd0	at atkbdc? irq 1
device		psm0	at atkbdc? irq 12
options		ATKBD_DFLT_KEYMAP
makeoptions	ATKBD_DFLT_KEYMAP="o1.phonetic.ctrlcaps"

device		vga0	at isa?

# splash screen/screen saver
pseudo-device	splash

# syscons is the default console driver, resembling an SCO console
device		sc0	at isa?
options		MAXCONS=16
options		SC_MOUSE_CHAR=3
options		SC_TWOBUTTON_MOUSE
options		SC_DFLT_FONT
makeoptions	SC_DFLT_FONT="cp1251"

# Floating point support - do not disable.
device		npx0	at nexus? port IO_NPX irq 13

# Serial (COM) ports
device		sio0	at isa? port IO_COM1 flags 0x10 irq 4
device		sio1	at isa? port IO_COM2 irq 3

# Parallel port
device		ppc0	at isa? irq 7
device		ppbus		# Parallel port bus (required)
device		lpt		# Printer

# PCI Ethernet NICs that use the common MII bus controller code.
# NOTE: Be sure to keep the 'device miibus' line in order to use these NICs!
device		miibus		# MII bus support
device		xl		# 3Com 3c90x (``Boomerang'', ``Cyclone'')

# Pseudo devices - the number indicates how many units to allocate.
pseudo-device	loop		# Network loopback
pseudo-device	ether		# Ethernet support
pseudo-device	sl	1	# Kernel SLIP
pseudo-device	ppp	1	# Kernel PPP
pseudo-device	tun		# Packet tunnel.
pseudo-device	pty		# Pseudo-ttys (telnet etc)
pseudo-device	md		# Memory "disks"
pseudo-device	speaker
pseudo-device	snp	4

# The `bpf' pseudo-device enables the Berkeley Packet Filter.
# Be aware of the administrative consequences of enabling this!
pseudo-device	bpf		#Berkeley packet filter

options		SC_HISTORY_SIZE=2000
-------------- next part --------------
A non-text attachment was scrubbed...
Name: not available
Type: application/pgp-signature
Size: 187 bytes
Desc: not available
Url : http://lists.freebsd.org/pipermail/freebsd-stable/attachments/20031105/81564612/attachment.bin


More information about the freebsd-stable mailing list