Processes hung in 'inode' state ...

[Marc G. Fournier]

Not sure what all to do, but doing a 'gdb -k kernel.debug /dev/mem', a
backtrack on one of the processes shows ... server has been up 12 days
now, and running a June 8th kernel ...

#0  0x20f4f0 in ?? ()
(kgdb) proc 67258
(kgdb) bt
#0  mi_switch () at machine/globals.h:119
#1  0x8014a1f9 in tsleep (ident=0x8a3d2200, priority=8, wmesg=0x80263d4a "inode", timo=0) at /usr/src/sys/kern/kern_synch.c:479
#2  0x80141507 in acquire (lkp=0x8a3d2200, extflags=16777280, wanted=1536) at /usr/src/sys/kern/kern_lock.c:147
#3  0x8014179c in lockmgr (lkp=0x8a3d2200, flags=16973826, interlkp=0xbdc27b6c, p=0xc26fa9c0) at /usr/src/sys/kern/kern_lock.c:355
#4  0x80172b73 in vop_stdlock (ap=0xc2a95d1c) at /usr/src/sys/kern/vfs_default.c:256
#5  0x801f0955 in ufs_vnoperate (ap=0xc2a95d1c) at /usr/src/sys/ufs/ufs/ufs_vnops.c:2376
#6  0x8017d450 in vn_lock (vp=0xbdc27b00, flags=131074, p=0xc26fa9c0) at vnode_if.h:861
#7  0x80173b1c in lookup (ndp=0xc2a95ec4) at /usr/src/sys/kern/vfs_lookup.c:301
#8  0x80173824 in namei (ndp=0xc2a95ec4) at /usr/src/sys/kern/vfs_lookup.c:153
#9  0x8017c867 in vn_open (ndp=0xc2a95ec4, fmode=1, cmode=420) at /usr/src/sys/kern/vfs_vnops.c:138
#10 0x8017899c in open (p=0xc26fa9c0, uap=0xc2a95f80) at /usr/src/sys/kern/vfs_syscalls.c:1029
#11 0x8023c485 in syscall2 (frame={tf_fs = 47, tf_es = 47, tf_ds = 2143223855, tf_edi = 4, tf_esi = 672253248, tf_ebp = 2143284976, tf_isp = -1029087276, tf_ebx = 672178060, tf_edx = 672253248,
      tf_ecx = 16, tf_eax = 5, tf_trapno = 12, tf_err = 2, tf_eip = 672086720, tf_cs = 31, tf_eflags = 535, tf_esp = 2143284932, tf_ss = 47}) at /usr/src/sys/i386/i386/trap.c:1175
#12 0x802299ab in Xint0x80_syscall ()
#13 0x2807ac6c in ?? ()
#14 0x2807abf0 in ?? ()
#15 0x804acbd in ?? ()
#16 0x8049e1e in ?? ()

and another one:

(kgdb) proc 66479
(kgdb) bt
#0  mi_switch () at machine/globals.h:119
#1  0x8014a1f9 in tsleep (ident=0x8a3d2200, priority=8, wmesg=0x80263d4a "inode", timo=0) at /usr/src/sys/kern/kern_synch.c:479
#2  0x80141507 in acquire (lkp=0x8a3d2200, extflags=16777280, wanted=1536) at /usr/src/sys/kern/kern_lock.c:147
#3  0x8014179c in lockmgr (lkp=0x8a3d2200, flags=16973826, interlkp=0xbdc27b6c, p=0xbcc74400) at /usr/src/sys/kern/kern_lock.c:355
#4  0x80172b73 in vop_stdlock (ap=0xbcebdcc8) at /usr/src/sys/kern/vfs_default.c:256
#5  0x801f0955 in ufs_vnoperate (ap=0xbcebdcc8) at /usr/src/sys/ufs/ufs/ufs_vnops.c:2376
#6  0x8017d450 in vn_lock (vp=0xbdc27b00, flags=131074, p=0xbcc74400) at vnode_if.h:861
#7  0x80173b1c in lookup (ndp=0xbcebdedc) at /usr/src/sys/kern/vfs_lookup.c:301
#8  0x80173824 in namei (ndp=0xbcebdedc) at /usr/src/sys/kern/vfs_lookup.c:153
#9  0x8013e1f6 in execve (p=0xbcc74400, uap=0xbcebdf80) at /usr/src/sys/kern/kern_exec.c:165
#10 0x8023c485 in syscall2 (frame={tf_fs = 2143223855, tf_es = 47, tf_ds = 2143223855, tf_edi = 134582432, tf_esi = 4, tf_ebp = 2143286828, tf_isp = -1125392428, tf_ebx = 673558412,
      tf_edx = 2143286892, tf_ecx = 134594560, tf_eax = 59, tf_trapno = 12, tf_err = 2, tf_eip = 673241568, tf_cs = 31, tf_eflags = 531, tf_esp = 2143286784, tf_ss = 47})
    at /usr/src/sys/i386/i386/trap.c:1175
#11 0x802299ab in Xint0x80_syscall ()
cannot open /proc/66479/mem
(kgdb)

vnodes, kvm, mallocs all appear to be normal:

Jun 23 12:33:00 neptune root: debug.numvnodes: 355550 - debug.freevnodes: 118314 - debug.vnlru_nowhere: 61 - vlruwt
Jun 23 12:33:00 neptune root:
Jun 23 12:33:00 neptune root:        unpath 40495   712K   1261K204800K 15310411    0     0  16,32,64
Jun 23 12:33:00 neptune root:          temp 61298  7542K  11330K204800K 81655865    0     0  16,32,64,128,256,512,1K,2K,4K,8K,16K,64K,128K
Jun 23 12:33:00 neptune root:
Jun 23 12:33:00 neptune root: vm.kvm_size: 2139095040
Jun 23 12:33:00 neptune root: vm.kvm_free: 822083584
Jun 23 12:33:00 neptune root:
Jun 23 12:33:00 neptune root: 567/1748/16384 mbuf clusters in use (current/peak/max)