RELENG_5 panic

[Robin P. Blanchard]

> Oh another of these wonderful races... can you go to that 
> frame and "print ts"?  If its NULL then someone has ripped 
> out the ts out from under us since it was checked for NULL in 
> the previous line!

Maybe this is a more useful kgdb session (I'm hoping)

# kgdb kernel.debug /usr/local/var/adm/crash/vmcore.44 
[GDB will not be able to debug user-mode threads: /usr/lib/libthread_db.so:
Undefined symbol "ps_pglobal_lookup"]
GNU gdb 6.1.1 [FreeBSD]
Copyright 2004 Free Software Foundation, Inc.
GDB is free software, covered by the GNU General Public License, and you are
welcome to change it and/or distribute copies of it under certain conditions.
Type "show copying" to see the conditions.
There is absolutely no warranty for GDB.  Type "show warranty" for details.
This GDB was configured as "i386-marcel-freebsd".
#0  doadump () at pcpu.h:160
160             __asm __volatile("movl %%fs:0,%0" : "=r" (td));
(kgdb) l *0xc0504808
0xc0504808 is in turnstile_wait (/usr/src/sys/kern/subr_turnstile.c:245).
240                     /*
241                      * Pick up the lock that td is blocked on.
242                      */
243                     ts = td->td_blocked;
244                     MPASS(ts != NULL);
245                     tc = TC_LOOKUP(ts->ts_lockobj);
246                     mtx_lock_spin(&tc->tc_lock);
247
248                     /*
249                      * This thread may not be blocked on this turnstile
anymore
(kgdb) bt 
#0  doadump () at pcpu.h:160
#1  0xc04dd58c in boot (howto=260) at /usr/src/sys/kern/kern_shutdown.c:410
#2  0xc04ddccd in panic (fmt=0xc066e594 "%s") at
/usr/src/sys/kern/kern_shutdown.c:566
#3  0xc0641e92 in trap_fatal (frame=0xc7ac0bc8, eva=36) at
/usr/src/sys/i386/i386/trap.c:817
#4  0xc0642535 in trap (frame=
      {tf_fs = 24, tf_es = -1066598384, tf_ds = -1066532848, tf_edi =
-1053916800, tf_esi = -1049515008, tf_ebp = -945025988, tf_isp = -945026060,
tf_ebx = -1053916800, tf_edx = -1053937024, tf_ecx = 56, tf_eax = 0,
tf_trapno = 12, tf_err = 0, tf_eip = -1068480504, tf_cs = 8, tf_eflags =
65683, tf_esp = -1053914880, tf_ss = 582}) at
/usr/src/sys/i386/i386/trap.c:255
#5  0xc062da3a in calltrap () at /usr/src/sys/i386/i386/exception.s:140
#6  0x00000018 in ?? ()
#7  0xc06d0010 in ipq ()
#8  0xc06e0010 in sc_buffer.3 ()
#9  0xc12e8180 in ?? ()
#10 0xc171ac00 in ?? ()
#11 0xc7ac0c3c in ?? ()
#12 0xc7ac0bf4 in ?? ()
#13 0xc12e8180 in ?? ()
#14 0xc12e3280 in ?? ()
#15 0x00000038 in ?? ()
#16 0x00000000 in ?? ()
#17 0x0000000c in ?? ()
#18 0x00000000 in ?? ()
#19 0xc0504808 in turnstile_wait (ts=0xc12e3280, lock=0xc06d022c,
owner=0xc171ac00)
    at /usr/src/sys/kern/subr_turnstile.c:243
#20 0xc04d2b7f in _mtx_lock_sleep (m=0xc06d022c, td=0xc12e8180, opts=0,
file=0x0, line=0)
    at /usr/src/sys/kern/kern_mutex.c:552
#21 0xc058a592 in tcp_isn_tick (xtp=0x0) at
/usr/src/sys/netinet/tcp_subr.c:1380
#22 0xc04ed069 in softclock (dummy=0x0) at
/usr/src/sys/kern/kern_timeout.c:279
#23 0xc04c460a in ithread_loop (arg=0xc12fd500) at
/usr/src/sys/kern/kern_intr.c:547
#24 0xc04c32c2 in fork_exit (callout=0xc04c4550 <ithread_loop>, arg=0x0,
frame=0x0)
    at /usr/src/sys/kern/kern_fork.c:791
#25 0xc062da9c in fork_trampoline () at
/usr/src/sys/i386/i386/exception.s:209
(kgdb) frame 19
#19 0xc0504808 in turnstile_wait (ts=0xc12e3280, lock=0xc06d022c,
owner=0xc171ac00)
    at /usr/src/sys/kern/subr_turnstile.c:243
243                     ts = td->td_blocked;
(kgdb) list
238                         ts->ts_lockobj->lo_name));
239
240                     /*
241                      * Pick up the lock that td is blocked on.
242                      */
243                     ts = td->td_blocked;
244                     MPASS(ts != NULL);
245                     tc = TC_LOOKUP(ts->ts_lockobj);
246                     mtx_lock_spin(&tc->tc_lock);
247
(kgdb) print ts
$1 = (struct turnstile *) 0xc12e3280
(kgdb) up
#20 0xc04d2b7f in _mtx_lock_sleep (m=0xc06d022c, td=0xc12e8180, opts=0,
file=0x0, line=0)
    at /usr/src/sys/kern/kern_mutex.c:552
552                     turnstile_wait(ts, &m->mtx_object, mtx_owner(m));
(kgdb) list
547     #endif
548
549                     /*
550                      * Block on the turnstile.
551                      */
552                     turnstile_wait(ts, &m->mtx_object, mtx_owner(m));
553             }
554
555     #ifdef KTR
556             if (cont_logged) {
(kgdb) print ts
$2 = (struct turnstile *) 0x0
(kgdb) 

---------------------------------------
Robin P. Blanchard
Systems Integration Specialist
Georgia Center for Continuing Education
fon: 706.542.2404 < > fax: 706.542.6546
---------------------------------------