Apache seg faults -- Possible problem with libc?

[Norbert Papke]

Environment:  FreeBSD 7.0 Stable (as of Apr 30), apache-2.0.63

I am experiencing Apache crashes on a fairly consistent and frequent basis.  
The crash occurs in strncmp().  To help with the diagnosis, I have rebuilt 
libc with debug symbols.  Here is a typical stack dump:

  #0  strncmp () at /usr/src/lib/libc/i386/string/strncmp.S:69
  #1  0x2832558c in getenv (name=0x28338648 "TZ") 
     at /usr/src/lib/libc/stdlib/getenv.c:144
  #2  0x2830ce3a in tzset_basic (rdlocked=0)
     at /usr/src/lib/libc/stdtime/localtime.c:1013
  #3  0x2830d42f in localtime (timep=0xbfbfc1d4)
     at /usr/src/lib/libc/stdtime/localtime.c:1158
  #4  0x28220e6c in explode_time () from /usr/local/lib/apache2/libapr-0.so.9
  #5  0x28220f13 in apr_time_exp_lt ()
     from /usr/local/lib/apache2/libapr-0.so.9
  #6  0x08070227 in cached_explode ()
  #7  0x28374519 in log_request_time ()
     from /usr/local/libexec/apache2/mod_log_config.so
  #8  0x2837358d in config_log_transaction ()
     from /usr/local/libexec/apache2/mod_log_config.so
  #9  0x28373654 in multi_log_transaction ()
     from /usr/local/libexec/apache2/mod_log_config.so
  #10 0x08074519 in ap_run_log_transaction ()
  #11 0x08063d62 in ap_process_request ()
  #12 0x0805e718 in ap_process_http_connection ()
  #13 0x08070817 in ap_run_process_connection ()
  #14 0x08064fde in child_main ()
  #15 0x08065283 in make_child ()
  #16 0x08065a51 in ap_mpm_run ()
  #17 0x0806bfb8 in main ()

Frames 0 - 4 are present in all crash scenarios.  However, similar crashes 
occur with different paths into explode_time().

  (gdb) frame 0
  #0  strncmp () at /usr/src/lib/libc/i386/string/strncmp.S:69
  69              movb    (%eax),%bl
  (gdb) p/x $eax
  $70 = 0x883a4bc
  (gdb) p/x *$eax
  Cannot access memory at address 0x883a4bc

eax contains the first string to be compared.  This is an invalid memory 
location.

  (gdb) frame 1
  #1  0x2832558c in getenv (name=0x28338648 "TZ")   
     at /usr/src/lib/libc/stdlib/getenv.c:144
  144             if (strncmp(nameValue, name, nameLen) == 0 && 
      nameValue[nameLen] == '=')
  Current language:  auto; currently c
  (gdb) p envVarsTotal
  $71 = 57
  (gdb) p envVars[56]
  $72 = {nameLen = 4294967295, valueSize = 4294967295, 
     name = 0x883a4bc <Address 0x883a4bc out of bounds>,
     value = 0x0, active = true, putenv = true}
  (gdb) p envVars[55]
  $73 = {nameLen = 16, valueSize = 4, 
     name =  0x8303f20 "KDE_FULL_SESSION=true", 
     value = 0x8303f31 "true",
     active = true, putenv = false}

Because of the inline functions used in getenv(), gdb's location information 
is somewhat incomplete.  However, I believe that line 144 was called by 
__findenv(),  The fault occurs when __findenv() tries to access what appears 
to be an invalid environment variable.  Most likely, envVarsTotal is too big 
by one.

For reference, here is the code for __findenv().  The calling line is marked.

static inline char *
__findenv(const char *name, size_t nameLen, int *envNdx, bool onlyActive)
{
	int ndx;

	/*
	 * Find environment variable from end of array (more likely to be
	 * active).  A variable created by putenv is always active or it is not
	 * tracked in the array.
	 */
	for (ndx = *envNdx; ndx >= 0; ndx--)
		if (envVars[ndx].putenv) {
/* ==> */		if (strncmpeq(envVars[ndx].name, name, nameLen)) {
				*envNdx = ndx;
				return (envVars[ndx].name + nameLen +
				    sizeof ("=") - 1);
			}
		} else if ((!onlyActive || envVars[ndx].active) &&
		    (envVars[ndx].nameLen == nameLen &&
		    strncmpeq(envVars[ndx].name, name, nameLen))) {
			*envNdx = ndx;
			return (envVars[ndx].value);
		}

	return (NULL);
}


which is called by

char *
getenv(const char *name)
{
	int envNdx;
	size_t nameLen;

	/* Check for malformed name. */
	if (name == NULL || (nameLen = __strleneq(name)) == 0) {
		errno = EINVAL;
		return (NULL);
	}

	/*
	 * Find environment variable via environ if no changes have been made
	 * via a *env() call or environ has been replaced by a running program,
	 * otherwise, use the rebuilt environment.
	 */
	if (envVars == NULL || environ != intEnviron)
		return (__findenv_environ(name, nameLen));
	else {
		envNdx = envVarsTotal - 1;
/* ==> */		return (__findenv(name, nameLen, &envNdx, true));
	}
}



Any suggestions on how I could nail down the cause?  

Cheers,

-- Norbert.