panic with heavy writing on md device

[Brian Fundakowski Feldman]

On Tue, Jun 29, 2004 at 11:11:51PM +0200, Ronald Klop wrote:
> Hello,
> 
> I got a panic while doing a lot of IO on a md device.
> I attached a gdb backtrace of the kernelcore and a dmesg output.
> 
> It happened while untarring a 30 MB tar on the background (&) and
> traversing this new directory in the meantime.
> 
> uname -a
> FreeBSD guido.thuis.klop.ws 5.2-CURRENT FreeBSD 5.2-CURRENT #1: Fri Jun 25  
> 04:03:35 CEST 2004      
> root at guido.thuis.klop.ws:/usr/obj/usr/src/sys/GUIDO  i386
> 
> On the same time I was playing music with mpg123, but that doesn't make  
> use of the md device.
> 
> I have had this more and always while doing a lot of IO on the md device.
> 
> Is there more info you need?

It looks like there is a memory leak -- in general, you are not supposed
to run out of kernel virtual memory address space unless you have a
dire misconfiguration somewhere.

Unfortunately, I think the most useful pieces of information (zone and
malloc allocation) are not fetchable via kvm anymore.  Here's a fix for
vmstat so that you can provide vmstat -m output using the -M and -N
args again.  I think vmstat -c output would also be useful.

-- 
Brian Fundakowski Feldman                           \'[ FreeBSD ]''''''''''\
  <> green at FreeBSD.org                               \  The Power to Serve! \
 Opinions expressed are my own.                       \,,,,,,,,,,,,,,,,,,,,,,\
-------------- next part --------------
Index: vmstat.c
===================================================================
RCS file: /usr/ncvs/src/usr.bin/vmstat/vmstat.c,v
retrieving revision 1.82
diff -u -r1.82 vmstat.c
--- vmstat.c	23 Apr 2004 13:10:29 -0000	1.82
+++ vmstat.c	29 Jun 2004 23:36:57 -0000
@@ -99,18 +99,22 @@
 	{ "_intrcnt" },
 #define	X_EINTRCNT	9
 	{ "_eintrcnt" },
+#define	X_KMEMSTATS	10
+	{ "_kmemstatistics" },
+#define	X_KMEMZONES	11
+	{ "_kmemzones" },
 #ifdef notyet
-#define	X_DEFICIT	10
+#define	X_DEFICIT	XXX
 	{ "_deficit" },
-#define X_REC		11
+#define X_REC		XXX
 	{ "_rectime" },
-#define X_PGIN		12
+#define X_PGIN		XXX
 	{ "_pgintime" },
-#define	X_XSTATS	13
+#define	X_XSTATS	XXX
 	{ "_xstats" },
-#define X_END		14
+#define X_END		XXX
 #else
-#define X_END		10
+#define X_END		12
 #endif
 	{ "" },
 };
@@ -153,6 +157,8 @@
 static void	dovmstat(unsigned int, int);
 static void	dozmem(void);
 static void	kread(int, void *, size_t);
+static void	kreado(int, void *, size_t, size_t);
+static char    *kgetstr(const char *);
 static void	needhdr(int);
 static void	printhdr(void);
 static void	usage(void);
@@ -894,9 +900,59 @@
 static void
 domem(void)
 {
-	if (kd != NULL)
-		errx(1, "not implemented");
-	dosysctl("kern.malloc");
+	struct malloc_type type;
+	
+	if (kd == NULL) {
+		dosysctl("kern.malloc");
+		return;
+	}
+	kread(X_KMEMSTATS, &type.ks_next, sizeof(type.ks_next));
+	(void)printf("\n        Type  InUse MemUse HighUse Requests"
+	    "  Size(s)\n");
+	do { 
+		/* XXX this should be exported in sys/malloc.h */
+		struct {
+			int kz_size;
+			char *kz_name;
+			/* uma_zone_t */ void *kz_zone;
+		} kz;
+		size_t kmemzonenum;
+		char *str;
+		int first;
+
+		if (kvm_read(kd, (u_long)type.ks_next, &type, sizeof(type)) !=
+		    sizeof(type))
+			errx(1, "%s: %p: %s", __func__, type.ks_next,
+			    kvm_geterr(kd));
+		if (type.ks_calls == 0)
+			continue;
+		str = kgetstr(type.ks_shortdesc);
+		(void)printf("%13s%6lu%6luK%7luK%9llu",
+		    str,
+		    type.ks_inuse,
+		    (type.ks_memuse + 1023) / 1024,
+		    (type.ks_maxused + 1023) / 1024,
+		    (long long unsigned)type.ks_calls);
+		free(str);
+		for (kmemzonenum = 0, first = 1; ; kmemzonenum++) {
+			kreado(X_KMEMZONES, &kz, sizeof(kz),
+			    kmemzonenum * sizeof(kz));
+			if (kz.kz_name == NULL) {
+				(void)printf("\n");
+				break;
+			}
+			if (!(type.ks_size & (1 << kmemzonenum)))
+				continue;
+			if (first)
+				(void)printf("  ");
+			else
+				(void)printf(",");
+			first = 0;
+			str = kgetstr(kz.kz_name);
+			(void)printf("%s", str);
+			free(str);
+		}
+	} while (type.ks_next != NULL);
 }
 
 static void
@@ -929,7 +985,7 @@
  * kread reads something from the kernel, given its nlist index.
  */
 static void
-kread(int nlx, void *addr, size_t size)
+kreado(int nlx, void *addr, size_t size, size_t offset)
 {
 	const char *sym;
 
@@ -939,12 +995,38 @@
 			++sym;
 		errx(1, "symbol %s not defined", sym);
 	}
-	if ((size_t)kvm_read(kd, namelist[nlx].n_value, addr, size) != size) {
+	if ((size_t)kvm_read(kd, namelist[nlx].n_value + offset, addr,
+	    size) != size) {
 		sym = namelist[nlx].n_name;
 		if (*sym == '_')
 			++sym;
 		errx(1, "%s: %s", sym, kvm_geterr(kd));
 	}
+}
+
+static void
+kread(int nlx, void *addr, size_t size)
+{
+	kreado(nlx, addr, size, 0);
+}
+
+static char *
+kgetstr(const char *strp)
+{
+	int n = 0, size = 1;
+	char *ret = NULL;
+
+	do {
+		if (size == n + 1) {
+			ret = realloc(ret, size);
+			if (ret == NULL)
+				err(1, "%s: realloc", __func__);
+			size *= 2;
+		}
+		if (kvm_read(kd, (u_long)strp + n, &ret[n], 1) != 1)
+			errx(1, "%s: %s", __func__, kvm_geterr(kd));
+	} while (ret[n++] != '\0');
+	return (ret);
 }
 
 static void