Panic in ZFS layer on 8.1-STABLE

[Dan Nelson]

In the last episode (Dec 15), Andriy Gapon said:
> on 15/12/2010 10:28 Jeremie Le Hen said the following:
> > Hi,
> > 
> > [ Please Cc: me when replying, as I'm not subscribed to -stable at . ]
> > 
> > My filer at home runs FreeBSD.  A single data RAID-1 zpool with 10~15
> > datasets, two of them using compression.  Over the night, I got the
> > following panic:
> 
> Thanks for the stack trace!
> But where is the promised panic message? :)
> 
> I suspect that you ran out of kernel address space.
> You'd probably have to tune your system and/or add more memory.
> Please research this topic via mailing lists archives.
> 
> > Tracing pid 0 tid 100111 td 0x86393a00
> > kdb_enter(809faa5b,809faa5b,80a12e84,cb114aec,0,...) at kdb_enter+0x3a
> > panic(80a12e84,1c000,2e3e8000,80a12e7e,7d0,...) at panic+0x131
> > kmem_malloc(8169008c,1c000,2,cb114b6c,80909a99,...) at kmem_malloc+0x285
> > page_alloc(0,1c000,cb114b5f,2,2f0c800,...) at page_alloc+0x27
> > uma_large_malloc(1c000,2,0,8609b3f0,30,...) at uma_large_malloc+0x4a
> > malloc(1c000,860b2120,2,cb114bb0,8601d36d,...) at malloc+0x7c
> > zfs_kmem_alloc(1c000,2,cb114bf0,8601f77b,1c000,...) at
> > zfs_kmem_alloc+0x20
> > zio_buf_alloc(1c000,cb114c30,86008817,92c33bd0,cb114bf0,...) at
> > zio_buf_alloc+0x44
> > zio_compress_data(3,b4264000,20000,0,cb114c58,...) at
> > zio_compress_data+0x8b

The following patch may help you.  It helps me :)  It converts the
zio_buf_alloc() call into a zio_buf_alloc_nowait(), so that if the alloc
fails, zio_compress_data() returns failure and zfs writes the block
uncompressed instead of panicing.

Index: sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c
===================================================================
--- sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c	(revision 216418)
+++ sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c	(working copy)
@@ -202,6 +202,20 @@ zio_buf_alloc(size_t size)
 		return (kmem_alloc(size, KM_SLEEP));
 }
 
+void *
+zio_buf_alloc_nowait(size_t size)
+{
+#ifdef ZIO_USE_UMA
+	size_t c = (size - 1) >> SPA_MINBLOCKSHIFT;
+
+	ASSERT(c < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT);
+
+	return (kmem_cache_alloc(zio_buf_cache[c], KM_NOSLEEP));
+#else
+	return (kmem_alloc(size, KM_NOSLEEP));
+#endif
+}
+
 /*
  * Use zio_data_buf_alloc to allocate data.  The data will not appear in a
  * crashdump if the kernel panics.  This exists so that we will limit the amount
Index: sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zio_compress.c
===================================================================
--- sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zio_compress.c	(revision 216418)
+++ sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zio_compress.c	(working copy)
@@ -32,6 +32,12 @@
 #include <sys/zio.h>
 #include <sys/zio_compress.h>
 
+int panics_avoided_by_not_compressing = 0;
+SYSCTL_DECL(_vfs_zfs);
+SYSCTL_INT(_vfs_zfs, OID_AUTO, compression_panics_avoided, CTLFLAG_RD, 
+	&panics_avoided_by_not_compressing, 0,
+    "kmem_map panics avoided by skipping compression when memory is low");
+
 /*
  * Compression vectors.
  */
@@ -109,7 +115,17 @@ zio_compress_data(int cpfunc, void *src, uint64_t
 	destbufsize = P2ALIGN(srcsize - (srcsize >> 3), SPA_MINBLOCKSIZE);
 	if (destbufsize == 0)
 		return (0);
+
+#if 1
+	dest = zio_buf_alloc_nowait(destbufsize);
+	if (dest == 0)
+	{
+		panics_avoided_by_not_compressing++;
+		return (0);
+	}
+#else
 	dest = zio_buf_alloc(destbufsize);
+#endif
 
 	ciosize = ci->ci_compress(src, dest, (size_t)srcsize,
 	    (size_t)destbufsize, ci->ci_level);
Index: sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/zio.h
===================================================================
--- sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/zio.h	(revision 216418)
+++ sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/zio.h	(working copy)
@@ -398,6 +398,7 @@ extern zio_t *zio_unique_parent(zio_t *cio);
 extern void zio_add_child(zio_t *pio, zio_t *cio);
 
 extern void *zio_buf_alloc(size_t size);
+extern void *zio_buf_alloc_nowait(size_t size);
 extern void zio_buf_free(void *buf, size_t size);
 extern void *zio_data_buf_alloc(size_t size);
 extern void zio_data_buf_free(void *buf, size_t size);



-- 
	Dan Nelson
	dnelson at allantgroup.com