svn commit: r258340 - head/sys/vm

[Alexander Motin]

Author: mav
Date: Tue Nov 19 10:51:46 2013
New Revision: 258340
URL: http://svnweb.freebsd.org/changeset/base/258340

Log:
  Implement mechanism to safely but slowly purge UMA per-CPU caches.
  
  This is a last resort for very low memory condition in case other measures
  to free memory were ineffective.  Sequentially cycle through all CPUs and
  extract per-CPU cache buckets into zone cache from where they can be freed.

Modified:
  head/sys/vm/uma_core.c

Modified: head/sys/vm/uma_core.c
==============================================================================
--- head/sys/vm/uma_core.c	Tue Nov 19 10:39:48 2013	(r258339)
+++ head/sys/vm/uma_core.c	Tue Nov 19 10:51:46 2013	(r258340)
@@ -75,6 +75,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/proc.h>
 #include <sys/rwlock.h>
 #include <sys/sbuf.h>
+#include <sys/sched.h>
 #include <sys/smp.h>
 #include <sys/vmmeter.h>
 
@@ -684,6 +685,78 @@ cache_drain(uma_zone_t zone)
 	ZONE_UNLOCK(zone);
 }
 
+static void
+cache_shrink(uma_zone_t zone)
+{
+
+	if (zone->uz_flags & UMA_ZFLAG_INTERNAL)
+		return;
+
+	ZONE_LOCK(zone);
+	zone->uz_count = (zone->uz_count_min + zone->uz_count) / 2;
+	ZONE_UNLOCK(zone);
+}
+
+static void
+cache_drain_safe_cpu(uma_zone_t zone)
+{
+	uma_cache_t cache;
+
+	if (zone->uz_flags & UMA_ZFLAG_INTERNAL)
+		return;
+
+	ZONE_LOCK(zone);
+	critical_enter();
+	cache = &zone->uz_cpu[curcpu];
+	if (cache->uc_allocbucket) {
+		LIST_INSERT_HEAD(&zone->uz_buckets, cache->uc_allocbucket,
+		    ub_link);
+		cache->uc_allocbucket = NULL;
+	}
+	if (cache->uc_freebucket) {
+		LIST_INSERT_HEAD(&zone->uz_buckets, cache->uc_freebucket,
+		    ub_link);
+		cache->uc_freebucket = NULL;
+	}
+	critical_exit();
+	ZONE_UNLOCK(zone);
+}
+
+/*
+ * Safely drain per-CPU caches of a zone(s) to alloc bucket.
+ * This is an expensive call because it needs to bind to all CPUs
+ * one by one and enter a critical section on each of them in order
+ * to safely access their cache buckets.
+ * Zone lock must not be held on call this function.
+ */
+static void
+cache_drain_safe(uma_zone_t zone)
+{
+	int cpu;
+
+	/*
+	 * Polite bucket sizes shrinking was not enouth, shrink aggressively.
+	 */
+	if (zone)
+		cache_shrink(zone);
+	else
+		zone_foreach(cache_shrink);
+
+	CPU_FOREACH(cpu) {
+		thread_lock(curthread);
+		sched_bind(curthread, cpu);
+		thread_unlock(curthread);
+
+		if (zone)
+			cache_drain_safe_cpu(zone);
+		else
+			zone_foreach(cache_drain_safe_cpu);
+	}
+	thread_lock(curthread);
+	sched_unbind(curthread);
+	thread_unlock(curthread);
+}
+
 /*
  * Drain the cached buckets from a zone.  Expects a locked zone on entry.
  */
@@ -3068,6 +3141,10 @@ uma_reclaim(void)
 #endif
 	bucket_enable();
 	zone_foreach(zone_drain);
+	if (vm_page_count_min()) {
+		cache_drain_safe(NULL);
+		zone_foreach(zone_drain);
+	}
 	/*
 	 * Some slabs may have been freed but this zone will be visited early
 	 * we visit again so that we can free pages that are empty once other