svn commit: r362740 - head/sys/vm

[Mark Johnston]

Author: markj
Date: Sun Jun 28 21:35:04 2020
New Revision: 362740
URL: https://svnweb.freebsd.org/changeset/base/362740

Log:
  Fix UMA's first-touch policy on systems with empty domains.
  
  Suppose a thread is running on a CPU in a NUMA domain with no physical
  RAM.  When an item is freed to a first-touch zone, it ends up in the
  cross-domain bucket.  When the bucket is full, it gets placed in another
  domain's bucket queue.  However, when allocating an item, UMA will
  always go to the keg upon a per-CPU cache miss because the empty
  domain's bucket queue will always be empty.  This means that a non-empty
  domain's bucket queues can grow very rapidly on such systems.  For
  example, it can easily cause mbuf allocation failures when the zone
  limit is reached.
  
  Change cache_alloc() to follow a round-robin policy when running on an
  empty domain.
  
  Sponsored by:	The FreeBSD Foundation
  Differential Revision:	https://reviews.freebsd.org/D25355

Modified:
  head/sys/vm/uma_core.c

Modified: head/sys/vm/uma_core.c
==============================================================================
--- head/sys/vm/uma_core.c	Sun Jun 28 21:34:38 2020	(r362739)
+++ head/sys/vm/uma_core.c	Sun Jun 28 21:35:04 2020	(r362740)
@@ -3398,7 +3398,7 @@ static __noinline bool
 cache_alloc(uma_zone_t zone, uma_cache_t cache, void *udata, int flags)
 {
 	uma_bucket_t bucket;
-	int domain;
+	int curdomain, domain;
 	bool new;
 
 	CRITICAL_ASSERT(curthread);
@@ -3445,7 +3445,8 @@ cache_alloc(uma_zone_t zone, uma_cache_t cache, void *
 	 * the critical section.
 	 */
 	domain = PCPU_GET(domain);
-	if ((cache_uz_flags(cache) & UMA_ZONE_ROUNDROBIN) != 0)
+	if ((cache_uz_flags(cache) & UMA_ZONE_ROUNDROBIN) != 0 ||
+	    VM_DOMAIN_EMPTY(domain))
 		domain = zone_domain_highest(zone, domain);
 	bucket = cache_fetch_bucket(zone, cache, domain);
 	if (bucket == NULL) {
@@ -3470,7 +3471,8 @@ cache_alloc(uma_zone_t zone, uma_cache_t cache, void *
 	cache = &zone->uz_cpu[curcpu];
 	if (cache->uc_allocbucket.ucb_bucket == NULL &&
 	    ((cache_uz_flags(cache) & UMA_ZONE_FIRSTTOUCH) == 0 ||
-	    domain == PCPU_GET(domain))) {
+	    (curdomain = PCPU_GET(domain)) == domain ||
+	    VM_DOMAIN_EMPTY(curdomain))) {
 		if (new)
 			atomic_add_long(&ZDOM_GET(zone, domain)->uzd_imax,
 			    bucket->ub_cnt);
@@ -3877,7 +3879,7 @@ zone_alloc_bucket(uma_zone_t zone, void *udata, int do
 	/* Avoid allocs targeting empty domains. */
 	if (domain != UMA_ANYDOMAIN && VM_DOMAIN_EMPTY(domain))
 		domain = UMA_ANYDOMAIN;
-	if ((zone->uz_flags & UMA_ZONE_ROUNDROBIN) != 0)
+	else if ((zone->uz_flags & UMA_ZONE_ROUNDROBIN) != 0)
 		domain = UMA_ANYDOMAIN;
 
 	if (zone->uz_max_items > 0)