git: 6337ca19a363 - main - uma: Factor out the implementations of uma_zfree_{arg,smr}()

The branch main has been updated by markj:

URL: https://cgit.FreeBSD.org/src/commit/?id=6337ca19a3637aa72eddf4d62a2eaf7d8df51638

commit 6337ca19a3637aa72eddf4d62a2eaf7d8df51638
Author:     Mark Johnston <markj@FreeBSD.org>
AuthorDate: 2026-07-24 21:11:58 +0000
Commit:     Mark Johnston <markj@FreeBSD.org>
CommitDate: 2026-07-24 21:11:58 +0000

    uma: Factor out the implementations of uma_zfree_{arg,smr}()
    
    The two function both free an item to a UMA zone, but uma_zfree_arg()
    does so in such as way as to ensure that the item will be the first one
    returned by a subsequent allocation, while uma_zfree_smr() must defer
    reuse of the item and therefore never frees to the per-CPU alloc bucket.
    
    When KASAN is enabled, we actually want uma_zfree_arg() to behave like
    uma_zfree_smr(): to improve the reliability of use-after-free detection,
    reuse of the newly freed item should be deferred for some time.
    
    Refactor a bit to make it easier to improve KASAN along these lines:
    introduce two helper functions, cache_free_item() and cache_free_smr(),
    which handle most of the work of interacting with the per-CPU caches.
    A subsequent commit will let uma_zfree_arg() use cache_free_smr() when
    KASAN is enabled.
    
    No functional change intended.
    
    Reviewed by:    rlibby
    MFC after:      1 month
    Sponsored by:   The FreeBSD Foundation
    Differential Revision:  https://reviews.freebsd.org/D58268
---
 sys/vm/uma_core.c | 159 +++++++++++++++++++++++++++++++-----------------------
 1 file changed, 91 insertions(+), 68 deletions(-)

diff --git a/sys/vm/uma_core.c b/sys/vm/uma_core.c
index 410b4f5a5142..02b2df89f636 100644
--- a/sys/vm/uma_core.c
+++ b/sys/vm/uma_core.c
@@ -4447,37 +4447,82 @@ fail:
 	return (NULL);
 }
 
-/* See uma.h */
-void
-uma_zfree_smr(uma_zone_t zone, void *item)
+static __always_inline bool
+cache_free_item(uma_zone_t zone, int uz_flags, void *item, void *udata)
 {
 	uma_cache_t cache;
-	uma_cache_bucket_t bucket;
 	int itemdomain;
+
+	/*
+	 * If possible, free to the per-CPU cache.  There are two
+	 * requirements for safe access to the per-CPU cache: (1) the thread
+	 * accessing the cache must not be preempted or yield during access,
+	 * and (2) the thread must not migrate CPUs without switching which
+	 * cache it accesses.  We rely on a critical section to prevent
+	 * preemption and migration.  We release the critical section in
+	 * order to acquire the zone mutex if we are unable to free to the
+	 * current cache; when we re-acquire the critical section, we must
+	 * detect and handle migration if it has occurred.
+	 */
+	itemdomain = 0;
 #ifdef NUMA
-	int uz_flags;
+	if ((uz_flags & UMA_ZONE_FIRSTTOUCH) != 0)
+		itemdomain = item_domain(item);
 #endif
 
-	CTR3(KTR_UMA, "uma_zfree_smr zone %s(%p) item %p",
-	    zone->uz_name, zone, item);
+	critical_enter();
+	do {
+		uma_cache_bucket_t bucket;
 
-#ifdef UMA_ZALLOC_DEBUG
-	KASSERT((zone->uz_flags & UMA_ZONE_SMR) != 0,
-	    ("uma_zfree_smr: called with non-SMR zone."));
-	KASSERT(item != NULL, ("uma_zfree_smr: Called with NULL pointer."));
-	SMR_ASSERT_NOT_ENTERED(zone->uz_smr);
-	if (uma_zfree_debug(zone, item, NULL) == EJUSTRETURN)
-		return;
+		cache = &zone->uz_cpu[curcpu];
+		/*
+		 * Try to free into the allocbucket first to give LIFO
+		 * ordering for cache-hot datastructures.  Spill over
+		 * into the freebucket if necessary.  Alloc will swap
+		 * them if one runs dry.
+		 */
+		bucket = &cache->uc_allocbucket;
+#ifdef NUMA
+		if ((uz_flags & UMA_ZONE_FIRSTTOUCH) != 0 &&
+		    PCPU_GET(domain) != itemdomain) {
+			bucket = &cache->uc_crossbucket;
+		} else
 #endif
-	cache = &zone->uz_cpu[curcpu];
+		if (bucket->ucb_cnt == bucket->ucb_entries &&
+		   cache->uc_freebucket.ucb_cnt <
+		   cache->uc_freebucket.ucb_entries)
+			cache_bucket_swap(&cache->uc_freebucket,
+			    &cache->uc_allocbucket);
+		if (__predict_true(bucket->ucb_cnt < bucket->ucb_entries)) {
+			cache_bucket_push(cache, bucket, item);
+			critical_exit();
+			return (true);
+		}
+	} while (cache_free(zone, cache, udata, itemdomain));
+	critical_exit();
+
+	return (false);
+}
+
+static __always_inline bool
+cache_free_smr(uma_zone_t zone, void *item, void *udata)
+{
+	uma_cache_t cache;
+	int itemdomain;
+#ifdef NUMA
+	int uz_flags;
+#endif
+
 	itemdomain = 0;
 #ifdef NUMA
-	uz_flags = cache_uz_flags(cache);
+	uz_flags = cache_uz_flags(&zone->uz_cpu[curcpu]);
 	if ((uz_flags & UMA_ZONE_FIRSTTOUCH) != 0)
 		itemdomain = item_domain(item);
 #endif
 	critical_enter();
 	do {
+		uma_cache_bucket_t bucket;
+
 		cache = &zone->uz_cpu[curcpu];
 		/* SMR Zones must free to the free bucket. */
 		bucket = &cache->uc_freebucket;
@@ -4490,11 +4535,33 @@ uma_zfree_smr(uma_zone_t zone, void *item)
 		if (__predict_true(bucket->ucb_cnt < bucket->ucb_entries)) {
 			cache_bucket_push(cache, bucket, item);
 			critical_exit();
-			return;
+			return (true);
 		}
-	} while (cache_free(zone, cache, NULL, itemdomain));
+	} while (cache_free(zone, cache, udata, itemdomain));
 	critical_exit();
 
+	return (false);
+}
+
+/* See uma.h */
+void
+uma_zfree_smr(uma_zone_t zone, void *item)
+{
+	CTR3(KTR_UMA, "uma_zfree_smr zone %s(%p) item %p",
+	    zone->uz_name, zone, item);
+
+#ifdef UMA_ZALLOC_DEBUG
+	KASSERT((zone->uz_flags & UMA_ZONE_SMR) != 0,
+	    ("uma_zfree_smr: called with non-SMR zone."));
+	KASSERT(item != NULL, ("uma_zfree_smr: Called with NULL pointer."));
+	SMR_ASSERT_NOT_ENTERED(zone->uz_smr);
+	if (uma_zfree_debug(zone, item, NULL) == EJUSTRETURN)
+		return;
+#endif
+
+	if (cache_free_smr(zone, item, NULL))
+		return;
+
 	/*
 	 * If nothing else caught this, we'll just do an internal free.
 	 */
@@ -4506,8 +4573,7 @@ void
 uma_zfree_arg(uma_zone_t zone, void *item, void *udata)
 {
 	uma_cache_t cache;
-	uma_cache_bucket_t bucket;
-	int itemdomain, uz_flags;
+	int uz_flags;
 
 	/* Enable entropy collection for RANDOM_ENABLE_UMA kernel option */
 	random_harvest_fast_uma(&zone, sizeof(zone), RANDOM_UMA);
@@ -4540,55 +4606,12 @@ uma_zfree_arg(uma_zone_t zone, void *item, void *udata)
 	 * The race here is acceptable.  If we miss it we'll just have to wait
 	 * a little longer for the limits to be reset.
 	 */
-	if (__predict_false(uz_flags & UMA_ZFLAG_LIMIT)) {
-		if (atomic_load_32(&zone->uz_sleepers) > 0)
-			goto zfree_item;
-	}
+	if (__predict_false(uz_flags & UMA_ZFLAG_LIMIT) &&
+	    atomic_load_32(&zone->uz_sleepers) > 0)
+		goto zfree_item;
 
-	/*
-	 * If possible, free to the per-CPU cache.  There are two
-	 * requirements for safe access to the per-CPU cache: (1) the thread
-	 * accessing the cache must not be preempted or yield during access,
-	 * and (2) the thread must not migrate CPUs without switching which
-	 * cache it accesses.  We rely on a critical section to prevent
-	 * preemption and migration.  We release the critical section in
-	 * order to acquire the zone mutex if we are unable to free to the
-	 * current cache; when we re-acquire the critical section, we must
-	 * detect and handle migration if it has occurred.
-	 */
-	itemdomain = 0;
-#ifdef NUMA
-	if ((uz_flags & UMA_ZONE_FIRSTTOUCH) != 0)
-		itemdomain = item_domain(item);
-#endif
-	critical_enter();
-	do {
-		cache = &zone->uz_cpu[curcpu];
-		/*
-		 * Try to free into the allocbucket first to give LIFO
-		 * ordering for cache-hot datastructures.  Spill over
-		 * into the freebucket if necessary.  Alloc will swap
-		 * them if one runs dry.
-		 */
-		bucket = &cache->uc_allocbucket;
-#ifdef NUMA
-		if ((uz_flags & UMA_ZONE_FIRSTTOUCH) != 0 &&
-		    PCPU_GET(domain) != itemdomain) {
-			bucket = &cache->uc_crossbucket;
-		} else
-#endif
-		if (bucket->ucb_cnt == bucket->ucb_entries &&
-		   cache->uc_freebucket.ucb_cnt <
-		   cache->uc_freebucket.ucb_entries)
-			cache_bucket_swap(&cache->uc_freebucket,
-			    &cache->uc_allocbucket);
-		if (__predict_true(bucket->ucb_cnt < bucket->ucb_entries)) {
-			cache_bucket_push(cache, bucket, item);
-			critical_exit();
-			return;
-		}
-	} while (cache_free(zone, cache, udata, itemdomain));
-	critical_exit();
+	if (cache_free_item(zone, uz_flags, item, udata))
+		return;
 
 	/*
 	 * If nothing else caught this, we'll just do an internal free.