svn commit: r355149 - head/sys/vm
Jeff Roberson
jeff at FreeBSD.org
Thu Nov 28 00:19:09 UTC 2019
Author: jeff
Date: Thu Nov 28 00:19:09 2019
New Revision: 355149
URL: https://svnweb.freebsd.org/changeset/base/355149
Log:
Implement a sysctl tree for uma zones to assist in debugging and provide
more statistcs than are exported via the ABI stable vmstat interface.
Rename uz_count to uz_bucket_size because even I was confused by the
name after returning to the source years later.
Reviewed by: rlibby
Differential Revision: https://reviews.freebsd.org/D22554
Modified:
head/sys/vm/uma_core.c
head/sys/vm/uma_int.h
Modified: head/sys/vm/uma_core.c
==============================================================================
--- head/sys/vm/uma_core.c Wed Nov 27 23:19:06 2019 (r355148)
+++ head/sys/vm/uma_core.c Thu Nov 28 00:19:09 2019 (r355149)
@@ -119,6 +119,7 @@ static uma_zone_t hashzone;
int uma_align_cache = 64 - 1;
static MALLOC_DEFINE(M_UMAHASH, "UMAHash", "UMA Hash Buckets");
+static MALLOC_DEFINE(M_UMA, "UMA", "UMA Misc");
/*
* Are we allowed to allocate buckets?
@@ -258,8 +259,8 @@ static void zone_dtor(void *, int, void *);
static int zero_init(void *, int, int);
static void keg_small_init(uma_keg_t keg);
static void keg_large_init(uma_keg_t keg);
-static void zone_foreach(void (*zfunc)(uma_zone_t));
-static void zone_timeout(uma_zone_t zone);
+static void zone_foreach(void (*zfunc)(uma_zone_t, void *), void *);
+static void zone_timeout(uma_zone_t zone, void *);
static int hash_alloc(struct uma_hash *, u_int);
static int hash_expand(struct uma_hash *, struct uma_hash *);
static void hash_free(struct uma_hash *hash);
@@ -285,10 +286,10 @@ static void uma_zero_item(void *, uma_zone_t);
static bool cache_alloc(uma_zone_t, uma_cache_t, void *, int);
static bool cache_free(uma_zone_t, uma_cache_t, void *, void *, int);
-void uma_print_zone(uma_zone_t);
-void uma_print_stats(void);
static int sysctl_vm_zone_count(SYSCTL_HANDLER_ARGS);
static int sysctl_vm_zone_stats(SYSCTL_HANDLER_ARGS);
+static int sysctl_handle_uma_zone_allocs(SYSCTL_HANDLER_ARGS);
+static int sysctl_handle_uma_zone_frees(SYSCTL_HANDLER_ARGS);
#ifdef INVARIANTS
static bool uma_dbg_kskip(uma_keg_t keg, void *mem);
@@ -296,6 +297,7 @@ static bool uma_dbg_zskip(uma_zone_t zone, void *mem);
static void uma_dbg_free(uma_zone_t zone, uma_slab_t slab, void *item);
static void uma_dbg_alloc(uma_zone_t zone, uma_slab_t slab, void *item);
+SYSCTL_NODE(_vm, OID_AUTO, uma, CTLFLAG_RW, 0, "Universal Memory Allocator");
static SYSCTL_NODE(_vm, OID_AUTO, debug, CTLFLAG_RD, 0,
"Memory allocation debugging");
@@ -458,7 +460,7 @@ bucket_alloc(uma_zone_t zone, void *udata, int flags)
}
if ((uintptr_t)udata & UMA_ZFLAG_CACHEONLY)
flags |= M_NOVM;
- ubz = bucket_zone_lookup(zone->uz_count);
+ ubz = bucket_zone_lookup(zone->uz_bucket_size);
if (ubz->ubz_zone == zone && (ubz + 1)->ubz_entries != 0)
ubz++;
bucket = uma_zalloc_arg(ubz->ubz_zone, udata, flags);
@@ -575,7 +577,7 @@ static void
uma_timeout(void *unused)
{
bucket_enable();
- zone_foreach(zone_timeout);
+ zone_foreach(zone_timeout, NULL);
/* Reschedule this event */
callout_reset(&uma_callout, UMA_TIMEOUT * hz, uma_timeout, NULL);
@@ -605,7 +607,7 @@ zone_domain_update_wss(uma_zone_domain_t zdom)
* Returns nothing.
*/
static void
-zone_timeout(uma_zone_t zone)
+zone_timeout(uma_zone_t zone, void *unused)
{
uma_keg_t keg;
u_int slabs;
@@ -845,19 +847,20 @@ cache_drain(uma_zone_t zone)
}
static void
-cache_shrink(uma_zone_t zone)
+cache_shrink(uma_zone_t zone, void *unused)
{
if (zone->uz_flags & UMA_ZFLAG_INTERNAL)
return;
ZONE_LOCK(zone);
- zone->uz_count = (zone->uz_count_min + zone->uz_count) / 2;
+ zone->uz_bucket_size =
+ (zone->uz_bucket_size_min + zone->uz_bucket_size) / 2;
ZONE_UNLOCK(zone);
}
static void
-cache_drain_safe_cpu(uma_zone_t zone)
+cache_drain_safe_cpu(uma_zone_t zone, void *unused)
{
uma_cache_t cache;
uma_bucket_t b1, b2, b3;
@@ -920,9 +923,9 @@ pcpu_cache_drain_safe(uma_zone_t zone)
* Polite bucket sizes shrinking was not enouth, shrink aggressively.
*/
if (zone)
- cache_shrink(zone);
+ cache_shrink(zone, NULL);
else
- zone_foreach(cache_shrink);
+ zone_foreach(cache_shrink, NULL);
CPU_FOREACH(cpu) {
thread_lock(curthread);
@@ -930,9 +933,9 @@ pcpu_cache_drain_safe(uma_zone_t zone)
thread_unlock(curthread);
if (zone)
- cache_drain_safe_cpu(zone);
+ cache_drain_safe_cpu(zone, NULL);
else
- zone_foreach(cache_drain_safe_cpu);
+ zone_foreach(cache_drain_safe_cpu, NULL);
}
thread_lock(curthread);
sched_unbind(curthread);
@@ -991,8 +994,8 @@ bucket_cache_reclaim(uma_zone_t zone, bool drain)
* Shrink the zone bucket size to ensure that the per-CPU caches
* don't grow too large.
*/
- if (zone->uz_count > zone->uz_count_min)
- zone->uz_count--;
+ if (zone->uz_bucket_size > zone->uz_bucket_size_min)
+ zone->uz_bucket_size--;
}
static void
@@ -1121,14 +1124,14 @@ out:
}
static void
-zone_drain(uma_zone_t zone)
+zone_drain(uma_zone_t zone, void *unused)
{
zone_reclaim(zone, M_NOWAIT, true);
}
static void
-zone_trim(uma_zone_t zone)
+zone_trim(uma_zone_t zone, void *unused)
{
zone_reclaim(zone, M_NOWAIT, false);
@@ -1818,7 +1821,7 @@ keg_ctor(void *mem, int size, void *udata, int flags)
}
static void
-zone_alloc_counters(uma_zone_t zone)
+zone_alloc_counters(uma_zone_t zone, void *unused)
{
zone->uz_allocs = counter_u64_alloc(M_WAITOK);
@@ -1826,6 +1829,169 @@ zone_alloc_counters(uma_zone_t zone)
zone->uz_fails = counter_u64_alloc(M_WAITOK);
}
+#define UMA_MAX_DUP 999
+static void
+zone_alloc_sysctl(uma_zone_t zone, void *unused)
+{
+ uma_zone_domain_t zdom;
+ uma_keg_t keg;
+ struct sysctl_oid *oid, *domainoid;
+ int domains, i;
+ static const char *nokeg = "cache zone";
+ char *c;
+
+ /*
+ * Make a sysctl safe copy of the zone name by removing
+ * any special characters and handling dups by appending
+ * an index.
+ */
+ if (zone->uz_namecnt != 0) {
+ if (zone->uz_namecnt > UMA_MAX_DUP)
+ zone->uz_namecnt = UMA_MAX_DUP;
+ zone->uz_ctlname = malloc(strlen(zone->uz_name) +
+ sizeof(__XSTRING(UMA_MAX_DUP)) + 1 , M_UMA, M_WAITOK);
+ sprintf(zone->uz_ctlname, "%s_%d", zone->uz_name,
+ zone->uz_namecnt);
+ } else
+ zone->uz_ctlname = strdup(zone->uz_name, M_UMA);
+ for (c = zone->uz_ctlname; *c != '\0'; c++)
+ if (strchr("./\\ -", *c) != NULL)
+ *c = '_';
+
+ /*
+ * Basic parameters at the root.
+ */
+ zone->uz_oid = SYSCTL_ADD_NODE(NULL, SYSCTL_STATIC_CHILDREN(_vm_uma),
+ OID_AUTO, zone->uz_ctlname, CTLFLAG_RD, NULL, "");
+ oid = zone->uz_oid;
+ SYSCTL_ADD_U32(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
+ "size", CTLFLAG_RD, &zone->uz_size, 0, "Allocation size");
+ SYSCTL_ADD_U32(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
+ "flags", CTLFLAG_RD, &zone->uz_flags, 0,
+ "Allocator configuration flags");
+ SYSCTL_ADD_U16(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
+ "bucket_size", CTLFLAG_RD, &zone->uz_bucket_size, 0,
+ "Desired per-cpu cache size");
+ SYSCTL_ADD_U16(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
+ "bucket_size_max", CTLFLAG_RD, &zone->uz_bucket_size_max, 0,
+ "Maximum allowed per-cpu cache size");
+
+ /*
+ * keg if present.
+ */
+ oid = SYSCTL_ADD_NODE(NULL, SYSCTL_CHILDREN(zone->uz_oid), OID_AUTO,
+ "keg", CTLFLAG_RD, NULL, "");
+ keg = zone->uz_keg;
+ if ((zone->uz_flags & UMA_ZFLAG_CACHEONLY) == 0) {
+ SYSCTL_ADD_CONST_STRING(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
+ "name", CTLFLAG_RD, keg->uk_name, "Keg name");
+ SYSCTL_ADD_U32(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
+ "rsize", CTLFLAG_RD, &keg->uk_rsize, 0,
+ "Real object size with alignment");
+ SYSCTL_ADD_U16(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
+ "ppera", CTLFLAG_RD, &keg->uk_ppera, 0,
+ "pages per-slab allocation");
+ SYSCTL_ADD_U16(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
+ "ipers", CTLFLAG_RD, &keg->uk_ipers, 0,
+ "items available per-slab");
+ SYSCTL_ADD_U32(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
+ "align", CTLFLAG_RD, &keg->uk_align, 0,
+ "item alignment mask");
+ SYSCTL_ADD_U32(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
+ "pages", CTLFLAG_RD, &keg->uk_pages, 0,
+ "Total pages currently allocated from VM");
+ SYSCTL_ADD_U32(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
+ "free", CTLFLAG_RD, &keg->uk_free, 0,
+ "items free in the slab layer");
+ } else
+ SYSCTL_ADD_CONST_STRING(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
+ "name", CTLFLAG_RD, nokeg, "Keg name");
+
+ /*
+ * Information about zone limits.
+ */
+ oid = SYSCTL_ADD_NODE(NULL, SYSCTL_CHILDREN(zone->uz_oid), OID_AUTO,
+ "limit", CTLFLAG_RD, NULL, "");
+ SYSCTL_ADD_U64(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
+ "items", CTLFLAG_RD, &zone->uz_items, 0,
+ "current number of cached items");
+ SYSCTL_ADD_U64(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
+ "max_items", CTLFLAG_RD, &zone->uz_max_items, 0,
+ "Maximum number of cached items");
+ SYSCTL_ADD_U32(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
+ "sleepers", CTLFLAG_RD, &zone->uz_sleepers, 0,
+ "Number of threads sleeping at limit");
+ SYSCTL_ADD_U64(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
+ "sleeps", CTLFLAG_RD, &zone->uz_sleeps, 0,
+ "Total zone limit sleeps");
+
+ /*
+ * Per-domain information.
+ */
+ if ((zone->uz_flags & UMA_ZONE_NUMA) != 0)
+ domains = vm_ndomains;
+ else
+ domains = 1;
+ domainoid = SYSCTL_ADD_NODE(NULL, SYSCTL_CHILDREN(zone->uz_oid),
+ OID_AUTO, "domain", CTLFLAG_RD, NULL, "");
+ for (i = 0; i < domains; i++) {
+ zdom = &zone->uz_domain[i];
+ oid = SYSCTL_ADD_NODE(NULL, SYSCTL_CHILDREN(domainoid),
+ OID_AUTO, VM_DOMAIN(i)->vmd_name, CTLFLAG_RD, NULL, "");
+ SYSCTL_ADD_LONG(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
+ "nitems", CTLFLAG_RD, &zdom->uzd_nitems,
+ "number of items in this domain");
+ SYSCTL_ADD_LONG(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
+ "imax", CTLFLAG_RD, &zdom->uzd_imax,
+ "maximum item count in this period");
+ SYSCTL_ADD_LONG(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
+ "imin", CTLFLAG_RD, &zdom->uzd_imin,
+ "minimum item count in this period");
+ SYSCTL_ADD_LONG(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
+ "wss", CTLFLAG_RD, &zdom->uzd_wss,
+ "Working set size");
+ }
+
+ /*
+ * General statistics.
+ */
+ oid = SYSCTL_ADD_NODE(NULL, SYSCTL_CHILDREN(zone->uz_oid), OID_AUTO,
+ "stats", CTLFLAG_RD, NULL, "");
+ SYSCTL_ADD_PROC(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
+ "current", CTLFLAG_RD | CTLTYPE_INT | CTLFLAG_MPSAFE,
+ zone, 1, sysctl_handle_uma_zone_cur, "I",
+ "Current number of allocated items");
+ SYSCTL_ADD_PROC(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
+ "allocs", CTLFLAG_RD | CTLTYPE_U64 | CTLFLAG_MPSAFE,
+ zone, 0, sysctl_handle_uma_zone_allocs, "QU",
+ "Total allocation calls");
+ SYSCTL_ADD_PROC(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
+ "frees", CTLFLAG_RD | CTLTYPE_U64 | CTLFLAG_MPSAFE,
+ zone, 0, sysctl_handle_uma_zone_frees, "QU",
+ "Total free calls");
+ SYSCTL_ADD_COUNTER_U64(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
+ "fails", CTLFLAG_RD, &zone->uz_fails,
+ "Number of allocation failures");
+ SYSCTL_ADD_U64(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
+ "xdomain", CTLFLAG_RD, &zone->uz_xdomain, 0,
+ "Free calls from the wrong domain");
+}
+
+struct uma_zone_count {
+ const char *name;
+ int count;
+};
+
+static void
+zone_count(uma_zone_t zone, void *arg)
+{
+ struct uma_zone_count *cnt;
+
+ cnt = arg;
+ if (strcmp(zone->uz_name, cnt->name) == 0)
+ cnt->count++;
+}
+
/*
* Zone header ctor. This initializes all fields, locks, etc.
*
@@ -1835,6 +2001,7 @@ zone_alloc_counters(uma_zone_t zone)
static int
zone_ctor(void *mem, int size, void *udata, int flags)
{
+ struct uma_zone_count cnt;
struct uma_zctor_args *arg = udata;
uma_zone_t zone = mem;
uma_zone_t z;
@@ -1849,9 +2016,9 @@ zone_ctor(void *mem, int size, void *udata, int flags)
zone->uz_fini = NULL;
zone->uz_sleeps = 0;
zone->uz_xdomain = 0;
- zone->uz_count = 0;
- zone->uz_count_min = 0;
- zone->uz_count_max = BUCKET_MAX;
+ zone->uz_bucket_size = 0;
+ zone->uz_bucket_size_min = 0;
+ zone->uz_bucket_size_max = BUCKET_MAX;
zone->uz_flags = 0;
zone->uz_warning = NULL;
/* The domain structures follow the cpu structures. */
@@ -1859,13 +2026,11 @@ zone_ctor(void *mem, int size, void *udata, int flags)
zone->uz_bkt_max = ULONG_MAX;
timevalclear(&zone->uz_ratecheck);
- if (__predict_true(booted == BOOT_RUNNING))
- zone_alloc_counters(zone);
- else {
- zone->uz_allocs = EARLY_COUNTER;
- zone->uz_frees = EARLY_COUNTER;
- zone->uz_fails = EARLY_COUNTER;
- }
+ /* Count the number of duplicate names. */
+ cnt.name = arg->name;
+ cnt.count = 0;
+ zone_foreach(zone_count, &cnt);
+ zone->uz_namecnt = cnt.count;
for (i = 0; i < vm_ndomains; i++)
TAILQ_INIT(&zone->uz_domain[i].uzd_buckets);
@@ -1903,6 +2068,8 @@ zone_ctor(void *mem, int size, void *udata, int flags)
keg = arg->keg;
if (arg->flags & UMA_ZONE_SECONDARY) {
+ KASSERT((zone->uz_flags & UMA_ZONE_SECONDARY) == 0,
+ ("Secondary zone requested UMA_ZFLAG_INTERNAL"));
KASSERT(arg->keg != NULL, ("Secondary zone on zero'd keg"));
zone->uz_init = arg->uminit;
zone->uz_fini = arg->fini;
@@ -1939,35 +2106,36 @@ zone_ctor(void *mem, int size, void *udata, int flags)
return (error);
}
+ /* Inherit properties from the keg. */
zone->uz_keg = keg;
zone->uz_size = keg->uk_size;
zone->uz_flags |= (keg->uk_flags &
(UMA_ZONE_INHERIT | UMA_ZFLAG_INHERIT));
- /*
- * Some internal zones don't have room allocated for the per cpu
- * caches. If we're internal, bail out here.
- */
- if (keg->uk_flags & UMA_ZFLAG_INTERNAL) {
- KASSERT((zone->uz_flags & UMA_ZONE_SECONDARY) == 0,
- ("Secondary zone requested UMA_ZFLAG_INTERNAL"));
- return (0);
+out:
+ if (__predict_true(booted == BOOT_RUNNING)) {
+ zone_alloc_counters(zone, NULL);
+ zone_alloc_sysctl(zone, NULL);
+ } else {
+ zone->uz_allocs = EARLY_COUNTER;
+ zone->uz_frees = EARLY_COUNTER;
+ zone->uz_fails = EARLY_COUNTER;
}
-out:
KASSERT((arg->flags & (UMA_ZONE_MAXBUCKET | UMA_ZONE_NOBUCKET)) !=
(UMA_ZONE_MAXBUCKET | UMA_ZONE_NOBUCKET),
("Invalid zone flag combination"));
- if ((arg->flags & UMA_ZONE_MAXBUCKET) != 0) {
- zone->uz_count = BUCKET_MAX;
- } else if ((arg->flags & UMA_ZONE_MINBUCKET) != 0) {
- zone->uz_count = BUCKET_MIN;
- zone->uz_count_max = BUCKET_MIN;
- } else if ((arg->flags & UMA_ZONE_NOBUCKET) != 0)
- zone->uz_count = 0;
+ if (arg->flags & UMA_ZFLAG_INTERNAL)
+ zone->uz_bucket_size_max = zone->uz_bucket_size = 0;
+ if ((arg->flags & UMA_ZONE_MAXBUCKET) != 0)
+ zone->uz_bucket_size = BUCKET_MAX;
+ else if ((arg->flags & UMA_ZONE_MINBUCKET) != 0)
+ zone->uz_bucket_size_max = zone->uz_bucket_size = BUCKET_MIN;
+ else if ((arg->flags & UMA_ZONE_NOBUCKET) != 0)
+ zone->uz_bucket_size = 0;
else
- zone->uz_count = bucket_select(zone->uz_size);
- zone->uz_count_min = zone->uz_count;
+ zone->uz_bucket_size = bucket_select(zone->uz_size);
+ zone->uz_bucket_size_min = zone->uz_bucket_size;
return (0);
}
@@ -2013,6 +2181,8 @@ zone_dtor(void *arg, int size, void *udata)
zone = (uma_zone_t)arg;
+ sysctl_remove_oid(zone->uz_oid, 1, 1);
+
if (!(zone->uz_flags & UMA_ZFLAG_INTERNAL))
cache_drain(zone);
@@ -2039,6 +2209,7 @@ zone_dtor(void *arg, int size, void *udata)
counter_u64_free(zone->uz_allocs);
counter_u64_free(zone->uz_frees);
counter_u64_free(zone->uz_fails);
+ free(zone->uz_ctlname, M_UMA);
if (zone->uz_lockptr == &zone->uz_lock)
ZONE_LOCK_FINI(zone);
}
@@ -2054,7 +2225,7 @@ zone_dtor(void *arg, int size, void *udata)
* Nothing
*/
static void
-zone_foreach(void (*zfunc)(uma_zone_t))
+zone_foreach(void (*zfunc)(uma_zone_t, void *arg), void *arg)
{
uma_keg_t keg;
uma_zone_t zone;
@@ -2068,10 +2239,10 @@ zone_foreach(void (*zfunc)(uma_zone_t))
rw_rlock(&uma_rwlock);
LIST_FOREACH(keg, &uma_kegs, uk_link) {
LIST_FOREACH(zone, &keg->uk_zones, uz_link)
- zfunc(zone);
+ zfunc(zone, arg);
}
LIST_FOREACH(zone, &uma_cachezones, uz_link)
- zfunc(zone);
+ zfunc(zone, arg);
if (__predict_true(booted == BOOT_RUNNING))
rw_runlock(&uma_rwlock);
}
@@ -2249,7 +2420,8 @@ uma_startup3(void)
uma_dbg_cnt = counter_u64_alloc(M_WAITOK);
uma_skip_cnt = counter_u64_alloc(M_WAITOK);
#endif
- zone_foreach(zone_alloc_counters);
+ zone_foreach(zone_alloc_counters, NULL);
+ zone_foreach(zone_alloc_sysctl, NULL);
callout_init(&uma_callout, 1);
callout_reset(&uma_callout, UMA_TIMEOUT * hz, uma_timeout, NULL);
booted = BOOT_RUNNING;
@@ -2665,7 +2837,7 @@ cache_alloc(uma_zone_t zone, uma_cache_t cache, void *
critical_enter();
/* Short-circuit for zones without buckets and low memory. */
- if (zone->uz_count == 0 || bucketdisable)
+ if (zone->uz_bucket_size == 0 || bucketdisable)
return (false);
cpu = curcpu;
@@ -2702,8 +2874,8 @@ cache_alloc(uma_zone_t zone, uma_cache_t cache, void *
* We bump the uz count when the cache size is insufficient to
* handle the working set.
*/
- if (lockfail && zone->uz_count < zone->uz_count_max)
- zone->uz_count++;
+ if (lockfail && zone->uz_bucket_size < zone->uz_bucket_size_max)
+ zone->uz_bucket_size++;
/*
* Fill a bucket and attempt to use it as the alloc bucket.
@@ -3005,11 +3177,11 @@ zone_alloc_bucket(uma_zone_t zone, void *udata, int do
if (zone->uz_max_items > 0) {
if (zone->uz_items >= zone->uz_max_items)
return (false);
- maxbucket = MIN(zone->uz_count,
+ maxbucket = MIN(zone->uz_bucket_size,
zone->uz_max_items - zone->uz_items);
zone->uz_items += maxbucket;
} else
- maxbucket = zone->uz_count;
+ maxbucket = zone->uz_bucket_size;
ZONE_UNLOCK(zone);
/* Don't wait for buckets, preserve caller's NOVM setting. */
@@ -3285,8 +3457,8 @@ zone_free_bucket(uma_zone_t zone, uma_bucket_t bucket,
if (ZONE_TRYLOCK(zone) == 0) {
/* Record contention to size the buckets. */
ZONE_LOCK(zone);
- if (zone->uz_count < zone->uz_count_max)
- zone->uz_count++;
+ if (zone->uz_bucket_size < zone->uz_bucket_size_max)
+ zone->uz_bucket_size++;
}
CTR3(KTR_UMA,
@@ -3323,7 +3495,7 @@ cache_free(uma_zone_t zone, uma_cache_t cache, void *u
CRITICAL_ASSERT(curthread);
- if (zone->uz_count == 0 || bucketdisable)
+ if (zone->uz_bucket_size == 0 || bucketdisable)
return false;
cpu = curcpu;
@@ -3523,9 +3695,9 @@ uma_zone_set_max(uma_zone_t zone, int nitems)
ZONE_LOCK(zone);
ubz = bucket_zone_max(zone, nitems);
count = ubz != NULL ? ubz->ubz_entries : 0;
- zone->uz_count_max = zone->uz_count = count;
- if (zone->uz_count_min > zone->uz_count_max)
- zone->uz_count_min = zone->uz_count_max;
+ zone->uz_bucket_size_max = zone->uz_bucket_size = count;
+ if (zone->uz_bucket_size_min > zone->uz_bucket_size_max)
+ zone->uz_bucket_size_min = zone->uz_bucket_size_max;
zone->uz_max_items = nitems;
ZONE_UNLOCK(zone);
@@ -3549,12 +3721,12 @@ uma_zone_set_maxcache(uma_zone_t zone, int nitems)
bpcpu++;
#endif
nitems -= ubz->ubz_entries * bpcpu * mp_ncpus;
- zone->uz_count_max = ubz->ubz_entries;
+ zone->uz_bucket_size_max = ubz->ubz_entries;
} else {
- zone->uz_count_max = zone->uz_count = 0;
+ zone->uz_bucket_size_max = zone->uz_bucket_size = 0;
}
- if (zone->uz_count_min > zone->uz_count_max)
- zone->uz_count_min = zone->uz_count_max;
+ if (zone->uz_bucket_size_min > zone->uz_bucket_size_max)
+ zone->uz_bucket_size_min = zone->uz_bucket_size_max;
zone->uz_bkt_max = nitems;
ZONE_UNLOCK(zone);
}
@@ -3602,20 +3774,71 @@ uma_zone_get_cur(uma_zone_t zone)
ZONE_LOCK(zone);
nitems = counter_u64_fetch(zone->uz_allocs) -
counter_u64_fetch(zone->uz_frees);
- CPU_FOREACH(i) {
- /*
- * See the comment in uma_vm_zone_stats() regarding the
- * safety of accessing the per-cpu caches. With the zone lock
- * held, it is safe, but can potentially result in stale data.
- */
- nitems += zone->uz_cpu[i].uc_allocs -
- zone->uz_cpu[i].uc_frees;
+ if ((zone->uz_flags & UMA_ZFLAG_INTERNAL) == 0) {
+ CPU_FOREACH(i) {
+ /*
+ * See the comment in uma_vm_zone_stats() regarding
+ * the safety of accessing the per-cpu caches. With
+ * the zone lock held, it is safe, but can potentially
+ * result in stale data.
+ */
+ nitems += zone->uz_cpu[i].uc_allocs -
+ zone->uz_cpu[i].uc_frees;
+ }
}
ZONE_UNLOCK(zone);
return (nitems < 0 ? 0 : nitems);
}
+static uint64_t
+uma_zone_get_allocs(uma_zone_t zone)
+{
+ uint64_t nitems;
+ u_int i;
+
+ ZONE_LOCK(zone);
+ nitems = counter_u64_fetch(zone->uz_allocs);
+ if ((zone->uz_flags & UMA_ZFLAG_INTERNAL) == 0) {
+ CPU_FOREACH(i) {
+ /*
+ * See the comment in uma_vm_zone_stats() regarding
+ * the safety of accessing the per-cpu caches. With
+ * the zone lock held, it is safe, but can potentially
+ * result in stale data.
+ */
+ nitems += zone->uz_cpu[i].uc_allocs;
+ }
+ }
+ ZONE_UNLOCK(zone);
+
+ return (nitems);
+}
+
+static uint64_t
+uma_zone_get_frees(uma_zone_t zone)
+{
+ uint64_t nitems;
+ u_int i;
+
+ ZONE_LOCK(zone);
+ nitems = counter_u64_fetch(zone->uz_frees);
+ if ((zone->uz_flags & UMA_ZFLAG_INTERNAL) == 0) {
+ CPU_FOREACH(i) {
+ /*
+ * See the comment in uma_vm_zone_stats() regarding
+ * the safety of accessing the per-cpu caches. With
+ * the zone lock held, it is safe, but can potentially
+ * result in stale data.
+ */
+ nitems += zone->uz_cpu[i].uc_frees;
+ }
+ }
+ ZONE_UNLOCK(zone);
+
+ return (nitems);
+}
+
/* See uma.h */
void
uma_zone_set_init(uma_zone_t zone, uma_init uminit)
@@ -3800,14 +4023,14 @@ uma_reclaim(int req)
switch (req) {
case UMA_RECLAIM_TRIM:
- zone_foreach(zone_trim);
+ zone_foreach(zone_trim, NULL);
break;
case UMA_RECLAIM_DRAIN:
case UMA_RECLAIM_DRAIN_CPU:
- zone_foreach(zone_drain);
+ zone_foreach(zone_drain, NULL);
if (req == UMA_RECLAIM_DRAIN_CPU) {
pcpu_cache_drain_safe(NULL);
- zone_foreach(zone_drain);
+ zone_foreach(zone_drain, NULL);
}
break;
default:
@@ -3819,7 +4042,7 @@ uma_reclaim(int req)
* we visit again so that we can free pages that are empty once other
* zones are drained. We have to do the same for buckets.
*/
- zone_drain(slabzone);
+ zone_drain(slabzone, NULL);
bucket_zone_drain();
sx_xunlock(&uma_reclaim_lock);
}
@@ -3859,14 +4082,14 @@ uma_zone_reclaim(uma_zone_t zone, int req)
switch (req) {
case UMA_RECLAIM_TRIM:
- zone_trim(zone);
+ zone_trim(zone, NULL);
break;
case UMA_RECLAIM_DRAIN:
- zone_drain(zone);
+ zone_drain(zone, NULL);
break;
case UMA_RECLAIM_DRAIN_CPU:
pcpu_cache_drain_safe(zone);
- zone_drain(zone);
+ zone_drain(zone, NULL);
break;
default:
panic("unhandled reclamation request %d", req);
@@ -3977,76 +4200,6 @@ uma_avail(void)
return (uma_kmem_limit - uma_size());
}
-void
-uma_print_stats(void)
-{
- zone_foreach(uma_print_zone);
-}
-
-static void
-slab_print(uma_slab_t slab)
-{
- printf("slab: keg %p, data %p, freecount %d\n",
- slab->us_keg, slab->us_data, slab->us_freecount);
-}
-
-static void
-cache_print(uma_cache_t cache)
-{
- printf("alloc: %p(%d), free: %p(%d), cross: %p(%d)j\n",
- cache->uc_allocbucket,
- cache->uc_allocbucket?cache->uc_allocbucket->ub_cnt:0,
- cache->uc_freebucket,
- cache->uc_freebucket?cache->uc_freebucket->ub_cnt:0,
- cache->uc_crossbucket,
- cache->uc_crossbucket?cache->uc_crossbucket->ub_cnt:0);
-}
-
-static void
-uma_print_keg(uma_keg_t keg)
-{
- uma_domain_t dom;
- uma_slab_t slab;
- int i;
-
- printf("keg: %s(%p) size %d(%d) flags %#x ipers %d ppera %d "
- "out %d free %d\n",
- keg->uk_name, keg, keg->uk_size, keg->uk_rsize, keg->uk_flags,
- keg->uk_ipers, keg->uk_ppera,
- (keg->uk_pages / keg->uk_ppera) * keg->uk_ipers - keg->uk_free,
- keg->uk_free);
- for (i = 0; i < vm_ndomains; i++) {
- dom = &keg->uk_domain[i];
- printf("Part slabs:\n");
- LIST_FOREACH(slab, &dom->ud_part_slab, us_link)
- slab_print(slab);
- printf("Free slabs:\n");
- LIST_FOREACH(slab, &dom->ud_free_slab, us_link)
- slab_print(slab);
- printf("Full slabs:\n");
- LIST_FOREACH(slab, &dom->ud_full_slab, us_link)
- slab_print(slab);
- }
-}
-
-void
-uma_print_zone(uma_zone_t zone)
-{
- uma_cache_t cache;
- int i;
-
- printf("zone: %s(%p) size %d maxitems %ju flags %#x\n",
- zone->uz_name, zone, zone->uz_size, (uintmax_t)zone->uz_max_items,
- zone->uz_flags);
- if (zone->uz_lockptr != &zone->uz_lock)
- uma_print_keg(zone->uz_keg);
- CPU_FOREACH(i) {
- cache = &zone->uz_cpu[i];
- printf("CPU %d Cache:\n", i);
- cache_print(cache);
- }
-}
-
#ifdef DDB
/*
* Generate statistics across both the zone and its per-cpu cache's. Return
@@ -4274,13 +4427,41 @@ sysctl_handle_uma_zone_max(SYSCTL_HANDLER_ARGS)
int
sysctl_handle_uma_zone_cur(SYSCTL_HANDLER_ARGS)
{
- uma_zone_t zone = *(uma_zone_t *)arg1;
+ uma_zone_t zone;
int cur;
+ /*
+ * Some callers want to add sysctls for global zones that
+ * may not yet exist so they pass a pointer to a pointer.
+ */
+ if (arg2 == 0)
+ zone = *(uma_zone_t *)arg1;
+ else
+ zone = arg1;
cur = uma_zone_get_cur(zone);
return (sysctl_handle_int(oidp, &cur, 0, req));
}
+static int
+sysctl_handle_uma_zone_allocs(SYSCTL_HANDLER_ARGS)
+{
+ uma_zone_t zone = arg1;
+ uint64_t cur;
+
+ cur = uma_zone_get_allocs(zone);
+ return (sysctl_handle_64(oidp, &cur, 0, req));
+}
+
+static int
+sysctl_handle_uma_zone_frees(SYSCTL_HANDLER_ARGS)
+{
+ uma_zone_t zone = arg1;
+ uint64_t cur;
+
+ cur = uma_zone_get_frees(zone);
+ return (sysctl_handle_64(oidp, &cur, 0, req));
+}
+
#ifdef INVARIANTS
static uma_slab_t
uma_dbg_getslab(uma_zone_t zone, void *item)
@@ -4500,7 +4681,8 @@ DB_SHOW_COMMAND(uma, db_show_uma)
db_printf(fmt_entry, cur_zone->uz_name,
(uintmax_t)cur_keg->uk_size, (intmax_t)used, cachefree,
(uintmax_t)allocs, (uintmax_t)sleeps,
- (unsigned)cur_zone->uz_count, (intmax_t)size, xdomain);
+ (unsigned)cur_zone->uz_bucket_size, (intmax_t)size,
+ xdomain);
if (db_pager_quit)
return;
@@ -4525,7 +4707,7 @@ DB_SHOW_COMMAND(umacache, db_show_umacache)
db_printf("%18s %8ju %8jd %8ld %12ju %8u\n",
z->uz_name, (uintmax_t)z->uz_size,
(intmax_t)(allocs - frees), cachefree,
- (uintmax_t)allocs, z->uz_count);
+ (uintmax_t)allocs, z->uz_bucket_size);
if (db_pager_quit)
return;
}
Modified: head/sys/vm/uma_int.h
==============================================================================
--- head/sys/vm/uma_int.h Wed Nov 27 23:19:06 2019 (r355148)
+++ head/sys/vm/uma_int.h Thu Nov 28 00:19:09 2019 (r355149)
@@ -338,8 +338,8 @@ struct uma_zone {
uint64_t uz_items; /* Total items count */
uint64_t uz_max_items; /* Maximum number of items to alloc */
uint32_t uz_sleepers; /* Number of sleepers on memory */
- uint16_t uz_count; /* Amount of items in full bucket */
- uint16_t uz_count_max; /* Maximum amount of items there */
+ uint16_t uz_bucket_size; /* Number of items in full bucket */
+ uint16_t uz_bucket_size_max; /* Maximum number of bucket items */
/* Offset 64, used in bucket replenish. */
uma_import uz_import; /* Import new memory to cache. */
@@ -364,14 +364,17 @@ struct uma_zone {
const char *uz_warning; /* Warning to print on failure */
struct timeval uz_ratecheck; /* Warnings rate-limiting */
struct task uz_maxaction; /* Task to run when at limit */
- uint16_t uz_count_min; /* Minimal amount of items in bucket */
+ uint16_t uz_bucket_size_min; /* Min number of items in bucket */
- /* Offset 256, stats. */
+ /* Offset 256+, stats and misc. */
counter_u64_t uz_allocs; /* Total number of allocations */
counter_u64_t uz_frees; /* Total number of frees */
counter_u64_t uz_fails; /* Total number of alloc failures */
uint64_t uz_sleeps; /* Total number of alloc sleeps */
uint64_t uz_xdomain; /* Total number of cross-domain frees */
+ char *uz_ctlname; /* sysctl safe name string. */
+ struct sysctl_oid *uz_oid; /* sysctl oid pointer. */
+ int uz_namecnt; /* duplicate name count. */
/*
* This HAS to be the last item because we adjust the zone size
More information about the svn-src-head
mailing list