svn commit: r356349 - in head: lib/libmemstat sys/vm
Jeff Roberson
jeff at FreeBSD.org
Sat Jan 4 03:30:09 UTC 2020
Author: jeff
Date: Sat Jan 4 03:30:08 2020
New Revision: 356349
URL: https://svnweb.freebsd.org/changeset/base/356349
Log:
Use per-domain keg locks. This provides both a lock and separate space
accounting for each NUMA domain. Independent keg domain locks are important
with cross-domain frees. Hashed zones are non-numa and use a single keg
lock to protect the hash table.
Reviewed by: markj, rlibby
Differential Revision: https://reviews.freebsd.org/D22829
Modified:
head/lib/libmemstat/memstat_uma.c
head/sys/vm/uma_core.c
head/sys/vm/uma_int.h
Modified: head/lib/libmemstat/memstat_uma.c
==============================================================================
--- head/lib/libmemstat/memstat_uma.c Sat Jan 4 03:15:34 2020 (r356348)
+++ head/lib/libmemstat/memstat_uma.c Sat Jan 4 03:30:08 2020 (r356349)
@@ -311,10 +311,12 @@ memstat_kvm_uma(struct memory_type_list *list, void *k
LIST_HEAD(, uma_keg) uma_kegs;
struct memory_type *mtp;
struct uma_zone_domain uzd;
+ struct uma_domain ukd;
struct uma_bucket *ubp, ub;
struct uma_cache *ucp, *ucp_array;
struct uma_zone *uzp, uz;
struct uma_keg *kzp, kz;
+ uint64_t kegfree;
int hint_dontsearch, i, mp_maxid, ndomains, ret;
char name[MEMTYPE_MAXNAME];
cpuset_t all_cpus;
@@ -454,18 +456,29 @@ skip_percpu:
for (i = 0; i < ndomains; i++) {
ret = kread(kvm, &uz.uz_domain[i], &uzd,
sizeof(uzd), 0);
+ if (ret != 0)
+ continue;
for (ubp =
TAILQ_FIRST(&uzd.uzd_buckets);
ubp != NULL;
ubp = TAILQ_NEXT(&ub, ub_link)) {
ret = kread(kvm, ubp, &ub,
sizeof(ub), 0);
+ if (ret != 0)
+ continue;
mtp->mt_zonefree += ub.ub_cnt;
}
}
if (!((kz.uk_flags & UMA_ZONE_SECONDARY) &&
LIST_FIRST(&kz.uk_zones) != uzp)) {
- mtp->mt_kegfree = kz.uk_free;
+ kegfree = 0;
+ for (i = 0; i < ndomains; i++) {
+ ret = kread(kvm, &kzp->uk_domain[i],
+ &ukd, sizeof(ukd), 0);
+ if (ret != 0)
+ kegfree += ukd.ud_free;
+ }
+ mtp->mt_kegfree = kegfree;
mtp->mt_free += mtp->mt_kegfree;
}
mtp->mt_free += mtp->mt_zonefree;
Modified: head/sys/vm/uma_core.c
==============================================================================
--- head/sys/vm/uma_core.c Sat Jan 4 03:15:34 2020 (r356348)
+++ head/sys/vm/uma_core.c Sat Jan 4 03:30:08 2020 (r356349)
@@ -740,23 +740,28 @@ static void
zone_timeout(uma_zone_t zone, void *unused)
{
uma_keg_t keg;
- u_int slabs;
+ u_int slabs, pages;
if ((zone->uz_flags & UMA_ZONE_HASH) == 0)
goto update_wss;
keg = zone->uz_keg;
- KEG_LOCK(keg);
+
/*
+ * Hash zones are non-numa by definition so the first domain
+ * is the only one present.
+ */
+ KEG_LOCK(keg, 0);
+ pages = keg->uk_domain[0].ud_pages;
+
+ /*
* Expand the keg hash table.
*
* This is done if the number of slabs is larger than the hash size.
* What I'm trying to do here is completely reduce collisions. This
* may be a little aggressive. Should I allow for two collisions max?
*/
- if (keg->uk_flags & UMA_ZONE_HASH &&
- (slabs = keg->uk_pages / keg->uk_ppera) >
- keg->uk_hash.uh_hashsize) {
+ if ((slabs = pages / keg->uk_ppera) > keg->uk_hash.uh_hashsize) {
struct uma_hash newhash;
struct uma_hash oldhash;
int ret;
@@ -767,9 +772,9 @@ zone_timeout(uma_zone_t zone, void *unused)
* I have to do everything in stages and check for
* races.
*/
- KEG_UNLOCK(keg);
+ KEG_UNLOCK(keg, 0);
ret = hash_alloc(&newhash, 1 << fls(slabs));
- KEG_LOCK(keg);
+ KEG_LOCK(keg, 0);
if (ret) {
if (hash_expand(&keg->uk_hash, &newhash)) {
oldhash = keg->uk_hash;
@@ -777,12 +782,12 @@ zone_timeout(uma_zone_t zone, void *unused)
} else
oldhash = newhash;
- KEG_UNLOCK(keg);
+ KEG_UNLOCK(keg, 0);
hash_free(&oldhash);
- return;
+ goto update_wss;
}
}
- KEG_UNLOCK(keg);
+ KEG_UNLOCK(keg, 0);
update_wss:
ZONE_LOCK(zone);
@@ -1166,7 +1171,7 @@ keg_drain(uma_keg_t keg)
struct slabhead freeslabs = { 0 };
uma_domain_t dom;
uma_slab_t slab, tmp;
- int i;
+ int i, n;
/*
* We don't want to take pages from statically allocated kegs at this
@@ -1175,33 +1180,27 @@ keg_drain(uma_keg_t keg)
if (keg->uk_flags & UMA_ZONE_NOFREE || keg->uk_freef == NULL)
return;
- CTR3(KTR_UMA, "keg_drain %s(%p) free items: %u",
- keg->uk_name, keg, keg->uk_free);
- KEG_LOCK(keg);
- if (keg->uk_free == 0)
- goto finished;
-
for (i = 0; i < vm_ndomains; i++) {
+ CTR4(KTR_UMA, "keg_drain %s(%p) domain %d free items: %u",
+ keg->uk_name, keg, i, dom->ud_free);
+ n = 0;
dom = &keg->uk_domain[i];
+ KEG_LOCK(keg, i);
LIST_FOREACH_SAFE(slab, &dom->ud_free_slab, us_link, tmp) {
/* We have nowhere to free these to. */
if (slab->us_flags & UMA_SLAB_BOOT)
continue;
-
- LIST_REMOVE(slab, us_link);
- keg->uk_pages -= keg->uk_ppera;
- keg->uk_free -= keg->uk_ipers;
-
if (keg->uk_flags & UMA_ZONE_HASH)
UMA_HASH_REMOVE(&keg->uk_hash, slab);
-
+ n++;
+ LIST_REMOVE(slab, us_link);
LIST_INSERT_HEAD(&freeslabs, slab, us_link);
}
+ dom->ud_pages -= n * keg->uk_ppera;
+ dom->ud_free -= n * keg->uk_ipers;
+ KEG_UNLOCK(keg, i);
}
-finished:
- KEG_UNLOCK(keg);
-
while ((slab = LIST_FIRST(&freeslabs)) != NULL) {
LIST_REMOVE(slab, us_link);
keg_free_slab(keg, slab, keg->uk_ipers);
@@ -1257,9 +1256,9 @@ zone_trim(uma_zone_t zone, void *unused)
}
/*
- * Allocate a new slab for a keg. This does not insert the slab onto a list.
- * The keg should be locked on entry and will be dropped and reacquired on
- * return.
+ * Allocate a new slab for a keg and inserts it into the partial slab list.
+ * The keg should be unlocked on entry. If the allocation succeeds it will
+ * be locked on return.
*
* Arguments:
* flags Wait flags for the item initialization routine
@@ -1273,6 +1272,7 @@ static uma_slab_t
keg_alloc_slab(uma_keg_t keg, uma_zone_t zone, int domain, int flags,
int aflags)
{
+ uma_domain_t dom;
uma_alloc allocf;
uma_slab_t slab;
unsigned long size;
@@ -1282,10 +1282,8 @@ keg_alloc_slab(uma_keg_t keg, uma_zone_t zone, int dom
KASSERT(domain >= 0 && domain < vm_ndomains,
("keg_alloc_slab: domain %d out of range", domain));
- KEG_LOCK_ASSERT(keg);
- allocf = keg->uk_allocf;
- KEG_UNLOCK(keg);
+ allocf = keg->uk_allocf;
slab = NULL;
mem = NULL;
if (keg->uk_flags & UMA_ZONE_OFFPAGE) {
@@ -1319,6 +1317,10 @@ keg_alloc_slab(uma_keg_t keg, uma_zone_t zone, int dom
}
uma_total_inc(size);
+ /* For HASH zones all pages go to the same uma_domain. */
+ if ((keg->uk_flags & UMA_ZONE_HASH) != 0)
+ domain = 0;
+
/* Point the slab into the allocated memory */
if (!(keg->uk_flags & UMA_ZONE_OFFPAGE))
slab = (uma_slab_t )(mem + keg->uk_pgoff);
@@ -1333,6 +1335,7 @@ keg_alloc_slab(uma_keg_t keg, uma_zone_t zone, int dom
slab->us_freecount = keg->uk_ipers;
slab->us_flags = sflags;
slab->us_domain = domain;
+
BIT_FILL(keg->uk_ipers, &slab->us_free);
#ifdef INVARIANTS
BIT_ZERO(keg->uk_ipers, slab_dbg_bits(slab, keg));
@@ -1348,7 +1351,7 @@ keg_alloc_slab(uma_keg_t keg, uma_zone_t zone, int dom
goto fail;
}
}
- KEG_LOCK(keg);
+ KEG_LOCK(keg, domain);
CTR3(KTR_UMA, "keg_alloc_slab: allocated slab %p for %s(%p)",
slab, keg->uk_name, keg);
@@ -1356,13 +1359,19 @@ keg_alloc_slab(uma_keg_t keg, uma_zone_t zone, int dom
if (keg->uk_flags & UMA_ZONE_HASH)
UMA_HASH_INSERT(&keg->uk_hash, slab, mem);
- keg->uk_pages += keg->uk_ppera;
- keg->uk_free += keg->uk_ipers;
+ /*
+ * If we got a slab here it's safe to mark it partially used
+ * and return. We assume that the caller is going to remove
+ * at least one item.
+ */
+ dom = &keg->uk_domain[domain];
+ LIST_INSERT_HEAD(&dom->ud_part_slab, slab, us_link);
+ dom->ud_pages += keg->uk_ppera;
+ dom->ud_free += keg->uk_ipers;
return (slab);
fail:
- KEG_LOCK(keg);
return (NULL);
}
@@ -1875,15 +1884,14 @@ keg_ctor(void *mem, int size, void *udata, int flags)
struct uma_kctor_args *arg = udata;
uma_keg_t keg = mem;
uma_zone_t zone;
+ int i;
bzero(keg, size);
keg->uk_size = arg->size;
keg->uk_init = arg->uminit;
keg->uk_fini = arg->fini;
keg->uk_align = arg->align;
- keg->uk_free = 0;
keg->uk_reserve = 0;
- keg->uk_pages = 0;
keg->uk_flags = arg->flags;
keg->uk_slabzone = NULL;
@@ -1926,6 +1934,15 @@ keg_ctor(void *mem, int size, void *udata, int flags)
keg_small_init(keg);
}
+ /*
+ * Sets all kegs with memory that comes from the page array to a
+ * first-touch domain policy.
+ */
+#ifdef UMA_FIRSTTOUCH
+ if ((keg->uk_flags & UMA_ZONE_HASH) == 0)
+ keg->uk_flags |= UMA_ZONE_NUMA;
+#endif
+
if (keg->uk_flags & UMA_ZONE_OFFPAGE)
keg->uk_slabzone = slabzone;
@@ -1954,9 +1971,10 @@ keg_ctor(void *mem, int size, void *udata, int flags)
keg->uk_freef = page_free;
/*
- * Initialize keg's lock
+ * Initialize keg's locks.
*/
- KEG_LOCK_INIT(keg, (arg->flags & UMA_ZONE_MTXCLASS));
+ for (i = 0; i < vm_ndomains; i++)
+ KEG_LOCK_INIT(keg, i, (arg->flags & UMA_ZONE_MTXCLASS));
/*
* If we're putting the slab header in the actual page we need to
@@ -1983,10 +2001,7 @@ keg_ctor(void *mem, int size, void *udata, int flags)
if (keg->uk_flags & UMA_ZONE_HASH)
hash_alloc(&keg->uk_hash, 0);
- CTR5(KTR_UMA, "keg_ctor %p zone %s(%p) out %d free %d\n",
- keg, zone->uz_name, zone,
- (keg->uk_pages / keg->uk_ppera) * keg->uk_ipers - keg->uk_free,
- keg->uk_free);
+ CTR3(KTR_UMA, "keg_ctor %p zone %s(%p)\n", keg, zone->uz_name, zone);
LIST_INSERT_HEAD(&keg->uk_zones, zone, uz_link);
@@ -2009,6 +2024,7 @@ static void
zone_alloc_sysctl(uma_zone_t zone, void *unused)
{
uma_zone_domain_t zdom;
+ uma_domain_t dom;
uma_keg_t keg;
struct sysctl_oid *oid, *domainoid;
int domains, i, cnt;
@@ -2056,6 +2072,10 @@ zone_alloc_sysctl(uma_zone_t zone, void *unused)
/*
* keg if present.
*/
+ if ((zone->uz_flags & UMA_ZONE_HASH) == 0)
+ domains = vm_ndomains;
+ else
+ domains = 1;
oid = SYSCTL_ADD_NODE(NULL, SYSCTL_CHILDREN(zone->uz_oid), OID_AUTO,
"keg", CTLFLAG_RD, NULL, "");
keg = zone->uz_keg;
@@ -2074,16 +2094,24 @@ zone_alloc_sysctl(uma_zone_t zone, void *unused)
SYSCTL_ADD_U32(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
"align", CTLFLAG_RD, &keg->uk_align, 0,
"item alignment mask");
- SYSCTL_ADD_U32(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
- "pages", CTLFLAG_RD, &keg->uk_pages, 0,
- "Total pages currently allocated from VM");
- SYSCTL_ADD_U32(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
- "free", CTLFLAG_RD, &keg->uk_free, 0,
- "items free in the slab layer");
SYSCTL_ADD_PROC(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
"efficiency", CTLFLAG_RD | CTLTYPE_INT | CTLFLAG_MPSAFE,
keg, 0, sysctl_handle_uma_slab_efficiency, "I",
"Slab utilization (100 - internal fragmentation %)");
+ domainoid = SYSCTL_ADD_NODE(NULL, SYSCTL_CHILDREN(oid),
+ OID_AUTO, "domain", CTLFLAG_RD, NULL, "");
+ for (i = 0; i < domains; i++) {
+ dom = &keg->uk_domain[i];
+ oid = SYSCTL_ADD_NODE(NULL, SYSCTL_CHILDREN(domainoid),
+ OID_AUTO, VM_DOMAIN(i)->vmd_name, CTLFLAG_RD,
+ NULL, "");
+ SYSCTL_ADD_U32(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
+ "pages", CTLFLAG_RD, &dom->ud_pages, 0,
+ "Total pages currently allocated from VM");
+ SYSCTL_ADD_U32(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
+ "free", CTLFLAG_RD, &dom->ud_free, 0,
+ "items free in the slab layer");
+ }
} else
SYSCTL_ADD_CONST_STRING(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
"name", CTLFLAG_RD, nokeg, "Keg name");
@@ -2114,14 +2142,12 @@ zone_alloc_sysctl(uma_zone_t zone, void *unused)
"Number of items in the bucket cache");
/*
- * Per-domain information.
+ * Per-domain zone information.
*/
- if ((zone->uz_flags & UMA_ZONE_NUMA) != 0)
- domains = vm_ndomains;
- else
- domains = 1;
domainoid = SYSCTL_ADD_NODE(NULL, SYSCTL_CHILDREN(zone->uz_oid),
OID_AUTO, "domain", CTLFLAG_RD, NULL, "");
+ if ((zone->uz_flags & UMA_ZONE_NUMA) == 0)
+ domains = 1;
for (i = 0; i < domains; i++) {
zdom = &zone->uz_domain[i];
oid = SYSCTL_ADD_NODE(NULL, SYSCTL_CHILDREN(domainoid),
@@ -2359,20 +2385,23 @@ static void
keg_dtor(void *arg, int size, void *udata)
{
uma_keg_t keg;
+ uint32_t free, pages;
+ int i;
keg = (uma_keg_t)arg;
- KEG_LOCK(keg);
- if (keg->uk_free != 0) {
- printf("Freed UMA keg (%s) was not empty (%d items). "
- " Lost %d pages of memory.\n",
- keg->uk_name ? keg->uk_name : "",
- keg->uk_free, keg->uk_pages);
+ free = pages = 0;
+ for (i = 0; i < vm_ndomains; i++) {
+ free += keg->uk_domain[i].ud_free;
+ pages += keg->uk_domain[i].ud_pages;
+ KEG_LOCK_FINI(keg, i);
}
- KEG_UNLOCK(keg);
+ if (free != 0)
+ printf("Freed UMA keg (%s) was not empty (%u items). "
+ " Lost %u pages of memory.\n",
+ keg->uk_name ? keg->uk_name : "",
+ free, pages);
hash_free(&keg->uk_hash);
-
- KEG_LOCK_FINI(keg);
}
/*
@@ -2659,11 +2688,6 @@ uma_zcreate(const char *name, size_t size, uma_ctor ct
KASSERT(powerof2(align + 1), ("invalid zone alignment %d for \"%s\"",
align, name));
- /* Sets all zones to a first-touch domain policy. */
-#ifdef UMA_FIRSTTOUCH
- flags |= UMA_ZONE_NUMA;
-#endif
-
/* This stuff is essential for the zone ctor */
memset(&args, 0, sizeof(args));
args.name = name;
@@ -3135,7 +3159,7 @@ keg_first_slab(uma_keg_t keg, int domain, bool rr)
KASSERT(domain >= 0 && domain < vm_ndomains,
("keg_first_slab: domain %d out of range", domain));
- KEG_LOCK_ASSERT(keg);
+ KEG_LOCK_ASSERT(keg, domain);
slab = NULL;
start = domain;
@@ -3156,31 +3180,39 @@ keg_first_slab(uma_keg_t keg, int domain, bool rr)
return (NULL);
}
+/*
+ * Fetch an existing slab from a free or partial list. Returns with the
+ * keg domain lock held if a slab was found or unlocked if not.
+ */
static uma_slab_t
keg_fetch_free_slab(uma_keg_t keg, int domain, bool rr, int flags)
{
+ uma_slab_t slab;
uint32_t reserve;
- KEG_LOCK_ASSERT(keg);
+ /* HASH has a single free list. */
+ if ((keg->uk_flags & UMA_ZONE_HASH) != 0)
+ domain = 0;
+ KEG_LOCK(keg, domain);
reserve = (flags & M_USE_RESERVE) != 0 ? 0 : keg->uk_reserve;
- if (keg->uk_free <= reserve)
+ if (keg->uk_domain[domain].ud_free <= reserve ||
+ (slab = keg_first_slab(keg, domain, rr)) == NULL) {
+ KEG_UNLOCK(keg, domain);
return (NULL);
- return (keg_first_slab(keg, domain, rr));
+ }
+ return (slab);
}
static uma_slab_t
keg_fetch_slab(uma_keg_t keg, uma_zone_t zone, int rdomain, const int flags)
{
struct vm_domainset_iter di;
- uma_domain_t dom;
uma_slab_t slab;
int aflags, domain;
bool rr;
restart:
- KEG_LOCK_ASSERT(keg);
-
/*
* Use the keg's policy if upper layers haven't already specified a
* domain (as happens with first-touch zones).
@@ -3211,23 +3243,13 @@ restart:
break;
slab = keg_alloc_slab(keg, zone, domain, flags, aflags);
- /*
- * If we got a slab here it's safe to mark it partially used
- * and return. We assume that the caller is going to remove
- * at least one item.
- */
- if (slab) {
- dom = &keg->uk_domain[slab->us_domain];
- LIST_INSERT_HEAD(&dom->ud_part_slab, slab, us_link);
+ if (slab != NULL)
return (slab);
- }
if (!rr && (flags & M_WAITOK) == 0)
break;
if (rr && vm_domainset_iter_policy(&di, &domain) != 0) {
if ((flags & M_WAITOK) != 0) {
- KEG_UNLOCK(keg);
vm_wait_doms(&keg->uk_dr.dr_policy->ds_mask);
- KEG_LOCK(keg);
goto restart;
}
break;
@@ -3239,9 +3261,9 @@ restart:
* could have while we were unlocked. Check again before we
* fail.
*/
- if ((slab = keg_fetch_free_slab(keg, domain, rr, flags)) != NULL) {
+ if ((slab = keg_fetch_free_slab(keg, domain, rr, flags)) != NULL)
return (slab);
- }
+
return (NULL);
}
@@ -3252,18 +3274,18 @@ slab_alloc_item(uma_keg_t keg, uma_slab_t slab)
void *item;
uint8_t freei;
- KEG_LOCK_ASSERT(keg);
+ KEG_LOCK_ASSERT(keg, slab->us_domain);
+ dom = &keg->uk_domain[slab->us_domain];
freei = BIT_FFS(keg->uk_ipers, &slab->us_free) - 1;
BIT_CLR(keg->uk_ipers, freei, &slab->us_free);
item = slab_item(slab, keg, freei);
slab->us_freecount--;
- keg->uk_free--;
+ dom->ud_free--;
/* Move this slab to the full list */
if (slab->us_freecount == 0) {
LIST_REMOVE(slab, us_link);
- dom = &keg->uk_domain[slab->us_domain];
LIST_INSERT_HEAD(&dom->ud_full_slab, slab, us_link);
}
@@ -3273,6 +3295,7 @@ slab_alloc_item(uma_keg_t keg, uma_slab_t slab)
static int
zone_import(void *arg, void **bucket, int max, int domain, int flags)
{
+ uma_domain_t dom;
uma_zone_t zone;
uma_slab_t slab;
uma_keg_t keg;
@@ -3284,7 +3307,6 @@ zone_import(void *arg, void **bucket, int max, int dom
zone = arg;
slab = NULL;
keg = zone->uz_keg;
- KEG_LOCK(keg);
/* Try to keep the buckets totally full */
for (i = 0; i < max; ) {
if ((slab = keg_fetch_slab(keg, zone, domain, flags)) == NULL)
@@ -3292,9 +3314,10 @@ zone_import(void *arg, void **bucket, int max, int dom
#ifdef NUMA
stripe = howmany(max, vm_ndomains);
#endif
+ dom = &keg->uk_domain[slab->us_domain];
while (slab->us_freecount && i < max) {
bucket[i++] = slab_alloc_item(keg, slab);
- if (keg->uk_free <= keg->uk_reserve)
+ if (dom->ud_free <= keg->uk_reserve)
break;
#ifdef NUMA
/*
@@ -3310,11 +3333,11 @@ zone_import(void *arg, void **bucket, int max, int dom
break;
#endif
}
+ KEG_UNLOCK(keg, slab->us_domain);
/* Don't block if we allocated any successfully. */
flags &= ~M_WAITOK;
flags |= M_NOWAIT;
}
- KEG_UNLOCK(keg);
return i;
}
@@ -3866,11 +3889,10 @@ slab_free_item(uma_zone_t zone, uma_slab_t slab, void
uint8_t freei;
keg = zone->uz_keg;
- KEG_LOCK_ASSERT(keg);
+ KEG_LOCK_ASSERT(keg, slab->us_domain);
- dom = &keg->uk_domain[slab->us_domain];
-
/* Do we need to remove from any lists? */
+ dom = &keg->uk_domain[slab->us_domain];
if (slab->us_freecount+1 == keg->uk_ipers) {
LIST_REMOVE(slab, us_link);
LIST_INSERT_HEAD(&dom->ud_free_slab, slab, us_link);
@@ -3885,37 +3907,45 @@ slab_free_item(uma_zone_t zone, uma_slab_t slab, void
slab->us_freecount++;
/* Keg statistics. */
- keg->uk_free++;
+ dom->ud_free++;
}
static void
zone_release(void *arg, void **bucket, int cnt)
{
+ struct mtx *lock;
uma_zone_t zone;
- void *item;
uma_slab_t slab;
uma_keg_t keg;
uint8_t *mem;
+ void *item;
int i;
zone = arg;
keg = zone->uz_keg;
- KEG_LOCK(keg);
+ lock = NULL;
+ if (__predict_false((zone->uz_flags & UMA_ZONE_HASH) != 0))
+ lock = KEG_LOCK(keg, 0);
for (i = 0; i < cnt; i++) {
item = bucket[i];
- if (!(zone->uz_flags & UMA_ZONE_VTOSLAB)) {
+ if (__predict_true((zone->uz_flags & UMA_ZONE_VTOSLAB) != 0)) {
+ slab = vtoslab((vm_offset_t)item);
+ } else {
mem = (uint8_t *)((uintptr_t)item & (~UMA_SLAB_MASK));
- if (zone->uz_flags & UMA_ZONE_HASH) {
+ if ((zone->uz_flags & UMA_ZONE_HASH) != 0)
slab = hash_sfind(&keg->uk_hash, mem);
- } else {
- mem += keg->uk_pgoff;
- slab = (uma_slab_t)mem;
- }
- } else
- slab = vtoslab((vm_offset_t)item);
+ else
+ slab = (uma_slab_t)(mem + keg->uk_pgoff);
+ }
+ if (lock != KEG_LOCKPTR(keg, slab->us_domain)) {
+ if (lock != NULL)
+ mtx_unlock(lock);
+ lock = KEG_LOCK(keg, slab->us_domain);
+ }
slab_free_item(zone, slab, item);
}
- KEG_UNLOCK(keg);
+ if (lock != NULL)
+ mtx_unlock(lock);
}
/*
@@ -4203,7 +4233,6 @@ uma_prealloc(uma_zone_t zone, int items)
int aflags, domain, slabs;
KEG_GET(zone, keg);
- KEG_LOCK(keg);
slabs = items / keg->uk_ipers;
if (slabs * keg->uk_ipers < items)
slabs++;
@@ -4216,18 +4245,16 @@ uma_prealloc(uma_zone_t zone, int items)
aflags);
if (slab != NULL) {
dom = &keg->uk_domain[slab->us_domain];
+ LIST_REMOVE(slab, us_link);
LIST_INSERT_HEAD(&dom->ud_free_slab, slab,
us_link);
+ KEG_UNLOCK(keg, slab->us_domain);
break;
}
- if (vm_domainset_iter_policy(&di, &domain) != 0) {
- KEG_UNLOCK(keg);
+ if (vm_domainset_iter_policy(&di, &domain) != 0)
vm_wait_doms(&keg->uk_dr.dr_policy->ds_mask);
- KEG_LOCK(keg);
- }
}
}
- KEG_UNLOCK(keg);
}
/* See uma.h */
@@ -4465,6 +4492,7 @@ sysctl_vm_zone_stats(SYSCTL_HANDLER_ARGS)
uma_keg_t kz;
uma_zone_t z;
uint64_t items;
+ uint32_t kfree, pages;
int count, error, i;
error = sysctl_wire_old_buffer(req, 0);
@@ -4494,6 +4522,11 @@ sysctl_vm_zone_stats(SYSCTL_HANDLER_ARGS)
(void)sbuf_bcat(&sbuf, &ush, sizeof(ush));
LIST_FOREACH(kz, &uma_kegs, uk_link) {
+ kfree = pages = 0;
+ for (i = 0; i < vm_ndomains; i++) {
+ kfree += kz->uk_domain[i].ud_free;
+ pages += kz->uk_domain[i].ud_pages;
+ }
LIST_FOREACH(z, &kz->uk_zones, uz_link) {
bzero(&uth, sizeof(uth));
ZONE_LOCK(z);
@@ -4506,11 +4539,11 @@ sysctl_vm_zone_stats(SYSCTL_HANDLER_ARGS)
uth.uth_pages = (items / kz->uk_ipers) *
kz->uk_ppera;
} else
- uth.uth_pages = kz->uk_pages;
+ uth.uth_pages = pages;
uth.uth_maxpages = (z->uz_max_items / kz->uk_ipers) *
kz->uk_ppera;
uth.uth_limit = z->uz_max_items;
- uth.uth_keg_free = z->uz_keg->uk_free;
+ uth.uth_keg_free = kfree;
/*
* A zone is secondary is it is not the first entry
@@ -4670,9 +4703,9 @@ uma_dbg_getslab(uma_zone_t zone, void *item)
keg = zone->uz_keg;
if ((keg->uk_flags & UMA_ZONE_HASH) == 0)
return ((uma_slab_t)(mem + keg->uk_pgoff));
- KEG_LOCK(keg);
+ KEG_LOCK(keg, 0);
slab = hash_sfind(&keg->uk_hash, mem);
- KEG_UNLOCK(keg);
+ KEG_UNLOCK(keg, 0);
return (slab);
}
@@ -4791,11 +4824,12 @@ get_uma_stats(uma_keg_t kz, uma_zone_t z, uint64_t *al
} else
uma_zone_sumstat(z, cachefree, allocs, &frees, sleeps,
xdomain);
- if (!((z->uz_flags & UMA_ZONE_SECONDARY) &&
- (LIST_FIRST(&kz->uk_zones) != z)))
- *cachefree += kz->uk_free;
- for (i = 0; i < vm_ndomains; i++)
+ for (i = 0; i < vm_ndomains; i++) {
*cachefree += z->uz_domain[i].uzd_nitems;
+ if (!((z->uz_flags & UMA_ZONE_SECONDARY) &&
+ (LIST_FIRST(&kz->uk_zones) != z)))
+ *cachefree += kz->uk_domain[i].ud_free;
+ }
*used = *allocs - frees;
return (((int64_t)*used + *cachefree) * kz->uk_size);
}
Modified: head/sys/vm/uma_int.h
==============================================================================
--- head/sys/vm/uma_int.h Sat Jan 4 03:15:34 2020 (r356348)
+++ head/sys/vm/uma_int.h Sat Jan 4 03:30:08 2020 (r356349)
@@ -254,9 +254,12 @@ cache_uz_size(uma_cache_t cache)
* Per-domain slab lists. Embedded in the kegs.
*/
struct uma_domain {
+ struct mtx_padalign ud_lock; /* Lock for the domain lists. */
struct slabhead ud_part_slab; /* partially allocated slabs */
struct slabhead ud_free_slab; /* completely unallocated slabs */
struct slabhead ud_full_slab; /* fully allocated slabs */
+ uint32_t ud_pages; /* Total page count */
+ uint32_t ud_free; /* Count of items free in slabs */
} __aligned(CACHE_LINE_SIZE);
typedef struct uma_domain * uma_domain_t;
@@ -268,14 +271,11 @@ typedef struct uma_domain * uma_domain_t;
*
*/
struct uma_keg {
- struct mtx uk_lock; /* Lock for the keg. */
struct uma_hash uk_hash;
LIST_HEAD(,uma_zone) uk_zones; /* Keg's zones */
struct domainset_ref uk_dr; /* Domain selection policy. */
uint32_t uk_align; /* Alignment mask */
- uint32_t uk_pages; /* Total page count */
- uint32_t uk_free; /* Count of items free in slabs */
uint32_t uk_reserve; /* Number of reserved items. */
uint32_t uk_size; /* Requested size of each item */
uint32_t uk_rsize; /* Real size of each item */
@@ -305,8 +305,8 @@ typedef struct uma_keg * uma_keg_t;
#ifdef _KERNEL
#define KEG_ASSERT_COLD(k) \
- KASSERT((k)->uk_pages == 0, ("keg %s initialization after use.",\
- (k)->uk_name))
+ KASSERT((k)->uk_domain[0].ud_pages == 0, \
+ ("keg %s initialization after use.", (k)->uk_name))
/*
* Free bits per-slab.
@@ -536,20 +536,22 @@ static __inline uma_slab_t hash_sfind(struct uma_hash
/* Lock Macros */
-#define KEG_LOCK_INIT(k, lc) \
- do { \
- if ((lc)) \
- mtx_init(&(k)->uk_lock, (k)->uk_name, \
- (k)->uk_name, MTX_DEF | MTX_DUPOK); \
- else \
- mtx_init(&(k)->uk_lock, (k)->uk_name, \
- "UMA zone", MTX_DEF | MTX_DUPOK); \
+#define KEG_LOCKPTR(k, d) (struct mtx *)&(k)->uk_domain[(d)].ud_lock
+#define KEG_LOCK_INIT(k, d, lc) \
+ do { \
+ if ((lc)) \
+ mtx_init(KEG_LOCKPTR(k, d), (k)->uk_name, \
+ (k)->uk_name, MTX_DEF | MTX_DUPOK); \
+ else \
+ mtx_init(KEG_LOCKPTR(k, d), (k)->uk_name, \
+ "UMA zone", MTX_DEF | MTX_DUPOK); \
} while (0)
-#define KEG_LOCK_FINI(k) mtx_destroy(&(k)->uk_lock)
-#define KEG_LOCK(k) mtx_lock(&(k)->uk_lock)
-#define KEG_UNLOCK(k) mtx_unlock(&(k)->uk_lock)
-#define KEG_LOCK_ASSERT(k) mtx_assert(&(k)->uk_lock, MA_OWNED)
+#define KEG_LOCK_FINI(k, d) mtx_destroy(KEG_LOCKPTR(k, d))
+#define KEG_LOCK(k, d) \
+ ({ mtx_lock(KEG_LOCKPTR(k, d)); KEG_LOCKPTR(k, d); })
+#define KEG_UNLOCK(k, d) mtx_unlock(KEG_LOCKPTR(k, d))
+#define KEG_LOCK_ASSERT(k, d) mtx_assert(KEG_LOCKPTR(k, d), MA_OWNED)
#define KEG_GET(zone, keg) do { \
(keg) = (zone)->uz_keg; \
More information about the svn-src-all
mailing list