svn commit: r252226 - head/sys/vm
Jeff Roberson
jeff at FreeBSD.org
Wed Jun 26 00:57:40 UTC 2013
Author: jeff
Date: Wed Jun 26 00:57:38 2013
New Revision: 252226
URL: http://svnweb.freebsd.org/changeset/base/252226
Log:
- Resolve bucket recursion issues by passing a cookie with zone flags
through bucket_alloc() to uma_zalloc_arg() and uma_zfree_arg().
- Make some smaller buckets for large zones to further reduce memory
waste.
- Implement uma_zone_reserve(). This holds aside a number of items only
for callers who specify M_USE_RESERVE. buckets will never be filled
from reserve allocations.
Sponsored by: EMC / Isilon Storage Division
Modified:
head/sys/vm/uma.h
head/sys/vm/uma_core.c
head/sys/vm/uma_int.h
Modified: head/sys/vm/uma.h
==============================================================================
--- head/sys/vm/uma.h Wed Jun 26 00:42:45 2013 (r252225)
+++ head/sys/vm/uma.h Wed Jun 26 00:57:38 2013 (r252226)
@@ -459,6 +459,12 @@ void uma_reclaim(void);
void uma_set_align(int align);
/*
+ * Set a reserved number of items to hold for M_USE_RESERVE allocations. All
+ * other requests must allocate new backing pages.
+ */
+void uma_zone_reserve(uma_zone_t zone, int nitems);
+
+/*
* Reserves the maximum KVA space required by the zone and configures the zone
* to use a VM_ALLOC_NOOBJ-based backend allocator.
*
Modified: head/sys/vm/uma_core.c
==============================================================================
--- head/sys/vm/uma_core.c Wed Jun 26 00:42:45 2013 (r252225)
+++ head/sys/vm/uma_core.c Wed Jun 26 00:57:38 2013 (r252226)
@@ -206,12 +206,14 @@ struct uma_bucket_zone {
#define BUCKET_MAX BUCKET_SIZE(128)
struct uma_bucket_zone bucket_zones[] = {
+ { NULL, "4 Bucket", BUCKET_SIZE(4), 4096 },
+ { NULL, "8 Bucket", BUCKET_SIZE(8), 2048 },
+ { NULL, "16 Bucket", BUCKET_SIZE(16), 1024 },
{ NULL, "32 Bucket", BUCKET_SIZE(32), 512 },
{ NULL, "64 Bucket", BUCKET_SIZE(64), 256 },
{ NULL, "128 Bucket", BUCKET_SIZE(128), 128 },
{ NULL, NULL, 0}
};
-static uma_zone_t largebucket;
/*
* Flags and enumerations to be passed to internal functions.
@@ -246,10 +248,10 @@ static void *zone_alloc_item(uma_zone_t,
static void zone_free_item(uma_zone_t, void *, void *, enum zfreeskip);
static void bucket_enable(void);
static void bucket_init(void);
-static uma_bucket_t bucket_alloc(uma_zone_t zone, int);
-static void bucket_free(uma_zone_t zone, uma_bucket_t);
+static uma_bucket_t bucket_alloc(uma_zone_t zone, void *, int);
+static void bucket_free(uma_zone_t zone, uma_bucket_t, void *);
static void bucket_zone_drain(void);
-static uma_bucket_t zone_alloc_bucket(uma_zone_t zone, int flags);
+static uma_bucket_t zone_alloc_bucket(uma_zone_t zone, void *, int flags);
static uma_slab_t zone_fetch_slab(uma_zone_t zone, uma_keg_t last, int flags);
static uma_slab_t zone_fetch_slab_multi(uma_zone_t zone, uma_keg_t last, int flags);
static void *slab_alloc_item(uma_keg_t keg, uma_slab_t slab);
@@ -304,17 +306,8 @@ bucket_init(void)
size += sizeof(void *) * ubz->ubz_entries;
ubz->ubz_zone = uma_zcreate(ubz->ubz_name, size,
NULL, NULL, NULL, NULL, UMA_ALIGN_PTR,
- UMA_ZONE_MAXBUCKET | UMA_ZONE_MTXCLASS);
+ UMA_ZONE_MTXCLASS | UMA_ZFLAG_BUCKET);
}
- /*
- * To avoid recursive bucket allocation loops we disable buckets
- * on the smallest bucket zone and use it for the largest zone.
- * The remainder of the zones all use the largest zone.
- */
- ubz--;
- ubz->ubz_zone->uz_count = bucket_zones[0].ubz_entries;
- bucket_zones[0].ubz_zone->uz_count = 0;
- largebucket = ubz->ubz_zone;
}
/*
@@ -350,7 +343,7 @@ bucket_select(int size)
}
static uma_bucket_t
-bucket_alloc(uma_zone_t zone, int flags)
+bucket_alloc(uma_zone_t zone, void *udata, int flags)
{
struct uma_bucket_zone *ubz;
uma_bucket_t bucket;
@@ -363,11 +356,26 @@ bucket_alloc(uma_zone_t zone, int flags)
*/
if (bucketdisable)
return (NULL);
-
- if (zone->uz_flags & UMA_ZFLAG_CACHEONLY)
+ /*
+ * To limit bucket recursion we store the original zone flags
+ * in a cookie passed via zalloc_arg/zfree_arg. This allows the
+ * NOVM flag to persist even through deep recursions. We also
+ * store ZFLAG_BUCKET once we have recursed attempting to allocate
+ * a bucket for a bucket zone so we do not allow infinite bucket
+ * recursion. This cookie will even persist to frees of unused
+ * buckets via the allocation path or bucket allocations in the
+ * free path.
+ */
+ if ((uintptr_t)udata & UMA_ZFLAG_BUCKET)
+ return (NULL);
+ if ((zone->uz_flags & UMA_ZFLAG_BUCKET) == 0)
+ udata = (void *)(uintptr_t)zone->uz_flags;
+ else
+ udata = (void *)((uintptr_t)udata | UMA_ZFLAG_BUCKET);
+ if ((uintptr_t)udata & UMA_ZFLAG_CACHEONLY)
flags |= M_NOVM;
ubz = bucket_zone_lookup(zone->uz_count);
- bucket = uma_zalloc(ubz->ubz_zone, flags);
+ bucket = uma_zalloc_arg(ubz->ubz_zone, udata, flags);
if (bucket) {
#ifdef INVARIANTS
bzero(bucket->ub_bucket, sizeof(void *) * ubz->ubz_entries);
@@ -380,14 +388,16 @@ bucket_alloc(uma_zone_t zone, int flags)
}
static void
-bucket_free(uma_zone_t zone, uma_bucket_t bucket)
+bucket_free(uma_zone_t zone, uma_bucket_t bucket, void *udata)
{
struct uma_bucket_zone *ubz;
KASSERT(bucket->ub_cnt == 0,
("bucket_free: Freeing a non free bucket."));
+ if ((zone->uz_flags & UMA_ZFLAG_BUCKET) == 0)
+ udata = (void *)(uintptr_t)zone->uz_flags;
ubz = bucket_zone_lookup(bucket->ub_entries);
- uma_zfree(ubz->ubz_zone, bucket);
+ uma_zfree_arg(ubz->ubz_zone, bucket, udata);
}
static void
@@ -662,9 +672,9 @@ cache_drain(uma_zone_t zone)
bucket_drain(zone, cache->uc_allocbucket);
bucket_drain(zone, cache->uc_freebucket);
if (cache->uc_allocbucket != NULL)
- bucket_free(zone, cache->uc_allocbucket);
+ bucket_free(zone, cache->uc_allocbucket, NULL);
if (cache->uc_freebucket != NULL)
- bucket_free(zone, cache->uc_freebucket);
+ bucket_free(zone, cache->uc_freebucket, NULL);
cache->uc_allocbucket = cache->uc_freebucket = NULL;
}
ZONE_LOCK(zone);
@@ -688,7 +698,7 @@ bucket_cache_drain(uma_zone_t zone)
LIST_REMOVE(bucket, ub_link);
ZONE_UNLOCK(zone);
bucket_drain(zone, bucket);
- bucket_free(zone, bucket);
+ bucket_free(zone, bucket, NULL);
ZONE_LOCK(zone);
}
}
@@ -1172,9 +1182,9 @@ keg_small_init(uma_keg_t keg)
/*
* We can't do OFFPAGE if we're internal or if we've been
* asked to not go to the VM for buckets. If we do this we
- * may end up going to the VM (kmem_map) for slabs which we
- * do not want to do if we're UMA_ZFLAG_CACHEONLY as a
- * result of UMA_ZONE_VM, which clearly forbids it.
+ * may end up going to the VM for slabs which we do not
+ * want to do if we're UMA_ZFLAG_CACHEONLY as a result
+ * of UMA_ZONE_VM, which clearly forbids it.
*/
if ((keg->uk_flags & UMA_ZFLAG_INTERNAL) ||
(keg->uk_flags & UMA_ZFLAG_CACHEONLY))
@@ -1301,6 +1311,7 @@ keg_ctor(void *mem, int size, void *udat
keg->uk_fini = arg->fini;
keg->uk_align = arg->align;
keg->uk_free = 0;
+ keg->uk_reserve = 0;
keg->uk_pages = 0;
keg->uk_flags = arg->flags;
keg->uk_allocf = page_alloc;
@@ -1472,6 +1483,9 @@ zone_ctor(void *mem, int size, void *uda
* This is a pure cache zone, no kegs.
*/
if (arg->import) {
+ if (arg->flags & UMA_ZONE_VM)
+ arg->flags |= UMA_ZFLAG_CACHEONLY;
+ zone->uz_flags = arg->flags;
zone->uz_size = arg->size;
zone->uz_import = arg->import;
zone->uz_release = arg->release;
@@ -2073,7 +2087,7 @@ zalloc_start:
cache->uc_allocbucket = NULL;
critical_exit();
if (bucket != NULL)
- bucket_free(zone, bucket);
+ bucket_free(zone, bucket, udata);
/* Short-circuit for zones without buckets and low memory. */
if (zone->uz_count == 0 || bucketdisable)
@@ -2131,8 +2145,7 @@ zalloc_start:
* We bump the uz count when the cache size is insufficient to
* handle the working set.
*/
- if (lockfail && zone->uz_count < BUCKET_MAX && zone->uz_count != 0 &&
- zone != largebucket)
+ if (lockfail && zone->uz_count < BUCKET_MAX)
zone->uz_count++;
ZONE_UNLOCK(zone);
@@ -2141,7 +2154,7 @@ zalloc_start:
* works we'll restart the allocation from the begining and it
* will use the just filled bucket.
*/
- bucket = zone_alloc_bucket(zone, flags);
+ bucket = zone_alloc_bucket(zone, udata, flags);
if (bucket != NULL) {
ZONE_LOCK(zone);
critical_enter();
@@ -2177,9 +2190,13 @@ static uma_slab_t
keg_fetch_slab(uma_keg_t keg, uma_zone_t zone, int flags)
{
uma_slab_t slab;
+ int reserve;
mtx_assert(&keg->uk_lock, MA_OWNED);
slab = NULL;
+ reserve = 0;
+ if ((flags & M_USE_RESERVE) == 0)
+ reserve = keg->uk_reserve;
for (;;) {
/*
@@ -2187,7 +2204,7 @@ keg_fetch_slab(uma_keg_t keg, uma_zone_t
* used over those that are totally full. This helps to reduce
* fragmentation.
*/
- if (keg->uk_free != 0) {
+ if (keg->uk_free > reserve) {
if (!LIST_EMPTY(&keg->uk_part_slab)) {
slab = LIST_FIRST(&keg->uk_part_slab);
} else {
@@ -2382,10 +2399,12 @@ zone_import(uma_zone_t zone, void **buck
if ((slab = zone->uz_slab(zone, keg, flags)) == NULL)
break;
keg = slab->us_keg;
- while (slab->us_freecount && i < max)
+ while (slab->us_freecount && i < max) {
bucket[i++] = slab_alloc_item(keg, slab);
-
- /* Don't block on the next fill */
+ if (keg->uk_free <= keg->uk_reserve)
+ break;
+ }
+ /* Don't grab more than one slab at a time. */
flags &= ~M_WAITOK;
flags |= M_NOWAIT;
}
@@ -2396,12 +2415,13 @@ zone_import(uma_zone_t zone, void **buck
}
static uma_bucket_t
-zone_alloc_bucket(uma_zone_t zone, int flags)
+zone_alloc_bucket(uma_zone_t zone, void *udata, int flags)
{
uma_bucket_t bucket;
int max;
- bucket = bucket_alloc(zone, M_NOWAIT | (flags & M_NOVM));
+ /* Don't wait for buckets, preserve caller's NOVM setting. */
+ bucket = bucket_alloc(zone, udata, M_NOWAIT | (flags & M_NOVM));
if (bucket == NULL)
goto out;
@@ -2437,7 +2457,7 @@ zone_alloc_bucket(uma_zone_t zone, int f
out:
if (bucket == NULL || bucket->ub_cnt == 0) {
if (bucket != NULL)
- bucket_free(zone, bucket);
+ bucket_free(zone, bucket, udata);
atomic_add_long(&zone->uz_fails, 1);
return (NULL);
}
@@ -2633,7 +2653,7 @@ zfree_start:
#ifdef UMA_DEBUG_ALLOC
printf("uma_zfree: Allocating new free bucket.\n");
#endif
- bucket = bucket_alloc(zone, M_NOWAIT);
+ bucket = bucket_alloc(zone, udata, M_NOWAIT);
if (bucket) {
critical_enter();
cpu = curcpu;
@@ -2647,7 +2667,7 @@ zfree_start:
* critical section to free the bucket.
*/
critical_exit();
- bucket_free(zone, bucket);
+ bucket_free(zone, bucket, udata);
goto zfree_restart;
}
@@ -2921,12 +2941,27 @@ uma_zone_set_allocf(uma_zone_t zone, uma
keg = zone_first_keg(zone);
KEG_LOCK(keg);
- keg->uk_flags |= UMA_ZFLAG_PRIVALLOC;
keg->uk_allocf = allocf;
KEG_UNLOCK(keg);
}
/* See uma.h */
+void
+uma_zone_reserve(uma_zone_t zone, int items)
+{
+ uma_keg_t keg;
+
+ keg = zone_first_keg(zone);
+ if (keg == NULL)
+ return;
+ KEG_LOCK(keg);
+ keg->uk_reserve = items;
+ KEG_UNLOCK(keg);
+
+ return;
+}
+
+/* See uma.h */
int
uma_zone_reserve_kva(uma_zone_t zone, int count)
{
@@ -2961,7 +2996,7 @@ uma_zone_reserve_kva(uma_zone_t zone, in
#else
keg->uk_allocf = noobj_alloc;
#endif
- keg->uk_flags |= UMA_ZONE_NOFREE | UMA_ZFLAG_PRIVALLOC;
+ keg->uk_flags |= UMA_ZONE_NOFREE;
KEG_UNLOCK(keg);
return (1);
Modified: head/sys/vm/uma_int.h
==============================================================================
--- head/sys/vm/uma_int.h Wed Jun 26 00:42:45 2013 (r252225)
+++ head/sys/vm/uma_int.h Wed Jun 26 00:57:38 2013 (r252226)
@@ -200,6 +200,7 @@ struct uma_keg {
uint32_t uk_align; /* Alignment mask */
uint32_t uk_pages; /* Total page count */
uint32_t uk_free; /* Count of items free in slabs */
+ uint32_t uk_reserve; /* Number of reserved items. */
uint32_t uk_size; /* Requested size of each item */
uint32_t uk_rsize; /* Real size of each item */
uint32_t uk_maxpages; /* Maximum number of pages to alloc */
@@ -325,12 +326,13 @@ struct uma_zone {
*/
#define UMA_ZFLAG_MULTI 0x04000000 /* Multiple kegs in the zone. */
#define UMA_ZFLAG_DRAINING 0x08000000 /* Running zone_drain. */
-#define UMA_ZFLAG_PRIVALLOC 0x10000000 /* Use uz_allocf. */
+#define UMA_ZFLAG_BUCKET 0x10000000 /* Bucket zone. */
#define UMA_ZFLAG_INTERNAL 0x20000000 /* No offpage no PCPU. */
#define UMA_ZFLAG_FULL 0x40000000 /* Reached uz_maxpages */
#define UMA_ZFLAG_CACHEONLY 0x80000000 /* Don't ask VM for buckets. */
-#define UMA_ZFLAG_INHERIT (UMA_ZFLAG_INTERNAL | UMA_ZFLAG_CACHEONLY)
+#define UMA_ZFLAG_INHERIT \
+ (UMA_ZFLAG_INTERNAL | UMA_ZFLAG_CACHEONLY | UMA_ZFLAG_BUCKET)
static inline uma_keg_t
zone_first_keg(uma_zone_t zone)
More information about the svn-src-head
mailing list