svn commit: r327014 - in user/jeff/numa/sys: kern sys vm
Jeff Roberson
jeff at FreeBSD.org
Wed Dec 20 04:03:01 UTC 2017
Author: jeff
Date: Wed Dec 20 04:02:58 2017
New Revision: 327014
URL: https://svnweb.freebsd.org/changeset/base/327014
Log:
Add domain specific malloc functions.
Modified:
user/jeff/numa/sys/kern/kern_malloc.c
user/jeff/numa/sys/sys/malloc.h
user/jeff/numa/sys/vm/uma_core.c
user/jeff/numa/sys/vm/uma_int.h
Modified: user/jeff/numa/sys/kern/kern_malloc.c
==============================================================================
--- user/jeff/numa/sys/kern/kern_malloc.c Wed Dec 20 01:03:34 2017 (r327013)
+++ user/jeff/numa/sys/kern/kern_malloc.c Wed Dec 20 04:02:58 2017 (r327014)
@@ -95,6 +95,11 @@ __FBSDID("$FreeBSD$");
dtrace_malloc_probe_func_t dtrace_malloc_probe;
#endif
+#if defined(INVARIANTS) || defined(MALLOC_MAKE_FAILURES) || \
+ defined(DEBUG_MEMGUARD) || defined(DEBUG_REDZONE)
+#define MALLOC_DEBUG 1
+#endif
+
/*
* When realloc() is called, if the new size is sufficiently smaller than
* the old size, realloc() will allocate a new, smaller block to avoid
@@ -416,6 +421,20 @@ contigmalloc(unsigned long size, struct malloc_type *t
return (ret);
}
+void *
+contigmalloc_domain(unsigned long size, struct malloc_type *type,
+ int domain, int flags, vm_paddr_t low, vm_paddr_t high,
+ unsigned long alignment, vm_paddr_t boundary)
+{
+ void *ret;
+
+ ret = (void *)kmem_alloc_contig_domain(domain, size, flags, low, high,
+ alignment, boundary, VM_MEMATTR_DEFAULT);
+ if (ret != NULL)
+ malloc_type_allocated(type, round_page(size));
+ return (ret);
+}
+
/*
* contigfree:
*
@@ -431,26 +450,14 @@ contigfree(void *addr, unsigned long size, struct mall
malloc_type_freed(type, round_page(size));
}
-/*
- * malloc:
- *
- * Allocate a block of memory.
- *
- * If M_NOWAIT is set, this routine will not block and return NULL if
- * the allocation fails.
- */
-void *
-malloc(unsigned long size, struct malloc_type *mtp, int flags)
+#ifdef MALLOC_DEBUG
+static int
+malloc_dbg(caddr_t *vap, unsigned long *sizep, struct malloc_type *mtp,
+ int flags)
{
+#ifdef INVARIANTS
int indx;
- struct malloc_type_internal *mtip;
- caddr_t va;
- uma_zone_t zone;
-#if defined(DIAGNOSTIC) || defined(DEBUG_REDZONE)
- unsigned long osize = size;
-#endif
-#ifdef INVARIANTS
KASSERT(mtp->ks_magic == M_MAGIC, ("malloc: bad malloc type magic"));
/*
* Check that exactly one of M_WAITOK or M_NOWAIT is specified.
@@ -473,7 +480,8 @@ malloc(unsigned long size, struct malloc_type *mtp, in
if ((malloc_nowait_count % malloc_failure_rate) == 0) {
atomic_add_int(&malloc_failure_count, 1);
t_malloc_fail = time_uptime;
- return (NULL);
+ *vap = NULL;
+ return (EJUSTRETURN);
}
}
#endif
@@ -486,16 +494,46 @@ malloc(unsigned long size, struct malloc_type *mtp, in
#ifdef DEBUG_MEMGUARD
if (memguard_cmp_mtp(mtp, size)) {
va = memguard_alloc(size, flags);
- if (va != NULL)
- return (va);
+ if (va != NULL) {
+ *vap = va;
+ return (EJUSTRETURN);
+ }
/* This is unfortunate but should not be fatal. */
}
#endif
#ifdef DEBUG_REDZONE
- size = redzone_size_ntor(size);
+ *sizep = redzone_size_ntor(*sizep);
#endif
+ return (0);
+}
+#endif
+
+/*
+ * malloc:
+ *
+ * Allocate a block of memory.
+ *
+ * If M_NOWAIT is set, this routine will not block and return NULL if
+ * the allocation fails.
+ */
+void *
+malloc(unsigned long size, struct malloc_type *mtp, int flags)
+{
+ int indx;
+ struct malloc_type_internal *mtip;
+ caddr_t va;
+ uma_zone_t zone;
+#if defined(DIAGNOSTIC) || defined(DEBUG_REDZONE)
+ unsigned long osize = size;
+#endif
+
+#ifdef MALLOC_DEBUG
+ if (malloc_dbg(&va, &size, mtp, flags) != 0)
+ return (va);
+#endif
+
if (size <= kmem_zmax) {
mtip = mtp->ks_handle;
if (size & KMEM_ZMASK)
@@ -522,11 +560,55 @@ malloc(unsigned long size, struct malloc_type *mtp, in
KASSERT(va != NULL, ("malloc(M_WAITOK) returned NULL"));
else if (va == NULL)
t_malloc_fail = time_uptime;
-#ifdef DIAGNOSTIC
- if (va != NULL && !(flags & M_ZERO)) {
- memset(va, 0x70, osize);
- }
+#ifdef DEBUG_REDZONE
+ if (va != NULL)
+ va = redzone_setup(va, osize);
#endif
+ return ((void *) va);
+}
+
+void *
+malloc_domain(unsigned long size, struct malloc_type *mtp, int domain,
+ int flags)
+{
+ int indx;
+ struct malloc_type_internal *mtip;
+ caddr_t va;
+ uma_zone_t zone;
+#if defined(DIAGNOSTIC) || defined(DEBUG_REDZONE)
+ unsigned long osize = size;
+#endif
+
+#ifdef MALLOC_DEBUG
+ if (malloc_dbg(&va, &size, mtp, flags) != 0)
+ return (va);
+#endif
+ if (size <= kmem_zmax) {
+ mtip = mtp->ks_handle;
+ if (size & KMEM_ZMASK)
+ size = (size & ~KMEM_ZMASK) + KMEM_ZBASE;
+ indx = kmemsize[size >> KMEM_ZSHIFT];
+ KASSERT(mtip->mti_zone < numzones,
+ ("mti_zone %u out of range %d",
+ mtip->mti_zone, numzones));
+ zone = kmemzones[indx].kz_zone[mtip->mti_zone];
+#ifdef MALLOC_PROFILE
+ krequests[size >> KMEM_ZSHIFT]++;
+#endif
+ va = uma_zalloc_domain(zone, NULL, domain, flags);
+ if (va != NULL)
+ size = zone->uz_size;
+ malloc_type_zone_allocated(mtp, va == NULL ? 0 : size, indx);
+ } else {
+ size = roundup(size, PAGE_SIZE);
+ zone = NULL;
+ va = uma_large_malloc_domain(size, domain, flags);
+ malloc_type_allocated(mtp, va == NULL ? 0 : size);
+ }
+ if (flags & M_WAITOK)
+ KASSERT(va != NULL, ("malloc(M_WAITOK) returned NULL"));
+ else if (va == NULL)
+ t_malloc_fail = time_uptime;
#ifdef DEBUG_REDZONE
if (va != NULL)
va = redzone_setup(va, osize);
@@ -534,66 +616,124 @@ malloc(unsigned long size, struct malloc_type *mtp, in
return ((void *) va);
}
-/*
- * free:
- *
- * Free a block of memory allocated by malloc.
- *
- * This routine may not block.
- */
-void
-free(void *addr, struct malloc_type *mtp)
+#ifdef INVARIANTS
+static void
+free_save_type(void *addr, struct malloc_type *mtp, u_long size)
{
- uma_slab_t slab;
- u_long size;
+ struct malloc_type **mtpp = addr;
+ /*
+ * Cache a pointer to the malloc_type that most recently freed
+ * this memory here. This way we know who is most likely to
+ * have stepped on it later.
+ *
+ * This code assumes that size is a multiple of 8 bytes for
+ * 64 bit machines
+ */
+ mtpp = (struct malloc_type **) ((unsigned long)mtpp & ~UMA_ALIGN_PTR);
+ mtpp += (size - sizeof(struct malloc_type *)) /
+ sizeof(struct malloc_type *);
+ *mtpp = mtp;
+}
+#endif
+
+#ifdef MALLOC_DEBUG
+static int
+free_dbg(void **addrp, struct malloc_type *mtp)
+{
+ void *addr;
+
+ addr = *addrp;
KASSERT(mtp->ks_magic == M_MAGIC, ("free: bad malloc type magic"));
KASSERT(curthread->td_critnest == 0 || SCHEDULER_STOPPED(),
("free: called with spinlock or critical section held"));
/* free(NULL, ...) does nothing */
if (addr == NULL)
- return;
+ return (EJUSTRETURN);
#ifdef DEBUG_MEMGUARD
if (is_memguard_addr(addr)) {
memguard_free(addr);
- return;
+ return (EJUSTRETURN);
}
#endif
#ifdef DEBUG_REDZONE
redzone_check(addr);
- addr = redzone_addr_ntor(addr);
+ *addrp = redzone_addr_ntor(addr);
#endif
- slab = vtoslab((vm_offset_t)addr & (~UMA_SLAB_MASK));
+ return (0);
+}
+#endif
+/*
+ * free:
+ *
+ * Free a block of memory allocated by malloc.
+ *
+ * This routine may not block.
+ */
+void
+free(void *addr, struct malloc_type *mtp)
+{
+ uma_slab_t slab;
+ u_long size;
+
+#ifdef MALLOC_DEBUG
+ if (free_dbg(&addr, mtp) != 0)
+ return;
+#endif
+ /* free(NULL, ...) does nothing */
+ if (addr == NULL)
+ return;
+
+ slab = vtoslab((vm_offset_t)addr & (~UMA_SLAB_MASK));
if (slab == NULL)
panic("free: address %p(%p) has not been allocated.\n",
addr, (void *)((u_long)addr & (~UMA_SLAB_MASK)));
if (!(slab->us_flags & UMA_SLAB_MALLOC)) {
+ size = slab->us_keg->uk_size;
#ifdef INVARIANTS
- struct malloc_type **mtpp = addr;
+ free_save_type(addr, mtp, size);
#endif
+ uma_zfree_arg(LIST_FIRST(&slab->us_keg->uk_zones), addr, slab);
+ } else {
+ size = slab->us_size;
+ uma_large_free(slab);
+ }
+ malloc_type_freed(mtp, size);
+}
+
+void
+free_domain(void *addr, struct malloc_type *mtp)
+{
+ uma_slab_t slab;
+ u_long size;
+
+#ifdef MALLOC_DEBUG
+ if (free_dbg(&addr, mtp) != 0)
+ return;
+#endif
+
+ /* free(NULL, ...) does nothing */
+ if (addr == NULL)
+ return;
+
+ slab = vtoslab((vm_offset_t)addr & (~UMA_SLAB_MASK));
+ if (slab == NULL)
+ panic("free_domain: address %p(%p) has not been allocated.\n",
+ addr, (void *)((u_long)addr & (~UMA_SLAB_MASK)));
+
+ if (!(slab->us_flags & UMA_SLAB_MALLOC)) {
size = slab->us_keg->uk_size;
#ifdef INVARIANTS
- /*
- * Cache a pointer to the malloc_type that most recently freed
- * this memory here. This way we know who is most likely to
- * have stepped on it later.
- *
- * This code assumes that size is a multiple of 8 bytes for
- * 64 bit machines
- */
- mtpp = (struct malloc_type **)
- ((unsigned long)mtpp & ~UMA_ALIGN_PTR);
- mtpp += (size - sizeof(struct malloc_type *)) /
- sizeof(struct malloc_type *);
- *mtpp = mtp;
+ free_save_type(addr, mtp, size);
#endif
- uma_zfree_arg(LIST_FIRST(&slab->us_keg->uk_zones), addr, slab);
+ uma_zfree_domain(LIST_FIRST(&slab->us_keg->uk_zones),
+ addr, slab);
} else {
size = slab->us_size;
uma_large_free(slab);
Modified: user/jeff/numa/sys/sys/malloc.h
==============================================================================
--- user/jeff/numa/sys/sys/malloc.h Wed Dec 20 01:03:34 2017 (r327013)
+++ user/jeff/numa/sys/sys/malloc.h Wed Dec 20 04:02:58 2017 (r327014)
@@ -174,8 +174,16 @@ void *contigmalloc(unsigned long size, struct malloc_t
vm_paddr_t low, vm_paddr_t high, unsigned long alignment,
vm_paddr_t boundary) __malloc_like __result_use_check
__alloc_size(1) __alloc_align(6);
+void *contigmalloc_domain(unsigned long size, struct malloc_type *type,
+ int domain, int flags, vm_paddr_t low, vm_paddr_t high,
+ unsigned long alignment, vm_paddr_t boundary)
+ __malloc_like __result_use_check __alloc_size(1) __alloc_align(6);
void free(void *addr, struct malloc_type *type);
+void free_domain(void *addr, struct malloc_type *type);
void *malloc(unsigned long size, struct malloc_type *type, int flags)
+ __malloc_like __result_use_check __alloc_size(1);
+void *malloc_domain(unsigned long size, struct malloc_type *type,
+ int domain, int flags)
__malloc_like __result_use_check __alloc_size(1);
void malloc_init(void *);
int malloc_last_fail(void);
Modified: user/jeff/numa/sys/vm/uma_core.c
==============================================================================
--- user/jeff/numa/sys/vm/uma_core.c Wed Dec 20 01:03:34 2017 (r327013)
+++ user/jeff/numa/sys/vm/uma_core.c Wed Dec 20 04:02:58 2017 (r327014)
@@ -221,6 +221,8 @@ struct uma_bucket_zone bucket_zones[] = {
*/
enum zfreeskip { SKIP_NONE = 0, SKIP_DTOR, SKIP_FINI };
+#define UMA_ANYDOMAIN -1 /* Special value for domain search. */
+
/* Prototypes.. */
static void *noobj_alloc(uma_zone_t, vm_size_t, int, uint8_t *, int);
@@ -564,8 +566,8 @@ hash_alloc(struct uma_hash *hash)
M_UMAHASH, M_NOWAIT);
} else {
alloc = sizeof(hash->uh_slab_hash[0]) * UMA_HASH_SIZE_INIT;
- hash->uh_slab_hash = zone_alloc_item(hashzone, NULL, 0,
- M_WAITOK);
+ hash->uh_slab_hash = zone_alloc_item(hashzone, NULL,
+ UMA_ANYDOMAIN, M_WAITOK);
hash->uh_hashsize = UMA_HASH_SIZE_INIT;
}
if (hash->uh_slab_hash) {
@@ -1878,7 +1880,7 @@ uma_kcreate(uma_zone_t zone, size_t size, uma_init umi
args.align = (align == UMA_ALIGN_CACHE) ? uma_align_cache : align;
args.flags = flags;
args.zone = zone;
- return (zone_alloc_item(kegs, &args, 0, M_WAITOK));
+ return (zone_alloc_item(kegs, &args, UMA_ANYDOMAIN, M_WAITOK));
}
/* See uma.h */
@@ -1935,7 +1937,7 @@ uma_zcreate(const char *name, size_t size, uma_ctor ct
sx_slock(&uma_drain_lock);
locked = true;
}
- res = zone_alloc_item(zones, &args, 0, M_WAITOK);
+ res = zone_alloc_item(zones, &args, UMA_ANYDOMAIN, M_WAITOK);
if (locked)
sx_sunlock(&uma_drain_lock);
return (res);
@@ -1970,7 +1972,7 @@ uma_zsecond_create(char *name, uma_ctor ctor, uma_dtor
locked = true;
}
/* XXX Attaches only one keg of potentially many. */
- res = zone_alloc_item(zones, &args, 0, M_WAITOK);
+ res = zone_alloc_item(zones, &args, UMA_ANYDOMAIN, M_WAITOK);
if (locked)
sx_sunlock(&uma_drain_lock);
return (res);
@@ -1997,7 +1999,7 @@ uma_zcache_create(char *name, int size, uma_ctor ctor,
args.align = 0;
args.flags = flags;
- return (zone_alloc_item(zones, &args, 0, M_WAITOK));
+ return (zone_alloc_item(zones, &args, UMA_ANYDOMAIN, M_WAITOK));
}
static void
@@ -2206,7 +2208,7 @@ zalloc_start:
if (zone->uz_flags & UMA_ZONE_NUMA)
domain = PCPU_GET(domain);
else
- domain = 0;
+ domain = UMA_ANYDOMAIN;
/* Short-circuit for zones without buckets and low memory. */
if (zone->uz_count == 0 || bucketdisable)
@@ -2248,7 +2250,10 @@ zalloc_start:
/*
* Check the zone's cache of buckets.
*/
- zdom = &zone->uz_domain[domain];
+ if (domain == UMA_ANYDOMAIN)
+ zdom = &zone->uz_domain[0];
+ else
+ zdom = &zone->uz_domain[domain];
if ((bucket = LIST_FIRST(&zdom->uzd_buckets)) != NULL) {
KASSERT(bucket->ub_cnt != 0,
("uma_zalloc_arg: Returning an empty bucket."));
@@ -2306,6 +2311,28 @@ zalloc_item:
return (item);
}
+void *
+uma_zalloc_domain(uma_zone_t zone, void *udata, int domain, int flags)
+{
+
+ /* Enable entropy collection for RANDOM_ENABLE_UMA kernel option */
+ random_harvest_fast_uma(&zone, sizeof(zone), 1, RANDOM_UMA);
+
+ /* This is the fast path allocation */
+ CTR5(KTR_UMA,
+ "uma_zalloc_domain thread %x zone %s(%p) domain %d flags %d",
+ curthread, zone->uz_name, zone, domain, flags);
+
+ if (flags & M_WAITOK) {
+ WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL,
+ "uma_zalloc_domain: zone \"%s\"", zone->uz_name);
+ }
+ KASSERT(curthread->td_critnest == 0 || SCHEDULER_STOPPED(),
+ ("uma_zalloc_domain: called with spinlock or critical section held"));
+
+ return (zone_alloc_item(zone, udata, domain, flags));
+}
+
/*
* Find a slab with some space. Prefer slabs that are partially used over those
* that are totally full. This helps to reduce fragmentation.
@@ -2360,7 +2387,9 @@ keg_fetch_slab(uma_keg_t keg, uma_zone_t zone, int rdo
* Round-robin for non first-touch zones when there is more than one
* domain.
*/
- rr = (zone->uz_flags & UMA_ZONE_NUMA) == 0 && vm_ndomains != 1;
+ if (vm_ndomains == 1)
+ rdomain = 0;
+ rr = rdomain == UMA_ANYDOMAIN;
if (rr) {
keg->uk_cursor = (keg->uk_cursor + 1) % vm_ndomains;
domain = start = keg->uk_cursor;
@@ -2665,6 +2694,7 @@ zone_alloc_bucket(uma_zone_t zone, void *udata, int do
* Arguments
* zone The zone to alloc for.
* udata The data to be passed to the constructor.
+ * domain The domain to allocate from or UMA_ANYDOMAIN.
* flags M_WAITOK, M_NOWAIT, M_ZERO.
*
* Returns
@@ -2896,6 +2926,25 @@ zfree_item:
return;
}
+void
+uma_zfree_domain(uma_zone_t zone, void *item, void *udata)
+{
+
+ /* Enable entropy collection for RANDOM_ENABLE_UMA kernel option */
+ random_harvest_fast_uma(&zone, sizeof(zone), 1, RANDOM_UMA);
+
+ CTR2(KTR_UMA, "uma_zfree_domain thread %x zone %s", curthread,
+ zone->uz_name);
+
+ KASSERT(curthread->td_critnest == 0 || SCHEDULER_STOPPED(),
+ ("uma_zfree_domain: called with spinlock or critical section held"));
+
+ /* uma_zfree(..., NULL) does nothing, to match free(9). */
+ if (item == NULL)
+ return;
+ zone_free_item(zone, item, udata, SKIP_NONE);
+}
+
static void
slab_free_item(uma_keg_t keg, uma_slab_t slab, void *item)
{
@@ -3342,15 +3391,18 @@ uma_zone_exhausted_nolock(uma_zone_t zone)
}
void *
-uma_large_malloc(vm_size_t size, int wait)
+uma_large_malloc_domain(vm_size_t size, int domain, int wait)
{
vm_offset_t addr;
uma_slab_t slab;
- slab = zone_alloc_item(slabzone, NULL, 0, wait);
+ slab = zone_alloc_item(slabzone, NULL, domain, wait);
if (slab == NULL)
return (NULL);
- addr = kmem_malloc(kernel_arena, size, wait);
+ if (domain == UMA_ANYDOMAIN)
+ addr = kmem_malloc(kernel_arena, size, wait);
+ else
+ addr = kmem_malloc_domain(domain, size, wait);
if (addr != 0) {
vsetslab(addr, slab);
slab->us_data = (void *)addr;
@@ -3364,6 +3416,13 @@ uma_large_malloc(vm_size_t size, int wait)
}
return ((void *)addr);
+}
+
+void *
+uma_large_malloc(vm_size_t size, int wait)
+{
+
+ return uma_large_malloc_domain(size, UMA_ANYDOMAIN, wait);
}
void
Modified: user/jeff/numa/sys/vm/uma_int.h
==============================================================================
--- user/jeff/numa/sys/vm/uma_int.h Wed Dec 20 01:03:34 2017 (r327013)
+++ user/jeff/numa/sys/vm/uma_int.h Wed Dec 20 04:02:58 2017 (r327014)
@@ -386,6 +386,7 @@ zone_first_keg(uma_zone_t zone)
/* Internal prototypes */
static __inline uma_slab_t hash_sfind(struct uma_hash *hash, uint8_t *data);
void *uma_large_malloc(vm_size_t size, int wait);
+void *uma_large_malloc_domain(vm_size_t size, int domain, int wait);
void uma_large_free(uma_slab_t slab);
/* Lock Macros */
More information about the svn-src-user
mailing list