svn commit: r326077 - in user/jeff/numa: lib/libmemstat sys/amd64/amd64 sys/arm64/arm64 sys/i386/i386 sys/kern sys/mips/mips sys/powerpc/aim sys/powerpc/powerpc sys/riscv/riscv sys/sparc64/sparc64 ...
Jeff Roberson
jeff at FreeBSD.org
Tue Nov 21 21:18:07 UTC 2017
Author: jeff
Date: Tue Nov 21 21:18:04 2017
New Revision: 326077
URL: https://svnweb.freebsd.org/changeset/base/326077
Log:
NUMAify UMA
This is a slightly refactored version from the original projects/numa branch
with better iterators and cleaner integration with the vm_page_alloc and
kmem_*alloc layers.
Modified:
user/jeff/numa/lib/libmemstat/memstat_uma.c
user/jeff/numa/sys/amd64/amd64/uma_machdep.c
user/jeff/numa/sys/arm64/arm64/uma_machdep.c
user/jeff/numa/sys/i386/i386/pmap.c
user/jeff/numa/sys/kern/kern_mbuf.c
user/jeff/numa/sys/kern/subr_busdma_bufalloc.c
user/jeff/numa/sys/kern/subr_vmem.c
user/jeff/numa/sys/kern/vfs_bio.c
user/jeff/numa/sys/mips/mips/uma_machdep.c
user/jeff/numa/sys/powerpc/aim/mmu_oea64.c
user/jeff/numa/sys/powerpc/aim/slb.c
user/jeff/numa/sys/powerpc/powerpc/uma_machdep.c
user/jeff/numa/sys/riscv/riscv/uma_machdep.c
user/jeff/numa/sys/sparc64/sparc64/vm_machdep.c
user/jeff/numa/sys/sys/busdma_bufalloc.h
user/jeff/numa/sys/vm/uma.h
user/jeff/numa/sys/vm/uma_core.c
user/jeff/numa/sys/vm/uma_int.h
Modified: user/jeff/numa/lib/libmemstat/memstat_uma.c
==============================================================================
--- user/jeff/numa/lib/libmemstat/memstat_uma.c Tue Nov 21 21:16:48 2017 (r326076)
+++ user/jeff/numa/lib/libmemstat/memstat_uma.c Tue Nov 21 21:18:04 2017 (r326077)
@@ -53,6 +53,8 @@ static struct nlist namelist[] = {
{ .n_name = "_mp_maxid" },
#define X_ALL_CPUS 2
{ .n_name = "_all_cpus" },
+#define X_VM_NDOMAINS 3
+ { .n_name = "_vm_ndomains" },
{ .n_name = "" },
};
@@ -295,11 +297,12 @@ memstat_kvm_uma(struct memory_type_list *list, void *k
{
LIST_HEAD(, uma_keg) uma_kegs;
struct memory_type *mtp;
+ struct uma_zone_domain uzd;
struct uma_bucket *ubp, ub;
struct uma_cache *ucp, *ucp_array;
struct uma_zone *uzp, uz;
struct uma_keg *kzp, kz;
- int hint_dontsearch, i, mp_maxid, ret;
+ int hint_dontsearch, i, mp_maxid, ndomains, ret;
char name[MEMTYPE_MAXNAME];
cpuset_t all_cpus;
long cpusetsize;
@@ -321,6 +324,12 @@ memstat_kvm_uma(struct memory_type_list *list, void *k
list->mtl_error = ret;
return (-1);
}
+ ret = kread_symbol(kvm, X_VM_NDOMAINS, &ndomains,
+ sizeof(ndomains), 0);
+ if (ret != 0) {
+ list->mtl_error = ret;
+ return (-1);
+ }
ret = kread_symbol(kvm, X_UMA_KEGS, &uma_kegs, sizeof(uma_kegs), 0);
if (ret != 0) {
list->mtl_error = ret;
@@ -445,10 +454,17 @@ skip_percpu:
kz.uk_ipers;
mtp->mt_byteslimit = mtp->mt_countlimit * mtp->mt_size;
mtp->mt_count = mtp->mt_numallocs - mtp->mt_numfrees;
- for (ubp = LIST_FIRST(&uz.uz_buckets); ubp !=
- NULL; ubp = LIST_NEXT(&ub, ub_link)) {
- ret = kread(kvm, ubp, &ub, sizeof(ub), 0);
- mtp->mt_zonefree += ub.ub_cnt;
+ for (i = 0; i < ndomains; i++) {
+ ret = kread(kvm, &uz.uz_domain[i], &uzd,
+ sizeof(uzd), 0);
+ for (ubp =
+ LIST_FIRST(&uzd.uzd_buckets);
+ ubp != NULL;
+ ubp = LIST_NEXT(&ub, ub_link)) {
+ ret = kread(kvm, ubp, &ub,
+ sizeof(ub), 0);
+ mtp->mt_zonefree += ub.ub_cnt;
+ }
}
if (!((kz.uk_flags & UMA_ZONE_SECONDARY) &&
LIST_FIRST(&kz.uk_zones) != uzp)) {
Modified: user/jeff/numa/sys/amd64/amd64/uma_machdep.c
==============================================================================
--- user/jeff/numa/sys/amd64/amd64/uma_machdep.c Tue Nov 21 21:16:48 2017 (r326076)
+++ user/jeff/numa/sys/amd64/amd64/uma_machdep.c Tue Nov 21 21:18:04 2017 (r326077)
@@ -42,14 +42,15 @@ __FBSDID("$FreeBSD$");
#include <machine/vmparam.h>
void *
-uma_small_alloc(uma_zone_t zone, vm_size_t bytes, u_int8_t *flags, int wait)
+uma_small_alloc(uma_zone_t zone, vm_size_t bytes, int domain, u_int8_t *flags,
+ int wait)
{
vm_page_t m;
vm_paddr_t pa;
void *va;
*flags = UMA_SLAB_PRIV;
- m = vm_page_alloc(NULL, 0,
+ m = vm_page_alloc_domain(NULL, 0, domain,
malloc2vm_flags(wait) | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED);
if (m == NULL)
return (NULL);
Modified: user/jeff/numa/sys/arm64/arm64/uma_machdep.c
==============================================================================
--- user/jeff/numa/sys/arm64/arm64/uma_machdep.c Tue Nov 21 21:16:48 2017 (r326076)
+++ user/jeff/numa/sys/arm64/arm64/uma_machdep.c Tue Nov 21 21:18:04 2017 (r326077)
@@ -42,14 +42,15 @@ __FBSDID("$FreeBSD$");
#include <machine/vmparam.h>
void *
-uma_small_alloc(uma_zone_t zone, vm_size_t bytes, u_int8_t *flags, int wait)
+uma_small_alloc(uma_zone_t zone, vm_size_t bytes, int domain, u_int8_t *flags,
+ int wait)
{
vm_page_t m;
vm_paddr_t pa;
void *va;
*flags = UMA_SLAB_PRIV;
- m = vm_page_alloc(NULL, 0,
+ m = vm_page_alloc_domain(NULL, 0, domain,
malloc2vm_flags(wait) | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED);
if (m == NULL)
return (NULL);
Modified: user/jeff/numa/sys/i386/i386/pmap.c
==============================================================================
--- user/jeff/numa/sys/i386/i386/pmap.c Tue Nov 21 21:16:48 2017 (r326076)
+++ user/jeff/numa/sys/i386/i386/pmap.c Tue Nov 21 21:18:04 2017 (r326077)
@@ -342,8 +342,8 @@ static pt_entry_t *pmap_pte_quick(pmap_t pmap, vm_offs
static void pmap_pte_release(pt_entry_t *pte);
static int pmap_unuse_pt(pmap_t, vm_offset_t, struct spglist *);
#if defined(PAE) || defined(PAE_TABLES)
-static void *pmap_pdpt_allocf(uma_zone_t zone, vm_size_t bytes, uint8_t *flags,
- int wait);
+static void *pmap_pdpt_allocf(uma_zone_t zone, vm_size_t bytes, int domain,
+ uint8_t *flags, int wait);
#endif
static void pmap_set_pg(void);
@@ -703,12 +703,13 @@ pmap_page_init(vm_page_t m)
#if defined(PAE) || defined(PAE_TABLES)
static void *
-pmap_pdpt_allocf(uma_zone_t zone, vm_size_t bytes, uint8_t *flags, int wait)
+pmap_pdpt_allocf(uma_zone_t zone, vm_size_t bytes, int domain, uint8_t *flags,
+ int wait)
{
/* Inform UMA that this allocator uses kernel_map/object. */
*flags = UMA_SLAB_KERNEL;
- return ((void *)kmem_alloc_contig(kernel_arena, bytes, wait, 0x0ULL,
+ return ((void *)kmem_alloc_contig_domain(domain, bytes, wait, 0x0ULL,
0xffffffffULL, 1, 0, VM_MEMATTR_DEFAULT));
}
#endif
Modified: user/jeff/numa/sys/kern/kern_mbuf.c
==============================================================================
--- user/jeff/numa/sys/kern/kern_mbuf.c Tue Nov 21 21:16:48 2017 (r326076)
+++ user/jeff/numa/sys/kern/kern_mbuf.c Tue Nov 21 21:18:04 2017 (r326077)
@@ -281,7 +281,7 @@ static void mb_dtor_pack(void *, int, void *);
static int mb_zinit_pack(void *, int, int);
static void mb_zfini_pack(void *, int);
static void mb_reclaim(uma_zone_t, int);
-static void *mbuf_jumbo_alloc(uma_zone_t, vm_size_t, uint8_t *, int);
+static void *mbuf_jumbo_alloc(uma_zone_t, vm_size_t, int, uint8_t *, int);
/* Ensure that MSIZE is a power of 2. */
CTASSERT((((MSIZE - 1) ^ MSIZE) + 1) >> 1 == MSIZE);
@@ -384,12 +384,13 @@ SYSINIT(mbuf, SI_SUB_MBUF, SI_ORDER_FIRST, mbuf_init,
* pages.
*/
static void *
-mbuf_jumbo_alloc(uma_zone_t zone, vm_size_t bytes, uint8_t *flags, int wait)
+mbuf_jumbo_alloc(uma_zone_t zone, vm_size_t bytes, int domain, uint8_t *flags,
+ int wait)
{
/* Inform UMA that this allocator uses kernel_map/object. */
*flags = UMA_SLAB_KERNEL;
- return ((void *)kmem_alloc_contig(kernel_arena, bytes, wait,
+ return ((void *)kmem_alloc_contig_domain(domain, bytes, wait,
(vm_paddr_t)0, ~(vm_paddr_t)0, 1, 0, VM_MEMATTR_DEFAULT));
}
Modified: user/jeff/numa/sys/kern/subr_busdma_bufalloc.c
==============================================================================
--- user/jeff/numa/sys/kern/subr_busdma_bufalloc.c Tue Nov 21 21:16:48 2017 (r326076)
+++ user/jeff/numa/sys/kern/subr_busdma_bufalloc.c Tue Nov 21 21:18:04 2017 (r326077)
@@ -147,7 +147,7 @@ busdma_bufalloc_findzone(busdma_bufalloc_t ba, bus_siz
}
void *
-busdma_bufalloc_alloc_uncacheable(uma_zone_t zone, vm_size_t size,
+busdma_bufalloc_alloc_uncacheable(uma_zone_t zone, vm_size_t size, int domain,
uint8_t *pflag, int wait)
{
#ifdef VM_MEMATTR_UNCACHEABLE
@@ -155,7 +155,7 @@ busdma_bufalloc_alloc_uncacheable(uma_zone_t zone, vm_
/* Inform UMA that this allocator uses kernel_arena/object. */
*pflag = UMA_SLAB_KERNEL;
- return ((void *)kmem_alloc_attr(kernel_arena, size, wait, 0,
+ return ((void *)kmem_alloc_attr_domain(domain, size, wait, 0,
BUS_SPACE_MAXADDR, VM_MEMATTR_UNCACHEABLE));
#else
Modified: user/jeff/numa/sys/kern/subr_vmem.c
==============================================================================
--- user/jeff/numa/sys/kern/subr_vmem.c Tue Nov 21 21:16:48 2017 (r326076)
+++ user/jeff/numa/sys/kern/subr_vmem.c Tue Nov 21 21:18:04 2017 (r326077)
@@ -497,7 +497,7 @@ bt_insfree(vmem_t *vm, bt_t *bt)
* Import from the arena into the quantum cache in UMA.
*/
static int
-qc_import(void *arg, void **store, int cnt, int flags)
+qc_import(void *arg, void **store, int cnt, int domain, int flags)
{
qcache_t *qc;
vmem_addr_t addr;
@@ -611,7 +611,8 @@ static struct mtx_padalign __exclusive_cache_line vmem
* we are really out of KVA.
*/
static void *
-vmem_bt_alloc(uma_zone_t zone, vm_size_t bytes, uint8_t *pflag, int wait)
+vmem_bt_alloc(uma_zone_t zone, vm_size_t bytes, int domain, uint8_t *pflag,
+ int wait)
{
vmem_addr_t addr;
@@ -625,7 +626,7 @@ vmem_bt_alloc(uma_zone_t zone, vm_size_t bytes, uint8_
if (vmem_xalloc(kernel_arena, bytes, 0, 0, 0, VMEM_ADDR_MIN,
VMEM_ADDR_MAX, M_NOWAIT | M_NOVM | M_USE_RESERVE | M_BESTFIT,
&addr) == 0) {
- if (kmem_back(kernel_object, addr, bytes,
+ if (kmem_back_domain(domain, kernel_object, addr, bytes,
M_NOWAIT | M_USE_RESERVE) == 0) {
mtx_unlock(&vmem_bt_lock);
return ((void *)addr);
Modified: user/jeff/numa/sys/kern/vfs_bio.c
==============================================================================
--- user/jeff/numa/sys/kern/vfs_bio.c Tue Nov 21 21:16:48 2017 (r326076)
+++ user/jeff/numa/sys/kern/vfs_bio.c Tue Nov 21 21:18:04 2017 (r326077)
@@ -131,7 +131,7 @@ static __inline void bd_wakeup(void);
static int sysctl_runningspace(SYSCTL_HANDLER_ARGS);
static void bufkva_reclaim(vmem_t *, int);
static void bufkva_free(struct buf *);
-static int buf_import(void *, void **, int, int);
+static int buf_import(void *, void **, int, int, int);
static void buf_release(void *, void **, int);
static void maxbcachebuf_adjust(void);
@@ -1417,7 +1417,7 @@ buf_free(struct buf *bp)
* only as a per-cpu cache of bufs still maintained on a global list.
*/
static int
-buf_import(void *arg, void **store, int cnt, int flags)
+buf_import(void *arg, void **store, int cnt, int domain, int flags)
{
struct buf *bp;
int i;
Modified: user/jeff/numa/sys/mips/mips/uma_machdep.c
==============================================================================
--- user/jeff/numa/sys/mips/mips/uma_machdep.c Tue Nov 21 21:16:48 2017 (r326076)
+++ user/jeff/numa/sys/mips/mips/uma_machdep.c Tue Nov 21 21:18:04 2017 (r326077)
@@ -42,7 +42,8 @@ __FBSDID("$FreeBSD$");
#include <machine/vmparam.h>
void *
-uma_small_alloc(uma_zone_t zone, vm_size_t bytes, u_int8_t *flags, int wait)
+uma_small_alloc(uma_zone_t zone, vm_size_t bytes, int domain, u_int8_t *flags,
+ int wait)
{
vm_paddr_t pa;
vm_page_t m;
@@ -57,7 +58,8 @@ uma_small_alloc(uma_zone_t zone, vm_size_t bytes, u_in
#endif
for (;;) {
- m = vm_page_alloc_freelist(VM_FREELIST_DIRECT, pflags);
+ m = vm_page_alloc_freelist_domain(domain, VM_FREELIST_DIRECT,
+ pflags);
#ifndef __mips_n64
if (m == NULL && vm_page_reclaim_contig(pflags, 1,
0, MIPS_KSEG0_LARGEST_PHYS, PAGE_SIZE, 0))
Modified: user/jeff/numa/sys/powerpc/aim/mmu_oea64.c
==============================================================================
--- user/jeff/numa/sys/powerpc/aim/mmu_oea64.c Tue Nov 21 21:16:48 2017 (r326076)
+++ user/jeff/numa/sys/powerpc/aim/mmu_oea64.c Tue Nov 21 21:18:04 2017 (r326077)
@@ -1500,8 +1500,8 @@ retry:
static mmu_t installed_mmu;
static void *
-moea64_uma_page_alloc(uma_zone_t zone, vm_size_t bytes, uint8_t *flags,
- int wait)
+moea64_uma_page_alloc(uma_zone_t zone, vm_size_t bytes, int domain,
+ uint8_t *flags, int wait)
{
struct pvo_entry *pvo;
vm_offset_t va;
@@ -1518,7 +1518,7 @@ moea64_uma_page_alloc(uma_zone_t zone, vm_size_t bytes
*flags = UMA_SLAB_PRIV;
needed_lock = !PMAP_LOCKED(kernel_pmap);
- m = vm_page_alloc(NULL, 0,
+ m = vm_page_alloc_domain(NULL, 0, domain,
malloc2vm_flags(wait) | VM_ALLOC_WIRED | VM_ALLOC_NOOBJ);
if (m == NULL)
return (NULL);
Modified: user/jeff/numa/sys/powerpc/aim/slb.c
==============================================================================
--- user/jeff/numa/sys/powerpc/aim/slb.c Tue Nov 21 21:16:48 2017 (r326076)
+++ user/jeff/numa/sys/powerpc/aim/slb.c Tue Nov 21 21:18:04 2017 (r326077)
@@ -478,7 +478,8 @@ slb_insert_user(pmap_t pm, struct slb *slb)
}
static void *
-slb_uma_real_alloc(uma_zone_t zone, vm_size_t bytes, u_int8_t *flags, int wait)
+slb_uma_real_alloc(uma_zone_t zone, vm_size_t bytes, int domain,
+ u_int8_t *flags, int wait)
{
static vm_offset_t realmax = 0;
void *va;
@@ -488,7 +489,7 @@ slb_uma_real_alloc(uma_zone_t zone, vm_size_t bytes, u
realmax = platform_real_maxaddr();
*flags = UMA_SLAB_PRIV;
- m = vm_page_alloc_contig(NULL, 0,
+ m = vm_page_alloc_contig_domain(NULL, 0, domain,
malloc2vm_flags(wait) | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED,
1, 0, realmax, PAGE_SIZE, PAGE_SIZE, VM_MEMATTR_DEFAULT);
if (m == NULL)
Modified: user/jeff/numa/sys/powerpc/powerpc/uma_machdep.c
==============================================================================
--- user/jeff/numa/sys/powerpc/powerpc/uma_machdep.c Tue Nov 21 21:16:48 2017 (r326076)
+++ user/jeff/numa/sys/powerpc/powerpc/uma_machdep.c Tue Nov 21 21:18:04 2017 (r326077)
@@ -51,7 +51,8 @@ SYSCTL_INT(_hw, OID_AUTO, uma_mdpages, CTLFLAG_RD, &hw
"UMA MD pages in use");
void *
-uma_small_alloc(uma_zone_t zone, vm_size_t bytes, u_int8_t *flags, int wait)
+uma_small_alloc(uma_zone_t zone, vm_size_t bytes, int domain, u_int8_t *flags,
+ int wait)
{
void *va;
vm_paddr_t pa;
@@ -59,7 +60,7 @@ uma_small_alloc(uma_zone_t zone, vm_size_t bytes, u_in
*flags = UMA_SLAB_PRIV;
- m = vm_page_alloc(NULL, 0,
+ m = vm_page_alloc_domain(NULL, 0, domain,
malloc2vm_flags(wait) | VM_ALLOC_WIRED | VM_ALLOC_NOOBJ);
if (m == NULL)
return (NULL);
Modified: user/jeff/numa/sys/riscv/riscv/uma_machdep.c
==============================================================================
--- user/jeff/numa/sys/riscv/riscv/uma_machdep.c Tue Nov 21 21:16:48 2017 (r326076)
+++ user/jeff/numa/sys/riscv/riscv/uma_machdep.c Tue Nov 21 21:18:04 2017 (r326077)
@@ -41,7 +41,8 @@ __FBSDID("$FreeBSD$");
#include <machine/vmparam.h>
void *
-uma_small_alloc(uma_zone_t zone, vm_size_t bytes, u_int8_t *flags, int wait)
+uma_small_alloc(uma_zone_t zone, vm_size_t bytes, int domain, u_int8_t *flags,
+ int wait)
{
panic("uma_small_alloc");
Modified: user/jeff/numa/sys/sparc64/sparc64/vm_machdep.c
==============================================================================
--- user/jeff/numa/sys/sparc64/sparc64/vm_machdep.c Tue Nov 21 21:16:48 2017 (r326076)
+++ user/jeff/numa/sys/sparc64/sparc64/vm_machdep.c Tue Nov 21 21:18:04 2017 (r326077)
@@ -390,7 +390,8 @@ swi_vm(void *v)
}
void *
-uma_small_alloc(uma_zone_t zone, vm_size_t bytes, u_int8_t *flags, int wait)
+uma_small_alloc(uma_zone_t zone, vm_size_t bytes, int domain, u_int8_t *flags,
+ int wait)
{
vm_paddr_t pa;
vm_page_t m;
@@ -400,7 +401,7 @@ uma_small_alloc(uma_zone_t zone, vm_size_t bytes, u_in
*flags = UMA_SLAB_PRIV;
- m = vm_page_alloc(NULL, 0,
+ m = vm_page_alloc_domain(NULL, 0, domain,
malloc2vm_flags(wait) | VM_ALLOC_WIRED | VM_ALLOC_NOOBJ);
if (m == NULL)
return (NULL);
Modified: user/jeff/numa/sys/sys/busdma_bufalloc.h
==============================================================================
--- user/jeff/numa/sys/sys/busdma_bufalloc.h Tue Nov 21 21:16:48 2017 (r326076)
+++ user/jeff/numa/sys/sys/busdma_bufalloc.h Tue Nov 21 21:18:04 2017 (r326077)
@@ -111,7 +111,7 @@ struct busdma_bufzone * busdma_bufalloc_findzone(busdm
* you can probably use these when you need uncacheable buffers.
*/
void * busdma_bufalloc_alloc_uncacheable(uma_zone_t zone, vm_size_t size,
- uint8_t *pflag, int wait);
+ int domain, uint8_t *pflag, int wait);
void busdma_bufalloc_free_uncacheable(void *item, vm_size_t size,
uint8_t pflag);
Modified: user/jeff/numa/sys/vm/uma.h
==============================================================================
--- user/jeff/numa/sys/vm/uma.h Tue Nov 21 21:16:48 2017 (r326076)
+++ user/jeff/numa/sys/vm/uma.h Tue Nov 21 21:18:04 2017 (r326077)
@@ -45,6 +45,7 @@
/* Types and type defs */
struct uma_zone;
+struct vm_domain_iterator;
/* Opaque type used as a handle to the zone */
typedef struct uma_zone * uma_zone_t;
@@ -126,7 +127,8 @@ typedef void (*uma_fini)(void *mem, int size);
/*
* Import new memory into a cache zone.
*/
-typedef int (*uma_import)(void *arg, void **store, int count, int flags);
+typedef int (*uma_import)(void *arg, void **store, int count, int domain,
+ int flags);
/*
* Free memory from a cache zone.
@@ -279,6 +281,10 @@ uma_zone_t uma_zcache_create(char *name, int size, uma
* Allocates mp_maxid + 1 slabs sized to
* sizeof(struct pcpu).
*/
+#define UMA_ZONE_NUMA 0x10000 /*
+ * NUMA aware Zone. Implements a best
+ * effort first-touch policy.
+ */
/*
* These flags are shared between the keg and zone. In zones wishing to add
@@ -371,25 +377,21 @@ uma_zfree(uma_zone_t zone, void *item)
void uma_zwait(uma_zone_t zone);
/*
- * XXX The rest of the prototypes in this header are h0h0 magic for the VM.
- * If you think you need to use it for a normal zone you're probably incorrect.
- */
-
-/*
* Backend page supplier routines
*
* Arguments:
* zone The zone that is requesting pages.
* size The number of bytes being requested.
* pflag Flags for these memory pages, see below.
+ * domain The NUMA domain that we prefer for this allocation.
* wait Indicates our willingness to block.
*
* Returns:
* A pointer to the allocated memory or NULL on failure.
*/
-typedef void *(*uma_alloc)(uma_zone_t zone, vm_size_t size, uint8_t *pflag,
- int wait);
+typedef void *(*uma_alloc)(uma_zone_t zone, vm_size_t size, int domain,
+ uint8_t *pflag, int wait);
/*
* Backend page free routines
@@ -403,8 +405,6 @@ typedef void *(*uma_alloc)(uma_zone_t zone, vm_size_t
* None
*/
typedef void (*uma_free)(void *item, vm_size_t size, uint8_t pflag);
-
-
/*
* Sets up the uma allocator. (Called by vm_mem_init)
Modified: user/jeff/numa/sys/vm/uma_core.c
==============================================================================
--- user/jeff/numa/sys/vm/uma_core.c Tue Nov 21 21:16:48 2017 (r326076)
+++ user/jeff/numa/sys/vm/uma_core.c Tue Nov 21 21:18:04 2017 (r326077)
@@ -76,10 +76,12 @@ __FBSDID("$FreeBSD$");
#include <sys/vmmeter.h>
#include <vm/vm.h>
+#include <vm/vm_domain.h>
#include <vm/vm_object.h>
#include <vm/vm_page.h>
#include <vm/vm_pageout.h>
#include <vm/vm_param.h>
+#include <vm/vm_phys.h>
#include <vm/vm_map.h>
#include <vm/vm_kern.h>
#include <vm/vm_extern.h>
@@ -94,17 +96,12 @@ __FBSDID("$FreeBSD$");
#endif
/*
- * This is the zone and keg from which all zones are spawned. The idea is that
- * even the zone & keg heads are allocated from the allocator, so we use the
- * bss section to bootstrap us.
+ * This is the zone and keg from which all zones are spawned.
*/
-static struct uma_keg masterkeg;
-static struct uma_zone masterzone_k;
-static struct uma_zone masterzone_z;
-static uma_zone_t kegs = &masterzone_k;
-static uma_zone_t zones = &masterzone_z;
+static uma_zone_t kegs;
+static uma_zone_t zones;
-/* This is the zone from which all of uma_slab_t's are allocated. */
+/* This is the zone from which all offpage uma_slab_ts are allocated. */
static uma_zone_t slabzone;
/*
@@ -225,11 +222,11 @@ enum zfreeskip { SKIP_NONE = 0, SKIP_DTOR, SKIP_FINI }
/* Prototypes.. */
-static void *noobj_alloc(uma_zone_t, vm_size_t, uint8_t *, int);
-static void *page_alloc(uma_zone_t, vm_size_t, uint8_t *, int);
-static void *startup_alloc(uma_zone_t, vm_size_t, uint8_t *, int);
+static void *noobj_alloc(uma_zone_t, vm_size_t, int, uint8_t *, int);
+static void *page_alloc(uma_zone_t, vm_size_t, int, uint8_t *, int);
+static void *startup_alloc(uma_zone_t, vm_size_t, int, uint8_t *, int);
static void page_free(void *, vm_size_t, uint8_t);
-static uma_slab_t keg_alloc_slab(uma_keg_t, uma_zone_t, int);
+static uma_slab_t keg_alloc_slab(uma_keg_t, uma_zone_t, int, int);
static void cache_drain(uma_zone_t);
static void bucket_drain(uma_zone_t, uma_bucket_t);
static void bucket_cache_drain(uma_zone_t zone);
@@ -247,23 +244,23 @@ static int hash_expand(struct uma_hash *, struct uma_h
static void hash_free(struct uma_hash *hash);
static void uma_timeout(void *);
static void uma_startup3(void);
-static void *zone_alloc_item(uma_zone_t, void *, int);
+static void *zone_alloc_item(uma_zone_t, void *, int, int);
static void zone_free_item(uma_zone_t, void *, void *, enum zfreeskip);
static void bucket_enable(void);
static void bucket_init(void);
static uma_bucket_t bucket_alloc(uma_zone_t zone, void *, int);
static void bucket_free(uma_zone_t zone, uma_bucket_t, void *);
static void bucket_zone_drain(void);
-static uma_bucket_t zone_alloc_bucket(uma_zone_t zone, void *, int flags);
-static uma_slab_t zone_fetch_slab(uma_zone_t zone, uma_keg_t last, int flags);
-static uma_slab_t zone_fetch_slab_multi(uma_zone_t zone, uma_keg_t last, int flags);
+static uma_bucket_t zone_alloc_bucket(uma_zone_t, void *, int, int);
+static uma_slab_t zone_fetch_slab(uma_zone_t, uma_keg_t, int, int);
+static uma_slab_t zone_fetch_slab_multi(uma_zone_t, uma_keg_t, int, int);
static void *slab_alloc_item(uma_keg_t keg, uma_slab_t slab);
static void slab_free_item(uma_keg_t keg, uma_slab_t slab, void *item);
static uma_keg_t uma_kcreate(uma_zone_t zone, size_t size, uma_init uminit,
uma_fini fini, int align, uint32_t flags);
-static int zone_import(uma_zone_t zone, void **bucket, int max, int flags);
-static void zone_release(uma_zone_t zone, void **bucket, int cnt);
-static void uma_zero_item(void *item, uma_zone_t zone);
+static int zone_import(uma_zone_t, void **, int, int, int);
+static void zone_release(uma_zone_t, void **, int);
+static void uma_zero_item(void *, uma_zone_t);
void uma_print_zone(uma_zone_t);
void uma_print_stats(void);
@@ -329,7 +326,7 @@ bucket_init(void)
size += sizeof(void *) * ubz->ubz_entries;
ubz->ubz_zone = uma_zcreate(ubz->ubz_name, size,
NULL, NULL, NULL, NULL, UMA_ALIGN_PTR,
- UMA_ZONE_MTXCLASS | UMA_ZFLAG_BUCKET);
+ UMA_ZONE_MTXCLASS | UMA_ZFLAG_BUCKET | UMA_ZONE_NUMA);
}
}
@@ -566,7 +563,7 @@ hash_alloc(struct uma_hash *hash)
M_UMAHASH, M_NOWAIT);
} else {
alloc = sizeof(hash->uh_slab_hash[0]) * UMA_HASH_SIZE_INIT;
- hash->uh_slab_hash = zone_alloc_item(hashzone, NULL,
+ hash->uh_slab_hash = zone_alloc_item(hashzone, NULL, 0,
M_WAITOK);
hash->uh_hashsize = UMA_HASH_SIZE_INIT;
}
@@ -733,6 +730,7 @@ cache_drain_safe_cpu(uma_zone_t zone)
{
uma_cache_t cache;
uma_bucket_t b1, b2;
+ int domain;
if (zone->uz_flags & UMA_ZFLAG_INTERNAL)
return;
@@ -740,10 +738,14 @@ cache_drain_safe_cpu(uma_zone_t zone)
b1 = b2 = NULL;
ZONE_LOCK(zone);
critical_enter();
+ if (zone->uz_flags & UMA_ZONE_NUMA)
+ domain = PCPU_GET(domain);
+ else
+ domain = 0;
cache = &zone->uz_cpu[curcpu];
if (cache->uc_allocbucket) {
if (cache->uc_allocbucket->ub_cnt != 0)
- LIST_INSERT_HEAD(&zone->uz_buckets,
+ LIST_INSERT_HEAD(&zone->uz_domain[domain].uzd_buckets,
cache->uc_allocbucket, ub_link);
else
b1 = cache->uc_allocbucket;
@@ -751,7 +753,7 @@ cache_drain_safe_cpu(uma_zone_t zone)
}
if (cache->uc_freebucket) {
if (cache->uc_freebucket->ub_cnt != 0)
- LIST_INSERT_HEAD(&zone->uz_buckets,
+ LIST_INSERT_HEAD(&zone->uz_domain[domain].uzd_buckets,
cache->uc_freebucket, ub_link);
else
b2 = cache->uc_freebucket;
@@ -806,18 +808,22 @@ cache_drain_safe(uma_zone_t zone)
static void
bucket_cache_drain(uma_zone_t zone)
{
+ uma_zone_domain_t zdom;
uma_bucket_t bucket;
+ int i;
/*
- * Drain the bucket queues and free the buckets, we just keep two per
- * cpu (alloc/free).
+ * Drain the bucket queues and free the buckets.
*/
- while ((bucket = LIST_FIRST(&zone->uz_buckets)) != NULL) {
- LIST_REMOVE(bucket, ub_link);
- ZONE_UNLOCK(zone);
- bucket_drain(zone, bucket);
- bucket_free(zone, bucket, NULL);
- ZONE_LOCK(zone);
+ for (i = 0; i < vm_ndomains; i++) {
+ zdom = &zone->uz_domain[i];
+ while ((bucket = LIST_FIRST(&zdom->uzd_buckets)) != NULL) {
+ LIST_REMOVE(bucket, ub_link);
+ ZONE_UNLOCK(zone);
+ bucket_drain(zone, bucket);
+ bucket_free(zone, bucket, NULL);
+ ZONE_LOCK(zone);
+ }
}
/*
@@ -862,7 +868,9 @@ static void
keg_drain(uma_keg_t keg)
{
struct slabhead freeslabs = { 0 };
+ uma_domain_t dom;
uma_slab_t slab, tmp;
+ int i;
/*
* We don't want to take pages from statically allocated kegs at this
@@ -877,20 +885,25 @@ keg_drain(uma_keg_t keg)
if (keg->uk_free == 0)
goto finished;
- LIST_FOREACH_SAFE(slab, &keg->uk_free_slab, us_link, tmp) {
- /* We have nowhere to free these to. */
- if (slab->us_flags & UMA_SLAB_BOOT)
- continue;
+ for (i = 0; i < vm_ndomains; i++) {
+ dom = &keg->uk_domain[i];
+ LIST_FOREACH_SAFE(slab, &dom->ud_free_slab, us_link, tmp) {
+ /* We have nowhere to free these to. */
+ if (slab->us_flags & UMA_SLAB_BOOT)
+ continue;
- LIST_REMOVE(slab, us_link);
- keg->uk_pages -= keg->uk_ppera;
- keg->uk_free -= keg->uk_ipers;
+ LIST_REMOVE(slab, us_link);
+ keg->uk_pages -= keg->uk_ppera;
+ keg->uk_free -= keg->uk_ipers;
- if (keg->uk_flags & UMA_ZONE_HASH)
- UMA_HASH_REMOVE(&keg->uk_hash, slab, slab->us_data);
+ if (keg->uk_flags & UMA_ZONE_HASH)
+ UMA_HASH_REMOVE(&keg->uk_hash, slab,
+ slab->us_data);
- SLIST_INSERT_HEAD(&freeslabs, slab, us_hlink);
+ SLIST_INSERT_HEAD(&freeslabs, slab, us_hlink);
+ }
}
+
finished:
KEG_UNLOCK(keg);
@@ -950,7 +963,7 @@ zone_drain(uma_zone_t zone)
* caller specified M_NOWAIT.
*/
static uma_slab_t
-keg_alloc_slab(uma_keg_t keg, uma_zone_t zone, int wait)
+keg_alloc_slab(uma_keg_t keg, uma_zone_t zone, int domain, int wait)
{
uma_alloc allocf;
uma_slab_t slab;
@@ -959,6 +972,8 @@ keg_alloc_slab(uma_keg_t keg, uma_zone_t zone, int wai
uint8_t flags;
int i;
+ KASSERT(domain >= 0 && domain < vm_ndomains,
+ ("keg_alloc_slab: domain %d out of range", domain));
mtx_assert(&keg->uk_lock, MA_OWNED);
slab = NULL;
mem = NULL;
@@ -968,7 +983,7 @@ keg_alloc_slab(uma_keg_t keg, uma_zone_t zone, int wai
size = keg->uk_ppera * PAGE_SIZE;
if (keg->uk_flags & UMA_ZONE_OFFPAGE) {
- slab = zone_alloc_item(keg->uk_slabzone, NULL, wait);
+ slab = zone_alloc_item(keg->uk_slabzone, NULL, domain, wait);
if (slab == NULL)
goto out;
}
@@ -989,7 +1004,7 @@ keg_alloc_slab(uma_keg_t keg, uma_zone_t zone, int wai
wait |= M_NODUMP;
/* zone is passed for legacy reasons. */
- mem = allocf(zone, size, &flags, wait);
+ mem = allocf(zone, size, domain, &flags, wait);
if (mem == NULL) {
if (keg->uk_flags & UMA_ZONE_OFFPAGE)
zone_free_item(keg->uk_slabzone, slab, NULL, SKIP_NONE);
@@ -1010,6 +1025,7 @@ keg_alloc_slab(uma_keg_t keg, uma_zone_t zone, int wai
slab->us_data = mem;
slab->us_freecount = keg->uk_ipers;
slab->us_flags = flags;
+ slab->us_domain = domain;
BIT_FILL(SLAB_SETSIZE, &slab->us_free);
#ifdef INVARIANTS
BIT_ZERO(SLAB_SETSIZE, &slab->us_debugfree);
@@ -1049,7 +1065,8 @@ out:
* the VM is ready.
*/
static void *
-startup_alloc(uma_zone_t zone, vm_size_t bytes, uint8_t *pflag, int wait)
+startup_alloc(uma_zone_t zone, vm_size_t bytes, int domain, uint8_t *pflag,
+ int wait)
{
uma_keg_t keg;
void *mem;
@@ -1082,7 +1099,7 @@ startup_alloc(uma_zone_t zone, vm_size_t bytes, uint8_
#else
keg->uk_allocf = page_alloc;
#endif
- return keg->uk_allocf(zone, bytes, pflag, wait);
+ return keg->uk_allocf(zone, bytes, domain, pflag, wait);
}
/*
@@ -1097,12 +1114,13 @@ startup_alloc(uma_zone_t zone, vm_size_t bytes, uint8_
* NULL if M_NOWAIT is set.
*/
static void *
-page_alloc(uma_zone_t zone, vm_size_t bytes, uint8_t *pflag, int wait)
+page_alloc(uma_zone_t zone, vm_size_t bytes, int domain, uint8_t *pflag,
+ int wait)
{
void *p; /* Returned page */
*pflag = UMA_SLAB_KERNEL;
- p = (void *) kmem_malloc(kernel_arena, bytes, wait);
+ p = (void *) kmem_malloc_domain(domain, bytes, wait);
return (p);
}
@@ -1119,7 +1137,8 @@ page_alloc(uma_zone_t zone, vm_size_t bytes, uint8_t *
* NULL if M_NOWAIT is set.
*/
static void *
-noobj_alloc(uma_zone_t zone, vm_size_t bytes, uint8_t *flags, int wait)
+noobj_alloc(uma_zone_t zone, vm_size_t bytes, int domain, uint8_t *flags,
+ int wait)
{
TAILQ_HEAD(, vm_page) alloctail;
u_long npages;
@@ -1132,7 +1151,7 @@ noobj_alloc(uma_zone_t zone, vm_size_t bytes, uint8_t
npages = howmany(bytes, PAGE_SIZE);
while (npages > 0) {
- p = vm_page_alloc(NULL, 0, VM_ALLOC_INTERRUPT |
+ p = vm_page_alloc_domain(NULL, 0, domain, VM_ALLOC_INTERRUPT |
VM_ALLOC_WIRED | VM_ALLOC_NOOBJ |
((wait & M_WAITOK) != 0 ? VM_ALLOC_WAITOK :
VM_ALLOC_NOWAIT));
@@ -1404,6 +1423,7 @@ keg_ctor(void *mem, int size, void *udata, int flags)
keg->uk_init = arg->uminit;
keg->uk_fini = arg->fini;
keg->uk_align = arg->align;
+ keg->uk_cursor = 0;
keg->uk_free = 0;
keg->uk_reserve = 0;
keg->uk_pages = 0;
@@ -1545,6 +1565,8 @@ zone_ctor(void *mem, int size, void *udata, int flags)
zone->uz_count_min = 0;
zone->uz_flags = 0;
zone->uz_warning = NULL;
+ /* The domain structures follow the cpu structures. */
+ zone->uz_domain = (struct uma_zone_domain *)&zone->uz_cpu[mp_ncpus];
timevalclear(&zone->uz_ratecheck);
keg = arg->keg;
@@ -1750,22 +1772,43 @@ void
uma_startup(void *mem, int npages)
{
struct uma_zctor_args args;
+ uma_keg_t masterkeg;
+ uintptr_t m;
+ int zsize;
+ int ksize;
rw_init(&uma_rwlock, "UMA lock");
+ ksize = sizeof(struct uma_keg) +
+ (sizeof(struct uma_domain) * vm_ndomains);
+ zsize = sizeof(struct uma_zone) +
+ (sizeof(struct uma_cache) * mp_ncpus) +
+ (sizeof(struct uma_zone_domain) * vm_ndomains);
+
+ /* Use bootpages memory for the zone of zones and zone of kegs. */
+ m = (uintptr_t)mem;
+ zones = (uma_zone_t)m;
+ m += roundup(zsize, CACHE_LINE_SIZE);
+ kegs = (uma_zone_t)m;
+ m += roundup(zsize, CACHE_LINE_SIZE);
+ masterkeg = (uma_keg_t)m;
+ m += roundup(ksize, CACHE_LINE_SIZE);
+ m = roundup(m, PAGE_SIZE);
+ npages -= (m - (uintptr_t)mem) / PAGE_SIZE;
+ mem = (void *)m;
+
/* "manually" create the initial zone */
memset(&args, 0, sizeof(args));
args.name = "UMA Kegs";
- args.size = sizeof(struct uma_keg);
+ args.size = ksize;
args.ctor = keg_ctor;
args.dtor = keg_dtor;
args.uminit = zero_init;
args.fini = NULL;
- args.keg = &masterkeg;
+ args.keg = masterkeg;
args.align = 32 - 1;
args.flags = UMA_ZFLAG_INTERNAL;
- /* The initial zone has no Per cpu queues so it's smaller */
- zone_ctor(kegs, sizeof(struct uma_zone), &args, M_WAITOK);
+ zone_ctor(kegs, zsize, &args, M_WAITOK);
mtx_init(&uma_boot_pages_mtx, "UMA boot pages", NULL, MTX_DEF);
bootmem = mem;
@@ -1773,7 +1816,8 @@ uma_startup(void *mem, int npages)
args.name = "UMA Zones";
args.size = sizeof(struct uma_zone) +
- (sizeof(struct uma_cache) * (mp_maxid + 1));
+ (sizeof(struct uma_cache) * (mp_maxid + 1)) +
+ (sizeof(struct uma_zone_domain) * vm_ndomains);
args.ctor = zone_ctor;
args.dtor = zone_dtor;
args.uminit = zero_init;
@@ -1781,8 +1825,7 @@ uma_startup(void *mem, int npages)
args.keg = NULL;
args.align = 32 - 1;
args.flags = UMA_ZFLAG_INTERNAL;
- /* The initial zone has no Per cpu queues so it's smaller */
- zone_ctor(zones, sizeof(struct uma_zone), &args, M_WAITOK);
+ zone_ctor(zones, zsize, &args, M_WAITOK);
/* Now make a zone for slab headers */
slabzone = uma_zcreate("UMA Slabs",
@@ -1834,7 +1877,7 @@ uma_kcreate(uma_zone_t zone, size_t size, uma_init umi
args.align = (align == UMA_ALIGN_CACHE) ? uma_align_cache : align;
args.flags = flags;
args.zone = zone;
- return (zone_alloc_item(kegs, &args, M_WAITOK));
+ return (zone_alloc_item(kegs, &args, 0, M_WAITOK));
}
/* See uma.h */
@@ -1891,7 +1934,7 @@ uma_zcreate(const char *name, size_t size, uma_ctor ct
sx_slock(&uma_drain_lock);
locked = true;
}
- res = zone_alloc_item(zones, &args, M_WAITOK);
+ res = zone_alloc_item(zones, &args, 0, M_WAITOK);
if (locked)
sx_sunlock(&uma_drain_lock);
return (res);
@@ -1926,7 +1969,7 @@ uma_zsecond_create(char *name, uma_ctor ctor, uma_dtor
locked = true;
}
/* XXX Attaches only one keg of potentially many. */
- res = zone_alloc_item(zones, &args, M_WAITOK);
+ res = zone_alloc_item(zones, &args, 0, M_WAITOK);
if (locked)
sx_sunlock(&uma_drain_lock);
return (res);
@@ -1953,7 +1996,7 @@ uma_zcache_create(char *name, int size, uma_ctor ctor,
args.align = 0;
args.flags = flags;
- return (zone_alloc_item(zones, &args, M_WAITOK));
+ return (zone_alloc_item(zones, &args, 0, M_WAITOK));
}
static void
@@ -2058,11 +2101,11 @@ uma_zwait(uma_zone_t zone)
void *
uma_zalloc_arg(uma_zone_t zone, void *udata, int flags)
{
- void *item;
- uma_cache_t cache;
+ uma_zone_domain_t zdom;
uma_bucket_t bucket;
- int lockfail;
- int cpu;
+ uma_cache_t cache;
+ void *item;
+ int cpu, domain, lockfail;
/* Enable entropy collection for RANDOM_ENABLE_UMA kernel option */
random_harvest_fast_uma(&zone, sizeof(zone), 1, RANDOM_UMA);
@@ -2159,6 +2202,11 @@ zalloc_start:
if (bucket != NULL)
bucket_free(zone, bucket, udata);
+ if (zone->uz_flags & UMA_ZONE_NUMA)
+ domain = PCPU_GET(domain);
+ else
+ domain = 0;
+
/* Short-circuit for zones without buckets and low memory. */
if (zone->uz_count == 0 || bucketdisable)
goto zalloc_item;
@@ -2199,7 +2247,8 @@ zalloc_start:
/*
* Check the zone's cache of buckets.
*/
- if ((bucket = LIST_FIRST(&zone->uz_buckets)) != NULL) {
+ zdom = &zone->uz_domain[domain];
+ if ((bucket = LIST_FIRST(&zdom->uzd_buckets)) != NULL) {
KASSERT(bucket->ub_cnt != 0,
("uma_zalloc_arg: Returning an empty bucket."));
@@ -2224,7 +2273,7 @@ zalloc_start:
* works we'll restart the allocation from the beginning and it
* will use the just filled bucket.
*/
- bucket = zone_alloc_bucket(zone, udata, flags);
+ bucket = zone_alloc_bucket(zone, udata, domain, flags);
CTR3(KTR_UMA, "uma_zalloc: zone %s(%p) bucket zone returned %p",
zone->uz_name, zone, bucket);
if (bucket != NULL) {
@@ -2237,10 +2286,12 @@ zalloc_start:
* initialized bucket to make this less likely or claim
* the memory directly.
*/
- if (cache->uc_allocbucket == NULL)
- cache->uc_allocbucket = bucket;
+ if (cache->uc_allocbucket != NULL ||
+ (zone->uz_flags & UMA_ZONE_NUMA &&
+ domain != PCPU_GET(domain)))
+ LIST_INSERT_HEAD(&zdom->uzd_buckets, bucket, ub_link);
else
- LIST_INSERT_HEAD(&zone->uz_buckets, bucket, ub_link);
+ cache->uc_allocbucket = bucket;
ZONE_UNLOCK(zone);
goto zalloc_start;
}
@@ -2249,38 +2300,79 @@ zalloc_start:
* We may not be able to get a bucket so return an actual item.
*/
zalloc_item:
- item = zone_alloc_item(zone, udata, flags);
+ item = zone_alloc_item(zone, udata, domain, flags);
return (item);
}
+/*
+ * Find a slab with some space. Prefer slabs that are partially used over those
+ * that are totally full. This helps to reduce fragmentation.
+ *
+ * If 'rr' is 1, search all domains starting from 'domain'. Otherwise check
+ * only 'domain'.
+ */
static uma_slab_t
-keg_fetch_slab(uma_keg_t keg, uma_zone_t zone, int flags)
+keg_first_slab(uma_keg_t keg, int domain, int rr)
{
+ uma_domain_t dom;
uma_slab_t slab;
- int reserve;
+ int start;
+ KASSERT(domain >= 0 && domain < vm_ndomains,
+ ("keg_first_slab: domain %d out of range", domain));
+
+ slab = NULL;
+ start = domain;
+ do {
+ dom = &keg->uk_domain[domain];
+ if (!LIST_EMPTY(&dom->ud_part_slab))
+ return (LIST_FIRST(&dom->ud_part_slab));
+ if (!LIST_EMPTY(&dom->ud_free_slab)) {
+ slab = LIST_FIRST(&dom->ud_free_slab);
+ LIST_REMOVE(slab, us_link);
+ LIST_INSERT_HEAD(&dom->ud_part_slab, slab, us_link);
+ return (slab);
+ }
+ if (rr)
+ domain = (domain + 1) % vm_ndomains;
+ } while (domain != start);
+
+ return (NULL);
+}
+
+static uma_slab_t
+keg_fetch_slab(uma_keg_t keg, uma_zone_t zone, int rdomain, int flags)
+{
+ uma_domain_t dom;
+ uma_slab_t slab;
+ int allocflags, domain, reserve, rr, start;
+
mtx_assert(&keg->uk_lock, MA_OWNED);
slab = NULL;
reserve = 0;
+ allocflags = flags;
if ((flags & M_USE_RESERVE) == 0)
reserve = keg->uk_reserve;
- for (;;) {
- /*
- * Find a slab with some space. Prefer slabs that are partially
- * used over those that are totally full. This helps to reduce
- * fragmentation.
- */
- if (keg->uk_free > reserve) {
- if (!LIST_EMPTY(&keg->uk_part_slab)) {
- slab = LIST_FIRST(&keg->uk_part_slab);
- } else {
- slab = LIST_FIRST(&keg->uk_free_slab);
- LIST_REMOVE(slab, us_link);
- LIST_INSERT_HEAD(&keg->uk_part_slab, slab,
- us_link);
- }
+ /*
+ * Round-robin for non first-touch zones when there is more than one
+ * domain.
+ */
*** DIFF OUTPUT TRUNCATED AT 1000 LINES ***
More information about the svn-src-user
mailing list