svn commit: r331754 - user/jeff/numa/sys/vm
Jeff Roberson
jeff at FreeBSD.org
Fri Mar 30 01:33:11 UTC 2018
Author: jeff
Date: Fri Mar 30 01:33:10 2018
New Revision: 331754
URL: https://svnweb.freebsd.org/changeset/base/331754
Log:
Re-implement the page free cache with UMA. Change the limits so the import function doesn't
grab the lock if it's not going to succeed.
Modified:
user/jeff/numa/sys/vm/uma.h
user/jeff/numa/sys/vm/uma_core.c
user/jeff/numa/sys/vm/vm_page.c
user/jeff/numa/sys/vm/vm_page.h
Modified: user/jeff/numa/sys/vm/uma.h
==============================================================================
--- user/jeff/numa/sys/vm/uma.h Fri Mar 30 01:31:06 2018 (r331753)
+++ user/jeff/numa/sys/vm/uma.h Fri Mar 30 01:33:10 2018 (r331754)
@@ -286,6 +286,10 @@ uma_zone_t uma_zcache_create(char *name, int size, uma
* NUMA aware Zone. Implements a best
* effort first-touch policy.
*/
+#define UMA_ZONE_NOBUCKETCACHE 0x20000 /*
+ * Don't cache full buckets. Limit
+ * UMA to per-cpu state.
+ */
/*
* These flags are shared between the keg and zone. In zones wishing to add
Modified: user/jeff/numa/sys/vm/uma_core.c
==============================================================================
--- user/jeff/numa/sys/vm/uma_core.c Fri Mar 30 01:31:06 2018 (r331753)
+++ user/jeff/numa/sys/vm/uma_core.c Fri Mar 30 01:33:10 2018 (r331754)
@@ -2965,7 +2965,13 @@ zfree_start:
/* ub_cnt is pointing to the last free item */
KASSERT(bucket->ub_cnt != 0,
("uma_zfree: Attempting to insert an empty bucket onto the full list.\n"));
- LIST_INSERT_HEAD(&zdom->uzd_buckets, bucket, ub_link);
+ if ((zone->uz_flags & UMA_ZONE_NOBUCKETCACHE) != 0) {
+ ZONE_UNLOCK(zone);
+ bucket_drain(zone, bucket);
+ bucket_free(zone, bucket, udata);
+ goto zfree_restart;
+ } else
+ LIST_INSERT_HEAD(&zdom->uzd_buckets, bucket, ub_link);
}
/*
Modified: user/jeff/numa/sys/vm/vm_page.c
==============================================================================
--- user/jeff/numa/sys/vm/vm_page.c Fri Mar 30 01:31:06 2018 (r331753)
+++ user/jeff/numa/sys/vm/vm_page.c Fri Mar 30 01:33:10 2018 (r331754)
@@ -134,7 +134,6 @@ extern int vmem_startup_count(void);
struct vm_domain vm_dom[MAXMEMDOM];
static DPCPU_DEFINE(struct vm_batchqueue, pqbatch[MAXMEMDOM][PQ_COUNT]);
-static DPCPU_DEFINE(struct vm_batchqueue, freeqbatch[MAXMEMDOM]);
struct mtx_padalign __exclusive_cache_line pa_lock[PA_LOCK_COUNT];
@@ -221,8 +220,7 @@ vm_page_init_cache_zones(void *dummy __unused)
vmd->vmd_pgcache = uma_zcache_create("vm pgcache",
sizeof(struct vm_page), NULL, NULL, NULL, NULL,
vm_page_import, vm_page_release, vmd,
- /* UMA_ZONE_NOBUCKETCACHE |*/
- UMA_ZONE_MAXBUCKET | UMA_ZONE_VM);
+ UMA_ZONE_NOBUCKETCACHE | UMA_ZONE_MAXBUCKET | UMA_ZONE_VM);
}
}
SYSINIT(vm_page2, SI_SUB_VM_CONF, SI_ORDER_ANY, vm_page_init_cache_zones, NULL);
@@ -1781,6 +1779,11 @@ again:
}
#endif
vmd = VM_DOMAIN(domain);
+ if (object != NULL && vmd->vmd_pgcache != NULL) {
+ m = uma_zalloc(vmd->vmd_pgcache, M_NOWAIT);
+ if (m != NULL)
+ goto found;
+ }
if (vm_domain_allocate(vmd, req, 1)) {
/*
* If not, allocate it from the free page queues.
@@ -2188,22 +2191,23 @@ vm_page_import(void *arg, void **store, int cnt, int d
int i, j, n;
vmd = arg;
+ /* Only import if we can bring in a full bucket. */
+ if (cnt == 1 || !vm_domain_allocate(vmd, VM_ALLOC_NORMAL, cnt))
+ return (0);
domain = vmd->vmd_domain;
- n = 64; /* Starting stride. */
+ n = 64; /* Starting stride, arbitrary. */
vm_domain_free_lock(vmd);
for (i = 0; i < cnt; i+=n) {
n = vm_phys_alloc_npages(domain, VM_FREELIST_DEFAULT, &m,
MIN(n, cnt-i));
if (n == 0)
break;
- if (!vm_domain_allocate(vmd, VM_ALLOC_NORMAL, n)) {
- vm_phys_free_contig(m, n);
- break;
- }
for (j = 0; j < n; j++)
store[i+j] = m++;
}
vm_domain_free_unlock(vmd);
+ if (cnt != i)
+ vm_domain_freecnt_inc(vmd, cnt - i);
return (i);
}
@@ -3431,36 +3435,6 @@ vm_page_free_prep(vm_page_t m)
return (true);
}
-void
-vm_page_free_phys_pglist(struct pglist *tq)
-{
- struct vm_domain *vmd;
- vm_page_t m;
- int cnt;
-
- if (TAILQ_EMPTY(tq))
- return;
- vmd = NULL;
- cnt = 0;
- TAILQ_FOREACH(m, tq, listq) {
- if (vmd != vm_pagequeue_domain(m)) {
- if (vmd != NULL) {
- vm_domain_free_unlock(vmd);
- vm_domain_freecnt_inc(vmd, cnt);
- cnt = 0;
- }
- vmd = vm_pagequeue_domain(m);
- vm_domain_free_lock(vmd);
- }
- vm_phys_free_pages(m, 0);
- cnt++;
- }
- if (vmd != NULL) {
- vm_domain_free_unlock(vmd);
- vm_domain_freecnt_inc(vmd, cnt);
- }
-}
-
/*
* vm_page_free_toq:
*
@@ -3473,35 +3447,20 @@ vm_page_free_phys_pglist(struct pglist *tq)
void
vm_page_free_toq(vm_page_t m)
{
- struct vm_batchqueue *cpubq, bq;
struct vm_domain *vmd;
- int domain, freed;
if (!vm_page_free_prep(m))
return;
- domain = vm_phys_domain(m);
- vmd = VM_DOMAIN(domain);
-
- critical_enter();
- cpubq = DPCPU_PTR(freeqbatch[domain]);
- if (vm_batchqueue_insert(cpubq, m)) {
- critical_exit();
+ vmd = vm_pagequeue_domain(m);
+ if (m->pool == VM_FREEPOOL_DEFAULT && vmd->vmd_pgcache != NULL) {
+ uma_zfree(vmd->vmd_pgcache, m);
return;
}
- memcpy(&bq, cpubq, sizeof(bq));
- vm_batchqueue_init(cpubq);
- critical_exit();
-
vm_domain_free_lock(vmd);
vm_phys_free_pages(m, 0);
- freed = 1;
- VM_BATCHQ_FOREACH(&bq, m) {
- vm_phys_free_pages(m, 0);
- freed++;
- }
vm_domain_free_unlock(vmd);
- vm_domain_freecnt_inc(vmd, freed);
+ vm_domain_freecnt_inc(vmd, 1);
}
/*
Modified: user/jeff/numa/sys/vm/vm_page.h
==============================================================================
--- user/jeff/numa/sys/vm/vm_page.h Fri Mar 30 01:31:06 2018 (r331753)
+++ user/jeff/numa/sys/vm/vm_page.h Fri Mar 30 01:33:10 2018 (r331754)
@@ -532,7 +532,6 @@ void vm_page_dequeue(vm_page_t m);
void vm_page_dequeue_lazy(vm_page_t m);
void vm_page_dequeue_locked(vm_page_t m);
vm_page_t vm_page_find_least(vm_object_t, vm_pindex_t);
-void vm_page_free_phys_pglist(struct pglist *tq);
bool vm_page_free_prep(vm_page_t m);
vm_page_t vm_page_getfake(vm_paddr_t paddr, vm_memattr_t memattr);
void vm_page_initfake(vm_page_t m, vm_paddr_t paddr, vm_memattr_t memattr);
More information about the svn-src-user
mailing list