svn commit: r364129 - head/sys/vm
Conrad Meyer
cem at FreeBSD.org
Tue Aug 11 20:37:46 UTC 2020
Author: cem
Date: Tue Aug 11 20:37:45 2020
New Revision: 364129
URL: https://svnweb.freebsd.org/changeset/base/364129
Log:
Add support for multithreading the inactive queue pageout within a domain.
In very high throughput workloads, the inactive scan can become overwhelmed
as you have many cores producing pages and a single core freeing. Since
Mark's introduction of batched pagequeue operations, we can now run multiple
inactive threads working on independent batches.
To avoid confusing the pid and other control algorithms, I (Jeff) do this in
a mpi-like fan out and collect model that is driven from the primary page
daemon. It decides whether the shortfall can be overcome with a single
thread and if not dispatches multiple threads and waits for their results.
The heuristic is based on timing the pageout activity and averaging a
pages-per-second variable which is exponentially decayed. This is visible in
sysctl and may be interesting for other purposes.
I (Jeff) have verified that this does indeed double our paging throughput
when used with two threads. With four we tend to run into other contention
problems. For now I would like to commit this infrastructure with only a
single thread enabled.
The number of worker threads per domain can be controlled with the
'vm.pageout_threads_per_domain' tunable.
Submitted by: jeff (earlier version)
Discussed with: markj
Tested by: pho
Sponsored by: probably Netflix (based on contemporary commits)
Differential Revision: https://reviews.freebsd.org/D21629
Modified:
head/sys/vm/vm_meter.c
head/sys/vm/vm_page.c
head/sys/vm/vm_page.h
head/sys/vm/vm_pageout.c
head/sys/vm/vm_pagequeue.h
Modified: head/sys/vm/vm_meter.c
==============================================================================
--- head/sys/vm/vm_meter.c Tue Aug 11 17:54:10 2020 (r364128)
+++ head/sys/vm/vm_meter.c Tue Aug 11 20:37:45 2020 (r364129)
@@ -552,6 +552,9 @@ vm_domain_stats_init(struct vm_domain *vmd, struct sys
SYSCTL_ADD_UINT(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
"free_severe", CTLFLAG_RD, &vmd->vmd_free_severe, 0,
"Severe free pages");
+ SYSCTL_ADD_UINT(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
+ "inactive_pps", CTLFLAG_RD, &vmd->vmd_inactive_pps, 0,
+ "inactive pages freed/second");
}
Modified: head/sys/vm/vm_page.c
==============================================================================
--- head/sys/vm/vm_page.c Tue Aug 11 17:54:10 2020 (r364128)
+++ head/sys/vm/vm_page.c Tue Aug 11 20:37:45 2020 (r364129)
@@ -421,7 +421,7 @@ sysctl_vm_page_blacklist(SYSCTL_HANDLER_ARGS)
* In principle, this function only needs to set the flag PG_MARKER.
* Nonetheless, it write busies the page as a safety precaution.
*/
-static void
+void
vm_page_init_marker(vm_page_t marker, int queue, uint16_t aflags)
{
@@ -2488,7 +2488,7 @@ vm_page_zone_import(void *arg, void **store, int cnt,
* main purpose is to replenish the store of free pages.
*/
if (vmd->vmd_severeset || curproc == pageproc ||
- !_vm_domain_allocate(vmd, VM_ALLOC_NORMAL, cnt))
+ !_vm_domain_allocate(vmd, VM_ALLOC_SYSTEM, cnt))
return (0);
domain = vmd->vmd_domain;
vm_domain_free_lock(vmd);
Modified: head/sys/vm/vm_page.h
==============================================================================
--- head/sys/vm/vm_page.h Tue Aug 11 17:54:10 2020 (r364128)
+++ head/sys/vm/vm_page.h Tue Aug 11 20:37:45 2020 (r364129)
@@ -630,6 +630,7 @@ vm_page_t vm_page_find_least(vm_object_t, vm_pindex_t)
void vm_page_free_invalid(vm_page_t);
vm_page_t vm_page_getfake(vm_paddr_t paddr, vm_memattr_t memattr);
void vm_page_initfake(vm_page_t m, vm_paddr_t paddr, vm_memattr_t memattr);
+void vm_page_init_marker(vm_page_t marker, int queue, uint16_t aflags);
int vm_page_insert (vm_page_t, vm_object_t, vm_pindex_t);
void vm_page_invalid(vm_page_t m);
void vm_page_launder(vm_page_t m);
Modified: head/sys/vm/vm_pageout.c
==============================================================================
--- head/sys/vm/vm_pageout.c Tue Aug 11 17:54:10 2020 (r364128)
+++ head/sys/vm/vm_pageout.c Tue Aug 11 20:37:45 2020 (r364129)
@@ -82,6 +82,7 @@ __FBSDID("$FreeBSD$");
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/kernel.h>
+#include <sys/blockcount.h>
#include <sys/eventhandler.h>
#include <sys/lock.h>
#include <sys/mutex.h>
@@ -163,6 +164,12 @@ SYSCTL_INT(_vm, OID_AUTO, panic_on_oom,
SYSCTL_INT(_vm, OID_AUTO, pageout_update_period,
CTLFLAG_RWTUN, &vm_pageout_update_period, 0,
"Maximum active LRU update period");
+
+/* Access with get_pageout_threads_per_domain(). */
+static int pageout_threads_per_domain = 1;
+SYSCTL_INT(_vm, OID_AUTO, pageout_threads_per_domain, CTLFLAG_RDTUN,
+ &pageout_threads_per_domain, 0,
+ "Number of worker threads comprising each per-domain pagedaemon");
SYSCTL_INT(_vm, OID_AUTO, lowmem_period, CTLFLAG_RWTUN, &lowmem_period, 0,
"Low memory callback period");
@@ -1414,23 +1421,23 @@ vm_pageout_reinsert_inactive(struct scan_state *ss, st
vm_batchqueue_init(bq);
}
-/*
- * Attempt to reclaim the requested number of pages from the inactive queue.
- * Returns true if the shortage was addressed.
- */
-static int
-vm_pageout_scan_inactive(struct vm_domain *vmd, int shortage,
- int *addl_shortage)
+static void
+vm_pageout_scan_inactive(struct vm_domain *vmd, int page_shortage)
{
+ struct timeval start, end;
struct scan_state ss;
struct vm_batchqueue rq;
+ struct vm_page marker_page;
vm_page_t m, marker;
struct vm_pagequeue *pq;
vm_object_t object;
vm_page_astate_t old, new;
- int act_delta, addl_page_shortage, deficit, page_shortage, refs;
- int starting_page_shortage;
+ int act_delta, addl_page_shortage, starting_page_shortage, refs;
+ object = NULL;
+ vm_batchqueue_init(&rq);
+ getmicrouptime(&start);
+
/*
* The addl_page_shortage is an estimate of the number of temporarily
* stuck pages in the inactive queue. In other words, the
@@ -1440,24 +1447,14 @@ vm_pageout_scan_inactive(struct vm_domain *vmd, int sh
addl_page_shortage = 0;
/*
- * vmd_pageout_deficit counts the number of pages requested in
- * allocations that failed because of a free page shortage. We assume
- * that the allocations will be reattempted and thus include the deficit
- * in our scan target.
- */
- deficit = atomic_readandclear_int(&vmd->vmd_pageout_deficit);
- starting_page_shortage = page_shortage = shortage + deficit;
-
- object = NULL;
- vm_batchqueue_init(&rq);
-
- /*
* Start scanning the inactive queue for pages that we can free. The
* scan will stop when we reach the target or we have scanned the
* entire queue. (Note that m->a.act_count is not used to make
* decisions for the inactive queue, only for the active queue.)
*/
- marker = &vmd->vmd_markers[PQ_INACTIVE];
+ starting_page_shortage = page_shortage;
+ marker = &marker_page;
+ vm_page_init_marker(marker, PQ_INACTIVE, 0);
pq = &vmd->vmd_pagequeues[PQ_INACTIVE];
vm_pagequeue_lock(pq);
vm_pageout_init_scan(&ss, pq, marker, NULL, pq->pq_cnt);
@@ -1637,9 +1634,99 @@ reinsert:
vm_pageout_end_scan(&ss);
vm_pagequeue_unlock(pq);
- VM_CNT_ADD(v_dfree, starting_page_shortage - page_shortage);
+ /*
+ * Record the remaining shortage and the progress and rate it was made.
+ */
+ atomic_add_int(&vmd->vmd_addl_shortage, addl_page_shortage);
+ getmicrouptime(&end);
+ timevalsub(&end, &start);
+ atomic_add_int(&vmd->vmd_inactive_us,
+ end.tv_sec * 1000000 + end.tv_usec);
+ atomic_add_int(&vmd->vmd_inactive_freed,
+ starting_page_shortage - page_shortage);
+}
+/*
+ * Dispatch a number of inactive threads according to load and collect the
+ * results to prevent a coherent (CEM: incoherent?) view of paging activity on
+ * this domain.
+ */
+static int
+vm_pageout_inactive_dispatch(struct vm_domain *vmd, int shortage)
+{
+ u_int freed, pps, threads, us;
+
+ vmd->vmd_inactive_shortage = shortage;
+
/*
+ * If we have more work than we can do in a quarter of our interval, we
+ * fire off multiple threads to process it.
+ */
+ if (vmd->vmd_inactive_threads > 1 && vmd->vmd_inactive_pps != 0 &&
+ shortage > vmd->vmd_inactive_pps / VM_INACT_SCAN_RATE / 4) {
+ threads = vmd->vmd_inactive_threads;
+ vm_domain_pageout_lock(vmd);
+ vmd->vmd_inactive_shortage /= threads;
+ blockcount_acquire(&vmd->vmd_inactive_starting, threads - 1);
+ blockcount_acquire(&vmd->vmd_inactive_running, threads - 1);
+ wakeup(&vmd->vmd_inactive_shortage);
+ vm_domain_pageout_unlock(vmd);
+ }
+
+ /* Run the local thread scan. */
+ vm_pageout_scan_inactive(vmd, vmd->vmd_inactive_shortage);
+
+ /*
+ * Block until helper threads report results and then accumulate
+ * totals.
+ */
+ blockcount_wait(&vmd->vmd_inactive_running, NULL, "vmpoid", PVM);
+ freed = atomic_readandclear_int(&vmd->vmd_inactive_freed);
+ VM_CNT_ADD(v_dfree, freed);
+
+ /*
+ * Calculate the per-thread paging rate with an exponential decay of
+ * prior results. Careful to avoid integer rounding errors with large
+ * us values.
+ */
+ us = max(atomic_readandclear_int(&vmd->vmd_inactive_us), 1);
+ if (us > 1000000)
+ /* Keep rounding to tenths */
+ pps = (freed * 10) / ((us * 10) / 1000000);
+ else
+ pps = (1000000 / us) * freed;
+ vmd->vmd_inactive_pps = (vmd->vmd_inactive_pps / 2) + (pps / 2);
+
+ return (shortage - freed);
+}
+
+/*
+ * Attempt to reclaim the requested number of pages from the inactive queue.
+ * Returns true if the shortage was addressed.
+ */
+static int
+vm_pageout_inactive(struct vm_domain *vmd, int shortage, int *addl_shortage)
+{
+ struct vm_pagequeue *pq;
+ u_int addl_page_shortage, deficit, page_shortage;
+ u_int starting_page_shortage;
+
+ /*
+ * vmd_pageout_deficit counts the number of pages requested in
+ * allocations that failed because of a free page shortage. We assume
+ * that the allocations will be reattempted and thus include the deficit
+ * in our scan target.
+ */
+ deficit = atomic_readandclear_int(&vmd->vmd_pageout_deficit);
+ starting_page_shortage = shortage + deficit;
+
+ /*
+ * Run the inactive scan on as many threads as is necessary.
+ */
+ page_shortage = vm_pageout_inactive_dispatch(vmd, starting_page_shortage);
+ addl_page_shortage = atomic_readandclear_int(&vmd->vmd_addl_shortage);
+
+ /*
* Wake up the laundry thread so that it can perform any needed
* laundering. If we didn't meet our target, we're in shortfall and
* need to launder more aggressively. If PQ_LAUNDRY is empty and no
@@ -2066,7 +2153,7 @@ vm_pageout_worker(void *arg)
if (vm_pageout_lowmem() && vmd->vmd_free_count > ofree)
shortage -= min(vmd->vmd_free_count - ofree,
(u_int)shortage);
- target_met = vm_pageout_scan_inactive(vmd, shortage,
+ target_met = vm_pageout_inactive(vmd, shortage,
&addl_shortage);
} else
addl_shortage = 0;
@@ -2082,6 +2169,72 @@ vm_pageout_worker(void *arg)
}
/*
+ * vm_pageout_helper runs additional pageout daemons in times of high paging
+ * activity.
+ */
+static void
+vm_pageout_helper(void *arg)
+{
+ struct vm_domain *vmd;
+ int domain;
+
+ domain = (uintptr_t)arg;
+ vmd = VM_DOMAIN(domain);
+
+ vm_domain_pageout_lock(vmd);
+ for (;;) {
+ msleep(&vmd->vmd_inactive_shortage,
+ vm_domain_pageout_lockptr(vmd), PVM, "psleep", 0);
+ blockcount_release(&vmd->vmd_inactive_starting, 1);
+
+ vm_domain_pageout_unlock(vmd);
+ vm_pageout_scan_inactive(vmd, vmd->vmd_inactive_shortage);
+ vm_domain_pageout_lock(vmd);
+
+ /*
+ * Release the running count while the pageout lock is held to
+ * prevent wakeup races.
+ */
+ blockcount_release(&vmd->vmd_inactive_running, 1);
+ }
+}
+
+static int
+get_pageout_threads_per_domain(void)
+{
+ static bool resolved = false;
+ int half_cpus_per_dom;
+
+ /*
+ * This is serialized externally by the sorted autoconfig portion of
+ * boot.
+ */
+ if (__predict_true(resolved))
+ return (pageout_threads_per_domain);
+
+ /*
+ * Semi-arbitrarily constrain pagedaemon threads to less than half the
+ * total number of threads in the system as an insane upper limit.
+ */
+ half_cpus_per_dom = (mp_ncpus / vm_ndomains) / 2;
+
+ if (pageout_threads_per_domain < 1) {
+ printf("Invalid tuneable vm.pageout_threads_per_domain value: "
+ "%d out of valid range: [1-%d]; clamping to 1\n",
+ pageout_threads_per_domain, half_cpus_per_dom);
+ pageout_threads_per_domain = 1;
+ } else if (pageout_threads_per_domain > half_cpus_per_dom) {
+ printf("Invalid tuneable vm.pageout_threads_per_domain value: "
+ "%d out of valid range: [1-%d]; clamping to %d\n",
+ pageout_threads_per_domain, half_cpus_per_dom,
+ half_cpus_per_dom);
+ pageout_threads_per_domain = half_cpus_per_dom;
+ }
+ resolved = true;
+ return (pageout_threads_per_domain);
+}
+
+/*
* Initialize basic pageout daemon settings. See the comment above the
* definition of vm_domain for some explanation of how these thresholds are
* used.
@@ -2134,6 +2287,8 @@ vm_pageout_init_domain(int domain)
oid = SYSCTL_ADD_NODE(NULL, SYSCTL_CHILDREN(vmd->vmd_oid), OID_AUTO,
"pidctrl", CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "");
pidctrl_init_sysctl(&vmd->vmd_pid, SYSCTL_CHILDREN(oid));
+
+ vmd->vmd_inactive_threads = get_pageout_threads_per_domain();
}
static void
@@ -2184,10 +2339,11 @@ vm_pageout(void)
{
struct proc *p;
struct thread *td;
- int error, first, i;
+ int error, first, i, j, pageout_threads;
p = curproc;
td = curthread;
+ pageout_threads = get_pageout_threads_per_domain();
mtx_init(&vm_oom_ratelim_mtx, "vmoomr", NULL, MTX_DEF);
swap_pager_swap_init();
@@ -2206,6 +2362,14 @@ vm_pageout(void)
if (error != 0)
panic("starting pageout for domain %d: %d\n",
i, error);
+ }
+ for (j = 0; j < pageout_threads - 1; j++) {
+ error = kthread_add(vm_pageout_helper,
+ (void *)(uintptr_t)i, p, NULL, 0, 0,
+ "dom%d helper%d", i, j);
+ if (error != 0)
+ panic("starting pageout helper %d for domain "
+ "%d: %d\n", j, i, error);
}
error = kthread_add(vm_pageout_laundry_worker,
(void *)(uintptr_t)i, p, NULL, 0, 0, "laundry: dom%d", i);
Modified: head/sys/vm/vm_pagequeue.h
==============================================================================
--- head/sys/vm/vm_pagequeue.h Tue Aug 11 17:54:10 2020 (r364128)
+++ head/sys/vm/vm_pagequeue.h Tue Aug 11 20:37:45 2020 (r364129)
@@ -84,6 +84,7 @@ struct vm_batchqueue {
} __aligned(CACHE_LINE_SIZE);
#include <vm/uma.h>
+#include <sys/_blockcount.h>
#include <sys/pidctrl.h>
struct sysctl_oid;
@@ -254,6 +255,14 @@ struct vm_domain {
/* Paging control variables, used within single threaded page daemon. */
struct pidctrl vmd_pid; /* Pageout controller. */
boolean_t vmd_oom;
+ u_int vmd_inactive_threads;
+ u_int vmd_inactive_shortage; /* Per-thread shortage. */
+ blockcount_t vmd_inactive_running; /* Number of inactive threads. */
+ blockcount_t vmd_inactive_starting; /* Number of threads started. */
+ volatile u_int vmd_addl_shortage; /* Shortage accumulator. */
+ volatile u_int vmd_inactive_freed; /* Successful inactive frees. */
+ volatile u_int vmd_inactive_us; /* Microseconds for above. */
+ u_int vmd_inactive_pps; /* Exponential decay frees/second. */
int vmd_oom_seq;
int vmd_last_active_scan;
struct vm_page vmd_markers[PQ_COUNT]; /* (q) markers for queue scans */
More information about the svn-src-all
mailing list