svn commit: r331369 - head/sys/vm
Jeff Roberson
jeff at FreeBSD.org
Thu Mar 22 19:21:12 UTC 2018
Author: jeff
Date: Thu Mar 22 19:21:11 2018
New Revision: 331369
URL: https://svnweb.freebsd.org/changeset/base/331369
Log:
Lock reservations with a dedicated lock in each reservation. Protect the
vmd_free_count with atomics.
This allows us to allocate and free from reservations without the free lock
except where a superpage is allocated from the physical layer, which is
roughly 1/512 of the operations on amd64.
Use the counter api to eliminate cache conention on counters.
Reviewed by: markj
Tested by: pho
Sponsored by: Netflix, Dell/EMC Isilon
Differential Revision: https://reviews.freebsd.org/D14707
Modified:
head/sys/vm/vm_page.c
head/sys/vm/vm_pagequeue.h
head/sys/vm/vm_reserv.c
head/sys/vm/vm_reserv.h
Modified: head/sys/vm/vm_page.c
==============================================================================
--- head/sys/vm/vm_page.c Thu Mar 22 19:11:43 2018 (r331368)
+++ head/sys/vm/vm_page.c Thu Mar 22 19:21:11 2018 (r331369)
@@ -177,7 +177,6 @@ static uma_zone_t fakepg_zone;
static void vm_page_alloc_check(vm_page_t m);
static void vm_page_clear_dirty_mask(vm_page_t m, vm_page_bits_t pagebits);
static void vm_page_enqueue(uint8_t queue, vm_page_t m);
-static void vm_page_free_phys(struct vm_domain *vmd, vm_page_t m);
static void vm_page_init(void *dummy);
static int vm_page_insert_after(vm_page_t m, vm_object_t object,
vm_pindex_t pindex, vm_page_t mpred);
@@ -1677,10 +1676,10 @@ vm_page_alloc_after(vm_object_t object, vm_pindex_t pi
* for the request class and false otherwise.
*/
int
-vm_domain_available(struct vm_domain *vmd, int req, int npages)
+vm_domain_allocate(struct vm_domain *vmd, int req, int npages)
{
+ u_int limit, old, new;
- vm_domain_free_assert_locked(vmd);
req = req & VM_ALLOC_CLASS_MASK;
/*
@@ -1688,15 +1687,34 @@ vm_domain_available(struct vm_domain *vmd, int req, in
*/
if (curproc == pageproc && req != VM_ALLOC_INTERRUPT)
req = VM_ALLOC_SYSTEM;
+ if (req == VM_ALLOC_INTERRUPT)
+ limit = 0;
+ else if (req == VM_ALLOC_SYSTEM)
+ limit = vmd->vmd_interrupt_free_min;
+ else
+ limit = vmd->vmd_free_reserved;
- if (vmd->vmd_free_count >= npages + vmd->vmd_free_reserved ||
- (req == VM_ALLOC_SYSTEM &&
- vmd->vmd_free_count >= npages + vmd->vmd_interrupt_free_min) ||
- (req == VM_ALLOC_INTERRUPT &&
- vmd->vmd_free_count >= npages))
- return (1);
+ /*
+ * Attempt to reserve the pages. Fail if we're below the limit.
+ */
+ limit += npages;
+ old = vmd->vmd_free_count;
+ do {
+ if (old < limit)
+ return (0);
+ new = old - npages;
+ } while (atomic_fcmpset_int(&vmd->vmd_free_count, &old, new) == 0);
- return (0);
+ /* Wake the page daemon if we've crossed the threshold. */
+ if (vm_paging_needed(vmd, new) && !vm_paging_needed(vmd, old))
+ pagedaemon_wakeup(vmd->vmd_domain);
+
+ /* Only update bitsets on transitions. */
+ if ((old >= vmd->vmd_free_min && new < vmd->vmd_free_min) ||
+ (old >= vmd->vmd_free_severe && new < vmd->vmd_free_severe))
+ vm_domain_set(vmd);
+
+ return (1);
}
vm_page_t
@@ -1723,44 +1741,34 @@ vm_page_alloc_domain_after(vm_object_t object, vm_pind
again:
m = NULL;
#if VM_NRESERVLEVEL > 0
+ /*
+ * Can we allocate the page from a reservation?
+ */
if (vm_object_reserv(object) &&
- (m = vm_reserv_extend(req, object, pindex, domain, mpred))
- != NULL) {
+ ((m = vm_reserv_extend(req, object, pindex, domain, mpred)) != NULL ||
+ (m = vm_reserv_alloc_page(req, object, pindex, domain, mpred)) != NULL)) {
domain = vm_phys_domain(m);
vmd = VM_DOMAIN(domain);
goto found;
}
#endif
vmd = VM_DOMAIN(domain);
- vm_domain_free_lock(vmd);
- if (vm_domain_available(vmd, req, 1)) {
+ if (vm_domain_allocate(vmd, req, 1)) {
/*
- * Can we allocate the page from a reservation?
+ * If not, allocate it from the free page queues.
*/
+ vm_domain_free_lock(vmd);
+ m = vm_phys_alloc_pages(domain, object != NULL ?
+ VM_FREEPOOL_DEFAULT : VM_FREEPOOL_DIRECT, 0);
+ vm_domain_free_unlock(vmd);
+ if (m == NULL) {
+ vm_domain_freecnt_inc(vmd, 1);
#if VM_NRESERVLEVEL > 0
- if (!vm_object_reserv(object) ||
- (m = vm_reserv_alloc_page(object, pindex,
- domain, mpred)) == NULL)
+ if (vm_reserv_reclaim_inactive(domain))
+ goto again;
#endif
- {
- /*
- * If not, allocate it from the free page queues.
- */
- m = vm_phys_alloc_pages(domain, object != NULL ?
- VM_FREEPOOL_DEFAULT : VM_FREEPOOL_DIRECT, 0);
-#if VM_NRESERVLEVEL > 0
- if (m == NULL && vm_reserv_reclaim_inactive(domain)) {
- m = vm_phys_alloc_pages(domain,
- object != NULL ?
- VM_FREEPOOL_DEFAULT : VM_FREEPOOL_DIRECT,
- 0);
- }
-#endif
}
}
- if (m != NULL)
- vm_domain_freecnt_dec(vmd, 1);
- vm_domain_free_unlock(vmd);
if (m == NULL) {
/*
* Not allocatable, give up.
@@ -1775,9 +1783,7 @@ again:
*/
KASSERT(m != NULL, ("missing page"));
-#if VM_NRESERVLEVEL > 0
found:
-#endif
vm_page_alloc_check(m);
/*
@@ -1934,9 +1940,14 @@ vm_page_alloc_contig_domain(vm_object_t object, vm_pin
*/
again:
#if VM_NRESERVLEVEL > 0
+ /*
+ * Can we allocate the pages from a reservation?
+ */
if (vm_object_reserv(object) &&
- (m_ret = vm_reserv_extend_contig(req, object, pindex, domain,
- npages, low, high, alignment, boundary, mpred)) != NULL) {
+ ((m_ret = vm_reserv_extend_contig(req, object, pindex, domain,
+ npages, low, high, alignment, boundary, mpred)) != NULL ||
+ (m_ret = vm_reserv_alloc_contig(req, object, pindex, domain,
+ npages, low, high, alignment, boundary, mpred)) != NULL)) {
domain = vm_phys_domain(m_ret);
vmd = VM_DOMAIN(domain);
goto found;
@@ -1944,31 +1955,23 @@ again:
#endif
m_ret = NULL;
vmd = VM_DOMAIN(domain);
- vm_domain_free_lock(vmd);
- if (vm_domain_available(vmd, req, npages)) {
+ if (vm_domain_allocate(vmd, req, npages)) {
/*
- * Can we allocate the pages from a reservation?
+ * allocate them from the free page queues.
*/
+ vm_domain_free_lock(vmd);
+ m_ret = vm_phys_alloc_contig(domain, npages, low, high,
+ alignment, boundary);
+ vm_domain_free_unlock(vmd);
+ if (m_ret == NULL) {
+ vm_domain_freecnt_inc(vmd, npages);
#if VM_NRESERVLEVEL > 0
-retry:
- if (!vm_object_reserv(object) ||
- (m_ret = vm_reserv_alloc_contig(object, pindex, domain,
- npages, low, high, alignment, boundary, mpred)) == NULL)
+ if (vm_reserv_reclaim_contig(domain, npages, low,
+ high, alignment, boundary))
+ goto again;
#endif
- /*
- * If not, allocate them from the free page queues.
- */
- m_ret = vm_phys_alloc_contig(domain, npages, low, high,
- alignment, boundary);
-#if VM_NRESERVLEVEL > 0
- if (m_ret == NULL && vm_reserv_reclaim_contig(
- domain, npages, low, high, alignment, boundary))
- goto retry;
-#endif
+ }
}
- if (m_ret != NULL)
- vm_domain_freecnt_dec(vmd, npages);
- vm_domain_free_unlock(vmd);
if (m_ret == NULL) {
if (vm_domain_alloc_fail(vmd, object, req))
goto again;
@@ -2109,13 +2112,14 @@ vm_page_alloc_freelist_domain(int domain, int freelist
*/
vmd = VM_DOMAIN(domain);
again:
- vm_domain_free_lock(vmd);
- if (vm_domain_available(vmd, req, 1))
+ if (vm_domain_allocate(vmd, req, 1)) {
+ vm_domain_free_lock(vmd);
m = vm_phys_alloc_freelist_pages(domain, freelist,
VM_FREEPOOL_DIRECT, 0);
- if (m != NULL)
- vm_domain_freecnt_dec(vmd, 1);
- vm_domain_free_unlock(vmd);
+ vm_domain_free_unlock(vmd);
+ if (m == NULL)
+ vm_domain_freecnt_inc(vmd, 1);
+ }
if (m == NULL) {
if (vm_domain_alloc_fail(vmd, NULL, req))
goto again;
@@ -2491,8 +2495,9 @@ retry:
vm_page_remque(m);
vm_page_replace_checked(m_new, object,
m->pindex, m);
- m->valid = 0;
- vm_page_undirty(m);
+ if (vm_page_free_prep(m, false))
+ SLIST_INSERT_HEAD(&free, m,
+ plinks.s.ss);
/*
* The new page must be deactivated
@@ -2504,10 +2509,12 @@ retry:
m->flags &= ~PG_ZERO;
vm_page_remque(m);
vm_page_remove(m);
+ if (vm_page_free_prep(m, false))
+ SLIST_INSERT_HEAD(&free, m,
+ plinks.s.ss);
KASSERT(m->dirty == 0,
("page %p is dirty", m));
}
- SLIST_INSERT_HEAD(&free, m, plinks.s.ss);
} else
error = EBUSY;
unlock:
@@ -2548,7 +2555,7 @@ unlock:
do {
MPASS(vm_phys_domain(m) == domain);
SLIST_REMOVE_HEAD(&free, plinks.s.ss);
- vm_page_free_phys(vmd, m);
+ vm_phys_free_pages(m, 0);
cnt++;
} while ((m = SLIST_FIRST(&free)) != NULL);
vm_domain_free_unlock(vmd);
@@ -3159,24 +3166,12 @@ vm_page_free_prep(vm_page_t m, bool pagequeue_locked)
if (pmap_page_get_memattr(m) != VM_MEMATTR_DEFAULT)
pmap_page_set_memattr(m, VM_MEMATTR_DEFAULT);
- return (true);
-}
-
-/*
- * Insert the page into the physical memory allocator's free page
- * queues. This is the last step to free a page. The caller is
- * responsible for adjusting the free page count.
- */
-static void
-vm_page_free_phys(struct vm_domain *vmd, vm_page_t m)
-{
-
- vm_domain_free_assert_locked(vmd);
-
#if VM_NRESERVLEVEL > 0
- if (!vm_reserv_free_page(m))
+ if (vm_reserv_free_page(m))
+ return (false);
#endif
- vm_phys_free_pages(m, 0);
+
+ return (true);
}
void
@@ -3200,7 +3195,7 @@ vm_page_free_phys_pglist(struct pglist *tq)
vmd = vm_pagequeue_domain(m);
vm_domain_free_lock(vmd);
}
- vm_page_free_phys(vmd, m);
+ vm_phys_free_pages(m, 0);
cnt++;
}
if (vmd != NULL) {
@@ -3227,7 +3222,7 @@ vm_page_free_toq(vm_page_t m)
return;
vmd = vm_pagequeue_domain(m);
vm_domain_free_lock(vmd);
- vm_page_free_phys(vmd, m);
+ vm_phys_free_pages(m, 0);
vm_domain_free_unlock(vmd);
vm_domain_freecnt_inc(vmd, 1);
}
Modified: head/sys/vm/vm_pagequeue.h
==============================================================================
--- head/sys/vm/vm_pagequeue.h Thu Mar 22 19:11:43 2018 (r331368)
+++ head/sys/vm/vm_pagequeue.h Thu Mar 22 19:21:11 2018 (r331369)
@@ -180,7 +180,7 @@ vm_pagequeue_cnt_add(struct vm_pagequeue *pq, int adde
void vm_domain_set(struct vm_domain *vmd);
void vm_domain_clear(struct vm_domain *vmd);
-int vm_domain_available(struct vm_domain *vmd, int req, int npages);
+int vm_domain_allocate(struct vm_domain *vmd, int req, int npages);
/*
* vm_pagequeue_domain:
@@ -265,23 +265,6 @@ vm_domain_freecnt_inc(struct vm_domain *vmd, int adj)
new >= vmd->vmd_pageout_free_min)))
vm_domain_clear(vmd);
}
-
-static inline void
-vm_domain_freecnt_dec(struct vm_domain *vmd, int adj)
-{
- u_int old, new;
-
- old = atomic_fetchadd_int(&vmd->vmd_free_count, -adj);
- new = old - adj;
- KASSERT(new >= 0, ("vm_domain_freecnt_dec: free count underflow"));
- if (vm_paging_needed(vmd, new) && !vm_paging_needed(vmd, old))
- pagedaemon_wakeup(vmd->vmd_domain);
- /* Only update bitsets on transitions. */
- if ((old >= vmd->vmd_free_min && new < vmd->vmd_free_min) ||
- (old >= vmd->vmd_free_severe && new < vmd->vmd_free_severe))
- vm_domain_set(vmd);
-}
-
#endif /* _KERNEL */
#endif /* !_VM_PAGEQUEUE_ */
Modified: head/sys/vm/vm_reserv.c
==============================================================================
--- head/sys/vm/vm_reserv.c Thu Mar 22 19:11:43 2018 (r331368)
+++ head/sys/vm/vm_reserv.c Thu Mar 22 19:21:11 2018 (r331369)
@@ -45,6 +45,8 @@ __FBSDID("$FreeBSD$");
#include <sys/param.h>
#include <sys/kernel.h>
+#include <sys/counter.h>
+#include <sys/ktr.h>
#include <sys/lock.h>
#include <sys/malloc.h>
#include <sys/mutex.h>
@@ -54,6 +56,7 @@ __FBSDID("$FreeBSD$");
#include <sys/sysctl.h>
#include <sys/systm.h>
#include <sys/vmmeter.h>
+#include <sys/smp.h>
#include <vm/vm.h>
#include <vm/vm_param.h>
@@ -166,22 +169,37 @@ popmap_is_set(popmap_t popmap[], int i)
*
* A partially populated reservation can be broken and reclaimed at any time.
*
- * f - vm_domain_free_lock
+ * r - vm_reserv_lock
+ * d - vm_reserv_domain_lock
* o - vm_reserv_object_lock
* c - constant after boot
*/
struct vm_reserv {
- TAILQ_ENTRY(vm_reserv) partpopq; /* (f) per-domain queue. */
- LIST_ENTRY(vm_reserv) objq; /* (o, f) object queue */
- vm_object_t object; /* (o, f) containing object */
- vm_pindex_t pindex; /* (o, f) offset in object */
+ struct mtx lock; /* reservation lock. */
+ TAILQ_ENTRY(vm_reserv) partpopq; /* (d) per-domain queue. */
+ LIST_ENTRY(vm_reserv) objq; /* (o, r) object queue */
+ vm_object_t object; /* (o, r) containing object */
+ vm_pindex_t pindex; /* (o, r) offset in object */
vm_page_t pages; /* (c) first page */
- int domain; /* (c) NUMA domain. */
- int popcnt; /* (f) # of pages in use */
- char inpartpopq; /* (f) */
- popmap_t popmap[NPOPMAP]; /* (f) bit vector, used pages */
+ uint16_t domain; /* (c) NUMA domain. */
+ uint16_t popcnt; /* (r) # of pages in use */
+ char inpartpopq; /* (d) */
+ popmap_t popmap[NPOPMAP]; /* (r) bit vector, used pages */
};
+#define vm_reserv_lockptr(rv) (&(rv)->lock)
+#define vm_reserv_assert_locked(rv) \
+ mtx_assert(vm_reserv_lockptr(rv), MA_OWNED)
+#define vm_reserv_lock(rv) mtx_lock(vm_reserv_lockptr(rv))
+#define vm_reserv_trylock(rv) mtx_trylock(vm_reserv_lockptr(rv))
+#define vm_reserv_unlock(rv) mtx_unlock(vm_reserv_lockptr(rv))
+
+static struct mtx_padalign vm_reserv_domain_locks[MAXMEMDOM];
+
+#define vm_reserv_domain_lockptr(d) &vm_reserv_domain_locks[(d)]
+#define vm_reserv_domain_lock(d) mtx_lock(vm_reserv_domain_lockptr(d))
+#define vm_reserv_domain_unlock(d) mtx_unlock(vm_reserv_domain_lockptr(d))
+
/*
* The reservation array
*
@@ -218,13 +236,13 @@ static TAILQ_HEAD(, vm_reserv) vm_rvq_partpop[MAXMEMDO
static SYSCTL_NODE(_vm, OID_AUTO, reserv, CTLFLAG_RD, 0, "Reservation Info");
-static long vm_reserv_broken;
-SYSCTL_LONG(_vm_reserv, OID_AUTO, broken, CTLFLAG_RD,
- &vm_reserv_broken, 0, "Cumulative number of broken reservations");
+static counter_u64_t vm_reserv_broken = EARLY_COUNTER;
+SYSCTL_COUNTER_U64(_vm_reserv, OID_AUTO, broken, CTLFLAG_RD,
+ &vm_reserv_broken, "Cumulative number of broken reservations");
-static long vm_reserv_freed;
-SYSCTL_LONG(_vm_reserv, OID_AUTO, freed, CTLFLAG_RD,
- &vm_reserv_freed, 0, "Cumulative number of freed reservations");
+static counter_u64_t vm_reserv_freed = EARLY_COUNTER;
+SYSCTL_COUNTER_U64(_vm_reserv, OID_AUTO, freed, CTLFLAG_RD,
+ &vm_reserv_freed, "Cumulative number of freed reservations");
static int sysctl_vm_reserv_fullpop(SYSCTL_HANDLER_ARGS);
@@ -236,9 +254,9 @@ static int sysctl_vm_reserv_partpopq(SYSCTL_HANDLER_AR
SYSCTL_OID(_vm_reserv, OID_AUTO, partpopq, CTLTYPE_STRING | CTLFLAG_RD, NULL, 0,
sysctl_vm_reserv_partpopq, "A", "Partially populated reservation queues");
-static long vm_reserv_reclaimed;
-SYSCTL_LONG(_vm_reserv, OID_AUTO, reclaimed, CTLFLAG_RD,
- &vm_reserv_reclaimed, 0, "Cumulative number of reclaimed reservations");
+static counter_u64_t vm_reserv_reclaimed = EARLY_COUNTER;
+SYSCTL_COUNTER_U64(_vm_reserv, OID_AUTO, reclaimed, CTLFLAG_RD,
+ &vm_reserv_reclaimed, "Cumulative number of reclaimed reservations");
/*
* The object lock pool is used to synchronize the rvq. We can not use a
@@ -313,12 +331,12 @@ sysctl_vm_reserv_partpopq(SYSCTL_HANDLER_ARGS)
for (level = -1; level <= VM_NRESERVLEVEL - 2; level++) {
counter = 0;
unused_pages = 0;
- vm_domain_free_lock(VM_DOMAIN(domain));
+ vm_reserv_domain_lock(domain);
TAILQ_FOREACH(rv, &vm_rvq_partpop[domain], partpopq) {
counter++;
unused_pages += VM_LEVEL_0_NPAGES - rv->popcnt;
}
- vm_domain_free_unlock(VM_DOMAIN(domain));
+ vm_reserv_domain_unlock(domain);
sbuf_printf(&sbuf, "%6d, %7d, %6dK, %6d\n",
domain, level,
unused_pages * ((int)PAGE_SIZE / 1024), counter);
@@ -337,6 +355,9 @@ vm_reserv_remove(vm_reserv_t rv)
{
vm_object_t object;
+ vm_reserv_assert_locked(rv);
+ CTR5(KTR_VM, "%s: rv %p object %p popcnt %d inpartpop %d",
+ __FUNCTION__, rv, rv->object, rv->popcnt, rv->inpartpopq);
KASSERT(rv->object != NULL,
("vm_reserv_remove: reserv %p is free", rv));
KASSERT(!rv->inpartpopq,
@@ -356,6 +377,11 @@ vm_reserv_insert(vm_reserv_t rv, vm_object_t object, v
{
int i;
+ vm_reserv_assert_locked(rv);
+ CTR6(KTR_VM,
+ "%s: rv %p(%p) object %p new %p popcnt %d",
+ __FUNCTION__, rv, rv->pages, rv->object, object,
+ rv->popcnt);
KASSERT(rv->object == NULL,
("vm_reserv_insert: reserv %p isn't free", rv));
KASSERT(rv->popcnt == 0,
@@ -377,14 +403,15 @@ vm_reserv_insert(vm_reserv_t rv, vm_object_t object, v
* becomes zero, the reservation is destroyed. Additionally, moves the
* reservation to the tail of the partially populated reservation queue if the
* population count is non-zero.
- *
- * The free page queue lock must be held.
*/
static void
vm_reserv_depopulate(vm_reserv_t rv, int index)
{
+ struct vm_domain *vmd;
- vm_domain_free_assert_locked(VM_DOMAIN(rv->domain));
+ vm_reserv_assert_locked(rv);
+ CTR5(KTR_VM, "%s: rv %p object %p popcnt %d inpartpop %d",
+ __FUNCTION__, rv, rv->object, rv->popcnt, rv->inpartpopq);
KASSERT(rv->object != NULL,
("vm_reserv_depopulate: reserv %p is free", rv));
KASSERT(popmap_is_set(rv->popmap, index),
@@ -395,10 +422,7 @@ vm_reserv_depopulate(vm_reserv_t rv, int index)
KASSERT(rv->domain >= 0 && rv->domain < vm_ndomains,
("vm_reserv_depopulate: reserv %p's domain is corrupted %d",
rv, rv->domain));
- if (rv->inpartpopq) {
- TAILQ_REMOVE(&vm_rvq_partpop[rv->domain], rv, partpopq);
- rv->inpartpopq = FALSE;
- } else {
+ if (rv->popcnt == VM_LEVEL_0_NPAGES) {
KASSERT(rv->pages->psind == 1,
("vm_reserv_depopulate: reserv %p is already demoted",
rv));
@@ -406,14 +430,25 @@ vm_reserv_depopulate(vm_reserv_t rv, int index)
}
popmap_clear(rv->popmap, index);
rv->popcnt--;
+ vm_reserv_domain_lock(rv->domain);
+ if (rv->inpartpopq) {
+ TAILQ_REMOVE(&vm_rvq_partpop[rv->domain], rv, partpopq);
+ rv->inpartpopq = FALSE;
+ }
+ if (rv->popcnt != 0) {
+ rv->inpartpopq = TRUE;
+ TAILQ_INSERT_TAIL(&vm_rvq_partpop[rv->domain], rv, partpopq);
+ }
+ vm_reserv_domain_unlock(rv->domain);
+ vmd = VM_DOMAIN(rv->domain);
if (rv->popcnt == 0) {
vm_reserv_remove(rv);
+ vm_domain_free_lock(vmd);
vm_phys_free_pages(rv->pages, VM_LEVEL_0_ORDER);
- vm_reserv_freed++;
- } else {
- rv->inpartpopq = TRUE;
- TAILQ_INSERT_TAIL(&vm_rvq_partpop[rv->domain], rv, partpopq);
+ vm_domain_free_unlock(vmd);
+ counter_u64_add(vm_reserv_freed, 1);
}
+ vm_domain_freecnt_inc(vmd, 1);
}
/*
@@ -484,7 +519,9 @@ static void
vm_reserv_populate(vm_reserv_t rv, int index)
{
- vm_domain_free_assert_locked(VM_DOMAIN(rv->domain));
+ vm_reserv_assert_locked(rv);
+ CTR5(KTR_VM, "%s: rv %p object %p popcnt %d inpartpop %d",
+ __FUNCTION__, rv, rv->object, rv->popcnt, rv->inpartpopq);
KASSERT(rv->object != NULL,
("vm_reserv_populate: reserv %p is free", rv));
KASSERT(popmap_is_clear(rv->popmap, index),
@@ -497,17 +534,23 @@ vm_reserv_populate(vm_reserv_t rv, int index)
KASSERT(rv->domain >= 0 && rv->domain < vm_ndomains,
("vm_reserv_populate: reserv %p's domain is corrupted %d",
rv, rv->domain));
+ popmap_set(rv->popmap, index);
+ rv->popcnt++;
+ vm_reserv_domain_lock(rv->domain);
if (rv->inpartpopq) {
TAILQ_REMOVE(&vm_rvq_partpop[rv->domain], rv, partpopq);
rv->inpartpopq = FALSE;
}
- popmap_set(rv->popmap, index);
- rv->popcnt++;
if (rv->popcnt < VM_LEVEL_0_NPAGES) {
rv->inpartpopq = TRUE;
TAILQ_INSERT_TAIL(&vm_rvq_partpop[rv->domain], rv, partpopq);
- } else
+ } else {
+ KASSERT(rv->pages->psind == 0,
+ ("vm_reserv_populate: reserv %p is already promoted",
+ rv));
rv->pages->psind = 1;
+ }
+ vm_reserv_domain_unlock(rv->domain);
}
/*
@@ -572,31 +615,29 @@ vm_reserv_extend_contig(int req, vm_object_t object, v
return (NULL);
domain = rv->domain;
vmd = VM_DOMAIN(domain);
- vm_domain_free_lock(vmd);
- if (rv->object != object || !vm_domain_available(vmd, req, npages)) {
- m = NULL;
+ vm_reserv_lock(rv);
+ if (rv->object != object)
goto out;
- }
m = &rv->pages[index];
pa = VM_PAGE_TO_PHYS(m);
if (pa < low || pa + size > high || (pa & (alignment - 1)) != 0 ||
- ((pa ^ (pa + size - 1)) & ~(boundary - 1)) != 0) {
- m = NULL;
+ ((pa ^ (pa + size - 1)) & ~(boundary - 1)) != 0)
goto out;
- }
/* Handle vm_page_rename(m, new_object, ...). */
for (i = 0; i < npages; i++) {
- if (popmap_is_set(rv->popmap, index + i)) {
- m = NULL;
+ if (popmap_is_set(rv->popmap, index + i))
goto out;
- }
}
+ if (!vm_domain_allocate(vmd, req, npages))
+ goto out;
for (i = 0; i < npages; i++)
vm_reserv_populate(rv, index + i);
- vm_domain_freecnt_dec(vmd, npages);
-out:
- vm_domain_free_unlock(vmd);
+ vm_reserv_unlock(rv);
return (m);
+
+out:
+ vm_reserv_unlock(rv);
+ return (NULL);
}
/*
@@ -618,10 +659,11 @@ out:
* The object and free page queue must be locked.
*/
vm_page_t
-vm_reserv_alloc_contig(vm_object_t object, vm_pindex_t pindex, int domain,
+vm_reserv_alloc_contig(int req, vm_object_t object, vm_pindex_t pindex, int domain,
u_long npages, vm_paddr_t low, vm_paddr_t high, u_long alignment,
vm_paddr_t boundary, vm_page_t mpred)
{
+ struct vm_domain *vmd;
vm_paddr_t pa, size;
vm_page_t m, m_ret, msucc;
vm_pindex_t first, leftcap, rightcap;
@@ -629,7 +671,6 @@ vm_reserv_alloc_contig(vm_object_t object, vm_pindex_t
u_long allocpages, maxpages, minpages;
int i, index, n;
- vm_domain_free_assert_locked(VM_DOMAIN(domain));
VM_OBJECT_ASSERT_WLOCKED(object);
KASSERT(npages != 0, ("vm_reserv_alloc_contig: npages is 0"));
@@ -737,9 +778,19 @@ vm_reserv_alloc_contig(vm_object_t object, vm_pindex_t
* specified index may not be the first page within the first new
* reservation.
*/
- m = vm_phys_alloc_contig(domain, allocpages, low, high, ulmax(alignment,
- VM_LEVEL_0_SIZE), boundary > VM_LEVEL_0_SIZE ? boundary : 0);
- if (m == NULL)
+ m = NULL;
+ vmd = VM_DOMAIN(domain);
+ if (vm_domain_allocate(vmd, req, npages)) {
+ vm_domain_free_lock(vmd);
+ m = vm_phys_alloc_contig(domain, allocpages, low, high,
+ ulmax(alignment, VM_LEVEL_0_SIZE),
+ boundary > VM_LEVEL_0_SIZE ? boundary : 0);
+ vm_domain_free_unlock(vmd);
+ if (m == NULL) {
+ vm_domain_freecnt_inc(vmd, npages);
+ return (NULL);
+ }
+ } else
return (NULL);
KASSERT(vm_phys_domain(m) == domain,
("vm_reserv_alloc_contig: Page domain does not match requested."));
@@ -757,6 +808,7 @@ vm_reserv_alloc_contig(vm_object_t object, vm_pindex_t
KASSERT(rv->pages == m,
("vm_reserv_alloc_contig: reserv %p's pages is corrupted",
rv));
+ vm_reserv_lock(rv);
vm_reserv_insert(rv, object, first);
n = ulmin(VM_LEVEL_0_NPAGES - index, npages);
for (i = 0; i < n; i++)
@@ -766,6 +818,7 @@ vm_reserv_alloc_contig(vm_object_t object, vm_pindex_t
m_ret = &rv->pages[index];
index = 0;
}
+ vm_reserv_unlock(rv);
m += VM_LEVEL_0_NPAGES;
first += VM_LEVEL_0_NPAGES;
allocpages -= VM_LEVEL_0_NPAGES;
@@ -813,18 +866,20 @@ vm_reserv_extend(int req, vm_object_t object, vm_pinde
vmd = VM_DOMAIN(domain);
index = VM_RESERV_INDEX(object, pindex);
m = &rv->pages[index];
- vm_domain_free_lock(vmd);
- if (vm_domain_available(vmd, req, 1) == 0 ||
- /* Handle reclaim race. */
- rv->object != object ||
+ vm_reserv_lock(rv);
+ /* Handle reclaim race. */
+ if (rv->object != object ||
/* Handle vm_page_rename(m, new_object, ...). */
- popmap_is_set(rv->popmap, index))
+ popmap_is_set(rv->popmap, index)) {
m = NULL;
- if (m != NULL) {
- vm_reserv_populate(rv, index);
- vm_domain_freecnt_dec(vmd, 1);
+ goto out;
}
- vm_domain_free_unlock(vmd);
+ if (vm_domain_allocate(vmd, req, 1) == 0)
+ m = NULL;
+ else
+ vm_reserv_populate(rv, index);
+out:
+ vm_reserv_unlock(rv);
return (m);
}
@@ -840,15 +895,15 @@ vm_reserv_extend(int req, vm_object_t object, vm_pinde
* The object and free page queue must be locked.
*/
vm_page_t
-vm_reserv_alloc_page(vm_object_t object, vm_pindex_t pindex, int domain,
+vm_reserv_alloc_page(int req, vm_object_t object, vm_pindex_t pindex, int domain,
vm_page_t mpred)
{
+ struct vm_domain *vmd;
vm_page_t m, msucc;
vm_pindex_t first, leftcap, rightcap;
vm_reserv_t rv;
int index;
- vm_domain_free_assert_locked(VM_DOMAIN(domain));
VM_OBJECT_ASSERT_WLOCKED(object);
/*
@@ -917,15 +972,28 @@ vm_reserv_alloc_page(vm_object_t object, vm_pindex_t p
/*
* Allocate and populate the new reservation.
*/
- m = vm_phys_alloc_pages(domain, VM_FREEPOOL_DEFAULT, VM_LEVEL_0_ORDER);
- if (m == NULL)
+ m = NULL;
+ vmd = VM_DOMAIN(domain);
+ if (vm_domain_allocate(vmd, req, 1)) {
+ vm_domain_free_lock(vmd);
+ m = vm_phys_alloc_pages(domain, VM_FREEPOOL_DEFAULT,
+ VM_LEVEL_0_ORDER);
+ vm_domain_free_unlock(vmd);
+ if (m == NULL) {
+ vm_domain_freecnt_inc(vmd, 1);
+ return (NULL);
+ }
+ } else
return (NULL);
rv = vm_reserv_from_page(m);
+ vm_reserv_lock(rv);
KASSERT(rv->pages == m,
("vm_reserv_alloc_page: reserv %p's pages is corrupted", rv));
vm_reserv_insert(rv, object, first);
index = VM_RESERV_INDEX(object, pindex);
vm_reserv_populate(rv, index);
+ vm_reserv_unlock(rv);
+
return (&rv->pages[index]);
}
@@ -942,7 +1010,9 @@ vm_reserv_break(vm_reserv_t rv)
{
int begin_zeroes, hi, i, lo;
- vm_domain_free_assert_locked(VM_DOMAIN(rv->domain));
+ vm_reserv_assert_locked(rv);
+ CTR5(KTR_VM, "%s: rv %p object %p popcnt %d inpartpop %d",
+ __FUNCTION__, rv, rv->object, rv->popcnt, rv->inpartpopq);
vm_reserv_remove(rv);
rv->pages->psind = 0;
i = hi = 0;
@@ -981,12 +1051,14 @@ vm_reserv_break(vm_reserv_t rv)
if (i != NPOPMAP)
/* Convert from ffsl() to ordinary bit numbering. */
hi--;
+ vm_domain_free_lock(VM_DOMAIN(rv->domain));
vm_phys_free_contig(&rv->pages[begin_zeroes], NBPOPMAP * i +
hi - begin_zeroes);
+ vm_domain_free_unlock(VM_DOMAIN(rv->domain));
} while (i < NPOPMAP);
KASSERT(rv->popcnt == 0,
("vm_reserv_break: reserv %p's popcnt is corrupted", rv));
- vm_reserv_broken++;
+ counter_u64_add(vm_reserv_broken, 1);
}
/*
@@ -996,7 +1068,6 @@ void
vm_reserv_break_all(vm_object_t object)
{
vm_reserv_t rv;
- struct vm_domain *vmd;
/*
* This access of object->rvq is unsynchronized so that the
@@ -1005,27 +1076,22 @@ vm_reserv_break_all(vm_object_t object)
* lock prevents new additions, so we are guaranteed that when
* it returns NULL the object is properly empty.
*/
- vmd = NULL;
while ((rv = LIST_FIRST(&object->rvq)) != NULL) {
- if (vmd != VM_DOMAIN(rv->domain)) {
- if (vmd != NULL)
- vm_domain_free_unlock(vmd);
- vmd = VM_DOMAIN(rv->domain);
- vm_domain_free_lock(vmd);
- }
+ vm_reserv_lock(rv);
/* Reclaim race. */
- if (rv->object != object)
+ if (rv->object != object) {
+ vm_reserv_unlock(rv);
continue;
- KASSERT(rv->object == object,
- ("vm_reserv_break_all: reserv %p is corrupted", rv));
+ }
+ vm_reserv_domain_lock(rv->domain);
if (rv->inpartpopq) {
TAILQ_REMOVE(&vm_rvq_partpop[rv->domain], rv, partpopq);
rv->inpartpopq = FALSE;
}
+ vm_reserv_domain_unlock(rv->domain);
vm_reserv_break(rv);
+ vm_reserv_unlock(rv);
}
- if (vmd != NULL)
- vm_domain_free_unlock(vmd);
}
/*
@@ -1038,13 +1104,21 @@ boolean_t
vm_reserv_free_page(vm_page_t m)
{
vm_reserv_t rv;
+ boolean_t ret;
rv = vm_reserv_from_page(m);
if (rv->object == NULL)
return (FALSE);
- vm_domain_free_assert_locked(VM_DOMAIN(rv->domain));
- vm_reserv_depopulate(rv, m - rv->pages);
- return (TRUE);
+ vm_reserv_lock(rv);
+ /* Re-validate after lock. */
+ if (rv->object != NULL) {
+ vm_reserv_depopulate(rv, m - rv->pages);
+ ret = TRUE;
+ } else
+ ret = FALSE;
+ vm_reserv_unlock(rv);
+
+ return (ret);
}
/*
@@ -1058,6 +1132,7 @@ vm_reserv_init(void)
{
vm_paddr_t paddr;
struct vm_phys_seg *seg;
+ struct vm_reserv *rv;
int i, segind;
/*
@@ -1068,15 +1143,22 @@ vm_reserv_init(void)
seg = &vm_phys_segs[segind];
paddr = roundup2(seg->start, VM_LEVEL_0_SIZE);
while (paddr + VM_LEVEL_0_SIZE <= seg->end) {
- vm_reserv_array[paddr >> VM_LEVEL_0_SHIFT].pages =
- PHYS_TO_VM_PAGE(paddr);
- vm_reserv_array[paddr >> VM_LEVEL_0_SHIFT].domain =
- seg->domain;
+ rv = &vm_reserv_array[paddr >> VM_LEVEL_0_SHIFT];
+ rv->pages = PHYS_TO_VM_PAGE(paddr);
+ rv->domain = seg->domain;
+ mtx_init(&rv->lock, "vm reserv", NULL, MTX_DEF);
paddr += VM_LEVEL_0_SIZE;
}
}
- for (i = 0; i < MAXMEMDOM; i++)
+ for (i = 0; i < MAXMEMDOM; i++) {
+ mtx_init(&vm_reserv_domain_locks[i], "VM reserv domain", NULL,
+ MTX_DEF);
TAILQ_INIT(&vm_rvq_partpop[i]);
+ }
+
+ for (i = 0; i < VM_RESERV_OBJ_LOCK_COUNT; i++)
+ mtx_init(&vm_reserv_object_mtx[i], "resv obj lock", NULL,
+ MTX_DEF);
}
/*
@@ -1091,7 +1173,6 @@ vm_reserv_is_page_free(vm_page_t m)
rv = vm_reserv_from_page(m);
if (rv->object == NULL)
return (false);
- vm_domain_free_assert_locked(VM_DOMAIN(rv->domain));
return (popmap_is_clear(rv->popmap, m - rv->pages));
}
@@ -1131,7 +1212,10 @@ static void
vm_reserv_reclaim(vm_reserv_t rv)
{
- vm_domain_free_assert_locked(VM_DOMAIN(rv->domain));
+ vm_reserv_assert_locked(rv);
+ CTR5(KTR_VM, "%s: rv %p object %p popcnt %d inpartpop %d",
+ __FUNCTION__, rv, rv->object, rv->popcnt, rv->inpartpopq);
+ vm_reserv_domain_lock(rv->domain);
KASSERT(rv->inpartpopq,
("vm_reserv_reclaim: reserv %p's inpartpopq is FALSE", rv));
KASSERT(rv->domain >= 0 && rv->domain < vm_ndomains,
@@ -1139,8 +1223,9 @@ vm_reserv_reclaim(vm_reserv_t rv)
rv, rv->domain));
TAILQ_REMOVE(&vm_rvq_partpop[rv->domain], rv, partpopq);
rv->inpartpopq = FALSE;
+ vm_reserv_domain_unlock(rv->domain);
vm_reserv_break(rv);
- vm_reserv_reclaimed++;
+ counter_u64_add(vm_reserv_reclaimed, 1);
}
/*
@@ -1155,9 +1240,14 @@ vm_reserv_reclaim_inactive(int domain)
{
vm_reserv_t rv;
- vm_domain_free_assert_locked(VM_DOMAIN(domain));
- if ((rv = TAILQ_FIRST(&vm_rvq_partpop[domain])) != NULL) {
+ while ((rv = TAILQ_FIRST(&vm_rvq_partpop[domain])) != NULL) {
+ vm_reserv_lock(rv);
+ if (rv != TAILQ_FIRST(&vm_rvq_partpop[domain])) {
+ vm_reserv_unlock(rv);
+ continue;
+ }
vm_reserv_reclaim(rv);
+ vm_reserv_unlock(rv);
return (TRUE);
}
return (FALSE);
@@ -1176,14 +1266,16 @@ vm_reserv_reclaim_contig(int domain, u_long npages, vm
vm_paddr_t high, u_long alignment, vm_paddr_t boundary)
{
vm_paddr_t pa, size;
- vm_reserv_t rv;
+ vm_reserv_t rv, rvn;
int hi, i, lo, low_index, next_free;
- vm_domain_free_assert_locked(VM_DOMAIN(domain));
if (npages > VM_LEVEL_0_NPAGES - 1)
return (FALSE);
size = npages << PAGE_SHIFT;
- TAILQ_FOREACH(rv, &vm_rvq_partpop[domain], partpopq) {
+ vm_reserv_domain_lock(domain);
+again:
+ for (rv = TAILQ_FIRST(&vm_rvq_partpop[domain]); rv != NULL; rv = rvn) {
+ rvn = TAILQ_NEXT(rv, partpopq);
pa = VM_PAGE_TO_PHYS(&rv->pages[VM_LEVEL_0_NPAGES - 1]);
if (pa + PAGE_SIZE - size < low) {
/* This entire reservation is too low; go to next. */
@@ -1194,6 +1286,17 @@ vm_reserv_reclaim_contig(int domain, u_long npages, vm
/* This entire reservation is too high; go to next. */
continue;
}
+ if (vm_reserv_trylock(rv) == 0) {
+ vm_reserv_domain_unlock(domain);
+ vm_reserv_lock(rv);
+ if (!rv->inpartpopq) {
+ vm_reserv_domain_lock(domain);
+ if (!rvn->inpartpopq)
+ goto again;
+ continue;
+ }
+ } else
+ vm_reserv_domain_unlock(domain);
if (pa < low) {
/* Start the search for free pages at "low". */
low_index = (low + PAGE_MASK - pa) >> PAGE_SHIFT;
@@ -1239,6 +1342,7 @@ vm_reserv_reclaim_contig(int domain, u_long npages, vm
if ((NBPOPMAP * i - next_free) * PAGE_SIZE >=
size) {
vm_reserv_reclaim(rv);
+ vm_reserv_unlock(rv);
return (TRUE);
}
hi = ffsl(rv->popmap[i]);
@@ -1249,10 +1353,16 @@ vm_reserv_reclaim_contig(int domain, u_long npages, vm
if ((NBPOPMAP * i + hi - next_free) * PAGE_SIZE >=
size) {
vm_reserv_reclaim(rv);
+ vm_reserv_unlock(rv);
return (TRUE);
}
} while (i < NPOPMAP);
+ vm_reserv_unlock(rv);
+ vm_reserv_domain_lock(domain);
+ if (rvn != NULL && !rvn->inpartpopq)
+ goto again;
}
+ vm_reserv_domain_unlock(domain);
return (FALSE);
}
@@ -1270,7 +1380,11 @@ vm_reserv_rename(vm_page_t m, vm_object_t new_object,
VM_OBJECT_ASSERT_WLOCKED(new_object);
rv = vm_reserv_from_page(m);
if (rv->object == old_object) {
- vm_domain_free_lock(VM_DOMAIN(rv->domain));
+ vm_reserv_lock(rv);
+ CTR6(KTR_VM,
+ "%s: rv %p object %p new %p popcnt %d inpartpop %d",
*** DIFF OUTPUT TRUNCATED AT 1000 LINES ***
More information about the svn-src-all
mailing list