svn commit: r330288 - user/markj/vm-playground/sys/vm
Mark Johnston
markj at FreeBSD.org
Fri Mar 2 18:12:27 UTC 2018
Author: markj
Date: Fri Mar 2 18:12:25 2018
New Revision: 330288
URL: https://svnweb.freebsd.org/changeset/base/330288
Log:
Add vm_page_alloc_pages_after().
This is a new page allocation which is intended to complement
vm_page_grab_pages(). It permits the allocation of multiple pages,
contiguous in the pindex space, with a single call. When
VM_ALLOC_{NOWAIT,WAITFAIL} are specified, the returned run may be
shorter than the one requested. In support of this function,
vm_reserv_extend() and vm_reserv_alloc_page() may now optionally
return a run of contiguous pages from the same reservation, and the
new vm_phys_alloc_npages() function is used to allocate pages from
the physical memory allocator.
Modified:
user/markj/vm-playground/sys/vm/swap_pager.c
user/markj/vm-playground/sys/vm/vm_page.c
user/markj/vm-playground/sys/vm/vm_page.h
user/markj/vm-playground/sys/vm/vm_pagequeue.h
user/markj/vm-playground/sys/vm/vm_reserv.c
user/markj/vm-playground/sys/vm/vm_reserv.h
user/markj/vm-playground/sys/vm/vnode_pager.c
Modified: user/markj/vm-playground/sys/vm/swap_pager.c
==============================================================================
--- user/markj/vm-playground/sys/vm/swap_pager.c Fri Mar 2 17:07:08 2018 (r330287)
+++ user/markj/vm-playground/sys/vm/swap_pager.c Fri Mar 2 18:12:25 2018 (r330288)
@@ -1097,10 +1097,10 @@ swap_pager_getpages(vm_object_t object, vm_page_t *ma,
int *rahead)
{
struct buf *bp;
- vm_page_t mpred, msucc, p;
+ vm_page_t mpred, msucc;
vm_pindex_t pindex;
daddr_t blk;
- int i, j, maxahead, maxbehind, reqcount, shift;
+ int i, maxahead, maxbehind, reqcount, shift;
reqcount = count;
@@ -1136,39 +1136,27 @@ swap_pager_getpages(vm_object_t object, vm_page_t *ma,
/*
* Allocate readahead and readbehind pages.
*/
- shift = rbehind != NULL ? *rbehind : 0;
- if (shift != 0) {
- for (i = 1; i <= shift; i++) {
- p = vm_page_alloc(object, ma[0]->pindex - i,
- VM_ALLOC_NORMAL);
- if (p == NULL) {
- /* Shift allocated pages to the left. */
- for (j = 0; j < i - 1; j++)
- bp->b_pages[j] =
- bp->b_pages[j + shift - i + 1];
- break;
- }
- bp->b_pages[shift - i] = p;
- }
- shift = i - 1;
- *rbehind = shift;
- }
+ if (rbehind != NULL && *rbehind > 0) {
+ shift = vm_page_alloc_pages_after(object,
+ ma[0]->pindex - *rbehind, VM_ALLOC_NORMAL, &bp->b_pages[0],
+ *rbehind, mpred);
+ if (shift != *rbehind) {
+ /* Drop a partially allocated run. */
+ for (i = 0; i < shift; i++)
+ vm_page_free(bp->b_pages[i]);
+ shift = *rbehind = 0;
+ } else
+ count += *rbehind;
+ } else
+ shift = 0;
for (i = 0; i < reqcount; i++)
bp->b_pages[i + shift] = ma[i];
- if (rahead != NULL) {
- for (i = 0; i < *rahead; i++) {
- p = vm_page_alloc(object,
- ma[reqcount - 1]->pindex + i + 1, VM_ALLOC_NORMAL);
- if (p == NULL)
- break;
- bp->b_pages[shift + reqcount + i] = p;
- }
- *rahead = i;
- }
- if (rbehind != NULL)
- count += *rbehind;
- if (rahead != NULL)
+ if (rahead != NULL && *rahead > 0) {
+ *rahead = vm_page_alloc_pages_after(object,
+ ma[reqcount - 1]->pindex + 1, VM_ALLOC_NORMAL,
+ &bp->b_pages[reqcount + shift], *rahead, ma[reqcount - 1]);
count += *rahead;
+ }
vm_object_pip_add(object, count);
Modified: user/markj/vm-playground/sys/vm/vm_page.c
==============================================================================
--- user/markj/vm-playground/sys/vm/vm_page.c Fri Mar 2 17:07:08 2018 (r330287)
+++ user/markj/vm-playground/sys/vm/vm_page.c Fri Mar 2 18:12:25 2018 (r330288)
@@ -1696,9 +1696,10 @@ vm_page_alloc_after(vm_object_t object, vm_pindex_t pi
* for the request class and false otherwise.
*/
int
-vm_domain_allocate(struct vm_domain *vmd, int req, int npages)
+vm_domain_allocate(struct vm_domain *vmd, int req, int npages, bool partial)
{
u_int limit, old, new;
+ int avail;
req = req & VM_ALLOC_CLASS_MASK;
@@ -1707,6 +1708,7 @@ vm_domain_allocate(struct vm_domain *vmd, int req, int
*/
if (curproc == pageproc && req != VM_ALLOC_INTERRUPT)
req = VM_ALLOC_SYSTEM;
+
if (req == VM_ALLOC_INTERRUPT)
limit = 0;
else if (req == VM_ALLOC_SYSTEM)
@@ -1719,9 +1721,12 @@ vm_domain_allocate(struct vm_domain *vmd, int req, int
*/
do {
old = vmd->vmd_free_count;
- new = old - npages;
- if (new < limit)
+ if (old <= limit)
return (0);
+ avail = min(old - limit, (u_int)npages);
+ if (avail != npages && !partial)
+ return (0);
+ new = old - avail;
} while (atomic_cmpset_int(&vmd->vmd_free_count, old, new) == 0);
/* Wake the page daemon if we've crossed the threshold. */
@@ -1733,7 +1738,7 @@ vm_domain_allocate(struct vm_domain *vmd, int req, int
(old >= vmd->vmd_free_severe && new < vmd->vmd_free_severe))
vm_domain_set(vmd);
- return (1);
+ return (avail);
}
vm_page_t
@@ -1764,21 +1769,22 @@ again:
* Can we allocate the page from a reservation?
*/
if (vm_object_reserv(object) &&
- ((m = vm_reserv_extend(req, object, pindex, domain, mpred)) != NULL ||
- (m = vm_reserv_alloc_page(req, object, pindex, domain, mpred)) != NULL)) {
+ ((m = vm_reserv_extend(req, object, pindex, domain, mpred, NULL)) !=
+ NULL ||
+ (m = vm_reserv_alloc_page(req, object, pindex, domain, mpred,
+ NULL)) != NULL)) {
domain = vm_phys_domain(m);
vmd = VM_DOMAIN(domain);
goto found;
}
#endif
vmd = VM_DOMAIN(domain);
- if (object != NULL && !vm_object_reserv(object) &&
- vmd->vmd_pgcache != NULL) {
+ if (!vm_object_reserv(object) && vmd->vmd_pgcache != NULL) {
m = uma_zalloc(vmd->vmd_pgcache, M_NOWAIT);
if (m != NULL)
goto found;
}
- if (vm_domain_allocate(vmd, req, 1)) {
+ if (vm_domain_allocate(vmd, req, 1, false) == 1) {
/*
* If not, allocate it from the free page queues.
*/
@@ -1828,7 +1834,7 @@ found:
m->busy_lock = VPB_SINGLE_EXCLUSIVER;
if ((req & VM_ALLOC_SBUSY) != 0)
m->busy_lock = VPB_SHARERS_WORD(1);
- if (req & VM_ALLOC_WIRED) {
+ if ((req & VM_ALLOC_WIRED) != 0) {
/*
* The page lock is not required for wiring a page until that
* page is inserted into the object.
@@ -1869,7 +1875,191 @@ found:
return (m);
}
+int
+vm_page_alloc_pages_after(vm_object_t object, vm_pindex_t pindex, int req,
+ vm_page_t *ma, int nreq, vm_page_t mpred)
+{
+ struct vm_domainset_iter di;
+ int domain, n;
+
+ vm_domainset_iter_page_init(&di, object, &domain, &req);
+ do {
+ n = vm_page_alloc_pages_domain_after(object, pindex, domain,
+ req, ma, nreq, mpred);
+ if (n > 0)
+ break;
+ } while (vm_domainset_iter_page(&di, &domain, &req) == 0);
+
+ return (n);
+}
+
/*
+ * vm_page_alloc_pages_after:
+ *
+ * Allocate a range of pages, contiguous in the pindex space. The
+ * number of pages actually allocated is returned and may be smaller
+ * than the number requested unless VM_ALLOC_WAITOK is specified.
+ * This function is otherwise identical to vm_page_alloc().
+ */
+int
+vm_page_alloc_pages_domain_after(vm_object_t object, vm_pindex_t pindex,
+ int domain, int req, vm_page_t *ma, int nreq, vm_page_t mpred)
+{
+ struct vm_domain *vmd;
+ vm_page_t m;
+ int avail, i, nalloc, pool;
+ u_int busy_lock, flags, oflags;
+
+ KASSERT(nreq > 0, ("invalid nreq %d", nreq));
+ KASSERT((object != NULL) == ((req & VM_ALLOC_NOOBJ) == 0) &&
+ (object != NULL || (req & VM_ALLOC_SBUSY) == 0) &&
+ ((req & (VM_ALLOC_NOBUSY | VM_ALLOC_SBUSY)) !=
+ (VM_ALLOC_NOBUSY | VM_ALLOC_SBUSY)),
+ ("inconsistent object(%p)/req(%x)", object, req));
+ KASSERT(object == NULL || (req & VM_ALLOC_WAITOK) == 0,
+ ("Can't sleep and retry object insertion."));
+ KASSERT(mpred == NULL || mpred->pindex < pindex,
+ ("mpred %p doesn't precede pindex 0x%jx", mpred,
+ (uintmax_t)pindex));
+ if (object != NULL)
+ VM_OBJECT_ASSERT_WLOCKED(object);
+
+ nalloc = 0;
+
+#if VM_NRESERVLEVEL > 0
+ if (vm_object_reserv(object)) {
+ avail = nreq;
+ m = vm_reserv_extend(req, object, pindex, domain, mpred,
+ &avail);
+ if (m == NULL)
+ m = vm_reserv_alloc_page(req, object, pindex, domain,
+ mpred, &avail);
+ if (m != NULL) {
+ domain = vm_phys_domain(m);
+ while (nalloc < avail)
+ ma[nalloc++] = m++;
+
+ /*
+ * We might have gotten a short run back because we
+ * reached the end of a reservation. If so, declare
+ * success now rather than trying to fill the rest of
+ * the array, in the hope that a subsequent allocation
+ * attempt will allocate a new reservation.
+ */
+ if (nalloc == nreq || (req & VM_ALLOC_WAITOK) == 0)
+ goto done;
+ }
+ }
+#endif
+
+again:
+ vmd = VM_DOMAIN(domain);
+ if ((avail = vm_domain_allocate(vmd, req, nreq - nalloc, true)) > 0) {
+ pool = object != NULL ? VM_FREEPOOL_DEFAULT :
+ VM_FREEPOOL_DIRECT;
+ vm_domain_free_lock(vmd);
+ do {
+ i = vm_phys_alloc_npages(domain, pool, &m,
+ avail - nalloc);
+ if (i == 0) {
+ vm_domain_freecnt_inc(vmd, avail - nalloc);
+ break;
+ }
+ for (; i > 0; i--)
+ ma[nalloc++] = m++;
+ } while (nalloc < avail);
+ vm_domain_free_unlock(vmd);
+ }
+ if (nalloc == 0 || (nalloc < nreq && (req & VM_ALLOC_WAITOK) != 0)) {
+#if VM_NRESERVLEVEL > 0
+ if (vm_reserv_reclaim_inactive(domain))
+ goto again;
+#endif
+
+ /*
+ * We failed to allocate at least one page, or the caller
+ * requested a blocking allocation and we weren't able to
+ * scrounge enough pages in the latest attempt.
+ */
+ if (vm_domain_alloc_fail(vmd, object, req))
+ goto again;
+ return (0);
+ }
+
+done:
+ for (i = 0; i < nalloc; i++)
+ vm_page_alloc_check(ma[i]);
+
+ /*
+ * Initialize the pages. Only the PG_ZERO flag is inherited.
+ */
+ flags = 0;
+ if ((req & VM_ALLOC_ZERO) != 0)
+ flags |= PG_ZERO;
+ if ((req & VM_ALLOC_NODUMP) != 0)
+ flags |= PG_NODUMP;
+ oflags = (object == NULL || (object->flags & OBJ_UNMANAGED) != 0) ?
+ VPO_UNMANAGED : 0;
+ busy_lock = VPB_UNBUSIED;
+ if ((req & (VM_ALLOC_NOBUSY | VM_ALLOC_NOOBJ | VM_ALLOC_SBUSY)) == 0)
+ busy_lock = VPB_SINGLE_EXCLUSIVER;
+ if ((req & VM_ALLOC_SBUSY) != 0)
+ busy_lock = VPB_SHARERS_WORD(1);
+
+ for (i = 0; i < nalloc; i++) {
+ m = ma[i];
+
+ m->flags = (m->flags | PG_NODUMP) & flags;
+ m->aflags = 0;
+ m->oflags = oflags;
+ m->busy_lock = busy_lock;
+ if ((req & VM_ALLOC_WIRED) != 0) {
+ /*
+ * The page lock is not required for wiring a page
+ * until that page is inserted into the object.
+ */
+ m->wire_count = 1;
+ }
+ m->act_count = 0;
+
+ if (object != NULL) {
+ if (vm_page_insert_after(m, object, pindex + i,
+ mpred)) {
+ avail = i;
+ for (; i < nalloc; i++) {
+ m = ma[i];
+ m->busy_lock = VPB_UNBUSIED;
+ m->oflags = VPO_UNMANAGED;
+ m->wire_count = 0;
+ KASSERT(m->object == NULL,
+ ("page %p has object", m));
+ /* Don't change PG_ZERO. */
+ vm_page_free_toq(m);
+ }
+ if ((req & VM_ALLOC_WAITFAIL) != 0) {
+ VM_OBJECT_WUNLOCK(object);
+ vm_radix_wait();
+ VM_OBJECT_WLOCK(object);
+ }
+ nalloc = avail;
+ break;
+ }
+
+ /* Ignore device objects; the pager sets "memattr" for them. */
+ if (object->memattr != VM_MEMATTR_DEFAULT &&
+ (object->flags & OBJ_FICTITIOUS) == 0)
+ pmap_page_set_memattr(m, object->memattr);
+ } else
+ m->pindex = pindex + i;
+ mpred = m;
+ }
+ if ((req & VM_ALLOC_WIRED) != 0)
+ VM_CNT_ADD(v_wire_count, nalloc);
+
+ return (nalloc);
+}
+
+/*
* vm_page_alloc_contig:
*
* Allocate a contiguous set of physical pages of the given size "npages"
@@ -1981,7 +2171,7 @@ again:
#endif
m_ret = NULL;
vmd = VM_DOMAIN(domain);
- if (vm_domain_allocate(vmd, req, npages)) {
+ if (vm_domain_allocate(vmd, req, npages, false) == npages) {
/*
* allocate them from the free page queues.
*/
@@ -2139,7 +2329,7 @@ vm_page_alloc_freelist_domain(int domain, int freelist
*/
vmd = VM_DOMAIN(domain);
again:
- if (vm_domain_allocate(vmd, req, 1)) {
+ if (vm_domain_allocate(vmd, req, 1, false) == 1) {
vm_domain_free_lock(vmd);
m = vm_phys_alloc_freelist_pages(domain, freelist,
VM_FREEPOOL_DIRECT, 0);
@@ -2191,7 +2381,7 @@ vm_page_import(void *arg, void **store, int cnt, int d
MIN(n, cnt-i));
if (n == 0)
break;
- if (!vm_domain_allocate(vmd, VM_ALLOC_NORMAL, n)) {
+ if (vm_domain_allocate(vmd, VM_ALLOC_NORMAL, n, false) == 0) {
vm_phys_free_contig(m, n);
break;
}
@@ -3189,14 +3379,14 @@ vm_page_free_prep(vm_page_t m, bool pagequeue_locked)
if ((m->oflags & VPO_UNMANAGED) == 0) {
vm_page_lock_assert(m, MA_OWNED);
KASSERT(!pmap_page_is_mapped(m),
- ("vm_page_free_toq: freeing mapped page %p", m));
+ ("vm_page_free_prep: freeing mapped page %p", m));
} else
KASSERT(m->queue == PQ_NONE,
- ("vm_page_free_toq: unmanaged page %p is queued", m));
+ ("vm_page_free_prep: unmanaged page %p is queued", m));
VM_CNT_INC(v_tfree);
if (vm_page_sbusied(m))
- panic("vm_page_free: freeing busy page %p", m);
+ panic("vm_page_free_prep: freeing busy page %p", m);
vm_page_remove(m);
@@ -3222,11 +3412,11 @@ vm_page_free_prep(vm_page_t m, bool pagequeue_locked)
vm_page_undirty(m);
if (m->wire_count != 0)
- panic("vm_page_free: freeing wired page %p", m);
+ panic("vm_page_free_prep: freeing wired page %p", m);
if (m->hold_count != 0) {
m->flags &= ~PG_ZERO;
KASSERT((m->flags & PG_UNHOLDFREE) == 0,
- ("vm_page_free: freeing PG_UNHOLDFREE page %p", m));
+ ("vm_page_free_prep: freeing PG_UNHOLDFREE page %p", m));
m->flags |= PG_UNHOLDFREE;
return (false);
}
@@ -3703,9 +3893,8 @@ int
vm_page_grab_pages(vm_object_t object, vm_pindex_t pindex, int allocflags,
vm_page_t *ma, int count)
{
- vm_page_t m, mpred;
- int pflags;
- int i;
+ vm_page_t m, mpred, msucc;
+ int i, pflags, run;
bool sleep;
VM_OBJECT_ASSERT_WLOCKED(object);
@@ -3717,6 +3906,7 @@ vm_page_grab_pages(vm_object_t object, vm_pindex_t pin
KASSERT((allocflags & VM_ALLOC_SBUSY) == 0 ||
(allocflags & VM_ALLOC_IGN_SBUSY) != 0,
("vm_page_grab_pages: VM_ALLOC_SBUSY/IGN_SBUSY mismatch"));
+
if (count == 0)
return (0);
pflags = allocflags & ~(VM_ALLOC_NOWAIT | VM_ALLOC_WAITOK |
@@ -3728,10 +3918,14 @@ retrylookup:
m = vm_radix_lookup_le(&object->rtree, pindex + i);
if (m == NULL || m->pindex != pindex + i) {
mpred = m;
+ msucc = mpred != NULL ? TAILQ_NEXT(mpred, listq) :
+ TAILQ_FIRST(&object->memq);
m = NULL;
- } else
+ } else {
mpred = TAILQ_PREV(m, pglist, listq);
- for (; i < count; i++) {
+ msucc = TAILQ_NEXT(m, listq);
+ }
+ while (i < count) {
if (m != NULL) {
sleep = (allocflags & VM_ALLOC_IGN_SBUSY) != 0 ?
vm_page_xbusied(m) : vm_page_busied(m);
@@ -3761,21 +3955,41 @@ retrylookup:
vm_page_xbusy(m);
if ((allocflags & VM_ALLOC_SBUSY) != 0)
vm_page_sbusy(m);
+ if (m->valid == 0 &&
+ (allocflags & VM_ALLOC_ZERO) != 0) {
+ if ((m->flags & PG_ZERO) == 0)
+ pmap_zero_page(m);
+ m->valid = VM_PAGE_BITS_ALL;
+ }
+ ma[i++] = m;
} else {
- m = vm_page_alloc_after(object, pindex + i,
- pflags | VM_ALLOC_COUNT(count - i), mpred);
- if (m == NULL) {
+ /*
+ * Try to allocate multiple consecutive pages. Use the
+ * succeeding page, if any, to bound the length of the
+ * requested run.
+ */
+ run = msucc == NULL || msucc->pindex >= pindex + count ?
+ count - i : msucc->pindex - (pindex + i);
+ run = vm_page_alloc_pages_after(object, pindex + i,
+ pflags | VM_ALLOC_COUNT(run), ma + i, run, mpred);
+ if (run == 0) {
if ((allocflags & VM_ALLOC_NOWAIT) != 0)
break;
goto retrylookup;
}
+ if ((allocflags & VM_ALLOC_ZERO) != 0) {
+ for (; run != 0; run--, i++) {
+ m = ma[i];
+ if ((m->flags & PG_ZERO) == 0)
+ pmap_zero_page(m);
+ m->valid = VM_PAGE_BITS_ALL;
+ }
+ } else
+ i += run;
+ m = ma[i - 1];
}
- if (m->valid == 0 && (allocflags & VM_ALLOC_ZERO) != 0) {
- if ((m->flags & PG_ZERO) == 0)
- pmap_zero_page(m);
- m->valid = VM_PAGE_BITS_ALL;
- }
- ma[i] = mpred = m;
+ mpred = m;
+ msucc = TAILQ_NEXT(m, listq);
m = vm_page_next(m);
}
return (i);
Modified: user/markj/vm-playground/sys/vm/vm_page.h
==============================================================================
--- user/markj/vm-playground/sys/vm/vm_page.h Fri Mar 2 17:07:08 2018 (r330287)
+++ user/markj/vm-playground/sys/vm/vm_page.h Fri Mar 2 18:12:25 2018 (r330288)
@@ -467,6 +467,10 @@ vm_page_t vm_page_alloc_domain(vm_object_t, vm_pindex_
vm_page_t vm_page_alloc_after(vm_object_t, vm_pindex_t, int, vm_page_t);
vm_page_t vm_page_alloc_domain_after(vm_object_t, vm_pindex_t, int, int,
vm_page_t);
+int vm_page_alloc_pages_after(vm_object_t, vm_pindex_t, int, vm_page_t *, int,
+ vm_page_t);
+int vm_page_alloc_pages_domain_after(vm_object_t, vm_pindex_t, int, int,
+ vm_page_t *, int, vm_page_t);
vm_page_t vm_page_alloc_contig(vm_object_t object, vm_pindex_t pindex, int req,
u_long npages, vm_paddr_t low, vm_paddr_t high, u_long alignment,
vm_paddr_t boundary, vm_memattr_t memattr);
Modified: user/markj/vm-playground/sys/vm/vm_pagequeue.h
==============================================================================
--- user/markj/vm-playground/sys/vm/vm_pagequeue.h Fri Mar 2 17:07:08 2018 (r330287)
+++ user/markj/vm-playground/sys/vm/vm_pagequeue.h Fri Mar 2 18:12:25 2018 (r330288)
@@ -183,7 +183,8 @@ vm_pagequeue_cnt_add(struct vm_pagequeue *pq, int adde
void vm_domain_set(struct vm_domain *vmd);
void vm_domain_clear(struct vm_domain *vmd);
-int vm_domain_allocate(struct vm_domain *vmd, int req, int npages);
+int vm_domain_allocate(struct vm_domain *vmd, int req, int npages,
+ bool partial);
/*
* vm_pagequeue_domain:
Modified: user/markj/vm-playground/sys/vm/vm_reserv.c
==============================================================================
--- user/markj/vm-playground/sys/vm/vm_reserv.c Fri Mar 2 17:07:08 2018 (r330287)
+++ user/markj/vm-playground/sys/vm/vm_reserv.c Fri Mar 2 18:12:25 2018 (r330288)
@@ -643,7 +643,7 @@ vm_reserv_extend_contig(int req, vm_object_t object, v
if (popmap_is_set(rv->popmap, index + i))
goto out;
}
- if (!vm_domain_allocate(vmd, req, npages))
+ if (vm_domain_allocate(vmd, req, npages, false) == 0)
goto out;
for (i = 0; i < npages; i++)
vm_reserv_populate(rv, index + i);
@@ -792,7 +792,7 @@ vm_reserv_alloc_contig(int req, vm_object_t object, vm
*/
m = NULL;
vmd = VM_DOMAIN(domain);
- if (vm_domain_allocate(vmd, req, allocpages)) {
+ if (vm_domain_allocate(vmd, req, allocpages, false) == allocpages) {
vm_domain_free_lock(vmd);
m = vm_phys_alloc_contig(domain, allocpages, low, high,
ulmax(alignment, VM_LEVEL_0_SIZE),
@@ -839,8 +839,10 @@ vm_reserv_alloc_contig(int req, vm_object_t object, vm
}
/*
- * Attempts to extend an existing reservation and allocate the page to the
- * object.
+ * Attempts to extend an existing reservation and allocate the request page to
+ * the object. Opportunistically returns up to "*countp" contiguous pages if
+ * the caller so requests. The number of pages allocated is returned in
+ * "*countp".
*
* The page "mpred" must immediately precede the offset "pindex" within the
* specified object.
@@ -849,12 +851,12 @@ vm_reserv_alloc_contig(int req, vm_object_t object, vm
*/
vm_page_t
vm_reserv_extend(int req, vm_object_t object, vm_pindex_t pindex, int domain,
- vm_page_t mpred)
+ vm_page_t mpred, int *countp)
{
struct vm_domain *vmd;
vm_page_t m, msucc;
vm_reserv_t rv;
- int index;
+ int avail, index, nalloc;
VM_OBJECT_ASSERT_WLOCKED(object);
@@ -886,10 +888,30 @@ vm_reserv_extend(int req, vm_object_t object, vm_pinde
m = NULL;
goto out;
}
- if (vm_domain_allocate(vmd, req, 1) == 0)
- m = NULL;
- else
+
+ /*
+ * If the caller is prepared to accept multiple pages, try to allocate
+ * them. We are constrained by:
+ * 1) the number of pages the caller can accept,
+ * 2) the number of free pages in the reservation succeeding "index",
+ * 3) the number of available free pages in the domain.
+ */
+ nalloc = countp != NULL ? imin(VM_LEVEL_0_NPAGES - index, *countp) : 1;
+ if ((avail = vm_domain_allocate(vmd, req, nalloc, true)) > 0) {
vm_reserv_populate(rv, index);
+ if (countp != NULL) {
+ for (nalloc = 1; nalloc < avail; nalloc++) {
+ if (popmap_is_set(rv->popmap, ++index))
+ break;
+ vm_reserv_populate(rv, index);
+ }
+ if (nalloc < avail)
+ /* Return leftover pages. */
+ vm_domain_freecnt_inc(vmd, avail - nalloc);
+ *countp = nalloc;
+ }
+ } else
+ m = NULL;
out:
vm_reserv_unlock(rv);
@@ -897,22 +919,25 @@ out:
}
/*
- * Allocates a page from an existing reservation.
+ * Allocates a new reservation for the object, and returns a page from that
+ * reservation. Opportunistically returns up to *"countp" contiguous pages if
+ * the caller so requests. The number of pages allocated is returned in
+ * "*countp".
*
* The page "mpred" must immediately precede the offset "pindex" within the
* specified object.
*
- * The object and free page queue must be locked.
+ * The object and per-domain free page queues must be locked.
*/
vm_page_t
vm_reserv_alloc_page(int req, vm_object_t object, vm_pindex_t pindex, int domain,
- vm_page_t mpred)
+ vm_page_t mpred, int *countp)
{
struct vm_domain *vmd;
vm_page_t m, msucc;
vm_pindex_t first, leftcap, rightcap;
vm_reserv_t rv;
- int index;
+ int avail, index, nalloc;
VM_OBJECT_ASSERT_WLOCKED(object);
@@ -981,16 +1006,24 @@ vm_reserv_alloc_page(int req, vm_object_t object, vm_p
/*
* Allocate and populate the new reservation.
+ *
+ * If the caller is prepared to accept multiple pages, try to allocate
+ * them. We are constrained by:
+ * 1) the number of pages the caller can accept,
+ * 2) the number of free pages in the reservation succeeding "index",
+ * 3) the number of available free pages in the domain.
*/
+ index = VM_RESERV_INDEX(object, pindex);
m = NULL;
+ nalloc = countp != NULL ? imin(VM_LEVEL_0_NPAGES - index, *countp) : 1;
vmd = VM_DOMAIN(domain);
- if (vm_domain_allocate(vmd, req, 1)) {
+ if ((avail = vm_domain_allocate(vmd, req, nalloc, true)) > 0) {
vm_domain_free_lock(vmd);
m = vm_phys_alloc_pages(domain, VM_FREEPOOL_DEFAULT,
VM_LEVEL_0_ORDER);
vm_domain_free_unlock(vmd);
if (m == NULL) {
- vm_domain_freecnt_inc(vmd, 1);
+ vm_domain_freecnt_inc(vmd, avail);
return (NULL);
}
} else
@@ -1000,11 +1033,22 @@ vm_reserv_alloc_page(int req, vm_object_t object, vm_p
KASSERT(rv->pages == m,
("vm_reserv_alloc_page: reserv %p's pages is corrupted", rv));
vm_reserv_insert(rv, object, first);
- index = VM_RESERV_INDEX(object, pindex);
vm_reserv_populate(rv, index);
+ m = &rv->pages[index];
+ if (countp != NULL) {
+ for (nalloc = 1; nalloc < avail; nalloc++) {
+ if (popmap_is_set(rv->popmap, ++index))
+ break;
+ vm_reserv_populate(rv, index);
+ }
+ if (nalloc < avail)
+ /* Return leftover pages. */
+ vm_domain_freecnt_inc(vmd, avail - nalloc);
+ *countp = nalloc;
+ }
vm_reserv_unlock(rv);
- return (&rv->pages[index]);
+ return (m);
}
/*
@@ -1227,15 +1271,16 @@ vm_reserv_reclaim(vm_reserv_t rv)
/*
* Breaks the reservation at the head of the partially populated reservation
- * queue, releasing its free pages to the physical memory allocator. Returns
- * TRUE if a reservation is broken and FALSE otherwise.
+ * queue, releasing its free pages to the physical memory allocator, and
+ * returns the number of pages released.
*
* The free page queue lock must be held.
*/
-boolean_t
+int
vm_reserv_reclaim_inactive(int domain)
{
vm_reserv_t rv;
+ int freed;
while ((rv = TAILQ_FIRST(&vm_rvq_partpop[domain])) != NULL) {
vm_reserv_lock(rv);
@@ -1243,11 +1288,12 @@ vm_reserv_reclaim_inactive(int domain)
vm_reserv_unlock(rv);
continue;
}
+ freed = VM_LEVEL_0_NPAGES - rv->popcnt;
vm_reserv_reclaim(rv);
vm_reserv_unlock(rv);
- return (TRUE);
+ return (freed);
}
- return (FALSE);
+ return (0);
}
/*
Modified: user/markj/vm-playground/sys/vm/vm_reserv.h
==============================================================================
--- user/markj/vm-playground/sys/vm/vm_reserv.h Fri Mar 2 17:07:08 2018 (r330287)
+++ user/markj/vm-playground/sys/vm/vm_reserv.h Fri Mar 2 18:12:25 2018 (r330288)
@@ -54,10 +54,12 @@ vm_page_t vm_reserv_extend_contig(int req, vm_object_t
vm_pindex_t pindex, int domain, u_long npages,
vm_paddr_t low, vm_paddr_t high, u_long alignment,
vm_paddr_t boundary, vm_page_t mpred);
-vm_page_t vm_reserv_alloc_page(int req, vm_object_t object, vm_pindex_t pindex,
- int domain, vm_page_t mpred);
+vm_page_t vm_reserv_alloc_page(int req, vm_object_t object,
+ vm_pindex_t pindex, int domain, vm_page_t mpred,
+ int *countp);
vm_page_t vm_reserv_extend(int req, vm_object_t object,
- vm_pindex_t pindex, int domain, vm_page_t mpred);
+ vm_pindex_t pindex, int domain, vm_page_t mpred,
+ int *countp);
void vm_reserv_break_all(vm_object_t object);
boolean_t vm_reserv_free_page(vm_page_t m);
void vm_reserv_init(void);
@@ -67,7 +69,7 @@ int vm_reserv_level_iffullpop(vm_page_t m);
boolean_t vm_reserv_reclaim_contig(int domain, u_long npages,
vm_paddr_t low, vm_paddr_t high, u_long alignment,
vm_paddr_t boundary);
-boolean_t vm_reserv_reclaim_inactive(int domain);
+int vm_reserv_reclaim_inactive(int domain);
void vm_reserv_rename(vm_page_t m, vm_object_t new_object,
vm_object_t old_object, vm_pindex_t old_object_offset);
int vm_reserv_size(int level);
Modified: user/markj/vm-playground/sys/vm/vnode_pager.c
==============================================================================
--- user/markj/vm-playground/sys/vm/vnode_pager.c Fri Mar 2 17:07:08 2018 (r330287)
+++ user/markj/vm-playground/sys/vm/vnode_pager.c Fri Mar 2 18:12:25 2018 (r330288)
@@ -897,35 +897,27 @@ vnode_pager_generic_getpages(struct vnode *vp, vm_page
/*
* Fill in the bp->b_pages[] array with requested and optional
- * read behind or read ahead pages. Read behind pages are looked
- * up in a backward direction, down to a first cached page. Same
- * for read ahead pages, but there is no need to shift the array
- * in case of encountering a cached page.
+ * read behind or read ahead pages.
*/
i = bp->b_npages = 0;
- if (rbehind) {
- vm_pindex_t startpindex, tpindex;
- vm_page_t p;
+ if (rbehind > 0) {
+ vm_pindex_t startpindex;
+ vm_page_t mpred;
VM_OBJECT_WLOCK(object);
startpindex = m[0]->pindex - rbehind;
- if ((p = TAILQ_PREV(m[0], pglist, listq)) != NULL &&
- p->pindex >= startpindex)
- startpindex = p->pindex + 1;
+ if ((mpred = TAILQ_PREV(m[0], pglist, listq)) != NULL &&
+ mpred->pindex >= startpindex)
+ startpindex = mpred->pindex + 1;
- /* tpindex is unsigned; beware of numeric underflow. */
- for (tpindex = m[0]->pindex - 1;
- tpindex >= startpindex && tpindex < m[0]->pindex;
- tpindex--, i++) {
- p = vm_page_alloc(object, tpindex, VM_ALLOC_NORMAL);
- if (p == NULL) {
- /* Shift the array. */
- for (int j = 0; j < i; j++)
- bp->b_pages[j] = bp->b_pages[j +
- tpindex + 1 - startpindex];
- break;
- }
- bp->b_pages[tpindex - startpindex] = p;
+ i = vm_page_alloc_pages_after(object, startpindex,
+ VM_ALLOC_NORMAL, &bp->b_pages[0],
+ m[0]->pindex - startpindex, mpred);
+ if (i < m[0]->pindex - startpindex) {
+ /* We have to drop the partially allocated run. */
+ for (int j = 0; j < i; j++)
+ vm_page_free(bp->b_pages[j]);
+ i = 0;
}
bp->b_pgbefore = i;
@@ -939,29 +931,24 @@ vnode_pager_generic_getpages(struct vnode *vp, vm_page
bp->b_pages[i] = m[j];
bp->b_npages += count;
- if (rahead) {
- vm_pindex_t endpindex, tpindex;
- vm_page_t p;
+ if (rahead > 0) {
+ vm_pindex_t endpindex, startpindex;
+ vm_page_t msucc;
if (!VM_OBJECT_WOWNED(object))
VM_OBJECT_WLOCK(object);
- endpindex = m[count - 1]->pindex + rahead + 1;
- if ((p = TAILQ_NEXT(m[count - 1], listq)) != NULL &&
- p->pindex < endpindex)
- endpindex = p->pindex;
+ startpindex = m[count - 1]->pindex + 1;
+ endpindex = startpindex + rahead;
+ if ((msucc = TAILQ_NEXT(m[count - 1], listq)) != NULL &&
+ msucc->pindex < endpindex)
+ endpindex = msucc->pindex;
if (endpindex > object->size)
endpindex = object->size;
- for (tpindex = m[count - 1]->pindex + 1;
- tpindex < endpindex; i++, tpindex++) {
- p = vm_page_alloc(object, tpindex, VM_ALLOC_NORMAL);
- if (p == NULL)
- break;
- bp->b_pages[i] = p;
- }
-
- bp->b_pgafter = i - bp->b_npages;
- bp->b_npages = i;
+ bp->b_pgafter = vm_page_alloc_pages_after(object, startpindex,
+ VM_ALLOC_NORMAL, &bp->b_pages[i], endpindex - startpindex,
+ m[count - 1]);
+ bp->b_npages += bp->b_pgafter;
} else
bp->b_pgafter = 0;
More information about the svn-src-user
mailing list