svn commit: r312994 - head/sys/vm
Mark Johnston
markj at FreeBSD.org
Mon Jan 30 18:51:45 UTC 2017
Author: markj
Date: Mon Jan 30 18:51:43 2017
New Revision: 312994
URL: https://svnweb.freebsd.org/changeset/base/312994
Log:
Avoid page lookups in the top-level object in vm_object_madvise().
We can iterate over consecutive resident pages in the top-level object
using the object's page list rather than by performing lookups in the
object radix tree. This extends one of the optimizations in r312208 to the
case where a shadow chain is present.
Suggested by: alc
Reviewed by: alc, kib (previous version)
MFC after: 2 weeks
Differential Revision: https://reviews.freebsd.org/D9282
Modified:
head/sys/vm/vm_object.c
Modified: head/sys/vm/vm_object.c
==============================================================================
--- head/sys/vm/vm_object.c Mon Jan 30 18:35:24 2017 (r312993)
+++ head/sys/vm/vm_object.c Mon Jan 30 18:51:43 2017 (r312994)
@@ -1075,6 +1075,33 @@ vm_object_sync(vm_object_t object, vm_oo
}
/*
+ * Determine whether the given advice can be applied to the object. Advice is
+ * not applied to unmanaged pages since they never belong to page queues, and
+ * since MADV_FREE is destructive, it can apply only to anonymous pages that
+ * have been mapped at most once.
+ */
+static bool
+vm_object_advice_applies(vm_object_t object, int advice)
+{
+
+ if ((object->flags & OBJ_UNMANAGED) != 0)
+ return (false);
+ if (advice != MADV_FREE)
+ return (true);
+ return ((object->type == OBJT_DEFAULT || object->type == OBJT_SWAP) &&
+ (object->flags & OBJ_ONEMAPPING) != 0);
+}
+
+static void
+vm_object_madvise_freespace(vm_object_t object, int advice, vm_pindex_t pindex,
+ vm_size_t size)
+{
+
+ if (advice == MADV_FREE && object->type == OBJT_SWAP)
+ swap_pager_freespace(object, pindex, size);
+}
+
+/*
* vm_object_madvise:
*
* Implements the madvise function at the object/page level.
@@ -1101,100 +1128,102 @@ vm_object_madvise(vm_object_t object, vm
{
vm_pindex_t tpindex;
vm_object_t backing_object, tobject;
- vm_page_t m;
+ vm_page_t m, tm;
if (object == NULL)
return;
- VM_OBJECT_WLOCK(object);
- for (m = NULL; pindex < end; pindex++) {
relookup:
+ VM_OBJECT_WLOCK(object);
+ if (!vm_object_advice_applies(object, advice)) {
+ VM_OBJECT_WUNLOCK(object);
+ return;
+ }
+ for (m = vm_page_find_least(object, pindex); pindex < end; pindex++) {
tobject = object;
- tpindex = pindex;
-shadowlookup:
- /*
- * MADV_FREE only operates on OBJT_DEFAULT or OBJT_SWAP pages
- * and those pages must be OBJ_ONEMAPPING.
- */
- if (advice == MADV_FREE) {
- if ((tobject->type != OBJT_DEFAULT &&
- tobject->type != OBJT_SWAP) ||
- (tobject->flags & OBJ_ONEMAPPING) == 0) {
- goto unlock_tobject;
- }
- } else if ((tobject->flags & OBJ_UNMANAGED) != 0)
- goto unlock_tobject;
/*
- * In the common case where the object has no backing object, we
- * can avoid performing lookups at each pindex. In either case,
- * when applying MADV_FREE we take care to release any swap
- * space used to store non-resident pages.
+ * If the next page isn't resident in the top-level object, we
+ * need to search the shadow chain. When applying MADV_FREE, we
+ * take care to release any swap space used to store
+ * non-resident pages.
*/
- if (object->backing_object == NULL) {
- m = (m != NULL) ? TAILQ_NEXT(m, listq) :
- vm_page_find_least(object, pindex);
- tpindex = (m != NULL && m->pindex < end) ?
- m->pindex : end;
- if (advice == MADV_FREE && object->type == OBJT_SWAP &&
- tpindex > pindex)
- swap_pager_freespace(object, pindex,
- tpindex - pindex);
- if ((pindex = tpindex) == end)
- break;
- } else if ((m = vm_page_lookup(tobject, tpindex)) == NULL) {
- if (advice == MADV_FREE && tobject->type == OBJT_SWAP)
- swap_pager_freespace(tobject, tpindex, 1);
+ if (m == NULL || pindex < m->pindex) {
/*
- * Prepare to search the next object in the chain.
+ * Optimize a common case: if the top-level object has
+ * no backing object, we can skip over the non-resident
+ * range in constant time.
*/
- backing_object = tobject->backing_object;
- if (backing_object == NULL)
- goto unlock_tobject;
- VM_OBJECT_WLOCK(backing_object);
- tpindex += OFF_TO_IDX(tobject->backing_object_offset);
- if (tobject != object)
- VM_OBJECT_WUNLOCK(tobject);
- tobject = backing_object;
- goto shadowlookup;
+ if (object->backing_object == NULL) {
+ tpindex = (m != NULL && m->pindex < end) ?
+ m->pindex : end;
+ vm_object_madvise_freespace(object, advice,
+ pindex, tpindex - pindex);
+ if ((pindex = tpindex) == end)
+ break;
+ goto next_page;
+ }
+
+ tpindex = pindex;
+ do {
+ vm_object_madvise_freespace(tobject, advice,
+ tpindex, 1);
+ /*
+ * Prepare to search the next object in the
+ * chain.
+ */
+ backing_object = tobject->backing_object;
+ if (backing_object == NULL)
+ goto next_pindex;
+ VM_OBJECT_WLOCK(backing_object);
+ tpindex +=
+ OFF_TO_IDX(tobject->backing_object_offset);
+ if (tobject != object)
+ VM_OBJECT_WUNLOCK(tobject);
+ tobject = backing_object;
+ if (!vm_object_advice_applies(tobject, advice))
+ goto next_pindex;
+ } while ((tm = vm_page_lookup(tobject, tpindex)) ==
+ NULL);
+ } else {
+next_page:
+ tm = m;
+ m = TAILQ_NEXT(m, listq);
}
/*
* If the page is not in a normal state, skip it.
*/
- if (m->valid != VM_PAGE_BITS_ALL)
- goto unlock_tobject;
- vm_page_lock(m);
- if (m->hold_count != 0 || m->wire_count != 0) {
- vm_page_unlock(m);
- goto unlock_tobject;
- }
- KASSERT((m->flags & PG_FICTITIOUS) == 0,
- ("vm_object_madvise: page %p is fictitious", m));
- KASSERT((m->oflags & VPO_UNMANAGED) == 0,
- ("vm_object_madvise: page %p is not managed", m));
- if (vm_page_busied(m)) {
+ if (tm->valid != VM_PAGE_BITS_ALL)
+ goto next_pindex;
+ vm_page_lock(tm);
+ if (tm->hold_count != 0 || tm->wire_count != 0) {
+ vm_page_unlock(tm);
+ goto next_pindex;
+ }
+ KASSERT((tm->flags & PG_FICTITIOUS) == 0,
+ ("vm_object_madvise: page %p is fictitious", tm));
+ KASSERT((tm->oflags & VPO_UNMANAGED) == 0,
+ ("vm_object_madvise: page %p is not managed", tm));
+ if (vm_page_busied(tm)) {
+ if (object != tobject)
+ VM_OBJECT_WUNLOCK(tobject);
+ VM_OBJECT_WUNLOCK(object);
if (advice == MADV_WILLNEED) {
/*
* Reference the page before unlocking and
* sleeping so that the page daemon is less
- * likely to reclaim it.
+ * likely to reclaim it.
*/
- vm_page_aflag_set(m, PGA_REFERENCED);
+ vm_page_aflag_set(tm, PGA_REFERENCED);
}
- if (object != tobject)
- VM_OBJECT_WUNLOCK(object);
- VM_OBJECT_WUNLOCK(tobject);
- vm_page_busy_sleep(m, "madvpo", false);
- m = NULL;
- VM_OBJECT_WLOCK(object);
+ vm_page_busy_sleep(tm, "madvpo", false);
goto relookup;
}
- vm_page_advise(m, advice);
- vm_page_unlock(m);
- if (advice == MADV_FREE && tobject->type == OBJT_SWAP)
- swap_pager_freespace(tobject, tpindex, 1);
-unlock_tobject:
+ vm_page_advise(tm, advice);
+ vm_page_unlock(tm);
+ vm_object_madvise_freespace(tobject, advice, tm->pindex, 1);
+next_pindex:
if (tobject != object)
VM_OBJECT_WUNLOCK(tobject);
}
More information about the svn-src-all
mailing list