svn commit: r248110 - in user/attilio/vmcontention/sys: conf fs/tmpfs i386/xen kern vm
Attilio Rao
attilio at FreeBSD.org
Sat Mar 9 17:25:34 UTC 2013
Author: attilio
Date: Sat Mar 9 17:25:31 2013
New Revision: 248110
URL: http://svnweb.freebsd.org/changeset/base/248110
Log:
Merge back vmc-playground into vmcontention.
vm_radix.{c, h} and _vm_radix.h are copied straight from the branch
to preserve history.
Added:
user/attilio/vmcontention/sys/vm/_vm_radix.h
- copied unchanged from r248107, user/attilio/vmc-playground/sys/vm/_vm_radix.h
Replaced:
user/attilio/vmcontention/sys/vm/vm_radix.c
- copied unchanged from r248107, user/attilio/vmc-playground/sys/vm/vm_radix.c
user/attilio/vmcontention/sys/vm/vm_radix.h
- copied unchanged from r248107, user/attilio/vmc-playground/sys/vm/vm_radix.h
Modified:
user/attilio/vmcontention/sys/conf/files
user/attilio/vmcontention/sys/fs/tmpfs/tmpfs_fifoops.c
user/attilio/vmcontention/sys/fs/tmpfs/tmpfs_vnops.c
user/attilio/vmcontention/sys/i386/xen/pmap.c
user/attilio/vmcontention/sys/kern/subr_uio.c
user/attilio/vmcontention/sys/kern/uipc_shm.c
user/attilio/vmcontention/sys/vm/device_pager.c
user/attilio/vmcontention/sys/vm/sg_pager.c
user/attilio/vmcontention/sys/vm/vm_fault.c
user/attilio/vmcontention/sys/vm/vm_init.c
user/attilio/vmcontention/sys/vm/vm_mmap.c
user/attilio/vmcontention/sys/vm/vm_object.c
user/attilio/vmcontention/sys/vm/vm_object.h
user/attilio/vmcontention/sys/vm/vm_page.c
user/attilio/vmcontention/sys/vm/vm_page.h
user/attilio/vmcontention/sys/vm/vm_reserv.c
user/attilio/vmcontention/sys/vm/vnode_pager.c
Modified: user/attilio/vmcontention/sys/conf/files
==============================================================================
--- user/attilio/vmcontention/sys/conf/files Sat Mar 9 16:58:19 2013 (r248109)
+++ user/attilio/vmcontention/sys/conf/files Sat Mar 9 17:25:31 2013 (r248110)
@@ -3630,7 +3630,7 @@ vm/vm_page.c standard
vm/vm_pageout.c standard
vm/vm_pager.c standard
vm/vm_phys.c standard
-vm/vm_radix.c standard
+vm/vm_radix.c standard
vm/vm_reserv.c standard
vm/vm_unix.c standard
vm/vm_zeroidle.c standard
Modified: user/attilio/vmcontention/sys/fs/tmpfs/tmpfs_fifoops.c
==============================================================================
--- user/attilio/vmcontention/sys/fs/tmpfs/tmpfs_fifoops.c Sat Mar 9 16:58:19 2013 (r248109)
+++ user/attilio/vmcontention/sys/fs/tmpfs/tmpfs_fifoops.c Sat Mar 9 17:25:31 2013 (r248110)
@@ -39,7 +39,6 @@
#include <sys/param.h>
#include <sys/filedesc.h>
#include <sys/proc.h>
-#include <sys/systm.h>
#include <sys/vnode.h>
#include <vm/vm.h>
Modified: user/attilio/vmcontention/sys/fs/tmpfs/tmpfs_vnops.c
==============================================================================
--- user/attilio/vmcontention/sys/fs/tmpfs/tmpfs_vnops.c Sat Mar 9 16:58:19 2013 (r248109)
+++ user/attilio/vmcontention/sys/fs/tmpfs/tmpfs_vnops.c Sat Mar 9 17:25:31 2013 (r248110)
@@ -638,34 +638,19 @@ tmpfs_mappedwrite(vm_object_t vobj, vm_o
VM_OBJECT_WLOCK(vobj);
lookupvpg:
- vpg = vm_radix_lookup(&vobj->rtree, idx, VM_RADIX_ANY);
- if (vpg != NULL) {
- if (vm_page_is_valid(vpg, offset, tlen)) {
- if ((vpg->oflags & VPO_BUSY) != 0) {
- /*
- * Reference the page before unlocking and
- * sleeping so that the page daemon is less
- * likely to reclaim it.
- */
- vm_page_reference(vpg);
- vm_page_sleep(vpg, "tmfsmw");
- goto lookupvpg;
- }
- vm_page_busy(vpg);
- vm_page_undirty(vpg);
- VM_OBJECT_UNLOCK(vobj);
- error = uiomove_fromphys(&vpg, offset, tlen, uio);
- } else {
- if (vpg->flags & PG_CACHED) {
- mtx_lock(&vm_page_queue_free_mtx);
- if (vpg->object == vobj)
- vm_page_cache_free(vpg);
- mtx_unlock(&vm_page_queue_free_mtx);
- }
- VM_OBJECT_UNLOCK(vobj);
- vpg = NULL;
+ if (((vpg = vm_page_lookup(vobj, idx)) != NULL) &&
+ vm_page_is_valid(vpg, offset, tlen)) {
+ if ((vpg->oflags & VPO_BUSY) != 0) {
+ /*
+ * Reference the page before unlocking and sleeping so
+ * that the page daemon is less likely to reclaim it.
+ */
+ vm_page_reference(vpg);
+ vm_page_sleep(vpg, "tmfsmw");
+ goto lookupvpg;
}
- } else
+ vm_page_busy(vpg);
+ vm_page_undirty(vpg);
VM_OBJECT_WUNLOCK(vobj);
error = uiomove_fromphys(&vpg, offset, tlen, uio);
} else {
Modified: user/attilio/vmcontention/sys/i386/xen/pmap.c
==============================================================================
--- user/attilio/vmcontention/sys/i386/xen/pmap.c Sat Mar 9 16:58:19 2013 (r248109)
+++ user/attilio/vmcontention/sys/i386/xen/pmap.c Sat Mar 9 17:25:31 2013 (r248110)
@@ -1335,7 +1335,8 @@ pmap_free_zero_pages(vm_page_t free)
while (free != NULL) {
m = free;
- free = m->right;
+ free = (void *)m->object;
+ m->object = NULL;
vm_page_free_zero(m);
}
}
@@ -1393,7 +1394,7 @@ _pmap_unwire_ptp(pmap_t pmap, vm_page_t
* Put page on a list so that it is released after
* *ALL* TLB shootdown is done
*/
- m->right = *free;
+ m->object = (void *)*free;
*free = m;
}
@@ -2090,7 +2091,7 @@ out:
}
if (m_pc == NULL && pv_vafree != 0 && free != NULL) {
m_pc = free;
- free = m_pc->right;
+ free = (void *)m_pc->object;
/* Recycle a freed page table page. */
m_pc->wire_count = 1;
atomic_add_int(&cnt.v_wire_count, 1);
Modified: user/attilio/vmcontention/sys/kern/subr_uio.c
==============================================================================
--- user/attilio/vmcontention/sys/kern/subr_uio.c Sat Mar 9 16:58:19 2013 (r248109)
+++ user/attilio/vmcontention/sys/kern/subr_uio.c Sat Mar 9 17:25:31 2013 (r248110)
@@ -85,7 +85,6 @@ vm_pgmoveco(vm_map_t mapa, vm_offset_t k
vm_map_entry_t entry;
vm_pindex_t upindex;
vm_prot_t prot;
- vm_page_bits_t vbits;
boolean_t wired;
KASSERT((uaddr & PAGE_MASK) == 0,
@@ -96,7 +95,6 @@ vm_pgmoveco(vm_map_t mapa, vm_offset_t k
* unwired in sf_buf_mext().
*/
kern_pg = PHYS_TO_VM_PAGE(vtophys(kaddr));
- vbits = kern_pg->valid;
kern_pg->valid = VM_PAGE_BITS_ALL;
KASSERT(kern_pg->queue == PQ_NONE && kern_pg->wire_count == 1,
("vm_pgmoveco: kern_pg is not correctly wired"));
@@ -107,13 +105,6 @@ vm_pgmoveco(vm_map_t mapa, vm_offset_t k
return(EFAULT);
}
VM_OBJECT_WLOCK(uobject);
- if (vm_page_insert(kern_pg, uobject, upindex) != 0) {
- kern_pg->valid = vbits;
- VM_OBJECT_UNLOCK(uobject);
- vm_map_lookup_done(map, entry);
- return(ENOMEM);
- }
- vm_page_dirty(kern_pg);
retry:
if ((user_pg = vm_page_lookup(uobject, upindex)) != NULL) {
if (vm_page_sleep_if_busy(user_pg, TRUE, "vm_pgmoveco"))
@@ -131,6 +122,8 @@ retry:
if (uobject->backing_object != NULL)
pmap_remove(map->pmap, uaddr, uaddr + PAGE_SIZE);
}
+ vm_page_insert(kern_pg, uobject, upindex);
+ vm_page_dirty(kern_pg);
VM_OBJECT_WUNLOCK(uobject);
vm_map_lookup_done(map, entry);
return(KERN_SUCCESS);
Modified: user/attilio/vmcontention/sys/kern/uipc_shm.c
==============================================================================
--- user/attilio/vmcontention/sys/kern/uipc_shm.c Sat Mar 9 16:58:19 2013 (r248109)
+++ user/attilio/vmcontention/sys/kern/uipc_shm.c Sat Mar 9 17:25:31 2013 (r248110)
@@ -279,8 +279,7 @@ shm_dotruncate(struct shmfd *shmfd, off_
if (base != 0) {
idx = OFF_TO_IDX(length);
retry:
- m = vm_radix_lookup(&object->rtree, idx,
- VM_RADIX_BLACK);
+ m = vm_page_lookup(object, idx);
if (m != NULL) {
if ((m->oflags & VPO_BUSY) != 0 ||
m->busy != 0) {
Copied: user/attilio/vmcontention/sys/vm/_vm_radix.h (from r248107, user/attilio/vmc-playground/sys/vm/_vm_radix.h)
==============================================================================
--- /dev/null 00:00:00 1970 (empty, because file is newly added)
+++ user/attilio/vmcontention/sys/vm/_vm_radix.h Sat Mar 9 17:25:31 2013 (r248110, copy of r248107, user/attilio/vmc-playground/sys/vm/_vm_radix.h)
@@ -0,0 +1,39 @@
+/*
+ * Copyright (c) 2011 Jeffrey Roberson <jeff at freebsd.org>
+ * Copyright (c) 2008 Mayur Shardul <mayur.shardul at gmail.com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ */
+
+#ifndef __VM_RADIX_H_
+#define __VM_RADIX_H_
+
+/*
+ * Radix tree root.
+ */
+struct vm_radix {
+ uintptr_t rt_root;
+};
+
+#endif /* !__VM_RADIX_H_ */
Modified: user/attilio/vmcontention/sys/vm/device_pager.c
==============================================================================
--- user/attilio/vmcontention/sys/vm/device_pager.c Sat Mar 9 16:58:19 2013 (r248109)
+++ user/attilio/vmcontention/sys/vm/device_pager.c Sat Mar 9 17:25:31 2013 (r248110)
@@ -55,8 +55,6 @@ __FBSDID("$FreeBSD$");
#include <vm/vm_phys.h>
#include <vm/uma.h>
-#include <machine/cpu.h>
-
static void dev_pager_init(void);
static vm_object_t dev_pager_alloc(void *, vm_ooffset_t, vm_prot_t,
vm_ooffset_t, struct ucred *);
@@ -302,7 +300,7 @@ old_dev_pager_fault(vm_object_t object,
struct file *fpop;
struct thread *td;
vm_memattr_t memattr;
- int i, ref, ret;
+ int ref, ret;
pidx = OFF_TO_IDX(offset);
memattr = object->memattr;
@@ -354,10 +352,7 @@ old_dev_pager_fault(vm_object_t object,
vm_page_free(*mres);
vm_page_unlock(*mres);
*mres = page;
- while (vm_page_insert(page, object, pidx) != 0) {
- for (i = 0; i < 10000000; i++)
- cpu_spinwait();
- }
+ vm_page_insert(page, object, pidx);
}
page->valid = VM_PAGE_BITS_ALL;
return (VM_PAGER_OK);
Modified: user/attilio/vmcontention/sys/vm/sg_pager.c
==============================================================================
--- user/attilio/vmcontention/sys/vm/sg_pager.c Sat Mar 9 16:58:19 2013 (r248109)
+++ user/attilio/vmcontention/sys/vm/sg_pager.c Sat Mar 9 17:25:31 2013 (r248110)
@@ -182,10 +182,6 @@ sg_pager_getpages(vm_object_t object, vm
/* Construct a new fake page. */
page = vm_page_getfake(paddr, memattr);
VM_OBJECT_WLOCK(object);
- if (vm_page_insert(page, object, offset) != 0) {
- vm_page_putfake(page);
- return (VM_PAGER_FAIL);
- }
TAILQ_INSERT_TAIL(&object->un_pager.sgp.sgp_pglist, page, pageq);
/* Free the original pages and insert this fake page into the object. */
@@ -194,6 +190,7 @@ sg_pager_getpages(vm_object_t object, vm
vm_page_free(m[i]);
vm_page_unlock(m[i]);
}
+ vm_page_insert(page, object, offset);
m[reqpage] = page;
page->valid = VM_PAGE_BITS_ALL;
Modified: user/attilio/vmcontention/sys/vm/vm_fault.c
==============================================================================
--- user/attilio/vmcontention/sys/vm/vm_fault.c Sat Mar 9 16:58:19 2013 (r248109)
+++ user/attilio/vmcontention/sys/vm/vm_fault.c Sat Mar 9 17:25:31 2013 (r248110)
@@ -741,7 +741,9 @@ vnode_locked:
* process'es object. The page is
* automatically made dirty.
*/
+ vm_page_lock(fs.m);
vm_page_rename(fs.m, fs.first_object, fs.first_pindex);
+ vm_page_unlock(fs.m);
vm_page_busy(fs.m);
fs.first_m = fs.m;
fs.m = NULL;
Modified: user/attilio/vmcontention/sys/vm/vm_init.c
==============================================================================
--- user/attilio/vmcontention/sys/vm/vm_init.c Sat Mar 9 16:58:19 2013 (r248109)
+++ user/attilio/vmcontention/sys/vm/vm_init.c Sat Mar 9 17:25:31 2013 (r248110)
@@ -82,7 +82,6 @@ __FBSDID("$FreeBSD$");
#include <vm/vm_kern.h>
#include <vm/vm_object.h>
#include <vm/vm_page.h>
-#include <vm/vm_radix.h>
#include <vm/vm_map.h>
#include <vm/vm_pager.h>
#include <vm/vm_extern.h>
@@ -124,7 +123,6 @@ vm_mem_init(dummy)
vm_object_init();
vm_map_startup();
kmem_init(virtual_avail, virtual_end);
- vm_radix_init();
pmap_init();
vm_pager_init();
}
Modified: user/attilio/vmcontention/sys/vm/vm_mmap.c
==============================================================================
--- user/attilio/vmcontention/sys/vm/vm_mmap.c Sat Mar 9 16:58:19 2013 (r248109)
+++ user/attilio/vmcontention/sys/vm/vm_mmap.c Sat Mar 9 17:25:31 2013 (r248110)
@@ -83,7 +83,6 @@ __FBSDID("$FreeBSD$");
#include <vm/vm_pageout.h>
#include <vm/vm_extern.h>
#include <vm/vm_page.h>
-#include <vm/vm_radix.h>
#include <vm/vnode_pager.h>
#ifdef HWPMC_HOOKS
@@ -914,15 +913,10 @@ RestartScan:
object->type == OBJT_VNODE) {
pindex = OFF_TO_IDX(current->offset +
(addr - current->start));
- m = vm_radix_lookup(&object->rtree,
- pindex, VM_RADIX_ANY);
-
- /* Lock just for consistency. */
- mtx_lock(&vm_page_queue_free_mtx);
- if (m != NULL &&
- (m->flags & PG_CACHED) != 0)
+ m = vm_page_lookup(object, pindex);
+ if (m == NULL &&
+ vm_page_is_cached(object, pindex))
mincoreinfo = MINCORE_INCORE;
- mtx_unlock(&vm_page_queue_free_mtx);
if (m != NULL && m->valid == 0)
m = NULL;
if (m != NULL)
Modified: user/attilio/vmcontention/sys/vm/vm_object.c
==============================================================================
--- user/attilio/vmcontention/sys/vm/vm_object.c Sat Mar 9 16:58:19 2013 (r248109)
+++ user/attilio/vmcontention/sys/vm/vm_object.c Sat Mar 9 17:25:31 2013 (r248110)
@@ -94,6 +94,7 @@ __FBSDID("$FreeBSD$");
#include <vm/swap_pager.h>
#include <vm/vm_kern.h>
#include <vm/vm_extern.h>
+#include <vm/vm_radix.h>
#include <vm/vm_reserv.h>
#include <vm/uma.h>
@@ -165,13 +166,10 @@ vm_object_zdtor(void *mem, int size, voi
vm_object_t object;
object = (vm_object_t)mem;
- KASSERT(object->resident_page_count == 0,
- ("object %p resident_page_count = %d",
- object, object->resident_page_count));
KASSERT(TAILQ_EMPTY(&object->memq),
("object %p has resident pages in its memq", object));
- KASSERT(object->root == NULL,
- ("object %p has resident pages in its tree", object));
+ KASSERT(object->rtree.rt_root == 0,
+ ("object %p has resident pages in its trie", object));
#if VM_NRESERVLEVEL > 0
KASSERT(LIST_EMPTY(&object->rvq),
("object %p has reservations",
@@ -183,6 +181,9 @@ vm_object_zdtor(void *mem, int size, voi
KASSERT(object->paging_in_progress == 0,
("object %p paging_in_progress = %d",
object, object->paging_in_progress));
+ KASSERT(object->resident_page_count == 0,
+ ("object %p resident_page_count = %d",
+ object, object->resident_page_count));
KASSERT(object->shadow_count == 0,
("object %p shadow_count = %d",
object, object->shadow_count));
@@ -199,11 +200,11 @@ vm_object_zinit(void *mem, int size, int
rw_init_flags(&object->lock, "vm object", RW_DUPOK);
/* These are true for any object that has been freed */
- object->root = NULL;
+ object->rtree.rt_root = 0;
object->paging_in_progress = 0;
object->resident_page_count = 0;
object->shadow_count = 0;
- object->cache = NULL;
+ object->cache.rt_root = 0;
return (0);
}
@@ -295,6 +296,8 @@ vm_object_init(void)
NULL,
#endif
vm_object_zinit, NULL, UMA_ALIGN_PTR, UMA_ZONE_VM|UMA_ZONE_NOFREE);
+
+ vm_radix_init();
}
void
@@ -327,7 +330,7 @@ vm_object_set_memattr(vm_object_t object
case OBJT_SG:
case OBJT_SWAP:
case OBJT_VNODE:
- if (object->resident_page_count == 0)
+ if (!TAILQ_EMPTY(&object->memq))
return (KERN_FAILURE);
break;
case OBJT_DEAD:
@@ -673,11 +676,7 @@ vm_object_destroy(vm_object_t object)
void
vm_object_terminate(vm_object_t object)
{
- vm_page_t pa[VM_RADIX_STACK];
- vm_page_t p;
- vm_pindex_t start;
- u_int exhausted;
- int n, i;
+ vm_page_t p, p_next;
VM_OBJECT_ASSERT_WLOCKED(object);
@@ -722,73 +721,36 @@ vm_object_terminate(vm_object_t object)
* from the object. Rather than incrementally removing each page from
* the object, the page and object are reset to any empty state.
*/
- start = 0;
- exhausted = 0;
- while (exhausted == 0 && (n = vm_radix_lookupn(&object->rtree, start,
- 0, VM_RADIX_ANY, (void **)pa, VM_RADIX_STACK, &start,
- &exhausted)) != 0) {
- for (i = 0; i < n; i++) {
- p = pa[i];
- /*
- * Another thread may allocate this cached page from
- * the queue before we acquire the page queue free
- * mtx.
- */
- if (p->flags & PG_CACHED) {
- mtx_lock(&vm_page_queue_free_mtx);
- if (p->object == object) {
- p->object = NULL;
- p->valid = 0;
- /* Clear PG_CACHED and set PG_FREE. */
- p->flags ^= PG_CACHED | PG_FREE;
- cnt.v_cache_count--;
- cnt.v_free_count++;
- }
- mtx_unlock(&vm_page_queue_free_mtx);
- continue;
- } else if (p->object != object)
- continue;
- KASSERT(!p->busy && (p->oflags & VPO_BUSY) == 0,
- ("vm_object_terminate: freeing busy page %p", p));
- vm_page_lock(p);
- /*
- * Optimize the page's removal from the object by
- * resetting its "object" field. Specifically, if
- * the page is not wired, then the effect of this
- * assignment is that vm_page_free()'s call to
- * vm_page_remove() will return immediately without
- * modifying the page or the object.
- * Anyway, the radix tree cannot be accessed anymore
- * from within the object, thus all the nodes need
- * to be reclaimed later on.
- */
- p->object = NULL;
- if (p->wire_count == 0) {
- vm_page_free(p);
- PCPU_INC(cnt.v_pfree);
- }
- vm_page_unlock(p);
+ TAILQ_FOREACH_SAFE(p, &object->memq, listq, p_next) {
+ KASSERT(!p->busy && (p->oflags & VPO_BUSY) == 0,
+ ("vm_object_terminate: freeing busy page %p", p));
+ vm_page_lock(p);
+ /*
+ * Optimize the page's removal from the object by resetting
+ * its "object" field. Specifically, if the page is not
+ * wired, then the effect of this assignment is that
+ * vm_page_free()'s call to vm_page_remove() will return
+ * immediately without modifying the page or the object.
+ */
+ p->object = NULL;
+ if (p->wire_count == 0) {
+ vm_page_free(p);
+ PCPU_INC(cnt.v_pfree);
}
- if (n < VM_RADIX_STACK)
- break;
+ vm_page_unlock(p);
}
- vm_radix_reclaim_allnodes(&object->rtree);
/*
* If the object contained any pages, then reset it to an empty state.
* None of the object's fields, including "resident_page_count", were
* modified by the preceding loop.
*/
if (object->resident_page_count != 0) {
+ vm_radix_reclaim_allnodes(&object->rtree);
TAILQ_INIT(&object->memq);
object->resident_page_count = 0;
if (object->type == OBJT_VNODE)
vdrop(object->handle);
}
- if (object->cached_page_count != 0) {
- object->cached_page_count = 0;
- if (object->type == OBJT_VNODE)
- vdrop(object->handle);
- }
#if VM_NRESERVLEVEL > 0
if (__predict_false(!LIST_EMPTY(&object->rvq)))
@@ -1306,13 +1268,10 @@ vm_object_shadow(
void
vm_object_split(vm_map_entry_t entry)
{
- vm_page_t ma[VM_RADIX_STACK];
- vm_page_t m;
+ vm_page_t m, m_next;
vm_object_t orig_object, new_object, source;
- vm_pindex_t idx, offidxstart, start;
+ vm_pindex_t idx, offidxstart;
vm_size_t size;
- u_int exhausted;
- int i, n;
orig_object = entry->object.vm_object;
if (orig_object->type != OBJT_DEFAULT && orig_object->type != OBJT_SWAP)
@@ -1365,65 +1324,46 @@ vm_object_split(vm_map_entry_t entry)
("orig_object->charge < 0"));
orig_object->charge -= ptoa(size);
}
- start = offidxstart;
retry:
- exhausted = 0;
- while (exhausted == 0 && (n = vm_radix_lookupn(&orig_object->rtree,
- start, offidxstart + size, VM_RADIX_ANY, (void **)ma,
- VM_RADIX_STACK, &start, &exhausted)) != 0) {
- for (i = 0; i < n; i++) {
- m = ma[i];
- idx = m->pindex - offidxstart;
- if (m->flags & PG_CACHED) {
- mtx_lock(&vm_page_queue_free_mtx);
- if (m->object == orig_object)
- vm_page_cache_rename(m, new_object,
- idx);
- mtx_unlock(&vm_page_queue_free_mtx);
- continue;
- } else if (m->object != orig_object)
- continue;
- /*
- * We must wait for pending I/O to complete before
- * we can rename the page.
- *
- * We do not have to VM_PROT_NONE the page as mappings
- * should not be changed by this operation.
- */
- if ((m->oflags & VPO_BUSY) || m->busy) {
- start = m->pindex;
- VM_OBJECT_UNLOCK(new_object);
- m->oflags |= VPO_WANTED;
- VM_OBJECT_SLEEP(orig_object, m, PVM, "spltwt", 0);
- VM_OBJECT_LOCK(new_object);
- goto retry;
- }
+ m = vm_page_find_least(orig_object, offidxstart);
+ for (; m != NULL && (idx = m->pindex - offidxstart) < size;
+ m = m_next) {
+ m_next = TAILQ_NEXT(m, listq);
+
+ /*
+ * We must wait for pending I/O to complete before we can
+ * rename the page.
+ *
+ * We do not have to VM_PROT_NONE the page as mappings should
+ * not be changed by this operation.
+ */
+ if ((m->oflags & VPO_BUSY) || m->busy) {
+ VM_OBJECT_WUNLOCK(new_object);
+ m->oflags |= VPO_WANTED;
+ VM_OBJECT_SLEEP(orig_object, m, PVM, "spltwt", 0);
+ VM_OBJECT_WLOCK(new_object);
+ goto retry;
+ }
#if VM_NRESERVLEVEL > 0
- /*
- * If some of the reservation's allocated pages remain
- * with the original object, then transferring the
- * reservation to the new object is neither
- * particularly beneficial nor particularly harmful as
- * compared to leaving the reservation with the
- * original object. If, however, all of the
- * reservation's allocated pages are transferred to
- * the new object, then transferring the reservation
- * is typically beneficial. Determining which of
- * these two cases applies would be more costly than
- * unconditionally renaming the reservation.
- */
- vm_reserv_rename(m, new_object, orig_object,
- offidxstart);
+ /*
+ * If some of the reservation's allocated pages remain with
+ * the original object, then transferring the reservation to
+ * the new object is neither particularly beneficial nor
+ * particularly harmful as compared to leaving the reservation
+ * with the original object. If, however, all of the
+ * reservation's allocated pages are transferred to the new
+ * object, then transferring the reservation is typically
+ * beneficial. Determining which of these two cases applies
+ * would be more costly than unconditionally renaming the
+ * reservation.
+ */
+ vm_reserv_rename(m, new_object, orig_object, offidxstart);
#endif
- vm_page_rename(m, new_object, idx);
- /*
- * page automatically made dirty by rename and
- * cache handled
- */
- vm_page_busy(m);
- }
- if (n < VM_RADIX_STACK)
- break;
+ vm_page_lock(m);
+ vm_page_rename(m, new_object, idx);
+ vm_page_unlock(m);
+ /* page automatically made dirty by rename and cache handled */
+ vm_page_busy(m);
}
if (orig_object->type == OBJT_SWAP) {
/*
@@ -1431,6 +1371,19 @@ retry:
* and new_object's locks are released and reacquired.
*/
swap_pager_copy(orig_object, new_object, offidxstart, 0);
+
+ /*
+ * Transfer any cached pages from orig_object to new_object.
+ * If swap_pager_copy() found swapped out pages within the
+ * specified range of orig_object, then it changed
+ * new_object's type to OBJT_SWAP when it transferred those
+ * pages to new_object. Otherwise, new_object's type
+ * should still be OBJT_DEFAULT and orig_object should not
+ * contain any cached pages within the specified range.
+ */
+ if (__predict_false(!vm_object_cache_is_empty(orig_object)))
+ vm_page_cache_transfer(orig_object, offidxstart,
+ new_object);
}
VM_OBJECT_WUNLOCK(orig_object);
TAILQ_FOREACH(m, &new_object->memq, listq)
@@ -1449,14 +1402,10 @@ retry:
static int
vm_object_backing_scan(vm_object_t object, int op)
{
- vm_page_t pa[VM_RADIX_STACK];
+ int r = 1;
vm_page_t p;
vm_object_t backing_object;
- vm_pindex_t backing_offset_index, new_pindex;
- vm_pindex_t start;
- u_int exhausted;
- int color, i, n;
- int r = 1;
+ vm_pindex_t backing_offset_index;
VM_OBJECT_ASSERT_WLOCKED(object);
VM_OBJECT_ASSERT_WLOCKED(object->backing_object);
@@ -1484,41 +1433,15 @@ vm_object_backing_scan(vm_object_t objec
if (op & OBSC_COLLAPSE_WAIT) {
vm_object_set_flag(backing_object, OBJ_DEAD);
}
- color = VM_RADIX_BLACK;
- if (op & OBSC_COLLAPSE_WAIT)
- color |= VM_RADIX_RED;
+
/*
* Our scan
*/
-restart:
- start = 0;
- i = n = VM_RADIX_STACK;
- exhausted = 0;
- for (;;) {
- if (i == n) {
- if (n < VM_RADIX_STACK)
- break;
- if (exhausted != 0 ||
- (n = vm_radix_lookupn(&backing_object->rtree,
- start, 0, color, (void **)pa, VM_RADIX_STACK,
- &start, &exhausted)) == 0)
- break;
- i = 0;
- }
- p = pa[i++];
- /*
- * Free cached pages. XXX Why? Emulating old behavior here.
- */
- if (p->flags & PG_CACHED) {
- mtx_lock(&vm_page_queue_free_mtx);
- if (p->object == backing_object)
- vm_page_cache_free(p);
- mtx_unlock(&vm_page_queue_free_mtx);
- continue;
- } else if (p->object != backing_object)
- continue;
+ p = TAILQ_FIRST(&backing_object->memq);
+ while (p) {
+ vm_page_t next = TAILQ_NEXT(p, listq);
+ vm_pindex_t new_pindex = p->pindex - backing_offset_index;
- new_pindex = p->pindex - backing_offset_index;
if (op & OBSC_TEST_ALL_SHADOWED) {
vm_page_t pp;
@@ -1530,9 +1453,13 @@ restart:
* note that we do not busy the backing object's
* page.
*/
- if (p->pindex < backing_offset_index ||
- new_pindex >= object->size)
+ if (
+ p->pindex < backing_offset_index ||
+ new_pindex >= object->size
+ ) {
+ p = next;
continue;
+ }
/*
* See if the parent has the page or if the parent's
@@ -1561,9 +1488,12 @@ restart:
vm_page_t pp;
if (op & OBSC_COLLAPSE_NOWAIT) {
- if ((p->oflags & VPO_BUSY) || !p->valid ||
- p->busy)
+ if ((p->oflags & VPO_BUSY) ||
+ !p->valid ||
+ p->busy) {
+ p = next;
continue;
+ }
} else if (op & OBSC_COLLAPSE_WAIT) {
if ((p->oflags & VPO_BUSY) || p->busy) {
VM_OBJECT_WUNLOCK(object);
@@ -1579,7 +1509,8 @@ restart:
* should not have changed so we
* just restart our scan.
*/
- goto restart;
+ p = TAILQ_FIRST(&backing_object->memq);
+ continue;
}
}
@@ -1615,6 +1546,7 @@ restart:
else
vm_page_remove(p);
vm_page_unlock(p);
+ p = next;
continue;
}
@@ -1634,6 +1566,7 @@ restart:
* page before we can (re)lock the parent.
* Hence we can get here.
*/
+ p = next;
continue;
}
if (
@@ -1655,6 +1588,7 @@ restart:
else
vm_page_remove(p);
vm_page_unlock(p);
+ p = next;
continue;
}
@@ -1673,9 +1607,12 @@ restart:
* If the page was mapped to a process, it can remain
* mapped through the rename.
*/
+ vm_page_lock(p);
vm_page_rename(p, object, new_pindex);
+ vm_page_unlock(p);
/* page automatically made dirty by rename */
}
+ p = next;
}
return (r);
}
@@ -1789,6 +1726,13 @@ vm_object_collapse(vm_object_t object)
backing_object,
object,
OFF_TO_IDX(object->backing_object_offset), TRUE);
+
+ /*
+ * Free any cached pages from backing_object.
+ */
+ if (__predict_false(
+ !vm_object_cache_is_empty(backing_object)))
+ vm_page_cache_free(backing_object, 0, 0);
}
/*
* Object now shadows whatever backing_object did.
@@ -1909,112 +1853,80 @@ void
vm_object_page_remove(vm_object_t object, vm_pindex_t start, vm_pindex_t end,
int options)
{
- struct vnode *vp;
- vm_page_t pa[VM_RADIX_STACK];
- vm_page_t p;
- u_int exhausted;
- int i, n;
+ vm_page_t p, next;
int wirings;
VM_OBJECT_ASSERT_WLOCKED(object);
KASSERT((object->flags & OBJ_UNMANAGED) == 0 ||
(options & (OBJPR_CLEANONLY | OBJPR_NOTMAPPED)) == OBJPR_NOTMAPPED,
("vm_object_page_remove: illegal options for object %p", object));
- if (object->resident_page_count == 0 && object->cached_page_count == 0)
- return;
- vp = NULL;
+ if (object->resident_page_count == 0)
+ goto skipmemq;
vm_object_pip_add(object, 1);
-restart:
- exhausted = 0;
- while (exhausted == 0 && (n = vm_radix_lookupn(&object->rtree, start,
- end, VM_RADIX_ANY, (void **)pa, VM_RADIX_STACK, &start,
- &exhausted)) != 0) {
- for (i = 0; i < n; i++) {
- p = pa[i];
- /*
- * Another thread may allocate this cached page from
- * the queue before we acquire the page queue free
- * mtx.
- */
- if (p->flags & PG_CACHED) {
- mtx_lock(&vm_page_queue_free_mtx);
- if (p->object == object) {
- vm_page_cache_free(p);
- if (object->type == OBJT_VNODE &&
- object->cached_page_count == 0)
- vp = object->handle;
- }
- mtx_unlock(&vm_page_queue_free_mtx);
- continue;
- } else if (p->object != object)
- continue;
- /*
- * If the page is wired for any reason besides
- * the existence of managed, wired mappings, then
- * it cannot be freed. For example, fictitious
- * pages, which represent device memory, are
- * inherently wired and cannot be freed. They can,
- * however, be invalidated if the option
- * OBJPR_CLEANONLY is not specified.
- */
- vm_page_lock(p);
- if ((wirings = p->wire_count) != 0 &&
- (wirings = pmap_page_wired_mappings(p)) !=
- p->wire_count) {
- if ((options & OBJPR_NOTMAPPED) == 0) {
- pmap_remove_all(p);
- /*
- * Account for removal of wired
- * mappings.
- */
- if (wirings != 0)
- p->wire_count -= wirings;
- }
- if ((options & OBJPR_CLEANONLY) == 0) {
- p->valid = 0;
- vm_page_undirty(p);
- }
- vm_page_unlock(p);
- continue;
- }
- if (vm_page_sleep_if_busy(p, TRUE, "vmopar")) {
- start = p->pindex;
- goto restart;
- }
- KASSERT((p->flags & PG_FICTITIOUS) == 0,
- ("vm_object_page_remove: page %p is fictitious",
- p));
- if ((options & OBJPR_CLEANONLY) != 0 && p->valid != 0) {
- if ((options & OBJPR_NOTMAPPED) == 0)
- pmap_remove_write(p);
- if (p->dirty) {
- vm_page_unlock(p);
- continue;
- }
- }
+again:
+ p = vm_page_find_least(object, start);
+
+ /*
+ * Here, the variable "p" is either (1) the page with the least pindex
+ * greater than or equal to the parameter "start" or (2) NULL.
+ */
+ for (; p != NULL && (p->pindex < end || end == 0); p = next) {
+ next = TAILQ_NEXT(p, listq);
+
+ /*
+ * If the page is wired for any reason besides the existence
+ * of managed, wired mappings, then it cannot be freed. For
+ * example, fictitious pages, which represent device memory,
+ * are inherently wired and cannot be freed. They can,
+ * however, be invalidated if the option OBJPR_CLEANONLY is
+ * not specified.
+ */
+ vm_page_lock(p);
+ if ((wirings = p->wire_count) != 0 &&
+ (wirings = pmap_page_wired_mappings(p)) != p->wire_count) {
if ((options & OBJPR_NOTMAPPED) == 0) {
pmap_remove_all(p);
/* Account for removal of wired mappings. */
- if (wirings != 0) {
- KASSERT(p->wire_count == wirings,
- ("inconsistent wire count %d %d %p",
- p->wire_count, wirings, p));
- p->wire_count = 0;
- atomic_subtract_int(&cnt.v_wire_count,
- 1);
- }
if (wirings != 0)
p->wire_count -= wirings;
}
- vm_page_free(p);
+ if ((options & OBJPR_CLEANONLY) == 0) {
+ p->valid = 0;
+ vm_page_undirty(p);
+ }
vm_page_unlock(p);
+ continue;
}
- if (n < VM_RADIX_STACK)
- break;
+ if (vm_page_sleep_if_busy(p, TRUE, "vmopar"))
+ goto again;
+ KASSERT((p->flags & PG_FICTITIOUS) == 0,
+ ("vm_object_page_remove: page %p is fictitious", p));
+ if ((options & OBJPR_CLEANONLY) != 0 && p->valid != 0) {
+ if ((options & OBJPR_NOTMAPPED) == 0)
+ pmap_remove_write(p);
+ if (p->dirty) {
+ vm_page_unlock(p);
+ continue;
+ }
+ }
+ if ((options & OBJPR_NOTMAPPED) == 0) {
+ pmap_remove_all(p);
+ /* Account for removal of wired mappings. */
+ if (wirings != 0) {
+ KASSERT(p->wire_count == wirings,
+ ("inconsistent wire count %d %d %p",
+ p->wire_count, wirings, p));
+ p->wire_count = 0;
+ atomic_subtract_int(&cnt.v_wire_count, 1);
+ }
+ }
+ vm_page_free(p);
+ vm_page_unlock(p);
}
vm_object_pip_wakeup(object);
- if (vp)
- vdrop(vp);
+skipmemq:
+ if (__predict_false(!vm_object_cache_is_empty(object)))
+ vm_page_cache_free(object, start, end);
}
/*
@@ -2392,9 +2304,8 @@ DB_SHOW_COMMAND(object, vm_object_print_
db_printf(",");
count++;
- db_printf("(off=0x%jx,page=0x%jx,obj=%p,flags=0x%X)",
- (uintmax_t)p->pindex, (uintmax_t)VM_PAGE_TO_PHYS(p),
- p->object, p->flags);
+ db_printf("(off=0x%jx,page=0x%jx)",
+ (uintmax_t)p->pindex, (uintmax_t)VM_PAGE_TO_PHYS(p));
}
if (count != 0)
db_printf("\n");
Modified: user/attilio/vmcontention/sys/vm/vm_object.h
==============================================================================
--- user/attilio/vmcontention/sys/vm/vm_object.h Sat Mar 9 16:58:19 2013 (r248109)
+++ user/attilio/vmcontention/sys/vm/vm_object.h Sat Mar 9 17:25:31 2013 (r248110)
@@ -72,7 +72,7 @@
#include <sys/_mutex.h>
#include <sys/_rwlock.h>
-#include <vm/vm_radix.h>
+#include <vm/_vm_radix.h>
/*
* Types defined:
@@ -81,10 +81,10 @@
*
* The root of cached pages pool is protected by both the per-object lock
* and the free pages queue mutex.
- * On insert in the cache splay tree, the per-object lock is expected
+ * On insert in the cache radix trie, the per-object lock is expected
* to be already held and the free pages queue mutex will be
* acquired during the operation too.
- * On remove and lookup from the cache splay tree, only the free
+ * On remove and lookup from the cache radix trie, only the free
* pages queue mutex is expected to be locked.
* These rules allow for reliably checking for the presence of cached
* pages with only the per-object lock held, thereby reducing contention
@@ -103,7 +103,7 @@ struct vm_object {
LIST_HEAD(, vm_object) shadow_head; /* objects that this is a shadow for */
LIST_ENTRY(vm_object) shadow_list; /* chain of shadow objects */
TAILQ_HEAD(, vm_page) memq; /* list of resident pages */
- struct vm_radix rtree; /* root of the resident page radix index tree */
+ struct vm_radix rtree; /* root of the resident page radix trie*/
vm_pindex_t size; /* Object size */
int generation; /* generation ID */
int ref_count; /* How many refs?? */
@@ -114,11 +114,11 @@ struct vm_object {
u_short pg_color; /* (c) color of first page in obj */
u_int paging_in_progress; /* Paging (in or out) so don't collapse or destroy */
int resident_page_count; /* number of resident pages */
- int cached_page_count; /* number of cached pages */
struct vm_object *backing_object; /* object that I'm a shadow of */
vm_ooffset_t backing_object_offset;/* Offset in backing object */
TAILQ_ENTRY(vm_object) pager_object_list; /* list of all objects of this pager type */
LIST_HEAD(, vm_reserv) rvq; /* list of reservations */
+ struct vm_radix cache; /* (o + f) root of the cache page radix trie */
void *handle;
*** DIFF OUTPUT TRUNCATED AT 1000 LINES ***
More information about the svn-src-user
mailing list