svn commit: r248260 - in user/attilio/vmobj-readlock/sys: cddl/contrib/opensolaris/uts/common/fs/zfs dev/drm2/i915 dev/drm2/ttm fs/fuse fs/tmpfs kern vm
Attilio Rao
attilio at FreeBSD.org
Thu Mar 14 01:53:36 UTC 2013
Author: attilio
Date: Thu Mar 14 01:53:32 2013
New Revision: 248260
URL: http://svnweb.freebsd.org/changeset/base/248260
Log:
Move the page handling of sleeping on a page for a specific condition
under the protection of the page lock.
In detail:
1) VPO_WANTED is moved to be a per-page flag, removing it and introducing
PG_WANTED.
2) The function vm_page_sleep_onpage() is introduced to work as an
underlying version of vm_page_sleep() / vm_page_sleep_if_busy() to
specify priority and timo of the sleeping operation.
3) Convert all the current sleepers on the page to not use anymore
VM_OBJECT_SLEEP() but use vm_page_sleep(). In one case, where special
timo, priority and return value are needed, vm_page_sleep_onpage() is
used directly.
4) The operation results without races because the object lock is now
dropped after the page-lock is held. This creates a chain of trust.
Further notes:
- vm_page_sleep() and neighboor functions now always want the page-lock
held. The semantic of the unlocking is left unchanged.
- The burden to deal with page-locking is on the callers of the function.
This is because soon more per-page attributes will be moved under the
per-page lock, making the protection path a single critical section.
- The vm_page_sleep_if_busy() is temporary made an hard function. This is
to avoid namespace pollution by the object locking. With more changes
the object locking won't be necessary anymore there and it will be made
inline again as it should be.
- vm_page_flash() and vm_page_sleep() lost the necessity to be called
with the object lock held.
Sponsored by: EMC / Isilon storage division
Modified:
user/attilio/vmobj-readlock/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c
user/attilio/vmobj-readlock/sys/dev/drm2/i915/i915_gem.c
user/attilio/vmobj-readlock/sys/dev/drm2/ttm/ttm_bo_vm.c
user/attilio/vmobj-readlock/sys/fs/fuse/fuse_vnops.c
user/attilio/vmobj-readlock/sys/fs/tmpfs/tmpfs_subr.c
user/attilio/vmobj-readlock/sys/fs/tmpfs/tmpfs_vnops.c
user/attilio/vmobj-readlock/sys/kern/subr_uio.c
user/attilio/vmobj-readlock/sys/kern/uipc_shm.c
user/attilio/vmobj-readlock/sys/kern/vfs_bio.c
user/attilio/vmobj-readlock/sys/vm/phys_pager.c
user/attilio/vmobj-readlock/sys/vm/swap_pager.c
user/attilio/vmobj-readlock/sys/vm/vm_fault.c
user/attilio/vmobj-readlock/sys/vm/vm_object.c
user/attilio/vmobj-readlock/sys/vm/vm_object.h
user/attilio/vmobj-readlock/sys/vm/vm_page.c
user/attilio/vmobj-readlock/sys/vm/vm_page.h
user/attilio/vmobj-readlock/sys/vm/vnode_pager.c
Modified: user/attilio/vmobj-readlock/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c
==============================================================================
--- user/attilio/vmobj-readlock/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c Thu Mar 14 00:27:53 2013 (r248259)
+++ user/attilio/vmobj-readlock/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c Thu Mar 14 01:53:32 2013 (r248260)
@@ -342,7 +342,10 @@ page_busy(vnode_t *vp, int64_t start, in
* likely to reclaim it.
*/
vm_page_reference(pp);
+ vm_page_lock(pp);
+ zfs_vmobject_wunlock(obj);
vm_page_sleep(pp, "zfsmwb");
+ zfs_vmobject_wlock(obj);
continue;
}
} else {
@@ -390,7 +393,10 @@ page_hold(vnode_t *vp, int64_t start)
* likely to reclaim it.
*/
vm_page_reference(pp);
+ vm_page_lock(pp);
+ zfs_vmobject_wunlock(obj);
vm_page_sleep(pp, "zfsmwb");
+ zfs_vmobject_wlock(obj);
continue;
}
Modified: user/attilio/vmobj-readlock/sys/dev/drm2/i915/i915_gem.c
==============================================================================
--- user/attilio/vmobj-readlock/sys/dev/drm2/i915/i915_gem.c Thu Mar 14 00:27:53 2013 (r248259)
+++ user/attilio/vmobj-readlock/sys/dev/drm2/i915/i915_gem.c Thu Mar 14 01:53:32 2013 (r248260)
@@ -1421,7 +1421,10 @@ unlocked_vmobj:
if ((m->flags & VPO_BUSY) != 0) {
DRM_UNLOCK(dev);
+ vm_page_lock(m);
+ VM_OBJECT_WUNLOCK(vm_obj);
vm_page_sleep(m, "915pbs");
+ VM_OBJECT_WLOCK(vm_obj);
goto retry;
}
m->valid = VM_PAGE_BITS_ALL;
@@ -2309,14 +2312,17 @@ i915_gem_release_mmap(struct drm_i915_ge
if (devobj != NULL) {
page_count = OFF_TO_IDX(obj->base.size);
- VM_OBJECT_WLOCK(devobj);
retry:
+ VM_OBJECT_WLOCK(devobj);
for (i = 0; i < page_count; i++) {
m = vm_page_lookup(devobj, i);
if (m == NULL)
continue;
+ vm_page_lock(m);
+ VM_OBJECT_WUNLOCK(devobj);
if (vm_page_sleep_if_busy(m, true, "915unm"))
goto retry;
+ vm_page_unlock(m);
cdev_pager_free_page(devobj, m);
}
VM_OBJECT_WUNLOCK(devobj);
Modified: user/attilio/vmobj-readlock/sys/dev/drm2/ttm/ttm_bo_vm.c
==============================================================================
--- user/attilio/vmobj-readlock/sys/dev/drm2/ttm/ttm_bo_vm.c Thu Mar 14 00:27:53 2013 (r248259)
+++ user/attilio/vmobj-readlock/sys/dev/drm2/ttm/ttm_bo_vm.c Thu Mar 14 01:53:32 2013 (r248260)
@@ -215,7 +215,10 @@ reserve:
VM_OBJECT_WLOCK(vm_obj);
if ((m->flags & VPO_BUSY) != 0) {
+ vm_page_lock(m);
+ VM_OBJECT_WUNLOCK(vm_obj);
vm_page_sleep(m, "ttmpbs");
+ VM_OBJECT_WLOCK(vm_obj);
ttm_mem_io_unlock(man);
ttm_bo_unreserve(bo);
goto retry;
Modified: user/attilio/vmobj-readlock/sys/fs/fuse/fuse_vnops.c
==============================================================================
--- user/attilio/vmobj-readlock/sys/fs/fuse/fuse_vnops.c Thu Mar 14 00:27:53 2013 (r248259)
+++ user/attilio/vmobj-readlock/sys/fs/fuse/fuse_vnops.c Thu Mar 14 01:53:32 2013 (r248260)
@@ -1868,15 +1868,12 @@ fuse_vnop_getpages(struct vop_getpages_a
* now tell them that it is ok to use.
*/
if (!error) {
- if (m->oflags & VPO_WANTED) {
- fuse_vm_page_lock(m);
+ fuse_vm_page_lock(m);
+ if (m->flags & PG_WANTED)
vm_page_activate(m);
- fuse_vm_page_unlock(m);
- } else {
- fuse_vm_page_lock(m);
+ else
vm_page_deactivate(m);
- fuse_vm_page_unlock(m);
- }
+ fuse_vm_page_unlock(m);
vm_page_wakeup(m);
} else {
fuse_vm_page_lock(m);
Modified: user/attilio/vmobj-readlock/sys/fs/tmpfs/tmpfs_subr.c
==============================================================================
--- user/attilio/vmobj-readlock/sys/fs/tmpfs/tmpfs_subr.c Thu Mar 14 00:27:53 2013 (r248259)
+++ user/attilio/vmobj-readlock/sys/fs/tmpfs/tmpfs_subr.c Thu Mar 14 01:53:32 2013 (r248260)
@@ -1285,7 +1285,10 @@ retry:
if (m != NULL) {
if ((m->oflags & VPO_BUSY) != 0 ||
m->busy != 0) {
+ vm_page_lock(m);
+ VM_OBJECT_WUNLOCK(uobj);
vm_page_sleep(m, "tmfssz");
+ VM_OBJECT_WLOCK(uobj);
goto retry;
}
MPASS(m->valid == VM_PAGE_BITS_ALL);
Modified: user/attilio/vmobj-readlock/sys/fs/tmpfs/tmpfs_vnops.c
==============================================================================
--- user/attilio/vmobj-readlock/sys/fs/tmpfs/tmpfs_vnops.c Thu Mar 14 00:27:53 2013 (r248259)
+++ user/attilio/vmobj-readlock/sys/fs/tmpfs/tmpfs_vnops.c Thu Mar 14 01:53:32 2013 (r248260)
@@ -513,8 +513,8 @@ tmpfs_mappedread(vm_object_t vobj, vm_ob
offset = addr & PAGE_MASK;
tlen = MIN(PAGE_SIZE - offset, len);
- VM_OBJECT_WLOCK(vobj);
lookupvpg:
+ VM_OBJECT_WLOCK(vobj);
if (((m = vm_page_lookup(vobj, idx)) != NULL) &&
vm_page_is_valid(m, offset, tlen)) {
if ((m->oflags & VPO_BUSY) != 0) {
@@ -523,6 +523,8 @@ lookupvpg:
* that the page daemon is less likely to reclaim it.
*/
vm_page_reference(m);
+ vm_page_lock(m);
+ VM_OBJECT_WUNLOCK(vobj);
vm_page_sleep(m, "tmfsmr");
goto lookupvpg;
}
@@ -542,6 +544,8 @@ lookupvpg:
* that the page daemon is less likely to reclaim it.
*/
vm_page_reference(m);
+ vm_page_lock(m);
+ VM_OBJECT_WUNLOCK(vobj);
vm_page_sleep(m, "tmfsmr");
goto lookupvpg;
}
@@ -636,8 +640,8 @@ tmpfs_mappedwrite(vm_object_t vobj, vm_o
offset = addr & PAGE_MASK;
tlen = MIN(PAGE_SIZE - offset, len);
- VM_OBJECT_WLOCK(vobj);
lookupvpg:
+ VM_OBJECT_WLOCK(vobj);
if (((vpg = vm_page_lookup(vobj, idx)) != NULL) &&
vm_page_is_valid(vpg, offset, tlen)) {
if ((vpg->oflags & VPO_BUSY) != 0) {
@@ -646,6 +650,8 @@ lookupvpg:
* that the page daemon is less likely to reclaim it.
*/
vm_page_reference(vpg);
+ vm_page_lock(vpg);
+ VM_OBJECT_WUNLOCK(vobj);
vm_page_sleep(vpg, "tmfsmw");
goto lookupvpg;
}
Modified: user/attilio/vmobj-readlock/sys/kern/subr_uio.c
==============================================================================
--- user/attilio/vmobj-readlock/sys/kern/subr_uio.c Thu Mar 14 00:27:53 2013 (r248259)
+++ user/attilio/vmobj-readlock/sys/kern/subr_uio.c Thu Mar 14 01:53:32 2013 (r248260)
@@ -107,9 +107,9 @@ vm_pgmoveco(vm_map_t mapa, vm_offset_t k
VM_OBJECT_WLOCK(uobject);
retry:
if ((user_pg = vm_page_lookup(uobject, upindex)) != NULL) {
+ vm_page_lock(user_pg);
if (vm_page_sleep_if_busy(user_pg, TRUE, "vm_pgmoveco"))
goto retry;
- vm_page_lock(user_pg);
pmap_remove_all(user_pg);
vm_page_free(user_pg);
vm_page_unlock(user_pg);
Modified: user/attilio/vmobj-readlock/sys/kern/uipc_shm.c
==============================================================================
--- user/attilio/vmobj-readlock/sys/kern/uipc_shm.c Thu Mar 14 00:27:53 2013 (r248259)
+++ user/attilio/vmobj-readlock/sys/kern/uipc_shm.c Thu Mar 14 01:53:32 2013 (r248260)
@@ -283,7 +283,10 @@ retry:
if (m != NULL) {
if ((m->oflags & VPO_BUSY) != 0 ||
m->busy != 0) {
+ vm_page_lock(m);
+ VM_OBJECT_WUNLOCK(object);
vm_page_sleep(m, "shmtrc");
+ VM_OBJECT_WLOCK(object);
goto retry;
}
} else if (vm_pager_has_page(object, idx, NULL, NULL)) {
Modified: user/attilio/vmobj-readlock/sys/kern/vfs_bio.c
==============================================================================
--- user/attilio/vmobj-readlock/sys/kern/vfs_bio.c Thu Mar 14 00:27:53 2013 (r248259)
+++ user/attilio/vmobj-readlock/sys/kern/vfs_bio.c Thu Mar 14 01:53:32 2013 (r248260)
@@ -3053,12 +3053,12 @@ allocbuf(struct buf *bp, int size)
m = bp->b_pages[i];
KASSERT(m != bogus_page,
("allocbuf: bogus page found"));
- while (vm_page_sleep_if_busy(m, TRUE,
- "biodep"))
- continue;
+ do {
+ vm_page_lock(m);
+ } while (vm_page_sleep_if_busy(m, TRUE,
+ "biodep"));
bp->b_pages[i] = NULL;
- vm_page_lock(m);
vm_page_unwire(m, 0);
vm_page_unlock(m);
}
@@ -3581,8 +3581,12 @@ vfs_drain_busy_pages(struct buf *bp)
if ((m->oflags & VPO_BUSY) != 0) {
for (; last_busied < i; last_busied++)
vm_page_busy(bp->b_pages[last_busied]);
- while ((m->oflags & VPO_BUSY) != 0)
+ while ((m->oflags & VPO_BUSY) != 0) {
+ vm_page_lock(m);
+ VM_OBJECT_WUNLOCK(bp->b_bufobj->bo_object);
vm_page_sleep(m, "vbpage");
+ VM_OBJECT_WLOCK(bp->b_bufobj->bo_object);
+ }
}
}
for (i = 0; i < last_busied; i++)
Modified: user/attilio/vmobj-readlock/sys/vm/phys_pager.c
==============================================================================
--- user/attilio/vmobj-readlock/sys/vm/phys_pager.c Thu Mar 14 00:27:53 2013 (r248259)
+++ user/attilio/vmobj-readlock/sys/vm/phys_pager.c Thu Mar 14 01:53:32 2013 (r248260)
@@ -38,6 +38,7 @@ __FBSDID("$FreeBSD$");
#include <sys/sysctl.h>
#include <vm/vm.h>
+#include <vm/vm_param.h>
#include <vm/vm_object.h>
#include <vm/vm_page.h>
#include <vm/vm_pager.h>
@@ -152,9 +153,11 @@ phys_pager_getpages(vm_object_t object,
KASSERT(m[i]->dirty == 0,
("phys_pager_getpages: dirty page %p", m[i]));
/* The requested page must remain busy, the others not. */
- if (i == reqpage)
+ if (i == reqpage) {
+ vm_page_lock(m[i]);
vm_page_flash(m[i]);
- else
+ vm_page_unlock(m[i]);
+ } else
vm_page_wakeup(m[i]);
}
return (VM_PAGER_OK);
Modified: user/attilio/vmobj-readlock/sys/vm/swap_pager.c
==============================================================================
--- user/attilio/vmobj-readlock/sys/vm/swap_pager.c Thu Mar 14 00:27:53 2013 (r248259)
+++ user/attilio/vmobj-readlock/sys/vm/swap_pager.c Thu Mar 14 01:53:32 2013 (r248260)
@@ -1211,13 +1211,15 @@ swap_pager_getpages(vm_object_t object,
*/
VM_OBJECT_WLOCK(object);
while ((mreq->oflags & VPO_SWAPINPROG) != 0) {
- mreq->oflags |= VPO_WANTED;
PCPU_INC(cnt.v_intrans);
- if (VM_OBJECT_SLEEP(object, mreq, PSWP, "swread", hz * 20)) {
+ vm_page_lock(mreq);
+ VM_OBJECT_WUNLOCK(object);
+ if (vm_page_sleep_onpage(mreq, PSWP, "swread", hz * 20)) {
printf(
"swap_pager: indefinite wait buffer: bufobj: %p, blkno: %jd, size: %ld\n",
bp->b_bufobj, (intmax_t)bp->b_blkno, bp->b_bcount);
}
+ VM_OBJECT_WLOCK(object);
}
/*
@@ -1531,8 +1533,11 @@ swp_pager_async_iodone(struct buf *bp)
m->valid = 0;
if (i != bp->b_pager.pg_reqpage)
swp_pager_free_nrpage(m);
- else
+ else {
+ vm_page_lock(m);
vm_page_flash(m);
+ vm_page_unlock(m);
+ }
/*
* If i == bp->b_pager.pg_reqpage, do not wake
* the page up. The caller needs to.
@@ -1585,8 +1590,11 @@ swp_pager_async_iodone(struct buf *bp)
vm_page_deactivate(m);
vm_page_unlock(m);
vm_page_wakeup(m);
- } else
+ } else {
+ vm_page_lock(m);
vm_page_flash(m);
+ vm_page_unlock(m);
+ }
} else {
/*
* For write success, clear the dirty
Modified: user/attilio/vmobj-readlock/sys/vm/vm_fault.c
==============================================================================
--- user/attilio/vmobj-readlock/sys/vm/vm_fault.c Thu Mar 14 00:27:53 2013 (r248259)
+++ user/attilio/vmobj-readlock/sys/vm/vm_fault.c Thu Mar 14 01:53:32 2013 (r248260)
@@ -379,8 +379,10 @@ RetryFault:;
unlock_map(&fs);
if (fs.m == vm_page_lookup(fs.object,
fs.pindex)) {
- vm_page_sleep_if_busy(fs.m, TRUE,
- "vmpfw");
+ vm_page_lock(fs.m);
+ if (!vm_page_sleep_if_busy(fs.m, TRUE,
+ "vmpfw"))
+ vm_page_unlock(fs.m);
}
vm_object_pip_wakeup(fs.object);
VM_OBJECT_WUNLOCK(fs.object);
Modified: user/attilio/vmobj-readlock/sys/vm/vm_object.c
==============================================================================
--- user/attilio/vmobj-readlock/sys/vm/vm_object.c Thu Mar 14 00:27:53 2013 (r248259)
+++ user/attilio/vmobj-readlock/sys/vm/vm_object.c Thu Mar 14 01:53:32 2013 (r248260)
@@ -391,7 +391,7 @@ vm_object_pip_wait(vm_object_t object, c
VM_OBJECT_ASSERT_WLOCKED(object);
while (object->paging_in_progress) {
object->flags |= OBJ_PIPWNT;
- VM_OBJECT_SLEEP(object, object, PVM, waitid, 0);
+ VM_OBJECT_SLEEP(object, object, PVM, waitid);
}
}
@@ -584,7 +584,7 @@ retry:
VM_OBJECT_WUNLOCK(robject);
object->flags |= OBJ_PIPWNT;
VM_OBJECT_SLEEP(object, object,
- PDROP | PVM, "objde2", 0);
+ PDROP | PVM, "objde2");
VM_OBJECT_WLOCK(robject);
temp = robject->backing_object;
if (object == temp) {
@@ -844,6 +844,7 @@ rescan:
np = TAILQ_NEXT(p, listq);
if (p->valid == 0)
continue;
+ vm_page_lock(p);
if (vm_page_sleep_if_busy(p, TRUE, "vpcwai")) {
if (object->generation != curgeneration) {
if ((flags & OBJPC_SYNC) != 0)
@@ -854,6 +855,7 @@ rescan:
np = vm_page_find_least(object, pi);
continue;
}
+ vm_page_unlock(p);
if (!vm_object_page_remove_write(p, flags, &clearobjflags))
continue;
@@ -1138,11 +1140,10 @@ shadowlookup:
*/
vm_page_aflag_set(m, PGA_REFERENCED);
}
- vm_page_unlock(m);
if (object != tobject)
VM_OBJECT_WUNLOCK(object);
- m->oflags |= VPO_WANTED;
- VM_OBJECT_SLEEP(tobject, m, PDROP | PVM, "madvpo", 0);
+ VM_OBJECT_WUNLOCK(tobject);
+ vm_page_sleep(m, "madvpo");
VM_OBJECT_WLOCK(object);
goto relookup;
}
@@ -1339,8 +1340,10 @@ retry:
*/
if ((m->oflags & VPO_BUSY) || m->busy) {
VM_OBJECT_WUNLOCK(new_object);
- m->oflags |= VPO_WANTED;
- VM_OBJECT_SLEEP(orig_object, m, PVM, "spltwt", 0);
+ vm_page_lock(m);
+ VM_OBJECT_WUNLOCK(orig_object);
+ vm_page_sleep(m, "spltwt");
+ VM_OBJECT_WLOCK(orig_object);
VM_OBJECT_WLOCK(new_object);
goto retry;
}
@@ -1497,9 +1500,9 @@ vm_object_backing_scan(vm_object_t objec
} else if (op & OBSC_COLLAPSE_WAIT) {
if ((p->oflags & VPO_BUSY) || p->busy) {
VM_OBJECT_WUNLOCK(object);
- p->oflags |= VPO_WANTED;
- VM_OBJECT_SLEEP(backing_object, p,
- PDROP | PVM, "vmocol", 0);
+ vm_page_lock(p);
+ VM_OBJECT_WUNLOCK(backing_object);
+ vm_page_sleep(p, "vmocol");
VM_OBJECT_WLOCK(object);
VM_OBJECT_WLOCK(backing_object);
/*
Modified: user/attilio/vmobj-readlock/sys/vm/vm_object.h
==============================================================================
--- user/attilio/vmobj-readlock/sys/vm/vm_object.h Thu Mar 14 00:27:53 2013 (r248259)
+++ user/attilio/vmobj-readlock/sys/vm/vm_object.h Thu Mar 14 01:53:32 2013 (r248260)
@@ -216,8 +216,8 @@ extern struct vm_object kmem_object_stor
rw_rlock(&(object)->lock)
#define VM_OBJECT_RUNLOCK(object) \
rw_runlock(&(object)->lock)
-#define VM_OBJECT_SLEEP(object, wchan, pri, wmesg, timo) \
- rw_sleep((wchan), &(object)->lock, (pri), (wmesg), (timo))
+#define VM_OBJECT_SLEEP(object, wchan, pri, wmesg) \
+ rw_sleep((wchan), &(object)->lock, (pri), (wmesg), 0)
#define VM_OBJECT_TRYRLOCK(object) \
rw_try_rlock(&(object)->lock)
#define VM_OBJECT_TRYWLOCK(object) \
Modified: user/attilio/vmobj-readlock/sys/vm/vm_page.c
==============================================================================
--- user/attilio/vmobj-readlock/sys/vm/vm_page.c Thu Mar 14 00:27:53 2013 (r248259)
+++ user/attilio/vmobj-readlock/sys/vm/vm_page.c Thu Mar 14 01:53:32 2013 (r248260)
@@ -485,9 +485,9 @@ void
vm_page_flash(vm_page_t m)
{
- VM_OBJECT_ASSERT_WLOCKED(m->object);
- if (m->oflags & VPO_WANTED) {
- m->oflags &= ~VPO_WANTED;
+ vm_page_lock_assert(m, MA_OWNED);
+ if (m->flags & PG_WANTED) {
+ m->flags &= ~PG_WANTED;
wakeup(m);
}
}
@@ -506,7 +506,9 @@ vm_page_wakeup(vm_page_t m)
VM_OBJECT_ASSERT_WLOCKED(m->object);
KASSERT(m->oflags & VPO_BUSY, ("vm_page_wakeup: page not busy!!!"));
m->oflags &= ~VPO_BUSY;
+ vm_page_lock(m);
vm_page_flash(m);
+ vm_page_unlock(m);
}
void
@@ -524,8 +526,11 @@ vm_page_io_finish(vm_page_t m)
VM_OBJECT_ASSERT_WLOCKED(m->object);
KASSERT(m->busy > 0, ("vm_page_io_finish: page %p is not busy", m));
m->busy--;
- if (m->busy == 0)
+ if (m->busy == 0) {
+ vm_page_lock(m);
vm_page_flash(m);
+ vm_page_unlock(m);
+ }
}
/*
@@ -719,15 +724,12 @@ vm_page_readahead_finish(vm_page_t m)
* deactivating the page is usually the best choice,
* unless the page is wanted by another thread.
*/
- if (m->oflags & VPO_WANTED) {
- vm_page_lock(m);
+ vm_page_lock(m);
+ if (m->flags & PG_WANTED)
vm_page_activate(m);
- vm_page_unlock(m);
- } else {
- vm_page_lock(m);
+ else
vm_page_deactivate(m);
- vm_page_unlock(m);
- }
+ vm_page_unlock(m);
vm_page_wakeup(m);
} else {
/*
@@ -743,29 +745,43 @@ vm_page_readahead_finish(vm_page_t m)
}
/*
- * vm_page_sleep:
+ * vm_page_sleep_if_busy:
*
- * Sleep and release the page lock.
+ * Sleep and release the page queues lock if VPO_BUSY is set or,
+ * if also_m_busy is TRUE, busy is non-zero. Returns TRUE if the
+ * thread slept and the page queues lock was released.
+ * Otherwise, retains the page queues lock and returns FALSE.
*
- * The object containing the given page must be locked.
+ * The given page and object containing it must be locked.
*/
-void
-vm_page_sleep(vm_page_t m, const char *msg)
+int
+vm_page_sleep_if_busy(vm_page_t m, int also_m_busy, const char *msg)
{
VM_OBJECT_ASSERT_WLOCKED(m->object);
- if (mtx_owned(vm_page_lockptr(m)))
- vm_page_unlock(m);
+ if ((m->oflags & VPO_BUSY) || (also_m_busy && m->busy)) {
+ VM_OBJECT_WUNLOCK(m->object);
+ vm_page_sleep(m, msg);
+ VM_OBJECT_WLOCK(m->object);
+ return (TRUE);
+ }
+ return (FALSE);
+}
- /*
- * It's possible that while we sleep, the page will get
- * unbusied and freed. If we are holding the object
- * lock, we will assume we hold a reference to the object
- * such that even if m->object changes, we can re-lock
- * it.
- */
- m->oflags |= VPO_WANTED;
- VM_OBJECT_SLEEP(m->object, m, PVM, msg, 0);
+/*
+ * vm_page_sleep_onpage:
+ *
+ * Sleep and release the page lock, using the page pointer as wchan.
+ *
+ * The given page must be locked.
+ */
+int
+vm_page_sleep_onpage(vm_page_t m, int pri, const char *wmesg, int timo)
+{
+
+ vm_page_lock_assert(m, MA_OWNED);
+ m->flags |= PG_WANTED;
+ return (msleep(m, vm_page_lockptr(m), pri | PDROP, wmesg, timo));
}
/*
@@ -2320,6 +2336,7 @@ retrylookup:
* likely to reclaim it.
*/
vm_page_aflag_set(m, PGA_REFERENCED);
+ vm_page_lock(m);
vm_page_sleep(m, "pgrbwt");
goto retrylookup;
} else {
Modified: user/attilio/vmobj-readlock/sys/vm/vm_page.h
==============================================================================
--- user/attilio/vmobj-readlock/sys/vm/vm_page.h Thu Mar 14 00:27:53 2013 (r248259)
+++ user/attilio/vmobj-readlock/sys/vm/vm_page.h Thu Mar 14 01:53:32 2013 (r248260)
@@ -166,7 +166,7 @@ struct vm_page {
*
*/
#define VPO_BUSY 0x01 /* page is in transit */
-#define VPO_WANTED 0x02 /* someone is waiting for page */
+#define VPO_UNUSED02 0x02 /* --available-- */
#define VPO_UNMANAGED 0x04 /* no PV management for page */
#define VPO_SWAPINPROG 0x08 /* swap I/O in progress on page */
#define VPO_NOSYNC 0x10 /* do not collect for syncer */
@@ -270,6 +270,7 @@ extern struct mtx_padalign pa_lock[];
#define PG_WINATCFLS 0x0040 /* flush dirty page on inactive q */
#define PG_NODUMP 0x0080 /* don't include this page in a dump */
#define PG_UNHOLDFREE 0x0100 /* delayed free of a held page */
+#define PG_WANTED 0x0200 /* someone is waiting for page */
/*
* Misc constants.
@@ -401,7 +402,8 @@ void vm_page_rename (vm_page_t, vm_objec
void vm_page_requeue(vm_page_t m);
void vm_page_requeue_locked(vm_page_t m);
void vm_page_set_valid_range(vm_page_t m, int base, int size);
-void vm_page_sleep(vm_page_t m, const char *msg);
+int vm_page_sleep_if_busy(vm_page_t m, int also_m_busy, const char *msg);
+int vm_page_sleep_onpage(vm_page_t m, int pri, const char *msg, int timo);
vm_offset_t vm_page_startup(vm_offset_t vaddr);
void vm_page_unhold_pages(vm_page_t *ma, int count);
void vm_page_unwire (vm_page_t, int);
@@ -527,40 +529,34 @@ vm_page_dirty(vm_page_t m)
}
/*
- * vm_page_remque:
+ * vm_page_sleep:
*
- * If the given page is in a page queue, then remove it from that page
- * queue.
+ * Convenience wrapper around vm_page_sleep_onpage(), passing
+ * PVM priority and 0 timeout values. Unlocks the page upon return.
*
* The page must be locked.
*/
-static inline void
-vm_page_remque(vm_page_t m)
+static __inline void
+vm_page_sleep(vm_page_t m, const char *msg)
{
- if (m->queue != PQ_NONE)
- vm_page_dequeue(m);
+ vm_page_sleep_onpage(m, PVM, msg, 0);
}
/*
- * vm_page_sleep_if_busy:
+ * vm_page_remque:
*
- * Sleep and release the page queues lock if VPO_BUSY is set or,
- * if also_m_busy is TRUE, busy is non-zero. Returns TRUE if the
- * thread slept and the page queues lock was released.
- * Otherwise, retains the page queues lock and returns FALSE.
+ * If the given page is in a page queue, then remove it from that page
+ * queue.
*
- * The object containing the given page must be locked.
+ * The page must be locked.
*/
-static __inline int
-vm_page_sleep_if_busy(vm_page_t m, int also_m_busy, const char *msg)
+static inline void
+vm_page_remque(vm_page_t m)
{
- if ((m->oflags & VPO_BUSY) || (also_m_busy && m->busy)) {
- vm_page_sleep(m, msg);
- return (TRUE);
- }
- return (FALSE);
+ if (m->queue != PQ_NONE)
+ vm_page_dequeue(m);
}
/*
Modified: user/attilio/vmobj-readlock/sys/vm/vnode_pager.c
==============================================================================
--- user/attilio/vmobj-readlock/sys/vm/vnode_pager.c Thu Mar 14 00:27:53 2013 (r248259)
+++ user/attilio/vmobj-readlock/sys/vm/vnode_pager.c Thu Mar 14 01:53:32 2013 (r248260)
@@ -117,7 +117,7 @@ vnode_create_vobject(struct vnode *vp, o
}
VOP_UNLOCK(vp, 0);
vm_object_set_flag(object, OBJ_DISCONNECTWNT);
- VM_OBJECT_SLEEP(object, object, PDROP | PVM, "vodead", 0);
+ VM_OBJECT_SLEEP(object, object, PDROP | PVM, "vodead");
vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
}
@@ -211,7 +211,7 @@ retry:
if ((object->flags & OBJ_DEAD) == 0)
break;
vm_object_set_flag(object, OBJ_DISCONNECTWNT);
- VM_OBJECT_SLEEP(object, object, PDROP | PVM, "vadead", 0);
+ VM_OBJECT_SLEEP(object, object, PDROP | PVM, "vadead");
}
if (vp->v_usecount == 0)
More information about the svn-src-user
mailing list