svn commit: r253487 - in user/attilio/vmobj-fullread/sys: cddl/contrib/opensolaris/uts/common/fs/zfs dev/drm2/i915 fs/tmpfs kern vm
Attilio Rao
attilio at FreeBSD.org
Sat Jul 20 08:38:37 UTC 2013
Author: attilio
Date: Sat Jul 20 08:38:34 2013
New Revision: 253487
URL: http://svnweb.freebsd.org/changeset/base/253487
Log:
Rework vm_page_sleep_if_busy() and vm_page_grab() to be possibly
called with the object lock in read-mode.
With the object lock held in read-mode the busy operation must be
time-invariant, which means VM_ALLOC_NOBUSY cannot be really passed.
Also, in the vm_page_sleep_if_busy() the VM_ALLOC_* flags don't make
much sense, maybe these 2 interfaces should grow their own set of
flags (and vm_page_grab() internally can play with VM_ALLOC_* flags
as appropriate).
Sponsored by: EMC / Isilon storage division
Tested by: pho
Modified:
user/attilio/vmobj-fullread/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c
user/attilio/vmobj-fullread/sys/dev/drm2/i915/i915_gem.c
user/attilio/vmobj-fullread/sys/fs/tmpfs/tmpfs_subr.c
user/attilio/vmobj-fullread/sys/kern/subr_uio.c
user/attilio/vmobj-fullread/sys/kern/uipc_shm.c
user/attilio/vmobj-fullread/sys/kern/vfs_bio.c
user/attilio/vmobj-fullread/sys/vm/vm_fault.c
user/attilio/vmobj-fullread/sys/vm/vm_object.c
user/attilio/vmobj-fullread/sys/vm/vm_object.h
user/attilio/vmobj-fullread/sys/vm/vm_page.c
user/attilio/vmobj-fullread/sys/vm/vm_page.h
Modified: user/attilio/vmobj-fullread/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c
==============================================================================
--- user/attilio/vmobj-fullread/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c Sat Jul 20 08:25:03 2013 (r253486)
+++ user/attilio/vmobj-fullread/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c Sat Jul 20 08:38:34 2013 (r253487)
@@ -336,19 +336,9 @@ page_busy(vnode_t *vp, int64_t start, in
for (;;) {
if ((pp = vm_page_lookup(obj, OFF_TO_IDX(start))) != NULL &&
pp->valid) {
- if (vm_page_busy_wlocked(pp)) {
- /*
- * Reference the page before unlocking and
- * sleeping so that the page daemon is less
- * likely to reclaim it.
- */
- vm_page_reference(pp);
- vm_page_lock(pp);
- zfs_vmobject_wunlock(obj);
- vm_page_busy_sleep(pp, "zfsmwb");
- zfs_vmobject_wlock(obj);
+ if (vm_page_sleep_if_busy(pp, "zfsmwb",
+ VM_ALLOC_NOBUSY, TRUE))
continue;
- }
vm_page_busy_rlock(pp);
} else if (pp == NULL) {
if (!alloc)
Modified: user/attilio/vmobj-fullread/sys/dev/drm2/i915/i915_gem.c
==============================================================================
--- user/attilio/vmobj-fullread/sys/dev/drm2/i915/i915_gem.c Sat Jul 20 08:25:03 2013 (r253486)
+++ user/attilio/vmobj-fullread/sys/dev/drm2/i915/i915_gem.c Sat Jul 20 08:38:34 2013 (r253487)
@@ -2333,7 +2333,8 @@ retry:
m = vm_page_lookup(devobj, i);
if (m == NULL)
continue;
- if (vm_page_sleep_if_busy(m, "915unm"))
+ if (vm_page_sleep_if_busy(m, "915unm", VM_ALLOC_NOBUSY,
+ FALSE))
goto retry;
cdev_pager_free_page(devobj, m);
}
Modified: user/attilio/vmobj-fullread/sys/fs/tmpfs/tmpfs_subr.c
==============================================================================
--- user/attilio/vmobj-fullread/sys/fs/tmpfs/tmpfs_subr.c Sat Jul 20 08:25:03 2013 (r253486)
+++ user/attilio/vmobj-fullread/sys/fs/tmpfs/tmpfs_subr.c Sat Jul 20 08:38:34 2013 (r253487)
@@ -1331,7 +1331,8 @@ tmpfs_reg_resize(struct vnode *vp, off_t
retry:
m = vm_page_lookup(uobj, idx);
if (m != NULL) {
- if (vm_page_sleep_if_busy(m, "tmfssz"))
+ if (vm_page_sleep_if_busy(m, "tmfssz",
+ VM_ALLOC_NOBUSY, FALSE))
goto retry;
MPASS(m->valid == VM_PAGE_BITS_ALL);
} else if (vm_pager_has_page(uobj, idx, NULL, NULL)) {
Modified: user/attilio/vmobj-fullread/sys/kern/subr_uio.c
==============================================================================
--- user/attilio/vmobj-fullread/sys/kern/subr_uio.c Sat Jul 20 08:25:03 2013 (r253486)
+++ user/attilio/vmobj-fullread/sys/kern/subr_uio.c Sat Jul 20 08:38:34 2013 (r253487)
@@ -107,7 +107,8 @@ vm_pgmoveco(vm_map_t mapa, vm_offset_t k
VM_OBJECT_WLOCK(uobject);
retry:
if ((user_pg = vm_page_lookup(uobject, upindex)) != NULL) {
- if (vm_page_sleep_if_busy(user_pg, "vm_pgmoveco"))
+ if (vm_page_sleep_if_busy(user_pg, "vm_pgmoveco",
+ VM_ALLOC_NOBUSY, FALSE))
goto retry;
vm_page_lock(user_pg);
pmap_remove_all(user_pg);
Modified: user/attilio/vmobj-fullread/sys/kern/uipc_shm.c
==============================================================================
--- user/attilio/vmobj-fullread/sys/kern/uipc_shm.c Sat Jul 20 08:25:03 2013 (r253486)
+++ user/attilio/vmobj-fullread/sys/kern/uipc_shm.c Sat Jul 20 08:38:34 2013 (r253487)
@@ -281,7 +281,8 @@ shm_dotruncate(struct shmfd *shmfd, off_
retry:
m = vm_page_lookup(object, idx);
if (m != NULL) {
- if (vm_page_sleep_if_busy(m, "shmtrc"))
+ if (vm_page_sleep_if_busy(m, "shmtrc",
+ VM_ALLOC_NOBUSY, FALSE))
goto retry;
} else if (vm_pager_has_page(object, idx, NULL, NULL)) {
m = vm_page_alloc(object, idx, VM_ALLOC_NORMAL);
Modified: user/attilio/vmobj-fullread/sys/kern/vfs_bio.c
==============================================================================
--- user/attilio/vmobj-fullread/sys/kern/vfs_bio.c Sat Jul 20 08:25:03 2013 (r253486)
+++ user/attilio/vmobj-fullread/sys/kern/vfs_bio.c Sat Jul 20 08:38:34 2013 (r253487)
@@ -3444,7 +3444,7 @@ allocbuf(struct buf *bp, int size)
KASSERT(m != bogus_page,
("allocbuf: bogus page found"));
while (vm_page_sleep_if_busy(m,
- "biodep"))
+ "biodep", VM_ALLOC_NOBUSY, FALSE))
continue;
bp->b_pages[i] = NULL;
Modified: user/attilio/vmobj-fullread/sys/vm/vm_fault.c
==============================================================================
--- user/attilio/vmobj-fullread/sys/vm/vm_fault.c Sat Jul 20 08:25:03 2013 (r253486)
+++ user/attilio/vmobj-fullread/sys/vm/vm_fault.c Sat Jul 20 08:38:34 2013 (r253487)
@@ -392,7 +392,8 @@ RetryFault:;
unlock_map(&fs);
if (fs.m == vm_page_lookup(fs.object,
fs.pindex)) {
- vm_page_sleep_if_busy(fs.m, "vmpfw");
+ vm_page_sleep_if_busy(fs.m, "vmpfw",
+ VM_ALLOC_NOBUSY, FALSE);
}
vm_object_pip_wakeup(fs.object);
VM_OBJECT_WUNLOCK(fs.object);
Modified: user/attilio/vmobj-fullread/sys/vm/vm_object.c
==============================================================================
--- user/attilio/vmobj-fullread/sys/vm/vm_object.c Sat Jul 20 08:25:03 2013 (r253486)
+++ user/attilio/vmobj-fullread/sys/vm/vm_object.c Sat Jul 20 08:38:34 2013 (r253487)
@@ -870,7 +870,8 @@ rescan:
np = TAILQ_NEXT(p, listq);
if (p->valid == 0)
continue;
- if (vm_page_sleep_if_busy(p, "vpcwai")) {
+ if (vm_page_sleep_if_busy(p, "vpcwai", VM_ALLOC_NOBUSY,
+ FALSE)) {
if (object->generation != curgeneration) {
if ((flags & OBJPC_SYNC) != 0)
goto rescan;
Modified: user/attilio/vmobj-fullread/sys/vm/vm_object.h
==============================================================================
--- user/attilio/vmobj-fullread/sys/vm/vm_object.h Sat Jul 20 08:25:03 2013 (r253486)
+++ user/attilio/vmobj-fullread/sys/vm/vm_object.h Sat Jul 20 08:38:34 2013 (r253487)
@@ -226,6 +226,8 @@ extern struct vm_object kmem_object_stor
rw_assert(&(object)->lock, RA_WLOCKED)
#define VM_OBJECT_LOCK_DOWNGRADE(object) \
rw_downgrade(&(object)->lock)
+#define VM_OBJECT_LOCK_TRYUPGRADE(object) \
+ rw_try_upgrade(&(object)->lock)
#define VM_OBJECT_RLOCK(object) \
rw_rlock(&(object)->lock)
#define VM_OBJECT_RUNLOCK(object) \
@@ -238,6 +240,8 @@ extern struct vm_object kmem_object_stor
rw_try_wlock(&(object)->lock)
#define VM_OBJECT_WLOCK(object) \
rw_wlock(&(object)->lock)
+#define VM_OBJECT_WOWNED(object) \
+ rw_wowned(&(object)->lock)
#define VM_OBJECT_WUNLOCK(object) \
rw_wunlock(&(object)->lock)
Modified: user/attilio/vmobj-fullread/sys/vm/vm_page.c
==============================================================================
--- user/attilio/vmobj-fullread/sys/vm/vm_page.c Sat Jul 20 08:25:03 2013 (r253486)
+++ user/attilio/vmobj-fullread/sys/vm/vm_page.c Sat Jul 20 08:38:34 2013 (r253487)
@@ -869,26 +869,53 @@ vm_page_readahead_finish(vm_page_t m)
* be locked.
*/
int
-vm_page_sleep_if_busy(vm_page_t m, const char *msg)
+vm_page_sleep_if_busy(vm_page_t m, const char *msg, int busyflags,
+ boolean_t pref)
{
vm_object_t obj;
+ int cond, iswowned;
vm_page_lock_assert(m, MA_NOTOWNED);
- VM_OBJECT_ASSERT_WLOCKED(m->object);
- if (vm_page_busy_locked(m)) {
+ /*
+ * The page-specific object must be cached because page
+ * identity can change during the sleep, causing the
+ * re-lock of a different object.
+ * It is assumed that a reference to the object is already
+ * held by the callers.
+ */
+ obj = m->object;
+ VM_OBJECT_ASSERT_LOCKED(obj);
+ iswowned = VM_OBJECT_WOWNED(obj);
+ KASSERT((busyflags & VM_ALLOC_NOBUSY) == 0 || iswowned,
+ ("vm_page_sleep_if_busy: VM_ALLOC_NOBUSY with read object lock"));
+
+ if ((busyflags & VM_ALLOC_NOBUSY) != 0) {
+ cond = (busyflags & VM_ALLOC_IGN_RBUSY) != 0 ?
+ vm_page_busy_wlocked(m) : vm_page_busy_locked(m);
+ } else if ((busyflags & VM_ALLOC_RBUSY) != 0)
+ cond = !vm_page_busy_tryrlock(m);
+ else
+ cond = !vm_page_busy_trywlock(m);
+ if (cond) {
+
/*
- * The page-specific object must be cached because page
- * identity can change during the sleep, causing the
- * re-lock of a different object.
- * It is assumed that a reference to the object is already
- * held by the callers.
+ * Some consumers may want to reference the page before
+ * unlocking and sleeping so that the page daemon is less
+ * likely to reclaim it.
*/
- obj = m->object;
+ if (pref)
+ vm_page_aflag_set(m, PGA_REFERENCED);
vm_page_lock(m);
- VM_OBJECT_WUNLOCK(obj);
+ if (iswowned)
+ VM_OBJECT_WUNLOCK(obj);
+ else
+ VM_OBJECT_RUNLOCK(obj);
vm_page_busy_sleep(m, msg);
- VM_OBJECT_WLOCK(obj);
+ if (iswowned)
+ VM_OBJECT_WLOCK(obj);
+ else
+ VM_OBJECT_RLOCK(obj);
return (TRUE);
}
return (FALSE);
@@ -2500,44 +2527,44 @@ vm_page_t
vm_page_grab(vm_object_t object, vm_pindex_t pindex, int allocflags)
{
vm_page_t m;
- int sleep;
+ int origwlock;
VM_OBJECT_ASSERT_WLOCKED(object);
+ origwlock = VM_OBJECT_WOWNED(object);
KASSERT((allocflags & VM_ALLOC_RETRY) != 0,
("vm_page_grab: VM_ALLOC_RETRY is required"));
KASSERT((allocflags & VM_ALLOC_RBUSY) == 0 ||
(allocflags & VM_ALLOC_IGN_RBUSY) != 0,
("vm_page_grab: VM_ALLOC_RBUSY/VM_ALLOC_IGN_RBUSY mismatch"));
+ KASSERT((allocflags & VM_ALLOC_NOBUSY) == 0 || origwlock != 0,
+ ("vm_page_grab: VM_ALLOC_NOBUSY with object read lock"));
retrylookup:
if ((m = vm_page_lookup(object, pindex)) != NULL) {
- sleep = (allocflags & VM_ALLOC_IGN_RBUSY) != 0 ?
- vm_page_busy_wlocked(m) : vm_page_busy_locked(m);
- if (sleep) {
- /*
- * Reference the page before unlocking and
- * sleeping so that the page daemon is less
- * likely to reclaim it.
- */
- vm_page_aflag_set(m, PGA_REFERENCED);
- vm_page_lock(m);
- VM_OBJECT_WUNLOCK(object);
- vm_page_busy_sleep(m, "pgrbwt");
- VM_OBJECT_WLOCK(object);
+ if (vm_page_sleep_if_busy(m, "pgrbwt", allocflags &
+ (VM_ALLOC_NOBUSY | VM_ALLOC_RBUSY | VM_ALLOC_IGN_RBUSY),
+ TRUE))
goto retrylookup;
- } else {
+ else {
if ((allocflags & VM_ALLOC_WIRED) != 0) {
vm_page_lock(m);
vm_page_wire(m);
vm_page_unlock(m);
}
- if ((allocflags &
- (VM_ALLOC_NOBUSY | VM_ALLOC_RBUSY)) == 0)
- vm_page_busy_wlock(m);
- if ((allocflags & VM_ALLOC_RBUSY) != 0)
- vm_page_busy_rlock(m);
+
+ /*
+ * If the lock state changed in the meanwhile,
+ * unwind back.
+ */
+ if (VM_OBJECT_WOWNED(object) != origwlock)
+ VM_OBJECT_LOCK_DOWNGRADE(object);
return (m);
}
}
+ if (!VM_OBJECT_WOWNED(object) && !VM_OBJECT_LOCK_TRYUPGRADE(object)) {
+ VM_OBJECT_RUNLOCK(object);
+ VM_OBJECT_WLOCK(object);
+ goto retrylookup;
+ }
m = vm_page_alloc(object, pindex, allocflags & ~(VM_ALLOC_RETRY |
VM_ALLOC_IGN_RBUSY));
if (m == NULL) {
@@ -2545,7 +2572,12 @@ retrylookup:
VM_WAIT;
VM_OBJECT_WLOCK(object);
goto retrylookup;
- } else if (m->valid != 0)
+ }
+
+ /* If the lock state changed in the meanwhile, unwind back. */
+ if (VM_OBJECT_WOWNED(object) != origwlock)
+ VM_OBJECT_LOCK_DOWNGRADE(object);
+ if (m->valid != 0)
return (m);
if (allocflags & VM_ALLOC_ZERO && (m->flags & PG_ZERO) == 0)
pmap_zero_page(m);
Modified: user/attilio/vmobj-fullread/sys/vm/vm_page.h
==============================================================================
--- user/attilio/vmobj-fullread/sys/vm/vm_page.h Sat Jul 20 08:25:03 2013 (r253486)
+++ user/attilio/vmobj-fullread/sys/vm/vm_page.h Sat Jul 20 08:38:34 2013 (r253487)
@@ -439,7 +439,8 @@ void vm_page_rename (vm_page_t, vm_objec
void vm_page_requeue(vm_page_t m);
void vm_page_requeue_locked(vm_page_t m);
void vm_page_set_valid_range(vm_page_t m, int base, int size);
-int vm_page_sleep_if_busy(vm_page_t m, const char *msg);
+int vm_page_sleep_if_busy(vm_page_t m, const char *msg, int busyflags,
+ boolean_t pref);
vm_offset_t vm_page_startup(vm_offset_t vaddr);
void vm_page_unhold_pages(vm_page_t *ma, int count);
void vm_page_unwire (vm_page_t, int);
More information about the svn-src-user
mailing list