svn commit: r252220 - in user/attilio/vmobj-readlock/sys: cddl/contrib/opensolaris/uts/common/fs/zfs dev/drm2/i915 dev/drm2/ttm fs/tmpfs kern vm
Attilio Rao
attilio at FreeBSD.org
Tue Jun 25 22:46:55 UTC 2013
Author: attilio
Date: Tue Jun 25 22:46:52 2013
New Revision: 252220
URL: http://svnweb.freebsd.org/changeset/base/252220
Log:
Reimplement the busy mechanism by using atomics and not depending by
object lock for the operation themselves.
This way the busy can be performed also within a path where the obj
is locked in read way.
vm_page_sleep() is renamed into vm_page_busy_sleep() in order to show it
is a busy specific operation.
Sponsored by: EMC / Isilon storage division
Modified:
user/attilio/vmobj-readlock/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c
user/attilio/vmobj-readlock/sys/dev/drm2/i915/i915_gem.c
user/attilio/vmobj-readlock/sys/dev/drm2/ttm/ttm_bo_vm.c
user/attilio/vmobj-readlock/sys/fs/tmpfs/tmpfs_vnops.c
user/attilio/vmobj-readlock/sys/kern/vfs_bio.c
user/attilio/vmobj-readlock/sys/vm/vm_fault.c
user/attilio/vmobj-readlock/sys/vm/vm_object.c
user/attilio/vmobj-readlock/sys/vm/vm_page.c
user/attilio/vmobj-readlock/sys/vm/vm_page.h
user/attilio/vmobj-readlock/sys/vm/vm_pageout.c
user/attilio/vmobj-readlock/sys/vm/vm_phys.c
user/attilio/vmobj-readlock/sys/vm/vnode_pager.c
Modified: user/attilio/vmobj-readlock/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c
==============================================================================
--- user/attilio/vmobj-readlock/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c Tue Jun 25 22:14:32 2013 (r252219)
+++ user/attilio/vmobj-readlock/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c Tue Jun 25 22:46:52 2013 (r252220)
@@ -345,7 +345,7 @@ page_busy(vnode_t *vp, int64_t start, in
vm_page_reference(pp);
vm_page_lock(pp);
zfs_vmobject_wunlock(obj);
- vm_page_sleep(pp, "zfsmwb");
+ vm_page_busy_sleep(pp, "zfsmwb");
zfs_vmobject_wlock(obj);
continue;
}
@@ -429,7 +429,7 @@ update_pages(vnode_t *vp, int64_t start,
("zfs update_pages: unaligned data in putpages case"));
KASSERT(pp->valid == VM_PAGE_BITS_ALL,
("zfs update_pages: invalid page in putpages case"));
- KASSERT(pp->busy > 0,
+ KASSERT(vm_page_busy_rlocked(pp),
("zfs update_pages: unbusy page in putpages case"));
KASSERT(!pmap_page_is_write_mapped(pp),
("zfs update_pages: writable page in putpages case"));
Modified: user/attilio/vmobj-readlock/sys/dev/drm2/i915/i915_gem.c
==============================================================================
--- user/attilio/vmobj-readlock/sys/dev/drm2/i915/i915_gem.c Tue Jun 25 22:14:32 2013 (r252219)
+++ user/attilio/vmobj-readlock/sys/dev/drm2/i915/i915_gem.c Tue Jun 25 22:46:52 2013 (r252220)
@@ -1382,7 +1382,7 @@ retry:
DRM_UNLOCK(dev);
vm_page_lock(m);
VM_OBJECT_WUNLOCK(vm_obj);
- vm_page_sleep(m, "915pee");
+ vm_page_busy_sleep(m, "915pee");
goto retry;
}
goto have_page;
@@ -1440,7 +1440,7 @@ retry:
DRM_UNLOCK(dev);
vm_page_lock(m);
VM_OBJECT_WUNLOCK(vm_obj);
- vm_page_sleep(m, "915pbs");
+ vm_page_busy_sleep(m, "915pbs");
goto retry;
}
m->valid = VM_PAGE_BITS_ALL;
Modified: user/attilio/vmobj-readlock/sys/dev/drm2/ttm/ttm_bo_vm.c
==============================================================================
--- user/attilio/vmobj-readlock/sys/dev/drm2/ttm/ttm_bo_vm.c Tue Jun 25 22:14:32 2013 (r252219)
+++ user/attilio/vmobj-readlock/sys/dev/drm2/ttm/ttm_bo_vm.c Tue Jun 25 22:46:52 2013 (r252220)
@@ -215,7 +215,7 @@ reserve:
if (vm_page_busy_locked(m)) {
vm_page_lock(m);
VM_OBJECT_WUNLOCK(vm_obj);
- vm_page_sleep(m, "ttmpbs");
+ vm_page_busy_sleep(m, "ttmpbs");
VM_OBJECT_WLOCK(vm_obj);
ttm_mem_io_unlock(man);
ttm_bo_unreserve(bo);
Modified: user/attilio/vmobj-readlock/sys/fs/tmpfs/tmpfs_vnops.c
==============================================================================
--- user/attilio/vmobj-readlock/sys/fs/tmpfs/tmpfs_vnops.c Tue Jun 25 22:14:32 2013 (r252219)
+++ user/attilio/vmobj-readlock/sys/fs/tmpfs/tmpfs_vnops.c Tue Jun 25 22:46:52 2013 (r252220)
@@ -479,8 +479,7 @@ tmpfs_nocacheread(vm_object_t tobj, vm_p
} else
vm_page_zero_invalid(m, TRUE);
}
- vm_page_busy_wunlock(m);
- vm_page_busy_rlock(m);
+ vm_page_busy_downgrade(m);
VM_OBJECT_WUNLOCK(tobj);
error = uiomove_fromphys(&m, offset, tlen, uio);
VM_OBJECT_WLOCK(tobj);
@@ -594,8 +593,7 @@ tmpfs_mappedwrite(vm_object_t tobj, size
} else
vm_page_zero_invalid(tpg, TRUE);
}
- vm_page_busy_wunlock(tpg);
- vm_page_busy_rlock(tpg);
+ vm_page_busy_downgrade(tpg);
VM_OBJECT_WUNLOCK(tobj);
error = uiomove_fromphys(&tpg, offset, tlen, uio);
VM_OBJECT_WLOCK(tobj);
Modified: user/attilio/vmobj-readlock/sys/kern/vfs_bio.c
==============================================================================
--- user/attilio/vmobj-readlock/sys/kern/vfs_bio.c Tue Jun 25 22:14:32 2013 (r252219)
+++ user/attilio/vmobj-readlock/sys/kern/vfs_bio.c Tue Jun 25 22:46:52 2013 (r252220)
@@ -4003,7 +4003,7 @@ vfs_drain_busy_pages(struct buf *bp)
while (vm_page_busy_wlocked(m)) {
vm_page_lock(m);
VM_OBJECT_WUNLOCK(bp->b_bufobj->bo_object);
- vm_page_sleep(m, "vbpage");
+ vm_page_busy_sleep(m, "vbpage");
VM_OBJECT_WLOCK(bp->b_bufobj->bo_object);
}
}
@@ -4269,7 +4269,7 @@ vm_hold_free_pages(struct buf *bp, int n
for (index = newnpages; index < bp->b_npages; index++) {
p = bp->b_pages[index];
bp->b_pages[index] = NULL;
- if (p->busy != 0)
+ if (vm_page_busy_rlocked(p))
printf("vm_hold_free_pages: blkno: %jd, lblkno: %jd\n",
(intmax_t)bp->b_blkno, (intmax_t)bp->b_lblkno);
p->wire_count--;
Modified: user/attilio/vmobj-readlock/sys/vm/vm_fault.c
==============================================================================
--- user/attilio/vmobj-readlock/sys/vm/vm_fault.c Tue Jun 25 22:14:32 2013 (r252219)
+++ user/attilio/vmobj-readlock/sys/vm/vm_fault.c Tue Jun 25 22:46:52 2013 (r252220)
@@ -926,12 +926,13 @@ vnode_locked:
if (m_hold != NULL) {
*m_hold = fs.m;
if (fault_flags & VM_FAULT_IOBUSY)
- vm_page_busy_rlock(fs.m);
+ vm_page_busy_downgrade(fs.m);
else
vm_page_hold(fs.m);
}
vm_page_unlock(fs.m);
- vm_page_busy_wunlock(fs.m);
+ if (m_hold == NULL || (fault_flags & VM_FAULT_IOBUSY) == 0)
+ vm_page_busy_wunlock(fs.m);
/*
* Unlock everything, and return
Modified: user/attilio/vmobj-readlock/sys/vm/vm_object.c
==============================================================================
--- user/attilio/vmobj-readlock/sys/vm/vm_object.c Tue Jun 25 22:14:32 2013 (r252219)
+++ user/attilio/vmobj-readlock/sys/vm/vm_object.c Tue Jun 25 22:46:52 2013 (r252220)
@@ -1167,7 +1167,7 @@ shadowlookup:
if (object != tobject)
VM_OBJECT_WUNLOCK(object);
VM_OBJECT_WUNLOCK(tobject);
- vm_page_sleep(m, "madvpo");
+ vm_page_busy_sleep(m, "madvpo");
VM_OBJECT_WLOCK(object);
goto relookup;
}
@@ -1346,7 +1346,7 @@ retry:
VM_OBJECT_WUNLOCK(new_object);
vm_page_lock(m);
VM_OBJECT_WUNLOCK(orig_object);
- vm_page_sleep(m, "spltwt");
+ vm_page_busy_sleep(m, "spltwt");
VM_OBJECT_WLOCK(orig_object);
VM_OBJECT_WLOCK(new_object);
goto retry;
@@ -1505,7 +1505,7 @@ vm_object_backing_scan(vm_object_t objec
VM_OBJECT_WUNLOCK(object);
vm_page_lock(p);
VM_OBJECT_WUNLOCK(backing_object);
- vm_page_sleep(p, "vmocol");
+ vm_page_busy_sleep(p, "vmocol");
VM_OBJECT_WLOCK(object);
VM_OBJECT_WLOCK(backing_object);
/*
@@ -1905,7 +1905,7 @@ again:
}
if (vm_page_busy_locked(p)) {
VM_OBJECT_WUNLOCK(object);
- vm_page_sleep(p, "vmopar");
+ vm_page_busy_sleep(p, "vmopar");
VM_OBJECT_WLOCK(object);
goto again;
}
Modified: user/attilio/vmobj-readlock/sys/vm/vm_page.c
==============================================================================
--- user/attilio/vmobj-readlock/sys/vm/vm_page.c Tue Jun 25 22:14:32 2013 (r252219)
+++ user/attilio/vmobj-readlock/sys/vm/vm_page.c Tue Jun 25 22:46:52 2013 (r252220)
@@ -470,86 +470,123 @@ vm_page_reference(vm_page_t m)
}
void
-vm_page_busy_assert_unlocked(vm_page_t m)
+vm_page_busy_downgrade(vm_page_t m)
{
+ u_int retry, x;
- KASSERT((m->oflags & VPO_BUSY) == 0 && m->busy == 0,
- ("vm_page_busy_assert_unlocked: page %p busy", (void *)m));
-}
+ vm_page_busy_assert_wlocked(m);
-void
-vm_page_busy_assert_wlocked(vm_page_t m)
-{
-
- KASSERT((m->oflags & VPO_BUSY) != 0,
- ("vm_page_busy_assert_wlocked: page %p not write busy", (void *)m));
+ retry = 0;
+ for (;;) {
+ if (retry++ > 0)
+ panic("vm_page_busy_downgrade: failed loop %p", m);
+ x = m->busy_lock;
+ x &= VPB_LOCK_WAITERS;
+ if (atomic_cmpset_rel_int(&m->busy_lock,
+ VPB_SINGLE_WRITER | x, VPB_READERS_LOCK(1) | x))
+ break;
+ }
}
int
-vm_page_busy_locked(vm_page_t m)
+vm_page_busy_rlocked(vm_page_t m)
{
+ u_int x;
- return ((m->oflags & VPO_BUSY) != 0 || m->busy != 0);
+ x = m->busy_lock;
+ return ((x & VPB_LOCK_READ) != 0 && x != VPB_UNLOCKED);
}
void
vm_page_busy_runlock(vm_page_t m)
{
+ u_int retry, x;
+
+ vm_page_busy_assert_rlocked(m);
+
+ retry = 0;
+ for (;;) {
+ if (retry++ > 10000)
+ panic("vm_page_busy_runlock: failed loop %p", m);
+ x = m->busy_lock;
+ if (VPB_READERS(x) > 1) {
+ if (atomic_cmpset_int(&m->busy_lock, x,
+ x - VPB_ONE_READER))
+ break;
+ continue;
+ }
+ if ((x & VPB_LOCK_WAITERS) == 0) {
+ KASSERT(x == VPB_READERS_LOCK(1),
+ ("vm_page_busy_runlock: invalid lock state"));
+ if (atomic_cmpset_int(&m->busy_lock,
+ VPB_READERS_LOCK(1), VPB_UNLOCKED))
+ break;
+ continue;
+ }
+ KASSERT(x == (VPB_READERS_LOCK(1) | VPB_LOCK_WAITERS),
+ ("vm_page_busy_runlock: invalid lock state for waiters"));
- VM_OBJECT_ASSERT_WLOCKED(m->object);
- KASSERT(m->busy > 0,
- ("vm_page_busy_runlock: page %p is not read busy", m));
- m->busy--;
- if (m->busy == 0) {
vm_page_lock(m);
- vm_page_flash(m);
+ if (atomic_cmpset_int(&m->busy_lock, x, VPB_UNLOCKED)) {
+ vm_page_unlock(m);
+ continue;
+ }
+ wakeup(m);
vm_page_unlock(m);
+ break;
}
}
-int
-vm_page_busy_tryrlock(vm_page_t m)
+/*
+ * vm_page_busy_sleep:
+ *
+ * Sleep and release the page lock, using the page pointer as wchan.
+ * This is used to implement the hard-path of busying mechanism.
+ *
+ * The given page must be locked.
+ */
+void
+vm_page_busy_sleep(vm_page_t m, const char *wmesg)
{
+ u_int x;
- VM_OBJECT_ASSERT_WLOCKED(m->object);
- if ((m->oflags & VPO_BUSY) != 0)
- return (0);
- m->busy++;
- return (1);
-}
-
-int
-vm_page_busy_trywlock(vm_page_t m)
-{
+ vm_page_lock_assert(m, MA_OWNED);
- VM_OBJECT_ASSERT_WLOCKED(m->object);
- if ((m->oflags & VPO_BUSY) != 0 || m->busy != 0)
- return (0);
- m->oflags |= VPO_BUSY;
- return (1);
+ x = m->busy_lock;
+ if (x == VPB_UNLOCKED) {
+ vm_page_unlock(m);
+ return;
+ }
+ if ((x & VPB_LOCK_WAITERS) == 0 &&
+ !atomic_cmpset_int(&m->busy_lock, x, x | VPB_LOCK_WAITERS)) {
+ vm_page_unlock(m);
+ return;
+ }
+ msleep(m, vm_page_lockptr(m), PVM | PDROP, wmesg, 0);
}
int
-vm_page_busy_wlocked(vm_page_t m)
+vm_page_busy_tryrlock(vm_page_t m)
{
+ u_int x;
- return ((m->oflags & VPO_BUSY) != 0);
+ x = m->busy_lock;
+ return ((x & VPB_LOCK_READ) != 0 &&
+ atomic_cmpset_acq_int(&m->busy_lock, x, x + VPB_ONE_READER));
}
void
-vm_page_busy_wunlock(vm_page_t m)
+vm_page_busy_wunlock_hard(vm_page_t m)
{
- VM_OBJECT_ASSERT_WLOCKED(m->object);
- KASSERT(m->oflags & VPO_BUSY,
- ("vm_page_busy_wunlock: page not busy"));
- m->oflags &= ~VPO_BUSY;
+ vm_page_busy_assert_wlocked(m);
+
vm_page_lock(m);
- vm_page_flash(m);
+ atomic_store_rel_int(&m->busy_lock, VPB_UNLOCKED);
+ wakeup(m);
vm_page_unlock(m);
}
-
/*
* vm_page_flash:
*
@@ -558,12 +595,22 @@ vm_page_busy_wunlock(vm_page_t m)
void
vm_page_flash(vm_page_t m)
{
+ u_int retry, x;
vm_page_lock_assert(m, MA_OWNED);
- if (m->flags & PG_WANTED) {
- m->flags &= ~PG_WANTED;
- wakeup(m);
+
+ retry = 0;
+ for (;;) {
+ if (retry++ > 1000)
+ panic("vm_page_flash: failed loop %p", m);
+ x = m->busy_lock;
+ if ((x & VPB_LOCK_WAITERS) == 0)
+ return;
+ if (atomic_cmpset_int(&m->busy_lock, x,
+ x & (~VPB_LOCK_WAITERS)))
+ break;
}
+ wakeup(m);
}
/*
@@ -678,7 +725,8 @@ vm_page_initfake(vm_page_t m, vm_paddr_t
/* Fictitious pages don't use "segind". */
m->flags = PG_FICTITIOUS;
/* Fictitious pages don't use "order" or "pool". */
- m->oflags = VPO_UNMANAGED | VPO_BUSY;
+ m->oflags = VPO_UNMANAGED;
+ m->busy_lock = VPB_SINGLE_WRITER;
m->wire_count = 1;
memattr:
pmap_page_set_memattr(m, memattr);
@@ -758,7 +806,7 @@ vm_page_readahead_finish(vm_page_t m)
* unless the page is wanted by another thread.
*/
vm_page_lock(m);
- if (m->flags & PG_WANTED)
+ if ((m->busy_lock & VPB_LOCK_WAITERS) != 0)
vm_page_activate(m);
else
vm_page_deactivate(m);
@@ -778,22 +826,6 @@ vm_page_readahead_finish(vm_page_t m)
}
/*
- * _vm_page_sleep_onpage:
- *
- * Sleep and release the page lock, using the page pointer as wchan.
- *
- * The given page must be locked.
- */
-static inline int
-_vm_page_sleep_onpage(vm_page_t m, int pri, const char *wmesg, int timo)
-{
-
- vm_page_lock_assert(m, MA_OWNED);
- m->flags |= PG_WANTED;
- return (msleep(m, vm_page_lockptr(m), pri | PDROP, wmesg, timo));
-}
-
-/*
* vm_page_sleep_if_busy:
*
* Sleep and release the page queues lock if the page is busied.
@@ -821,7 +853,7 @@ vm_page_sleep_if_busy(vm_page_t m, const
obj = m->object;
vm_page_lock(m);
VM_OBJECT_WUNLOCK(obj);
- _vm_page_sleep_onpage(m, PVM, msg, 0);
+ vm_page_busy_sleep(m, msg);
VM_OBJECT_WLOCK(obj);
return (TRUE);
}
@@ -829,20 +861,6 @@ vm_page_sleep_if_busy(vm_page_t m, const
}
/*
- * vm_page_sleep_onpage:
- *
- * External version of _vm_page_sleep_onpage().
- *
- * Check the inline version for comments.
- */
-int
-vm_page_sleep_onpage(vm_page_t m, int pri, const char *wmesg, int timo)
-{
-
- return (_vm_page_sleep_onpage(m, pri, wmesg, timo));
-}
-
-/*
* vm_page_dirty_KBI: [ internal use only ]
*
* Set all bits in the page's dirty field.
@@ -972,7 +990,6 @@ vm_page_remove(vm_page_t m)
return;
VM_OBJECT_ASSERT_WLOCKED(object);
if (vm_page_busy_wlocked(m)) {
- m->oflags &= ~VPO_BUSY;
lockacq = FALSE;
if ((m->oflags & VPO_UNMANAGED) != 0 &&
!mtx_owned(vm_page_lockptr(m))) {
@@ -980,6 +997,7 @@ vm_page_remove(vm_page_t m)
vm_page_lock(m);
}
vm_page_flash(m);
+ atomic_store_rel_int(&m->busy_lock, VPB_UNLOCKED);
if (lockacq)
vm_page_unlock(m);
}
@@ -1370,7 +1388,8 @@ vm_page_alloc(vm_object_t object, vm_pin
("vm_page_alloc: page %p has unexpected queue %d", m, m->queue));
KASSERT(m->wire_count == 0, ("vm_page_alloc: page %p is wired", m));
KASSERT(m->hold_count == 0, ("vm_page_alloc: page %p is held", m));
- KASSERT(m->busy == 0, ("vm_page_alloc: page %p is busy", m));
+ KASSERT(!vm_page_busy_rlocked(m),
+ ("vm_page_alloc: page %p is busy", m));
KASSERT(m->dirty == 0, ("vm_page_alloc: page %p is dirty", m));
KASSERT(pmap_page_get_memattr(m) == VM_MEMATTR_DEFAULT,
("vm_page_alloc: page %p has unexpected memattr %d", m,
@@ -1414,10 +1433,11 @@ vm_page_alloc(vm_object_t object, vm_pin
m->aflags = 0;
m->oflags = object == NULL || (object->flags & OBJ_UNMANAGED) != 0 ?
VPO_UNMANAGED : 0;
+ m->busy_lock = VPB_UNLOCKED;
if ((req & (VM_ALLOC_NOBUSY | VM_ALLOC_NOOBJ | VM_ALLOC_RBUSY)) == 0)
- m->oflags |= VPO_BUSY;
+ m->busy_lock = VPB_SINGLE_WRITER;
if ((req & VM_ALLOC_RBUSY) != 0)
- m->busy++;
+ m->busy_lock = VPB_READERS_LOCK(1);
if (req & VM_ALLOC_WIRED) {
/*
* The page lock is not required for wiring a page until that
@@ -1585,8 +1605,6 @@ retry:
atomic_add_int(&cnt.v_wire_count, npages);
oflags = VPO_UNMANAGED;
if (object != NULL) {
- if ((req & (VM_ALLOC_NOBUSY | VM_ALLOC_RBUSY)) == 0)
- oflags |= VPO_BUSY;
if (object->memattr != VM_MEMATTR_DEFAULT &&
memattr == VM_MEMATTR_DEFAULT)
memattr = object->memattr;
@@ -1594,8 +1612,13 @@ retry:
for (m = m_ret; m < &m_ret[npages]; m++) {
m->aflags = 0;
m->flags = (m->flags | PG_NODUMP) & flags;
- if ((req & VM_ALLOC_RBUSY) != 0)
- m->busy++;
+ m->busy_lock = VPB_UNLOCKED;
+ if (object != NULL) {
+ if ((req & (VM_ALLOC_NOBUSY | VM_ALLOC_RBUSY)) == 0)
+ m->busy_lock = VPB_SINGLE_WRITER;
+ if ((req & VM_ALLOC_RBUSY) != 0)
+ m->busy_lock = VPB_READERS_LOCK(1);
+ }
if ((req & VM_ALLOC_WIRED) != 0)
m->wire_count = 1;
/* Unmanaged pages don't use "act_count". */
@@ -1638,7 +1661,7 @@ vm_page_alloc_init(vm_page_t m)
("vm_page_alloc_init: page %p is wired", m));
KASSERT(m->hold_count == 0,
("vm_page_alloc_init: page %p is held", m));
- KASSERT(m->busy == 0,
+ KASSERT(!vm_page_busy_rlocked(m),
("vm_page_alloc_init: page %p is busy", m));
KASSERT(m->dirty == 0,
("vm_page_alloc_init: page %p is dirty", m));
@@ -1997,7 +2020,7 @@ vm_page_free_toq(vm_page_t m)
if (VM_PAGE_IS_FREE(m))
panic("vm_page_free: freeing free page %p", m);
- else if (m->busy != 0)
+ else if (vm_page_busy_rlocked(m))
panic("vm_page_free: freeing busy page %p", m);
/*
@@ -2443,6 +2466,7 @@ vm_page_t
vm_page_grab(vm_object_t object, vm_pindex_t pindex, int allocflags)
{
vm_page_t m;
+ int sleep;
VM_OBJECT_ASSERT_WLOCKED(object);
KASSERT((allocflags & VM_ALLOC_RETRY) != 0,
@@ -2452,8 +2476,9 @@ vm_page_grab(vm_object_t object, vm_pind
("vm_page_grab: VM_ALLOC_RBUSY/VM_ALLOC_IGN_RBUSY mismatch"));
retrylookup:
if ((m = vm_page_lookup(object, pindex)) != NULL) {
- if ((m->oflags & VPO_BUSY) != 0 ||
- ((allocflags & VM_ALLOC_IGN_RBUSY) == 0 && m->busy != 0)) {
+ sleep = (allocflags & VM_ALLOC_IGN_RBUSY) != 0 ?
+ vm_page_busy_wlocked(m) : vm_page_busy_locked(m);
+ if (sleep) {
/*
* Reference the page before unlocking and
* sleeping so that the page daemon is less
@@ -2462,7 +2487,7 @@ retrylookup:
vm_page_aflag_set(m, PGA_REFERENCED);
vm_page_lock(m);
VM_OBJECT_WUNLOCK(object);
- vm_page_sleep(m, "pgrbwt");
+ vm_page_busy_sleep(m, "pgrbwt");
VM_OBJECT_WLOCK(object);
goto retrylookup;
} else {
@@ -3020,9 +3045,9 @@ DB_SHOW_COMMAND(pginfo, vm_page_print_pg
m = (vm_page_t)addr;
db_printf(
"page %p obj %p pidx 0x%jx phys 0x%jx q %d hold %d wire %d\n"
- " af 0x%x of 0x%x f 0x%x act %d busy %d valid 0x%x dirty 0x%x\n",
+ " af 0x%x of 0x%x f 0x%x act %d busy %x valid 0x%x dirty 0x%x\n",
m, m->object, (uintmax_t)m->pindex, (uintmax_t)m->phys_addr,
m->queue, m->hold_count, m->wire_count, m->aflags, m->oflags,
- m->flags, m->act_count, m->busy, m->valid, m->dirty);
+ m->flags, m->act_count, m->busy_lock, m->valid, m->dirty);
}
#endif /* DDB */
Modified: user/attilio/vmobj-readlock/sys/vm/vm_page.h
==============================================================================
--- user/attilio/vmobj-readlock/sys/vm/vm_page.h Tue Jun 25 22:14:32 2013 (r252219)
+++ user/attilio/vmobj-readlock/sys/vm/vm_page.h Tue Jun 25 22:46:52 2013 (r252220)
@@ -155,11 +155,12 @@ struct vm_page {
uint8_t oflags; /* page VPO_* flags (O) */
uint16_t flags; /* page PG_* flags (P) */
u_char act_count; /* page usage count (P) */
- u_char busy; /* page busy count (O) */
+ u_char basy; /* page busy count (O) */
/* NOTE that these must support one bit per DEV_BSIZE in a page!!! */
/* so, on normal X86 kernels, they must be at least 8 bits wide */
vm_page_bits_t valid; /* map of valid DEV_BSIZE chunks (O) */
vm_page_bits_t dirty; /* map of dirty DEV_BSIZE chunks (M) */
+ u_int busy_lock; /* busy owners lock */
};
/*
@@ -176,12 +177,31 @@ struct vm_page {
* mappings, and such pages are also not on any PQ queue.
*
*/
-#define VPO_BUSY 0x01 /* page is in transit */
+#define VPO_UNUSED01 0x01 /* --available-- */
#define VPO_SWAPSLEEP 0x02 /* waiting for swap to finish */
#define VPO_UNMANAGED 0x04 /* no PV management for page */
#define VPO_SWAPINPROG 0x08 /* swap I/O in progress on page */
#define VPO_NOSYNC 0x10 /* do not collect for syncer */
+/*
+ * ARXXX: Insert comments for busy here.
+ */
+#define VPB_LOCK_READ 0x01
+#define VPB_LOCK_WRITE 0x02
+#define VPB_LOCK_WAITERS 0x04
+#define VPB_LOCK_FLAGMASK \
+ (VPB_LOCK_READ | VPB_LOCK_WRITE | VPB_LOCK_WAITERS)
+
+#define VPB_READERS_SHIFT 3
+#define VPB_READERS(x) \
+ (((x) & ~VPB_LOCK_FLAGMASK) >> VPB_READERS_SHIFT)
+#define VPB_READERS_LOCK(x) ((x) << VPB_READERS_SHIFT | VPB_LOCK_READ)
+#define VPB_ONE_READER (1 << VPB_READERS_SHIFT)
+
+#define VPB_SINGLE_WRITER VPB_LOCK_WRITE
+
+#define VPB_UNLOCKED VPB_READERS_LOCK(0)
+
#define PQ_NONE 255
#define PQ_INACTIVE 0
#define PQ_ACTIVE 1
@@ -282,7 +302,6 @@ extern struct mtx_padalign pa_lock[];
#define PG_WINATCFLS 0x0040 /* flush dirty page on inactive q */
#define PG_NODUMP 0x0080 /* don't include this page in a dump */
#define PG_UNHOLDFREE 0x0100 /* delayed free of a held page */
-#define PG_WANTED 0x0200 /* someone is waiting for page */
/*
* Misc constants.
@@ -372,14 +391,12 @@ malloc2vm_flags(int malloc_flags)
}
#endif
-void vm_page_busy_assert_unlocked(vm_page_t m);
-void vm_page_busy_assert_wlocked(vm_page_t m);
-int vm_page_busy_locked(vm_page_t m);
+void vm_page_busy_downgrade(vm_page_t m);
+int vm_page_busy_rlocked(vm_page_t m);
void vm_page_busy_runlock(vm_page_t m);
+void vm_page_busy_sleep(vm_page_t m, const char *msg);
int vm_page_busy_tryrlock(vm_page_t m);
-int vm_page_busy_trywlock(vm_page_t m);
-int vm_page_busy_wlocked(vm_page_t m);
-void vm_page_busy_wunlock(vm_page_t m);
+void vm_page_busy_wunlock_hard(vm_page_t m);
void vm_page_flash(vm_page_t m);
void vm_page_hold(vm_page_t mem);
void vm_page_unhold(vm_page_t mem);
@@ -420,7 +437,6 @@ void vm_page_requeue(vm_page_t m);
void vm_page_requeue_locked(vm_page_t m);
void vm_page_set_valid_range(vm_page_t m, int base, int size);
int vm_page_sleep_if_busy(vm_page_t m, const char *msg);
-int vm_page_sleep_onpage(vm_page_t m, int pri, const char *msg, int timo);
vm_offset_t vm_page_startup(vm_offset_t vaddr);
void vm_page_unhold_pages(vm_page_t *ma, int count);
void vm_page_unwire (vm_page_t, int);
@@ -448,16 +464,46 @@ void vm_page_assert_locked_KBI(vm_page_t
void vm_page_lock_assert_KBI(vm_page_t m, int a, const char *file, int line);
#endif
+#define vm_page_busy_assert_rlocked(m) \
+ KASSERT(vm_page_busy_rlocked(m), \
+ ("vm_page_busy_assert_rlocked: page %p not read busy @ %s:%d", \
+ (void *)m, __FILE__, __LINE__));
+
+#define vm_page_busy_assert_unlocked(m) \
+ KASSERT(!vm_page_busy_locked(m), \
+ ("vm_page_busy_assert_wlocked: page %p busy @ %s:%d", \
+ (void *)m, __FILE__, __LINE__));
+
+#define vm_page_busy_assert_wlocked(m) \
+ KASSERT(vm_page_busy_wlocked(m), \
+ ("vm_page_busy_assert_wlocked: page %p not write busy @ %s:%d", \
+ (void *)m, __FILE__, __LINE__));
+
+#define vm_page_busy_locked(m) \
+ ((m)->busy_lock != VPB_UNLOCKED)
+
#define vm_page_busy_rlock(m) do { \
if (!vm_page_busy_tryrlock(m)) \
panic("%s: page %p failed read busing", __func__, m); \
} while (0)
+#define vm_page_busy_trywlock(m) \
+ (atomic_cmpset_acq_int(&m->busy_lock, VPB_UNLOCKED, VPB_SINGLE_WRITER))
+
#define vm_page_busy_wlock(m) do { \
if (!vm_page_busy_trywlock(m)) \
panic("%s: page %p failed write busing", __func__, m); \
} while (0)
+#define vm_page_busy_wlocked(m) \
+ ((m->busy_lock & VPB_SINGLE_WRITER) != 0)
+
+#define vm_page_busy_wunlock(m) do { \
+ if (!atomic_cmpset_rel_int(&(m)->busy_lock, VPB_SINGLE_WRITER, \
+ VPB_UNLOCKED)) \
+ vm_page_busy_wunlock_hard(m); \
+} while (0)
+
#ifdef INVARIANTS
void vm_page_object_lock_assert(vm_page_t m);
#define VM_PAGE_OBJECT_LOCK_ASSERT(m) vm_page_object_lock_assert(m)
@@ -556,21 +602,6 @@ vm_page_dirty(vm_page_t m)
}
/*
- * vm_page_sleep:
- *
- * Convenience wrapper around vm_page_sleep_onpage(), passing
- * PVM priority and 0 timeout values. Unlocks the page upon return.
- *
- * The page must be locked.
- */
-static __inline void
-vm_page_sleep(vm_page_t m, const char *msg)
-{
-
- vm_page_sleep_onpage(m, PVM, msg, 0);
-}
-
-/*
* vm_page_remque:
*
* If the given page is in a page queue, then remove it from that page
Modified: user/attilio/vmobj-readlock/sys/vm/vm_pageout.c
==============================================================================
--- user/attilio/vmobj-readlock/sys/vm/vm_pageout.c Tue Jun 25 22:14:32 2013 (r252219)
+++ user/attilio/vmobj-readlock/sys/vm/vm_pageout.c Tue Jun 25 22:46:52 2013 (r252220)
@@ -241,7 +241,7 @@ vm_pageout_init_marker(vm_page_t marker,
bzero(marker, sizeof(*marker));
marker->flags = PG_MARKER;
- marker->oflags = VPO_BUSY;
+ marker->busy_lock = VPB_SINGLE_WRITER;
marker->queue = queue;
marker->hold_count = 1;
}
Modified: user/attilio/vmobj-readlock/sys/vm/vm_phys.c
==============================================================================
--- user/attilio/vmobj-readlock/sys/vm/vm_phys.c Tue Jun 25 22:14:32 2013 (r252219)
+++ user/attilio/vmobj-readlock/sys/vm/vm_phys.c Tue Jun 25 22:46:52 2013 (r252220)
@@ -561,7 +561,7 @@ vm_phys_fictitious_reg_range(vm_paddr_t
vm_page_initfake(&fp[i], start + PAGE_SIZE * i, memattr);
pmap_page_init(&fp[i]);
fp[i].oflags &= ~VPO_UNMANAGED;
- fp[i].oflags &= ~VPO_BUSY;
+ fp[i].busy_lock = VPB_UNLOCKED;
}
mtx_lock(&vm_phys_fictitious_reg_mtx);
for (segind = 0; segind < VM_PHYS_FICTITIOUS_NSEGS; segind++) {
Modified: user/attilio/vmobj-readlock/sys/vm/vnode_pager.c
==============================================================================
--- user/attilio/vmobj-readlock/sys/vm/vnode_pager.c Tue Jun 25 22:14:32 2013 (r252219)
+++ user/attilio/vmobj-readlock/sys/vm/vnode_pager.c Tue Jun 25 22:46:52 2013 (r252220)
@@ -1140,8 +1140,7 @@ vnode_pager_generic_putpages(struct vnod
* pmap operation.
*/
m = ma[ncount - 1];
- KASSERT(m->busy > 0,
- ("vnode_pager_generic_putpages: page %p is not busy", m));
+ vm_page_busy_assert_rlocked(m);
KASSERT(!pmap_page_is_write_mapped(m),
("vnode_pager_generic_putpages: page %p is not read-only", m));
vm_page_clear_dirty(m, pgoff, PAGE_SIZE -
More information about the svn-src-user
mailing list