git: 87b646630c48 - main - vm_page: Consolidate page busy sleep mechanisms
- Go to: [ bottom of page ] [ top of archives ] [ this month ]
Date: Mon, 15 Nov 2021 18:02:59 UTC
The branch main has been updated by markj:
URL: https://cgit.FreeBSD.org/src/commit/?id=87b646630c4892e21446cd096bea6bcaecea33ac
commit 87b646630c4892e21446cd096bea6bcaecea33ac
Author: Mark Johnston <markj@FreeBSD.org>
AuthorDate: 2021-11-15 16:35:44 +0000
Commit: Mark Johnston <markj@FreeBSD.org>
CommitDate: 2021-11-15 18:01:30 +0000
vm_page: Consolidate page busy sleep mechanisms
- Modify vm_page_busy_sleep() and vm_page_busy_sleep_unlocked() to take
a VM_ALLOC_* flag indicating whether to sleep on shared-busy, and fix
up callers.
- Modify vm_page_busy_sleep() to return a status indicating whether the
object lock was dropped, and fix up callers.
- Convert callers of vm_page_sleep_if_busy() to use vm_page_busy_sleep()
instead.
- Remove vm_page_sleep_if_(x)busy().
No functional change intended.
Obtained from: jeff (object_concurrency patches)
Reviewed by: kib
MFC after: 2 weeks
Differential Revision: https://reviews.freebsd.org/D32947
---
ObsoleteFiles.inc | 3 ++
share/man/man9/Makefile | 1 -
share/man/man9/vm_page_busy.9 | 40 +++++++++---------
sys/vm/vm_fault.c | 5 +--
sys/vm/vm_object.c | 27 ++++++++----
sys/vm/vm_page.c | 97 ++++++++-----------------------------------
sys/vm/vm_page.h | 6 +--
7 files changed, 62 insertions(+), 117 deletions(-)
diff --git a/ObsoleteFiles.inc b/ObsoleteFiles.inc
index aec721b740b9..a5ca6b8542a1 100644
--- a/ObsoleteFiles.inc
+++ b/ObsoleteFiles.inc
@@ -40,6 +40,9 @@
# xargs -n1 | sort | uniq -d;
# done
+# 20211115: vm_page_sleep_if_busy removed
+OLD_FILES+=share/man/man9/vm_page_sleep_if_busy.9.gz
+
# 20211113: new clang import which bumps version from 12.0.1 to 13.0.0.
OLD_FILES+=usr/lib/clang/12.0.1/include/cuda_wrappers/algorithm
OLD_FILES+=usr/lib/clang/12.0.1/include/cuda_wrappers/complex
diff --git a/share/man/man9/Makefile b/share/man/man9/Makefile
index eb4465259226..b22dd2dff262 100644
--- a/share/man/man9/Makefile
+++ b/share/man/man9/Makefile
@@ -2354,7 +2354,6 @@ MLINKS+=vm_page_busy.9 vm_page_busied.9 \
vm_page_busy.9 vm_page_busy_sleep.9 \
vm_page_busy.9 vm_page_sbusied.9 \
vm_page_busy.9 vm_page_sbusy.9 \
- vm_page_busy.9 vm_page_sleep_if_busy.9 \
vm_page_busy.9 vm_page_sunbusy.9 \
vm_page_busy.9 vm_page_trysbusy.9 \
vm_page_busy.9 vm_page_tryxbusy.9 \
diff --git a/share/man/man9/vm_page_busy.9 b/share/man/man9/vm_page_busy.9
index f4e922a8319f..3f08a467bcb1 100644
--- a/share/man/man9/vm_page_busy.9
+++ b/share/man/man9/vm_page_busy.9
@@ -24,7 +24,7 @@
.\" DAMAGE.
.\"
.\" $FreeBSD$
-.Dd August 07, 2013
+.Dd November 11, 2021
.Dt VM_PAGE_BUSY 9
.Os
.Sh NAME
@@ -33,7 +33,6 @@
.Nm vm_page_busy_sleep ,
.Nm vm_page_sbusied ,
.Nm vm_page_sbusy ,
-.Nm vm_page_sleep_if_busy ,
.Nm vm_page_sunbusy ,
.Nm vm_page_trysbusy ,
.Nm vm_page_tryxbusy ,
@@ -52,14 +51,12 @@
.Fn vm_page_busied "vm_page_t m"
.Ft void
.Fn vm_page_busy_downgrade "vm_page_t m"
-.Ft void
-.Fn vm_page_busy_sleep "vm_page_t m" "const char *msg"
+.Ft bool
+.Fn vm_page_busy_sleep "vm_page_t m" "const char *msg" "int allocflags"
.Ft int
.Fn vm_page_sbusied "vm_page_t m"
.Ft void
.Fn vm_page_sbusy "vm_page_t m"
-.Ft int
-.Fn vm_page_sleep_if_busy "vm_page_t m" "const char *msg"
.Ft void
.Fn vm_page_sunbusy "vm_page_t m"
.Ft int
@@ -114,8 +111,23 @@ from an exclusive busy state to a shared busy state.
.Pp
The
.Fn vm_page_busy_sleep
-function puts the invoking thread to sleep using the appropriate
-waitchannels for the busy mechanism.
+checks the busy state of the page
+.Fa m
+and puts the invoking thread to sleep if the page is busy.
+The VM object for the page must be locked.
+The
+.Fa allocflags
+parameter must be either
+.Dv 0 ,
+in which case the function will sleep if the page is busied,
+or
+.Dv VM_ALLOC_IGN_SBUSY ,
+in which case the function will sleep only if the page is exclusively
+busied.
+A return value of true indicates that the invoking thread was put to
+sleep and that the object was unlocked.
+A return value of false indicates that the invoking thread did not sleep
+and the object remains locked.
The parameter
.Fa msg
is a string describing the sleep condition for userland tools.
@@ -133,18 +145,6 @@ function shared busies
.Fa m .
.Pp
The
-.Fn vm_page_sleep_if_busy
-function puts the invoking thread to sleep, using the appropriate
-waitchannels for the busy mechanism, if
-.Fa m .
-is busied in either exclusive or shared mode.
-If the invoking thread slept a non-zero value is returned, otherwise
-0 is returned.
-The parameter
-.Fa msg
-is a string describing the sleep condition for userland tools.
-.Pp
-The
.Fn vm_page_sunbusy
function shared unbusies
.Fa m .
diff --git a/sys/vm/vm_fault.c b/sys/vm/vm_fault.c
index 6bc59222b50e..d459bad78dd8 100644
--- a/sys/vm/vm_fault.c
+++ b/sys/vm/vm_fault.c
@@ -1294,9 +1294,8 @@ vm_fault_busy_sleep(struct faultstate *fs)
}
vm_object_pip_wakeup(fs->object);
unlock_map(fs);
- if (fs->m == vm_page_lookup(fs->object, fs->pindex))
- vm_page_busy_sleep(fs->m, "vmpfw", false);
- else
+ if (fs->m != vm_page_lookup(fs->object, fs->pindex) ||
+ !vm_page_busy_sleep(fs->m, "vmpfw", 0))
VM_OBJECT_WUNLOCK(fs->object);
VM_CNT_INC(v_intrans);
vm_object_deallocate(fs->first_object);
diff --git a/sys/vm/vm_object.c b/sys/vm/vm_object.c
index 5bbe7faed50b..6c4df272f739 100644
--- a/sys/vm/vm_object.c
+++ b/sys/vm/vm_object.c
@@ -1397,7 +1397,8 @@ next_page:
*/
vm_page_aflag_set(tm, PGA_REFERENCED);
}
- vm_page_busy_sleep(tm, "madvpo", false);
+ if (!vm_page_busy_sleep(tm, "madvpo", 0))
+ VM_OBJECT_WUNLOCK(tobject);
goto relookup;
}
vm_page_advise(tm, advice);
@@ -1581,7 +1582,8 @@ retry:
*/
if (vm_page_tryxbusy(m) == 0) {
VM_OBJECT_WUNLOCK(new_object);
- vm_page_sleep_if_busy(m, "spltwt");
+ if (vm_page_busy_sleep(m, "spltwt", 0))
+ VM_OBJECT_WLOCK(orig_object);
VM_OBJECT_WLOCK(new_object);
goto retry;
}
@@ -1667,14 +1669,17 @@ vm_object_collapse_scan_wait(vm_object_t object, vm_page_t p)
VM_OBJECT_WUNLOCK(object);
VM_OBJECT_WUNLOCK(backing_object);
vm_radix_wait();
+ VM_OBJECT_WLOCK(object);
+ } else if (p->object == object) {
+ VM_OBJECT_WUNLOCK(backing_object);
+ if (vm_page_busy_sleep(p, "vmocol", 0))
+ VM_OBJECT_WLOCK(object);
} else {
- if (p->object == object)
+ VM_OBJECT_WUNLOCK(object);
+ if (!vm_page_busy_sleep(p, "vmocol", 0))
VM_OBJECT_WUNLOCK(backing_object);
- else
- VM_OBJECT_WUNLOCK(object);
- vm_page_busy_sleep(p, "vmocol", false);
+ VM_OBJECT_WLOCK(object);
}
- VM_OBJECT_WLOCK(object);
VM_OBJECT_WLOCK(backing_object);
return (TAILQ_FIRST(&backing_object->memq));
}
@@ -2103,7 +2108,8 @@ again:
* not specified.
*/
if (vm_page_tryxbusy(p) == 0) {
- vm_page_sleep_if_busy(p, "vmopar");
+ if (vm_page_busy_sleep(p, "vmopar", 0))
+ VM_OBJECT_WLOCK(object);
goto again;
}
if (vm_page_wired(p)) {
@@ -2407,7 +2413,10 @@ again:
VM_OBJECT_RUNLOCK(tobject);
tobject = t1object;
}
- vm_page_busy_sleep(tm, "unwbo", true);
+ tobject = tm->object;
+ if (!vm_page_busy_sleep(tm, "unwbo",
+ VM_ALLOC_IGN_SBUSY))
+ VM_OBJECT_RUNLOCK(tobject);
goto again;
}
vm_page_unwire(tm, queue);
diff --git a/sys/vm/vm_page.c b/sys/vm/vm_page.c
index ac710bf43d08..28eb95d9fb33 100644
--- a/sys/vm/vm_page.c
+++ b/sys/vm/vm_page.c
@@ -1000,24 +1000,26 @@ vm_page_sunbusy(vm_page_t m)
* vm_page_busy_sleep:
*
* Sleep if the page is busy, using the page pointer as wchan.
- * This is used to implement the hard-path of busying mechanism.
+ * This is used to implement the hard-path of the busying mechanism.
+ *
+ * If VM_ALLOC_IGN_SBUSY is specified in allocflags, the function
+ * will not sleep if the page is shared-busy.
*
- * If nonshared is true, sleep only if the page is xbusy.
+ * The object lock must be held on entry.
*
- * The object lock must be held on entry and will be released on exit.
+ * Returns true if it slept and dropped the object lock, or false
+ * if there was no sleep and the lock is still held.
*/
-void
-vm_page_busy_sleep(vm_page_t m, const char *wmesg, bool nonshared)
+bool
+vm_page_busy_sleep(vm_page_t m, const char *wmesg, int allocflags)
{
vm_object_t obj;
obj = m->object;
VM_OBJECT_ASSERT_LOCKED(obj);
- vm_page_lock_assert(m, MA_NOTOWNED);
- if (!_vm_page_busy_sleep(obj, m, m->pindex, wmesg,
- nonshared ? VM_ALLOC_SBUSY : 0 , true))
- VM_OBJECT_DROP(obj);
+ return (_vm_page_busy_sleep(obj, m, m->pindex, wmesg, allocflags,
+ true));
}
/*
@@ -1026,21 +1028,19 @@ vm_page_busy_sleep(vm_page_t m, const char *wmesg, bool nonshared)
* Sleep if the page is busy, using the page pointer as wchan.
* This is used to implement the hard-path of busying mechanism.
*
- * If nonshared is true, sleep only if the page is xbusy.
+ * If VM_ALLOC_IGN_SBUSY is specified in allocflags, the function
+ * will not sleep if the page is shared-busy.
*
* The object lock must not be held on entry. The operation will
* return if the page changes identity.
*/
void
vm_page_busy_sleep_unlocked(vm_object_t obj, vm_page_t m, vm_pindex_t pindex,
- const char *wmesg, bool nonshared)
+ const char *wmesg, int allocflags)
{
-
VM_OBJECT_ASSERT_UNLOCKED(obj);
- vm_page_lock_assert(m, MA_NOTOWNED);
- _vm_page_busy_sleep(obj, m, pindex, wmesg,
- nonshared ? VM_ALLOC_SBUSY : 0, false);
+ (void)_vm_page_busy_sleep(obj, m, pindex, wmesg, allocflags, false);
}
/*
@@ -1050,6 +1050,8 @@ vm_page_busy_sleep_unlocked(vm_object_t obj, vm_page_t m, vm_pindex_t pindex,
* lockstate against parameters. Returns true if it sleeps and
* false otherwise.
*
+ * allocflags uses VM_ALLOC_* flags to specify the lock required.
+ *
* If locked is true the lock will be dropped for any true returns
* and held for any false returns.
*/
@@ -1395,71 +1397,6 @@ vm_page_free_invalid(vm_page_t m)
vm_page_free(m);
}
-/*
- * vm_page_sleep_if_busy:
- *
- * Sleep and release the object lock if the page is busied.
- * Returns TRUE if the thread slept.
- *
- * The given page must be unlocked and object containing it must
- * be locked.
- */
-int
-vm_page_sleep_if_busy(vm_page_t m, const char *wmesg)
-{
- vm_object_t obj;
-
- vm_page_lock_assert(m, MA_NOTOWNED);
- VM_OBJECT_ASSERT_WLOCKED(m->object);
-
- /*
- * The page-specific object must be cached because page
- * identity can change during the sleep, causing the
- * re-lock of a different object.
- * It is assumed that a reference to the object is already
- * held by the callers.
- */
- obj = m->object;
- if (_vm_page_busy_sleep(obj, m, m->pindex, wmesg, 0, true)) {
- VM_OBJECT_WLOCK(obj);
- return (TRUE);
- }
- return (FALSE);
-}
-
-/*
- * vm_page_sleep_if_xbusy:
- *
- * Sleep and release the object lock if the page is xbusied.
- * Returns TRUE if the thread slept.
- *
- * The given page must be unlocked and object containing it must
- * be locked.
- */
-int
-vm_page_sleep_if_xbusy(vm_page_t m, const char *wmesg)
-{
- vm_object_t obj;
-
- vm_page_lock_assert(m, MA_NOTOWNED);
- VM_OBJECT_ASSERT_WLOCKED(m->object);
-
- /*
- * The page-specific object must be cached because page
- * identity can change during the sleep, causing the
- * re-lock of a different object.
- * It is assumed that a reference to the object is already
- * held by the callers.
- */
- obj = m->object;
- if (_vm_page_busy_sleep(obj, m, m->pindex, wmesg, VM_ALLOC_SBUSY,
- true)) {
- VM_OBJECT_WLOCK(obj);
- return (TRUE);
- }
- return (FALSE);
-}
-
/*
* vm_page_dirty_KBI: [ internal use only ]
*
diff --git a/sys/vm/vm_page.h b/sys/vm/vm_page.h
index 21e33facc0a8..7adeaa7ddbe2 100644
--- a/sys/vm/vm_page.h
+++ b/sys/vm/vm_page.h
@@ -599,9 +599,9 @@ malloc2vm_flags(int malloc_flags)
bool vm_page_busy_acquire(vm_page_t m, int allocflags);
void vm_page_busy_downgrade(vm_page_t m);
int vm_page_busy_tryupgrade(vm_page_t m);
-void vm_page_busy_sleep(vm_page_t m, const char *msg, bool nonshared);
+bool vm_page_busy_sleep(vm_page_t m, const char *msg, int allocflags);
void vm_page_busy_sleep_unlocked(vm_object_t obj, vm_page_t m,
- vm_pindex_t pindex, const char *wmesg, bool nonshared);
+ vm_pindex_t pindex, const char *wmesg, int allocflags);
void vm_page_free(vm_page_t m);
void vm_page_free_zero(vm_page_t m);
@@ -685,8 +685,6 @@ vm_page_t vm_page_scan_contig(u_long npages, vm_page_t m_start,
vm_page_t m_end, u_long alignment, vm_paddr_t boundary, int options);
vm_page_bits_t vm_page_set_dirty(vm_page_t m);
void vm_page_set_valid_range(vm_page_t m, int base, int size);
-int vm_page_sleep_if_busy(vm_page_t m, const char *msg);
-int vm_page_sleep_if_xbusy(vm_page_t m, const char *msg);
vm_offset_t vm_page_startup(vm_offset_t vaddr);
void vm_page_sunbusy(vm_page_t m);
bool vm_page_try_remove_all(vm_page_t m);