svn commit: r355062 - in head/sys: kern vm
Konstantin Belousov
kib at FreeBSD.org
Sun Nov 24 19:12:24 UTC 2019
Author: kib
Date: Sun Nov 24 19:12:23 2019
New Revision: 355062
URL: https://svnweb.freebsd.org/changeset/base/355062
Log:
Record part of the owner struct thread pointer into busy_lock.
Record as much bits from curthread into busy_lock as fits. Low bits
for struct thread * representation are zero due to struct and zone
alignment, and they leave space for busy flags (perhaps except
statically allocated thread0). Upper bits are not very interesting
for assert, and in most practical situations recorded value should
allow to manually identify the owner with certainity.
Assert that unbusy is performed by the owner, except few places where
unbusy is done in io completion handler. For this case, add
_unchecked variants of asserts and unbusy primitives.
Reviewed by: markj (previous version)
Tested by: pho
Sponsored by: The FreeBSD Foundation
Differential revision: https://reviews.freebsd.org/D22298
Modified:
head/sys/kern/kern_sendfile.c
head/sys/vm/vm_page.c
head/sys/vm/vm_page.h
Modified: head/sys/kern/kern_sendfile.c
==============================================================================
--- head/sys/kern/kern_sendfile.c Sun Nov 24 19:06:38 2019 (r355061)
+++ head/sys/kern/kern_sendfile.c Sun Nov 24 19:12:23 2019 (r355062)
@@ -261,7 +261,7 @@ sendfile_iodone(void *arg, vm_page_t *pg, int count, i
for (int i = 0; i < count; i++)
if (pg[i] != bogus_page)
- vm_page_xunbusy(pg[i]);
+ vm_page_xunbusy_unchecked(pg[i]);
if (error)
sfio->error = error;
Modified: head/sys/vm/vm_page.c
==============================================================================
--- head/sys/vm/vm_page.c Sun Nov 24 19:06:38 2019 (r355061)
+++ head/sys/vm/vm_page.c Sun Nov 24 19:12:23 2019 (r355062)
@@ -437,7 +437,7 @@ vm_page_init_marker(vm_page_t marker, int queue, uint1
bzero(marker, sizeof(*marker));
marker->flags = PG_MARKER;
marker->aflags = aflags;
- marker->busy_lock = VPB_SINGLE_EXCLUSIVER;
+ marker->busy_lock = VPB_CURTHREAD_EXCLUSIVE;
marker->queue = queue;
}
@@ -939,18 +939,19 @@ vm_page_busy_downgrade(vm_page_t m)
int
vm_page_busy_tryupgrade(vm_page_t m)
{
- u_int x;
+ u_int ce, x;
vm_page_assert_sbusied(m);
x = m->busy_lock;
+ ce = VPB_CURTHREAD_EXCLUSIVE;
for (;;) {
if (VPB_SHARERS(x) > 1)
return (0);
KASSERT((x & ~VPB_BIT_WAITERS) == VPB_SHARERS_WORD(1),
("vm_page_busy_tryupgrade: invalid lock state"));
if (!atomic_fcmpset_acq_int(&m->busy_lock, &x,
- VPB_SINGLE_EXCLUSIVER | (x & VPB_BIT_WAITERS)))
+ ce | (x & VPB_BIT_WAITERS)))
continue;
return (1);
}
@@ -1108,7 +1109,7 @@ vm_page_tryxbusy(vm_page_t m)
vm_object_t obj;
if (atomic_cmpset_acq_int(&(m)->busy_lock, VPB_UNBUSIED,
- VPB_SINGLE_EXCLUSIVER) == 0)
+ VPB_CURTHREAD_EXCLUSIVE) == 0)
return (0);
obj = m->object;
@@ -1119,6 +1120,14 @@ vm_page_tryxbusy(vm_page_t m)
return (1);
}
+static void
+vm_page_xunbusy_hard_tail(vm_page_t m)
+{
+ atomic_store_rel_int(&m->busy_lock, VPB_UNBUSIED);
+ /* Wake the waiter. */
+ wakeup(m);
+}
+
/*
* vm_page_xunbusy_hard:
*
@@ -1127,14 +1136,15 @@ vm_page_tryxbusy(vm_page_t m)
void
vm_page_xunbusy_hard(vm_page_t m)
{
-
vm_page_assert_xbusied(m);
+ vm_page_xunbusy_hard_tail(m);
+}
- /*
- * Wake the waiter.
- */
- atomic_store_rel_int(&m->busy_lock, VPB_UNBUSIED);
- wakeup(m);
+void
+vm_page_xunbusy_hard_unchecked(vm_page_t m)
+{
+ vm_page_assert_xbusied_unchecked(m);
+ vm_page_xunbusy_hard_tail(m);
}
/*
@@ -1228,7 +1238,7 @@ vm_page_initfake(vm_page_t m, vm_paddr_t paddr, vm_mem
m->flags = PG_FICTITIOUS;
/* Fictitious pages don't use "order" or "pool". */
m->oflags = VPO_UNMANAGED;
- m->busy_lock = VPB_SINGLE_EXCLUSIVER;
+ m->busy_lock = VPB_CURTHREAD_EXCLUSIVE;
/* Fictitious pages are unevictable. */
m->ref_count = 1;
pmap_page_init(m);
@@ -1318,7 +1328,7 @@ vm_page_readahead_finish(vm_page_t m)
else
vm_page_deactivate(m);
vm_page_unlock(m);
- vm_page_xunbusy(m);
+ vm_page_xunbusy_unchecked(m);
}
/*
@@ -1967,7 +1977,7 @@ found:
VPO_UNMANAGED : 0;
m->busy_lock = VPB_UNBUSIED;
if ((req & (VM_ALLOC_NOBUSY | VM_ALLOC_NOOBJ | VM_ALLOC_SBUSY)) == 0)
- m->busy_lock = VPB_SINGLE_EXCLUSIVER;
+ m->busy_lock = VPB_CURTHREAD_EXCLUSIVE;
if ((req & VM_ALLOC_SBUSY) != 0)
m->busy_lock = VPB_SHARERS_WORD(1);
if (req & VM_ALLOC_WIRED) {
@@ -2161,7 +2171,7 @@ found:
VPO_UNMANAGED : 0;
busy_lock = VPB_UNBUSIED;
if ((req & (VM_ALLOC_NOBUSY | VM_ALLOC_NOOBJ | VM_ALLOC_SBUSY)) == 0)
- busy_lock = VPB_SINGLE_EXCLUSIVER;
+ busy_lock = VPB_CURTHREAD_EXCLUSIVE;
if ((req & VM_ALLOC_SBUSY) != 0)
busy_lock = VPB_SHARERS_WORD(1);
if ((req & VM_ALLOC_WIRED) != 0)
Modified: head/sys/vm/vm_page.h
==============================================================================
--- head/sys/vm/vm_page.h Sun Nov 24 19:06:38 2019 (r355061)
+++ head/sys/vm/vm_page.h Sun Nov 24 19:12:23 2019 (r355062)
@@ -307,7 +307,13 @@ struct vm_page {
#define VPB_SHARERS_WORD(x) ((x) << VPB_SHARERS_SHIFT | VPB_BIT_SHARED)
#define VPB_ONE_SHARER (1 << VPB_SHARERS_SHIFT)
-#define VPB_SINGLE_EXCLUSIVER VPB_BIT_EXCLUSIVE
+#define VPB_SINGLE_EXCLUSIVE VPB_BIT_EXCLUSIVE
+#ifdef INVARIANTS
+#define VPB_CURTHREAD_EXCLUSIVE \
+ (VPB_BIT_EXCLUSIVE | ((u_int)(uintptr_t)curthread & ~VPB_BIT_FLAGMASK))
+#else
+#define VPB_CURTHREAD_EXCLUSIVE VPB_SINGLE_EXCLUSIVE
+#endif
#define VPB_UNBUSIED VPB_SHARERS_WORD(0)
@@ -649,6 +655,7 @@ void vm_page_updatefake(vm_page_t m, vm_paddr_t paddr,
void vm_page_wire(vm_page_t);
bool vm_page_wire_mapped(vm_page_t m);
void vm_page_xunbusy_hard(vm_page_t m);
+void vm_page_xunbusy_hard_unchecked(vm_page_t m);
void vm_page_set_validclean (vm_page_t, int, int);
void vm_page_clear_dirty(vm_page_t, int, int);
void vm_page_set_invalid(vm_page_t, int, int);
@@ -684,10 +691,19 @@ void vm_page_lock_assert_KBI(vm_page_t m, int a, const
("vm_page_assert_unbusied: page %p busy @ %s:%d", \
(m), __FILE__, __LINE__))
-#define vm_page_assert_xbusied(m) \
+#define vm_page_assert_xbusied_unchecked(m) do { \
KASSERT(vm_page_xbusied(m), \
("vm_page_assert_xbusied: page %p not exclusive busy @ %s:%d", \
- (m), __FILE__, __LINE__))
+ (m), __FILE__, __LINE__)); \
+} while (0)
+#define vm_page_assert_xbusied(m) do { \
+ vm_page_assert_xbusied_unchecked(m); \
+ KASSERT((m->busy_lock & ~VPB_BIT_WAITERS) == \
+ VPB_CURTHREAD_EXCLUSIVE, \
+ ("vm_page_assert_xbusied: page %p busy_lock %#x not owned" \
+ " by me @ %s:%d", \
+ (m), (m)->busy_lock, __FILE__, __LINE__)); \
+} while (0)
#define vm_page_busied(m) \
((m)->busy_lock != VPB_UNBUSIED)
@@ -699,7 +715,7 @@ void vm_page_lock_assert_KBI(vm_page_t m, int a, const
} while (0)
#define vm_page_xbusied(m) \
- (((m)->busy_lock & VPB_SINGLE_EXCLUSIVER) != 0)
+ (((m)->busy_lock & VPB_SINGLE_EXCLUSIVE) != 0)
#define vm_page_xbusy(m) do { \
if (!vm_page_tryxbusy(m)) \
@@ -710,8 +726,13 @@ void vm_page_lock_assert_KBI(vm_page_t m, int a, const
/* Note: page m's lock must not be owned by the caller. */
#define vm_page_xunbusy(m) do { \
if (!atomic_cmpset_rel_int(&(m)->busy_lock, \
- VPB_SINGLE_EXCLUSIVER, VPB_UNBUSIED)) \
+ VPB_CURTHREAD_EXCLUSIVE, VPB_UNBUSIED)) \
vm_page_xunbusy_hard(m); \
+} while (0)
+#define vm_page_xunbusy_unchecked(m) do { \
+ if (!atomic_cmpset_rel_int(&(m)->busy_lock, \
+ VPB_CURTHREAD_EXCLUSIVE, VPB_UNBUSIED)) \
+ vm_page_xunbusy_hard_unchecked(m); \
} while (0)
#ifdef INVARIANTS
More information about the svn-src-head
mailing list