git: 795fc5ba6f62 - main - vm_object: Fix synchronization between reinit and global list walks
- Go to: [ bottom of page ] [ top of archives ] [ this month ]
Date: Sun, 22 Nov 2026 17:41:28 UTC
The branch main has been updated by markj:
URL: https://cgit.FreeBSD.org/src/commit/?id=795fc5ba6f622c88e19ce311799d558ddda11869
commit 795fc5ba6f622c88e19ce311799d558ddda11869
Author: Mark Johnston <markj@FreeBSD.org>
AuthorDate: 2026-11-22 15:31:43 +0000
Commit: Mark Johnston <markj@FreeBSD.org>
CommitDate: 2026-11-22 17:41:07 +0000
vm_object: Fix synchronization between reinit and global list walks
Checking for OBJT_DEAD in vm_object_list_handler() is racy since pagers
may set the object type to OBJT_DEAD after having dropped and reacquired
the object lock. dev_pager_dealloc() does this, for instance. Check
for the OBJ_DEAD flag as well, as that is set earlier. Fix one rare
case in the vnode pager where we freed an object without setting
OBJ_DEAD, and add an assertion to the zone destructor.
Try to minimize accesses of the object after dropping the object lock in
vm_object_list_handler().
Add a comment to warn others to think carefully before adding more
cases.
Fix shm_get_path() to reject dead objects. Fix tmpfs to finish
initialization of the object under its lock; we cannot claim that the
object is thread-private since it lives on a global list.
Try to close races with _vm_object_allocate(). Update object->flags,
which clears OBJ_DEAD, only after performing other initialization. This
is more general than the pre-existing barriers.
Clear the union fields in _vm_object_allocate(). Otherwise they can be
left set to non-zero values, and some code is not careful to reset them.
For instance, cdev_pager_allocate() inserts a new object into
dev_pager_object_list without resetting its un_pager.devp.handle field.
This also lets us check for partially constructed objects in
cdev_pager_get_path().
Fix some latent bugs after the vm_object flags field was widened from
u_short to u_int: various functions kept the old type and would silently
truncate. Only the OBJ_SYSVSHM and OBJ_POSIXSHM flags would get
truncated as a result.
Reported by: Jubal Skaggs <jubal@vcontext.net>
Reviewed by: alc, kib
MFC after: 2 weeks
Sponsored by: The FreeBSD Foundation
Differential Revision: https://reviews.freebsd.org/D61754
---
sys/fs/tmpfs/tmpfs_subr.c | 20 ++++++++++++------
sys/kern/uipc_shm.c | 2 +-
sys/vm/device_pager.c | 15 ++++++++++++--
sys/vm/swap_pager.c | 22 +++++++-------------
sys/vm/vm_object.c | 52 +++++++++++++++++++++++++++--------------------
sys/vm/vm_object.h | 4 ++--
sys/vm/vnode_pager.c | 1 +
7 files changed, 68 insertions(+), 48 deletions(-)
diff --git a/sys/fs/tmpfs/tmpfs_subr.c b/sys/fs/tmpfs/tmpfs_subr.c
index b29b555dedc2..089a97814970 100644
--- a/sys/fs/tmpfs/tmpfs_subr.c
+++ b/sys/fs/tmpfs/tmpfs_subr.c
@@ -658,17 +658,25 @@ tmpfs_alloc_node(struct mount *mp, struct tmpfs_mount *tmp, __enum_uint8(vtype)
atomic_thread_fence_rel();
break;
- case VREG:
- nnode->tn_reg.tn_aobj =
- vm_pager_allocate(tmpfs_pager_type, NULL, 0,
+ case VREG: {
+ vm_object_t object;
+
+ object = vm_pager_allocate(tmpfs_pager_type, NULL, 0,
VM_PROT_DEFAULT, 0,
NULL /* XXXKIB - tmpfs needs swap reservation */);
- nnode->tn_reg.tn_aobj->un_pager.swp.swp_priv = nnode;
- vm_object_set_flag(nnode->tn_reg.tn_aobj, OBJ_TMPFS);
+ /*
+ * Walkers of vm_object_list may use swp_priv once OBJ_TMPFS is
+ * set, so set both with the object locked.
+ */
+ VM_OBJECT_WLOCK(object);
+ object->un_pager.swp.swp_priv = nnode;
+ vm_object_set_flag(object, OBJ_TMPFS);
+ VM_OBJECT_WUNLOCK(object);
+ nnode->tn_reg.tn_aobj = object;
nnode->tn_reg.tn_tmp = tmp;
nnode->tn_reg.tn_pages = 0;
break;
-
+ }
default:
panic("tmpfs_alloc_node: type %p %d", nnode,
(int)nnode->tn_type);
diff --git a/sys/kern/uipc_shm.c b/sys/kern/uipc_shm.c
index eefe6a63b53c..c71f216fbe08 100644
--- a/sys/kern/uipc_shm.c
+++ b/sys/kern/uipc_shm.c
@@ -2261,7 +2261,7 @@ shm_get_path(struct vm_object *obj, char *path, size_t sz)
shmfd = NULL;
sx_slock(&shm_dict_lock);
VM_OBJECT_RLOCK(obj);
- if ((obj->flags & OBJ_POSIXSHM) == 0) {
+ if ((obj->flags & (OBJ_DEAD | OBJ_POSIXSHM)) != OBJ_POSIXSHM) {
error = EINVAL;
} else {
if (obj->type == shmfd_pager_type)
diff --git a/sys/vm/device_pager.c b/sys/vm/device_pager.c
index 9277d6255418..edce61e73aab 100644
--- a/sys/vm/device_pager.c
+++ b/sys/vm/device_pager.c
@@ -233,9 +233,17 @@ again:
mtx_lock(&dev_pager_mtx);
} else {
mtx_lock(&dev_pager_mtx);
- object->flags |= OBJ_COLORED;
+ VM_OBJECT_WLOCK(object);
+ vm_object_set_flag(object, OBJ_COLORED);
object->pg_color = color;
+ /*
+ * Setting devp.handle marks the object as
+ * constructed. The below unlock also ensures
+ * that the earlier store to devp.ops is
+ * visible.
+ */
object->un_pager.devp.handle = handle;
+ VM_OBJECT_WUNLOCK(object);
wakeup(&object->un_pager.devp.handle);
}
}
@@ -268,7 +276,10 @@ dev_pager_alloc(void *handle, vm_ooffset_t size, vm_prot_t prot,
void
cdev_pager_get_path(vm_object_t object, char *path, size_t sz)
{
- if (object->un_pager.devp.ops->cdev_pg_path != NULL)
+ VM_OBJECT_ASSERT_LOCKED(object);
+
+ if (object->un_pager.devp.handle != NULL &&
+ object->un_pager.devp.ops->cdev_pg_path != NULL)
object->un_pager.devp.ops->cdev_pg_path(
object->un_pager.devp.handle, path, sz);
}
diff --git a/sys/vm/swap_pager.c b/sys/vm/swap_pager.c
index 3bd0a3545ee6..ff29d93c552a 100644
--- a/sys/vm/swap_pager.c
+++ b/sys/vm/swap_pager.c
@@ -2051,7 +2051,8 @@ full_rescan:
bp = uma_zalloc(swrbuf_zone, M_WAITOK);
mtx_lock(&vm_object_list_mtx);
TAILQ_FOREACH(object, &vm_object_list, object_list) {
- if ((object->flags & OBJ_SWAP) == 0)
+ if ((atomic_load_int(&object->flags) & (OBJ_DEAD | OBJ_SWAP)) !=
+ OBJ_SWAP)
continue;
mtx_unlock(&vm_object_list_mtx);
/* Depends on type-stability. */
@@ -2059,22 +2060,13 @@ full_rescan:
/*
* Dead objects are eventually terminated on their own.
+ *
+ * Synchronize with _vm_object_allocate().
*/
- if ((object->flags & OBJ_DEAD) != 0)
- goto next_obj;
-
- /*
- * Sync with fences placed after pctrie
- * initialization. We must not access pctrie below
- * unless we checked that our object is swap and not
- * dead.
- */
- atomic_thread_fence_acq();
- if ((object->flags & OBJ_SWAP) == 0)
- goto next_obj;
+ if ((atomic_load_acq_int(&object->flags) &
+ (OBJ_DEAD | OBJ_SWAP)) == OBJ_SWAP)
+ swap_pager_swapoff_object(sp, object, &bp);
- swap_pager_swapoff_object(sp, object, &bp);
-next_obj:
VM_OBJECT_WUNLOCK(object);
mtx_lock(&vm_object_list_mtx);
}
diff --git a/sys/vm/vm_object.c b/sys/vm/vm_object.c
index 1e6e25d26a54..b2ac9c37f9aa 100644
--- a/sys/vm/vm_object.c
+++ b/sys/vm/vm_object.c
@@ -176,6 +176,8 @@ vm_object_zdtor(void *mem, int size, void *arg)
vm_object_t object;
object = (vm_object_t)mem;
+ KASSERT((object->flags & OBJ_DEAD) != 0,
+ ("object %p not dead yet", object));
KASSERT(object->ref_count == 0,
("object %p ref_count = %d", object, object->ref_count));
KASSERT(vm_radix_is_empty(&object->rtree),
@@ -227,24 +229,14 @@ vm_object_zinit(void *mem, int size, int flags)
}
static void
-_vm_object_allocate(objtype_t type, vm_pindex_t size, u_short flags,
+_vm_object_allocate(objtype_t type, vm_pindex_t size, u_int flags,
vm_object_t object, void *handle)
{
LIST_INIT(&object->shadow_head);
- object->type = type;
- object->flags = flags;
- if ((flags & OBJ_SWAP) != 0) {
+ memset(&object->un_pager, 0, sizeof(object->un_pager));
+ if ((flags & OBJ_SWAP) != 0)
pctrie_init(&object->un_pager.swp.swp_blks);
- object->un_pager.swp.writemappings = 0;
- }
-
- /*
- * Ensure that swap_pager_swapoff() iteration over object_list
- * sees up to date type and pctrie head if it observed
- * non-dead object.
- */
- atomic_thread_fence_rel();
object->pg_color = 0;
object->size = size;
@@ -261,6 +253,14 @@ _vm_object_allocate(objtype_t type, vm_pindex_t size, u_short flags,
LIST_INIT(&object->rvq);
#endif
umtx_shm_object_init(object);
+
+ /*
+ * Clear OBJ_DEAD only after other initialization is done,
+ * to synchronize with global object list traversal, e.g., in
+ * swap_pager_swapoff() or vm_object_list_handler().
+ */
+ object->type = type;
+ atomic_store_rel_int(&object->flags, flags);
}
/*
@@ -304,7 +304,7 @@ vm_object_init(void)
}
void
-vm_object_clear_flag(vm_object_t object, u_short bits)
+vm_object_clear_flag(vm_object_t object, u_int bits)
{
VM_OBJECT_ASSERT_WLOCKED(object);
@@ -399,7 +399,7 @@ vm_object_t
vm_object_allocate(objtype_t type, vm_pindex_t size)
{
vm_object_t object;
- u_short flags;
+ u_int flags;
switch (type) {
case OBJT_DEAD:
@@ -431,7 +431,7 @@ vm_object_allocate(objtype_t type, vm_pindex_t size)
}
vm_object_t
-vm_object_allocate_dyn(objtype_t dyntype, vm_pindex_t size, u_short flags)
+vm_object_allocate_dyn(objtype_t dyntype, vm_pindex_t size, u_int flags)
{
vm_object_t object;
@@ -2523,6 +2523,7 @@ vm_object_list_handler(struct sysctl_req *req, bool swap_only)
vm_object_t obj;
vm_page_t m;
u_long sp;
+ u_int flags;
int count, error;
key_t key;
unsigned short seq;
@@ -2556,12 +2557,14 @@ vm_object_list_handler(struct sysctl_req *req, bool swap_only)
*/
mtx_lock(&vm_object_list_mtx);
TAILQ_FOREACH(obj, &vm_object_list, object_list) {
- if (obj->type == OBJT_DEAD ||
+ if (obj->type == OBJT_DEAD || (obj->flags & OBJ_DEAD) != 0 ||
(swap_only && (obj->flags & (OBJ_ANON | OBJ_SWAP)) == 0))
continue;
VM_OBJECT_RLOCK(obj);
- if (obj->type == OBJT_DEAD ||
- (swap_only && (obj->flags & (OBJ_ANON | OBJ_SWAP)) == 0)) {
+ /* Synchronize with _vm_object_allocate(). */
+ flags = atomic_load_acq_int(&obj->flags);
+ if (obj->type == OBJT_DEAD || (flags & OBJ_DEAD) != 0 ||
+ (swap_only && (flags & (OBJ_ANON | OBJ_SWAP)) == 0)) {
VM_OBJECT_RUNLOCK(obj);
continue;
}
@@ -2604,7 +2607,7 @@ vm_object_list_handler(struct sysctl_req *req, bool swap_only)
NULL);
if (vp != NULL) {
vref(vp);
- } else if ((obj->flags & OBJ_ANON) != 0) {
+ } else if ((flags & OBJ_ANON) != 0) {
MPASS(kvo->kvo_type == KVME_TYPE_SWAP);
kvo->kvo_me = (uintptr_t)obj;
/* tmpfs objs are reported as vnodes */
@@ -2617,13 +2620,18 @@ vm_object_list_handler(struct sysctl_req *req, bool swap_only)
sizeof(kvo->kvo_path));
}
VM_OBJECT_RUNLOCK(obj);
- if ((obj->flags & OBJ_SYSVSHM) != 0) {
+
+ /*
+ * Note, it is safe to invoke shmobjinfo() and shm_get_path()
+ * concurrent with teardown of the object.
+ */
+ if ((flags & OBJ_SYSVSHM) != 0) {
kvo->kvo_flags |= KVMO_FLAG_SYSVSHM;
shmobjinfo(obj, &key, &seq);
kvo->kvo_vn_fileid = key;
kvo->kvo_vn_fsid_freebsd11 = seq;
}
- if ((obj->flags & OBJ_POSIXSHM) != 0) {
+ if ((flags & OBJ_POSIXSHM) != 0) {
kvo->kvo_flags |= KVMO_FLAG_POSIXSHM;
shm_get_path(obj, kvo->kvo_path,
sizeof(kvo->kvo_path));
diff --git a/sys/vm/vm_object.h b/sys/vm/vm_object.h
index 5a3a78b5b5a1..8beab3aa2236 100644
--- a/sys/vm/vm_object.h
+++ b/sys/vm/vm_object.h
@@ -330,7 +330,7 @@ vm_object_reserv(vm_object_t object)
return (false);
}
-void vm_object_clear_flag(vm_object_t object, u_short bits);
+void vm_object_clear_flag(vm_object_t object, u_int bits);
void vm_object_pip_add(vm_object_t object, short i);
void vm_object_pip_wakeup(vm_object_t object);
void vm_object_pip_wakeupn(vm_object_t object, short i);
@@ -355,7 +355,7 @@ extern int umtx_shm_vnobj_persistent;
vm_object_t vm_object_allocate (objtype_t, vm_pindex_t);
vm_object_t vm_object_allocate_anon(vm_pindex_t, vm_object_t, struct ucred *);
-vm_object_t vm_object_allocate_dyn(objtype_t, vm_pindex_t, u_short);
+vm_object_t vm_object_allocate_dyn(objtype_t, vm_pindex_t, u_int);
boolean_t vm_object_coalesce(vm_object_t, vm_ooffset_t, vm_size_t, vm_size_t,
int);
void vm_object_collapse (vm_object_t);
diff --git a/sys/vm/vnode_pager.c b/sys/vm/vnode_pager.c
index 7dba3c8ab21c..625a3f6a3a5c 100644
--- a/sys/vm/vnode_pager.c
+++ b/sys/vm/vnode_pager.c
@@ -293,6 +293,7 @@ retry:
VM_OBJECT_WLOCK(object);
KASSERT(object->ref_count == 1,
("leaked ref %p %d", object, object->ref_count));
+ vm_object_set_flag(object, OBJ_DEAD);
object->type = OBJT_DEAD;
refcount_init(&object->ref_count, 0);
VM_OBJECT_WUNLOCK(object);