git: 795fc5ba6f62 - main - vm_object: Fix synchronization between reinit and global list walks

From: Mark Johnston <markj_at_FreeBSD.org>
Date: Sun, 22 Nov 2026 17:41:28 UTC
The branch main has been updated by markj:

URL: https://cgit.FreeBSD.org/src/commit/?id=795fc5ba6f622c88e19ce311799d558ddda11869

commit 795fc5ba6f622c88e19ce311799d558ddda11869
Author:     Mark Johnston <markj@FreeBSD.org>
AuthorDate: 2026-11-22 15:31:43 +0000
Commit:     Mark Johnston <markj@FreeBSD.org>
CommitDate: 2026-11-22 17:41:07 +0000

    vm_object: Fix synchronization between reinit and global list walks
    
    Checking for OBJT_DEAD in vm_object_list_handler() is racy since pagers
    may set the object type to OBJT_DEAD after having dropped and reacquired
    the object lock.  dev_pager_dealloc() does this, for instance.  Check
    for the OBJ_DEAD flag as well, as that is set earlier.  Fix one rare
    case in the vnode pager where we freed an object without setting
    OBJ_DEAD, and add an assertion to the zone destructor.
    
    Try to minimize accesses of the object after dropping the object lock in
    vm_object_list_handler().
    Add a comment to warn others to think carefully before adding more
    cases.
    
    Fix shm_get_path() to reject dead objects.  Fix tmpfs to finish
    initialization of the object under its lock; we cannot claim that the
    object is thread-private since it lives on a global list.
    
    Try to close races with _vm_object_allocate().  Update object->flags,
    which clears OBJ_DEAD, only after performing other initialization.  This
    is more general than the pre-existing barriers.
    
    Clear the union fields in _vm_object_allocate().  Otherwise they can be
    left set to non-zero values, and some code is not careful to reset them.
    For instance, cdev_pager_allocate() inserts a new object into
    dev_pager_object_list without resetting its un_pager.devp.handle field.
    This also lets us check for partially constructed objects in
    cdev_pager_get_path().
    
    Fix some latent bugs after the vm_object flags field was widened from
    u_short to u_int: various functions kept the old type and would silently
    truncate.  Only the OBJ_SYSVSHM and OBJ_POSIXSHM flags would get
    truncated as a result.
    
    Reported by:    Jubal Skaggs <jubal@vcontext.net>
    Reviewed by:    alc, kib
    MFC after:      2 weeks
    Sponsored by:   The FreeBSD Foundation
    Differential Revision:  https://reviews.freebsd.org/D61754
---
 sys/fs/tmpfs/tmpfs_subr.c | 20 ++++++++++++------
 sys/kern/uipc_shm.c       |  2 +-
 sys/vm/device_pager.c     | 15 ++++++++++++--
 sys/vm/swap_pager.c       | 22 +++++++-------------
 sys/vm/vm_object.c        | 52 +++++++++++++++++++++++++++--------------------
 sys/vm/vm_object.h        |  4 ++--
 sys/vm/vnode_pager.c      |  1 +
 7 files changed, 68 insertions(+), 48 deletions(-)

diff --git a/sys/fs/tmpfs/tmpfs_subr.c b/sys/fs/tmpfs/tmpfs_subr.c
index b29b555dedc2..089a97814970 100644
--- a/sys/fs/tmpfs/tmpfs_subr.c
+++ b/sys/fs/tmpfs/tmpfs_subr.c
@@ -658,17 +658,25 @@ tmpfs_alloc_node(struct mount *mp, struct tmpfs_mount *tmp, __enum_uint8(vtype)
 		atomic_thread_fence_rel();
 		break;
 
-	case VREG:
-		nnode->tn_reg.tn_aobj =
-		    vm_pager_allocate(tmpfs_pager_type, NULL, 0,
+	case VREG: {
+		vm_object_t object;
+
+		object = vm_pager_allocate(tmpfs_pager_type, NULL, 0,
 		    VM_PROT_DEFAULT, 0,
 		    NULL /* XXXKIB - tmpfs needs swap reservation */);
-		nnode->tn_reg.tn_aobj->un_pager.swp.swp_priv = nnode;
-		vm_object_set_flag(nnode->tn_reg.tn_aobj, OBJ_TMPFS);
+		/*
+		 * Walkers of vm_object_list may use swp_priv once OBJ_TMPFS is
+		 * set, so set both with the object locked.
+		 */
+		VM_OBJECT_WLOCK(object);
+		object->un_pager.swp.swp_priv = nnode;
+		vm_object_set_flag(object, OBJ_TMPFS);
+		VM_OBJECT_WUNLOCK(object);
+		nnode->tn_reg.tn_aobj = object;
 		nnode->tn_reg.tn_tmp = tmp;
 		nnode->tn_reg.tn_pages = 0;
 		break;
-
+	}
 	default:
 		panic("tmpfs_alloc_node: type %p %d", nnode,
 		    (int)nnode->tn_type);
diff --git a/sys/kern/uipc_shm.c b/sys/kern/uipc_shm.c
index eefe6a63b53c..c71f216fbe08 100644
--- a/sys/kern/uipc_shm.c
+++ b/sys/kern/uipc_shm.c
@@ -2261,7 +2261,7 @@ shm_get_path(struct vm_object *obj, char *path, size_t sz)
 	shmfd = NULL;
 	sx_slock(&shm_dict_lock);
 	VM_OBJECT_RLOCK(obj);
-	if ((obj->flags & OBJ_POSIXSHM) == 0) {
+	if ((obj->flags & (OBJ_DEAD | OBJ_POSIXSHM)) != OBJ_POSIXSHM) {
 		error = EINVAL;
 	} else {
 		if (obj->type == shmfd_pager_type)
diff --git a/sys/vm/device_pager.c b/sys/vm/device_pager.c
index 9277d6255418..edce61e73aab 100644
--- a/sys/vm/device_pager.c
+++ b/sys/vm/device_pager.c
@@ -233,9 +233,17 @@ again:
 				mtx_lock(&dev_pager_mtx);
 			} else {
 				mtx_lock(&dev_pager_mtx);
-				object->flags |= OBJ_COLORED;
+				VM_OBJECT_WLOCK(object);
+				vm_object_set_flag(object, OBJ_COLORED);
 				object->pg_color = color;
+				/*
+				 * Setting devp.handle marks the object as
+				 * constructed. The below unlock also ensures
+				 * that the earlier store to devp.ops is
+				 * visible.
+				 */
 				object->un_pager.devp.handle = handle;
+				VM_OBJECT_WUNLOCK(object);
 				wakeup(&object->un_pager.devp.handle);
 			}
 		}
@@ -268,7 +276,10 @@ dev_pager_alloc(void *handle, vm_ooffset_t size, vm_prot_t prot,
 void
 cdev_pager_get_path(vm_object_t object, char *path, size_t sz)
 {
-	if (object->un_pager.devp.ops->cdev_pg_path != NULL)
+	VM_OBJECT_ASSERT_LOCKED(object);
+
+	if (object->un_pager.devp.handle != NULL &&
+	    object->un_pager.devp.ops->cdev_pg_path != NULL)
 		object->un_pager.devp.ops->cdev_pg_path(
 		    object->un_pager.devp.handle, path, sz);
 }
diff --git a/sys/vm/swap_pager.c b/sys/vm/swap_pager.c
index 3bd0a3545ee6..ff29d93c552a 100644
--- a/sys/vm/swap_pager.c
+++ b/sys/vm/swap_pager.c
@@ -2051,7 +2051,8 @@ full_rescan:
 	bp = uma_zalloc(swrbuf_zone, M_WAITOK);
 	mtx_lock(&vm_object_list_mtx);
 	TAILQ_FOREACH(object, &vm_object_list, object_list) {
-		if ((object->flags & OBJ_SWAP) == 0)
+		if ((atomic_load_int(&object->flags) & (OBJ_DEAD | OBJ_SWAP)) !=
+		    OBJ_SWAP)
 			continue;
 		mtx_unlock(&vm_object_list_mtx);
 		/* Depends on type-stability. */
@@ -2059,22 +2060,13 @@ full_rescan:
 
 		/*
 		 * Dead objects are eventually terminated on their own.
+		 *
+		 * Synchronize with _vm_object_allocate().
 		 */
-		if ((object->flags & OBJ_DEAD) != 0)
-			goto next_obj;
-
-		/*
-		 * Sync with fences placed after pctrie
-		 * initialization.  We must not access pctrie below
-		 * unless we checked that our object is swap and not
-		 * dead.
-		 */
-		atomic_thread_fence_acq();
-		if ((object->flags & OBJ_SWAP) == 0)
-			goto next_obj;
+		if ((atomic_load_acq_int(&object->flags) &
+		    (OBJ_DEAD | OBJ_SWAP)) == OBJ_SWAP)
+			swap_pager_swapoff_object(sp, object, &bp);
 
-		swap_pager_swapoff_object(sp, object, &bp);
-next_obj:
 		VM_OBJECT_WUNLOCK(object);
 		mtx_lock(&vm_object_list_mtx);
 	}
diff --git a/sys/vm/vm_object.c b/sys/vm/vm_object.c
index 1e6e25d26a54..b2ac9c37f9aa 100644
--- a/sys/vm/vm_object.c
+++ b/sys/vm/vm_object.c
@@ -176,6 +176,8 @@ vm_object_zdtor(void *mem, int size, void *arg)
 	vm_object_t object;
 
 	object = (vm_object_t)mem;
+	KASSERT((object->flags & OBJ_DEAD) != 0,
+	    ("object %p not dead yet", object));
 	KASSERT(object->ref_count == 0,
 	    ("object %p ref_count = %d", object, object->ref_count));
 	KASSERT(vm_radix_is_empty(&object->rtree),
@@ -227,24 +229,14 @@ vm_object_zinit(void *mem, int size, int flags)
 }
 
 static void
-_vm_object_allocate(objtype_t type, vm_pindex_t size, u_short flags,
+_vm_object_allocate(objtype_t type, vm_pindex_t size, u_int flags,
     vm_object_t object, void *handle)
 {
 	LIST_INIT(&object->shadow_head);
 
-	object->type = type;
-	object->flags = flags;
-	if ((flags & OBJ_SWAP) != 0) {
+	memset(&object->un_pager, 0, sizeof(object->un_pager));
+	if ((flags & OBJ_SWAP) != 0)
 		pctrie_init(&object->un_pager.swp.swp_blks);
-		object->un_pager.swp.writemappings = 0;
-	}
-
-	/*
-	 * Ensure that swap_pager_swapoff() iteration over object_list
-	 * sees up to date type and pctrie head if it observed
-	 * non-dead object.
-	 */
-	atomic_thread_fence_rel();
 
 	object->pg_color = 0;
 	object->size = size;
@@ -261,6 +253,14 @@ _vm_object_allocate(objtype_t type, vm_pindex_t size, u_short flags,
 	LIST_INIT(&object->rvq);
 #endif
 	umtx_shm_object_init(object);
+
+	/*
+	 * Clear OBJ_DEAD only after other initialization is done,
+	 * to synchronize with global object list traversal, e.g., in
+	 * swap_pager_swapoff() or vm_object_list_handler().
+	 */
+	object->type = type;
+	atomic_store_rel_int(&object->flags, flags);
 }
 
 /*
@@ -304,7 +304,7 @@ vm_object_init(void)
 }
 
 void
-vm_object_clear_flag(vm_object_t object, u_short bits)
+vm_object_clear_flag(vm_object_t object, u_int bits)
 {
 
 	VM_OBJECT_ASSERT_WLOCKED(object);
@@ -399,7 +399,7 @@ vm_object_t
 vm_object_allocate(objtype_t type, vm_pindex_t size)
 {
 	vm_object_t object;
-	u_short flags;
+	u_int flags;
 
 	switch (type) {
 	case OBJT_DEAD:
@@ -431,7 +431,7 @@ vm_object_allocate(objtype_t type, vm_pindex_t size)
 }
 
 vm_object_t
-vm_object_allocate_dyn(objtype_t dyntype, vm_pindex_t size, u_short flags)
+vm_object_allocate_dyn(objtype_t dyntype, vm_pindex_t size, u_int flags)
 {
 	vm_object_t object;
 
@@ -2523,6 +2523,7 @@ vm_object_list_handler(struct sysctl_req *req, bool swap_only)
 	vm_object_t obj;
 	vm_page_t m;
 	u_long sp;
+	u_int flags;
 	int count, error;
 	key_t key;
 	unsigned short seq;
@@ -2556,12 +2557,14 @@ vm_object_list_handler(struct sysctl_req *req, bool swap_only)
 	 */
 	mtx_lock(&vm_object_list_mtx);
 	TAILQ_FOREACH(obj, &vm_object_list, object_list) {
-		if (obj->type == OBJT_DEAD ||
+		if (obj->type == OBJT_DEAD || (obj->flags & OBJ_DEAD) != 0 ||
 		    (swap_only && (obj->flags & (OBJ_ANON | OBJ_SWAP)) == 0))
 			continue;
 		VM_OBJECT_RLOCK(obj);
-		if (obj->type == OBJT_DEAD ||
-		    (swap_only && (obj->flags & (OBJ_ANON | OBJ_SWAP)) == 0)) {
+		/* Synchronize with _vm_object_allocate(). */
+		flags = atomic_load_acq_int(&obj->flags);
+		if (obj->type == OBJT_DEAD || (flags & OBJ_DEAD) != 0 ||
+		    (swap_only && (flags & (OBJ_ANON | OBJ_SWAP)) == 0)) {
 			VM_OBJECT_RUNLOCK(obj);
 			continue;
 		}
@@ -2604,7 +2607,7 @@ vm_object_list_handler(struct sysctl_req *req, bool swap_only)
 		    NULL);
 		if (vp != NULL) {
 			vref(vp);
-		} else if ((obj->flags & OBJ_ANON) != 0) {
+		} else if ((flags & OBJ_ANON) != 0) {
 			MPASS(kvo->kvo_type == KVME_TYPE_SWAP);
 			kvo->kvo_me = (uintptr_t)obj;
 			/* tmpfs objs are reported as vnodes */
@@ -2617,13 +2620,18 @@ vm_object_list_handler(struct sysctl_req *req, bool swap_only)
 			    sizeof(kvo->kvo_path));
 		}
 		VM_OBJECT_RUNLOCK(obj);
-		if ((obj->flags & OBJ_SYSVSHM) != 0) {
+
+		/*
+		 * Note, it is safe to invoke shmobjinfo() and shm_get_path()
+		 * concurrent with teardown of the object.
+		 */
+		if ((flags & OBJ_SYSVSHM) != 0) {
 			kvo->kvo_flags |= KVMO_FLAG_SYSVSHM;
 			shmobjinfo(obj, &key, &seq);
 			kvo->kvo_vn_fileid = key;
 			kvo->kvo_vn_fsid_freebsd11 = seq;
 		}
-		if ((obj->flags & OBJ_POSIXSHM) != 0) {
+		if ((flags & OBJ_POSIXSHM) != 0) {
 			kvo->kvo_flags |= KVMO_FLAG_POSIXSHM;
 			shm_get_path(obj, kvo->kvo_path,
 			    sizeof(kvo->kvo_path));
diff --git a/sys/vm/vm_object.h b/sys/vm/vm_object.h
index 5a3a78b5b5a1..8beab3aa2236 100644
--- a/sys/vm/vm_object.h
+++ b/sys/vm/vm_object.h
@@ -330,7 +330,7 @@ vm_object_reserv(vm_object_t object)
 	return (false);
 }
 
-void vm_object_clear_flag(vm_object_t object, u_short bits);
+void vm_object_clear_flag(vm_object_t object, u_int bits);
 void vm_object_pip_add(vm_object_t object, short i);
 void vm_object_pip_wakeup(vm_object_t object);
 void vm_object_pip_wakeupn(vm_object_t object, short i);
@@ -355,7 +355,7 @@ extern int umtx_shm_vnobj_persistent;
 
 vm_object_t vm_object_allocate (objtype_t, vm_pindex_t);
 vm_object_t vm_object_allocate_anon(vm_pindex_t, vm_object_t, struct ucred *);
-vm_object_t vm_object_allocate_dyn(objtype_t, vm_pindex_t, u_short);
+vm_object_t vm_object_allocate_dyn(objtype_t, vm_pindex_t, u_int);
 boolean_t vm_object_coalesce(vm_object_t, vm_ooffset_t, vm_size_t, vm_size_t,
    int);
 void vm_object_collapse (vm_object_t);
diff --git a/sys/vm/vnode_pager.c b/sys/vm/vnode_pager.c
index 7dba3c8ab21c..625a3f6a3a5c 100644
--- a/sys/vm/vnode_pager.c
+++ b/sys/vm/vnode_pager.c
@@ -293,6 +293,7 @@ retry:
 			VM_OBJECT_WLOCK(object);
 			KASSERT(object->ref_count == 1,
 			    ("leaked ref %p %d", object, object->ref_count));
+			vm_object_set_flag(object, OBJ_DEAD);
 			object->type = OBJT_DEAD;
 			refcount_init(&object->ref_count, 0);
 			VM_OBJECT_WUNLOCK(object);