git: 3701716fdc45 - stable/15 - vm_page: Fix dequeue on arches with weak ordering

The branch stable/15 has been updated by markj:

URL: https://cgit.FreeBSD.org/src/commit/?id=3701716fdc45c203951f44d01710da310204b629

commit 3701716fdc45c203951f44d01710da310204b629
Author:     Mark Johnston <markj@FreeBSD.org>
AuthorDate: 2026-07-17 12:57:06 +0000
Commit:     Mark Johnston <markj@FreeBSD.org>
CommitDate: 2026-07-27 17:34:32 +0000

    vm_page: Fix dequeue on arches with weak ordering
    
    A vm_page's a.queue field records the page queue index for the page
    queue to which the page belongs.  The PGA_ENQUEUED flag indicates
    whether the page is actually enqueued in that queue's TAILQ.  When
    modifying the a.queue field, you need to hold the page queue lock for
    the queue corresponding to the old value, unless the old value is
    PQ_NONE.
    
    Suppose a managed page is freed.  vm_page_free_prep() calls
    vm_page_dequeue_deferred(), which checks whether the page belongs to a
    queue; if so it schedules an asynchronous dequeue operation so that page
    queue lock acquisitions can be batched if possible.
    
    The dequeue operation must be completed before the page's plinks.q
    fields are reused.  So, during page allocation, we call
    vm_page_dequeue() to finish the dequeue operation.  Similarly, since the
    buddy allocator uses the plinks.q fields for its own internal linkage,
    vm_freelist_add() calls vm_page_dequeue().
    
    _vm_page_pqstate_commit_dequeue() is the function which actually removes
    the page from its queue.  It sets a.queue = PG_NONE and removes the page
    from its queue.  However, the update to the page's atomic state is
    relaxed, so on systems with store reordering, it may race with a
    concurrent enqueue of the page into the buddy queues (probably more
    likely) or a page queue.
    
    Fix this: use a release store to update the page's queue state in
    _vm_page_pqstate_commit_dequeue(), and make sure that vm_page_dequeue()
    uses an acquire load when comparing m->a.queue == PQ_NONE.
    
    PR:             296767
    Reported and tested by: pkubaj
    Reviewed by:    alc, kib
    MFC after:      1 week
    Differential Revision:  https://reviews.freebsd.org/D58261
    
    (cherry picked from commit d809a10218884162ed47c658233746c53a98b1aa)
---
 sys/vm/vm_page.c | 26 ++++++++++++++++++++++++--
 sys/vm/vm_page.h | 32 ++++++++++++++++++++++++++++++++
 2 files changed, 56 insertions(+), 2 deletions(-)

diff --git a/sys/vm/vm_page.c b/sys/vm/vm_page.c
index cd95cc620ad4..ce5748bd5f33 100644
--- a/sys/vm/vm_page.c
+++ b/sys/vm/vm_page.c
@@ -3670,6 +3670,22 @@ vm_page_pqstate_fcmpset(vm_page_t m, vm_page_astate_t *old,
 	return (false);
 }
 
+static __always_inline bool
+vm_page_pqstate_fcmpset_rel(vm_page_t m, vm_page_astate_t *old,
+    vm_page_astate_t new)
+{
+	vm_page_astate_t tmp;
+
+	tmp = *old;
+	do {
+		if (__predict_true(vm_page_astate_fcmpset_rel(m, old, new)))
+			return (true);
+		counter_u64_add(pqstate_commit_retries, 1);
+	} while (old->_bits == tmp._bits);
+
+	return (false);
+}
+
 /*
  * Do the work of committing a queue state update that moves the page out of
  * its current queue.
@@ -3700,7 +3716,8 @@ _vm_page_pqstate_commit_dequeue(struct vm_pagequeue *pq, vm_page_t m,
 		next = TAILQ_NEXT(m, plinks.q);
 		TAILQ_REMOVE(&pq->pq_pl, m, plinks.q);
 		vm_pagequeue_cnt_dec(pq);
-		if (!vm_page_pqstate_fcmpset(m, old, new)) {
+		/* See vm_page_dequeue(). */
+		if (!vm_page_pqstate_fcmpset_rel(m, old, new)) {
 			if (next == NULL)
 				TAILQ_INSERT_TAIL(&pq->pq_pl, m, plinks.q);
 			else
@@ -4004,7 +4021,12 @@ vm_page_dequeue(vm_page_t m)
 {
 	vm_page_astate_t new, old;
 
-	old = vm_page_astate_load(m);
+	/*
+	 * Synchronize with _vm_page_pqstate_commit_dequeue(): make sure
+	 * that the page's queue linkage field updates are visible before
+	 * returning.
+	 */
+	old = vm_page_astate_load_acq(m);
 	do {
 		if (__predict_true(old.queue == PQ_NONE)) {
 			KASSERT((old.flags & PGA_QUEUE_STATE_MASK) == 0,
diff --git a/sys/vm/vm_page.h b/sys/vm/vm_page.h
index a091310ffd17..9913afa1992c 100644
--- a/sys/vm/vm_page.h
+++ b/sys/vm/vm_page.h
@@ -771,6 +771,18 @@ vm_page_astate_load(vm_page_t m)
 	return (a);
 }
 
+/*
+ *	Load a snapshot of a page's 32-bit atomic state, with acquire semantics.
+ */
+static inline vm_page_astate_t
+vm_page_astate_load_acq(vm_page_t m)
+{
+	vm_page_astate_t a;
+
+	a._bits = atomic_load_acq_32(&m->a._bits);
+	return (a);
+}
+
 /*
  *	Atomically compare and set a page's atomic state.
  */
@@ -788,6 +800,26 @@ vm_page_astate_fcmpset(vm_page_t m, vm_page_astate_t *old, vm_page_astate_t new)
 	return (atomic_fcmpset_32(&m->a._bits, &old->_bits, new._bits) != 0);
 }
 
+/*
+ *	Atomically compare and set a page's atomic state, with release
+ *	semantics.
+ */
+static inline bool
+vm_page_astate_fcmpset_rel(vm_page_t m, vm_page_astate_t *old,
+    vm_page_astate_t new)
+{
+
+	KASSERT(new.queue == PQ_INACTIVE || (new.flags & PGA_REQUEUE_HEAD) == 0,
+	    ("%s: invalid head requeue request for page %p", __func__, m));
+	KASSERT((new.flags & PGA_ENQUEUED) == 0 || new.queue != PQ_NONE,
+	    ("%s: setting PGA_ENQUEUED with PQ_NONE in page %p", __func__, m));
+	KASSERT(new._bits != old->_bits,
+	    ("%s: bits are unchanged", __func__));
+
+	return (atomic_fcmpset_rel_32(&m->a._bits, &old->_bits, new._bits) !=
+	    0);
+}
+
 /*
  *	Clear the given bits in the specified page.
  */