git: 7fa19ee28c90 - main - vm_page: reset iterator after domainset drops lock
- Go to: [ bottom of page ] [ top of archives ] [ this month ]
Date: Sun, 18 May 2025 18:15:01 UTC
The branch main has been updated by dougm:
URL: https://cgit.FreeBSD.org/src/commit/?id=7fa19ee28c90c159fdb98d2339455a5201dc5993
commit 7fa19ee28c90c159fdb98d2339455a5201dc5993
Author: Doug Moore <dougm@FreeBSD.org>
AuthorDate: 2025-05-18 18:09:00 +0000
Commit: Doug Moore <dougm@FreeBSD.org>
CommitDate: 2025-05-18 18:09:00 +0000
vm_page: reset iterator after domainset drops lock
Restore the WAITFAIL check on resetting the iterator in
vm_page_alloc_domain_iter. It was dropped in 5239a9fb8662 ("vm_page:
alloc_domain_iter") and blamed for a stress test failure. In fact,
the failure was elsewhere, and removing the WAITFAIL check just made
it less likely to be discovered, without fixing it.
Modify vm_domainset_iter_page_init and vm_domainset_iter_page so that
they can take an iterator argument and reset the iterator if
vm_domainset_iter_releases and reacquires the object lock. Failure to
reset the iterator in this case was the real problem.
Reviewed by: alc, markj, kib
Tested by: pho
Differential Revision: https://reviews.freebsd.org/D50392
---
sys/vm/vm_domainset.c | 16 ++++++++++------
sys/vm/vm_domainset.h | 7 +++++--
sys/vm/vm_glue.c | 4 ++--
sys/vm/vm_page.c | 24 +++++++++++++-----------
4 files changed, 30 insertions(+), 21 deletions(-)
diff --git a/sys/vm/vm_domainset.c b/sys/vm/vm_domainset.c
index f6ac3c984cbf..7b8bf4c77663 100644
--- a/sys/vm/vm_domainset.c
+++ b/sys/vm/vm_domainset.c
@@ -39,6 +39,7 @@
#include <sys/mutex.h>
#include <sys/malloc.h>
#include <sys/rwlock.h>
+#include <sys/pctrie.h>
#include <sys/vmmeter.h>
#include <vm/vm.h>
@@ -199,7 +200,7 @@ vm_domainset_iter_first(struct vm_domainset_iter *di, int *domain)
void
vm_domainset_iter_page_init(struct vm_domainset_iter *di, struct vm_object *obj,
- vm_pindex_t pindex, int *domain, int *req)
+ vm_pindex_t pindex, int *domain, int *req, struct pctrie_iter *pages)
{
struct domainset_ref *dr;
@@ -218,12 +219,12 @@ vm_domainset_iter_page_init(struct vm_domainset_iter *di, struct vm_object *obj,
VM_ALLOC_NOWAIT;
vm_domainset_iter_first(di, domain);
if (vm_page_count_min_domain(*domain))
- vm_domainset_iter_page(di, obj, domain);
+ vm_domainset_iter_page(di, obj, domain, pages);
}
int
vm_domainset_iter_page(struct vm_domainset_iter *di, struct vm_object *obj,
- int *domain)
+ int *domain, struct pctrie_iter *pages)
{
if (__predict_false(DOMAINSET_EMPTY(&di->di_valid_mask)))
return (ENOMEM);
@@ -248,8 +249,11 @@ vm_domainset_iter_page(struct vm_domainset_iter *di, struct vm_object *obj,
return (ENOMEM);
/* Wait for one of the domains to accumulate some free pages. */
- if (obj != NULL)
+ if (obj != NULL) {
VM_OBJECT_WUNLOCK(obj);
+ if (pages != NULL)
+ pctrie_iter_reset(pages);
+ }
vm_wait_doms(&di->di_valid_mask, 0);
if (obj != NULL)
VM_OBJECT_WLOCK(obj);
@@ -339,7 +343,7 @@ vm_domainset_iter_ignore(struct vm_domainset_iter *di, int domain)
int
vm_domainset_iter_page(struct vm_domainset_iter *di, struct vm_object *obj,
- int *domain)
+ int *domain, struct pctrie_iter *pages)
{
return (EJUSTRETURN);
@@ -347,7 +351,7 @@ vm_domainset_iter_page(struct vm_domainset_iter *di, struct vm_object *obj,
void
vm_domainset_iter_page_init(struct vm_domainset_iter *di, struct vm_object *obj,
- vm_pindex_t pindex, int *domain, int *flags)
+ vm_pindex_t pindex, int *domain, int *flags, struct pctrie_iter *pages)
{
*domain = 0;
diff --git a/sys/vm/vm_domainset.h b/sys/vm/vm_domainset.h
index d2cfe362ae78..0d325a642f40 100644
--- a/sys/vm/vm_domainset.h
+++ b/sys/vm/vm_domainset.h
@@ -28,6 +28,8 @@
#ifndef __VM_DOMAINSET_H__
#define __VM_DOMAINSET_H__
+struct pctrie_iter;
+
struct vm_domainset_iter {
struct domainset *di_domain;
unsigned int *di_iter;
@@ -40,9 +42,10 @@ struct vm_domainset_iter {
};
int vm_domainset_iter_page(struct vm_domainset_iter *, struct vm_object *,
- int *);
+ int *, struct pctrie_iter *);
void vm_domainset_iter_page_init(struct vm_domainset_iter *,
- struct vm_object *, vm_pindex_t, int *, int *);
+ struct vm_object *, vm_pindex_t, int *, int *,
+ struct pctrie_iter *);
int vm_domainset_iter_policy(struct vm_domainset_iter *, int *);
void vm_domainset_iter_policy_init(struct vm_domainset_iter *,
struct domainset *, int *, int *);
diff --git a/sys/vm/vm_glue.c b/sys/vm/vm_glue.c
index c2a032b24000..94df2c2f9a9e 100644
--- a/sys/vm/vm_glue.c
+++ b/sys/vm/vm_glue.c
@@ -453,7 +453,7 @@ vm_thread_stack_create(struct domainset *ds, int pages)
obj = vm_thread_kstack_size_to_obj(pages);
if (vm_ndomains > 1)
obj->domain.dr_policy = ds;
- vm_domainset_iter_page_init(&di, obj, 0, &domain, &req);
+ vm_domainset_iter_page_init(&di, obj, 0, &domain, &req, NULL);
do {
/*
* Get a kernel virtual address for this thread's kstack.
@@ -480,7 +480,7 @@ vm_thread_stack_create(struct domainset *ds, int pages)
vm_page_valid(ma[i]);
pmap_qenter(ks, ma, pages);
return (ks);
- } while (vm_domainset_iter_page(&di, obj, &domain) == 0);
+ } while (vm_domainset_iter_page(&di, obj, &domain, NULL) == 0);
return (0);
}
diff --git a/sys/vm/vm_page.c b/sys/vm/vm_page.c
index 91250060190d..eec85a5eb6f4 100644
--- a/sys/vm/vm_page.c
+++ b/sys/vm/vm_page.c
@@ -2007,13 +2007,14 @@ vm_page_alloc_iter(vm_object_t object, vm_pindex_t pindex, int req,
vm_page_t m;
int domain;
- vm_domainset_iter_page_init(&di, object, pindex, &domain, &req);
+ vm_domainset_iter_page_init(&di, object, pindex, &domain, &req,
+ pages);
do {
m = vm_page_alloc_domain_iter(object, pindex, domain, req,
pages);
if (m != NULL)
break;
- } while (vm_domainset_iter_page(&di, object, &domain) == 0);
+ } while (vm_domainset_iter_page(&di, object, &domain, pages) == 0);
return (m);
}
@@ -2142,7 +2143,8 @@ again:
* Not allocatable, give up.
*/
(void)vm_domain_alloc_fail(vmd, object, req);
- pctrie_iter_reset(pages);
+ if ((req & VM_ALLOC_WAITFAIL) != 0)
+ pctrie_iter_reset(pages);
return (NULL);
}
@@ -2254,7 +2256,7 @@ vm_page_alloc_contig(vm_object_t object, vm_pindex_t pindex, int req,
start_segind = -1;
- vm_domainset_iter_page_init(&di, object, pindex, &domain, &req);
+ vm_domainset_iter_page_init(&di, object, pindex, &domain, &req, NULL);
do {
m = vm_page_alloc_contig_domain(object, pindex, domain, req,
npages, low, high, alignment, boundary, memattr);
@@ -2266,7 +2268,7 @@ vm_page_alloc_contig(vm_object_t object, vm_pindex_t pindex, int req,
npages, low, high) == -1) {
vm_domainset_iter_ignore(&di, domain);
}
- } while (vm_domainset_iter_page(&di, object, &domain) == 0);
+ } while (vm_domainset_iter_page(&di, object, &domain, NULL) == 0);
return (m);
}
@@ -2576,12 +2578,12 @@ vm_page_alloc_noobj(int req)
vm_page_t m;
int domain;
- vm_domainset_iter_page_init(&di, NULL, 0, &domain, &req);
+ vm_domainset_iter_page_init(&di, NULL, 0, &domain, &req, NULL);
do {
m = vm_page_alloc_noobj_domain(domain, req);
if (m != NULL)
break;
- } while (vm_domainset_iter_page(&di, NULL, &domain) == 0);
+ } while (vm_domainset_iter_page(&di, NULL, &domain, NULL) == 0);
return (m);
}
@@ -2595,13 +2597,13 @@ vm_page_alloc_noobj_contig(int req, u_long npages, vm_paddr_t low,
vm_page_t m;
int domain;
- vm_domainset_iter_page_init(&di, NULL, 0, &domain, &req);
+ vm_domainset_iter_page_init(&di, NULL, 0, &domain, &req, NULL);
do {
m = vm_page_alloc_noobj_contig_domain(domain, req, npages, low,
high, alignment, boundary, memattr);
if (m != NULL)
break;
- } while (vm_domainset_iter_page(&di, NULL, &domain) == 0);
+ } while (vm_domainset_iter_page(&di, NULL, &domain, NULL) == 0);
return (m);
}
@@ -3316,7 +3318,7 @@ vm_page_reclaim_contig(int req, u_long npages, vm_paddr_t low, vm_paddr_t high,
ret = ERANGE;
- vm_domainset_iter_page_init(&di, NULL, 0, &domain, &req);
+ vm_domainset_iter_page_init(&di, NULL, 0, &domain, &req, NULL);
do {
status = vm_page_reclaim_contig_domain(domain, req, npages, low,
high, alignment, boundary);
@@ -3329,7 +3331,7 @@ vm_page_reclaim_contig(int req, u_long npages, vm_paddr_t low, vm_paddr_t high,
"from vm_page_reclaim_contig_domain()", status));
ret = ENOMEM;
}
- } while (vm_domainset_iter_page(&di, NULL, &domain) == 0);
+ } while (vm_domainset_iter_page(&di, NULL, &domain, NULL) == 0);
return (ret);
}