git: bda42942b95f - stable/14 - Avoid waiting on physical allocations that can't possibly be satisfied
- Go to: [ bottom of page ] [ top of archives ] [ this month ]
Date: Fri, 19 Sep 2025 11:42:36 UTC
The branch stable/14 has been updated by olce:
URL: https://cgit.FreeBSD.org/src/commit/?id=bda42942b95faf7b4cc8f3fa825c76ca58820ceb
commit bda42942b95faf7b4cc8f3fa825c76ca58820ceb
Author: Jason A. Harmening <jah@FreeBSD.org>
AuthorDate: 2023-11-20 23:23:58 +0000
Commit: Olivier Certner <olce@FreeBSD.org>
CommitDate: 2025-09-19 11:41:37 +0000
Avoid waiting on physical allocations that can't possibly be satisfied
- Change vm_page_reclaim_contig[_domain] to return an errno instead
of a boolean. 0 indicates a successful reclaim, ENOMEM indicates
lack of available memory to reclaim, with any other error (currently
only ERANGE) indicating that reclamation is impossible for the
specified address range. Change all callers to only follow
up with vm_page_wait* in the ENOMEM case.
- Introduce vm_domainset_iter_ignore(), which marks the specified
domain as unavailable for further use by the iterator. Use this
function to ignore domains that can't possibly satisfy a physical
allocation request. Since WAITOK allocations run the iterators
repeatedly, this avoids the possibility of infinitely spinning
in domain iteration if no available domain can satisfy the
allocation request.
PR: 274252
Reported by: kevans
Tested by: kevans
Reviewed by: markj
Differential Revision: https://reviews.freebsd.org/D42706
(cherry picked from commit 2619c5ccfe1f7889f0241916bd17d06340142b05)
MFCed as a prerequisite for further MFC of VM domainset changes. Based
on analysis, it would not hurt, and I have been using it in productions
for months now.
Resolved the trivial conflict due to commit 718d1928f874 ("LinuxKPI:
make linux_alloc_pages() honor __GFP_NORETRY") having been MFCed before
this one.
---
sys/arm/nvidia/drm2/tegra_bo.c | 9 +++--
sys/compat/linuxkpi/common/src/linux_page.c | 8 ++--
sys/dev/drm2/ttm/ttm_bo.c | 4 +-
sys/dev/drm2/ttm/ttm_page_alloc.c | 9 +++--
sys/kern/uipc_ktls.c | 5 ++-
sys/kern/uipc_shm.c | 5 ++-
sys/vm/vm_domainset.c | 32 +++++++++++++---
sys/vm/vm_domainset.h | 2 +
sys/vm/vm_kern.c | 24 +++++++++++-
sys/vm/vm_page.c | 58 ++++++++++++++++++++++-------
sys/vm/vm_page.h | 6 +--
11 files changed, 123 insertions(+), 39 deletions(-)
diff --git a/sys/arm/nvidia/drm2/tegra_bo.c b/sys/arm/nvidia/drm2/tegra_bo.c
index ae0c48d6b677..afec9f50ff50 100644
--- a/sys/arm/nvidia/drm2/tegra_bo.c
+++ b/sys/arm/nvidia/drm2/tegra_bo.c
@@ -95,7 +95,7 @@ tegra_bo_alloc_contig(size_t npages, u_long alignment, vm_memattr_t memattr,
vm_page_t **ret_page)
{
vm_page_t m;
- int tries, i;
+ int err, i, tries;
vm_paddr_t low, high, boundary;
low = 0;
@@ -107,9 +107,12 @@ retry:
low, high, alignment, boundary, memattr);
if (m == NULL) {
if (tries < 3) {
- if (!vm_page_reclaim_contig(0, npages, low, high,
- alignment, boundary))
+ err = vm_page_reclaim_contig(0, npages, low, high,
+ alignment, boundary);
+ if (err == ENOMEM)
vm_wait(NULL);
+ else if (err != 0)
+ return (ENOMEM);
tries++;
goto retry;
}
diff --git a/sys/compat/linuxkpi/common/src/linux_page.c b/sys/compat/linuxkpi/common/src/linux_page.c
index bb5a1411eeec..b910ada90a4d 100644
--- a/sys/compat/linuxkpi/common/src/linux_page.c
+++ b/sys/compat/linuxkpi/common/src/linux_page.c
@@ -125,10 +125,12 @@ linux_alloc_pages(gfp_t flags, unsigned int order)
if (page == NULL) {
if ((flags & (M_WAITOK | __GFP_NORETRY)) ==
M_WAITOK) {
- if (!vm_page_reclaim_contig(req,
- npages, 0, pmax, PAGE_SIZE, 0)) {
+ int err = vm_page_reclaim_contig(req,
+ npages, 0, pmax, PAGE_SIZE, 0);
+ if (err == ENOMEM)
vm_wait(NULL);
- }
+ else if (err != 0)
+ return (NULL);
flags &= ~M_WAITOK;
goto retry;
}
diff --git a/sys/dev/drm2/ttm/ttm_bo.c b/sys/dev/drm2/ttm/ttm_bo.c
index f60cb4f37535..b57d961338f4 100644
--- a/sys/dev/drm2/ttm/ttm_bo.c
+++ b/sys/dev/drm2/ttm/ttm_bo.c
@@ -1498,8 +1498,8 @@ retry:
VM_MAX_ADDRESS, PAGE_SIZE, 0, VM_MEMATTR_UNCACHEABLE);
if (unlikely(glob->dummy_read_page == NULL)) {
- if (tries < 1 && vm_page_reclaim_contig(0, 1, 0,
- VM_MAX_ADDRESS, PAGE_SIZE, 0)) {
+ if (tries < 1 && (vm_page_reclaim_contig(0, 1, 0,
+ VM_MAX_ADDRESS, PAGE_SIZE, 0) == 0)) {
tries++;
goto retry;
}
diff --git a/sys/dev/drm2/ttm/ttm_page_alloc.c b/sys/dev/drm2/ttm/ttm_page_alloc.c
index 67c484218cc2..7518ecb4dfd1 100644
--- a/sys/dev/drm2/ttm/ttm_page_alloc.c
+++ b/sys/dev/drm2/ttm/ttm_page_alloc.c
@@ -158,16 +158,19 @@ static vm_page_t
ttm_vm_page_alloc_dma32(int req, vm_memattr_t memattr)
{
vm_page_t p;
- int tries;
+ int err, tries;
for (tries = 0; ; tries++) {
p = vm_page_alloc_noobj_contig(req, 1, 0, 0xffffffff, PAGE_SIZE,
0, memattr);
if (p != NULL || tries > 2)
return (p);
- if (!vm_page_reclaim_contig(req, 1, 0, 0xffffffff,
- PAGE_SIZE, 0))
+ err = vm_page_reclaim_contig(req, 1, 0, 0xffffffff,
+ PAGE_SIZE, 0);
+ if (err == ENOMEM)
vm_wait(NULL);
+ else if (err != 0)
+ return (NULL);
}
}
diff --git a/sys/kern/uipc_ktls.c b/sys/kern/uipc_ktls.c
index 60ec05d3d94a..64150086658a 100644
--- a/sys/kern/uipc_ktls.c
+++ b/sys/kern/uipc_ktls.c
@@ -3189,8 +3189,9 @@ ktls_reclaim_thread(void *ctx)
* backlogs of buffers to be encrypted, leading to
* surges of traffic and potential NIC output drops.
*/
- if (!vm_page_reclaim_contig_domain_ext(domain, VM_ALLOC_NORMAL,
- atop(ktls_maxlen), 0, ~0ul, PAGE_SIZE, 0, ktls_max_reclaim)) {
+ if (vm_page_reclaim_contig_domain_ext(domain, VM_ALLOC_NORMAL,
+ atop(ktls_maxlen), 0, ~0ul, PAGE_SIZE, 0,
+ ktls_max_reclaim) != 0) {
vm_wait_domain(domain);
} else {
sc->reclaims += ktls_max_reclaim;
diff --git a/sys/kern/uipc_shm.c b/sys/kern/uipc_shm.c
index c8d5521c5ab1..a07dd097f7b8 100644
--- a/sys/kern/uipc_shm.c
+++ b/sys/kern/uipc_shm.c
@@ -883,8 +883,9 @@ shm_dotruncate_largepage(struct shmfd *shmfd, off_t length, void *rl_cookie)
}
error = vm_page_reclaim_contig(aflags,
pagesizes[psind] / PAGE_SIZE, 0, ~0,
- pagesizes[psind], 0) ? 0 :
- vm_wait_intr(object);
+ pagesizes[psind], 0);
+ if (error == ENOMEM)
+ error = vm_wait_intr(object);
if (error != 0) {
VM_OBJECT_WLOCK(object);
return (error);
diff --git a/sys/vm/vm_domainset.c b/sys/vm/vm_domainset.c
index b666324db494..ff0e476a68b9 100644
--- a/sys/vm/vm_domainset.c
+++ b/sys/vm/vm_domainset.c
@@ -68,6 +68,7 @@ vm_domainset_iter_init(struct vm_domainset_iter *di, struct domainset *ds,
di->di_domain = ds;
di->di_iter = iter;
di->di_policy = ds->ds_policy;
+ DOMAINSET_COPY(&ds->ds_mask, &di->di_valid_mask);
if (di->di_policy == DOMAINSET_POLICY_INTERLEAVE) {
#if VM_NRESERVLEVEL > 0
if (vm_object_reserv(obj)) {
@@ -156,7 +157,7 @@ vm_domainset_iter_first(struct vm_domainset_iter *di, int *domain)
switch (di->di_policy) {
case DOMAINSET_POLICY_FIRSTTOUCH:
*domain = PCPU_GET(domain);
- if (DOMAINSET_ISSET(*domain, &di->di_domain->ds_mask)) {
+ if (DOMAINSET_ISSET(*domain, &di->di_valid_mask)) {
/*
* Add an extra iteration because we will visit the
* current domain a second time in the rr iterator.
@@ -217,11 +218,14 @@ int
vm_domainset_iter_page(struct vm_domainset_iter *di, struct vm_object *obj,
int *domain)
{
+ if (__predict_false(DOMAINSET_EMPTY(&di->di_valid_mask)))
+ return (ENOMEM);
/* If there are more domains to visit we run the iterator. */
while (--di->di_n != 0) {
vm_domainset_iter_next(di, domain);
- if (!di->di_minskip || !vm_page_count_min_domain(*domain))
+ if (DOMAINSET_ISSET(*domain, &di->di_valid_mask) &&
+ (!di->di_minskip || !vm_page_count_min_domain(*domain)))
return (0);
}
@@ -239,7 +243,7 @@ vm_domainset_iter_page(struct vm_domainset_iter *di, struct vm_object *obj,
/* Wait for one of the domains to accumulate some free pages. */
if (obj != NULL)
VM_OBJECT_WUNLOCK(obj);
- vm_wait_doms(&di->di_domain->ds_mask, 0);
+ vm_wait_doms(&di->di_valid_mask, 0);
if (obj != NULL)
VM_OBJECT_WLOCK(obj);
if ((di->di_flags & VM_ALLOC_WAITFAIL) != 0)
@@ -284,11 +288,14 @@ vm_domainset_iter_policy_ref_init(struct vm_domainset_iter *di,
int
vm_domainset_iter_policy(struct vm_domainset_iter *di, int *domain)
{
+ if (DOMAINSET_EMPTY(&di->di_valid_mask))
+ return (ENOMEM);
/* If there are more domains to visit we run the iterator. */
while (--di->di_n != 0) {
vm_domainset_iter_next(di, domain);
- if (!di->di_minskip || !vm_page_count_min_domain(*domain))
+ if (DOMAINSET_ISSET(*domain, &di->di_valid_mask) &&
+ (!di->di_minskip || !vm_page_count_min_domain(*domain)))
return (0);
}
@@ -304,7 +311,7 @@ vm_domainset_iter_policy(struct vm_domainset_iter *di, int *domain)
return (ENOMEM);
/* Wait for one of the domains to accumulate some free pages. */
- vm_wait_doms(&di->di_domain->ds_mask, 0);
+ vm_wait_doms(&di->di_valid_mask, 0);
/* Restart the search. */
vm_domainset_iter_first(di, domain);
@@ -312,6 +319,15 @@ vm_domainset_iter_policy(struct vm_domainset_iter *di, int *domain)
return (0);
}
+void
+vm_domainset_iter_ignore(struct vm_domainset_iter *di, int domain)
+{
+ KASSERT(DOMAINSET_ISSET(domain, &di->di_valid_mask),
+ ("%s: domain %d not present in di_valid_mask for di %p",
+ __func__, domain, di));
+ DOMAINSET_CLR(domain, &di->di_valid_mask);
+}
+
#else /* !NUMA */
int
@@ -353,4 +369,10 @@ vm_domainset_iter_policy_ref_init(struct vm_domainset_iter *di,
*domain = 0;
}
+void
+vm_domainset_iter_ignore(struct vm_domainset_iter *di __unused,
+ int domain __unused)
+{
+}
+
#endif /* NUMA */
diff --git a/sys/vm/vm_domainset.h b/sys/vm/vm_domainset.h
index 4ce619069dd0..d2cfe362ae78 100644
--- a/sys/vm/vm_domainset.h
+++ b/sys/vm/vm_domainset.h
@@ -31,6 +31,7 @@
struct vm_domainset_iter {
struct domainset *di_domain;
unsigned int *di_iter;
+ domainset_t di_valid_mask;
vm_pindex_t di_offset;
int di_flags;
uint16_t di_policy;
@@ -47,6 +48,7 @@ void vm_domainset_iter_policy_init(struct vm_domainset_iter *,
struct domainset *, int *, int *);
void vm_domainset_iter_policy_ref_init(struct vm_domainset_iter *,
struct domainset_ref *, int *, int *);
+void vm_domainset_iter_ignore(struct vm_domainset_iter *, int);
int vm_wait_doms(const domainset_t *, int mflags);
diff --git a/sys/vm/vm_kern.c b/sys/vm/vm_kern.c
index 387d73d2e46e..f11dfd745720 100644
--- a/sys/vm/vm_kern.c
+++ b/sys/vm/vm_kern.c
@@ -234,8 +234,8 @@ kmem_alloc_contig_pages(vm_object_t object, vm_pindex_t pindex, int domain,
break;
VM_OBJECT_WUNLOCK(object);
- if (!vm_page_reclaim_contig_domain(domain, pflags, npages,
- low, high, alignment, boundary) && wait)
+ if (vm_page_reclaim_contig_domain(domain, pflags, npages,
+ low, high, alignment, boundary) == ENOMEM && wait)
vm_wait_domain(domain);
VM_OBJECT_WLOCK(object);
}
@@ -308,8 +308,12 @@ kmem_alloc_attr_domainset(struct domainset *ds, vm_size_t size, int flags,
vm_paddr_t low, vm_paddr_t high, vm_memattr_t memattr)
{
struct vm_domainset_iter di;
+ vm_page_t bounds[2];
void *addr;
int domain;
+ int start_segind;
+
+ start_segind = -1;
vm_domainset_iter_policy_init(&di, ds, &domain, &flags);
do {
@@ -317,6 +321,12 @@ kmem_alloc_attr_domainset(struct domainset *ds, vm_size_t size, int flags,
memattr);
if (addr != NULL)
break;
+ if (start_segind == -1)
+ start_segind = vm_phys_lookup_segind(low);
+ if (vm_phys_find_range(bounds, start_segind, domain,
+ atop(round_page(size)), low, high) == -1) {
+ vm_domainset_iter_ignore(&di, domain);
+ }
} while (vm_domainset_iter_policy(&di, &domain) == 0);
return (addr);
@@ -392,8 +402,12 @@ kmem_alloc_contig_domainset(struct domainset *ds, vm_size_t size, int flags,
vm_memattr_t memattr)
{
struct vm_domainset_iter di;
+ vm_page_t bounds[2];
void *addr;
int domain;
+ int start_segind;
+
+ start_segind = -1;
vm_domainset_iter_policy_init(&di, ds, &domain, &flags);
do {
@@ -401,6 +415,12 @@ kmem_alloc_contig_domainset(struct domainset *ds, vm_size_t size, int flags,
alignment, boundary, memattr);
if (addr != NULL)
break;
+ if (start_segind == -1)
+ start_segind = vm_phys_lookup_segind(low);
+ if (vm_phys_find_range(bounds, start_segind, domain,
+ atop(round_page(size)), low, high) == -1) {
+ vm_domainset_iter_ignore(&di, domain);
+ }
} while (vm_domainset_iter_policy(&di, &domain) == 0);
return (addr);
diff --git a/sys/vm/vm_page.c b/sys/vm/vm_page.c
index 5b5282dc5c9b..6daaaa79163e 100644
--- a/sys/vm/vm_page.c
+++ b/sys/vm/vm_page.c
@@ -2171,8 +2171,12 @@ vm_page_alloc_contig(vm_object_t object, vm_pindex_t pindex, int req,
vm_paddr_t boundary, vm_memattr_t memattr)
{
struct vm_domainset_iter di;
+ vm_page_t bounds[2];
vm_page_t m;
int domain;
+ int start_segind;
+
+ start_segind = -1;
vm_domainset_iter_page_init(&di, object, pindex, &domain, &req);
do {
@@ -2180,6 +2184,12 @@ vm_page_alloc_contig(vm_object_t object, vm_pindex_t pindex, int req,
npages, low, high, alignment, boundary, memattr);
if (m != NULL)
break;
+ if (start_segind == -1)
+ start_segind = vm_phys_lookup_segind(low);
+ if (vm_phys_find_range(bounds, start_segind, domain,
+ npages, low, high) == -1) {
+ vm_domainset_iter_ignore(&di, domain);
+ }
} while (vm_domainset_iter_page(&di, object, &domain) == 0);
return (m);
@@ -3023,7 +3033,7 @@ unlock:
* "npages" must be greater than zero. Both "alignment" and "boundary"
* must be a power of two.
*/
-bool
+int
vm_page_reclaim_contig_domain_ext(int domain, int req, u_long npages,
vm_paddr_t low, vm_paddr_t high, u_long alignment, vm_paddr_t boundary,
int desired_runs)
@@ -3031,14 +3041,15 @@ vm_page_reclaim_contig_domain_ext(int domain, int req, u_long npages,
struct vm_domain *vmd;
vm_page_t bounds[2], m_run, _m_runs[NRUNS], *m_runs;
u_long count, minalign, reclaimed;
- int error, i, min_reclaim, nruns, options, req_class, segind;
- bool ret;
+ int error, i, min_reclaim, nruns, options, req_class;
+ int segind, start_segind;
+ int ret;
KASSERT(npages > 0, ("npages is 0"));
KASSERT(powerof2(alignment), ("alignment is not a power of 2"));
KASSERT(powerof2(boundary), ("boundary is not a power of 2"));
- ret = false;
+ ret = ENOMEM;
/*
* If the caller wants to reclaim multiple runs, try to allocate
@@ -3078,6 +3089,8 @@ vm_page_reclaim_contig_domain_ext(int domain, int req, u_long npages,
if (curproc == pageproc && req_class != VM_ALLOC_INTERRUPT)
req_class = VM_ALLOC_SYSTEM;
+ start_segind = vm_phys_lookup_segind(low);
+
/*
* Return if the number of free pages cannot satisfy the requested
* allocation.
@@ -3094,14 +3107,17 @@ vm_page_reclaim_contig_domain_ext(int domain, int req, u_long npages,
* the reclamation of reservations and superpages each time.
*/
for (options = VPSC_NORESERV;;) {
+ bool phys_range_exists = false;
+
/*
* Find the highest runs that satisfy the given constraints
* and restrictions, and record them in "m_runs".
*/
count = 0;
- segind = vm_phys_lookup_segind(low);
+ segind = start_segind;
while ((segind = vm_phys_find_range(bounds, segind, domain,
npages, low, high)) != -1) {
+ phys_range_exists = true;
while ((m_run = vm_page_scan_contig(npages, bounds[0],
bounds[1], alignment, boundary, options))) {
bounds[0] = m_run + npages;
@@ -3111,6 +3127,11 @@ vm_page_reclaim_contig_domain_ext(int domain, int req, u_long npages,
segind++;
}
+ if (!phys_range_exists) {
+ ret = ERANGE;
+ goto done;
+ }
+
/*
* Reclaim the highest runs in LIFO (descending) order until
* the number of reclaimed pages, "reclaimed", is at least
@@ -3127,7 +3148,7 @@ vm_page_reclaim_contig_domain_ext(int domain, int req, u_long npages,
if (error == 0) {
reclaimed += npages;
if (reclaimed >= min_reclaim) {
- ret = true;
+ ret = 0;
goto done;
}
}
@@ -3142,7 +3163,8 @@ vm_page_reclaim_contig_domain_ext(int domain, int req, u_long npages,
else if (options == VPSC_NOSUPER)
options = VPSC_ANY;
else if (options == VPSC_ANY) {
- ret = reclaimed != 0;
+ if (reclaimed != 0)
+ ret = 0;
goto done;
}
}
@@ -3152,7 +3174,7 @@ done:
return (ret);
}
-bool
+int
vm_page_reclaim_contig_domain(int domain, int req, u_long npages,
vm_paddr_t low, vm_paddr_t high, u_long alignment, vm_paddr_t boundary)
{
@@ -3160,20 +3182,28 @@ vm_page_reclaim_contig_domain(int domain, int req, u_long npages,
alignment, boundary, 1));
}
-bool
+int
vm_page_reclaim_contig(int req, u_long npages, vm_paddr_t low, vm_paddr_t high,
u_long alignment, vm_paddr_t boundary)
{
struct vm_domainset_iter di;
- int domain;
- bool ret;
+ int domain, ret, status;
+
+ ret = ERANGE;
vm_domainset_iter_page_init(&di, NULL, 0, &domain, &req);
do {
- ret = vm_page_reclaim_contig_domain(domain, req, npages, low,
+ status = vm_page_reclaim_contig_domain(domain, req, npages, low,
high, alignment, boundary);
- if (ret)
- break;
+ if (status == 0)
+ return (0);
+ else if (status == ERANGE)
+ vm_domainset_iter_ignore(&di, domain);
+ else {
+ KASSERT(status == ENOMEM, ("Unrecognized error %d "
+ "from vm_page_reclaim_contig_domain()", status));
+ ret = ENOMEM;
+ }
} while (vm_domainset_iter_page(&di, NULL, &domain) == 0);
return (ret);
diff --git a/sys/vm/vm_page.h b/sys/vm/vm_page.h
index 68f68d62d2be..e8d1497e2afc 100644
--- a/sys/vm/vm_page.h
+++ b/sys/vm/vm_page.h
@@ -662,11 +662,11 @@ vm_page_t vm_page_prev(vm_page_t m);
bool vm_page_ps_test(vm_page_t m, int flags, vm_page_t skip_m);
void vm_page_putfake(vm_page_t m);
void vm_page_readahead_finish(vm_page_t m);
-bool vm_page_reclaim_contig(int req, u_long npages, vm_paddr_t low,
+int vm_page_reclaim_contig(int req, u_long npages, vm_paddr_t low,
vm_paddr_t high, u_long alignment, vm_paddr_t boundary);
-bool vm_page_reclaim_contig_domain(int domain, int req, u_long npages,
+int vm_page_reclaim_contig_domain(int domain, int req, u_long npages,
vm_paddr_t low, vm_paddr_t high, u_long alignment, vm_paddr_t boundary);
-bool vm_page_reclaim_contig_domain_ext(int domain, int req, u_long npages,
+int vm_page_reclaim_contig_domain_ext(int domain, int req, u_long npages,
vm_paddr_t low, vm_paddr_t high, u_long alignment, vm_paddr_t boundary,
int desired_runs);
void vm_page_reference(vm_page_t m);