git: 8703b572b9f6 - main - vm_page_grab_pages: fetch page ranges
- Go to: [ bottom of page ] [ top of archives ] [ this month ]
Date: Fri, 09 May 2025 03:53:37 UTC
The branch main has been updated by dougm:
URL: https://cgit.FreeBSD.org/src/commit/?id=8703b572b9f6fa37018485e0188473f6097b5740
commit 8703b572b9f6fa37018485e0188473f6097b5740
Author: Doug Moore <dougm@FreeBSD.org>
AuthorDate: 2025-05-09 03:51:58 +0000
Commit: Doug Moore <dougm@FreeBSD.org>
CommitDate: 2025-05-09 03:51:58 +0000
vm_page_grab_pages: fetch page ranges
Define an iterator based function for reading a range of consecutive,
non-NULL leaves from a pctrie. Adapt it to vm pages. Use it in
vm_page_grab_pages to fetch more than one page at a time.
Reviewed by: markj
Differential Revision: https://reviews.freebsd.org/D50253
---
sys/kern/subr_pctrie.c | 31 ++++++++++++++++++++++++-------
sys/sys/pctrie.h | 16 +++++++++++++++-
sys/vm/vm_page.c | 14 ++++++++++----
sys/vm/vm_radix.h | 11 +++++++++++
4 files changed, 60 insertions(+), 12 deletions(-)
diff --git a/sys/kern/subr_pctrie.c b/sys/kern/subr_pctrie.c
index be1d57b4931f..e91966652f80 100644
--- a/sys/kern/subr_pctrie.c
+++ b/sys/kern/subr_pctrie.c
@@ -612,13 +612,14 @@ pctrie_iter_prev(struct pctrie_iter *it)
* array, starting at index.
*/
static __always_inline int
-_pctrie_lookup_range(struct pctrie *ptree, uint64_t index, uint64_t *value[],
- int count, smr_t smr, enum pctrie_access access)
+_pctrie_lookup_range(struct pctrie *ptree, struct pctrie_node *node,
+ uint64_t index, uint64_t *value[], int count,
+ struct pctrie_node **parent_out, smr_t smr, enum pctrie_access access)
{
- struct pctrie_node *parent, *node;
+ struct pctrie_node *parent;
int base, end, i;
- parent = NULL;
+ parent = node;
for (i = 0; i < count;) {
node = _pctrie_lookup_node(ptree, parent, index + i, &parent,
smr, access);
@@ -641,6 +642,8 @@ _pctrie_lookup_range(struct pctrie *ptree, uint64_t index, uint64_t *value[],
if (i < end)
break;
}
+ if (parent_out != NULL)
+ *parent_out = parent;
return (i);
}
@@ -653,8 +656,8 @@ int
pctrie_lookup_range(struct pctrie *ptree, uint64_t index,
uint64_t *value[], int count)
{
- return (_pctrie_lookup_range(ptree, index, value, count, NULL,
- PCTRIE_LOCKED));
+ return (_pctrie_lookup_range(ptree, NULL, index, value, count, NULL,
+ NULL, PCTRIE_LOCKED));
}
/*
@@ -671,11 +674,25 @@ pctrie_lookup_range_unlocked(struct pctrie *ptree, uint64_t index,
int res;
smr_enter(smr);
- res = _pctrie_lookup_range(ptree, index, value, count, smr, PCTRIE_SMR);
+ res = _pctrie_lookup_range(ptree, NULL, index, value, count, NULL,
+ smr, PCTRIE_SMR);
smr_exit(smr);
return (res);
}
+/*
+ * Returns the number of contiguous, non-NULL entries read into the value[]
+ * array, starting at index, assuming access is externally synchronized by a
+ * lock. Uses an iterator.
+ */
+int
+pctrie_iter_lookup_range(struct pctrie_iter *it, uint64_t index,
+ uint64_t *value[], int count)
+{
+ return (_pctrie_lookup_range(it->ptree, it->node, index, value, count,
+ &it->node, NULL, PCTRIE_LOCKED));
+}
+
/*
* Find first leaf >= index, and fill iter with the path to the parent of that
* leaf. Return NULL if there is no such leaf less than limit.
diff --git a/sys/sys/pctrie.h b/sys/sys/pctrie.h
index 0ccd4ec1e42e..3ff11f2c79ae 100644
--- a/sys/sys/pctrie.h
+++ b/sys/sys/pctrie.h
@@ -217,7 +217,7 @@ name##_PCTRIE_LOOKUP(struct pctrie *ptree, uint64_t key) \
\
static __inline __unused int \
name##_PCTRIE_LOOKUP_RANGE(struct pctrie *ptree, uint64_t key, \
- struct type *value[], int count) \
+ struct type *value[], int count) \
{ \
uint64_t **data = (uint64_t **)value; \
\
@@ -227,6 +227,18 @@ name##_PCTRIE_LOOKUP_RANGE(struct pctrie *ptree, uint64_t key, \
return (count); \
} \
\
+static __inline __unused int \
+name##_PCTRIE_ITER_LOOKUP_RANGE(struct pctrie_iter *it, uint64_t key, \
+ struct type *value[], int count) \
+{ \
+ uint64_t **data = (uint64_t **)value; \
+ \
+ count = pctrie_iter_lookup_range(it, key, data, count); \
+ for (int i = 0; i < count; i++) \
+ value[i] = name##_PCTRIE_NZVAL2PTR(data[i]); \
+ return (count); \
+} \
+ \
static __inline __unused struct type * \
name##_PCTRIE_LOOKUP_LE(struct pctrie *ptree, uint64_t key) \
{ \
@@ -399,6 +411,8 @@ int pctrie_lookup_range(struct pctrie *ptree,
uint64_t index, uint64_t *value[], int count);
int pctrie_lookup_range_unlocked(struct pctrie *ptree,
uint64_t index, uint64_t *value[], int count, smr_t smr);
+int pctrie_iter_lookup_range(struct pctrie_iter *it, uint64_t index,
+ uint64_t *value[], int count);
uint64_t *pctrie_iter_lookup(struct pctrie_iter *it, uint64_t index);
uint64_t *pctrie_iter_stride(struct pctrie_iter *it, int stride);
uint64_t *pctrie_iter_next(struct pctrie_iter *it);
diff --git a/sys/vm/vm_page.c b/sys/vm/vm_page.c
index 5ed1879c6a34..9ef9da737ded 100644
--- a/sys/vm/vm_page.c
+++ b/sys/vm/vm_page.c
@@ -5150,7 +5150,7 @@ vm_page_grab_pages(vm_object_t object, vm_pindex_t pindex, int allocflags,
struct pctrie_iter pages;
vm_page_t m;
int pflags;
- int i;
+ int ahead, i;
VM_OBJECT_ASSERT_WLOCKED(object);
KASSERT(((u_int)allocflags >> VM_ALLOC_COUNT_SHIFT) == 0,
@@ -5163,9 +5163,15 @@ vm_page_grab_pages(vm_object_t object, vm_pindex_t pindex, int allocflags,
i = 0;
vm_page_iter_init(&pages, object);
retrylookup:
+ ahead = 0;
for (; i < count; i++) {
- m = vm_radix_iter_lookup(&pages, pindex + i);
- if (m != NULL) {
+ if (ahead == 0) {
+ ahead = vm_radix_iter_lookup_range(
+ &pages, pindex + i, &ma[i], count - i);
+ }
+ if (ahead > 0) {
+ --ahead;
+ m = ma[i];
if (!vm_page_tryacquire(m, allocflags)) {
if (vm_page_grab_sleep(object, m, pindex + i,
"grbmaw", allocflags, true)) {
@@ -5185,6 +5191,7 @@ retrylookup:
break;
goto retrylookup;
}
+ ma[i] = m;
}
if (vm_page_none_valid(m) &&
(allocflags & VM_ALLOC_ZERO) != 0) {
@@ -5193,7 +5200,6 @@ retrylookup:
vm_page_valid(m);
}
vm_page_grab_release(m, allocflags);
- ma[i] = m;
}
return (i);
}
diff --git a/sys/vm/vm_radix.h b/sys/vm/vm_radix.h
index 1e9115f1a6ba..6359373305e1 100644
--- a/sys/vm/vm_radix.h
+++ b/sys/vm/vm_radix.h
@@ -113,6 +113,17 @@ vm_radix_lookup_range_unlocked(struct vm_radix *rtree, vm_pindex_t index,
ma, count));
}
+/*
+ * Returns the number of contiguous, non-NULL pages read into the ma[]
+ * array, without requiring an external lock.
+ */
+static __inline int
+vm_radix_iter_lookup_range(struct pctrie_iter *pages, vm_pindex_t index,
+ vm_page_t ma[], int count)
+{
+ return (VM_RADIX_PCTRIE_ITER_LOOKUP_RANGE(pages, index, ma, count));
+}
+
/*
* Initialize an iterator for vm_radix.
*/