git: bcd85e0150c9 - main - vm_page: make iter_insert() public
- Go to: [ bottom of page ] [ top of archives ] [ this month ]
Date: Fri, 09 May 2025 04:14:36 UTC
The branch main has been updated by dougm:
URL: https://cgit.FreeBSD.org/src/commit/?id=bcd85e0150c903848f6e0d54f5428931c2f2d575
commit bcd85e0150c903848f6e0d54f5428931c2f2d575
Author: Doug Moore <dougm@FreeBSD.org>
AuthorDate: 2025-05-09 04:13:32 +0000
Commit: Doug Moore <dougm@FreeBSD.org>
CommitDate: 2025-05-09 04:13:32 +0000
vm_page: make iter_insert() public
In places where vm_page_insert() is used after lookups, or for
consecutive pages, use vm_page_iter_insert instead, to exploit
locality.
Reviewed by: kib, markj
Differential Revision: https://reviews.freebsd.org/D50248
---
sys/amd64/sgx/sgx.c | 62 ++++++++++++++---------------
sys/arm/nvidia/drm2/tegra_bo.c | 4 +-
sys/compat/linuxkpi/common/src/linux_page.c | 11 +++--
sys/dev/drm2/ttm/ttm_bo_vm.c | 9 +++--
sys/vm/vm_page.c | 2 +-
sys/vm/vm_page.h | 2 +
6 files changed, 50 insertions(+), 40 deletions(-)
diff --git a/sys/amd64/sgx/sgx.c b/sys/amd64/sgx/sgx.c
index af8adb763214..60209b04f31f 100644
--- a/sys/amd64/sgx/sgx.c
+++ b/sys/amd64/sgx/sgx.c
@@ -191,9 +191,20 @@ sgx_put_epc_page(struct sgx_softc *sc, struct epc_page *epc)
vmem_free(sc->vmem_epc, addr, PAGE_SIZE);
}
+static void
+sgx_insert_epc_page_by_index(vm_page_t page, vm_object_t object,
+ vm_pindex_t pidx, struct pctrie_iter *pages)
+{
+
+ VM_OBJECT_ASSERT_WLOCKED(object);
+
+ page->valid = VM_PAGE_BITS_ALL;
+ vm_page_iter_insert(page, object, pidx, pages);
+}
+
static int
sgx_va_slot_init_by_index(struct sgx_softc *sc, vm_object_t object,
- uint64_t idx)
+ uint64_t idx, struct pctrie_iter *pages)
{
struct epc_page *epc;
vm_page_t page;
@@ -202,7 +213,7 @@ sgx_va_slot_init_by_index(struct sgx_softc *sc, vm_object_t object,
VM_OBJECT_ASSERT_WLOCKED(object);
- p = vm_page_lookup(object, idx);
+ p = vm_radix_iter_lookup(pages, idx);
if (p == NULL) {
ret = sgx_get_epc_page(sc, &epc);
if (ret) {
@@ -216,20 +227,16 @@ sgx_va_slot_init_by_index(struct sgx_softc *sc, vm_object_t object,
mtx_unlock(&sc->mtx_encls);
page = PHYS_TO_VM_PAGE(epc->phys);
-
- page->valid = VM_PAGE_BITS_ALL;
- vm_page_insert(page, object, idx);
+ sgx_insert_epc_page_by_index(page, object, idx, pages);
}
return (0);
}
static int
-sgx_va_slot_init(struct sgx_softc *sc,
- struct sgx_enclave *enclave,
- uint64_t addr)
+sgx_va_slot_init(struct sgx_softc *sc, struct sgx_enclave *enclave,
+ vm_pindex_t pidx, struct pctrie_iter *pages)
{
- vm_pindex_t pidx;
uint64_t va_page_idx;
uint64_t idx;
vm_object_t object;
@@ -239,12 +246,10 @@ sgx_va_slot_init(struct sgx_softc *sc,
VM_OBJECT_ASSERT_WLOCKED(object);
- pidx = OFF_TO_IDX(addr);
-
va_page_idx = pidx / SGX_VA_PAGE_SLOTS;
idx = - SGX_VA_PAGES_OFFS - va_page_idx;
- ret = sgx_va_slot_init_by_index(sc, object, idx);
+ ret = sgx_va_slot_init_by_index(sc, object, idx, pages);
return (ret);
}
@@ -614,19 +619,8 @@ static struct cdev_pager_ops sgx_pg_ops = {
};
static void
-sgx_insert_epc_page_by_index(vm_page_t page, vm_object_t object,
- vm_pindex_t pidx)
-{
-
- VM_OBJECT_ASSERT_WLOCKED(object);
-
- page->valid = VM_PAGE_BITS_ALL;
- vm_page_insert(page, object, pidx);
-}
-
-static void
-sgx_insert_epc_page(struct sgx_enclave *enclave,
- struct epc_page *epc, uint64_t addr)
+sgx_insert_epc_page(struct sgx_enclave *enclave, struct epc_page *epc,
+ uint64_t addr, struct pctrie_iter *pages)
{
vm_pindex_t pidx;
vm_page_t page;
@@ -636,12 +630,13 @@ sgx_insert_epc_page(struct sgx_enclave *enclave,
pidx = OFF_TO_IDX(addr);
page = PHYS_TO_VM_PAGE(epc->phys);
- sgx_insert_epc_page_by_index(page, enclave->object, pidx);
+ sgx_insert_epc_page_by_index(page, enclave->object, pidx, pages);
}
static int
sgx_ioctl_create(struct sgx_softc *sc, struct sgx_enclave_create *param)
{
+ struct pctrie_iter pages;
struct sgx_vm_handle *vmh;
vm_map_entry_t entry;
vm_page_t p;
@@ -712,8 +707,9 @@ sgx_ioctl_create(struct sgx_softc *sc, struct sgx_enclave_create *param)
}
enclave->secs_epc_page = epc;
+ vm_page_iter_init(&pages, object);
VM_OBJECT_WLOCK(object);
- p = vm_page_lookup(object, SGX_SECS_VM_OBJECT_INDEX);
+ p = vm_radix_iter_lookup(&pages, SGX_SECS_VM_OBJECT_INDEX);
if (p) {
VM_OBJECT_WUNLOCK(object);
/* SECS page already added. */
@@ -722,7 +718,7 @@ sgx_ioctl_create(struct sgx_softc *sc, struct sgx_enclave_create *param)
}
ret = sgx_va_slot_init_by_index(sc, object,
- - SGX_VA_PAGES_OFFS - SGX_SECS_VM_OBJECT_INDEX);
+ - SGX_VA_PAGES_OFFS - SGX_SECS_VM_OBJECT_INDEX, &pages);
if (ret) {
VM_OBJECT_WUNLOCK(object);
dprintf("%s: Can't init va slot.\n", __func__);
@@ -762,7 +758,7 @@ sgx_ioctl_create(struct sgx_softc *sc, struct sgx_enclave_create *param)
page = PHYS_TO_VM_PAGE(epc->phys);
sgx_insert_epc_page_by_index(page, enclave->object,
- SGX_SECS_VM_OBJECT_INDEX);
+ SGX_SECS_VM_OBJECT_INDEX, &pages);
VM_OBJECT_WUNLOCK(object);
@@ -786,6 +782,7 @@ static int
sgx_ioctl_add_page(struct sgx_softc *sc,
struct sgx_enclave_add_page *addp)
{
+ struct pctrie_iter pages;
struct epc_page *secs_epc_page;
struct sgx_enclave *enclave;
struct sgx_vm_handle *vmh;
@@ -857,8 +854,9 @@ sgx_ioctl_add_page(struct sgx_softc *sc,
addr = (addp->addr - vmh->base);
pidx = OFF_TO_IDX(addr);
+ vm_page_iter_init(&pages, object);
VM_OBJECT_WLOCK(object);
- p = vm_page_lookup(object, pidx);
+ p = vm_radix_iter_lookup(&pages, pidx);
if (p) {
VM_OBJECT_WUNLOCK(object);
/* Page already added. */
@@ -866,7 +864,7 @@ sgx_ioctl_add_page(struct sgx_softc *sc,
goto error;
}
- ret = sgx_va_slot_init(sc, enclave, addr);
+ ret = sgx_va_slot_init(sc, enclave, pidx, &pages);
if (ret) {
VM_OBJECT_WUNLOCK(object);
dprintf("%s: Can't init va slot.\n", __func__);
@@ -898,7 +896,7 @@ sgx_ioctl_add_page(struct sgx_softc *sc,
goto error;
}
- sgx_insert_epc_page(enclave, epc, addr);
+ sgx_insert_epc_page(enclave, epc, addr, &pages);
VM_OBJECT_WUNLOCK(object);
diff --git a/sys/arm/nvidia/drm2/tegra_bo.c b/sys/arm/nvidia/drm2/tegra_bo.c
index 8952d09eec3d..73ba82f2f883 100644
--- a/sys/arm/nvidia/drm2/tegra_bo.c
+++ b/sys/arm/nvidia/drm2/tegra_bo.c
@@ -133,6 +133,7 @@ retry:
static int
tegra_bo_init_pager(struct tegra_bo *bo)
{
+ struct pctrie_iter pages;
vm_page_t m;
size_t size;
int i;
@@ -143,6 +144,7 @@ tegra_bo_init_pager(struct tegra_bo *bo)
if (vmem_alloc(kernel_arena, size, M_WAITOK | M_BESTFIT, &bo->vbase))
return (ENOMEM);
+ vm_page_iter_init(&pages, bo->cdev_pager);
VM_OBJECT_WLOCK(bo->cdev_pager);
for (i = 0; i < bo->npages; i++) {
m = bo->m[i];
@@ -159,7 +161,7 @@ tegra_bo_init_pager(struct tegra_bo *bo)
*/
m->oflags &= ~VPO_UNMANAGED;
m->flags |= PG_FICTITIOUS;
- if (vm_page_insert(m, bo->cdev_pager, i) != 0)
+ if (vm_page_iter_insert(m, bo->cdev_pager, i, &pages) != 0)
return (EINVAL);
}
VM_OBJECT_WUNLOCK(bo->cdev_pager);
diff --git a/sys/compat/linuxkpi/common/src/linux_page.c b/sys/compat/linuxkpi/common/src/linux_page.c
index b5a0d34a6ad7..93aaea01f0df 100644
--- a/sys/compat/linuxkpi/common/src/linux_page.c
+++ b/sys/compat/linuxkpi/common/src/linux_page.c
@@ -296,23 +296,27 @@ vm_fault_t
lkpi_vmf_insert_pfn_prot_locked(struct vm_area_struct *vma, unsigned long addr,
unsigned long pfn, pgprot_t prot)
{
+ struct pctrie_iter pages;
vm_object_t vm_obj = vma->vm_obj;
vm_object_t tmp_obj;
vm_page_t page;
vm_pindex_t pindex;
VM_OBJECT_ASSERT_WLOCKED(vm_obj);
+ vm_page_iter_init(&pages, vm_obj);
pindex = OFF_TO_IDX(addr - vma->vm_start);
if (vma->vm_pfn_count == 0)
vma->vm_pfn_first = pindex;
MPASS(pindex <= OFF_TO_IDX(vma->vm_end));
retry:
- page = vm_page_grab(vm_obj, pindex, VM_ALLOC_NOCREAT);
+ page = vm_page_grab_iter(vm_obj, pindex, VM_ALLOC_NOCREAT, &pages);
if (page == NULL) {
page = PHYS_TO_VM_PAGE(IDX_TO_OFF(pfn));
- if (!vm_page_busy_acquire(page, VM_ALLOC_WAITFAIL))
+ if (!vm_page_busy_acquire(page, VM_ALLOC_WAITFAIL)) {
+ pctrie_iter_reset(&pages);
goto retry;
+ }
if (page->object != NULL) {
tmp_obj = page->object;
vm_page_xunbusy(page);
@@ -336,10 +340,11 @@ retry:
vm_page_remove(page);
}
VM_OBJECT_WUNLOCK(tmp_obj);
+ pctrie_iter_reset(&pages);
VM_OBJECT_WLOCK(vm_obj);
goto retry;
}
- if (vm_page_insert(page, vm_obj, pindex)) {
+ if (vm_page_iter_insert(page, vm_obj, pindex, &pages) != 0) {
vm_page_xunbusy(page);
return (VM_FAULT_OOM);
}
diff --git a/sys/dev/drm2/ttm/ttm_bo_vm.c b/sys/dev/drm2/ttm/ttm_bo_vm.c
index 38575fc8a74b..b10c5f87a394 100644
--- a/sys/dev/drm2/ttm/ttm_bo_vm.c
+++ b/sys/dev/drm2/ttm/ttm_bo_vm.c
@@ -100,7 +100,7 @@ static int
ttm_bo_vm_fault(vm_object_t vm_obj, vm_ooffset_t offset,
int prot, vm_page_t *mres)
{
-
+ struct pctrie_iter pages;
struct ttm_buffer_object *bo = vm_obj->handle;
struct ttm_bo_device *bdev = bo->bdev;
struct ttm_tt *ttm = NULL;
@@ -114,6 +114,7 @@ ttm_bo_vm_fault(vm_object_t vm_obj, vm_ooffset_t offset,
if (*mres != NULL) {
(void)vm_page_remove(*mres);
}
+ vm_page_iter_init(&pages, vm_obj);
retry:
VM_OBJECT_WUNLOCK(vm_obj);
m = NULL;
@@ -234,10 +235,12 @@ reserve:
ttm_bo_unreserve(bo);
goto retry;
}
- m1 = vm_page_lookup(vm_obj, OFF_TO_IDX(offset));
+ pctrie_iter_reset(&pages);
+ m1 = vm_radix_iter_lookup(&pages, OFF_TO_IDX(offset));
/* XXX This looks like it should just be vm_page_replace? */
if (m1 == NULL) {
- if (vm_page_insert(m, vm_obj, OFF_TO_IDX(offset))) {
+ if (vm_page_iter_insert(
+ m, vm_obj, OFF_TO_IDX(offset), &pages) != 0) {
vm_page_xunbusy(m);
VM_OBJECT_WUNLOCK(vm_obj);
vm_wait(vm_obj);
diff --git a/sys/vm/vm_page.c b/sys/vm/vm_page.c
index 9ef9da737ded..8e32da5ea610 100644
--- a/sys/vm/vm_page.c
+++ b/sys/vm/vm_page.c
@@ -1532,7 +1532,7 @@ vm_page_insert(vm_page_t m, vm_object_t object, vm_pindex_t pindex)
*
* The object must be locked.
*/
-static int
+int
vm_page_iter_insert(vm_page_t m, vm_object_t object, vm_pindex_t pindex,
struct pctrie_iter *pages)
{
diff --git a/sys/vm/vm_page.h b/sys/vm/vm_page.h
index 3d7257644ef6..6c1826bc68e6 100644
--- a/sys/vm/vm_page.h
+++ b/sys/vm/vm_page.h
@@ -653,6 +653,8 @@ int vm_page_insert (vm_page_t, vm_object_t, vm_pindex_t);
void vm_page_invalid(vm_page_t m);
void vm_page_iter_free(struct pctrie_iter *pages, vm_page_t m);
void vm_page_iter_init(struct pctrie_iter *, vm_object_t);
+int vm_page_iter_insert(vm_page_t m, vm_object_t, vm_pindex_t,
+ struct pctrie_iter *);
void vm_page_iter_limit_init(struct pctrie_iter *, vm_object_t, vm_pindex_t);
bool vm_page_iter_remove(struct pctrie_iter *pages, vm_page_t m);
bool vm_page_iter_rename(struct pctrie_iter *old_pages, vm_page_t m,