git: bcd85e0150c9 - main - vm_page: make iter_insert() public

From: Doug Moore <dougm_at_FreeBSD.org>
Date: Fri, 09 May 2025 04:14:36 UTC
The branch main has been updated by dougm:

URL: https://cgit.FreeBSD.org/src/commit/?id=bcd85e0150c903848f6e0d54f5428931c2f2d575

commit bcd85e0150c903848f6e0d54f5428931c2f2d575
Author:     Doug Moore <dougm@FreeBSD.org>
AuthorDate: 2025-05-09 04:13:32 +0000
Commit:     Doug Moore <dougm@FreeBSD.org>
CommitDate: 2025-05-09 04:13:32 +0000

    vm_page: make iter_insert() public
    
    In places where vm_page_insert() is used after lookups, or for
    consecutive pages, use vm_page_iter_insert instead, to exploit
    locality.
    
    Reviewed by:    kib, markj
    Differential Revision:  https://reviews.freebsd.org/D50248
---
 sys/amd64/sgx/sgx.c                         | 62 ++++++++++++++---------------
 sys/arm/nvidia/drm2/tegra_bo.c              |  4 +-
 sys/compat/linuxkpi/common/src/linux_page.c | 11 +++--
 sys/dev/drm2/ttm/ttm_bo_vm.c                |  9 +++--
 sys/vm/vm_page.c                            |  2 +-
 sys/vm/vm_page.h                            |  2 +
 6 files changed, 50 insertions(+), 40 deletions(-)

diff --git a/sys/amd64/sgx/sgx.c b/sys/amd64/sgx/sgx.c
index af8adb763214..60209b04f31f 100644
--- a/sys/amd64/sgx/sgx.c
+++ b/sys/amd64/sgx/sgx.c
@@ -191,9 +191,20 @@ sgx_put_epc_page(struct sgx_softc *sc, struct epc_page *epc)
 	vmem_free(sc->vmem_epc, addr, PAGE_SIZE);
 }
 
+static void
+sgx_insert_epc_page_by_index(vm_page_t page, vm_object_t object,
+    vm_pindex_t pidx, struct pctrie_iter *pages)
+{
+
+	VM_OBJECT_ASSERT_WLOCKED(object);
+
+	page->valid = VM_PAGE_BITS_ALL;
+	vm_page_iter_insert(page, object, pidx, pages);
+}
+
 static int
 sgx_va_slot_init_by_index(struct sgx_softc *sc, vm_object_t object,
-    uint64_t idx)
+    uint64_t idx, struct pctrie_iter *pages)
 {
 	struct epc_page *epc;
 	vm_page_t page;
@@ -202,7 +213,7 @@ sgx_va_slot_init_by_index(struct sgx_softc *sc, vm_object_t object,
 
 	VM_OBJECT_ASSERT_WLOCKED(object);
 
-	p = vm_page_lookup(object, idx);
+	p = vm_radix_iter_lookup(pages, idx);
 	if (p == NULL) {
 		ret = sgx_get_epc_page(sc, &epc);
 		if (ret) {
@@ -216,20 +227,16 @@ sgx_va_slot_init_by_index(struct sgx_softc *sc, vm_object_t object,
 		mtx_unlock(&sc->mtx_encls);
 
 		page = PHYS_TO_VM_PAGE(epc->phys);
-
-		page->valid = VM_PAGE_BITS_ALL;
-		vm_page_insert(page, object, idx);
+		sgx_insert_epc_page_by_index(page, object, idx, pages);
 	}
 
 	return (0);
 }
 
 static int
-sgx_va_slot_init(struct sgx_softc *sc,
-    struct sgx_enclave *enclave,
-    uint64_t addr)
+sgx_va_slot_init(struct sgx_softc *sc, struct sgx_enclave *enclave,
+    vm_pindex_t pidx, struct pctrie_iter *pages)
 {
-	vm_pindex_t pidx;
 	uint64_t va_page_idx;
 	uint64_t idx;
 	vm_object_t object;
@@ -239,12 +246,10 @@ sgx_va_slot_init(struct sgx_softc *sc,
 
 	VM_OBJECT_ASSERT_WLOCKED(object);
 
-	pidx = OFF_TO_IDX(addr);
-
 	va_page_idx = pidx / SGX_VA_PAGE_SLOTS;
 	idx = - SGX_VA_PAGES_OFFS - va_page_idx;
 
-	ret = sgx_va_slot_init_by_index(sc, object, idx);
+	ret = sgx_va_slot_init_by_index(sc, object, idx, pages);
 
 	return (ret);
 }
@@ -614,19 +619,8 @@ static struct cdev_pager_ops sgx_pg_ops = {
 };
 
 static void
-sgx_insert_epc_page_by_index(vm_page_t page, vm_object_t object,
-    vm_pindex_t pidx)
-{
-
-	VM_OBJECT_ASSERT_WLOCKED(object);
-
-	page->valid = VM_PAGE_BITS_ALL;
-	vm_page_insert(page, object, pidx);
-}
-
-static void
-sgx_insert_epc_page(struct sgx_enclave *enclave,
-    struct epc_page *epc, uint64_t addr)
+sgx_insert_epc_page(struct sgx_enclave *enclave, struct epc_page *epc,
+    uint64_t addr, struct pctrie_iter *pages)
 {
 	vm_pindex_t pidx;
 	vm_page_t page;
@@ -636,12 +630,13 @@ sgx_insert_epc_page(struct sgx_enclave *enclave,
 	pidx = OFF_TO_IDX(addr);
 	page = PHYS_TO_VM_PAGE(epc->phys);
 
-	sgx_insert_epc_page_by_index(page, enclave->object, pidx);
+	sgx_insert_epc_page_by_index(page, enclave->object, pidx, pages);
 }
 
 static int
 sgx_ioctl_create(struct sgx_softc *sc, struct sgx_enclave_create *param)
 {
+	struct pctrie_iter pages;
 	struct sgx_vm_handle *vmh;
 	vm_map_entry_t entry;
 	vm_page_t p;
@@ -712,8 +707,9 @@ sgx_ioctl_create(struct sgx_softc *sc, struct sgx_enclave_create *param)
 	}
 	enclave->secs_epc_page = epc;
 
+	vm_page_iter_init(&pages, object);
 	VM_OBJECT_WLOCK(object);
-	p = vm_page_lookup(object, SGX_SECS_VM_OBJECT_INDEX);
+	p = vm_radix_iter_lookup(&pages, SGX_SECS_VM_OBJECT_INDEX);
 	if (p) {
 		VM_OBJECT_WUNLOCK(object);
 		/* SECS page already added. */
@@ -722,7 +718,7 @@ sgx_ioctl_create(struct sgx_softc *sc, struct sgx_enclave_create *param)
 	}
 
 	ret = sgx_va_slot_init_by_index(sc, object,
-	    - SGX_VA_PAGES_OFFS - SGX_SECS_VM_OBJECT_INDEX);
+	    - SGX_VA_PAGES_OFFS - SGX_SECS_VM_OBJECT_INDEX, &pages);
 	if (ret) {
 		VM_OBJECT_WUNLOCK(object);
 		dprintf("%s: Can't init va slot.\n", __func__);
@@ -762,7 +758,7 @@ sgx_ioctl_create(struct sgx_softc *sc, struct sgx_enclave_create *param)
 
 	page = PHYS_TO_VM_PAGE(epc->phys);
 	sgx_insert_epc_page_by_index(page, enclave->object,
-	    SGX_SECS_VM_OBJECT_INDEX);
+	    SGX_SECS_VM_OBJECT_INDEX, &pages);
 
 	VM_OBJECT_WUNLOCK(object);
 
@@ -786,6 +782,7 @@ static int
 sgx_ioctl_add_page(struct sgx_softc *sc,
     struct sgx_enclave_add_page *addp)
 {
+	struct pctrie_iter pages;
 	struct epc_page *secs_epc_page;
 	struct sgx_enclave *enclave;
 	struct sgx_vm_handle *vmh;
@@ -857,8 +854,9 @@ sgx_ioctl_add_page(struct sgx_softc *sc,
 	addr = (addp->addr - vmh->base);
 	pidx = OFF_TO_IDX(addr);
 
+	vm_page_iter_init(&pages, object);
 	VM_OBJECT_WLOCK(object);
-	p = vm_page_lookup(object, pidx);
+	p = vm_radix_iter_lookup(&pages, pidx);
 	if (p) {
 		VM_OBJECT_WUNLOCK(object);
 		/* Page already added. */
@@ -866,7 +864,7 @@ sgx_ioctl_add_page(struct sgx_softc *sc,
 		goto error;
 	}
 
-	ret = sgx_va_slot_init(sc, enclave, addr);
+	ret = sgx_va_slot_init(sc, enclave, pidx, &pages);
 	if (ret) {
 		VM_OBJECT_WUNLOCK(object);
 		dprintf("%s: Can't init va slot.\n", __func__);
@@ -898,7 +896,7 @@ sgx_ioctl_add_page(struct sgx_softc *sc,
 		goto error;
 	}
 
-	sgx_insert_epc_page(enclave, epc, addr);
+	sgx_insert_epc_page(enclave, epc, addr, &pages);
 
 	VM_OBJECT_WUNLOCK(object);
 
diff --git a/sys/arm/nvidia/drm2/tegra_bo.c b/sys/arm/nvidia/drm2/tegra_bo.c
index 8952d09eec3d..73ba82f2f883 100644
--- a/sys/arm/nvidia/drm2/tegra_bo.c
+++ b/sys/arm/nvidia/drm2/tegra_bo.c
@@ -133,6 +133,7 @@ retry:
 static int
 tegra_bo_init_pager(struct tegra_bo *bo)
 {
+	struct pctrie_iter pages;
 	vm_page_t m;
 	size_t size;
 	int i;
@@ -143,6 +144,7 @@ tegra_bo_init_pager(struct tegra_bo *bo)
 	if (vmem_alloc(kernel_arena, size, M_WAITOK | M_BESTFIT, &bo->vbase))
 		return (ENOMEM);
 
+	vm_page_iter_init(&pages, bo->cdev_pager);
 	VM_OBJECT_WLOCK(bo->cdev_pager);
 	for (i = 0; i < bo->npages; i++) {
 		m = bo->m[i];
@@ -159,7 +161,7 @@ tegra_bo_init_pager(struct tegra_bo *bo)
 		 */
 		m->oflags &= ~VPO_UNMANAGED;
 		m->flags |= PG_FICTITIOUS;
-		if (vm_page_insert(m, bo->cdev_pager, i) != 0)
+		if (vm_page_iter_insert(m, bo->cdev_pager, i, &pages) != 0)
 			return (EINVAL);
 	}
 	VM_OBJECT_WUNLOCK(bo->cdev_pager);
diff --git a/sys/compat/linuxkpi/common/src/linux_page.c b/sys/compat/linuxkpi/common/src/linux_page.c
index b5a0d34a6ad7..93aaea01f0df 100644
--- a/sys/compat/linuxkpi/common/src/linux_page.c
+++ b/sys/compat/linuxkpi/common/src/linux_page.c
@@ -296,23 +296,27 @@ vm_fault_t
 lkpi_vmf_insert_pfn_prot_locked(struct vm_area_struct *vma, unsigned long addr,
     unsigned long pfn, pgprot_t prot)
 {
+	struct pctrie_iter pages;
 	vm_object_t vm_obj = vma->vm_obj;
 	vm_object_t tmp_obj;
 	vm_page_t page;
 	vm_pindex_t pindex;
 
 	VM_OBJECT_ASSERT_WLOCKED(vm_obj);
+	vm_page_iter_init(&pages, vm_obj);
 	pindex = OFF_TO_IDX(addr - vma->vm_start);
 	if (vma->vm_pfn_count == 0)
 		vma->vm_pfn_first = pindex;
 	MPASS(pindex <= OFF_TO_IDX(vma->vm_end));
 
 retry:
-	page = vm_page_grab(vm_obj, pindex, VM_ALLOC_NOCREAT);
+	page = vm_page_grab_iter(vm_obj, pindex, VM_ALLOC_NOCREAT, &pages);
 	if (page == NULL) {
 		page = PHYS_TO_VM_PAGE(IDX_TO_OFF(pfn));
-		if (!vm_page_busy_acquire(page, VM_ALLOC_WAITFAIL))
+		if (!vm_page_busy_acquire(page, VM_ALLOC_WAITFAIL)) {
+			pctrie_iter_reset(&pages);
 			goto retry;
+		}
 		if (page->object != NULL) {
 			tmp_obj = page->object;
 			vm_page_xunbusy(page);
@@ -336,10 +340,11 @@ retry:
 				vm_page_remove(page);
 			}
 			VM_OBJECT_WUNLOCK(tmp_obj);
+			pctrie_iter_reset(&pages);
 			VM_OBJECT_WLOCK(vm_obj);
 			goto retry;
 		}
-		if (vm_page_insert(page, vm_obj, pindex)) {
+		if (vm_page_iter_insert(page, vm_obj, pindex, &pages) != 0) {
 			vm_page_xunbusy(page);
 			return (VM_FAULT_OOM);
 		}
diff --git a/sys/dev/drm2/ttm/ttm_bo_vm.c b/sys/dev/drm2/ttm/ttm_bo_vm.c
index 38575fc8a74b..b10c5f87a394 100644
--- a/sys/dev/drm2/ttm/ttm_bo_vm.c
+++ b/sys/dev/drm2/ttm/ttm_bo_vm.c
@@ -100,7 +100,7 @@ static int
 ttm_bo_vm_fault(vm_object_t vm_obj, vm_ooffset_t offset,
     int prot, vm_page_t *mres)
 {
-
+	struct pctrie_iter pages;
 	struct ttm_buffer_object *bo = vm_obj->handle;
 	struct ttm_bo_device *bdev = bo->bdev;
 	struct ttm_tt *ttm = NULL;
@@ -114,6 +114,7 @@ ttm_bo_vm_fault(vm_object_t vm_obj, vm_ooffset_t offset,
 	if (*mres != NULL) {
 		(void)vm_page_remove(*mres);
 	}
+	vm_page_iter_init(&pages, vm_obj);
 retry:
 	VM_OBJECT_WUNLOCK(vm_obj);
 	m = NULL;
@@ -234,10 +235,12 @@ reserve:
 		ttm_bo_unreserve(bo);
 		goto retry;
 	}
-	m1 = vm_page_lookup(vm_obj, OFF_TO_IDX(offset));
+	pctrie_iter_reset(&pages);
+	m1 = vm_radix_iter_lookup(&pages, OFF_TO_IDX(offset));
 	/* XXX This looks like it should just be vm_page_replace? */
 	if (m1 == NULL) {
-		if (vm_page_insert(m, vm_obj, OFF_TO_IDX(offset))) {
+		if (vm_page_iter_insert(
+		    m, vm_obj, OFF_TO_IDX(offset), &pages) != 0) {
 			vm_page_xunbusy(m);
 			VM_OBJECT_WUNLOCK(vm_obj);
 			vm_wait(vm_obj);
diff --git a/sys/vm/vm_page.c b/sys/vm/vm_page.c
index 9ef9da737ded..8e32da5ea610 100644
--- a/sys/vm/vm_page.c
+++ b/sys/vm/vm_page.c
@@ -1532,7 +1532,7 @@ vm_page_insert(vm_page_t m, vm_object_t object, vm_pindex_t pindex)
  *
  *	The object must be locked.
  */
-static int
+int
 vm_page_iter_insert(vm_page_t m, vm_object_t object, vm_pindex_t pindex,
     struct pctrie_iter *pages)
 {
diff --git a/sys/vm/vm_page.h b/sys/vm/vm_page.h
index 3d7257644ef6..6c1826bc68e6 100644
--- a/sys/vm/vm_page.h
+++ b/sys/vm/vm_page.h
@@ -653,6 +653,8 @@ int vm_page_insert (vm_page_t, vm_object_t, vm_pindex_t);
 void vm_page_invalid(vm_page_t m);
 void vm_page_iter_free(struct pctrie_iter *pages, vm_page_t m);
 void vm_page_iter_init(struct pctrie_iter *, vm_object_t);
+int vm_page_iter_insert(vm_page_t m, vm_object_t, vm_pindex_t,
+    struct pctrie_iter *);
 void vm_page_iter_limit_init(struct pctrie_iter *, vm_object_t, vm_pindex_t);
 bool vm_page_iter_remove(struct pctrie_iter *pages, vm_page_t m);
 bool vm_page_iter_rename(struct pctrie_iter *old_pages, vm_page_t m,