git: ad0cd0101590 - main - arm64 pmap: Eliminate page table rewalks after demotions

From: Alan Cox <alc_at_FreeBSD.org>
Date: Thu, 19 Nov 2026 06:42:09 UTC
The branch main has been updated by alc:

URL: https://cgit.FreeBSD.org/src/commit/?id=ad0cd01015906ce421cd2c3b7251a0f615bc0a20

commit ad0cd01015906ce421cd2c3b7251a0f615bc0a20
Author:     Alan Cox <alc@FreeBSD.org>
AuthorDate: 2026-11-09 17:20:02 +0000
Commit:     Alan Cox <alc@FreeBSD.org>
CommitDate: 2026-11-19 06:40:23 +0000

    arm64 pmap: Eliminate page table rewalks after demotions
    
    pmap_demote_l1() and pmap_demote_l2_locked() return the virtual address
    of the page table page that replaces the demoted superpage mapping.  In
    pmap_advise(), pmap_change_props_locked(), and pmap_clear_modify() use
    that address instead of rewalking the page table from the L1 or L2
    entry.  Similarly, after pmap_advise() demotes an L3C superpage, compute
    the virtual address of the L3 entry that it removes from the virtual
    address of the current L3 entry.
    
    Reviewed by:    kib
    MFC after:      1 week
    Differential Revision:  https://reviews.freebsd.org/D61599
---
 sys/arm64/arm64/pmap.c | 30 +++++++++++++++++-------------
 1 file changed, 17 insertions(+), 13 deletions(-)

diff --git a/sys/arm64/arm64/pmap.c b/sys/arm64/arm64/pmap.c
index c4435d7dd3eb..7340627e6721 100644
--- a/sys/arm64/arm64/pmap.c
+++ b/sys/arm64/arm64/pmap.c
@@ -8242,7 +8242,8 @@ pmap_advise(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, int advice)
 			if ((oldl2 & ATTR_SW_MANAGED) == 0)
 				continue;
 			lock = NULL;
-			if (!pmap_demote_l2_locked(pmap, l2, sva, &lock)) {
+			l3 = pmap_demote_l2_locked(pmap, l2, sva, &lock);
+			if (l3 == NULL) {
 				if (lock != NULL)
 					rw_wunlock(lock);
 
@@ -8268,7 +8269,7 @@ pmap_advise(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, int advice)
 				va -= PAGE_SIZE;
 				KASSERT(va >= sva,
 				    ("pmap_advise: no address gap"));
-				l3 = pmap_l2_to_l3(l2, va);
+				l3 = &l3[pmap_l3_index(va)];
 				KASSERT(pmap_load(l3) != 0,
 				    ("pmap_advise: invalid PTE"));
 				pmap_remove_l3(pmap, l3, va, pmap_load(l2),
@@ -8314,10 +8315,14 @@ pmap_advise(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, int advice)
 					 * repromotion trigger.
 					 */
                                         if ((oldl3 & ATTR_SW_WIRED) == 0) {
+						/*
+						 * sva and l3 may not be at the
+						 * start of the L3C superpage.
+						 */
 						dva = MIN((sva & ~L3C_OFFSET) +
 						    L3C_SIZE - PAGE_SIZE,
 						    va_next - PAGE_SIZE);
-						dl3 = pmap_l2_to_l3(l2, dva);
+						dl3 = &l3[atop(dva - sva)];
 						KASSERT(pmap_load(dl3) != 0,
 						    ("pmap_advise: invalid PTE"));
 						lock = NULL;
@@ -8410,14 +8415,14 @@ restart:
 		oldl2 = pmap_load(l2);
 		/* If oldl2 has ATTR_SW_DBM set, then it is also dirty. */
 		if ((oldl2 & ATTR_SW_DBM) != 0 &&
-		    pmap_demote_l2_locked(pmap, l2, va, &lock) &&
+		    (l3 = pmap_demote_l2_locked(pmap, l2, va, &lock)) != NULL &&
 		    (oldl2 & ATTR_SW_WIRED) == 0) {
 			/*
 			 * Write protect the mapping to a single page so that
 			 * a subsequent write access may repromote.
 			 */
 			va += VM_PAGE_TO_PHYS(m) - PTE_TO_PHYS(oldl2);
-			l3 = pmap_l2_to_l3(l2, va);
+			l3 = &l3[pmap_l3_index(va)];
 			if ((pmap_load(l3) & ATTR_CONTIGUOUS) != 0)
 				(void)pmap_demote_l3c(pmap, l3, va);
 			oldl3 = pmap_load(l3);
@@ -8760,7 +8765,7 @@ pmap_change_props_locked(void *addr, vm_size_t size, vm_prot_t prot,
 	vm_offset_t base, offset, tmpva, va;
 	vm_size_t pte_size;
 	vm_paddr_t pa;
-	pt_entry_t pte, *ptep, *newpte;
+	pt_entry_t pte, *ptep;
 	pt_entry_t bits, mask, old_mode_bits, old_mode_mask;
 	char *tmpptep;
 	int lvl, rv;
@@ -8857,11 +8862,11 @@ pmap_change_props_locked(void *addr, vm_size_t size, vm_prot_t prot,
 					pte_size = L1_SIZE;
 					break;
 				}
-				newpte = pmap_demote_l1(kernel_pmap, ptep,
+				ptep = pmap_demote_l1(kernel_pmap, ptep,
 				    tmpva & ~L1_OFFSET);
-				if (newpte == NULL)
+				if (ptep == NULL)
 					return (EINVAL);
-				ptep = pmap_l1_to_l2(ptep, tmpva);
+				ptep = &ptep[pmap_l2_index(tmpva)];
 				/* FALLTHROUGH */
 			case 2:
 				if ((pmap_load(ptep) & ATTR_CONTIGUOUS) != 0) {
@@ -8879,11 +8884,10 @@ pmap_change_props_locked(void *addr, vm_size_t size, vm_prot_t prot,
 					pte_size = L2_SIZE;
 					break;
 				}
-				newpte = pmap_demote_l2(kernel_pmap, ptep,
-				    tmpva);
-				if (newpte == NULL)
+				ptep = pmap_demote_l2(kernel_pmap, ptep, tmpva);
+				if (ptep == NULL)
 					return (EINVAL);
-				ptep = pmap_l2_to_l3(ptep, tmpva);
+				ptep = &ptep[pmap_l3_index(tmpva)];
 				/* FALLTHROUGH */
 			case 3:
 				if ((pmap_load(ptep) & ATTR_CONTIGUOUS) != 0) {