git: ad0cd0101590 - main - arm64 pmap: Eliminate page table rewalks after demotions
- Go to: [ bottom of page ] [ top of archives ] [ this month ]
Date: Thu, 19 Nov 2026 06:42:09 UTC
The branch main has been updated by alc:
URL: https://cgit.FreeBSD.org/src/commit/?id=ad0cd01015906ce421cd2c3b7251a0f615bc0a20
commit ad0cd01015906ce421cd2c3b7251a0f615bc0a20
Author: Alan Cox <alc@FreeBSD.org>
AuthorDate: 2026-11-09 17:20:02 +0000
Commit: Alan Cox <alc@FreeBSD.org>
CommitDate: 2026-11-19 06:40:23 +0000
arm64 pmap: Eliminate page table rewalks after demotions
pmap_demote_l1() and pmap_demote_l2_locked() return the virtual address
of the page table page that replaces the demoted superpage mapping. In
pmap_advise(), pmap_change_props_locked(), and pmap_clear_modify() use
that address instead of rewalking the page table from the L1 or L2
entry. Similarly, after pmap_advise() demotes an L3C superpage, compute
the virtual address of the L3 entry that it removes from the virtual
address of the current L3 entry.
Reviewed by: kib
MFC after: 1 week
Differential Revision: https://reviews.freebsd.org/D61599
---
sys/arm64/arm64/pmap.c | 30 +++++++++++++++++-------------
1 file changed, 17 insertions(+), 13 deletions(-)
diff --git a/sys/arm64/arm64/pmap.c b/sys/arm64/arm64/pmap.c
index c4435d7dd3eb..7340627e6721 100644
--- a/sys/arm64/arm64/pmap.c
+++ b/sys/arm64/arm64/pmap.c
@@ -8242,7 +8242,8 @@ pmap_advise(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, int advice)
if ((oldl2 & ATTR_SW_MANAGED) == 0)
continue;
lock = NULL;
- if (!pmap_demote_l2_locked(pmap, l2, sva, &lock)) {
+ l3 = pmap_demote_l2_locked(pmap, l2, sva, &lock);
+ if (l3 == NULL) {
if (lock != NULL)
rw_wunlock(lock);
@@ -8268,7 +8269,7 @@ pmap_advise(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, int advice)
va -= PAGE_SIZE;
KASSERT(va >= sva,
("pmap_advise: no address gap"));
- l3 = pmap_l2_to_l3(l2, va);
+ l3 = &l3[pmap_l3_index(va)];
KASSERT(pmap_load(l3) != 0,
("pmap_advise: invalid PTE"));
pmap_remove_l3(pmap, l3, va, pmap_load(l2),
@@ -8314,10 +8315,14 @@ pmap_advise(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, int advice)
* repromotion trigger.
*/
if ((oldl3 & ATTR_SW_WIRED) == 0) {
+ /*
+ * sva and l3 may not be at the
+ * start of the L3C superpage.
+ */
dva = MIN((sva & ~L3C_OFFSET) +
L3C_SIZE - PAGE_SIZE,
va_next - PAGE_SIZE);
- dl3 = pmap_l2_to_l3(l2, dva);
+ dl3 = &l3[atop(dva - sva)];
KASSERT(pmap_load(dl3) != 0,
("pmap_advise: invalid PTE"));
lock = NULL;
@@ -8410,14 +8415,14 @@ restart:
oldl2 = pmap_load(l2);
/* If oldl2 has ATTR_SW_DBM set, then it is also dirty. */
if ((oldl2 & ATTR_SW_DBM) != 0 &&
- pmap_demote_l2_locked(pmap, l2, va, &lock) &&
+ (l3 = pmap_demote_l2_locked(pmap, l2, va, &lock)) != NULL &&
(oldl2 & ATTR_SW_WIRED) == 0) {
/*
* Write protect the mapping to a single page so that
* a subsequent write access may repromote.
*/
va += VM_PAGE_TO_PHYS(m) - PTE_TO_PHYS(oldl2);
- l3 = pmap_l2_to_l3(l2, va);
+ l3 = &l3[pmap_l3_index(va)];
if ((pmap_load(l3) & ATTR_CONTIGUOUS) != 0)
(void)pmap_demote_l3c(pmap, l3, va);
oldl3 = pmap_load(l3);
@@ -8760,7 +8765,7 @@ pmap_change_props_locked(void *addr, vm_size_t size, vm_prot_t prot,
vm_offset_t base, offset, tmpva, va;
vm_size_t pte_size;
vm_paddr_t pa;
- pt_entry_t pte, *ptep, *newpte;
+ pt_entry_t pte, *ptep;
pt_entry_t bits, mask, old_mode_bits, old_mode_mask;
char *tmpptep;
int lvl, rv;
@@ -8857,11 +8862,11 @@ pmap_change_props_locked(void *addr, vm_size_t size, vm_prot_t prot,
pte_size = L1_SIZE;
break;
}
- newpte = pmap_demote_l1(kernel_pmap, ptep,
+ ptep = pmap_demote_l1(kernel_pmap, ptep,
tmpva & ~L1_OFFSET);
- if (newpte == NULL)
+ if (ptep == NULL)
return (EINVAL);
- ptep = pmap_l1_to_l2(ptep, tmpva);
+ ptep = &ptep[pmap_l2_index(tmpva)];
/* FALLTHROUGH */
case 2:
if ((pmap_load(ptep) & ATTR_CONTIGUOUS) != 0) {
@@ -8879,11 +8884,10 @@ pmap_change_props_locked(void *addr, vm_size_t size, vm_prot_t prot,
pte_size = L2_SIZE;
break;
}
- newpte = pmap_demote_l2(kernel_pmap, ptep,
- tmpva);
- if (newpte == NULL)
+ ptep = pmap_demote_l2(kernel_pmap, ptep, tmpva);
+ if (ptep == NULL)
return (EINVAL);
- ptep = pmap_l2_to_l3(ptep, tmpva);
+ ptep = &ptep[pmap_l3_index(tmpva)];
/* FALLTHROUGH */
case 3:
if ((pmap_load(ptep) & ATTR_CONTIGUOUS) != 0) {