git: 976de6053426 - stable/15 - arm64 pmap: optimize TLB management by pmap_update_entry()

The branch stable/15 has been updated by alc:

URL: https://cgit.FreeBSD.org/src/commit/?id=976de6053426249753a4a741dbd382f63cd7daf6

commit 976de6053426249753a4a741dbd382f63cd7daf6
Author:     Alan Cox <alc@FreeBSD.org>
AuthorDate: 2026-08-15 17:43:43 +0000
Commit:     Alan Cox <alc@FreeBSD.org>
CommitDate: 2026-10-18 18:38:32 +0000

    arm64 pmap: optimize TLB management by pmap_update_entry()
    
    To date, pmap_update_entry() has unconditionally passed false as
    final_only to pmap_s1_invalidate_range().  Passing false means that we
    invalidate the intermediate "page walk cache" entries in the TLB as well
    as the leaf that is being replaced. However, invalidating intermediate
    entries is only necessary when doing a superpage promotion that replaces
    a pointer to a page table page by a large page mapping.
    
    (cherry picked from commit b2e6b6545ee6377a1c8bc2b0a5bcc4cb605a8283)
---
 sys/arm64/arm64/pmap.c | 30 +++++++++++++++++-------------
 1 file changed, 17 insertions(+), 13 deletions(-)

diff --git a/sys/arm64/arm64/pmap.c b/sys/arm64/arm64/pmap.c
index 753184574c09..2124d41c2fe4 100644
--- a/sys/arm64/arm64/pmap.c
+++ b/sys/arm64/arm64/pmap.c
@@ -519,7 +519,7 @@ static void _pmap_unwire_l3(pmap_t pmap, vm_offset_t va, vm_page_t m,
     struct spglist *free);
 static int pmap_unuse_pt(pmap_t, vm_offset_t, pd_entry_t, struct spglist *);
 static void pmap_update_entry(pmap_t pmap, pd_entry_t *pte, pd_entry_t newpte,
-    vm_offset_t va, vm_size_t size);
+    vm_offset_t va, vm_size_t size, bool final_only);
 static __inline vm_page_t pmap_remove_pt_page(pmap_t pmap, vm_offset_t va);
 
 static uma_zone_t pmap_bti_ranges_zone;
@@ -2436,7 +2436,7 @@ pmap_kenter(vm_offset_t sva, vm_size_t size, vm_paddr_t pa, int mode)
 				 */
 				pmap_update_entry(kernel_pmap, pde,
 				    PHYS_TO_PTE(pa) | attr | L2_BLOCK, va,
-				    PAGE_SIZE);
+				    PAGE_SIZE, false);
 			}
 			PMAP_UNLOCK(kernel_pmap);
 			if (error == 0) {
@@ -4866,11 +4866,15 @@ pmap_remove_pt_page(pmap_t pmap, vm_offset_t va)
 /*
  * Performs a break-before-make update of a pmap entry. This is needed when
  * either promoting or demoting pages to ensure the TLB doesn't get into an
- * inconsistent state.
+ * inconsistent state.  The caller must pass false for "final_only" when
+ * promoting, because the TLB might be caching an intermediate entry that
+ * references the L{1,2}_TABLE that is being replaced.  In contrast, when
+ * demoting or the PTE's type isn't changing, no cached intermediate entry
+ * needs to change, so the caller should pass true as an optimization.
  */
-static void
+static __always_inline void
 pmap_update_entry(pmap_t pmap, pd_entry_t *ptep, pd_entry_t newpte,
-    vm_offset_t va, vm_size_t size)
+    vm_offset_t va, vm_size_t size, bool final_only)
 {
 	register_t intr;
 
@@ -4893,11 +4897,10 @@ pmap_update_entry(pmap_t pmap, pd_entry_t *ptep, pd_entry_t newpte,
 	pmap_clear_bits(ptep, ATTR_DESCR_VALID);
 
 	/*
-	 * When promoting, the L{1,2}_TABLE entry that is being replaced might
-	 * be cached, so we invalidate intermediate entries as well as final
-	 * entries.
+	 * We always inline pmap_update_entry() so that constant propagation
+	 * and dead code elimination will specialize the following code.
 	 */
-	pmap_s1_invalidate_range(pmap, va, va + size, false);
+	pmap_s1_invalidate_range(pmap, va, va + size, final_only);
 
 	/* Create the new mapping */
 	pmap_store(ptep, newpte);
@@ -5131,7 +5134,8 @@ setl3:
 	if ((newl2 & ATTR_SW_MANAGED) != 0)
 		pmap_pv_promote_l2(pmap, va, PTE_TO_PHYS(newl2), lockp);
 
-	pmap_update_entry(pmap, l2, newl2 | L2_BLOCK, va & ~L2_OFFSET, L2_SIZE);
+	pmap_update_entry(pmap, l2, newl2 | L2_BLOCK, va & ~L2_OFFSET, L2_SIZE,
+	    false);
 
 	counter_u64_add(pmap_l2_promotions, 1);
 	CTR2(KTR_PMAP, "pmap_promote_l2: success for va %#lx in pmap %p", va,
@@ -8435,7 +8439,7 @@ pmap_change_props_locked(vm_offset_t va, vm_size_t size, vm_prot_t prot,
 				 * performed.
 				 */
 				pmap_update_entry(kernel_pmap, ptep, pte, tmpva,
-				    PAGE_SIZE);
+				    PAGE_SIZE, true);
 				break;
 			}
 
@@ -8528,7 +8532,7 @@ pmap_demote_l1(pmap_t pmap, pt_entry_t *l1, vm_offset_t va)
 		l1 = (pt_entry_t *)(tmpl1 + ((vm_offset_t)l1 & PAGE_MASK));
 	}
 
-	pmap_update_entry(pmap, l1, l2phys | L1_TABLE, va, PAGE_SIZE);
+	pmap_update_entry(pmap, l1, l2phys | L1_TABLE, va, PAGE_SIZE, true);
 
 	counter_u64_add(pmap_l1_demotions, 1);
 fail:
@@ -8738,7 +8742,7 @@ pmap_demote_l2_locked(pmap_t pmap, pt_entry_t *l2, vm_offset_t va,
 	 * Pass PAGE_SIZE so that a single TLB invalidation is performed on
 	 * the 2MB page mapping.
 	 */
-	pmap_update_entry(pmap, l2, l3phys | L2_TABLE, va, PAGE_SIZE);
+	pmap_update_entry(pmap, l2, l3phys | L2_TABLE, va, PAGE_SIZE, true);
 
 	/*
 	 * Demote the PV entry.