git: 5b8c01d13a09 - main - amd64 pmap: Optimize PKU lookups when creating superpage mappings

The branch main has been updated by alc:

URL: https://cgit.FreeBSD.org/src/commit/?id=5b8c01d13a0970b11f47503fcd627d249a6e638a

commit 5b8c01d13a0970b11f47503fcd627d249a6e638a
Author:     Alan Cox <alc@FreeBSD.org>
AuthorDate: 2024-07-25 06:57:53 +0000
Commit:     Alan Cox <alc@FreeBSD.org>
CommitDate: 2024-07-26 05:38:46 +0000

    amd64 pmap: Optimize PKU lookups when creating superpage mappings
    
    Modify pmap_pkru_same() to update the prototype PTE at the same time as
    checking the address range.  This eliminates the need for calling
    pmap_pkru_get() in addition to pmap_pkru_same().  pmap_pkru_same() was
    already doing most of the work of pmap_pkru_get().
    
    Reviewed by:    kib
    Differential Revision:  https://reviews.freebsd.org/D46135
---
 sys/amd64/amd64/pmap.c | 29 +++++++++++++++++------------
 1 file changed, 17 insertions(+), 12 deletions(-)

diff --git a/sys/amd64/amd64/pmap.c b/sys/amd64/amd64/pmap.c
index dcf9b4f5a4f3..778d07689ff0 100644
--- a/sys/amd64/amd64/pmap.c
+++ b/sys/amd64/amd64/pmap.c
@@ -573,7 +573,8 @@ struct pmap_pkru_range {
 };
 
 static uma_zone_t pmap_pkru_ranges_zone;
-static bool pmap_pkru_same(pmap_t pmap, vm_offset_t sva, vm_offset_t eva);
+static bool pmap_pkru_same(pmap_t pmap, vm_offset_t sva, vm_offset_t eva,
+    pt_entry_t *pte);
 static pt_entry_t pmap_pkru_get(pmap_t pmap, vm_offset_t va);
 static void pmap_pkru_on_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva);
 static void *pkru_dup_range(void *ctx, void *data);
@@ -7071,11 +7072,9 @@ pmap_enter_largepage(pmap_t pmap, vm_offset_t va, pt_entry_t newpte, int flags,
 	PG_V = pmap_valid_bit(pmap);
 
 restart:
-	if (!pmap_pkru_same(pmap, va, va + pagesizes[psind]))
-		return (KERN_PROTECTION_FAILURE);
 	pten = newpte;
-	if (va < VM_MAXUSER_ADDRESS && pmap->pm_type == PT_X86)
-		pten |= pmap_pkru_get(pmap, va);
+	if (!pmap_pkru_same(pmap, va, va + pagesizes[psind], &pten))
+		return (KERN_PROTECTION_FAILURE);
 
 	if (psind == 2) {	/* 1G */
 		pml4e = pmap_pml4e(pmap, va);
@@ -7529,14 +7528,10 @@ pmap_enter_pde(pmap_t pmap, vm_offset_t va, pd_entry_t newpde, u_int flags,
 	 * and let vm_fault() cope.  Check after pde allocation, since
 	 * it could sleep.
 	 */
-	if (!pmap_pkru_same(pmap, va, va + NBPDR)) {
+	if (!pmap_pkru_same(pmap, va, va + NBPDR, &newpde)) {
 		pmap_abort_ptp(pmap, va, pdpg);
 		return (KERN_PROTECTION_FAILURE);
 	}
-	if (va < VM_MAXUSER_ADDRESS && pmap->pm_type == PT_X86) {
-		newpde &= ~X86_PG_PKU_MASK;
-		newpde |= pmap_pkru_get(pmap, va);
-	}
 
 	/*
 	 * If there are existing mappings, either abort or remove them.
@@ -11460,13 +11455,21 @@ pmap_pkru_deassign_all(pmap_t pmap)
 		rangeset_remove_all(&pmap->pm_pkru);
 }
 
+/*
+ * Returns true if the PKU setting is the same across the specified address
+ * range, and false otherwise.  When returning true, updates the referenced PTE
+ * to reflect the PKU setting.
+ */
 static bool
-pmap_pkru_same(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
+pmap_pkru_same(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, pt_entry_t *pte)
 {
 	struct pmap_pkru_range *next_ppr, *ppr;
 	vm_offset_t va;
+	u_int keyidx;
 
 	PMAP_LOCK_ASSERT(pmap, MA_OWNED);
+	KASSERT(pmap->pm_type != PT_X86 || (*pte & X86_PG_PKU_MASK) == 0,
+	    ("pte %p has unexpected PKU %ld", pte, *pte & X86_PG_PKU_MASK));
 	if (pmap->pm_type != PT_X86 ||
 	    (cpu_stdext_feature2 & CPUID_STDEXT2_PKU) == 0 ||
 	    sva >= VM_MAXUSER_ADDRESS)
@@ -11478,14 +11481,16 @@ pmap_pkru_same(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
 		return (ppr == NULL ||
 		    ppr->pkru_rs_el.re_start >= eva);
 	}
+	keyidx = ppr->pkru_keyidx;
 	while ((va = ppr->pkru_rs_el.re_end) < eva) {
 		next_ppr = rangeset_next(&pmap->pm_pkru, va);
 		if (next_ppr == NULL ||
 		    va != next_ppr->pkru_rs_el.re_start ||
-		    ppr->pkru_keyidx != next_ppr->pkru_keyidx)
+		    keyidx != next_ppr->pkru_keyidx)
 			return (false);
 		ppr = next_ppr;
 	}
+	*pte |= X86_PG_PKU(keyidx);
 	return (true);
 }