git: f1d73aacdc47 - main - pmap: Skip some superpage promotion attempts that will fail

From: Alan Cox <alc_at_FreeBSD.org>
Date: Tue, 04 Jun 2024 05:39:15 UTC
The branch main has been updated by alc:

URL: https://cgit.FreeBSD.org/src/commit/?id=f1d73aacdc47529310e2302094685295c032e28f

commit f1d73aacdc47529310e2302094685295c032e28f
Author:     Alan Cox <alc@FreeBSD.org>
AuthorDate: 2024-06-02 08:56:47 +0000
Commit:     Alan Cox <alc@FreeBSD.org>
CommitDate: 2024-06-04 05:38:05 +0000

    pmap: Skip some superpage promotion attempts that will fail
    
    Implement a simple heuristic to skip pointless promotion attempts by
    pmap_enter_quick_locked() and moea64_enter().  Specifically, when
    vm_fault() calls pmap_enter_quick() to map neighboring pages at the end
    of a copy-on-write fault, there is no point in attempting promotion in
    pmap_enter_quick_locked() and moea64_enter().  Promotion will fail
    because the base pages have differing protection.
    
    Reviewed by:    kib
    Differential Revision:  https://reviews.freebsd.org/D45431
    MFC after:      1 week
---
 sys/amd64/amd64/pmap.c      |  3 ++-
 sys/arm64/arm64/pmap.c      |  3 ++-
 sys/i386/i386/pmap.c        |  3 ++-
 sys/powerpc/aim/mmu_oea64.c |  9 +++++++--
 sys/riscv/riscv/pmap.c      |  3 ++-
 sys/vm/vm.h                 |  1 +
 sys/vm/vm_fault.c           | 11 ++++++++++-
 7 files changed, 26 insertions(+), 7 deletions(-)

diff --git a/sys/amd64/amd64/pmap.c b/sys/amd64/amd64/pmap.c
index 8105c9d92478..2f3119aede67 100644
--- a/sys/amd64/amd64/pmap.c
+++ b/sys/amd64/amd64/pmap.c
@@ -7818,7 +7818,8 @@ pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m,
 	 * If both the PTP and the reservation are fully populated, then
 	 * attempt promotion.
 	 */
-	if ((mpte == NULL || mpte->ref_count == NPTEPG) &&
+	if ((prot & VM_PROT_NO_PROMOTE) == 0 &&
+	    (mpte == NULL || mpte->ref_count == NPTEPG) &&
 	    (m->flags & PG_FICTITIOUS) == 0 &&
 	    vm_reserv_level_iffullpop(m) == 0) {
 		if (pde == NULL)
diff --git a/sys/arm64/arm64/pmap.c b/sys/arm64/arm64/pmap.c
index aaba6ca189a1..b6bc113ba8a4 100644
--- a/sys/arm64/arm64/pmap.c
+++ b/sys/arm64/arm64/pmap.c
@@ -6052,7 +6052,8 @@ pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m,
 	 * If both the PTP and the reservation are fully populated, then
 	 * attempt promotion.
 	 */
-	if ((mpte == NULL || mpte->ref_count == NL3PG) &&
+	if ((prot & VM_PROT_NO_PROMOTE) == 0 &&
+	    (mpte == NULL || mpte->ref_count == NL3PG) &&
 	    (m->flags & PG_FICTITIOUS) == 0 &&
 	    vm_reserv_level_iffullpop(m) == 0) {
 		if (l2 == NULL)
diff --git a/sys/i386/i386/pmap.c b/sys/i386/i386/pmap.c
index 40d8ceaf42b9..5808c31a99af 100644
--- a/sys/i386/i386/pmap.c
+++ b/sys/i386/i386/pmap.c
@@ -4250,7 +4250,8 @@ pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m,
 	 * If both the PTP and the reservation are fully populated, then
 	 * attempt promotion.
 	 */
-	if ((mpte == NULL || mpte->ref_count == NPTEPG) &&
+	if ((prot & VM_PROT_NO_PROMOTE) == 0 &&
+	    (mpte == NULL || mpte->ref_count == NPTEPG) &&
 	    (m->flags & PG_FICTITIOUS) == 0 &&
 	    vm_reserv_level_iffullpop(m) == 0) {
 		if (pde == NULL)
diff --git a/sys/powerpc/aim/mmu_oea64.c b/sys/powerpc/aim/mmu_oea64.c
index 391f90bb04eb..273dc38214e2 100644
--- a/sys/powerpc/aim/mmu_oea64.c
+++ b/sys/powerpc/aim/mmu_oea64.c
@@ -1755,10 +1755,14 @@ out:
 	 * If the VA of the entered page is not aligned with its PA,
 	 * don't try page promotion as it is not possible.
 	 * This reduces the number of promotion failures dramatically.
+	 *
+	 * Ignore VM_PROT_NO_PROMOTE unless PMAP_ENTER_QUICK_LOCKED.
 	 */
 	if (moea64_ps_enabled(pmap) && pmap != kernel_pmap && pvo != NULL &&
 	    (pvo->pvo_vaddr & PVO_MANAGED) != 0 &&
 	    (va & HPT_SP_MASK) == (pa & HPT_SP_MASK) &&
+	    ((prot & VM_PROT_NO_PROMOTE) == 0 ||
+	    (flags & PMAP_ENTER_QUICK_LOCKED) == 0) &&
 	    (m->flags & PG_FICTITIOUS) == 0 &&
 	    vm_reserv_level_iffullpop(m) == 0)
 		moea64_sp_promote(pmap, va, m);
@@ -1850,8 +1854,9 @@ moea64_enter_quick(pmap_t pm, vm_offset_t va, vm_page_t m,
     vm_prot_t prot)
 {
 
-	moea64_enter(pm, va, m, prot & (VM_PROT_READ | VM_PROT_EXECUTE),
-	    PMAP_ENTER_NOSLEEP | PMAP_ENTER_QUICK_LOCKED, 0);
+	moea64_enter(pm, va, m, prot & (VM_PROT_READ | VM_PROT_EXECUTE |
+	    VM_PROT_NO_PROMOTE), PMAP_ENTER_NOSLEEP | PMAP_ENTER_QUICK_LOCKED,
+	    0);
 }
 
 vm_paddr_t
diff --git a/sys/riscv/riscv/pmap.c b/sys/riscv/riscv/pmap.c
index 1e507f62696e..e8504bcb0f59 100644
--- a/sys/riscv/riscv/pmap.c
+++ b/sys/riscv/riscv/pmap.c
@@ -3519,7 +3519,8 @@ pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m,
 	 * If both the PTP and the reservation are fully populated, then attempt
 	 * promotion.
 	 */
-	if ((mpte == NULL || mpte->ref_count == Ln_ENTRIES) &&
+	if ((prot & VM_PROT_NO_PROMOTE) == 0 &&
+	    (mpte == NULL || mpte->ref_count == Ln_ENTRIES) &&
 	    (m->flags & PG_FICTITIOUS) == 0 &&
 	    vm_reserv_level_iffullpop(m) == 0) {
 		if (l2 == NULL)
diff --git a/sys/vm/vm.h b/sys/vm/vm.h
index 5fac6b3f2e78..b7d149a2fca2 100644
--- a/sys/vm/vm.h
+++ b/sys/vm/vm.h
@@ -76,6 +76,7 @@ typedef u_char vm_prot_t;	/* protection codes */
 #define	VM_PROT_COPY		((vm_prot_t) 0x08)	/* copy-on-read */
 #define	VM_PROT_PRIV_FLAG	((vm_prot_t) 0x10)
 #define	VM_PROT_FAULT_LOOKUP	VM_PROT_PRIV_FLAG
+#define	VM_PROT_NO_PROMOTE	VM_PROT_PRIV_FLAG
 #define	VM_PROT_QUICK_NOFAULT	VM_PROT_PRIV_FLAG	/* same to save bits */
 
 #define	VM_PROT_ALL		(VM_PROT_READ|VM_PROT_WRITE|VM_PROT_EXECUTE)
diff --git a/sys/vm/vm_fault.c b/sys/vm/vm_fault.c
index fbe0a117a388..3da411643f26 100644
--- a/sys/vm/vm_fault.c
+++ b/sys/vm/vm_fault.c
@@ -1891,6 +1891,7 @@ vm_fault_prefault(const struct faultstate *fs, vm_offset_t addra,
 	vm_offset_t addr, starta;
 	vm_pindex_t pindex;
 	vm_page_t m;
+	vm_prot_t prot;
 	int i;
 
 	pmap = fs->map->pmap;
@@ -1906,6 +1907,14 @@ vm_fault_prefault(const struct faultstate *fs, vm_offset_t addra,
 		if (starta < entry->start)
 			starta = entry->start;
 	}
+	prot = entry->protection;
+
+	/*
+	 * If pmap_enter() has enabled write access on a nearby mapping, then
+	 * don't attempt promotion, because it will fail.
+	 */
+	if ((fs->prot & VM_PROT_WRITE) != 0)
+		prot |= VM_PROT_NO_PROMOTE;
 
 	/*
 	 * Generate the sequence of virtual addresses that are candidates for
@@ -1949,7 +1958,7 @@ vm_fault_prefault(const struct faultstate *fs, vm_offset_t addra,
 		}
 		if (vm_page_all_valid(m) &&
 		    (m->flags & PG_FICTITIOUS) == 0)
-			pmap_enter_quick(pmap, addr, m, entry->protection);
+			pmap_enter_quick(pmap, addr, m, prot);
 		if (!obj_locked || lobject != entry->object.vm_object)
 			VM_OBJECT_RUNLOCK(lobject);
 	}