git: 720a892ac60f - main - amd64 pmap: simplify vtopte() and vtopde()

The branch main has been updated by kib:

URL: https://cgit.FreeBSD.org/src/commit/?id=720a892ac60fc1fa7c784edc40f188c93ac7e0b7

commit 720a892ac60fc1fa7c784edc40f188c93ac7e0b7
Author:     Konstantin Belousov <kib@FreeBSD.org>
AuthorDate: 2022-01-05 00:26:12 +0000
Commit:     Konstantin Belousov <kib@FreeBSD.org>
CommitDate: 2022-01-08 04:34:18 +0000

    amd64 pmap: simplify vtopte() and vtopde()
    
    Pre-calculate masks and page table/page directory bases, for LA48 and
    LA57 cases, trading a conditional for the memory accesses.
    
    This shaves 672 bytes of .text, by the cost of 32 .data bytes.  Note
    that eliminated code contained one conditional, and several costly
    movabs instructions, which are traded by two memory fetches per
    vtop{t,d}e() inlines.
    
    Reviewed by:    markj
    Discussed with: alc
    Sponsored by:   The FreeBSD Foundation
    MFC after:      1 week
    Differential revision:  https://reviews.freebsd.org/D33776
---
 sys/amd64/amd64/pmap.c | 39 +++++++++++++++++----------------------
 1 file changed, 17 insertions(+), 22 deletions(-)

diff --git a/sys/amd64/amd64/pmap.c b/sys/amd64/amd64/pmap.c
index a2d55b219505..8b5d58c2a540 100644
--- a/sys/amd64/amd64/pmap.c
+++ b/sys/amd64/amd64/pmap.c
@@ -1516,40 +1516,28 @@ pmap_pt_page_count_adj(pmap_t pmap, int count)
 	}
 }
 
+pt_entry_t vtoptem __read_mostly = (1ul << (NPTEPGSHIFT + NPDEPGSHIFT +
+    NPDPEPGSHIFT + NPML4EPGSHIFT)) - 1;
+pt_entry_t *PTmap __read_mostly = P4Tmap;
+
 PMAP_INLINE pt_entry_t *
 vtopte(vm_offset_t va)
 {
-	u_int64_t mask;
-
 	KASSERT(va >= VM_MAXUSER_ADDRESS, ("vtopte on a uva/gpa 0x%0lx", va));
 
-	if (la57) {
-		mask = ((1ul << (NPTEPGSHIFT + NPDEPGSHIFT + NPDPEPGSHIFT +
-		    NPML4EPGSHIFT + NPML5EPGSHIFT)) - 1);
-		return (P5Tmap + ((va >> PAGE_SHIFT) & mask));
-	} else {
-		mask = ((1ul << (NPTEPGSHIFT + NPDEPGSHIFT + NPDPEPGSHIFT +
-		    NPML4EPGSHIFT)) - 1);
-		return (P4Tmap + ((va >> PAGE_SHIFT) & mask));
-	}
+	return (PTmap + ((va >> PAGE_SHIFT) & vtoptem));
 }
 
+pd_entry_t vtopdem __read_mostly = (1ul << (NPDEPGSHIFT + NPDPEPGSHIFT +
+    NPML4EPGSHIFT)) - 1;
+pd_entry_t *PDmap __read_mostly = P4Dmap;
+
 static __inline pd_entry_t *
 vtopde(vm_offset_t va)
 {
-	u_int64_t mask;
-
 	KASSERT(va >= VM_MAXUSER_ADDRESS, ("vtopde on a uva/gpa 0x%0lx", va));
 
-	if (la57) {
-		mask = ((1ul << (NPDEPGSHIFT + NPDPEPGSHIFT +
-		    NPML4EPGSHIFT + NPML5EPGSHIFT)) - 1);
-		return (P5Dmap + ((va >> PDRSHIFT) & mask));
-	} else {
-		mask = ((1ul << (NPDEPGSHIFT + NPDPEPGSHIFT +
-		    NPML4EPGSHIFT)) - 1);
-		return (P4Dmap + ((va >> PDRSHIFT) & mask));
-	}
+	return (PDmap + ((va >> PDRSHIFT) & vtopdem));
 }
 
 static u_int64_t
@@ -2241,6 +2229,13 @@ pmap_bootstrap_la57(void *arg __unused)
 	 */
 	v_pml5[PML5PML5I] = KPML5phys | X86_PG_RW | X86_PG_V | pg_nx;
 
+	vtoptem = (1ul << (NPTEPGSHIFT + NPDEPGSHIFT + NPDPEPGSHIFT +
+	    NPML4EPGSHIFT + NPML5EPGSHIFT)) - 1;
+	PTmap = P5Tmap;
+	vtopdem = (1ul << (NPDEPGSHIFT + NPDPEPGSHIFT +
+	    NPML4EPGSHIFT + NPML5EPGSHIFT)) - 1;
+	PDmap = P5Dmap;
+
 	kernel_pmap->pm_cr3 = KPML5phys;
 	kernel_pmap->pm_pmltop = v_pml5;
 	pmap_pt_page_count_adj(kernel_pmap, 1);