git: 89b4150af5b1 - stable/13 - riscv: Add pmap helper functions required by four-level page tables

The branch stable/13 has been updated by markj:

URL: https://cgit.FreeBSD.org/src/commit/?id=89b4150af5b169bf2db38a91d7bafc13b0bf28ea

commit 89b4150af5b169bf2db38a91d7bafc13b0bf28ea
Author:     Mark Johnston <markj@FreeBSD.org>
AuthorDate: 2022-03-01 14:06:15 +0000
Commit:     Mark Johnston <markj@FreeBSD.org>
CommitDate: 2022-03-14 14:45:40 +0000

    riscv: Add pmap helper functions required by four-level page tables
    
    No functional change intended.
    
    Reviewed by:    jhb
    Sponsored by:   The FreeBSD Foundation
    
    (cherry picked from commit 5cf3a8216e3adad06f1ab02c9a9b201cebcd4809)
---
 sys/riscv/riscv/pmap.c | 36 +++++++++++++++++++++++++++++++++++-
 1 file changed, 35 insertions(+), 1 deletion(-)

diff --git a/sys/riscv/riscv/pmap.c b/sys/riscv/riscv/pmap.c
index ac6b854ac3c3..74192a16341c 100644
--- a/sys/riscv/riscv/pmap.c
+++ b/sys/riscv/riscv/pmap.c
@@ -363,13 +363,45 @@ pagezero(void *p)
 #define	L2PTE_TO_PHYS(l2) \
     ((((l2) & ~PTE_HI_MASK) >> PTE_PPN1_S) << L2_SHIFT)
 
+static __inline pd_entry_t *
+pmap_l0(pmap_t pmap, vm_offset_t va)
+{
+	KASSERT(pmap_mode != PMAP_MODE_SV39, ("%s: in SV39 mode", __func__));
+	KASSERT(VIRT_IS_VALID(va),
+	    ("%s: malformed virtual address %#lx", __func__, va));
+	return (&pmap->pm_top[pmap_l0_index(va)]);
+}
+
+static __inline pd_entry_t *
+pmap_l0_to_l1(pd_entry_t *l0, vm_offset_t va)
+{
+	vm_paddr_t phys;
+	pd_entry_t *l1;
+
+	KASSERT(pmap_mode != PMAP_MODE_SV39, ("%s: in SV39 mode", __func__));
+	phys = PTE_TO_PHYS(pmap_load(l0));
+	l1 = (pd_entry_t *)PHYS_TO_DMAP(phys);
+
+	return (&l1[pmap_l1_index(va)]);
+}
+
 static __inline pd_entry_t *
 pmap_l1(pmap_t pmap, vm_offset_t va)
 {
+	pd_entry_t *l0;
 
 	KASSERT(VIRT_IS_VALID(va),
 	    ("%s: malformed virtual address %#lx", __func__, va));
-	return (&pmap->pm_top[pmap_l1_index(va)]);
+	if (pmap_mode == PMAP_MODE_SV39) {
+		return (&pmap->pm_top[pmap_l1_index(va)]);
+	} else {
+		l0 = pmap_l0(pmap, va);
+		if ((pmap_load(l0) & PTE_V) == 0)
+			return (NULL);
+		if ((pmap_load(l0) & PTE_RX) != 0)
+			return (NULL);
+		return (pmap_l0_to_l1(l0, va));
+	}
 }
 
 static __inline pd_entry_t *
@@ -390,6 +422,8 @@ pmap_l2(pmap_t pmap, vm_offset_t va)
 	pd_entry_t *l1;
 
 	l1 = pmap_l1(pmap, va);
+	if (l1 == NULL)
+		return (NULL);
 	if ((pmap_load(l1) & PTE_V) == 0)
 		return (NULL);
 	if ((pmap_load(l1) & PTE_RX) != 0)