git: 25b18f0a12c9 - stable/13 - riscv: Add support for dynamically allocating L1 page table pages
- Go to: [ bottom of page ] [ top of archives ] [ this month ]
Date: Mon, 14 Mar 2022 14:46:16 UTC
The branch stable/13 has been updated by markj:
URL: https://cgit.FreeBSD.org/src/commit/?id=25b18f0a12c9acc23baae4efde53a0311c1e43da
commit 25b18f0a12c9acc23baae4efde53a0311c1e43da
Author: Mark Johnston <markj@FreeBSD.org>
AuthorDate: 2022-03-01 14:06:57 +0000
Commit: Mark Johnston <markj@FreeBSD.org>
CommitDate: 2022-03-14 14:45:47 +0000
riscv: Add support for dynamically allocating L1 page table pages
This is required in SV48 mode.
Reviewed by: jhb
Sponsored by: The FreeBSD Foundation
(cherry picked from commit 6ce716f7c31a0a1b5ca8224cbb627632f44e4e6c)
---
sys/riscv/riscv/pmap.c | 127 +++++++++++++++++++++++++++++++++++++++----------
1 file changed, 101 insertions(+), 26 deletions(-)
diff --git a/sys/riscv/riscv/pmap.c b/sys/riscv/riscv/pmap.c
index 8aa99d52f2c9..a6b604d43984 100644
--- a/sys/riscv/riscv/pmap.c
+++ b/sys/riscv/riscv/pmap.c
@@ -1190,6 +1190,8 @@ pmap_remove_pt_page(pmap_t pmap, vm_offset_t va)
static inline boolean_t
pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free)
{
+ KASSERT(m->ref_count > 0,
+ ("%s: page %p ref count underflow", __func__, m));
--m->ref_count;
if (m->ref_count == 0) {
@@ -1206,7 +1208,11 @@ _pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free)
vm_paddr_t phys;
PMAP_LOCK_ASSERT(pmap, MA_OWNED);
- if (m->pindex >= NUL2E) {
+ if (m->pindex >= NUL2E + NUL1E) {
+ pd_entry_t *l0;
+ l0 = pmap_l0(pmap, va);
+ pmap_clear(l0);
+ } else if (m->pindex >= NUL2E) {
pd_entry_t *l1;
l1 = pmap_l1(pmap, va);
pmap_clear(l1);
@@ -1225,6 +1231,15 @@ _pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free)
phys = PTE_TO_PHYS(pmap_load(l1));
pdpg = PHYS_TO_VM_PAGE(phys);
pmap_unwire_ptp(pmap, va, pdpg, free);
+ } else if (m->pindex < NUL2E + NUL1E && pmap_mode != PMAP_MODE_SV39) {
+ pd_entry_t *l0;
+ vm_page_t pdpg;
+
+ MPASS(pmap_mode != PMAP_MODE_SV39);
+ l0 = pmap_l0(pmap, va);
+ phys = PTE_TO_PHYS(pmap_load(l0));
+ pdpg = PHYS_TO_VM_PAGE(phys);
+ pmap_unwire_ptp(pmap, va, pdpg, free);
}
pmap_invalidate_page(pmap, va);
@@ -1323,7 +1338,7 @@ pmap_pinit(pmap_t pmap)
static vm_page_t
_pmap_alloc_l3(pmap_t pmap, vm_pindex_t ptepindex, struct rwlock **lockp)
{
- vm_page_t m, /*pdppg, */pdpg;
+ vm_page_t m, pdpg;
pt_entry_t entry;
vm_paddr_t phys;
pn_t pn;
@@ -1356,39 +1371,96 @@ _pmap_alloc_l3(pmap_t pmap, vm_pindex_t ptepindex, struct rwlock **lockp)
* Map the pagetable page into the process address space, if
* it isn't already there.
*/
-
- if (ptepindex >= NUL2E) {
- pd_entry_t *l1;
- vm_pindex_t l1index;
+ pn = VM_PAGE_TO_PHYS(m) >> PAGE_SHIFT;
+ if (ptepindex >= NUL2E + NUL1E) {
+ pd_entry_t *l0;
+ vm_pindex_t l0index;
+
+ KASSERT(pmap_mode != PMAP_MODE_SV39,
+ ("%s: pindex %#lx in SV39 mode", __func__, ptepindex));
+ KASSERT(ptepindex < NUL2E + NUL1E + NUL0E,
+ ("%s: pindex %#lx out of range", __func__, ptepindex));
+
+ l0index = ptepindex - (NUL2E + NUL1E);
+ l0 = &pmap->pm_top[l0index];
+ KASSERT((pmap_load(l0) & PTE_V) == 0,
+ ("%s: L0 entry %#lx is valid", __func__, pmap_load(l0)));
+
+ entry = PTE_V | (pn << PTE_PPN0_S);
+ pmap_store(l0, entry);
+ } else if (ptepindex >= NUL2E) {
+ pd_entry_t *l0, *l1;
+ vm_pindex_t l0index, l1index;
l1index = ptepindex - NUL2E;
- l1 = &pmap->pm_top[l1index];
+ if (pmap_mode == PMAP_MODE_SV39) {
+ l1 = &pmap->pm_top[l1index];
+ } else {
+ l0index = l1index >> Ln_ENTRIES_SHIFT;
+ l0 = &pmap->pm_top[l0index];
+ if (pmap_load(l0) == 0) {
+ /* Recurse to allocate the L1 page. */
+ if (_pmap_alloc_l3(pmap,
+ NUL2E + NUL1E + l0index, lockp) == NULL)
+ goto fail;
+ phys = PTE_TO_PHYS(pmap_load(l0));
+ } else {
+ phys = PTE_TO_PHYS(pmap_load(l0));
+ pdpg = PHYS_TO_VM_PAGE(phys);
+ pdpg->ref_count++;
+ }
+ l1 = (pd_entry_t *)PHYS_TO_DMAP(phys);
+ l1 = &l1[ptepindex & Ln_ADDR_MASK];
+ }
KASSERT((pmap_load(l1) & PTE_V) == 0,
("%s: L1 entry %#lx is valid", __func__, pmap_load(l1)));
- pn = (VM_PAGE_TO_PHYS(m) / PAGE_SIZE);
- entry = (PTE_V);
- entry |= (pn << PTE_PPN0_S);
+ entry = PTE_V | (pn << PTE_PPN0_S);
pmap_store(l1, entry);
pmap_distribute_l1(pmap, l1index, entry);
} else {
- vm_pindex_t l1index;
- pd_entry_t *l1, *l2;
+ vm_pindex_t l0index, l1index;
+ pd_entry_t *l0, *l1, *l2;
l1index = ptepindex >> (L1_SHIFT - L2_SHIFT);
- l1 = &pmap->pm_top[l1index];
- if (pmap_load(l1) == 0) {
- /* recurse for allocating page dir */
- if (_pmap_alloc_l3(pmap, NUL2E + l1index,
- lockp) == NULL) {
- vm_page_unwire_noq(m);
- vm_page_free_zero(m);
- return (NULL);
+ if (pmap_mode == PMAP_MODE_SV39) {
+ l1 = &pmap->pm_top[l1index];
+ if (pmap_load(l1) == 0) {
+ /* recurse for allocating page dir */
+ if (_pmap_alloc_l3(pmap, NUL2E + l1index,
+ lockp) == NULL)
+ goto fail;
+ } else {
+ phys = PTE_TO_PHYS(pmap_load(l1));
+ pdpg = PHYS_TO_VM_PAGE(phys);
+ pdpg->ref_count++;
}
} else {
- phys = PTE_TO_PHYS(pmap_load(l1));
- pdpg = PHYS_TO_VM_PAGE(phys);
- pdpg->ref_count++;
+ l0index = l1index >> Ln_ENTRIES_SHIFT;
+ l0 = &pmap->pm_top[l0index];
+ if (pmap_load(l0) == 0) {
+ /* Recurse to allocate the L1 entry. */
+ if (_pmap_alloc_l3(pmap, NUL2E + l1index,
+ lockp) == NULL)
+ goto fail;
+ phys = PTE_TO_PHYS(pmap_load(l0));
+ l1 = (pd_entry_t *)PHYS_TO_DMAP(phys);
+ l1 = &l1[l1index & Ln_ADDR_MASK];
+ } else {
+ phys = PTE_TO_PHYS(pmap_load(l0));
+ l1 = (pd_entry_t *)PHYS_TO_DMAP(phys);
+ l1 = &l1[l1index & Ln_ADDR_MASK];
+ if (pmap_load(l1) == 0) {
+ /* Recurse to allocate the L2 page. */
+ if (_pmap_alloc_l3(pmap,
+ NUL2E + l1index, lockp) == NULL)
+ goto fail;
+ } else {
+ phys = PTE_TO_PHYS(pmap_load(l1));
+ pdpg = PHYS_TO_VM_PAGE(phys);
+ pdpg->ref_count++;
+ }
+ }
}
phys = PTE_TO_PHYS(pmap_load(l1));
@@ -1397,15 +1469,18 @@ _pmap_alloc_l3(pmap_t pmap, vm_pindex_t ptepindex, struct rwlock **lockp)
KASSERT((pmap_load(l2) & PTE_V) == 0,
("%s: L2 entry %#lx is valid", __func__, pmap_load(l2)));
- pn = (VM_PAGE_TO_PHYS(m) / PAGE_SIZE);
- entry = (PTE_V);
- entry |= (pn << PTE_PPN0_S);
+ entry = PTE_V | (pn << PTE_PPN0_S);
pmap_store(l2, entry);
}
pmap_resident_count_inc(pmap, 1);
return (m);
+
+fail:
+ vm_page_unwire_noq(m);
+ vm_page_free_zero(m);
+ return (NULL);
}
static vm_page_t