git: b83204edf101 - main - powerpc/pmap: Support booke64 kernel pmap growing
- Go to: [ bottom of page ] [ top of archives ] [ this month ]
Date: Fri, 07 Aug 2026 02:52:39 UTC
The branch main has been updated by jhibbits:
URL: https://cgit.FreeBSD.org/src/commit/?id=b83204edf101d2c8ad40005ca2fe9a76796cd4f1
commit b83204edf101d2c8ad40005ca2fe9a76796cd4f1
Author: Justin Hibbits <jhibbits@FreeBSD.org>
AuthorDate: 2026-08-05 03:47:27 +0000
Commit: Justin Hibbits <jhibbits@FreeBSD.org>
CommitDate: 2026-08-07 02:52:28 +0000
powerpc/pmap: Support booke64 kernel pmap growing
In preparation of increasing the KVA on powerpc64 to 2TB to mirror
amd64's, rework the 64-bit Book-E pmap to not allocate all page table
pages at boot time, since that would be a waste of a lot of memory.
Instead, allocate all page table pages for the higher levels, leaving
the leaves (page directories) for dynamic allocation. This cuts the
boot-time page table size down from ~64MB to ~8MB with the current 32GB
KVA size, and bumping to 2TB KVA the boot-time page table is still ~8MB
instead of ballooning to ~4GB of mostly wasted space.
---
sys/powerpc/booke/pmap.c | 18 +++++++--
sys/powerpc/booke/pmap_64.c | 95 ++++++++++++++++++++++++++++++++++++++++-----
2 files changed, 101 insertions(+), 12 deletions(-)
diff --git a/sys/powerpc/booke/pmap.c b/sys/powerpc/booke/pmap.c
index 545416921d09..ce16566a6f13 100644
--- a/sys/powerpc/booke/pmap.c
+++ b/sys/powerpc/booke/pmap.c
@@ -359,6 +359,9 @@ static int mmu_booke_decode_kernel_ptr(vm_offset_t addr,
static void mmu_booke_page_array_startup(long);
static bool mmu_booke_page_is_mapped(vm_page_t m);
static bool mmu_booke_ps_enabled(pmap_t pmap);
+#ifdef __powerpc64__
+static int mmu_booke_growkernel(vm_offset_t);
+#endif
static struct pmap_funcs mmu_booke_methods = {
/* pmap dispatcher interface */
@@ -402,6 +405,9 @@ static struct pmap_funcs mmu_booke_methods = {
.page_array_startup = mmu_booke_page_array_startup,
.page_is_mapped = mmu_booke_page_is_mapped,
.ps_enabled = mmu_booke_ps_enabled,
+#ifdef __powerpc64__
+ .growkernel_nopanic = mmu_booke_growkernel,
+#endif
/* Internal interfaces */
.bootstrap = mmu_booke_bootstrap,
@@ -700,7 +706,6 @@ mmu_booke_bootstrap(vm_offset_t start, vm_offset_t kernelend)
(uintptr_t)msgbufp, data_end);
data_end = round_page(data_end);
- data_end = round_page(mmu_booke_alloc_kernel_pgtables(data_end));
/* Retrieve phys/avail mem regions */
mem_regions(&physmem_regions, &physmem_regions_sz,
@@ -709,7 +714,6 @@ mmu_booke_bootstrap(vm_offset_t start, vm_offset_t kernelend)
if (PHYS_AVAIL_ENTRIES < availmem_regions_sz)
panic("mmu_booke_bootstrap: phys_avail too small");
- data_end = round_page(data_end);
vm_page_array = (vm_page_t)data_end;
/*
* Get a rough idea (upper bound) on the size of the page array. The
@@ -723,6 +727,14 @@ mmu_booke_bootstrap(vm_offset_t start, vm_offset_t kernelend)
sz = (round_page(sz) / (PAGE_SIZE + sizeof(struct vm_page)));
data_end += round_page(sz * sizeof(struct vm_page));
+ /*
+ * Reserve kernel page-table pages last, so their reservation size can
+ * be computed from the final bootstrap data_end (on 64-bit, only leaf
+ * ptbls covering [VM_MIN_KERNEL_ADDRESS, data_end + slack] are
+ * pre-allocated; the rest are added on demand by pmap_growkernel()).
+ */
+ data_end = round_page(mmu_booke_alloc_kernel_pgtables(data_end));
+
/* Pre-round up to 1MB. This wastes some space, but saves TLB entries */
data_end = roundup2(data_end, 1 << 20);
@@ -1216,7 +1228,7 @@ mmu_booke_kremove(vm_offset_t va)
pte = pte_find(kernel_pmap, va);
- if (!PTE_ISVALID(pte)) {
+ if (pte == NULL || !PTE_ISVALID(pte)) {
CTR1(KTR_PMAP, "%s: invalid pte", __func__);
return;
diff --git a/sys/powerpc/booke/pmap_64.c b/sys/powerpc/booke/pmap_64.c
index 97f9a5967167..3f2db07ce931 100644
--- a/sys/powerpc/booke/pmap_64.c
+++ b/sys/powerpc/booke/pmap_64.c
@@ -113,6 +113,13 @@ unsigned int kernel_pdirs;
static uma_zone_t ptbl_root_zone;
static pte_t ****kernel_ptbl_root;
+/*
+ * Slack space beyond the bootstrap data_end for which leaf ptbls are
+ * pre-allocated. Must cover post-alloc growth (1MB roundup + TLB1 grow +
+ * kstack + guard pages). Rounded up to PDIR_SIZE by the reservation code.
+ */
+#define BOOTSTRAP_LEAF_SLACK (2 * PDIR_SIZE)
+
/*
* Base of the pmap_mapdev() region. On 32-bit it immediately follows the
* userspace address range. On On 64-bit it's far above, at (1 << 63), and
@@ -262,14 +269,16 @@ get_pgtbl_page(pmap_t pmap, void **ptr_tbl, uint32_t index,
vm_page_t m;
page = ptr_tbl[index];
- KASSERT(page != 0 || pmap != kernel_pmap,
- ("NULL page table page found in kernel pmap!"));
if (page == NULL) {
page = mmu_booke_alloc_page(pmap, index, nosleep);
if (ptr_tbl[index] == NULL) {
*isnew = true;
ptr_tbl[index] = page;
- if (hold_parent) {
+ /*
+ * Kernel page-table pages are never freed, so we do
+ * not maintain refcounts on their parents.
+ */
+ if (hold_parent && pmap != kernel_pmap) {
m = PHYS_TO_VM_PAGE(pmap_kextract((vm_offset_t)ptr_tbl));
m->ref_count++;
}
@@ -518,7 +527,8 @@ kernel_pte_alloc(vm_offset_t data_end, vm_offset_t addr)
{
pte_t *pte;
vm_size_t kva_size;
- int kernel_pdirs, kernel_pgtbls, pdir_l1s;
+ int kernel_pdirs, pdir_l1s;
+ unsigned int kernel_pgtbls;
vm_offset_t va, l1_va, pdir_va, ptbl_va;
int i, j, k;
@@ -526,7 +536,8 @@ kernel_pte_alloc(vm_offset_t data_end, vm_offset_t addr)
kernel_pmap->pm_root = kernel_ptbl_root;
pdir_l1s = howmany(kva_size, PG_ROOT_SIZE);
kernel_pdirs = howmany(kva_size, PDIR_L1_SIZE);
- kernel_pgtbls = howmany(kva_size, PDIR_SIZE);
+ kernel_pgtbls = howmany(kernel_vm_end - VM_MIN_KERNEL_ADDRESS,
+ PDIR_SIZE);
/* Initialize kernel pdir */
l1_va = (vm_offset_t)kernel_ptbl_root +
@@ -537,7 +548,8 @@ kernel_pte_alloc(vm_offset_t data_end, vm_offset_t addr)
printf("ptbl_root_va: %#lx\n", (vm_offset_t)kernel_ptbl_root);
printf("l1_va: %#lx (%d entries)\n", l1_va, pdir_l1s);
printf("pdir_va: %#lx(%d entries)\n", pdir_va, kernel_pdirs);
- printf("ptbl_va: %#lx(%d entries)\n", ptbl_va, kernel_pgtbls);
+ printf("ptbl_va: %#lx(%u entries)\n", ptbl_va, kernel_pgtbls);
+ printf("kernel_vm_end: %#lx\n", kernel_vm_end);
}
va = VM_MIN_KERNEL_ADDRESS;
@@ -550,8 +562,18 @@ kernel_pte_alloc(vm_offset_t data_end, vm_offset_t addr)
kernel_pmap->pm_root[i][j] = (pte_t **)pdir_va;
for (k = 0;
k < PDIR_NENTRIES && va < VM_MAX_KERNEL_ADDRESS;
- k++, va += PDIR_SIZE, ptbl_va += PAGE_SIZE)
- kernel_pmap->pm_root[i][j][k] = (pte_t *)ptbl_va;
+ k++, va += PDIR_SIZE) {
+ /*
+ * Only wire up leaf ptbl pages for the
+ * pre-allocated bootstrap KVA range; the rest
+ * are populated lazily by mmu_booke_growkernel().
+ */
+ if (va < kernel_vm_end) {
+ kernel_pmap->pm_root[i][j][k] =
+ (pte_t *)ptbl_va;
+ ptbl_va += PAGE_SIZE;
+ }
+ }
}
}
/*
@@ -572,16 +594,71 @@ static vm_offset_t
mmu_booke_alloc_kernel_pgtables(vm_offset_t data_end)
{
vm_size_t kva_size = VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS;
+ unsigned int leaves;
+
kernel_ptbl_root = (pte_t ****)data_end;
data_end += round_page(PG_ROOT_NENTRIES * sizeof(pte_t ***));
data_end += howmany(kva_size, PG_ROOT_SIZE) * PAGE_SIZE;
data_end += howmany(kva_size, PDIR_L1_SIZE) * PAGE_SIZE;
- data_end += howmany(kva_size, PDIR_SIZE) * PAGE_SIZE;
+ /*
+ * Reserve leaf page-table pages only for the current bootstrap KVA
+ * range plus a small slack for post-alloc growth (1MB TLB roundup,
+ * kstack + guard). Leaves for the rest of KVA are allocated on
+ * demand by mmu_booke_growkernel(). The two-step howmany() converges
+ * because leaves themselves add at most one extra leaf per 512 leaves.
+ */
+ leaves = howmany(data_end - VM_MIN_KERNEL_ADDRESS +
+ BOOTSTRAP_LEAF_SLACK, PDIR_SIZE);
+ leaves = howmany(data_end - VM_MIN_KERNEL_ADDRESS +
+ BOOTSTRAP_LEAF_SLACK + leaves * PAGE_SIZE, PDIR_SIZE);
+ data_end += leaves * PAGE_SIZE;
+
+ kernel_vm_end = VM_MIN_KERNEL_ADDRESS + leaves * PDIR_SIZE;
return (data_end);
}
+/*
+ * Grow the kernel page-table by allocating additional leaf ptbl pages so
+ * that pte_find(kernel_pmap, va) can succeed for [kernel_vm_end, addr].
+ *
+ * All upper levels (root, pdir_l1, pdir) are pre-populated for the whole
+ * KVA at bootstrap, so only the leaf level needs to be allocated here.
+ */
+static int
+mmu_booke_growkernel(vm_offset_t addr)
+{
+ pte_t **pdir;
+ vm_page_t m;
+ vm_offset_t va;
+
+ if (addr <= kernel_vm_end)
+ return (KERN_SUCCESS);
+
+ addr = roundup2(addr, PDIR_SIZE);
+ if (addr - 1 >= vm_map_max(kernel_map))
+ addr = vm_map_max(kernel_map);
+
+ for (va = kernel_vm_end; va < addr; va += PDIR_SIZE) {
+ pdir = kernel_pmap->pm_root[PG_ROOT_IDX(va)][PDIR_L1_IDX(va)];
+ KASSERT(pdir != NULL,
+ ("mmu_booke_growkernel: NULL pdir at va %#lx", va));
+ if (pdir[PDIR_IDX(va)] != NULL)
+ continue;
+ m = vm_page_alloc_noobj(VM_ALLOC_INTERRUPT | VM_ALLOC_WIRED |
+ VM_ALLOC_ZERO);
+ if (m == NULL) {
+ kernel_vm_end = va;
+ return (KERN_RESOURCE_SHORTAGE);
+ }
+ m->pindex = va >> PDIR_SHIFT;
+ pdir[PDIR_IDX(va)] = (pte_t *)VM_PAGE_TO_DMAP(m);
+ }
+ kernel_vm_end = addr;
+ return (KERN_SUCCESS);
+}
+
/*
* Initialize a preallocated and zeroed pmap structure,
* such as one in a vmspace structure.