svn commit: r365518 - in head/sys: amd64/amd64 vm
Konstantin Belousov
kib at FreeBSD.org
Wed Sep 9 21:50:25 UTC 2020
Author: kib
Date: Wed Sep 9 21:50:24 2020
New Revision: 365518
URL: https://svnweb.freebsd.org/changeset/base/365518
Log:
Add pmap_enter(9) PMAP_ENTER_LARGEPAGE flag and implement it on amd64.
The flag requests entry of non-managed superpage mapping of size
pagesizes[psind] into the page table.
Pmap supports fake wiring of the largepage mappings. Only attributes
of the largepage mapping can be changed by calling pmap_enter(9) over
existing mapping, physical address of the page must be unchanged.
Reviewed by: markj
Tested by: pho
Sponsored by: The FreeBSD Foundation
MFC after: 1 week
Differential revision: https://reviews.freebsd.org/D24652
Modified:
head/sys/amd64/amd64/pmap.c
head/sys/vm/pmap.h
Modified: head/sys/amd64/amd64/pmap.c
==============================================================================
--- head/sys/amd64/amd64/pmap.c Wed Sep 9 21:45:18 2020 (r365517)
+++ head/sys/amd64/amd64/pmap.c Wed Sep 9 21:50:24 2020 (r365518)
@@ -6475,6 +6475,119 @@ setpte:
}
#endif /* VM_NRESERVLEVEL > 0 */
+static int
+pmap_enter_largepage(pmap_t pmap, vm_offset_t va, pt_entry_t newpte, int flags,
+ int psind)
+{
+ vm_page_t mp;
+ pt_entry_t origpte, *pml4e, *pdpe, *pde, pten, PG_V;
+
+ PMAP_LOCK_ASSERT(pmap, MA_OWNED);
+ KASSERT(psind > 0 && psind < MAXPAGESIZES,
+ ("psind %d unexpected", psind));
+ KASSERT(((newpte & PG_FRAME) & (pagesizes[psind] - 1)) == 0,
+ ("unaligned phys address %#lx newpte %#lx psind %d",
+ newpte & PG_FRAME, newpte, psind));
+ KASSERT((va & (pagesizes[psind] - 1)) == 0,
+ ("unaligned va %#lx psind %d", va, psind));
+ KASSERT(va < VM_MAXUSER_ADDRESS,
+ ("kernel mode non-transparent superpage")); /* XXXKIB */
+ KASSERT(va + pagesizes[psind] < VM_MAXUSER_ADDRESS,
+ ("overflowing user map va %#lx psind %d", va, psind)); /* XXXKIB */
+
+ PG_V = pmap_valid_bit(pmap);
+
+restart:
+ pten = newpte;
+ if (va < VM_MAXUSER_ADDRESS && pmap->pm_type == PT_X86)
+ pten |= pmap_pkru_get(pmap, va);
+
+ if (psind == 2) { /* 1G */
+ if (!pmap_pkru_same(pmap, va, va + NBPDP))
+ return (KERN_PROTECTION_FAILURE);
+ pml4e = pmap_pml4e(pmap, va);
+ if ((*pml4e & PG_V) == 0) {
+ mp = _pmap_allocpte(pmap, pmap_pml4e_pindex(va),
+ NULL, va);
+ if (mp == NULL) {
+ if ((flags & PMAP_ENTER_NOSLEEP) != 0)
+ return (KERN_RESOURCE_SHORTAGE);
+ PMAP_UNLOCK(pmap);
+ vm_wait(NULL);
+ PMAP_LOCK(pmap);
+
+ /*
+ * Restart at least to recalcuate the pkru
+ * key. Our caller must keep the map locked
+ * so no paging structure can be validated
+ * under us.
+ */
+ goto restart;
+ }
+ pdpe = pmap_pdpe(pmap, va);
+ KASSERT(pdpe != NULL, ("va %#lx lost pdpe", va));
+ origpte = *pdpe;
+ MPASS(origpte == 0);
+ } else {
+ mp = PHYS_TO_VM_PAGE(*pml4e & PG_FRAME);
+ pdpe = pmap_pdpe(pmap, va);
+ KASSERT(pdpe != NULL, ("va %#lx lost pdpe", va));
+ origpte = *pdpe;
+ if ((origpte & PG_V) == 0)
+ mp->ref_count++;
+ }
+ KASSERT((origpte & PG_V) == 0 || ((origpte & PG_PS) != 0 &&
+ (origpte & PG_FRAME) == (newpte & PG_FRAME)),
+ ("va %#lx changing 1G phys page pdpe %#lx newpte %#lx",
+ va, origpte, newpte));
+ if ((newpte & PG_W) != 0 && (origpte & PG_W) == 0)
+ pmap->pm_stats.wired_count += NBPDP / PAGE_SIZE;
+ else if ((newpte & PG_W) == 0 && (origpte & PG_W) != 0)
+ pmap->pm_stats.wired_count -= NBPDP / PAGE_SIZE;
+ *pdpe = newpte;
+ } else /* (psind == 1) */ { /* 2M */
+ if (!pmap_pkru_same(pmap, va, va + NBPDR))
+ return (KERN_PROTECTION_FAILURE);
+ pde = pmap_pde(pmap, va);
+ if (pde == NULL) {
+ mp = _pmap_allocpte(pmap, pmap_pdpe_pindex(va),
+ NULL, va);
+ if (mp == NULL) {
+ if ((flags & PMAP_ENTER_NOSLEEP) != 0)
+ return (KERN_RESOURCE_SHORTAGE);
+ PMAP_UNLOCK(pmap);
+ vm_wait(NULL);
+ PMAP_LOCK(pmap);
+ goto restart;
+ }
+ pde = (pd_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(mp));
+ pde = &pde[pmap_pde_index(va)];
+ origpte = *pde;
+ MPASS(origpte == 0);
+ } else {
+ pdpe = pmap_pdpe(pmap, va);
+ MPASS(pdpe != NULL && (*pdpe & PG_V) != 0);
+ mp = PHYS_TO_VM_PAGE(*pdpe & PG_FRAME);
+ origpte = *pde;
+ if ((origpte & PG_V) == 0)
+ mp->ref_count++;
+ }
+ KASSERT((origpte & PG_V) == 0 || ((origpte & PG_PS) != 0 &&
+ (origpte & PG_FRAME) == (newpte & PG_FRAME)),
+ ("va %#lx changing 2M phys page pde %#lx newpte %#lx",
+ va, origpte, newpte));
+ if ((newpte & PG_W) != 0 && (origpte & PG_W) == 0)
+ pmap->pm_stats.wired_count += NBPDR / PAGE_SIZE;
+ else if ((newpte & PG_W) == 0 && (origpte & PG_W) != 0)
+ pmap->pm_stats.wired_count -= NBPDR / PAGE_SIZE;
+ *pde = newpte;
+ }
+ if ((origpte & PG_V) == 0)
+ pmap_resident_count_inc(pmap, pagesizes[psind] / PAGE_SIZE);
+
+ return (KERN_SUCCESS);
+}
+
/*
* Insert the given physical page (p) at
* the specified virtual address (v) in the
@@ -6554,6 +6667,13 @@ pmap_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, v
lock = NULL;
PMAP_LOCK(pmap);
+ if ((flags & PMAP_ENTER_LARGEPAGE) != 0) {
+ KASSERT((m->oflags & VPO_UNMANAGED) != 0,
+ ("managed largepage va %#lx flags %#x", va, flags));
+ rv = pmap_enter_largepage(pmap, va, newpte | PG_PS, flags,
+ psind);
+ goto out;
+ }
if (psind == 1) {
/* Assert the required virtual and physical alignment. */
KASSERT((va & PDRMASK) == 0, ("pmap_enter: va unaligned"));
Modified: head/sys/vm/pmap.h
==============================================================================
--- head/sys/vm/pmap.h Wed Sep 9 21:45:18 2020 (r365517)
+++ head/sys/vm/pmap.h Wed Sep 9 21:50:24 2020 (r365518)
@@ -106,6 +106,7 @@ extern vm_offset_t kernel_vm_end;
*/
#define PMAP_ENTER_NOSLEEP 0x00000100
#define PMAP_ENTER_WIRED 0x00000200
+#define PMAP_ENTER_LARGEPAGE 0x00000400
#define PMAP_ENTER_RESERVED 0xFF000000
/*
More information about the svn-src-head
mailing list