svn commit: r264590 - projects/amd64_xen_pv/sys/amd64/xen
Cherry G. Mathew
cherry at FreeBSD.org
Thu Apr 17 10:53:11 UTC 2014
Author: cherry
Date: Thu Apr 17 10:53:10 2014
New Revision: 264590
URL: http://svnweb.freebsd.org/changeset/base/264590
Log:
This commit brings us closer to native.
Desegregate pmap_pv.c and pmap.c
Bring in backing table functions from native and reduce dependency
on mmu_map.[ch]
Boot to single usermode breaks with this commit, however, this is
a good checkpoint to commit.
Next stop - direct map using 4K pages.
Approved by: gibbs (implicit)
Modified:
projects/amd64_xen_pv/sys/amd64/xen/pmap.c
projects/amd64_xen_pv/sys/amd64/xen/pmap_pv.c
Modified: projects/amd64_xen_pv/sys/amd64/xen/pmap.c
==============================================================================
--- projects/amd64_xen_pv/sys/amd64/xen/pmap.c Thu Apr 17 10:43:38 2014 (r264589)
+++ projects/amd64_xen_pv/sys/amd64/xen/pmap.c Thu Apr 17 10:53:10 2014 (r264590)
@@ -83,6 +83,9 @@
* SUCH DAMAGE.
*/
+
+#define AMD64_NPT_AWARE
+
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
@@ -138,13 +141,16 @@ __FBSDID("$FreeBSD$");
#include <sys/sched.h>
#include <vm/vm.h>
-#include <vm/vm_extern.h>
-#include <vm/vm_object.h>
-#include <vm/vm_page.h>
#include <vm/vm_param.h>
-#include <vm/vm_map.h>
#include <vm/vm_kern.h>
-#include <vm/pmap.h>
+#include <vm/vm_page.h>
+#include <vm/vm_map.h>
+#include <vm/vm_object.h>
+#include <vm/vm_extern.h>
+#include <vm/vm_pageout.h>
+#include <vm/vm_pager.h>
+#include <vm/vm_radix.h>
+#include <vm/vm_reserv.h>
#include <vm/uma.h>
#include <machine/md_var.h>
@@ -156,6 +162,162 @@ __FBSDID("$FreeBSD$");
#include <amd64/xen/mmu_map.h>
#include <amd64/xen/pmap_pv.h>
+static __inline boolean_t
+pmap_emulate_ad_bits(pmap_t pmap)
+{
+
+ return ((pmap->pm_flags & PMAP_EMULATE_AD_BITS) != 0);
+}
+
+static __inline pt_entry_t
+pmap_valid_bit(pmap_t pmap)
+{
+ pt_entry_t mask;
+
+ switch (pmap->pm_type) {
+ case PT_X86:
+ mask = X86_PG_V;
+ break;
+ case PT_EPT:
+ if (pmap_emulate_ad_bits(pmap))
+ mask = EPT_PG_EMUL_V;
+ else
+ mask = EPT_PG_READ;
+ break;
+ default:
+ panic("pmap_valid_bit: invalid pm_type %d", pmap->pm_type);
+ }
+
+ return (mask);
+}
+
+static __inline pt_entry_t
+pmap_rw_bit(pmap_t pmap)
+{
+ pt_entry_t mask;
+
+ switch (pmap->pm_type) {
+ case PT_X86:
+ mask = X86_PG_RW;
+ break;
+ case PT_EPT:
+ if (pmap_emulate_ad_bits(pmap))
+ mask = EPT_PG_EMUL_RW;
+ else
+ mask = EPT_PG_WRITE;
+ break;
+ default:
+ panic("pmap_rw_bit: invalid pm_type %d", pmap->pm_type);
+ }
+
+ return (mask);
+}
+
+static __inline pt_entry_t
+pmap_global_bit(pmap_t pmap)
+{
+ pt_entry_t mask;
+
+ switch (pmap->pm_type) {
+ case PT_X86:
+ mask = X86_PG_G;
+ break;
+ case PT_EPT:
+ mask = 0;
+ break;
+ default:
+ panic("pmap_global_bit: invalid pm_type %d", pmap->pm_type);
+ }
+
+ return (mask);
+}
+
+static __inline pt_entry_t
+pmap_accessed_bit(pmap_t pmap)
+{
+ pt_entry_t mask;
+
+ switch (pmap->pm_type) {
+ case PT_X86:
+ mask = X86_PG_A;
+ break;
+ case PT_EPT:
+ if (pmap_emulate_ad_bits(pmap))
+ mask = EPT_PG_READ;
+ else
+ mask = EPT_PG_A;
+ break;
+ default:
+ panic("pmap_accessed_bit: invalid pm_type %d", pmap->pm_type);
+ }
+
+ return (mask);
+}
+
+static __inline pt_entry_t
+pmap_modified_bit(pmap_t pmap)
+{
+ pt_entry_t mask;
+
+ switch (pmap->pm_type) {
+ case PT_X86:
+ mask = X86_PG_M;
+ break;
+ case PT_EPT:
+ if (pmap_emulate_ad_bits(pmap))
+ mask = EPT_PG_WRITE;
+ else
+ mask = EPT_PG_M;
+ break;
+ default:
+ panic("pmap_modified_bit: invalid pm_type %d", pmap->pm_type);
+ }
+
+ return (mask);
+}
+
+#ifdef PV_STATS
+#define PV_STAT(x) do { x ; } while (0)
+#else
+#define PV_STAT(x) do { } while (0)
+#endif
+
+#define pa_index(pa) ((pa) >> PDRSHIFT)
+#define pa_to_pvh(pa) (&pv_table[pa_index(pa)])
+
+#define NPV_LIST_LOCKS MAXCPU
+
+#define PHYS_TO_PV_LIST_LOCK(pa) \
+ (&pv_list_locks[pa_index(pa) % NPV_LIST_LOCKS])
+
+#define CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, pa) do { \
+ struct rwlock **_lockp = (lockp); \
+ struct rwlock *_new_lock; \
+ \
+ _new_lock = PHYS_TO_PV_LIST_LOCK(pa); \
+ if (_new_lock != *_lockp) { \
+ if (*_lockp != NULL) \
+ rw_wunlock(*_lockp); \
+ *_lockp = _new_lock; \
+ rw_wlock(*_lockp); \
+ } \
+} while (0)
+
+#define CHANGE_PV_LIST_LOCK_TO_VM_PAGE(lockp, m) \
+ CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, VM_PAGE_TO_PHYS(m))
+
+#define RELEASE_PV_LIST_LOCK(lockp) do { \
+ struct rwlock **_lockp = (lockp); \
+ \
+ if (*_lockp != NULL) { \
+ rw_wunlock(*_lockp); \
+ *_lockp = NULL; \
+ } \
+} while (0)
+
+#define VM_PAGE_TO_PV_LIST_LOCK(m) \
+ PHYS_TO_PV_LIST_LOCK(VM_PAGE_TO_PHYS(m))
+
extern vm_offset_t pa_index; /* from machdep.c */
extern unsigned long physfree; /* from machdep.c */
@@ -201,6 +363,23 @@ static u_int64_t DMPTphys; /* phys addr
static u_int64_t DMPDphys; /* phys addr of direct mapped level 2 */
static u_int64_t DMPDPphys; /* phys addr of direct mapped level 3 */
#endif /* DMAPSUPPORT || DMAP4KSUPPORT */
+static struct rwlock_padalign pvh_global_lock;
+
+/*
+ * Data for the pv entry allocation mechanism
+ */
+TAILQ_HEAD(pch, pv_chunk) pv_chunks = TAILQ_HEAD_INITIALIZER(pv_chunks);
+struct mtx pv_chunks_mutex;
+struct rwlock pv_list_locks[NPV_LIST_LOCKS];
+static struct md_page *pv_table;
+
+static int pmap_flags = 0; // XXX: PMAP_PDE_SUPERPAGE; /* flags for x86 pmaps */
+
+static pv_entry_t get_pv_entry(pmap_t pmap, struct rwlock **lockp);
+static vm_page_t reclaim_pv_chunk(pmap_t locked_pmap, struct rwlock **lockp);
+static int pmap_unuse_pt(pmap_t, vm_offset_t, pd_entry_t, struct spglist *);
+static void _pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m,
+ struct spglist *free);
static vm_paddr_t boot_ptphys; /* phys addr of start of
* kernel bootstrap tables
@@ -222,6 +401,133 @@ extern struct xenstore_domain_interface
extern vm_map_t pv_map;
+/********************/
+/* Inline functions */
+/********************/
+
+/* XXX: */
+
+#define MACH_TO_DMAP(_m) PHYS_TO_DMAP(xpmap_mtop(_m))
+
+/* Return a non-clipped PD index for a given VA */
+static __inline vm_pindex_t
+pmap_pde_pindex(vm_offset_t va)
+{
+ return (va >> PDRSHIFT);
+}
+
+
+/* Return various clipped indexes for a given VA */
+static __inline vm_pindex_t
+pmap_pte_index(vm_offset_t va)
+{
+
+ return ((va >> PAGE_SHIFT) & ((1ul << NPTEPGSHIFT) - 1));
+}
+
+static __inline vm_pindex_t
+pmap_pde_index(vm_offset_t va)
+{
+
+ return ((va >> PDRSHIFT) & ((1ul << NPDEPGSHIFT) - 1));
+}
+
+static __inline vm_pindex_t
+pmap_pdpe_index(vm_offset_t va)
+{
+
+ return ((va >> PDPSHIFT) & ((1ul << NPDPEPGSHIFT) - 1));
+}
+
+static __inline vm_pindex_t
+pmap_pml4e_index(vm_offset_t va)
+{
+
+ return ((va >> PML4SHIFT) & ((1ul << NPML4EPGSHIFT) - 1));
+}
+
+/* Return a pointer to the PML4 slot that corresponds to a VA */
+static __inline pml4_entry_t *
+pmap_pml4e(pmap_t pmap, vm_offset_t va)
+{
+
+ return (&pmap->pm_pml4[pmap_pml4e_index(va)]);
+}
+
+/* Return a pointer to the PDP slot that corresponds to a VA */
+static __inline pdp_entry_t *
+pmap_pml4e_to_pdpe(pml4_entry_t *pml4e, vm_offset_t va)
+{
+ pdp_entry_t *pdpe;
+
+ pdpe = (pdp_entry_t *)MACH_TO_DMAP(*pml4e & PG_FRAME);
+ return (&pdpe[pmap_pdpe_index(va)]);
+}
+
+/* Return a pointer to the PDP slot that corresponds to a VA */
+static __inline pdp_entry_t *
+pmap_pdpe(pmap_t pmap, vm_offset_t va)
+{
+ pml4_entry_t *pml4e;
+ pt_entry_t PG_V;
+
+ PG_V = pmap_valid_bit(pmap);
+ pml4e = pmap_pml4e(pmap, va);
+ if ((*pml4e & PG_V) == 0)
+ return (NULL);
+ return (pmap_pml4e_to_pdpe(pml4e, va));
+}
+
+/* Return a pointer to the PD slot that corresponds to a VA */
+static __inline pd_entry_t *
+pmap_pdpe_to_pde(pdp_entry_t *pdpe, vm_offset_t va)
+{
+ pd_entry_t *pde;
+
+ pde = (pd_entry_t *)MACH_TO_DMAP(*pdpe & PG_FRAME);
+ return (&pde[pmap_pde_index(va)]);
+}
+
+/* Return a pointer to the PD slot that corresponds to a VA */
+static __inline pd_entry_t *
+pmap_pde(pmap_t pmap, vm_offset_t va)
+{
+ pdp_entry_t *pdpe;
+ pt_entry_t PG_V;
+
+ PG_V = pmap_valid_bit(pmap);
+ pdpe = pmap_pdpe(pmap, va);
+ if (pdpe == NULL || (*pdpe & PG_V) == 0)
+ return (NULL);
+ return (pmap_pdpe_to_pde(pdpe, va));
+}
+
+/* Return a pointer to the PT slot that corresponds to a VA */
+static __inline pt_entry_t *
+pmap_pde_to_pte(pd_entry_t *pde, vm_offset_t va)
+{
+ pt_entry_t *pte;
+
+ pte = (pt_entry_t *)MACH_TO_DMAP(*pde & PG_FRAME);
+ return (&pte[pmap_pte_index(va)]);
+}
+
+/* Return a pointer to the PT slot that corresponds to a VA */
+static __inline pt_entry_t *
+pmap_pte(pmap_t pmap, vm_offset_t va)
+{
+ pd_entry_t *pde;
+ pt_entry_t PG_V;
+
+ PG_V = pmap_valid_bit(pmap);
+ pde = pmap_pde(pmap, va);
+ if (pde == NULL || (*pde & PG_V) == 0)
+ return (NULL);
+ if ((*pde & PG_PS) != 0) /* compat with i386 pmap_pte() */
+ return ((pt_entry_t *)pde);
+ return (pmap_pde_to_pte(pde, va));
+}
+
/* Index offset into a pagetable, for a given va */
static int
pt_index(uintptr_t va)
@@ -229,10 +535,119 @@ pt_index(uintptr_t va)
return ((va & PDRMASK) >> PAGE_SHIFT);
}
+static __inline void
+pmap_resident_count_inc(pmap_t pmap, int count)
+{
+
+ PMAP_LOCK_ASSERT(pmap, MA_OWNED);
+ pmap->pm_stats.resident_count += count;
+}
+
+static __inline void
+pmap_resident_count_dec(pmap_t pmap, int count)
+{
+
+ PMAP_LOCK_ASSERT(pmap, MA_OWNED);
+ pmap->pm_stats.resident_count -= count;
+}
+
+/* XXX: Remove after merge */
+static pt_entry_t *
+pmap_vtopte_inspect(pmap_t pmap, uintptr_t va, void *addr)
+{
+ KASSERT(addr != NULL, ("addr == NULL"));
+
+ mmu_map_t tptr = *(mmu_map_t *)addr;
+
+ pd_entry_t *pte; /* PTE address to return */
+
+ struct mmu_map_mbackend mb = {
+ ptmb_mappedalloc,
+ ptmb_mappedfree,
+ ptmb_ptov,
+ ptmb_vtop
+ };
+
+ mmu_map_t_init(tptr, &mb);
+
+ if (!mmu_map_inspect_va(pmap, tptr, va)) {
+ return NULL; /* XXX: fix api, return some kind of #define */
+ }
+
+ pte = mmu_map_pt(tptr); /* Read out PT from mmu state */
+
+ /* add VA offset */
+ pte += pt_index(va);
+
+ return pte;
+}
+
+static pt_entry_t *
+vtopte_inspect(uintptr_t va, void *addr)
+{
+ return pmap_vtopte_inspect(kernel_pmap, va, addr);
+}
+
+static pt_entry_t *
+pmap_vtopte_hold(pmap_t pmap, uintptr_t va, void *addr)
+{
+ KASSERT(addr != NULL, ("addr == NULL"));
+
+ mmu_map_t tptr = *(mmu_map_t *)addr;
+
+ pd_entry_t *pte; /* PTE address to return */
+
+ struct mmu_map_mbackend mb = {
+ ptmb_mappedalloc,
+ ptmb_mappedfree,
+ ptmb_ptov,
+ ptmb_vtop
+ };
+
+ mmu_map_t_init(tptr, &mb);
+
+ if (!mmu_map_inspect_va(pmap, tptr, va)) {
+ mmu_map_hold_va(pmap, tptr, va); /* PT hierarchy */
+ xen_flush_queue();
+ }
+
+ pte = mmu_map_pt(tptr); /* Read out PT from mmu state */
+
+ /* add VA offset */
+ pte += pt_index(va);
+
+ return pte;
+}
+
+pt_entry_t *
+vtopte_hold(uintptr_t va, void *addr)
+{
+ return pmap_vtopte_hold(kernel_pmap, va, addr);
+}
+
+static void
+pmap_vtopte_release(pmap_t pmap, uintptr_t va, void *addr)
+{
+ mmu_map_t tptr = *(mmu_map_t *)addr;
+
+ mmu_map_release_va(pmap, tptr, va);
+ mmu_map_t_fini(tptr);
+
+}
+
+void
+vtopte_release(uintptr_t va, void *addr)
+{
+ pmap_vtopte_release(kernel_pmap, va, addr);
+}
+
+/* -> XXX: */
+
+
/* return kernel virtual address of 'n' claimed physical pages at boot. */
static uintptr_t
-vallocpages(vm_paddr_t *firstaddr, int n)
+allocpages(vm_paddr_t *firstaddr, int n)
{
uintptr_t ret = *firstaddr + KERNBASE;
bzero((void *)ret, n * PAGE_SIZE);
@@ -258,6 +673,10 @@ static void
pmap_xen_setpages_ro(uintptr_t va, vm_size_t npages)
{
vm_size_t i;
+ pt_entry_t PG_V;
+
+ PG_V = pmap_valid_bit(kernel_pmap);
+
for (i = 0; i < npages; i++) {
PT_SET_MA(va + PAGE_SIZE * i,
phystomach(ptmb_vtop(va + PAGE_SIZE * i)) | PG_U | PG_V);
@@ -269,6 +688,12 @@ static void
pmap_xen_setpages_rw(uintptr_t va, vm_size_t npages)
{
vm_size_t i;
+
+ pt_entry_t PG_V, PG_RW;
+
+ PG_V = pmap_valid_bit(kernel_pmap);
+ PG_RW = pmap_rw_bit(kernel_pmap);
+
for (i = 0; i < npages; i++) {
PT_SET_MA(va + PAGE_SIZE * i,
phystomach(ptmb_vtop(va + PAGE_SIZE * i)) | PG_U | PG_V | PG_RW);
@@ -281,6 +706,9 @@ extern int end; /* End of kernel binary
static pt_entry_t
pmap_xen_kernel_vaflags(uintptr_t va)
{
+ pt_entry_t PG_RW;
+ PG_RW = pmap_rw_bit(kernel_pmap);
+
if ((va > (uintptr_t) &etext && /* .data, .bss et. al */
(va < (uintptr_t) &end))
||
@@ -297,7 +725,7 @@ pmap_xen_kernel_vaflags(uintptr_t va)
uintptr_t tmpva;
static void
-create_boot_pagetables(vm_paddr_t *firstaddr)
+create_pagetables(vm_paddr_t *firstaddr)
{
int i;
int nkpdpe;
@@ -310,23 +738,23 @@ create_boot_pagetables(vm_paddr_t *first
/* Allocate pseudo-physical pages for kernel page tables. */
nkpt = howmany(nkmapped, NPTEPG);
nkpdpe = howmany(nkpt, NPDEPG);
- KPML4phys = vallocpages(firstaddr, 1);
- KPDPphys = vallocpages(firstaddr, NKPML4E);
- KPDphys = vallocpages(firstaddr, nkpdpe);
- KPTphys = vallocpages(firstaddr, nkpt);
+ KPML4phys = allocpages(firstaddr, 1);
+ KPDPphys = allocpages(firstaddr, NKPML4E);
+ KPDphys = allocpages(firstaddr, nkpdpe);
+ KPTphys = allocpages(firstaddr, nkpt);
#ifdef DMAPSUPPORT
int ndm1g;
ndmpdp = (ptoa(Maxmem) + NBPDP - 1) >> PDPSHIFT;
if (ndmpdp < 4) /* Minimum 4GB of dirmap */
ndmpdp = 4;
- DMPDPphys = vallocpages(firstaddr, NDMPML4E);
+ DMPDPphys = allocpages(firstaddr, NDMPML4E);
ndm1g = 0;
if ((amd_feature & AMDID_PAGE1GB) != 0)
ndm1g = ptoa(Maxmem) >> PDPSHIFT;
if (ndm1g < ndmpdp)
- DMPDphys = vallocpages(firstaddr, ndmpdp - ndm1g);
+ DMPDphys = allocpages(firstaddr, ndmpdp - ndm1g);
dmaplimit = (vm_paddr_t)ndmpdp << PDPSHIFT;
#endif /* DMAPSUPPORT */
#ifdef DMAP4KSUPPORT
@@ -335,13 +763,13 @@ create_boot_pagetables(vm_paddr_t *first
int ndmpdpe = howmany(ndmpt, NPDEPG);
tmpva = ptoa(ndmapped);
- DMPDPphys = vallocpages(firstaddr, NDMPML4E);
- DMPDphys = vallocpages(firstaddr, ndmpdpe);
- DMPTphys = vallocpages(firstaddr, ndmpt);
+ DMPDPphys = allocpages(firstaddr, NDMPML4E);
+ DMPDphys = allocpages(firstaddr, ndmpdpe);
+ DMPTphys = allocpages(firstaddr, ndmpt);
for (i = 0;ptoa(i) < ptoa(ndmapped); i++) {
((pt_entry_t *)DMPTphys)[i] = phystomach(i << PAGE_SHIFT);
- ((pt_entry_t *)DMPTphys)[i] |= PG_V | PG_U;
+ ((pt_entry_t *)DMPTphys)[i] |= X86_PG_V | X86_PG_U;
}
pmap_xen_setpages_ro(DMPTphys, (i - 1) / NPTEPG + 1);
@@ -349,7 +777,7 @@ create_boot_pagetables(vm_paddr_t *first
for (i = 0; i < ndmpt; i ++) {
((pd_entry_t *)DMPDphys)[i] = phystomach(VTOP(DMPTphys) +
(i << PAGE_SHIFT));
- ((pd_entry_t *)DMPDphys)[i] |= PG_V | PG_U;
+ ((pd_entry_t *)DMPDphys)[i] |= X86_PG_V | X86_PG_U | X86_PG_RW;
}
@@ -358,7 +786,7 @@ create_boot_pagetables(vm_paddr_t *first
for (i = 0; i < ndmpdpe; i++) {
((pdp_entry_t *)DMPDPphys)[i] = phystomach(VTOP(DMPDphys) +
(i << PAGE_SHIFT));
- ((pdp_entry_t *)DMPDPphys)[i] |= PG_V | PG_U;
+ ((pdp_entry_t *)DMPDPphys)[i] |= X86_PG_V | X86_PG_U | X86_PG_RW;
}
pmap_xen_setpages_ro(DMPDPphys, (ndmpdpe - 1) / NPDPEPG + 1);
@@ -367,7 +795,7 @@ create_boot_pagetables(vm_paddr_t *first
for (i = 0; i < NDMPML4E - 1; i++) {
((pdp_entry_t *)KPML4phys)[DMPML4I + i] = phystomach(VTOP(DMPDPphys) +
(i << PAGE_SHIFT));
- ((pdp_entry_t *)KPML4phys)[DMPML4I + i] |= PG_V | PG_U;
+ ((pdp_entry_t *)KPML4phys)[DMPML4I + i] |= X86_PG_V | X86_PG_U | X86_PG_RW;
}
#endif /* DMAP4KSUPPORT */
@@ -380,7 +808,7 @@ create_boot_pagetables(vm_paddr_t *first
/* Fill in the underlying page table pages */
for (i = 0; ptoa(i) < ptoa(nkmapped); i++) {
((pt_entry_t *)KPTphys)[i] = phystomach(i << PAGE_SHIFT);
- ((pt_entry_t *)KPTphys)[i] |= PG_V | PG_U;
+ ((pt_entry_t *)KPTphys)[i] |= X86_PG_V | X86_PG_U;
((pt_entry_t *)KPTphys)[i] |=
pmap_xen_kernel_vaflags(PTOV(i << PAGE_SHIFT));
}
@@ -391,7 +819,7 @@ create_boot_pagetables(vm_paddr_t *first
for (i = 0; i < nkpt; i++) {
((pd_entry_t *)KPDphys)[i] = phystomach(VTOP(KPTphys) +
(i << PAGE_SHIFT));
- ((pd_entry_t *)KPDphys)[i] |= PG_RW | PG_V | PG_U;
+ ((pd_entry_t *)KPDphys)[i] |= X86_PG_RW | X86_PG_V | X86_PG_U;
}
@@ -400,7 +828,7 @@ create_boot_pagetables(vm_paddr_t *first
/* This replaces some of the KPTphys entries above */
for (i = 0; (i << PDRSHIFT) < *firstaddr; i++) {
((pd_entry_t *)KPDphys)[i] = phystomach(i << PDRSHIFT);
- ((pd_entry_t *)KPDphys)[i] |= PG_U | PG_RW | PG_V | PG_PS | PG_G;
+ ((pd_entry_t *)KPDphys)[i] |= X86_PG_U | X86_PG_RW | X86_PG_V | X86_PG_PS | X86_PG_G;
}
#endif
@@ -410,7 +838,7 @@ create_boot_pagetables(vm_paddr_t *first
for (i = 0; i < nkpdpe; i++) {
((pdp_entry_t *)KPDPphys)[i + KPDPI] = phystomach(VTOP(KPDphys) +
(i << PAGE_SHIFT));
- ((pdp_entry_t *)KPDPphys)[i + KPDPI] |= PG_RW | PG_V | PG_U;
+ ((pdp_entry_t *)KPDPphys)[i + KPDPI] |= X86_PG_RW | X86_PG_V | X86_PG_U;
}
pmap_xen_setpages_ro(KPDPphys, (nkpdpe - 1) / NPDPEPG + 1);
@@ -431,7 +859,7 @@ create_boot_pagetables(vm_paddr_t *first
if ((i << PDRSHIFT) > ptoa(Maxmem)) {
/*
* Since the page is zeroed out at
- * vallocpages(), the remaining ptes will be
+ * allocpages(), the remaining ptes will be
* invalid.
*/
@@ -439,8 +867,8 @@ create_boot_pagetables(vm_paddr_t *first
}
((pd_entry_t *)DMPDphys)[j] = (vm_paddr_t)(phystomach(i << PDRSHIFT));
/* Preset PG_M and PG_A because demotion expects it. */
- ((pd_entry_t *)DMPDphys)[j] |= PG_U | PG_V | PG_PS /* | PG_G */ |
- PG_M | PG_A;
+ ((pd_entry_t *)DMPDphys)[j] |= X86_PG_U | X86_PG_V | X86_PG_PS /* | X86_PG_G */ |
+ X86_PG_M | X86_PG_A;
}
/* Mark pages R/O */
pmap_xen_setpages_ro(DMPDphys, ndmpdp - ndm1g);
@@ -450,7 +878,7 @@ create_boot_pagetables(vm_paddr_t *first
if ((i << PDPSHIFT) > ptoa(Maxmem)) {
/*
* Since the page is zeroed out at
- * vallocpages(), the remaining ptes will be
+ * allocpages(), the remaining ptes will be
* invalid.
*/
@@ -459,13 +887,13 @@ create_boot_pagetables(vm_paddr_t *first
((pdp_entry_t *)DMPDPphys)[i] = (vm_paddr_t)phystomach(i << PDPSHIFT);
/* Preset PG_M and PG_A because demotion expects it. */
- ((pdp_entry_t *)DMPDPphys)[i] |= PG_U | PG_V | PG_PS | PG_G |
- PG_M | PG_A;
+ ((pdp_entry_t *)DMPDPphys)[i] |= X86_PG_U | X86_PG_V | X86_PG_PS | X86_PG_G |
+ X86_PG_M | X86_PG_A;
}
for (j = 0; i < ndmpdp; i++, j++) {
((pdp_entry_t *)DMPDPphys)[i] = phystomach(VTOP(DMPDphys) + (j << PAGE_SHIFT));
- ((pdp_entry_t *)DMPDPphys)[i] |= PG_V | PG_U;
+ ((pdp_entry_t *)DMPDPphys)[i] |= X86_PG_V | X86_PG_U;
}
pmap_xen_setpages_ro(DMPDPphys, NDMPML4E);
@@ -474,17 +902,17 @@ create_boot_pagetables(vm_paddr_t *first
for (i = 0; i < NDMPML4E; i++) {
((pdp_entry_t *)KPML4phys)[DMPML4I + i] = phystomach(VTOP(DMPDPphys) +
(i << PAGE_SHIFT));
- ((pdp_entry_t *)KPML4phys)[DMPML4I + i] |= PG_V | PG_U;
+ ((pdp_entry_t *)KPML4phys)[DMPML4I + i] |= X86_PG_V | X86_PG_U;
}
#endif /* DMAPSUPPORT */
/* And recursively map PML4 to itself in order to get PTmap */
((pdp_entry_t *)KPML4phys)[PML4PML4I] = phystomach(VTOP(KPML4phys));
- ((pdp_entry_t *)KPML4phys)[PML4PML4I] |= PG_V | PG_U;
+ ((pdp_entry_t *)KPML4phys)[PML4PML4I] |= X86_PG_V | X86_PG_U;
/* Connect the KVA slot up to the PML4 */
((pdp_entry_t *)KPML4phys)[KPML4I] = phystomach(VTOP(KPDPphys));
- ((pdp_entry_t *)KPML4phys)[KPML4I] |= PG_RW | PG_V | PG_U;
+ ((pdp_entry_t *)KPML4phys)[KPML4I] |= X86_PG_RW | X86_PG_V | X86_PG_U;
pmap_xen_setpages_ro(KPML4phys, 1);
@@ -504,6 +932,11 @@ pmap_xen_bootpages(vm_paddr_t *firstaddr
uintptr_t va;
vm_paddr_t ma;
+ pt_entry_t PG_V, PG_RW;
+
+ PG_V = pmap_valid_bit(kernel_pmap);
+ PG_RW = pmap_rw_bit(kernel_pmap);
+
/* i) Share info */
ma = xen_start_info->shared_info;
@@ -523,14 +956,14 @@ pmap_xen_bootpages(vm_paddr_t *firstaddr
* bootup).
*/
- va = vallocpages(firstaddr, 1);
+ va = allocpages(firstaddr, 1);
PT_SET_MA(va, ma | PG_RW | PG_V | PG_U);
HYPERVISOR_shared_info = (void *) va;
#if 0
/* ii) Userland page table base */
- va = vallocpages(firstaddr, 1);
+ va = allocpages(firstaddr, 1);
bzero((void *)va, PAGE_SIZE);
/*
@@ -572,7 +1005,7 @@ mmu_alloc(void)
KASSERT(physfree != 0,
("physfree must have been set before using mmu_alloc"));
- va = vallocpages(&physfree, atop(PAGE_SIZE));
+ va = allocpages(&physfree, atop(PAGE_SIZE));
/*
* Xen requires the page table hierarchy to be R/O.
@@ -593,7 +1026,7 @@ pmap_bootstrap(vm_paddr_t *firstaddr)
ptmb_ptov = boot_ptov;
ptmb_vtop = boot_vtop;
- create_boot_pagetables(firstaddr);
+ create_pagetables(firstaddr);
/* Switch to the new kernel tables */
xen_pt_switch(xpmap_ptom(VTOP(KPML4phys)));
@@ -623,7 +1056,7 @@ pmap_bootstrap(vm_paddr_t *firstaddr)
/*
* Xen guarantees mapped virtual addresses at boot time upto
- * xenstack + 512KB. We want to use these for vallocpages()
+ * xenstack + 512KB. We want to use these for allocpages()
* and therefore don't want to touch these mappings since
* they're scarce resources. Move along to the end of
* guaranteed mapping.
@@ -640,13 +1073,25 @@ pmap_bootstrap(vm_paddr_t *firstaddr)
/*
* Initialize the kernel pmap (which is statically allocated).
*/
+
PMAP_LOCK_INIT(kernel_pmap);
+
kernel_pmap->pm_pml4 = (pdp_entry_t *)KPML4phys;
- kernel_pmap->pm_cr3 = pmap_kextract_ma((vm_offset_t) KPML4phys);
+ kernel_pmap->pm_cr3 = xpmap_ptom((vm_offset_t) VTOP(KPML4phys));
kernel_pmap->pm_root.rt_root = 0;
+
CPU_FILL(&kernel_pmap->pm_active); /* don't allow deactivation */
+
+ CPU_ZERO(&kernel_pmap->pm_save);
+ TAILQ_INIT(&kernel_pmap->pm_pvchunk);
+ kernel_pmap->pm_flags = pmap_flags;
+
+ /*
+ * Initialize the global pv list lock.
+ */
+ rw_init(&pvh_global_lock, "pmap pv global");
+
pmap_pv_init();
- pmap_pv_pmap_init(kernel_pmap);
tsz = mmu_map_t_size();
@@ -669,66 +1114,24 @@ pmap_page_init(vm_page_t m)
pmap_pv_vm_page_init(m);
}
-/*
- * Map in backing memory from kernel_vm_end to addr,
- * and update kernel_vm_end.
+/*
+ * Initialize the pmap module.
+ * Called by vm_init, to initialize any structures that the pmap
+ * system needs to map virtual memory.
*/
+
void
-pmap_growkernel(uintptr_t addr)
+pmap_init(void)
{
- KASSERT(kernel_vm_end < addr, ("trying to shrink kernel VA!"));
-
- addr = trunc_page(addr);
+ uintptr_t va;
- char tbuf[tsz]; /* Safe to do this on the stack since tsz is
- * effectively const.
- */
+ /* XXX: review the use of gdtset for the purpose below */
- mmu_map_t tptr = tbuf;
+ /* Get a va for console and map the console mfn into it */
+ vm_paddr_t ma = xen_start_info->console.domU.mfn << PAGE_SHIFT;
- struct mmu_map_mbackend mb = {
- ptmb_mappedalloc,
- ptmb_mappedfree,
- ptmb_ptov,
- ptmb_vtop
- };
-
- mmu_map_t_init(tptr, &mb);
-
- for (;addr <= kernel_vm_end;addr += PAGE_SIZE) {
-
- if (mmu_map_inspect_va(kernel_pmap, tptr, addr)) {
- continue;
- }
- int pflags = VM_ALLOC_INTERRUPT | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED;
- vm_page_t m = vm_page_alloc(NULL, 0, pflags);
- KASSERT(m != NULL, ("Backing page alloc failed!"));
- vm_paddr_t pa = VM_PAGE_TO_PHYS(m);
-
- pmap_kenter(addr, pa);
- }
-
- mmu_map_t_fini(tptr);
-}
-
-/*
- * Initialize the pmap module.
- * Called by vm_init, to initialize any structures that the pmap
- * system needs to map virtual memory.
- */
-
-void
-pmap_init(void)
-{
- uintptr_t va;
-
- /* XXX: review the use of gdtset for the purpose below */
-
- /* Get a va for console and map the console mfn into it */
- vm_paddr_t ma = xen_start_info->console.domU.mfn << PAGE_SHIFT;
-
- va = kva_alloc(PAGE_SIZE);
- KASSERT(va != 0, ("Could not allocate KVA for console page!\n"));
+ va = kva_alloc(PAGE_SIZE);
+ KASSERT(va != 0, ("Could not allocate KVA for console page!\n"));
pmap_kenter_ma(va, ma);
console_page = (void *)va;
@@ -750,142 +1153,169 @@ pmap_init(void)
}
void
-pmap_pinit0(pmap_t pmap)
+pmap_invalidate_page(pmap_t pmap, vm_offset_t va)
{
- PMAP_LOCK_INIT(pmap);
- pmap->pm_pml4 = (void *) KPML4phys;
- pmap->pm_cr3 = pmap_kextract_ma((vm_offset_t) KPML4phys);
- pmap->pm_root.rt_root = 0;
- CPU_ZERO(&pmap->pm_active);
- CPU_ZERO(&pmap->pm_save);
- PCPU_SET(curpmap, pmap);
- pmap_pv_pmap_init(pmap);
- bzero(&pmap->pm_stats, sizeof pmap->pm_stats);
- pmap->pm_pcid = -1;
+ pmap_invalidate_range(pmap, va, va + PAGE_SIZE);
}
-int
-pmap_pinit(pmap_t pmap)
+void
+pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
{
- KASSERT(pmap != kernel_pmap,
- ("%s: kernel map re-initialised!", __func__));
+ vm_offset_t addr;
+ /* XXX: TODO SMP */
+ sched_pin();
- /*
- * allocate the page directory page
- */
- pmap->pm_pml4 = (void *) kmem_malloc(kernel_arena, PAGE_SIZE, M_ZERO);
- if (pmap->pm_pml4 == NULL) return 0;
+ for (addr = sva; addr < eva; addr += PAGE_SIZE)
+ invlpg(addr);
- pmap->pm_cr3 = pmap_kextract_ma((vm_offset_t)pmap->pm_pml4);
+ sched_unpin();
+}
- /*
- * We do not wire in kernel space, or the self-referencial
- * entry in userspace pmaps becase both kernel and userland
- * share ring3 privilege. The user/kernel context switch is
- * arbitrated by the hypervisor by means of pre-loaded values
- * for kernel and user %cr3. The userland parts of kernel VA
- * may be conditionally overlaid with the VA of curthread,
- * since the kernel occasionally needs to access userland
- * process VA space.
- */
+void
+pmap_invalidate_all(pmap_t pmap)
+{
- pmap_xen_setpages_ro((uintptr_t)pmap->pm_pml4, 1);
+ if (pmap == kernel_pmap || !CPU_EMPTY(&pmap->pm_active))
+ invltlb();
+}
- xen_pgdir_pin(pmap->pm_cr3);
+/*
+ * Routine: pmap_extract
+ * Function:
+ * Extract the physical page address associated
+ * with the given map/virtual_address pair.
+ */
- pmap->pm_root.rt_root = 0;
- CPU_ZERO(&pmap->pm_active);
- pmap_pv_pmap_init(pmap);
- bzero(&pmap->pm_stats, sizeof pmap->pm_stats);
- pmap->pm_pcid = -1; /* No pcid for now */
- CPU_ZERO(&pmap->pm_save);
+vm_paddr_t
+pmap_extract(pmap_t pmap, vm_offset_t va)
+{
- return 1;
-}
+ pt_entry_t PG_V;
-void pmap_xen_userload(pmap_t pmap)
-{
- KASSERT(pmap != kernel_pmap,
- ("Kernel pmap requested on user load.\n"));
+ PG_V = pmap_valid_bit(pmap);
- int i;
- for (i = 0; i < NUPML4E; i++) {
- pml4_entry_t pml4e;
- pml4e = (pmap->pm_pml4[i]);
- PT_SET_VA_MA((pml4_entry_t *)KPML4phys + i, pml4e, false);
+ if (pmap == kernel_pmap) {
+ return pmap_kextract(va);
}
- PT_UPDATES_FLUSH();
- invltlb();
- /* Tell xen about user pmap switch */
- xen_pt_user_switch(pmap->pm_cr3);
-}
+ pt_entry_t *pte;
+ vm_paddr_t ma = 0;
-void
-pmap_release(pmap_t pmap)
-{
- KASSERT(pmap != kernel_pmap,
- ("%s: kernel pmap released", __func__));
+
+ /* Walk the PT hierarchy to get the ma */
+ char tbuf[tsz]; /* Safe to do this on the stack since tsz is
+ * effectively const.
+ */
- KASSERT(pmap->pm_stats.resident_count == 0,
- ("pmap_release: pmap resident count %ld != 0",
- pmap->pm_stats.resident_count));
+ mmu_map_t tptr = tbuf;
- KASSERT(vm_radix_is_empty(&pmap->pm_root),
- ("pmap_release: pmap has reserved page table page(s)"));
+ pte = pmap_vtopte_inspect(pmap, va, &tptr);
- xen_pgdir_unpin(pmap->pm_cr3);
- pmap_xen_setpages_rw((uintptr_t)pmap->pm_pml4, 1);
+ if (pte != NULL && (*pte & PG_V)) {
+ ma = (*pte & PG_FRAME) | (va & PAGE_MASK);
+ }
- bzero(pmap->pm_pml4, PAGE_SIZE);
- kmem_free(kernel_arena, (vm_offset_t)pmap->pm_pml4, PAGE_SIZE);
+ pmap_vtopte_release(pmap, va, &tptr);
+
+ return xpmap_mtop(ma);
}
*** DIFF OUTPUT TRUNCATED AT 1000 LINES ***
More information about the svn-src-projects
mailing list