svn commit: r360887 - in head/sys: conf powerpc/aim powerpc/booke powerpc/include powerpc/powerpc vm
Justin Hibbits
jhibbits at FreeBSD.org
Mon May 11 02:33:39 UTC 2020
Author: jhibbits
Date: Mon May 11 02:33:37 2020
New Revision: 360887
URL: https://svnweb.freebsd.org/changeset/base/360887
Log:
powerpc64: Implement Radix MMU for POWER9 CPUs
Summary:
POWER9 supports two MMU formats: traditional hashed page tables, and Radix
page tables, similar to what's presesnt on most other architectures. The
PowerISA also specifies a process table -- a table of page table pointers--
which on the POWER9 is only available with the Radix MMU, so we can take
advantage of it with the Radix MMU driver.
Written by Matt Macy.
Differential Revision: https://reviews.freebsd.org/D19516
Added:
head/sys/powerpc/aim/mmu_radix.c (contents, props changed)
Modified:
head/sys/conf/files.powerpc
head/sys/powerpc/aim/aim_machdep.c
head/sys/powerpc/aim/mmu_oea.c
head/sys/powerpc/aim/mmu_oea64.c
head/sys/powerpc/booke/pmap.c
head/sys/powerpc/include/cpufunc.h
head/sys/powerpc/include/mmuvar.h
head/sys/powerpc/include/param.h
head/sys/powerpc/include/pmap.h
head/sys/powerpc/include/proc.h
head/sys/powerpc/include/pte.h
head/sys/powerpc/include/spr.h
head/sys/powerpc/include/sr.h
head/sys/powerpc/include/vmparam.h
head/sys/powerpc/powerpc/machdep.c
head/sys/powerpc/powerpc/mmu_if.m
head/sys/powerpc/powerpc/pmap_dispatch.c
head/sys/powerpc/powerpc/trap.c
head/sys/vm/vm_fault.c
Modified: head/sys/conf/files.powerpc
==============================================================================
--- head/sys/conf/files.powerpc Mon May 11 01:20:40 2020 (r360886)
+++ head/sys/conf/files.powerpc Mon May 11 02:33:37 2020 (r360887)
@@ -135,6 +135,7 @@ powerpc/aim/locore.S optional aim no-obj
powerpc/aim/aim_machdep.c optional aim
powerpc/aim/mmu_oea.c optional aim powerpc
powerpc/aim/mmu_oea64.c optional aim
+powerpc/aim/mmu_radix.c optional aim powerpc64
powerpc/aim/moea64_if.m optional aim
powerpc/aim/moea64_native.c optional aim
powerpc/aim/mp_cpudep.c optional aim
Modified: head/sys/powerpc/aim/aim_machdep.c
==============================================================================
--- head/sys/powerpc/aim/aim_machdep.c Mon May 11 01:20:40 2020 (r360886)
+++ head/sys/powerpc/aim/aim_machdep.c Mon May 11 02:33:37 2020 (r360887)
@@ -136,6 +136,8 @@ __FBSDID("$FreeBSD$");
struct bat battable[16];
#endif
+int radix_mmu = 0;
+
#ifndef __powerpc64__
/* Bits for running on 64-bit systems in 32-bit mode. */
extern void *testppc64, *testppc64size;
@@ -451,7 +453,14 @@ aim_cpu_init(vm_offset_t toc)
* in case the platform module had a better idea of what we
* should do.
*/
- if (cpu_features & PPC_FEATURE_64)
+ if (cpu_features2 & PPC_FEATURE2_ARCH_3_00) {
+ radix_mmu = 0;
+ TUNABLE_INT_FETCH("radix_mmu", &radix_mmu);
+ if (radix_mmu)
+ pmap_mmu_install(MMU_TYPE_RADIX, BUS_PROBE_GENERIC);
+ else
+ pmap_mmu_install(MMU_TYPE_G5, BUS_PROBE_GENERIC);
+ } else if (cpu_features & PPC_FEATURE_64)
pmap_mmu_install(MMU_TYPE_G5, BUS_PROBE_GENERIC);
else
pmap_mmu_install(MMU_TYPE_OEA, BUS_PROBE_GENERIC);
Modified: head/sys/powerpc/aim/mmu_oea.c
==============================================================================
--- head/sys/powerpc/aim/mmu_oea.c Mon May 11 01:20:40 2020 (r360886)
+++ head/sys/powerpc/aim/mmu_oea.c Mon May 11 02:33:37 2020 (r360887)
@@ -322,6 +322,7 @@ void moea_dumpsys_map(mmu_t mmu, vm_paddr_t pa, size_t
void moea_scan_init(mmu_t mmu);
vm_offset_t moea_quick_enter_page(mmu_t mmu, vm_page_t m);
void moea_quick_remove_page(mmu_t mmu, vm_offset_t addr);
+boolean_t moea_page_is_mapped(mmu_t mmu, vm_page_t m);
static int moea_map_user_ptr(mmu_t mmu, pmap_t pm,
volatile const void *uaddr, void **kaddr, size_t ulen, size_t *klen);
static int moea_decode_kernel_ptr(mmu_t mmu, vm_offset_t addr,
@@ -364,6 +365,7 @@ static mmu_method_t moea_methods[] = {
MMUMETHOD(mmu_page_set_memattr, moea_page_set_memattr),
MMUMETHOD(mmu_quick_enter_page, moea_quick_enter_page),
MMUMETHOD(mmu_quick_remove_page, moea_quick_remove_page),
+ MMUMETHOD(mmu_page_is_mapped, moea_page_is_mapped),
/* Internal interfaces */
MMUMETHOD(mmu_bootstrap, moea_bootstrap),
@@ -1102,6 +1104,12 @@ moea_quick_enter_page(mmu_t mmu, vm_page_t m)
void
moea_quick_remove_page(mmu_t mmu, vm_offset_t addr)
{
+}
+
+boolean_t
+moea_page_is_mapped(mmu_t mmu, vm_page_t m)
+{
+ return (!LIST_EMPTY(&(m)->md.mdpg_pvoh));
}
/*
Modified: head/sys/powerpc/aim/mmu_oea64.c
==============================================================================
--- head/sys/powerpc/aim/mmu_oea64.c Mon May 11 01:20:40 2020 (r360886)
+++ head/sys/powerpc/aim/mmu_oea64.c Mon May 11 02:33:37 2020 (r360887)
@@ -120,8 +120,7 @@ uintptr_t moea64_get_unique_vsid(void);
*
*/
-#define PV_LOCK_PER_DOM (PA_LOCK_COUNT * 3)
-#define PV_LOCK_COUNT (PV_LOCK_PER_DOM * MAXMEMDOM)
+#define PV_LOCK_COUNT PA_LOCK_COUNT
static struct mtx_padalign pv_lock[PV_LOCK_COUNT];
/*
@@ -130,8 +129,7 @@ static struct mtx_padalign pv_lock[PV_LOCK_COUNT];
* index at (N << 45).
*/
#ifdef __powerpc64__
-#define PV_LOCK_IDX(pa) (pa_index(pa) % PV_LOCK_PER_DOM + \
- (((pa) >> 45) % MAXMEMDOM) * PV_LOCK_PER_DOM)
+#define PV_LOCK_IDX(pa) ((pa_index(pa) * (((pa) >> 45) + 1)) % PV_LOCK_COUNT)
#else
#define PV_LOCK_IDX(pa) (pa_index(pa) % PV_LOCK_COUNT)
#endif
@@ -305,6 +303,7 @@ void moea64_dumpsys_map(mmu_t mmu, vm_paddr_t pa, size
void moea64_scan_init(mmu_t mmu);
vm_offset_t moea64_quick_enter_page(mmu_t mmu, vm_page_t m);
void moea64_quick_remove_page(mmu_t mmu, vm_offset_t addr);
+boolean_t moea64_page_is_mapped(mmu_t mmu, vm_page_t m);
static int moea64_map_user_ptr(mmu_t mmu, pmap_t pm,
volatile const void *uaddr, void **kaddr, size_t ulen, size_t *klen);
static int moea64_decode_kernel_ptr(mmu_t mmu, vm_offset_t addr,
@@ -353,6 +352,7 @@ static mmu_method_t moea64_methods[] = {
MMUMETHOD(mmu_page_set_memattr, moea64_page_set_memattr),
MMUMETHOD(mmu_quick_enter_page, moea64_quick_enter_page),
MMUMETHOD(mmu_quick_remove_page, moea64_quick_remove_page),
+ MMUMETHOD(mmu_page_is_mapped, moea64_page_is_mapped),
#ifdef __powerpc64__
MMUMETHOD(mmu_page_array_startup, moea64_page_array_startup),
#endif
@@ -1423,6 +1423,12 @@ moea64_quick_remove_page(mmu_t mmu, vm_offset_t addr)
("moea64_quick_remove_page: invalid address"));
mtx_unlock(PCPU_PTR(aim.qmap_lock));
sched_unpin();
+}
+
+boolean_t
+moea64_page_is_mapped(mmu_t mmu, vm_page_t m)
+{
+ return (!LIST_EMPTY(&(m)->md.mdpg_pvoh));
}
/*
Added: head/sys/powerpc/aim/mmu_radix.c
==============================================================================
--- /dev/null 00:00:00 1970 (empty, because file is newly added)
+++ head/sys/powerpc/aim/mmu_radix.c Mon May 11 02:33:37 2020 (r360887)
@@ -0,0 +1,6507 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
+ *
+ * Copyright (c) 2018 Matthew Macy
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+
+#include <sys/param.h>
+#include <sys/kernel.h>
+#include <sys/systm.h>
+#include <sys/epoch.h>
+#include <sys/conf.h>
+#include <sys/bitstring.h>
+#include <sys/queue.h>
+#include <sys/cpuset.h>
+#include <sys/endian.h>
+#include <sys/kerneldump.h>
+#include <sys/ktr.h>
+#include <sys/lock.h>
+#include <sys/syslog.h>
+#include <sys/msgbuf.h>
+#include <sys/malloc.h>
+#include <sys/mman.h>
+#include <sys/mutex.h>
+#include <sys/proc.h>
+#include <sys/rwlock.h>
+#include <sys/sched.h>
+#include <sys/sysctl.h>
+#include <sys/systm.h>
+#include <sys/vmem.h>
+#include <sys/vmmeter.h>
+#include <sys/smp.h>
+
+#include <sys/kdb.h>
+
+#include <dev/ofw/openfirm.h>
+
+#include <vm/vm.h>
+#include <vm/pmap.h>
+#include <vm/vm_param.h>
+#include <vm/vm_kern.h>
+#include <vm/vm_page.h>
+#include <vm/vm_map.h>
+#include <vm/vm_object.h>
+#include <vm/vm_extern.h>
+#include <vm/vm_pageout.h>
+#include <vm/vm_phys.h>
+#include <vm/vm_reserv.h>
+#include <vm/uma.h>
+
+#include <machine/_inttypes.h>
+#include <machine/cpu.h>
+#include <machine/platform.h>
+#include <machine/frame.h>
+#include <machine/md_var.h>
+#include <machine/psl.h>
+#include <machine/bat.h>
+#include <machine/hid.h>
+#include <machine/pte.h>
+#include <machine/sr.h>
+#include <machine/trap.h>
+#include <machine/mmuvar.h>
+
+#ifdef INVARIANTS
+#include <vm/uma_dbg.h>
+#endif
+
+#define PPC_BITLSHIFT(bit) (sizeof(long)*NBBY - 1 - (bit))
+#define PPC_BIT(bit) (1UL << PPC_BITLSHIFT(bit))
+#define PPC_BITLSHIFT_VAL(val, bit) ((val) << PPC_BITLSHIFT(bit))
+
+#include "opt_ddb.h"
+#ifdef DDB
+static void pmap_pte_walk(pml1_entry_t *l1, vm_offset_t va);
+#endif
+
+#define PG_W RPTE_WIRED
+#define PG_V RPTE_VALID
+#define PG_MANAGED RPTE_MANAGED
+#define PG_PROMOTED RPTE_PROMOTED
+#define PG_M RPTE_C
+#define PG_A RPTE_R
+#define PG_X RPTE_EAA_X
+#define PG_RW RPTE_EAA_W
+#define PG_PTE_CACHE RPTE_ATTR_MASK
+
+#define RPTE_SHIFT 9
+#define NLS_MASK ((1UL<<5)-1)
+#define RPTE_ENTRIES (1UL<<RPTE_SHIFT)
+#define RPTE_MASK (RPTE_ENTRIES-1)
+
+#define NLB_SHIFT 0
+#define NLB_MASK (((1UL<<52)-1) << 8)
+
+extern int nkpt;
+extern caddr_t crashdumpmap;
+
+#define RIC_FLUSH_TLB 0
+#define RIC_FLUSH_PWC 1
+#define RIC_FLUSH_ALL 2
+
+#define POWER9_TLB_SETS_RADIX 128 /* # sets in POWER9 TLB Radix mode */
+
+#define PPC_INST_TLBIE 0x7c000264
+#define PPC_INST_TLBIEL 0x7c000224
+#define PPC_INST_SLBIA 0x7c0003e4
+
+#define ___PPC_RA(a) (((a) & 0x1f) << 16)
+#define ___PPC_RB(b) (((b) & 0x1f) << 11)
+#define ___PPC_RS(s) (((s) & 0x1f) << 21)
+#define ___PPC_RT(t) ___PPC_RS(t)
+#define ___PPC_R(r) (((r) & 0x1) << 16)
+#define ___PPC_PRS(prs) (((prs) & 0x1) << 17)
+#define ___PPC_RIC(ric) (((ric) & 0x3) << 18)
+
+#define PPC_SLBIA(IH) __XSTRING(.long PPC_INST_SLBIA | \
+ ((IH & 0x7) << 21))
+#define PPC_TLBIE_5(rb,rs,ric,prs,r) \
+ __XSTRING(.long PPC_INST_TLBIE | \
+ ___PPC_RB(rb) | ___PPC_RS(rs) | \
+ ___PPC_RIC(ric) | ___PPC_PRS(prs) | \
+ ___PPC_R(r))
+
+#define PPC_TLBIEL(rb,rs,ric,prs,r) \
+ __XSTRING(.long PPC_INST_TLBIEL | \
+ ___PPC_RB(rb) | ___PPC_RS(rs) | \
+ ___PPC_RIC(ric) | ___PPC_PRS(prs) | \
+ ___PPC_R(r))
+
+#define PPC_INVALIDATE_ERAT PPC_SLBIA(7)
+
+static __inline void
+ttusync(void)
+{
+ __asm __volatile("eieio; tlbsync; ptesync" ::: "memory");
+}
+
+#define TLBIEL_INVAL_SEL_MASK 0xc00 /* invalidation selector */
+#define TLBIEL_INVAL_PAGE 0x000 /* invalidate a single page */
+#define TLBIEL_INVAL_SET_PID 0x400 /* invalidate a set for the current PID */
+#define TLBIEL_INVAL_SET_LPID 0x800 /* invalidate a set for current LPID */
+#define TLBIEL_INVAL_SET 0xc00 /* invalidate a set for all LPIDs */
+
+#define TLBIE_ACTUAL_PAGE_MASK 0xe0
+#define TLBIE_ACTUAL_PAGE_4K 0x00
+#define TLBIE_ACTUAL_PAGE_64K 0xa0
+#define TLBIE_ACTUAL_PAGE_2M 0x20
+#define TLBIE_ACTUAL_PAGE_1G 0x40
+
+#define TLBIE_PRS_PARTITION_SCOPE 0x0
+#define TLBIE_PRS_PROCESS_SCOPE 0x1
+
+#define TLBIE_RIC_INVALIDATE_TLB 0x0 /* Invalidate just TLB */
+#define TLBIE_RIC_INVALIDATE_PWC 0x1 /* Invalidate just PWC */
+#define TLBIE_RIC_INVALIDATE_ALL 0x2 /* Invalidate TLB, PWC,
+ * cached {proc, part}tab entries
+ */
+#define TLBIE_RIC_INVALIDATE_SEQ 0x3 /* HPT - only:
+ * Invalidate a range of translations
+ */
+
+static __inline void
+radix_tlbie(uint8_t ric, uint8_t prs, uint16_t is, uint32_t pid, uint32_t lpid,
+ vm_offset_t va, uint16_t ap)
+{
+ uint64_t rb, rs;
+
+ MPASS((va & PAGE_MASK) == 0);
+
+ rs = ((uint64_t)pid << 32) | lpid;
+ rb = va | is | ap;
+ __asm __volatile(PPC_TLBIE_5(%0, %1, %2, %3, 1) : :
+ "r" (rb), "r" (rs), "i" (ric), "i" (prs));
+}
+
+static __inline void
+radix_tlbie_invlpg_user_4k(uint32_t pid, vm_offset_t va)
+{
+
+ radix_tlbie(TLBIE_RIC_INVALIDATE_TLB, TLBIE_PRS_PROCESS_SCOPE,
+ TLBIEL_INVAL_PAGE, pid, 0, va, TLBIE_ACTUAL_PAGE_4K);
+}
+
+static __inline void
+radix_tlbie_invlpg_user_2m(uint32_t pid, vm_offset_t va)
+{
+
+ radix_tlbie(TLBIE_RIC_INVALIDATE_TLB, TLBIE_PRS_PROCESS_SCOPE,
+ TLBIEL_INVAL_PAGE, pid, 0, va, TLBIE_ACTUAL_PAGE_2M);
+}
+
+static __inline void
+radix_tlbie_invlpwc_user(uint32_t pid)
+{
+
+ radix_tlbie(TLBIE_RIC_INVALIDATE_PWC, TLBIE_PRS_PROCESS_SCOPE,
+ TLBIEL_INVAL_SET_PID, pid, 0, 0, 0);
+}
+
+static __inline void
+radix_tlbie_flush_user(uint32_t pid)
+{
+
+ radix_tlbie(TLBIE_RIC_INVALIDATE_ALL, TLBIE_PRS_PROCESS_SCOPE,
+ TLBIEL_INVAL_SET_PID, pid, 0, 0, 0);
+}
+
+static __inline void
+radix_tlbie_invlpg_kernel_4k(vm_offset_t va)
+{
+
+ radix_tlbie(TLBIE_RIC_INVALIDATE_TLB, TLBIE_PRS_PROCESS_SCOPE,
+ TLBIEL_INVAL_PAGE, 0, 0, va, TLBIE_ACTUAL_PAGE_4K);
+}
+
+static __inline void
+radix_tlbie_invlpg_kernel_2m(vm_offset_t va)
+{
+
+ radix_tlbie(TLBIE_RIC_INVALIDATE_TLB, TLBIE_PRS_PROCESS_SCOPE,
+ TLBIEL_INVAL_PAGE, 0, 0, va, TLBIE_ACTUAL_PAGE_2M);
+}
+
+/* 1GB pages aren't currently supported. */
+static __inline __unused void
+radix_tlbie_invlpg_kernel_1g(vm_offset_t va)
+{
+
+ radix_tlbie(TLBIE_RIC_INVALIDATE_TLB, TLBIE_PRS_PROCESS_SCOPE,
+ TLBIEL_INVAL_PAGE, 0, 0, va, TLBIE_ACTUAL_PAGE_1G);
+}
+
+static __inline void
+radix_tlbie_invlpwc_kernel(void)
+{
+
+ radix_tlbie(TLBIE_RIC_INVALIDATE_PWC, TLBIE_PRS_PROCESS_SCOPE,
+ TLBIEL_INVAL_SET_LPID, 0, 0, 0, 0);
+}
+
+static __inline void
+radix_tlbie_flush_kernel(void)
+{
+
+ radix_tlbie(TLBIE_RIC_INVALIDATE_ALL, TLBIE_PRS_PROCESS_SCOPE,
+ TLBIEL_INVAL_SET_LPID, 0, 0, 0, 0);
+}
+
+static __inline vm_pindex_t
+pmap_l3e_pindex(vm_offset_t va)
+{
+ return ((va & PG_FRAME) >> L3_PAGE_SIZE_SHIFT);
+}
+
+static __inline vm_pindex_t
+pmap_pml3e_index(vm_offset_t va)
+{
+
+ return ((va >> L3_PAGE_SIZE_SHIFT) & RPTE_MASK);
+}
+
+static __inline vm_pindex_t
+pmap_pml2e_index(vm_offset_t va)
+{
+ return ((va >> L2_PAGE_SIZE_SHIFT) & RPTE_MASK);
+}
+
+static __inline vm_pindex_t
+pmap_pml1e_index(vm_offset_t va)
+{
+ return ((va & PG_FRAME) >> L1_PAGE_SIZE_SHIFT);
+}
+
+/* Return various clipped indexes for a given VA */
+static __inline vm_pindex_t
+pmap_pte_index(vm_offset_t va)
+{
+
+ return ((va >> PAGE_SHIFT) & RPTE_MASK);
+}
+
+/* Return a pointer to the PT slot that corresponds to a VA */
+static __inline pt_entry_t *
+pmap_l3e_to_pte(pt_entry_t *l3e, vm_offset_t va)
+{
+ pt_entry_t *pte;
+ vm_paddr_t ptepa;
+
+ ptepa = (*l3e & NLB_MASK);
+ pte = (pt_entry_t *)PHYS_TO_DMAP(ptepa);
+ return (&pte[pmap_pte_index(va)]);
+}
+
+/* Return a pointer to the PD slot that corresponds to a VA */
+static __inline pt_entry_t *
+pmap_l2e_to_l3e(pt_entry_t *l2e, vm_offset_t va)
+{
+ pt_entry_t *l3e;
+ vm_paddr_t l3pa;
+
+ l3pa = (*l2e & NLB_MASK);
+ l3e = (pml3_entry_t *)PHYS_TO_DMAP(l3pa);
+ return (&l3e[pmap_pml3e_index(va)]);
+}
+
+/* Return a pointer to the PD slot that corresponds to a VA */
+static __inline pt_entry_t *
+pmap_l1e_to_l2e(pt_entry_t *l1e, vm_offset_t va)
+{
+ pt_entry_t *l2e;
+ vm_paddr_t l2pa;
+
+ l2pa = (*l1e & NLB_MASK);
+
+ l2e = (pml2_entry_t *)PHYS_TO_DMAP(l2pa);
+ return (&l2e[pmap_pml2e_index(va)]);
+}
+
+static __inline pml1_entry_t *
+pmap_pml1e(pmap_t pmap, vm_offset_t va)
+{
+
+ return (&pmap->pm_pml1[pmap_pml1e_index(va)]);
+}
+
+static pt_entry_t *
+pmap_pml2e(pmap_t pmap, vm_offset_t va)
+{
+ pt_entry_t *l1e;
+
+ l1e = pmap_pml1e(pmap, va);
+ if (l1e == NULL || (*l1e & RPTE_VALID) == 0)
+ return (NULL);
+ return (pmap_l1e_to_l2e(l1e, va));
+}
+
+static __inline pt_entry_t *
+pmap_pml3e(pmap_t pmap, vm_offset_t va)
+{
+ pt_entry_t *l2e;
+
+ l2e = pmap_pml2e(pmap, va);
+ if (l2e == NULL || (*l2e & RPTE_VALID) == 0)
+ return (NULL);
+ return (pmap_l2e_to_l3e(l2e, va));
+}
+
+static __inline pt_entry_t *
+pmap_pte(pmap_t pmap, vm_offset_t va)
+{
+ pt_entry_t *l3e;
+
+ l3e = pmap_pml3e(pmap, va);
+ if (l3e == NULL || (*l3e & RPTE_VALID) == 0)
+ return (NULL);
+ return (pmap_l3e_to_pte(l3e, va));
+}
+
+int nkpt = 64;
+SYSCTL_INT(_machdep, OID_AUTO, nkpt, CTLFLAG_RD, &nkpt, 0,
+ "Number of kernel page table pages allocated on bootup");
+
+vm_paddr_t dmaplimit;
+
+SYSCTL_NODE(_vm, OID_AUTO, pmap, CTLFLAG_RD, 0, "VM/pmap parameters");
+
+static int pg_ps_enabled = 1;
+SYSCTL_INT(_vm_pmap, OID_AUTO, pg_ps_enabled, CTLFLAG_RDTUN | CTLFLAG_NOFETCH,
+ &pg_ps_enabled, 0, "Are large page mappings enabled?");
+#ifdef INVARIANTS
+#define VERBOSE_PMAP 0
+#define VERBOSE_PROTECT 0
+static int pmap_logging;
+SYSCTL_INT(_vm_pmap, OID_AUTO, pmap_logging, CTLFLAG_RWTUN,
+ &pmap_logging, 0, "verbose debug logging");
+#endif
+
+static u_int64_t KPTphys; /* phys addr of kernel level 1 */
+
+//static vm_paddr_t KERNend; /* phys addr of end of bootstrap data */
+
+static vm_offset_t qframe = 0;
+static struct mtx qframe_mtx;
+static epoch_t pmap_epoch;
+
+void mmu_radix_activate(mmu_t mmu, struct thread *);
+void mmu_radix_advise(mmu_t mmu, pmap_t, vm_offset_t, vm_offset_t, int);
+void mmu_radix_align_superpage(mmu_t mmu, vm_object_t, vm_ooffset_t, vm_offset_t *,
+ vm_size_t);
+void mmu_radix_clear_modify(mmu_t, vm_page_t);
+void mmu_radix_copy(mmu_t, pmap_t, pmap_t, vm_offset_t, vm_size_t, vm_offset_t);
+int mmu_radix_map_user_ptr(mmu_t mmu, pmap_t pm,
+ volatile const void *uaddr, void **kaddr, size_t ulen, size_t *klen);
+int mmu_radix_decode_kernel_ptr(mmu_t, vm_offset_t, int *, vm_offset_t *);
+int mmu_radix_enter(mmu_t, pmap_t, vm_offset_t, vm_page_t, vm_prot_t, u_int, int8_t);
+void mmu_radix_enter_object(mmu_t, pmap_t, vm_offset_t, vm_offset_t, vm_page_t,
+ vm_prot_t);
+void mmu_radix_enter_quick(mmu_t, pmap_t, vm_offset_t, vm_page_t, vm_prot_t);
+vm_paddr_t mmu_radix_extract(mmu_t, pmap_t pmap, vm_offset_t va);
+vm_page_t mmu_radix_extract_and_hold(mmu_t, pmap_t, vm_offset_t, vm_prot_t);
+void mmu_radix_kenter(mmu_t, vm_offset_t, vm_paddr_t);
+vm_paddr_t mmu_radix_kextract(mmu_t, vm_offset_t);
+void mmu_radix_kremove(mmu_t, vm_offset_t);
+boolean_t mmu_radix_is_modified(mmu_t, vm_page_t);
+boolean_t mmu_radix_is_prefaultable(mmu_t, pmap_t, vm_offset_t);
+boolean_t mmu_radix_is_referenced(mmu_t, vm_page_t);
+void mmu_radix_object_init_pt(mmu_t, pmap_t, vm_offset_t, vm_object_t,
+ vm_pindex_t, vm_size_t);
+boolean_t mmu_radix_page_exists_quick(mmu_t, pmap_t, vm_page_t);
+void mmu_radix_page_init(mmu_t, vm_page_t);
+boolean_t mmu_radix_page_is_mapped(mmu_t, vm_page_t m);
+void mmu_radix_page_set_memattr(mmu_t, vm_page_t, vm_memattr_t);
+int mmu_radix_page_wired_mappings(mmu_t, vm_page_t);
+void mmu_radix_pinit(mmu_t, pmap_t);
+void mmu_radix_protect(mmu_t, pmap_t, vm_offset_t, vm_offset_t, vm_prot_t);
+boolean_t mmu_radix_ps_enabled(mmu_t, pmap_t);
+void mmu_radix_qenter(mmu_t, vm_offset_t, vm_page_t *, int);
+void mmu_radix_qremove(mmu_t, vm_offset_t, int);
+vm_offset_t mmu_radix_quick_enter_page(mmu_t, vm_page_t);
+void mmu_radix_quick_remove_page(mmu_t, vm_offset_t);
+boolean_t mmu_radix_ts_referenced(mmu_t, vm_page_t);
+void mmu_radix_release(mmu_t, pmap_t);
+void mmu_radix_remove(mmu_t, pmap_t, vm_offset_t, vm_offset_t);
+void mmu_radix_remove_all(mmu_t, vm_page_t);
+void mmu_radix_remove_pages(mmu_t, pmap_t);
+void mmu_radix_remove_write(mmu_t, vm_page_t);
+void mmu_radix_unwire(mmu_t, pmap_t, vm_offset_t, vm_offset_t);
+void mmu_radix_zero_page(mmu_t, vm_page_t);
+void mmu_radix_zero_page_area(mmu_t, vm_page_t, int, int);
+int mmu_radix_change_attr(mmu_t, vm_offset_t, vm_size_t, vm_memattr_t);
+void mmu_radix_page_array_startup(mmu_t mmu, long pages);
+
+#include "mmu_oea64.h"
+#include "mmu_if.h"
+#include "moea64_if.h"
+
+/*
+ * Kernel MMU interface
+ */
+
+static void mmu_radix_bootstrap(mmu_t mmup,
+ vm_offset_t kernelstart, vm_offset_t kernelend);
+
+static void mmu_radix_copy_page(mmu_t, vm_page_t, vm_page_t);
+static void mmu_radix_copy_pages(mmu_t mmu, vm_page_t *ma, vm_offset_t a_offset,
+ vm_page_t *mb, vm_offset_t b_offset, int xfersize);
+static void mmu_radix_growkernel(mmu_t, vm_offset_t);
+static void mmu_radix_init(mmu_t);
+static int mmu_radix_mincore(mmu_t, pmap_t, vm_offset_t, vm_paddr_t *);
+static vm_offset_t mmu_radix_map(mmu_t, vm_offset_t *, vm_paddr_t, vm_paddr_t, int);
+static void mmu_radix_pinit0(mmu_t, pmap_t);
+
+static void *mmu_radix_mapdev(mmu_t, vm_paddr_t, vm_size_t);
+static void *mmu_radix_mapdev_attr(mmu_t, vm_paddr_t, vm_size_t, vm_memattr_t);
+static void mmu_radix_unmapdev(mmu_t, vm_offset_t, vm_size_t);
+static void mmu_radix_kenter_attr(mmu_t, vm_offset_t, vm_paddr_t, vm_memattr_t ma);
+static boolean_t mmu_radix_dev_direct_mapped(mmu_t, vm_paddr_t, vm_size_t);
+static void mmu_radix_dumpsys_map(mmu_t mmu, vm_paddr_t pa, size_t sz,
+ void **va);
+static void mmu_radix_scan_init(mmu_t mmu);
+static void mmu_radix_cpu_bootstrap(mmu_t, int ap);
+
+static mmu_method_t mmu_radix_methods[] = {
+ MMUMETHOD(mmu_bootstrap, mmu_radix_bootstrap),
+ MMUMETHOD(mmu_copy_page, mmu_radix_copy_page),
+ MMUMETHOD(mmu_copy_pages, mmu_radix_copy_pages),
+ MMUMETHOD(mmu_cpu_bootstrap, mmu_radix_cpu_bootstrap),
+ MMUMETHOD(mmu_growkernel, mmu_radix_growkernel),
+ MMUMETHOD(mmu_init, mmu_radix_init),
+ MMUMETHOD(mmu_map, mmu_radix_map),
+ MMUMETHOD(mmu_mincore, mmu_radix_mincore),
+ MMUMETHOD(mmu_pinit, mmu_radix_pinit),
+ MMUMETHOD(mmu_pinit0, mmu_radix_pinit0),
+
+ MMUMETHOD(mmu_mapdev, mmu_radix_mapdev),
+ MMUMETHOD(mmu_mapdev_attr, mmu_radix_mapdev_attr),
+ MMUMETHOD(mmu_unmapdev, mmu_radix_unmapdev),
+ MMUMETHOD(mmu_kenter_attr, mmu_radix_kenter_attr),
+ MMUMETHOD(mmu_dev_direct_mapped,mmu_radix_dev_direct_mapped),
+ MMUMETHOD(mmu_scan_init, mmu_radix_scan_init),
+ MMUMETHOD(mmu_dumpsys_map, mmu_radix_dumpsys_map),
+ MMUMETHOD(mmu_page_is_mapped, mmu_radix_page_is_mapped),
+ MMUMETHOD(mmu_ps_enabled, mmu_radix_ps_enabled),
+ MMUMETHOD(mmu_object_init_pt, mmu_radix_object_init_pt),
+ MMUMETHOD(mmu_protect, mmu_radix_protect),
+ /* pmap dispatcher interface */
+ MMUMETHOD(mmu_clear_modify, mmu_radix_clear_modify),
+ MMUMETHOD(mmu_copy, mmu_radix_copy),
+ MMUMETHOD(mmu_enter, mmu_radix_enter),
+ MMUMETHOD(mmu_enter_object, mmu_radix_enter_object),
+ MMUMETHOD(mmu_enter_quick, mmu_radix_enter_quick),
+ MMUMETHOD(mmu_extract, mmu_radix_extract),
+ MMUMETHOD(mmu_extract_and_hold, mmu_radix_extract_and_hold),
+ MMUMETHOD(mmu_is_modified, mmu_radix_is_modified),
+ MMUMETHOD(mmu_is_prefaultable, mmu_radix_is_prefaultable),
+ MMUMETHOD(mmu_is_referenced, mmu_radix_is_referenced),
+ MMUMETHOD(mmu_ts_referenced, mmu_radix_ts_referenced),
+ MMUMETHOD(mmu_page_exists_quick,mmu_radix_page_exists_quick),
+ MMUMETHOD(mmu_page_init, mmu_radix_page_init),
+ MMUMETHOD(mmu_page_wired_mappings, mmu_radix_page_wired_mappings),
+ MMUMETHOD(mmu_qenter, mmu_radix_qenter),
+ MMUMETHOD(mmu_qremove, mmu_radix_qremove),
+ MMUMETHOD(mmu_release, mmu_radix_release),
+ MMUMETHOD(mmu_remove, mmu_radix_remove),
+ MMUMETHOD(mmu_remove_all, mmu_radix_remove_all),
+ MMUMETHOD(mmu_remove_write, mmu_radix_remove_write),
+ MMUMETHOD(mmu_unwire, mmu_radix_unwire),
+ MMUMETHOD(mmu_zero_page, mmu_radix_zero_page),
+ MMUMETHOD(mmu_zero_page_area, mmu_radix_zero_page_area),
+ MMUMETHOD(mmu_activate, mmu_radix_activate),
+ MMUMETHOD(mmu_quick_enter_page, mmu_radix_quick_enter_page),
+ MMUMETHOD(mmu_quick_remove_page, mmu_radix_quick_remove_page),
+ MMUMETHOD(mmu_page_set_memattr, mmu_radix_page_set_memattr),
+ MMUMETHOD(mmu_page_array_startup, mmu_radix_page_array_startup),
+
+ /* Internal interfaces */
+ MMUMETHOD(mmu_kenter, mmu_radix_kenter),
+ MMUMETHOD(mmu_kextract, mmu_radix_kextract),
+ MMUMETHOD(mmu_kremove, mmu_radix_kremove),
+ MMUMETHOD(mmu_change_attr, mmu_radix_change_attr),
+ MMUMETHOD(mmu_map_user_ptr, mmu_radix_map_user_ptr),
+ MMUMETHOD(mmu_decode_kernel_ptr, mmu_radix_decode_kernel_ptr),
+ { 0, 0 }
+};
+
+MMU_DEF(mmu_radix, MMU_TYPE_RADIX, mmu_radix_methods, 0);
+
+#define METHODVOID(m) mmu_radix_ ## m(mmu_t mmup)
+
+static boolean_t pmap_demote_l3e_locked(pmap_t pmap, pml3_entry_t *l3e, vm_offset_t va,
+ struct rwlock **lockp);
+static boolean_t pmap_demote_l3e(pmap_t pmap, pml3_entry_t *pde, vm_offset_t va);
+static int pmap_unuse_pt(pmap_t, vm_offset_t, pml3_entry_t, struct spglist *);
+static int pmap_remove_l3e(pmap_t pmap, pml3_entry_t *pdq, vm_offset_t sva,
+ struct spglist *free, struct rwlock **lockp);
+static int pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq, vm_offset_t sva,
+ pml3_entry_t ptepde, struct spglist *free, struct rwlock **lockp);
+static vm_page_t pmap_remove_pt_page(pmap_t pmap, vm_offset_t va);
+static bool pmap_remove_page(pmap_t pmap, vm_offset_t va, pml3_entry_t *pde,
+ struct spglist *free);
+static bool pmap_remove_ptes(pmap_t pmap, vm_offset_t sva, vm_offset_t eva,
+ pml3_entry_t *l3e, struct spglist *free, struct rwlock **lockp);
+
+static bool pmap_pv_insert_l3e(pmap_t pmap, vm_offset_t va, pml3_entry_t l3e,
+ u_int flags, struct rwlock **lockp);
+#if VM_NRESERVLEVEL > 0
+static void pmap_pv_promote_l3e(pmap_t pmap, vm_offset_t va, vm_paddr_t pa,
+ struct rwlock **lockp);
+#endif
+static void pmap_pvh_free(struct md_page *pvh, pmap_t pmap, vm_offset_t va);
+static int pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte);
+static vm_page_t mmu_radix_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m,
+ vm_prot_t prot, vm_page_t mpte, struct rwlock **lockp, bool *invalidate);
+
+static bool pmap_enter_2mpage(pmap_t pmap, vm_offset_t va, vm_page_t m,
+ vm_prot_t prot, struct rwlock **lockp);
+static int pmap_enter_l3e(pmap_t pmap, vm_offset_t va, pml3_entry_t newpde,
+ u_int flags, vm_page_t m, struct rwlock **lockp);
+
+static vm_page_t reclaim_pv_chunk(pmap_t locked_pmap, struct rwlock **lockp);
+static void free_pv_chunk(struct pv_chunk *pc);
+static vm_page_t _pmap_allocpte(pmap_t pmap, vm_pindex_t ptepindex, struct rwlock **lockp);
+static vm_page_t pmap_allocl3e(pmap_t pmap, vm_offset_t va,
+ struct rwlock **lockp);
+static vm_page_t pmap_allocpte(pmap_t pmap, vm_offset_t va,
+ struct rwlock **lockp);
+static void _pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m,
+ struct spglist *free);
+static boolean_t pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free);
+
+static void pmap_invalidate_page(pmap_t pmap, vm_offset_t start);
+static void pmap_invalidate_all(pmap_t pmap);
+static int pmap_change_attr_locked(vm_offset_t va, vm_size_t size, int mode, bool flush);
+
+/*
+ * Internal flags for pmap_enter()'s helper functions.
+ */
+#define PMAP_ENTER_NORECLAIM 0x1000000 /* Don't reclaim PV entries. */
+#define PMAP_ENTER_NOREPLACE 0x2000000 /* Don't replace mappings. */
+
+#define UNIMPLEMENTED() panic("%s not implemented", __func__)
+#define UNTESTED() panic("%s not yet tested", __func__)
+
+
+
+/* Number of supported PID bits */
+static unsigned int isa3_pid_bits;
+
+/* PID to start allocating from */
+static unsigned int isa3_base_pid;
+
+#define PROCTAB_SIZE_SHIFT (isa3_pid_bits + 4)
+#define PROCTAB_ENTRIES (1ul << isa3_pid_bits)
+
+
+/*
+ * Map of physical memory regions.
+ */
+static struct mem_region *regions, *pregions;
+static struct numa_mem_region *numa_pregions;
+static u_int phys_avail_count;
+static int regions_sz, pregions_sz, numa_pregions_sz;
+static struct pate *isa3_parttab;
+static struct prte *isa3_proctab;
+static vmem_t *asid_arena;
+
+extern void bs_remap_earlyboot(void);
+
+#define RADIX_PGD_SIZE_SHIFT 16
+#define RADIX_PGD_SIZE (1UL << RADIX_PGD_SIZE_SHIFT)
+
+#define RADIX_PGD_INDEX_SHIFT (RADIX_PGD_SIZE_SHIFT-3)
+#define NL2EPG (PAGE_SIZE/sizeof(pml2_entry_t))
+#define NL3EPG (PAGE_SIZE/sizeof(pml3_entry_t))
+
+#define NUPML1E (RADIX_PGD_SIZE/sizeof(uint64_t)) /* number of userland PML1 pages */
+#define NUPDPE (NUPML1E * NL2EPG)/* number of userland PDP pages */
+#define NUPDE (NUPDPE * NL3EPG) /* number of userland PD entries */
+
+/* POWER9 only permits a 64k partition table size. */
+#define PARTTAB_SIZE_SHIFT 16
+#define PARTTAB_SIZE (1UL << PARTTAB_SIZE_SHIFT)
+
+#define PARTTAB_HR (1UL << 63) /* host uses radix */
+#define PARTTAB_GR (1UL << 63) /* guest uses radix must match host */
+
+/* TLB flush actions. Used as argument to tlbiel_all() */
+enum {
+ TLB_INVAL_SCOPE_LPID = 0, /* invalidate TLBs for current LPID */
+ TLB_INVAL_SCOPE_GLOBAL = 1, /* invalidate all TLBs */
+};
+
+#define NPV_LIST_LOCKS MAXCPU
+static int pmap_initialized;
+static vm_paddr_t proctab0pa;
+static vm_paddr_t parttab_phys;
+CTASSERT(sizeof(struct pv_chunk) == PAGE_SIZE);
+
+/*
+ * Data for the pv entry allocation mechanism.
+ * Updates to pv_invl_gen are protected by the pv_list_locks[]
+ * elements, but reads are not.
+ */
+static TAILQ_HEAD(pch, pv_chunk) pv_chunks = TAILQ_HEAD_INITIALIZER(pv_chunks);
+static struct mtx __exclusive_cache_line pv_chunks_mutex;
+static struct rwlock __exclusive_cache_line pv_list_locks[NPV_LIST_LOCKS];
+static struct md_page *pv_table;
+static struct md_page pv_dummy;
+
+#ifdef PV_STATS
+#define PV_STAT(x) do { x ; } while (0)
+#else
+#define PV_STAT(x) do { } while (0)
+#endif
+
+#define pa_radix_index(pa) ((pa) >> L3_PAGE_SIZE_SHIFT)
+#define pa_to_pvh(pa) (&pv_table[pa_radix_index(pa)])
+
+#define PHYS_TO_PV_LIST_LOCK(pa) \
+ (&pv_list_locks[pa_radix_index(pa) % NPV_LIST_LOCKS])
+
+#define CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, pa) do { \
+ struct rwlock **_lockp = (lockp); \
+ struct rwlock *_new_lock; \
+ \
+ _new_lock = PHYS_TO_PV_LIST_LOCK(pa); \
+ if (_new_lock != *_lockp) { \
+ if (*_lockp != NULL) \
+ rw_wunlock(*_lockp); \
+ *_lockp = _new_lock; \
+ rw_wlock(*_lockp); \
+ } \
+} while (0)
+
+#define CHANGE_PV_LIST_LOCK_TO_VM_PAGE(lockp, m) \
+ CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, VM_PAGE_TO_PHYS(m))
+
+#define RELEASE_PV_LIST_LOCK(lockp) do { \
+ struct rwlock **_lockp = (lockp); \
+ \
+ if (*_lockp != NULL) { \
+ rw_wunlock(*_lockp); \
+ *_lockp = NULL; \
+ } \
+} while (0)
+
+#define VM_PAGE_TO_PV_LIST_LOCK(m) \
+ PHYS_TO_PV_LIST_LOCK(VM_PAGE_TO_PHYS(m))
+
+/*
+ * We support 52 bits, hence:
+ * bits 52 - 31 = 21, 0b10101
+ * RTS encoding details
+ * bits 0 - 3 of rts -> bits 6 - 8 unsigned long
+ * bits 4 - 5 of rts -> bits 62 - 63 of unsigned long
+ */
+#define RTS_SIZE ((0x2UL << 61) | (0x5UL << 5))
+
+
+static int powernv_enabled = 1;
+
+static inline void
+tlbiel_radix_set_isa300(uint32_t set, uint32_t is,
+ uint32_t pid, uint32_t ric, uint32_t prs)
+{
+ uint64_t rb;
+ uint64_t rs;
+
+ rb = PPC_BITLSHIFT_VAL(set, 51) | PPC_BITLSHIFT_VAL(is, 53);
+ rs = PPC_BITLSHIFT_VAL((uint64_t)pid, 31);
+
+ __asm __volatile(PPC_TLBIEL(%0, %1, %2, %3, 1)
+ : : "r"(rb), "r"(rs), "i"(ric), "i"(prs)
+ : "memory");
+}
+
+static void
+tlbiel_flush_isa3(uint32_t num_sets, uint32_t is)
+{
+ uint32_t set;
+
+ __asm __volatile("ptesync": : :"memory");
+
+ /*
+ * Flush the first set of the TLB, and the entire Page Walk Cache
+ * and partition table entries. Then flush the remaining sets of the
+ * TLB.
+ */
+ tlbiel_radix_set_isa300(0, is, 0, RIC_FLUSH_ALL, 0);
+ for (set = 1; set < num_sets; set++)
+ tlbiel_radix_set_isa300(set, is, 0, RIC_FLUSH_TLB, 0);
+
+ /* Do the same for process scoped entries. */
+ tlbiel_radix_set_isa300(0, is, 0, RIC_FLUSH_ALL, 1);
+ for (set = 1; set < num_sets; set++)
+ tlbiel_radix_set_isa300(set, is, 0, RIC_FLUSH_TLB, 1);
+
+ __asm __volatile("ptesync": : :"memory");
+}
+
+static void
+mmu_radix_tlbiel_flush(int scope)
+{
+ int is;
+
+ MPASS(scope == TLB_INVAL_SCOPE_LPID ||
+ scope == TLB_INVAL_SCOPE_GLOBAL);
+ is = scope + 2;
+
+ tlbiel_flush_isa3(POWER9_TLB_SETS_RADIX, is);
+ __asm __volatile(PPC_INVALIDATE_ERAT "; isync" : : :"memory");
+}
+
+static void
+mmu_radix_init_amor(void)
+{
+ /*
+ * In HV mode, we init AMOR (Authority Mask Override Register) so that
+ * the hypervisor and guest can setup IAMR (Instruction Authority Mask
+ * Register), enable key 0 and set it to 1.
+ *
+ * AMOR = 0b1100 .... 0000 (Mask for key 0 is 11)
+ */
+ mtspr(SPR_AMOR, (3ul << 62));
+}
+
+static void
+mmu_radix_init_iamr(void)
+{
+ /*
+ * Radix always uses key0 of the IAMR to determine if an access is
+ * allowed. We set bit 0 (IBM bit 1) of key0, to prevent instruction
+ * fetch.
+ */
+ mtspr(SPR_IAMR, (1ul << 62));
+}
+
+static void
+mmu_radix_pid_set(pmap_t pmap)
+{
+
+ mtspr(SPR_PID, pmap->pm_pid);
+ isync();
+}
+
+/* Quick sort callout for comparing physical addresses. */
+static int
+pa_cmp(const void *a, const void *b)
+{
+ const vm_paddr_t *pa = a, *pb = b;
+
+ if (*pa < *pb)
+ return (-1);
+ else if (*pa > *pb)
+ return (1);
+ else
+ return (0);
+}
+
+#define pte_load_store(ptep, pte) atomic_swap_long(ptep, pte)
+#define pte_load_clear(ptep) atomic_swap_long(ptep, 0)
+#define pte_store(ptep, pte) do { \
+ MPASS((pte) & (RPTE_EAA_R | RPTE_EAA_W | RPTE_EAA_X)); \
+ *(u_long *)(ptep) = (u_long)((pte) | PG_V | RPTE_LEAF); \
+} while (0)
+/*
+ * NB: should only be used for adding directories - not for direct mappings
+ */
+#define pde_store(ptep, pa) do { \
+ *(u_long *)(ptep) = (u_long)(pa|RPTE_VALID|RPTE_SHIFT); \
+} while (0)
+
+#define pte_clear(ptep) do { \
+ *(u_long *)(ptep) = (u_long)(0); \
+} while (0)
+
+#define PMAP_PDE_SUPERPAGE (1 << 8) /* supports 2MB superpages */
+
+/*
+ * Promotion to a 2MB (PDE) page mapping requires that the corresponding 4KB
+ * (PTE) page mappings have identical settings for the following fields:
+ */
+#define PG_PTE_PROMOTE (PG_X | PG_MANAGED | PG_W | PG_PTE_CACHE | \
+ PG_M | PG_A | RPTE_EAA_MASK | PG_V)
+
+
+static void
+pmap_epoch_init(void *arg __unused)
+{
+ pmap_epoch = epoch_alloc("pmap", EPOCH_PREEMPT | EPOCH_LOCKED);
+}
+SYSINIT(epoch, SI_SUB_EPOCH + 1, SI_ORDER_ANY, pmap_epoch_init, NULL);
+
+static bool
+pmap_not_in_di(void)
+{
+
+ return (curthread->td_md.md_invl_gen.gen == 0);
+}
+
+#define PMAP_ASSERT_NOT_IN_DI() \
+ KASSERT(pmap_not_in_di(), ("DI already started"))
+
+static void
+pmap_delayed_invl_started(epoch_tracker_t et)
+{
+ epoch_enter_preempt(pmap_epoch, et);
+ curthread->td_md.md_invl_gen.gen = 1;
+}
+
+static void
+pmap_delayed_invl_finished(epoch_tracker_t et)
+{
*** DIFF OUTPUT TRUNCATED AT 1000 LINES ***
More information about the svn-src-all
mailing list