svn commit: r337463 - in head/sys: arm/arm arm/include vm
Alan Cox
alc at FreeBSD.org
Wed Aug 8 16:55:03 UTC 2018
Author: alc
Date: Wed Aug 8 16:55:01 2018
New Revision: 337463
URL: https://svnweb.freebsd.org/changeset/base/337463
Log:
Add support for pmap_enter(..., psind=1) to the armv6 pmap. In other words,
add support for explicitly requesting that pmap_enter() create a 1 MB page
mapping. (Essentially, this feature allows the machine-independent layer
to create superpage mappings preemptively, and not wait for automatic
promotion to occur.)
Export pmap_ps_enabled() to the machine-independent layer.
Add a flag to pmap_pv_insert_pte1() that specifies whether it should fail
or reclaim a PV entry when one is not available.
Refactor pmap_enter_pte1() into two functions, one by the same name, that
is a general-purpose function for creating pte1 mappings, and another,
pmap_enter_1mpage(), that is used to prefault 1 MB read- and/or execute-
only mappings for execve(2), mmap(2), and shmat(2).
In addition, as an optimization to pmap_enter(..., psind=0), eliminate the
use of pte2_is_managed() from pmap_enter(). Unlike the x86 pmap
implementations, armv6 does not have a managed bit defined within the PTE.
So, pte2_is_managed() is actually a call to PHYS_TO_VM_PAGE(), which is O(n)
in the number of vm_phys_segs[]. All but one call to PHYS_TO_VM_PAGE() in
pmap_enter() can be avoided.
Reviewed by: kib, markj, mmel
Tested by: mmel
MFC after: 6 weeks
Differential Revision: https://reviews.freebsd.org/D16555
Modified:
head/sys/arm/arm/pmap-v6.c
head/sys/arm/include/pmap-v6.h
head/sys/vm/vm_fault.c
Modified: head/sys/arm/arm/pmap-v6.c
==============================================================================
--- head/sys/arm/arm/pmap-v6.c Wed Aug 8 16:17:50 2018 (r337462)
+++ head/sys/arm/arm/pmap-v6.c Wed Aug 8 16:55:01 2018 (r337463)
@@ -323,9 +323,17 @@ SYSCTL_INT(_debug, OID_AUTO, PMAP1unchanged, CTLFLAG_R
"Number of times pmap_pte2_quick didn't change PMAP1");
static struct mtx PMAP2mutex;
+/*
+ * Internal flags for pmap_enter()'s helper functions.
+ */
+#define PMAP_ENTER_NORECLAIM 0x1000000 /* Don't reclaim PV entries. */
+#define PMAP_ENTER_NOREPLACE 0x2000000 /* Don't replace mappings. */
+
static __inline void pt2_wirecount_init(vm_page_t m);
static boolean_t pmap_demote_pte1(pmap_t pmap, pt1_entry_t *pte1p,
vm_offset_t va);
+static int pmap_enter_pte1(pmap_t pmap, vm_offset_t va, pt1_entry_t pte1,
+ u_int flags, vm_page_t m);
void cache_icache_sync_fresh(vm_offset_t va, vm_paddr_t pa, vm_size_t size);
/*
@@ -1557,6 +1565,13 @@ static int sp_enabled = 1;
SYSCTL_INT(_vm_pmap, OID_AUTO, sp_enabled, CTLFLAG_RDTUN | CTLFLAG_NOFETCH,
&sp_enabled, 0, "Are large page mappings enabled?");
+bool
+pmap_ps_enabled(pmap_t pmap __unused)
+{
+
+ return (sp_enabled != 0);
+}
+
static SYSCTL_NODE(_vm_pmap, OID_AUTO, pte1, CTLFLAG_RD, 0,
"1MB page mapping counters");
@@ -3227,21 +3242,22 @@ pmap_try_insert_pv_entry(pmap_t pmap, vm_offset_t va,
/*
* Create the pv entries for each of the pages within a section.
*/
-static boolean_t
-pmap_pv_insert_pte1(pmap_t pmap, vm_offset_t va, vm_paddr_t pa)
+static bool
+pmap_pv_insert_pte1(pmap_t pmap, vm_offset_t va, pt1_entry_t pte1, u_int flags)
{
struct md_page *pvh;
pv_entry_t pv;
+ bool noreclaim;
rw_assert(&pvh_global_lock, RA_WLOCKED);
- if (pv_entry_count < pv_entry_high_water &&
- (pv = get_pv_entry(pmap, TRUE)) != NULL) {
- pv->pv_va = va;
- pvh = pa_to_pvh(pa);
- TAILQ_INSERT_TAIL(&pvh->pv_list, pv, pv_next);
- return (TRUE);
- } else
- return (FALSE);
+ noreclaim = (flags & PMAP_ENTER_NORECLAIM) != 0;
+ if ((noreclaim && pv_entry_count >= pv_entry_high_water) ||
+ (pv = get_pv_entry(pmap, noreclaim)) == NULL)
+ return (false);
+ pv->pv_va = va;
+ pvh = pa_to_pvh(pte1_pa(pte1));
+ TAILQ_INSERT_TAIL(&pvh->pv_list, pv, pv_next);
+ return (true);
}
static inline void
@@ -3879,7 +3895,7 @@ pmap_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, v
if ((prot & VM_PROT_WRITE) == 0)
npte2 |= PTE2_RO;
KASSERT((npte2 & (PTE2_NM | PTE2_RO)) != PTE2_RO,
- ("pmap_enter: flags includes VM_PROT_WRITE but prot doesn't"));
+ ("%s: flags includes VM_PROT_WRITE but prot doesn't", __func__));
if ((prot & VM_PROT_EXECUTE) == 0)
npte2 |= PTE2_NX;
if ((flags & PMAP_ENTER_WIRED) != 0)
@@ -3892,6 +3908,15 @@ pmap_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, v
rw_wlock(&pvh_global_lock);
PMAP_LOCK(pmap);
sched_pin();
+ if (psind == 1) {
+ /* Assert the required virtual and physical alignment. */
+ KASSERT((va & PTE1_OFFSET) == 0,
+ ("%s: va unaligned", __func__));
+ KASSERT(m->psind > 0, ("%s: m->psind < psind", __func__));
+ rv = pmap_enter_pte1(pmap, va, PTE1_PA(pa) | ATTR_TO_L1(npte2) |
+ PTE1_V, flags, m);
+ goto out;
+ }
/*
* In the case that a page table page is not
@@ -3937,7 +3962,7 @@ pmap_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, v
*/
if (mpte2)
pt2_wirecount_dec(mpte2, pte1_index(va));
- if (pte2_is_managed(opte2))
+ if ((m->oflags & VPO_UNMANAGED) == 0)
om = m;
goto validate;
}
@@ -3962,10 +3987,12 @@ pmap_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, v
if (opa) {
if (pte2_is_wired(opte2))
pmap->pm_stats.wired_count--;
- if (pte2_is_managed(opte2)) {
- om = PHYS_TO_VM_PAGE(opa);
+ om = PHYS_TO_VM_PAGE(opa);
+ if (om != NULL && (om->oflags & VPO_UNMANAGED) != 0)
+ om = NULL;
+ if (om != NULL)
pv = pmap_pvh_remove(&om->md, pmap, va);
- }
+
/*
* Remove extra pte2 reference
*/
@@ -3997,7 +4024,7 @@ validate:
* Now validate mapping with desired protection/wiring.
*/
if (prot & VM_PROT_WRITE) {
- if (pte2_is_managed(npte2))
+ if ((m->oflags & VPO_UNMANAGED) == 0)
vm_page_aflag_set(m, PGA_WRITEABLE);
}
@@ -4034,19 +4061,18 @@ validate:
opte2 = pte2_load_clear(pte2p);
pmap_tlb_flush(pmap, va);
pte2_store(pte2p, npte2);
- if (opte2 & PTE2_A) {
- if (pte2_is_managed(opte2))
+ if (om != NULL) {
+ KASSERT((om->oflags & VPO_UNMANAGED) == 0,
+ ("%s: om %p unmanaged", __func__, om));
+ if ((opte2 & PTE2_A) != 0)
vm_page_aflag_set(om, PGA_REFERENCED);
- }
- if (pte2_is_dirty(opte2)) {
- if (pte2_is_managed(opte2))
+ if (pte2_is_dirty(opte2))
vm_page_dirty(om);
+ if (TAILQ_EMPTY(&om->md.pv_list) &&
+ ((om->flags & PG_FICTITIOUS) != 0 ||
+ TAILQ_EMPTY(&pa_to_pvh(opa)->pv_list)))
+ vm_page_aflag_clear(om, PGA_WRITEABLE);
}
- if (pte2_is_managed(opte2) &&
- TAILQ_EMPTY(&om->md.pv_list) &&
- ((om->flags & PG_FICTITIOUS) != 0 ||
- TAILQ_EMPTY(&pa_to_pvh(opa)->pv_list)))
- vm_page_aflag_clear(om, PGA_WRITEABLE);
} else
pte2_store(pte2p, npte2);
}
@@ -4652,66 +4678,124 @@ pmap_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_
}
/*
- * Tries to create 1MB page mapping. Returns TRUE if successful and
- * FALSE otherwise. Fails if (1) a page table page cannot be allocated without
- * blocking, (2) a mapping already exists at the specified virtual address, or
- * (3) a pv entry cannot be allocated without reclaiming another pv entry.
+ * Tries to create a read- and/or execute-only 1 MB page mapping. Returns
+ * true if successful. Returns false if (1) a mapping already exists at the
+ * specified virtual address or (2) a PV entry cannot be allocated without
+ * reclaiming another PV entry.
*/
-static boolean_t
-pmap_enter_pte1(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot)
+static bool
+pmap_enter_1mpage(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot)
{
- pt1_entry_t *pte1p;
+ pt1_entry_t pte1;
vm_paddr_t pa;
- uint32_t l1prot;
+ PMAP_LOCK_ASSERT(pmap, MA_OWNED);
+ pa = VM_PAGE_TO_PHYS(m);
+ pte1 = PTE1(pa, PTE1_NM | PTE1_RO, ATTR_TO_L1(vm_page_pte2_attr(m)));
+ if ((prot & VM_PROT_EXECUTE) == 0)
+ pte1 |= PTE1_NX;
+ if (va < VM_MAXUSER_ADDRESS)
+ pte1 |= PTE1_U;
+ if (pmap != kernel_pmap)
+ pte1 |= PTE1_NG;
+ return (pmap_enter_pte1(pmap, va, pte1, PMAP_ENTER_NOSLEEP |
+ PMAP_ENTER_NOREPLACE | PMAP_ENTER_NORECLAIM, m) == KERN_SUCCESS);
+}
+
+/*
+ * Tries to create the specified 1 MB page mapping. Returns KERN_SUCCESS if
+ * the mapping was created, and either KERN_FAILURE or KERN_RESOURCE_SHORTAGE
+ * otherwise. Returns KERN_FAILURE if PMAP_ENTER_NOREPLACE was specified and
+ * a mapping already exists at the specified virtual address. Returns
+ * KERN_RESOURCE_SHORTAGE if PMAP_ENTER_NORECLAIM was specified and PV entry
+ * allocation failed.
+ */
+static int
+pmap_enter_pte1(pmap_t pmap, vm_offset_t va, pt1_entry_t pte1, u_int flags,
+ vm_page_t m)
+{
+ struct spglist free;
+ pt1_entry_t opte1, *pte1p;
+ pt2_entry_t pte2, *pte2p;
+ vm_offset_t cur, end;
+ vm_page_t mt;
+
rw_assert(&pvh_global_lock, RA_WLOCKED);
+ KASSERT((pte1 & (PTE1_NM | PTE1_RO)) == 0 ||
+ (pte1 & (PTE1_NM | PTE1_RO)) == (PTE1_NM | PTE1_RO),
+ ("%s: pte1 has inconsistent NM and RO attributes", __func__));
PMAP_LOCK_ASSERT(pmap, MA_OWNED);
pte1p = pmap_pte1(pmap, va);
- if (pte1_is_valid(pte1_load(pte1p))) {
- CTR3(KTR_PMAP, "%s: failure for va %#lx in pmap %p", __func__,
- va, pmap);
- return (FALSE);
+ opte1 = pte1_load(pte1p);
+ if (pte1_is_valid(opte1)) {
+ if ((flags & PMAP_ENTER_NOREPLACE) != 0) {
+ CTR3(KTR_PMAP, "%s: failure for va %#lx in pmap %p",
+ __func__, va, pmap);
+ return (KERN_FAILURE);
+ }
+ /* Break the existing mapping(s). */
+ SLIST_INIT(&free);
+ if (pte1_is_section(opte1)) {
+ /*
+ * If the section resulted from a promotion, then a
+ * reserved PT page could be freed.
+ */
+ pmap_remove_pte1(pmap, pte1p, va, &free);
+ } else {
+ sched_pin();
+ end = va + PTE1_SIZE;
+ for (cur = va, pte2p = pmap_pte2_quick(pmap, va);
+ cur != end; cur += PAGE_SIZE, pte2p++) {
+ pte2 = pte2_load(pte2p);
+ if (!pte2_is_valid(pte2))
+ continue;
+ if (pmap_remove_pte2(pmap, pte2p, cur, &free))
+ break;
+ }
+ sched_unpin();
+ }
+ vm_page_free_pages_toq(&free, false);
}
if ((m->oflags & VPO_UNMANAGED) == 0) {
/*
* Abort this mapping if its PV entry could not be created.
*/
- if (!pmap_pv_insert_pte1(pmap, va, VM_PAGE_TO_PHYS(m))) {
+ if (!pmap_pv_insert_pte1(pmap, va, pte1, flags)) {
CTR3(KTR_PMAP, "%s: failure for va %#lx in pmap %p",
__func__, va, pmap);
- return (FALSE);
+ return (KERN_RESOURCE_SHORTAGE);
}
+ if ((pte1 & PTE1_RO) == 0) {
+ for (mt = m; mt < &m[PTE1_SIZE / PAGE_SIZE]; mt++)
+ vm_page_aflag_set(mt, PGA_WRITEABLE);
+ }
}
+
/*
* Increment counters.
*/
+ if (pte1_is_wired(pte1))
+ pmap->pm_stats.wired_count += PTE1_SIZE / PAGE_SIZE;
pmap->pm_stats.resident_count += PTE1_SIZE / PAGE_SIZE;
/*
+ * Sync icache if exec permission and attribute VM_MEMATTR_WB_WA
+ * is set. QQQ: For more info, see comments in pmap_enter().
+ */
+ if ((pte1 & PTE1_NX) == 0 && m->md.pat_mode == VM_MEMATTR_WB_WA &&
+ pmap != kernel_pmap && (!pte1_is_section(opte1) ||
+ pte1_pa(opte1) != VM_PAGE_TO_PHYS(m) || (opte1 & PTE2_NX) != 0))
+ cache_icache_sync_fresh(va, VM_PAGE_TO_PHYS(m), PTE1_SIZE);
+
+ /*
* Map the section.
- *
- * QQQ: Why VM_PROT_WRITE is not evaluated and the mapping is
- * made readonly?
*/
- pa = VM_PAGE_TO_PHYS(m);
- l1prot = PTE1_RO | PTE1_NM;
- if (va < VM_MAXUSER_ADDRESS)
- l1prot |= PTE1_U | PTE1_NG;
- if ((prot & VM_PROT_EXECUTE) == 0)
- l1prot |= PTE1_NX;
- else if (m->md.pat_mode == VM_MEMATTR_WB_WA && pmap != kernel_pmap) {
- /*
- * Sync icache if exec permission and attribute VM_MEMATTR_WB_WA
- * is set. QQQ: For more info, see comments in pmap_enter().
- */
- cache_icache_sync_fresh(va, pa, PTE1_SIZE);
- }
- pte1_store(pte1p, PTE1(pa, l1prot, ATTR_TO_L1(vm_page_pte2_attr(m))));
+ pte1_store(pte1p, pte1);
pmap_pte1_mappings++;
CTR3(KTR_PMAP, "%s: success for va %#lx in pmap %p", __func__, va,
pmap);
- return (TRUE);
+ return (KERN_SUCCESS);
}
/*
@@ -4747,7 +4831,7 @@ pmap_enter_object(pmap_t pmap, vm_offset_t start, vm_o
va = start + ptoa(diff);
if ((va & PTE1_OFFSET) == 0 && va + PTE1_SIZE <= end &&
m->psind == 1 && sp_enabled &&
- pmap_enter_pte1(pmap, va, m, prot))
+ pmap_enter_1mpage(pmap, va, m, prot))
m = &m[PTE1_SIZE / PAGE_SIZE - 1];
else
mpt2pg = pmap_enter_quick_locked(pmap, va, m, prot,
@@ -6020,8 +6104,8 @@ pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_
continue;
if (dst_pmap->pm_pt1[pte1_idx] == 0 &&
(!pte1_is_managed(src_pte1) ||
- pmap_pv_insert_pte1(dst_pmap, addr,
- pte1_pa(src_pte1)))) {
+ pmap_pv_insert_pte1(dst_pmap, addr, src_pte1,
+ PMAP_ENTER_NORECLAIM))) {
dst_pmap->pm_pt1[pte1_idx] = src_pte1 &
~PTE1_W;
dst_pmap->pm_stats.resident_count +=
Modified: head/sys/arm/include/pmap-v6.h
==============================================================================
--- head/sys/arm/include/pmap-v6.h Wed Aug 8 16:17:50 2018 (r337462)
+++ head/sys/arm/include/pmap-v6.h Wed Aug 8 16:55:01 2018 (r337463)
@@ -167,6 +167,7 @@ void pmap_bootstrap(vm_offset_t);
void pmap_kenter(vm_offset_t, vm_paddr_t);
void pmap_kremove(vm_offset_t);
boolean_t pmap_page_is_mapped(vm_page_t);
+bool pmap_ps_enabled(pmap_t pmap);
void pmap_tlb_flush(pmap_t, vm_offset_t);
void pmap_tlb_flush_range(pmap_t, vm_offset_t, vm_size_t);
Modified: head/sys/vm/vm_fault.c
==============================================================================
--- head/sys/vm/vm_fault.c Wed Aug 8 16:17:50 2018 (r337462)
+++ head/sys/vm/vm_fault.c Wed Aug 8 16:55:01 2018 (r337463)
@@ -270,8 +270,8 @@ vm_fault_soft_fast(struct faultstate *fs, vm_offset_t
int fault_type, int fault_flags, boolean_t wired, vm_page_t *m_hold)
{
vm_page_t m, m_map;
-#if (defined(__amd64__) || defined(__i386__) || defined(__aarch64__)) && \
- VM_NRESERVLEVEL > 0
+#if (defined(__aarch64__) || defined(__amd64__) || (defined(__arm__) && \
+ __ARM_ARCH >= 6) || defined(__i386__)) && VM_NRESERVLEVEL > 0
vm_page_t m_super;
int flags;
#endif
@@ -285,8 +285,8 @@ vm_fault_soft_fast(struct faultstate *fs, vm_offset_t
return (KERN_FAILURE);
m_map = m;
psind = 0;
-#if (defined(__amd64__) || defined(__i386__) || defined(__aarch64__)) && \
- VM_NRESERVLEVEL > 0
+#if (defined(__aarch64__) || defined(__amd64__) || (defined(__arm__) && \
+ __ARM_ARCH >= 6) || defined(__i386__)) && VM_NRESERVLEVEL > 0
if ((m->flags & PG_FICTITIOUS) == 0 &&
(m_super = vm_reserv_to_superpage(m)) != NULL &&
rounddown2(vaddr, pagesizes[m_super->psind]) >= fs->entry->start &&
@@ -462,7 +462,8 @@ vm_fault_populate(struct faultstate *fs, vm_prot_t pro
pidx <= pager_last;
pidx += npages, m = vm_page_next(&m[npages - 1])) {
vaddr = fs->entry->start + IDX_TO_OFF(pidx) - fs->entry->offset;
-#if defined(__amd64__) || defined(__i386__) || defined(__aarch64__)
+#if defined(__aarch64__) || defined(__amd64__) || (defined(__arm__) && \
+ __ARM_ARCH >= 6) || defined(__i386__)
psind = m->psind;
if (psind > 0 && ((vaddr & (pagesizes[psind] - 1)) != 0 ||
pidx + OFF_TO_IDX(pagesizes[psind]) - 1 > pager_last ||
More information about the svn-src-all
mailing list