svn commit: r355883 - in head/sys: amd64/amd64 arm64/arm64
Alan Cox
alc at FreeBSD.org
Wed Dec 18 18:21:40 UTC 2019
Author: alc
Date: Wed Dec 18 18:21:39 2019
New Revision: 355883
URL: https://svnweb.freebsd.org/changeset/base/355883
Log:
When pmap_enter_{l2,pde}() are called to create a kernel mapping, they are
incrementing (and decrementing) the ref_count on kernel page table pages.
They should not do this. Kernel page table pages are expected to have a
fixed ref_count. Address this problem by refactoring pmap_alloc{_l2,pde}()
and their callers. This also eliminates some duplicated code from the
callers.
Correctly implement PMAP_ENTER_NOREPLACE in pmap_enter_{l2,pde}() on kernel
mappings.
Reduce code duplication by defining a function, pmap_abort_ptp(), for
handling a common error case.
Handle a possible page table page leak in pmap_copy(). Suppose that we are
determining whether to copy a superpage mapping. If we abort because there
is already a mapping in the destination pmap at the current address, then
simply decrementing the page table page's ref_count is correct, because the
page table page must have a ref_count > 1. However, if we abort because we
failed to allocate a PV entry, this might be a just allocated page table
page that has a ref_count = 1, so we should call pmap_abort_ptp().
Simplify error handling in pmap_enter_quick_locked().
Reviewed by: kib, markj (an earlier)
MFC after: 2 weeks
Differential Revision: https://reviews.freebsd.org/D22763
Modified:
head/sys/amd64/amd64/pmap.c
head/sys/arm64/arm64/pmap.c
Modified: head/sys/amd64/amd64/pmap.c
==============================================================================
--- head/sys/amd64/amd64/pmap.c Wed Dec 18 16:01:15 2019 (r355882)
+++ head/sys/amd64/amd64/pmap.c Wed Dec 18 18:21:39 2019 (r355883)
@@ -1202,6 +1202,7 @@ static void pmap_pvh_free(struct md_page *pvh, pmap_t
static pv_entry_t pmap_pvh_remove(struct md_page *pvh, pmap_t pmap,
vm_offset_t va);
+static void pmap_abort_ptp(pmap_t pmap, vm_offset_t va, vm_page_t mpte);
static int pmap_change_props_locked(vm_offset_t va, vm_size_t size,
vm_prot_t prot, int mode, int flags);
static boolean_t pmap_demote_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va);
@@ -1256,7 +1257,7 @@ static void pmap_update_pde_invalidate(pmap_t, vm_offs
static vm_page_t _pmap_allocpte(pmap_t pmap, vm_pindex_t ptepindex,
struct rwlock **lockp);
-static vm_page_t pmap_allocpde(pmap_t pmap, vm_offset_t va,
+static pd_entry_t *pmap_alloc_pde(pmap_t pmap, vm_offset_t va, vm_page_t *pdpgp,
struct rwlock **lockp);
static vm_page_t pmap_allocpte(pmap_t pmap, vm_offset_t va,
struct rwlock **lockp);
@@ -3601,6 +3602,27 @@ pmap_unuse_pt(pmap_t pmap, vm_offset_t va, pd_entry_t
return (pmap_unwire_ptp(pmap, va, mpte, free));
}
+/*
+ * Release a page table page reference after a failed attempt to create a
+ * mapping.
+ */
+static void
+pmap_abort_ptp(pmap_t pmap, vm_offset_t va, vm_page_t mpte)
+{
+ struct spglist free;
+
+ SLIST_INIT(&free);
+ if (pmap_unwire_ptp(pmap, va, mpte, &free)) {
+ /*
+ * Although "va" was never mapped, paging-structure caches
+ * could nonetheless have entries that refer to the freed
+ * page table pages. Invalidate those entries.
+ */
+ pmap_invalidate_page(pmap, va);
+ vm_page_free_pages_toq(&free, true);
+ }
+}
+
void
pmap_pinit0(pmap_t pmap)
{
@@ -3906,30 +3928,44 @@ _pmap_allocpte(pmap_t pmap, vm_pindex_t ptepindex, str
return (m);
}
-static vm_page_t
-pmap_allocpde(pmap_t pmap, vm_offset_t va, struct rwlock **lockp)
+static pd_entry_t *
+pmap_alloc_pde(pmap_t pmap, vm_offset_t va, vm_page_t *pdpgp,
+ struct rwlock **lockp)
{
- vm_pindex_t pdpindex, ptepindex;
pdp_entry_t *pdpe, PG_V;
+ pd_entry_t *pde;
vm_page_t pdpg;
+ vm_pindex_t pdpindex;
PG_V = pmap_valid_bit(pmap);
retry:
pdpe = pmap_pdpe(pmap, va);
if (pdpe != NULL && (*pdpe & PG_V) != 0) {
- /* Add a reference to the pd page. */
- pdpg = PHYS_TO_VM_PAGE(*pdpe & PG_FRAME);
- pdpg->ref_count++;
- } else {
+ pde = pmap_pdpe_to_pde(pdpe, va);
+ if (va < VM_MAXUSER_ADDRESS) {
+ /* Add a reference to the pd page. */
+ pdpg = PHYS_TO_VM_PAGE(*pdpe & PG_FRAME);
+ pdpg->ref_count++;
+ } else
+ pdpg = NULL;
+ } else if (va < VM_MAXUSER_ADDRESS) {
/* Allocate a pd page. */
- ptepindex = pmap_pde_pindex(va);
- pdpindex = ptepindex >> NPDPEPGSHIFT;
+ pdpindex = pmap_pde_pindex(va) >> NPDPEPGSHIFT;
pdpg = _pmap_allocpte(pmap, NUPDE + pdpindex, lockp);
- if (pdpg == NULL && lockp != NULL)
- goto retry;
- }
- return (pdpg);
+ if (pdpg == NULL) {
+ if (lockp != NULL)
+ goto retry;
+ else
+ return (NULL);
+ }
+ pde = (pd_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(pdpg));
+ pde = &pde[pmap_pde_index(va)];
+ } else
+ panic("pmap_alloc_pde: missing page table page for va %#lx",
+ va);
+ *pdpgp = pdpg;
+ return (pde);
}
static vm_page_t
@@ -6218,6 +6254,24 @@ pmap_enter_2mpage(pmap_t pmap, vm_offset_t va, vm_page
}
/*
+ * Returns true if every page table entry in the specified page table page is
+ * zero.
+ */
+static bool
+pmap_every_pte_zero(vm_paddr_t pa)
+{
+ pt_entry_t *pt_end, *pte;
+
+ KASSERT((pa & PAGE_MASK) == 0, ("pa is misaligned"));
+ pte = (pt_entry_t *)PHYS_TO_DMAP(pa);
+ for (pt_end = pte + NPTEPG; pte < pt_end; pte++) {
+ if (*pte != 0)
+ return (false);
+ }
+ return (true);
+}
+
+/*
* Tries to create the specified 2MB page mapping. Returns KERN_SUCCESS if
* the mapping was created, and either KERN_FAILURE or KERN_RESOURCE_SHORTAGE
* otherwise. Returns KERN_FAILURE if PMAP_ENTER_NOREPLACE was specified and
@@ -6252,8 +6306,8 @@ pmap_enter_pde(pmap_t pmap, vm_offset_t va, pd_entry_t
" in pmap %p", va, pmap);
return (KERN_FAILURE);
}
- if ((pdpg = pmap_allocpde(pmap, va, (flags & PMAP_ENTER_NOSLEEP) != 0 ?
- NULL : lockp)) == NULL) {
+ if ((pde = pmap_alloc_pde(pmap, va, &pdpg, (flags &
+ PMAP_ENTER_NOSLEEP) != 0 ? NULL : lockp)) == NULL) {
CTR2(KTR_PMAP, "pmap_enter_pde: failure for va %#lx"
" in pmap %p", va, pmap);
return (KERN_RESOURCE_SHORTAGE);
@@ -6265,11 +6319,7 @@ pmap_enter_pde(pmap_t pmap, vm_offset_t va, pd_entry_t
* it could sleep.
*/
if (!pmap_pkru_same(pmap, va, va + NBPDR)) {
- SLIST_INIT(&free);
- if (pmap_unwire_ptp(pmap, va, pdpg, &free)) {
- pmap_invalidate_page(pmap, va);
- vm_page_free_pages_toq(&free, true);
- }
+ pmap_abort_ptp(pmap, va, pdpg);
return (KERN_FAILURE);
}
if (va < VM_MAXUSER_ADDRESS && pmap->pm_type == PT_X86) {
@@ -6277,14 +6327,18 @@ pmap_enter_pde(pmap_t pmap, vm_offset_t va, pd_entry_t
newpde |= pmap_pkru_get(pmap, va);
}
- pde = (pd_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(pdpg));
- pde = &pde[pmap_pde_index(va)];
+ /*
+ * If there are existing mappings, either abort or remove them.
+ */
oldpde = *pde;
if ((oldpde & PG_V) != 0) {
- KASSERT(pdpg->ref_count > 1,
+ KASSERT(pdpg == NULL || pdpg->ref_count > 1,
("pmap_enter_pde: pdpg's reference count is too low"));
- if ((flags & PMAP_ENTER_NOREPLACE) != 0) {
- pdpg->ref_count--;
+ if ((flags & PMAP_ENTER_NOREPLACE) != 0 && (va <
+ VM_MAXUSER_ADDRESS || (oldpde & PG_PS) != 0 ||
+ pmap_every_pte_zero(oldpde & PG_FRAME))) {
+ if (pdpg != NULL)
+ pdpg->ref_count--;
CTR2(KTR_PMAP, "pmap_enter_pde: failure for va %#lx"
" in pmap %p", va, pmap);
return (KERN_FAILURE);
@@ -6294,7 +6348,7 @@ pmap_enter_pde(pmap_t pmap, vm_offset_t va, pd_entry_t
if ((oldpde & PG_PS) != 0) {
/*
* The reference to the PD page that was acquired by
- * pmap_allocpde() ensures that it won't be freed.
+ * pmap_alloc_pde() ensures that it won't be freed.
* However, if the PDE resulted from a promotion, then
* a reserved PT page could be freed.
*/
@@ -6308,8 +6362,14 @@ pmap_enter_pde(pmap_t pmap, vm_offset_t va, pd_entry_t
pmap_invalidate_all(pmap);
pmap_delayed_invl_finish();
}
- vm_page_free_pages_toq(&free, true);
- if (va >= VM_MAXUSER_ADDRESS) {
+ if (va < VM_MAXUSER_ADDRESS) {
+ vm_page_free_pages_toq(&free, true);
+ KASSERT(*pde == 0, ("pmap_enter_pde: non-zero pde %p",
+ pde));
+ } else {
+ KASSERT(SLIST_EMPTY(&free),
+ ("pmap_enter_pde: freed kernel page table page"));
+
/*
* Both pmap_remove_pde() and pmap_remove_ptes() will
* leave the kernel page table page zero filled.
@@ -6317,26 +6377,16 @@ pmap_enter_pde(pmap_t pmap, vm_offset_t va, pd_entry_t
mt = PHYS_TO_VM_PAGE(*pde & PG_FRAME);
if (pmap_insert_pt_page(pmap, mt, false))
panic("pmap_enter_pde: trie insert failed");
- } else
- KASSERT(*pde == 0, ("pmap_enter_pde: non-zero pde %p",
- pde));
+ }
}
+
if ((newpde & PG_MANAGED) != 0) {
/*
* Abort this mapping if its PV entry could not be created.
*/
if (!pmap_pv_insert_pde(pmap, va, newpde, flags, lockp)) {
- SLIST_INIT(&free);
- if (pmap_unwire_ptp(pmap, va, pdpg, &free)) {
- /*
- * Although "va" is not mapped, paging-
- * structure caches could nonetheless have
- * entries that refer to the freed page table
- * pages. Invalidate those entries.
- */
- pmap_invalidate_page(pmap, va);
- vm_page_free_pages_toq(&free, true);
- }
+ if (pdpg != NULL)
+ pmap_abort_ptp(pmap, va, pdpg);
CTR2(KTR_PMAP, "pmap_enter_pde: failure for va %#lx"
" in pmap %p", va, pmap);
return (KERN_RESOURCE_SHORTAGE);
@@ -6361,8 +6411,8 @@ pmap_enter_pde(pmap_t pmap, vm_offset_t va, pd_entry_t
pde_store(pde, newpde);
atomic_add_long(&pmap_pde_mappings, 1);
- CTR2(KTR_PMAP, "pmap_enter_pde: success for va %#lx"
- " in pmap %p", va, pmap);
+ CTR2(KTR_PMAP, "pmap_enter_pde: success for va %#lx in pmap %p",
+ va, pmap);
return (KERN_SUCCESS);
}
@@ -6437,7 +6487,6 @@ static vm_page_t
pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m,
vm_prot_t prot, vm_page_t mpte, struct rwlock **lockp)
{
- struct spglist free;
pt_entry_t newpte, *pte, PG_V;
KASSERT(va < kmi.clean_sva || va >= kmi.clean_eva ||
@@ -6494,11 +6543,9 @@ pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, v
pte = vtopte(va);
}
if (*pte) {
- if (mpte != NULL) {
+ if (mpte != NULL)
mpte->ref_count--;
- mpte = NULL;
- }
- return (mpte);
+ return (NULL);
}
/*
@@ -6506,21 +6553,9 @@ pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, v
*/
if ((m->oflags & VPO_UNMANAGED) == 0 &&
!pmap_try_insert_pv_entry(pmap, va, m, lockp)) {
- if (mpte != NULL) {
- SLIST_INIT(&free);
- if (pmap_unwire_ptp(pmap, va, mpte, &free)) {
- /*
- * Although "va" is not mapped, paging-
- * structure caches could nonetheless have
- * entries that refer to the freed page table
- * pages. Invalidate those entries.
- */
- pmap_invalidate_page(pmap, va);
- vm_page_free_pages_toq(&free, true);
- }
- mpte = NULL;
- }
- return (mpte);
+ if (mpte != NULL)
+ pmap_abort_ptp(pmap, va, mpte);
+ return (NULL);
}
/*
@@ -6620,8 +6655,8 @@ pmap_object_init_pt(pmap_t pmap, vm_offset_t addr, vm_
PMAP_LOCK(pmap);
for (pa = ptepa | pmap_cache_bits(pmap, pat_mode, 1);
pa < ptepa + size; pa += NBPDR) {
- pdpg = pmap_allocpde(pmap, addr, NULL);
- if (pdpg == NULL) {
+ pde = pmap_alloc_pde(pmap, addr, &pdpg, NULL);
+ if (pde == NULL) {
/*
* The creation of mappings below is only an
* optimization. If a page directory page
@@ -6632,8 +6667,6 @@ pmap_object_init_pt(pmap_t pmap, vm_offset_t addr, vm_
addr += NBPDR;
continue;
}
- pde = (pd_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(pdpg));
- pde = &pde[pmap_pde_index(addr)];
if ((*pde & PG_V) == 0) {
pde_store(pde, pa | PG_PS | PG_M | PG_A |
PG_U | PG_RW | PG_V);
@@ -6747,7 +6780,6 @@ pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_
vm_offset_t src_addr)
{
struct rwlock *lock;
- struct spglist free;
pml4_entry_t *pml4e;
pdp_entry_t *pdpe;
pd_entry_t *pde, srcptepaddr;
@@ -6818,12 +6850,9 @@ pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_
if (srcptepaddr & PG_PS) {
if ((addr & PDRMASK) != 0 || addr + NBPDR > end_addr)
continue;
- dst_pdpg = pmap_allocpde(dst_pmap, addr, NULL);
- if (dst_pdpg == NULL)
+ pde = pmap_alloc_pde(dst_pmap, addr, &dst_pdpg, NULL);
+ if (pde == NULL)
break;
- pde = (pd_entry_t *)
- PHYS_TO_DMAP(VM_PAGE_TO_PHYS(dst_pdpg));
- pde = &pde[pmap_pde_index(addr)];
if (*pde == 0 && ((srcptepaddr & PG_MANAGED) == 0 ||
pmap_pv_insert_pde(dst_pmap, addr, srcptepaddr,
PMAP_ENTER_NORECLAIM, &lock))) {
@@ -6832,7 +6861,7 @@ pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_
PAGE_SIZE);
atomic_add_long(&pmap_pde_mappings, 1);
} else
- dst_pdpg->ref_count--;
+ pmap_abort_ptp(dst_pmap, addr, dst_pdpg);
continue;
}
@@ -6877,19 +6906,7 @@ pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_
*dst_pte = ptetemp & ~(PG_W | PG_M | PG_A);
pmap_resident_count_inc(dst_pmap, 1);
} else {
- SLIST_INIT(&free);
- if (pmap_unwire_ptp(dst_pmap, addr, dstmpte,
- &free)) {
- /*
- * Although "addr" is not mapped,
- * paging-structure caches could
- * nonetheless have entries that refer
- * to the freed page table pages.
- * Invalidate those entries.
- */
- pmap_invalidate_page(dst_pmap, addr);
- vm_page_free_pages_toq(&free, true);
- }
+ pmap_abort_ptp(dst_pmap, addr, dstmpte);
goto out;
}
/* Have we copied all of the valid mappings? */
Modified: head/sys/arm64/arm64/pmap.c
==============================================================================
--- head/sys/arm64/arm64/pmap.c Wed Dec 18 16:01:15 2019 (r355882)
+++ head/sys/arm64/arm64/pmap.c Wed Dec 18 18:21:39 2019 (r355883)
@@ -331,6 +331,7 @@ static void pmap_pvh_free(struct md_page *pvh, pmap_t
static pv_entry_t pmap_pvh_remove(struct md_page *pvh, pmap_t pmap,
vm_offset_t va);
+static void pmap_abort_ptp(pmap_t pmap, vm_offset_t va, vm_page_t mpte);
static bool pmap_activate_int(pmap_t pmap);
static void pmap_alloc_asid(pmap_t pmap);
static int pmap_change_attr_locked(vm_offset_t va, vm_size_t size, int mode);
@@ -1500,6 +1501,29 @@ pmap_unuse_pt(pmap_t pmap, vm_offset_t va, pd_entry_t
return (pmap_unwire_l3(pmap, va, mpte, free));
}
+/*
+ * Release a page table page reference after a failed attempt to create a
+ * mapping.
+ */
+static void
+pmap_abort_ptp(pmap_t pmap, vm_offset_t va, vm_page_t mpte)
+{
+ struct spglist free;
+
+ SLIST_INIT(&free);
+ if (pmap_unwire_l3(pmap, va, mpte, &free)) {
+ /*
+ * Although "va" was never mapped, the TLB could nonetheless
+ * have intermediate entries that refer to the freed page
+ * table pages. Invalidate those entries.
+ *
+ * XXX redundant invalidation (See _pmap_unwire_l3().)
+ */
+ pmap_invalidate_page(pmap, va);
+ vm_page_free_pages_toq(&free, true);
+ }
+}
+
void
pmap_pinit0(pmap_t pmap)
{
@@ -1677,27 +1701,41 @@ _pmap_alloc_l3(pmap_t pmap, vm_pindex_t ptepindex, str
return (m);
}
-static vm_page_t
-pmap_alloc_l2(pmap_t pmap, vm_offset_t va, struct rwlock **lockp)
+static pd_entry_t *
+pmap_alloc_l2(pmap_t pmap, vm_offset_t va, vm_page_t *l2pgp,
+ struct rwlock **lockp)
{
- pd_entry_t *l1;
+ pd_entry_t *l1, *l2;
vm_page_t l2pg;
vm_pindex_t l2pindex;
retry:
l1 = pmap_l1(pmap, va);
if (l1 != NULL && (pmap_load(l1) & ATTR_DESCR_MASK) == L1_TABLE) {
- /* Add a reference to the L2 page. */
- l2pg = PHYS_TO_VM_PAGE(pmap_load(l1) & ~ATTR_MASK);
- l2pg->ref_count++;
- } else {
+ l2 = pmap_l1_to_l2(l1, va);
+ if (va < VM_MAXUSER_ADDRESS) {
+ /* Add a reference to the L2 page. */
+ l2pg = PHYS_TO_VM_PAGE(pmap_load(l1) & ~ATTR_MASK);
+ l2pg->ref_count++;
+ } else
+ l2pg = NULL;
+ } else if (va < VM_MAXUSER_ADDRESS) {
/* Allocate a L2 page. */
l2pindex = pmap_l2_pindex(va) >> Ln_ENTRIES_SHIFT;
l2pg = _pmap_alloc_l3(pmap, NUL2E + l2pindex, lockp);
- if (l2pg == NULL && lockp != NULL)
- goto retry;
- }
- return (l2pg);
+ if (l2pg == NULL) {
+ if (lockp != NULL)
+ goto retry;
+ else
+ return (NULL);
+ }
+ l2 = (pd_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(l2pg));
+ l2 = &l2[pmap_l2_index(va)];
+ } else
+ panic("pmap_alloc_l2: missing page table page for va %#lx",
+ va);
+ *l2pgp = l2pg;
+ return (l2);
}
static vm_page_t
@@ -3553,6 +3591,24 @@ pmap_enter_2mpage(pmap_t pmap, vm_offset_t va, vm_page
}
/*
+ * Returns true if every page table entry in the specified page table is
+ * zero.
+ */
+static bool
+pmap_every_pte_zero(vm_paddr_t pa)
+{
+ pt_entry_t *pt_end, *pte;
+
+ KASSERT((pa & PAGE_MASK) == 0, ("pa is misaligned"));
+ pte = (pt_entry_t *)PHYS_TO_DMAP(pa);
+ for (pt_end = pte + Ln_ENTRIES; pte < pt_end; pte++) {
+ if (*pte != 0)
+ return (false);
+ }
+ return (true);
+}
+
+/*
* Tries to create the specified 2MB page mapping. Returns KERN_SUCCESS if
* the mapping was created, and either KERN_FAILURE or KERN_RESOURCE_SHORTAGE
* otherwise. Returns KERN_FAILURE if PMAP_ENTER_NOREPLACE was specified and
@@ -3573,23 +3629,26 @@ pmap_enter_l2(pmap_t pmap, vm_offset_t va, pd_entry_t
PMAP_LOCK_ASSERT(pmap, MA_OWNED);
- if ((l2pg = pmap_alloc_l2(pmap, va, (flags & PMAP_ENTER_NOSLEEP) != 0 ?
- NULL : lockp)) == NULL) {
+ if ((l2 = pmap_alloc_l2(pmap, va, &l2pg, (flags &
+ PMAP_ENTER_NOSLEEP) != 0 ? NULL : lockp)) == NULL) {
CTR2(KTR_PMAP, "pmap_enter_l2: failure for va %#lx in pmap %p",
va, pmap);
return (KERN_RESOURCE_SHORTAGE);
}
- l2 = (pd_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(l2pg));
- l2 = &l2[pmap_l2_index(va)];
+ /*
+ * If there are existing mappings, either abort or remove them.
+ */
if ((old_l2 = pmap_load(l2)) != 0) {
- KASSERT(l2pg->ref_count > 1,
+ KASSERT(l2pg == NULL || l2pg->ref_count > 1,
("pmap_enter_l2: l2pg's ref count is too low"));
- if ((flags & PMAP_ENTER_NOREPLACE) != 0) {
- l2pg->ref_count--;
- CTR2(KTR_PMAP,
- "pmap_enter_l2: failure for va %#lx in pmap %p",
- va, pmap);
+ if ((flags & PMAP_ENTER_NOREPLACE) != 0 && (va <
+ VM_MAXUSER_ADDRESS || (old_l2 & ATTR_DESCR_MASK) ==
+ L2_BLOCK || pmap_every_pte_zero(old_l2 & ~ATTR_MASK))) {
+ if (l2pg != NULL)
+ l2pg->ref_count--;
+ CTR2(KTR_PMAP, "pmap_enter_l2: failure for va %#lx"
+ " in pmap %p", va, pmap);
return (KERN_FAILURE);
}
SLIST_INIT(&free);
@@ -3599,8 +3658,14 @@ pmap_enter_l2(pmap_t pmap, vm_offset_t va, pd_entry_t
else
pmap_remove_l3_range(pmap, old_l2, va, va + L2_SIZE,
&free, lockp);
- vm_page_free_pages_toq(&free, true);
- if (va >= VM_MAXUSER_ADDRESS) {
+ if (va < VM_MAXUSER_ADDRESS) {
+ vm_page_free_pages_toq(&free, true);
+ KASSERT(pmap_load(l2) == 0,
+ ("pmap_enter_l2: non-zero L2 entry %p", l2));
+ } else {
+ KASSERT(SLIST_EMPTY(&free),
+ ("pmap_enter_l2: freed kernel page table page"));
+
/*
* Both pmap_remove_l2() and pmap_remove_l3_range()
* will leave the kernel page table page zero filled.
@@ -3612,9 +3677,7 @@ pmap_enter_l2(pmap_t pmap, vm_offset_t va, pd_entry_t
panic("pmap_enter_l2: trie insert failed");
pmap_clear(l2);
pmap_invalidate_page(pmap, va);
- } else
- KASSERT(pmap_load(l2) == 0,
- ("pmap_enter_l2: non-zero L2 entry %p", l2));
+ }
}
if ((new_l2 & ATTR_SW_MANAGED) != 0) {
@@ -3622,20 +3685,8 @@ pmap_enter_l2(pmap_t pmap, vm_offset_t va, pd_entry_t
* Abort this mapping if its PV entry could not be created.
*/
if (!pmap_pv_insert_l2(pmap, va, new_l2, flags, lockp)) {
- SLIST_INIT(&free);
- if (pmap_unwire_l3(pmap, va, l2pg, &free)) {
- /*
- * Although "va" is not mapped, the TLB could
- * nonetheless have intermediate entries that
- * refer to the freed page table pages.
- * Invalidate those entries.
- *
- * XXX redundant invalidation (See
- * _pmap_unwire_l3().)
- */
- pmap_invalidate_page(pmap, va);
- vm_page_free_pages_toq(&free, true);
- }
+ if (l2pg != NULL)
+ pmap_abort_ptp(pmap, va, l2pg);
CTR2(KTR_PMAP,
"pmap_enter_l2: failure for va %#lx in pmap %p",
va, pmap);
@@ -3736,7 +3787,6 @@ static vm_page_t
pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m,
vm_prot_t prot, vm_page_t mpte, struct rwlock **lockp)
{
- struct spglist free;
pd_entry_t *pde;
pt_entry_t *l2, *l3, l3_val;
vm_paddr_t pa;
@@ -3810,11 +3860,9 @@ pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, v
* Abort if a mapping already exists.
*/
if (pmap_load(l3) != 0) {
- if (mpte != NULL) {
+ if (mpte != NULL)
mpte->ref_count--;
- mpte = NULL;
- }
- return (mpte);
+ return (NULL);
}
/*
@@ -3822,15 +3870,9 @@ pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, v
*/
if ((m->oflags & VPO_UNMANAGED) == 0 &&
!pmap_try_insert_pv_entry(pmap, va, m, lockp)) {
- if (mpte != NULL) {
- SLIST_INIT(&free);
- if (pmap_unwire_l3(pmap, va, mpte, &free)) {
- pmap_invalidate_page(pmap, va);
- vm_page_free_pages_toq(&free, true);
- }
- mpte = NULL;
- }
- return (mpte);
+ if (mpte != NULL)
+ pmap_abort_ptp(pmap, va, mpte);
+ return (NULL);
}
/*
@@ -3984,7 +4026,6 @@ pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_
vm_offset_t src_addr)
{
struct rwlock *lock;
- struct spglist free;
pd_entry_t *l0, *l1, *l2, srcptepaddr;
pt_entry_t *dst_pte, mask, nbits, ptetemp, *src_pte;
vm_offset_t addr, end_addr, va_next;
@@ -4027,12 +4068,9 @@ pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_
if ((addr & L2_OFFSET) != 0 ||
addr + L2_SIZE > end_addr)
continue;
- dst_l2pg = pmap_alloc_l2(dst_pmap, addr, NULL);
- if (dst_l2pg == NULL)
+ l2 = pmap_alloc_l2(dst_pmap, addr, &dst_l2pg, NULL);
+ if (l2 == NULL)
break;
- l2 = (pd_entry_t *)
- PHYS_TO_DMAP(VM_PAGE_TO_PHYS(dst_l2pg));
- l2 = &l2[pmap_l2_index(addr)];
if (pmap_load(l2) == 0 &&
((srcptepaddr & ATTR_SW_MANAGED) == 0 ||
pmap_pv_insert_l2(dst_pmap, addr, srcptepaddr,
@@ -4046,7 +4084,7 @@ pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_
PAGE_SIZE);
atomic_add_long(&pmap_l2_mappings, 1);
} else
- dst_l2pg->ref_count--;
+ pmap_abort_ptp(dst_pmap, addr, dst_l2pg);
continue;
}
KASSERT((srcptepaddr & ATTR_DESCR_MASK) == L2_TABLE,
@@ -4093,21 +4131,7 @@ pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_
pmap_store(dst_pte, (ptetemp & ~mask) | nbits);
pmap_resident_count_inc(dst_pmap, 1);
} else {
- SLIST_INIT(&free);
- if (pmap_unwire_l3(dst_pmap, addr, dstmpte,
- &free)) {
- /*
- * Although "addr" is not mapped,
- * the TLB could nonetheless have
- * intermediate entries that refer
- * to the freed page table pages.
- * Invalidate those entries.
- *
- * XXX redundant invalidation
- */
- pmap_invalidate_page(dst_pmap, addr);
- vm_page_free_pages_toq(&free, true);
- }
+ pmap_abort_ptp(dst_pmap, addr, dstmpte);
goto out;
}
/* Have we copied all of the valid mappings? */
More information about the svn-src-head
mailing list