svn commit: r237083 - in user/alc/superpages/sys: amd64/amd64
amd64/include vm
Alan Cox
alc at FreeBSD.org
Thu Jun 14 16:53:09 UTC 2012
Author: alc
Date: Thu Jun 14 16:53:08 2012
New Revision: 237083
URL: http://svn.freebsd.org/changeset/base/237083
Log:
Speed up the destruction of superpage mappings on amd64 by eliminating the
frequent need for pmap_remove_pages() and pmap_remove_pde() to access each
struct vm_page that makes up a superpage. This is accomplished through a
combination of two changes:
1) Replace PGA_WRITEABLE with an exact count of write mappings at both the
4KB and 2MB granularity so that pmap_remove_pages() and pmap_remove_pde()
must no longer access each struct vm_page in order to clear
PGA_WRITEABLE. (These counts are stored in space that was previously lost
to alignment in struct md_page, so struct vm_page is no larger.)
2) Call vm_page_dirty() during promotion of write mappings rather than
pmap_remove_pages() and pmap_remove_pde(). Each struct vm_page is
already accessed on promotion, so adding a call to vm_page_dirty()
shouldn't result in additional cache misses.
For "buildworld", this reduces the average running time of
pmap_remove_pages() by 23%.
Modified:
user/alc/superpages/sys/amd64/amd64/pmap.c
user/alc/superpages/sys/amd64/include/pmap.h
user/alc/superpages/sys/vm/swap_pager.c
user/alc/superpages/sys/vm/vm_page.c
user/alc/superpages/sys/vm/vm_pageout.c
user/alc/superpages/sys/vm/vnode_pager.c
Modified: user/alc/superpages/sys/amd64/amd64/pmap.c
==============================================================================
--- user/alc/superpages/sys/amd64/amd64/pmap.c Thu Jun 14 16:25:10 2012 (r237082)
+++ user/alc/superpages/sys/amd64/amd64/pmap.c Thu Jun 14 16:53:08 2012 (r237083)
@@ -231,9 +231,11 @@ static caddr_t crashdumpmap;
static void free_pv_chunk(struct pv_chunk *pc);
static void free_pv_entry(pmap_t pmap, pv_entry_t pv);
static pv_entry_t get_pv_entry(pmap_t pmap, boolean_t try);
-static void pmap_pv_demote_pde(pmap_t pmap, vm_offset_t va, vm_paddr_t pa);
-static boolean_t pmap_pv_insert_pde(pmap_t pmap, vm_offset_t va, vm_paddr_t pa);
-static void pmap_pv_promote_pde(pmap_t pmap, vm_offset_t va, vm_paddr_t pa);
+static void pmap_pv_demote_pde(pmap_t pmap, vm_offset_t va, pd_entry_t pde);
+static boolean_t pmap_pv_insert_pde(pmap_t pmap, vm_offset_t va,
+ pd_entry_t pde);
+static void pmap_pv_promote_pde(pmap_t pmap, vm_offset_t va,
+ pd_entry_t pde);
static void pmap_pvh_free(struct md_page *pvh, pmap_t pmap, vm_offset_t va);
static pv_entry_t pmap_pvh_remove(struct md_page *pvh, pmap_t pmap,
vm_offset_t va);
@@ -265,11 +267,8 @@ static int pmap_remove_pte(pmap_t pmap,
static void pmap_remove_pt_page(pmap_t pmap, vm_page_t mpte);
static void pmap_remove_page(pmap_t pmap, vm_offset_t va, pd_entry_t *pde,
vm_page_t *free);
-static void pmap_remove_entry(struct pmap *pmap, vm_page_t m,
- vm_offset_t va);
-static void pmap_insert_entry(pmap_t pmap, vm_offset_t va, vm_page_t m);
static boolean_t pmap_try_insert_pv_entry(pmap_t pmap, vm_offset_t va,
- vm_page_t m);
+ vm_page_t m, boolean_t write_mapping);
static void pmap_update_pde(pmap_t pmap, vm_offset_t va, pd_entry_t *pde,
pd_entry_t newpde);
static void pmap_update_pde_invalidate(vm_offset_t va, pd_entry_t newpde);
@@ -2063,7 +2062,6 @@ pmap_pv_reclaim(pmap_t locked_pmap)
{
struct pch newtail;
struct pv_chunk *pc;
- struct md_page *pvh;
pd_entry_t *pde;
pmap_t pmap;
pt_entry_t *pte, tpte;
@@ -2117,19 +2115,15 @@ pmap_pv_reclaim(pmap_t locked_pmap)
if ((tpte & PG_G) != 0)
pmap_invalidate_page(pmap, va);
m = PHYS_TO_VM_PAGE(tpte & PG_FRAME);
- if ((tpte & (PG_M | PG_RW)) == (PG_M | PG_RW))
- vm_page_dirty(m);
if ((tpte & PG_A) != 0)
vm_page_aflag_set(m, PGA_REFERENCED);
- TAILQ_REMOVE(&m->md.pv_list, pv, pv_list);
- if (TAILQ_EMPTY(&m->md.pv_list) &&
- (m->flags & PG_FICTITIOUS) == 0) {
- pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
- if (TAILQ_EMPTY(&pvh->pv_list)) {
- vm_page_aflag_clear(m,
- PGA_WRITEABLE);
- }
+ if ((tpte & PG_RW) != 0) {
+ if ((tpte & PG_M) != 0)
+ vm_page_dirty(m);
+ atomic_subtract_int(
+ &m->md.write_mappings, 1);
}
+ TAILQ_REMOVE(&m->md.pv_list, pv, pv_list);
pc->pc_map[field] |= 1UL << bit;
pmap_unuse_pt(pmap, va, *pde, &free);
freed++;
@@ -2320,7 +2314,7 @@ pmap_pvh_remove(struct md_page *pvh, pma
* entries for each of the 4KB page mappings.
*/
static void
-pmap_pv_demote_pde(pmap_t pmap, vm_offset_t va, vm_paddr_t pa)
+pmap_pv_demote_pde(pmap_t pmap, vm_offset_t va, pd_entry_t pde)
{
struct md_page *pvh;
pv_entry_t pv;
@@ -2328,19 +2322,23 @@ pmap_pv_demote_pde(pmap_t pmap, vm_offse
vm_page_t m;
rw_assert(&pvh_global_lock, RA_WLOCKED);
- KASSERT((pa & PDRMASK) == 0,
- ("pmap_pv_demote_pde: pa is not 2mpage aligned"));
+ KASSERT((pde & PG_MANAGED) != 0,
+ ("pmap_pv_demote_pde: pde is not managed"));
/*
* Transfer the 2mpage's pv entry for this mapping to the first
- * page's pv list.
+ * page's pv list. Decrement the 2mpage's write mappings only
+ * after the page's write mappings are incremented so that both
+ * counts are never simultaneously zero for any page.
*/
- pvh = pa_to_pvh(pa);
+ pvh = pa_to_pvh(pde & PG_PS_FRAME);
va = trunc_2mpage(va);
pv = pmap_pvh_remove(pvh, pmap, va);
KASSERT(pv != NULL, ("pmap_pv_demote_pde: pv not found"));
- m = PHYS_TO_VM_PAGE(pa);
+ m = PHYS_TO_VM_PAGE(pde & PG_PS_FRAME);
TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_list);
+ if ((pde & PG_RW) != 0)
+ atomic_add_int(&m->md.write_mappings, 1);
/* Instantiate the remaining NPTEPG - 1 pv entries. */
va_last = va + NBPDR - PAGE_SIZE;
do {
@@ -2348,8 +2346,16 @@ pmap_pv_demote_pde(pmap_t pmap, vm_offse
KASSERT((m->oflags & VPO_UNMANAGED) == 0,
("pmap_pv_demote_pde: page %p is not managed", m));
va += PAGE_SIZE;
- pmap_insert_entry(pmap, va, m);
+ pv = get_pv_entry(pmap, FALSE);
+ pv->pv_va = va;
+ TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_list);
+ if ((pde & PG_RW) != 0)
+ atomic_add_int(&m->md.write_mappings, 1);
} while (va < va_last);
+ if ((pde & PG_RW) != 0)
+ atomic_subtract_int(&pvh->write_mappings, 1);
+ KASSERT(!TAILQ_EMPTY(&pvh->pv_list) || pvh->write_mappings == 0,
+ ("pmap_pv_demote_pde: pvh %p has invalid write mappings", pvh));
}
/*
@@ -2358,7 +2364,7 @@ pmap_pv_demote_pde(pmap_t pmap, vm_offse
* for the 2MB page mapping.
*/
static void
-pmap_pv_promote_pde(pmap_t pmap, vm_offset_t va, vm_paddr_t pa)
+pmap_pv_promote_pde(pmap_t pmap, vm_offset_t va, pd_entry_t pde)
{
struct md_page *pvh;
pv_entry_t pv;
@@ -2366,27 +2372,43 @@ pmap_pv_promote_pde(pmap_t pmap, vm_offs
vm_page_t m;
rw_assert(&pvh_global_lock, RA_WLOCKED);
- KASSERT((pa & PDRMASK) == 0,
- ("pmap_pv_promote_pde: pa is not 2mpage aligned"));
+ KASSERT((pde & PG_MANAGED) != 0,
+ ("pmap_pv_promote_pde: pde is not managed"));
/*
* Transfer the first page's pv entry for this mapping to the
* 2mpage's pv list. Aside from avoiding the cost of a call
* to get_pv_entry(), a transfer avoids the possibility that
- * get_pv_entry() calls pmap_collect() and that pmap_collect()
+ * get_pv_entry() calls pmap_pv_reclaim() and that pmap_pv_reclaim()
* removes one of the mappings that is being promoted.
*/
- m = PHYS_TO_VM_PAGE(pa);
+ m = PHYS_TO_VM_PAGE(pde & PG_PS_FRAME);
va = trunc_2mpage(va);
pv = pmap_pvh_remove(&m->md, pmap, va);
KASSERT(pv != NULL, ("pmap_pv_promote_pde: pv not found"));
- pvh = pa_to_pvh(pa);
+ pvh = pa_to_pvh(pde & PG_PS_FRAME);
TAILQ_INSERT_TAIL(&pvh->pv_list, pv, pv_list);
+ if ((pde & PG_RW) != 0) {
+ /*
+ * Decrement the page's write mappings only after the
+ * 2mpage's write mappings are incremented so that both
+ * counts are never simultaneously zero for any page.
+ */
+ atomic_add_int(&pvh->write_mappings, 1);
+ vm_page_dirty(m);
+ atomic_subtract_int(&m->md.write_mappings, 1);
+ }
+ KASSERT(!TAILQ_EMPTY(&m->md.pv_list) || m->md.write_mappings == 0,
+ ("pmap_pv_promote_pde: page %p has invalid write mappings", m));
/* Free the remaining NPTEPG - 1 pv entries. */
va_last = va + NBPDR - PAGE_SIZE;
do {
m++;
va += PAGE_SIZE;
+ if ((pde & PG_RW) != 0) {
+ vm_page_dirty(m);
+ atomic_subtract_int(&m->md.write_mappings, 1);
+ }
pmap_pvh_free(&m->md, pmap, va);
} while (va < va_last);
}
@@ -2403,44 +2425,17 @@ pmap_pvh_free(struct md_page *pvh, pmap_
pv = pmap_pvh_remove(pvh, pmap, va);
KASSERT(pv != NULL, ("pmap_pvh_free: pv not found"));
+ KASSERT(!TAILQ_EMPTY(&pvh->pv_list) || pvh->write_mappings == 0,
+ ("pmap_pvh_free: pvh %p has invalid write mappings", pvh));
free_pv_entry(pmap, pv);
}
-static void
-pmap_remove_entry(pmap_t pmap, vm_page_t m, vm_offset_t va)
-{
- struct md_page *pvh;
-
- rw_assert(&pvh_global_lock, RA_WLOCKED);
- pmap_pvh_free(&m->md, pmap, va);
- if (TAILQ_EMPTY(&m->md.pv_list) && (m->flags & PG_FICTITIOUS) == 0) {
- pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
- if (TAILQ_EMPTY(&pvh->pv_list))
- vm_page_aflag_clear(m, PGA_WRITEABLE);
- }
-}
-
-/*
- * Create a pv entry for page at pa for
- * (pmap, va).
- */
-static void
-pmap_insert_entry(pmap_t pmap, vm_offset_t va, vm_page_t m)
-{
- pv_entry_t pv;
-
- rw_assert(&pvh_global_lock, RA_WLOCKED);
- PMAP_LOCK_ASSERT(pmap, MA_OWNED);
- pv = get_pv_entry(pmap, FALSE);
- pv->pv_va = va;
- TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_list);
-}
-
/*
* Conditionally create a pv entry.
*/
static boolean_t
-pmap_try_insert_pv_entry(pmap_t pmap, vm_offset_t va, vm_page_t m)
+pmap_try_insert_pv_entry(pmap_t pmap, vm_offset_t va, vm_page_t m,
+ boolean_t write_mapping)
{
pv_entry_t pv;
@@ -2449,6 +2444,8 @@ pmap_try_insert_pv_entry(pmap_t pmap, vm
if ((pv = get_pv_entry(pmap, TRUE)) != NULL) {
pv->pv_va = va;
TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_list);
+ if (write_mapping)
+ atomic_add_int(&m->md.write_mappings, 1);
return (TRUE);
} else
return (FALSE);
@@ -2458,7 +2455,7 @@ pmap_try_insert_pv_entry(pmap_t pmap, vm
* Create the pv entry for a 2MB page mapping.
*/
static boolean_t
-pmap_pv_insert_pde(pmap_t pmap, vm_offset_t va, vm_paddr_t pa)
+pmap_pv_insert_pde(pmap_t pmap, vm_offset_t va, pd_entry_t pde)
{
struct md_page *pvh;
pv_entry_t pv;
@@ -2466,8 +2463,10 @@ pmap_pv_insert_pde(pmap_t pmap, vm_offse
rw_assert(&pvh_global_lock, RA_WLOCKED);
if ((pv = get_pv_entry(pmap, TRUE)) != NULL) {
pv->pv_va = va;
- pvh = pa_to_pvh(pa);
+ pvh = pa_to_pvh(pde & PG_PS_FRAME);
TAILQ_INSERT_TAIL(&pvh->pv_list, pv, pv_list);
+ if ((pde & PG_RW) != 0)
+ atomic_add_int(&pvh->write_mappings, 1);
return (TRUE);
} else
return (FALSE);
@@ -2595,7 +2594,7 @@ pmap_demote_pde(pmap_t pmap, pd_entry_t
* the 2mpage to referencing the page table page.
*/
if ((oldpde & PG_MANAGED) != 0)
- pmap_pv_demote_pde(pmap, va, oldpde & PG_PS_FRAME);
+ pmap_pv_demote_pde(pmap, va, oldpde);
pmap_pde_demotions++;
CTR2(KTR_PMAP, "pmap_demote_pde: success for va %#lx"
@@ -2627,22 +2626,25 @@ pmap_remove_pde(pmap_t pmap, pd_entry_t
* PG_G.
*/
if (oldpde & PG_G)
- pmap_invalidate_page(kernel_pmap, sva);
+ pmap_invalidate_page(pmap, sva);
pmap_resident_count_dec(pmap, NBPDR / PAGE_SIZE);
if (oldpde & PG_MANAGED) {
- pvh = pa_to_pvh(oldpde & PG_PS_FRAME);
- pmap_pvh_free(pvh, pmap, sva);
eva = sva + NBPDR;
for (va = sva, m = PHYS_TO_VM_PAGE(oldpde & PG_PS_FRAME);
va < eva; va += PAGE_SIZE, m++) {
if ((oldpde & (PG_M | PG_RW)) == (PG_M | PG_RW))
- vm_page_dirty(m);
+ KASSERT(m->dirty == VM_PAGE_BITS_ALL,
+ ("pmap_remove_pde: not dirty"));
if (oldpde & PG_A)
vm_page_aflag_set(m, PGA_REFERENCED);
- if (TAILQ_EMPTY(&m->md.pv_list) &&
- TAILQ_EMPTY(&pvh->pv_list))
- vm_page_aflag_clear(m, PGA_WRITEABLE);
}
+ pvh = pa_to_pvh(oldpde & PG_PS_FRAME);
+ if ((oldpde & PG_RW) != 0) {
+ KASSERT((oldpde & PG_M) != 0,
+ ("pmap_remove_pde: oldpde is missing PG_M"));
+ atomic_subtract_int(&pvh->write_mappings, 1);
+ }
+ pmap_pvh_free(pvh, pmap, sva);
}
if (pmap == kernel_pmap) {
if (!pmap_demote_pde(pmap, pdq, sva))
@@ -2679,11 +2681,14 @@ pmap_remove_pte(pmap_t pmap, pt_entry_t
pmap_resident_count_dec(pmap, 1);
if (oldpte & PG_MANAGED) {
m = PHYS_TO_VM_PAGE(oldpte & PG_FRAME);
- if ((oldpte & (PG_M | PG_RW)) == (PG_M | PG_RW))
- vm_page_dirty(m);
if (oldpte & PG_A)
vm_page_aflag_set(m, PGA_REFERENCED);
- pmap_remove_entry(pmap, m, va);
+ if ((oldpte & PG_RW) != 0) {
+ if ((oldpte & PG_M) != 0)
+ vm_page_dirty(m);
+ atomic_subtract_int(&m->md.write_mappings, 1);
+ }
+ pmap_pvh_free(&m->md, pmap, va);
}
return (pmap_unuse_pt(pmap, va, ptepde, free));
}
@@ -2885,6 +2890,8 @@ pmap_remove_all(vm_page_t m)
(void)pmap_demote_pde(pmap, pde, va);
PMAP_UNLOCK(pmap);
}
+ KASSERT(pvh->write_mappings == 0,
+ ("pmap_remove_all: pvh %p has invalid write mappings", pvh));
small_mappings:
while ((pv = TAILQ_FIRST(&m->md.pv_list)) != NULL) {
pmap = PV_PMAP(pv);
@@ -2899,19 +2906,19 @@ small_mappings:
pmap->pm_stats.wired_count--;
if (tpte & PG_A)
vm_page_aflag_set(m, PGA_REFERENCED);
-
- /*
- * Update the vm_page_t clean and reference bits.
- */
- if ((tpte & (PG_M | PG_RW)) == (PG_M | PG_RW))
- vm_page_dirty(m);
+ if ((tpte & PG_RW) != 0) {
+ if ((tpte & PG_M) != 0)
+ vm_page_dirty(m);
+ atomic_subtract_int(&m->md.write_mappings, 1);
+ }
pmap_unuse_pt(pmap, pv->pv_va, *pde, &free);
pmap_invalidate_page(pmap, pv->pv_va);
TAILQ_REMOVE(&m->md.pv_list, pv, pv_list);
free_pv_entry(pmap, pv);
PMAP_UNLOCK(pmap);
}
- vm_page_aflag_clear(m, PGA_WRITEABLE);
+ KASSERT(m->md.write_mappings == 0,
+ ("pmap_remove_all: page %p has invalid write mappings", m));
rw_wunlock(&pvh_global_lock);
pmap_free_zero_pages(free);
}
@@ -2938,7 +2945,8 @@ retry:
for (va = sva, m = PHYS_TO_VM_PAGE(oldpde & PG_PS_FRAME);
va < eva; va += PAGE_SIZE, m++)
if ((oldpde & (PG_M | PG_RW)) == (PG_M | PG_RW))
- vm_page_dirty(m);
+ KASSERT(m->dirty == VM_PAGE_BITS_ALL,
+ ("pmap_protect_pde: not dirty"));
}
if ((prot & VM_PROT_WRITE) == 0)
newpde &= ~(PG_RW | PG_M);
@@ -2947,6 +2955,9 @@ retry:
if (newpde != oldpde) {
if (!atomic_cmpset_long(pde, oldpde, newpde))
goto retry;
+ if ((newpde & (PG_MANAGED | PG_RW)) == PG_MANAGED &&
+ (oldpde & (PG_MANAGED | PG_RW)) == (PG_MANAGED | PG_RW))
+ atomic_subtract_int(&pa_to_pvh(oldpde & PG_PS_FRAME)->write_mappings, 1);
if (oldpde & PG_G)
pmap_invalidate_page(pmap, sva);
else
@@ -3079,6 +3090,11 @@ retry:
if (pbits != obits) {
if (!atomic_cmpset_long(pte, obits, pbits))
goto retry;
+ if ((pbits & (PG_MANAGED | PG_RW)) == PG_MANAGED &&
+ (obits & (PG_MANAGED | PG_RW)) == (PG_MANAGED | PG_RW)) {
+ m = PHYS_TO_VM_PAGE(obits & PG_FRAME);
+ atomic_subtract_int(&m->md.write_mappings, 1);
+ }
if (obits & PG_G)
pmap_invalidate_page(pmap, sva);
else
@@ -3132,6 +3148,10 @@ setpde:
if (!atomic_cmpset_long(firstpte, newpde, newpde & ~PG_RW))
goto setpde;
newpde &= ~PG_RW;
+ if (newpde & PG_MANAGED) {
+ vm_page_t m = PHYS_TO_VM_PAGE(newpde & PG_FRAME);
+ atomic_subtract_int(&m->md.write_mappings, 1);
+ }
}
/*
@@ -3157,6 +3177,10 @@ setpte:
if (!atomic_cmpset_long(pte, oldpte, oldpte & ~PG_RW))
goto setpte;
oldpte &= ~PG_RW;
+ if (oldpte & PG_MANAGED) {
+ vm_page_t m = PHYS_TO_VM_PAGE(oldpte & PG_FRAME);
+ atomic_subtract_int(&m->md.write_mappings, 1);
+ }
oldpteva = (oldpte & PG_FRAME & PDRMASK) |
(va & ~PDRMASK);
CTR2(KTR_PMAP, "pmap_promote_pde: protect for va %#lx"
@@ -3188,7 +3212,7 @@ setpte:
* Promote the pv entries.
*/
if ((newpde & PG_MANAGED) != 0)
- pmap_pv_promote_pde(pmap, va, newpde & PG_PS_FRAME);
+ pmap_pv_promote_pde(pmap, va, newpde);
/*
* Propagate the PAT index to its proper position.
@@ -3345,7 +3369,7 @@ validate:
if ((prot & VM_PROT_WRITE) != 0) {
newpte |= PG_RW;
if ((newpte & PG_MANAGED) != 0)
- vm_page_aflag_set(m, PGA_WRITEABLE);
+ atomic_add_int(&m->md.write_mappings, 1);
}
if ((prot & VM_PROT_EXECUTE) == 0)
newpte |= pg_nx;
@@ -3380,11 +3404,10 @@ validate:
if ((newpte & PG_RW) == 0)
invlva = TRUE;
}
- if ((origpte & PG_MANAGED) != 0 &&
- TAILQ_EMPTY(&om->md.pv_list) &&
- ((om->flags & PG_FICTITIOUS) != 0 ||
- TAILQ_EMPTY(&pa_to_pvh(opa)->pv_list)))
- vm_page_aflag_clear(om, PGA_WRITEABLE);
+ if ((origpte & (PG_MANAGED | PG_RW)) == (PG_MANAGED | PG_RW))
+ atomic_subtract_int(&om->md.write_mappings, 1);
+ if (TAILQ_EMPTY(&om->md.pv_list))
+ KASSERT(om->md.write_mappings == 0, ("pmap_enter: xx2 %d", om->md.write_mappings));
if (invlva)
pmap_invalidate_page(pmap, va);
} else
@@ -3603,7 +3626,7 @@ pmap_enter_quick_locked(pmap_t pmap, vm_
* Enter on the PV list if part of our managed memory.
*/
if ((m->oflags & VPO_UNMANAGED) == 0 &&
- !pmap_try_insert_pv_entry(pmap, va, m)) {
+ !pmap_try_insert_pv_entry(pmap, va, m, FALSE)) {
if (mpte != NULL) {
free = NULL;
if (pmap_unwire_pte_hold(pmap, va, mpte, &free)) {
@@ -3861,8 +3884,7 @@ pmap_copy(pmap_t dst_pmap, pmap_t src_pm
PHYS_TO_DMAP(VM_PAGE_TO_PHYS(dstmpde));
pde = &pde[pmap_pde_index(addr)];
if (*pde == 0 && ((srcptepaddr & PG_MANAGED) == 0 ||
- pmap_pv_insert_pde(dst_pmap, addr, srcptepaddr &
- PG_PS_FRAME))) {
+ pmap_pv_insert_pde(dst_pmap, addr, srcptepaddr))) {
*pde = srcptepaddr & ~PG_W;
pmap_resident_count_inc(dst_pmap, NBPDR / PAGE_SIZE);
} else
@@ -3899,7 +3921,8 @@ pmap_copy(pmap_t dst_pmap, pmap_t src_pm
dst_pte = &dst_pte[pmap_pte_index(addr)];
if (*dst_pte == 0 &&
pmap_try_insert_pv_entry(dst_pmap, addr,
- PHYS_TO_VM_PAGE(ptetemp & PG_FRAME))) {
+ PHYS_TO_VM_PAGE(ptetemp & PG_FRAME),
+ (ptetemp & PG_RW) != 0)) {
/*
* Clear the wired, modified, and
* accessed (referenced) bits
@@ -4096,11 +4119,32 @@ pmap_page_is_mapped(vm_page_t m)
rv = !TAILQ_EMPTY(&m->md.pv_list) ||
((m->flags & PG_FICTITIOUS) == 0 &&
!TAILQ_EMPTY(&pa_to_pvh(VM_PAGE_TO_PHYS(m))->pv_list));
+ if (!rv) {
+ KASSERT(m->md.write_mappings == 0, ("pmap_page_is_mapped: xx2 %d", m->md.write_mappings));
+ if ((m->flags & PG_FICTITIOUS) == 0)
+ KASSERT(pa_to_pvh(VM_PAGE_TO_PHYS(m))->write_mappings == 0,
+ ("pmap_page_is_mapped: xx1"));
+ }
rw_wunlock(&pvh_global_lock);
return (rv);
}
/*
+ * XXX
+ */
+boolean_t
+pmap_page_is_write_mapped(vm_page_t m)
+{
+
+ if ((m->oflags & VPO_UNMANAGED) != 0)
+ return (FALSE);
+ return (m->md.write_mappings != 0 ||
+ ((m->flags & PG_FICTITIOUS) == 0 &&
+ pa_to_pvh(VM_PAGE_TO_PHYS(m))->write_mappings != 0));
+
+}
+
+/*
* Remove all pages from specified address space
* this aids process exit speeds. Also, this code
* is special cased for current process only, but
@@ -4175,17 +4219,6 @@ pmap_remove_pages(pmap_t pmap)
pte_clear(pte);
- /*
- * Update the vm_page_t clean/reference bits.
- */
- if ((tpte & (PG_M | PG_RW)) == (PG_M | PG_RW)) {
- if ((tpte & PG_PS) != 0) {
- for (mt = m; mt < &m[NBPDR / PAGE_SIZE]; mt++)
- vm_page_dirty(mt);
- } else
- vm_page_dirty(m);
- }
-
/* Mark free */
PV_STAT(pv_entry_frees++);
PV_STAT(pv_entry_spare++);
@@ -4195,12 +4228,16 @@ pmap_remove_pages(pmap_t pmap)
pmap_resident_count_dec(pmap, NBPDR / PAGE_SIZE);
pvh = pa_to_pvh(tpte & PG_PS_FRAME);
TAILQ_REMOVE(&pvh->pv_list, pv, pv_list);
- if (TAILQ_EMPTY(&pvh->pv_list)) {
+ if ((tpte & PG_RW) != 0) {
+ KASSERT((tpte & PG_M) != 0,
+ ("pmap_remove_pages: not PG_M"));
for (mt = m; mt < &m[NBPDR / PAGE_SIZE]; mt++)
- if ((mt->aflags & PGA_WRITEABLE) != 0 &&
- TAILQ_EMPTY(&mt->md.pv_list))
- vm_page_aflag_clear(mt, PGA_WRITEABLE);
+ KASSERT(mt->dirty == VM_PAGE_BITS_ALL,
+ ("pmap_remove_pages: not dirty"));
+ atomic_subtract_int(&pvh->write_mappings, 1);
}
+ if (TAILQ_EMPTY(&pvh->pv_list))
+ KASSERT(pvh->write_mappings == 0, ("pmap_remove_pages: xx1 %d", pvh->write_mappings));
mpte = pmap_lookup_pt_page(pmap, pv->pv_va);
if (mpte != NULL) {
pmap_remove_pt_page(pmap, mpte);
@@ -4214,13 +4251,13 @@ pmap_remove_pages(pmap_t pmap)
} else {
pmap_resident_count_dec(pmap, 1);
TAILQ_REMOVE(&m->md.pv_list, pv, pv_list);
- if ((m->aflags & PGA_WRITEABLE) != 0 &&
- TAILQ_EMPTY(&m->md.pv_list) &&
- (m->flags & PG_FICTITIOUS) == 0) {
- pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
- if (TAILQ_EMPTY(&pvh->pv_list))
- vm_page_aflag_clear(m, PGA_WRITEABLE);
+ if ((tpte & PG_RW) != 0) {
+ if ((tpte & PG_M) != 0)
+ vm_page_dirty(m);
+ atomic_subtract_int(&m->md.write_mappings, 1);
}
+ if (TAILQ_EMPTY(&m->md.pv_list))
+ KASSERT(m->md.write_mappings == 0, ("pmap_remove_pages: xx2 %d", m->md.write_mappings));
}
pmap_unuse_pt(pmap, pv->pv_va, ptepde, &free);
}
@@ -4257,7 +4294,9 @@ pmap_is_modified(vm_page_t m)
*/
VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
if ((m->oflags & VPO_BUSY) == 0 &&
- (m->aflags & PGA_WRITEABLE) == 0)
+ m->md.write_mappings == 0 &&
+ ((m->flags & PG_FICTITIOUS) != 0 ||
+ pa_to_pvh(VM_PAGE_TO_PHYS(m))->write_mappings == 0))
return (FALSE);
rw_wlock(&pvh_global_lock);
rv = pmap_is_modified_pvh(&m->md) ||
@@ -4388,7 +4427,9 @@ pmap_remove_write(vm_page_t m)
*/
VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
if ((m->oflags & VPO_BUSY) == 0 &&
- (m->aflags & PGA_WRITEABLE) == 0)
+ m->md.write_mappings == 0 &&
+ ((m->flags & PG_FICTITIOUS) != 0 ||
+ pa_to_pvh(VM_PAGE_TO_PHYS(m))->write_mappings == 0))
return;
rw_wlock(&pvh_global_lock);
if ((m->flags & PG_FICTITIOUS) != 0)
@@ -4403,6 +4444,7 @@ pmap_remove_write(vm_page_t m)
(void)pmap_demote_pde(pmap, pde, va);
PMAP_UNLOCK(pmap);
}
+ KASSERT(pvh->write_mappings == 0, ("pmap_remove_all: xx1"));
small_mappings:
TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
pmap = PV_PMAP(pv);
@@ -4419,11 +4461,12 @@ retry:
goto retry;
if ((oldpte & PG_M) != 0)
vm_page_dirty(m);
+ atomic_subtract_int(&m->md.write_mappings, 1);
pmap_invalidate_page(pmap, pv->pv_va);
}
PMAP_UNLOCK(pmap);
}
- vm_page_aflag_clear(m, PGA_WRITEABLE);
+ KASSERT(m->md.write_mappings == 0, ("pmap_remove_write: xx2"));
rw_wunlock(&pvh_global_lock);
}
@@ -4538,7 +4581,9 @@ pmap_clear_modify(vm_page_t m)
* If the object containing the page is locked and the page is not
* VPO_BUSY, then PGA_WRITEABLE cannot be concurrently set.
*/
- if ((m->aflags & PGA_WRITEABLE) == 0)
+ if (m->md.write_mappings == 0 &&
+ ((m->flags & PG_FICTITIOUS) != 0 ||
+ pa_to_pvh(VM_PAGE_TO_PHYS(m))->write_mappings == 0))
return;
rw_wlock(&pvh_global_lock);
if ((m->flags & PG_FICTITIOUS) != 0)
@@ -4568,6 +4613,7 @@ pmap_clear_modify(vm_page_t m)
oldpte & ~(PG_M | PG_RW)))
oldpte = *pte;
vm_page_dirty(m);
+ atomic_subtract_int(&m->md.write_mappings, 1);
pmap_invalidate_page(pmap, va);
}
}
Modified: user/alc/superpages/sys/amd64/include/pmap.h
==============================================================================
--- user/alc/superpages/sys/amd64/include/pmap.h Thu Jun 14 16:25:10 2012 (r237082)
+++ user/alc/superpages/sys/amd64/include/pmap.h Thu Jun 14 16:53:08 2012 (r237083)
@@ -241,6 +241,7 @@ struct pv_chunk;
struct md_page {
TAILQ_HEAD(,pv_entry) pv_list;
+ int write_mappings;
int pat_mode;
};
@@ -323,6 +324,7 @@ void *pmap_mapbios(vm_paddr_t, vm_size_t
void *pmap_mapdev(vm_paddr_t, vm_size_t);
void *pmap_mapdev_attr(vm_paddr_t, vm_size_t, int);
boolean_t pmap_page_is_mapped(vm_page_t m);
+boolean_t pmap_page_is_write_mapped(vm_page_t m);
void pmap_page_set_memattr(vm_page_t m, vm_memattr_t ma);
void pmap_unmapdev(vm_offset_t, vm_size_t);
void pmap_invalidate_page(pmap_t, vm_offset_t);
Modified: user/alc/superpages/sys/vm/swap_pager.c
==============================================================================
--- user/alc/superpages/sys/vm/swap_pager.c Thu Jun 14 16:25:10 2012 (r237082)
+++ user/alc/superpages/sys/vm/swap_pager.c Thu Jun 14 16:53:08 2012 (r237083)
@@ -1593,7 +1593,7 @@ swp_pager_async_iodone(struct buf *bp)
* status, then finish the I/O ( which decrements the
* busy count and possibly wakes waiter's up ).
*/
- KASSERT((m->aflags & PGA_WRITEABLE) == 0,
+ KASSERT(!pmap_page_is_write_mapped(m),
("swp_pager_async_iodone: page %p is not write"
" protected", m));
vm_page_undirty(m);
Modified: user/alc/superpages/sys/vm/vm_page.c
==============================================================================
--- user/alc/superpages/sys/vm/vm_page.c Thu Jun 14 16:25:10 2012 (r237082)
+++ user/alc/superpages/sys/vm/vm_page.c Thu Jun 14 16:53:08 2012 (r237083)
@@ -930,7 +930,7 @@ vm_page_insert(vm_page_t m, vm_object_t
* Since we are inserting a new and possibly dirty page,
* update the object's OBJ_MIGHTBEDIRTY flag.
*/
- if (m->aflags & PGA_WRITEABLE)
+ if (pmap_page_is_write_mapped(m))
vm_object_set_writeable_dirty(object);
}
@@ -2679,7 +2679,7 @@ vm_page_clear_dirty_mask(vm_page_t m, vm
* set by a concurrent pmap operation.
*/
VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
- if ((m->oflags & VPO_BUSY) == 0 && (m->aflags & PGA_WRITEABLE) == 0)
+ if ((m->oflags & VPO_BUSY) == 0 && !pmap_page_is_write_mapped(m))
m->dirty &= ~pagebits;
else {
/*
Modified: user/alc/superpages/sys/vm/vm_pageout.c
==============================================================================
--- user/alc/superpages/sys/vm/vm_pageout.c Thu Jun 14 16:25:10 2012 (r237082)
+++ user/alc/superpages/sys/vm/vm_pageout.c Thu Jun 14 16:53:08 2012 (r237083)
@@ -503,7 +503,7 @@ vm_pageout_flush(vm_page_t *mc, int coun
vm_page_t mt = mc[i];
KASSERT(pageout_status[i] == VM_PAGER_PEND ||
- (mt->aflags & PGA_WRITEABLE) == 0,
+ !pmap_page_is_write_mapped(mt),
("vm_pageout_flush: page %p is not write protected", mt));
switch (pageout_status[i]) {
case VM_PAGER_OK:
@@ -899,7 +899,7 @@ rescan0:
* be updated.
*/
if (m->dirty != VM_PAGE_BITS_ALL &&
- (m->aflags & PGA_WRITEABLE) != 0) {
+ pmap_page_is_write_mapped(m)) {
/*
* Avoid a race condition: Unless write access is
* removed from the page, another processor could
Modified: user/alc/superpages/sys/vm/vnode_pager.c
==============================================================================
--- user/alc/superpages/sys/vm/vnode_pager.c Thu Jun 14 16:25:10 2012 (r237082)
+++ user/alc/superpages/sys/vm/vnode_pager.c Thu Jun 14 16:53:08 2012 (r237083)
@@ -1146,7 +1146,7 @@ vnode_pager_generic_putpages(struct vnod
m = ma[ncount - 1];
KASSERT(m->busy > 0,
("vnode_pager_generic_putpages: page %p is not busy", m));
- KASSERT((m->aflags & PGA_WRITEABLE) == 0,
+ KASSERT(!pmap_page_is_write_mapped(m),
("vnode_pager_generic_putpages: page %p is not read-only", m));
vm_page_clear_dirty(m, pgoff, PAGE_SIZE -
pgoff);
More information about the svn-src-user
mailing list