svn commit: r208990 - in head/sys: amd64/amd64 arm/arm i386/i386
i386/xen ia64/ia64 mips/mips powerpc/aim powerpc/booke
sparc64/sparc64 sun4v/sun4v vm
Alan Cox
alc at FreeBSD.org
Thu Jun 10 16:56:36 UTC 2010
Author: alc
Date: Thu Jun 10 16:56:35 2010
New Revision: 208990
URL: http://svn.freebsd.org/changeset/base/208990
Log:
Reduce the scope of the page queues lock and the number of
PG_REFERENCED changes in vm_pageout_object_deactivate_pages().
Simplify this function's inner loop using TAILQ_FOREACH(), and shorten
some of its overly long lines. Update a stale comment.
Assert that PG_REFERENCED may be cleared only if the object containing
the page is locked. Add a comment documenting this.
Assert that a caller to vm_page_requeue() holds the page queues lock,
and assert that the page is on a page queue.
Push down the page queues lock into pmap_ts_referenced() and
pmap_page_exists_quick(). (As of now, there are no longer any pmap
functions that expect to be called with the page queues lock held.)
Neither pmap_ts_referenced() nor pmap_page_exists_quick() should ever
be passed an unmanaged page. Assert this rather than returning "0"
and "FALSE" respectively.
ARM:
Simplify pmap_page_exists_quick() by switching to TAILQ_FOREACH().
Push down the page queues lock inside of pmap_clearbit(), simplifying
pmap_clear_modify(), pmap_clear_reference(), and pmap_remove_write().
Additionally, this allows for avoiding the acquisition of the page
queues lock in some cases.
PowerPC/AIM:
moea*_page_exits_quick() and moea*_page_wired_mappings() will never be
called before pmap initialization is complete. Therefore, the check
for moea_initialized can be eliminated.
Push down the page queues lock inside of moea*_clear_bit(),
simplifying moea*_clear_modify() and moea*_clear_reference().
The last parameter to moea*_clear_bit() is never used. Eliminate it.
PowerPC/BookE:
Simplify mmu_booke_page_exists_quick()'s control flow.
Reviewed by: kib@
Modified:
head/sys/amd64/amd64/pmap.c
head/sys/arm/arm/pmap.c
head/sys/i386/i386/pmap.c
head/sys/i386/xen/pmap.c
head/sys/ia64/ia64/pmap.c
head/sys/mips/mips/pmap.c
head/sys/powerpc/aim/mmu_oea.c
head/sys/powerpc/aim/mmu_oea64.c
head/sys/powerpc/booke/pmap.c
head/sys/sparc64/sparc64/pmap.c
head/sys/sun4v/sun4v/pmap.c
head/sys/vm/vm_page.c
head/sys/vm/vm_page.h
head/sys/vm/vm_pageout.c
Modified: head/sys/amd64/amd64/pmap.c
==============================================================================
--- head/sys/amd64/amd64/pmap.c Thu Jun 10 16:45:30 2010 (r208989)
+++ head/sys/amd64/amd64/pmap.c Thu Jun 10 16:56:35 2010 (r208990)
@@ -3899,30 +3899,35 @@ pmap_page_exists_quick(pmap_t pmap, vm_p
struct md_page *pvh;
pv_entry_t pv;
int loops = 0;
+ boolean_t rv;
- if (m->flags & PG_FICTITIOUS)
- return (FALSE);
-
- mtx_assert(&vm_page_queue_mtx, MA_OWNED);
+ KASSERT((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0,
+ ("pmap_page_exists_quick: page %p is not managed", m));
+ rv = FALSE;
+ vm_page_lock_queues();
TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
if (PV_PMAP(pv) == pmap) {
- return (TRUE);
+ rv = TRUE;
+ break;
}
loops++;
if (loops >= 16)
break;
}
- if (loops < 16) {
+ if (!rv && loops < 16) {
pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
TAILQ_FOREACH(pv, &pvh->pv_list, pv_list) {
- if (PV_PMAP(pv) == pmap)
- return (TRUE);
+ if (PV_PMAP(pv) == pmap) {
+ rv = TRUE;
+ break;
+ }
loops++;
if (loops >= 16)
break;
}
}
- return (FALSE);
+ vm_page_unlock_queues();
+ return (rv);
}
/*
@@ -4335,10 +4340,10 @@ pmap_ts_referenced(vm_page_t m)
vm_offset_t va;
int rtval = 0;
- if (m->flags & PG_FICTITIOUS)
- return (rtval);
- mtx_assert(&vm_page_queue_mtx, MA_OWNED);
+ KASSERT((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0,
+ ("pmap_ts_referenced: page %p is not managed", m));
pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
+ vm_page_lock_queues();
TAILQ_FOREACH_SAFE(pv, &pvh->pv_list, pv_list, pvn) {
pmap = PV_PMAP(pv);
PMAP_LOCK(pmap);
@@ -4362,7 +4367,7 @@ pmap_ts_referenced(vm_page_t m)
rtval++;
if (rtval > 4) {
PMAP_UNLOCK(pmap);
- return (rtval);
+ goto out;
}
}
}
@@ -4391,6 +4396,8 @@ pmap_ts_referenced(vm_page_t m)
PMAP_UNLOCK(pmap);
} while ((pv = pvn) != NULL && pv != pvf);
}
+out:
+ vm_page_unlock_queues();
return (rtval);
}
Modified: head/sys/arm/arm/pmap.c
==============================================================================
--- head/sys/arm/arm/pmap.c Thu Jun 10 16:45:30 2010 (r208989)
+++ head/sys/arm/arm/pmap.c Thu Jun 10 16:56:35 2010 (r208990)
@@ -1423,7 +1423,7 @@ pmap_clearbit(struct vm_page *pg, u_int
u_int oflags;
int count = 0;
- mtx_assert(&vm_page_queue_mtx, MA_OWNED);
+ vm_page_lock_queues();
if (maskbits & PVF_WRITE)
maskbits |= PVF_MOD;
@@ -1433,6 +1433,7 @@ pmap_clearbit(struct vm_page *pg, u_int
pg->md.pvh_attrs &= ~(maskbits & (PVF_MOD | PVF_REF));
if (TAILQ_EMPTY(&pg->md.pv_list)) {
+ vm_page_unlock_queues();
return (0);
}
@@ -1568,6 +1569,7 @@ pmap_clearbit(struct vm_page *pg, u_int
if (maskbits & PVF_WRITE)
vm_page_flag_clear(pg, PG_WRITEABLE);
+ vm_page_unlock_queues();
return (count);
}
@@ -4417,24 +4419,23 @@ pmap_page_exists_quick(pmap_t pmap, vm_p
{
pv_entry_t pv;
int loops = 0;
+ boolean_t rv;
- if (m->flags & PG_FICTITIOUS)
- return (FALSE);
-
- /*
- * Not found, check current mappings returning immediately
- */
- for (pv = TAILQ_FIRST(&m->md.pv_list);
- pv;
- pv = TAILQ_NEXT(pv, pv_list)) {
+ KASSERT((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0,
+ ("pmap_page_exists_quick: page %p is not managed", m));
+ rv = FALSE;
+ vm_page_lock_queues();
+ TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
if (pv->pv_pmap == pmap) {
- return (TRUE);
+ rv = TRUE;
+ break;
}
loops++;
if (loops >= 16)
break;
}
- return (FALSE);
+ vm_page_unlock_queues();
+ return (rv);
}
/*
@@ -4469,8 +4470,8 @@ int
pmap_ts_referenced(vm_page_t m)
{
- if (m->flags & PG_FICTITIOUS)
- return (0);
+ KASSERT((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0,
+ ("pmap_ts_referenced: page %p is not managed", m));
return (pmap_clearbit(m, PVF_REF));
}
@@ -4508,10 +4509,8 @@ pmap_clear_modify(vm_page_t m)
*/
if ((m->flags & PG_WRITEABLE) == 0)
return;
- vm_page_lock_queues();
if (m->md.pvh_attrs & PVF_MOD)
pmap_clearbit(m, PVF_MOD);
- vm_page_unlock_queues();
}
@@ -4541,10 +4540,8 @@ pmap_clear_reference(vm_page_t m)
KASSERT((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0,
("pmap_clear_reference: page %p is not managed", m));
- vm_page_lock_queues();
if (m->md.pvh_attrs & PVF_REF)
pmap_clearbit(m, PVF_REF);
- vm_page_unlock_queues();
}
@@ -4565,11 +4562,8 @@ pmap_remove_write(vm_page_t m)
*/
VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
if ((m->oflags & VPO_BUSY) != 0 ||
- (m->flags & PG_WRITEABLE) != 0) {
- vm_page_lock_queues();
+ (m->flags & PG_WRITEABLE) != 0)
pmap_clearbit(m, PVF_WRITE);
- vm_page_unlock_queues();
- }
}
Modified: head/sys/i386/i386/pmap.c
==============================================================================
--- head/sys/i386/i386/pmap.c Thu Jun 10 16:45:30 2010 (r208989)
+++ head/sys/i386/i386/pmap.c Thu Jun 10 16:56:35 2010 (r208990)
@@ -4061,30 +4061,35 @@ pmap_page_exists_quick(pmap_t pmap, vm_p
struct md_page *pvh;
pv_entry_t pv;
int loops = 0;
+ boolean_t rv;
- if (m->flags & PG_FICTITIOUS)
- return (FALSE);
-
- mtx_assert(&vm_page_queue_mtx, MA_OWNED);
+ KASSERT((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0,
+ ("pmap_page_exists_quick: page %p is not managed", m));
+ rv = FALSE;
+ vm_page_lock_queues();
TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
if (PV_PMAP(pv) == pmap) {
- return (TRUE);
+ rv = TRUE;
+ break;
}
loops++;
if (loops >= 16)
break;
}
- if (loops < 16) {
+ if (!rv && loops < 16) {
pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
TAILQ_FOREACH(pv, &pvh->pv_list, pv_list) {
- if (PV_PMAP(pv) == pmap)
- return (TRUE);
+ if (PV_PMAP(pv) == pmap) {
+ rv = TRUE;
+ break;
+ }
loops++;
if (loops >= 16)
break;
}
}
- return (FALSE);
+ vm_page_unlock_queues();
+ return (rv);
}
/*
@@ -4512,11 +4517,11 @@ pmap_ts_referenced(vm_page_t m)
vm_offset_t va;
int rtval = 0;
- if (m->flags & PG_FICTITIOUS)
- return (rtval);
- sched_pin();
- mtx_assert(&vm_page_queue_mtx, MA_OWNED);
+ KASSERT((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0,
+ ("pmap_ts_referenced: page %p is not managed", m));
pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
+ vm_page_lock_queues();
+ sched_pin();
TAILQ_FOREACH_SAFE(pv, &pvh->pv_list, pv_list, pvn) {
va = pv->pv_va;
pmap = PV_PMAP(pv);
@@ -4571,6 +4576,7 @@ pmap_ts_referenced(vm_page_t m)
}
out:
sched_unpin();
+ vm_page_unlock_queues();
return (rtval);
}
Modified: head/sys/i386/xen/pmap.c
==============================================================================
--- head/sys/i386/xen/pmap.c Thu Jun 10 16:45:30 2010 (r208989)
+++ head/sys/i386/xen/pmap.c Thu Jun 10 16:56:35 2010 (r208990)
@@ -3449,20 +3449,23 @@ pmap_page_exists_quick(pmap_t pmap, vm_p
{
pv_entry_t pv;
int loops = 0;
+ boolean_t rv;
- if (m->flags & PG_FICTITIOUS)
- return (FALSE);
-
- mtx_assert(&vm_page_queue_mtx, MA_OWNED);
+ KASSERT((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0,
+ ("pmap_page_exists_quick: page %p is not managed", m));
+ rv = FALSE;
+ vm_page_lock_queues();
TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
if (PV_PMAP(pv) == pmap) {
- return TRUE;
+ rv = TRUE;
+ break;
}
loops++;
if (loops >= 16)
break;
}
- return (FALSE);
+ vm_page_unlock_queues();
+ return (rv);
}
/*
@@ -3839,10 +3842,10 @@ pmap_ts_referenced(vm_page_t m)
pt_entry_t *pte;
int rtval = 0;
- if (m->flags & PG_FICTITIOUS)
- return (rtval);
+ KASSERT((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0,
+ ("pmap_ts_referenced: page %p is not managed", m));
+ vm_page_lock_queues();
sched_pin();
- mtx_assert(&vm_page_queue_mtx, MA_OWNED);
if ((pv = TAILQ_FIRST(&m->md.pv_list)) != NULL) {
pvf = pv;
do {
@@ -3867,6 +3870,7 @@ pmap_ts_referenced(vm_page_t m)
PT_SET_MA(PADDR1, 0);
sched_unpin();
+ vm_page_unlock_queues();
return (rtval);
}
Modified: head/sys/ia64/ia64/pmap.c
==============================================================================
--- head/sys/ia64/ia64/pmap.c Thu Jun 10 16:45:30 2010 (r208989)
+++ head/sys/ia64/ia64/pmap.c Thu Jun 10 16:56:35 2010 (r208990)
@@ -1837,23 +1837,23 @@ pmap_page_exists_quick(pmap_t pmap, vm_p
{
pv_entry_t pv;
int loops = 0;
+ boolean_t rv;
- if (m->flags & PG_FICTITIOUS)
- return FALSE;
-
- /*
- * Not found, check current mappings returning immediately if found.
- */
- mtx_assert(&vm_page_queue_mtx, MA_OWNED);
+ KASSERT((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0,
+ ("pmap_page_exists_quick: page %p is not managed", m));
+ rv = FALSE;
+ vm_page_lock_queues();
TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
if (pv->pv_pmap == pmap) {
- return TRUE;
+ rv = TRUE;
+ break;
}
loops++;
if (loops >= 16)
break;
}
- return (FALSE);
+ vm_page_unlock_queues();
+ return (rv);
}
/*
@@ -1949,9 +1949,9 @@ pmap_ts_referenced(vm_page_t m)
pv_entry_t pv;
int count = 0;
- if (m->flags & PG_FICTITIOUS)
- return 0;
-
+ KASSERT((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0,
+ ("pmap_ts_referenced: page %p is not managed", m));
+ vm_page_lock_queues();
TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
PMAP_LOCK(pv->pv_pmap);
oldpmap = pmap_switch(pv->pv_pmap);
@@ -1965,8 +1965,8 @@ pmap_ts_referenced(vm_page_t m)
pmap_switch(oldpmap);
PMAP_UNLOCK(pv->pv_pmap);
}
-
- return count;
+ vm_page_unlock_queues();
+ return (count);
}
/*
Modified: head/sys/mips/mips/pmap.c
==============================================================================
--- head/sys/mips/mips/pmap.c Thu Jun 10 16:45:30 2010 (r208989)
+++ head/sys/mips/mips/pmap.c Thu Jun 10 16:56:35 2010 (r208990)
@@ -2348,20 +2348,23 @@ pmap_page_exists_quick(pmap_t pmap, vm_p
{
pv_entry_t pv;
int loops = 0;
+ boolean_t rv;
- if (m->flags & PG_FICTITIOUS)
- return FALSE;
-
- mtx_assert(&vm_page_queue_mtx, MA_OWNED);
+ KASSERT((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0,
+ ("pmap_page_exists_quick: page %p is not managed", m));
+ rv = FALSE;
+ vm_page_lock_queues();
TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
if (pv->pv_pmap == pmap) {
- return TRUE;
+ rv = TRUE;
+ break;
}
loops++;
if (loops >= 16)
break;
}
- return (FALSE);
+ vm_page_unlock_queues();
+ return (rv);
}
/*
@@ -2594,14 +2597,16 @@ pmap_remove_write(vm_page_t m)
int
pmap_ts_referenced(vm_page_t m)
{
- if (m->flags & PG_FICTITIOUS)
- return (0);
+ KASSERT((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0,
+ ("pmap_ts_referenced: page %p is not managed", m));
if (m->md.pv_flags & PV_TABLE_REF) {
+ vm_page_lock_queues();
m->md.pv_flags &= ~PV_TABLE_REF;
- return 1;
+ vm_page_unlock_queues();
+ return (1);
}
- return 0;
+ return (0);
}
/*
Modified: head/sys/powerpc/aim/mmu_oea.c
==============================================================================
--- head/sys/powerpc/aim/mmu_oea.c Thu Jun 10 16:45:30 2010 (r208989)
+++ head/sys/powerpc/aim/mmu_oea.c Thu Jun 10 16:56:35 2010 (r208990)
@@ -286,7 +286,7 @@ static void moea_enter_locked(pmap_t, v
vm_prot_t, boolean_t);
static void moea_syncicache(vm_offset_t, vm_size_t);
static boolean_t moea_query_bit(vm_page_t, int);
-static u_int moea_clear_bit(vm_page_t, int, int *);
+static u_int moea_clear_bit(vm_page_t, int);
static void moea_kremove(mmu_t, vm_offset_t);
int moea_pte_spill(vm_offset_t);
@@ -1315,9 +1315,7 @@ moea_clear_reference(mmu_t mmu, vm_page_
KASSERT((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0,
("moea_clear_reference: page %p is not managed", m));
- vm_page_lock_queues();
- moea_clear_bit(m, PTE_REF, NULL);
- vm_page_unlock_queues();
+ moea_clear_bit(m, PTE_REF);
}
void
@@ -1337,9 +1335,7 @@ moea_clear_modify(mmu_t mmu, vm_page_t m
*/
if ((m->flags & PG_WRITEABLE) == 0)
return;
- vm_page_lock_queues();
- moea_clear_bit(m, PTE_CHG, NULL);
- vm_page_unlock_queues();
+ moea_clear_bit(m, PTE_CHG);
}
/*
@@ -1409,14 +1405,10 @@ moea_remove_write(mmu_t mmu, vm_page_t m
boolean_t
moea_ts_referenced(mmu_t mmu, vm_page_t m)
{
- int count;
-
- if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0)
- return (0);
- count = moea_clear_bit(m, PTE_REF, NULL);
-
- return (count);
+ KASSERT((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0,
+ ("moea_ts_referenced: page %p is not managed", m));
+ return (moea_clear_bit(m, PTE_REF));
}
/*
@@ -1531,19 +1523,23 @@ moea_page_exists_quick(mmu_t mmu, pmap_t
{
int loops;
struct pvo_entry *pvo;
+ boolean_t rv;
- if (!moea_initialized || (m->flags & PG_FICTITIOUS))
- return FALSE;
-
+ KASSERT((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0,
+ ("moea_page_exists_quick: page %p is not managed", m));
loops = 0;
+ rv = FALSE;
+ vm_page_lock_queues();
LIST_FOREACH(pvo, vm_page_to_pvoh(m), pvo_vlink) {
- if (pvo->pvo_pmap == pmap)
- return (TRUE);
+ if (pvo->pvo_pmap == pmap) {
+ rv = TRUE;
+ break;
+ }
if (++loops >= 16)
break;
}
-
- return (FALSE);
+ vm_page_unlock_queues();
+ return (rv);
}
/*
@@ -1557,7 +1553,7 @@ moea_page_wired_mappings(mmu_t mmu, vm_p
int count;
count = 0;
- if (!moea_initialized || (m->flags & PG_FICTITIOUS) != 0)
+ if ((m->flags & PG_FICTITIOUS) != 0)
return (count);
vm_page_lock_queues();
LIST_FOREACH(pvo, vm_page_to_pvoh(m), pvo_vlink)
@@ -2315,17 +2311,17 @@ moea_query_bit(vm_page_t m, int ptebit)
}
static u_int
-moea_clear_bit(vm_page_t m, int ptebit, int *origbit)
+moea_clear_bit(vm_page_t m, int ptebit)
{
u_int count;
struct pvo_entry *pvo;
struct pte *pt;
- int rv;
+
+ vm_page_lock_queues();
/*
* Clear the cached value.
*/
- rv = moea_attr_fetch(m);
moea_attr_clear(m, ptebit);
/*
@@ -2353,15 +2349,11 @@ moea_clear_bit(vm_page_t m, int ptebit,
}
mtx_unlock(&moea_table_mutex);
}
- rv |= pvo->pvo_pte.pte.pte_lo;
pvo->pvo_pte.pte.pte_lo &= ~ptebit;
MOEA_PVO_CHECK(pvo); /* sanity check */
}
- if (origbit != NULL) {
- *origbit = rv;
- }
-
+ vm_page_unlock_queues();
return (count);
}
Modified: head/sys/powerpc/aim/mmu_oea64.c
==============================================================================
--- head/sys/powerpc/aim/mmu_oea64.c Thu Jun 10 16:45:30 2010 (r208989)
+++ head/sys/powerpc/aim/mmu_oea64.c Thu Jun 10 16:56:35 2010 (r208990)
@@ -358,7 +358,7 @@ static void moea64_bridge_cpu_bootstrap
static void moea64_enter_locked(pmap_t, vm_offset_t, vm_page_t,
vm_prot_t, boolean_t);
static boolean_t moea64_query_bit(vm_page_t, u_int64_t);
-static u_int moea64_clear_bit(vm_page_t, u_int64_t, u_int64_t *);
+static u_int moea64_clear_bit(vm_page_t, u_int64_t);
static void moea64_kremove(mmu_t, vm_offset_t);
static void moea64_syncicache(pmap_t pmap, vm_offset_t va,
vm_offset_t pa, vm_size_t sz);
@@ -1510,9 +1510,7 @@ moea64_clear_reference(mmu_t mmu, vm_pag
KASSERT((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0,
("moea64_clear_reference: page %p is not managed", m));
- vm_page_lock_queues();
- moea64_clear_bit(m, LPTE_REF, NULL);
- vm_page_unlock_queues();
+ moea64_clear_bit(m, LPTE_REF);
}
void
@@ -1532,9 +1530,7 @@ moea64_clear_modify(mmu_t mmu, vm_page_t
*/
if ((m->flags & PG_WRITEABLE) == 0)
return;
- vm_page_lock_queues();
- moea64_clear_bit(m, LPTE_CHG, NULL);
- vm_page_unlock_queues();
+ moea64_clear_bit(m, LPTE_CHG);
}
/*
@@ -1605,14 +1601,10 @@ moea64_remove_write(mmu_t mmu, vm_page_t
boolean_t
moea64_ts_referenced(mmu_t mmu, vm_page_t m)
{
- int count;
-
- if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0)
- return (0);
- count = moea64_clear_bit(m, LPTE_REF, NULL);
-
- return (count);
+ KASSERT((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0,
+ ("moea64_ts_referenced: page %p is not managed", m));
+ return (moea64_clear_bit(m, LPTE_REF));
}
/*
@@ -1721,21 +1713,23 @@ moea64_page_exists_quick(mmu_t mmu, pmap
{
int loops;
struct pvo_entry *pvo;
+ boolean_t rv;
- if (!moea64_initialized || (m->flags & PG_FICTITIOUS))
- return FALSE;
-
- mtx_assert(&vm_page_queue_mtx, MA_OWNED);
-
+ KASSERT((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0,
+ ("moea64_page_exists_quick: page %p is not managed", m));
loops = 0;
+ rv = FALSE;
+ vm_page_lock_queues();
LIST_FOREACH(pvo, vm_page_to_pvoh(m), pvo_vlink) {
- if (pvo->pvo_pmap == pmap)
- return (TRUE);
+ if (pvo->pvo_pmap == pmap) {
+ rv = TRUE;
+ break;
+ }
if (++loops >= 16)
break;
}
-
- return (FALSE);
+ vm_page_unlock_queues();
+ return (rv);
}
/*
@@ -1749,7 +1743,7 @@ moea64_page_wired_mappings(mmu_t mmu, vm
int count;
count = 0;
- if (!moea64_initialized || (m->flags & PG_FICTITIOUS) != 0)
+ if ((m->flags & PG_FICTITIOUS) != 0)
return (count);
vm_page_lock_queues();
LIST_FOREACH(pvo, vm_page_to_pvoh(m), pvo_vlink)
@@ -2445,19 +2439,17 @@ moea64_query_bit(vm_page_t m, u_int64_t
}
static u_int
-moea64_clear_bit(vm_page_t m, u_int64_t ptebit, u_int64_t *origbit)
+moea64_clear_bit(vm_page_t m, u_int64_t ptebit)
{
u_int count;
struct pvo_entry *pvo;
struct lpte *pt;
- uint64_t rv;
- mtx_assert(&vm_page_queue_mtx, MA_OWNED);
+ vm_page_lock_queues();
/*
* Clear the cached value.
*/
- rv = moea64_attr_fetch(m);
moea64_attr_clear(m, ptebit);
/*
@@ -2486,16 +2478,12 @@ moea64_clear_bit(vm_page_t m, u_int64_t
moea64_pte_clear(pt, pvo->pvo_pmap, PVO_VADDR(pvo), ptebit);
}
}
- rv |= pvo->pvo_pte.lpte.pte_lo;
pvo->pvo_pte.lpte.pte_lo &= ~ptebit;
MOEA_PVO_CHECK(pvo); /* sanity check */
UNLOCK_TABLE();
}
- if (origbit != NULL) {
- *origbit = rv;
- }
-
+ vm_page_unlock_queues();
return (count);
}
Modified: head/sys/powerpc/booke/pmap.c
==============================================================================
--- head/sys/powerpc/booke/pmap.c Thu Jun 10 16:45:30 2010 (r208989)
+++ head/sys/powerpc/booke/pmap.c Thu Jun 10 16:56:35 2010 (r208990)
@@ -2293,17 +2293,14 @@ mmu_booke_ts_referenced(mmu_t mmu, vm_pa
pv_entry_t pv;
int count;
- mtx_assert(&vm_page_queue_mtx, MA_OWNED);
- if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0)
- return (0);
-
+ KASSERT((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0,
+ ("mmu_booke_ts_referenced: page %p is not managed", m));
count = 0;
+ vm_page_lock_queues();
TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) {
PMAP_LOCK(pv->pv_pmap);
- if ((pte = pte_find(mmu, pv->pv_pmap, pv->pv_va)) != NULL) {
- if (!PTE_ISVALID(pte))
- goto make_sure_to_unlock;
-
+ if ((pte = pte_find(mmu, pv->pv_pmap, pv->pv_va)) != NULL &&
+ PTE_ISVALID(pte)) {
if (PTE_ISREFERENCED(pte)) {
mtx_lock_spin(&tlbivax_mutex);
tlb_miss_lock();
@@ -2320,9 +2317,9 @@ mmu_booke_ts_referenced(mmu_t mmu, vm_pa
}
}
}
-make_sure_to_unlock:
PMAP_UNLOCK(pv->pv_pmap);
}
+ vm_page_unlock_queues();
return (count);
}
@@ -2394,20 +2391,23 @@ mmu_booke_page_exists_quick(mmu_t mmu, p
{
pv_entry_t pv;
int loops;
+ boolean_t rv;
- mtx_assert(&vm_page_queue_mtx, MA_OWNED);
- if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0)
- return (FALSE);
-
+ KASSERT((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0,
+ ("mmu_booke_page_exists_quick: page %p is not managed", m));
loops = 0;
+ rv = FALSE;
+ vm_page_lock_queues();
TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) {
- if (pv->pv_pmap == pmap)
- return (TRUE);
-
+ if (pv->pv_pmap == pmap) {
+ rv = TRUE;
+ break;
+ }
if (++loops >= 16)
break;
}
- return (FALSE);
+ vm_page_unlock_queues();
+ return (rv);
}
/*
Modified: head/sys/sparc64/sparc64/pmap.c
==============================================================================
--- head/sys/sparc64/sparc64/pmap.c Thu Jun 10 16:45:30 2010 (r208989)
+++ head/sys/sparc64/sparc64/pmap.c Thu Jun 10 16:56:35 2010 (r208990)
@@ -1789,20 +1789,25 @@ pmap_page_exists_quick(pmap_t pm, vm_pag
{
struct tte *tp;
int loops;
+ boolean_t rv;
- mtx_assert(&vm_page_queue_mtx, MA_OWNED);
- if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0)
- return (FALSE);
+ KASSERT((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0,
+ ("pmap_page_exists_quick: page %p is not managed", m));
loops = 0;
+ rv = FALSE;
+ vm_page_lock_queues();
TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) {
if ((tp->tte_data & TD_PV) == 0)
continue;
- if (TTE_GET_PMAP(tp) == pm)
- return (TRUE);
+ if (TTE_GET_PMAP(tp) == pm) {
+ rv = TRUE;
+ break;
+ }
if (++loops >= 16)
break;
}
- return (FALSE);
+ vm_page_unlock_queues();
+ return (rv);
}
/*
@@ -1878,10 +1883,10 @@ pmap_ts_referenced(vm_page_t m)
u_long data;
int count;
- mtx_assert(&vm_page_queue_mtx, MA_OWNED);
- if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0)
- return (0);
+ KASSERT((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0,
+ ("pmap_ts_referenced: page %p is not managed", m));
count = 0;
+ vm_page_lock_queues();
if ((tp = TAILQ_FIRST(&m->md.tte_list)) != NULL) {
tpf = tp;
do {
@@ -1895,6 +1900,7 @@ pmap_ts_referenced(vm_page_t m)
break;
} while ((tp = tpn) != NULL && tp != tpf);
}
+ vm_page_unlock_queues();
return (count);
}
Modified: head/sys/sun4v/sun4v/pmap.c
==============================================================================
--- head/sys/sun4v/sun4v/pmap.c Thu Jun 10 16:45:30 2010 (r208989)
+++ head/sys/sun4v/sun4v/pmap.c Thu Jun 10 16:56:35 2010 (r208990)
@@ -1738,20 +1738,23 @@ pmap_page_exists_quick(pmap_t pmap, vm_p
{
pv_entry_t pv;
int loops = 0;
+ boolean_t rv;
- if (m->flags & PG_FICTITIOUS)
- return FALSE;
-
- mtx_assert(&vm_page_queue_mtx, MA_OWNED);
+ KASSERT((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0,
+ ("pmap_page_exists_quick: page %p is not managed", m));
+ rv = FALSE;
+ vm_page_lock_queues();
TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
if (pv->pv_pmap == pmap) {
- return TRUE;
+ rv = TRUE;
+ break;
}
loops++;
if (loops >= 16)
break;
}
- return (FALSE);
+ vm_page_unlock_queues();
+ return (rv);
}
/*
@@ -2309,17 +2312,15 @@ pmap_tte_hash_resize(pmap_t pmap)
int
pmap_ts_referenced(vm_page_t m)
{
-
int rv;
pv_entry_t pv, pvf, pvn;
pmap_t pmap;
tte_t otte_data;
+ KASSERT((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0,
+ ("pmap_ts_referenced: page %p is not managed", m));
rv = 0;
- if (m->flags & PG_FICTITIOUS)
- return (rv);
-
- mtx_assert(&vm_page_queue_mtx, MA_OWNED);
+ vm_page_lock_queues();
if ((pv = TAILQ_FIRST(&m->md.pv_list)) != NULL) {
pvf = pv;
@@ -2347,6 +2348,7 @@ pmap_ts_referenced(vm_page_t m)
PMAP_UNLOCK(pmap);
} while ((pv = pvn) != NULL && pv != pvf);
}
+ vm_page_unlock_queues();
return (rv);
}
Modified: head/sys/vm/vm_page.c
==============================================================================
--- head/sys/vm/vm_page.c Thu Jun 10 16:45:30 2010 (r208989)
+++ head/sys/vm/vm_page.c Thu Jun 10 16:56:35 2010 (r208990)
@@ -502,6 +502,8 @@ vm_page_flag_clear(vm_page_t m, unsigned
{
mtx_assert(&vm_page_queue_mtx, MA_OWNED);
+ KASSERT((bits & PG_REFERENCED) == 0 || VM_OBJECT_LOCKED(m->object),
+ ("PG_REFERENCED and !VM_OBJECT_LOCKED"));
m->flags &= ~bits;
}
@@ -1333,8 +1335,7 @@ vm_waitpfault(void)
/*
* vm_page_requeue:
*
- * If the given page is contained within a page queue, move it to the tail
- * of that queue.
+ * Move the given page to the tail of its present page queue.
*
* The page queues must be locked.
*/
@@ -1344,11 +1345,12 @@ vm_page_requeue(vm_page_t m)
int queue = VM_PAGE_GETQUEUE(m);
struct vpgqueues *vpq;
- if (queue != PQ_NONE) {
- vpq = &vm_page_queues[queue];
- TAILQ_REMOVE(&vpq->pl, m, pageq);
- TAILQ_INSERT_TAIL(&vpq->pl, m, pageq);
- }
+ mtx_assert(&vm_page_queue_mtx, MA_OWNED);
+ KASSERT(queue != PQ_NONE,
+ ("vm_page_requeue: page %p is not queued", m));
+ vpq = &vm_page_queues[queue];
+ TAILQ_REMOVE(&vpq->pl, m, pageq);
+ TAILQ_INSERT_TAIL(&vpq->pl, m, pageq);
}
/*
Modified: head/sys/vm/vm_page.h
==============================================================================
--- head/sys/vm/vm_page.h Thu Jun 10 16:45:30 2010 (r208989)
+++ head/sys/vm/vm_page.h Thu Jun 10 16:56:35 2010 (r208990)
@@ -219,6 +219,9 @@ extern struct vpglocks pa_lock[];
* pte mappings, nor can they be removed from their objects via
* the object, and such pages are also not on any PQ queue.
*
+ * PG_REFERENCED may be cleared only if the object containing the page is
+ * locked.
+ *
* PG_WRITEABLE is set exclusively on managed pages by pmap_enter(). When it
* does so, the page must be VPO_BUSY.
*/
Modified: head/sys/vm/vm_pageout.c
==============================================================================
--- head/sys/vm/vm_pageout.c Thu Jun 10 16:45:30 2010 (r208989)
+++ head/sys/vm/vm_pageout.c Thu Jun 10 16:56:35 2010 (r208990)
@@ -547,21 +547,17 @@ vm_pageout_flush(vm_page_t *mc, int coun
/*
* vm_pageout_object_deactivate_pages
*
- * deactivate enough pages to satisfy the inactive target
- * requirements or if vm_page_proc_limit is set, then
- * deactivate all of the pages in the object and its
- * backing_objects.
+ * Deactivate enough pages to satisfy the inactive target
+ * requirements.
*
* The object and map must be locked.
*/
static void
-vm_pageout_object_deactivate_pages(pmap, first_object, desired)
- pmap_t pmap;
- vm_object_t first_object;
- long desired;
+vm_pageout_object_deactivate_pages(pmap_t pmap, vm_object_t first_object,
+ long desired)
{
vm_object_t backing_object, object;
- vm_page_t p, next;
+ vm_page_t p;
int actcount, remove_mode;
VM_OBJECT_LOCK_ASSERT(first_object, MA_OWNED);
@@ -579,61 +575,57 @@ vm_pageout_object_deactivate_pages(pmap,
if (object->shadow_count > 1)
remove_mode = 1;
/*
- * scan the objects entire memory queue
+ * Scan the object's entire memory queue.
*/
- p = TAILQ_FIRST(&object->memq);
- while (p != NULL) {
+ TAILQ_FOREACH(p, &object->memq, listq) {
if (pmap_resident_count(pmap) <= desired)
goto unlock_return;
- next = TAILQ_NEXT(p, listq);
- if ((p->oflags & VPO_BUSY) != 0 || p->busy != 0) {
- p = next;
+ if ((p->oflags & VPO_BUSY) != 0 || p->busy != 0)
continue;
- }
+ PCPU_INC(cnt.v_pdpages);
vm_page_lock(p);
- vm_page_lock_queues();
- cnt.v_pdpages++;
- if (p->wire_count != 0 ||
- p->hold_count != 0 ||
+ if (p->wire_count != 0 || p->hold_count != 0 ||
!pmap_page_exists_quick(pmap, p)) {
- vm_page_unlock_queues();
vm_page_unlock(p);
- p = next;
continue;
}
actcount = pmap_ts_referenced(p);
- if (actcount) {
- vm_page_flag_set(p, PG_REFERENCED);
- } else if (p->flags & PG_REFERENCED) {
- actcount = 1;
+ if ((p->flags & PG_REFERENCED) != 0) {
+ if (actcount == 0)
+ actcount = 1;
+ vm_page_lock_queues();
+ vm_page_flag_clear(p, PG_REFERENCED);
+ vm_page_unlock_queues();
}
- if ((p->queue != PQ_ACTIVE) &&
- (p->flags & PG_REFERENCED)) {
+ if (p->queue != PQ_ACTIVE && actcount != 0) {
vm_page_activate(p);
p->act_count += actcount;
- vm_page_flag_clear(p, PG_REFERENCED);
*** DIFF OUTPUT TRUNCATED AT 1000 LINES ***
More information about the svn-src-head
mailing list