svn commit: r254064 - in head/sys/amd64: amd64 include
Konstantin Belousov
kib at FreeBSD.org
Wed Aug 7 16:33:16 UTC 2013
Author: kib
Date: Wed Aug 7 16:33:15 2013
New Revision: 254064
URL: http://svnweb.freebsd.org/changeset/base/254064
Log:
Change the pmap_ts_referenced() method of amd64 pmap to use shared
pvh_global_lock. This allows the method to be executed in parallel,
avoiding undue contention on the pvh_global_lock for the multithreaded
pagedaemon.
The pmap_ts_referenced() function has to inspect the page mappings for
several pmaps, which need to be locked while pv list lock is owned.
This contradicts to the lock order, where pmap lock is before pv list
lock. Introduce the generation count for the pv list of the page or
superpage, which indicate any change in the pv list, and, as usual,
perform restart of the iteration if generation changed while pv lock
was dropped for blocking acquire of a pmap lock.
Reported and tested by: pho
Reviewed by: alc
Sponsored by: The FreeBSD Foundation
Modified:
head/sys/amd64/amd64/pmap.c
head/sys/amd64/include/pmap.h
Modified: head/sys/amd64/amd64/pmap.c
==============================================================================
--- head/sys/amd64/amd64/pmap.c Wed Aug 7 16:01:45 2013 (r254063)
+++ head/sys/amd64/amd64/pmap.c Wed Aug 7 16:33:15 2013 (r254064)
@@ -2183,6 +2183,7 @@ reclaim_pv_chunk(pmap_t locked_pmap, str
vm_page_aflag_set(m, PGA_REFERENCED);
CHANGE_PV_LIST_LOCK_TO_VM_PAGE(lockp, m);
TAILQ_REMOVE(&m->md.pv_list, pv, pv_next);
+ m->md.pv_gen++;
if (TAILQ_EMPTY(&m->md.pv_list) &&
(m->flags & PG_FICTITIOUS) == 0) {
pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
@@ -2470,6 +2471,7 @@ pmap_pvh_remove(struct md_page *pvh, pma
TAILQ_FOREACH(pv, &pvh->pv_list, pv_next) {
if (pmap == PV_PMAP(pv) && va == pv->pv_va) {
TAILQ_REMOVE(&pvh->pv_list, pv, pv_next);
+ pvh->pv_gen++;
break;
}
}
@@ -2509,6 +2511,7 @@ pmap_pv_demote_pde(pmap_t pmap, vm_offse
KASSERT(pv != NULL, ("pmap_pv_demote_pde: pv not found"));
m = PHYS_TO_VM_PAGE(pa);
TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next);
+ m->md.pv_gen++;
/* Instantiate the remaining NPTEPG - 1 pv entries. */
PV_STAT(atomic_add_long(&pv_entry_allocs, NPTEPG - 1));
va_last = va + NBPDR - PAGE_SIZE;
@@ -2527,6 +2530,7 @@ pmap_pv_demote_pde(pmap_t pmap, vm_offse
KASSERT((m->oflags & VPO_UNMANAGED) == 0,
("pmap_pv_demote_pde: page %p is not managed", m));
TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next);
+ m->md.pv_gen++;
if (va == va_last)
goto out;
}
@@ -2575,6 +2579,7 @@ pmap_pv_promote_pde(pmap_t pmap, vm_offs
KASSERT(pv != NULL, ("pmap_pv_promote_pde: pv not found"));
pvh = pa_to_pvh(pa);
TAILQ_INSERT_TAIL(&pvh->pv_list, pv, pv_next);
+ pvh->pv_gen++;
/* Free the remaining NPTEPG - 1 pv entries. */
va_last = va + NBPDR - PAGE_SIZE;
do {
@@ -2616,6 +2621,7 @@ pmap_try_insert_pv_entry(pmap_t pmap, vm
pv->pv_va = va;
CHANGE_PV_LIST_LOCK_TO_VM_PAGE(lockp, m);
TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next);
+ m->md.pv_gen++;
return (TRUE);
} else
return (FALSE);
@@ -2640,6 +2646,7 @@ pmap_pv_insert_pde(pmap_t pmap, vm_offse
CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, pa);
pvh = pa_to_pvh(pa);
TAILQ_INSERT_TAIL(&pvh->pv_list, pv, pv_next);
+ pvh->pv_gen++;
return (TRUE);
} else
return (FALSE);
@@ -3156,6 +3163,7 @@ small_mappings:
pmap_unuse_pt(pmap, pv->pv_va, *pde, &free);
pmap_invalidate_page(pmap, pv->pv_va);
TAILQ_REMOVE(&m->md.pv_list, pv, pv_next);
+ m->md.pv_gen++;
free_pv_entry(pmap, pv);
PMAP_UNLOCK(pmap);
}
@@ -3601,6 +3609,7 @@ retry:
pv->pv_va = va;
CHANGE_PV_LIST_LOCK_TO_PHYS(&lock, pa);
TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next);
+ m->md.pv_gen++;
if ((newpte & PG_RW) != 0)
vm_page_aflag_set(m, PGA_WRITEABLE);
}
@@ -4536,6 +4545,7 @@ pmap_remove_pages(pmap_t pmap)
pmap_resident_count_dec(pmap, NBPDR / PAGE_SIZE);
pvh = pa_to_pvh(tpte & PG_PS_FRAME);
TAILQ_REMOVE(&pvh->pv_list, pv, pv_next);
+ pvh->pv_gen++;
if (TAILQ_EMPTY(&pvh->pv_list)) {
for (mt = m; mt < &m[NBPDR / PAGE_SIZE]; mt++)
if ((mt->aflags & PGA_WRITEABLE) != 0 &&
@@ -4555,6 +4565,7 @@ pmap_remove_pages(pmap_t pmap)
} else {
pmap_resident_count_dec(pmap, 1);
TAILQ_REMOVE(&m->md.pv_list, pv, pv_next);
+ m->md.pv_gen++;
if ((m->aflags & PGA_WRITEABLE) != 0 &&
TAILQ_EMPTY(&m->md.pv_list) &&
(m->flags & PG_FICTITIOUS) == 0) {
@@ -4793,25 +4804,40 @@ pmap_ts_referenced(vm_page_t m)
struct md_page *pvh;
pv_entry_t pv, pvf, pvn;
pmap_t pmap;
+ struct rwlock *lock;
pd_entry_t oldpde, *pde;
pt_entry_t *pte;
vm_offset_t va;
- int rtval = 0;
+ int rtval, pvh_gen, md_gen;
KASSERT((m->oflags & VPO_UNMANAGED) == 0,
("pmap_ts_referenced: page %p is not managed", m));
- rw_wlock(&pvh_global_lock);
+ rw_rlock(&pvh_global_lock);
+ lock = VM_PAGE_TO_PV_LIST_LOCK(m);
+ pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
+ rtval = 0;
+retry:
+ rw_wlock(lock);
if ((m->flags & PG_FICTITIOUS) != 0)
goto small_mappings;
- pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
TAILQ_FOREACH_SAFE(pv, &pvh->pv_list, pv_next, pvn) {
pmap = PV_PMAP(pv);
- PMAP_LOCK(pmap);
+ if (!PMAP_TRYLOCK(pmap)) {
+ pvh_gen = pvh->pv_gen;
+ rw_wunlock(lock);
+ PMAP_LOCK(pmap);
+ rw_wlock(lock);
+ if (pvh_gen != pvh->pv_gen) {
+ PMAP_UNLOCK(pmap);
+ rw_wunlock(lock);
+ goto retry;
+ }
+ }
va = pv->pv_va;
pde = pmap_pde(pmap, va);
oldpde = *pde;
if ((oldpde & PG_A) != 0) {
- if (pmap_demote_pde(pmap, pde, va)) {
+ if (pmap_demote_pde_locked(pmap, pde, va, &lock)) {
if ((oldpde & PG_W) == 0) {
/*
* Remove the mapping to a single page
@@ -4823,7 +4849,10 @@ pmap_ts_referenced(vm_page_t m)
*/
va += VM_PAGE_TO_PHYS(m) - (oldpde &
PG_PS_FRAME);
- pmap_remove_page(pmap, va, pde, NULL);
+ pte = pmap_pde_to_pte(pde, va);
+ pmap_remove_pte(pmap, pte, va, *pde,
+ NULL, &lock);
+ pmap_invalidate_page(pmap, va);
rtval++;
if (rtval > 4) {
PMAP_UNLOCK(pmap);
@@ -4831,6 +4860,9 @@ pmap_ts_referenced(vm_page_t m)
}
}
}
+ KASSERT(lock == VM_PAGE_TO_PV_LIST_LOCK(m),
+ ("inconsistent pv lock %p %p for page %p",
+ lock, VM_PAGE_TO_PV_LIST_LOCK(m), m));
}
PMAP_UNLOCK(pmap);
}
@@ -4841,8 +4873,21 @@ small_mappings:
pvn = TAILQ_NEXT(pv, pv_next);
TAILQ_REMOVE(&m->md.pv_list, pv, pv_next);
TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next);
+ m->md.pv_gen++;
pmap = PV_PMAP(pv);
- PMAP_LOCK(pmap);
+ if (!PMAP_TRYLOCK(pmap)) {
+ pvh_gen = pvh->pv_gen;
+ md_gen = m->md.pv_gen;
+ rw_wunlock(lock);
+ PMAP_LOCK(pmap);
+ rw_wlock(lock);
+ if (pvh_gen != pvh->pv_gen ||
+ md_gen != m->md.pv_gen) {
+ PMAP_UNLOCK(pmap);
+ rw_wunlock(lock);
+ goto retry;
+ }
+ }
pde = pmap_pde(pmap, pv->pv_va);
KASSERT((*pde & PG_PS) == 0, ("pmap_ts_referenced:"
" found a 2mpage in page %p's pv list", m));
@@ -4858,7 +4903,8 @@ small_mappings:
} while ((pv = pvn) != NULL && pv != pvf);
}
out:
- rw_wunlock(&pvh_global_lock);
+ rw_wunlock(lock);
+ rw_runlock(&pvh_global_lock);
return (rtval);
}
Modified: head/sys/amd64/include/pmap.h
==============================================================================
--- head/sys/amd64/include/pmap.h Wed Aug 7 16:01:45 2013 (r254063)
+++ head/sys/amd64/include/pmap.h Wed Aug 7 16:33:15 2013 (r254064)
@@ -233,6 +233,7 @@ struct pv_chunk;
struct md_page {
TAILQ_HEAD(,pv_entry) pv_list;
+ int pv_gen;
int pat_mode;
};
More information about the svn-src-all
mailing list