PERFORCE change 84850 for review
Olivier Houchard
cognet at FreeBSD.org
Wed Oct 5 06:48:31 PDT 2005
http://perforce.freebsd.org/chv.cgi?CH=84850
Change 84850 by cognet at cognet on 2005/10/05 13:47:44
In pmap_remove(), pmap_remove_pages() and pmap_protect(), be smarter
when dealing with a superpage than blindly demoting it.
Affected files ...
.. //depot/projects/superpages/src/sys/arm/arm/pmap.c#4 edit
Differences ...
==== //depot/projects/superpages/src/sys/arm/arm/pmap.c#4 (text+ko) ====
@@ -189,7 +189,7 @@
#else /* PMAP_DEBUG */
#define PDEBUG(_lev_,_stat_) /* Nothing */
#define dprintf(x, arg...)
-#define PMAP_INLINE __inline
+#define PMAP_INLINE __inline
#endif /* PMAP_DEBUG */
extern struct pv_addr systempage;
@@ -1550,8 +1550,6 @@
pt_entry_t *ptep;
vm_paddr_t pa0;
- if (!reserv)
- return;
ptep = &l2b->l2b_kva[l2pte_index(va & L2_L_FRAME)];
pa0 = *ptep & L2_L_FRAME;
if ((*ptep & L2_S_FRAME) != pa0)
@@ -1596,8 +1594,6 @@
uint16_t l1idx;
int i;
- if (!reserv)
- return;
va0 = va & L1_S_ADDR_MASK;
l1idx = L1_IDX(va0);
pd = &pmap->pm_l1->l1_kva[l1idx];
@@ -1644,8 +1640,12 @@
if (*pt & L2_S_PROT_W)
pa |= L1_S_PROT_W;
*pd = L1_S_PROTO | pa | pte_l1_s_cache_mode | L1_S_DOM(pmap->pm_domain);
+#if 0
bzero(pt, 0x100 * sizeof(*pt));
+#endif
pmap_free_l2_bucket(pmap, &l2->l2_bucket[L2_BUCKET(l1idx)], 0x100);
+ if (pmap != pmap_kernel() && l2->l2_bucket[L2_BUCKET(l1idx)].l2b_kva != NULL)
+ panic("missed");
pmap_tlb_flushID(pmap);
}
@@ -2043,6 +2043,42 @@
return(pve); /* return removed pve */
}
+
+static void
+pmap_destroy_pv_range(pmap_t pm, vm_paddr_t pa, vm_offset_t sva,
+ vm_offset_t eva)
+{
+ struct pv_entry *pve, *pvetmp;
+ int i = 0;
+ pve = TAILQ_FIRST(&pm->pm_pvlist);
+
+ while (pve) {
+ pvetmp = TAILQ_NEXT(pve, pv_plist);
+ if (pve->pv_va >= sva && pve->pv_va < eva) {
+ pmap_nuke_pv(PHYS_TO_VM_PAGE(pa + (pve->pv_va - sva)),
+ pm, pve);
+ pmap_free_pv_entry(pve);
+ i++;
+ }
+ pve = pvetmp;
+ }
+}
+
+static int
+pmap_clearbit_pv_range(pmap_t pm, vm_offset_t sva, vm_offset_t eva, int bits)
+{
+ struct pv_entry *pve;
+ int ret = 0;
+
+ TAILQ_FOREACH(pve, &pm->pm_pvlist, pv_plist) {
+ if (pve->pv_va >= sva && pve->pv_va < eva) {
+ ret |= pve->pv_flags;
+ pve->pv_flags &= ~bits;
+ }
+ }
+ return (ret);
+}
+
/*
*
* pmap_modify_pv: Update pv flags
@@ -2173,6 +2209,8 @@
pt_entry_t *ptep, pte;
vm_paddr_t pa;
u_int l1idx;
+ struct pv_entry *pv;
+ struct vm_page *pg;
int rv = 0;
#if 0
@@ -2184,7 +2222,11 @@
if (l1pte_section_p(*pl1pd)) {
if (user && !(*pl1pd & L1_S_PROT_U))
goto out;
- if (ftype == VM_PROT_WRITE && !(*pl1pd & L1_S_PROT_W))
+ pg = PHYS_TO_VM_PAGE((*pl1pd & L1_S_FRAME) +
+ (va & L1_S_OFFSET));
+ pv = pmap_find_pv(pg, pm, va);
+ if (!pv ||
+ (ftype == VM_PROT_WRITE && !(pv->pv_flags & PVF_WRITE)))
goto out;
if ((*pl1pd & L1_S_DOM_MASK) != L1_S_DOM(pm->pm_domain)) {
*pl1pd &= ~L1_S_DOM_MASK;
@@ -2236,8 +2278,6 @@
* This looks like a good candidate for "page modified"
* emulation...
*/
- struct pv_entry *pv;
- struct vm_page *pg;
/* Extract the physical address of the page */
if ((pg = PHYS_TO_VM_PAGE(pa)) == NULL)
@@ -2802,9 +2842,6 @@
{
struct pcb *pcb;
- pmap_idcache_wbinv_all(pmap);
- pmap_tlb_flushID(pmap);
- cpu_cpwait();
LIST_REMOVE(pmap, pm_list);
if (vector_page < KERNBASE) {
struct pcb *curpcb = PCPU_GET(curpcb);
@@ -3043,27 +3080,70 @@
pd_entry_t *pd;
vm_paddr_t pa;
vm_page_t m;
-
+ int i;
+
vm_page_lock_queues();
for (pv = TAILQ_FIRST(&pmap->pm_pvlist); pv; pv = npv) {
if (pv->pv_va >= eva || pv->pv_va < sva) {
npv = TAILQ_NEXT(pv, pv_plist);
continue;
}
- pd = &pmap->pm_l1->l1_kva[L1_IDX(pv->pv_va)];
- if (l1pte_section_p(*pd))
- pmap_demote(pmap, pv->pv_va);
- if (pv->pv_flags & PVF_WIRED) {
+ if (pv->pv_flags & PVF_WIRED) {
/* The page is wired, cannot remove it now. */
npv = TAILQ_NEXT(pv, pv_plist);
continue;
}
+
+ pd = &pmap->pm_l1->l1_kva[L1_IDX(pv->pv_va)];
+ if (l1pte_section_p(*pd)) {
+ if ((pv->pv_va & L1_S_ADDR_MASK) >= sva &&
+ ((pv->pv_va & L1_S_ADDR_MASK) + L1_S_SIZE) <
+ eva) {
+ pmap_destroy_pv_range(pmap,
+ *pd & L1_S_ADDR_MASK,
+ pv->pv_va & L1_S_ADDR_MASK,
+ (pv->pv_va & L1_S_ADDR_MASK) +
+ L1_S_SIZE);
+ for (i = 0; i < 0x100; i++) {
+ m = PHYS_TO_VM_PAGE(
+ (*pd & L1_S_ADDR_MASK) +
+ i * PAGE_SIZE);
+ if (TAILQ_EMPTY(&m->md.pv_list))
+ vm_page_flag_clear(m,
+ PG_WRITEABLE);
+ }
+ *pd = 0;
+ npv = TAILQ_FIRST(&pmap->pm_pvlist);
+ continue;
+ } else
+ pmap_demote(pmap, pv->pv_va);
+ }
pmap->pm_stats.resident_count--;
l2b = pmap_get_l2_bucket(pmap, pv->pv_va);
KASSERT(l2b != NULL, ("No L2 bucket in pmap_remove_pages"));
pt = &l2b->l2b_kva[l2pte_index(pv->pv_va)];
- if ((*pt & L2_TYPE_MASK) == L2_TYPE_L)
- pmap_demote(pmap, pv->pv_va);
+ if ((*pt & L2_TYPE_MASK) == L2_TYPE_L) {
+ if ((pv->pv_va & L2_L_FRAME) >= sva &&
+ (pv->pv_va & L2_L_FRAME) + L2_L_SIZE < eva) {
+ pmap_destroy_pv_range(pmap, *pt & L2_L_FRAME,
+ pv->pv_va & L2_L_FRAME,
+ (pv->pv_va & L2_L_FRAME) + L2_L_SIZE);
+ pt = &l2b->l2b_kva[l2pte_index(pv->pv_va &
+ L2_L_FRAME)];
+ for (int i = 0; i < 0x10; i++) {
+ m = PHYS_TO_VM_PAGE((pt[i] &
+ L2_L_FRAME) + i * PAGE_SIZE);
+ if (TAILQ_EMPTY(&m->md.pv_list))
+ vm_page_flag_clear(m,
+ PG_WRITEABLE);
+ pt[i] = 0;
+ }
+ pmap_free_l2_bucket(pmap, l2b, 0x10);
+ npv = TAILQ_FIRST(&pmap->pm_pvlist);
+ continue;
+ } else
+ pmap_demote(pmap, pv->pv_va);
+ }
pa = *pt & L2_S_FRAME;
m = PHYS_TO_VM_PAGE(pa);
*pt = 0;
@@ -3073,6 +3153,7 @@
if (TAILQ_EMPTY(&m->md.pv_list))
vm_page_flag_clear(m, PG_WRITEABLE);
pmap_free_pv_entry(pv);
+ pmap_free_l2_bucket(pmap, l2b, 1);
}
vm_page_unlock_queues();
cpu_idcache_wbinv_all();
@@ -3449,6 +3530,8 @@
u_int flags;
int flush;
int demoted = 0;
+ int i;
+ pd_entry_t *pd;
if ((prot & VM_PROT_READ) == 0) {
mtx_lock(&Giant);
@@ -3487,9 +3570,25 @@
if ((sva & L1_S_OFFSET) == 0 &&
sva + L1_S_SIZE < eva) {
/* Change the whole 1MB superpage. */
- pm->pm_l1->l1_kva[L1_IDX(sva)] &= ~
- L1_S_PROT_W;
- flush++;
+ pd = &pm->pm_l1->l1_kva[L1_IDX(sva)];
+ if (*pd & L1_S_PROT_W) {
+ *pd &= ~L1_S_PROT_W;
+ PTE_SYNC(*pd);
+ flush++;
+ flags |= pmap_clearbit_pv_range(pm, sva,
+ sva + L1_S_SIZE, PVF_WRITE);
+ for (i = 0; i < 0x100; i++) {
+ vm_page_t m =
+ PHYS_TO_VM_PAGE(
+ (*pd & L1_S_FRAME) +
+ i * PAGE_SIZE);
+ pmap_vac_me_harder(m, pm,
+ sva + i * PAGE_SIZE);
+ if (pmap_track_modified(sva +
+ i * PAGE_SIZE))
+ vm_page_dirty(m);
+ }
+ }
sva += L1_S_SIZE;
continue;
}
@@ -3508,8 +3607,36 @@
while (sva < next_bucket) {
demoted = 0;
if ((*ptep & L2_TYPE_MASK) == L2_TYPE_L) {
- pmap_demote(pm, sva);
- demoted = 1;
+ if ((sva & L2_L_OFFSET) == 0 &&
+ sva + L2_L_SIZE < eva) {
+ if (*ptep & L2_L_PROT_W) {
+ flags |= pmap_clearbit_pv_range(
+ pm, sva,
+ sva + L2_L_SIZE, PVF_WRITE);
+ for (i = 0; i < 0x10; i++) {
+ vm_page_t m;
+
+ m = PHYS_TO_VM_PAGE(
+ (ptep[i] &
+ L2_L_FRAME) +
+ i * PAGE_SIZE);
+ ptep[i] &= ~L2_L_PROT_W;
+ PTE_SYNC(&ptep[i]);
+ pmap_vac_me_harder(m,
+ pm, sva +
+ i * PAGE_SIZE);
+ if (pmap_track_modified(
+ sva + i * PAGE_SIZE))
+ vm_page_dirty(m);
+ }
+ flush++;
+ }
+ sva += L2_L_SIZE;
+ continue;
+ } else {
+ pmap_demote(pm, sva);
+ demoted = 1;
+ }
}
if ((pte = *ptep) != 0 && (pte & L2_S_PROT_W) != 0) {
struct vm_page *pg;
@@ -3600,7 +3727,6 @@
/* There's already a superpage at this address, demote. */
pmap_demote(pmap, va);
} else if (l1pte_section_p(l1pd)) {
- pmap_free_l2_bucket(pmap, l2b, 0);
opg = m;
l2b = NULL;
}
@@ -4150,10 +4276,14 @@
pd_entry_t *pd = &pm->pm_l1->l1_kva[L1_IDX(sva)];
if (l1pte_section_p(*pd)) {
/* We can just remove the superpage. */
- if (0 && (sva == (sva & L1_S_ADDR_MASK)) &&
- (sva + 0x100000 < eva)) {
+ if ((sva == (sva & L1_S_ADDR_MASK)) &&
+ (sva + L1_S_SIZE < eva)) {
+ pmap_destroy_pv_range(pm,
+ *pd & L1_S_FRAME, sva,
+ sva + L1_S_SIZE);
*pd = 0;
- sva = sva + 0x100000;
+ flushall = 1;
+ sva += L1_S_SIZE;
continue;
} else {
pmap_demote(pm, sva);
@@ -4173,10 +4303,23 @@
pt_entry_t pte;
vm_paddr_t pa;
- if ((*ptep & L2_TYPE_MASK) == L2_TYPE_L)
+ if ((*ptep & L2_TYPE_MASK) == L2_TYPE_L) {
/* 64KB superpage. */
+ if ((sva & L2_L_FRAME) == sva &&
+ sva + L2_L_SIZE < eva) {
+ pmap_destroy_pv_range(pm,
+ *ptep & L2_L_FRAME,
+ sva, sva + L2_L_SIZE);
+ for (int i = 0; i < 0x10; i++)
+ ptep[i] = 0;
+ sva += L2_L_SIZE;
+ ptep += 0x10;
+ flushall = 1;
+ mappings += 0x10;
+ continue;
+ }
pmap_demote(pm, sva);
-
+ }
pte = *ptep;
if (pte == 0) {
@@ -4206,12 +4349,12 @@
pve = pmap_remove_pv(pg, pm, sva);
if (pve) {
#if 0
- simple_unlock(&pg->mdpage.pvh_slock);
+ simple_unlock(&pg->mdpage.pvh_slock);
#endif
- is_exec =
- PV_BEEN_EXECD(pve->pv_flags);
- is_refd =
- PV_BEEN_REFD(pve->pv_flags);
+ is_exec =
+ PV_BEEN_EXECD(pve->pv_flags);
+ is_refd =
+ PV_BEEN_REFD(pve->pv_flags);
pmap_free_pv_entry(pve);
}
}
@@ -4234,8 +4377,7 @@
} else
if (cleanlist_idx == PMAP_REMOVE_CLEAN_LIST_SIZE) {
/* Nuke everything if needed. */
- pmap_idcache_wbinv_all(pm);
- pmap_tlb_flushID(pm);
+ flushall = 1;
/*
* Roll back the previous PTE list,
@@ -4248,14 +4390,13 @@
*ptep = 0;
PTE_SYNC(ptep);
cleanlist_idx++;
- flushall = 1;
} else {
*ptep = 0;
PTE_SYNC(ptep);
- if (is_exec)
- pmap_tlb_flushID_SE(pm, sva);
- else
- if (is_refd)
+ if (!flushall && is_exec)
+ pmap_tlb_flushID_SE(pm, sva);
+ else
+ if (!flushall && is_refd)
pmap_tlb_flushD_SE(pm, sva);
}
@@ -4272,15 +4413,17 @@
for (cnt = 0; cnt < cleanlist_idx; cnt++) {
vm_offset_t clva =
cleanlist[cnt].va & ~1;
- if (cleanlist[cnt].va & 1) {
- pmap_idcache_wbinv_range(pm,
- clva, PAGE_SIZE);
- pmap_tlb_flushID_SE(pm, clva);
- } else {
- pmap_dcache_wb_range(pm,
- clva, PAGE_SIZE, TRUE,
- FALSE);
- pmap_tlb_flushD_SE(pm, clva);
+ if (!flushall) {
+ if (cleanlist[cnt].va & 1) {
+ pmap_idcache_wbinv_range(pm,
+ clva, PAGE_SIZE);
+ pmap_tlb_flushID_SE(pm, clva);
+ } else {
+ pmap_dcache_wb_range(pm,
+ clva, PAGE_SIZE, TRUE,
+ FALSE);
+ pmap_tlb_flushD_SE(pm, clva);
+ }
}
*cleanlist[cnt].pte = 0;
PTE_SYNC_CURRENT(pm, cleanlist[cnt].pte);
@@ -4294,7 +4437,6 @@
* easier to flush the whole cache.
*/
cleanlist_idx = PMAP_REMOVE_CLEAN_LIST_SIZE + 1;
- pmap_idcache_wbinv_all(pm);
flushall = 1;
}
}
@@ -4302,9 +4444,11 @@
pmap_free_l2_bucket(pm, l2b, mappings);
}
+ if (flushall) {
+ pmap_idcache_wbinv_all(pm);
+ pmap_tlb_flushID(pm);
+ }
vm_page_unlock_queues();
- if (flushall)
- cpu_tlb_flushID();
#if 0
pmap_release_pmap_lock(pm);
PMAP_MAP_TO_HEAD_UNLOCK();
More information about the p4-projects
mailing list