PERFORCE change 136143 for review
Kip Macy
kmacy at FreeBSD.org
Mon Feb 25 04:17:05 UTC 2008
http://perforce.freebsd.org/chv.cgi?CH=136143
Change 136143 by kmacy at pandemonium:kmacy:xen31 on 2008/02/25 04:16:08
add extensive KTR logging
revert to recursive page tables
eliminate most uses of writable page tables
disable pmap_copy for now (causes SIGBUS)
Affected files ...
.. //depot/projects/xen31/sys/i386/xen/pmap.c#39 edit
Differences ...
==== //depot/projects/xen31/sys/i386/xen/pmap.c#39 (text+ko) ====
@@ -103,6 +103,8 @@
* and to when physical maps must be made correct.
*/
+#define PMAP_DIAGNOSTIC
+
#include "opt_cpu.h"
#include "opt_pmap.h"
#include "opt_msgbuf.h"
@@ -112,6 +114,7 @@
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/kernel.h>
+#include <sys/ktr.h>
#include <sys/lock.h>
#include <sys/malloc.h>
#include <sys/mman.h>
@@ -193,8 +196,6 @@
#define pmap_pte_u(pte) ((*(int *)pte & PG_A) != 0)
#define pmap_pte_v(pte) ((*(int *)pte & PG_V) != 0)
-#define pmap_pte_set_w(pte, v) ((v) ? atomic_set_int((u_int *)(pte), PG_W) : \
- atomic_clear_int((u_int *)(pte), PG_W))
#define pmap_pte_set_prot(pte, v) ((*(int *)pte &= ~PG_PROT), (*(int *)pte |= (v)))
struct pmap kernel_pmap_store;
@@ -310,7 +311,6 @@
-
static __inline void
pagezero(void *page)
{
@@ -331,22 +331,27 @@
pd_set(struct pmap *pmap, int ptepindex, vm_paddr_t val, int type)
{
vm_paddr_t pdir_ma = vtomach(&pmap->pm_pdir[ptepindex]);
- vm_paddr_t shadow_pdir_ma = vtomach(&pmap->pm_pdir_shadow[ptepindex]);
switch (type) {
case SH_PD_SET_VA:
+#if 0
xen_queue_pt_update(shadow_pdir_ma,
xpmap_ptom(val & ~(PG_RW)));
+#endif
xen_queue_pt_update(pdir_ma,
xpmap_ptom(val));
break;
case SH_PD_SET_VA_MA:
+#if 0
xen_queue_pt_update(shadow_pdir_ma,
val & ~(PG_RW));
+#endif
xen_queue_pt_update(pdir_ma, val);
break;
case SH_PD_SET_VA_CLEAR:
+#if 0
xen_queue_pt_update(shadow_pdir_ma, 0);
+#endif
xen_queue_pt_update(pdir_ma, 0);
break;
}
@@ -409,7 +414,6 @@
#ifdef PAE
kernel_pmap->pm_pdpt = (pdpt_entry_t *) (KERNBASE + (u_int)IdlePDPT);
#endif
- kernel_pmap->pm_pdir_shadow = PCPU_GET(pdir_shadow);
kernel_pmap->pm_active = -1; /* don't allow deactivation */
TAILQ_INIT(&kernel_pmap->pm_pvchunk);
LIST_INIT(&allpmaps);
@@ -441,7 +445,7 @@
}
SYSMAP(caddr_t, CMAP1, CADDR1, 1)
SYSMAP(caddr_t, CMAP3, CADDR3, 1)
- *CMAP3 = 0;
+ PT_SET_MA(CADDR3, 0);
/*
* Crashdump maps.
@@ -467,9 +471,8 @@
mtx_init(&PMAP2mutex, "PMAP2", NULL, MTX_DEF);
virtual_avail = va;
+ PT_SET_MA(CADDR1, 0);
- *CMAP1 = 0;
-
/*
* Leave in place an identity mapping (virt == phys) for the low 1 MB
* physical memory region that is used by the ACPI wakeup code. This
@@ -809,6 +812,9 @@
u_int cpumask;
u_int other_cpus;
+ CTR2(KTR_PMAP, "pmap_invalidate_page: pmap=%p va=0x%x",
+ pmap, va);
+
sched_pin();
if (pmap == kernel_pmap || pmap->pm_active == all_cpus) {
invlpg(va);
@@ -832,6 +838,9 @@
u_int other_cpus;
vm_offset_t addr;
+ CTR3(KTR_PMAP, "pmap_invalidate_page: pmap=%p eva=0x%x sva=0x%x",
+ pmap, sva, eva);
+
sched_pin();
if (pmap == kernel_pmap || pmap->pm_active == all_cpus) {
for (addr = sva; addr < eva; addr += PAGE_SIZE)
@@ -857,6 +866,8 @@
u_int cpumask;
u_int other_cpus;
+ CTR1(KTR_PMAP, "pmap_invalidate_page: pmap=%p", pmap);
+
sched_pin();
if (pmap == kernel_pmap || pmap->pm_active == all_cpus) {
invltlb();
@@ -889,6 +900,8 @@
PMAP_INLINE void
pmap_invalidate_page(pmap_t pmap, vm_offset_t va)
{
+ CTR2(KTR_PMAP, "pmap_invalidate_page: pmap=%p va=0x%x",
+ pmap, va);
if (pmap == kernel_pmap || pmap->pm_active)
invlpg(va);
@@ -900,6 +913,10 @@
{
vm_offset_t addr;
+ if (eva - sva > PAGE_SIZE)
+ CTR3(KTR_PMAP, "pmap_invalidate_range: pmap=%p sva=0x%x eva=0x%x",
+ pmap, sva, eva);
+
if (pmap == kernel_pmap || pmap->pm_active)
for (addr = sva; addr < eva; addr += PAGE_SIZE)
invlpg(addr);
@@ -910,6 +927,8 @@
pmap_invalidate_all(pmap_t pmap)
{
+ CTR1(KTR_PMAP, "pmap_invalidate_all: pmap=%p", pmap);
+
if (pmap == kernel_pmap || pmap->pm_active)
invltlb();
}
@@ -933,8 +952,8 @@
{
return (pmap == kernel_pmap ||
- (pmap == vmspace_pmap(curthread->td_proc->p_vmspace) &&
- (pmap->pm_pdir[PTDPTDI] & PG_FRAME) == (PTDpde[0] & PG_FRAME)));
+ (pmap == vmspace_pmap(curthread->td_proc->p_vmspace) &&
+ (pmap->pm_pdir[PTDPTDI] & PG_FRAME) == (PTDpde[0] & PG_FRAME)));
}
/*
@@ -958,9 +977,11 @@
newpf = *pde & PG_FRAME;
if ((*PMAP2 & PG_FRAME) != newpf) {
PT_UPDATES_FLUSH();
- *PMAP2 = newpf | PG_V | PG_A | PG_M;
- pmap_invalidate_page(kernel_pmap, (vm_offset_t)PADDR2);
+ PT_SET_MA(PADDR2, newpf | PG_V | PG_A | PG_M);
+ CTR3(KTR_PMAP, "pmap_pte: pmap=%p va=0x%x newpte=0x%08x",
+ pmap, va, (*PMAP2 & 0xffffffff));
}
+
return (PADDR2 + (i386_btop(va) & (NPTEPG - 1)));
}
return (0);
@@ -975,7 +996,9 @@
{
if ((pt_entry_t *)((vm_offset_t)pte & ~PAGE_MASK) == PADDR2) {
- *PMAP2 = 0;
+ CTR1(KTR_PMAP, "pmap_pte_release: pte=0x%jx",
+ *PMAP2);
+ PT_SET_VA(PMAP2, 0, TRUE);
mtx_unlock(&PMAP2mutex);
}
}
@@ -1016,11 +1039,13 @@
newpf = *pde & PG_FRAME;
if ((*PMAP1 & PG_FRAME) != newpf) {
PT_UPDATES_FLUSH();
- *PMAP1 = newpf | PG_V | PG_A | PG_M;
+ PT_SET_MA(PADDR1, newpf | PG_V | PG_A | PG_M);
+ CTR3(KTR_PMAP, "pmap_pte_quick: pmap=%p va=0x%x newpte=0x%jx",
+ pmap, va, *PMAP1);
+
#ifdef SMP
PMAP1cpu = PCPU_GET(cpuid);
#endif
- invlcaddr(PADDR1);
PMAP1changed++;
} else
#ifdef SMP
@@ -1048,7 +1073,8 @@
vm_paddr_t rtval;
pt_entry_t *pte;
pd_entry_t pde;
-
+ pt_entry_t pteval;
+
rtval = 0;
PMAP_LOCK(pmap);
pde = pmap->pm_pdir[va >> PDRSHIFT];
@@ -1059,7 +1085,8 @@
return rtval;
}
pte = pmap_pte(pmap, va);
- rtval = (PT_GET(pte) & PG_FRAME) | (va & PAGE_MASK);
+ pteval = *pte ? xpmap_mtop(*pte) : 0;
+ rtval = (pteval & PG_FRAME) | (va & PAGE_MASK);
pmap_pte_release(pte);
}
PMAP_UNLOCK(pmap);
@@ -1124,7 +1151,7 @@
sched_pin();
pte = PT_GET(pmap_pte_quick(pmap, va));
if (*PMAP1)
- *PMAP1 = 0;
+ PT_SET_MA(PADDR1, 0);
if ((pte & PG_V) &&
((pte & PG_RW) || (prot & VM_PROT_WRITE) == 0)) {
m = PHYS_TO_VM_PAGE(pte & PG_FRAME);
@@ -1149,10 +1176,7 @@
PMAP_INLINE void
pmap_kenter(vm_offset_t va, vm_paddr_t pa)
{
- pt_entry_t *pte;
-
- pte = vtopte(va);
- pte_store(pte, pa | PG_RW | PG_V | pgeflag);
+ PT_SET_MA(va, xpmap_ptom(pa)| PG_RW | PG_V | pgeflag);
}
PMAP_INLINE void
@@ -1205,6 +1229,8 @@
vm_offset_t va, sva;
va = sva = *virt;
+ CTR4(KTR_PMAP, "pmap_map: va=0x%x start=0x%jx end=0x%jx prot=0x%x",
+ va, start, end, prot);
while (start < end) {
pmap_kenter(va, start);
va += PAGE_SIZE;
@@ -1235,7 +1261,8 @@
multicall_entry_t mcl[16];
multicall_entry_t *mclp = mcl;
int error;
-
+
+ CTR2(KTR_PMAP, "pmap_qenter:sva=0x%x count=%d", va, count);
pte = vtopte(sva);
endpte = pte + count;
while (pte < endpte) {
@@ -1266,7 +1293,7 @@
#ifdef INVARIANTS
for (pte = vtopte(sva), mclcount = 0; mclcount < count; mclcount++, pte++)
- KASSERT(*pte, ("pte not set for va=0x%x\n", sva + mclcount*PAGE_SIZE));
+ KASSERT(*pte, ("pte not set for va=0x%x", sva + mclcount*PAGE_SIZE));
#endif
}
@@ -1281,12 +1308,17 @@
{
vm_offset_t va;
+ CTR2(KTR_PMAP, "pmap_qremove: sva=0x%x count=%d", sva, count);
va = sva;
+ vm_page_lock_queues();
+ critical_enter();
while (count-- > 0) {
pmap_kremove(va);
va += PAGE_SIZE;
}
pmap_invalidate_range(kernel_pmap, sva, va);
+ critical_exit();
+ vm_page_unlock_queues();
}
/***************************************************
@@ -1324,6 +1356,7 @@
{
vm_offset_t pteva;
+ PT_UPDATES_FLUSH();
/*
* unmap the page table page
*/
@@ -1401,8 +1434,8 @@
int
pmap_pinit(pmap_t pmap)
{
- vm_page_t m, ptdpg[NPGPTD*2 + 1];
- int npgptd = NPGPTD*2 + 1;
+ vm_page_t m, ptdpg[NPGPTD + 1];
+ int npgptd = NPGPTD + 1;
static int color;
int i;
@@ -1419,17 +1452,9 @@
PMAP_LOCK_DESTROY(pmap);
return (0);
}
-#ifdef XEN
- pmap->pm_pdir_shadow = (pd_entry_t *)kmem_alloc_nofault(kernel_map,
- NBPTD);
- if (pmap->pm_pdir == NULL) {
- PMAP_LOCK_DESTROY(pmap);
- return (0);
- }
-#ifdef PAE
+#if defined(XEN) && defined(PAE)
pmap->pm_pdpt = (pd_entry_t *)kmem_alloc_nofault(kernel_map, 1);
#endif
-#endif
#if defined(PAE) && !defined(XEN)
pmap->pm_pdpt = uma_zalloc(pdptzone, M_WAITOK | M_ZERO);
@@ -1454,12 +1479,10 @@
ptdpg[i++] = m;
}
}
-
pmap_qenter((vm_offset_t)pmap->pm_pdir, ptdpg, NPGPTD);
-
for (i = 0; i < NPGPTD; i++) {
if ((ptdpg[i]->flags & PG_ZERO) == 0)
- bzero(pmap->pm_pdir + (i * NPDEPG), PAGE_SIZE);
+ pagezero(&pmap->pm_pdir[i*NPTEPG]);
}
mtx_lock_spin(&allpmaps_lock);
@@ -1470,8 +1493,8 @@
bcopy(PTD + KPTDI, pmap->pm_pdir + KPTDI, nkpt * sizeof(pd_entry_t));
#ifdef PAE
#ifdef XEN
- pmap_qenter((vm_offset_t)pmap->pm_pdpt, &ptdpg[NPGPTD*2], 1);
- if ((ptdpg[NPGPTD*2]->flags & PG_ZERO) == 0)
+ pmap_qenter((vm_offset_t)pmap->pm_pdpt, &ptdpg[NPGPTD], 1);
+ if ((ptdpg[NPGPTD]->flags & PG_ZERO) == 0)
bzero(pmap->pm_pdpt, PAGE_SIZE);
#endif
for (i = 0; i < NPGPTD; i++) {
@@ -1483,33 +1506,31 @@
}
#endif
#ifdef XEN
- pmap_qenter((vm_offset_t)pmap->pm_pdir_shadow, &ptdpg[NPGPTD], NPGPTD);
-
- for (i = 0; i < NPGPTD; i++)
- if ((ptdpg[i + NPGPTD]->flags & PG_ZERO) == 0)
- bzero(pmap->pm_pdir_shadow + (i * NPDEPG), PAGE_SIZE);
- for (i = 0; i < NPGPTD; i++)
- pmap->pm_pdir_shadow[PTDPTDI + i] =
- xpmap_ptom(VM_PAGE_TO_PHYS(ptdpg[i])) | PG_V;
- for (i = 0; i < NPGPTD; i++)
- pmap->pm_pdir[PTDPTDI + i] =
- xpmap_ptom(VM_PAGE_TO_PHYS(ptdpg[i + NPGPTD])) | PG_V;
- for (i = KPTDI; i < KPTDI + nkpt; i++)
- pmap->pm_pdir_shadow[i] = PTD[i] & ~(PG_RW);
for (i = 0; i < NPGPTD; i++) {
pt_entry_t *pd;
-
+ vm_paddr_t ma;
+
+ ma = xpmap_ptom(VM_PAGE_TO_PHYS(ptdpg[i]));
pd = pmap->pm_pdir + (i * NPDEPG);
PT_SET_MA(pd, *vtopte((vm_offset_t)pd) & ~(PG_M|PG_A|PG_U|PG_RW));
- pd = pmap->pm_pdir_shadow + (i * NPDEPG);
- PT_SET_MA(pd, *vtopte((vm_offset_t)pd) & ~(PG_M|PG_A|PG_U|PG_RW));
+#if 0
+ xen_pgd_pin(ma);
+#endif
}
+
#ifdef PAE
PT_SET_MA(pmap->pm_pdpt, *vtopte((vm_offset_t)pmap->pm_pdpt) & ~PG_RW);
#endif
+ vm_page_lock_queues();
xen_flush_queue();
+ xen_pgdpt_pin(xpmap_ptom(VM_PAGE_TO_PHYS(ptdpg[NPGPTD])));
+ for (i = 0; i < NPGPTD; i++) {
+ vm_paddr_t ma = xpmap_ptom(VM_PAGE_TO_PHYS(ptdpg[i]));
+ PT_SET_VA_MA(&pmap->pm_pdir[PTDPTDI + i], ma | PG_V | PG_A, FALSE);
+ }
+ xen_flush_queue();
+ vm_page_unlock_queues();
#endif
-
pmap->pm_active = 0;
TAILQ_INIT(&pmap->pm_pvchunk);
bzero(&pmap->pm_stats, sizeof pmap->pm_stats);
@@ -1522,7 +1543,7 @@
* mapped correctly.
*/
static vm_page_t
-_pmap_allocpte(pmap_t pmap, unsigned ptepindex, int flags)
+_pmap_allocpte(pmap_t pmap, unsigned int ptepindex, int flags)
{
vm_paddr_t ptema;
vm_page_t m;
@@ -1557,15 +1578,16 @@
* Map the pagetable page into the process address space, if
* it isn't already there.
*/
-
pmap->pm_stats.resident_count++;
ptema = xpmap_ptom(VM_PAGE_TO_PHYS(m));
xen_pt_pin(ptema);
- PD_SET_VA_MA(pmap, ptepindex,
+ PT_SET_VA_MA(&pmap->pm_pdir[ptepindex],
(ptema | PG_U | PG_RW | PG_V | PG_A | PG_M), TRUE);
-
- return m;
+
+ KASSERT(pmap->pm_pdir[ptepindex],
+ ("_pmap_allocpte: ptepindex=%d did not get mapped", ptepindex));
+ return (m);
}
static vm_page_t
@@ -1615,9 +1637,13 @@
* Here if the pte page isn't mapped, or if it has
* been deallocated.
*/
+ CTR3(KTR_PMAP, "pmap_allocpte: pmap=%p va=0x%08x flags=0x%x",
+ pmap, va, flags);
m = _pmap_allocpte(pmap, ptepindex, flags);
if (m == NULL && (flags & M_WAITOK))
goto retry;
+
+ KASSERT(pmap->pm_pdir[ptepindex], ("ptepindex=%d did not get mapped", ptepindex));
}
return (m);
}
@@ -1733,9 +1759,9 @@
int i;
#ifdef XEN
#ifdef PAE
- int npgptd = 2*NPGPTD + 1;
+ int npgptd = NPGPTD + 1;
#else
- int npgptd = 2*NPGPTD;
+ int npgptd = NPGPTD;
#endif
#else
int npgptd = NPGPTD;
@@ -1750,16 +1776,11 @@
LIST_REMOVE(pmap, pm_list);
mtx_unlock_spin(&allpmaps_lock);
-#ifdef XEN
for (i = 0; i < NPGPTD; i++)
- ptdpg[i] = PHYS_TO_VM_PAGE(vtophys(pmap->pm_pdir_shadow + (i*NPDEPG)) & PG_FRAME);
- pmap_qremove((vm_offset_t)pmap->pm_pdir_shadow, NPGPTD);
-#endif
- for (i = 0; i < NPGPTD; i++)
- ptdpg[NPGPTD + i] = PHYS_TO_VM_PAGE(vtophys(pmap->pm_pdir + (i*NPDEPG)) & PG_FRAME);
+ ptdpg[i] = PHYS_TO_VM_PAGE(vtophys(pmap->pm_pdir + (i*NPDEPG)) & PG_FRAME);
pmap_qremove((vm_offset_t)pmap->pm_pdir, NPGPTD);
#if defined(PAE) && defined(XEN)
- ptdpg[2*NPGPTD] = PHYS_TO_VM_PAGE(vtophys(pmap->pm_pdpt));
+ ptdpg[NPGPTD] = PHYS_TO_VM_PAGE(vtophys(pmap->pm_pdpt));
#endif
for (i = 0; i < npgptd; i++) {
@@ -1852,9 +1873,8 @@
mtx_lock_spin(&allpmaps_lock);
LIST_FOREACH(pmap, &allpmaps, pm_list)
- PD_SET_VA(pmap, (kernel_vm_end >> PDRSHIFT), newpdir, FALSE);
+ PD_SET_VA(pmap, (kernel_vm_end >> PDRSHIFT), newpdir, TRUE);
- PT_UPDATES_FLUSH();
mtx_unlock_spin(&allpmaps_lock);
kernel_vm_end = (kernel_vm_end + PAGE_SIZE * NPTEPG) & ~(PAGE_SIZE * NPTEPG - 1);
if (kernel_vm_end - 1 >= kernel_map->max_offset) {
@@ -2176,9 +2196,13 @@
pt_entry_t oldpte;
vm_page_t m;
+ CTR3(KTR_PMAP, "pmap_remove_pte: pmap=%p *ptq=0x%jx va=0x%x",
+ pmap, *ptq, va);
+
mtx_assert(&vm_page_queue_mtx, MA_OWNED);
PMAP_LOCK_ASSERT(pmap, MA_OWNED);
- oldpte = pte_load_clear(ptq);
+ oldpte = *ptq;
+ PT_SET_VA_MA(ptq, 0, TRUE);
if (oldpte & PG_W)
pmap->pm_stats.wired_count -= 1;
/*
@@ -2194,10 +2218,7 @@
* why, but I think the most likely explanation is that xen's writable
* page table implementation doesn't respect the unused bits.
*/
- if ((oldpte & PG_MANAGED)
-#ifdef PTEBROKEN
- || ((oldpte & PG_V) && (va < VM_MAXUSER_ADDRESS))
-#endif
+ if ((oldpte & PG_MANAGED) || ((oldpte & PG_V) && (va < VM_MAXUSER_ADDRESS))
) {
m = PHYS_TO_VM_PAGE(xpmap_mtop(oldpte) & PG_FRAME);
@@ -2227,6 +2248,9 @@
{
pt_entry_t *pte;
+ CTR2(KTR_PMAP, "pmap_remove_page: pmap=%p va=0x%x",
+ pmap, va);
+
mtx_assert(&vm_page_queue_mtx, MA_OWNED);
KASSERT(curthread->td_pinned > 0, ("curthread not pinned"));
PMAP_LOCK_ASSERT(pmap, MA_OWNED);
@@ -2235,7 +2259,7 @@
pmap_remove_pte(pmap, pte, va, free);
pmap_invalidate_page(pmap, va);
if (*PMAP1)
- *PMAP1 = 0;
+ PT_SET_MA(PADDR1, 0);
}
@@ -2253,7 +2277,10 @@
pt_entry_t *pte;
vm_page_t free = NULL;
int anyvalid;
-
+
+ CTR3(KTR_PMAP, "pmap_remove: pmap=%p sva=0x%x eva=0x%x",
+ pmap, sva, eva);
+
/*
* Perform an unsynchronized read. This is, however, safe.
*/
@@ -2330,12 +2357,13 @@
break;
}
}
+ PT_UPDATES_FLUSH();
if (*PMAP1)
- *PMAP1 = 0;
+ PT_SET_VA_MA(PMAP1, 0, TRUE);
out:
- sched_unpin();
if (anyvalid)
pmap_invalidate_all(pmap);
+ sched_unpin();
vm_page_unlock_queues();
PMAP_UNLOCK(pmap);
pmap_free_zero_pages(free);
@@ -2367,8 +2395,8 @@
* XXX This makes pmap_remove_all() illegal for non-managed pages!
*/
if (m->flags & PG_FICTITIOUS) {
- panic("pmap_remove_all: illegal for unmanaged page, va: 0x%x",
- VM_PAGE_TO_PHYS(m));
+ panic("pmap_remove_all: illegal for unmanaged page, va: 0x%jx",
+ VM_PAGE_TO_PHYS(m) & 0xffffffff);
}
#endif
mtx_assert(&vm_page_queue_mtx, MA_OWNED);
@@ -2378,7 +2406,9 @@
PMAP_LOCK(pmap);
pmap->pm_stats.resident_count--;
pte = pmap_pte_quick(pmap, pv->pv_va);
- tpte = pte_load_clear(pte);
+
+ tpte = *pte;
+ PT_SET_VA_MA(pte, 0, TRUE);
if (tpte & PG_W)
pmap->pm_stats.wired_count--;
if (tpte & PG_A)
@@ -2402,8 +2432,9 @@
PMAP_UNLOCK(pmap);
}
vm_page_flag_clear(m, PG_WRITEABLE);
+ PT_UPDATES_FLUSH();
if (*PMAP1)
- *PMAP1 = 0;
+ PT_SET_MA(PADDR1, 0);
sched_unpin();
}
@@ -2419,6 +2450,9 @@
pt_entry_t *pte;
int anychanged;
+ CTR4(KTR_PMAP, "pmap_protect: pmap=%p sva=0x%x eva=0x%x prot=0x%x",
+ pmap, sva, eva, prot);
+
if ((prot & VM_PROT_READ) == VM_PROT_NONE) {
pmap_remove(pmap, sva, eva);
return;
@@ -2506,6 +2540,12 @@
#endif
if (pbits != obits) {
+#ifdef XEN
+ obits = *pte;
+ PT_SET_VA_MA(pte, pbits, TRUE);
+ if (*pte != pbits)
+ goto retry;
+#else
#ifdef PAE
if (!atomic_cmpset_64(pte, obits, pbits))
goto retry;
@@ -2514,6 +2554,7 @@
pbits))
goto retry;
#endif
+#endif
if (obits & PG_G)
pmap_invalidate_page(pmap, sva);
else
@@ -2521,11 +2562,12 @@
}
}
}
+ PT_UPDATES_FLUSH();
if (*PMAP1)
- *PMAP1 = 0;
- sched_unpin();
+ PT_SET_VA_MA(PMAP1, 0, TRUE);
if (anychanged)
pmap_invalidate_all(pmap);
+ sched_unpin();
vm_page_unlock_queues();
PMAP_UNLOCK(pmap);
}
@@ -2554,6 +2596,8 @@
vm_page_t mpte, om;
boolean_t invlva;
+ CTR6(KTR_PMAP, "pmap_enter: pmap=%08p va=0x%08x access=0x%x m=%p prot=0x%x wired=%d",
+ pmap, va, access, m, prot, wired);
va = trunc_page(va);
#ifdef PMAP_DIAGNOSTIC
if (va > VM_MAX_KERNEL_ADDRESS)
@@ -2596,7 +2640,7 @@
*/
if (pte == NULL) {
panic("pmap_enter: invalid page directory pdir=%#jx, va=%#x\n",
- (uintmax_t)pmap->pm_pdir[PTDPTDI], va);
+ (uintmax_t)pmap->pm_pdir[va >> PDRSHIFT], va);
}
pa = VM_PAGE_TO_PHYS(m);
@@ -2606,11 +2650,12 @@
#if 0
KASSERT((*pte & PG_V) || (*pte == 0), ("address set but not valid pte=%p *pte=0x%016jx",
pte, *pte));
-#endif
- origpte = PT_GET(pte);
+#endif
+ origpte = *pte;
+ if (origpte)
+ origpte = xpmap_mtop(origpte);
opa = origpte & PG_FRAME;
-
/*
* Mapping has not changed, must be protection or wiring change.
*/
@@ -2700,6 +2745,7 @@
if (pmap == kernel_pmap)
newpte |= pgeflag;
+ critical_enter();
/*
* if the mapping or permission bits are different, we need
* to update the pte.
@@ -2707,7 +2753,8 @@
if ((origpte & ~(PG_M|PG_A)) != newpte) {
if (origpte) {
invlva = FALSE;
- origpte = pte_load_store(pte, newpte | PG_A);
+ origpte = *pte;
+ PT_SET_VA(pte, newpte | PG_A, FALSE);
if (origpte & PG_A) {
if (origpte & PG_MANAGED)
vm_page_flag_set(om, PG_REFERENCED);
@@ -2730,14 +2777,17 @@
}
if (invlva)
pmap_invalidate_page(pmap, va);
- } else
- pte_store(pte, newpte | PG_A);
+ } else{
+ PT_SET_VA(pte, newpte | PG_A, FALSE);
+ }
+
}
+ PT_UPDATES_FLUSH();
+ critical_exit();
if (*PMAP1)
- *PMAP1 = 0;
+ PT_SET_VA_MA(PMAP1, 0, TRUE);
sched_unpin();
vm_page_unlock_queues();
- KASSERT(*vtopte(va), ("pte not set for va=0x%x\n", va));
PMAP_UNLOCK(pmap);
}
@@ -2804,6 +2854,9 @@
int count = 0;
mclp = &mcl;
+ CTR4(KTR_PMAP, "pmap_enter_quick: pmap=%p va=0x%x m=%p prot=0x%x",
+ pmap, va, m, prot);
+
PMAP_LOCK(pmap);
(void) pmap_enter_quick_locked(&mclp, &count, pmap, va, m, prot, NULL);
if (count)
@@ -2900,6 +2953,7 @@
* quick entry into any pmap, one would likely use pmap_pte_quick.
* But that isn't as quick as vtopte.
*/
+ KASSERT(pmap_is_current(pmap), ("entering pages in non-current pmap"));
pte = vtopte(va);
if (*pte & PG_V) {
if (mpte != NULL) {
@@ -3040,6 +3094,7 @@
PMAP_LOCK(pmap);
pmap->pm_stats.resident_count += size >> PAGE_SHIFT;
npdes = size >> PDRSHIFT;
+ critical_enter();
for(i = 0; i < npdes; i++) {
PD_SET_VA(pmap, ptepindex,
ptepa | PG_U | PG_M | PG_RW | PG_V | PG_PS, FALSE);
@@ -3047,6 +3102,7 @@
ptepindex += 1;
}
pmap_invalidate_all(pmap);
+ critical_exit();
out:
PMAP_UNLOCK(pmap);
}
@@ -3064,21 +3120,25 @@
{
pt_entry_t *pte;
+ vm_page_lock_queues();
PMAP_LOCK(pmap);
pte = pmap_pte(pmap, va);
- if (wired && !pmap_pte_w(pte))
+ if (wired && !pmap_pte_w(pte)) {
+ PT_SET_VA_MA((pte), *(pte) | PG_W, TRUE);
pmap->pm_stats.wired_count++;
- else if (!wired && pmap_pte_w(pte))
+ } else if (!wired && pmap_pte_w(pte)) {
+ PT_SET_VA_MA((pte), *(pte) & ~PG_W, TRUE);
pmap->pm_stats.wired_count--;
-
+ }
+
/*
* Wiring is not a hardware characteristic so there is no need to
* invalidate TLB.
*/
- pmap_pte_set_w(pte, wired);
pmap_pte_release(pte);
PMAP_UNLOCK(pmap);
+ vm_page_unlock_queues();
}
@@ -3100,12 +3160,21 @@
vm_offset_t end_addr = src_addr + len;
vm_offset_t pdnxt;
+ return;
+
if (dst_addr != src_addr)
return;
- if (!pmap_is_current(src_pmap))
+ if (!pmap_is_current(src_pmap)) {
+ CTR2(KTR_PMAP,
+ "pmap_copy, skipping: pdir[PTDPTDI]=0x%jx PTDpde[0]=0x%jx",
+ (src_pmap->pm_pdir[PTDPTDI] & PG_FRAME), (PTDpde[0] & PG_FRAME));
+
return;
-
+ }
+ CTR5(KTR_PMAP, "pmap_copy: dst_pmap=%p src_pmap=%p dst_addr=0x%x len=%d src_addr=0x%x",
+ dst_pmap, src_pmap, dst_addr, len, src_addr);
+
vm_page_lock_queues();
if (dst_pmap < src_pmap) {
PMAP_LOCK(dst_pmap);
@@ -3168,11 +3237,7 @@
* accessed (referenced) bits
* during the copy.
*/
-#if 0
PT_SET_VA_MA(dst_pte, ptetemp & ~(PG_W | PG_M | PG_A), FALSE);
-#else
- *dst_pte = ptetemp & ~(PG_W | PG_M | PG_A);
-#endif
dst_pmap->pm_stats.resident_count++;
} else {
free = NULL;
@@ -3190,9 +3255,9 @@
src_pte++;
}
}
+ PT_UPDATES_FLUSH();
if (*PMAP1)
- *PMAP1 = 0;
- PT_UPDATES_FLUSH();
+ PT_SET_VA_MA(PMAP1, 0, TRUE);
sched_unpin();
vm_page_unlock_queues();
PMAP_UNLOCK(src_pmap);
@@ -3213,10 +3278,9 @@
if (*sysmaps->CMAP2)
panic("pmap_zero_page: CMAP2 busy");
sched_pin();
- *sysmaps->CMAP2 = PG_V | PG_RW | xpmap_ptom(VM_PAGE_TO_PHYS(m)) | PG_A | PG_M;
- invlcaddr(sysmaps->CADDR2);
+ PT_SET_MA(sysmaps->CADDR2, PG_V | PG_RW | xpmap_ptom(VM_PAGE_TO_PHYS(m)) | PG_A | PG_M);
pagezero(sysmaps->CADDR2);
- *sysmaps->CMAP2 = 0;
+ PT_SET_MA(sysmaps->CADDR2, 0);
sched_unpin();
mtx_unlock(&sysmaps->lock);
}
@@ -3237,14 +3301,13 @@
if (*sysmaps->CMAP2)
panic("pmap_zero_page: CMAP2 busy");
sched_pin();
- *sysmaps->CMAP2 = PG_V | PG_RW | xpmap_ptom(VM_PAGE_TO_PHYS(m)) | PG_A | PG_M;
+ PT_SET_MA(sysmaps->CADDR2, PG_V | PG_RW | xpmap_ptom(VM_PAGE_TO_PHYS(m)) | PG_A | PG_M);
- invlcaddr(sysmaps->CADDR2);
if (off == 0 && size == PAGE_SIZE)
pagezero(sysmaps->CADDR2);
else
bzero((char *)sysmaps->CADDR2 + off, size);
- *sysmaps->CMAP2 = 0;
+ PT_SET_MA(sysmaps->CADDR2, 0);
sched_unpin();
mtx_unlock(&sysmaps->lock);
}
@@ -3262,10 +3325,9 @@
if (*CMAP3)
panic("pmap_zero_page: CMAP3 busy");
sched_pin();
- *CMAP3 = PG_V | PG_RW | xpmap_ptom(VM_PAGE_TO_PHYS(m)) | PG_A | PG_M;
- invlcaddr(CADDR3);
+ PT_SET_MA(CADDR3, PG_V | PG_RW | xpmap_ptom(VM_PAGE_TO_PHYS(m)) | PG_A | PG_M);
pagezero(CADDR3);
- *CMAP3 = 0;
+ PT_SET_MA(CADDR3, 0);
sched_unpin();
}
@@ -3287,13 +3349,11 @@
if (*sysmaps->CMAP2)
panic("pmap_copy_page: CMAP2 busy");
sched_pin();
- invlpg((u_int)sysmaps->CADDR1);
- invlpg((u_int)sysmaps->CADDR2);
- *sysmaps->CMAP1 = PG_V | xpmap_ptom(VM_PAGE_TO_PHYS(src)) | PG_A;
- *sysmaps->CMAP2 = PG_V | PG_RW | xpmap_ptom(VM_PAGE_TO_PHYS(dst)) | PG_A | PG_M;
+ PT_SET_MA(sysmaps->CADDR1, PG_V | xpmap_ptom(VM_PAGE_TO_PHYS(src)) | PG_A);
+ PT_SET_MA(sysmaps->CADDR2, PG_V | PG_RW | xpmap_ptom(VM_PAGE_TO_PHYS(dst)) | PG_A | PG_M);
bcopy(sysmaps->CADDR1, sysmaps->CADDR2, PAGE_SIZE);
- *sysmaps->CMAP1 = 0;
- *sysmaps->CMAP2 = 0;
+ PT_SET_MA(sysmaps->CADDR1, 0);
+ PT_SET_MA(sysmaps->CADDR2, 0);
sched_unpin();
mtx_unlock(&sysmaps->lock);
}
@@ -3312,7 +3372,7 @@
int loops = 0;
if (m->flags & PG_FICTITIOUS)
- return FALSE;
+ return (FALSE);
mtx_assert(&vm_page_queue_mtx, MA_OWNED);
TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
@@ -3377,11 +3437,14 @@
uint32_t inuse, bitmask;
int allfree;
+ CTR1(KTR_PMAP, "pmap_remove_pages: pmap=%p", pmap);
+
if (pmap != vmspace_pmap(curthread->td_proc->p_vmspace)) {
printf("warning: pmap_remove_pages called with non-current pmap\n");
return;
}
vm_page_lock_queues();
+ KASSERT(pmap_is_current(pmap), ("removing pages from non-current pmap"));
PMAP_LOCK(pmap);
sched_pin();
TAILQ_FOREACH_SAFE(pc, &pmap->pm_pvchunk, pc_list, npc) {
@@ -3395,19 +3458,8 @@
pv = &pc->pc_pventry[idx];
inuse &= ~bitmask;
- if (pmap->pm_pdir_shadow[pv->pv_va >> PDRSHIFT] == 0) {
- printf("PDIR IS ZERO @ VA %08x\n", pv->pv_va);
- /* workaround insufficient wired count
- * on page directory - this only buys
- * us a little bit of time as the list
- * on one of the pages is now corrupt
- */
- allfree = 0;
- goto skip;
- }
-
pte = vtopte(pv->pv_va);
- tpte = PT_GET(pte);
+ tpte = *pte ? xpmap_mtop(*pte) : 0;
if (tpte == 0) {
printf(
@@ -3448,7 +3500,6 @@
vm_page_flag_clear(m, PG_WRITEABLE);
pmap_unuse_pt(pmap, pv->pv_va, &free);
- skip:
/* Mark free */
PV_STAT(pv_entry_frees++);
@@ -3471,8 +3522,10 @@
pmap_ptelist_free(&pv_vafree, (vm_offset_t)pc);
}
}
- if (*PMAP1)
- *PMAP1 = 0;
+ PT_UPDATES_FLUSH();
+ if (*PMAP1)
+ PT_SET_MA(PADDR1, 0);
+
sched_unpin();
pmap_invalidate_all(pmap);
vm_page_unlock_queues();
@@ -3510,7 +3563,7 @@
break;
}
if (*PMAP1)
- *PMAP1 = 0;
+ PT_SET_MA(PADDR1, 0);
sched_unpin();
return (rv);
}
@@ -3529,9 +3582,9 @@
return (rv);
- if (*pmap_pde(pmap, addr)) {
+ if (pmap_is_current(pmap) && *pmap_pde(pmap, addr)) {
pte = vtopte(addr);
- rv = ((*pte & PG_V) == 0);
+ rv = (*pte == 0);
}
return (rv);
}
@@ -3610,8 +3663,9 @@
PMAP_UNLOCK(pmap);
}
vm_page_flag_clear(m, PG_WRITEABLE);
+ PT_UPDATES_FLUSH();
if (*PMAP1)
- *PMAP1 = 0;
+ PT_SET_MA(PADDR1, 0);
sched_unpin();
}
@@ -3658,8 +3712,10 @@
>>> TRUNCATED FOR MAIL (1000 lines) <<<
More information about the p4-projects
mailing list