PERFORCE change 129014 for review
Kip Macy
kmacy at FreeBSD.org
Mon Nov 12 23:54:16 PST 2007
http://perforce.freebsd.org/chv.cgi?CH=129014
Change 129014 by kmacy at entropy_kmacy_xen31 on 2007/11/13 07:54:05
first cut at integrating xen support into latest pmap.c
Affected files ...
.. //depot/projects/xen31/sys/i386/xen/pmap.c#2 edit
Differences ...
==== //depot/projects/xen31/sys/i386/xen/pmap.c#2 (text+ko) ====
@@ -150,6 +150,9 @@
#include <machine/xbox.h>
#endif
+#include <machine/xen/xenvar.h>
+#include <machine/xen/xenfunc.h>
+
#if !defined(CPU_DISABLE_SSE) && defined(I686_CPU)
#define CPU_ENABLE_SSE
#endif
@@ -187,8 +190,12 @@
#define pmap_pte_u(pte) ((*(int *)pte & PG_A) != 0)
#define pmap_pte_v(pte) ((*(int *)pte & PG_V) != 0)
-#define pmap_pte_set_w(pte, v) ((v) ? atomic_set_int((u_int *)(pte), PG_W) : \
- atomic_clear_int((u_int *)(pte), PG_W))
+#define pmap_pte_set_w(pte, v) { \
+ if (v) \
+ PT_SET_VA_MA(pte, *pte | PG_W, TRUE); \
+ else \
+ PT_SET_VA_MA(pte, *pte & ~PG_W, TRUE); \
+}
#define pmap_pte_set_prot(pte, v) ((*(int *)pte &= ~PG_PROT), (*(int *)pte |= (v)))
struct pmap kernel_pmap_store;
@@ -248,6 +255,7 @@
static pt_entry_t *PMAP1 = 0, *PMAP2;
static pt_entry_t *PADDR1 = 0, *PADDR2;
#ifdef SMP
+static int PMAP1_inuse = 0, PMAP2_inuse = 0;
static int PMAP1cpu;
static int PMAP1changedcpu;
SYSCTL_INT(_debug, OID_AUTO, PMAP1changedcpu, CTLFLAG_RD,
@@ -287,6 +295,8 @@
static void pmap_pte_release(pt_entry_t *pte);
static int pmap_unuse_pt(pmap_t, vm_offset_t, vm_page_t *);
static vm_offset_t pmap_kmem_choose(vm_offset_t addr);
+static void pmap_copy_ma_range(vm_paddr_t src, vm_paddr_t dst, uint32_t soff, uint32_t eoff);
+
#ifdef PAE
static void *pmap_pdpt_allocf(uma_zone_t zone, int bytes, u_int8_t *flags, int wait);
#endif
@@ -294,6 +304,31 @@
CTASSERT(1 << PDESHIFT == sizeof(pd_entry_t));
CTASSERT(1 << PTESHIFT == sizeof(pt_entry_t));
+void
+pd_set(struct pmap *pmap, vm_paddr_t *ptr, vm_paddr_t val, int type)
+{
+ vm_paddr_t shadow_pdir_ma = pmap->pm_pdir[PTDPTDI] & ~0xFFF;
+ vm_paddr_t shadow_offset = (vm_paddr_t)(ptr - pmap->pm_pdir)*sizeof(vm_paddr_t);
+
+ switch (type) {
+ case SH_PD_SET_VA:
+ xen_queue_pt_update(shadow_pdir_ma + shadow_offset,
+ xpmap_ptom(val & ~(PG_RW|PG_M)));
+ xen_queue_pt_update(vtomach(ptr),
+ xpmap_ptom(val));
+ break;
+ case SH_PD_SET_VA_MA:
+ xen_queue_pt_update(shadow_pdir_ma + shadow_offset,
+ val & ~(PG_RW|PG_M));
+ xen_queue_pt_update(vtomach(ptr), val);
+ break;
+ case SH_PD_SET_VA_CLEAR:
+ xen_queue_pt_update(shadow_pdir_ma + shadow_offset, 0);
+ xen_queue_pt_update(vtomach(ptr), 0);
+ break;
+ }
+}
+
/*
* Move the kernel virtual free pointer to the next
* 4MB. This is used to help improve performance
@@ -382,7 +417,7 @@
}
SYSMAP(caddr_t, CMAP1, CADDR1, 1)
SYSMAP(caddr_t, CMAP3, CADDR3, 1)
- *CMAP3 = 0;
+ PT_CLEAR_VA(CMAP3, TRUE);
/*
* Crashdump maps.
@@ -408,14 +443,17 @@
mtx_init(&PMAP2mutex, "PMAP2", NULL, MTX_DEF);
virtual_avail = va;
-
- *CMAP1 = 0;
-
+ PT_CLEAR_VA(CMAP1, TRUE);
+
/*
* Leave in place an identity mapping (virt == phys) for the low 1 MB
* physical memory region that is used by the ACPI wakeup code. This
* mapping must not have PG_G set.
*/
+#ifndef XEN
+ /*
+ * leave here deliberately to show that this is not supported
+ */
#ifdef XBOX
/* FIXME: This is gross, but needed for the XBOX. Since we are in such
* an early stadium, we cannot yet neatly map video memory ... :-(
@@ -430,6 +468,7 @@
/* Turn on PG_G on kernel page(s) */
pmap_set_pg();
+#endif
}
/*
@@ -555,6 +594,11 @@
pt_entry_t *pte;
vm_offset_t va;
+ panic("IMPLEMENT XXX");
+
+
+
+
va = *head;
if (va == 0)
return (va); /* Out of memory */
@@ -754,6 +798,7 @@
smp_masked_invlpg(pmap->pm_active & other_cpus, va);
}
sched_unpin();
+ PT_UPDATES_FLUSH();
}
void
@@ -779,6 +824,7 @@
sva, eva);
}
sched_unpin();
+ PT_UPDATES_FLUSH();
}
void
@@ -822,6 +868,7 @@
if (pmap == kernel_pmap || pmap->pm_active)
invlpg(va);
+ PT_UPDATES_FLUSH();
}
PMAP_INLINE void
@@ -832,6 +879,7 @@
if (pmap == kernel_pmap || pmap->pm_active)
for (addr = sva; addr < eva; addr += PAGE_SIZE)
invlpg(addr);
+ PT_UPDATES_FLUSH();
}
PMAP_INLINE void
@@ -874,18 +922,22 @@
{
pd_entry_t newpf;
pd_entry_t *pde;
-
+ pd_entry_t tmppf;
+
pde = pmap_pde(pmap, va);
- if (*pde & PG_PS)
+ if (PT_GET(pde) & PG_PS)
return (pde);
- if (*pde != 0) {
+ if (PT_GET(pde) != 0) {
/* are we current address space or kernel? */
if (pmap_is_current(pmap))
return (vtopte(va));
mtx_lock(&PMAP2mutex);
- newpf = *pde & PG_FRAME;
- if ((*PMAP2 & PG_FRAME) != newpf) {
- *PMAP2 = newpf | PG_RW | PG_V | PG_A | PG_M;
+ newpf = PT_GET(pde) & PG_FRAME;
+ tmppf = PT_GET(PMAP2) & PG_FRAME;
+
+ if (tmppf != newpf) {
+ PT_SET_VA(PMAP2, newpf | PG_V | PG_A | PG_M/* XXX does PG_M cause problems? */, TRUE);
+
pmap_invalidate_page(kernel_pmap, (vm_offset_t)PADDR2);
}
return (PADDR2 + (i386_btop(va) & (NPTEPG - 1)));
@@ -910,6 +962,7 @@
{
invlpg((u_int)caddr);
+ PT_UPDATES_FLUSH();
}
/*
@@ -927,7 +980,8 @@
{
pd_entry_t newpf;
pd_entry_t *pde;
-
+ pd_entry_t tmppf;
+
pde = pmap_pde(pmap, va);
if (*pde & PG_PS)
return (pde);
@@ -937,9 +991,11 @@
return (vtopte(va));
mtx_assert(&vm_page_queue_mtx, MA_OWNED);
KASSERT(curthread->td_pinned > 0, ("curthread not pinned"));
- newpf = *pde & PG_FRAME;
- if ((*PMAP1 & PG_FRAME) != newpf) {
- *PMAP1 = newpf | PG_RW | PG_V | PG_A | PG_M;
+
+ newpf = PT_GET(pde) & PG_FRAME;
+ tmppf = PT_GET(PMAP1) & PG_FRAME;
+ if (tmppf != newpf) {
+ PT_SET_VA(PMAP1, newpf | PG_V | PG_A | PG_M/* ??? */, TRUE);
#ifdef SMP
PMAP1cpu = PCPU_GET(cpuid);
#endif
@@ -974,16 +1030,55 @@
rtval = 0;
PMAP_LOCK(pmap);
+ pde = PT_GET(&pmap->pm_pdir[va >> PDRSHIFT]);
+ if (pde != 0) {
+ if ((pde & PG_PS) != 0) {
+ rtval = (pde & PG_PS_FRAME) | (va & PDRMASK);
+ PMAP_UNLOCK(pmap);
+ return rtval;
+ }
+ pte = pmap_pte(pmap, va);
+ rtval = (PT_GET(pte) & PG_FRAME) | (va & PAGE_MASK);
+ pmap_pte_release(pte);
+ if (PMAP2_inuse) {
+ PT_CLEAR_VA(PMAP2, TRUE);
+ PMAP2_inuse = 0;
+ }
+ }
+ PMAP_UNLOCK(pmap);
+ return (rtval);
+}
+
+/*
+ * Routine: pmap_extract_ma
+ * Function:
+ * Like pmap_extract, but returns machine address
+ */
+vm_paddr_t
+pmap_extract_ma(pmap_t pmap, vm_offset_t va)
+{
+ vm_paddr_t rtval;
+ pt_entry_t *pte;
+ pd_entry_t pde;
+
+ rtval = 0;
+ PMAP_LOCK(pmap);
pde = pmap->pm_pdir[va >> PDRSHIFT];
if (pde != 0) {
if ((pde & PG_PS) != 0) {
- rtval = (pde & PG_PS_FRAME) | (va & PDRMASK);
+ rtval = (pde & ~PDRMASK) | (va & PDRMASK);
PMAP_UNLOCK(pmap);
return rtval;
}
pte = pmap_pte(pmap, va);
rtval = (*pte & PG_FRAME) | (va & PAGE_MASK);
pmap_pte_release(pte);
+#ifdef XEN
+ if (PMAP2_inuse) {
+ PT_CLEAR_VA(PMAP2, TRUE);
+ PMAP2_inuse = 0;
+ }
+#endif
}
PMAP_UNLOCK(pmap);
return (rtval);
@@ -1006,7 +1101,7 @@
m = NULL;
vm_page_lock_queues();
PMAP_LOCK(pmap);
- pde = *pmap_pde(pmap, va);
+ pde = PT_GET(pmap_pde(pmap, va));
if (pde != 0) {
if (pde & PG_PS) {
if ((pde & PG_RW) || (prot & VM_PROT_WRITE) == 0) {
@@ -1016,7 +1111,11 @@
}
} else {
sched_pin();
- pte = *pmap_pte_quick(pmap, va);
+ pte = PT_GET(pmap_pte_quick(pmap, va));
+ if (PMAP1_inuse) {
+ PT_CLEAR_VA(PMAP1, TRUE);
+ PMAP1_inuse = 0;
+ }
if (pte != 0 &&
((pte & PG_RW) || (prot & VM_PROT_WRITE) == 0)) {
m = PHYS_TO_VM_PAGE(pte & PG_FRAME);
@@ -1048,6 +1147,16 @@
}
PMAP_INLINE void
+pmap_kenter_ma(vm_offset_t va, vm_paddr_t ma)
+{
+ pt_entry_t *pte;
+
+ pte = vtopte(va);
+ PT_SET_VA_MA(pte, ma | PG_RW | PG_V | pgeflag, TRUE);
+}
+
+
+PMAP_INLINE void
pmap_kenter_attr(vm_offset_t va, vm_paddr_t pa, int mode)
{
pt_entry_t *pte;
@@ -1182,7 +1291,8 @@
/*
* unmap the page table page
*/
- pmap->pm_pdir[m->pindex] = 0;
+ xen_pt_unpin(pmap->pm_pdir[m->pindex]);
+ PD_CLEAR_VA(pmap, &pmap->pm_pdir[m->pindex], TRUE);
--pmap->pm_stats.resident_count;
/*
@@ -1221,7 +1331,7 @@
if (va >= VM_MAXUSER_ADDRESS)
return 0;
- ptepde = *pmap_pde(pmap, va);
+ ptepde = PT_GET(pmap_pde(pmap, va));
mpte = PHYS_TO_VM_PAGE(ptepde & PG_FRAME);
return pmap_unwire_pte_hold(pmap, mpte, free);
}
@@ -1251,8 +1361,9 @@
int
pmap_pinit(pmap_t pmap)
{
- vm_page_t m, ptdpg[NPGPTD];
- vm_paddr_t pa;
+ vm_page_t m, ptdpg[NPGPTD*2];
+ vm_paddr_t ma, ma_shadow;
+ int npgptd = NPGPTD*2;
static int color;
int i;
@@ -1295,30 +1406,53 @@
}
pmap_qenter((vm_offset_t)pmap->pm_pdir, ptdpg, NPGPTD);
-
+#ifndef XEN
for (i = 0; i < NPGPTD; i++) {
if ((ptdpg[i]->flags & PG_ZERO) == 0)
bzero(pmap->pm_pdir + (i * NPDEPG), PAGE_SIZE);
}
-
+#endif
mtx_lock_spin(&allpmaps_lock);
LIST_INSERT_HEAD(&allpmaps, pmap, pm_list);
mtx_unlock_spin(&allpmaps_lock);
/* Wire in kernel global address entries. */
/* XXX copies current process, does not fill in MPPTDI */
- bcopy(PTD + KPTDI, pmap->pm_pdir + KPTDI, nkpt * sizeof(pd_entry_t));
+ bcopy(kernel_pmap->pm_pdir + KPTDI, pmap->pm_pdir + KPTDI,
+ nkpt * sizeof(pd_entry_t));
+ /* XXX need to copy global address entries to page directory's L1 shadow */
+ ma = xpmap_ptom(VM_PAGE_TO_PHYS(ptdpg[NPGPTD]));
+ /* L1 pin shadow page director{y,ies} */
+ for (i = 0; i < NPGPTD; i++) {
+ ma = xpmap_ptom(VM_PAGE_TO_PHYS(ptdpg[NPGPTD + i]));
+ pmap_copy_ma_range(kernel_pmap->pm_pdir[PTDPTDI + i] & ~(PG_RW|PG_M), ma,
+ KPTDI*sizeof(pd_entry_t), nkpt*sizeof(pd_entry_t));
+ xen_pt_pin(ma);
+ }
+
#ifdef SMP
- pmap->pm_pdir[MPPTDI] = PTD[MPPTDI];
+ pmap->pm_pdir[MPPTDI] = kernel_pmap->pm_pdir[MPPTDI];
#endif
- /* install self-referential address mapping entry(s) */
+ /* pin and install L1 shadow */
for (i = 0; i < NPGPTD; i++) {
- pa = VM_PAGE_TO_PHYS(ptdpg[i]);
- pmap->pm_pdir[PTDPTDI + i] = pa | PG_V | PG_RW | PG_A | PG_M;
+ ma = xpmap_ptom(VM_PAGE_TO_PHYS(ptdpg[i]));
+ ma_shadow = xpmap_ptom(VM_PAGE_TO_PHYS(ptdpg[NPGPTD+i]));
+ /* re-map page directory read-only and pin */
+ PT_SET_MA(pmap->pm_pdir + i*PAGE_SIZE,
+ (ma | PG_V | PG_A));
+ xen_pgd_pin(ma);
+ /* add L1 shadow of L2 */
+ xen_queue_pt_update(vtomach(&pmap->pm_pdir[PTDPTDI + i]),
+ ma_shadow | PG_V | PG_A);
+ xen_queue_pt_update(ma_shadow + PTDPTDI*sizeof(vm_paddr_t),
+ vtomach(pmap->pm_pdir) | PG_V | PG_A);
+
#ifdef PAE
- pmap->pm_pdpt[i] = pa | PG_V;
+ #error "unsupported currently"
+ pmap->pm_pdpt[i] = ma | PG_V;
#endif
}
+ xen_flush_queue();
pmap->pm_active = 0;
TAILQ_INIT(&pmap->pm_pvchunk);
@@ -1371,8 +1505,9 @@
pmap->pm_stats.resident_count++;
ptepa = VM_PAGE_TO_PHYS(m);
- pmap->pm_pdir[ptepindex] =
- (pd_entry_t) (ptepa | PG_U | PG_RW | PG_V | PG_A | PG_M);
+ xen_pt_pin(xpmap_ptom(ptepa));
+ PD_SET_VA(pmap, &pmap->pm_pdir[ptepindex],
+ (ptepa | PG_U | PG_RW | PG_V | PG_A | PG_M), TRUE);
return m;
}
@@ -1396,7 +1531,7 @@
/*
* Get the page directory entry
*/
- ptepa = pmap->pm_pdir[ptepindex];
+ ptepa = PT_GET(&pmap->pm_pdir[ptepindex]);
/*
* This supports switching from a 4MB page to a
@@ -1534,8 +1669,9 @@
void
pmap_release(pmap_t pmap)
{
- vm_page_t m, ptdpg[NPGPTD];
- int i;
+ vm_page_t m, ptdpg[NPGPTD+1];
+ vm_paddr_t ma;
+ int i, npgptd = NPGPTD + 1;
KASSERT(pmap->pm_stats.resident_count == 0,
("pmap_release: pmap resident count %ld != 0",
@@ -1547,19 +1683,26 @@
mtx_unlock_spin(&allpmaps_lock);
for (i = 0; i < NPGPTD; i++)
- ptdpg[i] = PHYS_TO_VM_PAGE(pmap->pm_pdir[PTDPTDI + i] &
- PG_FRAME);
+ ptdpg[i] = PHYS_TO_VM_PAGE(PT_GET(&pmap->pm_pdir[PTDPTDI + i]) & PG_FRAME);
+
+ ptdpg[NPGPTD] = PHYS_TO_VM_PAGE(vtophys(pmap->pm_pdir));
+ for (i = 0; i < nkpt + NPGPTD; i++)
+ PD_CLEAR_VA(pmap, &pmap->pm_pdir[PTDPTDI + i], FALSE);
+
bzero(pmap->pm_pdir + PTDPTDI, (nkpt + NPGPTD) *
sizeof(*pmap->pm_pdir));
#ifdef SMP
- pmap->pm_pdir[MPPTDI] = 0;
+ PD_CLEAR_VA(pmap, &pmap->pm_pdir[MPPTDI], FALSE);
#endif
pmap_qremove((vm_offset_t)pmap->pm_pdir, NPGPTD);
- for (i = 0; i < NPGPTD; i++) {
+ for (i = 0; i < npgptd; i++) {
m = ptdpg[i];
+ ma = xpmap_ptom(VM_PAGE_TO_PHYS(m));
+ /* unpinning L1 and L2 treated the same */
+ xen_pgd_unpin(ma);
#ifdef PAE
KASSERT(VM_PAGE_TO_PHYS(m) == (pmap->pm_pdpt[i] & PG_FRAME),
("pmap_release: got wrong ptd page"));
@@ -1642,13 +1785,14 @@
pmap_zero_page(nkpg);
ptppaddr = VM_PAGE_TO_PHYS(nkpg);
newpdir = (pd_entry_t) (ptppaddr | PG_V | PG_RW | PG_A | PG_M);
- pdir_pde(PTD, kernel_vm_end) = newpdir;
+ PD_SET_VA(kernel_pmap, &pdir_pde(kernel_pmap->pm_pdir, kernel_vm_end), newpdir, TRUE);
mtx_lock_spin(&allpmaps_lock);
LIST_FOREACH(pmap, &allpmaps, pm_list) {
pde = pmap_pde(pmap, kernel_vm_end);
- pde_store(pde, newpdir);
+ PD_SET_VA(pmap, pde, newpdir, FALSE);
}
+ PT_UPDATES_FLUSH();
mtx_unlock_spin(&allpmaps_lock);
kernel_vm_end = (kernel_vm_end + PAGE_SIZE * NPTEPG) & ~(PAGE_SIZE * NPTEPG - 1);
if (kernel_vm_end - 1 >= kernel_map->max_offset) {
@@ -2024,6 +2168,10 @@
return;
pmap_remove_pte(pmap, pte, va, free);
pmap_invalidate_page(pmap, va);
+ if (PMAP1_inuse) {
+ PT_CLEAR_VA(PMAP1, TRUE);
+ PMAP1_inuse = 0;
+ }
}
/*
@@ -2088,7 +2236,7 @@
* Check for large page.
*/
if ((ptpaddr & PG_PS) != 0) {
- pmap->pm_pdir[pdirindex] = 0;
+ PD_CLEAR_VA(pmap, &pmap->pm_pdir[pdirindex], TRUE);
pmap->pm_stats.resident_count -= NBPDR / PAGE_SIZE;
anyvalid = 1;
continue;
@@ -2117,6 +2265,10 @@
break;
}
}
+ if (PMAP1_inuse) {
+ PT_CLEAR_VA(PMAP1, TRUE);
+ PMAP1_inuse = 0;
+ }
out:
sched_unpin();
if (anyvalid)
@@ -2188,6 +2340,10 @@
PMAP_UNLOCK(pmap);
}
vm_page_flag_clear(m, PG_WRITEABLE);
+ if (PMAP1_inuse) {
+ PT_CLEAR_VA(PMAP1, TRUE);
+ PMAP1_inuse = 0;
+ }
sched_unpin();
}
@@ -2259,7 +2415,9 @@
sva += PAGE_SIZE) {
vm_page_t m;
+#ifndef XEN
retry:
+#endif
/*
* Regardless of whether a pte is 32 or 64 bits in
* size, PG_RW, PG_A, and PG_M are among the least
@@ -2290,6 +2448,7 @@
#endif
if (pbits != obits) {
+#ifndef XEN
#ifdef PAE
if (!atomic_cmpset_64(pte, obits, pbits))
goto retry;
@@ -2298,6 +2457,9 @@
pbits))
goto retry;
#endif
+#else
+ PT_SET_VA(pte, pbits, FALSE);
+#endif
if (obits & PG_G)
pmap_invalidate_page(pmap, sva);
else
@@ -2305,6 +2467,10 @@
}
}
}
+ if (PMAP1_inuse) {
+ PT_CLEAR_VA(PMAP1, TRUE);
+ PMAP1_inuse = 0;
+ }
sched_unpin();
if (anychanged)
pmap_invalidate_all(pmap);
@@ -2360,7 +2526,7 @@
#if 0 && defined(PMAP_DIAGNOSTIC)
else {
pd_entry_t *pdeaddr = pmap_pde(pmap, va);
- origpte = *pdeaddr;
+ origpte = PT_GET(pdeaddr);
if ((origpte & PG_V) == 0) {
panic("pmap_enter: invalid kernel page table page, pdir=%p, pde=%p, va=%p\n",
pmap->pm_pdir[PTDPTDI], origpte, va);
@@ -2369,7 +2535,7 @@
#endif
pde = pmap_pde(pmap, va);
- if ((*pde & PG_PS) != 0)
+ if ((PT_GET(pde) & PG_PS) != 0)
panic("pmap_enter: attempted pmap_enter on 4MB page");
pte = pmap_pte_quick(pmap, va);
@@ -2383,7 +2549,7 @@
pa = VM_PAGE_TO_PHYS(m);
om = NULL;
- origpte = *pte;
+ origpte = PT_GET(pte);
opa = origpte & PG_FRAME;
/*
@@ -2506,6 +2672,10 @@
} else
pte_store(pte, newpte | PG_A);
}
+ if (PMAP1_inuse) {
+ PT_CLEAR_VA(PMAP1, TRUE);
+ PMAP1_inuse = 0;
+ }
sched_unpin();
vm_page_unlock_queues();
PMAP_UNLOCK(pmap);
@@ -2622,7 +2792,7 @@
* But that isn't as quick as vtopte.
*/
pte = vtopte(va);
- if (*pte) {
+ if (PT_GET(pte)) {
if (mpte != NULL) {
mpte->wire_count--;
mpte = NULL;
@@ -2744,8 +2914,8 @@
pmap->pm_stats.resident_count += size >> PAGE_SHIFT;
npdes = size >> PDRSHIFT;
for(i = 0; i < npdes; i++) {
- pde_store(&pmap->pm_pdir[ptepindex],
- ptepa | PG_U | PG_RW | PG_V | PG_PS);
+ PD_SET_VA(pmap, &pmap->pm_pdir[ptepindex],
+ ptepa | PG_U | PG_RW | PG_V | PG_PS, FALSE);
ptepa += NBPDR;
ptepindex += 1;
}
@@ -2781,6 +2951,10 @@
*/
pmap_pte_set_w(pte, wired);
pmap_pte_release(pte);
+ if (PMAP2_inuse) {
+ PT_CLEAR_VA(PMAP2, TRUE);
+ PMAP2_inuse = 0;
+ }
PMAP_UNLOCK(pmap);
}
@@ -2830,14 +3004,13 @@
pdnxt = (addr + NBPDR) & ~PDRMASK;
ptepindex = addr >> PDRSHIFT;
- srcptepaddr = src_pmap->pm_pdir[ptepindex];
+ srcptepaddr = PT_GET(&src_pmap->pm_pdir[ptepindex]);
if (srcptepaddr == 0)
continue;
if (srcptepaddr & PG_PS) {
if (dst_pmap->pm_pdir[ptepindex] == 0) {
- dst_pmap->pm_pdir[ptepindex] = srcptepaddr &
- ~PG_W;
+ PD_SET_VA(dst_pmap, &dst_pmap->pm_pdir[ptepindex], srcptepaddr & ~PG_W, TRUE);
dst_pmap->pm_stats.resident_count +=
NBPDR / PAGE_SIZE;
}
@@ -2872,8 +3045,8 @@
* accessed (referenced) bits
* during the copy.
*/
- *dst_pte = ptetemp & ~(PG_W | PG_M |
- PG_A);
+ PT_SET_VA(dst_pte, ptetemp & ~(PG_W | PG_M | PG_A), FALSE);
+
dst_pmap->pm_stats.resident_count++;
} else {
free = NULL;
@@ -2891,8 +3064,13 @@
src_pte++;
}
}
+ if (PMAP1_inuse) {
+ PT_CLEAR_VA(PMAP1, FALSE);
+ PMAP1_inuse = 0;
+ }
sched_unpin();
vm_page_unlock_queues();
+ PT_UPDATES_FLUSH();
PMAP_UNLOCK(src_pmap);
PMAP_UNLOCK(dst_pmap);
}
@@ -2928,9 +3106,11 @@
panic("pmap_zero_page: CMAP2 busy");
sched_pin();
*sysmaps->CMAP2 = PG_V | PG_RW | VM_PAGE_TO_PHYS(m) | PG_A | PG_M;
+ PT_SET_VA(sysmaps->CMAP2, PG_V | PG_RW | VM_PAGE_TO_PHYS(m) | PG_A | PG_M, TRUE);
+
invlcaddr(sysmaps->CADDR2);
pagezero(sysmaps->CADDR2);
- *sysmaps->CMAP2 = 0;
+ PT_CLEAR_VA(sysmaps->CMAP2, TRUE);
sched_unpin();
mtx_unlock(&sysmaps->lock);
}
@@ -2951,13 +3131,15 @@
if (*sysmaps->CMAP2)
panic("pmap_zero_page: CMAP2 busy");
sched_pin();
- *sysmaps->CMAP2 = PG_V | PG_RW | VM_PAGE_TO_PHYS(m) | PG_A | PG_M;
+ PT_SET_VA(sysmaps->CMAP2, PG_V | PG_RW | VM_PAGE_TO_PHYS(m) | PG_A | PG_M, TRUE);
+
invlcaddr(sysmaps->CADDR2);
if (off == 0 && size == PAGE_SIZE)
pagezero(sysmaps->CADDR2);
else
bzero((char *)sysmaps->CADDR2 + off, size);
- *sysmaps->CMAP2 = 0;
+
+ PT_CLEAR_VA(sysmaps->CMAP2, TRUE);
sched_unpin();
mtx_unlock(&sysmaps->lock);
}
@@ -2975,10 +3157,10 @@
if (*CMAP3)
panic("pmap_zero_page: CMAP3 busy");
sched_pin();
- *CMAP3 = PG_V | PG_RW | VM_PAGE_TO_PHYS(m) | PG_A | PG_M;
+ PT_SET_VA(CMAP3, PG_V | PG_RW | VM_PAGE_TO_PHYS(m) | PG_A | PG_M, TRUE);
invlcaddr(CADDR3);
pagezero(CADDR3);
- *CMAP3 = 0;
+ PT_CLEAR_VA(CMAP3, TRUE);
sched_unpin();
}
@@ -3002,15 +3184,45 @@
sched_pin();
invlpg((u_int)sysmaps->CADDR1);
invlpg((u_int)sysmaps->CADDR2);
- *sysmaps->CMAP1 = PG_V | VM_PAGE_TO_PHYS(src) | PG_A;
- *sysmaps->CMAP2 = PG_V | PG_RW | VM_PAGE_TO_PHYS(dst) | PG_A | PG_M;
+ PT_SET_VA(sysmaps->CMAP1, PG_V | VM_PAGE_TO_PHYS(src) | PG_A, FALSE);
+ PT_SET_VA(sysmaps->CMAP2, PG_V | PG_RW | VM_PAGE_TO_PHYS(dst) | PG_A | PG_M, TRUE);
bcopy(sysmaps->CADDR1, sysmaps->CADDR2, PAGE_SIZE);
- *sysmaps->CMAP1 = 0;
- *sysmaps->CMAP2 = 0;
+ PT_CLEAR_VA(sysmaps->CMAP1, FALSE);
+ PT_CLEAR_VA(sysmaps->CMAP2, TRUE);
+ sched_unpin();
+ mtx_unlock(&sysmaps->lock);
+}
+
+
+
+void
+pmap_copy_ma_range(vm_paddr_t src, vm_paddr_t dst, uint32_t soff, uint32_t len)
+{
+ struct sysmaps *sysmaps;
+
+ sysmaps = &sysmaps_pcpu[PCPU_GET(cpuid)];
+
+ mtx_lock(&sysmaps->lock);
+ if (*sysmaps->CMAP1)
+ panic("pmap_copy_ma: CMAP1 busy");
+ if (*sysmaps->CMAP2)
+ panic("pmap_copy_ma: CMAP2 busy");
+ sched_pin();
+
+ invlpg((u_int)sysmaps->CADDR1);
+ invlpg((u_int)sysmaps->CADDR2);
+
+ PT_SET_VA_MA(sysmaps->CMAP1, PG_V | src | PG_A, FALSE);
+ PT_SET_VA_MA(sysmaps->CMAP2, PG_V | PG_RW | dst | PG_A | PG_M, TRUE);
+
+ bcopy(sysmaps->CADDR1 + soff, sysmaps->CADDR2 + soff, len);
+ PT_CLEAR_VA(sysmaps->CMAP1, FALSE);
+ PT_CLEAR_VA(sysmaps->CMAP2, TRUE);
sched_unpin();
mtx_unlock(&sysmaps->lock);
}
+
/*
* Returns true if the pmap's pv is one of the first
* 16 pvs linked to from this page. This count may
@@ -3078,7 +3290,7 @@
inuse &= ~bitmask;
pte = vtopte(pv->pv_va);
- tpte = *pte;
+ tpte = PT_GET(pte);
if (tpte == 0) {
printf(
@@ -3140,6 +3352,10 @@
pmap_ptelist_free(&pv_vafree, (vm_offset_t)pc);
}
}
+ if (PMAP1_inuse) {
+ PT_CLEAR_VA(PMAP1, TRUE);
+ PMAP1_inuse = 0;
+ }
sched_unpin();
pmap_invalidate_all(pmap);
vm_page_unlock_queues();
@@ -3176,6 +3392,10 @@
if (rv)
break;
}
+ if (PMAP1_inuse) {
+ PT_CLEAR_VA(PMAP1, TRUE);
+ PMAP1_inuse = 0;
+ }
sched_unpin();
return (rv);
}
@@ -3193,6 +3413,13 @@
boolean_t rv;
rv = FALSE;
+ /* XXX
+ * in order for writable pagetables to help,
+ * this has to work - check if we aren't doing
+ * an invlpg on the page tables linear mappings
+ */
+ return (rv);
+
PMAP_LOCK(pmap);
if (*pmap_pde(pmap, addr)) {
pte = vtopte(addr);
@@ -3202,6 +3429,43 @@
return (rv);
}
+
+void
+pmap_map_readonly(pmap_t pmap, vm_offset_t va, int len)
+{
+ int i, npages = round_page(len) >> PAGE_SHIFT;
+ for (i = 0; i < npages; i++) {
+ pt_entry_t *pte;
+ pte = pmap_pte(pmap, (vm_offset_t)(va + i*PAGE_SIZE));
+ pte_store(pte, xpmap_mtop(*pte & ~(PG_RW|PG_M)));
+ PMAP_MARK_PRIV(xpmap_mtop(*pte));
+ pmap_pte_release(pte);
+ }
+ PT_UPDATES_FLUSH();
+ if (PMAP2_inuse) {
+ PT_CLEAR_VA(PMAP2, TRUE);
+ PMAP2_inuse = 0;
+ }
+}
+
+void
+pmap_map_readwrite(pmap_t pmap, vm_offset_t va, int len)
+{
+ int i, npages = round_page(len) >> PAGE_SHIFT;
+ for (i = 0; i < npages; i++) {
+ pt_entry_t *pte;
+ pte = pmap_pte(pmap, (vm_offset_t)(va + i*PAGE_SIZE));
+ PMAP_MARK_UNPRIV(xpmap_mtop(*pte));
+ pte_store(pte, xpmap_mtop(*pte) | (PG_RW|PG_M));
+ pmap_pte_release(pte);
+ }
+ PT_UPDATES_FLUSH();
+ if (PMAP2_inuse) {
+ PT_CLEAR_VA(PMAP2, TRUE);
+ PMAP2_inuse = 0;
+ }
+}
+
/*
* Clear the write and modified bits in each of the given page's mappings.
*/
@@ -3221,17 +3485,23 @@
pmap = PV_PMAP(pv);
PMAP_LOCK(pmap);
pte = pmap_pte_quick(pmap, pv->pv_va);
+#ifndef XEN
retry:
- oldpte = *pte;
+#endif
+ oldpte = PT_GET(pte);
if ((oldpte & PG_RW) != 0) {
/*
* Regardless of whether a pte is 32 or 64 bits
* in size, PG_RW and PG_M are among the least
* significant 32 bits.
*/
+#ifndef XEN
if (!atomic_cmpset_int((u_int *)pte, oldpte,
oldpte & ~(PG_RW | PG_M)))
goto retry;
+#else
+ PT_SET_VA(pte, oldpte & ~(PG_RW|PG_M), TRUE);
+#endif
if ((oldpte & PG_M) != 0)
vm_page_dirty(m);
pmap_invalidate_page(pmap, pv->pv_va);
@@ -3239,6 +3509,10 @@
PMAP_UNLOCK(pmap);
}
vm_page_flag_clear(m, PG_WRITEABLE);
+ if (PMAP1_inuse) {
+ PT_CLEAR_VA(PMAP1, TRUE);
+ PMAP1_inuse = 0;
+ }
sched_unpin();
}
@@ -3260,6 +3534,8 @@
pv_entry_t pv, pvf, pvn;
pmap_t pmap;
pt_entry_t *pte;
+ vm_paddr_t v;
+
int rtval = 0;
if (m->flags & PG_FICTITIOUS)
@@ -3275,8 +3551,13 @@
pmap = PV_PMAP(pv);
PMAP_LOCK(pmap);
pte = pmap_pte_quick(pmap, pv->pv_va);
- if ((*pte & PG_A) != 0) {
+ if (pte && ((v = PT_GET(pte)) & PG_A) != 0) {
+#ifndef XEN
atomic_clear_int((u_int *)pte, PG_A);
+#else
+ PT_SET_VA(pte, v & ~PG_A, TRUE);
+#endif
+
pmap_invalidate_page(pmap, pv->pv_va);
rtval++;
if (rtval > 4)
@@ -3285,6 +3566,10 @@
PMAP_UNLOCK(pmap);
} while ((pv = pvn) != NULL && pv != pvf);
}
+ if (PMAP1_inuse) {
+ PT_CLEAR_VA(PMAP1, TRUE);
+ PMAP1_inuse = 0;
+ }
sched_unpin();
return (rtval);
}
@@ -3298,7 +3583,8 @@
pv_entry_t pv;
pmap_t pmap;
pt_entry_t *pte;
-
+ vm_paddr_t val;
+
mtx_assert(&vm_page_queue_mtx, MA_OWNED);
if ((m->flags & PG_FICTITIOUS) != 0)
return;
@@ -3307,14 +3593,20 @@
pmap = PV_PMAP(pv);
PMAP_LOCK(pmap);
pte = pmap_pte_quick(pmap, pv->pv_va);
- if ((*pte & PG_M) != 0) {
+ val = PT_GET(pte);
+
+ if ((val & PG_M) != 0) {
/*
* Regardless of whether a pte is 32 or 64 bits
* in size, PG_M is among the least significant
* 32 bits.
*/
+#ifndef XEN
atomic_clear_int((u_int *)pte, PG_M);
+#else
+ PT_SET_VA(pv->pv_va, val & ~PG_M, FALSE);
pmap_invalidate_page(pmap, pv->pv_va);
+#endif
}
PMAP_UNLOCK(pmap);
}
@@ -3332,7 +3624,8 @@
pv_entry_t pv;
pmap_t pmap;
pt_entry_t *pte;
-
+ vm_paddr_t val;
+
mtx_assert(&vm_page_queue_mtx, MA_OWNED);
if ((m->flags & PG_FICTITIOUS) != 0)
return;
@@ -3341,13 +3634,18 @@
pmap = PV_PMAP(pv);
PMAP_LOCK(pmap);
pte = pmap_pte_quick(pmap, pv->pv_va);
- if ((*pte & PG_A) != 0) {
+ val = PT_GET(pte);
+ if ((val & PG_A) != 0) {
/*
* Regardless of whether a pte is 32 or 64 bits
* in size, PG_A is among the least significant
* 32 bits.
*/
>>> TRUNCATED FOR MAIL (1000 lines) <<<
More information about the p4-projects
mailing list