PERFORCE change 84785 for review
Olivier Houchard
cognet at FreeBSD.org
Tue Oct 4 06:13:10 PDT 2005
http://perforce.freebsd.org/chv.cgi?CH=84785
Change 84785 by cognet at cognet on 2005/10/04 13:13:08
Superpages bits for arm.
Affected files ...
.. //depot/projects/superpages/src/sys/arm/arm/pmap.c#3 edit
.. //depot/projects/superpages/src/sys/arm/include/pmap.h#3 edit
Differences ...
==== //depot/projects/superpages/src/sys/arm/arm/pmap.c#3 (text+ko) ====
@@ -1,4 +1,4 @@
-/* From: $NetBSD: pmap.c,v 1.148 2004/04/03 04:35:48 bsh Exp $ */
+//* From: $NetBSD: pmap.c,v 1.148 2004/04/03 04:35:48 bsh Exp $ */
/*-
* Copyright 2004 Olivier Houchard.
* Copyright 2003 Wasabi Systems, Inc.
@@ -168,6 +168,7 @@
#include <vm/vm_map.h>
#include <vm/vm_page.h>
#include <vm/vm_pageout.h>
+#include <vm/vm_reserve.h>
#include <vm/vm_extern.h>
#include <sys/lock.h>
#include <sys/mutex.h>
@@ -210,10 +211,16 @@
static int pmap_clearbit(struct vm_page *, u_int);
static struct l2_bucket *pmap_get_l2_bucket(pmap_t, vm_offset_t);
-static struct l2_bucket *pmap_alloc_l2_bucket(pmap_t, vm_offset_t);
+static struct l2_bucket *pmap_alloc_l2_bucket(pmap_t, vm_offset_t, int flags);
static void pmap_free_l2_bucket(pmap_t, struct l2_bucket *, u_int);
static vm_offset_t kernel_pt_lookup(vm_paddr_t);
+static void pmap_promote_small(pmap_t, struct l2_bucket *,
+ vm_offset_t, reservation_t, int);
+static void pmap_promote_large(pmap_t, vm_offset_t va,
+ reservation_t);
+static void pmap_demote(pmap_t, vm_offset_t);
+
static MALLOC_DEFINE(M_VMPMAP, "pmap", "PMAP L1");
vm_offset_t avail_end; /* PA of last available physical page */
@@ -910,7 +917,7 @@
* the bucket/page in the meantime.
*/
static struct l2_bucket *
-pmap_alloc_l2_bucket(pmap_t pm, vm_offset_t va)
+pmap_alloc_l2_bucket(pmap_t pm, vm_offset_t va, int flags)
{
struct l2_dtable *l2;
struct l2_bucket *l2b;
@@ -946,7 +953,7 @@
* No L2 page table has been allocated. Chances are, this
* is because we just allocated the l2_dtable, above.
*/
- ptep = (void*)uma_zalloc(l2zone, M_NOWAIT);
+ ptep = (void*)uma_zalloc(l2zone, flags);
l2b->l2b_phys = vtophys(ptep);
if (ptep == NULL) {
/*
@@ -1105,10 +1112,14 @@
*/
#ifdef ARM_USE_SMALL_ALLOC
pde = &kernel_pmap->pm_l1->l1_kva[L1_IDX(va)];
+ if (l1pte_section_p(*pde) && va < 0xd0000000 && va > virtual_avail)
+ panic("ouin");
if (!l1pte_section_p(*pde)) {
#endif
l2b = pmap_get_l2_bucket(pmap_kernel(), va);
ptep = &l2b->l2b_kva[l2pte_index(va)];
+ if ((*ptep & L2_TYPE_MASK) == L2_TYPE_L)
+ panic("fuxor");
pte = *ptep;
if ((pte & L2_S_CACHE_MASK) != pte_l2_s_cache_mode_pt) {
@@ -1454,8 +1465,18 @@
pv->pv_flags |= PVF_NC;
+ if (l1pte_section_p(
+ pm->pm_l1->l1_kva[L1_IDX(pv->pv_va)])) {
+ pmap_demote(pv->pv_pmap, pv->pv_va);
+ pv->pv_flags |= PVF_DEMOTED;
+ }
l2b = pmap_get_l2_bucket(pv->pv_pmap, pv->pv_va);
+
ptep = &l2b->l2b_kva[l2pte_index(pv->pv_va)];
+ if ((*ptep & L2_TYPE_MASK) == L2_TYPE_L) {
+ pmap_demote(pv->pv_pmap, pv->pv_va);
+ pv->pv_flags |= PVF_DEMOTED;
+ }
pte = *ptep & ~L2_S_CACHE_MASK;
if ((va != pv->pv_va || pm != pv->pv_pmap) &&
@@ -1509,10 +1530,177 @@
*ptep = pte;
PTE_SYNC_CURRENT(pv->pv_pmap, ptep);
+ if (pv->pv_flags & PVF_DEMOTED) {
+ pmap_promote_small(pv->pv_pmap, l2b, pv->pv_va,
+ NULL, 0);
+ pmap_promote_large(pv->pv_pmap, pv->pv_va,
+ NULL);
+ pv->pv_flags &= ~PVF_DEMOTED;
+ }
}
}
}
+/* XXX: misleading, promote_small actually promotes to large pages. */
+static void
+pmap_promote_small(pmap_t pmap, struct l2_bucket *l2b, vm_offset_t va,
+ reservation_t reserv, int off)
+{
+ int i;
+ pt_entry_t *ptep;
+ vm_paddr_t pa0;
+
+ if (!reserv)
+ return;
+ ptep = &l2b->l2b_kva[l2pte_index(va & L2_L_FRAME)];
+ pa0 = *ptep & L2_L_FRAME;
+ if ((*ptep & L2_S_FRAME) != pa0)
+ return;
+ if (reserv != NULL && ((pa0 - off * PAGE_SIZE)
+ != VM_PAGE_TO_PHYS(reserv->first_page)))
+ return;
+ /* First check it is eligible for promotion. */
+ for (i = 1; i < 0x10; i++) {
+ if (ptep[i] != (ptep[i - 1] + PAGE_SIZE)) {
+ /* There's no way we can do it. */
+ return;
+ }
+ if ((ptep[i] & pte_l2_s_cache_mask) ==
+ pte_l2_s_cache_mode_pt ||
+ (ptep[i - 1] & pte_l2_s_cache_mask) ==
+ pte_l2_s_cache_mode_pt)
+ panic("fuck");
+ }
+#ifdef SP_DEBUG
+ printf("promoting small %x\n", va);
+#endif
+ if (*ptep & L2_S_PROT_U)
+ pa0 |= L2_L_PROT_U;
+ if (*ptep & L2_S_PROT_W)
+ pa0 |= L2_L_PROT_W;
+ /* Let's do it. */
+ for (i = 0; i < 0x10; i++)
+ ptep[i] = pa0 | L2_L_PROTO |
+ pte_l2_l_cache_mode;
+ pmap_tlb_flushID(pmap);
+}
+
+static void
+pmap_promote_large(pmap_t pmap, vm_offset_t va, reservation_t reserv)
+{
+ pd_entry_t *pd;
+ pt_entry_t *pt;
+ vm_offset_t va0;
+ vm_paddr_t pa;
+ struct l2_dtable *l2;
+ uint16_t l1idx;
+ int i;
+
+ if (!reserv)
+ return;
+ va0 = va & L1_S_ADDR_MASK;
+ l1idx = L1_IDX(va0);
+ pd = &pmap->pm_l1->l1_kva[l1idx];
+ KASSERT(pd != NULL, ("NO pd_entry in pmap_promote_large"));
+ l2 = pmap->pm_l2[L2_IDX(l1idx)];
+ KASSERT(l2 != NULL, ("No l2 bucket in pmap_promote_large"));
+ pt = l2->l2_bucket[L2_BUCKET(l1idx)].l2b_kva;
+ if (l2pte_index(va0) != 0)
+ panic("rate");
+ KASSERT(pt != NULL, ("No pt in pmap_promote_large"));
+ if ((*pt & L2_TYPE_MASK) == L2_TYPE_L)
+ pa = *pt & L2_L_FRAME;
+ else
+ pa = *pt & L2_S_FRAME;
+ if (pa != (*pt & L1_S_FRAME))
+ return;
+ if (reserv != NULL && pa != VM_PAGE_TO_PHYS(reserv->first_page))
+ return;
+ for (i = 1; i < 0x100; i++) {
+ vm_paddr_t pa, pa2;
+
+ if ((pt[i] & L2_TYPE_MASK) == L2_TYPE_L)
+ pa = (pt[i] & L2_L_FRAME) + (i & 0xf) * PAGE_SIZE;
+ else
+ pa = pt[i] & L2_S_FRAME;
+ if ((pt[i - 1] & pte_l2_s_cache_mask) == pte_l2_s_cache_mode_pt
+ || (pt[i] & pte_l2_s_cache_mask) == pte_l2_s_cache_mode_pt)
+ panic("fuck2");
+ if ((pt[i - 1] & L2_TYPE_MASK) == L2_TYPE_L)
+ pa2 = (pt[i - 1] & L2_L_FRAME) +
+ ((i - 1) & 0xf) * PAGE_SIZE;
+ else
+ pa2 = pt[i - 1] & L2_S_FRAME;
+ if (pa != pa2 + PAGE_SIZE)
+ /* Nothing much we can do. */
+ return;
+ }
+#ifdef SP_DEBUG
+ printf("promoting large %x\n", va);
+#endif
+ /* Now free the unused pts. */
+ if (*pt & L2_S_PROT_U)
+ pa |= L1_S_PROT_U;
+ if (*pt & L2_S_PROT_W)
+ pa |= L1_S_PROT_W;
+ *pd = L1_S_PROTO | pa | pte_l1_s_cache_mode | L1_S_DOM(pmap->pm_domain);
+ bzero(pt, 0x100 * sizeof(*pt));
+ pmap_free_l2_bucket(pmap, &l2->l2_bucket[L2_BUCKET(l1idx)], 0x100);
+ pmap_tlb_flushID(pmap);
+}
+
+static void
+pmap_demote(pmap_t pmap, vm_offset_t va)
+{
+ pd_entry_t *pd;
+ pt_entry_t *pt;
+ struct l2_bucket *l2b;
+ vm_offset_t va0;
+ uint16_t l1idx;
+ uint16_t demote_size;
+ vm_paddr_t pa;
+ int i;
+
+#ifdef SP_DEBUG
+ printf("demoting %x\n", va);
+#endif
+ l1idx = L1_IDX(va);
+ pd = &pmap->pm_l1->l1_kva[l1idx];
+ if (l1pte_section_p(*pd)) {
+ va0 = va & L1_S_ADDR_MASK;
+ if (pmap == pmap_kernel())
+ l2b = pmap_get_l2_bucket(pmap, va0);
+ else
+ l2b = pmap_alloc_l2_bucket(pmap, va0, M_NOWAIT);
+ demote_size = 0x100;
+ pa = *pd & L1_S_ADDR_MASK;
+ if (*pd & L1_S_PROT_U)
+ pa |= L2_S_PROT_U;
+ if (*pd & L1_S_PROT_W)
+ pa |= L2_S_PROT_W;
+ pt = &l2b->l2b_kva[l2pte_index(va0)];
+ *pd = l2b->l2b_phys | L1_C_DOM(pmap->pm_domain) | L1_C_PROTO;
+ l2b->l2b_occupancy += 0x100;
+ } else {
+ va0 = va & L2_L_FRAME;
+ l2b = pmap_get_l2_bucket(pmap, va0);
+ demote_size = 0x10;
+ pt = &l2b->l2b_kva[l2pte_index(va0)];
+ if ((*pt & L2_TYPE_MASK) != L2_TYPE_L)
+ return;
+ pa = *pt & L2_L_FRAME;
+ if (*pt & L2_L_PROT_U)
+ pa |= L2_S_PROT_U;
+ if (*pt & L2_L_PROT_W)
+ pa |= L2_S_PROT_W;
+ }
+ pa |= L2_S_PROTO;
+ for (i = 0; i < demote_size; i++, pa += PAGE_SIZE)
+ pt[i] = (pa) | pte_l2_s_cache_mode;
+ pmap_tlb_flushID(pmap);
+}
+
+
/*
* Modify pte bits for all ptes corresponding to the given physical address.
* We use `maskbits' rather than `clearbits' because we're always passing
@@ -1559,9 +1747,17 @@
pmap_acquire_pmap_lock(pm);
#endif
+
+ if (l1pte_section_p(pm->pm_l1->l1_kva[L1_IDX(va)])) {
+ pmap_demote(pm, va);
+ pv->pv_flags |= PVF_DEMOTED;
+ }
l2b = pmap_get_l2_bucket(pm, va);
-
ptep = &l2b->l2b_kva[l2pte_index(va)];
+ if ((*ptep & L2_TYPE_MASK) == L2_TYPE_L) {
+ pmap_demote(pm, va);
+ pv->pv_flags |= PVF_DEMOTED;
+ }
npte = opte = *ptep;
if (maskbits & (PVF_WRITE|PVF_MOD) &&
@@ -1675,6 +1871,11 @@
if (PV_BEEN_REFD(oflags))
pmap_tlb_flushD_SE(pm, pv->pv_va);
}
+ if (pv->pv_flags & PVF_DEMOTED) {
+ pmap_promote_small(pm, l2b, va, NULL, 0);
+ pmap_promote_large(pm, va, NULL);
+ pv->pv_flags &= ~PVF_DEMOTED;
+ }
#if 0
pmap_release_pmap_lock(pm);
@@ -1713,8 +1914,6 @@
pmap_enter_pv(struct vm_page *pg, struct pv_entry *pve, pmap_t pm,
vm_offset_t va, u_int flags)
{
-
-
pve->pv_pmap = pm;
pve->pv_va = va;
pve->pv_flags = flags;
@@ -1981,6 +2180,19 @@
pmap_acquire_pmap_lock(pm);
#endif
l1idx = L1_IDX(va);
+ pl1pd = &pm->pm_l1->l1_kva[l1idx];
+ if (l1pte_section_p(*pl1pd)) {
+ if (user && !(*pl1pd & L1_S_PROT_U))
+ goto out;
+ if (ftype == VM_PROT_WRITE && !(*pl1pd & L1_S_PROT_W))
+ goto out;
+ if ((*pl1pd & L1_S_DOM_MASK) != L1_S_DOM(pm->pm_domain)) {
+ *pl1pd &= ~L1_S_DOM_MASK;
+ *pl1pd |= L1_S_DOM(pm->pm_domain);
+ rv = 1;
+ goto out;
+ }
+ }
/*
* If there is no l2_dtable for this address, then the process
@@ -2011,12 +2223,15 @@
/*
* Catch a userland access to the vector page mapped at 0x0
*/
- if (user && (pte & L2_S_PROT_U) == 0)
+ if (user && (((pte & L2_TYPE_MASK) == L2_TYPE_L &&
+ (pte & L2_L_PROT_U) == 0) || (pte & L2_S_PROT_U) == 0))
goto out;
- pa = l2pte_pa(pte);
+ pa = (pte & L2_TYPE_MASK) == L2_TYPE_L ?
+ ((pte & L2_L_FRAME) | (va & L2_L_OFFSET)) : l2pte_pa(pte);
- if ((ftype & VM_PROT_WRITE) && (pte & L2_S_PROT_W) == 0) {
+ if ((ftype & VM_PROT_WRITE) && (((pte & L2_TYPE_MASK) == L2_TYPE_L &&
+ (pte & L2_L_PROT_W) == 0) || (pte & L2_S_PROT_W) == 0)) {
/*
* This looks like a good candidate for "page modified"
* emulation...
@@ -2025,15 +2240,13 @@
struct vm_page *pg;
/* Extract the physical address of the page */
- if ((pg = PHYS_TO_VM_PAGE(pa)) == NULL) {
+ if ((pg = PHYS_TO_VM_PAGE(pa)) == NULL)
goto out;
- }
/* Get the current flags for this page. */
pv = pmap_find_pv(pg, pm, va);
- if (pv == NULL) {
+ if (pv == NULL)
goto out;
- }
/*
* Do the flags say this page is writable? If not then it
@@ -2042,9 +2255,8 @@
* PTE. Now we know a write has occurred we can correct this
* and also set the modified bit
*/
- if ((pv->pv_flags & PVF_WRITE) == 0) {
+ if ((pv->pv_flags & PVF_WRITE) == 0)
goto out;
- }
if (pmap_track_modified(pv->pv_va)) {
pg->md.pvh_attrs |= PVF_REF | PVF_MOD;
@@ -2059,7 +2271,12 @@
* changing. We've already set the cacheable bits based on
* the assumption that we can write to this page.
*/
- *ptep = (pte & ~L2_TYPE_MASK) | L2_S_PROTO | L2_S_PROT_W;
+ if ((pte & L2_TYPE_MASK) == L2_TYPE_L)
+ *ptep = (pte & ~L2_TYPE_MASK) | L2_L_PROTO |
+ L2_L_PROT_W;
+ else
+ *ptep = (pte & ~L2_TYPE_MASK) | L2_S_PROTO |
+ L2_S_PROT_W;
PTE_SYNC(ptep);
rv = 1;
} else
@@ -2088,7 +2305,6 @@
pg->md.pvh_attrs |= PVF_REF;
pv->pv_flags |= PVF_REF;
-
*ptep = (pte & ~L2_TYPE_MASK) | L2_S_PROTO;
PTE_SYNC(ptep);
rv = 1;
@@ -2212,7 +2428,7 @@
pv_entry_high_water = 9 * (pv_entry_max / 10);
l2zone = uma_zcreate("L2 Table", L2_TABLE_SIZE_REAL, pmap_l2ptp_ctor,
NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_VM | UMA_ZONE_NOFREE);
- uma_prealloc(l2zone, 4096);
+ /*uma_prealloc(l2zone, 4096);*/
l2table_zone = uma_zcreate("L2 Table", sizeof(struct l2_dtable),
NULL, NULL, NULL, NULL, UMA_ALIGN_PTR,
UMA_ZONE_VM | UMA_ZONE_NOFREE);
@@ -2823,8 +3039,10 @@
{
struct pv_entry *pv, *npv;
struct l2_bucket *l2b = NULL;
+ pt_entry_t *pt;
+ pd_entry_t *pd;
+ vm_paddr_t pa;
vm_page_t m;
- pt_entry_t *pt;
vm_page_lock_queues();
for (pv = TAILQ_FIRST(&pmap->pm_pvlist); pv; pv = npv) {
@@ -2832,6 +3050,9 @@
npv = TAILQ_NEXT(pv, pv_plist);
continue;
}
+ pd = &pmap->pm_l1->l1_kva[L1_IDX(pv->pv_va)];
+ if (l1pte_section_p(*pd))
+ pmap_demote(pmap, pv->pv_va);
if (pv->pv_flags & PVF_WIRED) {
/* The page is wired, cannot remove it now. */
npv = TAILQ_NEXT(pv, pv_plist);
@@ -2841,7 +3062,10 @@
l2b = pmap_get_l2_bucket(pmap, pv->pv_va);
KASSERT(l2b != NULL, ("No L2 bucket in pmap_remove_pages"));
pt = &l2b->l2b_kva[l2pte_index(pv->pv_va)];
- m = PHYS_TO_VM_PAGE(*pt & L2_ADDR_MASK);
+ if ((*pt & L2_TYPE_MASK) == L2_TYPE_L)
+ pmap_demote(pmap, pv->pv_va);
+ pa = *pt & L2_S_FRAME;
+ m = PHYS_TO_VM_PAGE(pa);
*pt = 0;
PTE_SYNC(pt);
npv = TAILQ_NEXT(pv, pv_plist);
@@ -3014,6 +3238,7 @@
TAILQ_FOREACH(pv, &m->md.pv_list, pv_list)
pmap_dcache_wb_range(pv->pv_pmap, pv->pv_va, PAGE_SIZE, TRUE, TRUE);
}
+
/*
* Add a list of wired pages to the kva
* this routine is only used for temporary
@@ -3084,6 +3309,8 @@
if (!pmap_get_pde_pte(pmap, addr, &pde, &pte))
return (FALSE);
+ if (!pte)
+ return (FALSE);
if (*pte == 0)
return (TRUE);
return (FALSE);
@@ -3154,6 +3381,7 @@
pmap_remove_all(vm_page_t m)
{
pv_entry_t pv;
+ pd_entry_t *pd;
pt_entry_t *ptep, pte;
struct l2_bucket *l2b;
boolean_t flush = FALSE;
@@ -3177,9 +3405,15 @@
if (flush == FALSE && (pv->pv_pmap == curpm ||
pv->pv_pmap == pmap_kernel()))
flush = TRUE;
+ pd = &pv->pv_pmap->pm_l1->l1_kva[L1_IDX(pv->pv_va)];
+ if (l1pte_section_p(*pd))
+ pmap_demote(pv->pv_pmap, pv->pv_va);
l2b = pmap_get_l2_bucket(pv->pv_pmap, pv->pv_va);
+
KASSERT(l2b != NULL, ("No l2 bucket"));
ptep = &l2b->l2b_kva[l2pte_index(pv->pv_va)];
+ if ((*ptep & L2_TYPE_MASK) == L2_TYPE_L)
+ pmap_demote(pv->pv_pmap, pv->pv_va);
pte = *ptep;
*ptep = 0;
PTE_SYNC_CURRENT(pv->pv_pmap, ptep);
@@ -3214,6 +3448,7 @@
vm_offset_t next_bucket;
u_int flags;
int flush;
+ int demoted = 0;
if ((prot & VM_PROT_READ) == 0) {
mtx_lock(&Giant);
@@ -3243,10 +3478,25 @@
vm_page_lock_queues();
while (sva < eva) {
+ demoted = 0;
next_bucket = L2_NEXT_BUCKET(sva);
if (next_bucket > eva)
next_bucket = eva;
+ if (l1pte_section_p(pm->pm_l1->l1_kva[L1_IDX(sva)])) {
+ if ((sva & L1_S_OFFSET) == 0 &&
+ sva + L1_S_SIZE < eva) {
+ /* Change the whole 1MB superpage. */
+ pm->pm_l1->l1_kva[L1_IDX(sva)] &= ~
+ L1_S_PROT_W;
+ flush++;
+ sva += L1_S_SIZE;
+ continue;
+ }
+ pmap_demote(pm, sva);
+ l2b = pmap_get_l2_bucket(pm, sva);
+ demoted = 1;
+ }
l2b = pmap_get_l2_bucket(pm, sva);
if (l2b == NULL) {
sva = next_bucket;
@@ -3256,6 +3506,11 @@
ptep = &l2b->l2b_kva[l2pte_index(sva)];
while (sva < next_bucket) {
+ demoted = 0;
+ if ((*ptep & L2_TYPE_MASK) == L2_TYPE_L) {
+ pmap_demote(pm, sva);
+ demoted = 1;
+ }
if ((pte = *ptep) != 0 && (pte & L2_S_PROT_W) != 0) {
struct vm_page *pg;
u_int f;
@@ -3285,6 +3540,10 @@
pmap_tlb_flushD_SE(pm, sva);
}
+ if (demoted) {
+ pmap_promote_small(pm, l2b, sva, NULL, 0);
+ pmap_promote_large(pm, sva, NULL);
+ }
sva += PAGE_SIZE;
ptep++;
}
@@ -3322,13 +3581,40 @@
boolean_t wired)
{
struct l2_bucket *l2b = NULL;
- struct vm_page *opg;
+ struct vm_page *opg = NULL;
struct pv_entry *pve = NULL;
pt_entry_t *ptep, npte, opte;
+ pd_entry_t l1pd;
u_int nflags;
u_int oflags;
vm_paddr_t pa;
+ l1pd = pmap->pm_l1->l1_kva[L1_IDX(va)];
+ if (l1pte_section_p(l1pd) &&
+ ((l1pd & L1_S_FRAME) != (VM_PAGE_TO_PHYS(m) & L1_S_FRAME) ||
+ (((VM_PAGE_TO_PHYS(m) & L2_S_FRAME) & L2_S_OFFSET) !=
+ (va & L2_S_OFFSET)) ||
+ ((l1pd & L1_S_PROT_U) != 0) != (pmap != pmap_kernel() && va !=
+ vector_page) || ((l1pd & L1_S_PROT_W) != 0) !=
+ ((prot & VM_PROT_WRITE) != 0))) {
+ /* There's already a superpage at this address, demote. */
+ pmap_demote(pmap, va);
+ } else if (l1pte_section_p(l1pd)) {
+ pmap_free_l2_bucket(pmap, l2b, 0);
+ opg = m;
+ l2b = NULL;
+ }
+
+ if (opg == NULL) {
+ if (pmap == pmap_kernel()) {
+ l2b = pmap_get_l2_bucket(pmap, va);
+ if (!l2b)
+ l2b = pmap_grow_l2_bucket(pmap, va);
+ } else
+ l2b = pmap_alloc_l2_bucket(pmap, va, M_NOWAIT);
+ KASSERT(l2b != NULL,
+ ("pmap_enter: failed to allocate l2 bucket"));
+ }
vm_page_lock_queues();
if (va == vector_page) {
pa = systempage.pv_pa;
@@ -3345,31 +3631,40 @@
PDEBUG(1, printf("pmap_enter: pmap = %08x, va = %08x, m = %08x, prot = %x, "
"wired = %x\n", (uint32_t) pmap, va, (uint32_t) m, prot, wired));
- if (pmap == pmap_kernel()) {
- l2b = pmap_get_l2_bucket(pmap, va);
- if (l2b == NULL)
- l2b = pmap_grow_l2_bucket(pmap, va);
- } else
- l2b = pmap_alloc_l2_bucket(pmap, va);
- KASSERT(l2b != NULL,
- ("pmap_enter: failed to allocate l2 bucket"));
- ptep = &l2b->l2b_kva[l2pte_index(va)];
+ if (l2b) {
+ ptep = &l2b->l2b_kva[l2pte_index(va)];
- opte = *ptep;
- npte = pa;
- oflags = 0;
+ opte = *ptep;
+ npte = pa;
+ oflags = 0;
+ } else {
+ npte = oflags = opte = 0;
+ ptep = NULL;
+ }
if (opte) {
+ if ((opte & L2_TYPE_MASK) == L2_TYPE_L) {
+ if (((opte & L2_L_FRAME) | (va & L2_L_OFFSET)) ==
+ VM_PAGE_TO_PHYS(m) &&
+ (((opte & L2_L_PROT_W) != 0) ==
+ (prot & VM_PROT_WRITE) != 0) &&
+ (((opte & L2_L_PROT_U) == 0) ==
+ (pmap == pmap_kernel() || va == vector_page)))
+ opg = m;
+ else {
+ pmap_demote(pmap, va);
+ opte = *ptep;
+ }
+ }
/*
* There is already a mapping at this address.
* If the physical address is different, lookup the
* vm_page.
*/
- if (l2pte_pa(opte) != pa)
+ if (!opg && l2pte_pa(opte) != pa)
opg = PHYS_TO_VM_PAGE(l2pte_pa(opte));
- else
+ else if (!opg)
opg = m;
- } else
- opg = NULL;
+ }
if ((prot & (VM_PROT_ALL)) ||
(!m || m->md.pvh_attrs & PVF_REF)) {
@@ -3492,7 +3787,7 @@
/*
* Keep the stats up to date
*/
- if (opte == 0) {
+ if (opte == 0 && l2b) {
l2b->l2b_occupancy++;
pmap->pm_stats.resident_count++;
}
@@ -3502,7 +3797,7 @@
* If this is just a wiring change, the two PTEs will be
* identical, so there's no need to update the page table.
*/
- if (npte != opte) {
+ if (l2b && npte != opte && ((opte & L2_TYPE_MASK) != L2_TYPE_L)) {
boolean_t is_cached = pmap_is_current(pmap);
*ptep = npte;
@@ -3539,6 +3834,17 @@
pmap_vac_me_harder(m, pmap, va);
+ if ((va < VM_MAXUSER_ADDRESS ||
+ m->object == kernel_object ||
+ (m->object == kmem_object && FALSE)) && m->reserv) {
+ if (m->reserv->refcnt > 0 &&
+ m->reserv->refcnt != SP_LARGE &&
+ (m->reserv->refcnt % SMALL) == 0)
+ pmap_promote_small(pmap, l2b, va, m->reserv,
+ m->reserv->refcnt - SMALL);
+ else if (m->reserv->refcnt == SP_LARGE)
+ pmap_promote_large(pmap, va, m->reserv);
+ }
}
vm_page_unlock_queues();
}
@@ -3579,15 +3885,19 @@
void
pmap_change_wiring(pmap_t pmap, vm_offset_t va, boolean_t wired)
{
+#if 0
struct l2_bucket *l2b;
pt_entry_t *ptep, pte;
+#endif
vm_page_t pg;
+#if 0
l2b = pmap_get_l2_bucket(pmap, va);
KASSERT(l2b, ("No l2b bucket in pmap_change_wiring"));
ptep = &l2b->l2b_kva[l2pte_index(va)];
pte = *ptep;
- pg = PHYS_TO_VM_PAGE(l2pte_pa(pte));
+#endif
+ pg = PHYS_TO_VM_PAGE(pmap_extract(pmap, va));
if (pg)
pmap_modify_pv(pg, pmap, va, PVF_WIRED, wired);
}
@@ -3626,10 +3936,6 @@
l1pd = *pl1pd;
if (l1pte_section_p(l1pd)) {
- /*
- * These should only happen for pmap_kernel()
- */
- KASSERT(pm == pmap_kernel(), ("huh"));
pa = (l1pd & L1_S_FRAME) | (va & L1_S_OFFSET);
} else {
/*
@@ -3685,10 +3991,6 @@
vm_page_lock_queues();
if (l1pte_section_p(l1pd)) {
- /*
- * These should only happen for pmap_kernel()
- */
- KASSERT(pmap == pmap_kernel(), ("huh"));
pa = (l1pd & L1_S_FRAME) | (va & L1_S_OFFSET);
if (l1pd & L1_S_PROT_W || (prot & VM_PROT_WRITE) == 0) {
m = PHYS_TO_VM_PAGE(pa);
@@ -3845,8 +4147,22 @@
l2b = pmap_get_l2_bucket(pm, sva);
if (l2b == NULL) {
- sva = next_bucket;
- continue;
+ pd_entry_t *pd = &pm->pm_l1->l1_kva[L1_IDX(sva)];
+ if (l1pte_section_p(*pd)) {
+ /* We can just remove the superpage. */
+ if (0 && (sva == (sva & L1_S_ADDR_MASK)) &&
+ (sva + 0x100000 < eva)) {
+ *pd = 0;
+ sva = sva + 0x100000;
+ continue;
+ } else {
+ pmap_demote(pm, sva);
+ l2b = pmap_get_l2_bucket(pm, sva);
+ }
+ } else {
+ sva = next_bucket;
+ continue;
+ }
}
ptep = &l2b->l2b_kva[l2pte_index(sva)];
@@ -3857,6 +4173,10 @@
pt_entry_t pte;
vm_paddr_t pa;
+ if ((*ptep & L2_TYPE_MASK) == L2_TYPE_L)
+ /* 64KB superpage. */
+ pmap_demote(pm, sva);
+
pte = *ptep;
if (pte == 0) {
@@ -4017,7 +4337,6 @@
_arm_bzero((void *)(phys + off), size, IS_PHYSICAL) == 0)
return;
-
/*
* Hook in the page, zero it, and purge the cache for that
* zeroed page. Invalidate the TLB as needed.
@@ -4085,7 +4404,11 @@
l2b = pmap_get_l2_bucket(pmap_kernel(), va);
+ if (!l2b)
+ panic("fuck2");
sptep = ptep = &l2b->l2b_kva[l2pte_index(va)];
+ if ((*ptep & L2_TYPE_MASK) == L2_TYPE_L)
+ panic("bordel");
while (va < next_bucket) {
pte = *ptep;
@@ -4826,4 +5149,3 @@
return (NULL);
}
-
==== //depot/projects/superpages/src/sys/arm/include/pmap.h#3 (text+ko) ====
@@ -479,6 +479,7 @@
#define PVF_EXEC 0x10 /* mapping is executable */
#define PVF_UNC 0x20 /* mapping is 'user' non-cacheable */
#define PVF_KNC 0x40 /* mapping is 'kernel' non-cacheable */
+#define PVF_DEMOTED 0x80 /* Page was demoted. */
#define PVF_NC (PVF_UNC|PVF_KNC)
void vector_page_setprot(int);
More information about the p4-projects
mailing list