PERFORCE change 94863 for review
Olivier Houchard
cognet at FreeBSD.org
Sun Apr 9 19:40:24 UTC 2006
http://perforce.freebsd.org/chv.cgi?CH=94863
Change 94863 by cognet at cognet on 2006/04/09 19:39:31
Be a bit smarter when promoting pages, and don't do it if different
small pages into the larger page have different cache mode.
This let us mapping PTE as write-through again, instead of forcing
writa-back, however that means we have to demote some kernel
superpage, if one of them contains PTEs.
Affected files ...
.. //depot/projects/superpages/src/sys/arm/arm/pmap.c#9 edit
.. //depot/projects/superpages/src/sys/arm/include/pmap.h#8 edit
Differences ...
==== //depot/projects/superpages/src/sys/arm/arm/pmap.c#9 (text+ko) ====
@@ -220,6 +220,7 @@
static void pmap_promote_large(pmap_t, vm_offset_t va,
reservation_t);
static void pmap_demote(pmap_t, vm_offset_t);
+static void pmap_demote_large(pmap_t, vm_offset_t);
static MALLOC_DEFINE(M_VMPMAP, "pmap", "PMAP L1");
@@ -286,6 +287,11 @@
char *_tmppt;
+#ifdef ARM_USE_SMALL_ALLOC
+extern struct mtx smallalloc_mtx;
+extern vm_offset_t alloc_curaddr;
+extern vm_offset_t alloc_firstaddr;
+#endif
/*
* Metadata for L1 translation tables.
*/
@@ -619,11 +625,9 @@
pte_l2_s_cache_mode = L2_B|L2_C;
pte_l2_s_cache_mask = L2_S_CACHE_MASK_xscale;
-#if 0
pte_l1_s_cache_mode_pt = L1_S_C;
pte_l2_l_cache_mode_pt = L2_C;
pte_l2_s_cache_mode_pt = L2_C;
-#endif
#ifdef XSCALE_CACHE_READ_WRITE_ALLOCATE
/*
* The XScale core has an enhanced mode where writes that
@@ -682,9 +686,6 @@
xscale_use_minidata = 1;
#endif
- pte_l1_s_cache_mode_pt = pte_l1_s_cache_mode;
- pte_l2_l_cache_mode_pt = pte_l2_s_cache_mode;
- pte_l2_s_cache_mode_pt = pte_l2_s_cache_mode;
pte_l2_s_prot_u = L2_S_PROT_U_xscale;
pte_l2_s_prot_w = L2_S_PROT_W_xscale;
pte_l2_s_prot_mask = L2_S_PROT_MASK_xscale;
@@ -1099,9 +1100,7 @@
#ifndef PMAP_INCLUDE_PTE_SYNC
struct l2_bucket *l2b;
pt_entry_t *ptep, pte;
-#ifdef ARM_USE_SMALL_ALLOC
pd_entry_t *pde;
-#endif
vm_offset_t va = (vm_offset_t)mem & ~PAGE_MASK;
/*
@@ -1112,33 +1111,38 @@
* page tables, we simply fix up the cache-mode here if it's not
* correct.
*/
+ pde = &kernel_pmap->pm_l1->l1_kva[L1_IDX(va)];
#ifdef ARM_USE_SMALL_ALLOC
- pde = &kernel_pmap->pm_l1->l1_kva[L1_IDX(va)];
- if (l1pte_section_p(*pde) && va < 0xd0000000 && va > virtual_avail)
- panic("ouin");
- if (!l1pte_section_p(*pde)) {
+ if (l1pte_section_p(*pde) && va >= alloc_firstaddr)
+ return (0);
#endif
+ if (l1pte_section_p(*pde) && (*pde & pte_l1_s_cache_mask) !=
+ pte_l1_s_cache_mode_pt)
+ pmap_demote_large(pmap_kernel(), va);
+ else if (l1pte_section_p(*pde))
+ return (0);
+
+ l2b = pmap_get_l2_bucket(pmap_kernel(), va);
+ ptep = &l2b->l2b_kva[l2pte_index(va)];
+ if ((*ptep & L2_TYPE_MASK) == L2_TYPE_L &&
+ (*ptep & L2_L_CACHE_MASK) != pte_l2_l_cache_mode_pt) {
+ pmap_demote(pmap_kernel(), va);
l2b = pmap_get_l2_bucket(pmap_kernel(), va);
- ptep = &l2b->l2b_kva[l2pte_index(va)];
- if ((*ptep & L2_TYPE_MASK) == L2_TYPE_L)
- panic("fuxor");
- pte = *ptep;
+ }
+ pte = *ptep;
- if ((pte & L2_S_CACHE_MASK) != pte_l2_s_cache_mode_pt) {
- /*
- * Page tables must have the cache-mode set to
- * Write-Thru.
- */
- *ptep = (pte & ~L2_S_CACHE_MASK) | pte_l2_s_cache_mode_pt;
- PTE_SYNC(ptep);
- cpu_tlb_flushD_SE(va);
- cpu_cpwait();
- }
+ if ((pte & L2_S_CACHE_MASK) != pte_l2_s_cache_mode_pt) {
+ /*
+ * Page tables must have the cache-mode set to
+ * Write-Thru.
+ */
+ *ptep = (pte & ~L2_S_CACHE_MASK) | pte_l2_s_cache_mode_pt;
+ PTE_SYNC(ptep);
+ cpu_tlb_flushD_SE(va);
+ cpu_cpwait();
-#ifdef ARM_USE_SMALL_ALLOC
}
#endif
-#endif
memset(mem, 0, L2_TABLE_SIZE_REAL);
PTE_SYNC_RANGE(mem, L2_TABLE_SIZE_REAL / sizeof(pt_entry_t));
return (0);
@@ -1551,6 +1555,7 @@
int i;
pt_entry_t *ptep;
vm_paddr_t pa0;
+ int cachemode;
ptep = &l2b->l2b_kva[l2pte_index(va & L2_L_FRAME)];
pa0 = *ptep & L2_L_FRAME;
@@ -1574,10 +1579,14 @@
if (*ptep & L2_S_PROT_W)
pa0 |= L2_L_PROT_W;
/* Let's do it. */
+ if ((*ptep & pte_l2_s_cache_mask) == pte_l2_s_cache_mode)
+ cachemode = pte_l2_l_cache_mode;
+ else
+ cachemode = pte_l2_l_cache_mode_pt;
for (i = 0; i < 0x10; i++) {
pmap_tlb_flushID_SE(pmap, va + i * PAGE_SIZE);
ptep[i] = pa0 | L2_L_PROTO |
- pte_l2_l_cache_mode;
+ cachemode;
}
}
@@ -1592,6 +1601,7 @@
struct l1_ttable *l1;
uint16_t l1idx;
int i;
+ int cachemode, cachemode2;
va0 = va & L1_S_ADDR_MASK;
l1idx = L1_IDX(va0);
@@ -1614,18 +1624,43 @@
for (i = 1; i < 0x100; i++) {
vm_paddr_t pa, pa2;
- if ((pt[i] & L2_TYPE_MASK) == L2_TYPE_L)
+ if ((pt[i] & L2_TYPE_MASK) == L2_TYPE_L) {
pa = (pt[i] & L2_L_FRAME) + (i & 0xf) * PAGE_SIZE;
- else
+ if ((pt[i] & pte_l2_l_cache_mask) ==
+ pte_l2_l_cache_mode)
+ cachemode = pte_l1_s_cache_mode;
+ else
+ cachemode = pte_l1_s_cache_mode_pt;
+ } else {
pa = pt[i] & L2_S_FRAME;
- if ((pt[i - 1] & L2_TYPE_MASK) == L2_TYPE_L)
+ if ((pt[i] & pte_l2_s_cache_mask) ==
+ pte_l2_s_cache_mode)
+ cachemode = pte_l1_s_cache_mode;
+ else
+ cachemode = pte_l1_s_cache_mode_pt;
+ }
+ if ((pt[i - 1] & L2_TYPE_MASK) == L2_TYPE_L) {
pa2 = (pt[i - 1] & L2_L_FRAME) +
((i - 1) & 0xf) * PAGE_SIZE;
- else
+ if ((pt[i - 1] & pte_l2_l_cache_mask) ==
+ pte_l2_l_cache_mode)
+ cachemode2 = pte_l1_s_cache_mode;
+ else
+ cachemode2 = pte_l1_s_cache_mode_pt;
+
+ } else {
pa2 = pt[i - 1] & L2_S_FRAME;
+ if ((pt[i - 1] & pte_l2_s_cache_mask) ==
+ pte_l2_s_cache_mode)
+ cachemode2 = pte_l1_s_cache_mode;
+ else
+ cachemode2 = pte_l1_s_cache_mode_pt;
+ }
if (pa != pa2 + PAGE_SIZE)
/* Nothing much we can do. */
return;
+ if (cachemode != cachemode2)
+ return;
}
#ifdef SP_DEBUG
printf("promoting large %x\n", va);
@@ -1635,7 +1670,7 @@
pa |= L1_S_PROT_U;
if (*pt & L2_S_PROT_W)
pa |= L1_S_PROT_W;
- *pd = L1_S_PROTO | pa | pte_l1_s_cache_mode | L1_S_DOM(pmap->pm_domain);
+ *pd = L1_S_PROTO | pa | cachemode | L1_S_DOM(pmap->pm_domain);
pmap_free_l2_bucket(pmap, &l2->l2_bucket[L2_BUCKET(l1idx)], 0x100);
if (pmap == kernel_pmap) {
SLIST_FOREACH(l1, &l1_list, l1_link) {
@@ -1645,6 +1680,69 @@
}
}
+/*
+ * Special case of pmap_demote(), where we know we want to demote a section
+ * mapping into large pages, except for the concerned page.
+ */
+
+static void
+pmap_demote_large(pmap_t pmap, vm_offset_t va)
+{
+ pd_entry_t *pd;
+ pt_entry_t *pt;
+ struct l2_bucket *l2b;
+ struct l1_ttable *l1;
+ vm_offset_t va0;
+ uint16_t l1idx;
+ vm_paddr_t pa;
+ int i;
+ int mode_l = 0, mode_s = 0;
+
+#ifdef SP_DEBUG
+ printf("demoting section mapping at %x\n", va);
+#endif
+ l1idx = L1_IDX(va);
+ pd = &pmap->pm_l1->l1_kva[l1idx];
+ va0 = va & L1_S_ADDR_MASK;
+ if (pmap == pmap_kernel())
+ l2b = pmap_get_l2_bucket(pmap, va0);
+ else
+ l2b = pmap_alloc_l2_bucket(pmap, va0, M_NOWAIT);
+ pa = *pd & L1_S_ADDR_MASK;
+ if (*pd & L1_S_PROT_U) {
+ mode_l |= L2_L_PROT_U;
+ mode_s |= L2_S_PROT_U;
+ }
+ if (*pd & L1_S_PROT_W) {
+ mode_l |= L2_L_PROT_W;
+ mode_s |= L2_S_PROT_W;
+ }
+ if ((*pd & pte_l1_s_cache_mask) == pte_l1_s_cache_mode) {
+ mode_l |= pte_l2_l_cache_mode;
+ mode_s |= pte_l2_s_cache_mode;
+ } else {
+ mode_l |= pte_l2_l_cache_mode_pt;
+ mode_s |= pte_l2_s_cache_mode_pt;
+ }
+ pt = &l2b->l2b_kva[l2pte_index(va0)];
+ *pd = l2b->l2b_phys | L1_C_DOM(pmap->pm_domain) | L1_C_PROTO;
+ l2b->l2b_occupancy += 0x100;
+ if (pmap == kernel_pmap) {
+ SLIST_FOREACH(l1, &l1_list, l1_link) {
+ l1->l1_kva[l1idx] = *pd;
+ PTE_SYNC(&l1->l1_kva[l1idx]);
+ }
+ }
+ for (i = 0; i < 0x100; i++, pa++) {
+ if (va0 + i * PAGE_SIZE != va)
+ pt[i] = (pa & L2_L_FRAME) | mode_l;
+ else
+ pt[i] = pa | mode_s;
+ }
+ pmap_tlb_flushID(pmap);
+
+}
+
static void
pmap_demote(pmap_t pmap, vm_offset_t va)
{
@@ -1657,6 +1755,7 @@
uint16_t demote_size;
vm_paddr_t pa;
int i;
+ int cachemode;
#ifdef SP_DEBUG
printf("demoting %x\n", va);
@@ -1675,6 +1774,10 @@
pa |= L2_S_PROT_U;
if (*pd & L1_S_PROT_W)
pa |= L2_S_PROT_W;
+ if ((*pd & pte_l1_s_cache_mask) == pte_l1_s_cache_mode)
+ cachemode = pte_l2_s_cache_mode;
+ else
+ cachemode = pte_l2_s_cache_mode_pt;
pt = &l2b->l2b_kva[l2pte_index(va0)];
*pd = l2b->l2b_phys | L1_C_DOM(pmap->pm_domain) | L1_C_PROTO;
l2b->l2b_occupancy += 0x100;
@@ -1696,10 +1799,14 @@
pa |= L2_S_PROT_U;
if (*pt & L2_L_PROT_W)
pa |= L2_S_PROT_W;
+ if ((*pt & pte_l2_l_cache_mask) == pte_l2_l_cache_mode)
+ cachemode = pte_l2_s_cache_mode;
+ else
+ cachemode = pte_l2_s_cache_mode_pt;
}
pa |= L2_S_PROTO;
for (i = 0; i < demote_size; i++, pa += PAGE_SIZE)
- pt[i] = (pa) | pte_l2_s_cache_mode;
+ pt[i] = (pa) | cachemode;
pmap_tlb_flushID(pmap);
}
@@ -2661,11 +2768,6 @@
* (physical) address starting relative to 0]
*/
#define PMAP_STATIC_L2_SIZE 16
-#ifdef ARM_USE_SMALL_ALLOC
-extern struct mtx smallalloc_mtx;
-extern vm_offset_t alloc_curaddr;
-extern vm_offset_t alloc_firstaddr;
-#endif
void
pmap_bootstrap(vm_offset_t firstaddr, vm_offset_t lastaddr, struct pv_addr *l1pt)
==== //depot/projects/superpages/src/sys/arm/include/pmap.h#8 (text+ko) ====
@@ -337,7 +337,7 @@
#define PMAP_NEEDS_PTE_SYNC 1
#define PMAP_INCLUDE_PTE_SYNC
#elif (ARM_MMU_SA1 == 0)
-#if 1
+#if defined(CPU_ARM9) && !defined(ARM9_CACHE_WRITE_THROUGH)
#define PMAP_NEEDS_PTE_SYNC 1
#define PMAP_INCLUDE_PTE_SYNC
#else
More information about the p4-projects
mailing list