svn commit: r196705 - in head/sys/i386: i386 include

From: John Baldwin <jhb_at_FreeBSD.org>
Date: Mon, 31 Aug 2009 17:42:52 +0000 (UTC)
Author: jhb
Date: Mon Aug 31 17:42:52 2009
New Revision: 196705
URL: http://svn.freebsd.org/changeset/base/196705

Log:
  Improve pmap_change_attr() so that it is able to demote a large (2/4MB)
  page into 4KB pages as needed.  This should be fairly rare in practice
  on i386.  This includes merging the following changes from the amd64 pmap:
  180430, 180485, 180845, 181043, 181077, and 196318.
  - Add basic support for changing attributes on PDEs to pmap_change_attr()
    similar to the support in the initial version of pmap_change_attr() on
    amd64 including inlines for pmap_pde_attr() and pmap_pte_attr().
  - Extend pmap_demote_pde() to include the ability to instantiate a new page
    table page where none existed before.
  - Enhance pmap_change_attr().  Use pmap_demote_pde() to demote a 2/4MB page
    mapping to 4KB page mappings when the specified attribute change only
    applies to a portion of the 2/4MB page.  Previously, in such cases,
    pmap_change_attr() gave up and returned an error.
  - Correct a critical accounting error in pmap_demote_pde().
  
  Reviewed by:	alc
  MFC after:	3 days

Modified:
  head/sys/i386/i386/pmap.c
  head/sys/i386/include/pmap.h

Modified: head/sys/i386/i386/pmap.c
==============================================================================
--- head/sys/i386/i386/pmap.c	Mon Aug 31 17:34:11 2009	(r196704)
+++ head/sys/i386/i386/pmap.c	Mon Aug 31 17:42:52 2009	(r196705)
_at__at_ -288,12 +288,15 _at__at_ static boolean_t pmap_enter_pde(pmap_t p
 static vm_page_t pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va,
     vm_page_t m, vm_prot_t prot, vm_page_t mpte);
 static void pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte);
+static void pmap_fill_ptp(pt_entry_t *firstpte, pt_entry_t newpte);
 static boolean_t pmap_is_modified_pvh(struct md_page *pvh);
 static void pmap_kenter_attr(vm_offset_t va, vm_paddr_t pa, int mode);
 static vm_page_t pmap_lookup_pt_page(pmap_t pmap, vm_offset_t va);
+static void pmap_pde_attr(pd_entry_t *pde, int cache_bits);
 static void pmap_promote_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va);
 static boolean_t pmap_protect_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t sva,
     vm_prot_t prot);
+static void pmap_pte_attr(pt_entry_t *pte, int cache_bits);
 static void pmap_remove_pde(pmap_t pmap, pd_entry_t *pdq, vm_offset_t sva,
     vm_page_t *free);
 static int pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq, vm_offset_t sva,
_at__at_ -2289,32 +2292,62 _at__at_ pmap_pv_insert_pde(pmap_t pmap, vm_offse
 }
 
 /*
- * Tries to demote a 2- or 4MB page mapping.
+ * Fills a page table page with mappings to consecutive physical pages.
+ */
+static void
+pmap_fill_ptp(pt_entry_t *firstpte, pt_entry_t newpte)
+{
+	pt_entry_t *pte;
+
+	for (pte = firstpte; pte < firstpte + NPTEPG; pte++) {
+		*pte = newpte;	
+		newpte += PAGE_SIZE;
+	}
+}
+
+/*
+ * Tries to demote a 2- or 4MB page mapping.  If demotion fails, the
+ * 2- or 4MB page mapping is invalidated.
  */
 static boolean_t
 pmap_demote_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va)
 {
 	pd_entry_t newpde, oldpde;
 	pmap_t allpmaps_entry;
-	pt_entry_t *firstpte, newpte, *pte;
+	pt_entry_t *firstpte, newpte;
 	vm_paddr_t mptepa;
 	vm_page_t free, mpte;
 
 	PMAP_LOCK_ASSERT(pmap, MA_OWNED);
+	oldpde = *pde;
+	KASSERT((oldpde & (PG_PS | PG_V)) == (PG_PS | PG_V),
+	    ("pmap_demote_pde: oldpde is missing PG_PS and/or PG_V"));
 	mpte = pmap_lookup_pt_page(pmap, va);
 	if (mpte != NULL)
 		pmap_remove_pt_page(pmap, mpte);
 	else {
-		KASSERT((*pde & PG_W) == 0,
+		KASSERT((oldpde & PG_W) == 0,
 		    ("pmap_demote_pde: page table page for a wired mapping"
 		    " is missing"));
-		free = NULL;
-		pmap_remove_pde(pmap, pde, trunc_4mpage(va), &free);
-		pmap_invalidate_page(pmap, trunc_4mpage(va));
-		pmap_free_zero_pages(free);
-		CTR2(KTR_PMAP, "pmap_demote_pde: failure for va %#x"
-		    " in pmap %p", va, pmap);
-		return (FALSE);
+
+		/*
+		 * Invalidate the 2- or 4MB page mapping and return
+		 * "failure" if the mapping was never accessed or the
+		 * allocation of the new page table page fails.
+		 */
+		if ((oldpde & PG_A) == 0 || (mpte = vm_page_alloc(NULL,
+		    va >> PDRSHIFT, VM_ALLOC_NOOBJ | VM_ALLOC_NORMAL |
+		    VM_ALLOC_WIRED)) == NULL) {
+			free = NULL;
+			pmap_remove_pde(pmap, pde, trunc_4mpage(va), &free);
+			pmap_invalidate_page(pmap, trunc_4mpage(va));
+			pmap_free_zero_pages(free);
+			CTR2(KTR_PMAP, "pmap_demote_pde: failure for va %#x"
+			    " in pmap %p", va, pmap);
+			return (FALSE);
+		}
+		if (va < VM_MAXUSER_ADDRESS)
+			pmap->pm_stats.resident_count++;
 	}
 	mptepa = VM_PAGE_TO_PHYS(mpte);
 
_at__at_ -2348,30 +2381,32 _at__at_ pmap_demote_pde(pmap_t pmap, pd_entry_t 
 		}
 		firstpte = PADDR2;
 	}
-	oldpde = *pde;
 	newpde = mptepa | PG_M | PG_A | (oldpde & PG_U) | PG_RW | PG_V;
-	KASSERT((oldpde & (PG_A | PG_V)) == (PG_A | PG_V),
-	    ("pmap_demote_pde: oldpde is missing PG_A and/or PG_V"));
+	KASSERT((oldpde & PG_A) != 0,
+	    ("pmap_demote_pde: oldpde is missing PG_A"));
 	KASSERT((oldpde & (PG_M | PG_RW)) != PG_RW,
 	    ("pmap_demote_pde: oldpde is missing PG_M"));
-	KASSERT((oldpde & PG_PS) != 0,
-	    ("pmap_demote_pde: oldpde is missing PG_PS"));
 	newpte = oldpde & ~PG_PS;
 	if ((newpte & PG_PDE_PAT) != 0)
 		newpte ^= PG_PDE_PAT | PG_PTE_PAT;
 
 	/*
-	 * If the mapping has changed attributes, update the page table
-	 * entries.
-	 */ 
+	 * If the page table page is new, initialize it.
+	 */
+	if (mpte->wire_count == 1) {
+		mpte->wire_count = NPTEPG;
+		pmap_fill_ptp(firstpte, newpte);
+	}
 	KASSERT((*firstpte & PG_FRAME) == (newpte & PG_FRAME),
 	    ("pmap_demote_pde: firstpte and newpte map different physical"
 	    " addresses"));
+
+	/*
+	 * If the mapping has changed attributes, update the page table
+	 * entries.
+	 */ 
 	if ((*firstpte & PG_PTE_PROMOTE) != (newpte & PG_PTE_PROMOTE))
-		for (pte = firstpte; pte < firstpte + NPTEPG; pte++) {
-			*pte = newpte;	
-			newpte += PAGE_SIZE;
-		}
+		pmap_fill_ptp(firstpte, newpte);
 	
 	/*
 	 * Demote the mapping.  This pmap is locked.  The old PDE has
_at__at_ -4426,6 +4461,40 _at__at_ pmap_clear_reference(vm_page_t m)
  * Miscellaneous support routines follow
  */
 
+/* Adjust the cache mode for a 4KB page mapped via a PTE. */
+static __inline void
+pmap_pte_attr(pt_entry_t *pte, int cache_bits)
+{
+	u_int opte, npte;
+
+	/*
+	 * The cache mode bits are all in the low 32-bits of the
+	 * PTE, so we can just spin on updating the low 32-bits.
+	 */
+	do {
+		opte = *(u_int *)pte;
+		npte = opte & ~PG_PTE_CACHE;
+		npte |= cache_bits;
+	} while (npte != opte && !atomic_cmpset_int((u_int *)pte, opte, npte));
+}
+
+/* Adjust the cache mode for a 2/4MB page mapped via a PDE. */
+static __inline void
+pmap_pde_attr(pd_entry_t *pde, int cache_bits)
+{
+	u_int opde, npde;
+
+	/*
+	 * The cache mode bits are all in the low 32-bits of the
+	 * PDE, so we can just spin on updating the low 32-bits.
+	 */
+	do {
+		opde = *(u_int *)pde;
+		npde = opde & ~PG_PDE_CACHE;
+		npde |= cache_bits;
+	} while (npde != opde && !atomic_cmpset_int((u_int *)pde, opde, npde));
+}
+
 /*
  * Map a set of physical memory pages into the kernel virtual
  * address space. Return a pointer to where it is mapped. This
_at__at_ -4537,13 +4606,23 _at__at_ pmap_page_set_memattr(vm_page_t m, vm_me
 	}
 }
 
+/*
+ * Changes the specified virtual address range's memory type to that given by
+ * the parameter "mode".  The specified virtual address range must be
+ * completely contained within either the kernel map.
+ *
+ * Returns zero if the change completed successfully, and either EINVAL or
+ * ENOMEM if the change failed.  Specifically, EINVAL is returned if some part
+ * of the virtual address range was not mapped, and ENOMEM is returned if
+ * there was insufficient memory available to complete the change.
+ */
 int
 pmap_change_attr(vm_offset_t va, vm_size_t size, int mode)
 {
 	vm_offset_t base, offset, tmpva;
-	pt_entry_t *pte;
-	u_int opte, npte;
 	pd_entry_t *pde;
+	pt_entry_t *pte;
+	int cache_bits_pte, cache_bits_pde;
 	boolean_t changed;
 
 	base = trunc_page(va);
_at__at_ -4556,47 +4635,93 _at__at_ pmap_change_attr(vm_offset_t va, vm_size
 	if (base < VM_MIN_KERNEL_ADDRESS)
 		return (EINVAL);
 
-	/* 4MB pages and pages that aren't mapped aren't supported. */
-	for (tmpva = base; tmpva < (base + size); tmpva += PAGE_SIZE) {
+	cache_bits_pde = cache_bits_pte = -1;
+	changed = FALSE;
+
+	/*
+	 * Pages that aren't mapped aren't supported.  Also break down
+	 * 2/4MB pages into 4KB pages if required.
+	 */
+	PMAP_LOCK(kernel_pmap);
+	for (tmpva = base; tmpva < base + size; ) {
 		pde = pmap_pde(kernel_pmap, tmpva);
-		if (*pde & PG_PS)
-			return (EINVAL);
-		if (*pde == 0)
+		if (*pde == 0) {
+			PMAP_UNLOCK(kernel_pmap);
 			return (EINVAL);
+		}
+		if (*pde & PG_PS) {
+			/*
+			 * If the current 2/4MB page already has
+			 * the required memory type, then we need not
+			 * demote this page.  Just increment tmpva to
+			 * the next 2/4MB page frame.
+			 */
+			if (cache_bits_pde < 0)
+				cache_bits_pde = pmap_cache_bits(mode, 1);
+			if ((*pde & PG_PDE_CACHE) == cache_bits_pde) {
+				tmpva = trunc_4mpage(tmpva) + NBPDR;
+				continue;
+			}
+
+			/*
+			 * If the current offset aligns with a 2/4MB
+			 * page frame and there is at least 2/4MB left
+			 * within the range, then we need not break
+			 * down this page into 4KB pages.
+			 */
+			if ((tmpva & PDRMASK) == 0 &&
+			    tmpva + PDRMASK < base + size) {
+				tmpva += NBPDR;
+				continue;
+			}
+			if (!pmap_demote_pde(kernel_pmap, pde, tmpva)) {
+				PMAP_UNLOCK(kernel_pmap);
+				return (ENOMEM);
+			}
+		}
 		pte = vtopte(tmpva);
-		if (*pte == 0)
+		if (*pte == 0) {
+			PMAP_UNLOCK(kernel_pmap);
 			return (EINVAL);
+		}
+		tmpva += PAGE_SIZE;
 	}
+	PMAP_UNLOCK(kernel_pmap);
 
 	changed = FALSE;
 
 	/*
-	 * Ok, all the pages exist and are 4k, so run through them updating
-	 * their cache mode.
+	 * Ok, all the pages exist, so run through them updating their
+	 * cache mode if required.
 	 */
-	for (tmpva = base; size > 0; ) {
-		pte = vtopte(tmpva);
-
-		/*
-		 * The cache mode bits are all in the low 32-bits of the
-		 * PTE, so we can just spin on updating the low 32-bits.
-		 */
-		do {
-			opte = *(u_int *)pte;
-			npte = opte & ~(PG_PTE_PAT | PG_NC_PCD | PG_NC_PWT);
-			npte |= pmap_cache_bits(mode, 0);
-		} while (npte != opte &&
-		    !atomic_cmpset_int((u_int *)pte, opte, npte));
-		if (npte != opte)
-			changed = TRUE;
-		tmpva += PAGE_SIZE;
-		size -= PAGE_SIZE;
+	for (tmpva = base; tmpva < base + size; ) {
+		pde = pmap_pde(kernel_pmap, tmpva);
+		if (*pde & PG_PS) {
+			if (cache_bits_pde < 0)
+				cache_bits_pde = pmap_cache_bits(mode, 1);
+			if ((*pde & PG_PDE_CACHE) != cache_bits_pde) {
+				pmap_pde_attr(pde, cache_bits_pde);
+				if (!changed)
+					changed = TRUE;
+			}
+			tmpva = trunc_4mpage(tmpva) + NBPDR;
+		} else {
+			if (cache_bits_pte < 0)
+				cache_bits_pte = pmap_cache_bits(mode, 0);
+			pte = vtopte(tmpva);
+			if ((*pte & PG_PTE_CACHE) != cache_bits_pte) {
+				pmap_pte_attr(pte, cache_bits_pte);
+				if (!changed)
+					changed = TRUE;
+			}
+			tmpva += PAGE_SIZE;
+		}
 	}
 
 	/*
-	 * Flush CPU caches to make sure any data isn't cached that shouldn't
-	 * be, etc.
-	 */    
+	 * Flush CPU caches to make sure any data isn't cached that
+	 * shouldn't be, etc.
+	 */
 	if (changed) {
 		pmap_invalidate_range(kernel_pmap, base, tmpva);
 		pmap_invalidate_cache_range(base, tmpva);

Modified: head/sys/i386/include/pmap.h
==============================================================================
--- head/sys/i386/include/pmap.h	Mon Aug 31 17:34:11 2009	(r196704)
+++ head/sys/i386/include/pmap.h	Mon Aug 31 17:42:52 2009	(r196705)
_at__at_ -81,6 +81,10 _at__at_
 #define	PG_PROT		(PG_RW|PG_U)	/* all protection bits . */
 #define PG_N		(PG_NC_PWT|PG_NC_PCD)	/* Non-cacheable */
 
+/* Page level cache control fields used to determine the PAT type */
+#define PG_PDE_CACHE	(PG_PDE_PAT | PG_NC_PWT | PG_NC_PCD)
+#define PG_PTE_CACHE	(PG_PTE_PAT | PG_NC_PWT | PG_NC_PCD)
+
 /*
  * Promotion to a 2 or 4MB (PDE) page mapping requires that the corresponding
  * 4KB (PTE) page mappings have identical settings for the following fields:
Received on Mon Aug 31 2009 - 17:42:53 UTC