svn commit: r210124 - head/sys/amd64/amd64

[Alan Cox]

Author: alc
Date: Thu Jul 15 16:25:51 2010
New Revision: 210124
URL: http://svn.freebsd.org/changeset/base/210124

Log:
  Optimize pmap_remove()'s handling of PG_G mappings.  Specifically,
  instead of calling pmap_invalidate_page() for each PG_G mapping, call
  pmap_invalidate_range() for each range of PG_G mappings.  In addition,
  eliminate a redundant call to pmap_invalidate_page().  Both
  pmap_remove_pte() and pmap_remove_page() called pmap_invalidate_page()
  when the mapping had the PG_G attribute.  Now, only pmap_remove_page()
  calls pmap_invalidate_page().  Altogether, these changes eliminate 53%
  of the TLB shootdowns for a "buildworld" on a ZFS file system.  On
  FFS, the reduction is 3%.
  
  MFC after:	6 weeks

Modified:
  head/sys/amd64/amd64/pmap.c

Modified: head/sys/amd64/amd64/pmap.c
==============================================================================
--- head/sys/amd64/amd64/pmap.c	Thu Jul 15 14:43:12 2010	(r210123)
+++ head/sys/amd64/amd64/pmap.c	Thu Jul 15 16:25:51 2010	(r210124)
@@ -2602,12 +2602,6 @@ pmap_remove_pte(pmap_t pmap, pt_entry_t 
 	oldpte = pte_load_clear(ptq);
 	if (oldpte & PG_W)
 		pmap->pm_stats.wired_count -= 1;
-	/*
-	 * Machines that don't support invlpg, also don't support
-	 * PG_G.
-	 */
-	if (oldpte & PG_G)
-		pmap_invalidate_page(kernel_pmap, va);
 	pmap_resident_count_dec(pmap, 1);
 	if (oldpte & PG_MANAGED) {
 		m = PHYS_TO_VM_PAGE(oldpte & PG_FRAME);
@@ -2647,7 +2641,7 @@ pmap_remove_page(pmap_t pmap, vm_offset_
 void
 pmap_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
 {
-	vm_offset_t va_next;
+	vm_offset_t va, va_next;
 	pml4_entry_t *pml4e;
 	pdp_entry_t *pdpe;
 	pd_entry_t ptpaddr, *pde;
@@ -2748,20 +2742,27 @@ pmap_remove(pmap_t pmap, vm_offset_t sva
 		if (va_next > eva)
 			va_next = eva;
 
+		va = va_next;
 		for (pte = pmap_pde_to_pte(pde, sva); sva != va_next; pte++,
 		    sva += PAGE_SIZE) {
-			if (*pte == 0)
+			if (*pte == 0) {
+				if (va != va_next) {
+					pmap_invalidate_range(pmap, va, sva);
+					va = va_next;
+				}
 				continue;
-
-			/*
-			 * The TLB entry for a PG_G mapping is invalidated
-			 * by pmap_remove_pte().
-			 */
+			}
 			if ((*pte & PG_G) == 0)
 				anyvalid = 1;
-			if (pmap_remove_pte(pmap, pte, sva, ptpaddr, &free))
+			else if (va == va_next)
+				va = sva;
+			if (pmap_remove_pte(pmap, pte, sva, ptpaddr, &free)) {
+				sva += PAGE_SIZE;
 				break;
+			}
 		}
+		if (va != va_next)
+			pmap_invalidate_range(pmap, va, sva);
 	}
 out:
 	if (anyvalid)