svn commit: r342769 - head/sys/i386/include

[Konstantin Belousov]

Author: kib
Date: Fri Jan  4 17:33:07 2019
New Revision: 342769
URL: https://svnweb.freebsd.org/changeset/base/342769

Log:
  i386: Use atomic 64bit load to read PDE value from PAE pagetables in
  pmap_kextract().
  
  pmap_kextract() can race with promotion/demotion on the kernel page
  table, in which case current non-atomic 64bit read would see torn
  value, breaking pmap_kextract().  pmap_kextract() would correctly
  handle either promoted or demoted PDE, but not a mix where one word
  is from a different state.
  
  It requires PAE and > 4G memory to reproduce.  We observed this in
  real loads, both for intensive use of malloc(9)/free(9) where
  vtoslab() returned invalid pointer to the slab, and with the use of
  busdma_bounce, where incorrect page was bounced.
  
  In collaboration with:	pho
  Reviewed by:	markj
  Sponsored by:	The FreeBSD Foundation
  MFC after:	1 week
  Differential revision:	https://reviews.freebsd.org/D18714

Modified:
  head/sys/i386/include/pmap.h

Modified: head/sys/i386/include/pmap.h
==============================================================================
--- head/sys/i386/include/pmap.h	Fri Jan  4 17:31:50 2019	(r342768)
+++ head/sys/i386/include/pmap.h	Fri Jan  4 17:33:07 2019	(r342769)
@@ -234,6 +234,32 @@ extern pd_entry_t *IdlePTD;	/* physical address of "Id
  */
 extern pt_entry_t *KPTmap;
 
+#if (defined(PAE) || defined(PAE_TABLES))
+
+#define	pde_cmpset(pdep, old, new)	atomic_cmpset_64_i586(pdep, old, new)
+#define	pte_load_store(ptep, pte)	atomic_swap_64_i586(ptep, pte)
+#define	pte_load_clear(ptep)		atomic_swap_64_i586(ptep, 0)
+#define	pte_store(ptep, pte)		atomic_store_rel_64_i586(ptep, pte)
+#define	pte_load(ptep)			atomic_load_acq_64_i586(ptep)
+
+extern pt_entry_t pg_nx;
+
+#else /* !(PAE || PAE_TABLES) */
+
+#define	pde_cmpset(pdep, old, new)	atomic_cmpset_int(pdep, old, new)
+#define	pte_load_store(ptep, pte)	atomic_swap_int(ptep, pte)
+#define	pte_load_clear(ptep)		atomic_swap_int(ptep, 0)
+#define	pte_store(ptep, pte) do { \
+	*(u_int *)(ptep) = (u_int)(pte); \
+} while (0)
+#define	pte_load(ptep)			atomic_load_acq_int(ptep)
+
+#endif /* !(PAE || PAE_TABLES) */
+
+#define	pte_clear(ptep)			pte_store(ptep, 0)
+
+#define	pde_store(pdep, pde)		pte_store(pdep, pde)
+
 /*
  * Extract from the kernel page table the physical address that is mapped by
  * the given virtual address "va".
@@ -245,7 +271,7 @@ pmap_kextract(vm_offset_t va)
 {
 	vm_paddr_t pa;
 
-	if ((pa = PTD[va >> PDRSHIFT]) & PG_PS) {
+	if ((pa = pte_load(&PTD[va >> PDRSHIFT])) & PG_PS) {
 		pa = (pa & PG_PS_FRAME) | (va & PDRMASK);
 	} else {
 		/*
@@ -260,30 +286,6 @@ pmap_kextract(vm_offset_t va)
 	}
 	return (pa);
 }
-
-#if (defined(PAE) || defined(PAE_TABLES))
-
-#define	pde_cmpset(pdep, old, new)	atomic_cmpset_64_i586(pdep, old, new)
-#define	pte_load_store(ptep, pte)	atomic_swap_64_i586(ptep, pte)
-#define	pte_load_clear(ptep)		atomic_swap_64_i586(ptep, 0)
-#define	pte_store(ptep, pte)		atomic_store_rel_64_i586(ptep, pte)
-
-extern pt_entry_t pg_nx;
-
-#else /* !(PAE || PAE_TABLES) */
-
-#define	pde_cmpset(pdep, old, new)	atomic_cmpset_int(pdep, old, new)
-#define	pte_load_store(ptep, pte)	atomic_swap_int(ptep, pte)
-#define	pte_load_clear(ptep)		atomic_swap_int(ptep, 0)
-#define	pte_store(ptep, pte) do { \
-	*(u_int *)(ptep) = (u_int)(pte); \
-} while (0)
-
-#endif /* !(PAE || PAE_TABLES) */
-
-#define	pte_clear(ptep)			pte_store(ptep, 0)
-
-#define	pde_store(pdep, pde)		pte_store(pdep, pde)
 
 #endif /* _KERNEL */