svn commit: r269050 - head/sys/i386/i386

[Marius Strobl]

Author: marius
Date: Thu Jul 24 10:08:02 2014
New Revision: 269050
URL: http://svnweb.freebsd.org/changeset/base/269050

Log:
  - Copying and zeroing pages via temporary mappings involves updating the
    corresponding page tables followed by accesses to the pages in question.
    This sequence is subject to the situation exactly described in the "AMD64
    Architecture Programmer's Manual Volume 2: System Programming" rev. 3.23,
    "7.3.1 Special Coherency Considerations" [1, p. 171 f.]. Therefore, issuing
    the INVLPG right after modifying the PTE bits is crucial.
    For pmap_copy_page(), this has been broken in r124956 and later on carried
    over to pmap_copy_pages() derived from the former, while all other places
    in the i386 PMAP code use the correct order of instructions in this regard.
    Fixing the latter breakage solves the problem of data corruption seen with
    unmapped I/O enabled when running at least bare metal on AMD R-268D APUs.
    However, this might also fix similar corruption reported for virtualized
    environments.
  - In pmap_copy_pages(), correctly set the cache bits on the source page being
    copied. This change is thought to be a NOP for the real world, though. [2]
  
  1: http://amd-dev.wpengine.netdna-cdn.com/wordpress/media/2012/10/24593_APM_v21.pdf
  
  Submitted by:	kib [2]
  Reviewed by:	alc, kib
  MFC after:	3 days
  Sponsored by:	Bally Wulff Games & Entertainment GmbH

Modified:
  head/sys/i386/i386/pmap.c

Modified: head/sys/i386/i386/pmap.c
==============================================================================
--- head/sys/i386/i386/pmap.c	Thu Jul 24 09:40:31 2014	(r269049)
+++ head/sys/i386/i386/pmap.c	Thu Jul 24 10:08:02 2014	(r269050)
@@ -1286,6 +1286,13 @@ pmap_pte_release(pt_entry_t *pte)
 		mtx_unlock(&PMAP2mutex);
 }
 
+/*
+ * NB:  The sequence of updating a page table followed by accesses to the
+ * corresponding pages is subject to the situation described in the "AMD64
+ * Architecture Programmer's Manual Volume 2: System Programming" rev. 3.23,
+ * "7.3.1 Special Coherency Considerations".  Therefore, issuing the INVLPG
+ * right after modifying the PTE bits is crucial.
+ */
 static __inline void
 invlcaddr(void *caddr)
 {
@@ -4333,12 +4340,12 @@ pmap_copy_page(vm_page_t src, vm_page_t 
 	if (*sysmaps->CMAP2)
 		panic("pmap_copy_page: CMAP2 busy");
 	sched_pin();
-	invlpg((u_int)sysmaps->CADDR1);
-	invlpg((u_int)sysmaps->CADDR2);
 	*sysmaps->CMAP1 = PG_V | VM_PAGE_TO_PHYS(src) | PG_A |
 	    pmap_cache_bits(src->md.pat_mode, 0);
+	invlcaddr(sysmaps->CADDR1);
 	*sysmaps->CMAP2 = PG_V | PG_RW | VM_PAGE_TO_PHYS(dst) | PG_A | PG_M |
 	    pmap_cache_bits(dst->md.pat_mode, 0);
+	invlcaddr(sysmaps->CADDR2);
 	bcopy(sysmaps->CADDR1, sysmaps->CADDR2, PAGE_SIZE);
 	*sysmaps->CMAP1 = 0;
 	*sysmaps->CMAP2 = 0;
@@ -4366,8 +4373,6 @@ pmap_copy_pages(vm_page_t ma[], vm_offse
 		panic("pmap_copy_pages: CMAP2 busy");
 	sched_pin();
 	while (xfersize > 0) {
-		invlpg((u_int)sysmaps->CADDR1);
-		invlpg((u_int)sysmaps->CADDR2);
 		a_pg = ma[a_offset >> PAGE_SHIFT];
 		a_pg_offset = a_offset & PAGE_MASK;
 		cnt = min(xfersize, PAGE_SIZE - a_pg_offset);
@@ -4375,9 +4380,11 @@ pmap_copy_pages(vm_page_t ma[], vm_offse
 		b_pg_offset = b_offset & PAGE_MASK;
 		cnt = min(cnt, PAGE_SIZE - b_pg_offset);
 		*sysmaps->CMAP1 = PG_V | VM_PAGE_TO_PHYS(a_pg) | PG_A |
-		    pmap_cache_bits(b_pg->md.pat_mode, 0);
+		    pmap_cache_bits(a_pg->md.pat_mode, 0);
+		invlcaddr(sysmaps->CADDR1);
 		*sysmaps->CMAP2 = PG_V | PG_RW | VM_PAGE_TO_PHYS(b_pg) | PG_A |
 		    PG_M | pmap_cache_bits(b_pg->md.pat_mode, 0);
+		invlcaddr(sysmaps->CADDR2);
 		a_cp = sysmaps->CADDR1 + a_pg_offset;
 		b_cp = sysmaps->CADDR2 + b_pg_offset;
 		bcopy(a_cp, b_cp, cnt);