svn commit: r220803 - in head/sys: amd64/amd64 amd64/include
i386/i386 i386/include
Konstantin Belousov
kib at FreeBSD.org
Mon Apr 18 21:24:43 UTC 2011
Author: kib
Date: Mon Apr 18 21:24:42 2011
New Revision: 220803
URL: http://svn.freebsd.org/changeset/base/220803
Log:
Make pmap_invalidate_cache_range() available for consumption on amd64.
Add pmap_invalidate_cache_pages() method on x86. It flushes the CPU
cache for the set of pages, which are not neccessary mapped. Since its
supposed use is to prepare the move of the pages ownership to a device
that does not snoop all CPU accesses to the main memory (read GPU in
GMCH), do not rely on CPU self-snoop feature.
amd64 implementation takes advantage of the direct map. On i386,
extract the helper pmap_flush_page() from pmap_page_set_memattr(), and
use it to make a temporary mapping of the flushed page.
Reviewed by: alc
Sponsored by: The FreeBSD Foundation
MFC after: 3 weeks
Modified:
head/sys/amd64/amd64/pmap.c
head/sys/amd64/include/pmap.h
head/sys/i386/i386/pmap.c
head/sys/i386/include/pmap.h
Modified: head/sys/amd64/amd64/pmap.c
==============================================================================
--- head/sys/amd64/amd64/pmap.c Mon Apr 18 21:18:22 2011 (r220802)
+++ head/sys/amd64/amd64/pmap.c Mon Apr 18 21:24:42 2011 (r220803)
@@ -239,7 +239,6 @@ static vm_page_t pmap_enter_quick_locked
vm_page_t m, vm_prot_t prot, vm_page_t mpte);
static void pmap_fill_ptp(pt_entry_t *firstpte, pt_entry_t newpte);
static void pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte);
-static void pmap_invalidate_cache_range(vm_offset_t sva, vm_offset_t eva);
static boolean_t pmap_is_modified_pvh(struct md_page *pvh);
static boolean_t pmap_is_referenced_pvh(struct md_page *pvh);
static void pmap_kenter_attr(vm_offset_t va, vm_paddr_t pa, int mode);
@@ -1105,7 +1104,9 @@ pmap_update_pde(pmap_t pmap, vm_offset_t
}
#endif /* !SMP */
-static void
+#define PMAP_CLFLUSH_THRESHOLD (2 * 1024 * 1024)
+
+void
pmap_invalidate_cache_range(vm_offset_t sva, vm_offset_t eva)
{
@@ -1117,7 +1118,7 @@ pmap_invalidate_cache_range(vm_offset_t
if (cpu_feature & CPUID_SS)
; /* If "Self Snoop" is supported, do nothing. */
else if ((cpu_feature & CPUID_CLFSH) != 0 &&
- eva - sva < 2 * 1024 * 1024) {
+ eva - sva < PMAP_CLFLUSH_THRESHOLD) {
/*
* Otherwise, do per-cache line flush. Use the mfence
@@ -1142,6 +1143,34 @@ pmap_invalidate_cache_range(vm_offset_t
}
/*
+ * Remove the specified set of pages from the data and instruction caches.
+ *
+ * In contrast to pmap_invalidate_cache_range(), this function does not
+ * rely on the CPU's self-snoop feature, because it is intended for use
+ * when moving pages into a different cache domain.
+ */
+void
+pmap_invalidate_cache_pages(vm_page_t *pages, int count)
+{
+ vm_offset_t daddr, eva;
+ int i;
+
+ if (count >= PMAP_CLFLUSH_THRESHOLD / PAGE_SIZE ||
+ (cpu_feature & CPUID_CLFSH) == 0)
+ pmap_invalidate_cache();
+ else {
+ mfence();
+ for (i = 0; i < count; i++) {
+ daddr = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(pages[i]));
+ eva = daddr + PAGE_SIZE;
+ for (; daddr < eva; daddr += cpu_clflush_line_size)
+ clflush(daddr);
+ }
+ mfence();
+ }
+}
+
+/*
* Are we current address space or kernel?
*/
static __inline int
Modified: head/sys/amd64/include/pmap.h
==============================================================================
--- head/sys/amd64/include/pmap.h Mon Apr 18 21:18:22 2011 (r220802)
+++ head/sys/amd64/include/pmap.h Mon Apr 18 21:24:42 2011 (r220803)
@@ -328,6 +328,8 @@ void pmap_invalidate_page(pmap_t, vm_off
void pmap_invalidate_range(pmap_t, vm_offset_t, vm_offset_t);
void pmap_invalidate_all(pmap_t);
void pmap_invalidate_cache(void);
+void pmap_invalidate_cache_pages(vm_page_t *pages, int count);
+void pmap_invalidate_cache_range(vm_offset_t sva, vm_offset_t eva);
#endif /* _KERNEL */
Modified: head/sys/i386/i386/pmap.c
==============================================================================
--- head/sys/i386/i386/pmap.c Mon Apr 18 21:18:22 2011 (r220802)
+++ head/sys/i386/i386/pmap.c Mon Apr 18 21:24:42 2011 (r220803)
@@ -297,6 +297,7 @@ static boolean_t pmap_enter_pde(pmap_t p
vm_prot_t prot);
static vm_page_t pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va,
vm_page_t m, vm_prot_t prot, vm_page_t mpte);
+static void pmap_flush_page(vm_page_t m);
static void pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte);
static void pmap_fill_ptp(pt_entry_t *firstpte, pt_entry_t newpte);
static boolean_t pmap_is_modified_pvh(struct md_page *pvh);
@@ -1136,6 +1137,8 @@ pmap_update_pde(pmap_t pmap, vm_offset_t
}
#endif /* !SMP */
+#define PMAP_CLFLUSH_THRESHOLD (2 * 1024 * 1024)
+
void
pmap_invalidate_cache_range(vm_offset_t sva, vm_offset_t eva)
{
@@ -1148,7 +1151,7 @@ pmap_invalidate_cache_range(vm_offset_t
if (cpu_feature & CPUID_SS)
; /* If "Self Snoop" is supported, do nothing. */
else if ((cpu_feature & CPUID_CLFSH) != 0 &&
- eva - sva < 2 * 1024 * 1024) {
+ eva - sva < PMAP_CLFLUSH_THRESHOLD) {
/*
* Otherwise, do per-cache line flush. Use the mfence
@@ -1172,6 +1175,20 @@ pmap_invalidate_cache_range(vm_offset_t
}
}
+void
+pmap_invalidate_cache_pages(vm_page_t *pages, int count)
+{
+ int i;
+
+ if (count >= PMAP_CLFLUSH_THRESHOLD / PAGE_SIZE ||
+ (cpu_feature & CPUID_CLFSH) == 0) {
+ pmap_invalidate_cache();
+ } else {
+ for (i = 0; i < count; i++)
+ pmap_flush_page(pages[i]);
+ }
+}
+
/*
* Are we current address space or kernel? N.B. We return FALSE when
* a pmap's page table is in use because a kernel thread is borrowing
@@ -4825,8 +4842,6 @@ pmap_unmapdev(vm_offset_t va, vm_size_t
void
pmap_page_set_memattr(vm_page_t m, vm_memattr_t ma)
{
- struct sysmaps *sysmaps;
- vm_offset_t sva, eva;
m->md.pat_mode = ma;
if ((m->flags & PG_FICTITIOUS) != 0)
@@ -4849,25 +4864,42 @@ pmap_page_set_memattr(vm_page_t m, vm_me
* invalidation. In the worst case, whole cache is flushed by
* pmap_invalidate_cache_range().
*/
- if ((cpu_feature & (CPUID_SS|CPUID_CLFSH)) == CPUID_CLFSH) {
+ if ((cpu_feature & CPUID_SS) == 0)
+ pmap_flush_page(m);
+}
+
+static void
+pmap_flush_page(vm_page_t m)
+{
+ struct sysmaps *sysmaps;
+ vm_offset_t sva, eva;
+
+ if ((cpu_feature & CPUID_CLFSH) != 0) {
sysmaps = &sysmaps_pcpu[PCPU_GET(cpuid)];
mtx_lock(&sysmaps->lock);
if (*sysmaps->CMAP2)
- panic("pmap_page_set_memattr: CMAP2 busy");
+ panic("pmap_flush_page: CMAP2 busy");
sched_pin();
*sysmaps->CMAP2 = PG_V | PG_RW | VM_PAGE_TO_PHYS(m) |
PG_A | PG_M | pmap_cache_bits(m->md.pat_mode, 0);
invlcaddr(sysmaps->CADDR2);
sva = (vm_offset_t)sysmaps->CADDR2;
eva = sva + PAGE_SIZE;
- } else
- sva = eva = 0; /* gcc */
- pmap_invalidate_cache_range(sva, eva);
- if (sva != 0) {
+
+ /*
+ * Use mfence despite the ordering implied by
+ * mtx_{un,}lock() because clflush is not guaranteed
+ * to be ordered by any other instruction.
+ */
+ mfence();
+ for (; sva < eva; sva += cpu_clflush_line_size)
+ clflush(sva);
+ mfence();
*sysmaps->CMAP2 = 0;
sched_unpin();
mtx_unlock(&sysmaps->lock);
- }
+ } else
+ pmap_invalidate_cache();
}
/*
Modified: head/sys/i386/include/pmap.h
==============================================================================
--- head/sys/i386/include/pmap.h Mon Apr 18 21:18:22 2011 (r220802)
+++ head/sys/i386/include/pmap.h Mon Apr 18 21:24:42 2011 (r220803)
@@ -522,7 +522,8 @@ void pmap_invalidate_page(pmap_t, vm_off
void pmap_invalidate_range(pmap_t, vm_offset_t, vm_offset_t);
void pmap_invalidate_all(pmap_t);
void pmap_invalidate_cache(void);
-void pmap_invalidate_cache_range(vm_offset_t, vm_offset_t);
+void pmap_invalidate_cache_pages(vm_page_t *pages, int count);
+void pmap_invalidate_cache_range(vm_offset_t sva, vm_offset_t eva);
#endif /* _KERNEL */
More information about the svn-src-head
mailing list