svn commit: r338807 - in head/sys: amd64/amd64 amd64/include dev/drm2 dev/drm2/i915 i386/i386 i386/include x86/iommu
Konstantin Belousov
kib at FreeBSD.org
Wed Sep 19 19:35:05 UTC 2018
Author: kib
Date: Wed Sep 19 19:35:02 2018
New Revision: 338807
URL: https://svnweb.freebsd.org/changeset/base/338807
Log:
Convert x86 cache invalidation functions to ifuncs.
This simplifies the runtime logic and reduces the number of
runtime-constant branches.
Reviewed by: alc, markj
Sponsored by: The FreeBSD Foundation
Approved by: re (gjb)
Differential revision: https://reviews.freebsd.org/D16736
Modified:
head/sys/amd64/amd64/pmap.c
head/sys/amd64/include/pmap.h
head/sys/dev/drm2/drm_os_freebsd.c
head/sys/dev/drm2/i915/intel_ringbuffer.c
head/sys/i386/i386/pmap.c
head/sys/i386/i386/vm_machdep.c
head/sys/i386/include/pmap.h
head/sys/x86/iommu/intel_utils.c
Modified: head/sys/amd64/amd64/pmap.c
==============================================================================
--- head/sys/amd64/amd64/pmap.c Wed Sep 19 19:13:43 2018 (r338806)
+++ head/sys/amd64/amd64/pmap.c Wed Sep 19 19:35:02 2018 (r338807)
@@ -648,6 +648,10 @@ static vm_page_t pmap_enter_quick_locked(pmap_t pmap,
vm_page_t m, vm_prot_t prot, vm_page_t mpte, struct rwlock **lockp);
static void pmap_fill_ptp(pt_entry_t *firstpte, pt_entry_t newpte);
static int pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte);
+static void pmap_invalidate_cache_range_selfsnoop(vm_offset_t sva,
+ vm_offset_t eva);
+static void pmap_invalidate_cache_range_all(vm_offset_t sva,
+ vm_offset_t eva);
static void pmap_invalidate_pde_page(pmap_t pmap, vm_offset_t va,
pd_entry_t pde);
static void pmap_kenter_attr(vm_offset_t va, vm_paddr_t pa, int mode);
@@ -2171,36 +2175,62 @@ pmap_invalidate_pde_page(pmap_t pmap, vm_offset_t va,
pmap_invalidate_page(pmap, va);
}
+DEFINE_IFUNC(, void, pmap_invalidate_cache_range,
+ (vm_offset_t sva, vm_offset_t eva), static)
+{
+
+ if ((cpu_feature & CPUID_SS) != 0)
+ return (pmap_invalidate_cache_range_selfsnoop);
+ if ((cpu_feature & CPUID_CLFSH) != 0)
+ return (pmap_force_invalidate_cache_range);
+ return (pmap_invalidate_cache_range_all);
+}
+
#define PMAP_CLFLUSH_THRESHOLD (2 * 1024 * 1024)
-void
-pmap_invalidate_cache_range(vm_offset_t sva, vm_offset_t eva, boolean_t force)
+static void
+pmap_invalidate_cache_range_check_align(vm_offset_t sva, vm_offset_t eva)
{
- if (force) {
- sva &= ~(vm_offset_t)(cpu_clflush_line_size - 1);
- } else {
- KASSERT((sva & PAGE_MASK) == 0,
- ("pmap_invalidate_cache_range: sva not page-aligned"));
- KASSERT((eva & PAGE_MASK) == 0,
- ("pmap_invalidate_cache_range: eva not page-aligned"));
- }
+ KASSERT((sva & PAGE_MASK) == 0,
+ ("pmap_invalidate_cache_range: sva not page-aligned"));
+ KASSERT((eva & PAGE_MASK) == 0,
+ ("pmap_invalidate_cache_range: eva not page-aligned"));
+}
- if ((cpu_feature & CPUID_SS) != 0 && !force)
- ; /* If "Self Snoop" is supported and allowed, do nothing. */
- else if ((cpu_stdext_feature & CPUID_STDEXT_CLFLUSHOPT) != 0 &&
- eva - sva < PMAP_CLFLUSH_THRESHOLD) {
+static void
+pmap_invalidate_cache_range_selfsnoop(vm_offset_t sva, vm_offset_t eva)
+{
+
+ pmap_invalidate_cache_range_check_align(sva, eva);
+}
+
+void
+pmap_force_invalidate_cache_range(vm_offset_t sva, vm_offset_t eva)
+{
+
+ sva &= ~(vm_offset_t)(cpu_clflush_line_size - 1);
+ if (eva - sva >= PMAP_CLFLUSH_THRESHOLD) {
/*
- * XXX: Some CPUs fault, hang, or trash the local APIC
- * registers if we use CLFLUSH on the local APIC
- * range. The local APIC is always uncached, so we
- * don't need to flush for that range anyway.
+ * The supplied range is bigger than 2MB.
+ * Globally invalidate cache.
*/
- if (pmap_kextract(sva) == lapic_paddr)
- return;
+ pmap_invalidate_cache();
+ return;
+ }
+ /*
+ * XXX: Some CPUs fault, hang, or trash the local APIC
+ * registers if we use CLFLUSH on the local APIC range. The
+ * local APIC is always uncached, so we don't need to flush
+ * for that range anyway.
+ */
+ if (pmap_kextract(sva) == lapic_paddr)
+ return;
+
+ if ((cpu_stdext_feature & CPUID_STDEXT_CLFLUSHOPT) != 0) {
/*
- * Otherwise, do per-cache line flush. Use the sfence
+ * Do per-cache line flush. Use the sfence
* instruction to insure that previous stores are
* included in the write-back. The processor
* propagates flush to other processors in the cache
@@ -2210,10 +2240,7 @@ pmap_invalidate_cache_range(vm_offset_t sva, vm_offset
for (; sva < eva; sva += cpu_clflush_line_size)
clflushopt(sva);
sfence();
- } else if ((cpu_feature & CPUID_CLFSH) != 0 &&
- eva - sva < PMAP_CLFLUSH_THRESHOLD) {
- if (pmap_kextract(sva) == lapic_paddr)
- return;
+ } else {
/*
* Writes are ordered by CLFLUSH on Intel CPUs.
*/
@@ -2223,17 +2250,17 @@ pmap_invalidate_cache_range(vm_offset_t sva, vm_offset
clflush(sva);
if (cpu_vendor_id != CPU_VENDOR_INTEL)
mfence();
- } else {
-
- /*
- * No targeted cache flush methods are supported by CPU,
- * or the supplied range is bigger than 2MB.
- * Globally invalidate cache.
- */
- pmap_invalidate_cache();
}
}
+static void
+pmap_invalidate_cache_range_all(vm_offset_t sva, vm_offset_t eva)
+{
+
+ pmap_invalidate_cache_range_check_align(sva, eva);
+ pmap_invalidate_cache();
+}
+
/*
* Remove the specified set of pages from the data and instruction caches.
*
@@ -6938,7 +6965,7 @@ pmap_mapdev_attr(vm_paddr_t pa, vm_size_t size, int mo
for (tmpsize = 0; tmpsize < size; tmpsize += PAGE_SIZE)
pmap_kenter_attr(va + tmpsize, pa + tmpsize, mode);
pmap_invalidate_range(kernel_pmap, va, va + tmpsize);
- pmap_invalidate_cache_range(va, va + tmpsize, FALSE);
+ pmap_invalidate_cache_range(va, va + tmpsize);
return ((void *)(va + offset));
}
@@ -7297,7 +7324,7 @@ pmap_change_attr_locked(vm_offset_t va, vm_size_t size
*/
if (changed) {
pmap_invalidate_range(kernel_pmap, base, tmpva);
- pmap_invalidate_cache_range(base, tmpva, FALSE);
+ pmap_invalidate_cache_range(base, tmpva);
}
return (error);
}
Modified: head/sys/amd64/include/pmap.h
==============================================================================
--- head/sys/amd64/include/pmap.h Wed Sep 19 19:13:43 2018 (r338806)
+++ head/sys/amd64/include/pmap.h Wed Sep 19 19:35:02 2018 (r338807)
@@ -431,8 +431,8 @@ void pmap_invalidate_range(pmap_t, vm_offset_t, vm_off
void pmap_invalidate_all(pmap_t);
void pmap_invalidate_cache(void);
void pmap_invalidate_cache_pages(vm_page_t *pages, int count);
-void pmap_invalidate_cache_range(vm_offset_t sva, vm_offset_t eva,
- boolean_t force);
+void pmap_invalidate_cache_range(vm_offset_t sva, vm_offset_t eva);
+void pmap_force_invalidate_cache_range(vm_offset_t sva, vm_offset_t eva);
void pmap_get_mapping(pmap_t pmap, vm_offset_t va, uint64_t *ptr, int *num);
boolean_t pmap_map_io_transient(vm_page_t *, vm_offset_t *, int, boolean_t);
void pmap_unmap_io_transient(vm_page_t *, vm_offset_t *, int, boolean_t);
Modified: head/sys/dev/drm2/drm_os_freebsd.c
==============================================================================
--- head/sys/dev/drm2/drm_os_freebsd.c Wed Sep 19 19:13:43 2018 (r338806)
+++ head/sys/dev/drm2/drm_os_freebsd.c Wed Sep 19 19:35:02 2018 (r338807)
@@ -395,8 +395,8 @@ drm_clflush_virt_range(char *addr, unsigned long lengt
{
#if defined(__i386__) || defined(__amd64__)
- pmap_invalidate_cache_range((vm_offset_t)addr,
- (vm_offset_t)addr + length, TRUE);
+ pmap_force_invalidate_cache_range((vm_offset_t)addr,
+ (vm_offset_t)addr + length);
#else
DRM_ERROR("drm_clflush_virt_range not implemented on this architecture");
#endif
Modified: head/sys/dev/drm2/i915/intel_ringbuffer.c
==============================================================================
--- head/sys/dev/drm2/i915/intel_ringbuffer.c Wed Sep 19 19:13:43 2018 (r338806)
+++ head/sys/dev/drm2/i915/intel_ringbuffer.c Wed Sep 19 19:35:02 2018 (r338807)
@@ -471,8 +471,8 @@ init_pipe_control(struct intel_ring_buffer *ring)
if (pc->cpu_page == NULL)
goto err_unpin;
pmap_qenter((uintptr_t)pc->cpu_page, &obj->pages[0], 1);
- pmap_invalidate_cache_range((vm_offset_t)pc->cpu_page,
- (vm_offset_t)pc->cpu_page + PAGE_SIZE, FALSE);
+ pmap_force_invalidate_cache_range((vm_offset_t)pc->cpu_page,
+ (vm_offset_t)pc->cpu_page + PAGE_SIZE);
pc->obj = obj;
ring->private = pc;
@@ -1102,8 +1102,9 @@ static int init_status_page(struct intel_ring_buffer *
}
pmap_qenter((vm_offset_t)ring->status_page.page_addr, &obj->pages[0],
1);
- pmap_invalidate_cache_range((vm_offset_t)ring->status_page.page_addr,
- (vm_offset_t)ring->status_page.page_addr + PAGE_SIZE, FALSE);
+ pmap_force_invalidate_cache_range(
+ (vm_offset_t)ring->status_page.page_addr,
+ (vm_offset_t)ring->status_page.page_addr + PAGE_SIZE);
ring->status_page.obj = obj;
memset(ring->status_page.page_addr, 0, PAGE_SIZE);
Modified: head/sys/i386/i386/pmap.c
==============================================================================
--- head/sys/i386/i386/pmap.c Wed Sep 19 19:13:43 2018 (r338806)
+++ head/sys/i386/i386/pmap.c Wed Sep 19 19:35:02 2018 (r338807)
@@ -148,6 +148,7 @@ __FBSDID("$FreeBSD$");
#include <machine/intr_machdep.h>
#include <x86/apicvar.h>
#endif
+#include <x86/ifunc.h>
#include <machine/bootinfo.h>
#include <machine/cpu.h>
#include <machine/cputypes.h>
@@ -314,6 +315,10 @@ static vm_page_t pmap_enter_quick_locked(pmap_t pmap,
vm_page_t m, vm_prot_t prot, vm_page_t mpte);
static void pmap_flush_page(vm_page_t m);
static int pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte);
+static void pmap_invalidate_cache_range_selfsnoop(vm_offset_t sva,
+ vm_offset_t eva);
+static void pmap_invalidate_cache_range_all(vm_offset_t sva,
+ vm_offset_t eva);
static void pmap_invalidate_pde_page(pmap_t pmap, vm_offset_t va,
pd_entry_t pde);
static void pmap_fill_ptp(pt_entry_t *firstpte, pt_entry_t newpte);
@@ -1407,37 +1412,62 @@ pmap_invalidate_pde_page(pmap_t pmap, vm_offset_t va,
pmap_invalidate_page(pmap, va);
}
+DEFINE_IFUNC(, void, pmap_invalidate_cache_range, (vm_offset_t, vm_offset_t),
+ static)
+{
+
+ if ((cpu_feature & CPUID_SS) != 0)
+ return (pmap_invalidate_cache_range_selfsnoop);
+ if ((cpu_feature & CPUID_CLFSH) != 0)
+ return (pmap_force_invalidate_cache_range);
+ return (pmap_invalidate_cache_range_all);
+}
+
#define PMAP_CLFLUSH_THRESHOLD (2 * 1024 * 1024)
+static void
+pmap_invalidate_cache_range_check_align(vm_offset_t sva, vm_offset_t eva)
+{
+
+ KASSERT((sva & PAGE_MASK) == 0,
+ ("pmap_invalidate_cache_range: sva not page-aligned"));
+ KASSERT((eva & PAGE_MASK) == 0,
+ ("pmap_invalidate_cache_range: eva not page-aligned"));
+}
+
+static void
+pmap_invalidate_cache_range_selfsnoop(vm_offset_t sva, vm_offset_t eva)
+{
+
+ pmap_invalidate_cache_range_check_align(sva, eva);
+}
+
void
-pmap_invalidate_cache_range(vm_offset_t sva, vm_offset_t eva, boolean_t force)
+pmap_force_invalidate_cache_range(vm_offset_t sva, vm_offset_t eva)
{
- if (force) {
- sva &= ~(vm_offset_t)(cpu_clflush_line_size - 1);
- } else {
- KASSERT((sva & PAGE_MASK) == 0,
- ("pmap_invalidate_cache_range: sva not page-aligned"));
- KASSERT((eva & PAGE_MASK) == 0,
- ("pmap_invalidate_cache_range: eva not page-aligned"));
+ sva &= ~(vm_offset_t)(cpu_clflush_line_size - 1);
+ if (eva - sva >= PMAP_CLFLUSH_THRESHOLD) {
+ /*
+ * The supplied range is bigger than 2MB.
+ * Globally invalidate cache.
+ */
+ pmap_invalidate_cache();
+ return;
}
- if ((cpu_feature & CPUID_SS) != 0 && !force)
- ; /* If "Self Snoop" is supported and allowed, do nothing. */
- else if ((cpu_stdext_feature & CPUID_STDEXT_CLFLUSHOPT) != 0 &&
- eva - sva < PMAP_CLFLUSH_THRESHOLD) {
-#ifdef DEV_APIC
+ /*
+ * XXX: Some CPUs fault, hang, or trash the local APIC
+ * registers if we use CLFLUSH on the local APIC
+ * range. The local APIC is always uncached, so we
+ * don't need to flush for that range anyway.
+ */
+ if (pmap_kextract(sva) == lapic_paddr)
+ return;
+
+ if ((cpu_stdext_feature & CPUID_STDEXT_CLFLUSHOPT) != 0) {
/*
- * XXX: Some CPUs fault, hang, or trash the local APIC
- * registers if we use CLFLUSH on the local APIC
- * range. The local APIC is always uncached, so we
- * don't need to flush for that range anyway.
- */
- if (pmap_kextract(sva) == lapic_paddr)
- return;
-#endif
- /*
- * Otherwise, do per-cache line flush. Use the sfence
+ * Do per-cache line flush. Use the sfence
* instruction to insure that previous stores are
* included in the write-back. The processor
* propagates flush to other processors in the cache
@@ -1447,12 +1477,7 @@ pmap_invalidate_cache_range(vm_offset_t sva, vm_offset
for (; sva < eva; sva += cpu_clflush_line_size)
clflushopt(sva);
sfence();
- } else if ((cpu_feature & CPUID_CLFSH) != 0 &&
- eva - sva < PMAP_CLFLUSH_THRESHOLD) {
-#ifdef DEV_APIC
- if (pmap_kextract(sva) == lapic_paddr)
- return;
-#endif
+ } else {
/*
* Writes are ordered by CLFLUSH on Intel CPUs.
*/
@@ -1462,17 +1487,17 @@ pmap_invalidate_cache_range(vm_offset_t sva, vm_offset
clflush(sva);
if (cpu_vendor_id != CPU_VENDOR_INTEL)
mfence();
- } else {
-
- /*
- * No targeted cache flush methods are supported by CPU,
- * or the supplied range is bigger than 2MB.
- * Globally invalidate cache.
- */
- pmap_invalidate_cache();
}
}
+static void
+pmap_invalidate_cache_range_all(vm_offset_t sva, vm_offset_t eva)
+{
+
+ pmap_invalidate_cache_range_check_align(sva, eva);
+ pmap_invalidate_cache();
+}
+
void
pmap_invalidate_cache_pages(vm_page_t *pages, int count)
{
@@ -5479,7 +5504,7 @@ pmap_mapdev_attr(vm_paddr_t pa, vm_size_t size, int mo
for (tmpsize = 0; tmpsize < size; tmpsize += PAGE_SIZE)
pmap_kenter_attr(va + tmpsize, pa + tmpsize, mode);
pmap_invalidate_range(kernel_pmap, va, va + tmpsize);
- pmap_invalidate_cache_range(va, va + size, FALSE);
+ pmap_invalidate_cache_range(va, va + size);
return ((void *)(va + offset));
}
@@ -5718,7 +5743,7 @@ pmap_change_attr(vm_offset_t va, vm_size_t size, int m
*/
if (changed) {
pmap_invalidate_range(kernel_pmap, base, tmpva);
- pmap_invalidate_cache_range(base, tmpva, FALSE);
+ pmap_invalidate_cache_range(base, tmpva);
}
return (0);
}
Modified: head/sys/i386/i386/vm_machdep.c
==============================================================================
--- head/sys/i386/i386/vm_machdep.c Wed Sep 19 19:13:43 2018 (r338806)
+++ head/sys/i386/i386/vm_machdep.c Wed Sep 19 19:35:02 2018 (r338807)
@@ -650,7 +650,7 @@ sf_buf_invalidate(struct sf_buf *sf)
* settings are recalculated.
*/
pmap_qenter(sf->kva, &m, 1);
- pmap_invalidate_cache_range(sf->kva, sf->kva + PAGE_SIZE, FALSE);
+ pmap_invalidate_cache_range(sf->kva, sf->kva + PAGE_SIZE);
}
/*
Modified: head/sys/i386/include/pmap.h
==============================================================================
--- head/sys/i386/include/pmap.h Wed Sep 19 19:13:43 2018 (r338806)
+++ head/sys/i386/include/pmap.h Wed Sep 19 19:35:02 2018 (r338807)
@@ -394,8 +394,8 @@ void pmap_invalidate_range(pmap_t, vm_offset_t, vm_off
void pmap_invalidate_all(pmap_t);
void pmap_invalidate_cache(void);
void pmap_invalidate_cache_pages(vm_page_t *pages, int count);
-void pmap_invalidate_cache_range(vm_offset_t sva, vm_offset_t eva,
- boolean_t force);
+void pmap_invalidate_cache_range(vm_offset_t sva, vm_offset_t eva);
+void pmap_force_invalidate_cache_range(vm_offset_t sva, vm_offset_t eva);
void *pmap_trm_alloc(size_t size, int flags);
void pmap_trm_free(void *addr, size_t size);
Modified: head/sys/x86/iommu/intel_utils.c
==============================================================================
--- head/sys/x86/iommu/intel_utils.c Wed Sep 19 19:13:43 2018 (r338806)
+++ head/sys/x86/iommu/intel_utils.c Wed Sep 19 19:35:02 2018 (r338807)
@@ -368,8 +368,7 @@ dmar_flush_transl_to_ram(struct dmar_unit *unit, void
* If DMAR does not snoop paging structures accesses, flush
* CPU cache to memory.
*/
- pmap_invalidate_cache_range((uintptr_t)dst, (uintptr_t)dst + sz,
- TRUE);
+ pmap_force_invalidate_cache_range((uintptr_t)dst, (uintptr_t)dst + sz);
}
void
More information about the svn-src-head
mailing list