svn commit: r347134 - in head/sys: amd64/include i386/include kern x86/include x86/x86
Conrad Meyer
cem at FreeBSD.org
Sat May 4 20:34:29 UTC 2019
Author: cem
Date: Sat May 4 20:34:26 2019
New Revision: 347134
URL: https://svnweb.freebsd.org/changeset/base/347134
Log:
x86: Implement MWAIT support for stopping a CPU
IPI_STOP is used after panic or when ddb is entered manually. MONITOR/
MWAIT allows CPUs that support the feature to sleep in a low power way
instead of spinning. Something similar is already used at idle.
It is perhaps especially useful in oversubscribed VM environments, and is
safe to use even if the panic/ddb thread is not the BSP. (Except in the
presence of MWAIT errata, which are detected automatically on platforms with
known wakeup problems.)
It can be tuned/sysctled with "machdep.stop_mwait," which defaults to 0
(off). This commit also introduces the tunable
"machdep.mwait_cpustop_broken," which defaults to 0, unless the CPU has
known errata, but may be set to "1" in loader.conf to signal that mwait
wakeup is broken on CPUs FreeBSD does not yet know about.
Unfortunately, Bhyve doesn't yet support MONITOR extensions, so this doesn't
help bhyve hypervisors running FreeBSD guests.
Submitted by: Anton Rang <rang AT acm.org> (earlier version)
Reviewed by: kib
Sponsored by: Dell EMC Isilon
Differential Revision: https://reviews.freebsd.org/D20135
Modified:
head/sys/amd64/include/pcpu.h
head/sys/i386/include/pcpu.h
head/sys/kern/subr_smp.c
head/sys/x86/include/x86_smp.h
head/sys/x86/x86/cpu_machdep.c
head/sys/x86/x86/mp_x86.c
Modified: head/sys/amd64/include/pcpu.h
==============================================================================
--- head/sys/amd64/include/pcpu.h Sat May 4 19:40:30 2019 (r347133)
+++ head/sys/amd64/include/pcpu.h Sat May 4 20:34:26 2019 (r347134)
@@ -39,7 +39,8 @@
struct monitorbuf {
int idle_state; /* Used by cpu_idle_mwait. */
- char padding[128 - (1 * sizeof(int))];
+ int stop_state; /* Used by cpustop_handler. */
+ char padding[128 - (2 * sizeof(int))];
};
_Static_assert(sizeof(struct monitorbuf) == 128, "2x cache line");
@@ -89,6 +90,9 @@ _Static_assert(sizeof(struct monitorbuf) == 128, "2x c
#define PC_DBREG_CMD_LOAD 1
#ifdef _KERNEL
+
+#define MONITOR_STOPSTATE_RUNNING 0
+#define MONITOR_STOPSTATE_STOPPED 1
#if defined(__GNUCLIKE_ASM) && defined(__GNUCLIKE___TYPEOF)
Modified: head/sys/i386/include/pcpu.h
==============================================================================
--- head/sys/i386/include/pcpu.h Sat May 4 19:40:30 2019 (r347133)
+++ head/sys/i386/include/pcpu.h Sat May 4 20:34:26 2019 (r347134)
@@ -43,7 +43,8 @@
struct monitorbuf {
int idle_state; /* Used by cpu_idle_mwait. */
- char padding[128 - (1 * sizeof(int))];
+ int stop_state; /* Used by cpustop_handler. */
+ char padding[128 - (2 * sizeof(int))];
};
_Static_assert(sizeof(struct monitorbuf) == 128, "2x cache line");
@@ -89,6 +90,9 @@ _Static_assert(sizeof(struct monitorbuf) == 128, "2x c
char __pad[3610]
#ifdef _KERNEL
+
+#define MONITOR_STOPSTATE_RUNNING 0
+#define MONITOR_STOPSTATE_STOPPED 1
#if defined(__GNUCLIKE_ASM) && defined(__GNUCLIKE___TYPEOF)
Modified: head/sys/kern/subr_smp.c
==============================================================================
--- head/sys/kern/subr_smp.c Sat May 4 19:40:30 2019 (r347133)
+++ head/sys/kern/subr_smp.c Sat May 4 20:34:26 2019 (r347134)
@@ -351,42 +351,68 @@ generic_restart_cpus(cpuset_t map, u_int type)
#endif
volatile cpuset_t *cpus;
- KASSERT(type == IPI_STOP || type == IPI_STOP_HARD
#if X86
- || type == IPI_SUSPEND
-#endif
- , ("%s: invalid stop type", __func__));
+ KASSERT(type == IPI_STOP || type == IPI_STOP_HARD
+ || type == IPI_SUSPEND, ("%s: invalid stop type", __func__));
if (!smp_started)
return (0);
CTR1(KTR_SMP, "restart_cpus(%s)", cpusetobj_strprint(cpusetbuf, &map));
-#if X86
if (type == IPI_SUSPEND)
cpus = &resuming_cpus;
else
-#endif
cpus = &stopped_cpus;
/* signal other cpus to restart */
-#if X86
if (type == IPI_SUSPEND)
CPU_COPY_STORE_REL(&map, &toresume_cpus);
else
-#endif
CPU_COPY_STORE_REL(&map, &started_cpus);
-#if X86
+ /*
+ * Wake up any CPUs stopped with MWAIT. From MI code we can't tell if
+ * MONITOR/MWAIT is enabled, but the potentially redundant writes are
+ * relatively inexpensive.
+ */
+ if (type == IPI_STOP) {
+ struct monitorbuf *mb;
+ u_int id;
+
+ CPU_FOREACH(id) {
+ if (!CPU_ISSET(id, &map))
+ continue;
+
+ mb = &pcpu_find(id)->pc_monitorbuf;
+ atomic_store_int(&mb->stop_state,
+ MONITOR_STOPSTATE_RUNNING);
+ }
+ }
+
if (!nmi_is_broadcast || nmi_kdb_lock == 0) {
-#endif
+ /* wait for each to clear its bit */
+ while (CPU_OVERLAP(cpus, &map))
+ cpu_spinwait();
+ }
+#else /* !X86 */
+ KASSERT(type == IPI_STOP || type == IPI_STOP_HARD,
+ ("%s: invalid stop type", __func__));
+
+ if (!smp_started)
+ return (0);
+
+ CTR1(KTR_SMP, "restart_cpus(%s)", cpusetobj_strprint(cpusetbuf, &map));
+
+ cpus = &stopped_cpus;
+
+ /* signal other cpus to restart */
+ CPU_COPY_STORE_REL(&map, &started_cpus);
+
/* wait for each to clear its bit */
while (CPU_OVERLAP(cpus, &map))
cpu_spinwait();
-#if X86
- }
#endif
-
return (1);
}
Modified: head/sys/x86/include/x86_smp.h
==============================================================================
--- head/sys/x86/include/x86_smp.h Sat May 4 19:40:30 2019 (r347133)
+++ head/sys/x86/include/x86_smp.h Sat May 4 20:34:26 2019 (r347134)
@@ -61,6 +61,11 @@ struct cpu_info {
};
extern struct cpu_info *cpu_info;
+/*
+ * Set if MWAIT does not reliably wake when the MONITORed address is written.
+ */
+extern bool mwait_cpustop_broken;
+
#ifdef COUNT_IPIS
extern u_long *ipi_invltlb_counts[MAXCPU];
extern u_long *ipi_invlrng_counts[MAXCPU];
Modified: head/sys/x86/x86/cpu_machdep.c
==============================================================================
--- head/sys/x86/x86/cpu_machdep.c Sat May 4 19:40:30 2019 (r347133)
+++ head/sys/x86/x86/cpu_machdep.c Sat May 4 20:34:26 2019 (r347134)
@@ -110,6 +110,13 @@ static u_int cpu_reset_proxyid;
static volatile u_int cpu_reset_proxy_active;
#endif
+/*
+ * Automatically initialized per CPU errata in cpu_idle_tun below.
+ */
+bool mwait_cpustop_broken = false;
+SYSCTL_BOOL(_machdep, OID_AUTO, mwait_cpustop_broken, CTLFLAG_RDTUN,
+ &mwait_cpustop_broken, 0,
+ "Can not reliably wake MONITOR/MWAIT cpus without interrupts");
/*
* Machine dependent boot() routine
@@ -358,6 +365,7 @@ void
cpu_reset(void)
{
#ifdef SMP
+ struct monitorbuf *mb;
cpuset_t map;
u_int cnt;
@@ -378,6 +386,9 @@ cpu_reset(void)
/* Restart CPU #0. */
CPU_SETOF(0, &started_cpus);
+ mb = &pcpu_find(0)->pc_monitorbuf;
+ atomic_store_int(&mb->stop_state,
+ MONITOR_STOPSTATE_RUNNING);
wmb();
cnt = 0;
@@ -716,6 +727,7 @@ cpu_idle_tun(void *unused __unused)
/* Ryzen erratas 1057, 1109. */
cpu_idle_selector("hlt");
idle_mwait = 0;
+ mwait_cpustop_broken = true;
}
if (cpu_vendor_id == CPU_VENDOR_INTEL && cpu_id == 0x506c9) {
@@ -727,6 +739,7 @@ cpu_idle_tun(void *unused __unused)
* sleep states.
*/
cpu_idle_apl31_workaround = 1;
+ mwait_cpustop_broken = true;
}
TUNABLE_INT_FETCH("machdep.idle_apl31", &cpu_idle_apl31_workaround);
}
Modified: head/sys/x86/x86/mp_x86.c
==============================================================================
--- head/sys/x86/x86/mp_x86.c Sat May 4 19:40:30 2019 (r347133)
+++ head/sys/x86/x86/mp_x86.c Sat May 4 20:34:26 2019 (r347134)
@@ -161,6 +161,10 @@ struct cache_info {
unsigned int boot_address;
+static bool stop_mwait = false;
+SYSCTL_BOOL(_machdep, OID_AUTO, stop_mwait, CTLFLAG_RWTUN, &stop_mwait, 0,
+ "Use MONITOR/MWAIT when stopping CPU, if available");
+
#define MiB(v) (v ## ULL << 20)
void
@@ -1390,23 +1394,41 @@ nmi_call_kdb_smp(u_int type, struct trapframe *frame)
}
/*
- * Handle an IPI_STOP by saving our current context and spinning until we
- * are resumed.
+ * Handle an IPI_STOP by saving our current context and spinning (or mwaiting,
+ * if available) until we are resumed.
*/
void
cpustop_handler(void)
{
+ struct monitorbuf *mb;
u_int cpu;
+ bool use_mwait;
cpu = PCPU_GET(cpuid);
savectx(&stoppcbs[cpu]);
+ use_mwait = (stop_mwait && (cpu_feature2 & CPUID2_MON) != 0 &&
+ !mwait_cpustop_broken);
+ if (use_mwait) {
+ mb = PCPU_PTR(monitorbuf);
+ atomic_store_int(&mb->stop_state,
+ MONITOR_STOPSTATE_STOPPED);
+ }
+
/* Indicate that we are stopped */
CPU_SET_ATOMIC(cpu, &stopped_cpus);
/* Wait for restart */
while (!CPU_ISSET(cpu, &started_cpus)) {
+ if (use_mwait) {
+ cpu_monitor(mb, 0, 0);
+ if (atomic_load_int(&mb->stop_state) ==
+ MONITOR_STOPSTATE_STOPPED)
+ cpu_mwait(0, MWAIT_C1);
+ continue;
+ }
+
ia32_pause();
/*
More information about the svn-src-head
mailing list