call suspend_cpus() under smp_ipi_mtx

[Andriy Gapon]

Could you please review and/or test the following patch?

The idea is exactly the same as for cpu_stop() invocation in the shutdown path.
Please note that I've kept intr_disable() just because potentially mtx_lock_spin
could be implemented in such a way that it wouldn't block all interrupts via CPU
flags, but would use LAPIC TPR, for example.

I've also decided to add smp_ipi_mtx assertions to suspend_cpus and stop_cpus.

diff --git a/sys/dev/acpica/acpi.c b/sys/dev/acpica/acpi.c
index f0c750f..9750d46 100644
--- a/sys/dev/acpica/acpi.c
+++ b/sys/dev/acpica/acpi.c
@@ -2741,6 +2741,15 @@ acpi_EnterSleepState(struct acpi_softc *sc, int state)
     if (sc->acpi_sleep_delay > 0)
 	DELAY(sc->acpi_sleep_delay * 1000000);

+#ifdef SMP
+    /*
+     * Prevent inter-CPU deadlock possibility between suspend_cpus()
+     * and other inter-CPU synchronous calls like smp_rendezvous and
+     * TLB shootdowns.
+     */
+    if (state != ACPI_STATE_S1)
+	mtx_lock_spin(&smp_ipi_mtx);
+#endif
     intr = intr_disable();
     if (state != ACPI_STATE_S1) {
 	sleep_result = acpi_sleep_machdep(sc, state);
@@ -2784,6 +2793,9 @@ acpi_EnterSleepState(struct acpi_softc *sc, int state)
 	}

 	intr_restore(intr);
+#ifdef SMP
+        mtx_unlock_spin(&smp_ipi_mtx);
+#endif

 	/* call acpi_wakeup_machdep() again with interrupt enabled */
 	acpi_wakeup_machdep(sc, state, sleep_result, 1);
diff --git a/sys/kern/subr_smp.c b/sys/kern/subr_smp.c
index 3614798..edf16db 100644
--- a/sys/kern/subr_smp.c
+++ b/sys/kern/subr_smp.c
@@ -260,6 +260,7 @@ int
 stop_cpus(cpuset_t map)
 {

+	mtx_assert(&smp_ipi_mtx, MA_OWNED);
 	return (generic_stop_cpus(map, IPI_STOP));
 }

@@ -275,6 +276,7 @@ int
 suspend_cpus(cpuset_t map)
 {

+	mtx_assert(&smp_ipi_mtx, MA_OWNED);
 	return (generic_stop_cpus(map, IPI_SUSPEND));
 }
 #endif

-- 
Andriy Gapon