git: 6f0c2938e895 - main - Fix latency spikes on return from stop on POWER9+

The branch main has been updated by jhibbits:

URL: https://cgit.FreeBSD.org/src/commit/?id=6f0c2938e895bd3ddb14f44acf36dba3c6156a3f

commit 6f0c2938e895bd3ddb14f44acf36dba3c6156a3f
Author:     Timothy Pearson <tpearson@raptorengineering.com>
AuthorDate: 2025-01-20 20:03:40 +0000
Commit:     Justin Hibbits <jhibbits@FreeBSD.org>
CommitDate: 2025-01-21 23:14:41 +0000

    Fix latency spikes on return from stop on POWER9+
    
    On POWER9 and higher platforms, cpu_idle_power9() directly control the external
    interrupt enable lines when entering / exiting stop states.  This does not
    provide needed information to the rest of the kernel regarding the core going
    into a stop state, and results in random, significant latency spikes (>200ms)
    due to the stopped core not receiving wakeup interrupts in a deterministic
    manner.
    
    Bring cpu_idle_power9() in line with cpu_idle_powerx() by using spinlock
    entry / exit to control the interrupt state vs. direct MSR read / write.
    
    Signed-off-by: Timothy Pearson <tpearson@raptorengineering.com>
---
 sys/powerpc/powerpc/cpu.c | 29 +++++++++++++++++++----------
 1 file changed, 19 insertions(+), 10 deletions(-)

diff --git a/sys/powerpc/powerpc/cpu.c b/sys/powerpc/powerpc/cpu.c
index 4b8da50cf642..37eb01e2b74f 100644
--- a/sys/powerpc/powerpc/cpu.c
+++ b/sys/powerpc/powerpc/cpu.c
@@ -855,7 +855,6 @@ cpu_idle_powerx(sbintime_t sbt)
 static void
 cpu_idle_power9(sbintime_t sbt)
 {
-	register_t msr;
 	int max_stop_state = cpu_idle_max_stop_state;
 
 	/* Limit maximum stop state to valid values */
@@ -877,20 +876,30 @@ cpu_idle_power9(sbintime_t sbt)
 		cpu_idle_max_stop_state = max_stop_state;
 	}
 
-	msr = mfmsr();
-	/* Suspend external interrupts until stop instruction completes. */
-	mtmsr(msr &  ~PSL_EE);
+	/*
+	 * Enter spinlock and suspend external interrupts until the stop
+	 * instruction completes.
+	 */
+	spinlock_enter();
+
+	/* Final scheduler checks before core shutdown */
+	if (sched_runnable()) {
+		/* Exit spinlock and re-enable external interrupts */
+		spinlock_exit();
+		return;
+	}
+
 	/* Set the stop state to lowest latency, wake up to next instruction */
-	/* Set maximum transition level to 2, for deepest lossless sleep. */
-	mtspr(SPR_PSSCR, (2 << PSSCR_MTL_S) | (0 << PSSCR_RL_S));
-	/* "stop" instruction (PowerISA 3.0) */
+	mtspr(SPR_PSSCR, (max_stop_state << PSSCR_MTL_S) | (0 << PSSCR_RL_S));
+
+	/* Shut down core using "stop" instruction (PowerISA 3.0) */
 	__asm __volatile (".long 0x4c0002e4");
+
 	/*
 	 * Re-enable external interrupts to capture the interrupt that caused
-	 * the wake up.
+	 * the wake up.  Exit spinlock.
 	 */
-	mtmsr(msr);
-
+	spinlock_exit();
 }
 #endif