git: 564fe16c972d - main - powerpc/booke: Use a dedicated critical exception stack
- Go to: [ bottom of page ] [ top of archives ] [ this month ]
Date: Sat, 05 Sep 2026 04:15:15 UTC
The branch main has been updated by jhibbits:
URL: https://cgit.FreeBSD.org/src/commit/?id=564fe16c972dc039fc7b01b38dd913fe13eb1278
commit 564fe16c972dc039fc7b01b38dd913fe13eb1278
Author: Justin Hibbits <jhibbits@FreeBSD.org>
AuthorDate: 2026-09-04 02:48:35 +0000
Commit: Justin Hibbits <jhibbits@FreeBSD.org>
CommitDate: 2026-09-05 04:09:22 +0000
powerpc/booke: Use a dedicated critical exception stack
A critical exception, such as a watchdog, can trigger at any time,
including the middle of a standard exception prologue or epilogue, so
GPRs, including %r1 (the stack pointer) cannot be trusted at all.
Instead, use a private stack pointer for critical interrupts. Each CPU
now has its own critical exception stack, with the boot stack in the
bss.
---
sys/powerpc/booke/booke_machdep.c | 18 ++++++++++++++
sys/powerpc/booke/trap_subr.S | 50 ++++++++++++++++++++++++++++++---------
sys/powerpc/include/pcpu.h | 12 ++++++++--
sys/powerpc/powerpc/genassym.c | 1 +
4 files changed, 68 insertions(+), 13 deletions(-)
diff --git a/sys/powerpc/booke/booke_machdep.c b/sys/powerpc/booke/booke_machdep.c
index 467eb2ab9638..99b14bc9e974 100644
--- a/sys/powerpc/booke/booke_machdep.c
+++ b/sys/powerpc/booke/booke_machdep.c
@@ -97,6 +97,7 @@
#include <sys/kdb.h>
#include <sys/kernel.h>
#include <sys/lock.h>
+#include <sys/malloc.h>
#include <sys/mutex.h>
#include <sys/rwlock.h>
#include <sys/sysctl.h>
@@ -390,13 +391,30 @@ booke_init(u_long arg1, u_long arg2)
#define RES_GRANULE cacheline_size
extern uintptr_t tlb0_miss_locks[];
+/*
+ * Stack for critical-class interrupts taken in kernel mode on the boot CPU.
+ * It is static because the boot CPU is initialised long before the VM is
+ * running; APs reach cpu_pcpu_init() at SI_SUB_CPU and can allocate. Sizing a
+ * MAXCPU array here would reserve megabytes for CPUs that do not exist.
+ */
+static char booke_boot_critstack[BOOKE_CRITSTACK_SIZE] __aligned(16);
+static bool booke_boot_critstack_used;
+
/* Initialise a struct pcpu. */
void
cpu_pcpu_init(struct pcpu *pcpu, int cpuid, size_t sz)
{
+ char *critstack;
pcpu->pc_booke.tid_next = TID_MIN;
+ if (!booke_boot_critstack_used) {
+ booke_boot_critstack_used = true;
+ critstack = booke_boot_critstack;
+ } else
+ critstack = malloc(BOOKE_CRITSTACK_SIZE, M_DEVBUF, M_WAITOK);
+ pcpu->pc_booke.critstack = critstack + BOOKE_CRITSTACK_SIZE;
+
#ifdef SMP
uintptr_t *ptr;
int words_per_gran = RES_GRANULE / sizeof(uintptr_t);
diff --git a/sys/powerpc/booke/trap_subr.S b/sys/powerpc/booke/trap_subr.S
index 4bfd2118fb8f..10babefbb2f8 100644
--- a/sys/powerpc/booke/trap_subr.S
+++ b/sys/powerpc/booke/trap_subr.S
@@ -145,23 +145,44 @@
mfspr %r31, SPR_ESR; \
STORE %r30, (savearea+CPUSAVE_BOOKE_DEAR)(%r1); \
STORE %r31, (savearea+CPUSAVE_BOOKE_ESR)(%r1); \
+ mfspr %r30, SPR_SRR0; \
+ mfspr %r31, SPR_SRR1; \
+ STORE %r30, (savearea+BOOKE_CRITSAVE_SRR0)(%r1); \
+ STORE %r31, (savearea+BOOKE_CRITSAVE_SRR1)(%r1); \
mfspr %r30, isrr0; \
mfspr %r31, isrr1; /* MSR at interrupt time */ \
STORE %r30, (savearea+CPUSAVE_SRR0)(%r1); \
STORE %r31, (savearea+CPUSAVE_SRR1)(%r1); \
- mfspr %r30, SPR_SRR0; \
- mfspr %r31, SPR_SRR1; /* MSR at interrupt time */ \
- STORE %r30, (savearea+BOOKE_CRITSAVE_SRR0)(%r1); \
- STORE %r31, (savearea+BOOKE_CRITSAVE_SRR1)(%r1); \
isync; \
- mfspr %r1, sprg_sp; /* Restore SP */ \
mfcr %r30; /* Save CR */ \
- /* switch to per-thread kstack if intr taken in user mode */ \
mtcr %r31; /* MSR at interrupt time */ \
- bf 17, 1f; \
+ /* \
+ * Never inherit the interrupted r1. A critical interrupt can \
+ * land partway through a non-critical prolog, where r1 holds \
+ * the pcpu pointer rather than a stack, and laying a frame \
+ * there walks straight into the adjacent CPU's pcpu. \
+ */ \
GET_CPUINFO(%r1); /* Per-cpu structure */ \
+ bf 17, 1f; \
LOAD %r1, PC_CURPCB(%r1); /* Per-thread kernel stack */ \
-1:
+ b 2f; \
+1: LOAD %r1, PC_BOOKE_CRITSTACK(%r1); \
+2:
+
+/*
+ * Put back the SRR0-1 saved by STANDARD_CRIT_PROLOG.
+ *
+ * A critical interrupt can land in the middle of a non-critical prolog,
+ * before it has stashed SRR0-1, or between the mtsrr0/mtsrr1 and the rfi of
+ * a TLB miss return. Nested exceptions taken by the C dispatcher clobber
+ * them either way, so restore before returning.
+ */
+#define CRIT_SRR_RESTORE \
+ GET_CPUINFO(%r3); \
+ LOAD %r4, (PC_BOOKE_CRITSAVE+BOOKE_CRITSAVE_SRR0)(%r3); \
+ LOAD %r5, (PC_BOOKE_CRITSAVE+BOOKE_CRITSAVE_SRR1)(%r3); \
+ mtspr SPR_SRR0, %r4; \
+ mtspr SPR_SRR1, %r5
/*
* FRAME_SETUP assumes:
@@ -519,6 +540,7 @@ INTERRUPT(int_critical_input)
addi %r3, %r1, CALLSIZE
bl CNAME(powerpc_interrupt)
TOC_RESTORE
+ CRIT_SRR_RESTORE
FRAME_LEAVE(SPR_CSRR0, SPR_CSRR1)
rfci
@@ -613,9 +635,15 @@ INTERRUPT(int_fixed_interval_timer)
* Watchdog interrupt
****************************************************************************/
INTERRUPT(int_watchdog)
- STANDARD_CRIT_PROLOG(SPR_SPRG1, PC_BOOKE_CRITSAVE, SPR_CSRR0, SPR_CSRR1)
- FRAME_SETUP(SPR_SPRG1, PC_BOOKE_CRITSAVE, EXC_WDOG)
- b trap_common
+ STANDARD_CRIT_PROLOG(SPR_SPRG2, PC_BOOKE_CRITSAVE, SPR_CSRR0, SPR_CSRR1)
+ FRAME_SETUP(SPR_SPRG2, PC_BOOKE_CRITSAVE, EXC_WDOG)
+ GET_TOCBASE(%r2)
+ addi %r3, %r1, CALLSIZE
+ bl CNAME(powerpc_interrupt)
+ TOC_RESTORE
+ CRIT_SRR_RESTORE
+ FRAME_LEAVE(SPR_CSRR0, SPR_CSRR1)
+ rfci
/*****************************************************************************
diff --git a/sys/powerpc/include/pcpu.h b/sys/powerpc/include/pcpu.h
index f2a9bd0080b5..18f0e3209b92 100644
--- a/sys/powerpc/include/pcpu.h
+++ b/sys/powerpc/include/pcpu.h
@@ -85,10 +85,17 @@ struct pvo_entry;
#define BOOKE_TLB_SAVELEN 16
#define BOOKE_TLBSAVE_LEN (BOOKE_TLB_SAVELEN * BOOKE_TLB_MAXNEST)
+/*
+ * Stack used by critical-class interrupts taken in kernel mode. Sized for the
+ * full panic path (powerpc_interrupt -> trap_fatal -> printf -> backtrace),
+ * since that is the only thing that ever runs on it.
+ */
+#define BOOKE_CRITSTACK_SIZE 16384
+
#ifdef __powerpc64__
-#define BOOKE_PCPU_PAD 901
+#define BOOKE_PCPU_PAD 893
#else
-#define BOOKE_PCPU_PAD 365
+#define BOOKE_PCPU_PAD 361
#endif
#define PCPU_MD_BOOKE_FIELDS \
register_t critsave[BOOKE_CRITSAVE_LEN]; \
@@ -96,6 +103,7 @@ struct pvo_entry;
register_t tlbsave[BOOKE_TLBSAVE_LEN]; \
register_t tlb_level; \
uintptr_t *tlb_lock; \
+ void *critstack; \
int tid_next; \
char __pad[BOOKE_PCPU_PAD];
diff --git a/sys/powerpc/powerpc/genassym.c b/sys/powerpc/powerpc/genassym.c
index bc39e086bfa0..e8c08867d778 100644
--- a/sys/powerpc/powerpc/genassym.c
+++ b/sys/powerpc/powerpc/genassym.c
@@ -69,6 +69,7 @@ ASSYM(PC_BOOKE_MCHKSAVE, offsetof(struct pcpu, pc_booke.mchksave));
ASSYM(PC_BOOKE_TLBSAVE, offsetof(struct pcpu, pc_booke.tlbsave));
ASSYM(PC_BOOKE_TLB_LEVEL, offsetof(struct pcpu, pc_booke.tlb_level));
ASSYM(PC_BOOKE_TLB_LOCK, offsetof(struct pcpu, pc_booke.tlb_lock));
+ASSYM(PC_BOOKE_CRITSTACK, offsetof(struct pcpu, pc_booke.critstack));
#endif
ASSYM(CPUSAVE_R27, CPUSAVE_R27*sizeof(register_t));