git: 6e93f5e4d693 - main - amd64: FRED support
- Reply: John Baldwin : "Re: git: 6e93f5e4d693 - main - amd64: FRED support"
- Go to: [ bottom of page ] [ top of archives ] [ this month ]
Date: Mon, 20 Jul 2026 21:51:38 UTC
The branch main has been updated by kib:
URL: https://cgit.FreeBSD.org/src/commit/?id=6e93f5e4d6932c423b89dff8fc08d86f8bdeb7b9
commit 6e93f5e4d6932c423b89dff8fc08d86f8bdeb7b9
Author: Konstantin Belousov <kib@FreeBSD.org>
AuthorDate: 2026-02-13 11:24:40 +0000
Commit: Konstantin Belousov <kib@FreeBSD.org>
CommitDate: 2026-07-20 21:51:02 +0000
amd64: FRED support
FRED support as defined starting from the SDM rev. 90, requires a new
'events' entry point to receive user and kernel mode exceptions and
interrupts notifications from the hardware. A minimal asm trampoline is
enough, rest can be implemented in C due to the clean FRED organization
of the event reporting.
The syscall entry is handled by a microptimized assembly path, directly
calling into the amd64_syscall() handler, instead of the generic events
entry point.
Tested by: emaste
Sponsored by: The FreeBSD Foundation
MFC after: 1 week
Differential revision: https://reviews.freebsd.org/D55829
---
sys/amd64/amd64/cpu_switch.S | 73 ++++-
sys/amd64/amd64/db_trace.c | 5 +-
sys/amd64/amd64/exception.S | 282 ++++++++++++++++++
sys/amd64/amd64/genassym.c | 15 +
sys/amd64/amd64/initcpu.c | 43 +++
sys/amd64/amd64/machdep.c | 104 +++++--
sys/amd64/amd64/mp_machdep.c | 82 +++--
sys/amd64/amd64/trap.c | 558 +++++++++++++++++++++++++++++++++++
sys/amd64/amd64/vm_machdep.c | 10 +-
sys/amd64/include/md_var.h | 2 +
sys/amd64/include/pcb.h | 8 +-
sys/amd64/vmm/intel/vmx.c | 2 +-
sys/amd64/vmm/vmm.c | 2 +-
sys/amd64/vmm/vmm_lapic.c | 5 +
sys/amd64/vmm/vmm_lapic.h | 2 +
sys/dev/hyperv/vmbus/x86/vmbus_x86.c | 3 +-
sys/i386/i386/machdep.c | 2 +
sys/kern/kern_thread.c | 2 +-
sys/x86/include/apicvar.h | 3 +-
sys/x86/include/frame.h | 1 -
sys/x86/include/x86_var.h | 3 +
sys/x86/x86/local_apic.c | 38 +--
22 files changed, 1158 insertions(+), 87 deletions(-)
diff --git a/sys/amd64/amd64/cpu_switch.S b/sys/amd64/amd64/cpu_switch.S
index 21e55e64d74a..774098d7def5 100644
--- a/sys/amd64/amd64/cpu_switch.S
+++ b/sys/amd64/amd64/cpu_switch.S
@@ -187,14 +187,22 @@ do_kthread:
cmpq %rax,%rdx
jne do_tss
done_tss:
+ movq %r8,PCPU(CURPCB)
+ cmpb $0,fred
+ jne 1f
movq TD_MD_STACK_BASE(%r12),%r9
movq %r9,PCPU(RSP0)
- movq %r8,PCPU(CURPCB)
movq PCPU(PTI_RSP0),%rax
cmpq $~0,PCPU(UCR3)
cmove %r9,%rax
movq %rax,TSS_RSP0(%rdx)
- movq %r12,PCPU(CURTHREAD) /* into next thread */
+ jmp 2f
+1: movq TD_MD_STACK_BASE(%r12),%rdx
+ movl %edx,%eax
+ shrq $32,%rdx
+ movl $MSR_FRED_RSP0,%ecx
+ wrmsr
+2: movq %r12,PCPU(CURTHREAD) /* into next thread */
/* Test if debug registers should be restored. */
testl $PCB_DBREGS,PCB_FLAGS(%r8)
@@ -365,8 +373,36 @@ ENTRY(savectx)
movl %eax,PCB_SFMASK(%rdi)
movl %edx,PCB_SFMASK+4(%rdi)
- sgdt PCB_GDT(%rdi)
- sidt PCB_IDT(%rdi)
+ cmpb $0, fred
+ je 1f
+ movl $MSR_FRED_RSP0,%ecx
+ rdmsr
+ movl %eax,PCB_FRED_RSP0(%rdi)
+ movl %edx,PCB_FRED_RSP0+4(%rdi)
+ movl $MSR_FRED_RSP1,%ecx
+ rdmsr
+ movl %eax,PCB_FRED_RSP1(%rdi)
+ movl %edx,PCB_FRED_RSP1+4(%rdi)
+ movl $MSR_FRED_RSP2,%ecx
+ rdmsr
+ movl %eax,PCB_FRED_RSP2(%rdi)
+ movl %edx,PCB_FRED_RSP2+4(%rdi)
+ movl $MSR_FRED_RSP3,%ecx
+ rdmsr
+ movl %eax,PCB_FRED_RSP3(%rdi)
+ movl %edx,PCB_FRED_RSP3+4(%rdi)
+ movl $MSR_FRED_STKLVLS,%ecx
+ rdmsr
+ movl %eax,PCB_FRED_STKLVLS(%rdi)
+ movl %edx,PCB_FRED_STKLVLS+4(%rdi)
+ movl $MSR_FRED_CONFIG,%ecx
+ rdmsr
+ movl %eax,PCB_FRED_CONFIG(%rdi)
+ movl %edx,PCB_FRED_CONFIG+4(%rdi)
+ jmp 2f
+
+1: sidt PCB_IDT(%rdi)
+2: sgdt PCB_GDT(%rdi)
sldt PCB_LDT(%rdi)
str PCB_TR(%rdi)
@@ -442,8 +478,35 @@ ENTRY(resumectx)
movq %rax,%cr3
/* Restore descriptor tables. */
+ cmpb $0,fred
+ jne 1f
lidt PCB_IDT(%rdi)
- lldt PCB_LDT(%rdi)
+ jmp 2f
+1: movl $MSR_FRED_RSP0,%ecx
+ movl PCB_FRED_RSP0(%rdi),%eax
+ movl 4 + PCB_FRED_RSP0(%rdi),%edx
+ wrmsr
+ movl $MSR_FRED_RSP1,%ecx
+ movl PCB_FRED_RSP1(%rdi),%eax
+ movl 4 + PCB_FRED_RSP1(%rdi),%edx
+ wrmsr
+ movl $MSR_FRED_RSP2,%ecx
+ movl PCB_FRED_RSP2(%rdi),%eax
+ movl 4 + PCB_FRED_RSP2(%rdi),%edx
+ wrmsr
+ movl $MSR_FRED_RSP3,%ecx
+ movl PCB_FRED_RSP3(%rdi),%eax
+ movl 4 + PCB_FRED_RSP3(%rdi),%edx
+ wrmsr
+ movl $MSR_FRED_STKLVLS,%ecx
+ movl PCB_FRED_STKLVLS(%rdi),%eax
+ movl 4 + PCB_FRED_STKLVLS(%rdi),%edx
+ wrmsr
+ movl $MSR_FRED_CONFIG,%ecx
+ movl PCB_FRED_CONFIG(%rdi),%eax
+ movl 4 + PCB_FRED_CONFIG(%rdi),%edx
+ wrmsr
+2: lldt PCB_LDT(%rdi)
#define SDT_SYSTSS 9
#define SDT_SYSBSY 11
diff --git a/sys/amd64/amd64/db_trace.c b/sys/amd64/amd64/db_trace.c
index 5f751906b7e6..e14cd55978de 100644
--- a/sys/amd64/amd64/db_trace.c
+++ b/sys/amd64/amd64/db_trace.c
@@ -167,7 +167,10 @@ db_nextframe(db_addr_t *fp, db_addr_t *ip, struct thread *td)
strcmp(name, "fork_trampoline") == 0 ||
strcmp(name, "mchk_calltrap") == 0 ||
strcmp(name, "nmi_calltrap") == 0 ||
- strcmp(name, "Xdblfault") == 0)
+ strcmp(name, "Xdblfault") == 0 ||
+ strcmp(name, "fred_events_handlers") == 0 ||
+ strcmp(name, "user_fred_syscall") == 0 ||
+ strcmp(name, "user_fred_non_syscall") == 0)
frame_type = TRAP;
else if (strncmp(name, "Xatpic_intr", 11) == 0 ||
strncmp(name, "Xapic_isr", 9) == 0 ||
diff --git a/sys/amd64/amd64/exception.S b/sys/amd64/amd64/exception.S
index 4d58e07caf4b..6dfb8a431fe7 100644
--- a/sys/amd64/amd64/exception.S
+++ b/sys/amd64/amd64/exception.S
@@ -1063,6 +1063,8 @@ ENTRY(fork_trampoline)
movq %rbx,%rsi /* arg1 */
movq %rsp,%rdx /* trapframe pointer */
call fork_exit
+ cmpb $0, fred
+ jne fred_checkast
jmp doreti /* Handle any ASTs */
/*
@@ -1401,6 +1403,286 @@ gsbase_load_fault:
movq $0,PCB_GSBASE(%r8)
jmp doreti
+/*
+ * FRED guarantees that on entry, %rflags is set to '2', which in
+ * particular means that interrupts are disabled, and DF/AC are clear.
+ * Interrupts are re-enabled by the C handler, if appropriate.
+ */
+ ALIGN_TEXT
+user_fred_syscall:
+ movq %rdx,TF_RDX(%rsp)
+ movq %rcx,TF_RCX(%rsp)
+ SAVE_SEGS
+ movq PCPU(CURPCB),%r11
+ andl $~PCB_FULL_IRET,PCB_FLAGS(%r11)
+ sti
+ movw $KUDSEL,TF_SS(%rsp)
+ movw $KUCSEL,TF_CS(%rsp)
+ movq %r8,TF_R8(%rsp)
+ movq %r9,TF_R9(%rsp)
+ movq %rbx,TF_RBX(%rsp)
+ movq %rbp,TF_RBP(%rsp)
+ movq %r12,TF_R12(%rsp)
+ movq %r13,TF_R13(%rsp)
+ movq %r14,TF_R14(%rsp)
+ movq %r15,TF_R15(%rsp)
+ movl $TF_HASSEGS,TF_FLAGS(%rsp)
+ andl $TF_FRED_EVINFO2_INSTLENMASK,%eax
+ shrl $TF_FRED_EVINFO2_INSTLENSHIFT,%eax
+ movq %rax,TF_ERR(%rsp)
+ movq PCPU(CURTHREAD),%rdi
+ movl TF_RFLAGS(%rsp),%esi
+ andl $PSL_T,%esi
+ call amd64_syscall
+ movq PCPU(CURPCB),%rax
+ cli
+ testl $PCB_FULL_IRET,PCB_FLAGS(%rax)
+ jnz fred_checkast
+ movq PCPU(CURTHREAD),%rax
+ cmpl $0,TD_AST(%rax)
+ jnz fred_checkast
+ movq TF_RDI(%rsp),%rdi
+ movq TF_RSI(%rsp),%rsi
+ movq TF_RDX(%rsp),%rdx
+ movq TF_RAX(%rsp),%rax
+ xorl %r8d,%r8d
+ xorl %r9d,%r9d
+ xorl %r10d,%r10d
+ xorl %r11d,%r11d
+ addq $TF_ERR,%rsp
+ .globl fred_syscall_eretu
+fred_syscall_eretu:
+ eretu
+
+ ALIGN_TEXT
+user_fred_non_syscall:
+ movq %rdx,TF_RDX(%rsp)
+ movq %rcx,TF_RCX(%rsp)
+ movq %r8,TF_R8(%rsp)
+ movq %r9,TF_R9(%rsp)
+ movq %rbx,TF_RBX(%rsp)
+ movq %rbp,TF_RBP(%rsp)
+ movq %r10,TF_R10(%rsp)
+ movq %r11,TF_R11(%rsp)
+ movq %r12,TF_R12(%rsp)
+ movq %r13,TF_R13(%rsp)
+ movq %r14,TF_R14(%rsp)
+ movq %r15,TF_R15(%rsp)
+ SAVE_SEGS
+ movq PCPU(CURTHREAD),%r9
+ movq TD_PCB(%r9),%r8
+ rdfsbase %rax
+ movq %rax,PCB_FSBASE(%r8)
+ rdgsbase %rax
+ movq %rax,PCB_GSBASE(%r8)
+ movl $TF_HASSEGS,TF_FLAGS(%rsp)
+ KMSAN_ENTER
+ movq %rsp, %rdi
+ call trap_fred_u
+ KMSAN_LEAVE
+fred_checkast:
+ cli
+ movq PCPU(CURTHREAD),%rax
+ movq TD_PCB(%rax),%r8
+ cmpl $0,TD_AST(%rax)
+ je fred_gouser
+ sti
+ movq %rsp,%rdi /* pass a pointer to the trapframe */
+ call ast
+ jmp fred_checkast
+fred_gouser:
+ testl $PCB_FULL_IRET,PCB_FLAGS(%r8)
+ jz fred_u_regs
+ andl $~PCB_FULL_IRET,PCB_FLAGS(%r8)
+ testl $TF_HASSEGS,TF_FLAGS(%rsp)
+ jne fred_do_segs
+ movw $KUDSEL,%ax
+ movw %ax,TF_DS(%rsp)
+ movw %ax,TF_ES(%rsp)
+ movw $KUF32SEL,TF_FS(%rsp)
+ movw $KUG32SEL,TF_GS(%rsp)
+fred_do_segs:
+ /* Restore %fs and fsbase */
+ movw TF_FS(%rsp),%ax
+ .globl fred_ld_fs
+fred_ld_fs:
+ movw %ax,%fs
+ testl $PCB_32BIT,PCB_FLAGS(%r8)
+ jnz after_fred_fs
+ movq PCB_FSBASE(%r8),%rax
+ .globl fred_ld_fsbase
+fred_ld_fsbase:
+ wrfsbase %rax
+after_fred_fs:
+ /* Restore %gs and kgsbase */
+ movw TF_GS(%rsp),%ax
+ .globl fred_lkgs
+fred_lkgs:
+ lkgs %ax
+ testl $PCB_32BIT,PCB_FLAGS(%r8)
+ jnz after_fred_gs
+ movl PCB_GSBASE(%r8),%eax
+ movl PCB_GSBASE+4(%r8),%edx
+ movl $MSR_KGSBASE,%ecx
+ .globl fred_ld_kgsbase
+fred_ld_kgsbase:
+ wrmsr
+after_fred_gs:
+ .globl fred_ld_es
+fred_ld_es:
+ movw TF_ES(%rsp),%es
+ .globl fred_ld_ds
+fred_ld_ds:
+ movw TF_DS(%rsp),%ds
+fred_u_regs:
+ RESTORE_REGS
+ addq $TF_ERR,%rsp
+ .globl fred_eretu
+fred_eretu:
+ eretu
+
+ .globl fred_ld_fs_fault
+fred_ld_fs_fault:
+ movl $(TF_FRED_EVINFO2_TYPE_EXC | IDT_NP), TF_FRED_EVINFO2(%rsp)
+ movq %rsp,%rdi
+ call trap_fred_u
+ movw $KUF32SEL,TF_FS(%rsp)
+ jmp fred_checkast
+
+ .globl fred_ld_fsbase_fault
+fred_ld_fsbase_fault:
+ movl $(TF_FRED_EVINFO2_TYPE_EXC | IDT_NP), TF_FRED_EVINFO2(%rsp)
+ movq %rsp,%rdi
+ call trap_fred_u
+ movq PCPU(CURTHREAD),%r9
+ movq TD_PCB(%r9),%r8
+ movq $0,PCB_FSBASE(%r8)
+ jmp fred_checkast
+
+ .globl fred_lkgs_fault
+fred_lkgs_fault:
+ movl $(TF_FRED_EVINFO2_TYPE_EXC | IDT_NP), TF_FRED_EVINFO2(%rsp)
+ movq %rsp,%rdi
+ call trap_fred_u
+ movw $KUG32SEL,TF_GS(%rsp)
+ jmp fred_checkast
+
+ .globl fred_ld_kgsbase_fault
+fred_ld_kgsbase_fault:
+ movl $(TF_FRED_EVINFO2_TYPE_EXC | IDT_NP), TF_FRED_EVINFO2(%rsp)
+ movq %rsp,%rdi
+ call trap_fred_u
+ movq PCPU(CURTHREAD),%r9
+ movq TD_PCB(%r9),%r8
+ movq $0,PCB_GSBASE(%r8)
+ jmp fred_checkast
+
+ .globl fred_ld_es_fault
+fred_ld_es_fault:
+ movl $(TF_FRED_EVINFO2_TYPE_EXC | IDT_NP), TF_FRED_EVINFO2(%rsp)
+ movq %rsp,%rdi
+ call trap_fred_u
+ movw $KUDSEL,TF_ES(%rsp)
+ jmp fred_checkast
+
+ .globl fred_ld_ds_fault
+fred_ld_ds_fault:
+ movl $(TF_FRED_EVINFO2_TYPE_EXC | IDT_NP), TF_FRED_EVINFO2(%rsp)
+ movq %rsp,%rdi
+ call trap_fred_u
+ movw $KUDSEL,TF_DS(%rsp)
+ jmp fred_checkast
+
+ .globl fred_eretu_fault
+fred_eretu_fault:
+ subq $TF_ERR,%rsp
+ movq %rdi,TF_RDI(%rsp)
+ movq %rdx,TF_RDX(%rsp)
+ movq %rcx,TF_RCX(%rsp)
+ movq %rsi,TF_RSI(%rsp)
+ movq %r8,TF_R8(%rsp)
+ movq %r9,TF_R9(%rsp)
+ movq %rbx,TF_RBX(%rsp)
+ movq %rbp,TF_RBP(%rsp)
+ movq %r10,TF_R10(%rsp)
+ movq %r11,TF_R11(%rsp)
+ movq %r12,TF_R12(%rsp)
+ movq %r13,TF_R13(%rsp)
+ movq %r14,TF_R14(%rsp)
+ movq %r15,TF_R15(%rsp)
+ SAVE_SEGS
+ movl $(TF_FRED_EVINFO2_TYPE_EXC | IDT_GP), TF_FRED_EVINFO2(%rsp)
+ movq %rsp, %rdi
+ KMSAN_ENTER
+ call trap_fred_u
+ KMSAN_LEAVE
+ jmp fred_checkast
+
+/*
+ * Content for the FRED entries page.
+*/
+
+ .p2align 12,0x90
+ .globl fred_events_handlers, fred_events_handlers_end
+fred_events_handlers:
+ subq $TF_ERR,%rsp
+ movq %rax,TF_RAX(%rsp)
+ movq %rdi,TF_RDI(%rsp)
+ movq %rsi,TF_RSI(%rsp)
+ movl TF_FRED_EVINFO2(%rsp),%edi
+ movl %edi,%eax
+ andl $TF_FRED_EVINFO2_TYPEMASK,%edi
+ cmpl $TF_FRED_EVINFO2_TYPE_SYSCALL,%edi
+ je user_fred_syscall
+ jmp user_fred_non_syscall
+
+ . = fred_events_handlers + 256
+ subq $TF_ERR,%rsp
+ movq %rax,TF_RAX(%rsp)
+ movq %rdi,TF_RDI(%rsp)
+ movq %rdx,TF_RDX(%rsp)
+ movq %rcx,TF_RCX(%rsp)
+ movq %rsi,TF_RSI(%rsp)
+ movq %r8,TF_R8(%rsp)
+ movq %r9,TF_R9(%rsp)
+ movq %rbx,TF_RBX(%rsp)
+ movq %rbp,TF_RBP(%rsp)
+ movq %r10,TF_R10(%rsp)
+ movq %r11,TF_R11(%rsp)
+ movq %r12,TF_R12(%rsp)
+ movq %r13,TF_R13(%rsp)
+ movq %r14,TF_R14(%rsp)
+ movq %r15,TF_R15(%rsp)
+ movq %rsp, %rdi
+ KMSAN_ENTER
+ call trap_fred_k
+ KMSAN_LEAVE
+ RESTORE_REGS
+ addq $TF_ERR,%rsp
+ erets
+fred_events_handlers_end:
+
+ .globl fred_out_of_nmi
+fred_out_of_nmi:
+ pushq %rbp
+ movq %rsp,%rbp
+ xorl %eax,%eax
+ pushq $0
+ pushq $0
+ movw %ss,%ax
+ orl $(1<<18),%eax /* NMI block release for ERETS */
+ pushq %rax
+ pushq %rbp
+ pushfq
+ xorl %eax,%eax
+ movw %cs,%ax
+ pushq %rax
+ pushq $1f
+ pushq $0
+ erets
+1: popq %rbp
+ retq
+
#ifdef HWPMC_HOOKS
ENTRY(end_exceptions)
#endif
diff --git a/sys/amd64/amd64/genassym.c b/sys/amd64/amd64/genassym.c
index 875c536166d1..9a5a74b3f6d6 100644
--- a/sys/amd64/amd64/genassym.c
+++ b/sys/amd64/amd64/genassym.c
@@ -146,6 +146,12 @@ ASSYM(PCB_STAR, offsetof(struct pcb, pcb_star));
ASSYM(PCB_LSTAR, offsetof(struct pcb, pcb_lstar));
ASSYM(PCB_CSTAR, offsetof(struct pcb, pcb_cstar));
ASSYM(PCB_SFMASK, offsetof(struct pcb, pcb_sfmask));
+ASSYM(PCB_FRED_RSP0, offsetof(struct pcb, pcb_fred_rsp0));
+ASSYM(PCB_FRED_RSP1, offsetof(struct pcb, pcb_fred_rsp1));
+ASSYM(PCB_FRED_RSP2, offsetof(struct pcb, pcb_fred_rsp2));
+ASSYM(PCB_FRED_RSP3, offsetof(struct pcb, pcb_fred_rsp3));
+ASSYM(PCB_FRED_STKLVLS, offsetof(struct pcb, pcb_fred_stklvls));
+ASSYM(PCB_FRED_CONFIG, offsetof(struct pcb, pcb_fred_config));
ASSYM(PCB_SIZE, sizeof(struct pcb));
ASSYM(PCB_FULL_IRET, PCB_FULL_IRET);
ASSYM(PCB_DBREGS, PCB_DBREGS);
@@ -180,9 +186,15 @@ ASSYM(TF_DS, offsetof(struct trapframe, tf_ds));
ASSYM(TF_ES, offsetof(struct trapframe, tf_es));
ASSYM(TF_FS, offsetof(struct trapframe, tf_fs));
ASSYM(TF_GS, offsetof(struct trapframe, tf_gs));
+ASSYM(TF_FRED_EVINFO2, offsetof(struct trapframe, tf_fred_evinfo2));
ASSYM(TF_FLAGS, offsetof(struct trapframe, tf_flags));
ASSYM(TF_SIZE, sizeof(struct trapframe));
ASSYM(TF_HASSEGS, TF_HASSEGS);
+ASSYM(TF_FRED_EVINFO2_TYPE_EXC, TF_FRED_EVINFO2_TYPE_EXC);
+ASSYM(TF_FRED_EVINFO2_TYPE_SYSCALL, TF_FRED_EVINFO2_TYPE_SYSCALL);
+ASSYM(TF_FRED_EVINFO2_TYPEMASK, TF_FRED_EVINFO2_TYPEMASK);
+ASSYM(TF_FRED_EVINFO2_INSTLENMASK, TF_FRED_EVINFO2_INSTLENMASK);
+ASSYM(TF_FRED_EVINFO2_INSTLENSHIFT, TF_FRED_EVINFO2_INSTLENSHIFT);
ASSYM(PTI_RDX, offsetof(struct pti_frame, pti_rdx));
ASSYM(PTI_RAX, offsetof(struct pti_frame, pti_rax));
@@ -272,6 +284,9 @@ ASSYM(TSSSEL, GSEL(GPROC0_SEL, SEL_KPL));
ASSYM(LDTSEL, GSEL(GUSERLDT_SEL, SEL_KPL));
ASSYM(SEL_RPL_MASK, SEL_RPL_MASK);
+ASSYM(IDT_NP, IDT_NP);
+ASSYM(IDT_GP, IDT_GP);
+
ASSYM(__FreeBSD_version, __FreeBSD_version);
#ifdef HWPMC_HOOKS
diff --git a/sys/amd64/amd64/initcpu.c b/sys/amd64/amd64/initcpu.c
index 7161cf631508..5b615be4b8dc 100644
--- a/sys/amd64/amd64/initcpu.c
+++ b/sys/amd64/amd64/initcpu.c
@@ -397,3 +397,46 @@ initializecpucache(void)
cpu_stdext_feature &= ~CPUID_STDEXT_CLFLUSHOPT;
}
}
+
+extern const char fred_events_handlers[], fred_events_handlers_end[];
+
+void
+amd64_cpu_init_fred(void)
+{
+ MPASS((((uintptr_t)fred_events_handlers) & PAGE_MASK) == 0);
+ /*
+ * 128b = 0x2 * 64b redzone
+ * SLC 0 for EXTINT
+ * SLC 0 current
+ */
+ wrmsr(MSR_FRED_CONFIG, (uintptr_t)fred_events_handlers |
+ (0x2 << IA32_FRED_CONFIG_REDZONESZ_SHIFT) |
+ (0 << IA32_FRED_CONFIG_EXTINT_SLC_SHIFT) | 0);
+
+ /*
+ * With FRED, there is no need for workarounds requiring
+ * dedicated stack for #DB. Also, there is no need to have
+ * separate stacks for #NM and #MC, because CSL rules ensure
+ * that interrupted frame is safe even if NMI was interrupted
+ * by machine check or vice versa.
+ *
+ * Still, we need to provide a stack for events that can occur
+ * with the interrupts disabled, to avoid caring about the
+ * ordering of the context update in cpu_switch.S. Use CSL 1
+ * for #NM and #MC.
+ *
+ * Since double fault handler is used to handle stack
+ * overflow, it gets the dedicated stack, of the highest used
+ * CSL 2.
+ *
+ * CSL 3 is unused, and we have some space to re-arrange this
+ * if a need in the future appears.
+ */
+ wrmsr(MSR_FRED_STKLVLS, (1ul << (IDT_NM * 2)) |
+ (1ul << (IDT_MC * 2)) | (2ul << (IDT_DF * 2)));
+ wrmsr(MSR_FRED_RSP3, 0);
+ load_cr4(rcr4() | CR4_FRED);
+ if (IS_BSP() && bootverbose)
+ printf("FRED enabled\n");
+
+}
diff --git a/sys/amd64/amd64/machdep.c b/sys/amd64/amd64/machdep.c
index ab9edfe2c68f..c6a77c5ef852 100644
--- a/sys/amd64/amd64/machdep.c
+++ b/sys/amd64/amd64/machdep.c
@@ -206,10 +206,14 @@ long realmem = 0;
int late_console = 1;
int lass_enabled = 0;
+int __read_frequently fred = 0;
+SYSCTL_INT(_hw, OID_AUTO, fred, CTLFLAG_RDTUN | CTLFLAG_NOFETCH,
+ &fred, 0,
+ "FRED is used");
+
struct kva_md_info kmi;
struct region_descriptor r_idt;
-
struct pcpu *__pcpu;
struct pcpu temp_bsp_pcpu;
@@ -347,9 +351,9 @@ cpu_setregs(void)
static struct gate_descriptor idt0[NIDT];
struct gate_descriptor *idt = &idt0[0]; /* interrupt descriptor table */
-static char dblfault_stack[DBLFAULT_STACK_SIZE] __aligned(16);
+static char dblfault_stack[DBLFAULT_STACK_SIZE] __aligned(64);
static char mce0_stack[MCE_STACK_SIZE] __aligned(16);
-static char nmi0_stack[NMI_STACK_SIZE] __aligned(16);
+static char nmi0_stack[NMI_STACK_SIZE] __aligned(64);
static char dbg0_stack[DBG_STACK_SIZE] __aligned(16);
CTASSERT(sizeof(struct nmi_pcpu) == 16);
@@ -485,6 +489,9 @@ setidt(int idx, inthand_t *func, int typ, int dpl, int ist)
{
struct gate_descriptor *ip;
+ if (fred)
+ return;
+
ip = idt + idx;
ip->gd_looffset = (uintptr_t)func;
ip->gd_selector = GSEL(GCODE_SEL, SEL_KPL);
@@ -1186,9 +1193,11 @@ amd64_conf_fast_syscall(void)
msr = rdmsr(MSR_EFER) | EFER_SCE;
wrmsr(MSR_EFER, msr);
- wrmsr(MSR_LSTAR, pti ? (u_int64_t)IDTVEC(fast_syscall_pti) :
- (u_int64_t)IDTVEC(fast_syscall));
- wrmsr(MSR_CSTAR, (u_int64_t)IDTVEC(fast_syscall32));
+ if (!fred) {
+ wrmsr(MSR_LSTAR, pti ? (u_int64_t)IDTVEC(fast_syscall_pti) :
+ (u_int64_t)IDTVEC(fast_syscall));
+ wrmsr(MSR_CSTAR, (u_int64_t)IDTVEC(fast_syscall32));
+ }
msr = ((u_int64_t)GSEL(GCODE_SEL, SEL_KPL) << 32) |
((u_int64_t)GSEL(GUCODE32_SEL, SEL_UPL) << 48);
wrmsr(MSR_STAR, msr);
@@ -1230,33 +1239,49 @@ amd64_bsp_ist_init(struct pcpu *pc)
tssp = &pc->pc_common_tss;
- /* doublefault stack space, runs on ist1 */
- np = ((struct nmi_pcpu *)&dblfault_stack[sizeof(dblfault_stack)]) - 1;
- np->np_pcpu = (register_t)pc;
- tssp->tss_ist1 = (long)np;
+ /* Doublefault stack space, runs on ist1 for IDT. */
+ if (fred) {
+ wrmsr(MSR_FRED_RSP2, (uint64_t)&dblfault_stack[
+ sizeof(dblfault_stack)]);
+ } else {
+ np = ((struct nmi_pcpu *)&dblfault_stack[sizeof(
+ dblfault_stack)]) - 1;
+ np->np_pcpu = (register_t)pc;
+ tssp->tss_ist1 = (long)np;
+ }
/*
- * NMI stack, runs on ist2. The pcpu pointer is stored just
- * above the start of the ist2 stack.
+ * NMI stack.
*/
- np = ((struct nmi_pcpu *)&nmi0_stack[sizeof(nmi0_stack)]) - 1;
- np->np_pcpu = (register_t)pc;
- tssp->tss_ist2 = (long)np;
+ if (fred) {
+ wrmsr(MSR_FRED_RSP1, (uint64_t)&nmi0_stack[
+ sizeof(nmi0_stack)]);
+ } else {
+ /*
+ * Runs on ist2 for IDT. The pcpu pointer is stored
+ * just above the start of the ist2 stack.
+ */
+ np = ((struct nmi_pcpu *)&nmi0_stack[sizeof(nmi0_stack)]) - 1;
+ np->np_pcpu = (register_t)pc;
+ tssp->tss_ist2 = (long)np;
+ }
- /*
- * MC# stack, runs on ist3. The pcpu pointer is stored just
- * above the start of the ist3 stack.
- */
- np = ((struct nmi_pcpu *)&mce0_stack[sizeof(mce0_stack)]) - 1;
- np->np_pcpu = (register_t)pc;
- tssp->tss_ist3 = (long)np;
+ if (!fred) {
+ /*
+ * MC# stack for IDT, runs on ist3. The pcpu pointer
+ * is stored just above the start of the ist3 stack.
+ */
+ np = ((struct nmi_pcpu *)&mce0_stack[sizeof(mce0_stack)]) - 1;
+ np->np_pcpu = (register_t)pc;
+ tssp->tss_ist3 = (long)np;
- /*
- * DB# stack, runs on ist4.
- */
- np = ((struct nmi_pcpu *)&dbg0_stack[sizeof(dbg0_stack)]) - 1;
- np->np_pcpu = (register_t)pc;
- tssp->tss_ist4 = (long)np;
+ /*
+ * DB# stack for IDT, runs on ist4.
+ */
+ np = ((struct nmi_pcpu *)&dbg0_stack[sizeof(dbg0_stack)]) - 1;
+ np->np_pcpu = (register_t)pc;
+ tssp->tss_ist4 = (long)np;
+ }
}
/*
@@ -1359,6 +1384,13 @@ hammer_time(u_int64_t modulep, u_int64_t physfree)
TUNABLE_INT_FETCH("hw.use_xsave", &use_xsave);
}
+ if ((cpu_stdext_feature4 & (CPUID_STDEXT4_FRED | CPUID_STDEXT4_LKGS)) ==
+ (CPUID_STDEXT4_FRED | CPUID_STDEXT4_LKGS) &&
+ (cpu_stdext_feature & CPUID_STDEXT_FSGSBASE) != 0 && !pti) {
+ fred = 1;
+ TUNABLE_INT_FETCH("hw.fred", &fred);
+ }
+
sched_instance_select();
link_elf_ireloc();
@@ -1473,9 +1505,11 @@ hammer_time(u_int64_t modulep, u_int64_t physfree)
setidt(IDT_EVTCHN, pti ? &IDTVEC(xen_intr_upcall_pti) :
&IDTVEC(xen_intr_upcall), SDT_SYSIGT, SEL_KPL, 0);
#endif
- r_idt.rd_limit = sizeof(idt0) - 1;
- r_idt.rd_base = (long) idt;
- lidt(&r_idt);
+ if (!fred) {
+ r_idt.rd_limit = sizeof(idt0) - 1;
+ r_idt.rd_base = (long) idt;
+ lidt(&r_idt);
+ }
TUNABLE_INT_FETCH("hw.ibrs_disable", &hw_ibrs_disable);
TUNABLE_INT_FETCH("machdep.mitigations.ibrs.disable", &hw_ibrs_disable);
@@ -1564,6 +1598,9 @@ hammer_time(u_int64_t modulep, u_int64_t physfree)
if (getenv_is_true("debug.dump_modinfo_at_boot"))
preload_dump();
+ if (fred)
+ amd64_cpu_init_fred();
+
#ifdef DEV_ISA
#ifdef DEV_ATPIC
elcr_probe();
@@ -1828,6 +1865,11 @@ clear_pcb_flags(struct pcb *pcb, const u_int flags)
extern const char wrmsr_early_safe_gp_handler[];
+/*
+ * What about FRED? wrmsr_early_safe_start() is used before we
+ * switched CPU to the FRED mode. We use IDT to catch #GP from MSR
+ * write even if BSP is switched to the FRED mode later.
+ */
void
wrmsr_early_safe_start(void)
{
diff --git a/sys/amd64/amd64/mp_machdep.c b/sys/amd64/amd64/mp_machdep.c
index 69490b8e275f..81a24a4f4df5 100644
--- a/sys/amd64/amd64/mp_machdep.c
+++ b/sys/amd64/amd64/mp_machdep.c
@@ -240,25 +240,51 @@ init_secondary(void)
IOPERM_BITMAP_SIZE;
pc->pc_common_tss.tss_rsp0 = 0;
- /* The doublefault stack runs on IST1. */
- np = ((struct nmi_pcpu *)&doublefault_stack[DBLFAULT_STACK_SIZE]) - 1;
- np->np_pcpu = (register_t)pc;
- pc->pc_common_tss.tss_ist1 = (long)np;
-
- /* The NMI stack runs on IST2. */
- np = ((struct nmi_pcpu *)&nmi_stack[NMI_STACK_SIZE]) - 1;
- np->np_pcpu = (register_t)pc;
- pc->pc_common_tss.tss_ist2 = (long)np;
-
- /* The MC# stack runs on IST3. */
- np = ((struct nmi_pcpu *)&mce_stack[MCE_STACK_SIZE]) - 1;
- np->np_pcpu = (register_t)pc;
- pc->pc_common_tss.tss_ist3 = (long)np;
-
- /* The DB# stack runs on IST4. */
- np = ((struct nmi_pcpu *)&dbg_stack[DBG_STACK_SIZE]) - 1;
- np->np_pcpu = (register_t)pc;
- pc->pc_common_tss.tss_ist4 = (long)np;
+ /*
+ * The doublefault stack.
+ * Runs on IST1 for IDT.
+ * Uses CSL 2 for FRED.
+ */
+ if (fred) {
+ wrmsr(MSR_FRED_RSP2, (uint64_t)&doublefault_stack[
+ DBLFAULT_STACK_SIZE]);
+ } else {
+ np = ((struct nmi_pcpu *)&doublefault_stack[
+ DBLFAULT_STACK_SIZE]) - 1;
+ np->np_pcpu = (register_t)pc;
+ pc->pc_common_tss.tss_ist1 = (long)np;
+ }
+
+ /*
+ * The NMI stack.
+ * Runs on IST2 for IDT.
+ * Uses CSL 1 for FRED.
+ */
+ if (fred) {
+ wrmsr(MSR_FRED_RSP1, (uint64_t)&nmi_stack[NMI_STACK_SIZE]);
+ } else {
+ np = ((struct nmi_pcpu *)&nmi_stack[NMI_STACK_SIZE]) - 1;
+ np->np_pcpu = (register_t)pc;
+ pc->pc_common_tss.tss_ist2 = (long)np;
+ }
+
+ /*
+ * The MC# stack.
+ * Runs on IST3 for IDT.
+ * Shares CSL 1 with NMI for FRED.
+ */
+ if (!fred) {
+ np = ((struct nmi_pcpu *)&mce_stack[MCE_STACK_SIZE]) - 1;
+ np->np_pcpu = (register_t)pc;
+ pc->pc_common_tss.tss_ist3 = (long)np;
+ }
+
+ if (!fred) {
+ /* The DB# stack, used for for IDT, runs on IST4. */
+ np = ((struct nmi_pcpu *)&dbg_stack[DBG_STACK_SIZE]) - 1;
+ np->np_pcpu = (register_t)pc;
+ pc->pc_common_tss.tss_ist4 = (long)np;
+ }
/* Prepare private GDT */
gdt_segs[GPROC0_SEL].ssd_base = (long)&pc->pc_common_tss;
@@ -278,7 +304,10 @@ init_secondary(void)
wrmsr(MSR_KGSBASE, 0); /* User value */
fix_cpuid();
- lidt(&r_idt);
+ if (fred)
+ amd64_cpu_init_fred();
+ else
+ lidt(&r_idt);
gsel_tss = GSEL(GPROC0_SEL, SEL_KPL);
ltr(gsel_tss);
@@ -430,12 +459,17 @@ start_all_aps(void)
M_WAITOK | M_ZERO);
doublefault_stack = kmem_malloc(DBLFAULT_STACK_SIZE,
M_WAITOK | M_ZERO);
- mce_stack = kmem_malloc(MCE_STACK_SIZE,
- M_WAITOK | M_ZERO);
+ if (!fred) {
+ mce_stack = kmem_malloc(MCE_STACK_SIZE,
+ M_WAITOK | M_ZERO);
+ }
nmi_stack = kmem_malloc_domainset(
DOMAINSET_PREF(domain), NMI_STACK_SIZE, M_WAITOK | M_ZERO);
- dbg_stack = kmem_malloc_domainset(
- DOMAINSET_PREF(domain), DBG_STACK_SIZE, M_WAITOK | M_ZERO);
+ if (!fred) {
+ dbg_stack = kmem_malloc_domainset(
+ DOMAINSET_PREF(domain), DBG_STACK_SIZE,
+ M_WAITOK | M_ZERO);
+ }
dpcpu = kmem_malloc_domainset(DOMAINSET_PREF(domain),
DPCPU_SIZE, M_WAITOK | M_ZERO);
diff --git a/sys/amd64/amd64/trap.c b/sys/amd64/amd64/trap.c
index df9cc44bcbb2..815d356217f9 100644
--- a/sys/amd64/amd64/trap.c
+++ b/sys/amd64/amd64/trap.c
@@ -41,6 +41,7 @@
* AMD64 Trap and System call handling
*/
+#include "opt_atpic.h"
#include "opt_clock.h"
#include "opt_cpu.h"
#include "opt_hwpmc_hooks.h"
@@ -61,6 +62,7 @@
#include <sys/mutex.h>
#include <sys/resourcevar.h>
#include <sys/signalvar.h>
+#include <sys/smp.h>
#include <sys/syscall.h>
#include <sys/sysctl.h>
#include <sys/sysent.h>
@@ -83,6 +85,8 @@ PMC_SOFT_DEFINE( , , page_fault, write);
#include <machine/cpu.h>
#include <machine/intr_machdep.h>
+#include <x86/apicreg.h>
+#include <x86/apicvar.h>
#include <x86/mca.h>
#include <machine/md_var.h>
#include <machine/pcb.h>
@@ -90,6 +94,7 @@ PMC_SOFT_DEFINE( , , page_fault, write);
#include <machine/stack.h>
#include <machine/trap.h>
#include <machine/tss.h>
+#include <x86/isa/icu.h>
#ifdef KDTRACE_HOOKS
#include <sys/dtrace_bsd.h>
@@ -995,6 +1000,14 @@ trap_diag(struct trapframe *frame, vm_offset_t eva, const char *type_str)
frame->tf_r11, frame->tf_r12);
printf("r13: %016lx r14: %016lx r15: %016lx\n", frame->tf_r13,
frame->tf_r14, frame->tf_r15);
+ if (fred) {
+ struct trapframe_fred *f;
+
+ f = __containerof(frame, struct trapframe_fred, tf_idt);
+ printf("evdata %016lx evinfo1 %04x evinfo2 %08x\n",
+ f->tf_fred_evdata, frame->tf_fred_evinfo1,
+ frame->tf_fred_evinfo2);
+ }
}
static void
@@ -1069,6 +1082,14 @@ dblfault_handler(struct trapframe *frame)
frame->tf_cs, frame->tf_ss, frame->tf_ds, frame->tf_es,
frame->tf_fs, frame->tf_gs,
rdmsr(MSR_FSBASE), rdmsr(MSR_GSBASE), rdmsr(MSR_KGSBASE));
+ if (fred) {
+ struct trapframe_fred *f;
+
+ f = __containerof(frame, struct trapframe_fred, tf_idt);
+ printf("evdata %016lx evinfo1 %04x evinfo2 %08x\n",
+ f->tf_fred_evdata, frame->tf_fred_evinfo1,
+ frame->tf_fred_evinfo2);
+ }
/* Print these separately in case pcpu accesses trap. */
printf("cpuid = %d; apic id = %02x\n", PCPU_GET(cpuid),
PCPU_GET(apic_id));
@@ -1295,3 +1316,540 @@ amd64_syscall(struct thread *td, int traced)
amd64_syscall_ret_flush_l1d_check_inline(td->td_errno);
}
+
+static void
+trap_fred_extint(struct trapframe_fred *frame, u_int vec)
+{
+ switch (vec) {
+ case IPI_RENDEZVOUS:
+#ifdef COUNT_IPIS
+ ipi_rendezvous_counts[PCPU_GET(cpuid)]++;
+#endif
+ smp_rendezvous_action();
+ lapic_eoi();
+ break;
+ case IPI_INVLOP:
+ invlop_handler();
+ lapic_eoi();
+ break;
+ case IPI_BITMAP_VECTOR:
+ lapic_eoi();
+ ipi_bitmap_handler(&frame->tf_idt);
+ break;
+ case IPI_STOP:
+ lapic_eoi();
+ cpustop_handler();
+ break;
+ case IPI_SUSPEND:
+ cpususpend_handler();
+ lapic_eoi();
+ break;
+ case IPI_SWI:
+ lapic_eoi();
+ ipi_swi_handler(&frame->tf_idt);
+ break;
+ case IPI_OFF:
+ cpuoff_handler();
+ lapic_eoi();
+ break;
+ case APIC_SPURIOUS_INT:
+ break;
+ case APIC_TIMER_INT:
+ lapic_handle_timer(&frame->tf_idt);
+ break;
+ case APIC_CMC_INT:
+ lapic_handle_cmc();
+ break;
+ case APIC_ERROR_INT:
+ lapic_handle_error();
+ break;
+ case APIC_THERMAL_INT:
+ lapic_handle_thermal();
+ break;
+#ifdef XENHVM
+ case IDT_EVTCHN:
+ xen_arch_intr_handle_upcall(&frame->tf_idt);
+ break;
+#endif
+ default:
+ if (vec >= IPI_DYN_FIRST && vec <= IPI_DYN_LAST) {
+ void (*fi)(struct trapframe *);
+
+ fi = fred_ipi_handlers[vec - IPI_DYN_FIRST];
+ if (fi != NULL) {
*** 749 LINES SKIPPED ***