git: d39f7430a6e1 - main - amd64: preserve %cr2 in NMI/MCE/DBG handlers.

[Konstantin Belousov]

The branch main has been updated by kib:

URL: https://cgit.FreeBSD.org/src/commit/?id=d39f7430a6e1da419d6e4fb871bca5ba7863f738

commit d39f7430a6e1da419d6e4fb871bca5ba7863f738
Author:     Konstantin Belousov <kib at FreeBSD.org>
AuthorDate: 2020-12-25 21:58:43 +0000
Commit:     Konstantin Belousov <kib at FreeBSD.org>
CommitDate: 2020-12-27 10:59:33 +0000

    amd64: preserve %cr2 in NMI/MCE/DBG handlers.
    
    These handlers could interrupt code which has interrupts disabled,
    and if a spurious page fault occurs during exception handler run,
    we get clobbered %cr2 in higher level stack.
    
    This is mostly a speculation, but it is based on hints from good sources.
    
    MFC after:      1 week
    Reviewed by:    markj
    Sponsored by:   The FreeBSD Foundation
    Differential Revision:  https://reviews.freebsd.org/D27772
---
 sys/amd64/amd64/exception.S | 21 +++++++++++++++------
 sys/amd64/amd64/support.S   |  3 ++-
 2 files changed, 17 insertions(+), 7 deletions(-)

diff --git a/sys/amd64/amd64/exception.S b/sys/amd64/amd64/exception.S
index 948f6c40e776..6b56a2c8a50e 100644
--- a/sys/amd64/amd64/exception.S
+++ b/sys/amd64/amd64/exception.S
@@ -667,9 +667,10 @@ IDTVEC(dbg)
 	jnz	dbg_fromuserspace
 	lfence
 	/*
-	 * We've interrupted the kernel.  Preserve GS.base in %r12,
-	 * %cr3 in %r13, and possibly lower half of MSR_IA32_SPEC_CTL in %r14d.
+	 * We've interrupted the kernel.  See comment in NMI handler about
+	 * registers use.
 	 */
+	movq	%cr2,%r15
 	movl	$MSR_GSBASE,%ecx
 	rdmsr
 	movq	%rax,%r12
@@ -710,6 +711,7 @@ IDTVEC(dbg)
 	shrq	$32,%rdx
 	wrmsr
 	movq	%r13,%cr3
+	movq	%r15,%cr2
 	RESTORE_REGS
 	addq	$TF_RIP,%rsp
 	jmp	doreti_iret
@@ -804,10 +806,14 @@ IDTVEC(nmi)
 	testb	$SEL_RPL_MASK,TF_CS(%rsp)
 	jnz	nmi_fromuserspace
 	/*
-	 * We've interrupted the kernel.  Preserve GS.base in %r12,
-	 * %cr3 in %r13, and possibly lower half of MSR_IA32_SPEC_CTL in %r14d.
+	 * We've interrupted the kernel.  Preserve in callee-saved regs:
+	 * GS.base in %r12,
+	 * %cr3 in %r13,
+	 * possibly lower half of MSR_IA32_SPEC_CTL in %r14d,
+	 * %cr2 in %r15.
 	 */
 	lfence
+	movq	%cr2,%r15
 	movl	$MSR_GSBASE,%ecx
 	rdmsr
 	movq	%rax,%r12
@@ -957,6 +963,7 @@ nocallchain:
 	je	2f
 	call	flush_l1d_sw		/* bhyve L1TF assist */
 2:	movq	%r13,%cr3
+	movq	%r15,%cr2
 	RESTORE_REGS
 	addq	$TF_RIP,%rsp
 	jmp	doreti_iret
@@ -1011,9 +1018,10 @@ IDTVEC(mchk)
 	testb	$SEL_RPL_MASK,TF_CS(%rsp)
 	jnz	mchk_fromuserspace
 	/*
-	 * We've interrupted the kernel.  Preserve GS.base in %r12,
-	 * %cr3 in %r13, and possibly lower half of MSR_IA32_SPEC_CTL in %r14d.
+	 * We've interrupted the kernel.  See comment in NMI handler about
+	 * registers use.
 	 */
+	movq	%cr2,%r15
 	movl	$MSR_GSBASE,%ecx
 	rdmsr
 	movq	%rax,%r12
@@ -1071,6 +1079,7 @@ mchk_calltrap:
 	shrq	$32,%rdx
 	wrmsr
 	movq	%r13,%cr3
+	movq	%r15,%cr2
 	RESTORE_REGS
 	addq	$TF_RIP,%rsp
 	jmp	doreti_iret
diff --git a/sys/amd64/amd64/support.S b/sys/amd64/amd64/support.S
index 5a25dc1d9695..b0ef54a27ca1 100644
--- a/sys/amd64/amd64/support.S
+++ b/sys/amd64/amd64/support.S
@@ -1651,7 +1651,8 @@ END(handle_ibrs_exit_rs)
  *
  * N.B. The function does not follow ABI calling conventions, it corrupts %rbx.
  * The vmm.ko caller expects that only %rax, %rdx, %rbx, %rcx, %r9, and %rflags
- * registers are clobbered.  The NMI handler caller only needs %r13 preserved.
+ * registers are clobbered.  The NMI handler caller only needs %r13 and %r15
+ * preserved.
  */
 ENTRY(flush_l1d_sw)
 #define	L1D_FLUSH_SIZE	(64 * 1024)