svn commit: r339507 - in head/sys: amd64/amd64 amd64/ia32 amd64/include dev/cpuctl
Konstantin Belousov
kib at FreeBSD.org
Sat Oct 20 23:17:26 UTC 2018
Author: kib
Date: Sat Oct 20 23:17:24 2018
New Revision: 339507
URL: https://svnweb.freebsd.org/changeset/base/339507
Log:
amd64: flush L1 data cache on syscall return with an error.
The knob allows to select the flushing mode or turn it off/on. The
idea, as well as the list of the ignored syscall errors, were taken
from https://www.openwall.com/lists/kernel-hardening/2018/10/11/10 .
I was not able to measure statistically significant difference between
flush enabled vs disabled using syscall_timing getuid.
Reviewed by: bwidawsk
Sponsored by: The FreeBSD Foundation
MFC after: 1 week
Differential revision: https://reviews.freebsd.org/D17536
Modified:
head/sys/amd64/amd64/initcpu.c
head/sys/amd64/amd64/machdep.c
head/sys/amd64/amd64/support.S
head/sys/amd64/amd64/trap.c
head/sys/amd64/ia32/ia32_syscall.c
head/sys/amd64/include/md_var.h
head/sys/dev/cpuctl/cpuctl.c
Modified: head/sys/amd64/amd64/initcpu.c
==============================================================================
--- head/sys/amd64/amd64/initcpu.c Sat Oct 20 23:08:04 2018 (r339506)
+++ head/sys/amd64/amd64/initcpu.c Sat Oct 20 23:17:24 2018 (r339507)
@@ -253,6 +253,7 @@ initializecpu(void)
}
hw_ibrs_recalculate();
hw_ssb_recalculate(false);
+ amd64_syscall_ret_flush_l1d_recalc();
switch (cpu_vendor_id) {
case CPU_VENDOR_AMD:
init_amd();
Modified: head/sys/amd64/amd64/machdep.c
==============================================================================
--- head/sys/amd64/amd64/machdep.c Sat Oct 20 23:08:04 2018 (r339506)
+++ head/sys/amd64/amd64/machdep.c Sat Oct 20 23:17:24 2018 (r339507)
@@ -1722,6 +1722,11 @@ hammer_time(u_int64_t modulep, u_int64_t physfree)
!= NULL)
vty_set_preferred(VTY_VT);
+ TUNABLE_INT_FETCH("hw.ibrs_disable", &hw_ibrs_disable);
+ TUNABLE_INT_FETCH("hw.spec_store_bypass_disable", &hw_ssb_disable);
+ TUNABLE_INT_FETCH("machdep.syscall_ret_l1d_flush",
+ &syscall_ret_l1d_flush_mode);
+
finishidentcpu(); /* Final stage of CPU initialization */
initializecpu(); /* Initialize CPU registers */
@@ -1864,9 +1869,6 @@ hammer_time(u_int64_t modulep, u_int64_t physfree)
x86_init_fdt();
#endif
thread0.td_critnest = 0;
-
- TUNABLE_INT_FETCH("hw.ibrs_disable", &hw_ibrs_disable);
- TUNABLE_INT_FETCH("hw.spec_store_bypass_disable", &hw_ssb_disable);
TSEXIT();
Modified: head/sys/amd64/amd64/support.S
==============================================================================
--- head/sys/amd64/amd64/support.S Sat Oct 20 23:08:04 2018 (r339506)
+++ head/sys/amd64/amd64/support.S Sat Oct 20 23:17:24 2018 (r339507)
@@ -1556,3 +1556,10 @@ ENTRY(flush_l1d_sw)
ret
#undef L1D_FLUSH_SIZE
END(flush_l1d_sw)
+
+ENTRY(flush_l1d_sw_abi)
+ pushq %rbx
+ call flush_l1d_sw
+ popq %rbx
+ ret
+END(flush_l1d_sw_abi)
Modified: head/sys/amd64/amd64/trap.c
==============================================================================
--- head/sys/amd64/amd64/trap.c Sat Oct 20 23:08:04 2018 (r339506)
+++ head/sys/amd64/amd64/trap.c Sat Oct 20 23:17:24 2018 (r339507)
@@ -1056,6 +1056,84 @@ cpu_fetch_syscall_args(struct thread *td)
#include "../../kern/subr_syscall.c"
+static void (*syscall_ret_l1d_flush)(void);
+int syscall_ret_l1d_flush_mode;
+
+static void
+flush_l1d_hw(void)
+{
+
+ wrmsr(MSR_IA32_FLUSH_CMD, IA32_FLUSH_CMD_L1D);
+}
+
+static void __inline
+amd64_syscall_ret_flush_l1d_inline(int error)
+{
+ void (*p)(void);
+
+ if (error != 0 && error != EEXIST && error != EAGAIN &&
+ error != EXDEV && error != ENOENT && error != ENOTCONN &&
+ error != EINPROGRESS) {
+ p = syscall_ret_l1d_flush;
+ if (p != NULL)
+ p();
+ }
+}
+
+void
+amd64_syscall_ret_flush_l1d(int error)
+{
+
+ amd64_syscall_ret_flush_l1d_inline(error);
+}
+
+void
+amd64_syscall_ret_flush_l1d_recalc(void)
+{
+ bool l1d_hw;
+
+ l1d_hw = (cpu_stdext_feature3 & CPUID_STDEXT3_L1D_FLUSH) != 0;
+again:
+ switch (syscall_ret_l1d_flush_mode) {
+ case 0:
+ syscall_ret_l1d_flush = NULL;
+ break;
+ case 1:
+ syscall_ret_l1d_flush = l1d_hw ? flush_l1d_hw :
+ flush_l1d_sw_abi;
+ break;
+ case 2:
+ syscall_ret_l1d_flush = l1d_hw ? flush_l1d_hw : NULL;
+ break;
+ case 3:
+ syscall_ret_l1d_flush = flush_l1d_sw_abi;
+ break;
+ default:
+ syscall_ret_l1d_flush_mode = 1;
+ goto again;
+ }
+}
+
+static int
+machdep_syscall_ret_flush_l1d(SYSCTL_HANDLER_ARGS)
+{
+ int error, val;
+
+ val = syscall_ret_l1d_flush_mode;
+ error = sysctl_handle_int(oidp, &val, 0, req);
+ if (error != 0 || req->newptr == NULL)
+ return (error);
+ syscall_ret_l1d_flush_mode = val;
+ amd64_syscall_ret_flush_l1d_recalc();
+ return (0);
+}
+SYSCTL_PROC(_machdep, OID_AUTO, syscall_ret_flush_l1d, CTLTYPE_INT |
+ CTLFLAG_RWTUN | CTLFLAG_NOFETCH | CTLFLAG_MPSAFE, NULL, 0,
+ machdep_syscall_ret_flush_l1d, "I",
+ "Flush L1D on syscall return with error (0 - off, 1 - on, "
+ "2 - use hw only, 3 - use sw only");
+
+
/*
* System call handler for native binaries. The trap frame is already
* set up by the assembler trampoline and a pointer to it is saved in
@@ -1110,4 +1188,6 @@ amd64_syscall(struct thread *td, int traced)
*/
if (__predict_false(td->td_frame->tf_rip >= VM_MAXUSER_ADDRESS))
set_pcb_flags(td->td_pcb, PCB_FULL_IRET);
+
+ amd64_syscall_ret_flush_l1d_inline(error);
}
Modified: head/sys/amd64/ia32/ia32_syscall.c
==============================================================================
--- head/sys/amd64/ia32/ia32_syscall.c Sat Oct 20 23:08:04 2018 (r339506)
+++ head/sys/amd64/ia32/ia32_syscall.c Sat Oct 20 23:17:24 2018 (r339507)
@@ -231,6 +231,7 @@ ia32_syscall(struct trapframe *frame)
}
syscallret(td, error);
+ amd64_syscall_ret_flush_l1d(error);
}
static void
Modified: head/sys/amd64/include/md_var.h
==============================================================================
--- head/sys/amd64/include/md_var.h Sat Oct 20 23:08:04 2018 (r339506)
+++ head/sys/amd64/include/md_var.h Sat Oct 20 23:17:24 2018 (r339507)
@@ -41,6 +41,7 @@ extern int hw_lower_amd64_sharedpage;
extern int hw_ibrs_disable;
extern int hw_ssb_disable;
extern int nmi_flush_l1d_sw;
+extern int syscall_ret_l1d_flush_mode;
/*
* The file "conf/ldscript.amd64" defines the symbol "kernphys". Its
@@ -55,8 +56,11 @@ void amd64_conf_fast_syscall(void);
void amd64_db_resume_dbreg(void);
void amd64_lower_shared_page(struct sysentvec *);
void amd64_syscall(struct thread *td, int traced);
+void amd64_syscall_ret_flush_l1d(int error);
+void amd64_syscall_ret_flush_l1d_recalc(void);
void doreti_iret(void) __asm(__STRING(doreti_iret));
void doreti_iret_fault(void) __asm(__STRING(doreti_iret_fault));
+void flush_l1d_sw_abi(void);
void ld_ds(void) __asm(__STRING(ld_ds));
void ld_es(void) __asm(__STRING(ld_es));
void ld_fs(void) __asm(__STRING(ld_fs));
Modified: head/sys/dev/cpuctl/cpuctl.c
==============================================================================
--- head/sys/dev/cpuctl/cpuctl.c Sat Oct 20 23:08:04 2018 (r339506)
+++ head/sys/dev/cpuctl/cpuctl.c Sat Oct 20 23:17:24 2018 (r339507)
@@ -521,6 +521,9 @@ cpuctl_do_eval_cpu_features(int cpu, struct thread *td
hw_ibrs_recalculate();
restore_cpu(oldcpu, is_bound, td);
hw_ssb_recalculate(true);
+#ifdef __amd64__
+ amd64_syscall_ret_flush_l1d_recalc();
+#endif
printcpuinfo();
return (0);
}
More information about the svn-src-head
mailing list