svn commit: r331988 - in stable/11/sys/arm: arm include
Michal Meloun
mmel at FreeBSD.org
Wed Apr 4 06:11:07 UTC 2018
Author: mmel
Date: Wed Apr 4 06:11:05 2018
New Revision: 331988
URL: https://svnweb.freebsd.org/changeset/base/331988
Log:
MFC r328467:
Implement mitigation for Spectre version 2 attacks on ARMv7.
Modified:
stable/11/sys/arm/arm/cpuinfo.c
stable/11/sys/arm/arm/genassym.c
stable/11/sys/arm/arm/machdep.c
stable/11/sys/arm/arm/mp_machdep.c
stable/11/sys/arm/arm/swtch-v6.S
stable/11/sys/arm/arm/trap-v6.c
stable/11/sys/arm/include/cpuinfo.h
stable/11/sys/arm/include/pcpu.h
Directory Properties:
stable/11/ (props changed)
Modified: stable/11/sys/arm/arm/cpuinfo.c
==============================================================================
--- stable/11/sys/arm/arm/cpuinfo.c Wed Apr 4 05:43:03 2018 (r331987)
+++ stable/11/sys/arm/arm/cpuinfo.c Wed Apr 4 06:11:05 2018 (r331988)
@@ -31,6 +31,8 @@ __FBSDID("$FreeBSD$");
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/kernel.h>
+#include <sys/pcpu.h>
+#include <sys/smp.h>
#include <sys/sysctl.h>
#include <machine/cpu.h>
@@ -40,6 +42,9 @@ __FBSDID("$FreeBSD$");
#if __ARM_ARCH >= 6
void reinit_mmu(uint32_t ttb, uint32_t aux_clr, uint32_t aux_set);
+
+int disable_bp_hardening;
+int spectre_v2_safe = 1;
#endif
struct cpuinfo cpuinfo =
@@ -253,6 +258,7 @@ cpuinfo_get_actlr_modifier(uint32_t *actlr_mask, uint3
if (cpuinfo.implementer == CPU_IMPLEMENTER_ARM) {
switch (cpuinfo.part_number) {
+ case CPU_ARCH_CORTEX_A75:
case CPU_ARCH_CORTEX_A73:
case CPU_ARCH_CORTEX_A72:
case CPU_ARCH_CORTEX_A57:
@@ -335,5 +341,198 @@ cpuinfo_reinit_mmu(uint32_t ttb)
actlr_set |= cpu_quirks_actlr_set;
reinit_mmu(ttb, actlr_mask, actlr_set);
}
+
+static bool
+modify_actlr(uint32_t clear, uint32_t set)
+{
+ uint32_t reg, newreg;
+
+ reg = cp15_actlr_get();
+ newreg = reg;
+ newreg &= ~clear;
+ newreg |= set;
+ if (reg == newreg)
+ return (true);
+ cp15_actlr_set(newreg);
+
+ reg = cp15_actlr_get();
+ if (reg == newreg)
+ return (true);
+ return (false);
+}
+
+/* Apply/restore BP hardening on current core. */
+static int
+apply_bp_hardening(bool enable, int kind, bool actrl, uint32_t set_mask)
+{
+ if (enable) {
+ if (actrl && !modify_actlr(0, set_mask))
+ return (-1);
+ PCPU_SET(bp_harden_kind, kind);
+ } else {
+ PCPU_SET(bp_harden_kind, PCPU_BP_HARDEN_KIND_NONE);
+ if (actrl)
+ modify_actlr(~0, PCPU_GET(original_actlr));
+ spectre_v2_safe = 0;
+ }
+ return (0);
+}
+
+static void
+handle_bp_hardening(bool enable)
+{
+ int kind;
+ char *kind_str;
+
+ kind = PCPU_BP_HARDEN_KIND_NONE;
+ /*
+ * Note: Access to ACTRL is locked to secure world on most boards.
+ * This means that full BP hardening depends on updated u-boot/firmware
+ * or is impossible at all (if secure monitor is in on-chip ROM).
+ */
+ if (cpuinfo.implementer == CPU_IMPLEMENTER_ARM) {
+ switch (cpuinfo.part_number) {
+ case CPU_ARCH_CORTEX_A8:
+ /*
+ * For Cortex-A8, IBE bit must be set otherwise
+ * BPIALL is effectively NOP.
+ * Unfortunately, Cortex-A is also affected by
+ * ARM erratum 687067 which causes non-working
+ * BPIALL if IBE bit is set and 'Instruction L1 System
+ * Array Debug Register 0' is not 0.
+ * This register is not reset on power-up and is
+ * accessible only from secure world, so we cannot do
+ * nothing (nor detect) to fix this issue.
+ * I afraid that on chip ROM based secure monitor on
+ * AM335x (BeagleBone) doesn't reset this debug
+ * register.
+ */
+ kind = PCPU_BP_HARDEN_KIND_BPIALL;
+ if (apply_bp_hardening(enable, kind, true, 1 << 6) != 0)
+ goto actlr_err;
+ break;
+ break;
+
+ case CPU_ARCH_CORTEX_A9:
+ case CPU_ARCH_CORTEX_A12:
+ case CPU_ARCH_CORTEX_A17:
+ case CPU_ARCH_CORTEX_A57:
+ case CPU_ARCH_CORTEX_A72:
+ case CPU_ARCH_CORTEX_A73:
+ case CPU_ARCH_CORTEX_A75:
+ kind = PCPU_BP_HARDEN_KIND_BPIALL;
+ if (apply_bp_hardening(enable, kind, false, 0) != 0)
+ goto actlr_err;
+ break;
+
+ case CPU_ARCH_CORTEX_A15:
+ /*
+ * For Cortex-A15, set 'Enable invalidates of BTB' bit.
+ * Despite this, the BPIALL is still effectively NOP,
+ * but with this bit set, the ICIALLU also flushes
+ * branch predictor as side effect.
+ */
+ kind = PCPU_BP_HARDEN_KIND_ICIALLU;
+ if (apply_bp_hardening(enable, kind, true, 1 << 0) != 0)
+ goto actlr_err;
+ break;
+
+ default:
+ break;
+ }
+ } else if (cpuinfo.implementer == CPU_IMPLEMENTER_QCOM) {
+ printf("!!!WARNING!!! CPU(%d) is vulnerable to speculative "
+ "branch attacks. !!!\n"
+ "Qualcomm Krait cores are known (or believed) to be "
+ "vulnerable to \n"
+ "speculative branch attacks, no mitigation exists yet.\n",
+ PCPU_GET(cpuid));
+ goto unkonown_mitigation;
+ } else {
+ goto unkonown_mitigation;
+ }
+
+ if (bootverbose) {
+ switch (kind) {
+ case PCPU_BP_HARDEN_KIND_NONE:
+ kind_str = "not necessary";
+ break;
+ case PCPU_BP_HARDEN_KIND_BPIALL:
+ kind_str = "BPIALL";
+ break;
+ case PCPU_BP_HARDEN_KIND_ICIALLU:
+ kind_str = "ICIALLU";
+ break;
+ default:
+ panic("Unknown BP hardering kind (%d).", kind);
+ }
+ printf("CPU(%d) applied BP hardening: %s\n", PCPU_GET(cpuid),
+ kind_str);
+ }
+
+ return;
+
+unkonown_mitigation:
+ PCPU_SET(bp_harden_kind, PCPU_BP_HARDEN_KIND_NONE);
+ spectre_v2_safe = 0;
+ return;
+
+actlr_err:
+ PCPU_SET(bp_harden_kind, PCPU_BP_HARDEN_KIND_NONE);
+ spectre_v2_safe = 0;
+ printf("!!!WARNING!!! CPU(%d) is vulnerable to speculative branch "
+ "attacks. !!!\n"
+ "We cannot enable required bit(s) in ACTRL register\n"
+ "because it's locked by secure monitor and/or firmware.\n",
+ PCPU_GET(cpuid));
+}
+
+void
+cpuinfo_init_bp_hardening(void)
+{
+
+ /*
+ * Store original unmodified ACTRL, so we can restore it when
+ * BP hardening is disabled by sysctl.
+ */
+ PCPU_SET(original_actlr, cp15_actlr_get());
+ handle_bp_hardening(true);
+}
+
+static void
+bp_hardening_action(void *arg)
+{
+
+ handle_bp_hardening(disable_bp_hardening == 0);
+}
+
+static int
+sysctl_disable_bp_hardening(SYSCTL_HANDLER_ARGS)
+{
+ int rv;
+
+ rv = sysctl_handle_int(oidp, oidp->oid_arg1, oidp->oid_arg2, req);
+
+ if (!rv && req->newptr) {
+ spectre_v2_safe = 1;
+ dmb();
+#ifdef SMP
+ smp_rendezvous_cpus(all_cpus, smp_no_rendezvous_barrier,
+ bp_hardening_action, NULL, NULL);
+#else
+ bp_hardening_action(NULL);
+#endif
+ }
+
+ return (rv);
+}
+
+SYSCTL_PROC(_machdep, OID_AUTO, disable_bp_hardening,
+ CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE,
+ &disable_bp_hardening, 0, sysctl_disable_bp_hardening, "I",
+ "Disable BP hardening mitigation.");
+
+SYSCTL_INT(_machdep, OID_AUTO, spectre_v2_safe, CTLFLAG_RD,
+ &spectre_v2_safe, 0, "System is safe to Spectre Version 2 attacks");
#endif /* __ARM_ARCH >= 6 */
Modified: stable/11/sys/arm/arm/genassym.c
==============================================================================
--- stable/11/sys/arm/arm/genassym.c Wed Apr 4 05:43:03 2018 (r331987)
+++ stable/11/sys/arm/arm/genassym.c Wed Apr 4 06:11:05 2018 (r331988)
@@ -31,6 +31,7 @@ __FBSDID("$FreeBSD$");
#include <sys/cpuset.h>
#include <sys/systm.h>
#include <sys/assym.h>
+#include <sys/pcpu.h>
#include <sys/proc.h>
#include <sys/mbuf.h>
#include <sys/vmmeter.h>
@@ -134,6 +135,10 @@ ASSYM(PCB_VFPSTATE, offsetof(struct pcb, pcb_vfpstate)
#if __ARM_ARCH >= 6
ASSYM(PC_CURPMAP, offsetof(struct pcpu, pc_curpmap));
+ASSYM(PC_BP_HARDEN_KIND, offsetof(struct pcpu, pc_bp_harden_kind));
+ASSYM(PCPU_BP_HARDEN_KIND_NONE, PCPU_BP_HARDEN_KIND_NONE);
+ASSYM(PCPU_BP_HARDEN_KIND_BPIALL, PCPU_BP_HARDEN_KIND_BPIALL);
+ASSYM(PCPU_BP_HARDEN_KIND_ICIALLU, PCPU_BP_HARDEN_KIND_ICIALLU);
#endif
ASSYM(PAGE_SIZE, PAGE_SIZE);
Modified: stable/11/sys/arm/arm/machdep.c
==============================================================================
--- stable/11/sys/arm/arm/machdep.c Wed Apr 4 05:43:03 2018 (r331987)
+++ stable/11/sys/arm/arm/machdep.c Wed Apr 4 06:11:05 2018 (r331988)
@@ -1260,6 +1260,8 @@ initarm(struct arm_boot_params *abp)
msgbufinit(msgbufp, msgbufsize);
dbg_monitor_init();
arm_kdb_init();
+ /* Apply possible BP hardening. */
+ cpuinfo_init_bp_hardening();
return ((void *)STACKALIGN(thread0.td_pcb));
}
Modified: stable/11/sys/arm/arm/mp_machdep.c
==============================================================================
--- stable/11/sys/arm/arm/mp_machdep.c Wed Apr 4 05:43:03 2018 (r331987)
+++ stable/11/sys/arm/arm/mp_machdep.c Wed Apr 4 06:11:05 2018 (r331988)
@@ -200,6 +200,9 @@ init_secondary(int cpu)
/* Configure the interrupt controller */
intr_pic_init_secondary();
+ /* Apply possible BP hardening */
+ cpuinfo_init_bp_hardening();
+
mtx_lock_spin(&ap_boot_mtx);
atomic_add_rel_32(&smp_cpus, 1);
Modified: stable/11/sys/arm/arm/swtch-v6.S
==============================================================================
--- stable/11/sys/arm/arm/swtch-v6.S Wed Apr 4 05:43:03 2018 (r331987)
+++ stable/11/sys/arm/arm/swtch-v6.S Wed Apr 4 06:11:05 2018 (r331988)
@@ -145,7 +145,16 @@ ENTRY(cpu_context_switch)
* predictors and Requirements for branch predictor maintenance
* operations sections.
*/
- mcr CP15_BPIALL /* flush entire Branch Target Cache */
+ /*
+ * Additionally, to mitigate mistrained branch predictor attack
+ * we must invalidate it on affected CPUs. Unfortunately, BPIALL
+ * is effectively NOP on Cortex-A15 so it needs special treatment.
+ */
+ ldr r0, [r8, #PC_BP_HARDEN_KIND]
+ cmp r0, #PCPU_BP_HARDEN_KIND_ICIALLU
+ mcrne CP15_BPIALL /* Flush entire Branch Target Cache */
+ mcreq CP15_ICIALLU /* This is the only way how to flush */
+ /* Branch Target Cache on Cortex-A15. */
DSB
mov pc, lr
END(cpu_context_switch)
Modified: stable/11/sys/arm/arm/trap-v6.c
==============================================================================
--- stable/11/sys/arm/arm/trap-v6.c Wed Apr 4 05:43:03 2018 (r331987)
+++ stable/11/sys/arm/arm/trap-v6.c Wed Apr 4 06:11:05 2018 (r331988)
@@ -287,6 +287,7 @@ abort_handler(struct trapframe *tf, int prefetch)
struct vmspace *vm;
vm_prot_t ftype;
bool usermode;
+ int bp_harden;
#ifdef INVARIANTS
void *onfault;
#endif
@@ -303,6 +304,20 @@ abort_handler(struct trapframe *tf, int prefetch)
idx = FSR_TO_FAULT(fsr);
usermode = TRAPF_USERMODE(tf); /* Abort came from user mode? */
+
+ /*
+ * Apply BP hardening by flushing the branch prediction cache
+ * for prefaults on kernel addresses.
+ */
+ if (__predict_false(prefetch && far > VM_MAXUSER_ADDRESS &&
+ (idx == FAULT_TRAN_L2 || idx == FAULT_PERM_L2))) {
+ bp_harden = PCPU_GET(bp_harden_kind);
+ if (bp_harden == PCPU_BP_HARDEN_KIND_BPIALL)
+ _CP15_BPIALL();
+ else if (bp_harden == PCPU_BP_HARDEN_KIND_ICIALLU)
+ _CP15_ICIALLU();
+ }
+
if (usermode)
td->td_frame = tf;
Modified: stable/11/sys/arm/include/cpuinfo.h
==============================================================================
--- stable/11/sys/arm/include/cpuinfo.h Wed Apr 4 05:43:03 2018 (r331987)
+++ stable/11/sys/arm/include/cpuinfo.h Wed Apr 4 06:11:05 2018 (r331988)
@@ -49,6 +49,7 @@
#define CPU_ARCH_CORTEX_A57 0xD07
#define CPU_ARCH_CORTEX_A72 0xD08
#define CPU_ARCH_CORTEX_A73 0xD09
+#define CPU_ARCH_CORTEX_A75 0xD0A
/* QCOM */
@@ -122,6 +123,7 @@ extern struct cpuinfo cpuinfo;
void cpuinfo_init(void);
#if __ARM_ARCH >= 6
+void cpuinfo_init_bp_hardening(void);
void cpuinfo_reinit_mmu(uint32_t ttb);
#endif
#endif /* _MACHINE_CPUINFO_H_ */
Modified: stable/11/sys/arm/include/pcpu.h
==============================================================================
--- stable/11/sys/arm/include/pcpu.h Wed Apr 4 05:43:03 2018 (r331987)
+++ stable/11/sys/arm/include/pcpu.h Wed Apr 4 06:11:05 2018 (r331988)
@@ -42,6 +42,10 @@ struct vmspace;
#endif /* _KERNEL */
#if __ARM_ARCH >= 6
+/* Branch predictor hardening method */
+#define PCPU_BP_HARDEN_KIND_NONE 0
+#define PCPU_BP_HARDEN_KIND_BPIALL 1
+#define PCPU_BP_HARDEN_KIND_ICIALLU 2
#define PCPU_MD_FIELDS \
unsigned int pc_vfpsid; \
@@ -57,7 +61,9 @@ struct vmspace;
void *pc_qmap_pte2p; \
unsigned int pc_dbreg[32]; \
int pc_dbreg_cmd; \
- char __pad[19]
+ int pc_bp_harden_kind; \
+ uint32_t pc_original_actlr; \
+ char __pad[11]
#else
#define PCPU_MD_FIELDS \
char __pad[157]
More information about the svn-src-all
mailing list