svn commit: r358579 - in stable/12/sys: amd64/amd64 dev/cpuctl i386/i386 x86/acpica x86/include x86/x86

[Konstantin Belousov]

Author: kib
Date: Tue Mar  3 15:02:07 2020
New Revision: 358579
URL: https://svnweb.freebsd.org/changeset/base/358579

Log:
  MFC r358315:
  Fix IBRS for machines with IBRS_ALL capability.

Modified:
  stable/12/sys/amd64/amd64/initcpu.c
  stable/12/sys/amd64/amd64/support.S
  stable/12/sys/dev/cpuctl/cpuctl.c
  stable/12/sys/i386/i386/support.s
  stable/12/sys/x86/acpica/acpi_wakeup.c
  stable/12/sys/x86/include/x86_var.h
  stable/12/sys/x86/x86/cpu_machdep.c
Directory Properties:
  stable/12/   (props changed)

Modified: stable/12/sys/amd64/amd64/initcpu.c
==============================================================================
--- stable/12/sys/amd64/amd64/initcpu.c	Tue Mar  3 14:58:53 2020	(r358578)
+++ stable/12/sys/amd64/amd64/initcpu.c	Tue Mar  3 15:02:07 2020	(r358579)
@@ -255,7 +255,7 @@ initializecpu(void)
 		wrmsr(MSR_EFER, msr);
 		pg_nx = PG_NX;
 	}
-	hw_ibrs_recalculate();
+	hw_ibrs_recalculate(false);
 	hw_ssb_recalculate(false);
 	amd64_syscall_ret_flush_l1d_recalc();
 	switch (cpu_vendor_id) {

Modified: stable/12/sys/amd64/amd64/support.S
==============================================================================
--- stable/12/sys/amd64/amd64/support.S	Tue Mar  3 14:58:53 2020	(r358578)
+++ stable/12/sys/amd64/amd64/support.S	Tue Mar  3 15:02:07 2020	(r358579)
@@ -1547,7 +1547,7 @@ handle_ibrs_\l:
 
 /* all callers already saved %rax, %rdx, and %rcx */
 ENTRY(handle_ibrs_entry)
-	cmpb	$0,hw_ibrs_active(%rip)
+	cmpb	$0,hw_ibrs_ibpb_active(%rip)
 	je	1f
 	movl	$MSR_IA32_SPEC_CTRL,%ecx
 	rdmsr

Modified: stable/12/sys/dev/cpuctl/cpuctl.c
==============================================================================
--- stable/12/sys/dev/cpuctl/cpuctl.c	Tue Mar  3 14:58:53 2020	(r358578)
+++ stable/12/sys/dev/cpuctl/cpuctl.c	Tue Mar  3 15:02:07 2020	(r358579)
@@ -538,8 +538,8 @@ cpuctl_do_eval_cpu_features(int cpu, struct thread *td
 	set_cpu(cpu, td);
 	identify_cpu1();
 	identify_cpu2();
-	hw_ibrs_recalculate();
 	restore_cpu(oldcpu, is_bound, td);
+	hw_ibrs_recalculate(true);
 	hw_ssb_recalculate(true);
 #ifdef __amd64__
 	amd64_syscall_ret_flush_l1d_recalc();

Modified: stable/12/sys/i386/i386/support.s
==============================================================================
--- stable/12/sys/i386/i386/support.s	Tue Mar  3 14:58:53 2020	(r358578)
+++ stable/12/sys/i386/i386/support.s	Tue Mar  3 15:02:07 2020	(r358579)
@@ -446,7 +446,7 @@ msr_onfault:
 	ret
 
 ENTRY(handle_ibrs_entry)
-	cmpb	$0,hw_ibrs_active
+	cmpb	$0,hw_ibrs_ibpb_active
 	je	1f
 	movl	$MSR_IA32_SPEC_CTRL,%ecx
 	rdmsr

Modified: stable/12/sys/x86/acpica/acpi_wakeup.c
==============================================================================
--- stable/12/sys/x86/acpica/acpi_wakeup.c	Tue Mar  3 14:58:53 2020	(r358578)
+++ stable/12/sys/x86/acpica/acpi_wakeup.c	Tue Mar  3 15:02:07 2020	(r358579)
@@ -245,7 +245,7 @@ acpi_sleep_machdep(struct acpi_softc *sc, int state)
 		}
 #endif
 #ifdef __amd64__
-		hw_ibrs_active = 0;
+		hw_ibrs_ibpb_active = 0;
 		hw_ssb_active = 0;
 		cpu_stdext_feature3 = 0;
 		CPU_FOREACH(i) {

Modified: stable/12/sys/x86/include/x86_var.h
==============================================================================
--- stable/12/sys/x86/include/x86_var.h	Tue Mar  3 14:58:53 2020	(r358578)
+++ stable/12/sys/x86/include/x86_var.h	Tue Mar  3 15:02:07 2020	(r358579)
@@ -84,7 +84,7 @@ extern	int	use_xsave;
 extern	uint64_t xsave_mask;
 extern	u_int	max_apic_id;
 extern	int	pti;
-extern	int	hw_ibrs_active;
+extern	int	hw_ibrs_ibpb_active;
 extern	int	hw_mds_disable;
 extern	int	hw_ssb_active;
 extern	int	x86_taa_enable;
@@ -141,7 +141,7 @@ int	is_physical_memory(vm_paddr_t addr);
 int	isa_nmi(int cd);
 void	handle_ibrs_entry(void);
 void	handle_ibrs_exit(void);
-void	hw_ibrs_recalculate(void);
+void	hw_ibrs_recalculate(bool all_cpus);
 void	hw_mds_recalculate(void);
 void	hw_ssb_recalculate(bool all_cpus);
 void	x86_taa_recalculate(void);

Modified: stable/12/sys/x86/x86/cpu_machdep.c
==============================================================================
--- stable/12/sys/x86/x86/cpu_machdep.c	Tue Mar  3 14:58:53 2020	(r358578)
+++ stable/12/sys/x86/x86/cpu_machdep.c	Tue Mar  3 15:02:07 2020	(r358579)
@@ -855,23 +855,27 @@ nmi_handle_intr(u_int type, struct trapframe *frame)
 	nmi_call_kdb(PCPU_GET(cpuid), type, frame);
 }
 
-int hw_ibrs_active;
+static int hw_ibrs_active;
+int hw_ibrs_ibpb_active;
 int hw_ibrs_disable = 1;
 
 SYSCTL_INT(_hw, OID_AUTO, ibrs_active, CTLFLAG_RD, &hw_ibrs_active, 0,
     "Indirect Branch Restricted Speculation active");
 
 void
-hw_ibrs_recalculate(void)
+hw_ibrs_recalculate(bool for_all_cpus)
 {
 	if ((cpu_ia32_arch_caps & IA32_ARCH_CAP_IBRS_ALL) != 0) {
-		x86_msr_op(MSR_IA32_SPEC_CTRL, MSR_OP_LOCAL |
-		    (hw_ibrs_disable ? MSR_OP_ANDNOT : MSR_OP_OR),
+		x86_msr_op(MSR_IA32_SPEC_CTRL, (for_all_cpus ?
+		    MSR_OP_RENDEZVOUS : MSR_OP_LOCAL) |
+		    (hw_ibrs_disable != 0 ? MSR_OP_ANDNOT : MSR_OP_OR),
 		    IA32_SPEC_CTRL_IBRS);
-		return;
+		hw_ibrs_active = hw_ibrs_disable == 0;
+		hw_ibrs_ibpb_active = 0;
+	} else {
+		hw_ibrs_active = hw_ibrs_ibpb_active = (cpu_stdext_feature3 &
+		    CPUID_STDEXT3_IBPB) != 0 && !hw_ibrs_disable;
 	}
-	hw_ibrs_active = (cpu_stdext_feature3 & CPUID_STDEXT3_IBPB) != 0 &&
-	    !hw_ibrs_disable;
 }
 
 static int
@@ -884,7 +888,7 @@ hw_ibrs_disable_handler(SYSCTL_HANDLER_ARGS)
 	if (error != 0 || req->newptr == NULL)
 		return (error);
 	hw_ibrs_disable = val != 0;
-	hw_ibrs_recalculate();
+	hw_ibrs_recalculate(true);
 	return (0);
 }
 SYSCTL_PROC(_hw, OID_AUTO, ibrs_disable, CTLTYPE_INT | CTLFLAG_RWTUN |