svn commit: r357914 - head/sys/amd64/amd64

[Mateusz Guzik]

Author: mjg
Date: Fri Feb 14 13:14:19 2020
New Revision: 357914
URL: https://svnweb.freebsd.org/changeset/base/357914

Log:
  amd64: only check for error != 0 in the inlined part of l1d flush check
  
  this replaces the following near the syscall exit:
  cmp    $0x39,%rax
  ja     0xffffffff8108f82c
  movabs $0x200001800060005,%rcx
  bt     %rax,%rcx
  jae    0xffffffff8108f82c
  
  with:
  test   %edi,%edi
  jne    0xffffffff8091a49c

Modified:
  head/sys/amd64/amd64/trap.c

Modified: head/sys/amd64/amd64/trap.c
==============================================================================
--- head/sys/amd64/amd64/trap.c	Fri Feb 14 13:10:52 2020	(r357913)
+++ head/sys/amd64/amd64/trap.c	Fri Feb 14 13:14:19 2020	(r357914)
@@ -1068,25 +1068,32 @@ flush_l1d_hw(void)
 	wrmsr(MSR_IA32_FLUSH_CMD, IA32_FLUSH_CMD_L1D);
 }
 
-static void __inline
-amd64_syscall_ret_flush_l1d_inline(int error)
+static void __noinline
+amd64_syscall_ret_flush_l1d_check(int error)
 {
 	void (*p)(void);
 
-	if (error != 0 && error != EEXIST && error != EAGAIN &&
-	    error != EXDEV && error != ENOENT && error != ENOTCONN &&
-	    error != EINPROGRESS) {
-		p = syscall_ret_l1d_flush;
+	if (error != EEXIST && error != EAGAIN && error != EXDEV &&
+	    error != ENOENT && error != ENOTCONN && error != EINPROGRESS) {
+		p = (void *)atomic_load_ptr(&syscall_ret_l1d_flush);
 		if (p != NULL)
 			p();
 	}
 }
 
+static void __inline
+amd64_syscall_ret_flush_l1d_check_inline(int error)
+{
+
+	if (__predict_false(error != 0))
+		amd64_syscall_ret_flush_l1d_check(error);
+}
+
 void
 amd64_syscall_ret_flush_l1d(int error)
 {
 
-	amd64_syscall_ret_flush_l1d_inline(error);
+	amd64_syscall_ret_flush_l1d_check_inline(error);
 }
 
 void
@@ -1190,5 +1197,5 @@ amd64_syscall(struct thread *td, int traced)
 	if (__predict_false(td->td_frame->tf_rip >= VM_MAXUSER_ADDRESS))
 		set_pcb_flags(td->td_pcb, PCB_FULL_IRET);
 
-	amd64_syscall_ret_flush_l1d_inline(td->td_errno);
+	amd64_syscall_ret_flush_l1d_check_inline(td->td_errno);
 }