svn commit: r366614 - in stable/12/sys/amd64: amd64 include
Konstantin Belousov
kib at FreeBSD.org
Sat Oct 10 13:46:49 UTC 2020
Author: kib
Date: Sat Oct 10 13:46:48 2020
New Revision: 366614
URL: https://svnweb.freebsd.org/changeset/base/366614
Log:
MFC r366417:
amd64: Store full 64bit of FIP/FDP for 64bit processes when using XSAVE.
PR: 250043
Modified:
stable/12/sys/amd64/amd64/cpu_switch.S
stable/12/sys/amd64/amd64/fpu.c
stable/12/sys/amd64/include/md_var.h
Directory Properties:
stable/12/ (props changed)
Modified: stable/12/sys/amd64/amd64/cpu_switch.S
==============================================================================
--- stable/12/sys/amd64/amd64/cpu_switch.S Sat Oct 10 13:39:13 2020 (r366613)
+++ stable/12/sys/amd64/amd64/cpu_switch.S Sat Oct 10 13:46:48 2020 (r366614)
@@ -128,22 +128,25 @@ done_store_dr:
/* have we used fp, and need a save? */
cmpq %rdi,PCPU(FPCURTHREAD)
- jne 2f
- movq PCB_SAVEFPU(%r8),%r8
+ jne ctx_switch_fpusave_done
+ movq PCB_SAVEFPU(%r8),%r9
clts
cmpl $0,use_xsave(%rip)
jne 1f
- fxsave (%r8)
- jmp 2f
+ fxsave (%r9)
+ jmp ctx_switch_fpusave_done
1: movq %rdx,%rcx
movl xsave_mask,%eax
movl xsave_mask+4,%edx
+ testl $PCB_32BIT,PCB_FLAGS(%r8)
+ jne ctx_switch_xsave32
.globl ctx_switch_xsave
ctx_switch_xsave:
/* This is patched to xsaveopt if supported, see fpuinit_bsp1() */
- xsave (%r8)
+ xsave64 (%r9)
+ctx_switch_xsave_done:
movq %rcx,%rdx
-2:
+ctx_switch_fpusave_done:
/* Save is done. Now fire up new thread. Leave old vmspace. */
movq %rsi,%r12
movq %rdi,%r13
@@ -306,6 +309,11 @@ do_ldt: movq PCPU(LDT),%rax
movq %rdx,8(%rax)
movl $LDTSEL,%eax
jmp ld_ldt
+
+ .globl ctx_switch_xsave32
+ctx_switch_xsave32:
+ xsave (%r9)
+ jmp ctx_switch_xsave_done
END(cpu_switch)
/*
Modified: stable/12/sys/amd64/amd64/fpu.c
==============================================================================
--- stable/12/sys/amd64/amd64/fpu.c Sat Oct 10 13:39:13 2020 (r366613)
+++ stable/12/sys/amd64/amd64/fpu.c Sat Oct 10 13:46:48 2020 (r366614)
@@ -46,6 +46,7 @@ __FBSDID("$FreeBSD$");
#include <sys/mutex.h>
#include <sys/proc.h>
#include <sys/sysctl.h>
+#include <sys/sysent.h>
#include <machine/bus.h>
#include <sys/rman.h>
#include <sys/signalvar.h>
@@ -80,7 +81,7 @@ __FBSDID("$FreeBSD$");
#define stmxcsr(addr) __asm __volatile("stmxcsr %0" : : "m" (*(addr)))
static __inline void
-xrstor(char *addr, uint64_t mask)
+xrstor32(char *addr, uint64_t mask)
{
uint32_t low, hi;
@@ -90,27 +91,59 @@ xrstor(char *addr, uint64_t mask)
}
static __inline void
-xsave(char *addr, uint64_t mask)
+xrstor64(char *addr, uint64_t mask)
{
uint32_t low, hi;
low = mask;
hi = mask >> 32;
+ __asm __volatile("xrstor64 %0" : : "m" (*addr), "a" (low), "d" (hi));
+}
+
+static __inline void
+xsave32(char *addr, uint64_t mask)
+{
+ uint32_t low, hi;
+
+ low = mask;
+ hi = mask >> 32;
__asm __volatile("xsave %0" : "=m" (*addr) : "a" (low), "d" (hi) :
"memory");
}
static __inline void
-xsaveopt(char *addr, uint64_t mask)
+xsave64(char *addr, uint64_t mask)
{
uint32_t low, hi;
low = mask;
hi = mask >> 32;
+ __asm __volatile("xsave64 %0" : "=m" (*addr) : "a" (low), "d" (hi) :
+ "memory");
+}
+
+static __inline void
+xsaveopt32(char *addr, uint64_t mask)
+{
+ uint32_t low, hi;
+
+ low = mask;
+ hi = mask >> 32;
__asm __volatile("xsaveopt %0" : "=m" (*addr) : "a" (low), "d" (hi) :
"memory");
}
+static __inline void
+xsaveopt64(char *addr, uint64_t mask)
+{
+ uint32_t low, hi;
+
+ low = mask;
+ hi = mask >> 32;
+ __asm __volatile("xsaveopt64 %0" : "=m" (*addr) : "a" (low), "d" (hi) :
+ "memory");
+}
+
#else /* !(__GNUCLIKE_ASM && !lint) */
void fldcw(u_short cw);
@@ -122,9 +155,12 @@ void fxsave(caddr_t addr);
void fxrstor(caddr_t addr);
void ldmxcsr(u_int csr);
void stmxcsr(u_int *csr);
-void xrstor(char *addr, uint64_t mask);
-void xsave(char *addr, uint64_t mask);
-void xsaveopt(char *addr, uint64_t mask);
+void xrstor32(char *addr, uint64_t mask);
+void xrstor64(char *addr, uint64_t mask);
+void xsave32(char *addr, uint64_t mask);
+void xsave64(char *addr, uint64_t mask);
+void xsaveopt32(char *addr, uint64_t mask);
+void xsaveopt64(char *addr, uint64_t mask);
#endif /* __GNUCLIKE_ASM && !lint */
@@ -170,24 +206,48 @@ struct xsave_area_elm_descr {
} *xsave_area_desc;
static void
-fpusave_xsaveopt(void *addr)
+fpusave_xsaveopt64(void *addr)
{
+ xsaveopt64((char *)addr, xsave_mask);
+}
- xsaveopt((char *)addr, xsave_mask);
+static void
+fpusave_xsaveopt3264(void *addr)
+{
+ if (SV_CURPROC_FLAG(SV_ILP32))
+ xsaveopt32((char *)addr, xsave_mask);
+ else
+ xsaveopt64((char *)addr, xsave_mask);
}
static void
-fpusave_xsave(void *addr)
+fpusave_xsave64(void *addr)
{
+ xsave64((char *)addr, xsave_mask);
+}
- xsave((char *)addr, xsave_mask);
+static void
+fpusave_xsave3264(void *addr)
+{
+ if (SV_CURPROC_FLAG(SV_ILP32))
+ xsave32((char *)addr, xsave_mask);
+ else
+ xsave64((char *)addr, xsave_mask);
}
static void
-fpurestore_xrstor(void *addr)
+fpurestore_xrstor64(void *addr)
{
+ xrstor64((char *)addr, xsave_mask);
+}
- xrstor((char *)addr, xsave_mask);
+static void
+fpurestore_xrstor3264(void *addr)
+{
+ if (SV_CURPROC_FLAG(SV_ILP32))
+ xrstor32((char *)addr, xsave_mask);
+ else
+ xrstor64((char *)addr, xsave_mask);
}
static void
@@ -220,17 +280,24 @@ DEFINE_IFUNC(, void, fpusave, (void *), static)
{
init_xsave();
- if (use_xsave)
- return ((cpu_stdext_feature & CPUID_EXTSTATE_XSAVEOPT) != 0 ?
- fpusave_xsaveopt : fpusave_xsave);
- return (fpusave_fxsave);
+ if (!use_xsave)
+ return (fpusave_fxsave);
+ if ((cpu_stdext_feature & CPUID_EXTSTATE_XSAVEOPT) != 0) {
+ return ((cpu_stdext_feature & CPUID_STDEXT_NFPUSG) != 0 ?
+ fpusave_xsaveopt64 : fpusave_xsaveopt3264);
+ }
+ return ((cpu_stdext_feature & CPUID_STDEXT_NFPUSG) != 0 ?
+ fpusave_xsave64 : fpusave_xsave3264);
}
DEFINE_IFUNC(, void, fpurestore, (void *), static)
{
init_xsave();
- return (use_xsave ? fpurestore_xrstor : fpurestore_fxrstor);
+ if (!use_xsave)
+ return (fpurestore_fxrstor);
+ return ((cpu_stdext_feature & CPUID_STDEXT_NFPUSG) != 0 ?
+ fpurestore_xrstor64 : fpurestore_xrstor3264);
}
void
@@ -297,6 +364,7 @@ fpuinit_bsp1(void)
* read-only before cpu_startup().
*/
old_wp = disable_wp();
+ ctx_switch_xsave32[3] |= 0x10;
ctx_switch_xsave[3] |= 0x10;
restore_wp(old_wp);
}
Modified: stable/12/sys/amd64/include/md_var.h
==============================================================================
--- stable/12/sys/amd64/include/md_var.h Sat Oct 10 13:39:13 2020 (r366613)
+++ stable/12/sys/amd64/include/md_var.h Sat Oct 10 13:46:48 2020 (r366614)
@@ -38,6 +38,7 @@
extern uint64_t *vm_page_dump;
extern char ctx_switch_xsave[];
+extern char ctx_switch_xsave32[];
extern int hw_lower_amd64_sharedpage;
extern int hw_ibrs_disable;
extern int hw_ssb_disable;
More information about the svn-src-stable
mailing list