svn commit: r366417 - in head/sys/amd64: amd64 include
Konstantin Belousov
kib at FreeBSD.org
Sat Oct 3 23:17:30 UTC 2020
Author: kib
Date: Sat Oct 3 23:17:29 2020
New Revision: 366417
URL: https://svnweb.freebsd.org/changeset/base/366417
Log:
amd64: Store full 64bit of FIP/FDP for 64bit processes when using XSAVE.
If current process is 64bit, use rex-prefixed version of XSAVE
(XSAVE64). If current process is 32bit and CPU supports saving
segment registers cs/ds in the FPU save area, use non-prefixed variant
of XSAVE.
Reported and tested by: Michał Górny <mgorny at mgorny@moritz.systems>
PR: 250043
Reviewed by: emaste, markj
Sponsored by: The FreeBSD Foundation
MFC after: 1 week
Differential revision: https://reviews.freebsd.org/D26643
Modified:
head/sys/amd64/amd64/cpu_switch.S
head/sys/amd64/amd64/fpu.c
head/sys/amd64/include/md_var.h
Modified: head/sys/amd64/amd64/cpu_switch.S
==============================================================================
--- head/sys/amd64/amd64/cpu_switch.S Sat Oct 3 23:11:20 2020 (r366416)
+++ head/sys/amd64/amd64/cpu_switch.S Sat Oct 3 23:17:29 2020 (r366417)
@@ -116,22 +116,25 @@ done_store_dr:
/* have we used fp, and need a save? */
cmpq %rdi,PCPU(FPCURTHREAD)
- jne 2f
- movq PCB_SAVEFPU(%r8),%r8
+ jne ctx_switch_fpusave_done
+ movq PCB_SAVEFPU(%r8),%r9
clts
cmpl $0,use_xsave(%rip)
jne 1f
- fxsave (%r8)
- jmp 2f
+ fxsave (%r9)
+ jmp ctx_switch_fpusave_done
1: movq %rdx,%rcx
movl xsave_mask,%eax
movl xsave_mask+4,%edx
+ testl $PCB_32BIT,PCB_FLAGS(%r8)
+ jne ctx_switch_xsave32
.globl ctx_switch_xsave
ctx_switch_xsave:
/* This is patched to xsaveopt if supported, see fpuinit_bsp1() */
- xsave (%r8)
+ xsave64 (%r9)
+ctx_switch_xsave_done:
movq %rcx,%rdx
-2:
+ctx_switch_fpusave_done:
/* Save is done. Now fire up new thread. Leave old vmspace. */
movq %rsi,%r12
movq %rdi,%r13
@@ -294,6 +297,11 @@ do_ldt: movq PCPU(LDT),%rax
movq %rdx,8(%rax)
movl $LDTSEL,%eax
jmp ld_ldt
+
+ .globl ctx_switch_xsave32
+ctx_switch_xsave32:
+ xsave (%r9)
+ jmp ctx_switch_xsave_done
END(cpu_switch)
/*
Modified: head/sys/amd64/amd64/fpu.c
==============================================================================
--- head/sys/amd64/amd64/fpu.c Sat Oct 3 23:11:20 2020 (r366416)
+++ head/sys/amd64/amd64/fpu.c Sat Oct 3 23:17:29 2020 (r366417)
@@ -47,6 +47,7 @@ __FBSDID("$FreeBSD$");
#include <sys/mutex.h>
#include <sys/proc.h>
#include <sys/sysctl.h>
+#include <sys/sysent.h>
#include <machine/bus.h>
#include <sys/rman.h>
#include <sys/signalvar.h>
@@ -81,7 +82,7 @@ __FBSDID("$FreeBSD$");
#define stmxcsr(addr) __asm __volatile("stmxcsr %0" : : "m" (*(addr)))
static __inline void
-xrstor(char *addr, uint64_t mask)
+xrstor32(char *addr, uint64_t mask)
{
uint32_t low, hi;
@@ -91,27 +92,59 @@ xrstor(char *addr, uint64_t mask)
}
static __inline void
-xsave(char *addr, uint64_t mask)
+xrstor64(char *addr, uint64_t mask)
{
uint32_t low, hi;
low = mask;
hi = mask >> 32;
+ __asm __volatile("xrstor64 %0" : : "m" (*addr), "a" (low), "d" (hi));
+}
+
+static __inline void
+xsave32(char *addr, uint64_t mask)
+{
+ uint32_t low, hi;
+
+ low = mask;
+ hi = mask >> 32;
__asm __volatile("xsave %0" : "=m" (*addr) : "a" (low), "d" (hi) :
"memory");
}
static __inline void
-xsaveopt(char *addr, uint64_t mask)
+xsave64(char *addr, uint64_t mask)
{
uint32_t low, hi;
low = mask;
hi = mask >> 32;
+ __asm __volatile("xsave64 %0" : "=m" (*addr) : "a" (low), "d" (hi) :
+ "memory");
+}
+
+static __inline void
+xsaveopt32(char *addr, uint64_t mask)
+{
+ uint32_t low, hi;
+
+ low = mask;
+ hi = mask >> 32;
__asm __volatile("xsaveopt %0" : "=m" (*addr) : "a" (low), "d" (hi) :
"memory");
}
+static __inline void
+xsaveopt64(char *addr, uint64_t mask)
+{
+ uint32_t low, hi;
+
+ low = mask;
+ hi = mask >> 32;
+ __asm __volatile("xsaveopt64 %0" : "=m" (*addr) : "a" (low), "d" (hi) :
+ "memory");
+}
+
#else /* !(__GNUCLIKE_ASM && !lint) */
void fldcw(u_short cw);
@@ -123,9 +156,12 @@ void fxsave(caddr_t addr);
void fxrstor(caddr_t addr);
void ldmxcsr(u_int csr);
void stmxcsr(u_int *csr);
-void xrstor(char *addr, uint64_t mask);
-void xsave(char *addr, uint64_t mask);
-void xsaveopt(char *addr, uint64_t mask);
+void xrstor32(char *addr, uint64_t mask);
+void xrstor64(char *addr, uint64_t mask);
+void xsave32(char *addr, uint64_t mask);
+void xsave64(char *addr, uint64_t mask);
+void xsaveopt32(char *addr, uint64_t mask);
+void xsaveopt64(char *addr, uint64_t mask);
#endif /* __GNUCLIKE_ASM && !lint */
@@ -166,24 +202,48 @@ static struct xsave_area_elm_descr {
} *xsave_area_desc;
static void
-fpusave_xsaveopt(void *addr)
+fpusave_xsaveopt64(void *addr)
{
+ xsaveopt64((char *)addr, xsave_mask);
+}
- xsaveopt((char *)addr, xsave_mask);
+static void
+fpusave_xsaveopt3264(void *addr)
+{
+ if (SV_CURPROC_FLAG(SV_ILP32))
+ xsaveopt32((char *)addr, xsave_mask);
+ else
+ xsaveopt64((char *)addr, xsave_mask);
}
static void
-fpusave_xsave(void *addr)
+fpusave_xsave64(void *addr)
{
+ xsave64((char *)addr, xsave_mask);
+}
- xsave((char *)addr, xsave_mask);
+static void
+fpusave_xsave3264(void *addr)
+{
+ if (SV_CURPROC_FLAG(SV_ILP32))
+ xsave32((char *)addr, xsave_mask);
+ else
+ xsave64((char *)addr, xsave_mask);
}
static void
-fpurestore_xrstor(void *addr)
+fpurestore_xrstor64(void *addr)
{
+ xrstor64((char *)addr, xsave_mask);
+}
- xrstor((char *)addr, xsave_mask);
+static void
+fpurestore_xrstor3264(void *addr)
+{
+ if (SV_CURPROC_FLAG(SV_ILP32))
+ xrstor32((char *)addr, xsave_mask);
+ else
+ xrstor64((char *)addr, xsave_mask);
}
static void
@@ -216,17 +276,24 @@ DEFINE_IFUNC(, void, fpusave, (void *))
{
init_xsave();
- if (use_xsave)
- return ((cpu_stdext_feature & CPUID_EXTSTATE_XSAVEOPT) != 0 ?
- fpusave_xsaveopt : fpusave_xsave);
- return (fpusave_fxsave);
+ if (!use_xsave)
+ return (fpusave_fxsave);
+ if ((cpu_stdext_feature & CPUID_EXTSTATE_XSAVEOPT) != 0) {
+ return ((cpu_stdext_feature & CPUID_STDEXT_NFPUSG) != 0 ?
+ fpusave_xsaveopt64 : fpusave_xsaveopt3264);
+ }
+ return ((cpu_stdext_feature & CPUID_STDEXT_NFPUSG) != 0 ?
+ fpusave_xsave64 : fpusave_xsave3264);
}
DEFINE_IFUNC(, void, fpurestore, (void *))
{
init_xsave();
- return (use_xsave ? fpurestore_xrstor : fpurestore_fxrstor);
+ if (!use_xsave)
+ return (fpurestore_fxrstor);
+ return ((cpu_stdext_feature & CPUID_STDEXT_NFPUSG) != 0 ?
+ fpurestore_xrstor64 : fpurestore_xrstor3264);
}
void
@@ -293,6 +360,7 @@ fpuinit_bsp1(void)
* read-only before cpu_startup().
*/
old_wp = disable_wp();
+ ctx_switch_xsave32[3] |= 0x10;
ctx_switch_xsave[3] |= 0x10;
restore_wp(old_wp);
}
Modified: head/sys/amd64/include/md_var.h
==============================================================================
--- head/sys/amd64/include/md_var.h Sat Oct 3 23:11:20 2020 (r366416)
+++ head/sys/amd64/include/md_var.h Sat Oct 3 23:17:29 2020 (r366417)
@@ -37,6 +37,7 @@
#include <x86/x86_var.h>
extern char ctx_switch_xsave[];
+extern char ctx_switch_xsave32[];
extern int hw_lower_amd64_sharedpage;
extern int hw_ibrs_disable;
extern int hw_ssb_disable;
More information about the svn-src-all
mailing list