svn commit: r337431 - in head/sys/amd64: linux linux32
Konstantin Belousov
kib at FreeBSD.org
Tue Aug 7 18:29:11 UTC 2018
Author: kib
Date: Tue Aug 7 18:29:10 2018
New Revision: 337431
URL: https://svnweb.freebsd.org/changeset/base/337431
Log:
Futex support functions in linux.ko and linux32.ko on amd64 should be
aware of SMAP.
Reported and tested by: Johannes Lundberg <johalun0 at gmail.com>, wulf
Sponsored by: The FreeBSD Foundation
Modified:
head/sys/amd64/linux/linux_machdep.c
head/sys/amd64/linux/linux_support.s
head/sys/amd64/linux32/linux32_machdep.c
head/sys/amd64/linux32/linux32_support.s
Modified: head/sys/amd64/linux/linux_machdep.c
==============================================================================
--- head/sys/amd64/linux/linux_machdep.c Tue Aug 7 18:26:46 2018 (r337430)
+++ head/sys/amd64/linux/linux_machdep.c Tue Aug 7 18:29:10 2018 (r337431)
@@ -78,6 +78,9 @@ __FBSDID("$FreeBSD$");
#include <vm/vm_kern.h>
#include <vm/vm_map.h>
+#include <x86/ifunc.h>
+#include <x86/sysarch.h>
+
#include <amd64/linux/linux.h>
#include <amd64/linux/linux_proto.h>
#include <compat/linux/linux_emul.h>
@@ -88,8 +91,6 @@ __FBSDID("$FreeBSD$");
#include <compat/linux/linux_signal.h>
#include <compat/linux/linux_util.h>
-#include <x86/include/sysarch.h>
-
int
linux_execve(struct thread *td, struct linux_execve_args *args)
{
@@ -275,4 +276,49 @@ linux_set_cloned_tls(struct thread *td, void *desc)
td->td_frame->tf_fs = _ufssel;
return (0);
+}
+
+int futex_xchgl_nosmap(int oparg, uint32_t *uaddr, int *oldval);
+int futex_xchgl_smap(int oparg, uint32_t *uaddr, int *oldval);
+DEFINE_IFUNC(, int, futex_xchgl, (int, uint32_t *, int *), static)
+{
+
+ return ((cpu_stdext_feature & CPUID_STDEXT_SMAP) != 0 ?
+ futex_xchgl_smap : futex_xchgl_nosmap);
+}
+
+int futex_addl_nosmap(int oparg, uint32_t *uaddr, int *oldval);
+int futex_addl_smap(int oparg, uint32_t *uaddr, int *oldval);
+DEFINE_IFUNC(, int, futex_addl, (int, uint32_t *, int *), static)
+{
+
+ return ((cpu_stdext_feature & CPUID_STDEXT_SMAP) != 0 ?
+ futex_addl_smap : futex_addl_nosmap);
+}
+
+int futex_orl_nosmap(int oparg, uint32_t *uaddr, int *oldval);
+int futex_orl_smap(int oparg, uint32_t *uaddr, int *oldval);
+DEFINE_IFUNC(, int, futex_orl, (int, uint32_t *, int *), static)
+{
+
+ return ((cpu_stdext_feature & CPUID_STDEXT_SMAP) != 0 ?
+ futex_orl_smap : futex_orl_nosmap);
+}
+
+int futex_andl_nosmap(int oparg, uint32_t *uaddr, int *oldval);
+int futex_andl_smap(int oparg, uint32_t *uaddr, int *oldval);
+DEFINE_IFUNC(, int, futex_andl, (int, uint32_t *, int *), static)
+{
+
+ return ((cpu_stdext_feature & CPUID_STDEXT_SMAP) != 0 ?
+ futex_andl_smap : futex_andl_nosmap);
+}
+
+int futex_xorl_nosmap(int oparg, uint32_t *uaddr, int *oldval);
+int futex_xorl_smap(int oparg, uint32_t *uaddr, int *oldval);
+DEFINE_IFUNC(, int, futex_xorl, (int, uint32_t *, int *), static)
+{
+
+ return ((cpu_stdext_feature & CPUID_STDEXT_SMAP) != 0 ?
+ futex_xorl_smap : futex_xorl_nosmap);
}
Modified: head/sys/amd64/linux/linux_support.s
==============================================================================
--- head/sys/amd64/linux/linux_support.s Tue Aug 7 18:26:46 2018 (r337430)
+++ head/sys/amd64/linux/linux_support.s Tue Aug 7 18:29:10 2018 (r337431)
@@ -38,7 +38,7 @@ futex_fault:
movl $-EFAULT,%eax
ret
-ENTRY(futex_xchgl)
+ENTRY(futex_xchgl_nosmap)
movq PCPU(CURPCB),%r8
movq $futex_fault,PCB_ONFAULT(%r8)
movq $VM_MAXUSER_ADDRESS-4,%rax
@@ -49,14 +49,29 @@ ENTRY(futex_xchgl)
xorl %eax,%eax
movq %rax,PCB_ONFAULT(%r8)
ret
-END(futex_xchgl)
+END(futex_xchgl_nosmap)
-ENTRY(futex_addl)
+ENTRY(futex_xchgl_smap)
movq PCPU(CURPCB),%r8
movq $futex_fault,PCB_ONFAULT(%r8)
movq $VM_MAXUSER_ADDRESS-4,%rax
cmpq %rax,%rsi
ja futex_fault
+ stac
+ xchgl %edi,(%rsi)
+ clac
+ movl %edi,(%rdx)
+ xorl %eax,%eax
+ movq %rax,PCB_ONFAULT(%r8)
+ ret
+END(futex_xchgl_smap)
+
+ENTRY(futex_addl_nosmap)
+ movq PCPU(CURPCB),%r8
+ movq $futex_fault,PCB_ONFAULT(%r8)
+ movq $VM_MAXUSER_ADDRESS-4,%rax
+ cmpq %rax,%rsi
+ ja futex_fault
#ifdef SMP
lock
#endif
@@ -65,14 +80,32 @@ ENTRY(futex_addl)
xorl %eax,%eax
movq %rax,PCB_ONFAULT(%r8)
ret
-END(futex_addl)
+END(futex_addl_nosmap)
-ENTRY(futex_orl)
+ENTRY(futex_addl_smap)
movq PCPU(CURPCB),%r8
movq $futex_fault,PCB_ONFAULT(%r8)
movq $VM_MAXUSER_ADDRESS-4,%rax
cmpq %rax,%rsi
ja futex_fault
+ stac
+#ifdef SMP
+ lock
+#endif
+ xaddl %edi,(%rsi)
+ clac
+ movl %edi,(%rdx)
+ xorl %eax,%eax
+ movq %rax,PCB_ONFAULT(%r8)
+ ret
+END(futex_addl_smap)
+
+ENTRY(futex_orl_nosmap)
+ movq PCPU(CURPCB),%r8
+ movq $futex_fault,PCB_ONFAULT(%r8)
+ movq $VM_MAXUSER_ADDRESS-4,%rax
+ cmpq %rax,%rsi
+ ja futex_fault
movl (%rsi),%eax
1: movl %eax,%ecx
orl %edi,%ecx
@@ -85,9 +118,9 @@ ENTRY(futex_orl)
xorl %eax,%eax
movq %rax,PCB_ONFAULT(%r8)
ret
-END(futex_orl)
+END(futex_orl_nosmap)
-ENTRY(futex_andl)
+ENTRY(futex_orl_smap)
movq PCPU(CURPCB),%r8
movq $futex_fault,PCB_ONFAULT(%r8)
movq $VM_MAXUSER_ADDRESS-4,%rax
@@ -95,6 +128,28 @@ ENTRY(futex_andl)
ja futex_fault
movl (%rsi),%eax
1: movl %eax,%ecx
+ orl %edi,%ecx
+ stac
+#ifdef SMP
+ lock
+#endif
+ cmpxchgl %ecx,(%rsi)
+ clac
+ jnz 1b
+ movl %eax,(%rdx)
+ xorl %eax,%eax
+ movq %rax,PCB_ONFAULT(%r8)
+ ret
+END(futex_orl_smap)
+
+ENTRY(futex_andl_nosmap)
+ movq PCPU(CURPCB),%r8
+ movq $futex_fault,PCB_ONFAULT(%r8)
+ movq $VM_MAXUSER_ADDRESS-4,%rax
+ cmpq %rax,%rsi
+ ja futex_fault
+ movl (%rsi),%eax
+1: movl %eax,%ecx
andl %edi,%ecx
#ifdef SMP
lock
@@ -105,9 +160,9 @@ ENTRY(futex_andl)
xorl %eax,%eax
movq %rax,PCB_ONFAULT(%r8)
ret
-END(futex_andl)
+END(futex_andl_nosmap)
-ENTRY(futex_xorl)
+ENTRY(futex_andl_smap)
movq PCPU(CURPCB),%r8
movq $futex_fault,PCB_ONFAULT(%r8)
movq $VM_MAXUSER_ADDRESS-4,%rax
@@ -115,6 +170,28 @@ ENTRY(futex_xorl)
ja futex_fault
movl (%rsi),%eax
1: movl %eax,%ecx
+ andl %edi,%ecx
+ stac
+#ifdef SMP
+ lock
+#endif
+ cmpxchgl %ecx,(%rsi)
+ clac
+ jnz 1b
+ movl %eax,(%rdx)
+ xorl %eax,%eax
+ movq %rax,PCB_ONFAULT(%r8)
+ ret
+END(futex_andl_smap)
+
+ENTRY(futex_xorl_nosmap)
+ movq PCPU(CURPCB),%r8
+ movq $futex_fault,PCB_ONFAULT(%r8)
+ movq $VM_MAXUSER_ADDRESS-4,%rax
+ cmpq %rax,%rsi
+ ja futex_fault
+ movl (%rsi),%eax
+1: movl %eax,%ecx
xorl %edi,%ecx
#ifdef SMP
lock
@@ -125,4 +202,26 @@ ENTRY(futex_xorl)
xorl %eax,%eax
movq %rax,PCB_ONFAULT(%r8)
ret
-END(futex_xorl)
+END(futex_xorl_nosmap)
+
+ENTRY(futex_xorl_smap)
+ movq PCPU(CURPCB),%r8
+ movq $futex_fault,PCB_ONFAULT(%r8)
+ movq $VM_MAXUSER_ADDRESS-4,%rax
+ cmpq %rax,%rsi
+ ja futex_fault
+ movl (%rsi),%eax
+1: movl %eax,%ecx
+ xorl %edi,%ecx
+ stac
+#ifdef SMP
+ lock
+#endif
+ cmpxchgl %ecx,(%rsi)
+ clac
+ jnz 1b
+ movl %eax,(%rdx)
+ xorl %eax,%eax
+ movq %rax,PCB_ONFAULT(%r8)
+ ret
+END(futex_xorl_smap)
Modified: head/sys/amd64/linux32/linux32_machdep.c
==============================================================================
--- head/sys/amd64/linux32/linux32_machdep.c Tue Aug 7 18:26:46 2018 (r337430)
+++ head/sys/amd64/linux32/linux32_machdep.c Tue Aug 7 18:29:10 2018 (r337431)
@@ -58,10 +58,12 @@ __FBSDID("$FreeBSD$");
#include <sys/wait.h>
#include <machine/frame.h>
+#include <machine/md_var.h>
#include <machine/pcb.h>
#include <machine/psl.h>
#include <machine/segments.h>
#include <machine/specialreg.h>
+#include <x86/ifunc.h>
#include <vm/pmap.h>
#include <vm/vm.h>
@@ -821,4 +823,49 @@ linux_set_thread_area(struct thread *td,
update_gdt_gsbase(td, info.base_addr);
return (0);
+}
+
+int futex_xchgl_nosmap(int oparg, uint32_t *uaddr, int *oldval);
+int futex_xchgl_smap(int oparg, uint32_t *uaddr, int *oldval);
+DEFINE_IFUNC(, int, futex_xchgl, (int, uint32_t *, int *), static)
+{
+
+ return ((cpu_stdext_feature & CPUID_STDEXT_SMAP) != 0 ?
+ futex_xchgl_smap : futex_xchgl_nosmap);
+}
+
+int futex_addl_nosmap(int oparg, uint32_t *uaddr, int *oldval);
+int futex_addl_smap(int oparg, uint32_t *uaddr, int *oldval);
+DEFINE_IFUNC(, int, futex_addl, (int, uint32_t *, int *), static)
+{
+
+ return ((cpu_stdext_feature & CPUID_STDEXT_SMAP) != 0 ?
+ futex_addl_smap : futex_addl_nosmap);
+}
+
+int futex_orl_nosmap(int oparg, uint32_t *uaddr, int *oldval);
+int futex_orl_smap(int oparg, uint32_t *uaddr, int *oldval);
+DEFINE_IFUNC(, int, futex_orl, (int, uint32_t *, int *), static)
+{
+
+ return ((cpu_stdext_feature & CPUID_STDEXT_SMAP) != 0 ?
+ futex_orl_smap : futex_orl_nosmap);
+}
+
+int futex_andl_nosmap(int oparg, uint32_t *uaddr, int *oldval);
+int futex_andl_smap(int oparg, uint32_t *uaddr, int *oldval);
+DEFINE_IFUNC(, int, futex_andl, (int, uint32_t *, int *), static)
+{
+
+ return ((cpu_stdext_feature & CPUID_STDEXT_SMAP) != 0 ?
+ futex_andl_smap : futex_andl_nosmap);
+}
+
+int futex_xorl_nosmap(int oparg, uint32_t *uaddr, int *oldval);
+int futex_xorl_smap(int oparg, uint32_t *uaddr, int *oldval);
+DEFINE_IFUNC(, int, futex_xorl, (int, uint32_t *, int *), static)
+{
+
+ return ((cpu_stdext_feature & CPUID_STDEXT_SMAP) != 0 ?
+ futex_xorl_smap : futex_xorl_nosmap);
}
Modified: head/sys/amd64/linux32/linux32_support.s
==============================================================================
--- head/sys/amd64/linux32/linux32_support.s Tue Aug 7 18:26:46 2018 (r337430)
+++ head/sys/amd64/linux32/linux32_support.s Tue Aug 7 18:29:10 2018 (r337431)
@@ -38,7 +38,7 @@ futex_fault:
movl $-EFAULT,%eax
ret
-ENTRY(futex_xchgl)
+ENTRY(futex_xchgl_nosmap)
movq PCPU(CURPCB),%r8
movq $futex_fault,PCB_ONFAULT(%r8)
movq $VM_MAXUSER_ADDRESS-4,%rax
@@ -49,14 +49,29 @@ ENTRY(futex_xchgl)
xorl %eax,%eax
movq %rax,PCB_ONFAULT(%r8)
ret
-END(futex_xchgl)
+END(futex_xchgl_nosmap)
-ENTRY(futex_addl)
+ENTRY(futex_xchgl_smap)
movq PCPU(CURPCB),%r8
movq $futex_fault,PCB_ONFAULT(%r8)
movq $VM_MAXUSER_ADDRESS-4,%rax
cmpq %rax,%rsi
ja futex_fault
+ stac
+ xchgl %edi,(%rsi)
+ clac
+ movl %edi,(%rdx)
+ xorl %eax,%eax
+ movq %rax,PCB_ONFAULT(%r8)
+ ret
+END(futex_xchgl_smap)
+
+ENTRY(futex_addl_nosmap)
+ movq PCPU(CURPCB),%r8
+ movq $futex_fault,PCB_ONFAULT(%r8)
+ movq $VM_MAXUSER_ADDRESS-4,%rax
+ cmpq %rax,%rsi
+ ja futex_fault
#ifdef SMP
lock
#endif
@@ -65,14 +80,32 @@ ENTRY(futex_addl)
xorl %eax,%eax
movq %rax,PCB_ONFAULT(%r8)
ret
-END(futex_addl)
+END(futex_addl_nosmap)
-ENTRY(futex_orl)
+ENTRY(futex_addl_smap)
movq PCPU(CURPCB),%r8
movq $futex_fault,PCB_ONFAULT(%r8)
movq $VM_MAXUSER_ADDRESS-4,%rax
cmpq %rax,%rsi
ja futex_fault
+ stac
+#ifdef SMP
+ lock
+#endif
+ xaddl %edi,(%rsi)
+ clac
+ movl %edi,(%rdx)
+ xorl %eax,%eax
+ movq %rax,PCB_ONFAULT(%r8)
+ ret
+END(futex_addl_smap)
+
+ENTRY(futex_orl_nosmap)
+ movq PCPU(CURPCB),%r8
+ movq $futex_fault,PCB_ONFAULT(%r8)
+ movq $VM_MAXUSER_ADDRESS-4,%rax
+ cmpq %rax,%rsi
+ ja futex_fault
movl (%rsi),%eax
1: movl %eax,%ecx
orl %edi,%ecx
@@ -85,9 +118,9 @@ ENTRY(futex_orl)
xorl %eax,%eax
movq %rax,PCB_ONFAULT(%r8)
ret
-END(futex_orl)
+END(futex_orl_nosmap)
-ENTRY(futex_andl)
+ENTRY(futex_orl_smap)
movq PCPU(CURPCB),%r8
movq $futex_fault,PCB_ONFAULT(%r8)
movq $VM_MAXUSER_ADDRESS-4,%rax
@@ -95,6 +128,28 @@ ENTRY(futex_andl)
ja futex_fault
movl (%rsi),%eax
1: movl %eax,%ecx
+ orl %edi,%ecx
+ stac
+#ifdef SMP
+ lock
+#endif
+ cmpxchgl %ecx,(%rsi)
+ clac
+ jnz 1b
+ movl %eax,(%rdx)
+ xorl %eax,%eax
+ movq %rax,PCB_ONFAULT(%r8)
+ ret
+END(futex_orl_smap)
+
+ENTRY(futex_andl_nosmap)
+ movq PCPU(CURPCB),%r8
+ movq $futex_fault,PCB_ONFAULT(%r8)
+ movq $VM_MAXUSER_ADDRESS-4,%rax
+ cmpq %rax,%rsi
+ ja futex_fault
+ movl (%rsi),%eax
+1: movl %eax,%ecx
andl %edi,%ecx
#ifdef SMP
lock
@@ -105,9 +160,9 @@ ENTRY(futex_andl)
xorl %eax,%eax
movq %rax,PCB_ONFAULT(%r8)
ret
-END(futex_andl)
+END(futex_andl_nosmap)
-ENTRY(futex_xorl)
+ENTRY(futex_andl_smap)
movq PCPU(CURPCB),%r8
movq $futex_fault,PCB_ONFAULT(%r8)
movq $VM_MAXUSER_ADDRESS-4,%rax
@@ -115,6 +170,28 @@ ENTRY(futex_xorl)
ja futex_fault
movl (%rsi),%eax
1: movl %eax,%ecx
+ andl %edi,%ecx
+ stac
+#ifdef SMP
+ lock
+#endif
+ cmpxchgl %ecx,(%rsi)
+ clac
+ jnz 1b
+ movl %eax,(%rdx)
+ xorl %eax,%eax
+ movq %rax,PCB_ONFAULT(%r8)
+ ret
+END(futex_andl_smap)
+
+ENTRY(futex_xorl_nosmap)
+ movq PCPU(CURPCB),%r8
+ movq $futex_fault,PCB_ONFAULT(%r8)
+ movq $VM_MAXUSER_ADDRESS-4,%rax
+ cmpq %rax,%rsi
+ ja futex_fault
+ movl (%rsi),%eax
+1: movl %eax,%ecx
xorl %edi,%ecx
#ifdef SMP
lock
@@ -125,4 +202,26 @@ ENTRY(futex_xorl)
xorl %eax,%eax
movq %rax,PCB_ONFAULT(%r8)
ret
-END(futex_xorl)
+END(futex_xorl_nosmap)
+
+ENTRY(futex_xorl_smap)
+ movq PCPU(CURPCB),%r8
+ movq $futex_fault,PCB_ONFAULT(%r8)
+ movq $VM_MAXUSER_ADDRESS-4,%rax
+ cmpq %rax,%rsi
+ ja futex_fault
+ movl (%rsi),%eax
+1: movl %eax,%ecx
+ xorl %edi,%ecx
+ stac
+#ifdef SMP
+ lock
+#endif
+ cmpxchgl %ecx,(%rsi)
+ clac
+ jnz 1b
+ movl %eax,(%rdx)
+ xorl %eax,%eax
+ movq %rax,PCB_ONFAULT(%r8)
+ ret
+END(futex_xorl_smap)
More information about the svn-src-head
mailing list