git: 60ae7f745da0 - stable/13 - x86: Always use clean FPU and segment base state for new kthreads.
John Baldwin
jhb at FreeBSD.org
Mon Mar 29 22:12:10 UTC 2021
The branch stable/13 has been updated by jhb:
URL: https://cgit.FreeBSD.org/src/commit/?id=60ae7f745da0ec03f9a08b92f4db2fc9101b7123
commit 60ae7f745da0ec03f9a08b92f4db2fc9101b7123
Author: John Baldwin <jhb at FreeBSD.org>
AuthorDate: 2021-03-12 17:48:36 +0000
Commit: John Baldwin <jhb at FreeBSD.org>
CommitDate: 2021-03-29 18:10:59 +0000
x86: Always use clean FPU and segment base state for new kthreads.
Sponsored by: Netflix
(cherry picked from commit c7b021352332a2f79907d68f971849f74b73e1c6)
---
sys/amd64/amd64/vm_machdep.c | 46 +++++++++++++++++++++++++---------
sys/i386/i386/sys_machdep.c | 6 +++--
sys/i386/i386/vm_machdep.c | 59 +++++++++++++++++++++++++++++++-------------
3 files changed, 81 insertions(+), 30 deletions(-)
diff --git a/sys/amd64/amd64/vm_machdep.c b/sys/amd64/amd64/vm_machdep.c
index f10d0339a65a..6e60f2b3faff 100644
--- a/sys/amd64/amd64/vm_machdep.c
+++ b/sys/amd64/amd64/vm_machdep.c
@@ -164,10 +164,12 @@ cpu_fork(struct thread *td1, struct proc *p2, struct thread *td2, int flags)
return;
}
- /* Ensure that td1's pcb is up to date. */
- fpuexit(td1);
- if (td1 == curthread)
+ /* Ensure that td1's pcb is up to date for user processes. */
+ if ((td2->td_pflags & TDP_KTHREAD) == 0) {
+ MPASS(td1 == curthread);
+ fpuexit(td1);
update_pcb_bases(td1->td_pcb);
+ }
/* Point the stack and pcb to the actual location */
set_top_of_stack_td(td2);
@@ -178,8 +180,18 @@ cpu_fork(struct thread *td1, struct proc *p2, struct thread *td2, int flags)
/* Properly initialize pcb_save */
pcb2->pcb_save = get_pcb_user_save_pcb(pcb2);
- bcopy(get_pcb_user_save_td(td1), get_pcb_user_save_pcb(pcb2),
- cpu_max_ext_state_size);
+
+ /* Kernel processes start with clean FPU and segment bases. */
+ if ((td2->td_pflags & TDP_KTHREAD) != 0) {
+ pcb2->pcb_fsbase = 0;
+ pcb2->pcb_gsbase = 0;
+ clear_pcb_flags(pcb2, PCB_FPUINITDONE | PCB_USERFPUINITDONE |
+ PCB_KERNFPU | PCB_KERNFPU_THR);
+ } else {
+ MPASS((pcb2->pcb_flags & (PCB_KERNFPU | PCB_KERNFPU_THR)) == 0);
+ bcopy(get_pcb_user_save_td(td1), get_pcb_user_save_pcb(pcb2),
+ cpu_max_ext_state_size);
+ }
/* Point mdproc and then copy over td1's contents */
mdp2 = &p2->p_md;
@@ -564,10 +576,12 @@ cpu_copy_thread(struct thread *td, struct thread *td0)
pcb2 = td->td_pcb;
- /* Ensure that td0's pcb is up to date. */
- fpuexit(td0);
- if (td0 == curthread)
+ /* Ensure that td0's pcb is up to date for user threads. */
+ if ((td->td_pflags & TDP_KTHREAD) == 0) {
+ MPASS(td0 == curthread);
+ fpuexit(td0);
update_pcb_bases(td0->td_pcb);
+ }
/*
* Copy the upcall pcb. This loads kernel regs.
@@ -575,12 +589,22 @@ cpu_copy_thread(struct thread *td, struct thread *td0)
* values here.
*/
bcopy(td0->td_pcb, pcb2, sizeof(*pcb2));
- clear_pcb_flags(pcb2, PCB_KERNFPU);
pcb2->pcb_save = get_pcb_user_save_pcb(pcb2);
- bcopy(get_pcb_user_save_td(td0), pcb2->pcb_save,
- cpu_max_ext_state_size);
+
+ /* Kernel threads start with clean FPU and segment bases. */
+ if ((td->td_pflags & TDP_KTHREAD) != 0) {
+ pcb2->pcb_fsbase = 0;
+ pcb2->pcb_gsbase = 0;
+ clear_pcb_flags(pcb2, PCB_FPUINITDONE | PCB_USERFPUINITDONE |
+ PCB_KERNFPU | PCB_KERNFPU_THR);
+ } else {
+ MPASS((pcb2->pcb_flags & (PCB_KERNFPU | PCB_KERNFPU_THR)) == 0);
+ bcopy(get_pcb_user_save_td(td0), pcb2->pcb_save,
+ cpu_max_ext_state_size);
+ }
set_pcb_flags_raw(pcb2, PCB_FULL_IRET);
+
/*
* Create a new fresh stack for the new thread.
*/
diff --git a/sys/i386/i386/sys_machdep.c b/sys/i386/i386/sys_machdep.c
index 3f650b65e160..a0a1c273f467 100644
--- a/sys/i386/i386/sys_machdep.c
+++ b/sys/i386/i386/sys_machdep.c
@@ -108,7 +108,8 @@ set_fsbase(struct thread *td, uint32_t base)
fill_based_sd(&sd, base);
critical_enter();
td->td_pcb->pcb_fsd = sd;
- PCPU_GET(fsgs_gdt)[0] = sd;
+ if (td == curthread)
+ PCPU_GET(fsgs_gdt)[0] = sd;
critical_exit();
}
@@ -120,7 +121,8 @@ set_gsbase(struct thread *td, uint32_t base)
fill_based_sd(&sd, base);
critical_enter();
td->td_pcb->pcb_gsd = sd;
- PCPU_GET(fsgs_gdt)[1] = sd;
+ if (td == curthread)
+ PCPU_GET(fsgs_gdt)[1] = sd;
critical_exit();
}
diff --git a/sys/i386/i386/vm_machdep.c b/sys/i386/i386/vm_machdep.c
index 502de6e7f38f..471128e1713d 100644
--- a/sys/i386/i386/vm_machdep.c
+++ b/sys/i386/i386/vm_machdep.c
@@ -167,13 +167,15 @@ cpu_fork(struct thread *td1, struct proc *p2, struct thread *td2, int flags)
return;
}
- /* Ensure that td1's pcb is up to date. */
- if (td1 == curthread)
+ /* Ensure that td1's pcb is up to date for user processes. */
+ if ((td2->td_pflags & TDP_KTHREAD) == 0) {
+ MPASS(td1 == curthread);
td1->td_pcb->pcb_gs = rgs();
- critical_enter();
- if (PCPU_GET(fpcurthread) == td1)
- npxsave(td1->td_pcb->pcb_save);
- critical_exit();
+ critical_enter();
+ if (PCPU_GET(fpcurthread) == td1)
+ npxsave(td1->td_pcb->pcb_save);
+ critical_exit();
+ }
/* Point the pcb to the top of the stack */
pcb2 = get_pcb_td(td2);
@@ -184,8 +186,19 @@ cpu_fork(struct thread *td1, struct proc *p2, struct thread *td2, int flags)
/* Properly initialize pcb_save */
pcb2->pcb_save = get_pcb_user_save_pcb(pcb2);
- bcopy(get_pcb_user_save_td(td1), get_pcb_user_save_pcb(pcb2),
- cpu_max_ext_state_size);
+
+ /* Kernel processes start with clean NPX and segment bases. */
+ if ((td2->td_pflags & TDP_KTHREAD) != 0) {
+ pcb2->pcb_gs = _udatasel;
+ set_fsbase(td2, 0);
+ set_gsbase(td2, 0);
+ pcb2->pcb_flags &= ~(PCB_NPXINITDONE | PCB_NPXUSERINITDONE |
+ PCB_KERNNPX | PCB_KERNNPX_THR);
+ } else {
+ MPASS((pcb2->pcb_flags & (PCB_KERNNPX | PCB_KERNNPX_THR)) == 0);
+ bcopy(get_pcb_user_save_td(td1), get_pcb_user_save_pcb(pcb2),
+ cpu_max_ext_state_size);
+ }
/* Point mdproc and then copy over td1's contents */
mdp2 = &p2->p_md;
@@ -428,13 +441,15 @@ cpu_copy_thread(struct thread *td, struct thread *td0)
/* Point the pcb to the top of the stack. */
pcb2 = td->td_pcb;
- /* Ensure that td0's pcb is up to date. */
- if (td0 == curthread)
+ /* Ensure that td0's pcb is up to date for user threads. */
+ if ((td->td_pflags & TDP_KTHREAD) == 0) {
+ MPASS(td0 == curthread);
td0->td_pcb->pcb_gs = rgs();
- critical_enter();
- if (PCPU_GET(fpcurthread) == td0)
- npxsave(td0->td_pcb->pcb_save);
- critical_exit();
+ critical_enter();
+ if (PCPU_GET(fpcurthread) == td0)
+ npxsave(td0->td_pcb->pcb_save);
+ critical_exit();
+ }
/*
* Copy the upcall pcb. This loads kernel regs.
@@ -442,10 +457,20 @@ cpu_copy_thread(struct thread *td, struct thread *td0)
* values here.
*/
bcopy(td0->td_pcb, pcb2, sizeof(*pcb2));
- pcb2->pcb_flags &= ~PCB_KERNNPX;
pcb2->pcb_save = get_pcb_user_save_pcb(pcb2);
- bcopy(get_pcb_user_save_td(td0), pcb2->pcb_save,
- cpu_max_ext_state_size);
+
+ /* Kernel threads start with clean NPX and segment bases. */
+ if ((td->td_pflags & TDP_KTHREAD) != 0) {
+ pcb2->pcb_gs = _udatasel;
+ set_fsbase(td, 0);
+ set_gsbase(td, 0);
+ pcb2->pcb_flags &= ~(PCB_NPXINITDONE | PCB_NPXUSERINITDONE |
+ PCB_KERNNPX | PCB_KERNNPX_THR);
+ } else {
+ MPASS((pcb2->pcb_flags & (PCB_KERNNPX | PCB_KERNNPX_THR)) == 0);
+ bcopy(get_pcb_user_save_td(td0), pcb2->pcb_save,
+ cpu_max_ext_state_size);
+ }
/*
* Create a new fresh stack for the new thread.
More information about the dev-commits-src-all
mailing list