svn commit: r345226 - head/sys/amd64/amd64

[Konstantin Belousov]

Author: kib
Date: Sat Mar 16 11:16:09 2019
New Revision: 345226
URL: https://svnweb.freebsd.org/changeset/base/345226

Log:
  amd64: fix switching to the pmap with pti disabled.
  
  When the pmap with pti disabled (i.e. pm_ucr3 == PMAP_NO_CR3) is
  activated, tss.rsp0 was not updated.  Any interrupt that happen before
  next context switch would use pti trampoline stack for hardware frame
  but fault and interrupt handlers are not prepared to this.  Correctly
  update tss.rsp0 for both PMAP_NO_CR3 and pti pmaps.
  
  Note that this case, pti = 1 but pmap->pm_ucr3 == PMAP_NO_CR3 is not
  used at the moment.
  
  Reviewed by:	markj
  Tested by:	pho
  Sponsored by:	The FreeBSD Foundation
  MFC after:	1 week
  Differential revision:	https://reviews.freebsd.org/D19514

Modified:
  head/sys/amd64/amd64/pmap.c

Modified: head/sys/amd64/amd64/pmap.c
==============================================================================
--- head/sys/amd64/amd64/pmap.c	Sat Mar 16 11:12:02 2019	(r345225)
+++ head/sys/amd64/amd64/pmap.c	Sat Mar 16 11:16:09 2019	(r345226)
@@ -7759,12 +7759,11 @@ pmap_pcid_alloc_checked(pmap_t pmap, u_int cpuid)
 }
 
 static void
-pmap_activate_sw_pti_post(pmap_t pmap)
+pmap_activate_sw_pti_post(struct thread *td, pmap_t pmap)
 {
 
-	if (pmap->pm_ucr3 != PMAP_NO_CR3)
-		PCPU_GET(tssp)->tss_rsp0 = ((vm_offset_t)PCPU_PTR(pti_stack) +
-		    PC_PTI_STACK_SZ * sizeof(uint64_t)) & ~0xful;
+	PCPU_GET(tssp)->tss_rsp0 = pmap->pm_ucr3 != PMAP_NO_CR3 ?
+	    PCPU_GET(pti_rsp0) : (uintptr_t)td->td_pcb;
 }
 
 static void inline
@@ -7811,15 +7810,16 @@ pmap_activate_sw_pcid_pti(pmap_t pmap, u_int cpuid, co
 }
 
 static void
-pmap_activate_sw_pcid_invpcid_pti(pmap_t pmap, u_int cpuid)
+pmap_activate_sw_pcid_invpcid_pti(struct thread *td, pmap_t pmap, u_int cpuid)
 {
 
 	pmap_activate_sw_pcid_pti(pmap, cpuid, true);
-	pmap_activate_sw_pti_post(pmap);
+	pmap_activate_sw_pti_post(td, pmap);
 }
 
 static void
-pmap_activate_sw_pcid_noinvpcid_pti(pmap_t pmap, u_int cpuid)
+pmap_activate_sw_pcid_noinvpcid_pti(struct thread *td, pmap_t pmap,
+    u_int cpuid)
 {
 	register_t rflags;
 
@@ -7843,11 +7843,12 @@ pmap_activate_sw_pcid_noinvpcid_pti(pmap_t pmap, u_int
 	rflags = intr_disable();
 	pmap_activate_sw_pcid_pti(pmap, cpuid, false);
 	intr_restore(rflags);
-	pmap_activate_sw_pti_post(pmap);
+	pmap_activate_sw_pti_post(td, pmap);
 }
 
 static void
-pmap_activate_sw_pcid_nopti(pmap_t pmap, u_int cpuid)
+pmap_activate_sw_pcid_nopti(struct thread *td __unused, pmap_t pmap,
+    u_int cpuid)
 {
 	uint64_t cached, cr3;
 
@@ -7862,17 +7863,19 @@ pmap_activate_sw_pcid_nopti(pmap_t pmap, u_int cpuid)
 }
 
 static void
-pmap_activate_sw_pcid_noinvpcid_nopti(pmap_t pmap, u_int cpuid)
+pmap_activate_sw_pcid_noinvpcid_nopti(struct thread *td __unused, pmap_t pmap,
+    u_int cpuid)
 {
 	register_t rflags;
 
 	rflags = intr_disable();
-	pmap_activate_sw_pcid_nopti(pmap, cpuid);
+	pmap_activate_sw_pcid_nopti(td, pmap, cpuid);
 	intr_restore(rflags);
 }
 
 static void
-pmap_activate_sw_nopcid_nopti(pmap_t pmap, u_int cpuid __unused)
+pmap_activate_sw_nopcid_nopti(struct thread *td __unused, pmap_t pmap,
+    u_int cpuid __unused)
 {
 
 	load_cr3(pmap->pm_cr3);
@@ -7880,16 +7883,18 @@ pmap_activate_sw_nopcid_nopti(pmap_t pmap, u_int cpuid
 }
 
 static void
-pmap_activate_sw_nopcid_pti(pmap_t pmap, u_int cpuid __unused)
+pmap_activate_sw_nopcid_pti(struct thread *td, pmap_t pmap,
+    u_int cpuid __unused)
 {
 
-	pmap_activate_sw_nopcid_nopti(pmap, cpuid);
+	pmap_activate_sw_nopcid_nopti(td, pmap, cpuid);
 	PCPU_SET(kcr3, pmap->pm_cr3);
 	PCPU_SET(ucr3, pmap->pm_ucr3);
-	pmap_activate_sw_pti_post(pmap);
+	pmap_activate_sw_pti_post(td, pmap);
 }
 
-DEFINE_IFUNC(static, void, pmap_activate_sw_mode, (pmap_t, u_int), static)
+DEFINE_IFUNC(static, void, pmap_activate_sw_mode, (struct thread *, pmap_t,
+    u_int), static)
 {
 
 	if (pmap_pcid_enabled && pti && invpcid_works)
@@ -7922,7 +7927,7 @@ pmap_activate_sw(struct thread *td)
 #else
 	CPU_SET(cpuid, &pmap->pm_active);
 #endif
-	pmap_activate_sw_mode(pmap, cpuid);
+	pmap_activate_sw_mode(td, pmap, cpuid);
 #ifdef SMP
 	CPU_CLR_ATOMIC(cpuid, &oldpmap->pm_active);
 #else