svn commit: r333527 - stable/11/sys/amd64/amd64

[Konstantin Belousov]

Author: kib
Date: Sat May 12 11:18:14 2018
New Revision: 333527
URL: https://svnweb.freebsd.org/changeset/base/333527

Log:
  MFC r332450:
  Optimize context switch for PTI on PCID pmap.
  
  Approved by:	re (marius)

Modified:
  stable/11/sys/amd64/amd64/pmap.c
Directory Properties:
  stable/11/   (props changed)

Modified: stable/11/sys/amd64/amd64/pmap.c
==============================================================================
--- stable/11/sys/amd64/amd64/pmap.c	Sat May 12 11:06:59 2018	(r333526)
+++ stable/11/sys/amd64/amd64/pmap.c	Sat May 12 11:18:14 2018	(r333527)
@@ -7293,8 +7293,9 @@ pmap_pcid_alloc(pmap_t pmap, u_int cpuid)
 
 	CRITICAL_ASSERT(curthread);
 	gen = PCPU_GET(pcid_gen);
-	if (!pti && (pmap->pm_pcids[cpuid].pm_pcid == PMAP_PCID_KERN ||
-	    pmap->pm_pcids[cpuid].pm_gen == gen))
+	if (pmap->pm_pcids[cpuid].pm_pcid == PMAP_PCID_KERN)
+		return (pti ? 0 : CR3_PCID_SAVE);
+	if (pmap->pm_pcids[cpuid].pm_gen == gen)
 		return (CR3_PCID_SAVE);
 	pcid_next = PCPU_GET(pcid_next);
 	KASSERT((!pti && pcid_next <= PMAP_PCID_OVERMAX) ||
@@ -7321,7 +7322,7 @@ pmap_activate_sw(struct thread *td)
 {
 	pmap_t oldpmap, pmap;
 	struct invpcid_descr d;
-	uint64_t cached, cr3, kcr3, ucr3;
+	uint64_t cached, cr3, kcr3, kern_pti_cached, ucr3;
 	register_t rflags;
 	u_int cpuid;
 
@@ -7370,11 +7371,10 @@ pmap_activate_sw(struct thread *td)
 		if (!invpcid_works)
 			rflags = intr_disable();
 
-		if (!cached || (cr3 & ~CR3_PCID_MASK) != pmap->pm_cr3) {
+		kern_pti_cached = pti ? 0 : cached;
+		if (!kern_pti_cached || (cr3 & ~CR3_PCID_MASK) != pmap->pm_cr3) {
 			load_cr3(pmap->pm_cr3 | pmap->pm_pcids[cpuid].pm_pcid |
-			    cached);
-			if (cached)
-				PCPU_INC(pm_save_cnt);
+			    kern_pti_cached);
 		}
 		PCPU_SET(curpmap, pmap);
 		if (pti) {
@@ -7382,13 +7382,13 @@ pmap_activate_sw(struct thread *td)
 			ucr3 = pmap->pm_ucr3 | pmap->pm_pcids[cpuid].pm_pcid |
 			    PMAP_PCID_USER_PT;
 
-			/*
-			 * Manually invalidate translations cached
-			 * from the user page table, which are not
-			 * flushed by reload of cr3 with the kernel
-			 * page table pointer above.
-			 */
-			if (pmap->pm_ucr3 != PMAP_NO_CR3) {
+			if (!cached && pmap->pm_ucr3 != PMAP_NO_CR3) {
+				/*
+				 * Manually invalidate translations cached
+				 * from the user page table.  They are not
+				 * flushed by reload of cr3 with the kernel
+				 * page table pointer above.
+				 */
 				if (invpcid_works) {
 					d.pcid = PMAP_PCID_USER_PT |
 					    pmap->pm_pcids[cpuid].pm_pcid;
@@ -7405,6 +7405,8 @@ pmap_activate_sw(struct thread *td)
 		}
 		if (!invpcid_works)
 			intr_restore(rflags);
+		if (cached)
+			PCPU_INC(pm_save_cnt);
 	} else if (cr3 != pmap->pm_cr3) {
 		load_cr3(pmap->pm_cr3);
 		PCPU_SET(curpmap, pmap);