svn commit: r332450 - head/sys/amd64/amd64
Konstantin Belousov
kib at FreeBSD.org
Thu Apr 12 19:59:37 UTC 2018
Author: kib
Date: Thu Apr 12 19:59:36 2018
New Revision: 332450
URL: https://svnweb.freebsd.org/changeset/base/332450
Log:
Optimize context switch for PTI on PCID pmap.
In pti-enabled pmap, the PCID allocation scheme assigns temporal id
for the kernel page table, and user page table twin PCID is
calculating by setting high bit in the kernel PCID. So the kernel AS
is mapped with per-vmspace PCID, and we must completely shut down all
mappings in KVA when switching contexts, so that newly switched thread
would see all changes in KVA occured while it was not executing.
After all, KVA is same between all threads.
Currently the pti context switch for the user part of the page table
gets its TLB entries flushed too. It is excessive. The same PCID
flushing algorithm that is used for non-pti pmap, correctly works for
the UVA mappings. The only shared TLB entries are the pages from KVA
accessed by the kernel entry trampoline. All of them are static
except per-thread TSS and LDT. For TSS and LDT, the lifetime of newly
allocated entries is the whole thread life, so it is fine as well. If
not fine, then explicit shutdowns for current pmap of the newly
allocated LDT and TSS pages would be enough.
Also restore the constant value for the pm_pcid for the kernel_pmap.
Before, for PTI pmap, pm_pcid was erronously rolled same as user
pmap's pm_pcid, but it was not used.
Reviewed by: markj (previous version)
Discussed with: alc
Tested by: pho
Sponsored by: The FreeBSD Foundation
MFC after: 1 month
Differential revision: https://reviews.freebsd.org/D14961
Modified:
head/sys/amd64/amd64/pmap.c
Modified: head/sys/amd64/amd64/pmap.c
==============================================================================
--- head/sys/amd64/amd64/pmap.c Thu Apr 12 19:44:04 2018 (r332449)
+++ head/sys/amd64/amd64/pmap.c Thu Apr 12 19:59:36 2018 (r332450)
@@ -7330,8 +7330,9 @@ pmap_pcid_alloc(pmap_t pmap, u_int cpuid)
CRITICAL_ASSERT(curthread);
gen = PCPU_GET(pcid_gen);
- if (!pti && (pmap->pm_pcids[cpuid].pm_pcid == PMAP_PCID_KERN ||
- pmap->pm_pcids[cpuid].pm_gen == gen))
+ if (pmap->pm_pcids[cpuid].pm_pcid == PMAP_PCID_KERN)
+ return (pti ? 0 : CR3_PCID_SAVE);
+ if (pmap->pm_pcids[cpuid].pm_gen == gen)
return (CR3_PCID_SAVE);
pcid_next = PCPU_GET(pcid_next);
KASSERT((!pti && pcid_next <= PMAP_PCID_OVERMAX) ||
@@ -7358,7 +7359,7 @@ pmap_activate_sw(struct thread *td)
{
pmap_t oldpmap, pmap;
struct invpcid_descr d;
- uint64_t cached, cr3, kcr3, ucr3;
+ uint64_t cached, cr3, kcr3, kern_pti_cached, ucr3;
register_t rflags;
u_int cpuid;
@@ -7407,11 +7408,10 @@ pmap_activate_sw(struct thread *td)
if (!invpcid_works)
rflags = intr_disable();
- if (!cached || (cr3 & ~CR3_PCID_MASK) != pmap->pm_cr3) {
+ kern_pti_cached = pti ? 0 : cached;
+ if (!kern_pti_cached || (cr3 & ~CR3_PCID_MASK) != pmap->pm_cr3) {
load_cr3(pmap->pm_cr3 | pmap->pm_pcids[cpuid].pm_pcid |
- cached);
- if (cached)
- PCPU_INC(pm_save_cnt);
+ kern_pti_cached);
}
PCPU_SET(curpmap, pmap);
if (pti) {
@@ -7419,13 +7419,13 @@ pmap_activate_sw(struct thread *td)
ucr3 = pmap->pm_ucr3 | pmap->pm_pcids[cpuid].pm_pcid |
PMAP_PCID_USER_PT;
- /*
- * Manually invalidate translations cached
- * from the user page table, which are not
- * flushed by reload of cr3 with the kernel
- * page table pointer above.
- */
- if (pmap->pm_ucr3 != PMAP_NO_CR3) {
+ if (!cached && pmap->pm_ucr3 != PMAP_NO_CR3) {
+ /*
+ * Manually invalidate translations cached
+ * from the user page table. They are not
+ * flushed by reload of cr3 with the kernel
+ * page table pointer above.
+ */
if (invpcid_works) {
d.pcid = PMAP_PCID_USER_PT |
pmap->pm_pcids[cpuid].pm_pcid;
@@ -7442,6 +7442,8 @@ pmap_activate_sw(struct thread *td)
}
if (!invpcid_works)
intr_restore(rflags);
+ if (cached)
+ PCPU_INC(pm_save_cnt);
} else if (cr3 != pmap->pm_cr3) {
load_cr3(pmap->pm_cr3);
PCPU_SET(curpmap, pmap);
More information about the svn-src-head
mailing list