git: 9e0143694ad3 - main - amd64: add pmap_get_pcid() helper
- Go to: [ bottom of page ] [ top of archives ] [ this month ]
Date: Tue, 02 May 2023 11:39:53 UTC
The branch main has been updated by kib:
URL: https://cgit.FreeBSD.org/src/commit/?id=9e0143694ad31b534310a473b993e520264f11e2
commit 9e0143694ad31b534310a473b993e520264f11e2
Author: Konstantin Belousov <kib@FreeBSD.org>
AuthorDate: 2023-05-01 11:02:59 +0000
Commit: Konstantin Belousov <kib@FreeBSD.org>
CommitDate: 2023-05-02 11:32:35 +0000
amd64: add pmap_get_pcid() helper
Reviewed by: markj
Tested by: pho
Sponsored by: The FreeBSD Foundation
Differential revision: https://reviews.freebsd.org/D39890
---
sys/amd64/amd64/efirt_machdep.c | 19 +++++++++++++------
sys/amd64/amd64/mp_machdep.c | 20 ++++++++------------
sys/amd64/amd64/pmap.c | 16 ++++------------
sys/amd64/include/pmap.h | 8 ++++++++
4 files changed, 33 insertions(+), 30 deletions(-)
diff --git a/sys/amd64/amd64/efirt_machdep.c b/sys/amd64/amd64/efirt_machdep.c
index 6fffd113f868..b7f7a91b8671 100644
--- a/sys/amd64/amd64/efirt_machdep.c
+++ b/sys/amd64/amd64/efirt_machdep.c
@@ -298,6 +298,7 @@ int
efi_arch_enter(void)
{
pmap_t curpmap;
+ uint64_t cr3;
curpmap = PCPU_GET(curpmap);
PMAP_LOCK_ASSERT(curpmap, MA_OWNED);
@@ -313,8 +314,10 @@ efi_arch_enter(void)
if (pmap_pcid_enabled && !invpcid_works)
PCPU_SET(curpmap, NULL);
- load_cr3(VM_PAGE_TO_PHYS(efi_pmltop_page) | (pmap_pcid_enabled ?
- curpmap->pm_pcids[PCPU_GET(cpuid)].pm_pcid : 0));
+ cr3 = VM_PAGE_TO_PHYS(efi_pmltop_page);
+ if (pmap_pcid_enabled)
+ cr3 |= pmap_get_pcid(curpmap);
+ load_cr3(cr3);
/*
* If PCID is enabled, the clear CR3_PCID_SAVE bit in the loaded %cr3
* causes TLB invalidation.
@@ -328,12 +331,16 @@ void
efi_arch_leave(void)
{
pmap_t curpmap;
+ uint64_t cr3;
curpmap = &curproc->p_vmspace->vm_pmap;
- if (pmap_pcid_enabled && !invpcid_works)
- PCPU_SET(curpmap, curpmap);
- load_cr3(curpmap->pm_cr3 | (pmap_pcid_enabled ?
- curpmap->pm_pcids[PCPU_GET(cpuid)].pm_pcid : 0));
+ cr3 = curpmap->pm_cr3;
+ if (pmap_pcid_enabled) {
+ cr3 |= pmap_get_pcid(curpmap);
+ if (!invpcid_works)
+ PCPU_SET(curpmap, curpmap);
+ }
+ load_cr3(cr3);
if (!pmap_pcid_enabled)
invltlb();
vm_fault_enable_pagefaults(curthread->td_md.md_efirt_dis_pf);
diff --git a/sys/amd64/amd64/mp_machdep.c b/sys/amd64/amd64/mp_machdep.c
index 5c60d301c1e7..94589f736637 100644
--- a/sys/amd64/amd64/mp_machdep.c
+++ b/sys/amd64/amd64/mp_machdep.c
@@ -767,7 +767,7 @@ invltlb_invpcid_handler(pmap_t smp_tlb_pmap)
(*ipi_invltlb_counts[PCPU_GET(cpuid)])++;
#endif /* COUNT_IPIS */
- d.pcid = smp_tlb_pmap->pm_pcids[PCPU_GET(cpuid)].pm_pcid;
+ d.pcid = pmap_get_pcid(smp_tlb_pmap);
d.pad = 0;
d.addr = 0;
invpcid(&d, smp_tlb_pmap == kernel_pmap ? INVPCID_CTXGLOB :
@@ -786,7 +786,7 @@ invltlb_invpcid_pti_handler(pmap_t smp_tlb_pmap)
(*ipi_invltlb_counts[PCPU_GET(cpuid)])++;
#endif /* COUNT_IPIS */
- d.pcid = smp_tlb_pmap->pm_pcids[PCPU_GET(cpuid)].pm_pcid;
+ d.pcid = pmap_get_pcid(smp_tlb_pmap);
d.pad = 0;
d.addr = 0;
if (smp_tlb_pmap == kernel_pmap) {
@@ -808,8 +808,6 @@ invltlb_invpcid_pti_handler(pmap_t smp_tlb_pmap)
static void
invltlb_pcid_handler(pmap_t smp_tlb_pmap)
{
- uint32_t pcid;
-
#ifdef COUNT_XINVLTLB_HITS
xhits_gbl[PCPU_GET(cpuid)]++;
#endif /* COUNT_XINVLTLB_HITS */
@@ -828,8 +826,8 @@ invltlb_pcid_handler(pmap_t smp_tlb_pmap)
* CPU.
*/
if (smp_tlb_pmap == PCPU_GET(curpmap)) {
- pcid = smp_tlb_pmap->pm_pcids[PCPU_GET(cpuid)].pm_pcid;
- load_cr3(smp_tlb_pmap->pm_cr3 | pcid);
+ load_cr3(smp_tlb_pmap->pm_cr3 |
+ pmap_get_pcid(smp_tlb_pmap));
if (smp_tlb_pmap->pm_ucr3 != PMAP_NO_CR3)
PCPU_SET(ucr3_load_mask, ~CR3_PCID_SAVE);
}
@@ -865,8 +863,7 @@ invlpg_invpcid_handler(pmap_t smp_tlb_pmap, vm_offset_t smp_tlb_addr1)
if (smp_tlb_pmap == PCPU_GET(curpmap) &&
smp_tlb_pmap->pm_ucr3 != PMAP_NO_CR3 &&
PCPU_GET(ucr3_load_mask) == PMAP_UCR3_NOMASK) {
- d.pcid = smp_tlb_pmap->pm_pcids[PCPU_GET(cpuid)].pm_pcid |
- PMAP_PCID_USER_PT;
+ d.pcid = pmap_get_pcid(smp_tlb_pmap) | PMAP_PCID_USER_PT;
d.pad = 0;
d.addr = smp_tlb_addr1;
invpcid(&d, INVPCID_ADDR);
@@ -890,7 +887,7 @@ invlpg_pcid_handler(pmap_t smp_tlb_pmap, vm_offset_t smp_tlb_addr1)
if (smp_tlb_pmap == PCPU_GET(curpmap) &&
(ucr3 = smp_tlb_pmap->pm_ucr3) != PMAP_NO_CR3 &&
PCPU_GET(ucr3_load_mask) == PMAP_UCR3_NOMASK) {
- pcid = smp_tlb_pmap->pm_pcids[PCPU_GET(cpuid)].pm_pcid;
+ pcid = pmap_get_pcid(smp_tlb_pmap);
kcr3 = smp_tlb_pmap->pm_cr3 | pcid | CR3_PCID_SAVE;
ucr3 |= pcid | PMAP_PCID_USER_PT | CR3_PCID_SAVE;
pmap_pti_pcid_invlpg(ucr3, kcr3, smp_tlb_addr1);
@@ -944,8 +941,7 @@ invlrng_invpcid_handler(pmap_t smp_tlb_pmap, vm_offset_t smp_tlb_addr1,
if (smp_tlb_pmap == PCPU_GET(curpmap) &&
smp_tlb_pmap->pm_ucr3 != PMAP_NO_CR3 &&
PCPU_GET(ucr3_load_mask) == PMAP_UCR3_NOMASK) {
- d.pcid = smp_tlb_pmap->pm_pcids[PCPU_GET(cpuid)].pm_pcid |
- PMAP_PCID_USER_PT;
+ d.pcid = pmap_get_pcid(smp_tlb_pmap) | PMAP_PCID_USER_PT;
d.pad = 0;
d.addr = smp_tlb_addr1;
do {
@@ -978,7 +974,7 @@ invlrng_pcid_handler(pmap_t smp_tlb_pmap, vm_offset_t smp_tlb_addr1,
if (smp_tlb_pmap == PCPU_GET(curpmap) &&
(ucr3 = smp_tlb_pmap->pm_ucr3) != PMAP_NO_CR3 &&
PCPU_GET(ucr3_load_mask) == PMAP_UCR3_NOMASK) {
- pcid = smp_tlb_pmap->pm_pcids[PCPU_GET(cpuid)].pm_pcid;
+ pcid = pmap_get_pcid(smp_tlb_pmap);
kcr3 = smp_tlb_pmap->pm_cr3 | pcid | CR3_PCID_SAVE;
ucr3 |= pcid | PMAP_PCID_USER_PT | CR3_PCID_SAVE;
pmap_pti_pcid_invlrng(ucr3, kcr3, smp_tlb_addr1, smp_tlb_addr2);
diff --git a/sys/amd64/amd64/pmap.c b/sys/amd64/amd64/pmap.c
index 3b125041bb29..16bfe9986c6b 100644
--- a/sys/amd64/amd64/pmap.c
+++ b/sys/amd64/amd64/pmap.c
@@ -3076,7 +3076,6 @@ pmap_invalidate_page_pcid_cb(pmap_t pmap, vm_offset_t va,
struct invpcid_descr d;
uint64_t kcr3, ucr3;
uint32_t pcid;
- u_int cpuid;
/*
* Because pm_pcid is recalculated on a context switch, we
@@ -3095,9 +3094,7 @@ pmap_invalidate_page_pcid_cb(pmap_t pmap, vm_offset_t va,
PCPU_GET(ucr3_load_mask) != PMAP_UCR3_NOMASK)
return;
- cpuid = PCPU_GET(cpuid);
-
- pcid = pmap->pm_pcids[cpuid].pm_pcid;
+ pcid = pmap_get_pcid(pmap);
if (invpcid_works1) {
d.pcid = pcid | PMAP_PCID_USER_PT;
d.pad = 0;
@@ -3172,7 +3169,6 @@ pmap_invalidate_range_pcid_cb(pmap_t pmap, vm_offset_t sva, vm_offset_t eva,
struct invpcid_descr d;
uint64_t kcr3, ucr3;
uint32_t pcid;
- u_int cpuid;
CRITICAL_ASSERT(curthread);
@@ -3181,9 +3177,7 @@ pmap_invalidate_range_pcid_cb(pmap_t pmap, vm_offset_t sva, vm_offset_t eva,
PCPU_GET(ucr3_load_mask) != PMAP_UCR3_NOMASK)
return;
- cpuid = PCPU_GET(cpuid);
-
- pcid = pmap->pm_pcids[cpuid].pm_pcid;
+ pcid = pmap_get_pcid(pmap);
if (invpcid_works1) {
d.pcid = pcid | PMAP_PCID_USER_PT;
d.pad = 0;
@@ -3273,7 +3267,6 @@ pmap_invalidate_all_pcid_cb(pmap_t pmap, bool invpcid_works1)
struct invpcid_descr d;
uint64_t kcr3;
uint32_t pcid;
- u_int cpuid;
if (pmap == kernel_pmap) {
if (invpcid_works1) {
@@ -3284,9 +3277,8 @@ pmap_invalidate_all_pcid_cb(pmap_t pmap, bool invpcid_works1)
}
} else if (pmap == PCPU_GET(curpmap)) {
CRITICAL_ASSERT(curthread);
- cpuid = PCPU_GET(cpuid);
- pcid = pmap->pm_pcids[cpuid].pm_pcid;
+ pcid = pmap_get_pcid(pmap);
if (invpcid_works1) {
d.pcid = pcid;
d.pad = 0;
@@ -10152,7 +10144,7 @@ pmap_activate_boot(pmap_t pmap)
if (pti) {
kcr3 = pmap->pm_cr3;
if (pmap_pcid_enabled)
- kcr3 |= pmap->pm_pcids[cpuid].pm_pcid | CR3_PCID_SAVE;
+ kcr3 |= pmap_get_pcid(pmap) | CR3_PCID_SAVE;
} else {
kcr3 = PMAP_NO_CR3;
}
diff --git a/sys/amd64/include/pmap.h b/sys/amd64/include/pmap.h
index 3b99b88742e5..cef718fb50d5 100644
--- a/sys/amd64/include/pmap.h
+++ b/sys/amd64/include/pmap.h
@@ -533,6 +533,14 @@ pmap_invlpg(pmap_t pmap, vm_offset_t va)
}
#endif /* sys/pcpu.h && machine/cpufunc.h */
+/* Return pcid for the pmap pmap on current cpu */
+static __inline uint32_t
+pmap_get_pcid(pmap_t pmap)
+{
+ MPASS(pmap_pcid_enabled);
+ return (pmap->pm_pcids[PCPU_GET(cpuid)].pm_pcid);
+}
+
#endif /* _KERNEL */
/* Return various clipped indexes for a given VA */