git: 47656cc1ef1c - main - amd64: use INVLPGB for kernel pmap invalidations

The branch main has been updated by kib:

URL: https://cgit.FreeBSD.org/src/commit/?id=47656cc1ef1cac307f24de88a4fe23a1389af44e

commit 47656cc1ef1cac307f24de88a4fe23a1389af44e
Author:     Konstantin Belousov <kib@FreeBSD.org>
AuthorDate: 2024-05-13 22:42:08 +0000
Commit:     Konstantin Belousov <kib@FreeBSD.org>
CommitDate: 2024-08-21 16:35:15 +0000

    amd64: use INVLPGB for kernel pmap invalidations
    
    avoiding broadcast IPIs.
    
    Reviewed by:    alc, markj
    Tested by:      pho
    Sponsored by:   The FreeBSD Foundation
    MFC after:      1 week
    Differential revision:  https://reviews.freebsd.org/D45191
---
 sys/amd64/amd64/mp_machdep.c | 54 ++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 54 insertions(+)

diff --git a/sys/amd64/amd64/mp_machdep.c b/sys/amd64/amd64/mp_machdep.c
index 91737637b714..12abb8b6bf8b 100644
--- a/sys/amd64/amd64/mp_machdep.c
+++ b/sys/amd64/amd64/mp_machdep.c
@@ -679,6 +679,20 @@ local_cb:
 void
 smp_masked_invltlb(pmap_t pmap, smp_invl_cb_t curcpu_cb)
 {
+	if (invlpgb_works && pmap == kernel_pmap) {
+		invlpgb(INVLPGB_GLOB, 0, 0);
+
+		/*
+		 * TLBSYNC syncs only against INVLPGB executed on the
+		 * same CPU.  Since current thread is pinned by
+		 * caller, we do not need to enter critical section to
+		 * prevent migration.
+		 */
+		tlbsync();
+		sched_unpin();
+		return;
+	}
+
 	smp_targeted_tlb_shootdown(pmap, 0, 0, curcpu_cb, invl_op_tlb);
 #ifdef COUNT_XINVLTLB_HITS
 	ipi_global++;
@@ -688,6 +702,13 @@ smp_masked_invltlb(pmap_t pmap, smp_invl_cb_t curcpu_cb)
 void
 smp_masked_invlpg(vm_offset_t addr, pmap_t pmap, smp_invl_cb_t curcpu_cb)
 {
+	if (invlpgb_works && pmap == kernel_pmap) {
+		invlpgb(INVLPGB_GLOB | INVLPGB_VA | trunc_page(addr), 0, 0);
+		tlbsync();
+		sched_unpin();
+		return;
+	}
+
 	smp_targeted_tlb_shootdown(pmap, addr, 0, curcpu_cb, invl_op_pg);
 #ifdef COUNT_XINVLTLB_HITS
 	ipi_page++;
@@ -698,6 +719,39 @@ void
 smp_masked_invlpg_range(vm_offset_t addr1, vm_offset_t addr2, pmap_t pmap,
     smp_invl_cb_t curcpu_cb)
 {
+	if (invlpgb_works && pmap == kernel_pmap) {
+		vm_offset_t va;
+		uint64_t cnt, total;
+
+		addr1 = trunc_page(addr1);
+		addr2 = round_page(addr2);
+		total = atop(addr2 - addr1);
+		for (va = addr1; total > 0;) {
+			if ((va & PDRMASK) != 0 || total < NPDEPG) {
+				cnt = atop(NBPDR - (va & PDRMASK));
+				if (cnt > total)
+					cnt = total;
+				if (cnt > invlpgb_maxcnt + 1)
+					cnt = invlpgb_maxcnt + 1;
+				invlpgb(INVLPGB_GLOB | INVLPGB_VA | va, 0,
+				    cnt - 1);
+				va += ptoa(cnt);
+				total -= cnt;
+			} else {
+				cnt = total / NPTEPG;
+				if (cnt > invlpgb_maxcnt + 1)
+					cnt = invlpgb_maxcnt + 1;
+				invlpgb(INVLPGB_GLOB | INVLPGB_VA | va, 0,
+				    INVLPGB_2M_CNT | (cnt - 1));
+				va += cnt << PDRSHIFT;
+				total -= cnt * NPTEPG;
+			}
+		}
+		tlbsync();
+		sched_unpin();
+		return;
+	}
+
 	smp_targeted_tlb_shootdown(pmap, addr1, addr2, curcpu_cb,
 	    invl_op_pgrng);
 #ifdef COUNT_XINVLTLB_HITS