git: 1d4a00d45561 - main - powerpc/pmap: Use dcbz to zero pages in the radix pmap

The branch main has been updated by pkubaj:

URL: https://cgit.FreeBSD.org/src/commit/?id=1d4a00d455617e150da5a7c3076579a7e612d33d

commit 1d4a00d455617e150da5a7c3076579a7e612d33d
Author:     Piotr Kubaj <pkubaj@FreeBSD.org>
AuthorDate: 2026-09-08 11:22:58 +0000
Commit:     Piotr Kubaj <pkubaj@FreeBSD.org>
CommitDate: 2026-09-17 08:08:11 +0000

    powerpc/pmap: Use dcbz to zero pages in the radix pmap
    
    pagezero() was a plain store loop (bzero), which under the kernel build
    flags (-mno-vsx -msoft-float) compiles to byte stores.  dcbz
    establishes a zeroed cache line directly in the cache without a
    read-for-ownership fetch from memory, roughly halving the memory
    transactions of page zeroing.
    
    Measured on POWER9 (Raptor Blackbird, DD2.3, bare metal), zeroing a
    cold 256 MB buffer with 128-byte scalar loops:
    
      byte stores (current libkern memset)     8.6 GB/s
      doubleword (std) stores                  26.8 GB/s
      dcbz                                     34.6 GB/s
    
    dcbz raises an alignment interrupt on caching-inhibited mappings, and
    the kernel does not emulate it, so mmu_radix_zero_page() falls back to
    bzero() for any page whose memattr is not the write-back default.  The
    internal pagezero() callers only touch freshly allocated page-table
    pages, which are always write-back.
    
    Note: dcbz helps only zeroing, where there is no source to read.  For
    page copying it is a pessimization (it adds a redundant zeroing pass on
    top of the mandatory source read), so mmu_radix_copy_page() is left as a
    plain bcopy().
    
    Differential Revision:  https://reviews.freebsd.org/D59507
    Reviewed by:    jhibbits
---
 sys/powerpc/aim/mmu_radix.c | 19 ++++++++++++++++---
 1 file changed, 16 insertions(+), 3 deletions(-)

diff --git a/sys/powerpc/aim/mmu_radix.c b/sys/powerpc/aim/mmu_radix.c
index 7d456a587d29..145b0ec5560b 100644
--- a/sys/powerpc/aim/mmu_radix.c
+++ b/sys/powerpc/aim/mmu_radix.c
@@ -882,12 +882,22 @@ pmap_resident_count_dec(pmap_t pmap, int count)
 	pmap->pm_stats.resident_count -= count;
 }
 
+/*
+ * dcbz establishes a zeroed cache line without fetching it from memory,
+ * but raises an alignment interrupt on caching-inhibited mappings, so it
+ * may only be used on write-back memory.  The internal callers only zero
+ * freshly allocated page table pages, which are always write-back; the
+ * vm_page facing entry point guards on the page's memattr.
+ */
 static void
 pagezero(void *va)
 {
-	va = trunc_page(va);
+	vm_offset_t off;
 
-	bzero(va, PAGE_SIZE);
+	va = trunc_page(va);
+	for (off = 0; off < PAGE_SIZE; off += cacheline_size)
+		__asm __volatile("dcbz 0,%0" ::
+		    "r"((char *)va + off) : "memory");
 }
 
 static uint64_t
@@ -5791,7 +5801,10 @@ mmu_radix_zero_page(vm_page_t m)
 
 	CTR2(KTR_PMAP, "%s(%p)", __func__, m);
 	addr = VM_PAGE_TO_DMAP(m);
-	pagezero(addr);
+	if (__predict_true(m->md.mdpg_cache_attrs == VM_MEMATTR_DEFAULT))
+		pagezero(addr);
+	else
+		bzero(addr, PAGE_SIZE);
 }
 
 void