git: 7bb6b62394d3 - main - arm64: mte: copy/save tags on copy-on-write

The branch main has been updated by andrew:

URL: https://cgit.FreeBSD.org/src/commit/?id=7bb6b62394d3036567cf1453395d9e8b63d3dda1

commit 7bb6b62394d3036567cf1453395d9e8b63d3dda1
Author:     Harry Moulton <harry.moulton@arm.com>
AuthorDate: 2026-05-18 09:29:03 +0000
Commit:     Andrew Turner <andrew@FreeBSD.org>
CommitDate: 2026-05-27 15:22:26 +0000

    arm64: mte: copy/save tags on copy-on-write
    
    On a copy-on-write, copy the memory tags from the source pages to the
    destination pages so the forked process can continue to use MTE.
    
    Reviewed by:    andrew
    Sponsored by:   Arm Ltd
    Signed-off-by:  Harry Moulton <harry.moulton@arm.com>
    Differential Revision:  https://reviews.freebsd.org/D55955
---
 sys/arm64/arm64/mte.c   | 41 +++++++++++++++++++++++++++++++++++++++++
 sys/arm64/arm64/pmap.c  | 13 +++++++++++++
 sys/arm64/include/cpu.h |  1 +
 3 files changed, 55 insertions(+)

diff --git a/sys/arm64/arm64/mte.c b/sys/arm64/arm64/mte.c
index ba5d9df3b01c..6e902858a8b9 100644
--- a/sys/arm64/arm64/mte.c
+++ b/sys/arm64/arm64/mte.c
@@ -49,6 +49,22 @@ static u_int __read_mostly mte_version = 0;
 
 struct thread *mte_switch(struct thread *);
 
+#define load_tags(addr) ({						\
+	uint64_t __val;							\
+	asm volatile(							\
+	    ".arch_extension memtag	\n"				\
+	    "ldgm %0, [%1]		\n"				\
+	    ".arch_extension nomemtag" : "=r" (__val) : "r" (addr));	\
+	__val;								\
+})
+
+#define set_tags(tags, addr) do {					\
+	asm volatile(							\
+	    ".arch_extension memtag	\n"				\
+	    "stgm %0, [%1]		\n"				\
+	    ".arch_extension nomemtag" : "=r" (tags) : "r" (addr));	\
+} while (0)
+
 /* Fetch the block size used by tag load and store instructions */
 static inline size_t
 mte_block_size(void)
@@ -95,6 +111,31 @@ mte_sync_tags(vm_page_t page)
 	page->md.pv_flags |= PV_MTE_TAGGED;
 }
 
+/**
+ * Copy the allocation tags from given target to destination page. This is called
+ * on a copy-on-write and anything that causes a pmap_copy_page call.
+ */
+void
+mte_copy_tags(vm_page_t srcpage, vm_page_t dstpage, char *src, char *dst)
+{
+	size_t block_size;
+	uint64_t tags;
+
+	MPASS((srcpage->md.pv_flags & PV_MTE_TAGGED) != 0);
+
+	/*
+	 * Copy the tags from the source page to the destination page,
+	 * incrementing by the block count read from GMID_EL1
+	 */
+	block_size = mte_block_size();
+	for (size_t count = 0; count < PAGE_SIZE;
+	    count += block_size, src += block_size, dst += block_size) {
+		tags = load_tags(src);
+		set_tags(tags, dst);
+	}
+	dstpage->md.pv_flags |= PV_MTE_TAGGED;
+}
+
 void
 mte_fork(struct thread *new_td, struct thread *orig_td)
 {
diff --git a/sys/arm64/arm64/pmap.c b/sys/arm64/arm64/pmap.c
index 6914fc13b065..1fb9ac2011aa 100644
--- a/sys/arm64/arm64/pmap.c
+++ b/sys/arm64/arm64/pmap.c
@@ -146,6 +146,7 @@
 #include <vm/uma.h>
 
 #include <machine/asan.h>
+#include <machine/cpu.h>
 #include <machine/cpu_feat.h>
 #include <machine/elf.h>
 #include <machine/ifunc.h>
@@ -6954,6 +6955,15 @@ pmap_copy_page(vm_page_t msrc, vm_page_t mdst)
 	void *src = VM_PAGE_TO_DMAP(msrc);
 	void *dst = VM_PAGE_TO_DMAP(mdst);
 
+	/*
+	 * On a page copy, check whether the src page is tagged. If it is,
+	 * we must copy the tags before copying the contents of the page.
+	 */
+	if ((msrc->md.pv_flags & PV_MTE_TAGGED) != 0)
+		mte_copy_tags(msrc, mdst, src, dst);
+	else
+		mdst->md.pv_flags &= ~PV_MTE_TAGGED;
+
 	pagecopy(src, dst);
 }
 
@@ -6970,6 +6980,9 @@ pmap_copy_pages(vm_page_t ma[], vm_offset_t a_offset, vm_page_t mb[],
 	int cnt;
 
 	while (xfersize > 0) {
+		KASSERT(ADDR_IS_CANONICAL(a_offset),
+		    ("%s: Address not in canonical form: %lx", __func__, a_offset));
+
 		a_pg_offset = a_offset & PAGE_MASK;
 		m_a = ma[a_offset >> PAGE_SHIFT];
 		p_a = m_a->phys_addr;
diff --git a/sys/arm64/include/cpu.h b/sys/arm64/include/cpu.h
index d36e1e56c91c..bdbc601edd26 100644
--- a/sys/arm64/include/cpu.h
+++ b/sys/arm64/include/cpu.h
@@ -285,6 +285,7 @@ void	mte_thread_alloc(struct thread *);
 void	mte_thread0(struct thread *);
 
 void	mte_sync_tags(vm_page_t page);
+void	mte_copy_tags(vm_page_t, vm_page_t, char *, char *);
 
 /* Functions to read the sanitised view of the special registers */
 void	update_special_regs(u_int);