git: e20f0db4a94a - stable/13 - Start to clean up arm64 address space selection
Andrew Turner
andrew at FreeBSD.org
Mon Aug 30 12:43:11 UTC 2021
The branch stable/13 has been updated by andrew:
URL: https://cgit.FreeBSD.org/src/commit/?id=e20f0db4a94ab5c29161fda4e6377878f78ab8d7
commit e20f0db4a94ab5c29161fda4e6377878f78ab8d7
Author: Andrew Turner <andrew at FreeBSD.org>
AuthorDate: 2021-07-14 15:19:06 +0000
Commit: Andrew Turner <andrew at FreeBSD.org>
CommitDate: 2021-08-30 11:22:21 +0000
Start to clean up arm64 address space selection
On arm64 we should use bit 55 of the address to decide if aan address
is a user or kernel address. Add a new macro with this check and a
second to ensure the address is in teh canonical form, i.e.
the top bits are all zero or all one.
This will help with supporting future cpu features, including Top
Byte Ignore, Pointer Authentication, and Memory Tagging.
Reviewed by: kib
Sponsored by: The FreeBSD Foundation
Differential Revision: https://reviews.freebsd.org/D31179
(cherry picked from commit b7a78d573a566c77117af5fe0b259620934a5c7f)
---
sys/arm64/arm64/machdep.c | 10 ++++--
sys/arm64/arm64/pmap.c | 75 ++++++++++++++++++++++++++++++++-------------
sys/arm64/arm64/trap.c | 9 +++++-
sys/arm64/include/vmparam.h | 7 +++++
4 files changed, 76 insertions(+), 25 deletions(-)
diff --git a/sys/arm64/arm64/machdep.c b/sys/arm64/arm64/machdep.c
index 6b7e4be7c9d1..a2db5e2b67af 100644
--- a/sys/arm64/arm64/machdep.c
+++ b/sys/arm64/arm64/machdep.c
@@ -377,7 +377,10 @@ set_dbregs(struct thread *td, struct dbreg *regs)
addr = regs->db_breakregs[i].dbr_addr;
ctrl = regs->db_breakregs[i].dbr_ctrl;
- /* Don't let the user set a breakpoint on a kernel address. */
+ /*
+ * Don't let the user set a breakpoint on a kernel or
+ * non-canonical user address.
+ */
if (addr >= VM_MAXUSER_ADDRESS)
return (EINVAL);
@@ -412,7 +415,10 @@ set_dbregs(struct thread *td, struct dbreg *regs)
addr = regs->db_watchregs[i].dbw_addr;
ctrl = regs->db_watchregs[i].dbw_ctrl;
- /* Don't let the user set a watchpoint on a kernel address. */
+ /*
+ * Don't let the user set a watchpoint on a kernel or
+ * non-canonical user address.
+ */
if (addr >= VM_MAXUSER_ADDRESS)
return (EINVAL);
diff --git a/sys/arm64/arm64/pmap.c b/sys/arm64/arm64/pmap.c
index e57a4033c3a3..343a3d69ddee 100644
--- a/sys/arm64/arm64/pmap.c
+++ b/sys/arm64/arm64/pmap.c
@@ -466,11 +466,13 @@ pmap_l1_to_l2(pd_entry_t *l1p, vm_offset_t va)
l1 = pmap_load(l1p);
+ KASSERT(ADDR_IS_CANONICAL(va),
+ ("%s: Address not in canonical form: %lx", __func__, va));
/*
* The valid bit may be clear if pmap_update_entry() is concurrently
* modifying the entry, so for KVA only the entry type may be checked.
*/
- KASSERT(va >= VM_MAX_USER_ADDRESS || (l1 & ATTR_DESCR_VALID) != 0,
+ KASSERT(ADDR_IS_KERNEL(va) || (l1 & ATTR_DESCR_VALID) != 0,
("%s: L1 entry %#lx for %#lx is invalid", __func__, l1, va));
KASSERT((l1 & ATTR_DESCR_TYPE_MASK) == ATTR_DESCR_TYPE_TABLE,
("%s: L1 entry %#lx for %#lx is a leaf", __func__, l1, va));
@@ -498,11 +500,13 @@ pmap_l2_to_l3(pd_entry_t *l2p, vm_offset_t va)
l2 = pmap_load(l2p);
+ KASSERT(ADDR_IS_CANONICAL(va),
+ ("%s: Address not in canonical form: %lx", __func__, va));
/*
* The valid bit may be clear if pmap_update_entry() is concurrently
* modifying the entry, so for KVA only the entry type may be checked.
*/
- KASSERT(va >= VM_MAX_USER_ADDRESS || (l2 & ATTR_DESCR_VALID) != 0,
+ KASSERT(ADDR_IS_KERNEL(va) || (l2 & ATTR_DESCR_VALID) != 0,
("%s: L2 entry %#lx for %#lx is invalid", __func__, l2, va));
KASSERT((l2 & ATTR_DESCR_TYPE_MASK) == ATTR_DESCR_TYPE_TABLE,
("%s: L2 entry %#lx for %#lx is a leaf", __func__, l2, va));
@@ -1580,7 +1584,9 @@ pmap_qremove(vm_offset_t sva, int count)
vm_offset_t va;
int lvl;
- KASSERT(sva >= VM_MIN_KERNEL_ADDRESS, ("usermode va %lx", sva));
+ KASSERT(ADDR_IS_CANONICAL(sva),
+ ("%s: Address not in canonical form: %lx", __func__, sva));
+ KASSERT(ADDR_IS_KERNEL(sva), ("usermode va %lx", sva));
va = sva;
while (count-- > 0) {
@@ -1700,7 +1706,9 @@ pmap_unuse_pt(pmap_t pmap, vm_offset_t va, pd_entry_t ptepde,
{
vm_page_t mpte;
- if (va >= VM_MAXUSER_ADDRESS)
+ KASSERT(ADDR_IS_CANONICAL(va),
+ ("%s: Address not in canonical form: %lx", __func__, va));
+ if (ADDR_IS_KERNEL(va))
return (0);
KASSERT(ptepde != 0, ("pmap_unuse_pt: ptepde != 0"));
mpte = PHYS_TO_VM_PAGE(ptepde & ~ATTR_MASK);
@@ -1960,17 +1968,20 @@ pmap_alloc_l2(pmap_t pmap, vm_offset_t va, vm_page_t *l2pgp,
vm_page_t l2pg;
vm_pindex_t l2pindex;
+ KASSERT(ADDR_IS_CANONICAL(va),
+ ("%s: Address not in canonical form: %lx", __func__, va));
+
retry:
l1 = pmap_l1(pmap, va);
if (l1 != NULL && (pmap_load(l1) & ATTR_DESCR_MASK) == L1_TABLE) {
l2 = pmap_l1_to_l2(l1, va);
- if (va < VM_MAXUSER_ADDRESS) {
+ if (!ADDR_IS_KERNEL(va)) {
/* Add a reference to the L2 page. */
l2pg = PHYS_TO_VM_PAGE(pmap_load(l1) & ~ATTR_MASK);
l2pg->ref_count++;
} else
l2pg = NULL;
- } else if (va < VM_MAXUSER_ADDRESS) {
+ } else if (!ADDR_IS_KERNEL(va)) {
/* Allocate a L2 page. */
l2pindex = pmap_l2_pindex(va) >> Ln_ENTRIES_SHIFT;
l2pg = _pmap_alloc_l3(pmap, NUL2E + l2pindex, lockp);
@@ -2940,11 +2951,15 @@ pmap_remove_l3_range(pmap_t pmap, pd_entry_t l2e, vm_offset_t sva,
vm_offset_t va;
vm_page_t l3pg, m;
+ KASSERT(ADDR_IS_CANONICAL(sva),
+ ("%s: Start address not in canonical form: %lx", __func__, sva));
+ KASSERT(ADDR_IS_CANONICAL(eva) || eva == VM_MAX_USER_ADDRESS,
+ ("%s: End address not in canonical form: %lx", __func__, eva));
+
PMAP_LOCK_ASSERT(pmap, MA_OWNED);
KASSERT(rounddown2(sva, L2_SIZE) + L2_SIZE == roundup2(eva, L2_SIZE),
("pmap_remove_l3_range: range crosses an L3 page table boundary"));
- l3pg = sva < VM_MAXUSER_ADDRESS ? PHYS_TO_VM_PAGE(l2e & ~ATTR_MASK) :
- NULL;
+ l3pg = !ADDR_IS_KERNEL(sva) ? PHYS_TO_VM_PAGE(l2e & ~ATTR_MASK) : NULL;
va = eva;
for (l3 = pmap_l2_to_l3(&l2e, sva); sva != eva; l3++, sva += L3_SIZE) {
if (!pmap_l3_valid(pmap_load(l3))) {
@@ -3885,6 +3900,9 @@ pmap_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot,
boolean_t nosleep;
int lvl, rv;
+ KASSERT(ADDR_IS_CANONICAL(va),
+ ("%s: Address not in canonical form: %lx", __func__, va));
+
va = trunc_page(va);
if ((m->oflags & VPO_UNMANAGED) == 0)
VM_PAGE_OBJECT_BUSY_ASSERT(m);
@@ -3896,7 +3914,7 @@ pmap_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot,
if ((flags & PMAP_ENTER_WIRED) != 0)
new_l3 |= ATTR_SW_WIRED;
if (pmap->pm_stage == PM_STAGE1) {
- if (va < VM_MAXUSER_ADDRESS)
+ if (!ADDR_IS_KERNEL(va))
new_l3 |= ATTR_S1_AP(ATTR_S1_AP_USER) | ATTR_S1_PXN;
else
new_l3 |= ATTR_S1_UXN;
@@ -3963,7 +3981,7 @@ retry:
pde = pmap_pde(pmap, va, &lvl);
if (pde != NULL && lvl == 2) {
l3 = pmap_l2_to_l3(pde, va);
- if (va < VM_MAXUSER_ADDRESS && mpte == NULL) {
+ if (!ADDR_IS_KERNEL(va) && mpte == NULL) {
mpte = PHYS_TO_VM_PAGE(pmap_load(pde) & ~ATTR_MASK);
mpte->ref_count++;
}
@@ -3973,7 +3991,7 @@ retry:
if ((pmap_load(l2) & ATTR_DESCR_MASK) == L2_BLOCK &&
(l3 = pmap_demote_l2_locked(pmap, l2, va, &lock)) != NULL) {
l3 = &l3[pmap_l3_index(va)];
- if (va < VM_MAXUSER_ADDRESS) {
+ if (!ADDR_IS_KERNEL(va)) {
mpte = PHYS_TO_VM_PAGE(
pmap_load(l2) & ~ATTR_MASK);
mpte->ref_count++;
@@ -3982,7 +4000,7 @@ retry:
}
/* We need to allocate an L3 table. */
}
- if (va < VM_MAXUSER_ADDRESS) {
+ if (!ADDR_IS_KERNEL(va)) {
nosleep = (flags & PMAP_ENTER_NOSLEEP) != 0;
/*
@@ -4207,6 +4225,8 @@ pmap_enter_2mpage(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot,
PMAP_LOCK_ASSERT(pmap, MA_OWNED);
PMAP_ASSERT_STAGE1(pmap);
+ KASSERT(ADDR_IS_CANONICAL(va),
+ ("%s: Address not in canonical form: %lx", __func__, va));
new_l2 = (pd_entry_t)(VM_PAGE_TO_PHYS(m) | ATTR_DEFAULT |
ATTR_S1_IDX(m->md.pv_memattr) | ATTR_S1_AP(ATTR_S1_AP_RO) |
@@ -4218,7 +4238,7 @@ pmap_enter_2mpage(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot,
if ((prot & VM_PROT_EXECUTE) == 0 ||
m->md.pv_memattr == VM_MEMATTR_DEVICE)
new_l2 |= ATTR_S1_XN;
- if (va < VM_MAXUSER_ADDRESS)
+ if (!ADDR_IS_KERNEL(va))
new_l2 |= ATTR_S1_AP(ATTR_S1_AP_USER) | ATTR_S1_PXN;
else
new_l2 |= ATTR_S1_UXN;
@@ -4267,6 +4287,8 @@ pmap_enter_l2(pmap_t pmap, vm_offset_t va, pd_entry_t new_l2, u_int flags,
vm_page_t l2pg, mt;
PMAP_LOCK_ASSERT(pmap, MA_OWNED);
+ KASSERT(ADDR_IS_CANONICAL(va),
+ ("%s: Address not in canonical form: %lx", __func__, va));
if ((l2 = pmap_alloc_l2(pmap, va, &l2pg, (flags &
PMAP_ENTER_NOSLEEP) != 0 ? NULL : lockp)) == NULL) {
@@ -4281,9 +4303,10 @@ pmap_enter_l2(pmap_t pmap, vm_offset_t va, pd_entry_t new_l2, u_int flags,
if ((old_l2 = pmap_load(l2)) != 0) {
KASSERT(l2pg == NULL || l2pg->ref_count > 1,
("pmap_enter_l2: l2pg's ref count is too low"));
- if ((flags & PMAP_ENTER_NOREPLACE) != 0 && (va <
- VM_MAXUSER_ADDRESS || (old_l2 & ATTR_DESCR_MASK) ==
- L2_BLOCK || !pmap_every_pte_zero(old_l2 & ~ATTR_MASK))) {
+ if ((flags & PMAP_ENTER_NOREPLACE) != 0 &&
+ (!ADDR_IS_KERNEL(va) ||
+ (old_l2 & ATTR_DESCR_MASK) == L2_BLOCK ||
+ !pmap_every_pte_zero(old_l2 & ~ATTR_MASK))) {
if (l2pg != NULL)
l2pg->ref_count--;
CTR2(KTR_PMAP, "pmap_enter_l2: failure for va %#lx"
@@ -4297,7 +4320,7 @@ pmap_enter_l2(pmap_t pmap, vm_offset_t va, pd_entry_t new_l2, u_int flags,
else
pmap_remove_l3_range(pmap, old_l2, va, va + L2_SIZE,
&free, lockp);
- if (va < VM_MAXUSER_ADDRESS) {
+ if (!ADDR_IS_KERNEL(va)) {
vm_page_free_pages_toq(&free, true);
KASSERT(pmap_load(l2) == 0,
("pmap_enter_l2: non-zero L2 entry %p", l2));
@@ -4436,13 +4459,15 @@ pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m,
("pmap_enter_quick_locked: managed mapping within the clean submap"));
PMAP_LOCK_ASSERT(pmap, MA_OWNED);
PMAP_ASSERT_STAGE1(pmap);
+ KASSERT(ADDR_IS_CANONICAL(va),
+ ("%s: Address not in canonical form: %lx", __func__, va));
CTR2(KTR_PMAP, "pmap_enter_quick_locked: %p %lx", pmap, va);
/*
* In the case that a page table page is not
* resident, we are creating it here.
*/
- if (va < VM_MAXUSER_ADDRESS) {
+ if (!ADDR_IS_KERNEL(va)) {
vm_pindex_t l2pindex;
/*
@@ -4526,7 +4551,7 @@ pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m,
if ((prot & VM_PROT_EXECUTE) == 0 ||
m->md.pv_memattr == VM_MEMATTR_DEVICE)
l3_val |= ATTR_S1_XN;
- if (va < VM_MAXUSER_ADDRESS)
+ if (!ADDR_IS_KERNEL(va))
l3_val |= ATTR_S1_AP(ATTR_S1_AP_USER) | ATTR_S1_PXN;
else
l3_val |= ATTR_S1_UXN;
@@ -6273,6 +6298,9 @@ pmap_demote_l2_locked(pmap_t pmap, pt_entry_t *l2, vm_offset_t va,
PMAP_LOCK_ASSERT(pmap, MA_OWNED);
PMAP_ASSERT_STAGE1(pmap);
+ KASSERT(ADDR_IS_CANONICAL(va),
+ ("%s: Address not in canonical form: %lx", __func__, va));
+
l3 = NULL;
oldl2 = pmap_load(l2);
KASSERT((oldl2 & ATTR_DESCR_MASK) == L2_BLOCK,
@@ -6312,7 +6340,7 @@ pmap_demote_l2_locked(pmap_t pmap, pt_entry_t *l2, vm_offset_t va,
* so the direct map region is the only part of the
* kernel address space that must be handled here.
*/
- KASSERT(va < VM_MAXUSER_ADDRESS || VIRT_IN_DMAP(va),
+ KASSERT(!ADDR_IS_KERNEL(va) || VIRT_IN_DMAP(va),
("pmap_demote_l2: No saved mpte for va %#lx", va));
/*
@@ -6337,7 +6365,7 @@ pmap_demote_l2_locked(pmap_t pmap, pt_entry_t *l2, vm_offset_t va,
goto fail;
}
- if (va < VM_MAXUSER_ADDRESS) {
+ if (!ADDR_IS_KERNEL(va)) {
ml3->ref_count = NL3PG;
pmap_resident_count_inc(pmap, 1);
}
@@ -6700,7 +6728,10 @@ pmap_sync_icache(pmap_t pmap, vm_offset_t va, vm_size_t sz)
{
PMAP_ASSERT_STAGE1(pmap);
- if (va >= VM_MIN_KERNEL_ADDRESS) {
+ KASSERT(ADDR_IS_CANONICAL(va),
+ ("%s: Address not in canonical form: %lx", __func__, va));
+
+ if (ADDR_IS_KERNEL(va)) {
cpu_icache_sync_range(va, sz);
} else {
u_int len, offset;
diff --git a/sys/arm64/arm64/trap.c b/sys/arm64/arm64/trap.c
index fa33f713a4fc..2c95239ec1c5 100644
--- a/sys/arm64/arm64/trap.c
+++ b/sys/arm64/arm64/trap.c
@@ -261,8 +261,14 @@ data_abort(struct thread *td, struct trapframe *frame, uint64_t esr,
else {
intr_enable();
+ /* We received a TBI/PAC/etc. fault from the kernel */
+ if (!ADDR_IS_CANONICAL(far)) {
+ error = KERN_INVALID_ADDRESS;
+ goto bad_far;
+ }
+
/* The top bit tells us which range to use */
- if (far >= VM_MAXUSER_ADDRESS) {
+ if (ADDR_IS_KERNEL(far)) {
map = kernel_map;
} else {
map = &p->p_vmspace->vm_map;
@@ -306,6 +312,7 @@ data_abort(struct thread *td, struct trapframe *frame, uint64_t esr,
/* Fault in the page. */
error = vm_fault_trap(map, far, ftype, VM_FAULT_NORMAL, &sig, &ucode);
if (error != KERN_SUCCESS) {
+bad_far:
if (lower) {
call_trapsignal(td, sig, ucode, (void *)far,
ESR_ELx_EXCEPTION(esr));
diff --git a/sys/arm64/include/vmparam.h b/sys/arm64/include/vmparam.h
index a42c68d52887..170fef61b70d 100644
--- a/sys/arm64/include/vmparam.h
+++ b/sys/arm64/include/vmparam.h
@@ -156,6 +156,13 @@
#define VM_MIN_KERNEL_ADDRESS (0xffff000000000000UL)
#define VM_MAX_KERNEL_ADDRESS (0xffff008000000000UL)
+/* If true addr is in the kernel address space */
+#define ADDR_IS_KERNEL(addr) (((addr) & (1ul << 55)) == (1ul << 55))
+/* If true addr is in its canonical form (i.e. no TBI, PAC, etc.) */
+#define ADDR_IS_CANONICAL(addr) \
+ (((addr) & 0xffff000000000000UL) == 0 || \
+ ((addr) & 0xffff000000000000UL) == 0xffff000000000000UL)
+
/* 95 TiB maximum for the direct map region */
#define DMAP_MIN_ADDRESS (0xffffa00000000000UL)
#define DMAP_MAX_ADDRESS (0xffffff0000000000UL)
More information about the dev-commits-src-branches
mailing list