git: e6be6dedeea1 - main - kinst/arm64: Handle an additional PC-relative instruction
- Go to: [ bottom of page ] [ top of archives ] [ this month ]
Date: Wed, 20 May 2026 14:54:06 UTC
The branch main has been updated by markj:
URL: https://cgit.FreeBSD.org/src/commit/?id=e6be6dedeea1e6d2e5206e1e7422e2d556a6da0c
commit e6be6dedeea1e6d2e5206e1e7422e2d556a6da0c
Author: Mark Johnston <markj@FreeBSD.org>
AuthorDate: 2026-05-20 14:49:41 +0000
Commit: Mark Johnston <markj@FreeBSD.org>
CommitDate: 2026-05-20 14:49:41 +0000
kinst/arm64: Handle an additional PC-relative instruction
"ldr <reg>, <literal>" loads a value from a literal memory address into
a register. It's PC-relative and so cannot be directly implemented
using the trampoline mechanism. Unfortunately, on arm64 it can't easily
be emulated either since the return-to-EL1 handler does not restore
callee-saved registers, so like adr/adrp, we simply don't handle it.
These instructions are fairly rare in an arm64 kernel.
While here, refactor the code so that all instruction decoding is done
in one place: introduce an enum type which characterizes the instruction
type, add a helper to map instructions to enum values, and store the
corresponding enum value in the probe description.
Reviewed by: christos
MFC after: 1 week
Differential Revision: https://reviews.freebsd.org/D56988
---
sys/cddl/dev/kinst/aarch64/kinst_isa.c | 121 +++++++++++++++------------------
sys/cddl/dev/kinst/aarch64/kinst_isa.h | 15 +++-
2 files changed, 69 insertions(+), 67 deletions(-)
diff --git a/sys/cddl/dev/kinst/aarch64/kinst_isa.c b/sys/cddl/dev/kinst/aarch64/kinst_isa.c
index 1ccfe20b8dcb..d9a8fd0276f2 100644
--- a/sys/cddl/dev/kinst/aarch64/kinst_isa.c
+++ b/sys/cddl/dev/kinst/aarch64/kinst_isa.c
@@ -18,6 +18,30 @@
DPCPU_DEFINE_STATIC(struct kinst_cpu_state, kinst_state);
+static enum kinst_instr
+kinst_instr_type(kinst_patchval_t instr)
+{
+ if (((instr >> 22) & 0xff) == 0b00100001)
+ return (KINST_INSTR_LDX);
+ else if (((instr >> 22) & 0xff) == 0b00100000)
+ return (KINST_INSTR_STX);
+ if (((instr >> 24) & 0x1f) == 0b10000)
+ return (KINST_INSTR_ADR);
+ else if (((instr >> 26) & 0x3f) == 0b000101)
+ return (KINST_INSTR_B);
+ else if (((instr >> 24) & 0xff) == 0b01010100)
+ return (KINST_INSTR_BCOND);
+ else if (((instr >> 26) & 0x3f) == 0b100101)
+ return (KINST_INSTR_BL);
+ else if (((instr >> 25) & 0x3f) == 0b011010)
+ return (KINST_INSTR_CBZ);
+ else if (((instr >> 25) & 0x3f) == 0b011011)
+ return (KINST_INSTR_TBZ);
+ else if (((instr >> 24) & 0xbf) == 0b11000)
+ return (KINST_INSTR_LDR_LITERAL);
+ return (KINST_INSTR_COMMON);
+}
+
static void
kinst_emulate(struct trapframe *frame, const struct kinst_probe *kp)
{
@@ -26,8 +50,8 @@ kinst_emulate(struct trapframe *frame, const struct kinst_probe *kp)
uint8_t cond, reg, bitpos;
bool res;
- if (((instr >> 24) & 0x1f) == 0b10000) {
- /* adr/adrp */
+ switch (kp->kp_md.kp_type) {
+ case KINST_INSTR_ADR:
reg = instr & 0x1f;
imm = (instr >> 29) & 0x3;
imm |= ((instr >> 5) & 0x0007ffff) << 2;
@@ -44,14 +68,14 @@ kinst_emulate(struct trapframe *frame, const struct kinst_probe *kp)
frame->tf_x[reg] = (frame->tf_elr & ~0xfff) + imm;
}
frame->tf_elr += INSN_SIZE;
- } else if (((instr >> 26) & 0x3f) == 0b000101) {
- /* b */
+ break;
+ case KINST_INSTR_B:
imm = instr & 0x03ffffff;
if (imm & 0x0000000002000000)
imm |= 0xfffffffffe000000;
frame->tf_elr += imm << 2;
- } else if (((instr >> 24) & 0xff) == 0b01010100) {
- /* b.cond */
+ break;
+ case KINST_INSTR_BCOND:
imm = (instr >> 5) & 0x0007ffff;
if (imm & 0x0000000000040000)
imm |= 0xfffffffffffc0000;
@@ -92,15 +116,15 @@ kinst_emulate(struct trapframe *frame, const struct kinst_probe *kp)
frame->tf_elr += imm << 2;
else
frame->tf_elr += INSN_SIZE;
- } else if (((instr >> 26) & 0x3f) == 0b100101) {
- /* bl */
+ break;
+ case KINST_INSTR_BL:
imm = instr & 0x03ffffff;
if (imm & 0x0000000002000000)
imm |= 0xfffffffffe000000;
frame->tf_lr = frame->tf_elr + INSN_SIZE;
frame->tf_elr += imm << 2;
- } else if (((instr >> 25) & 0x3f) == 0b011010) {
- /* cbnz/cbz */
+ break;
+ case KINST_INSTR_CBZ:
cond = (instr >> 24) & 0x1;
reg = instr & 0x1f;
imm = (instr >> 5) & 0x0007ffff;
@@ -114,8 +138,8 @@ kinst_emulate(struct trapframe *frame, const struct kinst_probe *kp)
frame->tf_elr += imm << 2;
else
frame->tf_elr += INSN_SIZE;
- } else if (((instr >> 25) & 0x3f) == 0b011011) {
- /* tbnz/tbz */
+ break;
+ case KINST_INSTR_TBZ:
cond = (instr >> 24) & 0x1;
reg = instr & 0x1f;
bitpos = (instr >> 19) & 0x1f;
@@ -131,6 +155,9 @@ kinst_emulate(struct trapframe *frame, const struct kinst_probe *kp)
frame->tf_elr += imm << 2;
else
frame->tf_elr += INSN_SIZE;
+ break;
+ default:
+ __assert_unreachable();
}
}
@@ -211,7 +238,7 @@ kinst_invop(uintptr_t addr, struct trapframe *frame, uintptr_t scratch)
dtrace_probe(kp->kp_id, 0, 0, 0, 0, 0);
cpu->cpu_dtrace_caller = 0;
- if (kp->kp_md.emulate) {
+ if (kp->kp_md.kp_type != KINST_INSTR_COMMON) {
kinst_emulate(frame, kp);
} else {
ks->state = KINST_PROBE_FIRED;
@@ -245,50 +272,6 @@ kinst_patch_tracepoint(struct kinst_probe *kp, kinst_patchval_t val)
cpu_icache_sync_range(kp->kp_patchpoint, INSN_SIZE);
}
-static void
-kinst_instr_dissect(struct kinst_probe *kp)
-{
- struct kinst_probe_md *kpmd;
- kinst_patchval_t instr = kp->kp_savedval;
-
- kpmd = &kp->kp_md;
- kpmd->emulate = false;
-
- if (((instr >> 24) & 0x1f) == 0b10000)
- kpmd->emulate = true; /* adr/adrp */
- else if (((instr >> 26) & 0x3f) == 0b000101)
- kpmd->emulate = true; /* b */
- else if (((instr >> 24) & 0xff) == 0b01010100)
- kpmd->emulate = true; /* b.cond */
- else if (((instr >> 26) & 0x3f) == 0b100101)
- kpmd->emulate = true; /* bl */
- else if (((instr >> 25) & 0x3f) == 0b011010)
- kpmd->emulate = true; /* cbnz/cbz */
- else if (((instr >> 25) & 0x3f) == 0b011011)
- kpmd->emulate = true; /* tbnz/tbz */
-
- if (!kpmd->emulate)
- kinst_trampoline_populate(kp);
-}
-
-static bool
-kinst_instr_ldx(kinst_patchval_t instr)
-{
- if (((instr >> 22) & 0xff) == 0b00100001)
- return (true);
-
- return (false);
-}
-
-static bool
-kinst_instr_stx(kinst_patchval_t instr)
-{
- if (((instr >> 22) & 0xff) == 0b00100000)
- return (true);
-
- return (false);
-}
-
int
kinst_make_probe(linker_file_t lf, int symindx, linker_symval_t *symval,
void *opaque)
@@ -359,6 +342,8 @@ kinst_make_probe(linker_file_t lf, int symindx, linker_symval_t *symval,
ldxstx_block = false;
for (n = 0; instr < limit; instr++) {
+ enum kinst_instr type;
+
off = (int)((uint8_t *)instr - (uint8_t *)symval->value);
/*
@@ -366,9 +351,10 @@ kinst_make_probe(linker_file_t lf, int symindx, linker_symval_t *symval,
* breakpoint is placed in a LDX/STX block, we violate the
* operation and the loop might fail.
*/
- if (kinst_instr_ldx(*instr))
+ type = kinst_instr_type(*instr);
+ if (type == KINST_INSTR_LDX)
ldxstx_block = true;
- else if (kinst_instr_stx(*instr)) {
+ else if (type == KINST_INSTR_STX) {
ldxstx_block = false;
continue;
}
@@ -376,13 +362,14 @@ kinst_make_probe(linker_file_t lf, int symindx, linker_symval_t *symval,
continue;
/*
- * XXX: Skip ADR and ADRP instructions. The arm64 exception
- * handler has a micro-optimization where it doesn't restore
- * callee-saved registers when returning from exceptions in
- * EL1. This results in a panic when the kinst emulation code
- * modifies one of those registers.
+ * XXX: The arm64 exception handler has a micro-optimization
+ * where it doesn't restore callee-saved registers when
+ * returning from exceptions in EL1. As a result, instruction
+ * emulation doesn't work if a (callee-saved) register is
+ * modified. Hence, exclude the position-dependent ADR/ADRP and
+ * LDR <literal> instructions.
*/
- if (((*instr >> 24) & 0x1f) == 0b10000)
+ if (type == KINST_INSTR_ADR || type == KINST_INSTR_LDR_LITERAL)
continue;
if (pd->kpd_off != -1 && off != pd->kpd_off)
@@ -408,12 +395,14 @@ kinst_make_probe(linker_file_t lf, int symindx, linker_symval_t *symval,
kp->kp_patchpoint = instr;
kp->kp_savedval = *instr;
kp->kp_patchval = KINST_PATCHVAL;
+ kp->kp_md.kp_type = type;
if ((kp->kp_tramp = kinst_trampoline_alloc(M_WAITOK)) == NULL) {
KINST_LOG("cannot allocate trampoline for %p", instr);
return (ENOMEM);
}
+ if (kp->kp_md.kp_type == KINST_INSTR_COMMON)
+ kinst_trampoline_populate(kp);
- kinst_instr_dissect(kp);
kinst_probe_create(kp, lf);
}
if (ldxstx_block)
diff --git a/sys/cddl/dev/kinst/aarch64/kinst_isa.h b/sys/cddl/dev/kinst/aarch64/kinst_isa.h
index 7e1fd8d123e9..39cf6d49290a 100644
--- a/sys/cddl/dev/kinst/aarch64/kinst_isa.h
+++ b/sys/cddl/dev/kinst/aarch64/kinst_isa.h
@@ -19,8 +19,21 @@
typedef uint32_t kinst_patchval_t;
+enum kinst_instr {
+ KINST_INSTR_ADR, /* adr/adrp */
+ KINST_INSTR_B,
+ KINST_INSTR_BCOND,
+ KINST_INSTR_BL,
+ KINST_INSTR_CBZ, /* cbz/cbnz */
+ KINST_INSTR_TBZ, /* tbnz/tbz */
+ KINST_INSTR_LDR_LITERAL,
+ KINST_INSTR_LDX,
+ KINST_INSTR_STX,
+ KINST_INSTR_COMMON,
+};
+
struct kinst_probe_md {
- bool emulate; /* emulate in sw */
+ enum kinst_instr kp_type;
};
#endif /* _KINST_ISA_H_ */