git: 5b2f5e9999a9 - main - bhyve: Do not abort when the VM is destroyed during an exit
- Go to: [ bottom of page ] [ top of archives ] [ this month ]
Date: Tue, 17 Nov 2026 19:03:40 UTC
The branch main has been updated by hayzam:
URL: https://cgit.FreeBSD.org/src/commit/?id=5b2f5e9999a97582d5ec6308d14dfc70e66c56d1
commit 5b2f5e9999a97582d5ec6308d14dfc70e66c56d1
Author: Wanpeng Qian <wanpengqian@gmail.com>
AuthorDate: 2026-11-17 18:22:31 +0000
Commit: Hayzam Sherif <hayzam@FreeBSD.org>
CommitDate: 2026-11-17 18:22:31 +0000
bhyve: Do not abort when the VM is destroyed during an exit
When a VM is destroyed (bhyvectl --destroy) while a vCPU thread is
handling an exit in bhyve, every call on the VM fails. vm_loop()
handles that for vm_run() and ends bhyve with BHYVE_EXIT_ERROR, but a
vCPU in the middle of an exit hits one of the assert(error == 0) after
a register update or an exception injection (emulate_inout(),
vmexit_rdmsr(), vm_inject_fault()), or reports "Failed to emulate
instruction" and returns VMEXIT_ABORT: bhyve dies of SIGABRT and
leaves a core. With a guest doing a lot of port I/O or MSR accesses
that bhyve emulates, a forced stop hits this regularly.
Add vm_check_destroyed(), which exits with BHYVE_EXIT_ERROR, as
vm_loop() does, if the VM is gone. Call it before those asserts and
on the instruction emulation failure paths on amd64, arm64 and
riscv. A failure on a VM that still exists asserts or aborts as
before.
Signed-off-by: Wanpeng Qian <wanpengqian@gmail.com>
Sponsored by: keelos.dev
Reviewed by: hayzam
Approved by: markj (mentor)
Differential Revision: https://reviews.freebsd.org/D61376
---
usr.sbin/bhyve/aarch64/vmexit.c | 1 +
usr.sbin/bhyve/amd64/inout.c | 8 ++++++++
usr.sbin/bhyve/amd64/vmexit.c | 8 ++++++++
usr.sbin/bhyve/bhyverun.c | 18 ++++++++++++++++++
usr.sbin/bhyve/bhyverun.h | 1 +
usr.sbin/bhyve/riscv/vmexit.c | 1 +
6 files changed, 37 insertions(+)
diff --git a/usr.sbin/bhyve/aarch64/vmexit.c b/usr.sbin/bhyve/aarch64/vmexit.c
index cab3125b9c52..3ca51a1e9e7c 100644
--- a/usr.sbin/bhyve/aarch64/vmexit.c
+++ b/usr.sbin/bhyve/aarch64/vmexit.c
@@ -81,6 +81,7 @@ vmexit_inst_emul(struct vmctx *ctx __unused, struct vcpu *vcpu,
return (VMEXIT_CONTINUE);
fail:
+ vm_check_destroyed(vcpu);
fprintf(stderr, "Failed to emulate instruction ");
FPRINTLN(stderr, "at 0x%lx", vme->pc);
return (VMEXIT_ABORT);
diff --git a/usr.sbin/bhyve/amd64/inout.c b/usr.sbin/bhyve/amd64/inout.c
index 478678aae1fd..64f36093ba91 100644
--- a/usr.sbin/bhyve/amd64/inout.c
+++ b/usr.sbin/bhyve/amd64/inout.c
@@ -199,6 +199,8 @@ emulate_inout(struct vmctx *ctx, struct vcpu *vcpu, struct vm_exit *vmexit)
/* Update index register */
error = vie_update_register(vcpu, idxreg, index, addrsize);
+ if (error != 0)
+ vm_check_destroyed(vcpu);
assert(error == 0);
/*
@@ -208,12 +210,16 @@ emulate_inout(struct vmctx *ctx, struct vcpu *vcpu, struct vm_exit *vmexit)
if (rep) {
error = vie_update_register(vcpu, VM_REG_GUEST_RCX,
count, addrsize);
+ if (error != 0)
+ vm_check_destroyed(vcpu);
assert(error == 0);
}
/* Restart the instruction if more iterations remain */
if (retval == 0 && count != 0) {
error = vm_restart_instruction(vcpu);
+ if (error != 0)
+ vm_check_destroyed(vcpu);
assert(error == 0);
}
} else {
@@ -225,6 +231,8 @@ emulate_inout(struct vmctx *ctx, struct vcpu *vcpu, struct vm_exit *vmexit)
eax |= val & vie_size2mask(bytes);
error = vm_set_register(vcpu, VM_REG_GUEST_RAX,
eax);
+ if (error != 0)
+ vm_check_destroyed(vcpu);
assert(error == 0);
}
}
diff --git a/usr.sbin/bhyve/amd64/vmexit.c b/usr.sbin/bhyve/amd64/vmexit.c
index fb64f9bbe1a8..40a6a1781c89 100644
--- a/usr.sbin/bhyve/amd64/vmexit.c
+++ b/usr.sbin/bhyve/amd64/vmexit.c
@@ -66,6 +66,8 @@ vm_inject_fault(struct vcpu *vcpu, int vector, int errcode_valid,
error = vm_inject_exception(vcpu, vector, errcode_valid, errcode,
restart_instruction);
+ if (error != 0)
+ vm_check_destroyed(vcpu);
assert(error == 0);
}
@@ -83,6 +85,7 @@ vmexit_inout(struct vmctx *ctx, struct vcpu *vcpu, struct vm_run *vmrun)
error = emulate_inout(ctx, vcpu, vme);
if (error) {
+ vm_check_destroyed(vcpu);
EPRINTLN("Unhandled %s%c 0x%04x at 0x%lx",
in ? "in" : "out",
bytes == 1 ? 'b' : (bytes == 2 ? 'w' : 'l'),
@@ -120,10 +123,14 @@ vmexit_rdmsr(struct vmctx *ctx __unused, struct vcpu *vcpu,
eax = val;
error = vm_set_register(vcpu, VM_REG_GUEST_RAX, eax);
+ if (error != 0)
+ vm_check_destroyed(vcpu);
assert(error == 0);
edx = val >> 32;
error = vm_set_register(vcpu, VM_REG_GUEST_RDX, edx);
+ if (error != 0)
+ vm_check_destroyed(vcpu);
assert(error == 0);
return (VMEXIT_CONTINUE);
@@ -387,6 +394,7 @@ vmexit_inst_emul(struct vmctx *ctx __unused, struct vcpu *vcpu,
return (VMEXIT_CONTINUE);
fail:
+ vm_check_destroyed(vcpu);
fprintf(stderr, "Failed to emulate instruction sequence [ ");
for (i = 0; i < vie->num_valid; i++)
fprintf(stderr, "%02x", vie->inst[i]);
diff --git a/usr.sbin/bhyve/bhyverun.c b/usr.sbin/bhyve/bhyverun.c
index eec5bf2fb629..0f52af6ef601 100644
--- a/usr.sbin/bhyve/bhyverun.c
+++ b/usr.sbin/bhyve/bhyverun.c
@@ -724,6 +724,24 @@ vm_loop(struct vmctx *ctx, struct vcpu *vcpu)
EPRINTLN("vm_run error %d, errno %d", error, errno);
}
+/*
+ * A vCPU thread handling an exit finds that a call on the VM failed. If the
+ * VM was destroyed under it (bhyvectl --destroy), every call fails: ENXIO
+ * while the destroy waits for the vCPU threads, ENOTTY once the device is
+ * gone. End as vm_loop() does when vm_run() finds the VM destroyed, instead
+ * of asserting or reporting an emulation failure and aborting. On a VM that
+ * still exists this returns and the caller handles the failure as before.
+ */
+void
+vm_check_destroyed(struct vcpu *vcpu)
+{
+ int tmp;
+
+ if (vm_get_capability(vcpu, VM_CAP_UNRESTRICTED_GUEST, &tmp) != 0 &&
+ (errno == ENXIO || errno == ENOTTY))
+ exit(BHYVE_EXIT_ERROR);
+}
+
static int
num_vcpus_allowed(struct vmctx *ctx, struct vcpu *vcpu)
{
diff --git a/usr.sbin/bhyve/bhyverun.h b/usr.sbin/bhyve/bhyverun.h
index a6fddd9f11f8..0ad168472b0a 100644
--- a/usr.sbin/bhyve/bhyverun.h
+++ b/usr.sbin/bhyve/bhyverun.h
@@ -87,6 +87,7 @@ int bhyve_numa_parse(const char *opt);
void bhyve_cfg_warn(const char *old, const char *new);
void bhyve_init_vcpu(struct vcpu *vcpu);
+void vm_check_destroyed(struct vcpu *vcpu);
void bhyve_start_vcpu(struct vcpu *vcpu, bool bsp);
int bhyve_init_platform(struct vmctx *ctx, struct vcpu *bsp);
int bhyve_init_platform_late(struct vmctx *ctx, struct vcpu *bsp);
diff --git a/usr.sbin/bhyve/riscv/vmexit.c b/usr.sbin/bhyve/riscv/vmexit.c
index 29afdcc25352..452f4aad646c 100644
--- a/usr.sbin/bhyve/riscv/vmexit.c
+++ b/usr.sbin/bhyve/riscv/vmexit.c
@@ -95,6 +95,7 @@ vmexit_inst_emul(struct vmctx *ctx __unused, struct vcpu *vcpu,
return (VMEXIT_CONTINUE);
fail:
+ vm_check_destroyed(vcpu);
fprintf(stderr, "Failed to emulate instruction ");
FPRINTLN(stderr, "at 0x%lx", vme->pc);
return (VMEXIT_ABORT);