git: 5b2f5e9999a9 - main - bhyve: Do not abort when the VM is destroyed during an exit

From: Hayzam Sherif <hayzam_at_FreeBSD.org>
Date: Tue, 17 Nov 2026 19:03:40 UTC
The branch main has been updated by hayzam:

URL: https://cgit.FreeBSD.org/src/commit/?id=5b2f5e9999a97582d5ec6308d14dfc70e66c56d1

commit 5b2f5e9999a97582d5ec6308d14dfc70e66c56d1
Author:     Wanpeng Qian <wanpengqian@gmail.com>
AuthorDate: 2026-11-17 18:22:31 +0000
Commit:     Hayzam Sherif <hayzam@FreeBSD.org>
CommitDate: 2026-11-17 18:22:31 +0000

    bhyve: Do not abort when the VM is destroyed during an exit
    
    When a VM is destroyed (bhyvectl --destroy) while a vCPU thread is
    handling an exit in bhyve, every call on the VM fails.  vm_loop()
    handles that for vm_run() and ends bhyve with BHYVE_EXIT_ERROR, but a
    vCPU in the middle of an exit hits one of the assert(error == 0) after
    a register update or an exception injection (emulate_inout(),
    vmexit_rdmsr(), vm_inject_fault()), or reports "Failed to emulate
    instruction" and returns VMEXIT_ABORT: bhyve dies of SIGABRT and
    leaves a core.  With a guest doing a lot of port I/O or MSR accesses
    that bhyve emulates, a forced stop hits this regularly.
    
    Add vm_check_destroyed(), which exits with BHYVE_EXIT_ERROR, as
    vm_loop() does, if the VM is gone.  Call it before those asserts and
    on the instruction emulation failure paths on amd64, arm64 and
    riscv.  A failure on a VM that still exists asserts or aborts as
    before.
    
    Signed-off-by:  Wanpeng Qian <wanpengqian@gmail.com>
    Sponsored by:   keelos.dev
    Reviewed by:    hayzam
    Approved by:    markj (mentor)
    Differential Revision:  https://reviews.freebsd.org/D61376
---
 usr.sbin/bhyve/aarch64/vmexit.c |  1 +
 usr.sbin/bhyve/amd64/inout.c    |  8 ++++++++
 usr.sbin/bhyve/amd64/vmexit.c   |  8 ++++++++
 usr.sbin/bhyve/bhyverun.c       | 18 ++++++++++++++++++
 usr.sbin/bhyve/bhyverun.h       |  1 +
 usr.sbin/bhyve/riscv/vmexit.c   |  1 +
 6 files changed, 37 insertions(+)

diff --git a/usr.sbin/bhyve/aarch64/vmexit.c b/usr.sbin/bhyve/aarch64/vmexit.c
index cab3125b9c52..3ca51a1e9e7c 100644
--- a/usr.sbin/bhyve/aarch64/vmexit.c
+++ b/usr.sbin/bhyve/aarch64/vmexit.c
@@ -81,6 +81,7 @@ vmexit_inst_emul(struct vmctx *ctx __unused, struct vcpu *vcpu,
 	return (VMEXIT_CONTINUE);
 
 fail:
+	vm_check_destroyed(vcpu);
 	fprintf(stderr, "Failed to emulate instruction ");
 	FPRINTLN(stderr, "at 0x%lx", vme->pc);
 	return (VMEXIT_ABORT);
diff --git a/usr.sbin/bhyve/amd64/inout.c b/usr.sbin/bhyve/amd64/inout.c
index 478678aae1fd..64f36093ba91 100644
--- a/usr.sbin/bhyve/amd64/inout.c
+++ b/usr.sbin/bhyve/amd64/inout.c
@@ -199,6 +199,8 @@ emulate_inout(struct vmctx *ctx, struct vcpu *vcpu, struct vm_exit *vmexit)
 
 		/* Update index register */
 		error = vie_update_register(vcpu, idxreg, index, addrsize);
+		if (error != 0)
+			vm_check_destroyed(vcpu);
 		assert(error == 0);
 
 		/*
@@ -208,12 +210,16 @@ emulate_inout(struct vmctx *ctx, struct vcpu *vcpu, struct vm_exit *vmexit)
 		if (rep) {
 			error = vie_update_register(vcpu, VM_REG_GUEST_RCX,
 			    count, addrsize);
+			if (error != 0)
+				vm_check_destroyed(vcpu);
 			assert(error == 0);
 		}
 
 		/* Restart the instruction if more iterations remain */
 		if (retval == 0 && count != 0) {
 			error = vm_restart_instruction(vcpu);
+			if (error != 0)
+				vm_check_destroyed(vcpu);
 			assert(error == 0);
 		}
 	} else {
@@ -225,6 +231,8 @@ emulate_inout(struct vmctx *ctx, struct vcpu *vcpu, struct vm_exit *vmexit)
 			eax |= val & vie_size2mask(bytes);
 			error = vm_set_register(vcpu, VM_REG_GUEST_RAX,
 			    eax);
+			if (error != 0)
+				vm_check_destroyed(vcpu);
 			assert(error == 0);
 		}
 	}
diff --git a/usr.sbin/bhyve/amd64/vmexit.c b/usr.sbin/bhyve/amd64/vmexit.c
index fb64f9bbe1a8..40a6a1781c89 100644
--- a/usr.sbin/bhyve/amd64/vmexit.c
+++ b/usr.sbin/bhyve/amd64/vmexit.c
@@ -66,6 +66,8 @@ vm_inject_fault(struct vcpu *vcpu, int vector, int errcode_valid,
 
 	error = vm_inject_exception(vcpu, vector, errcode_valid, errcode,
 	    restart_instruction);
+	if (error != 0)
+		vm_check_destroyed(vcpu);
 	assert(error == 0);
 }
 
@@ -83,6 +85,7 @@ vmexit_inout(struct vmctx *ctx, struct vcpu *vcpu, struct vm_run *vmrun)
 
 	error = emulate_inout(ctx, vcpu, vme);
 	if (error) {
+		vm_check_destroyed(vcpu);
 		EPRINTLN("Unhandled %s%c 0x%04x at 0x%lx",
 		    in ? "in" : "out",
 		    bytes == 1 ? 'b' : (bytes == 2 ? 'w' : 'l'),
@@ -120,10 +123,14 @@ vmexit_rdmsr(struct vmctx *ctx __unused, struct vcpu *vcpu,
 
 	eax = val;
 	error = vm_set_register(vcpu, VM_REG_GUEST_RAX, eax);
+	if (error != 0)
+		vm_check_destroyed(vcpu);
 	assert(error == 0);
 
 	edx = val >> 32;
 	error = vm_set_register(vcpu, VM_REG_GUEST_RDX, edx);
+	if (error != 0)
+		vm_check_destroyed(vcpu);
 	assert(error == 0);
 
 	return (VMEXIT_CONTINUE);
@@ -387,6 +394,7 @@ vmexit_inst_emul(struct vmctx *ctx __unused, struct vcpu *vcpu,
 	return (VMEXIT_CONTINUE);
 
 fail:
+	vm_check_destroyed(vcpu);
 	fprintf(stderr, "Failed to emulate instruction sequence [ ");
 	for (i = 0; i < vie->num_valid; i++)
 		fprintf(stderr, "%02x", vie->inst[i]);
diff --git a/usr.sbin/bhyve/bhyverun.c b/usr.sbin/bhyve/bhyverun.c
index eec5bf2fb629..0f52af6ef601 100644
--- a/usr.sbin/bhyve/bhyverun.c
+++ b/usr.sbin/bhyve/bhyverun.c
@@ -724,6 +724,24 @@ vm_loop(struct vmctx *ctx, struct vcpu *vcpu)
 	EPRINTLN("vm_run error %d, errno %d", error, errno);
 }
 
+/*
+ * A vCPU thread handling an exit finds that a call on the VM failed.  If the
+ * VM was destroyed under it (bhyvectl --destroy), every call fails: ENXIO
+ * while the destroy waits for the vCPU threads, ENOTTY once the device is
+ * gone.  End as vm_loop() does when vm_run() finds the VM destroyed, instead
+ * of asserting or reporting an emulation failure and aborting.  On a VM that
+ * still exists this returns and the caller handles the failure as before.
+ */
+void
+vm_check_destroyed(struct vcpu *vcpu)
+{
+	int tmp;
+
+	if (vm_get_capability(vcpu, VM_CAP_UNRESTRICTED_GUEST, &tmp) != 0 &&
+	    (errno == ENXIO || errno == ENOTTY))
+		exit(BHYVE_EXIT_ERROR);
+}
+
 static int
 num_vcpus_allowed(struct vmctx *ctx, struct vcpu *vcpu)
 {
diff --git a/usr.sbin/bhyve/bhyverun.h b/usr.sbin/bhyve/bhyverun.h
index a6fddd9f11f8..0ad168472b0a 100644
--- a/usr.sbin/bhyve/bhyverun.h
+++ b/usr.sbin/bhyve/bhyverun.h
@@ -87,6 +87,7 @@ int bhyve_numa_parse(const char *opt);
 void bhyve_cfg_warn(const char *old, const char *new);
 
 void bhyve_init_vcpu(struct vcpu *vcpu);
+void vm_check_destroyed(struct vcpu *vcpu);
 void bhyve_start_vcpu(struct vcpu *vcpu, bool bsp);
 int bhyve_init_platform(struct vmctx *ctx, struct vcpu *bsp);
 int bhyve_init_platform_late(struct vmctx *ctx, struct vcpu *bsp);
diff --git a/usr.sbin/bhyve/riscv/vmexit.c b/usr.sbin/bhyve/riscv/vmexit.c
index 29afdcc25352..452f4aad646c 100644
--- a/usr.sbin/bhyve/riscv/vmexit.c
+++ b/usr.sbin/bhyve/riscv/vmexit.c
@@ -95,6 +95,7 @@ vmexit_inst_emul(struct vmctx *ctx __unused, struct vcpu *vcpu,
 	return (VMEXIT_CONTINUE);
 
 fail:
+	vm_check_destroyed(vcpu);
 	fprintf(stderr, "Failed to emulate instruction ");
 	FPRINTLN(stderr, "at 0x%lx", vme->pc);
 	return (VMEXIT_ABORT);