git: f7bce8e1826c - stable/15 - vmm: Suspend the VM before destroying it
- Go to: [ bottom of page ] [ top of archives ] [ this month ]
Date: Tue, 30 Sep 2025 15:23:50 UTC
The branch stable/15 has been updated by markj:
URL: https://cgit.FreeBSD.org/src/commit/?id=f7bce8e1826c362d848d9fb2caffbc80da2c73a3
commit f7bce8e1826c362d848d9fb2caffbc80da2c73a3
Author: Mark Johnston <markj@FreeBSD.org>
AuthorDate: 2025-09-10 16:00:36 +0000
Commit: Mark Johnston <markj@FreeBSD.org>
CommitDate: 2025-09-30 09:43:08 +0000
vmm: Suspend the VM before destroying it
Otherwise we don't do anything to kick vcpu threads out of a sleep
state when destroying a VM. For instance, suppose a guest executes hlt
on amd64 or wfi on arm64 with interrupts disabled. Then,
bhyvectl --destroy will hang until the vcpu thread somehow comes out of
vm_handle_hlt()/vm_handle_wfi() since destroy_dev() is waiting for vCPU
threads to drain.
Note that on amd64, if hw.vmm.halt_detection is set to 1 (the default),
the guest will automatically exit in this case since it's treated as a
shutdown. But, the above should not hang if halt_detection is set to 0.
Here, vm_suspend() wakes up vcpu threads, and a subsequent attempt to
run the vCPU will result in an error which gets propagated to userspace,
allowing destroy_dev() to proceed.
Add a new suspend code for this purpose. Modify bhyve to exit with
status 4 ("exited due to an error") when it's received, since that's
what'll happen generally when the VM is destroyed asynchronously.
Reported by: def
MFC after: 2 weeks
Sponsored by: Innovate UK
Differential Revision: https://reviews.freebsd.org/D51761
(cherry picked from commit 3d39856d4dfeab5b5a5e6bbdb6ce965db5bc4dc1)
---
sys/amd64/include/vmm.h | 1 +
sys/arm64/include/vmm.h | 1 +
sys/arm64/vmm/vmm.c | 6 ++++++
sys/dev/vmm/vmm_dev.c | 1 +
sys/riscv/include/vmm.h | 1 +
sys/riscv/vmm/vmm.c | 6 +++++-
usr.sbin/bhyve/aarch64/vmexit.c | 2 ++
usr.sbin/bhyve/amd64/vmexit.c | 2 ++
usr.sbin/bhyve/bhyve.8 | 2 +-
usr.sbin/bhyve/bhyverun.c | 6 ++----
usr.sbin/bhyve/riscv/vmexit.c | 2 ++
11 files changed, 24 insertions(+), 6 deletions(-)
diff --git a/sys/amd64/include/vmm.h b/sys/amd64/include/vmm.h
index 0b3daed4f69e..e35119af8572 100644
--- a/sys/amd64/include/vmm.h
+++ b/sys/amd64/include/vmm.h
@@ -46,6 +46,7 @@ enum vm_suspend_how {
VM_SUSPEND_POWEROFF,
VM_SUSPEND_HALT,
VM_SUSPEND_TRIPLEFAULT,
+ VM_SUSPEND_DESTROY,
VM_SUSPEND_LAST
};
diff --git a/sys/arm64/include/vmm.h b/sys/arm64/include/vmm.h
index 73b5b4a09591..e839b5dd92c9 100644
--- a/sys/arm64/include/vmm.h
+++ b/sys/arm64/include/vmm.h
@@ -42,6 +42,7 @@ enum vm_suspend_how {
VM_SUSPEND_RESET,
VM_SUSPEND_POWEROFF,
VM_SUSPEND_HALT,
+ VM_SUSPEND_DESTROY,
VM_SUSPEND_LAST
};
diff --git a/sys/arm64/vmm/vmm.c b/sys/arm64/vmm/vmm.c
index 3082d2941221..1dcefa1489e9 100644
--- a/sys/arm64/vmm/vmm.c
+++ b/sys/arm64/vmm/vmm.c
@@ -1342,8 +1342,14 @@ vm_handle_smccc_call(struct vcpu *vcpu, struct vm_exit *vme, bool *retu)
static int
vm_handle_wfi(struct vcpu *vcpu, struct vm_exit *vme, bool *retu)
{
+ struct vm *vm;
+
+ vm = vcpu->vm;
vcpu_lock(vcpu);
while (1) {
+ if (vm->suspend)
+ break;
+
if (vgic_has_pending_irq(vcpu->cookie))
break;
diff --git a/sys/dev/vmm/vmm_dev.c b/sys/dev/vmm/vmm_dev.c
index 9f2b009d02ec..460a508a60dc 100644
--- a/sys/dev/vmm/vmm_dev.c
+++ b/sys/dev/vmm/vmm_dev.c
@@ -901,6 +901,7 @@ vmmdev_lookup_and_destroy(const char *name, struct ucred *cred)
sc->cdev = NULL;
sx_xunlock(&vmmdev_mtx);
+ vm_suspend(sc->vm, VM_SUSPEND_DESTROY);
destroy_dev(cdev);
vmmdev_destroy(sc);
diff --git a/sys/riscv/include/vmm.h b/sys/riscv/include/vmm.h
index 1221521be368..de7119dd534a 100644
--- a/sys/riscv/include/vmm.h
+++ b/sys/riscv/include/vmm.h
@@ -49,6 +49,7 @@ enum vm_suspend_how {
VM_SUSPEND_RESET,
VM_SUSPEND_POWEROFF,
VM_SUSPEND_HALT,
+ VM_SUSPEND_DESTROY,
VM_SUSPEND_LAST
};
diff --git a/sys/riscv/vmm/vmm.c b/sys/riscv/vmm/vmm.c
index 7528ef6e4698..ec4514f70fa6 100644
--- a/sys/riscv/vmm/vmm.c
+++ b/sys/riscv/vmm/vmm.c
@@ -1036,10 +1036,14 @@ vm_raise_msi(struct vm *vm, uint64_t msg, uint64_t addr, int bus, int slot,
static int
vm_handle_wfi(struct vcpu *vcpu, struct vm_exit *vme, bool *retu)
{
+ struct vm *vm;
+ vm = vcpu->vm;
vcpu_lock(vcpu);
-
while (1) {
+ if (vm->suspend)
+ break;
+
if (aplic_check_pending(vcpu->cookie))
break;
diff --git a/usr.sbin/bhyve/aarch64/vmexit.c b/usr.sbin/bhyve/aarch64/vmexit.c
index 3acad4020a3c..2457cbe76b5e 100644
--- a/usr.sbin/bhyve/aarch64/vmexit.c
+++ b/usr.sbin/bhyve/aarch64/vmexit.c
@@ -122,6 +122,8 @@ vmexit_suspend(struct vmctx *ctx, struct vcpu *vcpu, struct vm_run *vmrun)
exit(1);
case VM_SUSPEND_HALT:
exit(2);
+ case VM_SUSPEND_DESTROY:
+ exit(4);
default:
fprintf(stderr, "vmexit_suspend: invalid reason %d\n", how);
exit(100);
diff --git a/usr.sbin/bhyve/amd64/vmexit.c b/usr.sbin/bhyve/amd64/vmexit.c
index 944f5de34645..14f89563fd0f 100644
--- a/usr.sbin/bhyve/amd64/vmexit.c
+++ b/usr.sbin/bhyve/amd64/vmexit.c
@@ -418,6 +418,8 @@ vmexit_suspend(struct vmctx *ctx, struct vcpu *vcpu, struct vm_run *vmrun)
exit(2);
case VM_SUSPEND_TRIPLEFAULT:
exit(3);
+ case VM_SUSPEND_DESTROY:
+ exit(4);
default:
EPRINTLN("vmexit_suspend: invalid reason %d", how);
exit(100);
diff --git a/usr.sbin/bhyve/bhyve.8 b/usr.sbin/bhyve/bhyve.8
index 89c0b23961a8..c902c265da9e 100644
--- a/usr.sbin/bhyve/bhyve.8
+++ b/usr.sbin/bhyve/bhyve.8
@@ -1126,7 +1126,7 @@ powered off
.It 2
halted
.It 3
-triple fault
+triple fault (amd64 only)
.It 4
exited due to an error
.El
diff --git a/usr.sbin/bhyve/bhyverun.c b/usr.sbin/bhyve/bhyverun.c
index 9ead49582a7d..bfc0b949a75d 100644
--- a/usr.sbin/bhyve/bhyverun.c
+++ b/usr.sbin/bhyve/bhyverun.c
@@ -561,10 +561,8 @@ fbsdrun_start_thread(void *param)
#endif
vm_loop(vi->ctx, vi->vcpu);
-
- /* not reached */
- exit(1);
- return (NULL);
+ /* We get here if the VM was destroyed asynchronously. */
+ exit(4);
}
void
diff --git a/usr.sbin/bhyve/riscv/vmexit.c b/usr.sbin/bhyve/riscv/vmexit.c
index 3bc83b3bef4e..985f8e4e9065 100644
--- a/usr.sbin/bhyve/riscv/vmexit.c
+++ b/usr.sbin/bhyve/riscv/vmexit.c
@@ -121,6 +121,8 @@ vmexit_suspend(struct vmctx *ctx, struct vcpu *vcpu, struct vm_run *vmrun)
exit(1);
case VM_SUSPEND_HALT:
exit(2);
+ case VM_SUSPEND_DESTROY:
+ exit(4);
default:
fprintf(stderr, "vmexit_suspend: invalid reason %d\n", how);
exit(100);