git: 5d3ee23b91f3 - stable/13 - vmm: don't lock a mtx in the icr_low write handler

The branch stable/13 has been updated by jhb:

URL: https://cgit.FreeBSD.org/src/commit/?id=5d3ee23b91f354f6dea2707cdf4a0fb813488106

commit 5d3ee23b91f354f6dea2707cdf4a0fb813488106
Author:     Corvin Köhne <corvink@FreeBSD.org>
AuthorDate: 2022-11-21 14:00:04 +0000
Commit:     John Baldwin <jhb@FreeBSD.org>
CommitDate: 2023-01-26 22:06:03 +0000

    vmm: don't lock a mtx in the icr_low write handler
    
    x2apic accesses are handled by a wrmsr exit. This handler is called in a
    critical section. So, we can't lock a mtx in the icr_low handler.
    
    Reported by:            kp, pho
    Tested by:              kp, pho
    Approved by:            manu (mentor)
    Fixes:                  c0f35dbf19c3c8825bd2b321d8efd582807d1940 vmm: Use a cpuset_t for vCPUs waiting for STARTUP IPIs.
    MFC after:              1 week
    MFC with:               c0f35dbf19c3c8825bd2b321d8efd582807d1940
    Sponsored by:           Beckhoff Automation GmbH & Co. KG
    Differential Revision:  https://reviews.freebsd.org/D37452
    
    (cherry picked from commit 7c326ab5bb9aced8dcbc2465ac1c9ff8df2ba46b)
---
 sys/amd64/vmm/io/vlapic.c | 61 +++++++++++++++++++++++++++--------------------
 1 file changed, 35 insertions(+), 26 deletions(-)

diff --git a/sys/amd64/vmm/io/vlapic.c b/sys/amd64/vmm/io/vlapic.c
index e13cdcc63d57..fd716fcc0b61 100644
--- a/sys/amd64/vmm/io/vlapic.c
+++ b/sys/amd64/vmm/io/vlapic.c
@@ -1127,9 +1127,8 @@ vlapic_icrlo_write_handler(struct vlapic *vlapic, bool *retu)
 			    i == vlapic->vcpuid)
 				break;
 
-			/* vCPU i is waiting for SIPI. */
-			CPU_SETOF(i, &dmask);
-			vm_await_start(vlapic->vm, &dmask);
+			CPU_SETOF(i, &ipimask);
+
 			break;
 		}
 
@@ -1140,36 +1139,17 @@ vlapic_icrlo_write_handler(struct vlapic *vlapic, bool *retu)
 			if (!phys)
 				break;
 
-			/*
-			 * Old bhyve versions don't support the IPI
-			 * exit. Translate it into the old style.
-			 */
 			i = vm_apicid2vcpuid(vlapic->vm, dest);
 			if (i >= vm_get_maxcpus(vlapic->vm) ||
 			    i == vlapic->vcpuid)
 				break;
 
-			/*
-			 * Ignore SIPIs in any state other than wait-for-SIPI
-			 */
-			CPU_SETOF(i, &dmask);
-			dmask = vm_start_cpus(vlapic->vm, &dmask);
-			if (CPU_EMPTY(&dmask))
-				break;
-
-			vmexit = vm_exitinfo(vlapic->vcpu);
-			vmexit->exitcode = VM_EXITCODE_SPINUP_AP;
-			vmexit->u.spinup_ap.vcpu = i;
-			vmexit->u.spinup_ap.rip = vec << PAGE_SHIFT;
+			CPU_SETOF(i, &ipimask);
 
-			*retu = true;
 			break;
 		}
 
-		/*
-		 * Ignore SIPIs in any state other than wait-for-SIPI
-		 */
-		ipimask = vm_start_cpus(vlapic->vm, &dmask);
+		CPU_COPY(&dmask, &ipimask);
 		break;
 	default:
 		return (1);
@@ -1199,14 +1179,43 @@ vlapic_handle_init(struct vcpu *vcpu, void *arg)
 int
 vm_handle_ipi(struct vcpu *vcpu, struct vm_exit *vme, bool *retu)
 {
+	struct vlapic *vlapic = vm_lapic(vcpu);
+	cpuset_t *dmask = &vme->u.ipi.dmask;
+	uint8_t vec = vme->u.ipi.vector;
+
 	*retu = true;
 	switch (vme->u.ipi.mode) {
 	case APIC_DELMODE_INIT:
-		vm_smp_rendezvous(vcpu, vme->u.ipi.dmask, vlapic_handle_init,
+		vm_smp_rendezvous(vcpu, *dmask, vlapic_handle_init,
 		    NULL);
-		vm_await_start(vcpu_vm(vcpu), &vme->u.ipi.dmask);
+		vm_await_start(vcpu_vm(vcpu), dmask);
+
+		if (!vlapic->ipi_exit) {
+			*retu = false;
+		}
+
 		break;
 	case APIC_DELMODE_STARTUP:
+		/*
+		 * Ignore SIPIs in any state other than wait-for-SIPI
+		 */
+		*dmask = vm_start_cpus(vcpu_vm(vcpu), dmask);
+
+		if (CPU_EMPTY(dmask)) {
+			*retu = false;
+			break;
+		}
+
+		/*
+		 * Old bhyve versions don't support the IPI
+		 * exit. Translate it into the old style.
+		 */
+		if (!vlapic->ipi_exit) {
+			vme->exitcode = VM_EXITCODE_SPINUP_AP;
+			vme->u.spinup_ap.vcpu = CPU_FFS(dmask);
+			vme->u.spinup_ap.rip = vec << PAGE_SHIFT;
+		}
+
 		break;
 	default:
 		return (1);