git: 5e8827063b9f - releng/13.2 - vmm: Fix AP startup compatibility for old bhyve executables

The branch releng/13.2 has been updated by markj:

URL: https://cgit.FreeBSD.org/src/commit/?id=5e8827063b9f565697135b33399a64ca9f000a02

commit 5e8827063b9f565697135b33399a64ca9f000a02
Author:     Mark Johnston <markj@FreeBSD.org>
AuthorDate: 2023-02-09 20:52:35 +0000
Commit:     Mark Johnston <markj@FreeBSD.org>
CommitDate: 2023-02-23 17:07:47 +0000

    vmm: Fix AP startup compatibility for old bhyve executables
    
    These changes unbreak AP startup when using a 13.1-RELEASE bhyve
    executable with a newer kernel:
    - Correct the destination mask for the VM_EXITCODE_IPI message generated
      by an INIT or STARTUP IPI in vlapic_icrlo_write_handler().
    - Only initialize vlapics on active vCPUs.  13.1-RELEASE bhyve activates
      AP vCPUs only after the BSP starts them with an IPI, and vmm now
      allocates vcpu structures lazily, so the STARTUP handling in
      vm_handle_ipi() could trigger a page fault.
    - Fix an off-by-one setting the vcpuid in a VM_EXITCODE_SPINUP_AP
      message.
    
    Approved by:    re (cperciva)
    Fixes:  7c326ab5bb9a ("vmm: don't lock a mtx in the icr_low write handler")
    Reviewed by:    jhb, corvink
    MFC after:      2 weeks
    Sponsored by:   The FreeBSD Foundation
    Differential Revision:  https://reviews.freebsd.org/D38446
    
    (cherry picked from commit b265a2e0d76422f4007e96dd7295ed0aeb846e2d)
    (cherry picked from commit 577a666c3c33def1d4b996575dc43f8567fdac5c)
---
 sys/amd64/vmm/io/vlapic.c | 22 ++++++++++++++--------
 1 file changed, 14 insertions(+), 8 deletions(-)

diff --git a/sys/amd64/vmm/io/vlapic.c b/sys/amd64/vmm/io/vlapic.c
index 44641cc29035..f7370e440d66 100644
--- a/sys/amd64/vmm/io/vlapic.c
+++ b/sys/amd64/vmm/io/vlapic.c
@@ -1160,7 +1160,7 @@ vlapic_icrlo_write_handler(struct vlapic *vlapic, bool *retu)
 		vmexit->exitcode = VM_EXITCODE_IPI;
 		vmexit->u.ipi.mode = mode;
 		vmexit->u.ipi.vector = vec;
-		vmexit->u.ipi.dmask = dmask;
+		vmexit->u.ipi.dmask = ipimask;
 
 		*retu = true;
 	}
@@ -1185,16 +1185,22 @@ vm_handle_ipi(struct vcpu *vcpu, struct vm_exit *vme, bool *retu)
 
 	*retu = true;
 	switch (vme->u.ipi.mode) {
-	case APIC_DELMODE_INIT:
-		vm_smp_rendezvous(vcpu, *dmask, vlapic_handle_init,
-		    NULL);
+	case APIC_DELMODE_INIT: {
+		cpuset_t active, reinit;
+
+		active = vm_active_cpus(vcpu_vm(vcpu));
+		CPU_AND(&reinit, &active, dmask);
+		if (!CPU_EMPTY(&reinit)) {
+			vm_smp_rendezvous(vcpu, reinit, vlapic_handle_init,
+			    NULL);
+		}
 		vm_await_start(vcpu_vm(vcpu), dmask);
 
-		if (!vlapic->ipi_exit) {
+		if (!vlapic->ipi_exit)
 			*retu = false;
-		}
 
 		break;
+	}
 	case APIC_DELMODE_STARTUP:
 		/*
 		 * Ignore SIPIs in any state other than wait-for-SIPI
@@ -1212,13 +1218,13 @@ vm_handle_ipi(struct vcpu *vcpu, struct vm_exit *vme, bool *retu)
 		 */
 		if (!vlapic->ipi_exit) {
 			vme->exitcode = VM_EXITCODE_SPINUP_AP;
-			vme->u.spinup_ap.vcpu = CPU_FFS(dmask);
+			vme->u.spinup_ap.vcpu = CPU_FFS(dmask) - 1;
 			vme->u.spinup_ap.rip = vec << PAGE_SHIFT;
 		}
 
 		break;
 	default:
-		return (1);
+		__assert_unreachable();
 	}
 
 	return (0);