git: cd60f1af560a - stable/14 - bhyve: Move vcpu initialization into a MD source file

From: Mark Johnston <markj_at_FreeBSD.org>
Date: Wed, 11 Oct 2023 13:25:20 UTC
The branch stable/14 has been updated by markj:

URL: https://cgit.FreeBSD.org/src/commit/?id=cd60f1af560ae0c63dd309417e0db0660f11fe14

commit cd60f1af560ae0c63dd309417e0db0660f11fe14
Author:     Mark Johnston <markj@FreeBSD.org>
AuthorDate: 2023-10-04 16:27:07 +0000
Commit:     Mark Johnston <markj@FreeBSD.org>
CommitDate: 2023-10-11 13:24:15 +0000

    bhyve: Move vcpu initialization into a MD source file
    
    - Make handling of x86 config options, like x86.x2apic, conditional to
      amd64.
    - Move fbsdrun_set_capabilities() and spinup_vcpu() to a new file,
      bhyverun_machdep.c.  The moved code is all highly x86 specific.
    
    I'm not sure how best to handle the namespace.  I'm using "bhyve_" for
    MD functions called from MI code.  We also have "fbsdrun_" for some MI
    routines that are typically called from MD code.  The file name is
    prefixed by "bhyverun_".
    
    Reviewed by:    corvink
    MFC after:      1 week
    Sponsored by:   Innovate UK
    Differential Revision:  https://reviews.freebsd.org/D40987
    
    (cherry picked from commit e20b74da223d675321618fe6d67858084d152c9e)
---
 usr.sbin/bhyve/Makefile                 |   1 +
 usr.sbin/bhyve/amd64/bhyverun_machdep.c | 125 ++++++++++++++++++++++++++++++++
 usr.sbin/bhyve/bhyverun.c               | 111 ++++------------------------
 usr.sbin/bhyve/bhyverun.h               |  12 ++-
 4 files changed, 152 insertions(+), 97 deletions(-)

diff --git a/usr.sbin/bhyve/Makefile b/usr.sbin/bhyve/Makefile
index 1a8191f9fd3f..de8e87d2ad49 100644
--- a/usr.sbin/bhyve/Makefile
+++ b/usr.sbin/bhyve/Makefile
@@ -21,6 +21,7 @@ SRCS=	\
 	basl.c			\
 	bhyvegc.c		\
 	bhyverun.c		\
+	bhyverun_machdep.c	\
 	block_if.c		\
 	bootrom.c		\
 	config.c		\
diff --git a/usr.sbin/bhyve/amd64/bhyverun_machdep.c b/usr.sbin/bhyve/amd64/bhyverun_machdep.c
new file mode 100644
index 000000000000..f995c7aef195
--- /dev/null
+++ b/usr.sbin/bhyve/amd64/bhyverun_machdep.c
@@ -0,0 +1,125 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2011 NetApp, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <assert.h>
+#include <err.h>
+#include <stdbool.h>
+#include <stdlib.h>
+
+#include <vmmapi.h>
+
+#include "bhyverun.h"
+#include "config.h"
+#include "pci_lpc.h"
+
+void
+bhyve_init_config(void)
+{
+	init_config();
+
+	/* Set default values prior to option parsing. */
+	set_config_bool("acpi_tables", false);
+	set_config_bool("acpi_tables_in_memory", true);
+	set_config_value("memory.size", "256M");
+	set_config_bool("x86.strictmsr", true);
+	set_config_value("lpc.fwcfg", "bhyve");
+}
+
+void
+bhyve_init_vcpu(struct vcpu *vcpu)
+{
+	int err, tmp;
+
+	if (get_config_bool_default("x86.vmexit_on_hlt", false)) {
+		err = vm_get_capability(vcpu, VM_CAP_HALT_EXIT, &tmp);
+		if (err < 0) {
+			fprintf(stderr, "VM exit on HLT not supported\n");
+			exit(4);
+		}
+		vm_set_capability(vcpu, VM_CAP_HALT_EXIT, 1);
+	}
+
+	if (get_config_bool_default("x86.vmexit_on_pause", false)) {
+		/*
+		 * pause exit support required for this mode
+		 */
+		err = vm_get_capability(vcpu, VM_CAP_PAUSE_EXIT, &tmp);
+		if (err < 0) {
+			fprintf(stderr,
+			    "SMP mux requested, no pause support\n");
+			exit(4);
+		}
+		vm_set_capability(vcpu, VM_CAP_PAUSE_EXIT, 1);
+	}
+
+	if (get_config_bool_default("x86.x2apic", false))
+		err = vm_set_x2apic_state(vcpu, X2APIC_ENABLED);
+	else
+		err = vm_set_x2apic_state(vcpu, X2APIC_DISABLED);
+
+	if (err) {
+		fprintf(stderr, "Unable to set x2apic state (%d)\n", err);
+		exit(4);
+	}
+
+	vm_set_capability(vcpu, VM_CAP_ENABLE_INVPCID, 1);
+
+	err = vm_set_capability(vcpu, VM_CAP_IPI_EXIT, 1);
+	assert(err == 0);
+}
+
+void
+bhyve_start_vcpu(struct vcpu *vcpu, bool bsp)
+{
+	int error;
+
+	if (bsp) {
+		if (lpc_bootrom()) {
+			error = vm_set_capability(vcpu,
+			    VM_CAP_UNRESTRICTED_GUEST, 1);
+			if (error != 0) {
+				err(4, "ROM boot failed: unrestricted guest "
+				    "capability not available");
+			}
+			error = vcpu_reset(vcpu);
+			assert(error == 0);
+		}
+	} else {
+		bhyve_init_vcpu(vcpu);
+
+		/*
+		 * Enable the 'unrestricted guest' mode for APs.
+		 *
+		 * APs startup in power-on 16-bit mode.
+		 */
+		error = vm_set_capability(vcpu, VM_CAP_UNRESTRICTED_GUEST, 1);
+		assert(error == 0);
+	}
+
+	fbsdrun_addcpu(vcpu_id(vcpu));
+}
diff --git a/usr.sbin/bhyve/bhyverun.c b/usr.sbin/bhyve/bhyverun.c
index b38c53da63eb..d2bdba2e4a90 100644
--- a/usr.sbin/bhyve/bhyverun.c
+++ b/usr.sbin/bhyve/bhyverun.c
@@ -476,17 +476,20 @@ fbsdrun_start_thread(void *param)
 	return (NULL);
 }
 
-static void
-fbsdrun_addcpu(struct vcpu_info *vi)
+void
+fbsdrun_addcpu(int vcpuid)
 {
+	struct vcpu_info *vi;
 	pthread_t thr;
 	int error;
 
+	vi = &vcpu_info[vcpuid];
+
 	error = vm_activate_cpu(vi->vcpu);
 	if (error != 0)
 		err(EX_OSERR, "could not activate CPU %d", vi->vcpuid);
 
-	CPU_SET_ATOMIC(vi->vcpuid, &cpumask);
+	CPU_SET_ATOMIC(vcpuid, &cpumask);
 
 	vm_suspend_cpu(vi->vcpu);
 
@@ -590,49 +593,6 @@ num_vcpus_allowed(struct vmctx *ctx, struct vcpu *vcpu)
 		return (1);
 }
 
-static void
-fbsdrun_set_capabilities(struct vcpu *vcpu)
-{
-	int err, tmp;
-
-	if (get_config_bool_default("x86.vmexit_on_hlt", false)) {
-		err = vm_get_capability(vcpu, VM_CAP_HALT_EXIT, &tmp);
-		if (err < 0) {
-			fprintf(stderr, "VM exit on HLT not supported\n");
-			exit(4);
-		}
-		vm_set_capability(vcpu, VM_CAP_HALT_EXIT, 1);
-	}
-
-	if (get_config_bool_default("x86.vmexit_on_pause", false)) {
-		/*
-		 * pause exit support required for this mode
-		 */
-		err = vm_get_capability(vcpu, VM_CAP_PAUSE_EXIT, &tmp);
-		if (err < 0) {
-			fprintf(stderr,
-			    "SMP mux requested, no pause support\n");
-			exit(4);
-		}
-		vm_set_capability(vcpu, VM_CAP_PAUSE_EXIT, 1);
-	}
-
-	if (get_config_bool_default("x86.x2apic", false))
-		err = vm_set_x2apic_state(vcpu, X2APIC_ENABLED);
-	else
-		err = vm_set_x2apic_state(vcpu, X2APIC_DISABLED);
-
-	if (err) {
-		fprintf(stderr, "Unable to set x2apic state (%d)\n", err);
-		exit(4);
-	}
-
-	vm_set_capability(vcpu, VM_CAP_ENABLE_INVPCID, 1);
-
-	err = vm_set_capability(vcpu, VM_CAP_IPI_EXIT, 1);
-	assert(err == 0);
-}
-
 static struct vmctx *
 do_open(const char *vmname)
 {
@@ -697,26 +657,6 @@ do_open(const char *vmname)
 	return (ctx);
 }
 
-static void
-spinup_vcpu(struct vcpu_info *vi, bool bsp)
-{
-	int error;
-
-	if (!bsp) {
-		fbsdrun_set_capabilities(vi->vcpu);
-
-		/*
-		 * Enable the 'unrestricted guest' mode for APs.
-		 *
-		 * APs startup in power-on 16-bit mode.
-		 */
-		error = vm_set_capability(vi->vcpu, VM_CAP_UNRESTRICTED_GUEST, 1);
-		assert(error == 0);
-	}
-
-	fbsdrun_addcpu(vi);
-}
-
 static bool
 parse_config_option(const char *option)
 {
@@ -787,17 +727,6 @@ parse_gdb_options(const char *opt)
 }
 #endif
 
-static void
-set_defaults(void)
-{
-
-	set_config_bool("acpi_tables", false);
-	set_config_bool("acpi_tables_in_memory", true);
-	set_config_value("memory.size", "256M");
-	set_config_bool("x86.strictmsr", true);
-	set_config_value("lpc.fwcfg", "bhyve");
-}
-
 int
 main(int argc, char *argv[])
 {
@@ -814,8 +743,8 @@ main(int argc, char *argv[])
 	restore_file = NULL;
 #endif
 
-	init_config();
-	set_defaults();
+	bhyve_init_config();
+
 	progname = basename(argv[0]);
 
 #ifdef BHYVE_SNAPSHOT
@@ -825,9 +754,11 @@ main(int argc, char *argv[])
 #endif
 	while ((c = getopt(argc, argv, optstr)) != -1) {
 		switch (c) {
+#ifdef __amd64__
 		case 'a':
 			set_config_bool("x86.x2apic", false);
 			break;
+#endif
 		case 'A':
 			set_config_bool("acpi_tables", true);
 			break;
@@ -899,6 +830,7 @@ main(int argc, char *argv[])
 			if (!parse_config_option(optarg))
 				errx(EX_USAGE, "invalid configuration option '%s'", optarg);
 			break;
+#ifdef __amd64__
 		case 'H':
 			set_config_bool("x86.vmexit_on_hlt", true);
 			break;
@@ -917,7 +849,6 @@ main(int argc, char *argv[])
 		case 'e':
 			set_config_bool("x86.strictio", true);
 			break;
-#ifdef __amd64__
 		case 'u':
 			set_config_bool("rtc.use_localtime", false);
 			break;
@@ -925,16 +856,18 @@ main(int argc, char *argv[])
 		case 'U':
 			set_config_value("uuid", optarg);
 			break;
+#ifdef __amd64__
 		case 'w':
 			set_config_bool("x86.strictmsr", false);
 			break;
+#endif
 		case 'W':
 			set_config_bool("virtio_msix", false);
 			break;
+#ifdef __amd64__
 		case 'x':
 			set_config_bool("x86.x2apic", true);
 			break;
-#ifdef __amd64__
 		case 'Y':
 			set_config_bool("x86.mptable", false);
 			break;
@@ -1008,7 +941,7 @@ main(int argc, char *argv[])
 		exit(4);
 	}
 
-	fbsdrun_set_capabilities(bsp);
+	bhyve_init_vcpu(bsp);
 
 	/* Allocate per-VCPU resources. */
 	vcpu_info = calloc(guest_ncpus, sizeof(*vcpu_info));
@@ -1099,23 +1032,11 @@ main(int argc, char *argv[])
 	init_gdb(ctx);
 #endif
 
-#ifdef __amd64__
-	if (lpc_bootrom()) {
-		if (vm_set_capability(bsp, VM_CAP_UNRESTRICTED_GUEST, 1)) {
-			fprintf(stderr, "ROM boot failed: unrestricted guest "
-			    "capability not available\n");
-			exit(4);
-		}
-		error = vcpu_reset(bsp);
-		assert(error == 0);
-	}
-#endif
-
 	/*
 	 * Add all vCPUs.
 	 */
 	for (int vcpuid = 0; vcpuid < guest_ncpus; vcpuid++)
-		spinup_vcpu(&vcpu_info[vcpuid], vcpuid == BSP);
+		bhyve_start_vcpu(vcpu_info[vcpuid].vcpu, vcpuid == BSP);
 
 #ifdef BHYVE_SNAPSHOT
 	if (restore_file != NULL) {
diff --git a/usr.sbin/bhyve/bhyverun.h b/usr.sbin/bhyve/bhyverun.h
index b1cfb99a964e..39e0916f08ef 100644
--- a/usr.sbin/bhyve/bhyverun.h
+++ b/usr.sbin/bhyve/bhyverun.h
@@ -26,8 +26,10 @@
  * SUCH DAMAGE.
  */
 
-#ifndef	_FBSDRUN_H_
-#define	_FBSDRUN_H_
+#ifndef	_BHYVERUN_H_
+#define	_BHYVERUN_H_
+
+#include <stdbool.h>
 
 #define	VMEXIT_CONTINUE		(0)
 #define	VMEXIT_ABORT		(-1)
@@ -46,6 +48,7 @@ uintptr_t paddr_host2guest(struct vmctx *ctx, void *addr);
 
 struct vcpu;
 struct vcpu *fbsdrun_vcpu(int vcpuid);
+void fbsdrun_addcpu(int vcpuid);
 void fbsdrun_deletecpu(int vcpuid);
 int fbsdrun_suspendcpu(int vcpuid);
 
@@ -53,4 +56,9 @@ int  fbsdrun_virtio_msix(void);
 
 typedef int (*vmexit_handler_t)(struct vmctx *, struct vcpu *, struct vm_run *);
 
+/* Interfaces implemented by machine-dependent code. */
+void bhyve_init_config(void);
+void bhyve_init_vcpu(struct vcpu *vcpu);
+void bhyve_start_vcpu(struct vcpu *vcpu, bool bsp);
+
 #endif