git: 75b1f9677268 - main - bhyve: restore the BARs at the addresses the guest gave them

From: Bojan Novković <bnovkov_at_FreeBSD.org>
Date: Wed, 04 Nov 2026 17:34:38 UTC
The branch main has been updated by bnovkov:

URL: https://cgit.FreeBSD.org/src/commit/?id=75b1f96772688c4feb698bd5aece8793324b9e23

commit 75b1f96772688c4feb698bd5aece8793324b9e23
Author:     Wanpeng Qian <wanpengqian@gmail.com>
AuthorDate: 2026-11-04 17:32:43 +0000
Commit:     Bojan Novković <bnovkov@FreeBSD.org>
CommitDate: 2026-11-04 17:33:44 +0000

    bhyve: restore the BARs at the addresses the guest gave them
    
    A restore set the addresses in the BARs and the command register to
    those of the snapshot, but left the regions registered where
    pci_emul_alloc_bar() had put them when bhyve started.  That is only
    right for a guest that kept bhyve's addresses.  The UEFI firmware moves
    64-bit BARs above 4 GB, and after a restore every access to such a BAR
    went nowhere:
    
      Emulating access to non-existent address to 0x800001014
    
    Before the devices are restored, unregister the regions of all of them;
    when a device is restored, register its regions at the restored
    addresses, if the restored command register enables them.  All devices
    go first because a restored BAR may be where another device's BAR is
    before the restore.
    
    Signed-off-by: Wanpeng Qian <wanpengqian@gmail.com>
    Sponsored by:   keelos.dev
    Reviewed by:    bnovkov
    Differential Revision:  https://reviews.freebsd.org/D60235
    MFC after:      2 weeks
    Tested by:      bnovkov
---
 usr.sbin/bhyve/pci_emul.c | 44 ++++++++++++++++++++++++++++++++++++++++++++
 usr.sbin/bhyve/pci_emul.h |  1 +
 usr.sbin/bhyve/snapshot.c |  2 ++
 3 files changed, 47 insertions(+)

diff --git a/usr.sbin/bhyve/pci_emul.c b/usr.sbin/bhyve/pci_emul.c
index 60128a735de1..1ce9478bdebc 100644
--- a/usr.sbin/bhyve/pci_emul.c
+++ b/usr.sbin/bhyve/pci_emul.c
@@ -2510,6 +2510,48 @@ INOUT_PORT(pci_cfgdata, CONF1_DATA_PORT+3, IOPORT_F_INOUT, pci_emul_cfgdata);
 #endif
 
 #ifdef BHYVE_SNAPSHOT
+/*
+ * Register or unregister the regions of the BARs that the device decodes.
+ */
+static void
+pci_restore_register_bars(struct pci_devinst *pi, int registration)
+{
+	int decode, i;
+
+	for (i = 0; i <= PCI_BARMAX; i++) {
+		switch (pi->pi_bar[i].type) {
+		case PCIBAR_IO:
+			decode = porten(pi);
+			break;
+		case PCIBAR_MEM32:
+		case PCIBAR_MEM64:
+			decode = memen(pi);
+			break;
+		default:
+			continue;
+		}
+		if (decode && pi->pi_bar[i].size != 0)
+			modify_bar_registration(pi, i, registration);
+	}
+}
+
+/*
+ * To be called before the devices are restored.  The regions of the BARs are
+ * registered at the addresses pci_emul_alloc_bar() chose; the restore of a
+ * device registers them at the addresses the guest had given them.  All
+ * devices give up their regions first, because a restored BAR may be where
+ * the BAR of another device is now.
+ */
+void
+pci_restore_prepare(void)
+{
+	struct pci_devinst *pi;
+
+	pi = NULL;
+	while ((pi = pci_next(pi)) != NULL)
+		pci_restore_register_bars(pi, 0);
+}
+
 /*
  * Saves/restores PCI device emulated state. Returns 0 on success.
  */
@@ -2544,6 +2586,8 @@ pci_snapshot_pci_dev(struct vm_snapshot_meta *meta)
 		SNAPSHOT_VAR_OR_LEAVE(pi->pi_bar[i].size, meta, ret, done);
 		SNAPSHOT_VAR_OR_LEAVE(pi->pi_bar[i].addr, meta, ret, done);
 	}
+	if (meta->op == VM_SNAPSHOT_RESTORE)
+		pci_restore_register_bars(pi, 1);
 
 	/* Restore MSI-X table. */
 	for (i = 0; i < pi->pi_msix.table_count; i++) {
diff --git a/usr.sbin/bhyve/pci_emul.h b/usr.sbin/bhyve/pci_emul.h
index b7ddfd73bda1..d536e53d01e3 100644
--- a/usr.sbin/bhyve/pci_emul.h
+++ b/usr.sbin/bhyve/pci_emul.h
@@ -268,6 +268,7 @@ int	pci_bus_configured(int bus);
 #ifdef BHYVE_SNAPSHOT
 struct pci_devinst *pci_next(const struct pci_devinst *cursor);
 int	pci_snapshot(struct vm_snapshot_meta *meta);
+void	pci_restore_prepare(void);
 int	pci_pause(struct pci_devinst *pdi);
 int	pci_resume(struct pci_devinst *pdi);
 #endif
diff --git a/usr.sbin/bhyve/snapshot.c b/usr.sbin/bhyve/snapshot.c
index 1d3d337ec9b7..067b05622e68 100644
--- a/usr.sbin/bhyve/snapshot.c
+++ b/usr.sbin/bhyve/snapshot.c
@@ -876,6 +876,8 @@ vm_restore_devices(struct restore_state *rstate)
 	int ret;
 	struct pci_devinst *pdi = NULL;
 
+	pci_restore_prepare();
+
 	while ((pdi = pci_next(pdi)) != NULL) {
 		ret = vm_restore_device(rstate, pci_snapshot, pdi->pi_name, pdi);
 		if (ret)