git: 9da9560c4dd3 - main - virtio: Add VirtIO PCI modern (V1) support

Bryan Venteicher bryanv at FreeBSD.org
Tue Jan 19 05:08:01 UTC 2021


The branch main has been updated by bryanv:

URL: https://cgit.FreeBSD.org/src/commit/?id=9da9560c4dd3556519cd391a04f0db157dc3c295

commit 9da9560c4dd3556519cd391a04f0db157dc3c295
Author:     Bryan Venteicher <bryanv at FreeBSD.org>
AuthorDate: 2021-01-19 04:55:23 +0000
Commit:     Bryan Venteicher <bryanv at FreeBSD.org>
CommitDate: 2021-01-19 04:55:23 +0000

    virtio: Add VirtIO PCI modern (V1) support
    
    Use the existing legacy PCI driver as the basis for shared code
    between the legacy and modern PCI drivers. The existing virtio_pci
    kernel module will contain both the legacy and modern drivers.
    
    Changes to the virtqueue and each device driver (network, block, etc)
    for V1 support come in later commits.
    
    Update the MMIO driver to reflect the VirtIO bus method changes, but
    the modern compliance can be improved on later.
    
    Note that the modern PCI driver requires bus_map_resource() to be
    implemented, which is not the case on all archs.
    
    The hw.virtio.pci.transitional tunable default value is zero so
    transitional devices will continue to be driven via the legacy
    driver.
    
    Reviewed by: grehan (mentor)
    Differential Revision: https://reviews.freebsd.org/D27856
---
 sys/conf/files                             |    3 +
 sys/dev/virtio/mmio/virtio_mmio.c          |   23 +-
 sys/dev/virtio/pci/virtio_pci.c            | 1297 +++++++++----------------
 sys/dev/virtio/pci/virtio_pci.h            |  173 ++--
 sys/dev/virtio/pci/virtio_pci_if.m         |   71 ++
 sys/dev/virtio/pci/virtio_pci_legacy.c     |  733 ++++++++++++++
 sys/dev/virtio/pci/virtio_pci_legacy_var.h |   78 ++
 sys/dev/virtio/pci/virtio_pci_modern.c     | 1448 ++++++++++++++++++++++++++++
 sys/dev/virtio/pci/virtio_pci_modern_var.h |  135 +++
 sys/dev/virtio/pci/virtio_pci_var.h        |   55 ++
 sys/dev/virtio/virtio.c                    |   76 +-
 sys/dev/virtio/virtio.h                    |    8 +
 sys/dev/virtio/virtio_bus_if.m             |   11 +
 sys/dev/virtio/virtio_endian.h             |  106 ++
 sys/dev/virtio/virtqueue.c                 |   10 +-
 sys/dev/virtio/virtqueue.h                 |    4 +-
 sys/modules/virtio/pci/Makefile            |    5 +-
 17 files changed, 3292 insertions(+), 944 deletions(-)

diff --git a/sys/conf/files b/sys/conf/files
index e641b79d9c46..79f3b7ca54e6 100644
--- a/sys/conf/files
+++ b/sys/conf/files
@@ -3457,6 +3457,9 @@ dev/virtio/virtqueue.c			optional	virtio
 dev/virtio/virtio_bus_if.m		optional	virtio
 dev/virtio/virtio_if.m			optional	virtio
 dev/virtio/pci/virtio_pci.c		optional	virtio_pci
+dev/virtio/pci/virtio_pci_if.m		optional	virtio_pci
+dev/virtio/pci/virtio_pci_legacy.c	optional	virtio_pci
+dev/virtio/pci/virtio_pci_modern.c	optional	virtio_pci
 dev/virtio/mmio/virtio_mmio.c		optional	virtio_mmio
 dev/virtio/mmio/virtio_mmio_acpi.c	optional	virtio_mmio acpi
 dev/virtio/mmio/virtio_mmio_fdt.c	optional	virtio_mmio fdt
diff --git a/sys/dev/virtio/mmio/virtio_mmio.c b/sys/dev/virtio/mmio/virtio_mmio.c
index bb5d84754f09..694d2a232fdd 100644
--- a/sys/dev/virtio/mmio/virtio_mmio.c
+++ b/sys/dev/virtio/mmio/virtio_mmio.c
@@ -84,7 +84,7 @@ static void	vtmmio_stop(device_t);
 static void	vtmmio_poll(device_t);
 static int	vtmmio_reinit(device_t, uint64_t);
 static void	vtmmio_reinit_complete(device_t);
-static void	vtmmio_notify_virtqueue(device_t, uint16_t);
+static void	vtmmio_notify_virtqueue(device_t, uint16_t, bus_size_t);
 static uint8_t	vtmmio_get_status(device_t);
 static void	vtmmio_set_status(device_t, uint8_t);
 static void	vtmmio_read_dev_config(device_t, bus_size_t, void *, int);
@@ -352,6 +352,13 @@ vtmmio_read_ivar(device_t dev, device_t child, int index, uintptr_t *result)
 		 */
 		*result = 0;
 		break;
+	case VIRTIO_IVAR_MODERN:
+		/*
+		 * There are several modern (aka MMIO v2) spec compliance
+		 * issues with this driver, but keep the status quo.
+		 */
+		*result = sc->vtmmio_version > 1;
+		break;
 	default:
 		return (ENOENT);
 	}
@@ -388,6 +395,10 @@ vtmmio_negotiate_features(device_t dev, uint64_t child_features)
 
 	sc = device_get_softc(dev);
 
+	if (sc->vtmmio_version > 1) {
+		child_features |= VIRTIO_F_VERSION_1;
+	}
+
 	vtmmio_write_config_4(sc, VIRTIO_MMIO_HOST_FEATURES_SEL, 1);
 	host_features = vtmmio_read_config_4(sc, VIRTIO_MMIO_HOST_FEATURES);
 	host_features <<= 32;
@@ -402,7 +413,7 @@ vtmmio_negotiate_features(device_t dev, uint64_t child_features)
 	 * host all support.
 	 */
 	features = host_features & child_features;
-	features = virtqueue_filter_features(features);
+	features = virtio_filter_transport_features(features);
 	sc->vtmmio_features = features;
 
 	vtmmio_describe_features(sc, "negotiated", features);
@@ -504,7 +515,8 @@ vtmmio_alloc_virtqueues(device_t dev, int flags, int nvqs,
 		size = vtmmio_read_config_4(sc, VIRTIO_MMIO_QUEUE_NUM_MAX);
 
 		error = virtqueue_alloc(dev, idx, size,
-		    VIRTIO_MMIO_VRING_ALIGN, ~(vm_paddr_t)0, info, &vq);
+		    VIRTIO_MMIO_QUEUE_NOTIFY, VIRTIO_MMIO_VRING_ALIGN,
+		    ~(vm_paddr_t)0, info, &vq);
 		if (error) {
 			device_printf(dev,
 			    "cannot allocate virtqueue %d: %d\n",
@@ -586,13 +598,14 @@ vtmmio_reinit_complete(device_t dev)
 }
 
 static void
-vtmmio_notify_virtqueue(device_t dev, uint16_t queue)
+vtmmio_notify_virtqueue(device_t dev, uint16_t queue, bus_size_t offset)
 {
 	struct vtmmio_softc *sc;
 
 	sc = device_get_softc(dev);
+	MPASS(offset == VIRTIO_MMIO_QUEUE_NOTIFY);
 
-	vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_NOTIFY, queue);
+	vtmmio_write_config_4(sc, offset, queue);
 }
 
 static uint8_t
diff --git a/sys/dev/virtio/pci/virtio_pci.c b/sys/dev/virtio/pci/virtio_pci.c
index 05a632f526a8..4d6fa929ef19 100644
--- a/sys/dev/virtio/pci/virtio_pci.c
+++ b/sys/dev/virtio/pci/virtio_pci.c
@@ -1,7 +1,7 @@
 /*-
  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
  *
- * Copyright (c) 2011, Bryan Venteicher <bryanv at FreeBSD.org>
+ * Copyright (c) 2017, Bryan Venteicher <bryanv at FreeBSD.org>
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -26,8 +26,6 @@
  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
-/* Driver for the VirtIO PCI interface. */
-
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
@@ -37,7 +35,6 @@ __FBSDID("$FreeBSD$");
 #include <sys/kernel.h>
 #include <sys/module.h>
 #include <sys/malloc.h>
-#include <sys/endian.h>
 
 #include <machine/bus.h>
 #include <machine/resource.h>
@@ -50,369 +47,236 @@ __FBSDID("$FreeBSD$");
 #include <dev/virtio/virtio.h>
 #include <dev/virtio/virtqueue.h>
 #include <dev/virtio/pci/virtio_pci.h>
+#include <dev/virtio/pci/virtio_pci_var.h>
 
-#include "virtio_bus_if.h"
+#include "virtio_pci_if.h"
 #include "virtio_if.h"
 
-struct vtpci_interrupt {
-	struct resource		*vti_irq;
-	int			 vti_rid;
-	void			*vti_handler;
-};
-
-struct vtpci_virtqueue {
-	struct virtqueue	*vtv_vq;
-	int			 vtv_no_intr;
-};
-
-struct vtpci_softc {
-	device_t			 vtpci_dev;
-	struct resource			*vtpci_res;
-	struct resource			*vtpci_msix_res;
-	uint64_t			 vtpci_features;
-	uint32_t			 vtpci_flags;
-#define VTPCI_FLAG_NO_MSI		0x0001
-#define VTPCI_FLAG_NO_MSIX		0x0002
-#define VTPCI_FLAG_LEGACY		0x1000
-#define VTPCI_FLAG_MSI			0x2000
-#define VTPCI_FLAG_MSIX			0x4000
-#define VTPCI_FLAG_SHARED_MSIX		0x8000
-#define VTPCI_FLAG_ITYPE_MASK		0xF000
-
-	/* This "bus" will only ever have one child. */
-	device_t			 vtpci_child_dev;
-	struct virtio_feature_desc	*vtpci_child_feat_desc;
-
-	int				 vtpci_nvqs;
-	struct vtpci_virtqueue		*vtpci_vqs;
-
-	/*
-	 * Ideally, each virtqueue that the driver provides a callback for will
-	 * receive its own MSIX vector. If there are not sufficient vectors
-	 * available, then attempt to have all the VQs share one vector. For
-	 * MSIX, the configuration changed notifications must be on their own
-	 * vector.
-	 *
-	 * If MSIX is not available, we will attempt to have the whole device
-	 * share one MSI vector, and then, finally, one legacy interrupt.
-	 */
-	struct vtpci_interrupt		 vtpci_device_interrupt;
-	struct vtpci_interrupt		*vtpci_msix_vq_interrupts;
-	int				 vtpci_nmsix_resources;
-};
-
-static int	vtpci_probe(device_t);
-static int	vtpci_attach(device_t);
-static int	vtpci_detach(device_t);
-static int	vtpci_suspend(device_t);
-static int	vtpci_resume(device_t);
-static int	vtpci_shutdown(device_t);
-static void	vtpci_driver_added(device_t, driver_t *);
-static void	vtpci_child_detached(device_t, device_t);
-static int	vtpci_read_ivar(device_t, device_t, int, uintptr_t *);
-static int	vtpci_write_ivar(device_t, device_t, int, uintptr_t);
-
-static uint64_t	vtpci_negotiate_features(device_t, uint64_t);
-static int	vtpci_with_feature(device_t, uint64_t);
-static int	vtpci_alloc_virtqueues(device_t, int, int,
-		    struct vq_alloc_info *);
-static int	vtpci_setup_intr(device_t, enum intr_type);
-static void	vtpci_stop(device_t);
-static int	vtpci_reinit(device_t, uint64_t);
-static void	vtpci_reinit_complete(device_t);
-static void	vtpci_notify_virtqueue(device_t, uint16_t);
-static uint8_t	vtpci_get_status(device_t);
-static void	vtpci_set_status(device_t, uint8_t);
-static void	vtpci_read_dev_config(device_t, bus_size_t, void *, int);
-static void	vtpci_write_dev_config(device_t, bus_size_t, void *, int);
-
-static void	vtpci_describe_features(struct vtpci_softc *, const char *,
+static void	vtpci_describe_features(struct vtpci_common *, const char *,
 		    uint64_t);
-static void	vtpci_probe_and_attach_child(struct vtpci_softc *);
-
-static int	vtpci_alloc_msix(struct vtpci_softc *, int);
-static int	vtpci_alloc_msi(struct vtpci_softc *);
-static int	vtpci_alloc_intr_msix_pervq(struct vtpci_softc *);
-static int	vtpci_alloc_intr_msix_shared(struct vtpci_softc *);
-static int	vtpci_alloc_intr_msi(struct vtpci_softc *);
-static int	vtpci_alloc_intr_legacy(struct vtpci_softc *);
-static int	vtpci_alloc_interrupt(struct vtpci_softc *, int, int,
+static int	vtpci_alloc_msix(struct vtpci_common *, int);
+static int	vtpci_alloc_msi(struct vtpci_common *);
+static int	vtpci_alloc_intr_msix_pervq(struct vtpci_common *);
+static int	vtpci_alloc_intr_msix_shared(struct vtpci_common *);
+static int	vtpci_alloc_intr_msi(struct vtpci_common *);
+static int	vtpci_alloc_intr_intx(struct vtpci_common *);
+static int	vtpci_alloc_interrupt(struct vtpci_common *, int, int,
+		    struct vtpci_interrupt *);
+static void	vtpci_free_interrupt(struct vtpci_common *,
 		    struct vtpci_interrupt *);
-static int	vtpci_alloc_intr_resources(struct vtpci_softc *);
 
-static int	vtpci_setup_legacy_interrupt(struct vtpci_softc *,
+static void	vtpci_free_interrupts(struct vtpci_common *);
+static void	vtpci_free_virtqueues(struct vtpci_common *);
+static void	vtpci_cleanup_setup_intr_attempt(struct vtpci_common *);
+static int	vtpci_alloc_intr_resources(struct vtpci_common *);
+static int	vtpci_setup_intx_interrupt(struct vtpci_common *,
 		    enum intr_type);
-static int	vtpci_setup_pervq_msix_interrupts(struct vtpci_softc *,
+static int	vtpci_setup_pervq_msix_interrupts(struct vtpci_common *,
 		    enum intr_type);
-static int	vtpci_setup_msix_interrupts(struct vtpci_softc *,
+static int	vtpci_set_host_msix_vectors(struct vtpci_common *);
+static int	vtpci_setup_msix_interrupts(struct vtpci_common *,
 		    enum intr_type);
-static int	vtpci_setup_interrupts(struct vtpci_softc *, enum intr_type);
-
-static int	vtpci_register_msix_vector(struct vtpci_softc *, int,
-		    struct vtpci_interrupt *);
-static int	vtpci_set_host_msix_vectors(struct vtpci_softc *);
-static int	vtpci_reinit_virtqueue(struct vtpci_softc *, int);
-
-static void	vtpci_free_interrupt(struct vtpci_softc *,
-		    struct vtpci_interrupt *);
-static void	vtpci_free_interrupts(struct vtpci_softc *);
-static void	vtpci_free_virtqueues(struct vtpci_softc *);
-static void	vtpci_release_child_resources(struct vtpci_softc *);
-static void	vtpci_cleanup_setup_intr_attempt(struct vtpci_softc *);
-static void	vtpci_reset(struct vtpci_softc *);
-
-static void	vtpci_select_virtqueue(struct vtpci_softc *, int);
-
-static void	vtpci_legacy_intr(void *);
+static int	vtpci_setup_intrs(struct vtpci_common *, enum intr_type);
+static int	vtpci_reinit_virtqueue(struct vtpci_common *, int);
+static void	vtpci_intx_intr(void *);
 static int	vtpci_vq_shared_intr_filter(void *);
 static void	vtpci_vq_shared_intr(void *);
 static int	vtpci_vq_intr_filter(void *);
 static void	vtpci_vq_intr(void *);
 static void	vtpci_config_intr(void *);
 
-#define vtpci_setup_msi_interrupt vtpci_setup_legacy_interrupt
-
-#define VIRTIO_PCI_CONFIG(_sc) \
-    VIRTIO_PCI_CONFIG_OFF((((_sc)->vtpci_flags & VTPCI_FLAG_MSIX)) != 0)
-
-/*
- * I/O port read/write wrappers.
- */
-#define vtpci_read_config_1(sc, o)	bus_read_1((sc)->vtpci_res, (o))
-#define vtpci_write_config_1(sc, o, v)	bus_write_1((sc)->vtpci_res, (o), (v))
+#define vtpci_setup_msi_interrupt vtpci_setup_intx_interrupt
 
 /*
- * Virtio-pci specifies that PCI Configuration area is guest endian. However,
- * since PCI devices are inherently little-endian, on BE systems the bus layer
- * transparently converts it to BE. For virtio-legacy, this conversion is
- * undesired, so an extra byte swap is required to fix it.
+ * This module contains two drivers:
+ *   - virtio_pci_legacy for pre-V1 support
+ *   - virtio_pci_modern for V1 support
  */
-#define vtpci_read_config_2(sc, o)      le16toh(bus_read_2((sc)->vtpci_res, (o)))
-#define vtpci_read_config_4(sc, o)      le32toh(bus_read_4((sc)->vtpci_res, (o)))
-#define vtpci_write_config_2(sc, o, v)  bus_write_2((sc)->vtpci_res, (o), (htole16(v)))
-#define vtpci_write_config_4(sc, o, v)  bus_write_4((sc)->vtpci_res, (o), (htole32(v)))
-
-/* PCI Header LE. On BE systems the bus layer takes care of byte swapping */
-#define vtpci_read_header_2(sc, o)      (bus_read_2((sc)->vtpci_res, (o)))
-#define vtpci_read_header_4(sc, o)      (bus_read_4((sc)->vtpci_res, (o)))
-#define vtpci_write_header_2(sc, o, v)  bus_write_2((sc)->vtpci_res, (o), (v))
-#define vtpci_write_header_4(sc, o, v)  bus_write_4((sc)->vtpci_res, (o), (v))
-
-/* Tunables. */
-static int vtpci_disable_msix = 0;
-TUNABLE_INT("hw.virtio.pci.disable_msix", &vtpci_disable_msix);
-
-static device_method_t vtpci_methods[] = {
-	/* Device interface. */
-	DEVMETHOD(device_probe,			  vtpci_probe),
-	DEVMETHOD(device_attach,		  vtpci_attach),
-	DEVMETHOD(device_detach,		  vtpci_detach),
-	DEVMETHOD(device_suspend,		  vtpci_suspend),
-	DEVMETHOD(device_resume,		  vtpci_resume),
-	DEVMETHOD(device_shutdown,		  vtpci_shutdown),
-
-	/* Bus interface. */
-	DEVMETHOD(bus_driver_added,		  vtpci_driver_added),
-	DEVMETHOD(bus_child_detached,		  vtpci_child_detached),
-	DEVMETHOD(bus_child_pnpinfo_str,	  virtio_child_pnpinfo_str),
-	DEVMETHOD(bus_read_ivar,		  vtpci_read_ivar),
-	DEVMETHOD(bus_write_ivar,		  vtpci_write_ivar),
-
-	/* VirtIO bus interface. */
-	DEVMETHOD(virtio_bus_negotiate_features,  vtpci_negotiate_features),
-	DEVMETHOD(virtio_bus_with_feature,	  vtpci_with_feature),
-	DEVMETHOD(virtio_bus_alloc_virtqueues,	  vtpci_alloc_virtqueues),
-	DEVMETHOD(virtio_bus_setup_intr,	  vtpci_setup_intr),
-	DEVMETHOD(virtio_bus_stop,		  vtpci_stop),
-	DEVMETHOD(virtio_bus_reinit,		  vtpci_reinit),
-	DEVMETHOD(virtio_bus_reinit_complete,	  vtpci_reinit_complete),
-	DEVMETHOD(virtio_bus_notify_vq,		  vtpci_notify_virtqueue),
-	DEVMETHOD(virtio_bus_read_device_config,  vtpci_read_dev_config),
-	DEVMETHOD(virtio_bus_write_device_config, vtpci_write_dev_config),
-
-	DEVMETHOD_END
-};
-
-static driver_t vtpci_driver = {
-	"virtio_pci",
-	vtpci_methods,
-	sizeof(struct vtpci_softc)
-};
-
-devclass_t vtpci_devclass;
-
-DRIVER_MODULE(virtio_pci, pci, vtpci_driver, vtpci_devclass, 0, 0);
 MODULE_VERSION(virtio_pci, 1);
 MODULE_DEPEND(virtio_pci, pci, 1, 1, 1);
 MODULE_DEPEND(virtio_pci, virtio, 1, 1, 1);
 
-static int
-vtpci_probe(device_t dev)
-{
-	char desc[36];
-	const char *name;
+int vtpci_disable_msix = 0;
+TUNABLE_INT("hw.virtio.pci.disable_msix", &vtpci_disable_msix);
 
-	if (pci_get_vendor(dev) != VIRTIO_PCI_VENDORID)
-		return (ENXIO);
+static uint8_t
+vtpci_read_isr(struct vtpci_common *cn)
+{
+	return (VIRTIO_PCI_READ_ISR(cn->vtpci_dev));
+}
 
-	if (pci_get_device(dev) < VIRTIO_PCI_DEVICEID_MIN ||
-	    pci_get_device(dev) > VIRTIO_PCI_DEVICEID_MAX)
-		return (ENXIO);
+static uint16_t
+vtpci_get_vq_size(struct vtpci_common *cn, int idx)
+{
+	return (VIRTIO_PCI_GET_VQ_SIZE(cn->vtpci_dev, idx));
+}
 
-	if (pci_get_revid(dev) != VIRTIO_PCI_ABI_VERSION)
-		return (ENXIO);
+static bus_size_t
+vtpci_get_vq_notify_off(struct vtpci_common *cn, int idx)
+{
+	return (VIRTIO_PCI_GET_VQ_NOTIFY_OFF(cn->vtpci_dev, idx));
+}
 
-	name = virtio_device_name(pci_get_subdevice(dev));
-	if (name == NULL)
-		name = "Unknown";
+static void
+vtpci_set_vq(struct vtpci_common *cn, struct virtqueue *vq)
+{
+	VIRTIO_PCI_SET_VQ(cn->vtpci_dev, vq);
+}
 
-	snprintf(desc, sizeof(desc), "VirtIO PCI %s adapter", name);
-	device_set_desc_copy(dev, desc);
+static void
+vtpci_disable_vq(struct vtpci_common *cn, int idx)
+{
+	VIRTIO_PCI_DISABLE_VQ(cn->vtpci_dev, idx);
+}
 
-	return (BUS_PROBE_DEFAULT);
+static int
+vtpci_register_cfg_msix(struct vtpci_common *cn, struct vtpci_interrupt *intr)
+{
+	return (VIRTIO_PCI_REGISTER_CFG_MSIX(cn->vtpci_dev, intr));
 }
 
 static int
-vtpci_attach(device_t dev)
+vtpci_register_vq_msix(struct vtpci_common *cn, int idx,
+    struct vtpci_interrupt *intr)
 {
-	struct vtpci_softc *sc;
-	device_t child;
-	int rid;
+	return (VIRTIO_PCI_REGISTER_VQ_MSIX(cn->vtpci_dev, idx, intr));
+}
 
-	sc = device_get_softc(dev);
-	sc->vtpci_dev = dev;
+void
+vtpci_init(struct vtpci_common *cn, device_t dev, bool modern)
+{
 
-	pci_enable_busmaster(dev);
+	cn->vtpci_dev = dev;
 
-	rid = PCIR_BAR(0);
-	sc->vtpci_res = bus_alloc_resource_any(dev, SYS_RES_IOPORT, &rid,
-	    RF_ACTIVE);
-	if (sc->vtpci_res == NULL) {
-		device_printf(dev, "cannot map I/O space\n");
-		return (ENXIO);
-	}
+	pci_enable_busmaster(dev);
 
+	if (modern)
+		cn->vtpci_flags |= VTPCI_FLAG_MODERN;
 	if (pci_find_cap(dev, PCIY_MSI, NULL) != 0)
-		sc->vtpci_flags |= VTPCI_FLAG_NO_MSI;
-
-	if (pci_find_cap(dev, PCIY_MSIX, NULL) == 0) {
-		rid = PCIR_BAR(1);
-		sc->vtpci_msix_res = bus_alloc_resource_any(dev,
-		    SYS_RES_MEMORY, &rid, RF_ACTIVE);
-	}
-
-	if (sc->vtpci_msix_res == NULL)
-		sc->vtpci_flags |= VTPCI_FLAG_NO_MSIX;
+		cn->vtpci_flags |= VTPCI_FLAG_NO_MSI;
+	if (pci_find_cap(dev, PCIY_MSIX, NULL) != 0)
+		cn->vtpci_flags |= VTPCI_FLAG_NO_MSIX;
+}
 
-	vtpci_reset(sc);
+int
+vtpci_add_child(struct vtpci_common *cn)
+{
+	device_t dev, child;
 
-	/* Tell the host we've noticed this device. */
-	vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_ACK);
+	dev = cn->vtpci_dev;
 
-	if ((child = device_add_child(dev, NULL, -1)) == NULL) {
+	child = device_add_child(dev, NULL, -1);
+	if (child == NULL) {
 		device_printf(dev, "cannot create child device\n");
-		vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_FAILED);
-		vtpci_detach(dev);
 		return (ENOMEM);
 	}
 
-	sc->vtpci_child_dev = child;
-	vtpci_probe_and_attach_child(sc);
+	cn->vtpci_child_dev = child;
 
 	return (0);
 }
 
-static int
-vtpci_detach(device_t dev)
+int
+vtpci_delete_child(struct vtpci_common *cn)
 {
-	struct vtpci_softc *sc;
-	device_t child;
+	device_t dev, child;
 	int error;
 
-	sc = device_get_softc(dev);
+	dev = cn->vtpci_dev;
 
-	if ((child = sc->vtpci_child_dev) != NULL) {
+	child = cn->vtpci_child_dev;
+	if (child != NULL) {
 		error = device_delete_child(dev, child);
 		if (error)
 			return (error);
-		sc->vtpci_child_dev = NULL;
-	}
-
-	vtpci_reset(sc);
-
-	if (sc->vtpci_msix_res != NULL) {
-		bus_release_resource(dev, SYS_RES_MEMORY, PCIR_BAR(1),
-		    sc->vtpci_msix_res);
-		sc->vtpci_msix_res = NULL;
-	}
-
-	if (sc->vtpci_res != NULL) {
-		bus_release_resource(dev, SYS_RES_IOPORT, PCIR_BAR(0),
-		    sc->vtpci_res);
-		sc->vtpci_res = NULL;
+		cn->vtpci_child_dev = NULL;
 	}
 
 	return (0);
 }
 
-static int
-vtpci_suspend(device_t dev)
+void
+vtpci_child_detached(struct vtpci_common *cn)
 {
 
-	return (bus_generic_suspend(dev));
-}
-
-static int
-vtpci_resume(device_t dev)
-{
+	vtpci_release_child_resources(cn);
 
-	return (bus_generic_resume(dev));
+	cn->vtpci_child_feat_desc = NULL;
+	cn->vtpci_features = 0;
 }
 
-static int
-vtpci_shutdown(device_t dev)
+int
+vtpci_reinit(struct vtpci_common *cn)
 {
+	int idx, error;
 
-	(void) bus_generic_shutdown(dev);
-	/* Forcibly stop the host device. */
-	vtpci_stop(dev);
+	for (idx = 0; idx < cn->vtpci_nvqs; idx++) {
+		error = vtpci_reinit_virtqueue(cn, idx);
+		if (error)
+			return (error);
+	}
+
+	if (vtpci_is_msix_enabled(cn)) {
+		error = vtpci_set_host_msix_vectors(cn);
+		if (error)
+			return (error);
+	}
 
 	return (0);
 }
 
 static void
-vtpci_driver_added(device_t dev, driver_t *driver)
+vtpci_describe_features(struct vtpci_common *cn, const char *msg,
+    uint64_t features)
 {
-	struct vtpci_softc *sc;
+	device_t dev, child;
+
+	dev = cn->vtpci_dev;
+	child = cn->vtpci_child_dev;
 
-	sc = device_get_softc(dev);
+	if (device_is_attached(child) || bootverbose == 0)
+		return;
 
-	vtpci_probe_and_attach_child(sc);
+	virtio_describe(dev, msg, features, cn->vtpci_child_feat_desc);
 }
 
-static void
-vtpci_child_detached(device_t dev, device_t child)
+uint64_t
+vtpci_negotiate_features(struct vtpci_common *cn,
+    uint64_t child_features, uint64_t host_features)
 {
-	struct vtpci_softc *sc;
+	uint64_t features;
 
-	sc = device_get_softc(dev);
+	vtpci_describe_features(cn, "host", host_features);
 
-	vtpci_reset(sc);
-	vtpci_release_child_resources(sc);
+	/*
+	 * Limit negotiated features to what the driver, virtqueue, and
+	 * host all support.
+	 */
+	features = host_features & child_features;
+	features = virtio_filter_transport_features(features);
+	vtpci_describe_features(cn, "negotiated", features);
+
+	cn->vtpci_features = features;
+
+	return (features);
 }
 
-static int
-vtpci_read_ivar(device_t dev, device_t child, int index, uintptr_t *result)
+int
+vtpci_with_feature(struct vtpci_common *cn, uint64_t feature)
 {
-	struct vtpci_softc *sc;
+	return ((cn->vtpci_features & feature) != 0);
+}
 
-	sc = device_get_softc(dev);
+int
+vtpci_read_ivar(struct vtpci_common *cn, int index, uintptr_t *result)
+{
+	device_t dev;
+	int error;
 
-	if (sc->vtpci_child_dev != child)
-		return (ENOENT);
+	dev = cn->vtpci_dev;
+	error = 0;
 
 	switch (index) {
-	case VIRTIO_IVAR_DEVTYPE:
 	case VIRTIO_IVAR_SUBDEVICE:
 		*result = pci_get_subdevice(dev);
 		break;
@@ -425,100 +289,74 @@ vtpci_read_ivar(device_t dev, device_t child, int index, uintptr_t *result)
 	case VIRTIO_IVAR_SUBVENDOR:
 		*result = pci_get_subvendor(dev);
 		break;
+	case VIRTIO_IVAR_MODERN:
+		*result = vtpci_is_modern(cn);
+		break;
 	default:
-		return (ENOENT);
+		error = ENOENT;
 	}
 
-	return (0);
+	return (error);
 }
 
-static int
-vtpci_write_ivar(device_t dev, device_t child, int index, uintptr_t value)
+int
+vtpci_write_ivar(struct vtpci_common *cn, int index, uintptr_t value)
 {
-	struct vtpci_softc *sc;
-
-	sc = device_get_softc(dev);
+	int error;
 
-	if (sc->vtpci_child_dev != child)
-		return (ENOENT);
+	error = 0;
 
 	switch (index) {
 	case VIRTIO_IVAR_FEATURE_DESC:
-		sc->vtpci_child_feat_desc = (void *) value;
+		cn->vtpci_child_feat_desc = (void *) value;
 		break;
 	default:
-		return (ENOENT);
+		error = ENOENT;
 	}
 
-	return (0);
+	return (error);
 }
 
-static uint64_t
-vtpci_negotiate_features(device_t dev, uint64_t child_features)
+int
+vtpci_alloc_virtqueues(struct vtpci_common *cn, int flags, int nvqs,
+    struct vq_alloc_info *vq_info)
 {
-	struct vtpci_softc *sc;
-	uint64_t host_features, features;
-
-	sc = device_get_softc(dev);
+	device_t dev;
+	int idx, align, error;
 
-	host_features = vtpci_read_header_4(sc, VIRTIO_PCI_HOST_FEATURES);
-	vtpci_describe_features(sc, "host", host_features);
+	dev = cn->vtpci_dev;
 
 	/*
-	 * Limit negotiated features to what the driver, virtqueue, and
-	 * host all support.
+	 * This is VIRTIO_PCI_VRING_ALIGN from legacy VirtIO. In modern VirtIO,
+	 * the tables do not have to be allocated contiguously, but we do so
+	 * anyways.
 	 */
-	features = host_features & child_features;
-	features = virtqueue_filter_features(features);
-	sc->vtpci_features = features;
-
-	vtpci_describe_features(sc, "negotiated", features);
-	vtpci_write_header_4(sc, VIRTIO_PCI_GUEST_FEATURES, features);
-
-	return (features);
-}
-
-static int
-vtpci_with_feature(device_t dev, uint64_t feature)
-{
-	struct vtpci_softc *sc;
+	align = 4096;
 
-	sc = device_get_softc(dev);
-
-	return ((sc->vtpci_features & feature) != 0);
-}
-
-static int
-vtpci_alloc_virtqueues(device_t dev, int flags, int nvqs,
-    struct vq_alloc_info *vq_info)
-{
-	struct vtpci_softc *sc;
-	struct virtqueue *vq;
-	struct vtpci_virtqueue *vqx;
-	struct vq_alloc_info *info;
-	int idx, error;
-	uint16_t size;
-
-	sc = device_get_softc(dev);
-
-	if (sc->vtpci_nvqs != 0)
+	if (cn->vtpci_nvqs != 0)
 		return (EALREADY);
 	if (nvqs <= 0)
 		return (EINVAL);
 
-	sc->vtpci_vqs = malloc(nvqs * sizeof(struct vtpci_virtqueue),
+	cn->vtpci_vqs = malloc(nvqs * sizeof(struct vtpci_virtqueue),
 	    M_DEVBUF, M_NOWAIT | M_ZERO);
-	if (sc->vtpci_vqs == NULL)
+	if (cn->vtpci_vqs == NULL)
 		return (ENOMEM);
 
 	for (idx = 0; idx < nvqs; idx++) {
-		vqx = &sc->vtpci_vqs[idx];
+		struct vtpci_virtqueue *vqx;
+		struct vq_alloc_info *info;
+		struct virtqueue *vq;
+		bus_size_t notify_offset;
+		uint16_t size;
+
+		vqx = &cn->vtpci_vqs[idx];
 		info = &vq_info[idx];
 
-		vtpci_select_virtqueue(sc, idx);
-		size = vtpci_read_header_2(sc, VIRTIO_PCI_QUEUE_NUM);
+		size = vtpci_get_vq_size(cn, idx);
+		notify_offset = vtpci_get_vq_notify_off(cn, idx);
 
-		error = virtqueue_alloc(dev, idx, size, VIRTIO_PCI_VRING_ALIGN,
+		error = virtqueue_alloc(dev, idx, size, notify_offset, align,
 		    ~(vm_paddr_t)0, info, &vq);
 		if (error) {
 			device_printf(dev,
@@ -526,270 +364,27 @@ vtpci_alloc_virtqueues(device_t dev, int flags, int nvqs,
 			break;
 		}
 
-		vtpci_write_header_4(sc, VIRTIO_PCI_QUEUE_PFN,
-		    virtqueue_paddr(vq) >> VIRTIO_PCI_QUEUE_ADDR_SHIFT);
+		vtpci_set_vq(cn, vq);
 
 		vqx->vtv_vq = *info->vqai_vq = vq;
 		vqx->vtv_no_intr = info->vqai_intr == NULL;
 
-		sc->vtpci_nvqs++;
+		cn->vtpci_nvqs++;
 	}
 
 	if (error)
-		vtpci_free_virtqueues(sc);
+		vtpci_free_virtqueues(cn);
 
 	return (error);
 }
 
 static int
-vtpci_setup_intr(device_t dev, enum intr_type type)
-{
-	struct vtpci_softc *sc;
-	int attempt, error;
-
-	sc = device_get_softc(dev);
-
-	for (attempt = 0; attempt < 5; attempt++) {
-		/*
-		 * Start with the most desirable interrupt configuration and
-		 * fallback towards less desirable ones.
-		 */
-		switch (attempt) {
-		case 0:
-			error = vtpci_alloc_intr_msix_pervq(sc);
-			break;
-		case 1:
-			error = vtpci_alloc_intr_msix_shared(sc);
-			break;
-		case 2:
-			error = vtpci_alloc_intr_msi(sc);
-			break;
-		case 3:
-			error = vtpci_alloc_intr_legacy(sc);
-			break;
-		default:
-			device_printf(dev,
-			    "exhausted all interrupt allocation attempts\n");
-			return (ENXIO);
-		}
-
-		if (error == 0 && vtpci_setup_interrupts(sc, type) == 0)
-			break;
-
-		vtpci_cleanup_setup_intr_attempt(sc);
-	}
-
-	if (bootverbose) {
-		if (sc->vtpci_flags & VTPCI_FLAG_LEGACY)
-			device_printf(dev, "using legacy interrupt\n");
-		else if (sc->vtpci_flags & VTPCI_FLAG_MSI)
-			device_printf(dev, "using MSI interrupt\n");
-		else if (sc->vtpci_flags & VTPCI_FLAG_SHARED_MSIX)
-			device_printf(dev, "using shared MSIX interrupts\n");
-		else
-			device_printf(dev, "using per VQ MSIX interrupts\n");
-	}
-
-	return (0);
-}
-
-static void
-vtpci_stop(device_t dev)
-{
-
-	vtpci_reset(device_get_softc(dev));
-}
-
-static int
-vtpci_reinit(device_t dev, uint64_t features)
-{
-	struct vtpci_softc *sc;
-	int idx, error;
-
-	sc = device_get_softc(dev);
-
-	/*
-	 * Redrive the device initialization. This is a bit of an abuse of
-	 * the specification, but VirtualBox, QEMU/KVM, and BHyVe seem to
-	 * play nice.
-	 *
-	 * We do not allow the host device to change from what was originally
-	 * negotiated beyond what the guest driver changed. MSIX state should
-	 * not change, number of virtqueues and their size remain the same, etc.
-	 * This will need to be rethought when we want to support migration.
-	 */
-
-	if (vtpci_get_status(dev) != VIRTIO_CONFIG_STATUS_RESET)
-		vtpci_stop(dev);
-
-	/*
-	 * Quickly drive the status through ACK and DRIVER. The device
-	 * does not become usable again until vtpci_reinit_complete().
-	 */
-	vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_ACK);
-	vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER);
-
-	vtpci_negotiate_features(dev, features);
-
-	for (idx = 0; idx < sc->vtpci_nvqs; idx++) {
-		error = vtpci_reinit_virtqueue(sc, idx);
-		if (error)
-			return (error);
-	}
-
-	if (sc->vtpci_flags & VTPCI_FLAG_MSIX) {
-		error = vtpci_set_host_msix_vectors(sc);
-		if (error)
-			return (error);
-	}
-
-	return (0);
-}
-
-static void
-vtpci_reinit_complete(device_t dev)
-{
-
-	vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER_OK);
-}
-
-static void
-vtpci_notify_virtqueue(device_t dev, uint16_t queue)
-{
-	struct vtpci_softc *sc;
-
-	sc = device_get_softc(dev);
-
-	vtpci_write_header_2(sc, VIRTIO_PCI_QUEUE_NOTIFY, queue);
-}
-
-static uint8_t
-vtpci_get_status(device_t dev)
-{
-	struct vtpci_softc *sc;
-
-	sc = device_get_softc(dev);
-
-	return (vtpci_read_config_1(sc, VIRTIO_PCI_STATUS));
-}
-
-static void
-vtpci_set_status(device_t dev, uint8_t status)
-{
-	struct vtpci_softc *sc;
-
-	sc = device_get_softc(dev);
-
-	if (status != VIRTIO_CONFIG_STATUS_RESET)
-		status |= vtpci_get_status(dev);
-
-	vtpci_write_config_1(sc, VIRTIO_PCI_STATUS, status);
-}
-
-static void
-vtpci_read_dev_config(device_t dev, bus_size_t offset,
*** 4018 LINES SKIPPED ***


More information about the dev-commits-src-all mailing list