svn commit: r305497 - in head: share/man/man9 sys/amd64/vmm/io sys/dev/pci
John Baldwin
jhb at FreeBSD.org
Tue Sep 6 20:17:55 UTC 2016
Author: jhb
Date: Tue Sep 6 20:17:54 2016
New Revision: 305497
URL: https://svnweb.freebsd.org/changeset/base/305497
Log:
Update the I/O MMU in bhyve when PCI devices are added and removed.
When the I/O MMU is active in bhyve, all PCI devices need valid entries
in the DMAR context tables. The I/O MMU code does a single enumeration
of the available PCI devices during initialization to add all existing
devices to a domain representing the host. The ppt(4) driver then moves
pass through devices in and out of domains for virtual machines as needed.
However, when new PCI devices were added at runtime either via SR-IOV or
HotPlug, the I/O MMU tables were not updated.
This change adds a new set of EVENTHANDLERS that are invoked when PCI
devices are added and deleted. The I/O MMU driver in bhyve installs
handlers for these events which it uses to add and remove devices to
the "host" domain.
Reviewed by: imp
Sponsored by: Chelsio Communications
Differential Revision: https://reviews.freebsd.org/D7667
Modified:
head/share/man/man9/pci.9
head/sys/amd64/vmm/io/iommu.c
head/sys/dev/pci/pci.c
head/sys/dev/pci/pcivar.h
Modified: head/share/man/man9/pci.9
==============================================================================
--- head/share/man/man9/pci.9 Tue Sep 6 20:01:15 2016 (r305496)
+++ head/share/man/man9/pci.9 Tue Sep 6 20:17:54 2016 (r305497)
@@ -25,7 +25,7 @@
.\"
.\" $FreeBSD$
.\"
-.Dd September 1, 2016
+.Dd September 6, 2016
.Dt PCI 9
.Os
.Sh NAME
@@ -149,6 +149,10 @@
.Fn pcie_read_config "device_t dev" "int reg" "int width"
.Ft void
.Fn pcie_write_config "device_t dev" "int reg" "uint32_t val" "int width"
+.Ft void
+.Fn pci_event_fn "void *arg" "device_t dev"
+.Fn EVENTHANDLER_REGISTER "pci_add_device" "pci_event_fn"
+.Fn EVENTHANDLER_DEREGISTER "pci_delete_resource" "pci_event_fn"
.In dev/pci/pci_iov.h
.Ft int
.Fn pci_iov_attach "device_t dev" "nvlist_t *pf_schema" "nvlist_t *vf_schema"
@@ -910,6 +914,24 @@ with one in the new distribution.
The
.Fn pci_remap_msix
function will fail if this condition is not met.
+.Ss Device Events
+The
+.Va pci_add_device
+event handler is invoked every time a new PCI device is added to the system.
+This includes the creation of Virtual Functions via SR-IOV.
+.Pp
+The
+.Va pci_delete_device
+event handler is invoked every time a PCI device is removed from the system.
+.Pp
+Both event handlers pass the
+.Vt device_t
+object of the relevant PCI device as
+.Fa dev
+to each callback function.
+Both event handlers are invoked while
+.Fa dev
+is unattached but with valid instance variables.
.Sh SEE ALSO
.Xr pci 4 ,
.Xr pciconf 8 ,
@@ -921,6 +943,7 @@ function will fail if this condition is
.Xr devclass 9 ,
.Xr device 9 ,
.Xr driver 9 ,
+.Xr eventhandler 9 ,
.Xr rman 9
.Rs
.%B FreeBSD Developers' Handbook
Modified: head/sys/amd64/vmm/io/iommu.c
==============================================================================
--- head/sys/amd64/vmm/io/iommu.c Tue Sep 6 20:01:15 2016 (r305496)
+++ head/sys/amd64/vmm/io/iommu.c Tue Sep 6 20:17:54 2016 (r305497)
@@ -58,6 +58,7 @@ SYSCTL_INT(_hw_vmm_iommu, OID_AUTO, enab
static struct iommu_ops *ops;
static void *host_domain;
+static eventhandler_tag add_tag, delete_tag;
static __inline int
IOMMU_INIT(void)
@@ -154,6 +155,21 @@ IOMMU_DISABLE(void)
}
static void
+iommu_pci_add(void *arg, device_t dev)
+{
+
+ /* Add new devices to the host domain. */
+ iommu_add_device(host_domain, pci_get_rid(dev));
+}
+
+static void
+iommu_pci_delete(void *arg, device_t dev)
+{
+
+ iommu_remove_device(host_domain, pci_get_rid(dev));
+}
+
+static void
iommu_init(void)
{
int error, bus, slot, func;
@@ -195,6 +211,9 @@ iommu_init(void)
*/
iommu_create_mapping(host_domain, 0, 0, maxaddr);
+ add_tag = EVENTHANDLER_REGISTER(pci_add_device, iommu_pci_add, NULL, 0);
+ delete_tag = EVENTHANDLER_REGISTER(pci_delete_device, iommu_pci_delete,
+ NULL, 0);
for (bus = 0; bus <= PCI_BUSMAX; bus++) {
for (slot = 0; slot <= PCI_SLOTMAX; slot++) {
for (func = 0; func <= PCI_FUNCMAX; func++) {
@@ -215,6 +234,15 @@ iommu_init(void)
void
iommu_cleanup(void)
{
+
+ if (add_tag != NULL) {
+ EVENTHANDLER_DEREGISTER(pci_add_device, add_tag);
+ add_tag = NULL;
+ }
+ if (delete_tag != NULL) {
+ EVENTHANDLER_DEREGISTER(pci_delete_device, delete_tag);
+ delete_tag = NULL;
+ }
IOMMU_DISABLE();
IOMMU_DESTROY_DOMAIN(host_domain);
IOMMU_CLEANUP();
Modified: head/sys/dev/pci/pci.c
==============================================================================
--- head/sys/dev/pci/pci.c Tue Sep 6 20:01:15 2016 (r305496)
+++ head/sys/dev/pci/pci.c Tue Sep 6 20:17:54 2016 (r305497)
@@ -4071,6 +4071,7 @@ pci_add_child(device_t bus, struct pci_d
pci_print_verbose(dinfo);
pci_add_resources(bus, dinfo->cfg.dev, 0, 0);
pci_child_added(dinfo->cfg.dev);
+ EVENTHANDLER_INVOKE(pci_add_device, dinfo->cfg.dev);
}
void
@@ -5312,6 +5313,8 @@ pci_child_deleted(device_t dev, device_t
dinfo = device_get_ivars(child);
rl = &dinfo->resources;
+ EVENTHANDLER_INVOKE(pci_delete_device, child);
+
/* Turn off access to resources we're about to free */
if (bus_child_present(child) != 0) {
pci_write_config(child, PCIR_COMMAND, pci_read_config(child,
Modified: head/sys/dev/pci/pcivar.h
==============================================================================
--- head/sys/dev/pci/pcivar.h Tue Sep 6 20:01:15 2016 (r305496)
+++ head/sys/dev/pci/pcivar.h Tue Sep 6 20:17:54 2016 (r305497)
@@ -31,6 +31,7 @@
#define _PCIVAR_H_
#include <sys/queue.h>
+#include <sys/eventhandler.h>
/* some PCI bus constants */
#define PCI_MAXMAPS_0 6 /* max. no. of memory/port maps */
@@ -631,4 +632,12 @@ void * vga_pci_map_bios(device_t dev, si
void vga_pci_unmap_bios(device_t dev, void *bios);
int vga_pci_repost(device_t dev);
+/**
+ * Global eventhandlers invoked when PCI devices are added or removed
+ * from the system.
+ */
+typedef void (*pci_event_fn)(void *arg, device_t dev);
+EVENTHANDLER_DECLARE(pci_add_device, pci_event_fn);
+EVENTHANDLER_DECLARE(pci_delete_device, pci_event_fn);
+
#endif /* _PCIVAR_H_ */
More information about the svn-src-all
mailing list