svn commit: r303205 - in head/sys: conf dev/cxgbe modules/cxgbe/if_cxgbe
John Baldwin
jhb at FreeBSD.org
Fri Jul 22 22:46:43 UTC 2016
Author: jhb
Date: Fri Jul 22 22:46:41 2016
New Revision: 303205
URL: https://svnweb.freebsd.org/changeset/base/303205
Log:
Add a driver to create VF devices on Chelsio T4/T5 NICs.
Chelsio NICs are a bit unique compared to some other NICs in that they
expose different functionality on different physical functions. In
particular, PF4 is used to manage the NIC interfaces ('t4nex' and 't5nex').
However, PF4 is not able to create VF devices. Instead, VFs are only
supported by physical functions 0 through 3. This commit adds 't4iov'
and 't5iov' drivers that attach to PF0-3.
One extra wrinkle is that the iov devices cannot enable SR-IOV until the
firwmare has been initialized by the main PF4 driver. To handle this
case, a new t4_if kobj interface has been added to permit cross-calls
between the PF drivers. The PF4 driver notifies sibling drivers when it
is fully attached. It also requests sibling drivers to detach before it
detaches. Sibling drivers query the PF4 driver during their attach
routine to see if it is attached. If not, the sibling drivers defer
their attach actions until the PF4 driver informs them it is attached.
VF devices are associated with a single port on the NIC. VF devices
created from PF0 are associated with the first port on the NIC, VFs
from PF1 are associated with the second port, etc. VF devices can
only be created from a PF device that has an associated port. Thus,
on a 2-port card, VFs are only supported on PF0 and PF1.
Reviewed by: np (earlier versions)
MFC after: 1 month
Sponsored by: Chelsio Communications
Added:
head/sys/dev/cxgbe/t4_if.m (contents, props changed)
head/sys/dev/cxgbe/t4_iov.c (contents, props changed)
Modified:
head/sys/conf/files
head/sys/dev/cxgbe/t4_main.c
head/sys/modules/cxgbe/if_cxgbe/Makefile
Modified: head/sys/conf/files
==============================================================================
--- head/sys/conf/files Fri Jul 22 21:52:07 2016 (r303204)
+++ head/sys/conf/files Fri Jul 22 22:46:41 2016 (r303205)
@@ -1243,6 +1243,8 @@ dev/cxgb/sys/uipc_mvec.c optional cxgb p
compile-with "${NORMAL_C} -I$S/dev/cxgb"
dev/cxgb/cxgb_t3fw.c optional cxgb cxgb_t3fw \
compile-with "${NORMAL_C} -I$S/dev/cxgb"
+dev/cxgbe/t4_iov.c optional cxgbe pci \
+ compile-with "${NORMAL_C} -I$S/dev/cxgbe"
dev/cxgbe/t4_mp_ring.c optional cxgbe pci \
compile-with "${NORMAL_C} -I$S/dev/cxgbe"
dev/cxgbe/t4_main.c optional cxgbe pci \
Added: head/sys/dev/cxgbe/t4_if.m
==============================================================================
--- /dev/null 00:00:00 1970 (empty, because file is newly added)
+++ head/sys/dev/cxgbe/t4_if.m Fri Jul 22 22:46:41 2016 (r303205)
@@ -0,0 +1,65 @@
+#-
+# Copyright (c) 2015-2016 Chelsio Communications, Inc.
+# All rights reserved.
+# Written by: John Baldwin <jhb at FreeBSD.org>
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
+#
+# $FreeBSD$
+#
+
+#include <sys/bus.h>
+
+INTERFACE t4;
+
+# The "main" device of a T4/T5 NIC is the PF4 device. Drivers for other
+# functions on the NIC need to wait for the main device to be initialized
+# before finishing attach. These routines allow drivers for other devices
+# to coordinate with the main driver for the PF4.
+
+# Called by a driver during attach to determine if the PF4 driver is
+# initialized. If the main driver is not ready, the driver should defer
+# further initialization until 'attach_child'.
+METHOD int is_main_ready {
+ device_t dev;
+};
+
+# Called by the PF4 driver on each sibling device when the PF4 driver is
+# initialized.
+METHOD int attach_child {
+ device_t dev;
+};
+
+# Called by the PF4 driver on each sibling device when the PF4 driver is
+# preparing to detach.
+METHOD int detach_child {
+ device_t dev;
+};
+
+# Called by a driver to query the PF4 driver for the unit number to use
+# for a given port. If the port is not enabled on the adapter, this
+# will fail.
+METHOD int read_port_unit {
+ device_t dev;
+ int port;
+ int *unit;
+};
Added: head/sys/dev/cxgbe/t4_iov.c
==============================================================================
--- /dev/null 00:00:00 1970 (empty, because file is newly added)
+++ head/sys/dev/cxgbe/t4_iov.c Fri Jul 22 22:46:41 2016 (r303205)
@@ -0,0 +1,289 @@
+/*-
+ * Copyright (c) 2015-2016 Chelsio Communications, Inc.
+ * All rights reserved.
+ * Written by: John Baldwin <jhb at FreeBSD.org>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/bus.h>
+#include <sys/kernel.h>
+#include <sys/module.h>
+#include <sys/systm.h>
+#include <dev/pci/pcivar.h>
+
+#ifdef PCI_IOV
+#include <sys/nv.h>
+#include <sys/iov_schema.h>
+#include <dev/pci/pci_iov.h>
+#endif
+
+#include "t4_if.h"
+
+struct t4iov_softc {
+ device_t sc_dev;
+ device_t sc_main;
+ bool sc_attached;
+};
+
+struct {
+ uint16_t device;
+ char *desc;
+} t4iov_pciids[] = {
+ {0x4000, "Chelsio T440-dbg"},
+ {0x4001, "Chelsio T420-CR"},
+ {0x4002, "Chelsio T422-CR"},
+ {0x4003, "Chelsio T440-CR"},
+ {0x4004, "Chelsio T420-BCH"},
+ {0x4005, "Chelsio T440-BCH"},
+ {0x4006, "Chelsio T440-CH"},
+ {0x4007, "Chelsio T420-SO"},
+ {0x4008, "Chelsio T420-CX"},
+ {0x4009, "Chelsio T420-BT"},
+ {0x400a, "Chelsio T404-BT"},
+ {0x400e, "Chelsio T440-LP-CR"},
+}, t5iov_pciids[] = {
+ {0x5000, "Chelsio T580-dbg"},
+ {0x5001, "Chelsio T520-CR"}, /* 2 x 10G */
+ {0x5002, "Chelsio T522-CR"}, /* 2 x 10G, 2 X 1G */
+ {0x5003, "Chelsio T540-CR"}, /* 4 x 10G */
+ {0x5007, "Chelsio T520-SO"}, /* 2 x 10G, nomem */
+ {0x5009, "Chelsio T520-BT"}, /* 2 x 10GBaseT */
+ {0x500a, "Chelsio T504-BT"}, /* 4 x 1G */
+ {0x500d, "Chelsio T580-CR"}, /* 2 x 40G */
+ {0x500e, "Chelsio T540-LP-CR"}, /* 4 x 10G */
+ {0x5010, "Chelsio T580-LP-CR"}, /* 2 x 40G */
+ {0x5011, "Chelsio T520-LL-CR"}, /* 2 x 10G */
+ {0x5012, "Chelsio T560-CR"}, /* 1 x 40G, 2 x 10G */
+ {0x5014, "Chelsio T580-LP-SO-CR"}, /* 2 x 40G, nomem */
+ {0x5015, "Chelsio T502-BT"}, /* 2 x 1G */
+#ifdef notyet
+ {0x5004, "Chelsio T520-BCH"},
+ {0x5005, "Chelsio T540-BCH"},
+ {0x5006, "Chelsio T540-CH"},
+ {0x5008, "Chelsio T520-CX"},
+ {0x500b, "Chelsio B520-SR"},
+ {0x500c, "Chelsio B504-BT"},
+ {0x500f, "Chelsio Amsterdam"},
+ {0x5013, "Chelsio T580-CHR"},
+#endif
+};
+
+static int t4iov_attach_child(device_t dev);
+
+static int
+t4iov_probe(device_t dev)
+{
+ uint16_t d;
+ size_t i;
+
+ d = pci_get_device(dev);
+ for (i = 0; i < nitems(t4iov_pciids); i++) {
+ if (d == t4iov_pciids[i].device) {
+ device_set_desc(dev, t4iov_pciids[i].desc);
+ return (BUS_PROBE_DEFAULT);
+ }
+ }
+ return (ENXIO);
+}
+
+static int
+t5iov_probe(device_t dev)
+{
+ uint16_t d;
+ size_t i;
+
+ d = pci_get_device(dev);
+ for (i = 0; i < nitems(t5iov_pciids); i++) {
+ if (d == t5iov_pciids[i].device) {
+ device_set_desc(dev, t5iov_pciids[i].desc);
+ return (BUS_PROBE_DEFAULT);
+ }
+ }
+ return (ENXIO);
+}
+
+static int
+t4iov_attach(device_t dev)
+{
+ struct t4iov_softc *sc;
+
+ sc = device_get_softc(dev);
+ sc->sc_dev = dev;
+
+ sc->sc_main = pci_find_dbsf(pci_get_domain(dev), pci_get_bus(dev),
+ pci_get_slot(dev), 4);
+ if (T4_IS_MAIN_READY(sc->sc_main) == 0)
+ return (t4iov_attach_child(dev));
+ return (0);
+}
+
+static int
+t4iov_attach_child(device_t dev)
+{
+ struct t4iov_softc *sc;
+#ifdef PCI_IOV
+ nvlist_t *pf_schema, *vf_schema;
+#endif
+ int error, unit;
+
+ sc = device_get_softc(dev);
+ MPASS(!sc->sc_attached);
+
+ /*
+ * PF0-3 are associated with a specific port on the NIC (PF0
+ * with port 0, etc.). Ask the PF4 driver for the unit number
+ * for this function's associated port to determine if the port
+ * is present.
+ */
+ error = T4_READ_PORT_UNIT(sc->sc_main, pci_get_function(dev), &unit);
+ if (error)
+ return (0);
+
+#ifdef PCI_IOV
+ pf_schema = pci_iov_schema_alloc_node();
+ vf_schema = pci_iov_schema_alloc_node();
+ error = pci_iov_attach(dev, pf_schema, vf_schema);
+ if (error) {
+ device_printf(dev, "Failed to initialize SR-IOV: %d\n", error);
+ return (0);
+ }
+#endif
+
+ sc->sc_attached = true;
+ return (0);
+}
+
+static int
+t4iov_detach_child(device_t dev)
+{
+ struct t4iov_softc *sc;
+#ifdef PCI_IOV
+ int error;
+#endif
+
+ sc = device_get_softc(dev);
+ if (!sc->sc_attached)
+ return (0);
+
+#ifdef PCI_IOV
+ error = pci_iov_detach(dev);
+ if (error != 0) {
+ device_printf(dev, "Failed to disable SR-IOV\n");
+ return (error);
+ }
+#endif
+
+ sc->sc_attached = false;
+ return (0);
+}
+
+static int
+t4iov_detach(device_t dev)
+{
+ struct t4iov_softc *sc;
+
+ sc = device_get_softc(dev);
+ if (sc->sc_attached)
+ return (t4iov_detach_child(dev));
+ return (0);
+}
+
+#ifdef PCI_IOV
+static int
+t4iov_iov_init(device_t dev, uint16_t num_vfs, const struct nvlist *config)
+{
+
+ /* XXX: The Linux driver sets up a vf_monitor task on T4 adapters. */
+ return (0);
+}
+
+static void
+t4iov_iov_uninit(device_t dev)
+{
+}
+
+static int
+t4iov_add_vf(device_t dev, uint16_t vfnum, const struct nvlist *config)
+{
+
+ return (0);
+}
+#endif
+
+static device_method_t t4iov_methods[] = {
+ DEVMETHOD(device_probe, t4iov_probe),
+ DEVMETHOD(device_attach, t4iov_attach),
+ DEVMETHOD(device_detach, t4iov_detach),
+
+#ifdef PCI_IOV
+ DEVMETHOD(pci_iov_init, t4iov_iov_init),
+ DEVMETHOD(pci_iov_uninit, t4iov_iov_uninit),
+ DEVMETHOD(pci_iov_add_vf, t4iov_add_vf),
+#endif
+
+ DEVMETHOD(t4_attach_child, t4iov_attach_child),
+ DEVMETHOD(t4_detach_child, t4iov_detach_child),
+
+ DEVMETHOD_END
+};
+
+static driver_t t4iov_driver = {
+ "t4iov",
+ t4iov_methods,
+ sizeof(struct t4iov_softc)
+};
+
+static device_method_t t5iov_methods[] = {
+ DEVMETHOD(device_probe, t5iov_probe),
+ DEVMETHOD(device_attach, t4iov_attach),
+ DEVMETHOD(device_detach, t4iov_detach),
+
+#ifdef PCI_IOV
+ DEVMETHOD(pci_iov_init, t4iov_iov_init),
+ DEVMETHOD(pci_iov_uninit, t4iov_iov_uninit),
+ DEVMETHOD(pci_iov_add_vf, t4iov_add_vf),
+#endif
+
+ DEVMETHOD(t4_attach_child, t4iov_attach_child),
+ DEVMETHOD(t4_detach_child, t4iov_detach_child),
+
+ DEVMETHOD_END
+};
+
+static driver_t t5iov_driver = {
+ "t5iov",
+ t5iov_methods,
+ sizeof(struct t4iov_softc)
+};
+
+static devclass_t t4iov_devclass, t5iov_devclass;
+
+DRIVER_MODULE(t4iov, pci, t4iov_driver, t4iov_devclass, 0, 0);
+MODULE_VERSION(t4iov, 1);
+
+DRIVER_MODULE(t5iov, pci, t5iov_driver, t5iov_devclass, 0, 0);
+MODULE_VERSION(t5iov, 1);
Modified: head/sys/dev/cxgbe/t4_main.c
==============================================================================
--- head/sys/dev/cxgbe/t4_main.c Fri Jul 22 21:52:07 2016 (r303204)
+++ head/sys/dev/cxgbe/t4_main.c Fri Jul 22 22:46:41 2016 (r303205)
@@ -76,16 +76,22 @@ __FBSDID("$FreeBSD$");
#include "t4_ioctl.h"
#include "t4_l2t.h"
#include "t4_mp_ring.h"
+#include "t4_if.h"
/* T4 bus driver interface */
static int t4_probe(device_t);
static int t4_attach(device_t);
static int t4_detach(device_t);
+static int t4_ready(device_t);
+static int t4_read_port_unit(device_t, int, int *);
static device_method_t t4_methods[] = {
DEVMETHOD(device_probe, t4_probe),
DEVMETHOD(device_attach, t4_attach),
DEVMETHOD(device_detach, t4_detach),
+ DEVMETHOD(t4_is_main_ready, t4_ready),
+ DEVMETHOD(t4_read_port_unit, t4_read_port_unit),
+
DEVMETHOD_END
};
static driver_t t4_driver = {
@@ -147,6 +153,9 @@ static device_method_t t5_methods[] = {
DEVMETHOD(device_attach, t4_attach),
DEVMETHOD(device_detach, t4_detach),
+ DEVMETHOD(t4_is_main_ready, t4_ready),
+ DEVMETHOD(t4_read_port_unit, t4_read_port_unit),
+
DEVMETHOD_END
};
static driver_t t5_driver = {
@@ -533,6 +542,7 @@ static int set_sched_queue(struct adapte
static int toe_capability(struct vi_info *, int);
#endif
static int mod_event(module_t, int, void *);
+static int notify_siblings(device_t, int);
struct {
uint16_t device;
@@ -1062,6 +1072,8 @@ t4_attach(device_t dev)
t4_set_desc(sc);
+ notify_siblings(dev, 0);
+
done:
if (rc != 0 && sc->cdev) {
/* cdev was created and so cxgbetool works; recover that way. */
@@ -1078,6 +1090,57 @@ done:
return (rc);
}
+static int
+t4_ready(device_t dev)
+{
+ struct adapter *sc;
+
+ sc = device_get_softc(dev);
+ if (sc->flags & FW_OK)
+ return (0);
+ return (ENXIO);
+}
+
+static int
+t4_read_port_unit(device_t dev, int port, int *unit)
+{
+ struct adapter *sc;
+ struct port_info *pi;
+
+ sc = device_get_softc(dev);
+ if (port < 0 || port >= MAX_NPORTS)
+ return (EINVAL);
+ pi = sc->port[port];
+ if (pi == NULL || pi->dev == NULL)
+ return (ENXIO);
+ *unit = device_get_unit(pi->dev);
+ return (0);
+}
+
+static int
+notify_siblings(device_t dev, int detaching)
+{
+ device_t sibling;
+ int error, i;
+
+ error = 0;
+ for (i = 0; i < PCI_FUNCMAX; i++) {
+ if (i == pci_get_function(dev))
+ continue;
+ sibling = pci_find_dbsf(pci_get_domain(dev), pci_get_bus(dev),
+ pci_get_slot(dev), i);
+ if (sibling == NULL || !device_is_attached(sibling))
+ continue;
+ if (detaching)
+ error = T4_DETACH_CHILD(sibling);
+ else
+ (void)T4_ATTACH_CHILD(sibling);
+ if (error)
+ break;
+ }
+ return (error);
+}
+
/*
* Idempotent
*/
@@ -1090,6 +1153,13 @@ t4_detach(device_t dev)
sc = device_get_softc(dev);
+ rc = notify_siblings(dev, 1);
+ if (rc) {
+ device_printf(dev,
+ "failed to detach sibling devices: %d\n", rc);
+ return (rc);
+ }
+
if (sc->flags & FULL_INIT_DONE)
t4_intr_disable(sc);
Modified: head/sys/modules/cxgbe/if_cxgbe/Makefile
==============================================================================
--- head/sys/modules/cxgbe/if_cxgbe/Makefile Fri Jul 22 21:52:07 2016 (r303204)
+++ head/sys/modules/cxgbe/if_cxgbe/Makefile Fri Jul 22 22:46:41 2016 (r303205)
@@ -13,8 +13,10 @@ SRCS+= opt_inet.h
SRCS+= opt_inet6.h
SRCS+= opt_ofed.h
SRCS+= opt_rss.h
-SRCS+= pci_if.h
+SRCS+= pci_if.h pci_iov_if.h
+SRCS+= t4_if.c t4_if.h
SRCS+= t4_hw.c
+SRCS+= t4_iov.c
SRCS+= t4_l2t.c
SRCS+= t4_main.c
SRCS+= t4_mp_ring.c
More information about the svn-src-head
mailing list