svn commit: r234432 - in user/np/toe_iwarp/sys: dev/cxgb dev/cxgb/common dev/cxgb/sys dev/cxgb/ulp/iw_cxgb dev/cxgb/ulp/tom modules/cxgb modules/cxgb/cxgb modules/cxgb/iw_cxgb modules/cxgb/toecore ...

Navdeep Parhar np at FreeBSD.org
Wed Apr 18 18:43:02 UTC 2012


Author: np
Date: Wed Apr 18 18:43:01 2012
New Revision: 234432
URL: http://svn.freebsd.org/changeset/base/234432

Log:
  T3 TCP offload and iWARP driver.

Added:
  user/np/toe_iwarp/sys/dev/cxgb/ulp/iw_cxgb/iw_cxgb_ib_intfc.h   (contents, props changed)
Deleted:
  user/np/toe_iwarp/sys/dev/cxgb/cxgb_offload.c
  user/np/toe_iwarp/sys/dev/cxgb/t3cdev.h
  user/np/toe_iwarp/sys/dev/cxgb/ulp/tom/cxgb_cpl_socket.c
  user/np/toe_iwarp/sys/dev/cxgb/ulp/tom/cxgb_ddp.c
  user/np/toe_iwarp/sys/dev/cxgb/ulp/tom/cxgb_defs.h
  user/np/toe_iwarp/sys/dev/cxgb/ulp/tom/cxgb_t3_ddp.h
  user/np/toe_iwarp/sys/dev/cxgb/ulp/tom/cxgb_tcp.h
  user/np/toe_iwarp/sys/dev/cxgb/ulp/tom/cxgb_tcp_offload.c
  user/np/toe_iwarp/sys/dev/cxgb/ulp/tom/cxgb_tcp_offload.h
  user/np/toe_iwarp/sys/dev/cxgb/ulp/tom/cxgb_tom_sysctl.c
  user/np/toe_iwarp/sys/modules/cxgb/toecore/
Modified:
  user/np/toe_iwarp/sys/dev/cxgb/common/cxgb_ctl_defs.h
  user/np/toe_iwarp/sys/dev/cxgb/cxgb_adapter.h
  user/np/toe_iwarp/sys/dev/cxgb/cxgb_main.c
  user/np/toe_iwarp/sys/dev/cxgb/cxgb_offload.h
  user/np/toe_iwarp/sys/dev/cxgb/cxgb_osdep.h
  user/np/toe_iwarp/sys/dev/cxgb/cxgb_sge.c
  user/np/toe_iwarp/sys/dev/cxgb/sys/mvec.h
  user/np/toe_iwarp/sys/dev/cxgb/ulp/iw_cxgb/iw_cxgb.c
  user/np/toe_iwarp/sys/dev/cxgb/ulp/iw_cxgb/iw_cxgb.h
  user/np/toe_iwarp/sys/dev/cxgb/ulp/iw_cxgb/iw_cxgb_cm.c
  user/np/toe_iwarp/sys/dev/cxgb/ulp/iw_cxgb/iw_cxgb_cm.h
  user/np/toe_iwarp/sys/dev/cxgb/ulp/iw_cxgb/iw_cxgb_cq.c
  user/np/toe_iwarp/sys/dev/cxgb/ulp/iw_cxgb/iw_cxgb_dbg.c
  user/np/toe_iwarp/sys/dev/cxgb/ulp/iw_cxgb/iw_cxgb_ev.c
  user/np/toe_iwarp/sys/dev/cxgb/ulp/iw_cxgb/iw_cxgb_hal.c
  user/np/toe_iwarp/sys/dev/cxgb/ulp/iw_cxgb/iw_cxgb_hal.h
  user/np/toe_iwarp/sys/dev/cxgb/ulp/iw_cxgb/iw_cxgb_mem.c
  user/np/toe_iwarp/sys/dev/cxgb/ulp/iw_cxgb/iw_cxgb_provider.c
  user/np/toe_iwarp/sys/dev/cxgb/ulp/iw_cxgb/iw_cxgb_provider.h
  user/np/toe_iwarp/sys/dev/cxgb/ulp/iw_cxgb/iw_cxgb_qp.c
  user/np/toe_iwarp/sys/dev/cxgb/ulp/iw_cxgb/iw_cxgb_resource.c
  user/np/toe_iwarp/sys/dev/cxgb/ulp/iw_cxgb/iw_cxgb_user.h
  user/np/toe_iwarp/sys/dev/cxgb/ulp/iw_cxgb/iw_cxgb_wr.h
  user/np/toe_iwarp/sys/dev/cxgb/ulp/tom/cxgb_cpl_io.c
  user/np/toe_iwarp/sys/dev/cxgb/ulp/tom/cxgb_l2t.c
  user/np/toe_iwarp/sys/dev/cxgb/ulp/tom/cxgb_l2t.h
  user/np/toe_iwarp/sys/dev/cxgb/ulp/tom/cxgb_listen.c
  user/np/toe_iwarp/sys/dev/cxgb/ulp/tom/cxgb_toepcb.h
  user/np/toe_iwarp/sys/dev/cxgb/ulp/tom/cxgb_tom.c
  user/np/toe_iwarp/sys/dev/cxgb/ulp/tom/cxgb_tom.h
  user/np/toe_iwarp/sys/modules/cxgb/Makefile
  user/np/toe_iwarp/sys/modules/cxgb/cxgb/Makefile
  user/np/toe_iwarp/sys/modules/cxgb/iw_cxgb/Makefile
  user/np/toe_iwarp/sys/modules/cxgb/tom/Makefile

Modified: user/np/toe_iwarp/sys/dev/cxgb/common/cxgb_ctl_defs.h
==============================================================================
--- user/np/toe_iwarp/sys/dev/cxgb/common/cxgb_ctl_defs.h	Wed Apr 18 18:37:56 2012	(r234431)
+++ user/np/toe_iwarp/sys/dev/cxgb/common/cxgb_ctl_defs.h	Wed Apr 18 18:43:01 2012	(r234432)
@@ -60,14 +60,12 @@ struct mtutab {
 	const unsigned short *mtus; /* the MTU table values */
 };
 
-struct net_device;
-
 /*
- * Structure used to request the adapter net_device owning a given MAC address.
+ * Structure used to request the ifnet that owns a given MAC address.
  */
 struct iff_mac {
-	struct net_device *dev;          /* the net_device */
-	const unsigned char *mac_addr;   /* MAC address to lookup */
+	struct ifnet *dev;
+	const unsigned char *mac_addr;
 	u16 vlan_tag;
 };
 
@@ -85,7 +83,7 @@ struct ddp_params {
 
 struct adap_ports {
 	unsigned int nports;     /* number of ports on this adapter */
-	struct net_device *lldevs[MAX_NPORTS];
+	struct ifnet *lldevs[MAX_NPORTS];
 };
 
 /*

Modified: user/np/toe_iwarp/sys/dev/cxgb/cxgb_adapter.h
==============================================================================
--- user/np/toe_iwarp/sys/dev/cxgb/cxgb_adapter.h	Wed Apr 18 18:37:56 2012	(r234431)
+++ user/np/toe_iwarp/sys/dev/cxgb/cxgb_adapter.h	Wed Apr 18 18:43:01 2012	(r234432)
@@ -56,7 +56,6 @@ $FreeBSD$
 #include <dev/pci/pcivar.h>
 
 #include <cxgb_osdep.h>
-#include <t3cdev.h>
 #include <sys/mbufq.h>
 
 struct adapter;
@@ -129,6 +128,7 @@ enum {
 	CXGB_OFLD_INIT	= (1 << 7),
 	TP_PARITY_INIT	= (1 << 8),
 	CXGB_BUSY	= (1 << 9),
+	TOM_INIT_DONE	= (1 << 10),
 
 	/* port flags */
 	DOOMED		= (1 << 0),
@@ -178,7 +178,6 @@ struct sge_rspq {
 	uint32_t        async_notif;
 	uint32_t	cntxt_id;
 	uint32_t        offload_pkts;
-	uint32_t        offload_bundles;
 	uint32_t        pure_rsps;
 	uint32_t        unhandled_irqs;
 	uint32_t        starved;
@@ -290,6 +289,7 @@ struct sge_qset {
 	uint32_t                txq_stopped;       /* which Tx queues are stopped */
 	uint64_t                port_stats[SGE_PSTAT_MAX];
 	struct port_info        *port;
+	struct adapter          *adap;
 	int                     idx; /* qset # */
 	int                     qs_flags;
 	int			coalescing;
@@ -306,10 +306,13 @@ struct sge {
 
 struct filter_info;
 
+typedef int (*cpl_handler_t)(struct sge_qset *, struct rsp_desc *,
+    struct mbuf *);
+
 struct adapter {
+	SLIST_ENTRY(adapter)	link;
 	device_t		dev;
 	int			flags;
-	TAILQ_ENTRY(adapter)    adapter_entry;
 
 	/* PCI register resources */
 	int			regs_rid;
@@ -375,11 +378,16 @@ struct adapter {
 
 	struct port_info	port[MAX_NPORTS];
 	device_t		portdev[MAX_NPORTS];
-	struct t3cdev           tdev;
+#ifdef TCP_OFFLOAD
+	void 			*tom_softc;
+	void 			*iwarp_softc;
+#endif
 	char                    fw_version[64];
 	char                    port_types[MAX_NPORTS + 1];
 	uint32_t                open_device_map;
-	uint32_t                registered_device_map;
+#ifdef TCP_OFFLOAD
+	int			offload_map;
+#endif
 	struct mtx              lock;
 	driver_intr_t           *cxgb_intr;
 	int                     msi_count;
@@ -391,6 +399,11 @@ struct adapter {
 	char                    elmerlockbuf[ADAPTER_LOCK_NAME_LEN];
 
 	int			timestamp;
+
+#ifdef TCP_OFFLOAD
+#define NUM_CPL_HANDLERS	0xa7
+	cpl_handler_t cpl_handler[NUM_CPL_HANDLERS] __aligned(CACHE_LINE_SIZE);
+#endif
 };
 
 struct t3_rx_mode {
@@ -501,10 +514,12 @@ void t3_os_link_changed(adapter_t *adapt
 			int speed, int duplex, int fc, int mac_was_reset);
 void t3_os_phymod_changed(struct adapter *adap, int port_id);
 void t3_sge_err_intr_handler(adapter_t *adapter);
-int t3_offload_tx(struct t3cdev *, struct mbuf *);
+#ifdef TCP_OFFLOAD
+int t3_offload_tx(struct adapter *, struct mbuf *);
+#endif
 void t3_os_set_hw_addr(adapter_t *adapter, int port_idx, u8 hw_addr[]);
 int t3_mgmt_tx(adapter_t *adap, struct mbuf *m);
-
+int t3_register_cpl_handler(struct adapter *, int, cpl_handler_t);
 
 int t3_sge_alloc(struct adapter *);
 int t3_sge_free(struct adapter *);
@@ -555,15 +570,9 @@ txq_to_qset(struct sge_txq *q, int qidx)
 	return container_of(q, struct sge_qset, txq[qidx]);
 }
 
-static __inline struct adapter *
-tdev2adap(struct t3cdev *d)
-{
-	return container_of(d, struct adapter, tdev);
-}
-
 #undef container_of
 
-#define OFFLOAD_DEVMAP_BIT 15
+#define OFFLOAD_DEVMAP_BIT (1 << MAX_NPORTS)
 static inline int offload_running(adapter_t *adapter)
 {
         return isset(&adapter->open_device_map, OFFLOAD_DEVMAP_BIT);
@@ -572,4 +581,5 @@ static inline int offload_running(adapte
 void cxgb_tx_watchdog(void *arg);
 int cxgb_transmit(struct ifnet *ifp, struct mbuf *m);
 void cxgb_qflush(struct ifnet *ifp);
+void t3_iterate(void (*)(struct adapter *, void *), void *);
 #endif

Modified: user/np/toe_iwarp/sys/dev/cxgb/cxgb_main.c
==============================================================================
--- user/np/toe_iwarp/sys/dev/cxgb/cxgb_main.c	Wed Apr 18 18:37:56 2012	(r234431)
+++ user/np/toe_iwarp/sys/dev/cxgb/cxgb_main.c	Wed Apr 18 18:43:01 2012	(r234432)
@@ -30,6 +30,8 @@ POSSIBILITY OF SUCH DAMAGE.
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
+#include "opt_inet.h"
+
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/kernel.h>
@@ -107,6 +109,9 @@ static inline void mk_set_tcb_field(stru
     unsigned int, u64, u64);
 static inline void set_tcb_field_ulp(struct cpl_set_tcb_field *, unsigned int,
     unsigned int, u64, u64);
+#ifdef TCP_OFFLOAD
+static int cpl_not_handled(struct sge_qset *, struct rsp_desc *, struct mbuf *);
+#endif
 
 /* Attachment glue for the PCI controller end of the device.  Each port of
  * the device is attached separately, as defined later.
@@ -119,10 +124,11 @@ static __inline void reg_block_dump(stru
     unsigned int end);
 static void cxgb_get_regs(adapter_t *sc, struct ch_ifconf_regs *regs, uint8_t *buf);
 static int cxgb_get_regs_len(void);
-static int offload_open(struct port_info *pi);
 static void touch_bars(device_t dev);
-static int offload_close(struct t3cdev *tdev);
 static void cxgb_update_mac_settings(struct port_info *p);
+#ifdef TCP_OFFLOAD
+static int toe_capability(struct port_info *, int);
+#endif
 
 static device_method_t cxgb_controller_methods[] = {
 	DEVMETHOD(device_probe,		cxgb_controller_probe),
@@ -138,8 +144,11 @@ static driver_t cxgb_controller_driver =
 	sizeof(struct adapter)
 };
 
+static int cxgbc_mod_event(module_t, int, void *);
 static devclass_t	cxgb_controller_devclass;
-DRIVER_MODULE(cxgbc, pci, cxgb_controller_driver, cxgb_controller_devclass, 0, 0);
+DRIVER_MODULE(cxgbc, pci, cxgb_controller_driver, cxgb_controller_devclass,
+    cxgbc_mod_event, 0);
+MODULE_VERSION(cxgbc, 1);
 
 /*
  * Attachment glue for the ports.  Attachment is done directly to the
@@ -177,6 +186,14 @@ static struct cdevsw cxgb_cdevsw = {
 
 static devclass_t	cxgb_port_devclass;
 DRIVER_MODULE(cxgb, cxgbc, cxgb_port_driver, cxgb_port_devclass, 0, 0);
+MODULE_VERSION(cxgb, 1);
+
+static struct mtx t3_list_lock;
+static SLIST_HEAD(, adapter) t3_list;
+#ifdef TCP_OFFLOAD
+static struct mtx t3_uld_list_lock;
+static SLIST_HEAD(, uld_info) t3_uld_list;
+#endif
 
 /*
  * The driver uses the best interrupt scheme available on a platform in the
@@ -195,15 +212,6 @@ SYSCTL_INT(_hw_cxgb, OID_AUTO, msi_allow
     "MSI-X, MSI, INTx selector");
 
 /*
- * The driver enables offload as a default.
- * To disable it, use ofld_disable = 1.
- */
-static int ofld_disable = 0;
-TUNABLE_INT("hw.cxgb.ofld_disable", &ofld_disable);
-SYSCTL_INT(_hw_cxgb, OID_AUTO, ofld_disable, CTLFLAG_RDTUN, &ofld_disable, 0,
-    "disable ULP offload");
-
-/*
  * The driver uses an auto-queue algorithm by default.
  * To disable it and force a single queue-set per port, use multiq = 0
  */
@@ -445,6 +453,25 @@ cxgb_controller_attach(device_t dev)
 	sc->msi_count = 0;
 	ai = cxgb_get_adapter_info(dev);
 
+	snprintf(sc->lockbuf, ADAPTER_LOCK_NAME_LEN, "cxgb controller lock %d",
+	    device_get_unit(dev));
+	ADAPTER_LOCK_INIT(sc, sc->lockbuf);
+
+	snprintf(sc->reglockbuf, ADAPTER_LOCK_NAME_LEN, "SGE reg lock %d",
+	    device_get_unit(dev));
+	snprintf(sc->mdiolockbuf, ADAPTER_LOCK_NAME_LEN, "cxgb mdio lock %d",
+	    device_get_unit(dev));
+	snprintf(sc->elmerlockbuf, ADAPTER_LOCK_NAME_LEN, "cxgb elmer lock %d",
+	    device_get_unit(dev));
+	
+	MTX_INIT(&sc->sge.reg_lock, sc->reglockbuf, NULL, MTX_SPIN);
+	MTX_INIT(&sc->mdio_lock, sc->mdiolockbuf, NULL, MTX_DEF);
+	MTX_INIT(&sc->elmer_lock, sc->elmerlockbuf, NULL, MTX_DEF);
+
+	mtx_lock(&t3_list_lock);
+	SLIST_INSERT_HEAD(&t3_list, sc, link);
+	mtx_unlock(&t3_list_lock);
+
 	/* find the PCIe link width and set max read request to 4KB*/
 	if (pci_find_cap(dev, PCIY_EXPRESS, &reg) == 0) {
 		uint16_t lnk;
@@ -471,24 +498,10 @@ cxgb_controller_attach(device_t dev)
 	if ((sc->regs_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
 	    &sc->regs_rid, RF_ACTIVE)) == NULL) {
 		device_printf(dev, "Cannot allocate BAR region 0\n");
-		return (ENXIO);
+		error = ENXIO;
+		goto out;
 	}
 
-	snprintf(sc->lockbuf, ADAPTER_LOCK_NAME_LEN, "cxgb controller lock %d",
-	    device_get_unit(dev));
-	ADAPTER_LOCK_INIT(sc, sc->lockbuf);
-
-	snprintf(sc->reglockbuf, ADAPTER_LOCK_NAME_LEN, "SGE reg lock %d",
-	    device_get_unit(dev));
-	snprintf(sc->mdiolockbuf, ADAPTER_LOCK_NAME_LEN, "cxgb mdio lock %d",
-	    device_get_unit(dev));
-	snprintf(sc->elmerlockbuf, ADAPTER_LOCK_NAME_LEN, "cxgb elmer lock %d",
-	    device_get_unit(dev));
-	
-	MTX_INIT(&sc->sge.reg_lock, sc->reglockbuf, NULL, MTX_SPIN);
-	MTX_INIT(&sc->mdio_lock, sc->mdiolockbuf, NULL, MTX_DEF);
-	MTX_INIT(&sc->elmer_lock, sc->elmerlockbuf, NULL, MTX_DEF);
-	
 	sc->bt = rman_get_bustag(sc->regs_res);
 	sc->bh = rman_get_bushandle(sc->regs_res);
 	sc->mmio_len = rman_get_size(sc->regs_res);
@@ -604,7 +617,7 @@ cxgb_controller_attach(device_t dev)
 	} else {
 		sc->flags |= TPS_UPTODATE;
 	}
-	
+
 	/*
 	 * Create a child device for each MAC.  The ethernet attachment
 	 * will be done in these children.
@@ -636,12 +649,7 @@ cxgb_controller_attach(device_t dev)
 	t3_sge_init_adapter(sc);
 
 	t3_led_ready(sc);
-	
-	cxgb_offload_init();
-	if (is_offload(sc)) {
-		setbit(&sc->registered_device_map, OFFLOAD_DEVMAP_BIT);
-		cxgb_adapter_ofld(sc);
-        }
+
 	error = t3_get_fw_version(sc, &vers);
 	if (error)
 		goto out;
@@ -662,6 +670,11 @@ cxgb_controller_attach(device_t dev)
 	device_printf(sc->dev, "Firmware Version %s\n", &sc->fw_version[0]);
 	callout_reset(&sc->cxgb_tick_ch, hz, cxgb_tick, sc);
 	t3_add_attach_sysctls(sc);
+
+#ifdef TCP_OFFLOAD
+	for (i = 0; i < NUM_CPL_HANDLERS; i++)
+		sc->cpl_handler[i] = cpl_not_handled;
+#endif
 out:
 	if (error)
 		cxgb_free(sc);
@@ -775,20 +788,9 @@ cxgb_free(struct adapter *sc)
 		sc->tq = NULL;
 	}
 	
-	if (is_offload(sc)) {
-		clrbit(&sc->registered_device_map, OFFLOAD_DEVMAP_BIT);
-		cxgb_adapter_unofld(sc);
-	}
-
-#ifdef notyet
-	if (sc->flags & CXGB_OFLD_INIT)
-		cxgb_offload_deactivate(sc);
-#endif
 	free(sc->filters, M_DEVBUF);
 	t3_sge_free(sc);
 
-	cxgb_offload_exit();
-
 	if (sc->udbs_res != NULL)
 		bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->udbs_rid,
 		    sc->udbs_res);
@@ -800,6 +802,9 @@ cxgb_free(struct adapter *sc)
 	MTX_DESTROY(&sc->mdio_lock);
 	MTX_DESTROY(&sc->sge.reg_lock);
 	MTX_DESTROY(&sc->elmer_lock);
+	mtx_lock(&t3_list_lock);
+	SLIST_REMOVE(&t3_list, sc, adapter, link);
+	mtx_unlock(&t3_list_lock);
 	ADAPTER_LOCK_DEINIT(sc);
 }
 
@@ -1017,6 +1022,10 @@ cxgb_port_attach(device_t dev)
 	ifp->if_qflush = cxgb_qflush;
 
 	ifp->if_capabilities = CXGB_CAP;
+#ifdef TCP_OFFLOAD
+	if (is_offload(sc))
+		ifp->if_capabilities |= IFCAP_TOE4;
+#endif
 	ifp->if_capenable = CXGB_CAP_ENABLE;
 	ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO;
 
@@ -1420,65 +1429,6 @@ setup_rss(adapter_t *adap)
 	              cpus, rspq_map);
 
 }
-
-/*
- * Sends an mbuf to an offload queue driver
- * after dealing with any active network taps.
- */
-static inline int
-offload_tx(struct t3cdev *tdev, struct mbuf *m)
-{
-	int ret;
-
-	ret = t3_offload_tx(tdev, m);
-	return (ret);
-}
-
-static int
-write_smt_entry(struct adapter *adapter, int idx)
-{
-	struct port_info *pi = &adapter->port[idx];
-	struct cpl_smt_write_req *req;
-	struct mbuf *m;
-
-	if ((m = m_gethdr(M_NOWAIT, MT_DATA)) == NULL)
-		return (ENOMEM);
-
-	req = mtod(m, struct cpl_smt_write_req *);
-	m->m_pkthdr.len = m->m_len = sizeof(struct cpl_smt_write_req);
-	
-	req->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
-	OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SMT_WRITE_REQ, idx));
-	req->mtu_idx = NMTUS - 1;  /* should be 0 but there's a T3 bug */
-	req->iff = idx;
-	memset(req->src_mac1, 0, sizeof(req->src_mac1));
-	memcpy(req->src_mac0, pi->hw_addr, ETHER_ADDR_LEN);
-
-	m_set_priority(m, 1);
-
-	offload_tx(&adapter->tdev, m);
-
-	return (0);
-}
-
-static int
-init_smt(struct adapter *adapter)
-{
-	int i;
-
-	for_each_port(adapter, i)
-		write_smt_entry(adapter, i);
-	return 0;
-}
-
-static void
-init_port_mtus(adapter_t *adapter)
-{
-	unsigned int mtus = ETHERMTU | (ETHERMTU << 16);
-
-	t3_write_reg(adapter, A_TP_MTU_PORT_TABLE, mtus);
-}
-
 static void
 send_pktsched_cmd(struct adapter *adap, int sched, int qidx, int lo,
 			      int hi, int port)
@@ -1705,45 +1655,6 @@ cxgb_down(struct adapter *sc)
 	t3_intr_disable(sc);
 }
 
-static int
-offload_open(struct port_info *pi)
-{
-	struct adapter *sc = pi->adapter;
-	struct t3cdev *tdev = &sc->tdev;
-
-	setbit(&sc->open_device_map, OFFLOAD_DEVMAP_BIT);
-
-	t3_tp_set_offload_mode(sc, 1);
-	tdev->lldev = pi->ifp;
-	init_port_mtus(sc);
-	t3_load_mtus(sc, sc->params.mtus, sc->params.a_wnd, sc->params.b_wnd,
-		     sc->params.rev == 0 ?  sc->port[0].ifp->if_mtu : 0xffff);
-	init_smt(sc);
-	cxgb_add_clients(tdev);
-
-	return (0);
-}
-
-static int
-offload_close(struct t3cdev *tdev)
-{
-	struct adapter *adapter = tdev2adap(tdev);
-
-	if (!isset(&adapter->open_device_map, OFFLOAD_DEVMAP_BIT))
-		return (0);
-
-	/* Call back all registered clients */
-	cxgb_remove_clients(tdev);
-
-	tdev->lldev = NULL;
-	cxgb_set_dummy_ops(tdev);
-	t3_tp_set_offload_mode(adapter, 0);
-
-	clrbit(&adapter->open_device_map, OFFLOAD_DEVMAP_BIT);
-
-	return (0);
-}
-
 /*
  * if_init for cxgb ports.
  */
@@ -1793,15 +1704,9 @@ cxgb_init_locked(struct port_info *p)
 		ADAPTER_UNLOCK(sc);
 	}
 
-	if (sc->open_device_map == 0) {
-		if ((rc = cxgb_up(sc)) != 0)
+	if (sc->open_device_map == 0 && ((rc = cxgb_up(sc)) != 0))
 			goto done;
 
-		if (is_offload(sc) && !ofld_disable && offload_open(p))
-			log(LOG_WARNING,
-			    "Could not initialize offload capabilities\n");
-	}
-
 	PORT_LOCK(p);
 	if (isset(&sc->open_device_map, p->port_id) &&
 	    (ifp->if_drv_flags & IFF_DRV_RUNNING)) {
@@ -1929,7 +1834,6 @@ cxgb_uninit_synchronized(struct port_inf
 	DELAY(100 * 1000);
 	t3_mac_disable(&pi->mac, MAC_DIRECTION_RX);
 
-
 	pi->phy.ops->power_down(&pi->phy, 1);
 
 	PORT_UNLOCK(pi);
@@ -1937,9 +1841,6 @@ cxgb_uninit_synchronized(struct port_inf
 	pi->link_config.link_ok = 0;
 	t3_os_link_changed(sc, pi->port_id, 0, 0, 0, 0, 0);
 
-	if ((sc->open_device_map & PORT_MASK) == 0)
-		offload_close(&sc->tdev);
-
 	if (sc->open_device_map == 0)
 		cxgb_down(pi->adapter);
 
@@ -2081,6 +1982,15 @@ fail:
 			/* Safe to do this even if cxgb_up not called yet */
 			cxgb_set_lro(p, ifp->if_capenable & IFCAP_LRO);
 		}
+#ifdef TCP_OFFLOAD
+		if (mask & IFCAP_TOE4) {
+			int enable = (ifp->if_capenable ^ mask) & IFCAP_TOE4;
+
+			error = toe_capability(p, enable);
+			if (error == 0)
+				ifp->if_capenable ^= mask;
+		}
+#endif
 		if (mask & IFCAP_VLAN_HWTAGGING) {
 			ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
 			if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
@@ -3362,3 +3272,235 @@ set_tcb_field_ulp(struct cpl_set_tcb_fie
 	txpkt->len = htonl(V_ULPTX_NFLITS(sizeof(*req) / 8));
 	mk_set_tcb_field(req, tid, word, mask, val);
 }
+
+void
+t3_iterate(void (*func)(struct adapter *, void *), void *arg)
+{
+	struct adapter *sc;
+
+	mtx_lock(&t3_list_lock);
+	SLIST_FOREACH(sc, &t3_list, link) {
+		/*
+		 * func should not make any assumptions about what state sc is
+		 * in - the only guarantee is that sc->sc_lock is a valid lock.
+		 */
+		func(sc, arg);
+	}
+	mtx_unlock(&t3_list_lock);
+}
+
+#ifdef TCP_OFFLOAD
+static int
+toe_capability(struct port_info *pi, int enable)
+{
+	int rc;
+	struct adapter *sc = pi->adapter;
+
+	ADAPTER_LOCK_ASSERT_OWNED(sc);
+
+	if (!is_offload(sc))
+		return (ENODEV);
+
+	if (enable) {
+		if (!(sc->flags & FULL_INIT_DONE)) {
+			log(LOG_WARNING,
+			    "You must enable a cxgb interface first\n");
+			return (EAGAIN);
+		}
+
+		if (isset(&sc->offload_map, pi->port_id))
+			return (0);
+
+		if (!(sc->flags & TOM_INIT_DONE)) {
+			rc = t3_activate_uld(sc, ULD_TOM);
+			if (rc == EAGAIN) {
+				log(LOG_WARNING,
+				    "You must kldload t3_tom.ko before trying "
+				    "to enable TOE on a cxgb interface.\n");
+			}
+			if (rc != 0)
+				return (rc);
+			KASSERT(sc->tom_softc != NULL,
+			    ("%s: TOM activated but softc NULL", __func__));
+			KASSERT(sc->flags & TOM_INIT_DONE,
+			    ("%s: TOM activated but flag not set", __func__));
+		}
+
+		setbit(&sc->offload_map, pi->port_id);
+
+		/*
+		 * XXX: Temporary code to allow iWARP to be enabled when TOE is
+		 * enabled on any port.  Need to figure out how to enable,
+		 * disable, load, and unload iWARP cleanly.
+		 */
+		if (!isset(&sc->offload_map, MAX_NPORTS) &&
+		    t3_activate_uld(sc, ULD_IWARP) == 0)
+			setbit(&sc->offload_map, MAX_NPORTS);
+	} else {
+		if (!isset(&sc->offload_map, pi->port_id))
+			return (0);
+
+		KASSERT(sc->flags & TOM_INIT_DONE,
+		    ("%s: TOM never initialized?", __func__));
+		clrbit(&sc->offload_map, pi->port_id);
+	}
+
+	return (0);
+}
+
+/*
+ * Add an upper layer driver to the global list.
+ */
+int
+t3_register_uld(struct uld_info *ui)
+{
+	int rc = 0;
+	struct uld_info *u;
+
+	mtx_lock(&t3_uld_list_lock);
+	SLIST_FOREACH(u, &t3_uld_list, link) {
+	    if (u->uld_id == ui->uld_id) {
+		    rc = EEXIST;
+		    goto done;
+	    }
+	}
+
+	SLIST_INSERT_HEAD(&t3_uld_list, ui, link);
+	ui->refcount = 0;
+done:
+	mtx_unlock(&t3_uld_list_lock);
+	return (rc);
+}
+
+int
+t3_unregister_uld(struct uld_info *ui)
+{
+	int rc = EINVAL;
+	struct uld_info *u;
+
+	mtx_lock(&t3_uld_list_lock);
+
+	SLIST_FOREACH(u, &t3_uld_list, link) {
+	    if (u == ui) {
+		    if (ui->refcount > 0) {
+			    rc = EBUSY;
+			    goto done;
+		    }
+
+		    SLIST_REMOVE(&t3_uld_list, ui, uld_info, link);
+		    rc = 0;
+		    goto done;
+	    }
+	}
+done:
+	mtx_unlock(&t3_uld_list_lock);
+	return (rc);
+}
+
+int
+t3_activate_uld(struct adapter *sc, int id)
+{
+	int rc = EAGAIN;
+	struct uld_info *ui;
+
+	mtx_lock(&t3_uld_list_lock);
+
+	SLIST_FOREACH(ui, &t3_uld_list, link) {
+		if (ui->uld_id == id) {
+			rc = ui->activate(sc);
+			if (rc == 0)
+				ui->refcount++;
+			goto done;
+		}
+	}
+done:
+	mtx_unlock(&t3_uld_list_lock);
+
+	return (rc);
+}
+
+int
+t3_deactivate_uld(struct adapter *sc, int id)
+{
+	int rc = EINVAL;
+	struct uld_info *ui;
+
+	mtx_lock(&t3_uld_list_lock);
+
+	SLIST_FOREACH(ui, &t3_uld_list, link) {
+		if (ui->uld_id == id) {
+			rc = ui->deactivate(sc);
+			if (rc == 0)
+				ui->refcount--;
+			goto done;
+		}
+	}
+done:
+	mtx_unlock(&t3_uld_list_lock);
+
+	return (rc);
+}
+
+static int
+cpl_not_handled(struct sge_qset *qs __unused, struct rsp_desc *r __unused,
+    struct mbuf *m)
+{
+	m_freem(m);
+	return (EDOOFUS);
+}
+
+int
+t3_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h)
+{
+	uintptr_t *loc, new;
+
+	if (opcode >= NUM_CPL_HANDLERS)
+		return (EINVAL);
+
+	new = h ? (uintptr_t)h : (uintptr_t)cpl_not_handled;
+	loc = (uintptr_t *) &sc->cpl_handler[opcode];
+	atomic_store_rel_ptr(loc, new);
+
+	return (0);
+}
+#endif
+
+static int
+cxgbc_mod_event(module_t mod, int cmd, void *arg)
+{
+	int rc = 0;
+
+	switch (cmd) {
+	case MOD_LOAD:
+		mtx_init(&t3_list_lock, "T3 adapters", 0, MTX_DEF);
+		SLIST_INIT(&t3_list);
+#ifdef TCP_OFFLOAD
+		mtx_init(&t3_uld_list_lock, "T3 ULDs", 0, MTX_DEF);
+		SLIST_INIT(&t3_uld_list);
+#endif
+		break;
+
+	case MOD_UNLOAD:
+#ifdef TCP_OFFLOAD
+		mtx_lock(&t3_uld_list_lock);
+		if (!SLIST_EMPTY(&t3_uld_list)) {
+			rc = EBUSY;
+			mtx_unlock(&t3_uld_list_lock);
+			break;
+		}
+		mtx_unlock(&t3_uld_list_lock);
+		mtx_destroy(&t3_uld_list_lock);
+#endif
+		mtx_lock(&t3_list_lock);
+		if (!SLIST_EMPTY(&t3_list)) {
+			rc = EBUSY;
+			mtx_unlock(&t3_list_lock);
+			break;
+		}
+		mtx_unlock(&t3_list_lock);
+		mtx_destroy(&t3_list_lock);
+		break;
+	}
+
+	return (rc);
+}

Modified: user/np/toe_iwarp/sys/dev/cxgb/cxgb_offload.h
==============================================================================
--- user/np/toe_iwarp/sys/dev/cxgb/cxgb_offload.h	Wed Apr 18 18:37:56 2012	(r234431)
+++ user/np/toe_iwarp/sys/dev/cxgb/cxgb_offload.h	Wed Apr 18 18:43:01 2012	(r234432)
@@ -1,4 +1,3 @@
-
 /**************************************************************************
 
 Copyright (c) 2007-2008, Chelsio Inc.
@@ -33,221 +32,99 @@ $FreeBSD$
 #ifndef _CXGB_OFFLOAD_H
 #define _CXGB_OFFLOAD_H
 
-#include <common/cxgb_tcb.h>
-#include <t3cdev.h>
-
-MALLOC_DECLARE(M_CXGB);
+#ifdef TCP_OFFLOAD
+enum {
+	ULD_TOM = 1,
+	ULD_IWARP = 2,
+};
 
 struct adapter;
-struct cxgb_client;
-
-void cxgb_offload_init(void);
-void cxgb_offload_exit(void);
-
-void cxgb_adapter_ofld(struct adapter *adapter);
-void cxgb_adapter_unofld(struct adapter *adapter);
-int cxgb_offload_activate(struct adapter *adapter);
-void cxgb_offload_deactivate(struct adapter *adapter);
-int cxgb_ofld_recv(struct t3cdev *dev, struct mbuf **m, int n);
-
-void cxgb_set_dummy_ops(struct t3cdev *dev);
-
-
-/*
- * Client registration.  Users of T3 driver must register themselves.
- * The T3 driver will call the add function of every client for each T3
- * adapter activated, passing up the t3cdev ptr.  Each client fills out an
- * array of callback functions to process CPL messages.
- */
-
-void cxgb_register_client(struct cxgb_client *client);
-void cxgb_unregister_client(struct cxgb_client *client);
-void cxgb_add_clients(struct t3cdev *tdev);
-void cxgb_remove_clients(struct t3cdev *tdev);
-
-typedef int (*cxgb_cpl_handler_func)(struct t3cdev *dev,
-				      struct mbuf *m, void *ctx);
-
-struct l2t_entry;
-struct cxgb_client {
-	char 			*name;
-	void 			(*add) (struct t3cdev *);
-	void 			(*remove) (struct t3cdev *);
-	cxgb_cpl_handler_func 	*handlers;
-	int			(*redirect)(void *ctx, struct rtentry *old,
-					    struct rtentry *new,
-					    struct l2t_entry *l2t);
-	TAILQ_ENTRY(cxgb_client)         client_entry;
+struct uld_info {
+	SLIST_ENTRY(uld_info) link;
+	int refcount;
+	int uld_id;
+	int (*activate)(struct adapter *);
+	int (*deactivate)(struct adapter *);
 };
 
-/*
- * TID allocation services.
- */
-int cxgb_alloc_atid(struct t3cdev *dev, struct cxgb_client *client,
-		     void *ctx);
-int cxgb_alloc_stid(struct t3cdev *dev, struct cxgb_client *client,
-		     void *ctx);
-void *cxgb_free_atid(struct t3cdev *dev, int atid);
-void cxgb_free_stid(struct t3cdev *dev, int stid);
-void *cxgb_get_lctx(struct t3cdev *tdev, int stid);
-void cxgb_insert_tid(struct t3cdev *dev, struct cxgb_client *client,
-		      void *ctx,
-	unsigned int tid);
-void cxgb_queue_tid_release(struct t3cdev *dev, unsigned int tid);
-void cxgb_remove_tid(struct t3cdev *dev, void *ctx, unsigned int tid);
-
-struct toe_tid_entry {
-	struct cxgb_client 	*client;
-	void 			*ctx;
+struct tom_tunables {
+	int sndbuf;
+	int ddp;
+	int indsz;
+	int ddp_thres;
 };
 
 /* CPL message priority levels */
 enum {
 	CPL_PRIORITY_DATA = 0,     /* data messages */
+	CPL_PRIORITY_CONTROL = 1   /* offload control messages */
+#ifdef notyet
 	CPL_PRIORITY_SETUP = 1,	   /* connection setup messages */
 	CPL_PRIORITY_TEARDOWN = 0, /* connection teardown messages */
 	CPL_PRIORITY_LISTEN = 1,   /* listen start/stop messages */
 	CPL_PRIORITY_ACK = 1,      /* RX ACK messages */
-	CPL_PRIORITY_CONTROL = 1   /* offload control messages */
+#endif
 };
 
-/* Flags for return value of CPL message handlers */
-enum {
-	CPL_RET_BUF_DONE = 1,   // buffer processing done, buffer may be freed
-	CPL_RET_BAD_MSG = 2,    // bad CPL message (e.g., unknown opcode)
-	CPL_RET_UNKNOWN_TID = 4	// unexpected unknown TID
-};
+#define S_HDR_NDESC	0
+#define M_HDR_NDESC	0xf
+#define V_HDR_NDESC(x)	((x) << S_HDR_NDESC)
+#define G_HDR_NDESC(x)	(((x) >> S_HDR_NDESC) & M_HDR_NDESC)
 
-typedef int (*cpl_handler_func)(struct t3cdev *dev, struct mbuf *m);
+#define S_HDR_QSET	4
+#define M_HDR_QSET	0xf
+#define V_HDR_QSET(x)	((x) << S_HDR_QSET)
+#define G_HDR_QSET(x)	(((x) >> S_HDR_QSET) & M_HDR_QSET)
 
-/*
- * Returns a pointer to the first byte of the CPL header in an sk_buff that
- * contains a CPL message.
- */
-static inline void *cplhdr(struct mbuf *m)
-{
-	return mtod(m, uint8_t *);
-}
+#define S_HDR_CTRL	8
+#define V_HDR_CTRL(x)	((x) << S_HDR_CTRL)
+#define F_HDR_CTRL	V_HDR_CTRL(1U)
 
-void t3_register_cpl_handler(unsigned int opcode, cpl_handler_func h);
+#define S_HDR_DF	9
+#define V_HDR_DF(x)	((x) << S_HDR_DF)
+#define F_HDR_DF	V_HDR_DF(1U)
 
-union listen_entry {
-	struct toe_tid_entry toe_tid;
-	union listen_entry *next;
-};
+#define S_HDR_SGL	10
+#define V_HDR_SGL(x)	((x) << S_HDR_SGL)
+#define F_HDR_SGL	V_HDR_SGL(1U)
 
-union active_open_entry {
-	struct toe_tid_entry toe_tid;
-	union active_open_entry *next;
-};
-
-/*
- * Holds the size, base address, free list start, etc of the TID, server TID,
- * and active-open TID tables for a offload device.
- * The tables themselves are allocated dynamically.
- */
-struct tid_info {
-	struct toe_tid_entry *tid_tab;
-	unsigned int ntids;
-	volatile unsigned int tids_in_use;
-
-	union listen_entry *stid_tab;
-	unsigned int nstids;
-	unsigned int stid_base;
-
-	union active_open_entry *atid_tab;
-	unsigned int natids;
-	unsigned int atid_base;
-
-	/*
-	 * The following members are accessed R/W so we put them in their own
-	 * cache lines.
-	 *
-	 * XXX We could combine the atid fields above with the lock here since
-	 * atids are use once (unlike other tids).  OTOH the above fields are
-	 * usually in cache due to tid_tab.
-	 */
-	struct mtx atid_lock /* ____cacheline_aligned_in_smp */;
-	union active_open_entry *afree;
-	unsigned int atids_in_use;
-
-	struct mtx stid_lock /*____cacheline_aligned */;
-	union listen_entry *sfree;
-	unsigned int stids_in_use;
-};
-
-struct t3c_data {
-	struct t3cdev *dev;
-	unsigned int tx_max_chunk;  /* max payload for TX_DATA */
-	unsigned int max_wrs;       /* max in-flight WRs per connection */
-	unsigned int nmtus;
-	const unsigned short *mtus;
-	struct tid_info tid_maps;
-
-	struct toe_tid_entry *tid_release_list;
-	struct mtx tid_release_lock;
-	struct task tid_release_task;
+struct ofld_hdr
+{
+	void *sgl;	/* SGL, if F_HDR_SGL set in flags */
+	int plen;	/* amount of payload (in bytes) */
+	int flags;
 };
 
 /*
- * t3cdev -> toe_data accessor
+ * Convenience function for fixed size CPLs that fit in 1 desc.
  */
-#define T3C_DATA(dev) (*(struct t3c_data **)&(dev)->l4opt)
-
-/*
- * Map an ATID or STID to their entries in the corresponding TID tables.
- */
-static inline union active_open_entry *atid2entry(const struct tid_info *t,
-                                                  unsigned int atid)
+#define M_GETHDR_OFLD(qset, ctrl, cpl) \
+    m_gethdr_ofld(qset, ctrl, sizeof(*cpl), (void **)&cpl)
+static inline struct mbuf *
+m_gethdr_ofld(int qset, int ctrl, int cpllen, void **cpl)
 {
-        return &t->atid_tab[atid - t->atid_base];
-}
-
+	struct mbuf *m;
+	struct ofld_hdr *oh;
 
-static inline union listen_entry *stid2entry(const struct tid_info *t,
-                                             unsigned int stid)
-{
-        return &t->stid_tab[stid - t->stid_base];
-}
+	m = m_gethdr(M_NOWAIT, MT_DATA);
+	if (m == NULL)
+		return (NULL);
 
-/*
- * Find the connection corresponding to a TID.
- */
-static inline struct toe_tid_entry *lookup_tid(const struct tid_info *t,
-                                               unsigned int tid)
-{

*** DIFF OUTPUT TRUNCATED AT 1000 LINES ***


More information about the svn-src-user mailing list