svn commit: r254429 - user/np/cxl_tuning/sys/dev/cxgbe

[Navdeep Parhar]

Author: np
Date: Fri Aug 16 21:04:58 2013
New Revision: 254429
URL: http://svnweb.freebsd.org/changeset/base/254429

Log:
  Enable buffer packing by default for T5 cards.  At this point I
  consistently get full 40G line rate for a single TCP connection (both
  1500 MTU and 9000 MTU, both single direction and bidirectional 40G tx +
  40G rx simultaneously) out of the box.
  
  For example, this is on a recent HEAD with changes from this branch.
  No tuning.  sysctl.conf is empty and loader.conf has just one line
  setting a high hw.intr_storm_threshold.  No special buffer sizes or r/w
  sizes are given to the benchmark programs either.
  
  # sysctl hw.model
  hw.model: Intel(R) Xeon(R) CPU E5-1620 0 @ 3.60GHz
  # iperf -v
  iperf version 2.0.5 (08 Jul 2010) pthreads
  
  /* MTU 1500. */
  # iperf -c ha1 -t60
  ------------------------------------------------------------
  Client connecting to ha1, TCP port 5001
  TCP window size: 32.5 KByte (default)
  ------------------------------------------------------------
  [  3] local 10.3.166.121 port 32906 connected with 10.3.166.122 port 5001
  [ ID] Interval       Transfer     Bandwidth
  [  3]  0.0-60.0 sec   263 GBytes  37.6 Gbits/sec
  
  # iperf -c ha1 -t60 -d
  ------------------------------------------------------------
  Server listening on TCP port 5001
  TCP window size: 64.0 KByte (default)
  ------------------------------------------------------------
  ------------------------------------------------------------
  Client connecting to ha1, TCP port 5001
  TCP window size:  265 KByte (default)
  ------------------------------------------------------------
  [  5] local 10.3.166.121 port 54353 connected with 10.3.166.122 port 5001
  [  4] local 10.3.166.121 port 5001 connected with 10.3.166.122 port 33091
  [ ID] Interval       Transfer     Bandwidth
  [  4]  0.0-60.0 sec   262 GBytes  37.5 Gbits/sec
  [  5]  0.0-60.0 sec   263 GBytes  37.6 Gbits/sec
  
  /* MTU 9000. */
  # iperf -c ha0 -t60
  ------------------------------------------------------------
  Client connecting to ha0, TCP port 5001
  TCP window size: 35.0 KByte (default)
  ------------------------------------------------------------
  [  3] local 10.2.166.121 port 44734 connected with 10.2.166.122 port 5001
  [ ID] Interval       Transfer     Bandwidth
  [  3]  0.0-60.0 sec   276 GBytes  39.6 Gbits/sec
  
  # iperf -c ha0 -t60 -d
  ------------------------------------------------------------
  Server listening on TCP port 5001
  TCP window size: 64.0 KByte (default)
  ------------------------------------------------------------
  ------------------------------------------------------------
  Client connecting to ha0, TCP port 5001
  TCP window size:  291 KByte (default)
  ------------------------------------------------------------
  [  5] local 10.2.166.121 port 53293 connected with 10.2.166.122 port 5001
  [  4] local 10.2.166.121 port 5001 connected with 10.2.166.122 port 14311
  [ ID] Interval       Transfer     Bandwidth
  [  5]  0.0-60.0 sec   276 GBytes  39.5 Gbits/sec
  [  4]  0.0-60.0 sec   276 GBytes  39.5 Gbits/sec

Modified:
  user/np/cxl_tuning/sys/dev/cxgbe/t4_sge.c

Modified: user/np/cxl_tuning/sys/dev/cxgbe/t4_sge.c
==============================================================================
--- user/np/cxl_tuning/sys/dev/cxgbe/t4_sge.c	Fri Aug 16 20:35:03 2013	(r254428)
+++ user/np/cxl_tuning/sys/dev/cxgbe/t4_sge.c	Fri Aug 16 21:04:58 2013	(r254429)
@@ -819,7 +819,7 @@ t4_setup_port_queues(struct port_info *p
 	struct ifnet *ifp = pi->ifp;
 	struct sysctl_oid *oid = device_get_sysctl_tree(pi->dev);
 	struct sysctl_oid_list *children = SYSCTL_CHILDREN(oid);
-	int bufsize;
+	int bufsize, pack;
 
 	oid = SYSCTL_ADD_NODE(&pi->ctx, children, OID_AUTO, "rxq", CTLFLAG_RD,
 	    NULL, "rx queues");
@@ -841,6 +841,12 @@ t4_setup_port_queues(struct port_info *p
 	 * b) allocate queue iff it will take direct interrupts.
 	 */
 	bufsize = mtu_to_bufsize(ifp->if_mtu);
+	if (sc->flags & BUF_PACKING_OK &&
+	    ((is_t5(sc) && buffer_packing) ||	/* 1 or -1 both ok for T5 */
+	    (is_t4(sc) && buffer_packing == 1)))
+		pack = 1;
+	else
+		pack = 0;
 	for_each_rxq(pi, i, rxq) {
 
 		init_iq(&rxq->iq, sc, pi->tmr_idx, pi->pktc_idx, pi->qsize_rxq,
@@ -848,7 +854,7 @@ t4_setup_port_queues(struct port_info *p
 
 		snprintf(name, sizeof(name), "%s rxq%d-fl",
 		    device_get_nameunit(pi->dev), i);
-		init_fl(sc, &rxq->fl, pi->qsize_rxq / 8, bufsize, 0, name);
+		init_fl(sc, &rxq->fl, pi->qsize_rxq / 8, bufsize, pack, name);
 
 		if (sc->flags & INTR_DIRECT
 #ifdef TCP_OFFLOAD
@@ -865,6 +871,7 @@ t4_setup_port_queues(struct port_info *p
 
 #ifdef TCP_OFFLOAD
 	bufsize = mtu_to_bufsize_toe(sc, ifp->if_mtu);
+	pack = 0;	/* XXX: think about this some more */
 	for_each_ofld_rxq(pi, i, ofld_rxq) {
 
 		init_iq(&ofld_rxq->iq, sc, pi->tmr_idx, pi->pktc_idx,
@@ -872,7 +879,8 @@ t4_setup_port_queues(struct port_info *p
 
 		snprintf(name, sizeof(name), "%s ofld_rxq%d-fl",
 		    device_get_nameunit(pi->dev), i);
-		init_fl(sc, &ofld_rxq->fl, pi->qsize_rxq / 8, bufsize, 0, name);
+		init_fl(sc, &ofld_rxq->fl, pi->qsize_rxq / 8, bufsize, pack,
+		    name);
 
 		if (sc->flags & INTR_DIRECT ||
 		    (sc->intr_count > 1 && pi->nofldrxq > pi->nrxq)) {