svn commit: r323954 - head/sys/net

[Stephen Hurd]

Author: shurd
Date: Sat Sep 23 16:46:30 2017
New Revision: 323954
URL: https://svnweb.freebsd.org/changeset/base/323954

Log:
  Have ifmp_ring_enqueue() abdicate instead of switch to a consumer
  
  Move TX out of the enqueue() path. As a result, we need
  to have ifmp_ring_check_drainage() pick up from the abdicate state.
  
  We also need to either enqueue the TX task, or check drainage
  after calling ifmp_ring_enqueue() to ensure it's sent.
  
  This change results in a 30% small packet forwarding improvement.
  
  Reviewed by:	olivier, sbruno
  Approved by:	sbruno (mentor)
  Sponsored by:	Limelight Networks
  Differential Revision:	https://reviews.freebsd.org/D12439

Modified:
  head/sys/net/iflib.c
  head/sys/net/mp_ring.c

Modified: head/sys/net/iflib.c
==============================================================================
--- head/sys/net/iflib.c	Sat Sep 23 13:37:02 2017	(r323953)
+++ head/sys/net/iflib.c	Sat Sep 23 16:46:30 2017	(r323954)
@@ -3515,8 +3515,7 @@ _task_fn_tx(void *context)
 	}
 	if (txq->ift_db_pending)
 		ifmp_ring_enqueue(txq->ift_br, (void **)&txq, 1, TX_BATCH_SIZE);
-	else
-		ifmp_ring_check_drainage(txq->ift_br, TX_BATCH_SIZE);
+	ifmp_ring_check_drainage(txq->ift_br, TX_BATCH_SIZE);
 	if (ctx->ifc_flags & IFC_LEGACY)
 		IFDI_INTR_ENABLE(ctx);
 	else {
@@ -3718,16 +3717,14 @@ iflib_if_transmit(if_t ifp, struct mbuf *m)
 	DBG_COUNTER_INC(tx_seen);
 	err = ifmp_ring_enqueue(txq->ift_br, (void **)&m, 1, TX_BATCH_SIZE);
 
+	GROUPTASK_ENQUEUE(&txq->ift_task);
 	if (err) {
-		GROUPTASK_ENQUEUE(&txq->ift_task);
 		/* support forthcoming later */
 #ifdef DRIVER_BACKPRESSURE
 		txq->ift_closed = TRUE;
 #endif
 		ifmp_ring_check_drainage(txq->ift_br, TX_BATCH_SIZE);
 		m_freem(m);
-	} else if (TXQ_AVAIL(txq) < (txq->ift_size >> 1)) {
-		GROUPTASK_ENQUEUE(&txq->ift_task);
 	}
 
 	return (err);

Modified: head/sys/net/mp_ring.c
==============================================================================
--- head/sys/net/mp_ring.c	Sat Sep 23 13:37:02 2017	(r323953)
+++ head/sys/net/mp_ring.c	Sat Sep 23 16:46:30 2017	(r323954)
@@ -454,18 +454,12 @@ ifmp_ring_enqueue(struct ifmp_ring *r, void **items, i
 	do {
 		os.state = ns.state = r->state;
 		ns.pidx_tail = pidx_stop;
-		ns.flags = BUSY;
+		if (os.flags == IDLE)
+			ns.flags = ABDICATED;
 	} while (atomic_cmpset_rel_64(&r->state, os.state, ns.state) == 0);
 	critical_exit();
 	counter_u64_add(r->enqueues, n);
 
-	/*
-	 * Turn into a consumer if some other thread isn't active as a consumer
-	 * already.
-	 */
-	if (os.flags != BUSY)
-		drain_ring_lockless(r, ns, os.flags, budget);
-
 	return (0);
 }
 #endif
@@ -476,7 +470,9 @@ ifmp_ring_check_drainage(struct ifmp_ring *r, int budg
 	union ring_state os, ns;
 
 	os.state = r->state;
-	if (os.flags != STALLED || os.pidx_head != os.pidx_tail || r->can_drain(r) == 0)
+	if ((os.flags != STALLED && os.flags != ABDICATED) ||	// Only continue in STALLED and ABDICATED
+	    os.pidx_head != os.pidx_tail ||			// Require work to be available
+	    (os.flags != ABDICATED && r->can_drain(r) == 0))	// Can either drain, or everyone left
 		return;
 
 	MPASS(os.cidx != os.pidx_tail);	/* implied by STALLED */