git: aefd61a1de86 - main - dpaa: Add LRO and receive callback batching
- Go to: [ bottom of page ] [ top of archives ] [ this month ]
Date: Mon, 24 Aug 2026 03:11:54 UTC
The branch main has been updated by jhibbits:
URL: https://cgit.FreeBSD.org/src/commit/?id=aefd61a1de86521abc1551aa98ed94316fc83d86
commit aefd61a1de86521abc1551aa98ed94316fc83d86
Author: Justin Hibbits <jhibbits@FreeBSD.org>
AuthorDate: 2026-08-10 18:03:00 +0000
Commit: Justin Hibbits <jhibbits@FreeBSD.org>
CommitDate: 2026-08-24 03:11:27 +0000
dpaa: Add LRO and receive callback batching
Reduce the code executed in the DQRR dequeue loop, and move the
heavy-weight operations to post-dequeue loop.
* Batch if_input() after DQRR dispatch loop completes. Only do the
DQRR_CI_CINH write at the end of the loop, so only up to 16 entries
will be processed.
* Add software LRO per FQ. Each per-CPU RX FQ gets its own LRO tracking
structure.
Since LRO is configured at FQ initialization time, allocate the ifnet
earlier in attach to prevent a panic.
---
sys/dev/dpaa/dpaa_eth.c | 58 ++++++++++++++++++++++++++++++++++++++++++++-
sys/dev/dpaa/dpaa_eth.h | 13 ++++++++++
sys/dev/dpaa/if_dtsec.c | 7 +++---
sys/dev/dpaa/if_memac.c | 10 +++++---
sys/dev/dpaa/qman.c | 7 ++++++
sys/dev/dpaa/qman.h | 20 ++++++++++++++++
sys/dev/dpaa/qman_portals.c | 38 ++++++++++++++++++++++-------
sys/dev/dpaa/qman_var.h | 4 ++++
8 files changed, 142 insertions(+), 15 deletions(-)
diff --git a/sys/dev/dpaa/dpaa_eth.c b/sys/dev/dpaa/dpaa_eth.c
index 23ebc78c1a6f..67e9da45b1d7 100644
--- a/sys/dev/dpaa/dpaa_eth.c
+++ b/sys/dev/dpaa/dpaa_eth.c
@@ -46,6 +46,7 @@
#include <net/if_arp.h>
#include <netinet/ip.h>
#include <netinet/ip6.h>
+#include <netinet/tcp_lro.h>
#include <dev/mii/mii.h>
#include <dev/mii/miivar.h>
@@ -507,7 +508,19 @@ dpaa_eth_fq_rx_callback(device_t portal, struct qman_fq *fq,
m->m_len = frame->length;
m_fixhdr(m);
- if_input(sc->sc_ifnet, m);
+ /*
+ * Offer to LRO first.
+ */
+ if (rxfq->lro_inited &&
+ (if_getcapenable(sc->sc_ifnet) & IFCAP_LRO) != 0 &&
+ (m->m_pkthdr.csum_flags & (CSUM_L4_CALC | CSUM_L4_VALID)) ==
+ (CSUM_L4_CALC | CSUM_L4_VALID) &&
+ tcp_lro_rx(&rxfq->lro, m, 0) == 0)
+ return (1);
+
+ m->m_nextpkt = NULL;
+ *rxfq->rx_tailp = m;
+ rxfq->rx_tailp = &m->m_nextpkt;
return (1);
@@ -519,6 +532,32 @@ err:
return (1);
}
+/*
+ * Post-poll flush hook invoked once per QMan portal poll on any
+ * RX FQ that dispatched at least one frame this cycle. Runs on
+ * the FQ's affine CPU, outside the DQRR dispatch loop.
+ */
+static void
+dpaa_eth_fq_rx_flush(struct qman_fq *fq __unused, void *ctx)
+{
+ struct dpaa_eth_rx_fq *rxfq = ctx;
+
+ if (rxfq->rx_head != NULL) {
+ struct mbuf *chain = rxfq->rx_head;
+
+ rxfq->rx_head = NULL;
+ rxfq->rx_tailp = &rxfq->rx_head;
+ if_input(rxfq->sc->sc_ifnet, chain);
+ }
+ /*
+ * Flush LRO whenever initialised. If the user disabled
+ * IFCAP_LRO between the last callback and this flush, entries
+ * queued in that window still need to be drained.
+ */
+ if (rxfq->lro_inited)
+ tcp_lro_flush_all(&rxfq->lro);
+}
+
static int
dpaa_eth_fq_tx_confirm_callback(device_t portal, struct qman_fq *fq,
struct qman_fd *frame, void *app)
@@ -592,6 +631,14 @@ dpaa_eth_fq_rx_free(struct dpaa_eth_softc *sc)
for (i = 0; i < sc->sc_nrxfqs; i++) {
if (sc->sc_rx_fqs[i].fq != NULL)
qman_fq_free(sc->sc_rx_fqs[i].fq);
+ /*
+ * Any pending non-LRO mbufs on the batch chain
+ * are dropped here rather than delivered late.
+ */
+ if (sc->sc_rx_fqs[i].rx_head != NULL)
+ m_freem(sc->sc_rx_fqs[i].rx_head);
+ if (sc->sc_rx_fqs[i].lro_inited)
+ tcp_lro_free(&sc->sc_rx_fqs[i].lro);
}
free(sc->sc_rx_fqs, M_DEVBUF);
sc->sc_rx_fqs = NULL;
@@ -664,6 +711,14 @@ dpaa_eth_fq_rx_init(struct dpaa_eth_softc *sc)
sc->sc_rx_fqs[i].fqid = base_fqid + i;
sc->sc_rx_fqs[i].cpu = i;
sc->sc_rx_fqs[i].sc = sc;
+ sc->sc_rx_fqs[i].rx_head = NULL;
+ sc->sc_rx_fqs[i].rx_tailp = &sc->sc_rx_fqs[i].rx_head;
+
+ /* Best-effort LRO per FQ. */
+ if (tcp_lro_init(&sc->sc_rx_fqs[i].lro) == 0) {
+ sc->sc_rx_fqs[i].lro.ifp = sc->sc_ifnet;
+ sc->sc_rx_fqs[i].lro_inited = true;
+ }
error = qman_fq_register_cb(fq, dpaa_eth_fq_rx_callback,
&sc->sc_rx_fqs[i]);
@@ -672,6 +727,7 @@ dpaa_eth_fq_rx_init(struct dpaa_eth_softc *sc)
"could not register RX callback for FQ %d\n", i);
goto err;
}
+ (void)qman_fq_register_flush_cb(fq, dpaa_eth_fq_rx_flush);
}
/*
diff --git a/sys/dev/dpaa/dpaa_eth.h b/sys/dev/dpaa/dpaa_eth.h
index 4da52a84617e..43724ba391a5 100644
--- a/sys/dev/dpaa/dpaa_eth.h
+++ b/sys/dev/dpaa/dpaa_eth.h
@@ -27,6 +27,8 @@
#ifndef DPAA_ETH_H_
#define DPAA_ETH_H_
+#include <netinet/tcp_lro.h>
+
/* * TX csum-offload hwassist mask for dTSEC and mEMAC. */
#define DPAA_CSUM_TX_OFFLOAD \
(CSUM_IP | CSUM_DELAY_DATA | CSUM_DELAY_DATA_IPV6)
@@ -54,6 +56,17 @@ struct dpaa_eth_rx_fq {
* Sysctl readers get advisory (torn on 32-bit hosts) values.
*/
uint64_t frames_in;
+
+ /*
+ * Batched-input state. The RX callback appends non-LRO mbufs
+ * onto rx_head via m_nextpkt; the per-FQ flush hook hands the
+ * whole chain to if_input() and then runs tcp_lro_flush_all().
+ * Both fields are only touched from the affine CPU.
+ */
+ struct mbuf *rx_head;
+ struct mbuf **rx_tailp;
+ struct lro_ctrl lro;
+ bool lro_inited;
};
struct dpaa_eth_softc {
diff --git a/sys/dev/dpaa/if_dtsec.c b/sys/dev/dpaa/if_dtsec.c
index a0ea6e69dee3..7ef7de69da31 100644
--- a/sys/dev/dpaa/if_dtsec.c
+++ b/sys/dev/dpaa/if_dtsec.c
@@ -465,6 +465,9 @@ dtsec_attach(device_t dev)
/* Init callouts */
callout_init(&sc->sc_base.sc_tick_callout, CALLOUT_MPSAFE);
+ /* Create network interface for upper layers */
+ ifp = sc->sc_base.sc_ifnet = if_alloc(IFT_ETHER);
+
/* Create RX buffer pool */
error = dpaa_eth_pool_rx_init(&sc->sc_base);
if (error != 0)
@@ -507,8 +510,6 @@ dtsec_attach(device_t dev)
return (ENXIO);
}
- /* Create network interface for upper layers */
- ifp = sc->sc_base.sc_ifnet = if_alloc(IFT_ETHER);
if_setsoftc(ifp, sc);
if_setflags(ifp, IFF_SIMPLEX | IFF_BROADCAST | IFF_MULTICAST);
@@ -532,7 +533,7 @@ dtsec_attach(device_t dev)
if_setcapabilities(ifp, IFCAP_JUMBO_MTU | IFCAP_VLAN_MTU |
IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6 |
- IFCAP_TXCSUM | IFCAP_TXCSUM_IPV6);
+ IFCAP_TXCSUM | IFCAP_TXCSUM_IPV6 | IFCAP_LRO);
if_setcapenable(ifp, if_getcapabilities(ifp));
if_sethwassist(ifp, DPAA_CSUM_TX_OFFLOAD);
diff --git a/sys/dev/dpaa/if_memac.c b/sys/dev/dpaa/if_memac.c
index 047cfea323a6..2e2ac116709a 100644
--- a/sys/dev/dpaa/if_memac.c
+++ b/sys/dev/dpaa/if_memac.c
@@ -354,6 +354,8 @@ memac_if_ioctl(if_t ifp, u_long command, caddr_t data)
IFCAP_TXCSUM | IFCAP_TXCSUM_IPV6);
if_togglehwassist(ifp, DPAA_CSUM_TX_OFFLOAD);
}
+ if ((changed & IFCAP_LRO) != 0)
+ if_togglecapenable(ifp, IFCAP_LRO);
break;
case SIOCGIFMEDIA:
@@ -566,6 +568,9 @@ memac_attach(device_t dev)
/* Init callouts */
callout_init(&sc->sc_base.sc_tick_callout, CALLOUT_MPSAFE);
+ /* Create network interface for upper layers */
+ ifp = sc->sc_base.sc_ifnet = if_alloc(IFT_ETHER);
+
/* Create RX buffer pool */
error = dpaa_eth_pool_rx_init(&sc->sc_base);
if (error != 0)
@@ -596,8 +601,6 @@ memac_attach(device_t dev)
dpaa_eth_fm_port_rx_init(&sc->sc_base);
dpaa_eth_fm_port_tx_init(&sc->sc_base);
- /* Create network interface for upper layers */
- ifp = sc->sc_base.sc_ifnet = if_alloc(IFT_ETHER);
if_setsoftc(ifp, sc);
if_setflags(ifp, IFF_SIMPLEX | IFF_BROADCAST | IFF_MULTICAST);
@@ -618,7 +621,8 @@ memac_attach(device_t dev)
if_setcapabilities(ifp, IFCAP_JUMBO_MTU |
IFCAP_VLAN_MTU | IFCAP_VLAN_HWCSUM |
IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6 |
- IFCAP_TXCSUM | IFCAP_TXCSUM_IPV6);
+ IFCAP_TXCSUM | IFCAP_TXCSUM_IPV6 |
+ IFCAP_LRO);
if_setcapenable(ifp, if_getcapabilities(ifp));
if_sethwassist(ifp, DPAA_CSUM_TX_OFFLOAD);
diff --git a/sys/dev/dpaa/qman.c b/sys/dev/dpaa/qman.c
index 4c4c53beacfc..9da6e45d3b49 100644
--- a/sys/dev/dpaa/qman.c
+++ b/sys/dev/dpaa/qman.c
@@ -665,6 +665,13 @@ qman_fq_register_cb(struct qman_fq *fq, qman_cb_dqrr callback,
return (0);
}
+int
+qman_fq_register_flush_cb(struct qman_fq *fq, qman_cb_flush flush)
+{
+ fq->cb.flush = flush;
+ return (0);
+}
+
int
qman_fq_enqueue(struct qman_fq *fq, struct dpaa_fd *frame)
{
diff --git a/sys/dev/dpaa/qman.h b/sys/dev/dpaa/qman.h
index 25ddc80eeedb..b009760d6257 100644
--- a/sys/dev/dpaa/qman.h
+++ b/sys/dev/dpaa/qman.h
@@ -164,11 +164,21 @@ typedef int (*qman_cb_dqrr)(device_t, struct qman_fq *,
struct qman_fd *, void *);
typedef void (*qman_cb_mr)(device_t, struct qman_fq *,
struct qman_mr_entry *);
+typedef void (*qman_cb_flush)(struct qman_fq *, void *);
struct qman_cb {
qman_cb_dqrr dqrr;
qman_cb_mr ern;
qman_cb_mr fqscn;
+ /*
+ * Optional post-poll hook. If set, qman_portal_loop_dqrr()
+ * calls it once per poll cycle on every FQ that had a frame
+ * dispatched, after all DQRR entries have been drained.
+ * Consumers use this for aggregation flushes (e.g., LRO
+ * flush, batched if_input) that must happen outside the
+ * per-frame dispatch path.
+ */
+ qman_cb_flush flush;
void *ctx;
};
/**
@@ -265,6 +275,16 @@ int qman_fq_free(struct qman_fq *fq);
* @param app A pointer to the user's data.
* @return E_OK on success; error code otherwise.
*/
+/*
+ * Register a post-poll flush callback on @fq. Called once per poll
+ * cycle on any FQ that had at least one dispatched frame during
+ * that cycle, after all DQRR entries were drained. Reuses the
+ * per-FQ ctx registered via qman_fq_register_cb(). May be called
+ * before or after qman_fq_register_cb() but only makes sense if the
+ * DQRR callback is also set.
+ */
+int qman_fq_register_flush_cb(struct qman_fq *fq, qman_cb_flush flush);
+
int qman_fq_register_cb(struct qman_fq *fq, qman_cb_dqrr callback,
void *ctx);
diff --git a/sys/dev/dpaa/qman_portals.c b/sys/dev/dpaa/qman_portals.c
index be271c24bf06..7b5d795eb2a2 100644
--- a/sys/dev/dpaa/qman_portals.c
+++ b/sys/dev/dpaa/qman_portals.c
@@ -369,6 +369,7 @@ qman_portal_fq_enqueue(device_t dev, struct qman_fq *fq, struct dpaa_fd *frame)
static int
qman_portal_loop_dqrr(struct qman_portal_softc *sc)
{
+ SLIST_HEAD(, qman_fq) dirty_fqs = SLIST_HEAD_INITIALIZER(dirty_fqs);
struct qman_dqrr_entry *dqrr;
struct qman_dqrr_entry *base;
struct qman_fq *fq;
@@ -376,24 +377,45 @@ qman_portal_loop_dqrr(struct qman_portal_softc *sc)
DQRR_CI_CI_M;
int pi = bus_read_4(sc->sc_base.sc_mres[1], QCSP_DQRR_PI_CINH) &
DQRR_PI_PI_M;
+ int start_ci = ci;
base = sc->sc_dqrr.ring;
do {
dqrr = &base[ci];
dpaa_flush_line(dqrr);
dpaa_touch_line(dqrr);
- if ((dqrr->stat & QMAN_DQRR_STAT_HAS_FRAME)) {
- fq = qman_fq_from_index(dqrr->fqid);
- if (fq != NULL && fq->cb.dqrr != NULL) {
- fq->cb.dqrr(sc->sc_base.sc_dev, fq,
- &dqrr->fd, fq->cb.ctx);
- }
- } else
+ if ((dqrr->stat & QMAN_DQRR_STAT_HAS_FRAME) == 0)
break;
+
+ fq = qman_fq_from_index(dqrr->fqid);
+ if (fq != NULL && fq->cb.dqrr != NULL) {
+ fq->cb.dqrr(sc->sc_base.sc_dev, fq,
+ &dqrr->fd, fq->cb.ctx);
+ /*
+ * Track the FQ for post-poll flush. Only need to track
+ * once.
+ */
+ if (fq->cb.flush != NULL && !fq->dirty) {
+ fq->dirty = true;
+ SLIST_INSERT_HEAD(&dirty_fqs, fq,
+ dirty_next);
+ }
+ }
ci = (ci + 1) & DQRR_CI_CI_M;
- bus_write_4(sc->sc_base.sc_mres[1], QCSP_DQRR_CI_CINH, ci);
} while (ci != pi);
+ /* Update CI after the loop. CI writes are expensive. */
+ if (ci != start_ci)
+ bus_write_4(sc->sc_base.sc_mres[1], QCSP_DQRR_CI_CINH, ci);
+
+ /* Drain the dirty list, invoking the per-FQ flush hook. */
+ while (!SLIST_EMPTY(&dirty_fqs)) {
+ fq = SLIST_FIRST(&dirty_fqs);
+ SLIST_REMOVE_HEAD(&dirty_fqs, dirty_next);
+ fq->dirty = false;
+ fq->cb.flush(fq, fq->cb.ctx);
+ }
+
return (0);
}
diff --git a/sys/dev/dpaa/qman_var.h b/sys/dev/dpaa/qman_var.h
index 2d64c10d534c..ac18e6acecb9 100644
--- a/sys/dev/dpaa/qman_var.h
+++ b/sys/dev/dpaa/qman_var.h
@@ -7,6 +7,8 @@
#ifndef QMAN_VAR_H
#define QMAN_VAR_H
+#include <sys/queue.h>
+
#include "dpaa_common.h"
#include "portals.h"
@@ -181,6 +183,8 @@ struct qman_fq {
uint32_t fqid; /* base FQID of the range */
uint32_t fqid_count; /* length of the range (>=1) */
bool force_fqid; /* caller owns the FQID allocation */
+ bool dirty; /* on a portal's dirty list this poll */
+ SLIST_ENTRY(qman_fq) dirty_next;/* DQRR loop dirty list. */
struct qman_cb cb;
};