git: d7397e95685b - main - dpaa/eth: distribute RX across per-CPU FQs via FMan KeyGen
- Go to: [ bottom of page ] [ top of archives ] [ this month ]
Date: Mon, 24 Aug 2026 03:11:53 UTC
The branch main has been updated by jhibbits:
URL: https://cgit.FreeBSD.org/src/commit/?id=d7397e95685b30414cc63d6f60acaf8242ee9155
commit d7397e95685b30414cc63d6f60acaf8242ee9155
Author: Justin Hibbits <jhibbits@FreeBSD.org>
AuthorDate: 2026-08-10 14:11:53 +0000
Commit: Justin Hibbits <jhibbits@FreeBSD.org>
CommitDate: 2026-08-24 03:11:26 +0000
dpaa/eth: distribute RX across per-CPU FQs via FMan KeyGen
Grow sc_nrxfqs from 1 to the CPU total, and hash the RX
5-tuple across the range with the KG driver from the prior commit.
Each FQ lands on its own per-CPU QMan channel, so a given core
drains only its own share of RX work and gets frame annotation +
data-head stashed into its cache.
* Add alignment parameter to qman_alloc_fqid_range() to meet KeyGen
requirements.
* Initialize 1 frame queue (FQ) per CPU in dpaa_eth_fm_port_rx_init(),
using a 5-tuple to spread the load across CPUs.
* Channel ownership for TX confirms moved from rx_init/free to
tx_init/free -- sc_rx_channel is now a TX-confirm-only per-port
pool channel.
Fallbacks/degradation:
* If any per-CPU channel is -1 (no portal attached)
the port fails to attach with a clear message.
* If the FQID range can't be allocated aligned, the port fails
attach.
* If KG scheme allocation fails at port setup, the port keeps
its N FQs but only FQ #0 sees traffic.
---
sys/dev/dpaa/dpaa_eth.c | 262 +++++++++++++++++++++++++++++++++++++++--------
sys/dev/dpaa/dpaa_eth.h | 6 ++
sys/dev/dpaa/fman_port.c | 23 +++++
sys/dev/dpaa/fman_port.h | 12 +++
sys/dev/dpaa/qman.c | 5 +-
sys/dev/dpaa/qman.h | 8 +-
6 files changed, 267 insertions(+), 49 deletions(-)
diff --git a/sys/dev/dpaa/dpaa_eth.c b/sys/dev/dpaa/dpaa_eth.c
index 29161d528f71..23ebc78c1a6f 100644
--- a/sys/dev/dpaa/dpaa_eth.c
+++ b/sys/dev/dpaa/dpaa_eth.c
@@ -59,6 +59,7 @@
#include "dpaa_common.h"
#include "dpaa_eth.h"
#include "fman.h"
+#include "fman_keygen.h"
#include "fman_parser.h"
#include "fman_port.h"
#include "fman_if.h"
@@ -112,6 +113,19 @@ struct dpaa_eth_frame_info {
struct dpaa_sgte fi_sgt[DPAA_NUM_OF_SG_TABLE_ENTRY];
};
+/*
+ * Loader-tunable override for the per-port RX FQ count. 0 (default)
+ * means "auto". Otherwise must be a power of two.
+ */
+static int dpaa_eth_nrxfqs_tunable = 0;
+SYSCTL_NODE(_hw, OID_AUTO, dpaa, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
+ "DPAA driver tunables");
+static SYSCTL_NODE(_hw_dpaa, OID_AUTO, eth, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
+ "DPAA Ethernet driver");
+SYSCTL_INT(_hw_dpaa_eth, OID_AUTO, nrxfqs, CTLFLAG_RDTUN,
+ &dpaa_eth_nrxfqs_tunable, 0,
+ "Per-port RX FQ count override (0=auto, else power-of-2 in [1,8])");
+
enum dpaa_eth_pool_params {
DTSEC_RM_POOL_RX_LOW_MARK = 16,
DTSEC_RM_POOL_RX_HIGH_MARK = 64,
@@ -129,8 +143,6 @@ enum dpaa_eth_pool_params {
DTSEC_RM_POOL_FI_MAX_SIZE = 256,
};
-#define DTSEC_RM_FQR_RX_CHANNEL 0x401
-#define DTSEC_RM_FQR_TX_CONF_CHANNEL 0
enum dpaa_eth_fq_params {
DTSEC_RM_FQR_RX_WQ = 1,
DTSEC_RM_FQR_TX_WQ = 1,
@@ -194,6 +206,10 @@ dpaa_eth_fm_port_rx_init(struct dpaa_eth_softc *sc)
struct fman_port_params params;
int error;
+ /*
+ * dflt/err FQID is the base of the RSS range: non-hashable
+ * frames (ARP, IP fragments, non-IP) fall through to FQ #0.
+ */
params.dflt_fqid = sc->sc_rx_fqid_base;
params.err_fqid = sc->sc_rx_fqid_base;
params.rx_params.num_pools = 1;
@@ -206,6 +222,43 @@ dpaa_eth_fm_port_rx_init(struct dpaa_eth_softc *sc)
return (ENXIO);
}
+ /*
+ * The RX port's own FMan hardware port ID (cell-index in the
+ * OFW node) is the index KG uses for scheme-binding. It was
+ * previously left at zero, which made every port fight for KG
+ * port 0 -- only the first attach won, the rest got EBUSY.
+ */
+ sc->sc_port_rx_hw_id = fman_port_get_id(sc->sc_rx_port);
+
+ /*
+ * Wire up FMan KeyGen 5-tuple hashing across the sc_nrxfqs FQs
+ * created in dpaa_eth_fq_rx_init. Skipped for the trivial N=1
+ * case (single-core system or forced fallback): with one FQ
+ * there's nothing to distribute. Only flip the port's parser
+ * output to KG on success -- routing to KG with no bound scheme
+ * drops frames.
+ */
+ if (sc->sc_nrxfqs > 1) {
+ struct fman_softc *fman_sc =
+ device_get_softc(device_get_parent(sc->sc_rx_port));
+
+ error = fman_kg_alloc_hash_scheme(fman_sc,
+ sc->sc_port_rx_hw_id, sc->sc_rx_fqid_base,
+ sc->sc_nrxfqs);
+ if (error != 0) {
+ device_printf(sc->sc_dev,
+ "fman_kg_alloc_hash_scheme failed: %d\n", error);
+ /*
+ * Non-fatal: FMan port still delivers to
+ * dflt_fqid == sc_rx_fqid_base (FQ #0), which is
+ * a valid single-queue fallback. Leave RFPNE at
+ * the BMI-enqueue default.
+ */
+ } else {
+ fman_port_rx_use_kg(sc->sc_rx_port, true);
+ }
+ }
+
return (0);
}
@@ -407,6 +460,7 @@ dpaa_eth_fq_rx_callback(device_t portal, struct qman_fq *fq,
m = NULL;
rxfq = app;
sc = rxfq->sc;
+ rxfq->frames_in++;
frame_va = DPAA_FD_GET_ADDR(frame);
frame_ic = frame_va; /* internal context at head of the frame */
@@ -516,7 +570,23 @@ dpaa_eth_fq_tx_confirm_callback(device_t portal, struct qman_fq *fq,
void
dpaa_eth_fq_rx_free(struct dpaa_eth_softc *sc)
{
- int cpu, i;
+ int i;
+
+ /*
+ * Tear down the KG scheme first so no new frames land on FQs
+ * about to be retired. Point the parser output back at BMI-
+ * enqueue before freeing the scheme so the port keeps
+ * delivering to dflt_fqid instead of dropping through an
+ * emptied KG.
+ */
+ if (sc->sc_nrxfqs > 1 && sc->sc_rx_port != NULL) {
+ struct fman_softc *fman_sc =
+ device_get_softc(device_get_parent(sc->sc_rx_port));
+
+ fman_port_rx_use_kg(sc->sc_rx_port, false);
+ (void)fman_kg_free_hash_scheme(fman_sc,
+ sc->sc_port_rx_hw_id);
+ }
if (sc->sc_rx_fqs != NULL) {
for (i = 0; i < sc->sc_nrxfqs; i++) {
@@ -525,15 +595,11 @@ dpaa_eth_fq_rx_free(struct dpaa_eth_softc *sc)
}
free(sc->sc_rx_fqs, M_DEVBUF);
sc->sc_rx_fqs = NULL;
- sc->sc_nrxfqs = 0;
}
- if (sc->sc_rx_channel != 0) {
- CPU_FOREACH(cpu) {
- device_t portal = DPCPU_ID_GET(cpu, qman_affine_portal);
- QMAN_PORTAL_STATIC_DEQUEUE_RM_CHANNEL(portal,
- sc->sc_rx_channel);
- }
- qman_free_channel(sc->sc_rx_channel);
+ if (sc->sc_nrxfqs > 0) {
+ qman_free_fqid_range(sc->sc_rx_fqid_base, sc->sc_nrxfqs);
+ sc->sc_nrxfqs = 0;
+ sc->sc_rx_fqid_base = 0;
}
}
@@ -541,65 +607,157 @@ int
dpaa_eth_fq_rx_init(struct dpaa_eth_softc *sc)
{
struct qman_fq *fq;
- int error;
- int cpu;
+ uint32_t base_fqid;
+ int align, nfqs;
+ int error, i;
- /* Default Frame Queue */
- if (sc->sc_rx_channel == 0)
- sc->sc_rx_channel = qman_alloc_channel();
- /*
- * Stash 1 cacheline of frame annotation (parse result / IC) and
- * 1 of frame data head into the destination core's cache when
- * QMan dequeues an RX frame -- the RX callback reads both.
- */
- fq = qman_fq_create(1, sc->sc_rx_channel, DTSEC_RM_FQR_RX_WQ,
- false, 0, false, false, true, false, 0, 0, 0, 1, 1);
- if (fq == NULL) {
+ if (dpaa_eth_nrxfqs_tunable > 0)
+ nfqs = dpaa_eth_nrxfqs_tunable;
+ else
+ nfqs = mp_ncpus;
+
+ align = 1 << ilog2(nfqs);
+ error = qman_alloc_fqid_range(nfqs, align, &base_fqid);
+ if (error != 0) {
device_printf(sc->sc_dev,
- "could not create default RX queue\n");
+ "could not reserve %d contiguous FQIDs (aligned): %d\n",
+ nfqs, error);
return (EIO);
}
- CPU_FOREACH(cpu) {
- device_t portal = DPCPU_ID_GET(cpu, qman_affine_portal);
- QMAN_PORTAL_STATIC_DEQUEUE_CHANNEL(portal, sc->sc_rx_channel);
+ sc->sc_nrxfqs = nfqs;
+ sc->sc_rx_fqid_base = base_fqid;
+ sc->sc_rx_fqs = malloc(nfqs * sizeof(*sc->sc_rx_fqs),
+ M_DEVBUF, M_WAITOK | M_ZERO);
+
+ /*
+ * Create the N RX FQs, one per per-CPU channel. QMan portal
+ * attach has already subscribed each portal to its own
+ * per-CPU channel, so frames land on the right core without
+ * per-driver static-dequeue plumbing.
+ *
+ * Stash 1 cacheline of frame annotation (parse result / IC)
+ * and 1 of frame data head into the destination core's cache
+ * when QMan dequeues an RX frame -- the RX callback reads
+ * both.
+ */
+ for (i = 0; i < nfqs; i++) {
+ int chan = qman_percpu_channel(i);
+
+ if (chan == -1) {
+ device_printf(sc->sc_dev,
+ "no per-CPU QMan channel for CPU %d\n", i);
+ error = EIO;
+ goto err;
+ }
+ fq = qman_fq_create(1, chan, DTSEC_RM_FQR_RX_WQ,
+ /*force_fqid=*/true, base_fqid + i,
+ false, false, true, false, 0, 0, 0, 1, 1);
+ if (fq == NULL) {
+ device_printf(sc->sc_dev,
+ "could not create RX FQ %d (fqid 0x%x)\n",
+ i, base_fqid + i);
+ error = EIO;
+ goto err;
+ }
+ sc->sc_rx_fqs[i].fq = fq;
+ sc->sc_rx_fqs[i].fqid = base_fqid + i;
+ sc->sc_rx_fqs[i].cpu = i;
+ sc->sc_rx_fqs[i].sc = sc;
+
+ error = qman_fq_register_cb(fq, dpaa_eth_fq_rx_callback,
+ &sc->sc_rx_fqs[i]);
+ if (error != 0) {
+ device_printf(sc->sc_dev,
+ "could not register RX callback for FQ %d\n", i);
+ goto err;
+ }
}
- sc->sc_nrxfqs = 1;
- sc->sc_rx_fqs = malloc(sc->sc_nrxfqs * sizeof(*sc->sc_rx_fqs),
- M_DEVBUF, M_WAITOK | M_ZERO);
- sc->sc_rx_fqs[0].fq = fq;
- sc->sc_rx_fqs[0].fqid = qman_fq_get_fqid(fq);
- sc->sc_rx_fqs[0].cpu = -1; /* not pinned; any portal may drain */
- sc->sc_rx_fqs[0].sc = sc;
- sc->sc_rx_fqid_base = sc->sc_rx_fqs[0].fqid;
-
- error = qman_fq_register_cb(fq, dpaa_eth_fq_rx_callback,
- &sc->sc_rx_fqs[0]);
- if (error != 0) {
- device_printf(sc->sc_dev, "could not register RX callback\n");
- dpaa_eth_fq_rx_free(sc);
- return (EIO);
+ /*
+ * Expose per-FQ observability under dev.<port>.rx_fq.<i>.{cpu,
+ * fqid, frames}. The sysctl_ctx owned by sc_dev handles all
+ * teardown at device detach, so nothing to unwind on the free
+ * path.
+ */
+ {
+ struct sysctl_ctx_list *ctx =
+ device_get_sysctl_ctx(sc->sc_dev);
+ struct sysctl_oid *tree = device_get_sysctl_tree(sc->sc_dev);
+ struct sysctl_oid *rxnode = SYSCTL_ADD_NODE(ctx,
+ SYSCTL_CHILDREN(tree), OID_AUTO, "rx_fq",
+ CTLFLAG_RD | CTLFLAG_MPSAFE, NULL,
+ "Per-FQ RX statistics");
+
+ for (i = 0; i < nfqs; i++) {
+ char name[8];
+ struct sysctl_oid *fqnode;
+
+ snprintf(name, sizeof(name), "%d", i);
+ fqnode = SYSCTL_ADD_NODE(ctx,
+ SYSCTL_CHILDREN(rxnode), OID_AUTO, name,
+ CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "");
+ SYSCTL_ADD_UINT(ctx, SYSCTL_CHILDREN(fqnode),
+ OID_AUTO, "fqid", CTLFLAG_RD,
+ &sc->sc_rx_fqs[i].fqid, 0, "FQID");
+ SYSCTL_ADD_INT(ctx, SYSCTL_CHILDREN(fqnode),
+ OID_AUTO, "cpu", CTLFLAG_RD,
+ &sc->sc_rx_fqs[i].cpu, 0,
+ "CPU affine to this FQ");
+ SYSCTL_ADD_UQUAD(ctx, SYSCTL_CHILDREN(fqnode),
+ OID_AUTO, "frames", CTLFLAG_RD,
+ &sc->sc_rx_fqs[i].frames_in,
+ "Frames dispatched to the RX callback");
+ }
+ }
+
+ if (bootverbose) {
+ device_printf(sc->sc_dev,
+ "RSS: %d RX FQ%s starting at fqid 0x%x, %s\n",
+ nfqs, nfqs == 1 ? "" : "s", base_fqid,
+ nfqs == 1 ? "no distribution" :
+ "IP 5-tuple hash via KeyGen");
}
return (0);
+
+err:
+ dpaa_eth_fq_rx_free(sc);
+ return (error);
}
void
dpaa_eth_fq_tx_free(struct dpaa_eth_softc *sc)
{
+ int cpu;
if (sc->sc_tx_fq)
qman_fq_free(sc->sc_tx_fq);
if (sc->sc_tx_conf_fq)
qman_fq_free(sc->sc_tx_conf_fq);
+
+ /*
+ * The port's pool channel now hosts only TX confirms (RX has
+ * moved to per-CPU pool channels). Unsubscribe every portal
+ * and release the channel here rather than in fq_rx_free.
+ */
+ if (sc->sc_rx_channel != 0) {
+ CPU_FOREACH(cpu) {
+ device_t portal = DPCPU_ID_GET(cpu, qman_affine_portal);
+ QMAN_PORTAL_STATIC_DEQUEUE_RM_CHANNEL(portal,
+ sc->sc_rx_channel);
+ }
+ qman_free_channel(sc->sc_rx_channel);
+ sc->sc_rx_channel = 0;
+ }
}
int
dpaa_eth_fq_tx_init(struct dpaa_eth_softc *sc)
{
int error;
+ int cpu;
void *fq;
/* TX Frame Queue */
@@ -614,8 +772,24 @@ dpaa_eth_fq_tx_init(struct dpaa_eth_softc *sc)
sc->sc_tx_fq = fq;
- if (sc->sc_rx_channel == 0)
+ /*
+ * TX confirms need a pool channel with at least one subscriber.
+ * Historically the RX path allocated and subscribed sc_rx_channel
+ * for both RX and TX confirms; now RX uses per-CPU channels, so
+ * this path is the sole owner. Allocate + subscribe here.
+ * (Confirm handling doesn't benefit from CPU pinning today; a
+ * follow-on could route TX confirms to the enqueue CPU via the
+ * same per-CPU channel infrastructure.)
+ */
+ if (sc->sc_rx_channel == 0) {
sc->sc_rx_channel = qman_alloc_channel();
+ CPU_FOREACH(cpu) {
+ device_t portal = DPCPU_ID_GET(cpu, qman_affine_portal);
+ QMAN_PORTAL_STATIC_DEQUEUE_CHANNEL(portal,
+ sc->sc_rx_channel);
+ }
+ }
+
/* TX Confirmation Frame Queue */
fq = qman_fq_create(1, sc->sc_rx_channel,
DTSEC_RM_FQR_TX_CONF_WQ, false, 0, false, false, true, false, 0, 0,
diff --git a/sys/dev/dpaa/dpaa_eth.h b/sys/dev/dpaa/dpaa_eth.h
index e4090f8521cb..4da52a84617e 100644
--- a/sys/dev/dpaa/dpaa_eth.h
+++ b/sys/dev/dpaa/dpaa_eth.h
@@ -48,6 +48,12 @@ struct dpaa_eth_rx_fq {
uint32_t fqid;
int cpu; /* CPU pin, or -1 if unpinned */
struct dpaa_eth_softc *sc;
+ /*
+ * Frame count. Written by the RX callback which is affine to
+ * one CPU per FQ, so plain uint64 without atomics is safe.
+ * Sysctl readers get advisory (torn on 32-bit hosts) values.
+ */
+ uint64_t frames_in;
};
struct dpaa_eth_softc {
diff --git a/sys/dev/dpaa/fman_port.c b/sys/dev/dpaa/fman_port.c
index 04ddf22c1250..833be542055e 100644
--- a/sys/dev/dpaa/fman_port.c
+++ b/sys/dev/dpaa/fman_port.c
@@ -667,6 +667,29 @@ fman_port_enable(device_t dev)
return (0);
}
+int
+fman_port_get_id(device_t dev)
+{
+ struct fman_port_softc *sc = device_get_softc(dev);
+
+ return (sc->sc_port_id);
+}
+
+/*
+ * Runtime toggle for whether the praser sends frames to KeyGen (HWK)
+ * or straight to BMI-enqueue. Called by dpaa_eth after successfully
+ * binding a KG scheme to this port -- flipping this without a bound scheme
+ * drops all Rx traffic on the port.
+ */
+void
+fman_port_rx_use_kg(device_t dev, bool enable)
+{
+ struct fman_port_softc *sc = device_get_softc(dev);
+
+ bus_write_4(sc->sc_mem, FMBM_RFPNE,
+ enable ? NIA_ENG_HWK : (NIA_ENG_BMI | NIA_BMI_AC_ENQ_FRAME));
+}
+
static device_method_t fman_port_methods[] = {
DEVMETHOD(device_probe, fman_port_probe),
DEVMETHOD(device_attach, fman_port_attach),
diff --git a/sys/dev/dpaa/fman_port.h b/sys/dev/dpaa/fman_port.h
index 8803c5e5f7c9..57a3dc63fa36 100644
--- a/sys/dev/dpaa/fman_port.h
+++ b/sys/dev/dpaa/fman_port.h
@@ -26,4 +26,16 @@ struct fman_port_params {
};
};
+int fman_port_get_id(device_t dev);
+
+/*
+ * Runtime toggle for the RX port's parser->KeyGen routing. Default
+ * (RX-init time) is parser->BMI-enqueue with no KG involvement; a
+ * consumer flips this to true only *after* successfully binding a
+ * KG scheme via fman_kg_alloc_hash_scheme(). Routing to KG with no
+ * bound scheme drops the port's Rx traffic on the floor. Only valid
+ * on FMAN_PORT_TYPE_RX devices; no-op / undefined on TX/OH.
+ */
+void fman_port_rx_use_kg(device_t rx_port, bool enable);
+
#endif
diff --git a/sys/dev/dpaa/qman.c b/sys/dev/dpaa/qman.c
index f0964502413d..4c4c53beacfc 100644
--- a/sys/dev/dpaa/qman.c
+++ b/sys/dev/dpaa/qman.c
@@ -462,13 +462,14 @@ qman_percpu_channel(int cpu)
}
int
-qman_alloc_fqid_range(uint32_t count, uint32_t *basep)
+qman_alloc_fqid_range(uint32_t count, uint32_t align, uint32_t *basep)
{
struct qman_softc *sc = qman_sc;
vmem_addr_t base;
int error;
- error = vmem_alloc(sc->sc_fqalloc, count, M_BESTFIT | M_NOWAIT, &base);
+ error = vmem_xalloc(sc->sc_fqalloc, count, align, 0, 0,
+ VMEM_ADDR_MIN, VMEM_ADDR_MAX, M_BESTFIT | M_NOWAIT, &base);
if (error != 0)
return (error);
*basep = base;
diff --git a/sys/dev/dpaa/qman.h b/sys/dev/dpaa/qman.h
index f09d57dc45c6..25ddc80eeedb 100644
--- a/sys/dev/dpaa/qman.h
+++ b/sys/dev/dpaa/qman.h
@@ -323,10 +323,12 @@ int qman_percpu_channel(int cpu);
/*
* Reserve a contiguous range of @count FQIDs (needed by callers that
* program a KeyGen-style base+mask distribution and then create the
- * individual FQs one-by-one with force_fqid=true). Returns 0 on
- * success and writes the base FQID to *basep.
+ * individual FQs one-by-one with force_fqid=true). @align is the
+ * required base alignment in FQIDs (0 for no requirement, or the
+ * range size for the power-of-two-aligned base FMan KeyGen wants).
+ * Returns 0 on success and writes the base FQID to *basep.
*/
-int qman_alloc_fqid_range(uint32_t count, uint32_t *basep);
+int qman_alloc_fqid_range(uint32_t count, uint32_t align, uint32_t *basep);
void qman_free_fqid_range(uint32_t base, uint32_t count);
/**