git: 94a6919efda0 - main - dpaa2: Allocate DMA tag per Rx buffer
- Go to: [ bottom of page ] [ top of archives ] [ this month ]
Date: Thu, 10 Sep 2026 10:29:38 UTC
The branch main has been updated by dsl:
URL: https://cgit.FreeBSD.org/src/commit/?id=94a6919efda0638bd35a2cae2c4748a91c7fce66
commit 94a6919efda0638bd35a2cae2c4748a91c7fce66
Author: Dmitry Salychev <dsl@FreeBSD.org>
AuthorDate: 2026-08-26 13:34:36 +0000
Commit: Dmitry Salychev <dsl@FreeBSD.org>
CommitDate: 2026-09-10 10:29:22 +0000
dpaa2: Allocate DMA tag per Rx buffer
Previously each channel allocated its own set of buffers using the
channel's DMA tag which causes a DMA lock contention under load.
Even a single saturated 1 Gbps link caused ~50,000 adaptive
mutex spin events (as per lockstat) per second. With the proposed
approach there's no more contention on the channel's DMA mutex and the
adaptive mutex spin events dropped to ~6,000/s.
Stress test where iperf3 pushed as much traffic as possible to the 4
ports revealed that throughput drops from expected 940 Mbps down to
600-800 on each link with the "bounce pages lock" generating ~110,000
adaptive mutex spin events per second, but this is to be addressed
later on.
Tested by: flo_purplekraken.com, gnikl_justmail.de, bofh@
MFC after: 3 weeks
Differential Revision: reviews.freebsd.org/D59463
Event: EuroBSDcon Devsummit 2026
---
sys/dev/dpaa2/dpaa2_buf.c | 62 +++++++++++++++++++++++++++++++------------
sys/dev/dpaa2/dpaa2_buf.h | 13 ++++++---
sys/dev/dpaa2/dpaa2_channel.c | 14 ++--------
sys/dev/dpaa2/dpaa2_channel.h | 6 +++++
sys/dev/dpaa2/dpaa2_ni.c | 31 +++++++++++++---------
5 files changed, 80 insertions(+), 46 deletions(-)
diff --git a/sys/dev/dpaa2/dpaa2_buf.c b/sys/dev/dpaa2/dpaa2_buf.c
index 228e4448210d..2f727c65924d 100644
--- a/sys/dev/dpaa2/dpaa2_buf.c
+++ b/sys/dev/dpaa2/dpaa2_buf.c
@@ -44,7 +44,10 @@
#include "dpaa2_ni.h"
#include "dpaa2_frame.h"
-MALLOC_DEFINE(M_DPAA2_RXB, "dpaa2_rxb", "DPAA2 DMA-mapped buffer (Rx)");
+MALLOC_DEFINE(M_DPAA2_RXB, "dpaa2_rxb",
+ "DPAA2 DMA-mapped buffer (Rx)");
+MALLOC_DEFINE(M_DPAA2_RXB_EXT, "dpaa2_rxb_ext",
+ "DPAA2 DMA-mapped buffer (Rx extension)");
/**
* @brief Allocate Rx buffers visible to QBMan and release them to the
@@ -52,24 +55,19 @@ MALLOC_DEFINE(M_DPAA2_RXB, "dpaa2_rxb", "DPAA2 DMA-mapped buffer (Rx)");
*/
int
dpaa2_buf_seed_pool(device_t dev, device_t bpdev, void *arg, uint32_t count,
- int size, struct mtx *dma_mtx)
+ int size)
{
struct dpaa2_ni_softc *sc = device_get_softc(dev);
struct dpaa2_bp_softc *bpsc = device_get_softc(bpdev);
struct dpaa2_channel *ch = (struct dpaa2_channel *)arg;
struct dpaa2_buf *buf;
+ struct dpaa2_bufext_rx *bext;
const int alloc = DPAA2_ATOMIC_READ(&sc->buf_num);
const uint16_t bpid = bpsc->attr.bpid;
+ bus_dma_tag_t dmat;
bus_addr_t paddr[DPAA2_SWP_BUFS_PER_CMD];
int error, bufn = 0;
-#if defined(INVARIANTS)
- KASSERT(ch->rx_dmat != NULL, ("%s: no DMA tag?", __func__));
- if (dma_mtx != NULL) {
- mtx_assert(dma_mtx, MA_OWNED);
- }
-#endif /* INVARIANTS */
-
#ifdef _notyet_
/* Limit amount of buffers released to the pool */
count = (alloc + count > DPAA2_NI_BUFS_MAX)
@@ -93,12 +91,44 @@ dpaa2_buf_seed_pool(device_t dev, device_t bpdev, void *arg, uint32_t count,
buf = malloc(sizeof(struct dpaa2_buf), M_DPAA2_RXB, M_NOWAIT);
if (buf == NULL) {
- device_printf(dev, "%s: malloc() failed\n", __func__);
+ device_printf(dev, "%s: Rx buf malloc() failed\n",
+ __func__);
+ return (ENOMEM);
+ }
+ bext = malloc(sizeof(struct dpaa2_bufext_rx), M_DPAA2_RXB_EXT,
+ M_NOWAIT);
+ if (bext == NULL) {
+ device_printf(dev, "%s: Rx bext malloc() failed\n",
+ __func__);
return (ENOMEM);
}
- DPAA2_BUF_INIT_TAGOPT(buf, ch->rx_dmat, ch);
+ bext->ch = ch;
+ mtx_init(&bext->dma_mtx, "dpaa2_buf_dma_mtx", NULL,
+ MTX_DEF | MTX_NEW);
+ error = bus_dma_tag_create(
+ bus_get_dma_tag(dev), /* parent */
+ sc->buf_align, 0, /* alignment, boundary */
+ BUS_SPACE_MAXADDR, /* low restricted addr */
+ BUS_SPACE_MAXADDR, /* high restricted addr */
+ NULL, NULL, /* filter, filterarg */
+ RX_SEG_MAXSZ, /* maxsize */
+ RX_SEG_N, /* nsegments */
+ RX_SEG_SZ, /* maxsegsize */
+ 0, /* flags */
+ NULL, /* lockfunc */
+ NULL, /* lockarg */
+ &dmat);
+ if (error) {
+ device_printf(dev, "%s: failed to create Rx buf dmat\n",
+ __func__);
+ return (error);
+ }
+ DPAA2_BUF_INIT_TAGOPT(buf, dmat, bext);
- error = dpaa2_buf_seed_rxb(dev, buf, size, dma_mtx);
+ mtx_assert(&bext->dma_mtx, MA_NOTOWNED);
+ mtx_lock(&bext->dma_mtx);
+ error = dpaa2_buf_seed_rxb(dev, buf, size);
+ mtx_unlock(&bext->dma_mtx);
if (error != 0) {
device_printf(dev, "%s: dpaa2_buf_seed_rxb() failed: "
"error=%d/n", __func__, error);
@@ -126,8 +156,7 @@ dpaa2_buf_seed_pool(device_t dev, device_t bpdev, void *arg, uint32_t count,
* @brief Prepare Rx buffer to be released to the buffer pool.
*/
int
-dpaa2_buf_seed_rxb(device_t dev, struct dpaa2_buf *buf, int size,
- struct mtx *dma_mtx)
+dpaa2_buf_seed_rxb(device_t dev, struct dpaa2_buf *buf, int size)
{
struct dpaa2_ni_softc *sc = device_get_softc(dev);
struct dpaa2_swa *swa;
@@ -137,9 +166,8 @@ dpaa2_buf_seed_rxb(device_t dev, struct dpaa2_buf *buf, int size,
#if defined(INVARIANTS)
DPAA2_BUF_ASSERT_RXPREP(buf);
- if (dma_mtx != NULL) {
- mtx_assert(dma_mtx, MA_OWNED);
- }
+ struct dpaa2_bufext_rx *bext = (struct dpaa2_bufext_rx *)buf->opt;
+ mtx_assert(&bext->dma_mtx, MA_OWNED);
#endif /* INVARIANTS */
if (__predict_false(buf->dmap == NULL)) {
diff --git a/sys/dev/dpaa2/dpaa2_buf.h b/sys/dev/dpaa2/dpaa2_buf.h
index 16ea7e1905ac..719f8d5acd0f 100644
--- a/sys/dev/dpaa2/dpaa2_buf.h
+++ b/sys/dev/dpaa2/dpaa2_buf.h
@@ -52,6 +52,11 @@ struct dpaa2_buf {
void *opt;
};
+struct dpaa2_bufext_rx {
+ struct mtx dma_mtx;
+ struct dpaa2_channel *ch;
+};
+
#define DPAA2_BUF_INIT_TAGOPT(__buf, __tag, __opt) do { \
KASSERT((__buf) != NULL, ("%s: buf is NULL", __func__)); \
\
@@ -147,7 +152,7 @@ struct dpaa2_buf {
KASSERT((__buf)->nseg == 0, ("%s: nseg > 0?", __func__)); \
KASSERT((__buf)->m == NULL, ("%s: mbuf set?", __func__)); \
KASSERT((__buf)->sgt == NULL, ("%s: S/G table set?", __func__)); \
- KASSERT((__buf)->opt != NULL, ("%s: no channel?", __func__)); \
+ KASSERT((__buf)->opt != NULL, ("%s: no Rx ext?", __func__)); \
} while(0)
#define DPAA2_BUF_ASSERT_RXREADY(__buf) do { \
KASSERT((__buf)->paddr != 0, ("%s: paddr not set?", __func__)); \
@@ -159,7 +164,7 @@ struct dpaa2_buf {
KASSERT((__buf)->nseg == 1, ("%s: nseg != 1?", __func__)); \
KASSERT((__buf)->m != NULL, ("%s: no mbuf?", __func__)); \
KASSERT((__buf)->sgt == NULL, ("%s: S/G table set?", __func__)); \
- KASSERT((__buf)->opt != NULL, ("%s: no channel?", __func__)); \
+ KASSERT((__buf)->opt != NULL, ("%s: no Rx ext?", __func__)); \
} while(0)
#else /* !INVARIANTS */
#define DPAA2_BUF_ASSERT_RXPREP(__buf) do { \
@@ -168,8 +173,8 @@ struct dpaa2_buf {
} while(0)
#endif /* INVARIANTS */
-int dpaa2_buf_seed_pool(device_t, device_t, void *, uint32_t, int, struct mtx *);
-int dpaa2_buf_seed_rxb(device_t, struct dpaa2_buf *, int, struct mtx *);
+int dpaa2_buf_seed_pool(device_t, device_t, void *, uint32_t, int);
+int dpaa2_buf_seed_rxb(device_t, struct dpaa2_buf *, int);
int dpaa2_buf_seed_txb(device_t, struct dpaa2_buf *);
#endif /* _DPAA2_BUF_H */
diff --git a/sys/dev/dpaa2/dpaa2_channel.c b/sys/dev/dpaa2/dpaa2_channel.c
index 654c6f2baf70..b0372ef7ba86 100644
--- a/sys/dev/dpaa2/dpaa2_channel.c
+++ b/sys/dev/dpaa2/dpaa2_channel.c
@@ -78,12 +78,6 @@
MALLOC_DEFINE(M_DPAA2_CH, "dpaa2_ch", "DPAA2 QBMan Channel");
-#define RX_SEG_N (1u)
-#define RX_SEG_SZ (((MJUM9BYTES - 1) / PAGE_SIZE + 1) * PAGE_SIZE)
-#define RX_SEG_MAXSZ (((MJUM9BYTES - 1) / PAGE_SIZE + 1) * PAGE_SIZE)
-CTASSERT(RX_SEG_SZ % PAGE_SIZE == 0);
-CTASSERT(RX_SEG_MAXSZ % PAGE_SIZE == 0);
-
#define TX_SEG_N (16u) /* XXX-DSL: does DPAA2 limit exist? */
#define TX_SEG_SZ (PAGE_SIZE)
#define TX_SEG_MAXSZ (TX_SEG_N * TX_SEG_SZ)
@@ -212,7 +206,7 @@ dpaa2_chan_setup(device_t dev, device_t iodev, device_t condev, device_t bpdev,
/* Allocate initial # of Rx buffers and a channel storage */
error = dpaa2_buf_seed_pool(dev, bpdev, ch, DPAA2_NI_BUFS_INIT,
- DPAA2_RX_BUF_SIZE, NULL);
+ DPAA2_RX_BUF_SIZE);
if (error) {
device_printf(dev, "%s: failed to seed buffer pool\n",
__func__);
@@ -539,12 +533,8 @@ dpaa2_chan_bp_task(void *arg, int count)
/* Double allocated Rx buffers if amount of free buffers is < 25% */
if (bpconf.free_bufn < (buf_num >> 2)) {
- mtx_assert(&ch->dma_mtx, MA_NOTOWNED);
- mtx_lock(&ch->dma_mtx);
(void)dpaa2_buf_seed_pool(ch->ni_dev, bpdev, ch, buf_num,
- DPAA2_RX_BUF_SIZE, &ch->dma_mtx);
- mtx_unlock(&ch->dma_mtx);
-
+ DPAA2_RX_BUF_SIZE);
DPAA2_ATOMIC_XCHG(&sc->buf_free, bpconf.free_bufn);
}
}
diff --git a/sys/dev/dpaa2/dpaa2_channel.h b/sys/dev/dpaa2/dpaa2_channel.h
index 67b0285b7db6..28f96022b8c0 100644
--- a/sys/dev/dpaa2/dpaa2_channel.h
+++ b/sys/dev/dpaa2/dpaa2_channel.h
@@ -45,6 +45,12 @@
#define DPAA2_TX_BUFRING_SZ (4096u)
+#define RX_SEG_N (1u)
+#define RX_SEG_SZ (((MJUM9BYTES - 1) / PAGE_SIZE + 1) * PAGE_SIZE)
+#define RX_SEG_MAXSZ (((MJUM9BYTES - 1) / PAGE_SIZE + 1) * PAGE_SIZE)
+CTASSERT(RX_SEG_SZ % PAGE_SIZE == 0);
+CTASSERT(RX_SEG_MAXSZ % PAGE_SIZE == 0);
+
/**
* @brief QBMan channel to process ingress traffic.
*
diff --git a/sys/dev/dpaa2/dpaa2_ni.c b/sys/dev/dpaa2/dpaa2_ni.c
index 435b0fdf4085..a07ece428e60 100644
--- a/sys/dev/dpaa2/dpaa2_ni.c
+++ b/sys/dev/dpaa2/dpaa2_ni.c
@@ -132,10 +132,10 @@ MALLOC_DEFINE(M_DPAA2_TXB, "dpaa2_txb", "DPAA2 DMA-mapped buffer (Tx)");
* How many times channel cleanup routine will be repeated if the RX or TX
* budget was depleted.
*/
-#define DPAA2_CLEAN_BUDGET 64 /* sysctl(9)? */
+#define DPAA2_CLEAN_BUDGET 128 /* sysctl(9)? */
/* TX/RX budget for the channel cleanup task */
-#define DPAA2_TX_BUDGET 128 /* sysctl(9)? */
-#define DPAA2_RX_BUDGET 256 /* sysctl(9)? */
+#define DPAA2_TX_BUDGET 256 /* sysctl(9)? */
+#define DPAA2_RX_BUDGET 512 /* sysctl(9)? */
#define DPNI_IRQ_INDEX 0 /* Index of the only DPNI IRQ. */
#define DPNI_IRQ_LINK_CHANGED 1 /* Link state changed */
@@ -3165,6 +3165,7 @@ dpaa2_ni_rx(struct dpaa2_channel *ch, struct dpaa2_ni_fq *fq,
bus_addr_t paddr;
struct dpaa2_swa *swa;
struct dpaa2_buf *buf;
+ struct dpaa2_bufext_rx *bext;
struct dpaa2_channel *bch;
struct dpaa2_ni_softc *sc;
struct dpaa2_bp_softc *bpsc;
@@ -3182,7 +3183,8 @@ dpaa2_ni_rx(struct dpaa2_channel *ch, struct dpaa2_ni_fq *fq,
paddr = (bus_addr_t)fd->addr;
buf = swa->buf;
- bch = (struct dpaa2_channel *)buf->opt;
+ bext = (struct dpaa2_bufext_rx *)buf->opt;
+ bch = bext->ch;
sc = device_get_softc(bch->ni_dev);
update_csum_flags = true;
@@ -3230,8 +3232,8 @@ dpaa2_ni_rx(struct dpaa2_channel *ch, struct dpaa2_ni_fq *fq,
break;
}
- mtx_assert(&bch->dma_mtx, MA_NOTOWNED);
- mtx_lock(&bch->dma_mtx);
+ mtx_assert(&bext->dma_mtx, MA_NOTOWNED);
+ mtx_lock(&bext->dma_mtx);
bus_dmamap_sync(buf->dmat, buf->dmap, BUS_DMASYNC_POSTREAD);
bus_dmamap_unload(buf->dmat, buf->dmap);
@@ -3248,7 +3250,7 @@ dpaa2_ni_rx(struct dpaa2_channel *ch, struct dpaa2_ni_fq *fq,
buf->seg.ds_len = 0;
buf->nseg = 0;
- mtx_unlock(&bch->dma_mtx);
+ mtx_unlock(&bext->dma_mtx);
m->m_flags |= M_PKTHDR;
m->m_data = buf_data;
@@ -3293,13 +3295,14 @@ dpaa2_ni_rx(struct dpaa2_channel *ch, struct dpaa2_ni_fq *fq,
for (int i = 0; i < ch->recycled_n; i++) {
buf = ch->recycled[i];
- bch = (struct dpaa2_channel *)buf->opt;
+ bext = (struct dpaa2_bufext_rx *)buf->opt;
+ bch = bext->ch;
- mtx_assert(&bch->dma_mtx, MA_NOTOWNED);
- mtx_lock(&bch->dma_mtx);
+ mtx_assert(&bext->dma_mtx, MA_NOTOWNED);
+ mtx_lock(&bext->dma_mtx);
error = dpaa2_buf_seed_rxb(sc->dev, buf,
- DPAA2_RX_BUF_SIZE, &bch->dma_mtx);
- mtx_unlock(&bch->dma_mtx);
+ DPAA2_RX_BUF_SIZE);
+ mtx_unlock(&bext->dma_mtx);
if (__predict_false(error != 0)) {
/* TODO: What else to do with the buffer? */
@@ -3338,6 +3341,7 @@ dpaa2_ni_rx_err(struct dpaa2_channel *ch, struct dpaa2_ni_fq *fq,
bus_addr_t paddr;
struct dpaa2_swa *swa;
struct dpaa2_buf *buf;
+ struct dpaa2_bufext_rx *bext;
struct dpaa2_channel *bch;
struct dpaa2_ni_softc *sc;
device_t bpdev;
@@ -3351,7 +3355,8 @@ dpaa2_ni_rx_err(struct dpaa2_channel *ch, struct dpaa2_ni_fq *fq,
paddr = (bus_addr_t)fd->addr;
buf = swa->buf;
- bch = (struct dpaa2_channel *)buf->opt;
+ bext = (struct dpaa2_bufext_rx *)buf->opt;
+ bch = bext->ch;
sc = device_get_softc(bch->ni_dev);
KASSERT(swa->magic == DPAA2_MAGIC, ("%s: wrong magic", __func__));