git: d00f3505efad - main - mlx5en: do now waste ipsec_accel_in_tag on non-ipsec packets rx

From: Konstantin Belousov <kib_at_FreeBSD.org>
Date: Tue, 20 Aug 2024 12:43:45 UTC
The branch main has been updated by kib:

URL: https://cgit.FreeBSD.org/src/commit/?id=d00f3505efad7c323f74bee63f7d2527daf0f534

commit d00f3505efad7c323f74bee63f7d2527daf0f534
Author:     Konstantin Belousov <kib@FreeBSD.org>
AuthorDate: 2024-07-31 22:23:09 +0000
Commit:     Konstantin Belousov <kib@FreeBSD.org>
CommitDate: 2024-08-20 12:42:12 +0000

    mlx5en: do now waste ipsec_accel_in_tag on non-ipsec packets rx
    
    Do not prepend ipsec tags into mbuf head when preparing rx wqe, store it
    separately.  Only prepend (and clear the store) when received packed was
    indeed offloaded by ipsec engine.  Then we do not need to refill tags
    for slots that received non-ipsec packets.
    
    This should solve some minimal degradation of the rx CPU usage due to
    unneeded tag allocation for each packet.
    
    Sponsored by:   NVidia networking
---
 sys/dev/mlx5/mlx5_accel/ipsec.h           | 23 +++++++-----------
 sys/dev/mlx5/mlx5_accel/mlx5_ipsec_rxtx.c | 39 +++++++++++++++++--------------
 sys/dev/mlx5/mlx5_en/en.h                 |  3 +++
 sys/dev/mlx5/mlx5_en/mlx5_en_main.c       |  2 ++
 sys/dev/mlx5/mlx5_en/mlx5_en_rx.c         | 12 +++++-----
 5 files changed, 42 insertions(+), 37 deletions(-)

diff --git a/sys/dev/mlx5/mlx5_accel/ipsec.h b/sys/dev/mlx5/mlx5_accel/ipsec.h
index 1658542fc9c6..2abd68d9770a 100644
--- a/sys/dev/mlx5/mlx5_accel/ipsec.h
+++ b/sys/dev/mlx5/mlx5_accel/ipsec.h
@@ -251,27 +251,22 @@ void mlx5e_accel_ipsec_fs_rx_tables_destroy(struct mlx5e_priv *priv);
 int mlx5e_accel_ipsec_fs_rx_tables_create(struct mlx5e_priv *priv);
 void mlx5e_accel_ipsec_fs_rx_catchall_rules_destroy(struct mlx5e_priv *priv);
 int mlx5e_accel_ipsec_fs_rx_catchall_rules(struct mlx5e_priv *priv);
-int mlx5_accel_ipsec_rx_tag_add(if_t ifp, struct mbuf *mb);
-int mlx5e_accel_ipsec_handle_rx_cqe(struct mbuf *mb, struct mlx5_cqe64 *cqe);
+int mlx5_accel_ipsec_rx_tag_add(if_t ifp, struct mlx5e_rq_mbuf *mr);
+void mlx5e_accel_ipsec_handle_rx_cqe(struct mbuf *mb, struct mlx5_cqe64 *cqe,
+    struct mlx5e_rq_mbuf *mr);
+
 static inline int mlx5e_accel_ipsec_flow(struct mlx5_cqe64 *cqe)
 {
 	return MLX5_IPSEC_METADATA_MARKER(be32_to_cpu(cqe->ft_metadata));
 }
 
-static inline void mlx5e_accel_ipsec_handle_rx(struct mbuf *mb, struct mlx5_cqe64 *cqe)
+static inline void
+mlx5e_accel_ipsec_handle_rx(struct mbuf *mb, struct mlx5_cqe64 *cqe,
+    struct mlx5e_rq_mbuf *mr)
 {
 	u32 ipsec_meta_data = be32_to_cpu(cqe->ft_metadata);
 
-	if (!MLX5_IPSEC_METADATA_MARKER(ipsec_meta_data)) {
-		struct m_tag *mtag;
-
-		mtag = m_tag_find(mb, PACKET_TAG_IPSEC_ACCEL_IN, NULL);
-		if (mtag != NULL)
-			m_tag_delete(mb, mtag);
-
-		return;
-	}
-
-	mlx5e_accel_ipsec_handle_rx_cqe(mb, cqe);
+	if (MLX5_IPSEC_METADATA_MARKER(ipsec_meta_data))
+		mlx5e_accel_ipsec_handle_rx_cqe(mb, cqe, mr);
 }
 #endif	/* __MLX5_ACCEL_IPSEC_H__ */
diff --git a/sys/dev/mlx5/mlx5_accel/mlx5_ipsec_rxtx.c b/sys/dev/mlx5/mlx5_accel/mlx5_ipsec_rxtx.c
index 5ff8e021b196..0883cfb2d510 100644
--- a/sys/dev/mlx5/mlx5_accel/mlx5_ipsec_rxtx.c
+++ b/sys/dev/mlx5/mlx5_accel/mlx5_ipsec_rxtx.c
@@ -35,36 +35,41 @@
 
 #define MLX5_IPSEC_METADATA_HANDLE(ipsec_metadata) (ipsec_metadata & 0xFFFFFF)
 
-int mlx5_accel_ipsec_rx_tag_add(if_t ifp, struct mbuf *mb)
+int
+mlx5_accel_ipsec_rx_tag_add(if_t ifp, struct mlx5e_rq_mbuf *mr)
 {
 	struct mlx5e_priv *priv;
-	struct ipsec_accel_in_tag *tag;
-	struct m_tag *mtag;
+	struct ipsec_accel_in_tag *mtag;
 
 	priv = if_getsoftc(ifp);
 	if (priv->ipsec == NULL)
 		return (0);
+	if (mr->ipsec_mtag != NULL)
+		return (0);
 
-	mtag = m_tag_get(PACKET_TAG_IPSEC_ACCEL_IN, sizeof(*tag), M_NOWAIT);
+	mtag = (struct ipsec_accel_in_tag *)m_tag_get(
+	    PACKET_TAG_IPSEC_ACCEL_IN, sizeof(*mtag), M_NOWAIT);
 	if (mtag == NULL)
-		return -ENOMEM;
-
-	m_tag_prepend(mb, mtag);
-	return 0;
+		return (-ENOMEM);
+	mr->ipsec_mtag = mtag;
+	return (0);
 }
 
-int mlx5e_accel_ipsec_handle_rx_cqe(struct mbuf *mb, struct mlx5_cqe64 *cqe)
+void
+mlx5e_accel_ipsec_handle_rx_cqe(struct mbuf *mb, struct mlx5_cqe64 *cqe,
+    struct mlx5e_rq_mbuf *mr)
 {
-	struct ipsec_accel_in_tag *tag;
-	u32  drv_spi;
+	struct ipsec_accel_in_tag *mtag;
+	u32 drv_spi;
 
 	drv_spi = MLX5_IPSEC_METADATA_HANDLE(be32_to_cpu(cqe->ft_metadata));
-	tag = (struct ipsec_accel_in_tag *) m_tag_find(mb, PACKET_TAG_IPSEC_ACCEL_IN, NULL);
-	WARN_ON(tag == NULL);
-	if (tag)
-		tag->drv_spi = drv_spi;
-
-	return 0;
+	mtag = mr->ipsec_mtag;
+	WARN_ON(mtag == NULL);
+	mr->ipsec_mtag = NULL;
+	if (mtag != NULL) {
+		mtag->drv_spi = drv_spi;
+		m_tag_prepend(mb, &mtag->tag);
+	}
 }
 
 void
diff --git a/sys/dev/mlx5/mlx5_en/en.h b/sys/dev/mlx5/mlx5_en/en.h
index 8966aeacb890..80e0b7fbdedb 100644
--- a/sys/dev/mlx5/mlx5_en/en.h
+++ b/sys/dev/mlx5/mlx5_en/en.h
@@ -747,10 +747,13 @@ struct mlx5e_cq {
 	struct mlx5_wq_ctrl wq_ctrl;
 } __aligned(MLX5E_CACHELINE_SIZE);
 
+struct ipsec_accel_in_tag;
+
 struct mlx5e_rq_mbuf {
 	bus_dmamap_t	dma_map;
 	caddr_t		data;
 	struct mbuf	*mbuf;
+	struct ipsec_accel_in_tag *ipsec_mtag;
 };
 
 struct mlx5e_rq {
diff --git a/sys/dev/mlx5/mlx5_en/mlx5_en_main.c b/sys/dev/mlx5/mlx5_en/mlx5_en_main.c
index acb9f72b15c3..5081c1a0b782 100644
--- a/sys/dev/mlx5/mlx5_en/mlx5_en_main.c
+++ b/sys/dev/mlx5/mlx5_en/mlx5_en_main.c
@@ -1325,6 +1325,8 @@ mlx5e_destroy_rq(struct mlx5e_rq *rq)
 	wq_sz = mlx5_wq_ll_get_size(&rq->wq);
 	for (i = 0; i != wq_sz; i++) {
 		if (rq->mbuf[i].mbuf != NULL) {
+			if (rq->mbuf[i].ipsec_mtag != NULL)
+				m_tag_free(&rq->mbuf[i].ipsec_mtag->tag);
 			bus_dmamap_unload(rq->dma_tag, rq->mbuf[i].dma_map);
 			m_freem(rq->mbuf[i].mbuf);
 		}
diff --git a/sys/dev/mlx5/mlx5_en/mlx5_en_rx.c b/sys/dev/mlx5/mlx5_en/mlx5_en_rx.c
index 3d4b75884354..a24bbe3d193e 100644
--- a/sys/dev/mlx5/mlx5_en/mlx5_en_rx.c
+++ b/sys/dev/mlx5/mlx5_en/mlx5_en_rx.c
@@ -70,7 +70,7 @@ mlx5e_alloc_rx_wqe(struct mlx5e_rq *rq,
 	/* get IP header aligned */
 	m_adj(mb, MLX5E_NET_IP_ALIGN);
 
-	err = mlx5_accel_ipsec_rx_tag_add(rq->ifp, mb);
+	err = mlx5_accel_ipsec_rx_tag_add(rq->ifp, &rq->mbuf[ix]);
 	if (err)
 		goto err_free_mbuf;
 	err = -bus_dmamap_load_mbuf_sg(rq->dma_tag, rq->mbuf[ix].dma_map,
@@ -277,9 +277,8 @@ mlx5e_mbuf_tstmp(struct mlx5e_priv *priv, uint64_t hw_tstmp)
 }
 
 static inline void
-mlx5e_build_rx_mbuf(struct mlx5_cqe64 *cqe,
-    struct mlx5e_rq *rq, struct mbuf *mb,
-    u32 cqe_bcnt)
+mlx5e_build_rx_mbuf(struct mlx5_cqe64 *cqe, struct mlx5e_rq *rq,
+    struct mbuf *mb, struct mlx5e_rq_mbuf *mr, u32 cqe_bcnt)
 {
 	if_t ifp = rq->ifp;
 	struct mlx5e_channel *c;
@@ -423,7 +422,7 @@ mlx5e_build_rx_mbuf(struct mlx5_cqe64 *cqe,
 		break;
 	}
 
-	mlx5e_accel_ipsec_handle_rx(mb, cqe);
+	mlx5e_accel_ipsec_handle_rx(mb, cqe, mr);
 }
 
 static inline void
@@ -588,7 +587,8 @@ mlx5e_poll_rx_cq(struct mlx5e_rq *rq, int budget)
 			    rq->mbuf[wqe_counter].dma_map);
 		}
 rx_common:
-		mlx5e_build_rx_mbuf(cqe, rq, mb, byte_cnt);
+		mlx5e_build_rx_mbuf(cqe, rq, mb, &rq->mbuf[wqe_counter],
+		    byte_cnt);
 		rq->stats.bytes += byte_cnt;
 		rq->stats.packets++;
 #ifdef NUMA