git: 4b32c67bbfdb - main - verbs/mlxs: mlx5: Add tunnel offloads support in direct verbs
- Go to: [ bottom of page ] [ top of archives ] [ this month ]
Date: Wed, 16 Sep 2026 14:13:25 UTC
The branch main has been updated by kib:
URL: https://cgit.FreeBSD.org/src/commit/?id=4b32c67bbfdbbd8ae557f18324a7c115668ca01a
commit 4b32c67bbfdbbd8ae557f18324a7c115668ca01a
Author: Vag Singh <vag.singh@oneconvergence.com>
AuthorDate: 2025-08-14 07:02:06 +0000
Commit: Konstantin Belousov <kib@FreeBSD.org>
CommitDate: 2026-09-16 14:12:09 +0000
verbs/mlxs: mlx5: Add tunnel offloads support in direct verbs
[PATCH 28/31] FreeBSD OFED support for DPDK MLX5 PMD
a) In order to enable offloading such as checksum and LRO for incoming
tunneling traffic, the QP should be created with tunnel offloads flag -
MLX5DV_QP_CREATE_TUNNEL_OFFLOAD.
b) Reports capability of which tunneling type supports
the tunneling offloads.
c) verbs: Add support in RSS of the inner packet
Some user space application would like to do RSS on the inner packet
fields instead of the outer. When user will set the IBV_RX_HASH_INNER
bit with one of the other hash fields, then the RSS will be on the inner
packet.
Differential revision: https://reviews.freebsd.org/D32203
MFC after: 1 month
---
contrib/ofed/libibverbs/man/ibv_create_qp_ex.3 | 20 ++++++++++
contrib/ofed/libibverbs/verbs.h | 3 +-
contrib/ofed/libmlx5/libmlx5.map | 1 +
contrib/ofed/libmlx5/mlx5-abi.h | 5 ++-
contrib/ofed/libmlx5/mlx5.c | 4 ++
contrib/ofed/libmlx5/mlx5.h | 1 +
contrib/ofed/libmlx5/mlx5dv.h | 28 +++++++++++++-
contrib/ofed/libmlx5/mlx5dv_query_device.3 | 28 ++++++++------
contrib/ofed/libmlx5/verbs.c | 53 +++++++++++++++++++++++---
9 files changed, 123 insertions(+), 20 deletions(-)
diff --git a/contrib/ofed/libibverbs/man/ibv_create_qp_ex.3 b/contrib/ofed/libibverbs/man/ibv_create_qp_ex.3
index 99ae49757683..4957a69258ec 100644
--- a/contrib/ofed/libibverbs/man/ibv_create_qp_ex.3
+++ b/contrib/ofed/libibverbs/man/ibv_create_qp_ex.3
@@ -69,6 +69,26 @@ uint64_t rx_hash_fields_mask; /* RX fields that should particip
.in -8
};
.fi
+
+.nf
+enum ibv_rx_hash_fields {
+.in +8
+IBV_RX_HASH_SRC_IPV4 = 1 << 0,
+IBV_RX_HASH_DST_IPV4 = 1 << 1,
+IBV_RX_HASH_SRC_IPV6 = 1 << 2,
+IBV_RX_HASH_DST_IPV6 = 1 << 3,
+IBV_RX_HASH_SRC_PORT_TCP = 1 << 4,
+IBV_RX_HASH_DST_PORT_TCP = 1 << 5,
+IBV_RX_HASH_SRC_PORT_UDP = 1 << 6,
+IBV_RX_HASH_DST_PORT_UDP = 1 << 7,
+/* When using tunneling protocol, e.g. VXLAN, then we have an inner (encapsulated packet) and outer.
+ * For applying RSS on the inner packet, then the following field should be set with one of the L3/L4 fields.
+*/
+IBV_RX_HASH_INNER = (1UL << 31),
+.in -8
+};
+.fi
+
.PP
The function
.B ibv_create_qp_ex()
diff --git a/contrib/ofed/libibverbs/verbs.h b/contrib/ofed/libibverbs/verbs.h
index e7769e072044..54a8a2d39342 100644
--- a/contrib/ofed/libibverbs/verbs.h
+++ b/contrib/ofed/libibverbs/verbs.h
@@ -236,7 +236,8 @@ enum ibv_rx_hash_fields {
IBV_RX_HASH_SRC_PORT_TCP = 1 << 4,
IBV_RX_HASH_DST_PORT_TCP = 1 << 5,
IBV_RX_HASH_SRC_PORT_UDP = 1 << 6,
- IBV_RX_HASH_DST_PORT_UDP = 1 << 7
+ IBV_RX_HASH_DST_PORT_UDP = 1 << 7,
+ IBV_RX_HASH_INNER = (1UL << 31),
};
struct ibv_rss_caps {
diff --git a/contrib/ofed/libmlx5/libmlx5.map b/contrib/ofed/libmlx5/libmlx5.map
index b1402dc26e70..f7978224dc6f 100644
--- a/contrib/ofed/libmlx5/libmlx5.map
+++ b/contrib/ofed/libmlx5/libmlx5.map
@@ -20,5 +20,6 @@ MLX5_1.2 {
MLX5_1.3 {
global:
+ mlx5dv_create_qp;
mlx5dv_create_wq;
} MLX5_1.2;
diff --git a/contrib/ofed/libmlx5/mlx5-abi.h b/contrib/ofed/libmlx5/mlx5-abi.h
index 13a953a0bc50..8e8e71d9bd1c 100644
--- a/contrib/ofed/libmlx5/mlx5-abi.h
+++ b/contrib/ofed/libmlx5/mlx5-abi.h
@@ -43,6 +43,7 @@
enum {
MLX5_QP_FLAG_SIGNATURE = 1 << 0,
MLX5_QP_FLAG_SCATTER_CQE = 1 << 1,
+ MLX5_QP_FLAG_TUNNEL_OFFLOADS = 1 << 2,
};
enum {
@@ -178,7 +179,7 @@ struct mlx5_create_qp_ex_rss {
__u8 reserved[6];
__u8 rx_hash_key[128];
__u32 comp_mask;
- __u32 reserved1;
+ __u32 create_flags;
};
struct mlx5_create_qp_resp_ex {
@@ -302,6 +303,8 @@ struct mlx5_query_device_ex_resp {
__u32 reserved;
struct mlx5dv_sw_parsing_caps sw_parsing_caps;
struct mlx5_striding_rq_caps striding_rq_caps;
+ __u32 tunnel_offloads_caps;
+ __u32 reserved4;
};
#endif /* MLX5_ABI_H */
diff --git a/contrib/ofed/libmlx5/mlx5.c b/contrib/ofed/libmlx5/mlx5.c
index c2daf25287f8..b36c0c763fa0 100644
--- a/contrib/ofed/libmlx5/mlx5.c
+++ b/contrib/ofed/libmlx5/mlx5.c
@@ -671,6 +671,10 @@ int mlx5dv_query_device(struct ibv_context *ctx_in,
comp_mask_out |= MLX5DV_CONTEXT_MASK_STRIDING_RQ;
}
+ if (attrs_out->comp_mask & MLX5DV_CONTEXT_MASK_TUNNEL_OFFLOADS) {
+ attrs_out->tunnel_offloads_caps = mctx->tunnel_offloads_caps;
+ comp_mask_out |= MLX5DV_CONTEXT_MASK_TUNNEL_OFFLOADS;
+ }
attrs_out->comp_mask = comp_mask_out;
diff --git a/contrib/ofed/libmlx5/mlx5.h b/contrib/ofed/libmlx5/mlx5.h
index e65847b37e31..bcb43ed4791f 100644
--- a/contrib/ofed/libmlx5/mlx5.h
+++ b/contrib/ofed/libmlx5/mlx5.h
@@ -274,6 +274,7 @@ struct mlx5_context {
struct mlx5dv_ctx_allocators extern_alloc;
struct mlx5dv_sw_parsing_caps sw_parsing_caps;
struct mlx5dv_striding_rq_caps striding_rq_caps;
+ uint32_t tunnel_offloads_caps;
};
struct mlx5_bitmap {
diff --git a/contrib/ofed/libmlx5/mlx5dv.h b/contrib/ofed/libmlx5/mlx5dv.h
index b8e7b204f536..fb6180b50104 100644
--- a/contrib/ofed/libmlx5/mlx5dv.h
+++ b/contrib/ofed/libmlx5/mlx5dv.h
@@ -60,7 +60,8 @@ enum mlx5dv_context_comp_mask {
MLX5DV_CONTEXT_MASK_CQE_COMPRESION = 1 << 0,
MLX5DV_CONTEXT_MASK_SWP = 1 << 1,
MLX5DV_CONTEXT_MASK_STRIDING_RQ = 1 << 2,
- MLX5DV_CONTEXT_MASK_RESERVED = 1 << 3,
+ MLX5DV_CONTEXT_MASK_TUNNEL_OFFLOADS = 1 << 3,
+ MLX5DV_CONTEXT_MASK_RESERVED = 1 << 4,
};
struct mlx5dv_cqe_comp_caps {
@@ -81,6 +82,13 @@ struct mlx5dv_striding_rq_caps {
uint32_t supported_qpts;
};
+enum mlx5dv_tunnel_offloads {
+ MLX5DV_RAW_PACKET_CAP_TUNNELED_OFFLOAD_VXLAN = 1 << 0,
+ MLX5DV_RAW_PACKET_CAP_TUNNELED_OFFLOAD_GRE = 1 << 1,
+ MLX5DV_RAW_PACKET_CAP_TUNNELED_OFFLOAD_GENEVE = 1 << 2,
+};
+
+
/*
* Direct verbs device-specific attributes
*/
@@ -91,6 +99,7 @@ struct mlx5dv_context {
struct mlx5dv_cqe_comp_caps cqe_comp_caps;
struct mlx5dv_sw_parsing_caps sw_parsing_caps;
struct mlx5dv_striding_rq_caps striding_rq_caps;
+ uint32_t tunnel_offloads_caps;
};
enum mlx5dv_context_flags {
@@ -116,6 +125,23 @@ struct mlx5dv_cq_init_attr {
struct ibv_cq_ex *mlx5dv_create_cq(struct ibv_context *context,
struct ibv_cq_init_attr_ex *cq_attr,
struct mlx5dv_cq_init_attr *mlx5_cq_attr);
+
+enum mlx5dv_qp_create_flags {
+ MLX5DV_QP_CREATE_TUNNEL_OFFLOADS = 1 << 0,
+};
+
+enum mlx5dv_qp_init_attr_mask {
+ MLX5DV_QP_INIT_ATTR_MASK_QP_CREATE_FLAGS = 1 << 0,
+};
+
+struct mlx5dv_qp_init_attr {
+ uint64_t comp_mask; /* Use enum mlx5dv_qp_init_attr_mask */
+ uint32_t create_flags; /* Use enum mlx5dv_qp_create_flags */
+};
+
+struct ibv_qp *mlx5dv_create_qp(struct ibv_context *context,
+ struct ibv_qp_init_attr_ex *qp_attr,
+ struct mlx5dv_qp_init_attr *mlx5_qp_attr);
/*
* Most device capabilities are exported by ibv_query_device(...),
* but there is HW device-specific information which is important
diff --git a/contrib/ofed/libmlx5/mlx5dv_query_device.3 b/contrib/ofed/libmlx5/mlx5dv_query_device.3
index 2e6d45b32c27..ba812ba06a4c 100644
--- a/contrib/ofed/libmlx5/mlx5dv_query_device.3
+++ b/contrib/ofed/libmlx5/mlx5dv_query_device.3
@@ -47,13 +47,8 @@ uint64_t flags;
uint64_t comp_mask;
struct mlx5dv_cqe_comp_caps cqe_comp_caps;
struct mlx5dv_sw_parsing_caps sw_parsing_caps;
-.in -8
-};
-
-enum mlx5dv_context_comp_mask {
-.in +8
- MLX5DV_CONTEXT_MASK_CQE_COMPRESION = (1 << 0),
- MLX5DV_CONTEXT_MASK_SWP = (1 << 1),
+struct mlx5dv_striding_rq_caps striding_rq_caps;
+uint32_t tunnel_offloads_caps;
.in -8
};
@@ -73,10 +68,21 @@ enum mlx5dv_context_flags {
.nf
enum mlx5dv_context_comp_mask {
.in +8
- MLX5DV_CONTEXT_MASK_CQE_COMPRESION = 1 << 0,
- MLX5DV_CONTEXT_MASK_SWP = 1 << 1,
- MLX5DV_CONTEXT_MASK_STRIDING_RQ = 1 << 2,
- MLX5DV_CONTEXT_MASK_RESERVED = 1 << 3,
+ MLX5DV_CONTEXT_MASK_CQE_COMPRESION = 1 << 0,
+ MLX5DV_CONTEXT_MASK_SWP = 1 << 1,
+ MLX5DV_CONTEXT_MASK_STRIDING_RQ = 1 << 2,
+ MLX5DV_CONTEXT_MASK_TUNNEL_OFFLOADS = 1 << 3,
+ MLX5DV_CONTEXT_MASK_RESERVED = 1 << 4,
+.in -8
+};
+
+.PP
+.nf
+enum mlx5dv_tunnel_offloads {
+.in +8
+ MLX5DV_RAW_PACKET_CAP_TUNNELED_OFFLOAD_VXLAN = 1 << 0,
+ MLX5DV_RAW_PACKET_CAP_TUNNELED_OFFLOAD_GRE = 1 << 1,
+ MLX5DV_RAW_PACKET_CAP_TUNNELED_OFFLOAD_GENEVE = 1 << 2,
.in -8
};
diff --git a/contrib/ofed/libmlx5/verbs.c b/contrib/ofed/libmlx5/verbs.c
index 4f3859abd93b..5e91dcaf5f7a 100644
--- a/contrib/ofed/libmlx5/verbs.c
+++ b/contrib/ofed/libmlx5/verbs.c
@@ -1193,7 +1193,8 @@ static void mlx5_free_qp_buf(struct mlx5_qp *qp)
static int mlx5_cmd_create_rss_qp(struct ibv_context *context,
struct ibv_qp_init_attr_ex *attr,
- struct mlx5_qp *qp)
+ struct mlx5_qp *qp,
+ uint32_t mlx5_create_flags)
{
struct mlx5_create_qp_ex_rss cmd_ex_rss = {};
struct mlx5_create_qp_resp_ex resp = {};
@@ -1207,6 +1208,7 @@ static int mlx5_cmd_create_rss_qp(struct ibv_context *context,
cmd_ex_rss.rx_hash_fields_mask = attr->rx_hash_conf.rx_hash_fields_mask;
cmd_ex_rss.rx_hash_function = attr->rx_hash_conf.rx_hash_function;
cmd_ex_rss.rx_key_len = attr->rx_hash_conf.rx_hash_key_len;
+ cmd_ex_rss.create_flags = mlx5_create_flags;
memcpy(cmd_ex_rss.rx_hash_key, attr->rx_hash_conf.rx_hash_key,
attr->rx_hash_conf.rx_hash_key_len);
@@ -1259,6 +1261,10 @@ enum {
IBV_QP_INIT_ATTR_RX_HASH),
};
+enum {
+ MLX5_DV_CREATE_QP_SUP_COMP_MASK = MLX5DV_QP_INIT_ATTR_MASK_QP_CREATE_FLAGS
+};
+
enum {
MLX5_CREATE_QP_EX2_COMP_MASK = (IBV_QP_INIT_ATTR_CREATE_FLAGS |
IBV_QP_INIT_ATTR_MAX_TSO_HEADER |
@@ -1267,7 +1273,8 @@ enum {
};
static struct ibv_qp *create_qp(struct ibv_context *context,
- struct ibv_qp_init_attr_ex *attr)
+ struct ibv_qp_init_attr_ex *attr,
+ struct mlx5dv_qp_init_attr *mlx5_qp_attr)
{
struct mlx5_create_qp cmd;
struct mlx5_create_qp_resp resp;
@@ -1278,6 +1285,7 @@ static struct ibv_qp *create_qp(struct ibv_context *context,
struct ibv_qp *ibqp;
int32_t usr_idx = 0;
uint32_t uuar_index;
+ uint32_t mlx5_create_flags = 0;
FILE *fp = ctx->dbg_fp;
if (attr->comp_mask & ~MLX5_CREATE_QP_SUP_COMP_MASK)
@@ -1299,14 +1307,39 @@ static struct ibv_qp *create_qp(struct ibv_context *context,
memset(&resp, 0, sizeof(resp));
memset(&resp_ex, 0, sizeof(resp_ex));
+ if (mlx5_qp_attr) {
+ if (!check_comp_mask(mlx5_qp_attr->comp_mask,
+ MLX5_DV_CREATE_QP_SUP_COMP_MASK)) {
+ mlx5_dbg(fp, MLX5_DBG_QP,
+ "Unsupported vendor comp_mask for create_qp\n");
+ errno = EINVAL;
+ goto err;
+ }
+
+ if (mlx5_qp_attr->comp_mask &
+ MLX5DV_QP_INIT_ATTR_MASK_QP_CREATE_FLAGS) {
+ if (mlx5_qp_attr->create_flags &
+ MLX5DV_QP_CREATE_TUNNEL_OFFLOADS) {
+ mlx5_create_flags = MLX5_QP_FLAG_TUNNEL_OFFLOADS;
+ } else {
+ mlx5_dbg(fp, MLX5_DBG_QP,
+ "Unsupported creation flags requested for create_qp\n");
+ errno = EINVAL;
+ goto err;
+ }
+ }
+ }
+
if (attr->comp_mask & IBV_QP_INIT_ATTR_RX_HASH) {
- ret = mlx5_cmd_create_rss_qp(context, attr, qp);
+ ret = mlx5_cmd_create_rss_qp(context, attr, qp,
+ mlx5_create_flags);
if (ret)
goto err;
return ibqp;
}
+ cmd.flags = mlx5_create_flags;
qp->wq_sig = qp_sig_enabled();
if (qp->wq_sig)
cmd.flags |= MLX5_QP_FLAG_SIGNATURE;
@@ -1465,7 +1498,7 @@ struct ibv_qp *mlx5_create_qp(struct ibv_pd *pd,
memcpy(&attrx, attr, sizeof(*attr));
attrx.comp_mask = IBV_QP_INIT_ATTR_PD;
attrx.pd = pd;
- qp = create_qp(pd->context, &attrx);
+ qp = create_qp(pd->context, &attrx, NULL);
if (qp)
memcpy(attr, &attrx, sizeof(*attr));
@@ -1793,7 +1826,14 @@ int mlx5_detach_mcast(struct ibv_qp *qp, const union ibv_gid *gid, uint16_t lid)
struct ibv_qp *mlx5_create_qp_ex(struct ibv_context *context,
struct ibv_qp_init_attr_ex *attr)
{
- return create_qp(context, attr);
+ return create_qp(context, attr, NULL);
+}
+
+struct ibv_qp *mlx5dv_create_qp(struct ibv_context *context,
+ struct ibv_qp_init_attr_ex *qp_attr,
+ struct mlx5dv_qp_init_attr *mlx5_qp_attr)
+{
+ return create_qp(context, qp_attr, mlx5_qp_attr);
}
int mlx5_get_srq_num(struct ibv_srq *srq, uint32_t *srq_num)
@@ -1887,7 +1927,7 @@ create_cmd_qp(struct ibv_context *context,
init_attr.send_cq = srq_attr->cq;
init_attr.recv_cq = srq_attr->cq;
- qp = create_qp(context, &init_attr);
+ qp = create_qp(context, &init_attr, NULL);
if (!qp)
return NULL;
@@ -2140,6 +2180,7 @@ int mlx5_query_device_ex(struct ibv_context *context,
mctx->cqe_comp_caps = resp.cqe_comp_caps;
mctx->sw_parsing_caps = resp.sw_parsing_caps;
mctx->striding_rq_caps = resp.striding_rq_caps.caps;
+ mctx->tunnel_offloads_caps = resp.tunnel_offloads_caps;
major = (raw_fw_ver >> 32) & 0xffff;
minor = (raw_fw_ver >> 16) & 0xffff;