git: c6b66002cbf3 - main - mlx5/mlx5ib: Azure/HyperV OFED failing to create IB verbs device
- Go to: [ bottom of page ] [ top of archives ] [ this month ]
Date: Wed, 16 Sep 2026 14:12:47 UTC
The branch main has been updated by kib:
URL: https://cgit.FreeBSD.org/src/commit/?id=c6b66002cbf30b78fcd37c5b017c30965cb213f6
commit c6b66002cbf30b78fcd37c5b017c30965cb213f6
Author: Vag Singh <vag.singh@oneconvergence.com>
AuthorDate: 2025-08-14 05:37:58 +0000
Commit: Konstantin Belousov <kib@FreeBSD.org>
CommitDate: 2026-09-16 14:12:07 +0000
mlx5/mlx5ib: Azure/HyperV OFED failing to create IB verbs device
[PATCH 08/31] FreeBSD OFED support for DPDK MLX5 PMD
a) Refactor the netdev notifier registration into a small helper
function.
This is a pre-step towards having mlx5 IB device over an Ethernet port
which doesn't support RoCE. Also, renamed the de-registration helper
and the new helper as netdev notifier and not roce, to make it clear
this is not only used with roce.
b) Rename RoCE related helpers to reflect being Eth ones
This is a pre-step towards having mlx5 IB device also over Eth ports
where
RoCE is not supported. We change the roce enable/disable and roce_lag
init/fini function names to have _eth instead of _roce.
c) Support RAW Ethernet when RoCE is disabled
On some environments, such as certain SRIOV VF configurations, RoCE is
not supported for mlx5 Ethernet ports. Currently, the driver will not
open IB device on that port.
This is problematic, since we do want user-space RAW Ethernet
(RAW_PACKET QPs) functionality to remain in place. For that end,
enhance the relevant
driver flows such that we do create a device instance in that case.
Differential revision: https://reviews.freebsd.org/D32182
MFC after: 1 month
---
sys/dev/mlx5/mlx5_core/mlx5_core.h | 6 +++++
sys/dev/mlx5/mlx5_core/mlx5_main.c | 27 ++++++++++----------
sys/dev/mlx5/mlx5_ib/mlx5_ib_main.c | 51 ++++++++++++++++++++++---------------
3 files changed, 51 insertions(+), 33 deletions(-)
diff --git a/sys/dev/mlx5/mlx5_core/mlx5_core.h b/sys/dev/mlx5/mlx5_core/mlx5_core.h
index f63bb2070bcf..15b22027fede 100644
--- a/sys/dev/mlx5/mlx5_core/mlx5_core.h
+++ b/sys/dev/mlx5/mlx5_core/mlx5_core.h
@@ -40,6 +40,9 @@
extern int mlx5_core_debug_mask;
+extern struct list_head mlx5_dev_list;
+extern struct mutex mlx5_intf_mutex;
+
#define mlx5_core_dbg(dev, format, ...) \
pr_debug("%s:%s:%d:(pid %d): " format, \
(dev)->priv.name, __func__, __LINE__, curthread->td_proc->p_pid, \
@@ -66,6 +69,9 @@ do { \
__func__, __LINE__, curthread->td_proc->p_pid, \
##__VA_ARGS__)
+#define mlx5_core_for_each_priv(__priv) \
+ list_for_each_entry(__priv, &mlx5_dev_list, dev_list)
+
enum {
MLX5_CMD_DATA, /* print command payload only */
MLX5_CMD_TIME, /* print command execution time */
diff --git a/sys/dev/mlx5/mlx5_core/mlx5_main.c b/sys/dev/mlx5/mlx5_core/mlx5_main.c
index 184457b7d0b5..32e6ba9b509d 100644
--- a/sys/dev/mlx5/mlx5_core/mlx5_main.c
+++ b/sys/dev/mlx5/mlx5_core/mlx5_main.c
@@ -98,8 +98,9 @@ SYSCTL_INT(_hw_mlx5, OID_AUTO, comp_eq_size, CTLFLAG_RDTUN | CTLFLAG_MPSAFE,
"Set default completion EQ size between 1024 and 16384 inclusivly. Value should be power of two.");
static LIST_HEAD(intf_list);
-static LIST_HEAD(dev_list);
-static DEFINE_MUTEX(intf_mutex);
+
+LIST_HEAD(mlx5_dev_list);
+DEFINE_MUTEX(mlx5_intf_mutex);
struct mlx5_device_context {
struct list_head list;
@@ -834,11 +835,11 @@ mlx5_register_device(struct mlx5_core_dev *dev)
struct mlx5_priv *priv = &dev->priv;
struct mlx5_interface *intf;
- mutex_lock(&intf_mutex);
- list_add_tail(&priv->dev_list, &dev_list);
+ mutex_lock(&mlx5_intf_mutex);
+ list_add_tail(&priv->dev_list, &mlx5_dev_list);
list_for_each_entry(intf, &intf_list, list)
mlx5_add_device(intf, priv);
- mutex_unlock(&intf_mutex);
+ mutex_unlock(&mlx5_intf_mutex);
return 0;
}
@@ -849,11 +850,11 @@ mlx5_unregister_device(struct mlx5_core_dev *dev)
struct mlx5_priv *priv = &dev->priv;
struct mlx5_interface *intf;
- mutex_lock(&intf_mutex);
+ mutex_lock(&mlx5_intf_mutex);
list_for_each_entry(intf, &intf_list, list)
mlx5_remove_device(intf, priv);
list_del(&priv->dev_list);
- mutex_unlock(&intf_mutex);
+ mutex_unlock(&mlx5_intf_mutex);
}
int mlx5_register_interface(struct mlx5_interface *intf)
@@ -863,11 +864,11 @@ int mlx5_register_interface(struct mlx5_interface *intf)
if (!intf->add || !intf->remove)
return -EINVAL;
- mutex_lock(&intf_mutex);
+ mutex_lock(&mlx5_intf_mutex);
list_add_tail(&intf->list, &intf_list);
- list_for_each_entry(priv, &dev_list, dev_list)
+ list_for_each_entry(priv, &mlx5_dev_list, dev_list)
mlx5_add_device(intf, priv);
- mutex_unlock(&intf_mutex);
+ mutex_unlock(&mlx5_intf_mutex);
return 0;
}
@@ -877,11 +878,11 @@ void mlx5_unregister_interface(struct mlx5_interface *intf)
{
struct mlx5_priv *priv;
- mutex_lock(&intf_mutex);
- list_for_each_entry(priv, &dev_list, dev_list)
+ mutex_lock(&mlx5_intf_mutex);
+ list_for_each_entry(priv, &mlx5_dev_list, dev_list)
mlx5_remove_device(intf, priv);
list_del(&intf->list);
- mutex_unlock(&intf_mutex);
+ mutex_unlock(&mlx5_intf_mutex);
}
EXPORT_SYMBOL(mlx5_unregister_interface);
diff --git a/sys/dev/mlx5/mlx5_ib/mlx5_ib_main.c b/sys/dev/mlx5/mlx5_ib/mlx5_ib_main.c
index e598291685d7..484ce0a786b9 100644
--- a/sys/dev/mlx5/mlx5_ib/mlx5_ib_main.c
+++ b/sys/dev/mlx5/mlx5_ib/mlx5_ib_main.c
@@ -3195,16 +3195,30 @@ static void get_dev_fw_str(struct ib_device *ibdev, char *str,
fw_rev_min(dev->mdev), fw_rev_sub(dev->mdev));
}
-static int mlx5_roce_lag_init(struct mlx5_ib_dev *dev)
+static int mlx5_eth_lag_init(struct mlx5_ib_dev *dev)
{
return 0;
}
-static void mlx5_roce_lag_cleanup(struct mlx5_ib_dev *dev)
+static void mlx5_eth_lag_cleanup(struct mlx5_ib_dev *dev)
{
}
-static void mlx5_remove_roce_notifier(struct mlx5_ib_dev *dev)
+static int mlx5_add_netdev_notifier(struct mlx5_ib_dev *dev)
+{
+ int err;
+
+ dev->roce.nb.notifier_call = mlx5_netdev_event;
+ err = register_netdevice_notifier(&dev->roce.nb);
+ if (err) {
+ dev->roce.nb.notifier_call = NULL;
+ return err;
+ }
+
+ return 0;
+}
+
+static void mlx5_remove_netdev_notifier(struct mlx5_ib_dev *dev)
{
if (dev->roce.nb.notifier_call) {
unregister_netdevice_notifier(&dev->roce.nb);
@@ -3228,7 +3242,7 @@ mlx5_enable_roce_if_cb(if_t ifp, void *arg)
return (0);
}
-static int mlx5_enable_roce(struct mlx5_ib_dev *dev)
+static int mlx5_enable_eth(struct mlx5_ib_dev *dev)
{
struct epoch_tracker et;
VNET_ITERATOR_DECL(vnet_iter);
@@ -3245,12 +3259,9 @@ static int mlx5_enable_roce(struct mlx5_ib_dev *dev)
NET_EPOCH_EXIT(et);
VNET_LIST_RUNLOCK();
- dev->roce.nb.notifier_call = mlx5_netdev_event;
- err = register_netdevice_notifier(&dev->roce.nb);
- if (err) {
- dev->roce.nb.notifier_call = NULL;
+ err = mlx5_add_netdev_notifier(dev);
+ if (err)
return err;
- }
if (MLX5_CAP_GEN(dev->mdev, roce)) {
err = mlx5_nic_vport_enable_roce(dev->mdev);
@@ -3258,7 +3269,7 @@ static int mlx5_enable_roce(struct mlx5_ib_dev *dev)
goto err_unregister_netdevice_notifier;
}
- err = mlx5_roce_lag_init(dev);
+ err = mlx5_eth_lag_init(dev);
if (err)
goto err_disable_roce;
@@ -3269,13 +3280,13 @@ err_disable_roce:
mlx5_nic_vport_disable_roce(dev->mdev);
err_unregister_netdevice_notifier:
- mlx5_remove_roce_notifier(dev);
+ mlx5_remove_netdev_notifier(dev);
return err;
}
-static void mlx5_disable_roce(struct mlx5_ib_dev *dev)
+static void mlx5_disable_eth(struct mlx5_ib_dev *dev)
{
- mlx5_roce_lag_cleanup(dev);
+ mlx5_eth_lag_cleanup(dev);
if (MLX5_CAP_GEN(dev->mdev, roce))
mlx5_nic_vport_disable_roce(dev->mdev);
}
@@ -3773,14 +3784,14 @@ static void *mlx5_ib_add(struct mlx5_core_dev *mdev)
spin_lock_init(&dev->reset_flow_resource_lock);
if (ll == IB_LINK_LAYER_ETHERNET) {
- err = mlx5_enable_roce(dev);
+ err = mlx5_enable_eth(dev);
if (err)
goto err_free_port;
}
err = create_dev_resources(&dev->devr);
if (err)
- goto err_disable_roce;
+ goto err_disable_eth;
err = mlx5_ib_odp_init_one(dev);
if (err)
@@ -3846,10 +3857,10 @@ err_odp:
err_rsrc:
destroy_dev_resources(&dev->devr);
-err_disable_roce:
+err_disable_eth:
if (ll == IB_LINK_LAYER_ETHERNET) {
- mlx5_disable_roce(dev);
- mlx5_remove_roce_notifier(dev);
+ mlx5_disable_eth(dev);
+ mlx5_remove_netdev_notifier(dev);
}
err_free_port:
@@ -3868,7 +3879,7 @@ static void mlx5_ib_remove(struct mlx5_core_dev *mdev, void *context)
mlx5_ib_devx_cleanup_event_table(dev);
mlx5_ib_cleanup_congestion(dev);
- mlx5_remove_roce_notifier(dev);
+ mlx5_remove_netdev_notifier(dev);
ib_unregister_device(&dev->ib_dev);
mlx5_ib_stage_bfreg_cleanup(dev);
mlx5_ib_dealloc_q_counters(dev);
@@ -3876,7 +3887,7 @@ static void mlx5_ib_remove(struct mlx5_core_dev *mdev, void *context)
mlx5_ib_odp_remove_one(dev);
destroy_dev_resources(&dev->devr);
if (ll == IB_LINK_LAYER_ETHERNET)
- mlx5_disable_roce(dev);
+ mlx5_disable_eth(dev);
kfree(dev->port);
ib_dealloc_device(&dev->ib_dev);
}