git: ea00d7e8cab9 - main - mlx5: Add raw ethernet local loopback support.

The branch main has been updated by hselasky:

URL: https://cgit.FreeBSD.org/src/commit/?id=ea00d7e8cab97f818c9793a112249ab79a2f8322

commit ea00d7e8cab97f818c9793a112249ab79a2f8322
Author:     Hans Petter Selasky <hselasky@FreeBSD.org>
AuthorDate: 2022-02-01 15:20:15 +0000
Commit:     Hans Petter Selasky <hselasky@FreeBSD.org>
CommitDate: 2022-02-01 15:21:16 +0000

    mlx5: Add raw ethernet local loopback support.
    
    Currently, unicast/multicast loopback raw ethernet (non-RDMA) packets
    are sent back to the vport.  A unicast loopback packet is the packet
    with destination MAC address the same as the source MAC address.  For
    multicast, the destination MAC address is in the vport's multicast
    filter list.
    
    Moreover, the local loopback is not needed if there is one or none
    user space context.
    
    After this patch, the raw ethernet unicast and multicast local
    loopback are disabled by default. When there is more than one user
    space context, the local loopback is enabled.
    
    Note that when local loopback is disabled, raw ethernet packets are
    not looped back to the vport and are forwarded to the next routing
    level (eswitch, or multihost switch, or out to the wire depending on
    the configuration).
    
    Linux commits:
    c85023e153e3824661d07307138fdeff41f6d86a
    8978cc921fc7fad3f4d6f91f1da01352aeeeff25
    
    MFC after:      1 week
    Sponsored by:   NVIDIA Networking
---
 sys/dev/mlx5/mlx5_core/mlx5_main.c     | 17 +++++++++++++++++
 sys/dev/mlx5/mlx5_en/mlx5_en_ethtool.c | 12 ++++++++++++
 sys/dev/mlx5/mlx5_ib/mlx5_ib.h         |  4 ++++
 sys/dev/mlx5/mlx5_ib/mlx5_ib_main.c    | 32 +++++++++++++++++++++++++++++++-
 4 files changed, 64 insertions(+), 1 deletion(-)

diff --git a/sys/dev/mlx5/mlx5_core/mlx5_main.c b/sys/dev/mlx5/mlx5_core/mlx5_main.c
index 0745d19b2725..aaa8f657432a 100644
--- a/sys/dev/mlx5/mlx5_core/mlx5_main.c
+++ b/sys/dev/mlx5/mlx5_core/mlx5_main.c
@@ -532,6 +532,17 @@ static int set_hca_ctrl(struct mlx5_core_dev *dev)
 	return err;
 }
 
+static int mlx5_core_set_hca_defaults(struct mlx5_core_dev *dev)
+{
+	int ret = 0;
+
+	/* Disable local_lb by default */
+	if (MLX5_CAP_GEN(dev, port_type) == MLX5_CAP_PORT_TYPE_ETH)
+		ret = mlx5_nic_vport_update_local_lb(dev, false);
+
+       return ret;
+}
+
 static int mlx5_core_enable_hca(struct mlx5_core_dev *dev, u16 func_id)
 {
 	u32 out[MLX5_ST_SZ_DW(enable_hca_out)] = {0};
@@ -1135,6 +1146,12 @@ static int mlx5_load_one(struct mlx5_core_dev *dev, struct mlx5_priv *priv,
 		goto err_free_comp_eqs;
 	}
 
+	err = mlx5_core_set_hca_defaults(dev);
+	if (err) {
+		mlx5_core_err(dev, "Failed to set HCA defaults %d\n", err);
+		goto err_free_comp_eqs;
+	}
+
 	err = mlx5_mpfs_init(dev);
 	if (err) {
 		mlx5_core_err(dev, "mpfs init failed %d\n", err);
diff --git a/sys/dev/mlx5/mlx5_en/mlx5_en_ethtool.c b/sys/dev/mlx5/mlx5_en/mlx5_en_ethtool.c
index d07aee1f0793..f57d70080a46 100644
--- a/sys/dev/mlx5/mlx5_en/mlx5_en_ethtool.c
+++ b/sys/dev/mlx5/mlx5_en/mlx5_en_ethtool.c
@@ -1230,6 +1230,12 @@ mlx5e_ethtool_handler(SYSCTL_HANDLER_ARGS)
 		break;
 
 	case MLX5_PARAM_OFFSET(mc_local_lb):
+		/* check if mlx5ib is managing this feature */
+		if (MLX5_CAP_GEN(priv->mdev, port_type) != MLX5_CAP_PORT_TYPE_ETH) {
+			error = EOPNOTSUPP;
+			break;
+		}
+
 		priv->params_ethtool.mc_local_lb =
 		    priv->params_ethtool.mc_local_lb ? 1 : 0;
 
@@ -1242,6 +1248,12 @@ mlx5e_ethtool_handler(SYSCTL_HANDLER_ARGS)
 		break;
 
 	case MLX5_PARAM_OFFSET(uc_local_lb):
+		/* check if mlx5ib is managing this feature */
+		if (MLX5_CAP_GEN(priv->mdev, port_type) != MLX5_CAP_PORT_TYPE_ETH) {
+			error = EOPNOTSUPP;
+			break;
+		}
+
 		priv->params_ethtool.uc_local_lb =
 		    priv->params_ethtool.uc_local_lb ? 1 : 0;
 
diff --git a/sys/dev/mlx5/mlx5_ib/mlx5_ib.h b/sys/dev/mlx5/mlx5_ib/mlx5_ib.h
index ba4b49f24831..316a45806966 100644
--- a/sys/dev/mlx5/mlx5_ib/mlx5_ib.h
+++ b/sys/dev/mlx5/mlx5_ib/mlx5_ib.h
@@ -795,6 +795,10 @@ struct mlx5_ib_dev {
 	struct mlx5_ib_congestion congestion;
 
 	struct mlx5_async_ctx	async_ctx;
+
+	/* protect the user_td */
+	struct mutex		lb_mutex;
+	u32			user_td;
 };
 
 static inline struct mlx5_ib_cq *to_mibcq(struct mlx5_core_cq *mcq)
diff --git a/sys/dev/mlx5/mlx5_ib/mlx5_ib_main.c b/sys/dev/mlx5/mlx5_ib/mlx5_ib_main.c
index 598a506b3b70..70a7fcd758de 100644
--- a/sys/dev/mlx5/mlx5_ib/mlx5_ib_main.c
+++ b/sys/dev/mlx5/mlx5_ib/mlx5_ib_main.c
@@ -1206,7 +1206,22 @@ static int mlx5_ib_alloc_transport_domain(struct mlx5_ib_dev *dev, u32 *tdn,
 	if (err)
 		return err;
 
-	return 0;
+	if ((MLX5_CAP_GEN(dev->mdev, port_type) != MLX5_CAP_PORT_TYPE_ETH) ||
+	    (!MLX5_CAP_GEN(dev->mdev, disable_local_lb_uc) &&
+	     !MLX5_CAP_GEN(dev->mdev, disable_local_lb_mc)))
+		return 0;
+
+	mutex_lock(&dev->lb_mutex);
+	dev->user_td++;
+
+	if (dev->user_td == 2)
+		err = mlx5_nic_vport_update_local_lb(dev->mdev, true);
+
+	mutex_unlock(&dev->lb_mutex);
+
+	if (err != 0)
+		mlx5_dealloc_transport_domain(dev->mdev, *tdn, uid);
+	return err;
 }
 
 static void mlx5_ib_dealloc_transport_domain(struct mlx5_ib_dev *dev, u32 tdn,
@@ -1216,6 +1231,19 @@ static void mlx5_ib_dealloc_transport_domain(struct mlx5_ib_dev *dev, u32 tdn,
 		return;
 
 	mlx5_dealloc_transport_domain(dev->mdev, tdn, uid);
+
+	if ((MLX5_CAP_GEN(dev->mdev, port_type) != MLX5_CAP_PORT_TYPE_ETH) ||
+	    (!MLX5_CAP_GEN(dev->mdev, disable_local_lb_uc) &&
+	     !MLX5_CAP_GEN(dev->mdev, disable_local_lb_mc)))
+		return;
+
+	mutex_lock(&dev->lb_mutex);
+	dev->user_td--;
+
+	if (dev->user_td < 2)
+		mlx5_nic_vport_update_local_lb(dev->mdev, false);
+
+	mutex_unlock(&dev->lb_mutex);
 }
 
 static int mlx5_ib_alloc_ucontext(struct ib_ucontext *uctx,
@@ -3284,6 +3312,8 @@ static void *mlx5_ib_add(struct mlx5_core_dev *mdev)
 
 	MLX5_INIT_DOORBELL_LOCK(&dev->uar_lock);
 
+	mutex_init(&dev->lb_mutex);
+
 	INIT_IB_DEVICE_OPS(&dev->ib_dev.ops, mlx5, MLX5);
 	snprintf(dev->ib_dev.name, IB_DEVICE_NAME_MAX, "mlx5_%d", device_get_unit(mdev->pdev->dev.bsddev));
 	dev->ib_dev.owner		= THIS_MODULE;