svn commit: r352956 - head/sys/dev/mlx5/mlx5_ib

[Hans Petter Selasky]

Author: hselasky
Date: Wed Oct  2 09:09:28 2019
New Revision: 352956
URL: https://svnweb.freebsd.org/changeset/base/352956

Log:
  Fix reported max SGE calculation in mlx5ib.
  
  Add the 512 bytes limit of RDMA READ and the size of remote address to the max
  SGE calculation.
  
  Submitted by:	slavash@
  Linux commit:	288c01b746aa
  MFC after:	3 days
  Sponsored by:	Mellanox Technologies

Modified:
  head/sys/dev/mlx5/mlx5_ib/mlx5_ib_main.c
  head/sys/dev/mlx5/mlx5_ib/mlx5_ib_qp.c

Modified: head/sys/dev/mlx5/mlx5_ib/mlx5_ib_main.c
==============================================================================
--- head/sys/dev/mlx5/mlx5_ib/mlx5_ib_main.c	Wed Oct  2 09:06:13 2019	(r352955)
+++ head/sys/dev/mlx5/mlx5_ib/mlx5_ib_main.c	Wed Oct  2 09:09:28 2019	(r352956)
@@ -636,6 +636,7 @@ static int mlx5_ib_query_device(struct ib_device *ibde
 	struct mlx5_ib_dev *dev = to_mdev(ibdev);
 	struct mlx5_core_dev *mdev = dev->mdev;
 	int err = -ENOMEM;
+	int max_sq_desc;
 	int max_rq_sg;
 	int max_sq_sg;
 	u64 min_page_size = 1ull << MLX5_CAP_GEN(mdev, log_pg_sz);
@@ -758,9 +759,10 @@ static int mlx5_ib_query_device(struct ib_device *ibde
 	props->max_qp_wr	   = 1 << MLX5_CAP_GEN(mdev, log_max_qp_sz);
 	max_rq_sg =  MLX5_CAP_GEN(mdev, max_wqe_sz_rq) /
 		     sizeof(struct mlx5_wqe_data_seg);
-	max_sq_sg = (MLX5_CAP_GEN(mdev, max_wqe_sz_sq) -
-		     sizeof(struct mlx5_wqe_ctrl_seg)) /
-		     sizeof(struct mlx5_wqe_data_seg);
+	max_sq_desc = min_t(int, MLX5_CAP_GEN(mdev, max_wqe_sz_sq), 512);
+	max_sq_sg = (max_sq_desc - sizeof(struct mlx5_wqe_ctrl_seg) -
+		     sizeof(struct mlx5_wqe_raddr_seg)) /
+		sizeof(struct mlx5_wqe_data_seg);
 	props->max_sge = min(max_rq_sg, max_sq_sg);
 	props->max_sge_rd	   = MLX5_MAX_SGE_RD;
 	props->max_cq		   = 1 << MLX5_CAP_GEN(mdev, log_max_cq);

Modified: head/sys/dev/mlx5/mlx5_ib/mlx5_ib_qp.c
==============================================================================
--- head/sys/dev/mlx5/mlx5_ib/mlx5_ib_qp.c	Wed Oct  2 09:06:13 2019	(r352955)
+++ head/sys/dev/mlx5/mlx5_ib/mlx5_ib_qp.c	Wed Oct  2 09:09:28 2019	(r352956)
@@ -346,6 +346,29 @@ static int calc_send_wqe(struct ib_qp_init_attr *attr)
 		return ALIGN(max_t(int, inl_size, size), MLX5_SEND_WQE_BB);
 }
 
+static int get_send_sge(struct ib_qp_init_attr *attr, int wqe_size)
+{
+	int max_sge;
+
+	if (attr->qp_type == IB_QPT_RC)
+		max_sge = (min_t(int, wqe_size, 512) -
+			   sizeof(struct mlx5_wqe_ctrl_seg) -
+			   sizeof(struct mlx5_wqe_raddr_seg)) /
+			sizeof(struct mlx5_wqe_data_seg);
+	else if (attr->qp_type == IB_QPT_XRC_INI)
+		max_sge = (min_t(int, wqe_size, 512) -
+			   sizeof(struct mlx5_wqe_ctrl_seg) -
+			   sizeof(struct mlx5_wqe_xrc_seg) -
+			   sizeof(struct mlx5_wqe_raddr_seg)) /
+			sizeof(struct mlx5_wqe_data_seg);
+	else
+		max_sge = (wqe_size - sq_overhead(attr)) /
+			sizeof(struct mlx5_wqe_data_seg);
+
+	return min_t(int, max_sge, wqe_size - sq_overhead(attr) /
+		     sizeof(struct mlx5_wqe_data_seg));
+}
+
 static int calc_sq_size(struct mlx5_ib_dev *dev, struct ib_qp_init_attr *attr,
 			struct mlx5_ib_qp *qp)
 {
@@ -382,7 +405,11 @@ static int calc_sq_size(struct mlx5_ib_dev *dev, struc
 		return -ENOMEM;
 	}
 	qp->sq.wqe_shift = ilog2(MLX5_SEND_WQE_BB);
-	qp->sq.max_gs = attr->cap.max_send_sge;
+	qp->sq.max_gs = get_send_sge(attr, wqe_size);
+	if (qp->sq.max_gs < attr->cap.max_send_sge)
+		return -ENOMEM;
+
+	attr->cap.max_send_sge = qp->sq.max_gs;
 	qp->sq.max_post = wq_size / wqe_size;
 	attr->cap.max_send_wr = qp->sq.max_post;