git: 823f158a0133 - main - bnxt_re/libbnxtre: Add support for Variable WQE on BCM576xx controllers
- Go to: [ bottom of page ] [ top of archives ] [ this month ]
Date: Tue, 27 Jan 2026 12:16:27 UTC
The branch main has been updated by ssaxena:
URL: https://cgit.FreeBSD.org/src/commit/?id=823f158a01330484f83f69c31d695035c607f8ee
commit 823f158a01330484f83f69c31d695035c607f8ee
Author: Sreekanth Reddy <sreekanth.reddy@broadcom.com>
AuthorDate: 2026-01-23 16:40:13 +0000
Commit: Sumit Saxena <ssaxena@FreeBSD.org>
CommitDate: 2026-01-27 12:13:47 +0000
bnxt_re/libbnxtre: Add support for Variable WQE on BCM576xx controllers
Variable size WQE means that each send Work Queue Entry to
HW can use different WQE sizes as opposed to the static WQE
size on the current devices. Set variable WQE mode for BCM576xx
devices. Depth of the Queue will be a multiple of slot which
is 16 bytes. The number of slots should be a multiple of 256
as per the HW requirement.
Initialize the Software shadow queue to hold requests equal to
the number of slots.
Reviewed by: ssaxena
Differential Revision: https://reviews.freebsd.org/D54525
MFC after: 3 days
---
contrib/ofed/libbnxtre/abi.h | 10 ++-
contrib/ofed/libbnxtre/main.c | 1 +
contrib/ofed/libbnxtre/memory.h | 1 +
contrib/ofed/libbnxtre/verbs.c | 31 +++++++---
sys/dev/bnxt/bnxt_re/bnxt_re-abi.h | 12 ++--
sys/dev/bnxt/bnxt_re/bnxt_re.h | 11 +++-
sys/dev/bnxt/bnxt_re/ib_verbs.c | 123 +++++++++++++++++++++++--------------
sys/dev/bnxt/bnxt_re/ib_verbs.h | 5 ++
sys/dev/bnxt/bnxt_re/main.c | 20 +++---
sys/dev/bnxt/bnxt_re/qplib_fp.c | 74 +++++++++++++++++-----
sys/dev/bnxt/bnxt_re/qplib_fp.h | 12 ++++
sys/dev/bnxt/bnxt_re/qplib_sp.c | 16 +++--
sys/dev/bnxt/bnxt_re/qplib_sp.h | 8 +++
13 files changed, 227 insertions(+), 97 deletions(-)
diff --git a/contrib/ofed/libbnxtre/abi.h b/contrib/ofed/libbnxtre/abi.h
index e3533c122688..a729c16f4db9 100644
--- a/contrib/ofed/libbnxtre/abi.h
+++ b/contrib/ofed/libbnxtre/abi.h
@@ -46,7 +46,7 @@
#define __aligned_u64 __attribute__((aligned(8))) u64
-#define BNXT_RE_ABI_VERSION 6
+#define BNXT_RE_ABI_VERSION 7
#define BNXT_RE_MAX_INLINE_SIZE 0x60
#define BNXT_RE_MAX_INLINE_SIZE_VAR_WQE 0x1E0
#define BNXT_RE_MAX_PUSH_SIZE_VAR_WQE 0xD0
@@ -267,12 +267,14 @@ enum {
BNXT_RE_COMP_MASK_UCNTX_MQP_EX_SUPPORTED = 0x8,
BNXT_RE_COMP_MASK_UCNTX_DBR_PACING_ENABLED = 0x10,
BNXT_RE_COMP_MASK_UCNTX_DBR_RECOVERY_ENABLED = 0x20,
- BNXT_RE_COMP_MASK_UCNTX_HW_RETX_ENABLED = 0x40
+ BNXT_RE_COMP_MASK_UCNTX_HW_RETX_ENABLED = 0x40,
+ BNXT_RE_COMP_MASK_UCNTX_CMASK_HAVE_MODE = 0x80
};
enum bnxt_re_req_to_drv {
BNXT_RE_COMP_MASK_REQ_UCNTX_POW2_SUPPORT = 0x01,
- BNXT_RE_COMP_MASK_REQ_UCNTX_RSVD_WQE = 0x02
+ BNXT_RE_COMP_MASK_REQ_UCNTX_RSVD_WQE = 0x02,
+ BNXT_RE_COMP_MASK_REQ_UCNTX_VAR_WQE_SUPPORT = 0x03
};
#define BNXT_RE_WQE_MODES_WQE_MODE_MASK 0x01
@@ -410,6 +412,8 @@ struct bnxt_re_qp_req {
__u64 qpsva;
__u64 qprva;
__u64 qp_handle;
+ __u64 comp_mask;
+ __u32 sq_slots;
} __attribute__((packed));
struct bnxt_re_qp_resp {
diff --git a/contrib/ofed/libbnxtre/main.c b/contrib/ofed/libbnxtre/main.c
index 8e1903688a53..ee08f891dd1e 100644
--- a/contrib/ofed/libbnxtre/main.c
+++ b/contrib/ofed/libbnxtre/main.c
@@ -297,6 +297,7 @@ int bnxt_re_init_context(struct verbs_device *vdev, struct ibv_context *ibvctx,
req.comp_mask |= BNXT_RE_COMP_MASK_REQ_UCNTX_POW2_SUPPORT;
req.comp_mask |= BNXT_RE_COMP_MASK_REQ_UCNTX_RSVD_WQE;
+ req.comp_mask |= BNXT_RE_COMP_MASK_REQ_UCNTX_VAR_WQE_SUPPORT;
ret = ibv_cmd_get_context(ibvctx, &req.cmd, sizeof(req),
&resp.resp, sizeof(resp));
diff --git a/contrib/ofed/libbnxtre/memory.h b/contrib/ofed/libbnxtre/memory.h
index fe18f30ad1a5..875d10f3769f 100644
--- a/contrib/ofed/libbnxtre/memory.h
+++ b/contrib/ofed/libbnxtre/memory.h
@@ -51,6 +51,7 @@ struct bnxt_re_qattr {
uint32_t nwr;
uint32_t sz_ring;
uint32_t sz_shad;
+ uint32_t sw_nwr;
};
/* spin lock wrapper struct */
diff --git a/contrib/ofed/libbnxtre/verbs.c b/contrib/ofed/libbnxtre/verbs.c
index 320b374abe7e..602d792a1ce0 100644
--- a/contrib/ofed/libbnxtre/verbs.c
+++ b/contrib/ofed/libbnxtre/verbs.c
@@ -1224,6 +1224,8 @@ static int bnxt_re_get_rq_slots(struct bnxt_re_dev *rdev, uint8_t qpmode,
return slots;
}
+#define BNXT_VAR_MAX_SLOT_ALIGN 256
+
static int bnxt_re_get_sq_slots(struct bnxt_re_dev *rdev,
uint8_t qpmode, uint32_t nswr,
uint32_t nsge, uint32_t ils, uint32_t *esize)
@@ -1235,17 +1237,21 @@ static int bnxt_re_get_sq_slots(struct bnxt_re_dev *rdev,
uint32_t ilsize;
uint32_t hdr_sz;
uint32_t slots;
+ uint32_t align;
hdr_sz = bnxt_re_get_sqe_hdr_sz();
stride = sizeof(struct bnxt_re_sge);
+ align = hdr_sz;
+ if (qpmode == BNXT_RE_WQE_MODE_VARIABLE)
+ align = stride;
max_wqesz = bnxt_re_calc_wqe_sz(rdev->devattr.max_sge);
- ilsize = get_aligned(ils, hdr_sz);
+ ilsize = get_aligned(ils, align);
wqe_size = bnxt_re_calc_wqe_sz(nsge);
if (ilsize) {
cal_ils = hdr_sz + ilsize;
wqe_size = MAX(cal_ils, wqe_size);
- wqe_size = get_aligned(wqe_size, hdr_sz);
+ wqe_size = get_aligned(wqe_size, align);
}
if (wqe_size > max_wqesz)
return -EINVAL;
@@ -1256,6 +1262,8 @@ static int bnxt_re_get_sq_slots(struct bnxt_re_dev *rdev,
if (esize)
*esize = wqe_size;
slots = (nswr * wqe_size) / stride;
+ if (qpmode == BNXT_RE_WQE_MODE_VARIABLE)
+ slots = get_aligned(slots, BNXT_VAR_MAX_SLOT_ALIGN);
return slots;
}
@@ -1288,11 +1296,15 @@ static int bnxt_re_get_sqmem_size(struct bnxt_re_context *cntx,
qattr->nwr = nswr;
qattr->slots = nslots;
qattr->esize = esz;
+ if (mode)
+ qattr->sw_nwr = nslots;
+ else
+ qattr->sw_nwr = nswr;
bytes = nslots * sizeof(struct bnxt_re_sge); /* ring */
bytes += npsn * bnxt_re_get_psne_size(cntx); /* psn */
qattr->sz_ring = get_aligned(bytes, cntx->rdev->pg_size);
- qattr->sz_shad = nswr * sizeof(struct bnxt_re_wrid); /* shadow */
+ qattr->sz_shad = qattr->sw_nwr * sizeof(struct bnxt_re_wrid); /* shadow */
return 0;
}
@@ -1315,6 +1327,7 @@ static int bnxt_re_get_rqmem_size(struct bnxt_re_context *cntx,
qattr->nwr = nrwr;
qattr->slots = nslots;
qattr->esize = esz;
+ qattr->sw_nwr = nrwr;
bytes = nslots * sizeof(struct bnxt_re_sge);
qattr->sz_ring = get_aligned(bytes, cntx->rdev->pg_size);
@@ -1406,8 +1419,8 @@ static int bnxt_re_alloc_init_swque(struct bnxt_re_joint_queue *jqq,
if (!jqq->swque)
return -ENOMEM;
jqq->start_idx = 0;
- jqq->last_idx = qattr->nwr - 1;
- for (indx = 0; indx < qattr->nwr; indx++)
+ jqq->last_idx = qattr->sw_nwr - 1;
+ for (indx = 0; indx < qattr->sw_nwr; indx++)
jqq->swque[indx].next_idx = indx + 1;
jqq->swque[jqq->last_idx].next_idx = 0;
jqq->last_idx = 0;
@@ -1457,13 +1470,13 @@ static int bnxt_re_alloc_queues(struct bnxt_re_qp *qp,
*/
que->pad = (que->va + que->depth * que->stride);
psn_size = bnxt_re_get_psne_size(qp->cntx);
- que->pad_stride_log2 = (uint32_t)bnxt_log2((double)psn_size);
+ que->pad_stride_log2 = (uint32_t)bnxt_log2((double)(psn_size - 1));
ret = bnxt_re_alloc_init_swque(qp->jsqq, qp->mem, &qattr[indx]);
if (ret)
goto fail;
- qp->cap.max_swr = qattr[indx].nwr;
+ qp->cap.max_swr = qattr[indx].sw_nwr;
qp->jsqq->cntx = qp->cntx;
que->dbtail = (qp->qpmode == BNXT_RE_WQE_MODE_VARIABLE) ?
&que->tail : &qp->jsqq->start_idx;
@@ -1473,7 +1486,7 @@ static int bnxt_re_alloc_queues(struct bnxt_re_qp *qp,
goto skip_msn;
que->msn = 0;
que->msn_tbl_sz = 0;
- if (qp->qpmode & BNXT_RE_WQE_MODE_VARIABLE)
+ if (qp->qpmode == BNXT_RE_WQE_MODE_VARIABLE)
que->msn_tbl_sz = roundup_pow_of_two(qattr->slots) / 2;
else
que->msn_tbl_sz = roundup_pow_of_two(qattr->nwr);
@@ -1575,6 +1588,8 @@ struct ibv_qp *bnxt_re_create_qp(struct ibv_pd *ibvpd,
req.qpsva = (uint64_t)qp->jsqq->hwque->va;
req.qprva = qp->jrqq ? (uint64_t)qp->jrqq->hwque->va : 0;
req.qp_handle = (uint64_t)qp;
+ if (qp->qpmode == BNXT_RE_WQE_MODE_VARIABLE)
+ req.sq_slots = qattr[BNXT_RE_QATTR_SQ_INDX].slots;
if (ibv_cmd_create_qp(ibvpd, &qp->ibvqp, attr, &req.cmd, sizeof(req),
&resp.resp, sizeof(resp)))
diff --git a/sys/dev/bnxt/bnxt_re/bnxt_re-abi.h b/sys/dev/bnxt/bnxt_re/bnxt_re-abi.h
index 8f48609e7f6f..c7ca19f29046 100644
--- a/sys/dev/bnxt/bnxt_re/bnxt_re-abi.h
+++ b/sys/dev/bnxt/bnxt_re/bnxt_re-abi.h
@@ -34,7 +34,7 @@
#include <asm/types.h>
#include <linux/types.h>
-#define BNXT_RE_ABI_VERSION 6
+#define BNXT_RE_ABI_VERSION 7
enum {
BNXT_RE_COMP_MASK_UCNTX_WC_DPI_ENABLED = 0x01,
@@ -43,12 +43,14 @@ enum {
BNXT_RE_COMP_MASK_UCNTX_MQP_EX_SUPPORTED = 0x08,
BNXT_RE_COMP_MASK_UCNTX_DBR_PACING_ENABLED = 0x10,
BNXT_RE_COMP_MASK_UCNTX_DBR_RECOVERY_ENABLED = 0x20,
- BNXT_RE_COMP_MASK_UCNTX_HW_RETX_ENABLED = 0x40
+ BNXT_RE_COMP_MASK_UCNTX_HW_RETX_ENABLED = 0x40,
+ BNXT_RE_COMP_MASK_UCNTX_CMASK_HAVE_MODE = 0x80,
};
enum {
BNXT_RE_COMP_MASK_REQ_UCNTX_POW2_SUPPORT = 0x01,
- BNXT_RE_COMP_MASK_REQ_UCNTX_RSVD_WQE = 0x02
+ BNXT_RE_COMP_MASK_REQ_UCNTX_RSVD_WQE = 0x02,
+ BNXT_RE_COMP_MASK_REQ_UCNTX_VAR_WQE_SUPPORT = 0x03
};
struct bnxt_re_uctx_req {
@@ -66,7 +68,7 @@ struct bnxt_re_uctx_resp {
__u32 max_cqd;
__u32 chip_id0;
__u32 chip_id1;
- __u32 modes;
+ __u32 mode;
__aligned_u64 comp_mask;
} __attribute__((packed));
@@ -134,6 +136,8 @@ struct bnxt_re_qp_req {
__u64 qpsva;
__u64 qprva;
__u64 qp_handle;
+ __u64 comp_mask;
+ __u32 sq_slots;
} __attribute__((packed));
struct bnxt_re_qp_resp {
diff --git a/sys/dev/bnxt/bnxt_re/bnxt_re.h b/sys/dev/bnxt/bnxt_re/bnxt_re.h
index c0bc11e490b2..0afc8566c020 100644
--- a/sys/dev/bnxt/bnxt_re/bnxt_re.h
+++ b/sys/dev/bnxt/bnxt_re/bnxt_re.h
@@ -717,7 +717,7 @@ void bnxt_re_remove_device(struct bnxt_re_dev *rdev, u8 removal_type,
void bnxt_re_destroy_lag(struct bnxt_re_dev **rdev);
int bnxt_re_add_device(struct bnxt_re_dev **rdev,
struct ifnet *netdev,
- u8 qp_mode, u8 op_type, u8 wqe_mode, u32 num_msix_requested,
+ u8 qp_mode, u8 op_type, u32 num_msix_requested,
struct auxiliary_device *aux_dev);
void bnxt_re_create_base_interface(bool primary);
int bnxt_re_schedule_work(struct bnxt_re_dev *rdev, unsigned long event,
@@ -1070,6 +1070,15 @@ static inline void bnxt_re_set_def_do_pacing(struct bnxt_re_dev *rdev)
rdev->qplib_res.pacing_data->do_pacing = rdev->dbr_def_do_pacing;
}
+static inline bool bnxt_re_is_var_size_supported(struct bnxt_re_dev *rdev,
+ struct bnxt_re_ucontext *uctx)
+{
+ if (uctx)
+ return uctx->cmask & BNXT_RE_UCNTX_CAP_VAR_WQE_ENABLED;
+ else
+ return rdev->chip_ctx->modes.wqe_mode;
+}
+
static inline void bnxt_re_set_pacing_dev_state(struct bnxt_re_dev *rdev)
{
rdev->qplib_res.pacing_data->dev_err_state =
diff --git a/sys/dev/bnxt/bnxt_re/ib_verbs.c b/sys/dev/bnxt/bnxt_re/ib_verbs.c
index 37b77c5af909..32899abab9ff 100644
--- a/sys/dev/bnxt/bnxt_re/ib_verbs.c
+++ b/sys/dev/bnxt/bnxt_re/ib_verbs.c
@@ -1667,15 +1667,18 @@ static int bnxt_re_setup_swqe_size(struct bnxt_re_qp *qp,
align = sizeof(struct sq_send_hdr);
ilsize = ALIGN(init_attr->cap.max_inline_data, align);
- sq->wqe_size = bnxt_re_get_swqe_size(ilsize, sq->max_sge);
- if (sq->wqe_size > _get_swqe_sz(dev_attr->max_qp_sges))
- return -EINVAL;
- /* For Cu/Wh and gen p5 backward compatibility mode
- * wqe size is fixed to 128 bytes
+ /* For gen p4 and gen p5 fixed wqe compatibility mode
+ * wqe size is fixed to 128 bytes - ie 6 SGEs
*/
- if (sq->wqe_size < _get_swqe_sz(dev_attr->max_qp_sges) &&
- qplqp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC)
- sq->wqe_size = _get_swqe_sz(dev_attr->max_qp_sges);
+ if (qplqp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) {
+ sq->wqe_size = _get_swqe_sz(BNXT_STATIC_MAX_SGE);
+ sq->max_sge = BNXT_STATIC_MAX_SGE;
+ } else {
+ sq->wqe_size = bnxt_re_get_swqe_size(ilsize, sq->max_sge);
+ if (sq->wqe_size > _get_swqe_sz(dev_attr->max_qp_sges))
+ return -EINVAL;
+ }
+
if (init_attr->cap.max_inline_data) {
qplqp->max_inline_data = sq->wqe_size -
@@ -1728,10 +1731,14 @@ static int bnxt_re_init_user_qp(struct bnxt_re_dev *rdev,
sizeof(struct sq_psn_search);
if (rdev->dev_attr && _is_host_msn_table(rdev->dev_attr->dev_cap_ext_flags2))
psn_sz = sizeof(struct sq_msn_search);
- psn_nume = (qplib_qp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) ?
- qplib_qp->sq.max_wqe :
- ((qplib_qp->sq.max_wqe * qplib_qp->sq.wqe_size) /
- sizeof(struct bnxt_qplib_sge));
+ if (cntx && bnxt_re_is_var_size_supported(rdev, cntx)) {
+ psn_nume = ureq.sq_slots;
+ } else {
+ psn_nume = (qplib_qp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) ?
+ qplib_qp->sq.max_wqe :
+ ((qplib_qp->sq.max_wqe * qplib_qp->sq.wqe_size) /
+ sizeof(struct bnxt_qplib_sge));
+ }
if (rdev->dev_attr && _is_host_msn_table(rdev->dev_attr->dev_cap_ext_flags2))
psn_nume = roundup_pow_of_two(psn_nume);
@@ -1912,6 +1919,7 @@ static struct bnxt_re_qp *bnxt_re_create_shadow_qp(struct bnxt_re_pd *pd,
/* Shadow QP SQ depth should be same as QP1 RQ depth */
qp->qplib_qp.sq.wqe_size = bnxt_re_get_swqe_size(0, 6);
qp->qplib_qp.sq.max_wqe = qp1_qp->rq.max_wqe;
+ qp->qplib_qp.sq.max_sw_wqe = qp1_qp->rq.max_wqe;
qp->qplib_qp.sq.max_sge = 2;
/* Q full delta can be 1 since it is internal QP */
qp->qplib_qp.sq.q_full_delta = 1;
@@ -1923,6 +1931,7 @@ static struct bnxt_re_qp *bnxt_re_create_shadow_qp(struct bnxt_re_pd *pd,
qp->qplib_qp.rq.wqe_size = _max_rwqe_sz(6); /* 128 Byte wqe size */
qp->qplib_qp.rq.max_wqe = qp1_qp->rq.max_wqe;
+ qp->qplib_qp.rq.max_sw_wqe = qp1_qp->rq.max_wqe;
qp->qplib_qp.rq.max_sge = qp1_qp->rq.max_sge;
qp->qplib_qp.rq.sginfo.pgsize = PAGE_SIZE;
qp->qplib_qp.rq.sginfo.pgshft = PAGE_SHIFT;
@@ -1995,6 +2004,7 @@ static int bnxt_re_init_rq_attr(struct bnxt_re_qp *qp,
entries = init_attr->cap.max_recv_wr + 1;
entries = bnxt_re_init_depth(entries, cntx);
rq->max_wqe = min_t(u32, entries, dev_attr->max_qp_wqes + 1);
+ rq->max_sw_wqe = rq->max_wqe;
rq->q_full_delta = 0;
rq->sginfo.pgsize = PAGE_SIZE;
rq->sginfo.pgshft = PAGE_SHIFT;
@@ -2019,10 +2029,11 @@ static void bnxt_re_adjust_gsi_rq_attr(struct bnxt_re_qp *qp)
static int bnxt_re_init_sq_attr(struct bnxt_re_qp *qp,
struct ib_qp_init_attr *init_attr,
- void *cntx)
+ void *cntx, struct ib_udata *udata)
{
struct bnxt_qplib_dev_attr *dev_attr;
struct bnxt_qplib_qp *qplqp;
+ struct bnxt_re_qp_req ureq;
struct bnxt_re_dev *rdev;
struct bnxt_qplib_q *sq;
int diff = 0;
@@ -2034,35 +2045,53 @@ static int bnxt_re_init_sq_attr(struct bnxt_re_qp *qp,
sq = &qplqp->sq;
dev_attr = rdev->dev_attr;
- sq->max_sge = init_attr->cap.max_send_sge;
- if (sq->max_sge > dev_attr->max_qp_sges) {
- sq->max_sge = dev_attr->max_qp_sges;
- init_attr->cap.max_send_sge = sq->max_sge;
+ if (udata) {
+ rc = ib_copy_from_udata(&ureq, udata,
+ min(udata->inlen, sizeof(ureq)));
+ if (rc)
+ return -EINVAL;
}
- rc = bnxt_re_setup_swqe_size(qp, init_attr);
- if (rc)
- return rc;
- /*
- * Change the SQ depth if user has requested minimum using
- * configfs. Only supported for kernel consumers. Setting
- * min_tx_depth to 4096 to handle iser SQ full condition
- * in most of the newer OS distros
- */
+
+ sq->max_sge = init_attr->cap.max_send_sge;
entries = init_attr->cap.max_send_wr;
- if (!cntx && rdev->min_tx_depth && init_attr->qp_type != IB_QPT_GSI) {
+ if (cntx && udata && qplqp->wqe_mode == BNXT_QPLIB_WQE_MODE_VARIABLE) {
+ sq->max_wqe = ureq.sq_slots;
+ sq->max_sw_wqe = ureq.sq_slots;
+ sq->wqe_size = sizeof(struct sq_sge);
+ } else {
+ if (sq->max_sge > dev_attr->max_qp_sges) {
+ sq->max_sge = dev_attr->max_qp_sges;
+ init_attr->cap.max_send_sge = sq->max_sge;
+ }
+ rc = bnxt_re_setup_swqe_size(qp, init_attr);
+ if (rc)
+ return rc;
/*
- * If users specify any value greater than 1 use min_tx_depth
- * provided by user for comparison. Else, compare it with the
- * BNXT_RE_MIN_KERNEL_QP_TX_DEPTH and adjust it accordingly.
+ * Change the SQ depth if user has requested minimum using
+ * configfs. Only supported for kernel consumers. Setting
+ * min_tx_depth to 4096 to handle iser SQ full condition
+ * in most of the newer OS distros
*/
- if (rdev->min_tx_depth > 1 && entries < rdev->min_tx_depth)
- entries = rdev->min_tx_depth;
- else if (entries < BNXT_RE_MIN_KERNEL_QP_TX_DEPTH)
- entries = BNXT_RE_MIN_KERNEL_QP_TX_DEPTH;
- }
- diff = bnxt_re_get_diff(cntx, rdev->chip_ctx);
- entries = bnxt_re_init_depth(entries + diff + 1, cntx);
- sq->max_wqe = min_t(u32, entries, dev_attr->max_qp_wqes + diff + 1);
+
+ if (!cntx && rdev->min_tx_depth && init_attr->qp_type != IB_QPT_GSI) {
+ /*
+ * If users specify any value greater than 1 use min_tx_depth
+ * provided by user for comparison. Else, compare it with the
+ * BNXT_RE_MIN_KERNEL_QP_TX_DEPTH and adjust it accordingly.
+ */
+ if (rdev->min_tx_depth > 1 && entries < rdev->min_tx_depth)
+ entries = rdev->min_tx_depth;
+ else if (entries < BNXT_RE_MIN_KERNEL_QP_TX_DEPTH)
+ entries = BNXT_RE_MIN_KERNEL_QP_TX_DEPTH;
+ }
+ diff = bnxt_re_get_diff(cntx, rdev->chip_ctx);
+ entries = bnxt_re_init_depth(entries + diff + 1, cntx);
+ sq->max_wqe = min_t(u32, entries, dev_attr->max_qp_wqes + diff + 1);
+ if (qplqp->wqe_mode == BNXT_QPLIB_WQE_MODE_VARIABLE)
+ sq->max_sw_wqe = bnxt_qplib_get_depth(sq, qplqp->wqe_mode, true);
+ else
+ sq->max_sw_wqe = sq->max_wqe;
+ }
sq->q_full_delta = diff + 1;
/*
* Reserving one slot for Phantom WQE. Application can
@@ -2128,11 +2157,6 @@ out:
return qptype;
}
-static int bnxt_re_init_qp_wqe_mode(struct bnxt_re_dev *rdev)
-{
- return rdev->chip_ctx->modes.wqe_mode;
-}
-
static int bnxt_re_init_qp_attr(struct bnxt_re_qp *qp, struct bnxt_re_pd *pd,
struct ib_qp_init_attr *init_attr,
struct ib_udata *udata)
@@ -2166,7 +2190,7 @@ static int bnxt_re_init_qp_attr(struct bnxt_re_qp *qp, struct bnxt_re_pd *pd,
goto out;
}
qplqp->type = (u8)qptype;
- qplqp->wqe_mode = bnxt_re_init_qp_wqe_mode(rdev);
+ qplqp->wqe_mode = bnxt_re_is_var_size_supported(rdev, cntx);
ether_addr_copy(qplqp->smac, rdev->dev_addr);
if (init_attr->qp_type == IB_QPT_RC) {
@@ -2213,7 +2237,7 @@ static int bnxt_re_init_qp_attr(struct bnxt_re_qp *qp, struct bnxt_re_pd *pd,
bnxt_re_adjust_gsi_rq_attr(qp);
/* Setup SQ */
- rc = bnxt_re_init_sq_attr(qp, init_attr, cntx);
+ rc = bnxt_re_init_sq_attr(qp, init_attr, cntx, udata);
if (rc)
goto out;
if (init_attr->qp_type == IB_QPT_GSI)
@@ -2849,6 +2873,7 @@ int bnxt_re_modify_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr,
if (entries > dev_attr->max_qp_wqes)
entries = dev_attr->max_qp_wqes;
qp->qplib_qp.rq.max_wqe = entries;
+ qp->qplib_qp.rq.max_sw_wqe = qp->qplib_qp.rq.max_wqe;
qp->qplib_qp.rq.q_full_delta = qp->qplib_qp.rq.max_wqe -
qp_attr->cap.max_recv_wr;
qp->qplib_qp.rq.max_sge = qp_attr->cap.max_recv_sge;
@@ -5349,7 +5374,7 @@ int bnxt_re_alloc_ucontext(struct ib_ucontext *uctx_in,
}
genp5 = _is_chip_gen_p5_p7(cctx);
- resp.modes = genp5 ? cctx->modes.wqe_mode : 0;
+ resp.mode = genp5 ? cctx->modes.wqe_mode : 0;
if (rdev->dev_attr && _is_host_msn_table(rdev->dev_attr->dev_cap_ext_flags2))
resp.comp_mask = BNXT_RE_COMP_MASK_UCNTX_HW_RETX_ENABLED;
@@ -5384,6 +5409,12 @@ int bnxt_re_alloc_ucontext(struct ib_ucontext *uctx_in,
if (bnxt_re_init_rsvd_wqe_flag(&ureq, &resp, genp5))
dev_warn(rdev_to_dev(rdev),
"Rsvd wqe in use! Try the updated library.\n");
+ if (ureq.comp_mask & BNXT_RE_COMP_MASK_REQ_UCNTX_VAR_WQE_SUPPORT) {
+ resp.comp_mask |= BNXT_RE_COMP_MASK_UCNTX_CMASK_HAVE_MODE;
+ resp.mode = rdev->chip_ctx->modes.wqe_mode;
+ if (resp.mode == BNXT_QPLIB_WQE_MODE_VARIABLE)
+ resp.comp_mask |= BNXT_RE_UCNTX_CAP_VAR_WQE_ENABLED;
+ }
} else {
dev_warn(rdev_to_dev(rdev),
"Enabled roundup logic. Update the library!\n");
diff --git a/sys/dev/bnxt/bnxt_re/ib_verbs.h b/sys/dev/bnxt/bnxt_re/ib_verbs.h
index f3b7f44c828d..66d15dd2b767 100644
--- a/sys/dev/bnxt/bnxt_re/ib_verbs.h
+++ b/sys/dev/bnxt/bnxt_re/ib_verbs.h
@@ -508,6 +508,11 @@ static inline int bnxt_re_init_pow2_flag(struct bnxt_re_uctx_req *req,
return 0;
}
+enum {
+ BNXT_RE_UCNTX_CAP_POW2_DISABLED = 0x1ULL,
+ BNXT_RE_UCNTX_CAP_VAR_WQE_ENABLED = 0x2ULL,
+};
+
static inline u32 bnxt_re_init_depth(u32 ent, struct bnxt_re_ucontext *uctx)
{
return uctx ? (uctx->cmask & BNXT_RE_COMP_MASK_UCNTX_POW2_DISABLED) ?
diff --git a/sys/dev/bnxt/bnxt_re/main.c b/sys/dev/bnxt/bnxt_re/main.c
index dc68854157a0..01c2710bc3ea 100644
--- a/sys/dev/bnxt/bnxt_re/main.c
+++ b/sys/dev/bnxt/bnxt_re/main.c
@@ -1101,7 +1101,6 @@ static int bnxt_re_handle_start(struct auxiliary_device *adev)
rc = bnxt_re_add_device(&rdev, real_dev,
en_info->gsi_mode,
BNXT_RE_POST_RECOVERY_INIT,
- en_info->wqe_mode,
en_info->num_msix_requested, adev);
if (rc) {
/* Add device failed. Unregister the device.
@@ -1435,15 +1434,15 @@ static void bnxt_re_set_db_offset(struct bnxt_re_dev *rdev)
return;
}
-static void bnxt_re_set_drv_mode(struct bnxt_re_dev *rdev, u8 mode)
+static void bnxt_re_set_drv_mode(struct bnxt_re_dev *rdev)
{
struct bnxt_qplib_chip_ctx *cctx;
struct bnxt_en_dev *en_dev;
en_dev = rdev->en_dev;
cctx = rdev->chip_ctx;
- cctx->modes.wqe_mode = _is_chip_gen_p5_p7(rdev->chip_ctx) ?
- mode : BNXT_QPLIB_WQE_MODE_STATIC;
+ cctx->modes.wqe_mode = _is_chip_p7(rdev->chip_ctx) ?
+ BNXT_QPLIB_WQE_MODE_VARIABLE : BNXT_QPLIB_WQE_MODE_STATIC;
cctx->modes.te_bypass = false;
if (bnxt_re_hwrm_qcaps(rdev))
dev_err(rdev_to_dev(rdev),
@@ -1492,7 +1491,7 @@ static void bnxt_re_destroy_chip_ctx(struct bnxt_re_dev *rdev)
kfree(chip_ctx);
}
-static int bnxt_re_setup_chip_ctx(struct bnxt_re_dev *rdev, u8 wqe_mode)
+static int bnxt_re_setup_chip_ctx(struct bnxt_re_dev *rdev)
{
struct bnxt_qplib_chip_ctx *chip_ctx;
struct bnxt_en_dev *en_dev;
@@ -1527,7 +1526,7 @@ static int bnxt_re_setup_chip_ctx(struct bnxt_re_dev *rdev, u8 wqe_mode)
rc = -ENOMEM;
goto fail;
}
- bnxt_re_set_drv_mode(rdev, wqe_mode);
+ bnxt_re_set_drv_mode(rdev);
bnxt_re_set_db_offset(rdev);
rc = bnxt_qplib_map_db_bar(&rdev->qplib_res);
@@ -3551,7 +3550,7 @@ static void bnxt_re_dev_uninit(struct bnxt_re_dev *rdev, u8 op_type)
}
}
-static int bnxt_re_dev_init(struct bnxt_re_dev *rdev, u8 op_type, u8 wqe_mode)
+static int bnxt_re_dev_init(struct bnxt_re_dev *rdev, u8 op_type)
{
struct bnxt_re_ring_attr rattr = {};
struct bnxt_qplib_creq_ctx *creq;
@@ -3566,7 +3565,7 @@ static int bnxt_re_dev_init(struct bnxt_re_dev *rdev, u8 op_type, u8 wqe_mode)
}
set_bit(BNXT_RE_FLAG_NETDEV_REGISTERED, &rdev->flags);
- rc = bnxt_re_setup_chip_ctx(rdev, wqe_mode);
+ rc = bnxt_re_setup_chip_ctx(rdev);
if (rc) {
dev_err(rdev_to_dev(rdev), "Failed to get chip context rc 0x%x", rc);
bnxt_re_unregister_netdev(rdev);
@@ -3879,7 +3878,7 @@ void bnxt_re_remove_device(struct bnxt_re_dev *rdev, u8 op_type,
int bnxt_re_add_device(struct bnxt_re_dev **rdev,
struct ifnet *netdev,
- u8 qp_mode, u8 op_type, u8 wqe_mode,
+ u8 qp_mode, u8 op_type,
u32 num_msix_requested,
struct auxiliary_device *aux_dev)
{
@@ -3952,7 +3951,7 @@ int bnxt_re_add_device(struct bnxt_re_dev **rdev,
rtnl_lock();
en_info->rdev = *rdev;
rtnl_unlock();
- rc = bnxt_re_dev_init(*rdev, op_type, wqe_mode);
+ rc = bnxt_re_dev_init(*rdev, op_type);
if (rc) {
ref_error:
bnxt_re_dev_unreg(*rdev);
@@ -4401,7 +4400,6 @@ static int bnxt_re_probe(struct auxiliary_device *adev,
rc = bnxt_re_add_device(&rdev, en_dev->net,
BNXT_RE_GSI_MODE_ALL,
BNXT_RE_COMPLETE_INIT,
- BNXT_QPLIB_WQE_MODE_STATIC,
BNXT_RE_MSIX_FROM_MOD_PARAM, adev);
if (rc) {
mutex_unlock(&bnxt_re_mutex);
diff --git a/sys/dev/bnxt/bnxt_re/qplib_fp.c b/sys/dev/bnxt/bnxt_re/qplib_fp.c
index d4b439b05588..19708302198b 100644
--- a/sys/dev/bnxt/bnxt_re/qplib_fp.c
+++ b/sys/dev/bnxt/bnxt_re/qplib_fp.c
@@ -842,15 +842,15 @@ static int bnxt_qplib_alloc_init_swq(struct bnxt_qplib_q *que)
int rc = 0;
int indx;
- que->swq = kcalloc(que->max_wqe, sizeof(*que->swq), GFP_KERNEL);
+ que->swq = kcalloc(que->max_sw_wqe, sizeof(*que->swq), GFP_KERNEL);
if (!que->swq) {
rc = -ENOMEM;
goto out;
}
que->swq_start = 0;
- que->swq_last = que->max_wqe - 1;
- for (indx = 0; indx < que->max_wqe; indx++)
+ que->swq_last = que->max_sw_wqe - 1;
+ for (indx = 0; indx < que->max_sw_wqe; indx++)
que->swq[indx].next_idx = indx + 1;
que->swq[que->swq_last].next_idx = 0; /* Make it circular */
que->swq_last = 0;
@@ -879,19 +879,23 @@ static u32 bnxt_qplib_get_stride(void)
return sizeof(struct sq_sge);
}
-static u32 bnxt_qplib_get_depth(struct bnxt_qplib_q *que)
+u32 bnxt_qplib_get_depth(struct bnxt_qplib_q *que, u8 wqe_mode, bool is_sq)
{
- u8 stride;
+ u32 slots;
- stride = bnxt_qplib_get_stride();
- return (que->wqe_size * que->max_wqe) / stride;
+ /* Queue depth is the number of slots. */
+ slots = (que->wqe_size * que->max_wqe) / sizeof(struct sq_sge);
+ /* For variable WQE mode, need to align the slots to 256 */
+ if (wqe_mode == BNXT_QPLIB_WQE_MODE_VARIABLE && is_sq)
+ slots = ALIGN(slots, BNXT_VAR_MAX_SLOT_ALIGN);
+ return slots;
}
static u32 _set_sq_size(struct bnxt_qplib_q *que, u8 wqe_mode)
{
/* For Variable mode supply number of 16B slots */
return (wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) ?
- que->max_wqe : bnxt_qplib_get_depth(que);
+ que->max_wqe : bnxt_qplib_get_depth(que, wqe_mode, true);
}
static u32 _set_sq_max_slot(u8 wqe_mode)
@@ -929,7 +933,7 @@ int bnxt_qplib_create_qp1(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
hwq_attr.res = res;
hwq_attr.sginfo = &sq->sginfo;
hwq_attr.stride = bnxt_qplib_get_stride();
- hwq_attr.depth = bnxt_qplib_get_depth(sq);
+ hwq_attr.depth = bnxt_qplib_get_depth(sq, qp->wqe_mode, false);
hwq_attr.type = HWQ_TYPE_QUEUE;
rc = bnxt_qplib_alloc_init_hwq(&sq->hwq, &hwq_attr);
if (rc)
@@ -953,7 +957,7 @@ int bnxt_qplib_create_qp1(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
hwq_attr.res = res;
hwq_attr.sginfo = &rq->sginfo;
hwq_attr.stride = bnxt_qplib_get_stride();
- hwq_attr.depth = bnxt_qplib_get_depth(rq);
+ hwq_attr.depth = bnxt_qplib_get_depth(rq, qp->wqe_mode, false);
hwq_attr.type = HWQ_TYPE_QUEUE;
rc = bnxt_qplib_alloc_init_hwq(&rq->hwq, &hwq_attr);
if (rc)
@@ -1105,7 +1109,7 @@ int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
hwq_attr.res = res;
hwq_attr.sginfo = &sq->sginfo;
hwq_attr.stride = bnxt_qplib_get_stride();
- hwq_attr.depth = bnxt_qplib_get_depth(sq);
+ hwq_attr.depth = bnxt_qplib_get_depth(sq, qp->wqe_mode, true);
hwq_attr.aux_stride = psn_sz;
hwq_attr.aux_depth = (psn_sz) ?
_set_sq_size(sq, qp->wqe_mode) : 0;
@@ -1147,7 +1151,7 @@ int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
hwq_attr.res = res;
hwq_attr.sginfo = &rq->sginfo;
hwq_attr.stride = bnxt_qplib_get_stride();
- hwq_attr.depth = bnxt_qplib_get_depth(rq);
+ hwq_attr.depth = bnxt_qplib_get_depth(rq, qp->wqe_mode, false);
hwq_attr.aux_stride = 0;
hwq_attr.aux_depth = 0;
hwq_attr.type = HWQ_TYPE_QUEUE;
@@ -2878,6 +2882,32 @@ out:
return rc;
}
+static int bnxt_qplib_get_cqe_sq_cons(struct bnxt_qplib_q *sq, u32 cqe_slot)
+{
+ struct bnxt_qplib_hwq *sq_hwq;
+ struct bnxt_qplib_swq *swq;
+ int cqe_sq_cons = -1;
+ u32 start, last;
+
+ sq_hwq = &sq->hwq;
+
+ start = sq->swq_start;
+ last = sq->swq_last;
+
+ while (last != start) {
+ swq = &sq->swq[last];
+ if (swq->slot_idx == cqe_slot) {
+ cqe_sq_cons = swq->next_idx;
+ dev_err(&sq_hwq->pdev->dev, "%s: Found cons wqe = %d slot = %d\n",
+ __func__, cqe_sq_cons, cqe_slot);
+ break;
+ }
+
+ last = swq->next_idx;
+ }
+ return cqe_sq_cons;
+}
+
static int bnxt_qplib_cq_process_req(struct bnxt_qplib_cq *cq,
struct cq_req *hwcqe,
struct bnxt_qplib_cqe **pcqe, int *budget,
@@ -2886,8 +2916,9 @@ static int bnxt_qplib_cq_process_req(struct bnxt_qplib_cq *cq,
struct bnxt_qplib_qp *qp;
struct bnxt_qplib_q *sq;
struct bnxt_qplib_cqe *cqe;
- u32 cqe_sq_cons;
+ u32 cqe_sq_cons, slot_num;
struct bnxt_qplib_swq *swq;
+ int cqe_cons;
int rc = 0;
qp = (struct bnxt_qplib_qp *)le64_to_cpu(hwcqe->qp_handle);
@@ -2899,13 +2930,26 @@ static int bnxt_qplib_cq_process_req(struct bnxt_qplib_cq *cq,
}
sq = &qp->sq;
- cqe_sq_cons = le16_to_cpu(hwcqe->sq_cons_idx) % sq->max_wqe;
+ cqe_sq_cons = le16_to_cpu(hwcqe->sq_cons_idx) % sq->max_sw_wqe;
if (qp->sq.flushed) {
dev_dbg(&cq->hwq.pdev->dev,
"%s: QPLIB: QP in Flush QP = %p\n", __func__, qp);
goto done;
}
+ if (__is_err_cqe_for_var_wqe(qp, hwcqe->status)) {
+ slot_num = le16_to_cpu(hwcqe->sq_cons_idx);
+ cqe_cons = bnxt_qplib_get_cqe_sq_cons(sq, slot_num);
+ if (cqe_cons < 0) {
+ dev_dbg(&cq->hwq.pdev->dev, "%s: Wrong SQ cons cqe_slot_indx = %d\n",
+ __func__, slot_num);
+ goto done;
+ }
+ cqe_sq_cons = cqe_cons;
+ dev_dbg(&cq->hwq.pdev->dev, "%s: cqe_sq_cons = %d swq_last = %d swq_start = %d\n",
+ __func__, cqe_sq_cons, sq->swq_last, sq->swq_start);
+ }
+
/* Require to walk the sq's swq to fabricate CQEs for all previously
* signaled SWQEs due to CQE aggregation from the current sq cons
* to the cqe_sq_cons
@@ -3348,7 +3392,7 @@ static int bnxt_qplib_cq_process_terminal(struct bnxt_qplib_cq *cq,
if (cqe_cons == 0xFFFF)
goto do_rq;
- cqe_cons %= sq->max_wqe;
+ cqe_cons %= sq->max_sw_wqe;
if (qp->sq.flushed) {
dev_dbg(&cq->hwq.pdev->dev,
"%s: QPLIB: QP in Flush QP = %p\n", __func__, qp);
diff --git a/sys/dev/bnxt/bnxt_re/qplib_fp.h b/sys/dev/bnxt/bnxt_re/qplib_fp.h
index fd2d8a08c818..542a26782c62 100644
--- a/sys/dev/bnxt/bnxt_re/qplib_fp.h
+++ b/sys/dev/bnxt/bnxt_re/qplib_fp.h
@@ -300,6 +300,7 @@ struct bnxt_qplib_q {
struct bnxt_qplib_sg_info sginfo;
struct bnxt_qplib_hwq hwq;
u32 max_wqe;
+ u32 max_sw_wqe;
u16 max_sge;
u16 wqe_size;
u16 q_full_delta;
@@ -635,5 +636,16 @@ static inline uint64_t bnxt_re_update_msn_tbl(uint32_t st_idx, uint32_t npsn, ui
SQ_MSN_SEARCH_START_PSN_MASK));
}
+static inline bool __is_var_wqe(struct bnxt_qplib_qp *qp)
+{
+ return (qp->wqe_mode == BNXT_QPLIB_WQE_MODE_VARIABLE);
+}
+
+static inline bool __is_err_cqe_for_var_wqe(struct bnxt_qplib_qp *qp, u8 status)
+{
+ return (status != CQ_REQ_STATUS_OK) && __is_var_wqe(qp);
+}
+
void bnxt_re_schedule_dbq_event(struct bnxt_qplib_res *res);
+u32 bnxt_qplib_get_depth(struct bnxt_qplib_q *que, u8 wqe_mode, bool is_sq);
#endif
diff --git a/sys/dev/bnxt/bnxt_re/qplib_sp.c b/sys/dev/bnxt/bnxt_re/qplib_sp.c
index 8c382c7a645a..f876573ce69f 100644
--- a/sys/dev/bnxt/bnxt_re/qplib_sp.c
+++ b/sys/dev/bnxt/bnxt_re/qplib_sp.c
@@ -129,11 +129,6 @@ int bnxt_qplib_get_dev_attr(struct bnxt_qplib_rcfw *rcfw)
* one extra entry while creating the qp
*/
attr->max_qp_wqes = le16_to_cpu(sb->max_qp_wr) - 1;
- /* Adjust for max_qp_wqes for variable wqe */
- if (cctx->modes.wqe_mode == BNXT_QPLIB_WQE_MODE_VARIABLE) {
- attr->max_qp_wqes = (BNXT_MAX_SQ_SIZE) /
- (BNXT_MAX_VAR_WQE_SIZE / BNXT_SGE_SIZE) - 1;
- }
if (!_is_chip_gen_p5_p7(cctx)) {
/*
* 128 WQEs needs to be reserved for the HW (8916). Prevent
@@ -141,10 +136,13 @@ int bnxt_qplib_get_dev_attr(struct bnxt_qplib_rcfw *rcfw)
*/
attr->max_qp_wqes -= BNXT_QPLIB_RESERVED_QP_WRS;
}
- attr->max_qp_sges = sb->max_sge;
- if (_is_chip_gen_p5_p7(cctx) &&
- cctx->modes.wqe_mode == BNXT_QPLIB_WQE_MODE_VARIABLE)
- attr->max_qp_sges = sb->max_sge_var_wqe;
+
+ /* Adjust for max_qp_wqes for variable wqe */
+ if (cctx->modes.wqe_mode == BNXT_QPLIB_WQE_MODE_VARIABLE)
+ attr->max_qp_wqes = BNXT_VAR_MAX_WQE - 1;
+
+ attr->max_qp_sges = cctx->modes.wqe_mode == BNXT_QPLIB_WQE_MODE_VARIABLE ?
+ min_t(u32, sb->max_sge_var_wqe, BNXT_VAR_MAX_SGE) : sb->max_sge;
attr->max_cq = bnxt_re_cap_fw_res(le32_to_cpu(sb->max_cq),
dev_res.max_cq, sw_max_en);
diff --git a/sys/dev/bnxt/bnxt_re/qplib_sp.h b/sys/dev/bnxt/bnxt_re/qplib_sp.h
index fa7dd890501c..5a5485dc5250 100644
--- a/sys/dev/bnxt/bnxt_re/qplib_sp.h
+++ b/sys/dev/bnxt/bnxt_re/qplib_sp.h
@@ -32,6 +32,7 @@
#define __BNXT_QPLIB_SP_H__
#include <rdma/ib_verbs.h>
+#include "bnxt_re-abi.h"
#define BNXT_QPLIB_RESERVED_QP_WRS 128
@@ -395,6 +396,13 @@ bool ib_modify_qp_is_ok_compat(enum ib_qp_state cur_state, enum ib_qp_state next
#define BNXT_MAX_VAR_WQE_SIZE 512
#define BNXT_SGE_SIZE 16
+#define BNXT_VAR_MAX_WQE 4352
+#define BNXT_VAR_MAX_SLOT_ALIGN 256
+#define BNXT_VAR_MAX_SGE 13
+#define BNXT_RE_MAX_RQ_WQES 65536
+
+#define BNXT_STATIC_MAX_SGE 6
+
/* PF defines */
#define BNXT_RE_MAX_QP_SUPPORTED(chip_gen) \
chip_gen == BNXT_RE_DEFAULT ? (64 * 1024) : 0