git: 913e454f3061 - main - bnxt_en/re: Use FW defined resource limit for RoCE
- Go to: [ bottom of page ] [ top of archives ] [ this month ]
Date: Tue, 27 Jan 2026 12:16:24 UTC
The branch main has been updated by ssaxena:
URL: https://cgit.FreeBSD.org/src/commit/?id=913e454f306124c83797bfa71681dcdf806532c6
commit 913e454f306124c83797bfa71681dcdf806532c6
Author: Sreekanth Reddy <sreekanth.reddy@broadcom.com>
AuthorDate: 2026-01-23 16:32:51 +0000
Commit: Sumit Saxena <ssaxena@FreeBSD.org>
CommitDate: 2026-01-27 12:11:41 +0000
bnxt_en/re: Use FW defined resource limit for RoCE
Check FW flags for status of Resource Limits. If bit
FUNC_QCAPS_RESP_FLAGS_EXT2_SW_MAX_RESOURCE_LIMITS_SUPPORTED
is set, that means FW set the resource limit for L2 and RoCE.
We'll then do the following:
L2 driver would allocate context memory based on what FW reported.
RoCE driver uses FW reported values without capping. These values
are the total FW reported value minus L2 and other components shares.
For example:
FW reported max_qps = 137217 in L2 query, this includes:
5120 for L2
1025 for QP1
131072 for RoCE
L2 will allocate backing store memory for the total.
In RoCE query, we'll get max_qp = 131072 and use it
without further capping.
Reviewed by: ssaxena
Differential Revision: https://reviews.freebsd.org/D54522
MFC after: 3 days
---
sys/dev/bnxt/bnxt_en/bnxt.h | 4 ++++
sys/dev/bnxt/bnxt_en/bnxt_hwrm.c | 3 +++
sys/dev/bnxt/bnxt_en/bnxt_ulp.c | 2 ++
sys/dev/bnxt/bnxt_en/bnxt_ulp.h | 2 ++
sys/dev/bnxt/bnxt_en/if_bnxt.c | 9 +++++++--
sys/dev/bnxt/bnxt_re/qplib_res.h | 7 +++++++
sys/dev/bnxt/bnxt_re/qplib_sp.c | 31 +++++++++++++++++--------------
7 files changed, 42 insertions(+), 16 deletions(-)
diff --git a/sys/dev/bnxt/bnxt_en/bnxt.h b/sys/dev/bnxt/bnxt_en/bnxt.h
index 78e98eb23beb..dc2ea1fdc09e 100644
--- a/sys/dev/bnxt/bnxt_en/bnxt.h
+++ b/sys/dev/bnxt/bnxt_en/bnxt.h
@@ -1250,6 +1250,10 @@ struct bnxt_softc {
#define BNXT_FW_CAP_CFA_NTUPLE_RX_EXT_IP_PROTO BIT_ULL(47)
#define BNXT_FW_CAP_ENABLE_RDMA_SRIOV BIT_ULL(48)
#define BNXT_FW_CAP_RSS_TCAM BIT_ULL(49)
+
+ #define BNXT_FW_CAP_SW_MAX_RESOURCE_LIMITS BIT_ULL(61)
+ #define BNXT_SW_RES_LMT(bp) ((bp)->fw_cap & BNXT_FW_CAP_SW_MAX_RESOURCE_LIMITS)
+
uint32_t lpi_tmr_lo;
uint32_t lpi_tmr_hi;
/* copied from flags and flags2 in hwrm_port_phy_qcaps_output */
diff --git a/sys/dev/bnxt/bnxt_en/bnxt_hwrm.c b/sys/dev/bnxt/bnxt_en/bnxt_hwrm.c
index 3e5dc0bf4b6d..e540cc74578d 100644
--- a/sys/dev/bnxt/bnxt_en/bnxt_hwrm.c
+++ b/sys/dev/bnxt/bnxt_en/bnxt_hwrm.c
@@ -1218,6 +1218,9 @@ bnxt_hwrm_func_qcaps(struct bnxt_softc *softc)
flags_ext & HWRM_FUNC_QCAPS_OUTPUT_FLAGS_EXT_DBR_PACING_SUPPORTED)
softc->fw_cap |= BNXT_FW_CAP_DBR_PACING_SUPPORTED;
+ if (flags_ext2 & HWRM_FUNC_QCAPS_OUTPUT_FLAGS_EXT2_SW_MAX_RESOURCE_LIMITS_SUPPORTED)
+ softc->fw_cap |= BNXT_FW_CAP_SW_MAX_RESOURCE_LIMITS;
+
if (flags_ext2 & HWRM_FUNC_QCAPS_OUTPUT_FLAGS_EXT2_GENERIC_STATS_SUPPORTED)
softc->fw_cap |= BNXT_FW_CAP_GENERIC_STATS;
func->fw_fid = le16toh(resp->fid);
diff --git a/sys/dev/bnxt/bnxt_en/bnxt_ulp.c b/sys/dev/bnxt/bnxt_en/bnxt_ulp.c
index 9950858f0eef..677c9c99b74e 100644
--- a/sys/dev/bnxt/bnxt_en/bnxt_ulp.c
+++ b/sys/dev/bnxt/bnxt_en/bnxt_ulp.c
@@ -458,6 +458,8 @@ static inline void bnxt_set_edev_info(struct bnxt_en_dev *edev, struct bnxt_soft
edev->flags |= BNXT_EN_FLAG_ROCEV2_CAP;
if (bp->is_asym_q)
edev->flags |= BNXT_EN_FLAG_ASYM_Q;
+ if (BNXT_SW_RES_LMT(bp))
+ edev->flags |= BNXT_EN_FLAG_SW_RES_LMT;
edev->hwrm_bar = bp->hwrm_bar;
edev->port_partition_type = bp->port_partition_type;
edev->ulp_version = BNXT_ULP_VERSION;
diff --git a/sys/dev/bnxt/bnxt_en/bnxt_ulp.h b/sys/dev/bnxt/bnxt_en/bnxt_ulp.h
index bbf16304456a..04c3f558fd20 100644
--- a/sys/dev/bnxt/bnxt_en/bnxt_ulp.h
+++ b/sys/dev/bnxt/bnxt_en/bnxt_ulp.h
@@ -90,8 +90,10 @@ struct bnxt_en_dev {
#define BNXT_EN_FLAG_ULP_STOPPED 0x8
#define BNXT_EN_FLAG_ASYM_Q 0x10
#define BNXT_EN_FLAG_MULTI_HOST 0x20
+ #define BNXT_EN_FLAG_SW_RES_LMT 0x400
#define BNXT_EN_ASYM_Q(edev) ((edev)->flags & BNXT_EN_FLAG_ASYM_Q)
#define BNXT_EN_MH(edev) ((edev)->flags & BNXT_EN_FLAG_MULTI_HOST)
+#define BNXT_EN_SW_RES_LMT(edev) ((edev)->flags & BNXT_EN_FLAG_SW_RES_LMT)
const struct bnxt_en_ops *en_ops;
struct bnxt_ulp ulp_tbl[BNXT_MAX_ULP];
int l2_db_offset; /* Doorbell BAR offset
diff --git a/sys/dev/bnxt/bnxt_en/if_bnxt.c b/sys/dev/bnxt/bnxt_en/if_bnxt.c
index a34bd1a7e511..af6a4c7b01c1 100644
--- a/sys/dev/bnxt/bnxt_en/if_bnxt.c
+++ b/sys/dev/bnxt/bnxt_en/if_bnxt.c
@@ -1198,8 +1198,13 @@ static int bnxt_alloc_ctx_mem(struct bnxt_softc *softc)
max_srqs = ctxm->max_entries;
if (softc->flags & BNXT_FLAG_ROCE_CAP) {
pg_lvl = 2;
- extra_qps = min_t(u32, 65536, max_qps - l2_qps - qp1_qps);
- extra_srqs = min_t(u32, 8192, max_srqs - srqs);
+ if (BNXT_SW_RES_LMT(softc)) {
+ extra_qps = max_qps - l2_qps - qp1_qps;
+ extra_srqs = max_srqs - srqs;
+ } else {
+ extra_qps = min_t(uint32_t, 65536, max_qps - l2_qps - qp1_qps);
+ extra_srqs = min_t(uint32_t, 8192, max_srqs - srqs);
+ }
}
ctxm = &ctx->ctx_arr[BNXT_CTX_QP];
diff --git a/sys/dev/bnxt/bnxt_re/qplib_res.h b/sys/dev/bnxt/bnxt_re/qplib_res.h
index 914694ee3b8c..59a8a43ecef6 100644
--- a/sys/dev/bnxt/bnxt_re/qplib_res.h
+++ b/sys/dev/bnxt/bnxt_re/qplib_res.h
@@ -843,4 +843,11 @@ static inline void bnxt_qplib_max_res_supported(struct bnxt_qplib_chip_ctx *cctx
break;
}
}
+
+static inline u32 bnxt_re_cap_fw_res(u32 fw_val, u32 drv_cap, bool sw_max_en)
+{
+ if (sw_max_en)
+ return fw_val;
+ return min_t(u32, fw_val, drv_cap);
+}
#endif
diff --git a/sys/dev/bnxt/bnxt_re/qplib_sp.c b/sys/dev/bnxt/bnxt_re/qplib_sp.c
index 31a0878e421b..8c382c7a645a 100644
--- a/sys/dev/bnxt/bnxt_re/qplib_sp.c
+++ b/sys/dev/bnxt/bnxt_re/qplib_sp.c
@@ -40,6 +40,7 @@
#include "qplib_res.h"
#include "qplib_rcfw.h"
#include "qplib_sp.h"
+#include "bnxt_ulp.h"
const struct bnxt_qplib_gid bnxt_qplib_gid_zero = {{ 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0 }};
@@ -79,6 +80,7 @@ static void bnxt_qplib_query_version(struct bnxt_qplib_rcfw *rcfw, char *fw_ver)
int bnxt_qplib_get_dev_attr(struct bnxt_qplib_rcfw *rcfw)
{
+ struct bnxt_qplib_max_res dev_res = {};
struct creq_query_func_resp resp = {};
struct bnxt_qplib_cmdqmsg msg = {};
struct creq_query_func_resp_sb *sb;
@@ -86,10 +88,10 @@ int bnxt_qplib_get_dev_attr(struct bnxt_qplib_rcfw *rcfw)
struct bnxt_qplib_dev_attr *attr;
struct bnxt_qplib_chip_ctx *cctx;
struct cmdq_query_func req = {};
+ bool sw_max_en;
u8 *tqm_alloc;
int i, rc = 0;
u32 temp;
- u8 chip_gen = BNXT_RE_DEFAULT;
cctx = rcfw->res->cctx;
attr = rcfw->res->dattr;
@@ -110,10 +112,11 @@ int bnxt_qplib_get_dev_attr(struct bnxt_qplib_rcfw *rcfw)
rc = bnxt_qplib_rcfw_send_message(rcfw, &msg);
if (rc)
goto bail;
+ bnxt_qplib_max_res_supported(cctx, rcfw->res, &dev_res, false);
+ sw_max_en = BNXT_EN_SW_RES_LMT(rcfw->res->en_dev);
/* Extract the context from the side buffer */
- chip_gen = _get_chip_gen_p5_type(cctx);
- attr->max_qp = le32_to_cpu(sb->max_qp);
- attr->max_qp = min_t(u32, attr->max_qp, BNXT_RE_MAX_QP_SUPPORTED(chip_gen));
+ attr->max_qp = bnxt_re_cap_fw_res(le32_to_cpu(sb->max_qp),
+ dev_res.max_qp, sw_max_en);
/* max_qp value reported by FW does not include the QP1 */
attr->max_qp += 1;
attr->max_qp_rd_atom =
@@ -142,29 +145,29 @@ int bnxt_qplib_get_dev_attr(struct bnxt_qplib_rcfw *rcfw)
if (_is_chip_gen_p5_p7(cctx) &&
cctx->modes.wqe_mode == BNXT_QPLIB_WQE_MODE_VARIABLE)
attr->max_qp_sges = sb->max_sge_var_wqe;
- attr->max_cq = le32_to_cpu(sb->max_cq);
- attr->max_cq = min_t(u32, attr->max_cq, BNXT_RE_MAX_CQ_SUPPORTED(chip_gen));
+ attr->max_cq = bnxt_re_cap_fw_res(le32_to_cpu(sb->max_cq),
+ dev_res.max_cq, sw_max_en);
attr->max_cq_wqes = le32_to_cpu(sb->max_cqe);
attr->max_cq_wqes = min_t(u32, BNXT_QPLIB_MAX_CQ_WQES, attr->max_cq_wqes);
attr->max_cq_sges = attr->max_qp_sges;
- attr->max_mr = le32_to_cpu(sb->max_mr);
- attr->max_mr = min_t(u32, attr->max_mr, BNXT_RE_MAX_MRW_SUPPORTED(chip_gen));
- attr->max_mw = le32_to_cpu(sb->max_mw);
- attr->max_mw = min_t(u32, attr->max_mw, BNXT_RE_MAX_MRW_SUPPORTED(chip_gen));
+ attr->max_mr = bnxt_re_cap_fw_res(le32_to_cpu(sb->max_mr),
+ dev_res.max_mr, sw_max_en);
+ attr->max_mw = bnxt_re_cap_fw_res(le32_to_cpu(sb->max_mw),
+ dev_res.max_mr, sw_max_en);
attr->max_mr_size = le64_to_cpu(sb->max_mr_size);
attr->max_pd = BNXT_QPLIB_MAX_PD;
attr->max_raw_ethy_qp = le32_to_cpu(sb->max_raw_eth_qp);
- attr->max_ah = le32_to_cpu(sb->max_ah);
- attr->max_ah = min_t(u32, attr->max_ah, BNXT_RE_MAX_AH_SUPPORTED(chip_gen));
+ attr->max_ah = bnxt_re_cap_fw_res(le32_to_cpu(sb->max_ah),
+ dev_res.max_ah, sw_max_en);
attr->max_fmr = le32_to_cpu(sb->max_fmr);
attr->max_map_per_fmr = sb->max_map_per_fmr;
- attr->max_srq = le16_to_cpu(sb->max_srq);
- attr->max_srq = min_t(u32, attr->max_srq, BNXT_RE_MAX_SRQ_SUPPORTED(chip_gen));
+ attr->max_srq = bnxt_re_cap_fw_res(le16_to_cpu(sb->max_srq),
+ dev_res.max_srq, sw_max_en);
attr->max_srq_wqes = le32_to_cpu(sb->max_srq_wr) - 1;
attr->max_srq_sges = sb->max_srq_sge;
attr->max_pkey = 1;