git: 088a42c0ff76 - main - if_bnxt: add MPC (Mid-Path Channel) ring infrastructure
- Go to: [ bottom of page ] [ top of archives ] [ this month ]
Date: Fri, 09 Oct 2026 10:12:38 UTC
The branch main has been updated by ssaxena:
URL: https://cgit.FreeBSD.org/src/commit/?id=088a42c0ff7620463a870bc789f940fdf112802f
commit 088a42c0ff7620463a870bc789f940fdf112802f
Author: Sumit Saxena <sumit.saxena@broadcom.com>
AuthorDate: 2026-07-29 09:12:26 +0000
Commit: Sumit Saxena <ssaxena@FreeBSD.org>
CommitDate: 2026-10-09 08:24:43 +0000
if_bnxt: add MPC (Mid-Path Channel) ring infrastructure
Add MPC ring support: a pair of extra, driver-private
TX/completion/notification rings the firmware uses as a side-channel
for offload commands. The only current consumer is kTLS TX crypto
command submission; RCE/CFA channel types are not yet implemented by
any consumer.
Adds MPC ring allocation/teardown, HWRM ring alloc/free, a private
interrupt path for the MPC NQ ring, and the TX submit/completion
path. The RoCE-only IRQ table builder is split into a generic,
exported bnxt_populate_irq(softc, irq_count) that both MPC and RoCE
use, plus an incremental bnxt_populate_irq_roce() that grows the
table on top of whatever is already there, so IRQ rids stay
consistent regardless of attach order.
This patch does not link standalone: bnxt_mpc_cmp() calls into
kTLS's completion handler, added by a later patch.
Reviewed by: gallatin
MFC after: 2 weeks
Sponsored by: Broadcom Inc.
Differential Revision: https://reviews.freebsd.org/D58603
---
sys/conf/files | 1 +
sys/dev/bnxt/bnxt_en/bnxt.h | 28 +
sys/dev/bnxt/bnxt_en/bnxt_hwrm.c | 54 +-
sys/dev/bnxt/bnxt_en/bnxt_mpc.c | 1075 +++++++++++++++++++++++++++++++++++++
sys/dev/bnxt/bnxt_en/bnxt_mpc.h | 182 +++++++
sys/dev/bnxt/bnxt_en/bnxt_ulp.c | 69 ++-
sys/dev/bnxt/bnxt_en/if_bnxt.c | 54 +-
sys/modules/bnxt/bnxt_en/Makefile | 1 +
8 files changed, 1450 insertions(+), 14 deletions(-)
diff --git a/sys/conf/files b/sys/conf/files
index 73a5d5fa8408..0a6e5874896f 100644
--- a/sys/conf/files
+++ b/sys/conf/files
@@ -1333,6 +1333,7 @@ dev/bnxt/bnxt_en/bnxt_log.c optional bnxt iflib pci compile-with "${BNXT_C}"
dev/bnxt/bnxt_en/bnxt_log_data.c optional bnxt iflib pci compile-with "${BNXT_C}"
dev/bnxt/bnxt_en/bnxt_coredump.c optional bnxt iflib pci compile-with "${BNXT_C}"
dev/bnxt/bnxt_en/bnxt_ptp.c optional bnxt iflib pci compile-with "${BNXT_C}"
+dev/bnxt/bnxt_en/bnxt_mpc.c optional bnxt iflib pci compile-with "${BNXT_C}"
dev/bnxt/bnxt_en/if_bnxt.c optional bnxt iflib pci compile-with "${BNXT_C}"
dev/bwi/bwimac.c optional bwi
dev/bwi/bwiphy.c optional bwi
diff --git a/sys/dev/bnxt/bnxt_en/bnxt.h b/sys/dev/bnxt/bnxt_en/bnxt.h
index 80cc8905d66b..1de100915f97 100644
--- a/sys/dev/bnxt/bnxt_en/bnxt.h
+++ b/sys/dev/bnxt/bnxt_en/bnxt.h
@@ -608,6 +608,11 @@ struct tx_bd_opaque {
#define TX_RING_MASK(txr) ((txr)->ring_size - 1)
+struct bnxt_sw_mpc_tx_bd {
+ uint8_t inline_bds;
+ unsigned long handle;
+};
+
struct bnxt_ring {
uint64_t paddr;
vm_offset_t doorbell;
@@ -618,6 +623,15 @@ struct bnxt_ring {
uint16_t phys_id;
uint16_t idx;
uint16_t running_bds; /* host TX-completion-coalescing accumulator */
+ uint8_t queue_id; /* CoS queue, or BNXT_MPC_QUEUE_ID below */
+#define BNXT_MPC_QUEUE_ID 0xff
+ uint8_t mpc_chnl_type; /* valid when queue_id == BNXT_MPC_QUEUE_ID */
+ struct iflib_dma_info ring_mem; /* manual DMA alloc for driver-private rings (MPC) */
+ struct bnxt_cp_ring *cp_ring; /* completion ring this TX ring reports to (MPC) */
+ uint16_t prod; /* MPC TX ring producer index */
+ uint16_t cons; /* MPC TX ring consumer index */
+ struct mtx tx_lock; /* MPC TX ring lock */
+ struct bnxt_sw_mpc_tx_bd *tx_mpc_buf_ring;
struct bnxt_full_tpa_start *tpa_start;
union {
u64 db_key64;
@@ -649,6 +663,10 @@ struct bnxt_cp_ring {
uint8_t type;
#define Q_TYPE_TX 1
#define Q_TYPE_RX 2
+#define TX_CP_NQ 3 /* MPC-private NQ ring (bnxt_mpc.c) */
+ int msix_vec; /* MPC-private ring: bus_setup_intr vector */
+ struct resource *irq_res; /* MPC-private ring: bus_setup_intr resource */
+ void *irq_cookie; /* MPC-private ring: bus_setup_intr cookie */
};
struct bnxt_full_tpa_start {
@@ -718,6 +736,10 @@ struct bnxt_func_qcfg {
uint16_t alloc_completion_rings;
uint16_t alloc_tx_rings;
uint16_t alloc_rx_rings;
+ /* Firmware-granted counts before the MPC reservation is subtracted;
+ * bnxt_set_dflt_mpc_rings() needs these unreduced values. */
+ uint16_t orig_alloc_completion_rings;
+ uint16_t orig_alloc_tx_rings;
uint16_t alloc_vnics;
uint16_t alloc_rss_ctx;
uint16_t alloc_l2_ctx;
@@ -1435,6 +1457,8 @@ struct bnxt_softc {
struct bnxt_ptp_cfg *ptp_cfg;
bool rx_ts_enabled;
+ struct bnxt_mpc_info *mpc_info;
+
#define BNXT_CAGR_CQCOAL_OFFSET 0xc00
#define BNXT_CAGR_NQAGG_MAXTIMER 0x5930000
#define BNXT_CAGR_TICK_RES_DEFAULT 0x2
@@ -1523,6 +1547,10 @@ int bnxt_alloc_ctx_pg_tbls(struct bnxt_softc *softc,
uint32_t mem_size, uint8_t depth,
struct bnxt_ctx_mem_type *ctxm);
int bnxt_hwrm_ptp_qcfg(struct bnxt_softc *bp);
+int bnxt_populate_irq(struct bnxt_softc *softc, int irq_count);
+void bnxt_mark_cpr_invalid(struct bnxt_cp_ring *cpr);
+void bnxt_set_db_mask(struct bnxt_softc *bp, struct bnxt_ring *db,
+ uint32_t ring_type);
static inline u32
readl_fbsd(struct bnxt_softc *bp, u32 reg_off, u8 bar_idx)
diff --git a/sys/dev/bnxt/bnxt_en/bnxt_hwrm.c b/sys/dev/bnxt/bnxt_en/bnxt_hwrm.c
index 2649100e9b86..fb549bc05c10 100644
--- a/sys/dev/bnxt/bnxt_en/bnxt_hwrm.c
+++ b/sys/dev/bnxt/bnxt_en/bnxt_hwrm.c
@@ -32,6 +32,7 @@
#include "bnxt.h"
#include "bnxt_hwrm.h"
+#include "bnxt_mpc.h"
#include "hsi_struct_def.h"
#include "bnxt_coredump.h"
@@ -1205,6 +1206,7 @@ bnxt_hwrm_func_qcaps(struct bnxt_softc *softc)
(void *)softc->hwrm_cmd_resp.idi_vaddr;
struct bnxt_func_info *func = &softc->func;
uint32_t flags, flags_ext, flags_ext2;
+ uint8_t mpc_chnls_cap = 0;
bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_FUNC_QCAPS);
req.fid = htole16(0xffff);
@@ -1354,8 +1356,18 @@ bnxt_hwrm_func_qcaps(struct bnxt_softc *softc)
}
}
+ /*
+ * Only extract the field needed here; the actual alloc/free work
+ * (malloc(M_NOWAIT)/free() in bnxt_alloc_mpc_info()) happens after
+ * BNXT_HWRM_UNLOCK() below instead of under hwrm_lock.
+ */
+ if (BNXT_PF(softc))
+ mpc_chnls_cap = resp->mpc_chnls_cap;
+
fail:
BNXT_HWRM_UNLOCK(softc);
+ if (!rc && BNXT_PF(softc))
+ bnxt_alloc_mpc_info(softc, mpc_chnls_cap);
return rc;
}
@@ -1468,6 +1480,22 @@ bnxt_hwrm_func_qcfg(struct bnxt_softc *softc)
fn_qcfg->alloc_stat_ctx = le16toh(resp->alloc_stat_ctx);
fn_qcfg->alloc_msix = le16toh(resp->alloc_msix);
+ fn_qcfg->orig_alloc_completion_rings = fn_qcfg->alloc_completion_rings;
+ fn_qcfg->orig_alloc_tx_rings = fn_qcfg->alloc_tx_rings;
+
+ /* Reserve minimum rings for MPC */
+ if (BNXT_PF(softc)) {
+ if (softc->mpc_info) {
+ uint16_t mpc_min = BNXT_MIN_MPC_TCE + BNXT_MIN_MPC_RCE;
+
+ /* Clamp to 0 instead of underflowing these unsigned counts. */
+ fn_qcfg->alloc_completion_rings -=
+ min(fn_qcfg->alloc_completion_rings, mpc_min);
+ fn_qcfg->alloc_tx_rings -=
+ min(fn_qcfg->alloc_tx_rings, mpc_min);
+ }
+ }
+
switch (resp->port_partition_type) {
case HWRM_FUNC_QCFG_OUTPUT_PORT_PARTITION_TYPE_NPAR1_0:
case HWRM_FUNC_QCFG_OUTPUT_PORT_PARTITION_TYPE_NPAR1_2:
@@ -1976,17 +2004,24 @@ bnxt_hwrm_ring_alloc(struct bnxt_softc *softc, uint8_t type,
switch (type) {
case HWRM_RING_ALLOC_INPUT_RING_TYPE_TX:
- cp_ring = &softc->tx_cp_rings[idx];
+ if (ring->queue_id == BNXT_MPC_QUEUE_ID) {
+ cp_ring = ring->cp_ring;
+ req.mpc_chnls_type = ring->mpc_chnl_type;
+ req.enables |= htole32(HWRM_RING_ALLOC_INPUT_ENABLES_MPC_CHNLS_TYPE);
+ } else {
+ cp_ring = &softc->tx_cp_rings[idx];
- req.cmpl_ring_id = htole16(cp_ring->ring.phys_id);
- /* queue_id - what CoS queue the TX ring is associated with */
- req.queue_id = htole16(softc->tx_q_info[0].queue_id);
+ /* queue_id - what CoS queue the TX ring is associated with */
+ req.queue_id = htole16(softc->tx_q_info[0].queue_id);
+ req.enables |= htole32(
+ HWRM_RING_ALLOC_INPUT_ENABLES_STAT_CTX_ID_VALID);
+ }
req.stat_ctx_id = htole32(cp_ring->stats_ctx_id);
- req.enables |= htole32(
- HWRM_RING_ALLOC_INPUT_ENABLES_STAT_CTX_ID_VALID);
- if (softc->fw_cap & BNXT_FLAG_TX_COAL_CMPL)
+ if (softc->fw_cap & BNXT_FLAG_TX_COAL_CMPL &&
+ ring->queue_id != BNXT_MPC_QUEUE_ID)
req.cmpl_coal_cnt = softc->tx_hw_coal_cnt;
+ req.cmpl_ring_id = htole16(cp_ring->ring.phys_id);
break;
case HWRM_RING_ALLOC_INPUT_RING_TYPE_RX:
if (!BNXT_CHIP_P5_PLUS(softc))
@@ -2023,7 +2058,10 @@ bnxt_hwrm_ring_alloc(struct bnxt_softc *softc, uint8_t type,
}
req.cq_handle = htole64(ring->id);
- req.nq_ring_id = htole16(softc->nq_rings[idx].ring.phys_id);
+ if (ring->queue_id == BNXT_MPC_QUEUE_ID)
+ req.nq_ring_id = htole16(softc->mpc_info->mpc_nq_rings[idx].ring.phys_id);
+ else
+ req.nq_ring_id = htole16(softc->nq_rings[idx].ring.phys_id);
req.enables |= htole32(
HWRM_RING_ALLOC_INPUT_ENABLES_NQ_RING_ID_VALID);
break;
diff --git a/sys/dev/bnxt/bnxt_en/bnxt_mpc.c b/sys/dev/bnxt/bnxt_en/bnxt_mpc.c
new file mode 100644
index 000000000000..4e35b3bdca9c
--- /dev/null
+++ b/sys/dev/bnxt/bnxt_en/bnxt_mpc.c
@@ -0,0 +1,1075 @@
+/*
+ * Broadcom NetXtreme-C/E network driver.
+ *
+ * Copyright (c) 2026 Broadcom, All Rights Reserved.
+ * The term Broadcom refers to Broadcom Limited and/or its subsidiaries
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <sys/param.h>
+#include <sys/endian.h>
+#include <sys/bus.h>
+#include <sys/rman.h>
+#include <sys/pciio.h>
+#include <sys/time.h>
+#include <machine/atomic.h>
+#include <machine/resource.h>
+#include <dev/pci/pcivar.h>
+
+#include "bnxt.h"
+#include "bnxt_mpc.h"
+#include "bnxt_hwrm.h"
+#include "bnxt_log.h"
+
+/*
+ * bnxt_ktls_mpc_cmp() is defined in bnxt_ktls.c, not yet part of this
+ * branch; forward-declared here so this file builds standalone.
+ */
+void bnxt_ktls_mpc_cmp(struct bnxt_softc *softc, uint32_t client,
+ unsigned long handle, struct bnxt_cmpl_entry *cmpl_entry_arr,
+ uint32_t cmpl_num);
+
+void bnxt_alloc_mpc_info(struct bnxt_softc *softc, uint8_t mpc_chnls_cap)
+{
+ if (mpc_chnls_cap) {
+ if (!softc->mpc_info)
+ softc->mpc_info = malloc(sizeof(*softc->mpc_info),
+ M_DEVBUF, M_NOWAIT | M_ZERO);
+ } else {
+ bnxt_free_mpc_info(softc);
+ }
+ if (softc->mpc_info)
+ softc->mpc_info->mpc_chnls_cap = mpc_chnls_cap;
+}
+
+void bnxt_free_mpc_info(struct bnxt_softc *softc)
+{
+ if (softc->mpc_info) {
+ free(softc->mpc_info, M_DEVBUF);
+ softc->mpc_info = NULL;
+ }
+}
+
+static void __bnxt_set_dflt_mpc_rings(struct bnxt_softc *softc,
+ enum bnxt_mpc_type type, int *avail, int avail_cp)
+{
+ struct bnxt_mpc_info *mpc = softc->mpc_info;
+ int dflt1, dflt2;
+ int idx1, idx2;
+ int min1, min2;
+ int val1, val2;
+
+ if (type == BNXT_MPC_CRYPTO) {
+ min1 = BNXT_MIN_MPC_TCE;
+ min2 = BNXT_MIN_MPC_RCE;
+ dflt1 = BNXT_DFLT_MPC_TCE;
+ dflt2 = BNXT_DFLT_MPC_RCE;
+ idx1 = BNXT_MPC_TCE_TYPE;
+ idx2 = BNXT_MPC_RCE_TYPE;
+ } else {
+ min1 = BNXT_MIN_MPC_TE_CFA;
+ min2 = BNXT_MIN_MPC_RE_CFA;
+ dflt1 = BNXT_DFLT_MPC_TE_CFA;
+ dflt2 = BNXT_DFLT_MPC_RE_CFA;
+ idx1 = BNXT_MPC_TE_CFA_TYPE;
+ idx2 = BNXT_MPC_RE_CFA_TYPE;
+ }
+ if (*avail < (min1 + min2))
+ return;
+
+ val1 = min(*avail / 2, softc->ntxqsets);
+ val2 = val1;
+
+ val1 = min(val1, dflt1);
+ val2 = min(val2, dflt2);
+
+ if (avail_cp < min1 || avail_cp < min2)
+ return;
+
+ val1 = min(val1, avail_cp);
+ val2 = min(val2, avail_cp);
+
+ mpc->mpc_ring_count[idx1] = val1;
+ mpc->mpc_ring_count[idx2] = val2;
+
+ *avail = *avail - val1 - val2;
+}
+
+void bnxt_set_dflt_mpc_rings(struct bnxt_softc *softc)
+{
+ struct bnxt_func_qcfg *fn_qcfg = &softc->fn_qcfg;
+ struct bnxt_mpc_info *mpc = softc->mpc_info;
+ int avail, mpc_cp, i;
+ int avail_cp;
+
+ if (!mpc)
+ return;
+
+ for (i = 0; i < BNXT_MPC_TYPE_MAX; i++)
+ mpc->mpc_ring_count[i] = 0;
+ mpc->mpc_cp_rings_count = 0;
+
+ avail = fn_qcfg->orig_alloc_tx_rings - softc->ntxqsets;
+
+ avail_cp = fn_qcfg->orig_alloc_completion_rings - softc->ntxqsets;
+ avail_cp = min(avail_cp,
+ ((pci_msix_count(softc->dev) - 1) - softc->ntxqsets));
+
+ if (BNXT_MPC_CRYPTO_CAPABLE(softc))
+ __bnxt_set_dflt_mpc_rings(softc, BNXT_MPC_CRYPTO, &avail,
+ avail_cp);
+
+ for (i = 0, mpc_cp = 0; i < BNXT_MPC_TYPE_MAX; i++) {
+ if (mpc_cp < mpc->mpc_ring_count[i])
+ mpc_cp = mpc->mpc_ring_count[i];
+ }
+ mpc->mpc_cp_rings_count = mpc_cp;
+}
+
+int bnxt_alloc_mpcs(struct bnxt_softc *softc)
+{
+ struct bnxt_mpc_info *mpc = softc->mpc_info;
+ int i;
+
+ if (!BNXT_MPC_CRYPTO_CAPABLE(softc))
+ return 0;
+
+ for (i = 0; i < BNXT_MPC_TYPE_MAX; i++) {
+ int num;
+ struct bnxt_ring *txr;
+
+ /* Currently only KTLS TX is supported */
+ if (i != BNXT_MPC_TCE_TYPE)
+ continue;
+
+ num = mpc->mpc_ring_count[i];
+ if (!num)
+ continue;
+ txr = malloc(num * sizeof(*txr), M_DEVBUF, M_NOWAIT | M_ZERO);
+ if (!txr) {
+ device_printf(softc->dev,
+ "%s: Failed to allocate %d MPC rings\n",
+ __func__, num);
+ return ENOMEM;
+ }
+ mpc->mpc_rings[i] = txr;
+ }
+
+ mpc->mpc_cp_rings =
+ malloc(mpc->mpc_cp_rings_count * sizeof(struct bnxt_cp_ring),
+ M_DEVBUF, M_NOWAIT | M_ZERO);
+ if (!mpc->mpc_cp_rings) {
+ device_printf(softc->dev,
+ "%s: Failed to allocate %d MPC completion rings\n",
+ __func__, mpc->mpc_cp_rings_count);
+ return ENOMEM;
+ }
+
+ mpc->mpc_nq_rings =
+ malloc(mpc->mpc_cp_rings_count * sizeof(struct bnxt_cp_ring),
+ M_DEVBUF, M_NOWAIT | M_ZERO);
+ if (!mpc->mpc_nq_rings) {
+ device_printf(softc->dev,
+ "%s: Failed to allocate %d MPC notification rings\n",
+ __func__, mpc->mpc_cp_rings_count);
+ return ENOMEM;
+ }
+ return 0;
+}
+
+void bnxt_free_mpcs(struct bnxt_softc *softc)
+{
+ struct bnxt_mpc_info *mpc = softc->mpc_info;
+ int i;
+
+ if (!mpc)
+ return;
+
+ for (i = 0; i < BNXT_MPC_TYPE_MAX; i++) {
+ /* Currently only KTLS TX is supported */
+ if (i != BNXT_MPC_TCE_TYPE)
+ continue;
+
+ if (mpc->mpc_rings[i]) {
+ free(mpc->mpc_rings[i], M_DEVBUF);
+ mpc->mpc_rings[i] = NULL;
+ }
+ }
+
+ if (mpc->mpc_cp_rings) {
+ free(mpc->mpc_cp_rings, M_DEVBUF);
+ mpc->mpc_cp_rings = NULL;
+ }
+
+ if (mpc->mpc_nq_rings) {
+ free(mpc->mpc_nq_rings, M_DEVBUF);
+ mpc->mpc_nq_rings = NULL;
+ }
+}
+
+int bnxt_alloc_mpc_rings(struct bnxt_softc *softc)
+{
+ struct bnxt_mpc_info *mpc = softc->mpc_info;
+ int i, j, mpc_idx = 0;
+ int mpc_base_ring_id =
+ (softc->scctx->isc_nrxqsets * 2) + 1 + softc->ntxqsets;
+
+ if (!mpc)
+ return 0;
+
+ for (i = 0; i < mpc->mpc_cp_rings_count; i++) {
+ struct bnxt_cp_ring *cpr = &mpc->mpc_cp_rings[i];
+ int rc;
+
+ cpr->ring.queue_id = BNXT_MPC_QUEUE_ID;
+
+ /* Set up the completion ring */
+ cpr->stats_ctx_id = HWRM_NA_SIGNATURE;
+ cpr->ring.phys_id = (uint16_t)HWRM_NA_SIGNATURE;
+ cpr->ring.softc = softc;
+ cpr->ring.idx = i;
+ cpr->ring.id = mpc_base_ring_id + i;
+ cpr->ring.doorbell = softc->legacy_db_size;
+ cpr->ring.ring_size = softc->scctx->isc_ntxd[0];
+ cpr->ring.db_ring_mask = cpr->ring.ring_size - 1;
+
+ rc = iflib_dma_alloc(softc->ctx,
+ sizeof(struct cmpl_base) * cpr->ring.ring_size,
+ &cpr->ring.ring_mem, 0);
+ if (rc) {
+ device_printf(softc->dev,
+ "%s: Failed to allocate DMA memory for completion ring %d (size=%zu), rc=%d\n",
+ __func__, i, sizeof(struct cmpl_base) * cpr->ring.ring_size, rc);
+ return rc;
+ }
+
+ cpr->ring.vaddr = cpr->ring.ring_mem.idi_vaddr;
+ cpr->ring.paddr = cpr->ring.ring_mem.idi_paddr;
+ }
+
+ for (i = 0; i < BNXT_MPC_TYPE_MAX; i++) {
+ int num = mpc->mpc_ring_count[i], rc;
+
+ /* Currently only KTLS TX is supported */
+ if (i != BNXT_MPC_TCE_TYPE)
+ continue;
+
+ for (j = 0; j < num; j++) {
+ struct bnxt_ring *txr = &mpc->mpc_rings[i][j];
+ /* tx_stats[] is indexed across all MPC types, so use
+ * the flat mpc_idx, not the intra-type counter j. */
+ int stat_idx = mpc_idx;
+
+ txr->queue_id = BNXT_MPC_QUEUE_ID;
+ txr->mpc_chnl_type = i;
+
+ /* alloc mpc TX ring */
+ txr->phys_id = (uint16_t)HWRM_NA_SIGNATURE;
+ txr->softc = softc;
+ txr->idx = i << BNXT_RING_MPC_IDX_TYPE_SFT | j;
+ txr->id = mpc_base_ring_id + mpc_idx++;
+ txr->doorbell = softc->legacy_db_size;
+ txr->ring_size = softc->scctx->isc_ntxd[1];
+ txr->db_ring_mask = txr->ring_size - 1;
+
+ /* Init before any early return: bnxt_free_mpc_rings()
+ * unconditionally mtx_destroy()s every txr. */
+ mtx_init(&txr->tx_lock, "MPC TX lock", NULL, MTX_SPIN);
+
+ rc = iflib_dma_alloc(softc->ctx,
+ sizeof(struct tx_bd_short) * txr->ring_size,
+ &txr->ring_mem, 0);
+ if (rc) {
+ device_printf(softc->dev,
+ "%s: Failed to allocate DMA memory for TX ring %d (size=%zu), rc=%d\n",
+ __func__, j, sizeof(struct tx_bd_short) * txr->ring_size, rc);
+ return rc;
+ }
+
+ txr->vaddr = txr->ring_mem.idi_vaddr;
+ txr->paddr = txr->ring_mem.idi_paddr;
+
+ txr->tx_mpc_buf_ring =
+ malloc(txr->ring_size * sizeof(struct bnxt_sw_mpc_tx_bd),
+ M_DEVBUF, M_NOWAIT | M_ZERO);
+ if (!txr->tx_mpc_buf_ring) {
+ device_printf(softc->dev,
+ "%s: Failed to allocate buffer ring for TX ring %d (size=%zu)\n",
+ __func__, j, txr->ring_size * sizeof(struct bnxt_sw_mpc_tx_bd));
+ return ENOMEM;
+ }
+
+ rc = iflib_dma_alloc(softc->ctx, sizeof(struct ctx_hw_stats),
+ &mpc->tx_stats[stat_idx], 0);
+ if (rc) {
+ device_printf(softc->dev,
+ "%s: Failed to allocate DMA memory for TX stats ring %d, rc=%d\n",
+ __func__, j, rc);
+ return rc;
+ }
+ bus_dmamap_sync(mpc->tx_stats[stat_idx].idi_tag,
+ mpc->tx_stats[stat_idx].idi_map, BUS_DMASYNC_PREREAD);
+ }
+ }
+
+ /* Set up the Notification ring (NQ) */
+ for (i = 0; i < mpc->mpc_cp_rings_count; i++) {
+ struct bnxt_cp_ring *nqr = &mpc->mpc_nq_rings[i];
+ int rc;
+
+ nqr->ring.queue_id = BNXT_MPC_QUEUE_ID;
+
+ nqr->stats_ctx_id = HWRM_NA_SIGNATURE;
+ nqr->ring.phys_id = (uint16_t)HWRM_NA_SIGNATURE;
+ nqr->ring.softc = softc;
+ nqr->ring.idx = i;
+ nqr->ring.id = softc->ntxqsets + i;
+ nqr->ring.doorbell = softc->legacy_db_size;
+ nqr->ring.ring_size = softc->scctx->isc_ntxd[2];
+ nqr->ring.db_ring_mask = nqr->ring.ring_size - 1;
+
+ rc = iflib_dma_alloc(softc->ctx,
+ sizeof(struct cmpl_base) * nqr->ring.ring_size,
+ &nqr->ring.ring_mem, 0);
+ if (rc) {
+ device_printf(softc->dev,
+ "%s: Failed to allocate DMA memory for notification ring %d (size=%zu), rc=%d\n",
+ __func__, i, sizeof(struct cmpl_base) * nqr->ring.ring_size, rc);
+ return rc;
+ }
+
+ nqr->ring.vaddr = nqr->ring.ring_mem.idi_vaddr;
+ nqr->ring.paddr = nqr->ring.ring_mem.idi_paddr;
+ nqr->type = TX_CP_NQ;
+ }
+
+ return 0;
+}
+
+void bnxt_free_mpc_rings(struct bnxt_softc *softc)
+{
+ struct bnxt_mpc_info *mpc = softc->mpc_info;
+ int i, j, mpc_idx = 0;
+
+ if (!mpc)
+ return;
+
+ /* A partial alloc failure can leave these NULL; guard each array. */
+ if (mpc->mpc_nq_rings) {
+ for (i = 0; i < mpc->mpc_cp_rings_count; i++)
+ iflib_dma_free(&mpc->mpc_nq_rings[i].ring.ring_mem);
+ }
+
+ if (mpc->mpc_cp_rings) {
+ for (i = 0; i < mpc->mpc_cp_rings_count; i++)
+ iflib_dma_free(&mpc->mpc_cp_rings[i].ring.ring_mem);
+ }
+
+ for (i = 0; i < BNXT_MPC_TYPE_MAX; i++) {
+ int num = mpc->mpc_ring_count[i];
+
+ /* Currently only KTLS TX is supported */
+ if (i != BNXT_MPC_TCE_TYPE)
+ continue;
+
+ /*
+ * Same partial-allocation case as above: mpc_ring_count[i] can
+ * be nonzero while mpc_rings[i] itself failed to allocate.
+ */
+ if (!mpc->mpc_rings[i])
+ continue;
+
+ for (j = 0; j < num; j++) {
+ struct bnxt_ring *txr = &mpc->mpc_rings[i][j];
+
+ /* TCE ring vaddr holds cleartext AES-GCM keys; scrub
+ * before freeing the pages. */
+ if (i == BNXT_MPC_TCE_TYPE && txr->ring_mem.idi_vaddr)
+ explicit_bzero(txr->ring_mem.idi_vaddr,
+ txr->ring_mem.idi_size);
+
+ iflib_dma_free(&txr->ring_mem);
+ if (txr->tx_mpc_buf_ring) {
+ free(txr->tx_mpc_buf_ring, M_DEVBUF);
+ txr->tx_mpc_buf_ring = NULL;
+ }
+ /* A ring past a partial alloc failure never had its
+ * tx_lock mtx_init()'d; guard the destroy. */
+ if (mtx_initialized(&txr->tx_lock))
+ mtx_destroy(&txr->tx_lock);
+ /* Must match the flat stat_idx bnxt_alloc_mpc_rings() used. */
+ iflib_dma_free(&mpc->tx_stats[mpc_idx++]);
+ }
+ }
+}
+
+void bnxt_clear_mpc_rings_ids(struct bnxt_softc *softc)
+{
+ struct bnxt_mpc_info *mpc = softc->mpc_info;
+ int i, j;
+
+ if (!mpc)
+ return;
+
+ for (i = 0; i < mpc->mpc_cp_rings_count; i++) {
+ mpc->mpc_nq_rings[i].ring.phys_id = (uint16_t)HWRM_NA_SIGNATURE;
+ mpc->mpc_cp_rings[i].ring.phys_id = (uint16_t)HWRM_NA_SIGNATURE;
+ mpc->mpc_cp_rings[i].stats_ctx_id = HWRM_NA_SIGNATURE;
+ }
+
+ for (i = 0; i < BNXT_MPC_TYPE_MAX; i++) {
+ int num = mpc->mpc_ring_count[i];
+
+ /* Currently only KTLS TX is supported */
+ if (i != BNXT_MPC_TCE_TYPE)
+ continue;
+
+ for (j = 0; j < num; j++) {
+ struct bnxt_ring *txr = &mpc->mpc_rings[i][j];
+
+ txr->phys_id = (uint16_t)HWRM_NA_SIGNATURE;
+ }
+ }
+}
+
+static void bnxt_map_tx_to_cp(struct bnxt_softc *softc)
+{
+ struct bnxt_mpc_info *mpc = softc->mpc_info;
+ int cp_count = mpc->mpc_cp_rings_count;
+ int i, j;
+
+ for (i = 0; i < BNXT_MPC_TYPE_MAX; i++) {
+ int num = mpc->mpc_ring_count[i];
+
+ if (i != BNXT_MPC_TCE_TYPE)
+ continue;
+
+ for (j = 0; j < num; j++) {
+ struct bnxt_ring *txr = &mpc->mpc_rings[i][j];
+ int cp_idx = j % cp_count;
+
+ txr->cp_ring = &mpc->mpc_cp_rings[cp_idx];
+ }
+ }
+}
+
+struct bnxt_ring *bnxt_select_mpc_ring(struct bnxt_mpc_info *mpc)
+{
+ int idx;
+
+ /* Callers should gate on a non-zero TCE ring count already, but
+ * guard the divide/index here too since this feeds a live TX path. */
+ if (!mpc || !mpc->mpc_ring_count[BNXT_MPC_TCE_TYPE])
+ return NULL;
+
+ idx = PCPU_GET(cpuid) % mpc->mpc_ring_count[BNXT_MPC_TCE_TYPE];
+
+ return &mpc->mpc_rings[BNXT_MPC_TCE_TYPE][idx];
+}
+
+int bnxt_hwrm_mpc_ring_alloc(struct bnxt_softc *softc)
+{
+ struct bnxt_mpc_info *mpc = softc->mpc_info;
+ int i, j, rc;
+
+ if (!mpc)
+ return 0;
+
+ bnxt_map_tx_to_cp(softc);
+ for (i = 0; i < mpc->mpc_cp_rings_count; i++) {
+ struct bnxt_cp_ring *nqr = &mpc->mpc_nq_rings[i];
+
+ nqr->cons = 0;
+ nqr->v_bit = 1;
+ nqr->raw_cons = 0;
+ nqr->toggle = 0;
+ nqr->last_idx = UINT32_MAX;
+ bnxt_mark_cpr_invalid(nqr);
+
+ rc = bnxt_hwrm_ring_alloc(
+ softc, HWRM_RING_ALLOC_INPUT_RING_TYPE_NQ, &nqr->ring);
+
+ bnxt_set_db_mask(softc, &nqr->ring,
+ HWRM_RING_ALLOC_INPUT_RING_TYPE_NQ);
+ if (rc) {
+ device_printf(softc->dev,
+ "%s: Failed to allocate HWRM notification ring %d, rc=%d\n",
+ __func__, i, rc);
+ return rc;
+ }
+
+ softc->db_ops.bnxt_db_nq(nqr, 1);
+ }
+
+ for (i = 0; i < mpc->mpc_cp_rings_count; i++) {
+ struct bnxt_cp_ring *cpr = &mpc->mpc_cp_rings[i];
+
+ rc = bnxt_hwrm_stat_ctx_alloc(softc, cpr,
+ mpc->tx_stats[i].idi_paddr);
+ if (rc) {
+ device_printf(softc->dev,
+ "%s: Failed to allocate HWRM stats context for completion ring %d, rc=%d\n",
+ __func__, i, rc);
+ return rc;
+ }
+
+ cpr->v_bit = 1;
+ cpr->cons = 0;
+ cpr->raw_cons = 0;
+ cpr->toggle = 0;
+ bnxt_mark_cpr_invalid(cpr);
+ rc = bnxt_hwrm_ring_alloc(
+ softc, HWRM_RING_ALLOC_INPUT_RING_TYPE_L2_CMPL, &cpr->ring);
+ bnxt_set_db_mask(softc, &cpr->ring,
+ HWRM_RING_ALLOC_INPUT_RING_TYPE_L2_CMPL);
+ if (rc) {
+ device_printf(softc->dev,
+ "%s: Failed to allocate HWRM completion ring %d, rc=%d\n",
+ __func__, i, rc);
+ return rc;
+ }
+
+ softc->db_ops.bnxt_db_tx_cq(cpr, 1);
+ }
+
+ for (i = 0; i < BNXT_MPC_TYPE_MAX; i++) {
+ int num = mpc->mpc_ring_count[i];
+
+ /* Currently only KTLS TX is supported */
+ if (i != BNXT_MPC_TCE_TYPE)
+ continue;
+
+ for (j = 0; j < num; j++) {
+ struct bnxt_ring *txr = &mpc->mpc_rings[i][j];
+
+ txr->prod = 0;
+ txr->cons = 0;
+ memset(txr->tx_mpc_buf_ring, 0,
+ txr->ring_size * sizeof(struct bnxt_sw_mpc_tx_bd));
+
+ rc = bnxt_hwrm_ring_alloc(
+ softc, HWRM_RING_ALLOC_INPUT_RING_TYPE_TX, txr);
+ bnxt_set_db_mask(softc, txr,
+ HWRM_RING_ALLOC_INPUT_RING_TYPE_TX);
+ if (rc) {
+ device_printf(softc->dev,
+ "%s: Failed to allocate HWRM TX ring %d, rc=%d\n",
+ __func__, j, rc);
+ return rc;
+ }
+ softc->db_ops.bnxt_db_tx(txr, 0);
+ }
+ }
+ return 0;
+}
+
+int bnxt_hwrm_mpc_ring_free(struct bnxt_softc *softc)
+{
+ struct bnxt_mpc_info *mpc = softc->mpc_info;
+ int i, j, rc;
+
+ if (!mpc)
+ return 0;
+
+ for (i = 0; i < BNXT_MPC_TYPE_MAX; i++) {
+ int num = mpc->mpc_ring_count[i];
+
+ /* Currently only KTLS TX is supported */
+ if (i != BNXT_MPC_TCE_TYPE)
+ continue;
+
+ for (j = 0; j < num; j++) {
+ struct bnxt_ring *txr = &mpc->mpc_rings[i][j];
+ struct bnxt_cp_ring *cpr = txr->cp_ring;
+
+ if (!cpr)
+ continue;
+
+ rc = bnxt_hwrm_ring_free(
+ softc, HWRM_RING_ALLOC_INPUT_RING_TYPE_TX, txr,
+ cpr->ring.phys_id);
+ if (rc) {
+ device_printf(softc->dev,
+ "%s: Failed to free HWRM TX ring %d (ring_id=%d, phys_id=%d), rc=%d\n",
+ __func__, j, txr->id, cpr->ring.phys_id, rc);
+ return rc;
+ }
+ txr->phys_id = (uint16_t)HWRM_NA_SIGNATURE;
+ }
+ }
+
+ for (i = 0; i < mpc->mpc_cp_rings_count; i++) {
+ struct bnxt_cp_ring *cpr = &mpc->mpc_cp_rings[i];
+
+ rc = bnxt_hwrm_ring_free(
+ softc, HWRM_RING_ALLOC_INPUT_RING_TYPE_L2_CMPL,
+ &cpr->ring, (uint16_t)HWRM_NA_SIGNATURE);
+ if (rc) {
+ device_printf(softc->dev,
+ "%s: Failed to free HWRM completion ring %d (ring_id=%d, phys_id=%d), rc=%d\n",
+ __func__, i, cpr->ring.id, cpr->ring.phys_id, rc);
+ return rc;
+ }
+
+ rc = bnxt_hwrm_stat_ctx_free(softc, cpr);
+ if (rc) {
+ device_printf(softc->dev,
+ "%s: Failed to free HWRM stats context id %d for completion ring %d, rc=%d\n",
+ __func__, cpr->stats_ctx_id, i, rc);
+ return rc;
+ }
+
+ cpr->ring.phys_id = (uint16_t)HWRM_NA_SIGNATURE;
+ }
+
+ for (i = 0; i < mpc->mpc_cp_rings_count; i++) {
+ struct bnxt_cp_ring *nqr = &mpc->mpc_nq_rings[i];
+
+ rc = bnxt_hwrm_ring_free(
+ softc, HWRM_RING_ALLOC_INPUT_RING_TYPE_NQ, &nqr->ring,
+ (uint16_t)HWRM_NA_SIGNATURE);
+ if (rc) {
+ device_printf(softc->dev,
+ "%s: Failed to free HWRM notification ring %d (ring_id=%d, phys_id=%d), rc=%d\n",
+ __func__, i, nqr->ring.id, nqr->ring.phys_id, rc);
+ return rc;
+ }
+ nqr->ring.phys_id = (uint16_t)HWRM_NA_SIGNATURE;
+ }
+ return 0;
+}
+
+static void bnxt_mpc_process_cpr(struct bnxt_cp_ring *cpr)
+{
+ struct bnxt_softc *softc = cpr->ring.softc;
+
+ /* Handle completions on the mpc completion ring */
+ uint32_t raw_cons = cpr->raw_cons;
+ uint32_t cons = cpr->cons;
+ struct cmpl_base *cmpl;
+ bool v_bit = cpr->v_bit;
+ bool last_v_bit;
+ uint32_t last_cons;
+ uint16_t type;
+
+ for (;;) {
+ last_cons = cons;
+ last_v_bit = v_bit;
+ cmpl = &((struct cmpl_base *)cpr->ring.vaddr)[cons];
+
+ if (!CMP_VALID(cmpl, v_bit))
+ break;
+
+ type = le16toh(cmpl->type) & CMPL_BASE_TYPE_MASK;
+ switch (type) {
+ case CMPL_BASE_TYPE_MID_PATH_SHORT:
+ case CMPL_BASE_TYPE_MID_PATH_LONG: {
+ int rc = bnxt_mpc_cmp(softc, cpr, &raw_cons);
+
+ /* EBUSY: 2nd half of a MID_PATH_LONG completion hasn't
+ * landed yet; stop and retry later. Any other error is
+ * non-transient and already logged/skipped, so fall
+ * through and advance like the success case. */
+ if (rc == EBUSY)
+ goto done;
+ break;
+ }
+ case CMPL_BASE_TYPE_HWRM_DONE:
+ /* ignore this completion */
+ break;
+ default: {
+ /* Rate-limit prints so a stuck ring feeding
+ * unrecognized types can't flood the console.
+ * Statics are unlocked; a best-effort limiter. */
+ static struct timeval last;
+ static int curpps;
+
+ if (ppsratecheck(&last, &curpps, 5))
+ device_printf(softc->dev,
+ "%s: Unknown completion type %u on ring %d (raw_cons=%u, cons=%u)\n",
+ __func__, type, cpr->ring.idx, raw_cons, cons);
+ break;
+ }
+ }
+ raw_cons++;
+ /* Recompute from raw_cons rather than NEXT_CP_CONS_V(): a
+ * MID_PATH_LONG completion can wrap raw_cons inside
+ * bnxt_mpc_cmp(), which would desync v_bit otherwise. */
+ cons = MPC_RING_CMP(cpr, raw_cons);
+ v_bit = !(raw_cons & cpr->ring.ring_size);
+ }
+done:
+ cpr->cons = last_cons;
+ cpr->raw_cons = raw_cons;
+ cpr->v_bit = last_v_bit;
+ softc->db_ops.bnxt_db_tx_cq(cpr, 1);
+}
+
+static void bnxt_mpc_irq(void *arg)
+{
+ struct bnxt_cp_ring *cpr = arg;
+ struct bnxt_softc *softc = cpr->ring.softc;
+ nq_cn_t *cmp = (nq_cn_t *)cpr->ring.vaddr;
+ uint32_t raw_cons = cpr->raw_cons;
+ uint16_t nq_type, nqe_cnt = 0;
+ uint32_t cons = cpr->cons;
+ bool v_bit = cpr->v_bit;
+
+ while (1) {
+ if (!NQ_VALID(&cmp[cons], v_bit))
+ goto done;
+
+ nq_type = NQ_CN_TYPE_MASK & cmp[cons].type;
+
+ if (NQE_CN_TYPE(nq_type) == NQ_CN_TYPE_CQ_NOTIFICATION) {
*** 817 LINES SKIPPED ***