Re: git: 088a42c0ff76 - main - if_bnxt: add MPC (Mid-Path Channel) ring infrastructure
- Go to: [ bottom of page ] [ top of archives ] [ this month ]
Date: Tue, 13 Oct 2026 06:13:10 UTC
On Tue, 13 Oct 2026 at 8:25 AM, Minsoo Choo <minsoo@minsoo.io> wrote:
> On 2026-10-09 06:12, Sumit Saxena wrote:
>
> > The branch main has been updated by ssaxena:
> >
> > URL:
> https://cgit.FreeBSD.org/src/commit/?id=088a42c0ff7620463a870bc789f940fdf112802f
> >
> > commit 088a42c0ff7620463a870bc789f940fdf112802f
> > Author: Sumit Saxena <sumit.saxena@broadcom.com>
> > AuthorDate: 2026-07-29 09:12:26 +0000
> > Commit: Sumit Saxena <ssaxena@FreeBSD.org>
> > CommitDate: 2026-10-09 08:24:43 +0000
> >
> > if_bnxt: add MPC (Mid-Path Channel) ring infrastructure
> >
> > Add MPC ring support: a pair of extra, driver-private
> > TX/completion/notification rings the firmware uses as a side-channel
> > for offload commands. The only current consumer is kTLS TX crypto
> > command submission; RCE/CFA channel types are not yet implemented by
> > any consumer.
> >
> > Adds MPC ring allocation/teardown, HWRM ring alloc/free, a private
> > interrupt path for the MPC NQ ring, and the TX submit/completion
> > path. The RoCE-only IRQ table builder is split into a generic,
> > exported bnxt_populate_irq(softc, irq_count) that both MPC and RoCE
> > use, plus an incremental bnxt_populate_irq_roce() that grows the
> > table on top of whatever is already there, so IRQ rids stay
> > consistent regardless of attach order.
> >
> > This patch does not link standalone: bnxt_mpc_cmp() calls into
> > kTLS's completion handler, added by a later patch.
> >
> > Reviewed by: gallatin
> > MFC after: 2 weeks
> > Sponsored by: Broadcom Inc.
> > Differential Revision: https://reviews.freebsd.org/D58603
> > ---
> > sys/conf/files | 1 +
> > sys/dev/bnxt/bnxt_en/bnxt.h | 28 +
> > sys/dev/bnxt/bnxt_en/bnxt_hwrm.c | 54 +-
> > sys/dev/bnxt/bnxt_en/bnxt_mpc.c | 1075
> +++++++++++++++++++++++++++++++++++++
> > sys/dev/bnxt/bnxt_en/bnxt_mpc.h | 182 +++++++
> > sys/dev/bnxt/bnxt_en/bnxt_ulp.c | 69 ++-
> > sys/dev/bnxt/bnxt_en/if_bnxt.c | 54 +-
> > sys/modules/bnxt/bnxt_en/Makefile | 1 +
> > 8 files changed, 1450 insertions(+), 14 deletions(-)
> >
> > diff --git a/sys/conf/files b/sys/conf/files
> > index 73a5d5fa8408..0a6e5874896f 100644
> > --- a/sys/conf/files
> > +++ b/sys/conf/files
> > @@ -1333,6 +1333,7 @@ dev/bnxt/bnxt_en/bnxt_log.c optional
> bnxt iflib pci compile-with "${BNXT_C}"
> > dev/bnxt/bnxt_en/bnxt_log_data.c optional bnxt iflib pci
> compile-with "${BNXT_C}"
> > dev/bnxt/bnxt_en/bnxt_coredump.c optional bnxt iflib pci
> compile-with "${BNXT_C}"
> > dev/bnxt/bnxt_en/bnxt_ptp.c optional bnxt iflib pci
> compile-with "${BNXT_C}"
> > +dev/bnxt/bnxt_en/bnxt_mpc.c optional bnxt iflib pci
> compile-with "${BNXT_C}"
> > dev/bnxt/bnxt_en/if_bnxt.c optional bnxt iflib pci
> compile-with "${BNXT_C}"
> > dev/bwi/bwimac.c optional bwi
> > dev/bwi/bwiphy.c optional bwi
> > diff --git a/sys/dev/bnxt/bnxt_en/bnxt.h b/sys/dev/bnxt/bnxt_en/bnxt.h
> > index 80cc8905d66b..1de100915f97 100644
> > --- a/sys/dev/bnxt/bnxt_en/bnxt.h
> > +++ b/sys/dev/bnxt/bnxt_en/bnxt.h
> > @@ -608,6 +608,11 @@ struct tx_bd_opaque {
> >
> > #define TX_RING_MASK(txr) ((txr)->ring_size - 1)
> >
> > +struct bnxt_sw_mpc_tx_bd {
> > + uint8_t inline_bds;
> > + unsigned long handle;
> > +};
> > +
> > struct bnxt_ring {
> > uint64_t paddr;
> > vm_offset_t doorbell;
> > @@ -618,6 +623,15 @@ struct bnxt_ring {
> > uint16_t phys_id;
> > uint16_t idx;
> > uint16_t running_bds; /* host
> TX-completion-coalescing accumulator */
> > + uint8_t queue_id; /* CoS queue, or
> BNXT_MPC_QUEUE_ID below */
> > +#define BNXT_MPC_QUEUE_ID 0xff
> > + uint8_t mpc_chnl_type; /* valid when queue_id ==
> BNXT_MPC_QUEUE_ID */
> > + struct iflib_dma_info ring_mem; /* manual DMA alloc for
> driver-private rings (MPC) */
> > + struct bnxt_cp_ring *cp_ring; /* completion ring this TX
> ring reports to (MPC) */
> > + uint16_t prod; /* MPC TX ring producer
> index */
> > + uint16_t cons; /* MPC TX ring consumer
> index */
> > + struct mtx tx_lock; /* MPC TX ring lock */
> > + struct bnxt_sw_mpc_tx_bd *tx_mpc_buf_ring;
> > struct bnxt_full_tpa_start *tpa_start;
> > union {
> > u64 db_key64;
> > @@ -649,6 +663,10 @@ struct bnxt_cp_ring {
> > uint8_t type;
> > #define Q_TYPE_TX 1
> > #define Q_TYPE_RX 2
> > +#define TX_CP_NQ 3 /* MPC-private NQ ring
> (bnxt_mpc.c) */
> > + int msix_vec; /* MPC-private ring:
> bus_setup_intr vector */
> > + struct resource *irq_res; /* MPC-private ring:
> bus_setup_intr resource */
> > + void *irq_cookie; /* MPC-private ring:
> bus_setup_intr cookie */
> > };
> >
> > struct bnxt_full_tpa_start {
> > @@ -718,6 +736,10 @@ struct bnxt_func_qcfg {
> > uint16_t alloc_completion_rings;
> > uint16_t alloc_tx_rings;
> > uint16_t alloc_rx_rings;
> > + /* Firmware-granted counts before the MPC reservation is
> subtracted;
> > + * bnxt_set_dflt_mpc_rings() needs these unreduced values. */
> > + uint16_t orig_alloc_completion_rings;
> > + uint16_t orig_alloc_tx_rings;
> > uint16_t alloc_vnics;
> > uint16_t alloc_rss_ctx;
> > uint16_t alloc_l2_ctx;
> > @@ -1435,6 +1457,8 @@ struct bnxt_softc {
> > struct bnxt_ptp_cfg *ptp_cfg;
> > bool rx_ts_enabled;
> >
> > + struct bnxt_mpc_info *mpc_info;
> > +
> > #define BNXT_CAGR_CQCOAL_OFFSET 0xc00
> > #define BNXT_CAGR_NQAGG_MAXTIMER 0x5930000
> > #define BNXT_CAGR_TICK_RES_DEFAULT 0x2
> > @@ -1523,6 +1547,10 @@ int bnxt_alloc_ctx_pg_tbls(struct bnxt_softc
> *softc,
> > uint32_t mem_size, uint8_t depth,
> > struct bnxt_ctx_mem_type *ctxm);
> > int bnxt_hwrm_ptp_qcfg(struct bnxt_softc *bp);
> > +int bnxt_populate_irq(struct bnxt_softc *softc, int irq_count);
> > +void bnxt_mark_cpr_invalid(struct bnxt_cp_ring *cpr);
> > +void bnxt_set_db_mask(struct bnxt_softc *bp, struct bnxt_ring *db,
> > + uint32_t ring_type);
> >
> > static inline u32
> > readl_fbsd(struct bnxt_softc *bp, u32 reg_off, u8 bar_idx)
> > diff --git a/sys/dev/bnxt/bnxt_en/bnxt_hwrm.c
> b/sys/dev/bnxt/bnxt_en/bnxt_hwrm.c
> > index 2649100e9b86..fb549bc05c10 100644
> > --- a/sys/dev/bnxt/bnxt_en/bnxt_hwrm.c
> > +++ b/sys/dev/bnxt/bnxt_en/bnxt_hwrm.c
> > @@ -32,6 +32,7 @@
> >
> > #include "bnxt.h"
> > #include "bnxt_hwrm.h"
> > +#include "bnxt_mpc.h"
> > #include "hsi_struct_def.h"
> > #include "bnxt_coredump.h"
> >
> > @@ -1205,6 +1206,7 @@ bnxt_hwrm_func_qcaps(struct bnxt_softc *softc)
> > (void *)softc->hwrm_cmd_resp.idi_vaddr;
> > struct bnxt_func_info *func = &softc->func;
> > uint32_t flags, flags_ext, flags_ext2;
> > + uint8_t mpc_chnls_cap = 0;
> >
> > bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_FUNC_QCAPS);
> > req.fid = htole16(0xffff);
> > @@ -1354,8 +1356,18 @@ bnxt_hwrm_func_qcaps(struct bnxt_softc *softc)
> > }
> > }
> >
> > + /*
> > + * Only extract the field needed here; the actual alloc/free work
> > + * (malloc(M_NOWAIT)/free() in bnxt_alloc_mpc_info()) happens after
> > + * BNXT_HWRM_UNLOCK() below instead of under hwrm_lock.
> > + */
> > + if (BNXT_PF(softc))
> > + mpc_chnls_cap = resp->mpc_chnls_cap;
> > +
> > fail:
> > BNXT_HWRM_UNLOCK(softc);
> > + if (!rc && BNXT_PF(softc))
> > + bnxt_alloc_mpc_info(softc, mpc_chnls_cap);
> > return rc;
> > }
> >
> > @@ -1468,6 +1480,22 @@ bnxt_hwrm_func_qcfg(struct bnxt_softc *softc)
> > fn_qcfg->alloc_stat_ctx = le16toh(resp->alloc_stat_ctx);
> > fn_qcfg->alloc_msix = le16toh(resp->alloc_msix);
> >
> > + fn_qcfg->orig_alloc_completion_rings =
> fn_qcfg->alloc_completion_rings;
> > + fn_qcfg->orig_alloc_tx_rings = fn_qcfg->alloc_tx_rings;
> > +
> > + /* Reserve minimum rings for MPC */
> > + if (BNXT_PF(softc)) {
> > + if (softc->mpc_info) {
> > + uint16_t mpc_min = BNXT_MIN_MPC_TCE +
> BNXT_MIN_MPC_RCE;
> > +
> > + /* Clamp to 0 instead of underflowing these
> unsigned counts. */
> > + fn_qcfg->alloc_completion_rings -=
> > + min(fn_qcfg->alloc_completion_rings, mpc_min);
> > + fn_qcfg->alloc_tx_rings -=
> > + min(fn_qcfg->alloc_tx_rings, mpc_min);
> > + }
> > + }
> > +
> > switch (resp->port_partition_type) {
> > case HWRM_FUNC_QCFG_OUTPUT_PORT_PARTITION_TYPE_NPAR1_0:
> > case HWRM_FUNC_QCFG_OUTPUT_PORT_PARTITION_TYPE_NPAR1_2:
> > @@ -1976,17 +2004,24 @@ bnxt_hwrm_ring_alloc(struct bnxt_softc *softc,
> uint8_t type,
> >
> > switch (type) {
> > case HWRM_RING_ALLOC_INPUT_RING_TYPE_TX:
> > - cp_ring = &softc->tx_cp_rings[idx];
> > + if (ring->queue_id == BNXT_MPC_QUEUE_ID) {
> > + cp_ring = ring->cp_ring;
> > + req.mpc_chnls_type = ring->mpc_chnl_type;
> > + req.enables |=
> htole32(HWRM_RING_ALLOC_INPUT_ENABLES_MPC_CHNLS_TYPE);
> > + } else {
> > + cp_ring = &softc->tx_cp_rings[idx];
> >
> > - req.cmpl_ring_id = htole16(cp_ring->ring.phys_id);
> > - /* queue_id - what CoS queue the TX ring is associated
> with */
> > - req.queue_id = htole16(softc->tx_q_info[0].queue_id);
> > + /* queue_id - what CoS queue the TX ring is
> associated with */
> > + req.queue_id =
> htole16(softc->tx_q_info[0].queue_id);
> >
> > + req.enables |= htole32(
> > +
> HWRM_RING_ALLOC_INPUT_ENABLES_STAT_CTX_ID_VALID);
> > + }
> > req.stat_ctx_id = htole32(cp_ring->stats_ctx_id);
> > - req.enables |= htole32(
> > - HWRM_RING_ALLOC_INPUT_ENABLES_STAT_CTX_ID_VALID);
> > - if (softc->fw_cap & BNXT_FLAG_TX_COAL_CMPL)
> > + if (softc->fw_cap & BNXT_FLAG_TX_COAL_CMPL &&
> > + ring->queue_id != BNXT_MPC_QUEUE_ID)
> > req.cmpl_coal_cnt = softc->tx_hw_coal_cnt;
> > + req.cmpl_ring_id = htole16(cp_ring->ring.phys_id);
> > break;
> > case HWRM_RING_ALLOC_INPUT_RING_TYPE_RX:
> > if (!BNXT_CHIP_P5_PLUS(softc))
> > @@ -2023,7 +2058,10 @@ bnxt_hwrm_ring_alloc(struct bnxt_softc *softc,
> uint8_t type,
> > }
> >
> > req.cq_handle = htole64(ring->id);
> > - req.nq_ring_id =
> htole16(softc->nq_rings[idx].ring.phys_id);
> > + if (ring->queue_id == BNXT_MPC_QUEUE_ID)
> > + req.nq_ring_id =
> htole16(softc->mpc_info->mpc_nq_rings[idx].ring.phys_id);
> > + else
> > + req.nq_ring_id =
> htole16(softc->nq_rings[idx].ring.phys_id);
> > req.enables |= htole32(
> > HWRM_RING_ALLOC_INPUT_ENABLES_NQ_RING_ID_VALID);
> > break;
> > diff --git a/sys/dev/bnxt/bnxt_en/bnxt_mpc.c
> b/sys/dev/bnxt/bnxt_en/bnxt_mpc.c
> > new file mode 100644
> > index 000000000000..4e35b3bdca9c
> > --- /dev/null
> > +++ b/sys/dev/bnxt/bnxt_en/bnxt_mpc.c
> > @@ -0,0 +1,1075 @@
> > +/*
> > + * Broadcom NetXtreme-C/E network driver.
> > + *
> > + * Copyright (c) 2026 Broadcom, All Rights Reserved.
> > + * The term Broadcom refers to Broadcom Limited and/or its subsidiaries
> > + *
> > + * Redistribution and use in source and binary forms, with or without
> > + * modification, are permitted provided that the following conditions
> > + * are met:
> > + * 1. Redistributions of source code must retain the above copyright
> > + * notice, this list of conditions and the following disclaimer.
> > + * 2. Redistributions in binary form must reproduce the above copyright
> > + * notice, this list of conditions and the following disclaimer in
> the
> > + * documentation and/or other materials provided with the
> distribution.
> > + *
> > + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
> "AS IS"
> > + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
> TO, THE
> > + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
> PURPOSE
> > + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
> CONTRIBUTORS
> > + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
> OR
> > + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
> > + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
> BUSINESS
> > + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
> IN
> > + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
> OTHERWISE)
> > + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
> OF
> > + * THE POSSIBILITY OF SUCH DAMAGE.
> > + */
> > +
> > +#include <sys/param.h>
> > +#include <sys/endian.h>
> > +#include <sys/bus.h>
> > +#include <sys/rman.h>
> > +#include <sys/pciio.h>
> > +#include <sys/time.h>
> > +#include <machine/atomic.h>
> > +#include <machine/resource.h>
> > +#include <dev/pci/pcivar.h>
> > +
> > +#include "bnxt.h"
> > +#include "bnxt_mpc.h"
> > +#include "bnxt_hwrm.h"
> > +#include "bnxt_log.h"
> > +
> > +/*
> > + * bnxt_ktls_mpc_cmp() is defined in bnxt_ktls.c, not yet part of this
> > + * branch; forward-declared here so this file builds standalone.
> > + */
> > +void bnxt_ktls_mpc_cmp(struct bnxt_softc *softc, uint32_t client,
> > + unsigned long handle, struct bnxt_cmpl_entry *cmpl_entry_arr,
> > + uint32_t cmpl_num);
> > +
> > +void bnxt_alloc_mpc_info(struct bnxt_softc *softc, uint8_t
> mpc_chnls_cap)
> > +{
> > + if (mpc_chnls_cap) {
> > + if (!softc->mpc_info)
> > + softc->mpc_info = malloc(sizeof(*softc->mpc_info),
> > + M_DEVBUF, M_NOWAIT |
> M_ZERO);
> > + } else {
> > + bnxt_free_mpc_info(softc);
> > + }
> > + if (softc->mpc_info)
> > + softc->mpc_info->mpc_chnls_cap = mpc_chnls_cap;
> > +}
> > +
> > +void bnxt_free_mpc_info(struct bnxt_softc *softc)
> > +{
> > + if (softc->mpc_info) {
> > + free(softc->mpc_info, M_DEVBUF);
> > + softc->mpc_info = NULL;
> > + }
> > +}
> > +
> > +static void __bnxt_set_dflt_mpc_rings(struct bnxt_softc *softc,
> > + enum bnxt_mpc_type type, int *avail, int avail_cp)
> > +{
> > + struct bnxt_mpc_info *mpc = softc->mpc_info;
> > + int dflt1, dflt2;
> > + int idx1, idx2;
> > + int min1, min2;
> > + int val1, val2;
> > +
> > + if (type == BNXT_MPC_CRYPTO) {
> > + min1 = BNXT_MIN_MPC_TCE;
> > + min2 = BNXT_MIN_MPC_RCE;
> > + dflt1 = BNXT_DFLT_MPC_TCE;
> > + dflt2 = BNXT_DFLT_MPC_RCE;
> > + idx1 = BNXT_MPC_TCE_TYPE;
> > + idx2 = BNXT_MPC_RCE_TYPE;
> > + } else {
> > + min1 = BNXT_MIN_MPC_TE_CFA;
> > + min2 = BNXT_MIN_MPC_RE_CFA;
> > + dflt1 = BNXT_DFLT_MPC_TE_CFA;
> > + dflt2 = BNXT_DFLT_MPC_RE_CFA;
> > + idx1 = BNXT_MPC_TE_CFA_TYPE;
> > + idx2 = BNXT_MPC_RE_CFA_TYPE;
> > + }
> > + if (*avail < (min1 + min2))
> > + return;
> > +
> > + val1 = min(*avail / 2, softc->ntxqsets);
> > + val2 = val1;
> > +
> > + val1 = min(val1, dflt1);
> > + val2 = min(val2, dflt2);
> > +
> > + if (avail_cp < min1 || avail_cp < min2)
> > + return;
> > +
> > + val1 = min(val1, avail_cp);
> > + val2 = min(val2, avail_cp);
> > +
> > + mpc->mpc_ring_count[idx1] = val1;
> > + mpc->mpc_ring_count[idx2] = val2;
> > +
> > + *avail = *avail - val1 - val2;
> > +}
> > +
> > +void bnxt_set_dflt_mpc_rings(struct bnxt_softc *softc)
> > +{
> > + struct bnxt_func_qcfg *fn_qcfg = &softc->fn_qcfg;
> > + struct bnxt_mpc_info *mpc = softc->mpc_info;
> > + int avail, mpc_cp, i;
> > + int avail_cp;
> > +
> > + if (!mpc)
> > + return;
> > +
> > + for (i = 0; i < BNXT_MPC_TYPE_MAX; i++)
> > + mpc->mpc_ring_count[i] = 0;
> > + mpc->mpc_cp_rings_count = 0;
> > +
> > + avail = fn_qcfg->orig_alloc_tx_rings - softc->ntxqsets;
> > +
> > + avail_cp = fn_qcfg->orig_alloc_completion_rings - softc->ntxqsets;
> > + avail_cp = min(avail_cp,
> > + ((pci_msix_count(softc->dev) - 1) - softc->ntxqsets));
> > +
> > + if (BNXT_MPC_CRYPTO_CAPABLE(softc))
> > + __bnxt_set_dflt_mpc_rings(softc, BNXT_MPC_CRYPTO, &avail,
> > + avail_cp);
> > +
> > + for (i = 0, mpc_cp = 0; i < BNXT_MPC_TYPE_MAX; i++) {
> > + if (mpc_cp < mpc->mpc_ring_count[i])
> > + mpc_cp = mpc->mpc_ring_count[i];
> > + }
> > + mpc->mpc_cp_rings_count = mpc_cp;
> > +}
> > +
> > +int bnxt_alloc_mpcs(struct bnxt_softc *softc)
> > +{
> > + struct bnxt_mpc_info *mpc = softc->mpc_info;
> > + int i;
> > +
> > + if (!BNXT_MPC_CRYPTO_CAPABLE(softc))
> > + return 0;
> > +
> > + for (i = 0; i < BNXT_MPC_TYPE_MAX; i++) {
> > + int num;
> > + struct bnxt_ring *txr;
> > +
> > + /* Currently only KTLS TX is supported */
> > + if (i != BNXT_MPC_TCE_TYPE)
> > + continue;
> > +
> > + num = mpc->mpc_ring_count[i];
> > + if (!num)
> > + continue;
> > + txr = malloc(num * sizeof(*txr), M_DEVBUF, M_NOWAIT |
> M_ZERO);
> > + if (!txr) {
> > + device_printf(softc->dev,
> > + "%s: Failed to allocate %d MPC rings\n",
> > + __func__, num);
> > + return ENOMEM;
> > + }
> > + mpc->mpc_rings[i] = txr;
> > + }
> > +
> > + mpc->mpc_cp_rings =
> > + malloc(mpc->mpc_cp_rings_count * sizeof(struct bnxt_cp_ring),
> > + M_DEVBUF, M_NOWAIT | M_ZERO);
> > + if (!mpc->mpc_cp_rings) {
> > + device_printf(softc->dev,
> > + "%s: Failed to allocate %d MPC completion rings\n",
> > + __func__, mpc->mpc_cp_rings_count);
> > + return ENOMEM;
> > + }
> > +
> > + mpc->mpc_nq_rings =
> > + malloc(mpc->mpc_cp_rings_count * sizeof(struct bnxt_cp_ring),
> > + M_DEVBUF, M_NOWAIT | M_ZERO);
> > + if (!mpc->mpc_nq_rings) {
> > + device_printf(softc->dev,
> > + "%s: Failed to allocate %d MPC notification rings\n",
> > + __func__, mpc->mpc_cp_rings_count);
> > + return ENOMEM;
> > + }
> > + return 0;
> > +}
> > +
> > +void bnxt_free_mpcs(struct bnxt_softc *softc)
> > +{
> > + struct bnxt_mpc_info *mpc = softc->mpc_info;
> > + int i;
> > +
> > + if (!mpc)
> > + return;
> > +
> > + for (i = 0; i < BNXT_MPC_TYPE_MAX; i++) {
> > + /* Currently only KTLS TX is supported */
> > + if (i != BNXT_MPC_TCE_TYPE)
> > + continue;
> > +
> > + if (mpc->mpc_rings[i]) {
> > + free(mpc->mpc_rings[i], M_DEVBUF);
> > + mpc->mpc_rings[i] = NULL;
> > + }
> > + }
> > +
> > + if (mpc->mpc_cp_rings) {
> > + free(mpc->mpc_cp_rings, M_DEVBUF);
> > + mpc->mpc_cp_rings = NULL;
> > + }
> > +
> > + if (mpc->mpc_nq_rings) {
> > + free(mpc->mpc_nq_rings, M_DEVBUF);
> > + mpc->mpc_nq_rings = NULL;
> > + }
> > +}
> > +
> > +int bnxt_alloc_mpc_rings(struct bnxt_softc *softc)
> > +{
> > + struct bnxt_mpc_info *mpc = softc->mpc_info;
> > + int i, j, mpc_idx = 0;
> > + int mpc_base_ring_id =
> > + (softc->scctx->isc_nrxqsets * 2) + 1 + softc->ntxqsets;
> > +
> > + if (!mpc)
> > + return 0;
> > +
> > + for (i = 0; i < mpc->mpc_cp_rings_count; i++) {
> > + struct bnxt_cp_ring *cpr = &mpc->mpc_cp_rings[i];
> > + int rc;
> > +
> > + cpr->ring.queue_id = BNXT_MPC_QUEUE_ID;
> > +
> > + /* Set up the completion ring */
> > + cpr->stats_ctx_id = HWRM_NA_SIGNATURE;
> > + cpr->ring.phys_id = (uint16_t)HWRM_NA_SIGNATURE;
> > + cpr->ring.softc = softc;
> > + cpr->ring.idx = i;
> > + cpr->ring.id = mpc_base_ring_id + i;
> > + cpr->ring.doorbell = softc->legacy_db_size;
> > + cpr->ring.ring_size = softc->scctx->isc_ntxd[0];
> > + cpr->ring.db_ring_mask = cpr->ring.ring_size - 1;
> > +
> > + rc = iflib_dma_alloc(softc->ctx,
> > + sizeof(struct cmpl_base) * cpr->ring.ring_size,
> > + &cpr->ring.ring_mem, 0);
> > + if (rc) {
> > + device_printf(softc->dev,
> > + "%s: Failed to allocate DMA memory for
> completion ring %d (size=%zu), rc=%d\n",
> > + __func__, i, sizeof(struct cmpl_base) *
> cpr->ring.ring_size, rc);
> > + return rc;
> > + }
> > +
> > + cpr->ring.vaddr = cpr->ring.ring_mem.idi_vaddr;
> > + cpr->ring.paddr = cpr->ring.ring_mem.idi_paddr;
> > + }
> > +
> > + for (i = 0; i < BNXT_MPC_TYPE_MAX; i++) {
> > + int num = mpc->mpc_ring_count[i], rc;
> > +
> > + /* Currently only KTLS TX is supported */
> > + if (i != BNXT_MPC_TCE_TYPE)
> > + continue;
> > +
> > + for (j = 0; j < num; j++) {
> > + struct bnxt_ring *txr = &mpc->mpc_rings[i][j];
> > + /* tx_stats[] is indexed across all MPC types, so
> use
> > + * the flat mpc_idx, not the intra-type counter j.
> */
> > + int stat_idx = mpc_idx;
> > +
> > + txr->queue_id = BNXT_MPC_QUEUE_ID;
> > + txr->mpc_chnl_type = i;
> > +
> > + /* alloc mpc TX ring */
> > + txr->phys_id = (uint16_t)HWRM_NA_SIGNATURE;
> > + txr->softc = softc;
> > + txr->idx = i << BNXT_RING_MPC_IDX_TYPE_SFT | j;
> > + txr->id = mpc_base_ring_id + mpc_idx++;
> > + txr->doorbell = softc->legacy_db_size;
> > + txr->ring_size = softc->scctx->isc_ntxd[1];
> > + txr->db_ring_mask = txr->ring_size - 1;
> > +
> > + /* Init before any early return:
> bnxt_free_mpc_rings()
> > + * unconditionally mtx_destroy()s every txr. */
> > + mtx_init(&txr->tx_lock, "MPC TX lock", NULL,
> MTX_SPIN);
> > +
> > + rc = iflib_dma_alloc(softc->ctx,
> > + sizeof(struct tx_bd_short) * txr->ring_size,
> > + &txr->ring_mem, 0);
> > + if (rc) {
> > + device_printf(softc->dev,
> > + "%s: Failed to allocate DMA memory for
> TX ring %d (size=%zu), rc=%d\n",
> > + __func__, j, sizeof(struct
> tx_bd_short) * txr->ring_size, rc);
> > + return rc;
> > + }
> > +
> > + txr->vaddr = txr->ring_mem.idi_vaddr;
> > + txr->paddr = txr->ring_mem.idi_paddr;
> > +
> > + txr->tx_mpc_buf_ring =
> > + malloc(txr->ring_size * sizeof(struct
> bnxt_sw_mpc_tx_bd),
> > + M_DEVBUF, M_NOWAIT | M_ZERO);
> > + if (!txr->tx_mpc_buf_ring) {
> > + device_printf(softc->dev,
> > + "%s: Failed to allocate buffer ring
> for TX ring %d (size=%zu)\n",
> > + __func__, j, txr->ring_size *
> sizeof(struct bnxt_sw_mpc_tx_bd));
> > + return ENOMEM;
> > + }
> > +
> > + rc = iflib_dma_alloc(softc->ctx, sizeof(struct
> ctx_hw_stats),
> > + &mpc->tx_stats[stat_idx], 0);
> > + if (rc) {
> > + device_printf(softc->dev,
> > + "%s: Failed to allocate DMA memory for
> TX stats ring %d, rc=%d\n",
> > + __func__, j, rc);
> > + return rc;
> > + }
> > + bus_dmamap_sync(mpc->tx_stats[stat_idx].idi_tag,
> > + mpc->tx_stats[stat_idx].idi_map,
> BUS_DMASYNC_PREREAD);
> > + }
> > + }
> > +
> > + /* Set up the Notification ring (NQ) */
> > + for (i = 0; i < mpc->mpc_cp_rings_count; i++) {
> > + struct bnxt_cp_ring *nqr = &mpc->mpc_nq_rings[i];
> > + int rc;
> > +
> > + nqr->ring.queue_id = BNXT_MPC_QUEUE_ID;
> > +
> > + nqr->stats_ctx_id = HWRM_NA_SIGNATURE;
> > + nqr->ring.phys_id = (uint16_t)HWRM_NA_SIGNATURE;
> > + nqr->ring.softc = softc;
> > + nqr->ring.idx = i;
> > + nqr->ring.id = softc->ntxqsets + i;
> > + nqr->ring.doorbell = softc->legacy_db_size;
> > + nqr->ring.ring_size = softc->scctx->isc_ntxd[2];
> > + nqr->ring.db_ring_mask = nqr->ring.ring_size - 1;
> > +
> > + rc = iflib_dma_alloc(softc->ctx,
> > + sizeof(struct cmpl_base) * nqr->ring.ring_size,
> > + &nqr->ring.ring_mem, 0);
> > + if (rc) {
> > + device_printf(softc->dev,
> > + "%s: Failed to allocate DMA memory for
> notification ring %d (size=%zu), rc=%d\n",
> > + __func__, i, sizeof(struct cmpl_base) *
> nqr->ring.ring_size, rc);
> > + return rc;
> > + }
> > +
> > + nqr->ring.vaddr = nqr->ring.ring_mem.idi_vaddr;
> > + nqr->ring.paddr = nqr->ring.ring_mem.idi_paddr;
> > + nqr->type = TX_CP_NQ;
> > + }
> > +
> > + return 0;
> > +}
> > +
> > +void bnxt_free_mpc_rings(struct bnxt_softc *softc)
> > +{
> > + struct bnxt_mpc_info *mpc = softc->mpc_info;
> > + int i, j, mpc_idx = 0;
> > +
> > + if (!mpc)
> > + return;
> > +
> > + /* A partial alloc failure can leave these NULL; guard each array.
> */
> > + if (mpc->mpc_nq_rings) {
> > + for (i = 0; i < mpc->mpc_cp_rings_count; i++)
> > +
> iflib_dma_free(&mpc->mpc_nq_rings[i].ring.ring_mem);
> > + }
> > +
> > + if (mpc->mpc_cp_rings) {
> > + for (i = 0; i < mpc->mpc_cp_rings_count; i++)
> > +
> iflib_dma_free(&mpc->mpc_cp_rings[i].ring.ring_mem);
> > + }
> > +
> > + for (i = 0; i < BNXT_MPC_TYPE_MAX; i++) {
> > + int num = mpc->mpc_ring_count[i];
> > +
> > + /* Currently only KTLS TX is supported */
> > + if (i != BNXT_MPC_TCE_TYPE)
> > + continue;
> > +
> > + /*
> > + * Same partial-allocation case as above:
> mpc_ring_count[i] can
> > + * be nonzero while mpc_rings[i] itself failed to allocate.
> > + */
> > + if (!mpc->mpc_rings[i])
> > + continue;
> > +
> > + for (j = 0; j < num; j++) {
> > + struct bnxt_ring *txr = &mpc->mpc_rings[i][j];
> > +
> > + /* TCE ring vaddr holds cleartext AES-GCM keys;
> scrub
> > + * before freeing the pages. */
> > + if (i == BNXT_MPC_TCE_TYPE &&
> txr->ring_mem.idi_vaddr)
> > + explicit_bzero(txr->ring_mem.idi_vaddr,
> > + txr->ring_mem.idi_size);
> > +
> > + iflib_dma_free(&txr->ring_mem);
> > + if (txr->tx_mpc_buf_ring) {
> > + free(txr->tx_mpc_buf_ring, M_DEVBUF);
> > + txr->tx_mpc_buf_ring = NULL;
> > + }
> > + /* A ring past a partial alloc failure never had
> its
> > + * tx_lock mtx_init()'d; guard the destroy. */
> > + if (mtx_initialized(&txr->tx_lock))
> > + mtx_destroy(&txr->tx_lock);
> > + /* Must match the flat stat_idx
> bnxt_alloc_mpc_rings() used. */
> > + iflib_dma_free(&mpc->tx_stats[mpc_idx++]);
> > + }
> > + }
> > +}
> > +
> > +void bnxt_clear_mpc_rings_ids(struct bnxt_softc *softc)
> > +{
> > + struct bnxt_mpc_info *mpc = softc->mpc_info;
> > + int i, j;
> > +
> > + if (!mpc)
> > + return;
> > +
> > + for (i = 0; i < mpc->mpc_cp_rings_count; i++) {
> > + mpc->mpc_nq_rings[i].ring.phys_id =
> (uint16_t)HWRM_NA_SIGNATURE;
> > + mpc->mpc_cp_rings[i].ring.phys_id =
> (uint16_t)HWRM_NA_SIGNATURE;
> > + mpc->mpc_cp_rings[i].stats_ctx_id = HWRM_NA_SIGNATURE;
> > + }
> > +
> > + for (i = 0; i < BNXT_MPC_TYPE_MAX; i++) {
> > + int num = mpc->mpc_ring_count[i];
> > +
> > + /* Currently only KTLS TX is supported */
> > + if (i != BNXT_MPC_TCE_TYPE)
> > + continue;
> > +
> > + for (j = 0; j < num; j++) {
> > + struct bnxt_ring *txr = &mpc->mpc_rings[i][j];
> > +
> > + txr->phys_id = (uint16_t)HWRM_NA_SIGNATURE;
> > + }
> > + }
> > +}
> > +
> > +static void bnxt_map_tx_to_cp(struct bnxt_softc *softc)
> > +{
> > + struct bnxt_mpc_info *mpc = softc->mpc_info;
> > + int cp_count = mpc->mpc_cp_rings_count;
> > + int i, j;
> > +
> > + for (i = 0; i < BNXT_MPC_TYPE_MAX; i++) {
> > + int num = mpc->mpc_ring_count[i];
> > +
> > + if (i != BNXT_MPC_TCE_TYPE)
> > + continue;
> > +
> > + for (j = 0; j < num; j++) {
> > + struct bnxt_ring *txr = &mpc->mpc_rings[i][j];
> > + int cp_idx = j % cp_count;
> > +
> > + txr->cp_ring = &mpc->mpc_cp_rings[cp_idx];
> > + }
> > + }
> > +}
> > +
> > +struct bnxt_ring *bnxt_select_mpc_ring(struct bnxt_mpc_info *mpc)
> > +{
> > + int idx;
> > +
> > + /* Callers should gate on a non-zero TCE ring count already, but
> > + * guard the divide/index here too since this feeds a live TX
> path. */
> > + if (!mpc || !mpc->mpc_ring_count[BNXT_MPC_TCE_TYPE])
> > + return NULL;
> > +
> > + idx = PCPU_GET(cpuid) % mpc->mpc_ring_count[BNXT_MPC_TCE_TYPE];
> > +
> > + return &mpc->mpc_rings[BNXT_MPC_TCE_TYPE][idx];
> > +}
> > +
> > +int bnxt_hwrm_mpc_ring_alloc(struct bnxt_softc *softc)
> > +{
> > + struct bnxt_mpc_info *mpc = softc->mpc_info;
> > + int i, j, rc;
> > +
> > + if (!mpc)
> > + return 0;
> > +
> > + bnxt_map_tx_to_cp(softc);
> > + for (i = 0; i < mpc->mpc_cp_rings_count; i++) {
> > + struct bnxt_cp_ring *nqr = &mpc->mpc_nq_rings[i];
> > +
> > + nqr->cons = 0;
> > + nqr->v_bit = 1;
> > + nqr->raw_cons = 0;
> > + nqr->toggle = 0;
> > + nqr->last_idx = UINT32_MAX;
> > + bnxt_mark_cpr_invalid(nqr);
> > +
> > + rc = bnxt_hwrm_ring_alloc(
> > + softc, HWRM_RING_ALLOC_INPUT_RING_TYPE_NQ, &nqr->ring);
> > +
> > + bnxt_set_db_mask(softc, &nqr->ring,
> > + HWRM_RING_ALLOC_INPUT_RING_TYPE_NQ);
> > + if (rc) {
> > + device_printf(softc->dev,
> > + "%s: Failed to allocate HWRM notification ring
> %d, rc=%d\n",
> > + __func__, i, rc);
> > + return rc;
> > + }
> > +
> > + softc->db_ops.bnxt_db_nq(nqr, 1);
> > + }
> > +
> > + for (i = 0; i < mpc->mpc_cp_rings_count; i++) {
> > + struct bnxt_cp_ring *cpr = &mpc->mpc_cp_rings[i];
> > +
> > + rc = bnxt_hwrm_stat_ctx_alloc(softc, cpr,
> > + mpc->tx_stats[i].idi_paddr);
> > + if (rc) {
> > + device_printf(softc->dev,
> > + "%s: Failed to allocate HWRM stats context for
> completion ring %d, rc=%d\n",
> > + __func__, i, rc);
> > + return rc;
> > + }
> > +
> > + cpr->v_bit = 1;
> > + cpr->cons = 0;
> > + cpr->raw_cons = 0;
> > + cpr->toggle = 0;
> > + bnxt_mark_cpr_invalid(cpr);
> > + rc = bnxt_hwrm_ring_alloc(
> > + softc, HWRM_RING_ALLOC_INPUT_RING_TYPE_L2_CMPL,
> &cpr->ring);
> > + bnxt_set_db_mask(softc, &cpr->ring,
> > + HWRM_RING_ALLOC_INPUT_RING_TYPE_L2_CMPL);
> > + if (rc) {
> > + device_printf(softc->dev,
> > + "%s: Failed to allocate HWRM completion ring
> %d, rc=%d\n",
> > + __func__, i, rc);
> > + return rc;
> > + }
> > +
> > + softc->db_ops.bnxt_db_tx_cq(cpr, 1);
> > + }
> > +
> > + for (i = 0; i < BNXT_MPC_TYPE_MAX; i++) {
> > + int num = mpc->mpc_ring_count[i];
> > +
> > + /* Currently only KTLS TX is supported */
> > + if (i != BNXT_MPC_TCE_TYPE)
> > + continue;
> > +
> > + for (j = 0; j < num; j++) {
> > + struct bnxt_ring *txr = &mpc->mpc_rings[i][j];
> > +
> > + txr->prod = 0;
> > + txr->cons = 0;
> > + memset(txr->tx_mpc_buf_ring, 0,
> > + txr->ring_size * sizeof(struct
> bnxt_sw_mpc_tx_bd));
> > +
> > + rc = bnxt_hwrm_ring_alloc(
> > + softc, HWRM_RING_ALLOC_INPUT_RING_TYPE_TX,
> txr);
> > + bnxt_set_db_mask(softc, txr,
> > + HWRM_RING_ALLOC_INPUT_RING_TYPE_TX);
> > + if (rc) {
> > + device_printf(softc->dev,
> > + "%s: Failed to allocate HWRM TX ring
> %d, rc=%d\n",
> > + __func__, j, rc);
> > + return rc;
> > + }
> > + softc->db_ops.bnxt_db_tx(txr, 0);
> > + }
> > + }
> > + return 0;
> > +}
> > +
> > +int bnxt_hwrm_mpc_ring_free(struct bnxt_softc *softc)
> > +{
> > + struct bnxt_mpc_info *mpc = softc->mpc_info;
> > + int i, j, rc;
> > +
> > + if (!mpc)
> > + return 0;
> > +
> > + for (i = 0; i < BNXT_MPC_TYPE_MAX; i++) {
> > + int num = mpc->mpc_ring_count[i];
> > +
> > + /* Currently only KTLS TX is supported */
> > + if (i != BNXT_MPC_TCE_TYPE)
> > + continue;
> > +
> > + for (j = 0; j < num; j++) {
> > + struct bnxt_ring *txr = &mpc->mpc_rings[i][j];
> > + struct bnxt_cp_ring *cpr = txr->cp_ring;
> > +
> > + if (!cpr)
> > + continue;
> > +
> > + rc = bnxt_hwrm_ring_free(
> > + softc, HWRM_RING_ALLOC_INPUT_RING_TYPE_TX, txr,
> > + cpr->ring.phys_id);
> > + if (rc) {
> > + device_printf(softc->dev,
> > + "%s: Failed to free HWRM TX ring %d
> (ring_id=%d, phys_id=%d), rc=%d\n",
> > + __func__, j, txr->id,
> cpr->ring.phys_id, rc);
> > + return rc;
> > + }
> > + txr->phys_id = (uint16_t)HWRM_NA_SIGNATURE;
> > + }
> > + }
> > +
> > + for (i = 0; i < mpc->mpc_cp_rings_count; i++) {
> > + struct bnxt_cp_ring *cpr = &mpc->mpc_cp_rings[i];
> > +
> > + rc = bnxt_hwrm_ring_free(
> > + softc, HWRM_RING_ALLOC_INPUT_RING_TYPE_L2_CMPL,
> > + &cpr->ring, (uint16_t)HWRM_NA_SIGNATURE);
> > + if (rc) {
> > + device_printf(softc->dev,
> > + "%s: Failed to free HWRM completion ring %d
> (ring_id=%d, phys_id=%d), rc=%d\n",
> > + __func__, i, cpr->ring.id, cpr->ring.phys_id,
> rc);
> > + return rc;
> > + }
> > +
> > + rc = bnxt_hwrm_stat_ctx_free(softc, cpr);
> > + if (rc) {
> > + device_printf(softc->dev,
> > + "%s: Failed to free HWRM stats context id %d
> for completion ring %d, rc=%d\n",
> > + __func__, cpr->stats_ctx_id, i, rc);
> > + return rc;
> > + }
> > +
> > + cpr->ring.phys_id = (uint16_t)HWRM_NA_SIGNATURE;
> > + }
> > +
> > + for (i = 0; i < mpc->mpc_cp_rings_count; i++) {
> > + struct bnxt_cp_ring *nqr = &mpc->mpc_nq_rings[i];
> > +
> > + rc = bnxt_hwrm_ring_free(
> > + softc, HWRM_RING_ALLOC_INPUT_RING_TYPE_NQ, &nqr->ring,
> > + (uint16_t)HWRM_NA_SIGNATURE);
> > + if (rc) {
> > + device_printf(softc->dev,
> > + "%s: Failed to free HWRM notification ring %d
> (ring_id=%d, phys_id=%d), rc=%d\n",
> > + __func__, i, nqr->ring.id, nqr->ring.phys_id,
> rc);
> > + return rc;
> > + }
> > + nqr->ring.phys_id = (uint16_t)HWRM_NA_SIGNATURE;
> > + }
> > + return 0;
> > +}
> > +
> > +static void bnxt_mpc_process_cpr(struct bnxt_cp_ring *cpr)
> > +{
> > + struct bnxt_softc *softc = cpr->ring.softc;
> > +
> > + /* Handle completions on the mpc completion ring */
> > + uint32_t raw_cons = cpr->raw_cons;
> > + uint32_t cons = cpr->cons;
> > + struct cmpl_base *cmpl;
> > + bool v_bit = cpr->v_bit;
> > + bool last_v_bit;
> > + uint32_t last_cons;
> > + uint16_t type;
> > +
> > + for (;;) {
> > + last_cons = cons;
> > + last_v_bit = v_bit;
> > + cmpl = &((struct cmpl_base *)cpr->ring.vaddr)[cons];
> > +
> > + if (!CMP_VALID(cmpl, v_bit))
> > + break;
> > +
> > + type = le16toh(cmpl->type) & CMPL_BASE_TYPE_MASK;
> > + switch (type) {
> > + case CMPL_BASE_TYPE_MID_PATH_SHORT:
> > + case CMPL_BASE_TYPE_MID_PATH_LONG: {
> > + int rc = bnxt_mpc_cmp(softc, cpr, &raw_cons);
> > +
> > + /* EBUSY: 2nd half of a MID_PATH_LONG completion
> hasn't
> > + * landed yet; stop and retry later. Any other
> error is
> > + * non-transient and already logged/skipped, so
> fall
> > + * through and advance like the success case. */
> > + if (rc == EBUSY)
> > + goto done;
> > + break;
> > + }
> > + case CMPL_BASE_TYPE_HWRM_DONE:
> > + /* ignore this completion */
> > + break;
> > + default: {
> > + /* Rate-limit prints so a stuck ring feeding
> > + * unrecognized types can't flood the console.
> > + * Statics are unlocked; a best-effort limiter. */
> > + static struct timeval last;
> > + static int curpps;
> > +
> > + if (ppsratecheck(&last, &curpps, 5))
> > + device_printf(softc->dev,
> > + "%s: Unknown completion type %u on
> ring %d (raw_cons=%u, cons=%u)\n",
> > + __func__, type, cpr->ring.idx,
> raw_cons, cons);
> > + break;
> > + }
> > + }
> > + raw_cons++;
> > + /* Recompute from raw_cons rather than NEXT_CP_CONS_V(): a
> > + * MID_PATH_LONG completion can wrap raw_cons inside
> > + * bnxt_mpc_cmp(), which would desync v_bit otherwise. */
> > + cons = MPC_RING_CMP(cpr, raw_cons);
> > + v_bit = !(raw_cons & cpr->ring.ring_size);
> > + }
> > +done:
> > + cpr->cons = last_cons;
> > + cpr->raw_cons = raw_cons;
> > + cpr->v_bit = last_v_bit;
> > + softc->db_ops.bnxt_db_tx_cq(cpr, 1);
> > +}
> > +
> > +static void bnxt_mpc_irq(void *arg)
> > +{
> > + struct bnxt_cp_ring *cpr = arg;
> > + struct bnxt_softc *softc = cpr->ring.softc;
> > + nq_cn_t *cmp = (nq_cn_t *)cpr->ring.vaddr;
> > + uint32_t raw_cons = cpr->raw_cons;
> > + uint16_t nq_type, nqe_cnt = 0;
> > + uint32_t cons = cpr->cons;
> > + bool v_bit = cpr->v_bit;
> > +
> > + while (1) {
> > + if (!NQ_VALID(&cmp[cons], v_bit))
> > + goto done;
> > +
> > + nq_type = NQ_CN_TYPE_MASK & cmp[cons].type;
> > +
> > + if (NQE_CN_TYPE(nq_type) == NQ_CN_TYPE_CQ_NOTIFICATION) {
> > *** 817 LINES SKIPPED ***
> This causes build failure on powerpc because they don't have 16-bit
> atomic operations. Can we switch to atomic_*_32 and relevant variable to
> 32-bit?
I will look into it.
Thanks,
Sumit
>
>
> --
> Minsoo Choo
>
>
>