git: 70d389bede66 - main - f_bnxt: add kTLS (kernel TLS) TX offload
- Go to: [ bottom of page ] [ top of archives ] [ this month ]
Date: Fri, 09 Oct 2026 10:12:41 UTC
The branch main has been updated by ssaxena:
URL: https://cgit.FreeBSD.org/src/commit/?id=70d389bede66da585508c1813c10a873d42612e7
commit 70d389bede66da585508c1813c10a873d42612e7
Author: Sumit Saxena <sumit.saxena@broadcom.com>
AuthorDate: 2026-07-29 12:16:07 +0000
Commit: Sumit Saxena <ssaxena@FreeBSD.org>
CommitDate: 2026-10-09 08:24:43 +0000
f_bnxt: add kTLS (kernel TLS) TX offload
Add kTLS TX crypto offload, layered on MPC (crypto add/delete
commands submitted over its TCE channel) and TX completion coalescing
(shared tx_bd_opaque encoding).
Adds kTLS session lifecycle management, a key-context ID allocator,
HWRM key-context alloc/free with partition-mode support, and the TX
submit path: in-order packets go out on the normal iflib path with
CSUM_SND_TAG set, while out-of-order or retransmitted TLS records get
a presync command plus a replayed copy of the record built from the
driver's own replay ring. Also adds kTLS/MPC completion-time counter
sysctls and the max_ktls_entries tunable.
Reviewed by: gallatin
MFC after: 2 weeks
Sponsored by: Broadcom Inc.
Differential Revision: https://reviews.freebsd.org/D58606
---
sys/conf/files | 1 +
sys/dev/bnxt/bnxt_en/bnxt.h | 77 +-
sys/dev/bnxt/bnxt_en/bnxt_hwrm.c | 58 +-
sys/dev/bnxt/bnxt_en/bnxt_ktls.c | 2233 ++++++++++++++++++++++++++++++++++++
sys/dev/bnxt/bnxt_en/bnxt_ktls.h | 421 +++++++
sys/dev/bnxt/bnxt_en/bnxt_sysctl.c | 218 ++++
sys/dev/bnxt/bnxt_en/bnxt_sysctl.h | 3 +
sys/dev/bnxt/bnxt_en/bnxt_txrx.c | 64 +-
sys/dev/bnxt/bnxt_en/if_bnxt.c | 180 ++-
sys/modules/bnxt/bnxt_en/Makefile | 3 +-
10 files changed, 3246 insertions(+), 12 deletions(-)
diff --git a/sys/conf/files b/sys/conf/files
index 0a6e5874896f..ee56f70e706b 100644
--- a/sys/conf/files
+++ b/sys/conf/files
@@ -1334,6 +1334,7 @@ dev/bnxt/bnxt_en/bnxt_log_data.c optional bnxt iflib pci compile-with "${BNXT_C}
dev/bnxt/bnxt_en/bnxt_coredump.c optional bnxt iflib pci compile-with "${BNXT_C}"
dev/bnxt/bnxt_en/bnxt_ptp.c optional bnxt iflib pci compile-with "${BNXT_C}"
dev/bnxt/bnxt_en/bnxt_mpc.c optional bnxt iflib pci compile-with "${BNXT_C}"
+dev/bnxt/bnxt_en/bnxt_ktls.c optional bnxt iflib pci compile-with "${BNXT_C}"
dev/bnxt/bnxt_en/if_bnxt.c optional bnxt iflib pci compile-with "${BNXT_C}"
dev/bwi/bwimac.c optional bwi
dev/bwi/bwiphy.c optional bwi
diff --git a/sys/dev/bnxt/bnxt_en/bnxt.h b/sys/dev/bnxt/bnxt_en/bnxt.h
index 7b03aa1da866..f0b9d114ad72 100644
--- a/sys/dev/bnxt/bnxt_en/bnxt.h
+++ b/sys/dev/bnxt/bnxt_en/bnxt.h
@@ -238,6 +238,11 @@
#define BNXT_MAX_NUM_QUEUES (BNXT_MAX_L2_QUEUES + BNXT_ROCE_IRQ_COUNT)
+#define BNXT_DEBUG(dev, fmt, ...) do { \
+ if (bootverbose) \
+ device_printf(dev, "" fmt, ##__VA_ARGS__); \
+} while (0)
+
/* Completion related defines */
#define CMP_VALID(cmp, v_bit) \
((!!(((struct cmpl_base *)(cmp))->info3_v & htole32(CMPL_BASE_V))) == !!(v_bit) )
@@ -322,6 +327,7 @@
(offsetof(struct tx_port_stats_ext, counter) / 8)
extern const char bnxt_driver_version[];
+extern const uint16_t bnxt_tx_lhint[];
typedef void (*bnxt_doorbell_tx)(void *, uint16_t idx);
typedef void (*bnxt_doorbell_rx)(void *, uint16_t idx);
typedef void (*bnxt_doorbell_rx_cq)(void *, bool);
@@ -609,11 +615,47 @@ struct tx_bd_opaque {
#define TX_RING_MASK(txr) ((txr)->ring_size - 1)
+#define SET_TX_OPAQUE_KTLS(opq, txr, replay, prod, segs) \
+ do { \
+ opq->ktls_replay = (replay); \
+ opq->idx = (prod) & TX_RING_MASK(txr); \
+ opq->bds = (segs); \
+ } while(0)
+
+#define KID_HIGH(kid) ((kid & 0x000fff80) >> 7) /*Upper 13 bits of KID*/
+#define KID_LOW(kid) (kid & 0x0000007f) /*Lower 7 bits of KID*/
+#define TX_BD_FLAGS_CRYPTO_EN (1 << 15)
+
+/* Hand-rolled generic BD views used only for byte-addressable copies of
+ * the kTLS presync command (see bnxt_ktls_pre_xmit()); distinct from the
+ * full HSI tx_bd_short/tx_bd_long types used for normal TX BDs. */
+struct tx_bd {
+ uint32_t tx_bd_len_flags_type;
+#define TX_BD_TYPE (0x3f << 0)
+#define TX_BD_FLAGS_BD_CNT_SHIFT 8
+#define TX_BD_LEN_SHIFT 16
+ uint32_t tx_bd_opaque;
+ uint64_t tx_bd_haddr;
+} __packed;
+
+struct tx_bd_presync {
+ uint32_t tx_bd_len_flags_type;
+#define TX_BD_TYPE_PRESYNC_TX_BD (0x09 << 0)
+ uint32_t tx_bd_opaque;
+ uint32_t tx_bd_kid;
+ uint32_t tx_bd_unused;
+} __packed;
+
struct bnxt_sw_mpc_tx_bd {
uint8_t inline_bds;
unsigned long handle;
};
+struct bnxt_sw_tx_bd {
+ uint8_t is_replay;
+ uint8_t inline_bds;
+};
+
struct bnxt_ring {
uint64_t paddr;
vm_offset_t doorbell;
@@ -632,7 +674,11 @@ struct bnxt_ring {
uint16_t prod; /* MPC TX ring producer index */
uint16_t cons; /* MPC TX ring consumer index */
struct mtx tx_lock; /* MPC TX ring lock */
- struct bnxt_sw_mpc_tx_bd *tx_mpc_buf_ring;
+ union {
+ struct bnxt_sw_tx_bd *tx_buf_ring;
+ struct bnxt_sw_mpc_tx_bd *tx_mpc_buf_ring;
+ };
+ struct bnxt_replay_pkt *replay_pkt;
struct bnxt_full_tpa_start *tpa_start;
union {
u64 db_key64;
@@ -865,6 +911,7 @@ struct bnxt_ctx_mem_type {
#define BNXT_CTX_FTQM HWRM_FUNC_BACKING_STORE_CFG_V2_INPUT_TYPE_FP_TQM_RING
#define BNXT_CTX_MRAV HWRM_FUNC_BACKING_STORE_CFG_V2_INPUT_TYPE_MRAV
#define BNXT_CTX_TIM HWRM_FUNC_BACKING_STORE_CFG_V2_INPUT_TYPE_TIM
+#define BNXT_CTX_TCK HWRM_FUNC_BACKING_STORE_CFG_V2_INPUT_TYPE_TX_CK
#define BNXT_CTX_TKC HWRM_FUNC_BACKING_STORE_CFG_V2_INPUT_TYPE_TKC
#define BNXT_CTX_RKC HWRM_FUNC_BACKING_STORE_CFG_V2_INPUT_TYPE_RKC
#define BNXT_CTX_MTQM HWRM_FUNC_BACKING_STORE_CFG_V2_INPUT_TYPE_MP_TQM_RING
@@ -914,6 +961,20 @@ struct bnxt_ctx_mem_info {
struct bnxt_ctx_mem_type ctx_arr[BNXT_CTX_V2_MAX];
};
+enum bnxt_tls_rtpe {
+ BNXT_CRYPTO_TYPE_KTLS = 0,
+ BNXT_CRYPTO_TYPE_QUIC,
+};
+
+struct bnxt_hw_tls_resc {
+ uint32_t min_tx_key_ctxs;
+ uint32_t max_tx_key_ctxs;
+ uint32_t resv_tx_key_ctxs;
+ uint32_t min_rx_key_ctxs;
+ uint32_t max_rx_key_ctxs;
+ uint32_t resv_rx_key_ctxs;
+};
+
struct bnxt_hw_resc {
uint16_t min_rsscos_ctxs;
uint16_t max_rsscos_ctxs;
@@ -941,6 +1002,7 @@ struct bnxt_hw_resc {
uint16_t max_nqs;
uint16_t max_irqs;
uint16_t resv_irqs;
+ struct bnxt_hw_tls_resc tls_resc[2];
};
enum bnxt_type_ets {
@@ -1354,6 +1416,7 @@ struct bnxt_softc {
#define BNXT_FW_CAP_CFA_NTUPLE_RX_EXT_IP_PROTO BIT_ULL(47)
#define BNXT_FW_CAP_ENABLE_RDMA_SRIOV BIT_ULL(48)
#define BNXT_FW_CAP_RSS_TCAM BIT_ULL(49)
+ #define BNXT_FW_CAP_KTLS_SUPPORTED BIT_ULL(50)
#define BNXT_FLAG_TX_COAL_CMPL BIT_ULL(51)
#define BNXT_FW_CAP_SW_MAX_RESOURCE_LIMITS BIT_ULL(61)
@@ -1459,6 +1522,12 @@ struct bnxt_softc {
bool rx_ts_enabled;
struct bnxt_mpc_info *mpc_info;
+ struct bnxt_tls_info *ktls_info;
+ struct sysctl_ctx_list ktls_stats;
+ struct sysctl_oid *ktls_stats_oid;
+ struct sysctl_ctx_list mpc_cmp_time_stats;
+ struct sysctl_oid *mpc_cmp_time_stats_oid;
+ uint32_t max_ktls_entries;
#define BNXT_CAGR_CQCOAL_OFFSET 0xc00
#define BNXT_CAGR_NQAGG_MAXTIMER 0x5930000
@@ -1604,4 +1673,10 @@ bnxt_test_and_clear_bit_nonatomic(int nr, bitstr_t *bmap)
bit_clear(bmap, nr);
return (1);
}
+
+int bnxt_tls_snd_tag_alloc(if_t ifp,
+ union if_snd_tag_alloc_params *params,
+ struct m_snd_tag **ppmt);
+int bnxt_ktls_xmit(struct bnxt_softc* softc, struct bnxt_ring* txr,
+ struct mbuf** ppmb, uint16_t* lflags, uint32_t* kid, if_pkt_info_t pi);
#endif /* _BNXT_H */
diff --git a/sys/dev/bnxt/bnxt_en/bnxt_hwrm.c b/sys/dev/bnxt/bnxt_en/bnxt_hwrm.c
index fb549bc05c10..57af44ec38c5 100644
--- a/sys/dev/bnxt/bnxt_en/bnxt_hwrm.c
+++ b/sys/dev/bnxt/bnxt_en/bnxt_hwrm.c
@@ -30,9 +30,12 @@
#include <sys/endian.h>
#include <linux/pci.h>
+#include "opt_kern_tls.h"
+
#include "bnxt.h"
#include "bnxt_hwrm.h"
#include "bnxt_mpc.h"
+#include "bnxt_ktls.h"
#include "hsi_struct_def.h"
#include "bnxt_coredump.h"
@@ -789,11 +792,21 @@ int bnxt_hwrm_func_resc_qcaps(struct bnxt_softc *softc, bool all)
hw_resc->min_stat_ctxs = le16toh(resp->min_stat_ctx);
hw_resc->max_stat_ctxs = le16toh(resp->max_stat_ctx);
+ {
+ struct bnxt_hw_tls_resc *tls_resc =
+ &hw_resc->tls_resc[BNXT_CRYPTO_TYPE_KTLS];
+
+ tls_resc->min_tx_key_ctxs = le32toh(resp->min_ktls_tx_key_ctxs);
+ tls_resc->max_tx_key_ctxs = le32toh(resp->max_ktls_tx_key_ctxs);
+ tls_resc->min_rx_key_ctxs = le32toh(resp->min_ktls_rx_key_ctxs);
+ tls_resc->max_rx_key_ctxs = le32toh(resp->max_ktls_rx_key_ctxs);
+ }
+
if (BNXT_CHIP_P5_PLUS(softc)) {
hw_resc->max_nqs = hw_resc->max_irqs = le16toh(resp->max_msix);
hw_resc->max_hw_ring_grps = hw_resc->max_rx_rings;
}
-
+
if (BNXT_PF(softc)) {
struct bnxt_pf_info *pf = &softc->pf;
@@ -1206,6 +1219,11 @@ bnxt_hwrm_func_qcaps(struct bnxt_softc *softc)
(void *)softc->hwrm_cmd_resp.idi_vaddr;
struct bnxt_func_info *func = &softc->func;
uint32_t flags, flags_ext, flags_ext2;
+#if defined(KERN_TLS) && defined(KTLS_IFLIB_SUPPORT)
+ bool ktls_fw_supported = false;
+ uint16_t ktls_max_keys = 0, ktls_ctxs_per_partition = 0;
+ bool ktls_partition_cap = false;
+#endif
uint8_t mpc_chnls_cap = 0;
bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_FUNC_QCAPS);
@@ -1259,6 +1277,16 @@ bnxt_hwrm_func_qcaps(struct bnxt_softc *softc)
if (BNXT_PF(softc) && (flags_ext & HWRM_FUNC_QCAPS_OUTPUT_FLAGS_EXT_ECN_STATS_SUPPORTED))
softc->fw_cap |= BNXT_FW_CAP_ECN_STATS;
+#if defined(KERN_TLS) && defined(KTLS_IFLIB_SUPPORT)
+ if (flags_ext & HWRM_FUNC_QCAPS_OUTPUT_FLAGS_EXT_KTLS_SUPPORTED) {
+ softc->fw_cap |= BNXT_FW_CAP_KTLS_SUPPORTED;
+ /* Actual alloc/free is deferred until after BNXT_HWRM_UNLOCK(); it can sleep. */
+ ktls_fw_supported = true;
+ ktls_max_keys = le16toh(resp->max_key_ctxs_alloc);
+ ktls_partition_cap = BNXT_PARTITION_CAP(resp);
+ ktls_ctxs_per_partition = le16toh(resp->ctxs_per_partition);
+ }
+#endif
if (flags_ext & HWRM_FUNC_QCAPS_OUTPUT_FLAGS_EXT_PTP_PPS_SUPPORTED)
softc->fw_cap |= BNXT_FW_CAP_PTP_PPS;
if (flags_ext & HWRM_FUNC_QCAPS_OUTPUT_FLAGS_EXT_PTP_PTM_SUPPORTED)
@@ -1364,10 +1392,27 @@ bnxt_hwrm_func_qcaps(struct bnxt_softc *softc)
if (BNXT_PF(softc))
mpc_chnls_cap = resp->mpc_chnls_cap;
-fail:
BNXT_HWRM_UNLOCK(softc);
- if (!rc && BNXT_PF(softc))
+ if (BNXT_PF(softc))
bnxt_alloc_mpc_info(softc, mpc_chnls_cap);
+#if defined(KERN_TLS) && defined(KTLS_IFLIB_SUPPORT)
+ if (ktls_fw_supported) {
+ bnxt_alloc_ktls_info(softc, ktls_max_keys, ktls_partition_cap,
+ ktls_ctxs_per_partition);
+ /*
+ * HW Tx completion coalescing is not used with HW KTLS; the
+ * driver expects ordinary per-packet TX completions on that path.
+ */
+ if (softc->ktls_info != NULL)
+ softc->fw_cap &= ~BNXT_FLAG_TX_COAL_CMPL;
+ } else {
+ bnxt_free_ktls_info(softc);
+ }
+#endif
+ return rc;
+
+fail:
+ BNXT_HWRM_UNLOCK(softc);
return rc;
}
@@ -1461,6 +1506,8 @@ bnxt_hwrm_func_qcfg(struct bnxt_softc *softc)
struct hwrm_func_qcfg_output *resp =
(void *)softc->hwrm_cmd_resp.idi_vaddr;
struct bnxt_func_qcfg *fn_qcfg = &softc->fn_qcfg;
+ struct bnxt_hw_resc *hw_resc = &softc->hw_resc;
+ struct bnxt_hw_tls_resc *tls_resc;
bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_FUNC_QCFG);
req.fid = htole16(0xffff);
@@ -1535,6 +1582,10 @@ bnxt_hwrm_func_qcfg(struct bnxt_softc *softc)
if (flags & HWRM_FUNC_QCFG_OUTPUT_FLAGS_ENABLE_RDMA_SRIOV)
softc->fw_cap |= BNXT_FW_CAP_ENABLE_RDMA_SRIOV;
+ tls_resc = &hw_resc->tls_resc[BNXT_CRYPTO_TYPE_KTLS];
+ tls_resc->resv_tx_key_ctxs = le32toh(resp->num_ktls_tx_key_ctxs);
+ tls_resc->resv_rx_key_ctxs = le32toh(resp->num_ktls_rx_key_ctxs);
+
if (softc->db_size)
goto end;
@@ -2486,6 +2537,7 @@ bnxt_hwrm_reserve_pf_rings(struct bnxt_softc *softc)
req.num_rx_rings = htole16(BNXT_MAX_NUM_QUEUES);
req.num_vnics = htole16(BNXT_MAX_NUM_QUEUES);
req.num_stat_ctxs = htole16(BNXT_MAX_NUM_QUEUES * 2);
+ bnxt_hwrm_reserve_pf_key_ctxs(softc, &req, BNXT_CRYPTO_TYPE_KTLS);
return hwrm_send_message(softc, &req, sizeof(req));
}
diff --git a/sys/dev/bnxt/bnxt_en/bnxt_ktls.c b/sys/dev/bnxt/bnxt_en/bnxt_ktls.c
new file mode 100644
index 000000000000..0578de559910
--- /dev/null
+++ b/sys/dev/bnxt/bnxt_en/bnxt_ktls.c
@@ -0,0 +1,2233 @@
+/*
+ * Broadcom NetXtreme-C/E network driver.
+ *
+ * Copyright (c) 2026 Broadcom, All Rights Reserved.
+ * The term Broadcom refers to Broadcom Limited and/or its subsidiaries
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+#include <sys/param.h>
+#include <sys/stdatomic.h>
+#include <sys/ktls.h>
+#include <sys/malloc.h>
+#include <sys/socket.h>
+#include <sys/socketvar.h>
+#include <sys/kernel.h>
+#include <sys/counter.h>
+#include <sys/kthread.h>
+#include <sys/bitstring.h>
+#include <sys/queue.h>
+#include <machine/atomic.h>
+#include <vm/uma.h>
+
+#include <net/if.h>
+#include <net/ethernet.h>
+#include <net/pfil.h>
+
+#include <netinet/in.h>
+#include <netinet/ip.h>
+#include <netinet/udp.h>
+#include <netinet/ip6.h>
+#include <netinet/tcp.h>
+#include <netinet/tcp_lro.h>
+#include <netinet/in_pcb.h>
+#include <netinet/tcp_var.h>
+
+#include "opt_kern_tls.h"
+#include "opt_rss.h"
+#include "opt_ratelimit.h"
+
+#include "bnxt.h"
+#include "bnxt_hwrm.h"
+#include "bnxt_ktls.h"
+#include "bnxt_mpc.h"
+
+#include <opencrypto/cryptodev.h>
+
+#ifdef KERN_TLS
+
+static if_snd_tag_free_t bnxt_tls_snd_tag_free;
+
+static const struct if_snd_tag_sw bnxt_tls_snd_tag_sw = {
+ .snd_tag_free = bnxt_tls_snd_tag_free,
+ .type = IF_SND_TAG_TYPE_TLS
+};
+static int bnxt_ktls_dev_add(struct bnxt_softc* softc,
+ const struct tls_session_params* en,
+ struct bnxt_ktls_offload_ctx_tx* kctx_tx);
+static void bnxt_ktls_dev_del(struct bnxt_softc* softc,
+ struct bnxt_ktls_offload_ctx_tx* kctx_tx);
+static int bnxt_crypto_del(struct bnxt_softc* softc,
+ struct bnxt_ktls_offload_ctx_tx* kctx_tx,
+ uint8_t type);
+static void bnxt_tls_work(void *context, int pending);
+static inline void
+bnxt_calc_mpc_cmp_time(struct bnxt_softc* softc, unsigned long rtt);
+
+static int
+bnxt_tls_tag_import(void *arg, void **store, int cnt, int domain, int flags)
+{
+ struct bnxt_ktls_offload_ctx_tx *kctx_tx;
+ int i;
+
+ for (i = 0; i != cnt; i++) {
+ kctx_tx = malloc(sizeof(*kctx_tx), M_DEVBUF, flags | M_ZERO);
+ if (kctx_tx == NULL)
+ return (i);
+ TASK_INIT(&kctx_tx->work_task, 0, bnxt_tls_work, kctx_tx);
+ store[i] = kctx_tx;
+ }
+ return (i);
+}
+
+static void
+bnxt_tls_tag_release(void *arg, void **store, int cnt)
+{
+ struct bnxt_ktls_offload_ctx_tx *kctx_tx;
+ if_ctx_t ctx;
+ struct bnxt_softc* priv;
+ int i;
+
+ for (i = 0; i != cnt; i++) {
+ kctx_tx = store[i];
+ if (kctx_tx->tag.ifp) {
+ ctx = if_getsoftc(kctx_tx->tag.ifp);
+ priv = iflib_get_softc(ctx);
+ if (priv->ktls_info->tq)
+ taskqueue_drain(priv->ktls_info->tq, &kctx_tx->work_task);
+ }
+ free(kctx_tx, M_DEVBUF);
+ }
+}
+
+static void
+bnxt_tls_tag_zfree(struct bnxt_ktls_offload_ctx_tx *kctx_tx)
+{
+ /* make sure any unhandled taskqueue events are ignored */
+ kctx_tx->state = BNXT_TLS_STATE_FREED;
+
+ /* avoid leaking keys */
+ memset(&kctx_tx->crypto_info, 0, sizeof(kctx_tx->crypto_info));
+
+ /* return tag to UMA */
+ uma_zfree(kctx_tx->ktls->zone, kctx_tx);
+}
+
+static void bnxt_tls_snd_tag_free(struct m_snd_tag* pmt)
+{
+ if_ctx_t ctx;
+ struct bnxt_softc* priv;
+ struct bnxt_ktls_offload_ctx_tx* kctx_tx =
+ __containerof(pmt, struct bnxt_ktls_offload_ctx_tx, tag);
+
+ kctx_tx->state = BNXT_TLS_STATE_FREE;
+ /* Hold a reference until bnxt_tls_work() runs, to avoid a free-vs-read race. */
+ atomic_add_32(&kctx_tx->mpc_inflight, 1);
+
+ ctx = if_getsoftc(kctx_tx->tag.ifp);
+ priv = iflib_get_softc(ctx);
+ bnxt_ktls_dev_del(priv, kctx_tx);
+
+ taskqueue_enqueue(priv->ktls_info->tq, &kctx_tx->work_task);
+}
+
+static void
+bnxt_tls_work(void *context, int pending)
+{
+ struct bnxt_ktls_offload_ctx_tx *kctx_tx = context;
+
+ /* Drop the reference taken in bnxt_tls_snd_tag_free(); free once last. */
+ if (atomic_fetchadd_32(&kctx_tx->mpc_inflight, -1) == 1)
+ bnxt_tls_tag_zfree(kctx_tx);
+}
+
+int bnxt_tls_snd_tag_alloc(if_t ifp, union if_snd_tag_alloc_params* params,
+ struct m_snd_tag** ppmt)
+{
+ const struct if_snd_tag_sw* snd_tag_sw;
+ struct bnxt_softc* priv;
+ struct bnxt_tls_info* ktls;
+ if_ctx_t ctx;
+ struct bnxt_ktls_offload_ctx_tx* kctx_tx = NULL;
+ const struct tls_session_params* en;
+ int error;
+
+ ctx = if_getsoftc(ifp);
+ priv = iflib_get_softc(ctx);
+ ktls = priv->ktls_info;
+
+ if (!test_bit(BNXT_STATE_OPEN, &priv->state))
+ return (EPROTONOSUPPORT);
+
+ atomic_add_32(&priv->ktls_info->snd_tag_alloc_ref, 1);
+
+ if (priv->ktls_info->init == 0) {
+ counter_u64_add(ktls->err_counters[BNXT_KTLS_TX_UNINITIALIZED], 1);
+ atomic_subtract_32(&priv->ktls_info->snd_tag_alloc_ref, 1);
+ BNXT_DEBUG(priv->dev, "%s:%d: ifp:(%pa) ktls is not initialized\n",
+ __func__, __LINE__, ifp);
+ return (EPROTONOSUPPORT);
+ }
+
+ /* allocate new tag from zone, if any */
+ kctx_tx = uma_zalloc(priv->ktls_info->zone, M_NOWAIT);
+ if (kctx_tx == NULL) {
+ counter_u64_add(ktls->err_counters[BNXT_KTLS_TX_CTX_ALLOC_FAILED], 1);
+ atomic_subtract_32(&priv->ktls_info->snd_tag_alloc_ref, 1);
+ BNXT_DEBUG(priv->dev, "%s:%d: ifp:(%pa) kctx_tx uma_zalloc failed\n",
+ __func__, __LINE__, ifp);
+ return (ENOMEM);
+ }
+ /* Set before any goto failure: bnxt_tls_tag_zfree() needs it. */
+ kctx_tx->ktls = priv->ktls_info;
+
+ en = ¶ms->tls.tls->params;
+
+ /* only TLS v1.2 and v1.3 is currently supported */
+ if (en->tls_vmajor != TLS_MAJOR_VER_ONE ||
+ (en->tls_vminor != TLS_MINOR_VER_TWO
+#ifdef TLS_MINOR_VER_THREE
+ && en->tls_vminor != TLS_MINOR_VER_THREE
+#endif
+ )) {
+ BNXT_DEBUG(priv->dev, "kTLS snd_tag_alloc: unsupported TLS version %u.%u\n",
+ en->tls_vmajor, en->tls_vminor);
+ error = EPROTONOSUPPORT;
+ goto failure;
+ }
+
+ switch (en->cipher_algorithm) {
+ case CRYPTO_AES_NIST_GCM_16:
+ if ((en->cipher_key_len != 128 / 8) &&
+ (en->cipher_key_len != 256 / 8)) {
+ BNXT_DEBUG(priv->dev, "kTLS snd_tag_alloc: unsupported cipher key len %u\n",
+ en->cipher_key_len * 8);
+ error = EPROTONOSUPPORT;
+ goto failure;
+ }
+ break;
+
+ default:
+ BNXT_DEBUG(priv->dev, "kTLS snd_tag_alloc: unsupported cipher algorithm %u\n",
+ en->cipher_algorithm);
+ error = EPROTONOSUPPORT;
+ goto failure;
+ }
+
+ /* Fetch TCP seq number from struct tcpcb */
+ kctx_tx->tcp_seq_no = intotcpcb(params->tls.tls->inp)->snd_nxt;
+
+ /*
+ * next_seqno currently only carries a valid TLS record seq number
+ * for TLS v1.0; kernel support for other versions is still needed.
+ */
+#ifdef KTLS_REC_SEQ_NUMBER
+ kctx_tx->rec_seq = params->tls.tls->next_seqno;
+#endif
+
+ kctx_tx->state = BNXT_TLS_STATE_INIT;
+
+ error = bnxt_ktls_dev_add(priv, en, kctx_tx);
+ if (error) {
+ counter_u64_add(ktls->err_counters[BNXT_KTLS_TX_DEV_ADD_FAILED], 1);
+ error = EINVAL;
+ goto failure;
+ }
+
+ if (params->hdr.type == IF_SND_TAG_TYPE_TLS) {
+ snd_tag_sw = &bnxt_tls_snd_tag_sw;
+ } else {
+ counter_u64_add(ktls->err_counters[BNXT_KTLS_TX_UNSUPPORTED_TLS_HDR], 1);
+ BNXT_DEBUG(priv->dev, "kTLS snd_tag_alloc: unsupported snd_tag type %u\n",
+ params->hdr.type);
+ bnxt_ktls_dev_del(priv, kctx_tx);
+ error = EOPNOTSUPP;
+ goto failure;
+ }
+
+ MPASS(kctx_tx->tag.refcount == 0);
+ m_snd_tag_init(&kctx_tx->tag, ifp, snd_tag_sw);
+ *ppmt = &kctx_tx->tag;
+ atomic_subtract_32(&priv->ktls_info->snd_tag_alloc_ref, 1);
+ BNXT_DEBUG(priv->dev, "[KID:%u]: TAG_ALLOC: tcp_seqno: %u"
+ " record_no: %lu\n", kctx_tx->kid, kctx_tx->tcp_seq_no, kctx_tx->rec_seq);
+
+ return (0);
+
+failure:
+ kctx_tx->state = BNXT_TLS_STATE_FREE;
+ /* Same reference-then-release pattern as bnxt_tls_work(), to avoid a double free. */
+ atomic_add_32(&kctx_tx->mpc_inflight, 1);
+ if (atomic_fetchadd_32(&kctx_tx->mpc_inflight, -1) == 1)
+ bnxt_tls_tag_zfree(kctx_tx);
+ atomic_subtract_32(&priv->ktls_info->snd_tag_alloc_ref, 1);
+ return (error);
+}
+
+void bnxt_alloc_ktls_info(struct bnxt_softc* softc, uint16_t max_keys,
+ bool partition_cap, uint16_t ctxs_per_partition)
+{
+ struct bnxt_tls_info* ktls = softc->ktls_info;
+
+ if (BNXT_VF(softc))
+ return;
+
+ if (!ktls) {
+ bool partition_mode = false;
+ struct bnxt_kctx* kctx;
+ uint16_t batch_sz = 0;
+ int i;
+
+ ktls = malloc(sizeof(*ktls), M_DEVBUF, M_NOWAIT | M_ZERO);
+
+ if (ktls == NULL) {
+ device_printf(softc->dev, "kTLS info alloc failed\n");
+ return;
+ }
+
+ if (partition_cap) {
+ batch_sz = ctxs_per_partition;
+ if (batch_sz && batch_sz <= BNXT_KID_BATCH_SIZE)
+ partition_mode = true;
+ }
+ for (i = 0; i < BNXT_MAX_CRYPTO_KEY_TYPE; i++) {
+ kctx = &ktls->kctx[i];
+ kctx->type = i;
+ kctx->max_ctx = softc->max_ktls_entries;
+ TAILQ_INIT(&kctx->list);
+ mtx_init(&kctx->lock, "BNXT KCTL LOCK", NULL,
+ MTX_DEF | MTX_NOWITNESS);
+ cv_init(&kctx->alloc_pending_cv, "bnxt_ktls_alloc");
+ atomic_store_32(&kctx->alloc_pending, 0);
+ if (partition_mode) {
+ int bmap_sz;
+
+ bmap_sz = howmany(kctx->max_ctx, batch_sz);
+ kctx->partition_bmap_nbits = bmap_sz;
+ kctx->partition_bmap =
+ bit_alloc(bmap_sz, M_DEVBUF, M_NOWAIT);
+ if (kctx->partition_bmap == NULL) {
+ BNXT_DEBUG(softc->dev,
+ "kTLS partition_bmap alloc failed, disabling partition mode\n");
+ partition_mode = false;
+ }
+ }
+ }
+ atomic_store_32(&ktls->snd_tag_alloc_ref, 0);
+ ktls->partition_mode = partition_mode;
+ ktls->ctxs_per_partition = batch_sz;
+
+ atomic_store_32(&ktls->pending, 0);
+
+ softc->ktls_info = ktls;
+
+ /* Create DMA descriptor TAG */
+ if ((bus_dma_tag_create(
+ bus_get_dma_tag(softc->dev),
+ 1, /* any alignment */
+ 0, /* no boundary */
+ BUS_SPACE_MAXADDR, /* lowaddr */
+ BUS_SPACE_MAXADDR, /* highaddr */
+ NULL, NULL, /* filter, filterarg */
+ 65536,
+ BNXT_MAX_NUM_SEGS,
+ 16384,
+ 0, /* flags */
+ NULL, NULL, /* lockfunc, lockfuncarg */
+ &ktls->dma_tag))) {
+ device_printf(softc->dev, "kTLS dma tag alloc failed\n");
+ /*
+ * Back out ktls_info entirely (not via bnxt_free_ktls_info(),
+ * which would call bus_dma_tag_destroy() on this NULL tag).
+ */
+ softc->ktls_info = NULL;
+ for (i = 0; i < BNXT_MAX_CRYPTO_KEY_TYPE; i++) {
+ kctx = &ktls->kctx[i];
+ cv_destroy(&kctx->alloc_pending_cv);
+ mtx_destroy(&kctx->lock);
+ if (kctx->partition_bmap != NULL)
+ free(kctx->partition_bmap, M_DEVBUF);
+ }
+ free(ktls, M_DEVBUF);
+ return;
+ }
+ }
+ ktls->max_key_ctxs_alloc = max_keys;
+}
+
+void bnxt_ktls_del_all(struct bnxt_softc *softc)
+{
+ struct bnxt_tls_info *ktls = softc->ktls_info;
+ struct bnxt_kid_info *kid;
+ struct bnxt_kctx *kctx;
+ int i, j;
+
+ if (!ktls)
+ return;
+
+ /*
+ * Called during FW reset, before bnxt_clear_ktls(). FW reinitialises
+ * its key-context table, so just mark every KID free in software.
+ */
+ /* Shutting down; no need to hold kctx locks. */
+ for (i = 0; i < BNXT_MAX_CRYPTO_KEY_TYPE; i++) {
+ kctx = &ktls->kctx[i];
+ TAILQ_FOREACH(kid, &kctx->list, list) {
+ for (j = 0; j < (int)kid->count; j++)
+ bit_set(kid->ids, j);
+ }
+ }
+}
+
+void bnxt_clear_ktls(struct bnxt_softc *softc)
+{
+ struct bnxt_tls_info *ktls = softc->ktls_info;
+ struct bnxt_kid_info *kid, *tmp;
+ struct bnxt_kctx *kctx;
+ int i;
+
+ if (!ktls)
+ return;
+
+ /* Shutting down or FW reset, no need to protect the lists. */
+ for (i = 0; i < BNXT_MAX_CRYPTO_KEY_TYPE; i++) {
+ kctx = &ktls->kctx[i];
+ TAILQ_FOREACH_SAFE(kid, &kctx->list, list, tmp) {
+ TAILQ_REMOVE(&kctx->list, kid, list);
+ if (kid->ids != NULL)
+ free(kid->ids, M_DEVBUF);
+ free(kid, M_DEVBUF);
+ }
+ if (kctx->partition_bmap != NULL) {
+ free(kctx->partition_bmap, M_DEVBUF);
+ kctx->partition_bmap = NULL;
+ }
+ kctx->total_alloc = 0;
+ }
+
+ /*
+ * partition_bmap is now NULL for every kctx above; without this, a
+ * later bnxt_hwrm_key_ctx_alloc() would still take the partition-mode
+ * path (nothing re-negotiates it after a reset) and bit_ffc_at() a
+ * NULL bitmap. Also bump reset_gen so bnxt_ktls_xmit() can catch
+ * READY tags whose firmware KID this reset erased.
+ */
+ ktls->partition_mode = false;
+ atomic_add_32(&ktls->reset_gen, 1);
+}
+
+void bnxt_free_ktls_counters(struct bnxt_softc* softc)
+{
+ uint8_t i;
+ struct bnxt_tls_info *ktls = softc->ktls_info;
+
+ if (!ktls)
+ return;
+
+ for (i = 0; i < BNXT_MPC_CMP_TIME_MAX_COUNTERS; i++) {
+ if (ktls->mpc_cmp_time[i]) {
+ counter_u64_free(ktls->mpc_cmp_time[i]);
+ ktls->mpc_cmp_time[i] = NULL;
+ }
+ }
+
+ for (i = 0; i < BNXT_KTLS_MAX_COUNTERS; i++) {
+ if (ktls->counters[i]) {
+ counter_u64_free(ktls->counters[i]);
+ ktls->counters[i] = NULL;
+ }
+ }
+
+ for (i = 0; i < BNXT_KTLS_ERR_MAX_COUNTERS; i++) {
+ if (ktls->err_counters[i]) {
+ counter_u64_free(ktls->err_counters[i]);
+ ktls->err_counters[i] = NULL;
+ }
+ }
+}
+
+void bnxt_alloc_ktls_counters(struct bnxt_softc* softc)
+{
+ uint8_t i;
+ struct bnxt_tls_info *ktls = softc->ktls_info;
+
+ if (!ktls)
+ return;
+
+ for (i = 0; i < BNXT_MPC_CMP_TIME_MAX_COUNTERS; i++) {
+ if (!ktls->mpc_cmp_time[i])
+ ktls->mpc_cmp_time[i] = counter_u64_alloc(M_WAITOK);
+ }
+
+ for (i = 0; i < BNXT_KTLS_MAX_COUNTERS; i++) {
+ if (!ktls->counters[i])
+ ktls->counters[i] = counter_u64_alloc(M_WAITOK);
+ }
+
+ for (i = 0; i < BNXT_KTLS_ERR_MAX_COUNTERS; i++) {
+ if (!ktls->err_counters[i])
+ ktls->err_counters[i] = counter_u64_alloc(M_WAITOK);
+ }
+
+}
+
+void bnxt_free_ktls_info(struct bnxt_softc* softc)
+{
+ struct bnxt_tls_info* ktls = softc->ktls_info;
+ int i;
+
+ if (!ktls)
+ return;
+
+ /* Retry a busy sysctl_ctx_free(); don't free ktls while its sysctl nodes are still live. */
+ if (softc->ktls_stats_oid != NULL) {
+ for (i = 0; i < 10; i++) {
+ if (sysctl_ctx_free(&softc->ktls_stats) == 0) {
+ softc->ktls_stats_oid = NULL;
+ break;
+ }
+ pause("bnxtktlssysctl", hz / 10);
+ }
+ }
+ if (softc->mpc_cmp_time_stats_oid != NULL) {
+ for (i = 0; i < 10; i++) {
+ if (sysctl_ctx_free(&softc->mpc_cmp_time_stats) == 0) {
+ softc->mpc_cmp_time_stats_oid = NULL;
+ break;
+ }
+ pause("bnxtktlssysctl", hz / 10);
+ }
+ }
+ if (softc->ktls_stats_oid != NULL || softc->mpc_cmp_time_stats_oid != NULL) {
+ device_printf(softc->dev,
+ "kTLS: stats sysctl busy, leaking kTLS info to avoid a use-after-free\n");
+ return;
+ }
+
+ bnxt_clear_ktls(softc);
+ bus_dma_tag_destroy(ktls->dma_tag);
+ if (ktls->mpc_zone != NULL)
+ uma_zdestroy(ktls->mpc_zone);
+
+ if (ktls->init) {
+ ktls->init = 0;
+ if (ktls->tq != NULL) {
+ taskqueue_quiesce(ktls->tq);
+ taskqueue_free(ktls->tq);
+ ktls->tq = NULL;
+ }
+ uma_zdestroy(ktls->zone);
+ }
+
+ for (i = 0; i < BNXT_MAX_CRYPTO_KEY_TYPE; i++) {
+ cv_destroy(&ktls->kctx[i].alloc_pending_cv);
+ if (mtx_initialized(&ktls->kctx[i].lock))
+ mtx_destroy(&ktls->kctx[i].lock);
+ }
+
+ free(ktls, M_DEVBUF);
+ softc->ktls_info = NULL;
+}
+
+void bnxt_hwrm_reserve_pf_key_ctxs(struct bnxt_softc* softc,
+ struct hwrm_func_cfg_input* req,
+ uint8_t type)
+{
+ struct bnxt_tls_info* tls = softc->ktls_info;
+ struct bnxt_hw_resc* hw_resc = &softc->hw_resc;
+ struct bnxt_hw_tls_resc* tls_resc;
+ uint32_t tx, rx;
+
+ if (!tls)
+ return;
+
+ tls_resc = &hw_resc->tls_resc[type];
+ tx = min(BNXT_TCK(tls).max_ctx, tls_resc->max_tx_key_ctxs);
+ rx = min(BNXT_RCK(tls).max_ctx, tls_resc->max_rx_key_ctxs);
+ if (type == BNXT_CRYPTO_TYPE_KTLS) {
+ req->num_ktls_tx_key_ctxs = htole32(tx);
+ req->num_ktls_rx_key_ctxs = htole32(rx);
+ req->enables |= htole32(
+ HWRM_FUNC_CFG_INPUT_ENABLES_KTLS_TX_KEY_CTXS |
+ HWRM_FUNC_CFG_INPUT_ENABLES_KTLS_RX_KEY_CTXS);
+ }
+}
+
+static int __bnxt_partition_alloc(struct bnxt_kctx* kctx, uint32_t* id)
+{
+ int max = kctx->partition_bmap_nbits;
+ ssize_t next;
+
+ bit_ffc_at(kctx->partition_bmap, kctx->next, max, &next);
+ if (next == -1)
+ bit_ffc(kctx->partition_bmap, max, &next);
+ if (next == -1)
+ return ENOSPC;
+ *id = next;
+ kctx->next = next;
+ return 0;
+}
+
+static int bnxt_partition_alloc(struct bnxt_kctx* kctx, uint32_t* id)
+{
+ int rc;
+
+ mtx_lock(&kctx->lock);
+ do {
+ rc = __bnxt_partition_alloc(kctx, id);
+ if (rc) {
+ mtx_unlock(&kctx->lock);
+ return rc;
+ }
+ } while (bnxt_test_and_set_bit_nonatomic(*id, kctx->partition_bmap));
+ mtx_unlock(&kctx->lock);
+ return 0;
+}
+
+static int bnxt_key_ctx_store(struct bnxt_softc* softc, __le32* key_buf,
+ uint32_t num, bool contig, struct bnxt_kctx* kctx,
+ uint32_t* id)
+{
+ struct bnxt_kid_info* kid;
+ uint32_t i;
+ int j;
+
+ for (i = 0; i < num;) {
+ kid = malloc(sizeof(*kid), M_DEVBUF, M_NOWAIT | M_ZERO);
+ if (kid == NULL) {
+ device_printf(softc->dev, "kTLS kid alloc failed\n");
+ return ENOMEM;
+ }
+ kid->ids = bit_alloc(BNXT_KID_BATCH_SIZE, M_DEVBUF, M_NOWAIT);
+ if (kid->ids == NULL) {
+ free(kid, M_DEVBUF);
+ device_printf(softc->dev, "kTLS kid ids alloc failed\n");
+ return ENOMEM;
+ }
+ kid->start_id = le32toh(key_buf[i]);
+ if (contig)
+ kid->count = (num > BNXT_KID_BATCH_SIZE) ? BNXT_KID_BATCH_SIZE : num;
+ else
+ /*
+ * Non-contiguous IDs can't share one start_id+count
+ * range, so each gets its own bnxt_kid_info node.
+ */
+ kid->count = 1;
+ for (j = 0; j < kid->count; j++)
+ bit_set(kid->ids, j);
+ if (id != NULL && i == 0) {
+ bit_clear(kid->ids, 0);
+ *id = kid->start_id;
+ }
+ mtx_lock(&kctx->lock);
+ TAILQ_INSERT_TAIL(&kctx->list, kid, list);
+ kctx->total_alloc += kid->count;
+ mtx_unlock(&kctx->lock);
*** 2716 LINES SKIPPED ***