git: 71d82199a111 - main - cxgbe: Add support to the base driver for NVMe/TCP PDU offload
- Go to: [ bottom of page ] [ top of archives ] [ this month ]
Date: Mon, 10 Nov 2025 15:51:29 UTC
The branch main has been updated by jhb:
URL: https://cgit.FreeBSD.org/src/commit/?id=71d82199a111af67cba73e32a27a900d74c1a1cc
commit 71d82199a111af67cba73e32a27a900d74c1a1cc
Author: John Baldwin <jhb@FreeBSD.org>
AuthorDate: 2025-11-10 15:50:48 +0000
Commit: John Baldwin <jhb@FreeBSD.org>
CommitDate: 2025-11-10 15:50:48 +0000
cxgbe: Add support to the base driver for NVMe/TCP PDU offload
- Adds various per-queue counters similar to iSCSI PDU offload as well
as a hook in the adapter softc for a reference to the NVMe/TCP softc.
- Instruct the firmware to include a DDP indicator in the status field
for NVMe/TCP PDU completion messages. This flag indicates if the
payload data for a PDU has been received in the free list or if it
was placed directly into a kernel I/O data buffer via DDP.
Sponsored by: Chelsio Communications
---
sys/dev/cxgbe/adapter.h | 11 ++++++++
sys/dev/cxgbe/t4_main.c | 16 ++++++++++++
sys/dev/cxgbe/t4_sge.c | 68 +++++++++++++++++++++++++++++++++++++++++++++++--
3 files changed, 93 insertions(+), 2 deletions(-)
diff --git a/sys/dev/cxgbe/adapter.h b/sys/dev/cxgbe/adapter.h
index 57618b1ceb15..0946c3110817 100644
--- a/sys/dev/cxgbe/adapter.h
+++ b/sys/dev/cxgbe/adapter.h
@@ -723,6 +723,16 @@ struct sge_ofld_rxq {
uint64_t rx_iscsi_padding_errors;
uint64_t rx_iscsi_header_digest_errors;
uint64_t rx_iscsi_data_digest_errors;
+ counter_u64_t rx_nvme_ddp_setup_ok;
+ counter_u64_t rx_nvme_ddp_setup_no_stag;
+ counter_u64_t rx_nvme_ddp_setup_error;
+ counter_u64_t rx_nvme_ddp_pdus;
+ counter_u64_t rx_nvme_ddp_octets;
+ counter_u64_t rx_nvme_fl_pdus;
+ counter_u64_t rx_nvme_fl_octets;
+ counter_u64_t rx_nvme_invalid_headers;
+ counter_u64_t rx_nvme_header_digest_errors;
+ counter_u64_t rx_nvme_data_digest_errors;
uint64_t rx_aio_ddp_jobs;
uint64_t rx_aio_ddp_octets;
u_long rx_toe_tls_records;
@@ -1000,6 +1010,7 @@ struct adapter {
void *iwarp_softc; /* (struct c4iw_dev *) */
struct iw_tunables iwt;
void *iscsi_ulp_softc; /* (struct cxgbei_data *) */
+ void *nvme_ulp_softc; /* (struct nvmf_che_adapter *) */
struct l2t_data *l2t; /* L2 table */
struct smt_data *smt; /* Source MAC Table */
struct tid_info tids;
diff --git a/sys/dev/cxgbe/t4_main.c b/sys/dev/cxgbe/t4_main.c
index 2f6eb6062c20..6133d810c003 100644
--- a/sys/dev/cxgbe/t4_main.c
+++ b/sys/dev/cxgbe/t4_main.c
@@ -13009,6 +13009,22 @@ clear_stats(struct adapter *sc, u_int port_id)
ofld_rxq->rx_iscsi_ddp_octets = 0;
ofld_rxq->rx_iscsi_fl_pdus = 0;
ofld_rxq->rx_iscsi_fl_octets = 0;
+ counter_u64_zero(
+ ofld_rxq->rx_nvme_ddp_setup_ok);
+ counter_u64_zero(
+ ofld_rxq->rx_nvme_ddp_setup_no_stag);
+ counter_u64_zero(
+ ofld_rxq->rx_nvme_ddp_setup_error);
+ counter_u64_zero(ofld_rxq->rx_nvme_ddp_pdus);
+ counter_u64_zero(ofld_rxq->rx_nvme_ddp_octets);
+ counter_u64_zero(ofld_rxq->rx_nvme_fl_pdus);
+ counter_u64_zero(ofld_rxq->rx_nvme_fl_octets);
+ counter_u64_zero(
+ ofld_rxq->rx_nvme_invalid_headers);
+ counter_u64_zero(
+ ofld_rxq->rx_nvme_header_digest_errors);
+ counter_u64_zero(
+ ofld_rxq->rx_nvme_data_digest_errors);
ofld_rxq->rx_aio_ddp_jobs = 0;
ofld_rxq->rx_aio_ddp_octets = 0;
ofld_rxq->rx_toe_tls_records = 0;
diff --git a/sys/dev/cxgbe/t4_sge.c b/sys/dev/cxgbe/t4_sge.c
index 8cd5b0543d8a..4932a3abdeca 100644
--- a/sys/dev/cxgbe/t4_sge.c
+++ b/sys/dev/cxgbe/t4_sge.c
@@ -852,6 +852,11 @@ t4_tweak_chip_settings(struct adapter *sc)
/* We use multiple DDP page sizes both in plain-TOE and ISCSI modes. */
m = v = F_TDDPTAGTCB | F_ISCSITAGTCB;
+ if (sc->nvmecaps != 0) {
+ /* Request DDP status bit for NVMe PDU completions. */
+ m |= F_NVME_TCP_DDP_VAL_EN;
+ v |= F_NVME_TCP_DDP_VAL_EN;
+ }
t4_set_reg_field(sc, A_ULP_RX_CTL, m, v);
m = V_INDICATESIZE(M_INDICATESIZE) | F_REARMDDPOFFSET |
@@ -4170,6 +4175,20 @@ alloc_ofld_rxq(struct vi_info *vi, struct sge_ofld_rxq *ofld_rxq, int idx,
ofld_rxq->rx_iscsi_ddp_setup_ok = counter_u64_alloc(M_WAITOK);
ofld_rxq->rx_iscsi_ddp_setup_error =
counter_u64_alloc(M_WAITOK);
+ ofld_rxq->rx_nvme_ddp_setup_ok = counter_u64_alloc(M_WAITOK);
+ ofld_rxq->rx_nvme_ddp_setup_no_stag =
+ counter_u64_alloc(M_WAITOK);
+ ofld_rxq->rx_nvme_ddp_setup_error =
+ counter_u64_alloc(M_WAITOK);
+ ofld_rxq->rx_nvme_ddp_octets = counter_u64_alloc(M_WAITOK);
+ ofld_rxq->rx_nvme_ddp_pdus = counter_u64_alloc(M_WAITOK);
+ ofld_rxq->rx_nvme_fl_octets = counter_u64_alloc(M_WAITOK);
+ ofld_rxq->rx_nvme_fl_pdus = counter_u64_alloc(M_WAITOK);
+ ofld_rxq->rx_nvme_invalid_headers = counter_u64_alloc(M_WAITOK);
+ ofld_rxq->rx_nvme_header_digest_errors =
+ counter_u64_alloc(M_WAITOK);
+ ofld_rxq->rx_nvme_data_digest_errors =
+ counter_u64_alloc(M_WAITOK);
ofld_rxq->ddp_buffer_alloc = counter_u64_alloc(M_WAITOK);
ofld_rxq->ddp_buffer_reuse = counter_u64_alloc(M_WAITOK);
ofld_rxq->ddp_buffer_free = counter_u64_alloc(M_WAITOK);
@@ -4207,6 +4226,16 @@ free_ofld_rxq(struct vi_info *vi, struct sge_ofld_rxq *ofld_rxq)
MPASS(!(ofld_rxq->iq.flags & IQ_SW_ALLOCATED));
counter_u64_free(ofld_rxq->rx_iscsi_ddp_setup_ok);
counter_u64_free(ofld_rxq->rx_iscsi_ddp_setup_error);
+ counter_u64_free(ofld_rxq->rx_nvme_ddp_setup_ok);
+ counter_u64_free(ofld_rxq->rx_nvme_ddp_setup_no_stag);
+ counter_u64_free(ofld_rxq->rx_nvme_ddp_setup_error);
+ counter_u64_free(ofld_rxq->rx_nvme_ddp_octets);
+ counter_u64_free(ofld_rxq->rx_nvme_ddp_pdus);
+ counter_u64_free(ofld_rxq->rx_nvme_fl_octets);
+ counter_u64_free(ofld_rxq->rx_nvme_fl_pdus);
+ counter_u64_free(ofld_rxq->rx_nvme_invalid_headers);
+ counter_u64_free(ofld_rxq->rx_nvme_header_digest_errors);
+ counter_u64_free(ofld_rxq->rx_nvme_data_digest_errors);
counter_u64_free(ofld_rxq->ddp_buffer_alloc);
counter_u64_free(ofld_rxq->ddp_buffer_reuse);
counter_u64_free(ofld_rxq->ddp_buffer_free);
@@ -4218,12 +4247,12 @@ static void
add_ofld_rxq_sysctls(struct sysctl_ctx_list *ctx, struct sysctl_oid *oid,
struct sge_ofld_rxq *ofld_rxq)
{
- struct sysctl_oid_list *children;
+ struct sysctl_oid_list *children, *top;
if (ctx == NULL || oid == NULL)
return;
- children = SYSCTL_CHILDREN(oid);
+ top = children = SYSCTL_CHILDREN(oid);
SYSCTL_ADD_U64(ctx, children, OID_AUTO, "rx_aio_ddp_jobs",
CTLFLAG_RD, &ofld_rxq->rx_aio_ddp_jobs, 0,
"# of aio_read(2) jobs completed via DDP");
@@ -4280,6 +4309,41 @@ add_ofld_rxq_sysctls(struct sysctl_ctx_list *ctx, struct sysctl_oid *oid,
SYSCTL_ADD_U64(ctx, children, OID_AUTO, "data_digest_errors",
CTLFLAG_RD, &ofld_rxq->rx_iscsi_data_digest_errors, 0,
"# of PDUs with invalid data digests");
+
+ oid = SYSCTL_ADD_NODE(ctx, top, OID_AUTO, "nvme",
+ CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "TOE NVMe statistics");
+ children = SYSCTL_CHILDREN(oid);
+
+ SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "ddp_setup_ok",
+ CTLFLAG_RD, &ofld_rxq->rx_nvme_ddp_setup_ok,
+ "# of times DDP buffer was setup successfully");
+ SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "ddp_setup_no_stag",
+ CTLFLAG_RD, &ofld_rxq->rx_nvme_ddp_setup_no_stag,
+ "# of times STAG was not available for DDP buffer setup");
+ SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "ddp_setup_error",
+ CTLFLAG_RD, &ofld_rxq->rx_nvme_ddp_setup_error,
+ "# of times DDP buffer setup failed");
+ SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "ddp_octets",
+ CTLFLAG_RD, &ofld_rxq->rx_nvme_ddp_octets,
+ "# of octets placed directly");
+ SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "ddp_pdus",
+ CTLFLAG_RD, &ofld_rxq->rx_nvme_ddp_pdus,
+ "# of PDUs with data placed directly");
+ SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "fl_octets",
+ CTLFLAG_RD, &ofld_rxq->rx_nvme_fl_octets,
+ "# of data octets delivered in freelist");
+ SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "fl_pdus",
+ CTLFLAG_RD, &ofld_rxq->rx_nvme_fl_pdus,
+ "# of PDUs with data delivered in freelist");
+ SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "invalid_headers",
+ CTLFLAG_RD, &ofld_rxq->rx_nvme_invalid_headers,
+ "# of PDUs with invalid header field");
+ SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "header_digest_errors",
+ CTLFLAG_RD, &ofld_rxq->rx_nvme_header_digest_errors,
+ "# of PDUs with invalid header digests");
+ SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "data_digest_errors",
+ CTLFLAG_RD, &ofld_rxq->rx_nvme_data_digest_errors,
+ "# of PDUs with invalid data digests");
}
#endif