git: 613e7ce43f81 - main - cxgbe: Greatly simplify ktls_wr_len for T7

The branch main has been updated by jhb:

URL: https://cgit.FreeBSD.org/src/commit/?id=613e7ce43f814c3fb3e076f8509b969b5c12808f

commit 613e7ce43f814c3fb3e076f8509b969b5c12808f
Author:     John Baldwin <jhb@FreeBSD.org>
AuthorDate: 2026-09-30 19:19:11 +0000
Commit:     John Baldwin <jhb@FreeBSD.org>
CommitDate: 2026-09-30 19:19:11 +0000

    cxgbe: Greatly simplify ktls_wr_len for T7
    
    Don't try to fine-tune the WR size when estimating the work request
    length when parsing the packet.  Use a much simpler worst-case
    estimate that only depends on a few fields in the mbuf metadata.
    
    Sponsored by:   Chelsio Communications
---
 sys/dev/cxgbe/crypto/t7_kern_tls.c | 134 ++++++++++++-------------------------
 1 file changed, 44 insertions(+), 90 deletions(-)

diff --git a/sys/dev/cxgbe/crypto/t7_kern_tls.c b/sys/dev/cxgbe/crypto/t7_kern_tls.c
index 3ba4c47235f1..f993b160191f 100644
--- a/sys/dev/cxgbe/crypto/t7_kern_tls.c
+++ b/sys/dev/cxgbe/crypto/t7_kern_tls.c
@@ -573,14 +573,10 @@ ktls_gcm_aad_len(struct tlspcb *tlsp)
 }
 
 static int
-ktls_wr_len(struct tlspcb *tlsp, struct mbuf *m, struct mbuf *m_tls,
-    int *nsegsp)
+ktls_wr_len(struct tlspcb *tlsp, struct mbuf *m, struct mbuf *m_tls)
 {
-	const struct tls_record_layer *hdr;
-	u_int header_len, imm_len, offset, plen, rlen, tlen, wr_len;
-	u_int leading_waste, trailing_waste;
-	bool inline_key, last_ghash_frag, request_ghash, send_partial_ghash;
-	bool short_record;
+	u_int imm_len, nsegs, tlen, wr_len;
+	bool inline_key, use_ghash;
 
 	M_ASSERTEXTPG(m_tls);
 
@@ -605,104 +601,54 @@ ktls_wr_len(struct tlspcb *tlsp, struct mbuf *m, struct mbuf *m_tls,
 
 		/* This should always be the last TLS record in a chain. */
 		MPASS(m_tls->m_next == NULL);
-		*nsegsp = 0;
 		return (wr_len);
 	}
 
-	hdr = (void *)m_tls->m_epg_hdr;
-	rlen = TLS_HEADER_LENGTH + ntohs(hdr->tls_length);
-
-	/*
-	 * See if this request might make use of GHASH state.  This
-	 * errs on the side of over-budgeting the WR size.
-	 */
-	last_ghash_frag = false;
-	request_ghash = false;
-	send_partial_ghash = false;
-	if (tlsp->enc_mode == SCMD_CIPH_MODE_AES_GCM &&
-	    tlsp->sc->tlst.partial_ghash && tlsp->sc->tlst.short_records) {
-		u_int trailer_len;
-
-		trailer_len = m_tls->m_epg_trllen;
-		if (tlsp->tls13)
-			trailer_len--;
-		KASSERT(trailer_len == AES_GMAC_HASH_LEN,
-		    ("invalid trailer length for AES-GCM"));
+	/* AES-GCM records might use GHASH state. */
+	use_ghash = (tlsp->enc_mode == SCMD_CIPH_MODE_AES_GCM);
 
-		/* Is this the start of a TLS record? */
-		if (mtod(m_tls, vm_offset_t) <= m_tls->m_epg_hdrlen) {
-			/*
-			 * Might use partial GHASH if this doesn't
-			 * send the full record.
-			 */
-			if (tlen < rlen) {
-				if (tlen < (rlen - trailer_len))
-					send_partial_ghash = true;
-				request_ghash = true;
-			}
-		} else {
-			send_partial_ghash = true;
-			if (tlen < rlen)
-				request_ghash = true;
-			if (tlen >= (rlen - trailer_len))
-				last_ghash_frag = true;
-		}
-	}
+	inline_key = use_ghash || tlsp->inline_key;
 
-	/*
-	 * Assume not sending partial GHASH for this call to get the
-	 * larger size.
-	 */
-	short_record = ktls_is_short_record(tlsp, m_tls, tlen, rlen,
-	    &header_len, &offset, &plen, &leading_waste, &trailing_waste,
-	    false, request_ghash);
-
-	inline_key = send_partial_ghash || tlsp->inline_key;
-
-	/* Calculate the size of the work request. */
 	wr_len = ktls_base_wr_size(tlsp, inline_key);
 
-	if (send_partial_ghash)
+	/* Assume sending a partial GHASH. */
+	if (use_ghash)
 		wr_len += AES_GMAC_HASH_LEN;
 
-	if (leading_waste != 0 || trailing_waste != 0) {
-		/*
-		 * Partial records might require a SplitMode
-		 * CPL_RX_PHYS_DSGL.
-		 */
-		wr_len += sizeof(struct cpl_t7_rx_phys_dsgl);
-	}
+	/* Assume a SplitMode CPL_RX_PHYS_DSGL. */
+	wr_len += sizeof(struct cpl_t7_rx_phys_dsgl);
 
-	/* Budget for an LSO header even if we don't use it. */
+	/* Assume an LSO header even if we don't use it. */
 	wr_len += sizeof(struct cpl_tx_pkt_lso_core);
 
 	/*
 	 * Headers (including the TLS header) are always sent as
-	 * immediate data.  Short records include a raw AES IV as
-	 * immediate data.  TLS 1.3 non-short records include a
-	 * placeholder for the sequence number as immediate data.
-	 * Short records using a partial hash may also need to send
-	 * TLS AAD.  If a partial hash might be sent, assume a short
-	 * record to get the larger size.
+	 * immediate data.  In the worse case, a short record can send
+	 * a full AES IV and AAD.
 	 */
-	imm_len = m->m_len + header_len;
-	if (short_record || send_partial_ghash) {
-		imm_len += AES_BLOCK_LEN;
-		if (send_partial_ghash && header_len != 0)
-			imm_len += ktls_gcm_aad_len(tlsp);
-	} else if (tlsp->tls13)
-		imm_len += sizeof(uint64_t);
+	imm_len = m->m_len + m_tls->m_epg_hdrlen;
+	imm_len += AES_BLOCK_LEN;
+	if (use_ghash)
+		imm_len += ktls_gcm_aad_len(tlsp);
+
 	wr_len += roundup2(imm_len, 16);
 
 	/*
-	 * TLS record payload via DSGL.  For partial GCM mode we
-	 * might need an extra SG entry for a placeholder.
+	 * Compute an upper bound on DSGL elements assuming the entire
+	 * TLS record payload including header type in trailer for TLS
+	 * 1.3.
 	 */
-	*nsegsp = sglist_count_mbuf_epg(m_tls, m_tls->m_epg_hdrlen + offset,
-	    plen);
-	wr_len += ktls_sgl_size(*nsegsp + (last_ghash_frag ? 1 : 0));
+	nsegs = m_tls->m_epg_npgs;
+	if (tlsp->tls13)
+		nsegs++;
 
-	if (request_ghash) {
+	/* Extra SGL for GHASH placeholder. */
+	nsegs++;
+
+	MPASS(nsegs <= TX_SGL_SEGS);
+	wr_len += ktls_sgl_size(nsegs);
+
+	if (use_ghash) {
 		/* AES-GCM records might return a partial hash. */
 		wr_len += sizeof(struct ulp_txpkt);
 		wr_len += sizeof(struct ulptx_idata);
@@ -713,6 +659,10 @@ ktls_wr_len(struct tlspcb *tlsp, struct mbuf *m, struct mbuf *m_tls,
 	}
 
 	wr_len = roundup2(wr_len, 16);
+
+	/* This estimate can be a bit too conservative, so clamp it. */
+	wr_len = MIN(wr_len, SGE_MAX_WR_LEN);
+
 	return (wr_len);
 }
 
@@ -777,7 +727,7 @@ t7_ktls_parse_pkt(struct mbuf *m)
 	struct tcphdr *tcp;
 	struct mbuf *m_tls;
 	void *items[1];
-	int error, nsegs;
+	int error;
 	u_int wr_len, tot_len;
 	uint16_t eh_type;
 
@@ -862,12 +812,11 @@ t7_ktls_parse_pkt(struct mbuf *m)
 	for (m_tls = m->m_next; m_tls != NULL; m_tls = m_tls->m_next) {
 		MPASS(m_tls->m_flags & M_EXTPG);
 
-		wr_len = ktls_wr_len(tlsp, m, m_tls, &nsegs);
+		wr_len = ktls_wr_len(tlsp, m, m_tls);
 #ifdef VERBOSE_TRACES
-		CTR(KTR_CXGBE, "%s: %p wr_len %d nsegs %d", __func__, tlsp,
-		    wr_len, nsegs);
+		CTR(KTR_CXGBE, "%s: %p wr_len %d", __func__, tlsp, wr_len);
 #endif
-		if (wr_len > SGE_MAX_WR_LEN || nsegs > TX_SGL_SEGS)
+		if (wr_len > SGE_MAX_WR_LEN)
 			return (EFBIG);
 		tot_len += roundup2(wr_len, EQ_ESIZE);
 	}
@@ -1512,6 +1461,7 @@ ktls_write_tls_wr(struct tlspcb *tlsp, struct sge_txq *txq,
 	} else
 		txpkt_lens[1] = 0;
 
+	MPASS(wr_len <= SGE_MAX_WR_LEN);
 	ndesc = howmany(wr_len, EQ_ESIZE);
 	MPASS(ndesc <= available);
 
@@ -2024,6 +1974,9 @@ t7_ktls_write_wr(struct sge_txq *txq, void *dst, struct mbuf *m,
 	struct ether_header *eh;
 	tcp_seq tcp_seqno;
 	u_int ndesc, pidx, totdesc;
+#ifdef INVARIANTS
+	u_int len16 = mbuf_len16(m);
+#endif
 	uint16_t eh_type, mss;
 
 	TXQ_LOCK_ASSERT_OWNED(txq);
@@ -2077,6 +2030,7 @@ t7_ktls_write_wr(struct sge_txq *txq, void *dst, struct mbuf *m,
 
 		tcp_seqno += m_tls->m_len;
 	}
+	MPASS(totdesc * EQ_ESIZE / 16 <= len16);
 
 	/*
 	 * Queue another packet if this was a GCM request that didn't