git: 58d69f4ecfee - stable/13 - crypto: Add a new type of crypto buffer for a single mbuf.

From: John Baldwin <jhb_at_FreeBSD.org>
Date: Thu, 21 Oct 2021 17:07:13 UTC
The branch stable/13 has been updated by jhb:

URL: https://cgit.FreeBSD.org/src/commit/?id=58d69f4ecfeef7cc811c21ac1d3901c4fdc4f610

commit 58d69f4ecfeef7cc811c21ac1d3901c4fdc4f610
Author:     John Baldwin <jhb@FreeBSD.org>
AuthorDate: 2021-05-25 23:59:18 +0000
Commit:     John Baldwin <jhb@FreeBSD.org>
CommitDate: 2021-10-21 15:51:26 +0000

    crypto: Add a new type of crypto buffer for a single mbuf.
    
    This is intended for use in KTLS transmit where each TLS record is
    described by a single mbuf that is itself queued in the socket buffer.
    Using the existing CRYPTO_BUF_MBUF would result in
    bus_dmamap_load_crp() walking additional mbufs in the socket buffer
    that are not relevant, but generating a S/G list that potentially
    exceeds the limit of the tag (while also wasting CPU cycles).
    
    Reviewed by:    markj
    Sponsored by:   Netflix
    Differential Revision:  https://reviews.freebsd.org/D30136
    
    (cherry picked from commit 883a0196b629a07e52562b4103cc0f6391083080)
---
 share/man/man9/crypto_buffer.9   | 11 ++++++++---
 sys/crypto/ccp/ccp.c             |  3 +++
 sys/dev/cxgbe/crypto/t4_crypto.c |  3 +++
 sys/dev/sec/sec.c                |  3 +++
 sys/kern/subr_bus_dma.c          | 25 +++++++++++++++++++++++++
 sys/mips/nlm/dev/sec/nlmsec.c    |  3 +++
 sys/mips/nlm/dev/sec/nlmseclib.c |  3 +++
 sys/opencrypto/criov.c           | 21 +++++++++++++++++++++
 sys/opencrypto/crypto.c          |  2 ++
 sys/opencrypto/cryptodev.h       | 22 +++++++++++++++++++++-
 sys/opencrypto/cryptosoft.c      |  1 +
 11 files changed, 93 insertions(+), 4 deletions(-)

diff --git a/share/man/man9/crypto_buffer.9 b/share/man/man9/crypto_buffer.9
index 333fc5ec449f..df6128650cd4 100644
--- a/share/man/man9/crypto_buffer.9
+++ b/share/man/man9/crypto_buffer.9
@@ -30,7 +30,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd September 24, 2020
+.Dd May 25, 2021
 .Dt CRYPTO_BUFFER 9
 .Os
 .Sh NAME
@@ -195,7 +195,10 @@ An array of bytes mapped into the kernel's address space.
 A scatter/gather list of kernel buffers as described in
 .Xr uio 9 .
 .It Dv CRYPTO_BUF_MBUF
-A network memory buffer as described in
+A chain of network memory buffers as described in
+.Xr mbuf 9 .
+.It Dv CRYPTO_BUF_SINGLE_MBUF
+A single network memory buffer as described in
 .Xr mbuf 9 .
 .It Dv CRYPTO_BUF_VMPAGE
 A scatter/gather list of
@@ -220,7 +223,9 @@ data buffer
 A pointer to a
 .Vt struct mbuf
 for
-.Dv CRYPTO_BUF_MBUF .
+.Dv CRYPTO_BUF_MBUF
+and
+.Dv CRYPTO_BUF_SINGLE_MBUF .
 .It Fa cb_uio
 A pointer to a
 .Vt struct uio
diff --git a/sys/crypto/ccp/ccp.c b/sys/crypto/ccp/ccp.c
index 63967d7f8e3c..7cc38b14f3fd 100644
--- a/sys/crypto/ccp/ccp.c
+++ b/sys/crypto/ccp/ccp.c
@@ -101,6 +101,9 @@ ccp_populate_sglist(struct sglist *sg, struct crypto_buffer *cb)
 	case CRYPTO_BUF_MBUF:
 		error = sglist_append_mbuf(sg, cb->cb_mbuf);
 		break;
+	case CRYPTO_BUF_SINGLE_MBUF:
+		error = sglist_append_single_mbuf(sg, cb->cb_mbuf);
+		break;
 	case CRYPTO_BUF_UIO:
 		error = sglist_append_uio(sg, cb->cb_uio);
 		break;
diff --git a/sys/dev/cxgbe/crypto/t4_crypto.c b/sys/dev/cxgbe/crypto/t4_crypto.c
index a3c25eb7df47..ea1272758eaf 100644
--- a/sys/dev/cxgbe/crypto/t4_crypto.c
+++ b/sys/dev/cxgbe/crypto/t4_crypto.c
@@ -271,6 +271,9 @@ ccr_populate_sglist(struct sglist *sg, struct crypto_buffer *cb)
 	case CRYPTO_BUF_MBUF:
 		error = sglist_append_mbuf(sg, cb->cb_mbuf);
 		break;
+	case CRYPTO_BUF_SINGLE_MBUF:
+		error = sglist_append_single_mbuf(sg, cb->cb_mbuf);
+		break;
 	case CRYPTO_BUF_UIO:
 		error = sglist_append_uio(sg, cb->cb_uio);
 		break;
diff --git a/sys/dev/sec/sec.c b/sys/dev/sec/sec.c
index fa3e20806124..f485d932727b 100644
--- a/sys/dev/sec/sec.c
+++ b/sys/dev/sec/sec.c
@@ -850,6 +850,9 @@ sec_desc_map_dma(struct sec_softc *sc, struct sec_dma_mem *dma_mem,
 	case CRYPTO_BUF_MBUF:
 		size = m_length(crp->crp_buf.cb_mbuf, NULL);
 		break;
+	case CRYPTO_BUF_SINGLE_MBUF:
+		size = crp->crp_buf.cb_mbuf->m_len;
+		break;
 	case CRYPTO_BUF_VMPAGE:
 		size = PAGE_SIZE - crp->crp_buf.cb_vm_page_offset;
 		break;
diff --git a/sys/kern/subr_bus_dma.c b/sys/kern/subr_bus_dma.c
index 6248dd520511..a7ad30e77715 100644
--- a/sys/kern/subr_bus_dma.c
+++ b/sys/kern/subr_bus_dma.c
@@ -172,6 +172,27 @@ _bus_dmamap_load_mbuf_epg(bus_dma_tag_t dmat, bus_dmamap_t map,
 	return (error);
 }
 
+/*
+ * Load a single mbuf.
+ */
+static int
+_bus_dmamap_load_single_mbuf(bus_dma_tag_t dmat, bus_dmamap_t map,
+    struct mbuf *m, bus_dma_segment_t *segs, int *nsegs, int flags)
+{
+	int error;
+
+	error = 0;
+	if ((m->m_flags & M_EXTPG) != 0)
+		error = _bus_dmamap_load_mbuf_epg(dmat, map, m, segs, nsegs,
+		    flags);
+	else
+		error = _bus_dmamap_load_buffer(dmat, map, m->m_data, m->m_len,
+		    kernel_pmap, flags | BUS_DMA_LOAD_MBUF, segs, nsegs);
+	CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d",
+	    __func__, dmat, flags, error, *nsegs);
+	return (error);
+}
+
 /*
  * Load an mbuf chain.
  */
@@ -658,6 +679,10 @@ bus_dmamap_load_crp_buffer(bus_dma_tag_t dmat, bus_dmamap_t map,
 		error = _bus_dmamap_load_mbuf_sg(dmat, map, cb->cb_mbuf,
 		    NULL, &nsegs, flags);
 		break;
+	case CRYPTO_BUF_SINGLE_MBUF:
+		error = _bus_dmamap_load_single_mbuf(dmat, map, cb->cb_mbuf,
+		    NULL, &nsegs, flags);
+		break;
 	case CRYPTO_BUF_UIO:
 		error = _bus_dmamap_load_uio(dmat, map, cb->cb_uio, &nsegs,
 		    flags);
diff --git a/sys/mips/nlm/dev/sec/nlmsec.c b/sys/mips/nlm/dev/sec/nlmsec.c
index c28eaf334613..d5d83ba796bd 100644
--- a/sys/mips/nlm/dev/sec/nlmsec.c
+++ b/sys/mips/nlm/dev/sec/nlmsec.c
@@ -468,12 +468,15 @@ xlp_get_nsegs(struct cryptop *crp, unsigned int *nsegs)
 
 	switch (crp->crp_buf.cb_type) {
 	case CRYPTO_BUF_MBUF:
+	case CRYPTO_BUF_SINGLE_MBUF:
 	{
 		struct mbuf *m = NULL;
 
 		m = crp->crp_buf.cb_mbuf;
 		while (m != NULL) {
 			*nsegs += NLM_CRYPTO_NUM_SEGS_REQD(m->m_len);
+			if (crp->crp_buf.cb_type == CRYPTO_BUF_SINGLE_MBUF)
+				break;
 			m = m->m_next;
 		}
 		break;
diff --git a/sys/mips/nlm/dev/sec/nlmseclib.c b/sys/mips/nlm/dev/sec/nlmseclib.c
index bb154b5d3d79..6e5e715c28f8 100644
--- a/sys/mips/nlm/dev/sec/nlmseclib.c
+++ b/sys/mips/nlm/dev/sec/nlmseclib.c
@@ -112,6 +112,7 @@ nlm_crypto_form_srcdst_segs(struct xlp_sec_command *cmd,
 
 	switch (crp->crp_buf.cb_type) {
 	case CRYPTO_BUF_MBUF:
+	case CRYPTO_BUF_SINGLE_MBUF:
 	{
 		struct mbuf *m = NULL;
 
@@ -123,6 +124,8 @@ nlm_crypto_form_srcdst_segs(struct xlp_sec_command *cmd,
 				dstseg = nlm_crypto_fill_dst_seg(cmd->paramp,
 				    dstseg, mtod(m,caddr_t), m->m_len);
 			}
+			if (crp->crp_buf.cb_type == CRYPTO_BUF_SINGLE_MBUF)
+				break;
 			m = m->m_next;
 		}
 		break;
diff --git a/sys/opencrypto/criov.c b/sys/opencrypto/criov.c
index bf7965032084..df053b314d41 100644
--- a/sys/opencrypto/criov.c
+++ b/sys/opencrypto/criov.c
@@ -344,6 +344,7 @@ crypto_cursor_init(struct crypto_buffer_cursor *cc,
 		cc->cc_buf_len = cb->cb_buf_len;
 		break;
 	case CRYPTO_BUF_MBUF:
+	case CRYPTO_BUF_SINGLE_MBUF:
 		cc->cc_mbuf = cb->cb_mbuf;
 		break;
 	case CRYPTO_BUF_VMPAGE:
@@ -389,6 +390,10 @@ crypto_cursor_advance(struct crypto_buffer_cursor *cc, size_t amount)
 				break;
 		}
 		break;
+	case CRYPTO_BUF_SINGLE_MBUF:
+		MPASS(cc->cc_mbuf->m_len >= cc->cc_offset + amount);
+		cc->cc_offset += amount;
+		break;
 	case CRYPTO_BUF_VMPAGE:
 		for (;;) {
 			SDT_PROBE2(opencrypto, criov, cursor_advance, vmpage,
@@ -436,6 +441,7 @@ crypto_cursor_segbase(struct crypto_buffer_cursor *cc)
 	case CRYPTO_BUF_CONTIG:
 		return (cc->cc_buf);
 	case CRYPTO_BUF_MBUF:
+	case CRYPTO_BUF_SINGLE_MBUF:
 		if (cc->cc_mbuf == NULL)
 			return (NULL);
 		if (cc->cc_mbuf->m_flags & M_EXTPG)
@@ -463,6 +469,7 @@ crypto_cursor_seglen(struct crypto_buffer_cursor *cc)
 	case CRYPTO_BUF_VMPAGE:
 		return (PAGE_SIZE - cc->cc_offset);
 	case CRYPTO_BUF_MBUF:
+	case CRYPTO_BUF_SINGLE_MBUF:
 		if (cc->cc_mbuf == NULL)
 			return (0);
 		if (cc->cc_mbuf->m_flags & M_EXTPG)
@@ -516,6 +523,11 @@ crypto_cursor_copyback(struct crypto_buffer_cursor *cc, int size,
 				break;
 		}
 		break;
+	case CRYPTO_BUF_SINGLE_MBUF:
+		MPASS(cc->cc_mbuf->m_len >= cc->cc_offset + size);
+		m_copyback(cc->cc_mbuf, cc->cc_offset, size, src);
+		cc->cc_offset += size;
+		break;
 	case CRYPTO_BUF_VMPAGE:
 		for (;;) {
 			dst = (char *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(
@@ -599,6 +611,11 @@ crypto_cursor_copydata(struct crypto_buffer_cursor *cc, int size, void *vdst)
 				break;
 		}
 		break;
+	case CRYPTO_BUF_SINGLE_MBUF:
+		MPASS(cc->cc_mbuf->m_len >= cc->cc_offset + size);
+		m_copydata(cc->cc_mbuf, cc->cc_offset, size, dst);
+		cc->cc_offset += size;
+		break;
 	case CRYPTO_BUF_VMPAGE:
 		for (;;) {
 			src = (char *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(
@@ -699,6 +716,7 @@ crypto_copyback(struct cryptop *crp, int off, int size, const void *src)
 		cb = &crp->crp_buf;
 	switch (cb->cb_type) {
 	case CRYPTO_BUF_MBUF:
+	case CRYPTO_BUF_SINGLE_MBUF:
 		m_copyback(cb->cb_mbuf, off, size, src);
 		break;
 #if CRYPTO_MAY_HAVE_VMPAGE
@@ -731,6 +749,7 @@ crypto_copydata(struct cryptop *crp, int off, int size, void *dst)
 
 	switch (crp->crp_buf.cb_type) {
 	case CRYPTO_BUF_MBUF:
+	case CRYPTO_BUF_SINGLE_MBUF:
 		m_copydata(crp->crp_buf.cb_mbuf, off, size, dst);
 		break;
 #if CRYPTO_MAY_HAVE_VMPAGE
@@ -765,6 +784,7 @@ crypto_apply_buf(struct crypto_buffer *cb, int off, int len,
 
 	switch (cb->cb_type) {
 	case CRYPTO_BUF_MBUF:
+	case CRYPTO_BUF_SINGLE_MBUF:
 		error = m_apply(cb->cb_mbuf, off, len,
 		    (int (*)(void *, void *, u_int))f, arg);
 		break;
@@ -843,6 +863,7 @@ crypto_buffer_contiguous_subsegment(struct crypto_buffer *cb, size_t skip,
 
 	switch (cb->cb_type) {
 	case CRYPTO_BUF_MBUF:
+	case CRYPTO_BUF_SINGLE_MBUF:
 		return (m_contiguous_subsegment(cb->cb_mbuf, skip, len));
 	case CRYPTO_BUF_UIO:
 		return (cuio_contiguous_segment(cb->cb_uio, skip, len));
diff --git a/sys/opencrypto/crypto.c b/sys/opencrypto/crypto.c
index 704077043bb8..bcde910728e1 100644
--- a/sys/opencrypto/crypto.c
+++ b/sys/opencrypto/crypto.c
@@ -1224,6 +1224,8 @@ crypto_buffer_len(struct crypto_buffer *cb)
 		if (cb->cb_mbuf->m_flags & M_PKTHDR)
 			return (cb->cb_mbuf->m_pkthdr.len);
 		return (m_length(cb->cb_mbuf, NULL));
+	case CRYPTO_BUF_SINGLE_MBUF:
+		return (cb->cb_mbuf->m_len);
 	case CRYPTO_BUF_VMPAGE:
 		return (cb->cb_vm_page_len);
 	case CRYPTO_BUF_UIO:
diff --git a/sys/opencrypto/cryptodev.h b/sys/opencrypto/cryptodev.h
index 277867734841..f60f4e63735a 100644
--- a/sys/opencrypto/cryptodev.h
+++ b/sys/opencrypto/cryptodev.h
@@ -392,7 +392,8 @@ enum crypto_buffer_type {
 	CRYPTO_BUF_UIO,
 	CRYPTO_BUF_MBUF,
 	CRYPTO_BUF_VMPAGE,
-	CRYPTO_BUF_LAST = CRYPTO_BUF_VMPAGE
+	CRYPTO_BUF_SINGLE_MBUF,
+	CRYPTO_BUF_LAST = CRYPTO_BUF_SINGLE_MBUF
 };
 
 /*
@@ -524,6 +525,13 @@ _crypto_use_mbuf(struct crypto_buffer *cb, struct mbuf *m)
 	cb->cb_type = CRYPTO_BUF_MBUF;
 }
 
+static __inline void
+_crypto_use_single_mbuf(struct crypto_buffer *cb, struct mbuf *m)
+{
+	cb->cb_mbuf = m;
+	cb->cb_type = CRYPTO_BUF_SINGLE_MBUF;
+}
+
 static __inline void
 _crypto_use_vmpage(struct crypto_buffer *cb, vm_page_t *pages, int len,
     int offset)
@@ -553,6 +561,12 @@ crypto_use_mbuf(struct cryptop *crp, struct mbuf *m)
 	_crypto_use_mbuf(&crp->crp_buf, m);
 }
 
+static __inline void
+crypto_use_single_mbuf(struct cryptop *crp, struct mbuf *m)
+{
+	_crypto_use_single_mbuf(&crp->crp_buf, m);
+}
+
 static __inline void
 crypto_use_vmpage(struct cryptop *crp, vm_page_t *pages, int len, int offset)
 {
@@ -577,6 +591,12 @@ crypto_use_output_mbuf(struct cryptop *crp, struct mbuf *m)
 	_crypto_use_mbuf(&crp->crp_obuf, m);
 }
 
+static __inline void
+crypto_use_output_single_mbuf(struct cryptop *crp, struct mbuf *m)
+{
+	_crypto_use_single_mbuf(&crp->crp_obuf, m);
+}
+
 static __inline void
 crypto_use_output_vmpage(struct cryptop *crp, vm_page_t *pages, int len,
     int offset)
diff --git a/sys/opencrypto/cryptosoft.c b/sys/opencrypto/cryptosoft.c
index a09f085a1eef..4040a014881e 100644
--- a/sys/opencrypto/cryptosoft.c
+++ b/sys/opencrypto/cryptosoft.c
@@ -1105,6 +1105,7 @@ swcr_compdec(struct swcr_session *ses, struct cryptop *crp)
 	if (result < crp->crp_payload_length) {
 		switch (crp->crp_buf.cb_type) {
 		case CRYPTO_BUF_MBUF:
+		case CRYPTO_BUF_SINGLE_MBUF:
 			adj = result - crp->crp_payload_length;
 			m_adj(crp->crp_buf.cb_mbuf, adj);
 			break;