git: 883a0196b629 - main - crypto: Add a new type of crypto buffer for a single mbuf.
John Baldwin
jhb at FreeBSD.org
Tue May 25 23:59:40 UTC 2021
The branch main has been updated by jhb:
URL: https://cgit.FreeBSD.org/src/commit/?id=883a0196b629a07e52562b4103cc0f6391083080
commit 883a0196b629a07e52562b4103cc0f6391083080
Author: John Baldwin <jhb at FreeBSD.org>
AuthorDate: 2021-05-25 23:59:18 +0000
Commit: John Baldwin <jhb at FreeBSD.org>
CommitDate: 2021-05-25 23:59:18 +0000
crypto: Add a new type of crypto buffer for a single mbuf.
This is intended for use in KTLS transmit where each TLS record is
described by a single mbuf that is itself queued in the socket buffer.
Using the existing CRYPTO_BUF_MBUF would result in
bus_dmamap_load_crp() walking additional mbufs in the socket buffer
that are not relevant, but generating a S/G list that potentially
exceeds the limit of the tag (while also wasting CPU cycles).
Reviewed by: markj
Sponsored by: Netflix
Differential Revision: https://reviews.freebsd.org/D30136
---
share/man/man9/crypto_buffer.9 | 11 ++++++++---
sys/crypto/ccp/ccp.c | 3 +++
sys/dev/cxgbe/crypto/t4_crypto.c | 3 +++
sys/dev/sec/sec.c | 3 +++
sys/kern/subr_bus_dma.c | 25 +++++++++++++++++++++++++
sys/mips/nlm/dev/sec/nlmsec.c | 3 +++
sys/mips/nlm/dev/sec/nlmseclib.c | 3 +++
sys/opencrypto/criov.c | 21 +++++++++++++++++++++
sys/opencrypto/crypto.c | 2 ++
sys/opencrypto/cryptodev.h | 22 +++++++++++++++++++++-
sys/opencrypto/cryptosoft.c | 1 +
11 files changed, 93 insertions(+), 4 deletions(-)
diff --git a/share/man/man9/crypto_buffer.9 b/share/man/man9/crypto_buffer.9
index 333fc5ec449f..df6128650cd4 100644
--- a/share/man/man9/crypto_buffer.9
+++ b/share/man/man9/crypto_buffer.9
@@ -30,7 +30,7 @@
.\"
.\" $FreeBSD$
.\"
-.Dd September 24, 2020
+.Dd May 25, 2021
.Dt CRYPTO_BUFFER 9
.Os
.Sh NAME
@@ -195,7 +195,10 @@ An array of bytes mapped into the kernel's address space.
A scatter/gather list of kernel buffers as described in
.Xr uio 9 .
.It Dv CRYPTO_BUF_MBUF
-A network memory buffer as described in
+A chain of network memory buffers as described in
+.Xr mbuf 9 .
+.It Dv CRYPTO_BUF_SINGLE_MBUF
+A single network memory buffer as described in
.Xr mbuf 9 .
.It Dv CRYPTO_BUF_VMPAGE
A scatter/gather list of
@@ -220,7 +223,9 @@ data buffer
A pointer to a
.Vt struct mbuf
for
-.Dv CRYPTO_BUF_MBUF .
+.Dv CRYPTO_BUF_MBUF
+and
+.Dv CRYPTO_BUF_SINGLE_MBUF .
.It Fa cb_uio
A pointer to a
.Vt struct uio
diff --git a/sys/crypto/ccp/ccp.c b/sys/crypto/ccp/ccp.c
index 63967d7f8e3c..7cc38b14f3fd 100644
--- a/sys/crypto/ccp/ccp.c
+++ b/sys/crypto/ccp/ccp.c
@@ -101,6 +101,9 @@ ccp_populate_sglist(struct sglist *sg, struct crypto_buffer *cb)
case CRYPTO_BUF_MBUF:
error = sglist_append_mbuf(sg, cb->cb_mbuf);
break;
+ case CRYPTO_BUF_SINGLE_MBUF:
+ error = sglist_append_single_mbuf(sg, cb->cb_mbuf);
+ break;
case CRYPTO_BUF_UIO:
error = sglist_append_uio(sg, cb->cb_uio);
break;
diff --git a/sys/dev/cxgbe/crypto/t4_crypto.c b/sys/dev/cxgbe/crypto/t4_crypto.c
index cdd14fcee2f9..0fc806dc2eb8 100644
--- a/sys/dev/cxgbe/crypto/t4_crypto.c
+++ b/sys/dev/cxgbe/crypto/t4_crypto.c
@@ -271,6 +271,9 @@ ccr_populate_sglist(struct sglist *sg, struct crypto_buffer *cb)
case CRYPTO_BUF_MBUF:
error = sglist_append_mbuf(sg, cb->cb_mbuf);
break;
+ case CRYPTO_BUF_SINGLE_MBUF:
+ error = sglist_append_single_mbuf(sg, cb->cb_mbuf);
+ break;
case CRYPTO_BUF_UIO:
error = sglist_append_uio(sg, cb->cb_uio);
break;
diff --git a/sys/dev/sec/sec.c b/sys/dev/sec/sec.c
index fa3e20806124..f485d932727b 100644
--- a/sys/dev/sec/sec.c
+++ b/sys/dev/sec/sec.c
@@ -850,6 +850,9 @@ sec_desc_map_dma(struct sec_softc *sc, struct sec_dma_mem *dma_mem,
case CRYPTO_BUF_MBUF:
size = m_length(crp->crp_buf.cb_mbuf, NULL);
break;
+ case CRYPTO_BUF_SINGLE_MBUF:
+ size = crp->crp_buf.cb_mbuf->m_len;
+ break;
case CRYPTO_BUF_VMPAGE:
size = PAGE_SIZE - crp->crp_buf.cb_vm_page_offset;
break;
diff --git a/sys/kern/subr_bus_dma.c b/sys/kern/subr_bus_dma.c
index 6248dd520511..a7ad30e77715 100644
--- a/sys/kern/subr_bus_dma.c
+++ b/sys/kern/subr_bus_dma.c
@@ -172,6 +172,27 @@ _bus_dmamap_load_mbuf_epg(bus_dma_tag_t dmat, bus_dmamap_t map,
return (error);
}
+/*
+ * Load a single mbuf.
+ */
+static int
+_bus_dmamap_load_single_mbuf(bus_dma_tag_t dmat, bus_dmamap_t map,
+ struct mbuf *m, bus_dma_segment_t *segs, int *nsegs, int flags)
+{
+ int error;
+
+ error = 0;
+ if ((m->m_flags & M_EXTPG) != 0)
+ error = _bus_dmamap_load_mbuf_epg(dmat, map, m, segs, nsegs,
+ flags);
+ else
+ error = _bus_dmamap_load_buffer(dmat, map, m->m_data, m->m_len,
+ kernel_pmap, flags | BUS_DMA_LOAD_MBUF, segs, nsegs);
+ CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d",
+ __func__, dmat, flags, error, *nsegs);
+ return (error);
+}
+
/*
* Load an mbuf chain.
*/
@@ -658,6 +679,10 @@ bus_dmamap_load_crp_buffer(bus_dma_tag_t dmat, bus_dmamap_t map,
error = _bus_dmamap_load_mbuf_sg(dmat, map, cb->cb_mbuf,
NULL, &nsegs, flags);
break;
+ case CRYPTO_BUF_SINGLE_MBUF:
+ error = _bus_dmamap_load_single_mbuf(dmat, map, cb->cb_mbuf,
+ NULL, &nsegs, flags);
+ break;
case CRYPTO_BUF_UIO:
error = _bus_dmamap_load_uio(dmat, map, cb->cb_uio, &nsegs,
flags);
diff --git a/sys/mips/nlm/dev/sec/nlmsec.c b/sys/mips/nlm/dev/sec/nlmsec.c
index 9318206a04fd..fb3e0b65b5a1 100644
--- a/sys/mips/nlm/dev/sec/nlmsec.c
+++ b/sys/mips/nlm/dev/sec/nlmsec.c
@@ -468,12 +468,15 @@ xlp_get_nsegs(struct cryptop *crp, unsigned int *nsegs)
switch (crp->crp_buf.cb_type) {
case CRYPTO_BUF_MBUF:
+ case CRYPTO_BUF_SINGLE_MBUF:
{
struct mbuf *m = NULL;
m = crp->crp_buf.cb_mbuf;
while (m != NULL) {
*nsegs += NLM_CRYPTO_NUM_SEGS_REQD(m->m_len);
+ if (crp->crp_buf.cb_type == CRYPTO_BUF_SINGLE_MBUF)
+ break;
m = m->m_next;
}
break;
diff --git a/sys/mips/nlm/dev/sec/nlmseclib.c b/sys/mips/nlm/dev/sec/nlmseclib.c
index bb154b5d3d79..6e5e715c28f8 100644
--- a/sys/mips/nlm/dev/sec/nlmseclib.c
+++ b/sys/mips/nlm/dev/sec/nlmseclib.c
@@ -112,6 +112,7 @@ nlm_crypto_form_srcdst_segs(struct xlp_sec_command *cmd,
switch (crp->crp_buf.cb_type) {
case CRYPTO_BUF_MBUF:
+ case CRYPTO_BUF_SINGLE_MBUF:
{
struct mbuf *m = NULL;
@@ -123,6 +124,8 @@ nlm_crypto_form_srcdst_segs(struct xlp_sec_command *cmd,
dstseg = nlm_crypto_fill_dst_seg(cmd->paramp,
dstseg, mtod(m,caddr_t), m->m_len);
}
+ if (crp->crp_buf.cb_type == CRYPTO_BUF_SINGLE_MBUF)
+ break;
m = m->m_next;
}
break;
diff --git a/sys/opencrypto/criov.c b/sys/opencrypto/criov.c
index bf7965032084..df053b314d41 100644
--- a/sys/opencrypto/criov.c
+++ b/sys/opencrypto/criov.c
@@ -344,6 +344,7 @@ crypto_cursor_init(struct crypto_buffer_cursor *cc,
cc->cc_buf_len = cb->cb_buf_len;
break;
case CRYPTO_BUF_MBUF:
+ case CRYPTO_BUF_SINGLE_MBUF:
cc->cc_mbuf = cb->cb_mbuf;
break;
case CRYPTO_BUF_VMPAGE:
@@ -389,6 +390,10 @@ crypto_cursor_advance(struct crypto_buffer_cursor *cc, size_t amount)
break;
}
break;
+ case CRYPTO_BUF_SINGLE_MBUF:
+ MPASS(cc->cc_mbuf->m_len >= cc->cc_offset + amount);
+ cc->cc_offset += amount;
+ break;
case CRYPTO_BUF_VMPAGE:
for (;;) {
SDT_PROBE2(opencrypto, criov, cursor_advance, vmpage,
@@ -436,6 +441,7 @@ crypto_cursor_segbase(struct crypto_buffer_cursor *cc)
case CRYPTO_BUF_CONTIG:
return (cc->cc_buf);
case CRYPTO_BUF_MBUF:
+ case CRYPTO_BUF_SINGLE_MBUF:
if (cc->cc_mbuf == NULL)
return (NULL);
if (cc->cc_mbuf->m_flags & M_EXTPG)
@@ -463,6 +469,7 @@ crypto_cursor_seglen(struct crypto_buffer_cursor *cc)
case CRYPTO_BUF_VMPAGE:
return (PAGE_SIZE - cc->cc_offset);
case CRYPTO_BUF_MBUF:
+ case CRYPTO_BUF_SINGLE_MBUF:
if (cc->cc_mbuf == NULL)
return (0);
if (cc->cc_mbuf->m_flags & M_EXTPG)
@@ -516,6 +523,11 @@ crypto_cursor_copyback(struct crypto_buffer_cursor *cc, int size,
break;
}
break;
+ case CRYPTO_BUF_SINGLE_MBUF:
+ MPASS(cc->cc_mbuf->m_len >= cc->cc_offset + size);
+ m_copyback(cc->cc_mbuf, cc->cc_offset, size, src);
+ cc->cc_offset += size;
+ break;
case CRYPTO_BUF_VMPAGE:
for (;;) {
dst = (char *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(
@@ -599,6 +611,11 @@ crypto_cursor_copydata(struct crypto_buffer_cursor *cc, int size, void *vdst)
break;
}
break;
+ case CRYPTO_BUF_SINGLE_MBUF:
+ MPASS(cc->cc_mbuf->m_len >= cc->cc_offset + size);
+ m_copydata(cc->cc_mbuf, cc->cc_offset, size, dst);
+ cc->cc_offset += size;
+ break;
case CRYPTO_BUF_VMPAGE:
for (;;) {
src = (char *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(
@@ -699,6 +716,7 @@ crypto_copyback(struct cryptop *crp, int off, int size, const void *src)
cb = &crp->crp_buf;
switch (cb->cb_type) {
case CRYPTO_BUF_MBUF:
+ case CRYPTO_BUF_SINGLE_MBUF:
m_copyback(cb->cb_mbuf, off, size, src);
break;
#if CRYPTO_MAY_HAVE_VMPAGE
@@ -731,6 +749,7 @@ crypto_copydata(struct cryptop *crp, int off, int size, void *dst)
switch (crp->crp_buf.cb_type) {
case CRYPTO_BUF_MBUF:
+ case CRYPTO_BUF_SINGLE_MBUF:
m_copydata(crp->crp_buf.cb_mbuf, off, size, dst);
break;
#if CRYPTO_MAY_HAVE_VMPAGE
@@ -765,6 +784,7 @@ crypto_apply_buf(struct crypto_buffer *cb, int off, int len,
switch (cb->cb_type) {
case CRYPTO_BUF_MBUF:
+ case CRYPTO_BUF_SINGLE_MBUF:
error = m_apply(cb->cb_mbuf, off, len,
(int (*)(void *, void *, u_int))f, arg);
break;
@@ -843,6 +863,7 @@ crypto_buffer_contiguous_subsegment(struct crypto_buffer *cb, size_t skip,
switch (cb->cb_type) {
case CRYPTO_BUF_MBUF:
+ case CRYPTO_BUF_SINGLE_MBUF:
return (m_contiguous_subsegment(cb->cb_mbuf, skip, len));
case CRYPTO_BUF_UIO:
return (cuio_contiguous_segment(cb->cb_uio, skip, len));
diff --git a/sys/opencrypto/crypto.c b/sys/opencrypto/crypto.c
index b56d0fb70698..b6c4441b4284 100644
--- a/sys/opencrypto/crypto.c
+++ b/sys/opencrypto/crypto.c
@@ -1155,6 +1155,8 @@ crypto_buffer_len(struct crypto_buffer *cb)
if (cb->cb_mbuf->m_flags & M_PKTHDR)
return (cb->cb_mbuf->m_pkthdr.len);
return (m_length(cb->cb_mbuf, NULL));
+ case CRYPTO_BUF_SINGLE_MBUF:
+ return (cb->cb_mbuf->m_len);
case CRYPTO_BUF_VMPAGE:
return (cb->cb_vm_page_len);
case CRYPTO_BUF_UIO:
diff --git a/sys/opencrypto/cryptodev.h b/sys/opencrypto/cryptodev.h
index 862ad1d4f6e5..1adc81f2e4d3 100644
--- a/sys/opencrypto/cryptodev.h
+++ b/sys/opencrypto/cryptodev.h
@@ -354,7 +354,8 @@ enum crypto_buffer_type {
CRYPTO_BUF_UIO,
CRYPTO_BUF_MBUF,
CRYPTO_BUF_VMPAGE,
- CRYPTO_BUF_LAST = CRYPTO_BUF_VMPAGE
+ CRYPTO_BUF_SINGLE_MBUF,
+ CRYPTO_BUF_LAST = CRYPTO_BUF_SINGLE_MBUF
};
/*
@@ -480,6 +481,13 @@ _crypto_use_mbuf(struct crypto_buffer *cb, struct mbuf *m)
cb->cb_type = CRYPTO_BUF_MBUF;
}
+static __inline void
+_crypto_use_single_mbuf(struct crypto_buffer *cb, struct mbuf *m)
+{
+ cb->cb_mbuf = m;
+ cb->cb_type = CRYPTO_BUF_SINGLE_MBUF;
+}
+
static __inline void
_crypto_use_vmpage(struct crypto_buffer *cb, vm_page_t *pages, int len,
int offset)
@@ -509,6 +517,12 @@ crypto_use_mbuf(struct cryptop *crp, struct mbuf *m)
_crypto_use_mbuf(&crp->crp_buf, m);
}
+static __inline void
+crypto_use_single_mbuf(struct cryptop *crp, struct mbuf *m)
+{
+ _crypto_use_single_mbuf(&crp->crp_buf, m);
+}
+
static __inline void
crypto_use_vmpage(struct cryptop *crp, vm_page_t *pages, int len, int offset)
{
@@ -533,6 +547,12 @@ crypto_use_output_mbuf(struct cryptop *crp, struct mbuf *m)
_crypto_use_mbuf(&crp->crp_obuf, m);
}
+static __inline void
+crypto_use_output_single_mbuf(struct cryptop *crp, struct mbuf *m)
+{
+ _crypto_use_single_mbuf(&crp->crp_obuf, m);
+}
+
static __inline void
crypto_use_output_vmpage(struct cryptop *crp, vm_page_t *pages, int len,
int offset)
diff --git a/sys/opencrypto/cryptosoft.c b/sys/opencrypto/cryptosoft.c
index a09f085a1eef..4040a014881e 100644
--- a/sys/opencrypto/cryptosoft.c
+++ b/sys/opencrypto/cryptosoft.c
@@ -1105,6 +1105,7 @@ swcr_compdec(struct swcr_session *ses, struct cryptop *crp)
if (result < crp->crp_payload_length) {
switch (crp->crp_buf.cb_type) {
case CRYPTO_BUF_MBUF:
+ case CRYPTO_BUF_SINGLE_MBUF:
adj = result - crp->crp_payload_length;
m_adj(crp->crp_buf.cb_mbuf, adj);
break;
More information about the dev-commits-src-all
mailing list