git: 240b7bfe56f2 - main - ipsec_offload: offload inner checksums calculations for UDP/TCP/TSO
- Go to: [ bottom of page ] [ top of archives ] [ this month ]
Date: Fri, 12 Jul 2024 11:25:09 UTC
The branch main has been updated by kib:
URL: https://cgit.FreeBSD.org/src/commit/?id=240b7bfe56f25b20ee248e4fb4a3232c5f13390d
commit 240b7bfe56f25b20ee248e4fb4a3232c5f13390d
Author: Konstantin Belousov <kib@FreeBSD.org>
AuthorDate: 2024-02-14 12:52:56 +0000
Commit: Konstantin Belousov <kib@FreeBSD.org>
CommitDate: 2024-07-12 04:27:58 +0000
ipsec_offload: offload inner checksums calculations for UDP/TCP/TSO
and allow the interface driver to declare such support.
Sponsored by: NVIDIA networking
Differential revision: https://reviews.freebsd.org/D44221
---
sys/net/if_var.h | 3 ++
sys/netipsec/ipsec_offload.c | 5 ++-
sys/netipsec/ipsec_offload.h | 7 +++-
sys/netipsec/ipsec_output.c | 90 +++++++++++++++++++++++---------------------
4 files changed, 60 insertions(+), 45 deletions(-)
diff --git a/sys/net/if_var.h b/sys/net/if_var.h
index 3e094dcb3cd5..579585b25dd2 100644
--- a/sys/net/if_var.h
+++ b/sys/net/if_var.h
@@ -148,6 +148,8 @@ enum IF_SA_CNT_WHICH {
};
typedef int (*if_sa_cnt_fn_t)(if_t ifp, void *sa,
uint32_t drv_spi, void *priv, struct seclifetime *lt);
+typedef int (*if_ipsec_hwassist_fn_t)(if_t ifp, void *sav,
+ u_int drv_spi,void *priv);
struct ifnet_hw_tsomax {
u_int tsomaxbytes; /* TSO total burst length limit in bytes */
@@ -727,6 +729,7 @@ struct if_ipsec_accel_methods {
if_sa_newkey_fn_t if_sa_newkey;
if_sa_deinstall_fn_t if_sa_deinstall;
if_sa_cnt_fn_t if_sa_cnt;
+ if_ipsec_hwassist_fn_t if_hwassist;
};
void if_setipsec_accel_methods(if_t ifp, const struct if_ipsec_accel_methods *);
diff --git a/sys/netipsec/ipsec_offload.c b/sys/netipsec/ipsec_offload.c
index 851bacaf4ea1..7f63a5e0ccb6 100644
--- a/sys/netipsec/ipsec_offload.c
+++ b/sys/netipsec/ipsec_offload.c
@@ -799,12 +799,13 @@ ipsec_accel_output_tag(struct mbuf *m, u_int drv_spi)
bool
ipsec_accel_output(struct ifnet *ifp, struct mbuf *m, struct inpcb *inp,
- struct secpolicy *sp, struct secasvar *sav, int af, int mtu)
+ struct secpolicy *sp, struct secasvar *sav, int af, int mtu, int *hwassist)
{
struct ifp_handle_sav *i;
struct ip *ip;
u_long ip_len, skip;
+ *hwassist = 0;
if (ifp == NULL)
return (false);
@@ -845,6 +846,8 @@ ipsec_accel_output(struct ifnet *ifp, struct mbuf *m, struct inpcb *inp,
if (sp != NULL)
key_freesp(&sp);
+ *hwassist = ifp->if_ipsec_accel_m->if_hwassist(ifp, sav,
+ i->drv_spi, i->ifdata);
return (true);
}
diff --git a/sys/netipsec/ipsec_offload.h b/sys/netipsec/ipsec_offload.h
index 87e2a33288be..27b9c938832e 100644
--- a/sys/netipsec/ipsec_offload.h
+++ b/sys/netipsec/ipsec_offload.h
@@ -173,11 +173,14 @@ void ipsec_accel_spddel_impl(struct secpolicy *sp);
int ipsec_accel_input(struct mbuf *m, int offset, int proto);
bool ipsec_accel_output(struct ifnet *ifp, struct mbuf *m,
struct inpcb *inp, struct secpolicy *sp, struct secasvar *sav, int af,
- int mtu);
+ int mtu, int *hwassist);
void ipsec_accel_forget_sav(struct secasvar *sav);
#else
#define ipsec_accel_input(a, b, c) (ENXIO)
-#define ipsec_accel_output(a, b, c, d, e, f, g) (false)
+#define ipsec_accel_output(a, b, c, d, e, f, g, h) ({ \
+ *h = 0; \
+ false; \
+})
#define ipsec_accel_forget_sav(a)
#endif
diff --git a/sys/netipsec/ipsec_output.c b/sys/netipsec/ipsec_output.c
index 8f49bc8fce24..10f1728f72ac 100644
--- a/sys/netipsec/ipsec_output.c
+++ b/sys/netipsec/ipsec_output.c
@@ -195,7 +195,8 @@ ipsec4_perform_request(struct ifnet *ifp, struct mbuf *m, struct secpolicy *sp,
union sockaddr_union *dst;
struct secasvar *sav;
struct ip *ip;
- int error, i, off;
+ int error, hwassist, i, off;
+ bool accel;
IPSEC_ASSERT(idx < sp->tcount, ("Wrong IPsec request index %d", idx));
@@ -212,7 +213,7 @@ ipsec4_perform_request(struct ifnet *ifp, struct mbuf *m, struct secpolicy *sp,
if (sav == NULL) {
if (error == EJUSTRETURN) { /* No IPsec required */
(void)ipsec_accel_output(ifp, m, inp, sp, NULL,
- AF_INET, mtu);
+ AF_INET, mtu, &hwassist);
key_freesp(&sp);
return (error);
}
@@ -225,7 +226,28 @@ ipsec4_perform_request(struct ifnet *ifp, struct mbuf *m, struct secpolicy *sp,
if ((error = ipsec_run_hhooks(&ctx, HHOOK_TYPE_IPSEC_OUT)) != 0)
goto bad;
- if (ipsec_accel_output(ifp, m, inp, sp, sav, AF_INET, mtu))
+ hwassist = 0;
+ accel = ipsec_accel_output(ifp, m, inp, sp, sav, AF_INET, mtu,
+ &hwassist);
+
+ /*
+ * Do delayed checksums now because we send before
+ * this is done in the normal processing path.
+ */
+ if ((m->m_pkthdr.csum_flags & CSUM_DELAY_DATA & ~hwassist) != 0) {
+ in_delayed_cksum(m);
+ m->m_pkthdr.csum_flags &= ~CSUM_DELAY_DATA;
+ }
+#if defined(SCTP) || defined(SCTP_SUPPORT)
+ if ((m->m_pkthdr.csum_flags & CSUM_SCTP & ~hwassist) != 0) {
+ struct ip *ip;
+
+ ip = mtod(m, struct ip *);
+ sctp_delayed_cksum(m, (uint32_t)(ip->ip_hl << 2));
+ m->m_pkthdr.csum_flags &= ~CSUM_SCTP;
+ }
+#endif
+ if (accel)
return (EJUSTRETURN);
ip = mtod(m, struct ip *);
@@ -401,25 +423,7 @@ ipsec4_common_output(struct ifnet *ifp, struct mbuf *m, struct inpcb *inp,
* packets, and thus, even if they are forwarded, the replies will
* return back to us.
*/
- if (!forwarding) {
- /*
- * Do delayed checksums now because we send before
- * this is done in the normal processing path.
- */
- if (m->m_pkthdr.csum_flags & CSUM_DELAY_DATA) {
- in_delayed_cksum(m);
- m->m_pkthdr.csum_flags &= ~CSUM_DELAY_DATA;
- }
-#if defined(SCTP) || defined(SCTP_SUPPORT)
- if (m->m_pkthdr.csum_flags & CSUM_SCTP) {
- struct ip *ip;
- ip = mtod(m, struct ip *);
- sctp_delayed_cksum(m, (uint32_t)(ip->ip_hl << 2));
- m->m_pkthdr.csum_flags &= ~CSUM_SCTP;
- }
-#endif
- }
/* NB: callee frees mbuf and releases reference to SP */
error = ipsec4_check_pmtu(ifp, m, sp, forwarding);
if (error != 0) {
@@ -596,7 +600,8 @@ ipsec6_perform_request(struct ifnet *ifp, struct mbuf *m, struct secpolicy *sp,
union sockaddr_union *dst;
struct secasvar *sav;
struct ip6_hdr *ip6;
- int error, i, off;
+ int error, hwassist, i, off;
+ bool accel;
IPSEC_ASSERT(idx < sp->tcount, ("Wrong IPsec request index %d", idx));
@@ -604,7 +609,7 @@ ipsec6_perform_request(struct ifnet *ifp, struct mbuf *m, struct secpolicy *sp,
if (sav == NULL) {
if (error == EJUSTRETURN) { /* No IPsec required */
(void)ipsec_accel_output(ifp, m, inp, sp, NULL,
- AF_INET6, mtu);
+ AF_INET6, mtu, &hwassist);
key_freesp(&sp);
return (error);
}
@@ -619,7 +624,26 @@ ipsec6_perform_request(struct ifnet *ifp, struct mbuf *m, struct secpolicy *sp,
if ((error = ipsec_run_hhooks(&ctx, HHOOK_TYPE_IPSEC_OUT)) != 0)
goto bad;
- if (ipsec_accel_output(ifp, m, inp, sp, sav, AF_INET6, mtu))
+ hwassist = 0;
+ accel = ipsec_accel_output(ifp, m, inp, sp, sav, AF_INET6, mtu,
+ &hwassist);
+
+ /*
+ * Do delayed checksums now because we send before
+ * this is done in the normal processing path.
+ */
+ if ((m->m_pkthdr.csum_flags & CSUM_DELAY_DATA_IPV6 & ~hwassist) != 0) {
+ in6_delayed_cksum(m, m->m_pkthdr.len -
+ sizeof(struct ip6_hdr), sizeof(struct ip6_hdr));
+ m->m_pkthdr.csum_flags &= ~CSUM_DELAY_DATA_IPV6;
+ }
+#if defined(SCTP) || defined(SCTP_SUPPORT)
+ if ((m->m_pkthdr.csum_flags & CSUM_SCTP_IPV6 & ~hwassist) != 0) {
+ sctp_delayed_cksum(m, sizeof(struct ip6_hdr));
+ m->m_pkthdr.csum_flags &= ~CSUM_SCTP_IPV6;
+ }
+#endif
+ if (accel)
return (EJUSTRETURN);
ip6 = mtod(m, struct ip6_hdr *); /* pfil can change mbuf */
@@ -778,24 +802,6 @@ ipsec6_common_output(struct ifnet *ifp, struct mbuf *m, struct inpcb *inp,
return (0); /* No IPsec required. */
}
- if (!forwarding) {
- /*
- * Do delayed checksums now because we send before
- * this is done in the normal processing path.
- */
- if (m->m_pkthdr.csum_flags & CSUM_DELAY_DATA_IPV6) {
- in6_delayed_cksum(m, m->m_pkthdr.len -
- sizeof(struct ip6_hdr), sizeof(struct ip6_hdr));
- m->m_pkthdr.csum_flags &= ~CSUM_DELAY_DATA_IPV6;
- }
-#if defined(SCTP) || defined(SCTP_SUPPORT)
- if (m->m_pkthdr.csum_flags & CSUM_SCTP_IPV6) {
- sctp_delayed_cksum(m, sizeof(struct ip6_hdr));
- m->m_pkthdr.csum_flags &= ~CSUM_SCTP_IPV6;
- }
-#endif
- }
-
error = ipsec6_check_pmtu(ifp, m, sp, forwarding);
if (error != 0) {
if (error == EJUSTRETURN)