svn commit: r360417 - head/sys/dev/cxgbe/tom
John Baldwin
jhb at FreeBSD.org
Mon Apr 27 23:59:43 UTC 2020
Author: jhb
Date: Mon Apr 27 23:59:42 2020
New Revision: 360417
URL: https://svnweb.freebsd.org/changeset/base/360417
Log:
Add support for KTLS RX over TOE to T6.
This largely reuses the TLS TOE support added in r330884. However,
this uses the KTLS framework in upstream OpenSSL rather than requiring
Chelsio-specific patches to OpenSSL. As with the existing TLS TOE
support, use of RX offload requires setting the tls_rx_ports sysctl.
Reviewed by: np
Sponsored by: Chelsio Communications
Differential Revision: https://reviews.freebsd.org/D24453
Modified:
head/sys/dev/cxgbe/tom/t4_tls.c
head/sys/dev/cxgbe/tom/t4_tom.c
head/sys/dev/cxgbe/tom/t4_tom.h
Modified: head/sys/dev/cxgbe/tom/t4_tls.c
==============================================================================
--- head/sys/dev/cxgbe/tom/t4_tls.c Mon Apr 27 23:55:09 2020 (r360416)
+++ head/sys/dev/cxgbe/tom/t4_tls.c Mon Apr 27 23:59:42 2020 (r360417)
@@ -379,7 +379,7 @@ prepare_rxkey_wr(struct tls_keyctx *kwr, struct tls_ke
int proto_ver = kctx->proto_ver;
kwr->u.rxhdr.flitcnt_hmacctrl =
- ((kctx->tx_key_info_size >> 4) << 3) | kctx->hmac_ctrl;
+ ((kctx->rx_key_info_size >> 4) << 3) | kctx->hmac_ctrl;
kwr->u.rxhdr.protover_ciphmode =
V_TLS_KEYCTX_TX_WR_PROTOVER(get_proto_ver(proto_ver)) |
@@ -408,7 +408,7 @@ prepare_rxkey_wr(struct tls_keyctx *kwr, struct tls_ke
(IPAD_SIZE + OPAD_SIZE));
} else {
memcpy(kwr->keys.edkey, kctx->rx.key,
- (kctx->tx_key_info_size - SALT_SIZE));
+ (kctx->rx_key_info_size - SALT_SIZE));
memcpy(kwr->u.rxhdr.rxsalt, kctx->rx.salt, SALT_SIZE);
}
}
@@ -674,6 +674,13 @@ program_key_context(struct tcpcb *tp, struct toepcb *t
if ((G_KEY_GET_LOC(k_ctx->l_p_key) == KEY_WRITE_RX) ||
(tls_ofld->key_location == TLS_SFO_WR_CONTEXTLOC_DDR)) {
+
+ /*
+ * XXX: The userland library sets tx_key_info_size, not
+ * rx_key_info_size.
+ */
+ k_ctx->rx_key_info_size = k_ctx->tx_key_info_size;
+
error = tls_program_key_id(toep, k_ctx);
if (error) {
/* XXX: Only clear quiesce for KEY_WRITE_RX? */
@@ -866,31 +873,37 @@ t4_ctloutput_tls(struct socket *so, struct sockopt *so
#ifdef KERN_TLS
static void
-init_ktls_key_context(struct ktls_session *tls, struct tls_key_context *k_ctx)
+init_ktls_key_context(struct ktls_session *tls, struct tls_key_context *k_ctx,
+ int direction)
{
struct auth_hash *axf;
- u_int mac_key_size;
- char *hash;
+ u_int key_info_size, mac_key_size;
+ char *hash, *key;
- k_ctx->l_p_key = V_KEY_GET_LOC(KEY_WRITE_TX);
- if (tls->params.tls_vminor == TLS_MINOR_VER_ONE)
- k_ctx->proto_ver = SCMD_PROTO_VERSION_TLS_1_1;
- else
- k_ctx->proto_ver = SCMD_PROTO_VERSION_TLS_1_2;
+ k_ctx->l_p_key = V_KEY_GET_LOC(direction == KTLS_TX ? KEY_WRITE_TX :
+ KEY_WRITE_RX);
+ k_ctx->proto_ver = tls->params.tls_vmajor << 8 | tls->params.tls_vminor;
k_ctx->cipher_secret_size = tls->params.cipher_key_len;
- k_ctx->tx_key_info_size = sizeof(struct tx_keyctx_hdr) +
+ key_info_size = sizeof(struct tx_keyctx_hdr) +
k_ctx->cipher_secret_size;
- memcpy(k_ctx->tx.key, tls->params.cipher_key,
- tls->params.cipher_key_len);
- hash = k_ctx->tx.key + tls->params.cipher_key_len;
+ if (direction == KTLS_TX)
+ key = k_ctx->tx.key;
+ else
+ key = k_ctx->rx.key;
+ memcpy(key, tls->params.cipher_key, tls->params.cipher_key_len);
+ hash = key + tls->params.cipher_key_len;
if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16) {
k_ctx->state.auth_mode = SCMD_AUTH_MODE_GHASH;
k_ctx->state.enc_mode = SCMD_CIPH_MODE_AES_GCM;
k_ctx->iv_size = 4;
k_ctx->mac_first = 0;
k_ctx->hmac_ctrl = SCMD_HMAC_CTRL_NOP;
- k_ctx->tx_key_info_size += GMAC_BLOCK_LEN;
- memcpy(k_ctx->tx.salt, tls->params.iv, SALT_SIZE);
+ key_info_size += GMAC_BLOCK_LEN;
+ k_ctx->mac_secret_size = 0;
+ if (direction == KTLS_TX)
+ memcpy(k_ctx->tx.salt, tls->params.iv, SALT_SIZE);
+ else
+ memcpy(k_ctx->rx.salt, tls->params.iv, SALT_SIZE);
t4_init_gmac_hash(tls->params.cipher_key,
tls->params.cipher_key_len, hash);
} else {
@@ -917,29 +930,38 @@ init_ktls_key_context(struct ktls_session *tls, struct
k_ctx->iv_size = 8; /* for CBC, iv is 16B, unit of 2B */
k_ctx->mac_first = 1;
k_ctx->hmac_ctrl = SCMD_HMAC_CTRL_NO_TRUNC;
- k_ctx->tx_key_info_size += roundup2(mac_key_size, 16) * 2;
+ key_info_size += roundup2(mac_key_size, 16) * 2;
k_ctx->mac_secret_size = mac_key_size;
t4_init_hmac_digest(axf, mac_key_size, tls->params.auth_key,
tls->params.auth_key_len, hash);
}
+ if (direction == KTLS_TX)
+ k_ctx->tx_key_info_size = key_info_size;
+ else
+ k_ctx->rx_key_info_size = key_info_size;
k_ctx->frag_size = tls->params.max_frame_len;
k_ctx->iv_ctrl = 1;
}
int
-tls_alloc_ktls(struct toepcb *toep, struct ktls_session *tls)
+tls_alloc_ktls(struct toepcb *toep, struct ktls_session *tls, int direction)
{
+ struct adapter *sc = td_adapter(toep->td);
struct tls_key_context *k_ctx;
- int error;
+ int error, key_offset;
if (toep->tls.mode == TLS_MODE_TLSOM)
return (EINVAL);
if (!can_tls_offload(td_adapter(toep->td)))
return (EINVAL);
switch (ulp_mode(toep)) {
+ case ULP_MODE_TLS:
+ break;
case ULP_MODE_NONE:
case ULP_MODE_TCPDDP:
+ if (direction != KTLS_TX)
+ return (EINVAL);
break;
default:
return (EINVAL);
@@ -987,48 +1009,81 @@ tls_alloc_ktls(struct toepcb *toep, struct ktls_sessio
tls->params.tls_vminor > TLS_MINOR_VER_TWO)
return (EPROTONOSUPPORT);
+ /* Bail if we already have a key. */
+ if (direction == KTLS_TX) {
+ if (toep->tls.tx_key_addr != -1)
+ return (EOPNOTSUPP);
+ } else {
+ if (toep->tls.rx_key_addr != -1)
+ return (EOPNOTSUPP);
+ }
+
/*
* XXX: This assumes no key renegotation. If KTLS ever supports
* that we will want to allocate TLS sessions dynamically rather
* than as a static member of toep.
*/
k_ctx = &toep->tls.k_ctx;
- init_ktls_key_context(tls, k_ctx);
+ init_ktls_key_context(tls, k_ctx, direction);
- toep->tls.scmd0.seqno_numivs =
- (V_SCMD_SEQ_NO_CTRL(3) |
- V_SCMD_PROTO_VERSION(k_ctx->proto_ver) |
- V_SCMD_ENC_DEC_CTRL(SCMD_ENCDECCTRL_ENCRYPT) |
- V_SCMD_CIPH_AUTH_SEQ_CTRL((k_ctx->mac_first == 0)) |
- V_SCMD_CIPH_MODE(k_ctx->state.enc_mode) |
- V_SCMD_AUTH_MODE(k_ctx->state.auth_mode) |
- V_SCMD_HMAC_CTRL(k_ctx->hmac_ctrl) |
- V_SCMD_IV_SIZE(k_ctx->iv_size));
+ error = tls_program_key_id(toep, k_ctx);
+ if (error)
+ return (error);
- toep->tls.scmd0.ivgen_hdrlen =
- (V_SCMD_IV_GEN_CTRL(k_ctx->iv_ctrl) |
- V_SCMD_KEY_CTX_INLINE(0) |
- V_SCMD_TLS_FRAG_ENABLE(1));
+ if (direction == KTLS_TX) {
+ toep->tls.scmd0.seqno_numivs =
+ (V_SCMD_SEQ_NO_CTRL(3) |
+ V_SCMD_PROTO_VERSION(get_proto_ver(k_ctx->proto_ver)) |
+ V_SCMD_ENC_DEC_CTRL(SCMD_ENCDECCTRL_ENCRYPT) |
+ V_SCMD_CIPH_AUTH_SEQ_CTRL((k_ctx->mac_first == 0)) |
+ V_SCMD_CIPH_MODE(k_ctx->state.enc_mode) |
+ V_SCMD_AUTH_MODE(k_ctx->state.auth_mode) |
+ V_SCMD_HMAC_CTRL(k_ctx->hmac_ctrl) |
+ V_SCMD_IV_SIZE(k_ctx->iv_size));
- if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16)
- toep->tls.iv_len = 8;
- else
- toep->tls.iv_len = AES_BLOCK_LEN;
+ toep->tls.scmd0.ivgen_hdrlen =
+ (V_SCMD_IV_GEN_CTRL(k_ctx->iv_ctrl) |
+ V_SCMD_KEY_CTX_INLINE(0) |
+ V_SCMD_TLS_FRAG_ENABLE(1));
- toep->tls.mac_length = k_ctx->mac_secret_size;
+ if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16)
+ toep->tls.iv_len = 8;
+ else
+ toep->tls.iv_len = AES_BLOCK_LEN;
- toep->tls.tx_key_addr = -1;
+ toep->tls.mac_length = k_ctx->mac_secret_size;
- error = tls_program_key_id(toep, k_ctx);
- if (error)
- return (error);
+ toep->tls.fcplenmax = get_tp_plen_max(&toep->tls);
+ toep->tls.expn_per_ulp = tls->params.tls_hlen +
+ tls->params.tls_tlen;
+ toep->tls.pdus_per_ulp = 1;
+ toep->tls.adjusted_plen = toep->tls.expn_per_ulp +
+ toep->tls.k_ctx.frag_size;
+ } else {
+ /* Stop timer on handshake completion */
+ tls_stop_handshake_timer(toep);
- toep->tls.fcplenmax = get_tp_plen_max(&toep->tls);
- toep->tls.expn_per_ulp = tls->params.tls_hlen + tls->params.tls_tlen;
- toep->tls.pdus_per_ulp = 1;
- toep->tls.adjusted_plen = toep->tls.expn_per_ulp +
- toep->tls.k_ctx.frag_size;
+ toep->flags &= ~TPF_FORCE_CREDITS;
+ /*
+ * RX key tags are an index into the key portion of MA
+ * memory stored as an offset from the base address in
+ * units of 64 bytes.
+ */
+ key_offset = toep->tls.rx_key_addr - sc->vres.key.start;
+ t4_set_tls_keyid(toep, key_offset / 64);
+ t4_set_tls_tcb_field(toep, W_TCB_ULP_RAW,
+ V_TCB_ULP_RAW(M_TCB_ULP_RAW),
+ V_TCB_ULP_RAW((V_TF_TLS_KEY_SIZE(3) |
+ V_TF_TLS_CONTROL(1) |
+ V_TF_TLS_ACTIVE(1) |
+ V_TF_TLS_ENABLE(1))));
+ t4_set_tls_tcb_field(toep, W_TCB_TLS_SEQ,
+ V_TCB_TLS_SEQ(M_TCB_TLS_SEQ),
+ V_TCB_TLS_SEQ(0));
+ t4_clear_rx_quiesce(toep);
+ }
+
toep->tls.mode = TLS_MODE_KTLS;
return (0);
@@ -1671,7 +1726,7 @@ t4_push_ktls(struct adapter *sc, struct toepcb *toep,
("%s: flowc_wr not sent for tid %u.", __func__, toep->tid));
KASSERT(ulp_mode(toep) == ULP_MODE_NONE ||
- ulp_mode(toep) == ULP_MODE_TCPDDP,
+ ulp_mode(toep) == ULP_MODE_TCPDDP || ulp_mode(toep) == ULP_MODE_TLS,
("%s: ulp_mode %u for toep %p", __func__, ulp_mode(toep), toep));
KASSERT(tls_tx_key(toep),
("%s: TX key not set for toep %p", __func__, toep));
@@ -1956,6 +2011,10 @@ do_rx_tls_cmp(struct sge_iq *iq, const struct rss_head
struct socket *so;
struct sockbuf *sb;
struct mbuf *tls_data;
+#ifdef KERN_TLS
+ struct tls_get_record *tgr;
+ struct mbuf *control;
+#endif
int len, pdu_length, rx_credits;
KASSERT(toep->tid == tid, ("%s: toep tid/atid mismatch", __func__));
@@ -1982,6 +2041,7 @@ do_rx_tls_cmp(struct sge_iq *iq, const struct rss_head
pdu_length = G_CPL_RX_TLS_CMP_PDULENGTH(be32toh(cpl->pdulength_length));
+ so = inp_inpcbtosocket(inp);
tp = intotcpcb(inp);
#ifdef VERBOSE_TRACES
@@ -2006,35 +2066,94 @@ do_rx_tls_cmp(struct sge_iq *iq, const struct rss_head
("%s: payload too small", __func__));
tls_hdr_pkt = mtod(m, void *);
- /*
- * Only the TLS header is sent to OpenSSL, so report errors by
- * altering the record type.
- */
- if ((tls_hdr_pkt->res_to_mac_error & M_TLSRX_HDR_PKT_ERROR) != 0)
- tls_hdr_pkt->type = CONTENT_TYPE_ERROR;
-
- /* Trim this CPL's mbuf to only include the TLS header. */
- KASSERT(m->m_len == len && m->m_next == NULL,
- ("%s: CPL spans multiple mbufs", __func__));
- m->m_len = TLS_HEADER_LENGTH;
- m->m_pkthdr.len = TLS_HEADER_LENGTH;
-
tls_data = mbufq_dequeue(&toep->ulp_pdu_reclaimq);
if (tls_data != NULL) {
KASSERT(be32toh(cpl->seq) == tls_data->m_pkthdr.tls_tcp_seq,
("%s: sequence mismatch", __func__));
+ }
+#ifdef KERN_TLS
+ if (toep->tls.mode == TLS_MODE_KTLS) {
+ /* Report decryption errors as EBADMSG. */
+ if ((tls_hdr_pkt->res_to_mac_error & M_TLSRX_HDR_PKT_ERROR) !=
+ 0) {
+ m_freem(m);
+ m_freem(tls_data);
+
+ CURVNET_SET(toep->vnet);
+ so->so_error = EBADMSG;
+ sorwakeup(so);
+
+ INP_WUNLOCK(inp);
+ CURVNET_RESTORE();
+
+ return (0);
+ }
+
+ /* Allocate the control message mbuf. */
+ control = sbcreatecontrol(NULL, sizeof(*tgr), TLS_GET_RECORD,
+ IPPROTO_TCP);
+ if (control == NULL) {
+ m_freem(m);
+ m_freem(tls_data);
+
+ CURVNET_SET(toep->vnet);
+ so->so_error = ENOBUFS;
+ sorwakeup(so);
+
+ INP_WUNLOCK(inp);
+ CURVNET_RESTORE();
+
+ return (0);
+ }
+
+ tgr = (struct tls_get_record *)
+ CMSG_DATA(mtod(control, struct cmsghdr *));
+ tgr->tls_type = tls_hdr_pkt->type;
+ tgr->tls_vmajor = be16toh(tls_hdr_pkt->version) >> 8;
+ tgr->tls_vminor = be16toh(tls_hdr_pkt->version) & 0xff;
+
+ m_freem(m);
+
+ if (tls_data != NULL) {
+ m_last(tls_data)->m_flags |= M_EOR;
+ tgr->tls_length = htobe16(tls_data->m_pkthdr.len);
+ } else
+ tgr->tls_length = 0;
+ m = tls_data;
+ } else
+#endif
+ {
/*
- * Update the TLS header length to be the length of
- * the payload data.
+ * Only the TLS header is sent to OpenSSL, so report
+ * errors by altering the record type.
*/
- tls_hdr_pkt->length = htobe16(tls_data->m_pkthdr.len);
+ if ((tls_hdr_pkt->res_to_mac_error & M_TLSRX_HDR_PKT_ERROR) !=
+ 0)
+ tls_hdr_pkt->type = CONTENT_TYPE_ERROR;
- m->m_next = tls_data;
- m->m_pkthdr.len += tls_data->m_len;
+ /* Trim this CPL's mbuf to only include the TLS header. */
+ KASSERT(m->m_len == len && m->m_next == NULL,
+ ("%s: CPL spans multiple mbufs", __func__));
+ m->m_len = TLS_HEADER_LENGTH;
+ m->m_pkthdr.len = TLS_HEADER_LENGTH;
+
+ if (tls_data != NULL) {
+ /*
+ * Update the TLS header length to be the length of
+ * the payload data.
+ */
+ tls_hdr_pkt->length = htobe16(tls_data->m_pkthdr.len);
+
+ m->m_next = tls_data;
+ m->m_pkthdr.len += tls_data->m_len;
+ }
+
+#ifdef KERN_TLS
+ control = NULL;
+#endif
}
- so = inp_inpcbtosocket(inp);
sb = &so->so_rcv;
SOCKBUF_LOCK(sb);
@@ -2044,6 +2163,9 @@ do_rx_tls_cmp(struct sge_iq *iq, const struct rss_head
CTR3(KTR_CXGBE, "%s: tid %u, excess rx (%d bytes)",
__func__, tid, pdu_length);
m_freem(m);
+#ifdef KERN_TLS
+ m_freem(control);
+#endif
SOCKBUF_UNLOCK(sb);
INP_WUNLOCK(inp);
@@ -2080,7 +2202,12 @@ do_rx_tls_cmp(struct sge_iq *iq, const struct rss_head
sb->sb_flags &= ~SB_AUTOSIZE;
}
- sbappendstream_locked(sb, m, 0);
+#ifdef KERN_TLS
+ if (control != NULL)
+ sbappendcontrol_locked(sb, m, control, 0);
+ else
+#endif
+ sbappendstream_locked(sb, m, 0);
rx_credits = sbspace(sb) > tp->rcv_wnd ? sbspace(sb) - tp->rcv_wnd : 0;
#ifdef VERBOSE_TRACES
CTR4(KTR_CXGBE, "%s: tid %u rx_credits %u rcv_wnd %u",
Modified: head/sys/dev/cxgbe/tom/t4_tom.c
==============================================================================
--- head/sys/dev/cxgbe/tom/t4_tom.c Mon Apr 27 23:55:09 2020 (r360416)
+++ head/sys/dev/cxgbe/tom/t4_tom.c Mon Apr 27 23:59:42 2020 (r360417)
@@ -40,9 +40,6 @@ __FBSDID("$FreeBSD$");
#include <sys/systm.h>
#include <sys/kernel.h>
#include <sys/ktr.h>
-#ifdef KERN_TLS
-#include <sys/ktls.h>
-#endif
#include <sys/lock.h>
#include <sys/limits.h>
#include <sys/module.h>
@@ -824,10 +821,7 @@ t4_alloc_tls_session(struct toedev *tod, struct tcpcb
INP_WLOCK_ASSERT(tp->t_inpcb);
MPASS(tls != NULL);
- if (direction != KTLS_TX)
- return (EOPNOTSUPP);
-
- return (tls_alloc_ktls(toep, tls));
+ return (tls_alloc_ktls(toep, tls, direction));
}
#endif
Modified: head/sys/dev/cxgbe/tom/t4_tom.h
==============================================================================
--- head/sys/dev/cxgbe/tom/t4_tom.h Mon Apr 27 23:55:09 2020 (r360416)
+++ head/sys/dev/cxgbe/tom/t4_tom.h Mon Apr 27 23:59:42 2020 (r360417)
@@ -452,6 +452,6 @@ int tls_rx_key(struct toepcb *);
void tls_stop_handshake_timer(struct toepcb *);
int tls_tx_key(struct toepcb *);
void tls_uninit_toep(struct toepcb *);
-int tls_alloc_ktls(struct toepcb *, struct ktls_session *);
+int tls_alloc_ktls(struct toepcb *, struct ktls_session *, int);
#endif
More information about the svn-src-head
mailing list