svn commit: r355873 - head/sys/opencrypto
John Baldwin
jhb at FreeBSD.org
Wed Dec 18 01:37:00 UTC 2019
Author: jhb
Date: Wed Dec 18 01:37:00 2019
New Revision: 355873
URL: https://svnweb.freebsd.org/changeset/base/355873
Log:
Add support for TLS 1.3 using AES-GCM to the OCF backend for KTLS.
Reviewed by: gallatin
Sponsored by: Netflix
Differential Revision: https://reviews.freebsd.org/D22802
Modified:
head/sys/opencrypto/ktls_ocf.c
Modified: head/sys/opencrypto/ktls_ocf.c
==============================================================================
--- head/sys/opencrypto/ktls_ocf.c Wed Dec 18 01:33:20 2019 (r355872)
+++ head/sys/opencrypto/ktls_ocf.c Wed Dec 18 01:37:00 2019 (r355873)
@@ -60,13 +60,21 @@ static MALLOC_DEFINE(M_KTLS_OCF, "ktls_ocf", "OCF KTLS
SYSCTL_DECL(_kern_ipc_tls);
SYSCTL_DECL(_kern_ipc_tls_stats);
-static counter_u64_t ocf_gcm_crypts;
-SYSCTL_COUNTER_U64(_kern_ipc_tls_stats, OID_AUTO, ocf_gcm_crypts, CTLFLAG_RD,
- &ocf_gcm_crypts,
- "Total number of OCF GCM encryption operations");
+static SYSCTL_NODE(_kern_ipc_tls_stats, OID_AUTO, ocf, CTLFLAG_RD, 0,
+ "Kernel TLS offload via OCF stats");
+static counter_u64_t ocf_tls12_gcm_crypts;
+SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, tls12_gcm_crypts,
+ CTLFLAG_RD, &ocf_tls12_gcm_crypts,
+ "Total number of OCF TLS 1.2 GCM encryption operations");
+
+static counter_u64_t ocf_tls13_gcm_crypts;
+SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, tls13_gcm_crypts,
+ CTLFLAG_RD, &ocf_tls13_gcm_crypts,
+ "Total number of OCF TLS 1.3 GCM encryption operations");
+
static counter_u64_t ocf_retries;
-SYSCTL_COUNTER_U64(_kern_ipc_tls_stats, OID_AUTO, ocf_retries, CTLFLAG_RD,
+SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, retries, CTLFLAG_RD,
&ocf_retries,
"Number of OCF encryption operation retries");
@@ -84,9 +92,10 @@ ktls_ocf_callback(struct cryptop *crp)
}
static int
-ktls_ocf_encrypt(struct ktls_session *tls, const struct tls_record_layer *hdr,
- uint8_t *trailer, struct iovec *iniov, struct iovec *outiov, int iovcnt,
- uint64_t seqno, uint8_t record_type __unused)
+ktls_ocf_tls12_gcm_encrypt(struct ktls_session *tls,
+ const struct tls_record_layer *hdr, uint8_t *trailer, struct iovec *iniov,
+ struct iovec *outiov, int iovcnt, uint64_t seqno,
+ uint8_t record_type __unused)
{
struct uio uio;
struct tls_aead_data ad;
@@ -127,7 +136,7 @@ ktls_ocf_encrypt(struct ktls_session *tls, const struc
iov[0].iov_base = &ad;
iov[0].iov_len = sizeof(ad);
uio.uio_resid = sizeof(ad);
-
+
/*
* OCF always does encryption in place, so copy the data if
* needed. Ugh.
@@ -171,7 +180,7 @@ ktls_ocf_encrypt(struct ktls_session *tls, const struc
crde->crd_flags = CRD_F_ENCRYPT | CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT;
memcpy(crde->crd_iv, &nd, sizeof(nd));
- counter_u64_add(ocf_gcm_crypts, 1);
+ counter_u64_add(ocf_tls12_gcm_crypts, 1);
for (;;) {
error = crypto_dispatch(crp);
if (error)
@@ -198,6 +207,118 @@ ktls_ocf_encrypt(struct ktls_session *tls, const struc
return (error);
}
+static int
+ktls_ocf_tls13_gcm_encrypt(struct ktls_session *tls,
+ const struct tls_record_layer *hdr, uint8_t *trailer, struct iovec *iniov,
+ struct iovec *outiov, int iovcnt, uint64_t seqno, uint8_t record_type)
+{
+ struct uio uio;
+ struct tls_aead_data_13 ad;
+ char nonce[12];
+ struct cryptodesc *crde, *crda;
+ struct cryptop *crp;
+ struct ocf_session *os;
+ struct ocf_operation *oo;
+ struct iovec *iov;
+ int i, error;
+
+ os = tls->cipher;
+
+ oo = malloc(sizeof(*oo) + (iovcnt + 2) * sizeof(*iov), M_KTLS_OCF,
+ M_WAITOK | M_ZERO);
+ oo->os = os;
+ iov = oo->iov;
+
+ crp = crypto_getreq(2);
+ if (crp == NULL) {
+ free(oo, M_KTLS_OCF);
+ return (ENOMEM);
+ }
+
+ /* Setup the nonce. */
+ memcpy(nonce, tls->params.iv, tls->params.iv_len);
+ *(uint64_t *)(nonce + 4) ^= htobe64(seqno);
+
+ /* Setup the AAD. */
+ ad.type = hdr->tls_type;
+ ad.tls_vmajor = hdr->tls_vmajor;
+ ad.tls_vminor = hdr->tls_vminor;
+ ad.tls_length = hdr->tls_length;
+ iov[0].iov_base = &ad;
+ iov[0].iov_len = sizeof(ad);
+ uio.uio_resid = sizeof(ad);
+
+ /*
+ * OCF always does encryption in place, so copy the data if
+ * needed. Ugh.
+ */
+ for (i = 0; i < iovcnt; i++) {
+ iov[i + 1] = outiov[i];
+ if (iniov[i].iov_base != outiov[i].iov_base)
+ memcpy(outiov[i].iov_base, iniov[i].iov_base,
+ outiov[i].iov_len);
+ uio.uio_resid += outiov[i].iov_len;
+ }
+
+ trailer[0] = record_type;
+ iov[iovcnt + 1].iov_base = trailer;
+ iov[iovcnt + 1].iov_len = AES_GMAC_HASH_LEN + 1;
+ uio.uio_resid += AES_GMAC_HASH_LEN + 1;
+
+ uio.uio_iov = iov;
+ uio.uio_iovcnt = iovcnt + 2;
+ uio.uio_offset = 0;
+ uio.uio_segflg = UIO_SYSSPACE;
+ uio.uio_td = curthread;
+
+ crp->crp_session = os->sid;
+ crp->crp_flags = CRYPTO_F_IOV | CRYPTO_F_CBIMM;
+ crp->crp_uio = &uio;
+ crp->crp_ilen = uio.uio_resid;
+ crp->crp_opaque = oo;
+ crp->crp_callback = ktls_ocf_callback;
+
+ crde = crp->crp_desc;
+ crda = crde->crd_next;
+
+ crda->crd_alg = os->crda_alg;
+ crda->crd_skip = 0;
+ crda->crd_len = sizeof(ad);
+ crda->crd_inject = crp->crp_ilen - AES_GMAC_HASH_LEN;
+
+ crde->crd_alg = CRYPTO_AES_NIST_GCM_16;
+ crde->crd_skip = sizeof(ad);
+ crde->crd_len = crp->crp_ilen - (sizeof(ad) + AES_GMAC_HASH_LEN);
+ crde->crd_flags = CRD_F_ENCRYPT | CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT;
+ memcpy(crde->crd_iv, nonce, sizeof(nonce));
+
+ counter_u64_add(ocf_tls13_gcm_crypts, 1);
+ for (;;) {
+ error = crypto_dispatch(crp);
+ if (error)
+ break;
+
+ mtx_lock(&os->lock);
+ while (!oo->done)
+ mtx_sleep(oo, &os->lock, 0, "ocfktls", 0);
+ mtx_unlock(&os->lock);
+
+ if (crp->crp_etype != EAGAIN) {
+ error = crp->crp_etype;
+ break;
+ }
+
+ crp->crp_etype = 0;
+ crp->crp_flags &= ~CRYPTO_F_DONE;
+ oo->done = false;
+ counter_u64_add(ocf_retries, 1);
+ }
+
+ crypto_freereq(crp);
+ free(oo, M_KTLS_OCF);
+ return (error);
+}
+
static void
ktls_ocf_free(struct ktls_session *tls)
{
@@ -221,8 +342,6 @@ ktls_ocf_try(struct socket *so, struct ktls_session *t
switch (tls->params.cipher_algorithm) {
case CRYPTO_AES_NIST_GCM_16:
- if (tls->params.iv_len != TLS_AEAD_GCM_LEN)
- return (EINVAL);
switch (tls->params.cipher_key_len) {
case 128 / 8:
cria.cri_alg = CRYPTO_AES_128_NIST_GMAC;
@@ -240,10 +359,10 @@ ktls_ocf_try(struct socket *so, struct ktls_session *t
return (EPROTONOSUPPORT);
}
- /* Only TLS 1.1 and TLS 1.2 are currently supported. */
+ /* Only TLS 1.2 and 1.3 are supported. */
if (tls->params.tls_vmajor != TLS_MAJOR_VER_ONE ||
- tls->params.tls_vminor < TLS_MINOR_VER_ONE ||
- tls->params.tls_vminor > TLS_MINOR_VER_TWO)
+ tls->params.tls_vminor < TLS_MINOR_VER_TWO ||
+ tls->params.tls_vminor > TLS_MINOR_VER_THREE)
return (EPROTONOSUPPORT);
os = malloc(sizeof(*os), M_KTLS_OCF, M_NOWAIT | M_ZERO);
@@ -265,7 +384,10 @@ ktls_ocf_try(struct socket *so, struct ktls_session *t
os->crda_alg = cria.cri_alg;
mtx_init(&os->lock, "ktls_ocf", NULL, MTX_DEF);
tls->cipher = os;
- tls->sw_encrypt = ktls_ocf_encrypt;
+ if (tls->params.tls_vminor == TLS_MINOR_VER_THREE)
+ tls->sw_encrypt = ktls_ocf_tls13_gcm_encrypt;
+ else
+ tls->sw_encrypt = ktls_ocf_tls12_gcm_encrypt;
tls->free = ktls_ocf_free;
return (0);
}
@@ -284,14 +406,16 @@ ktls_ocf_modevent(module_t mod, int what, void *arg)
switch (what) {
case MOD_LOAD:
- ocf_gcm_crypts = counter_u64_alloc(M_WAITOK);
+ ocf_tls12_gcm_crypts = counter_u64_alloc(M_WAITOK);
+ ocf_tls13_gcm_crypts = counter_u64_alloc(M_WAITOK);
ocf_retries = counter_u64_alloc(M_WAITOK);
return (ktls_crypto_backend_register(&ocf_backend));
case MOD_UNLOAD:
error = ktls_crypto_backend_deregister(&ocf_backend);
if (error)
return (error);
- counter_u64_free(ocf_gcm_crypts);
+ counter_u64_free(ocf_tls12_gcm_crypts);
+ counter_u64_free(ocf_tls13_gcm_crypts);
counter_u64_free(ocf_retries);
return (0);
default:
More information about the svn-src-all
mailing list