git: 12978b1c9996 - stable/13 - ccr: Use a software OCF session for requests which fallback to software.
- Go to: [ bottom of page ] [ top of archives ] [ this month ]
Date: Fri, 29 Apr 2022 23:12:30 UTC
The branch stable/13 has been updated by jhb:
URL: https://cgit.FreeBSD.org/src/commit/?id=12978b1c9996607ee1788fefc81ab6ad1b6eea60
commit 12978b1c9996607ee1788fefc81ab6ad1b6eea60
Author: John Baldwin <jhb@FreeBSD.org>
AuthorDate: 2022-01-04 22:22:32 +0000
Commit: John Baldwin <jhb@FreeBSD.org>
CommitDate: 2022-04-29 22:50:29 +0000
ccr: Use a software OCF session for requests which fallback to software.
Previously the driver duplicated code from cryptosoft.c to handle
certain edge case AES-CCM and AES-GCM requests. However, this
approach has a few downsides:
1) It only uses "plain" software and not accelerated software since it
uses enc_xform directly.
2) It performs the operation synchronously even though the caller
believes it is invoking an async driver. This was fine for the
original use case of requests with only AAD and no payload that
execute quickly, but is a bit more disingenuous for large requests
which fall back due to exceeding the size of a firmware work
request (e.g. due to large scatter/gather lists).
3) It has required several updates since ccr(4) was added to the tree.
Instead, allocate a software session for AES-CCM and AES-GCM sessions
and dispatch a cloned request asynchronusly to the software session.
Reviewed by: markj
Sponsored by: Chelsio Communications
Differential Revision: https://reviews.freebsd.org/D33608
(cherry picked from commit e43cf698d93c9f3007ade4884c6ace38eec43a52)
---
sys/dev/cxgbe/crypto/t4_crypto.c | 387 +++++----------------------------------
1 file changed, 44 insertions(+), 343 deletions(-)
diff --git a/sys/dev/cxgbe/crypto/t4_crypto.c b/sys/dev/cxgbe/crypto/t4_crypto.c
index 231756f542ac..3d667ac5a405 100644
--- a/sys/dev/cxgbe/crypto/t4_crypto.c
+++ b/sys/dev/cxgbe/crypto/t4_crypto.c
@@ -191,6 +191,13 @@ struct ccr_session {
struct ccr_session_cipher cipher;
struct mtx lock;
+ /*
+ * A fallback software session is used for certain GCM/CCM
+ * requests that the hardware can't handle such as requests
+ * with only AAD and no payload.
+ */
+ crypto_session_t sw_session;
+
/*
* Pre-allocate S/G lists used when preparing a work request.
* 'sg_input' contains an sglist describing the entire input
@@ -1381,151 +1388,6 @@ ccr_gcm_done(struct ccr_softc *sc, struct ccr_session *s,
return (error);
}
-/*
- * Handle a GCM request that is not supported by the crypto engine by
- * performing the operation in software. Derived from swcr_authenc().
- */
-static void
-ccr_gcm_soft(struct ccr_session *s, struct cryptop *crp)
-{
- struct auth_hash *axf;
- struct enc_xform *exf;
- void *auth_ctx, *kschedule;
- char block[GMAC_BLOCK_LEN];
- char digest[GMAC_DIGEST_LEN];
- int error, i, len;
-
- auth_ctx = NULL;
- kschedule = NULL;
-
- /* Initialize the MAC. */
- switch (s->cipher.key_len) {
- case 16:
- axf = &auth_hash_nist_gmac_aes_128;
- break;
- case 24:
- axf = &auth_hash_nist_gmac_aes_192;
- break;
- case 32:
- axf = &auth_hash_nist_gmac_aes_256;
- break;
- default:
- error = EINVAL;
- goto out;
- }
- auth_ctx = malloc(axf->ctxsize, M_CCR, M_NOWAIT);
- if (auth_ctx == NULL) {
- error = ENOMEM;
- goto out;
- }
- axf->Init(auth_ctx);
- axf->Setkey(auth_ctx, s->cipher.enckey, s->cipher.key_len);
-
- /* Initialize the cipher. */
- exf = &enc_xform_aes_nist_gcm;
- kschedule = malloc(exf->ctxsize, M_CCR, M_NOWAIT);
- if (kschedule == NULL) {
- error = ENOMEM;
- goto out;
- }
- error = exf->setkey(kschedule, s->cipher.enckey,
- s->cipher.key_len);
- if (error)
- goto out;
-
- if ((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0) {
- error = EINVAL;
- goto out;
- }
-
- axf->Reinit(auth_ctx, crp->crp_iv, AES_GCM_IV_LEN);
-
- /* MAC the AAD. */
- if (crp->crp_aad != NULL) {
- len = rounddown(crp->crp_aad_length, sizeof(block));
- if (len != 0)
- axf->Update(auth_ctx, crp->crp_aad, len);
- if (crp->crp_aad_length != len) {
- memset(block, 0, sizeof(block));
- memcpy(block, (char *)crp->crp_aad + len,
- crp->crp_aad_length - len);
- axf->Update(auth_ctx, block, sizeof(block));
- }
- } else {
- for (i = 0; i < crp->crp_aad_length; i += sizeof(block)) {
- len = imin(crp->crp_aad_length - i, sizeof(block));
- crypto_copydata(crp, crp->crp_aad_start + i, len,
- block);
- bzero(block + len, sizeof(block) - len);
- axf->Update(auth_ctx, block, sizeof(block));
- }
- }
-
- exf->reinit(kschedule, crp->crp_iv, AES_GCM_IV_LEN);
-
- /* Do encryption with MAC */
- for (i = 0; i < crp->crp_payload_length; i += sizeof(block)) {
- len = imin(crp->crp_payload_length - i, sizeof(block));
- crypto_copydata(crp, crp->crp_payload_start + i, len, block);
- bzero(block + len, sizeof(block) - len);
- if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) {
- exf->encrypt(kschedule, block, block);
- axf->Update(auth_ctx, block, len);
- crypto_copyback(crp, crp->crp_payload_start + i, len,
- block);
- } else {
- axf->Update(auth_ctx, block, len);
- }
- }
-
- /* Length block. */
- bzero(block, sizeof(block));
- ((uint32_t *)block)[1] = htobe32(crp->crp_aad_length * 8);
- ((uint32_t *)block)[3] = htobe32(crp->crp_payload_length * 8);
- axf->Update(auth_ctx, block, sizeof(block));
-
- /* Finalize MAC. */
- axf->Final(digest, auth_ctx);
-
- /* Inject or validate tag. */
- if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) {
- crypto_copyback(crp, crp->crp_digest_start, sizeof(digest),
- digest);
- error = 0;
- } else {
- char digest2[GMAC_DIGEST_LEN];
-
- crypto_copydata(crp, crp->crp_digest_start, sizeof(digest2),
- digest2);
- if (timingsafe_bcmp(digest, digest2, sizeof(digest)) == 0) {
- error = 0;
-
- /* Tag matches, decrypt data. */
- for (i = 0; i < crp->crp_payload_length;
- i += sizeof(block)) {
- len = imin(crp->crp_payload_length - i,
- sizeof(block));
- crypto_copydata(crp, crp->crp_payload_start + i,
- len, block);
- bzero(block + len, sizeof(block) - len);
- exf->decrypt(kschedule, block, block);
- crypto_copyback(crp, crp->crp_payload_start + i,
- len, block);
- }
- } else
- error = EBADMSG;
- explicit_bzero(digest2, sizeof(digest2));
- }
-
-out:
- zfree(kschedule, M_CCR);
- zfree(auth_ctx, M_CCR);
- explicit_bzero(block, sizeof(block));
- explicit_bzero(digest, sizeof(digest));
- crp->crp_etype = error;
- crypto_done(crp);
-}
-
static int
ccr_ccm_hmac_ctrl(unsigned int authsize)
{
@@ -1894,208 +1756,41 @@ ccr_ccm_done(struct ccr_softc *sc, struct ccr_session *s,
}
/*
- * Handle a CCM request that is not supported by the crypto engine by
- * performing the operation in software. Derived from swcr_ccm().
+ * Use the software session for requests not supported by the crypto
+ * engine (e.g. CCM and GCM requests with an empty payload).
*/
-static void
-build_ccm_b0(const char *nonce, u_int nonce_length, u_int aad_length,
- u_int data_length, u_int tag_length, uint8_t *b0)
-{
- uint8_t *bp;
- uint8_t flags, L;
-
- KASSERT(nonce_length >= 7 && nonce_length <= 13,
- ("nonce_length must be between 7 and 13 bytes"));
-
- /*
- * Need to determine the L field value. This is the number of
- * bytes needed to specify the length of the message; the length
- * is whatever is left in the 16 bytes after specifying flags and
- * the nonce.
- */
- L = 15 - nonce_length;
-
- flags = ((aad_length > 0) << 6) +
- (((tag_length - 2) / 2) << 3) +
- L - 1;
-
- /*
- * Now we need to set up the first block, which has flags, nonce,
- * and the message length.
- */
- b0[0] = flags;
- memcpy(b0 + 1, nonce, nonce_length);
- bp = b0 + 1 + nonce_length;
-
- /* Need to copy L' [aka L-1] bytes of data_length */
- for (uint8_t *dst = b0 + CCM_CBC_BLOCK_LEN - 1; dst >= bp; dst--) {
- *dst = data_length;
- data_length >>= 8;
- }
-}
-
-/* NB: OCF only supports AAD lengths < 2^32. */
static int
-build_ccm_aad_length(u_int aad_length, uint8_t *blk)
+ccr_soft_done(struct cryptop *crp)
{
- if (aad_length < ((1 << 16) - (1 << 8))) {
- be16enc(blk, aad_length);
- return (sizeof(uint16_t));
- } else {
- blk[0] = 0xff;
- blk[1] = 0xfe;
- be32enc(blk + 2, aad_length);
- return (2 + sizeof(uint32_t));
- }
+ struct cryptop *orig;
+
+ orig = crp->crp_opaque;
+ orig->crp_etype = crp->crp_etype;
+ crypto_freereq(crp);
+ crypto_done(orig);
+ return (0);
}
static void
-ccr_ccm_soft(struct ccr_session *s, struct cryptop *crp)
+ccr_soft(struct ccr_session *s, struct cryptop *crp)
{
- const struct crypto_session_params *csp;
- struct auth_hash *axf;
- struct enc_xform *exf;
- union authctx *auth_ctx;
- void *kschedule;
- char block[CCM_CBC_BLOCK_LEN];
- char tag[AES_CBC_MAC_HASH_LEN];
- u_int taglen;
- int error, i, len;
-
- auth_ctx = NULL;
- kschedule = NULL;
- taglen = s->ccm_mac.hash_len;
-
- csp = crypto_get_params(crp->crp_session);
- if (crp->crp_payload_length > ccm_max_payload_length(csp)) {
- error = EMSGSIZE;
- goto out;
- }
-
- /* Initialize the MAC. */
- switch (s->cipher.key_len) {
- case 16:
- axf = &auth_hash_ccm_cbc_mac_128;
- break;
- case 24:
- axf = &auth_hash_ccm_cbc_mac_192;
- break;
- case 32:
- axf = &auth_hash_ccm_cbc_mac_256;
- break;
- default:
- error = EINVAL;
- goto out;
- }
- auth_ctx = malloc(axf->ctxsize, M_CCR, M_NOWAIT);
- if (auth_ctx == NULL) {
- error = ENOMEM;
- goto out;
- }
- axf->Init(auth_ctx);
- axf->Setkey(auth_ctx, s->cipher.enckey, s->cipher.key_len);
-
- /* Initialize the cipher. */
- exf = &enc_xform_ccm;
- kschedule = malloc(exf->ctxsize, M_CCR, M_NOWAIT);
- if (kschedule == NULL) {
- error = ENOMEM;
- goto out;
- }
- error = exf->setkey(kschedule, s->cipher.enckey,
- s->cipher.key_len);
- if (error)
- goto out;
-
- if ((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0) {
- error = EINVAL;
- goto out;
- }
-
- axf->Reinit(auth_ctx, crp->crp_iv, csp->csp_ivlen);
-
- /* Supply MAC with b0. */
- build_ccm_b0(crp->crp_iv, csp->csp_ivlen, crp->crp_aad_length,
- crp->crp_payload_length, taglen, block);
- axf->Update(auth_ctx, block, CCM_CBC_BLOCK_LEN);
-
- /* MAC the AAD. */
- if (crp->crp_aad_length != 0) {
- len = build_ccm_aad_length(crp->crp_aad_length, block);
- axf->Update(auth_ctx, block, len);
- if (crp->crp_aad != NULL)
- axf->Update(auth_ctx, crp->crp_aad,
- crp->crp_aad_length);
- else
- crypto_apply(crp, crp->crp_aad_start,
- crp->crp_aad_length, axf->Update, auth_ctx);
-
- /* Pad the AAD (including length field) to a full block. */
- len = (len + crp->crp_aad_length) % CCM_CBC_BLOCK_LEN;
- if (len != 0) {
- len = CCM_CBC_BLOCK_LEN - len;
- memset(block, 0, CCM_CBC_BLOCK_LEN);
- axf->Update(auth_ctx, block, len);
- }
- }
+ struct cryptop *new;
+ int error;
- exf->reinit(kschedule, crp->crp_iv, csp->csp_ivlen);
-
- /* Do encryption/decryption with MAC */
- for (i = 0; i < crp->crp_payload_length; i += sizeof(block)) {
- len = imin(crp->crp_payload_length - i, sizeof(block));
- crypto_copydata(crp, crp->crp_payload_start + i, len, block);
- bzero(block + len, sizeof(block) - len);
- if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) {
- axf->Update(auth_ctx, block, len);
- exf->encrypt(kschedule, block, block);
- crypto_copyback(crp, crp->crp_payload_start + i, len,
- block);
- } else {
- exf->decrypt(kschedule, block, block);
- axf->Update(auth_ctx, block, len);
- }
+ new = crypto_clonereq(crp, s->sw_session, M_NOWAIT);
+ if (new == NULL) {
+ crp->crp_etype = ENOMEM;
+ crypto_done(crp);
+ return;
}
- /* Finalize MAC. */
- axf->Final(tag, auth_ctx);
-
- /* Inject or validate tag. */
- if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) {
- crypto_copyback(crp, crp->crp_digest_start, taglen, tag);
- error = 0;
- } else {
- char tag2[AES_CBC_MAC_HASH_LEN];
-
- crypto_copydata(crp, crp->crp_digest_start, taglen, tag2);
- if (timingsafe_bcmp(tag, tag2, taglen) == 0) {
- error = 0;
-
- /* Tag matches, decrypt data. */
- exf->reinit(kschedule, crp->crp_iv, csp->csp_ivlen);
- for (i = 0; i < crp->crp_payload_length;
- i += sizeof(block)) {
- len = imin(crp->crp_payload_length - i,
- sizeof(block));
- crypto_copydata(crp, crp->crp_payload_start + i,
- len, block);
- bzero(block + len, sizeof(block) - len);
- exf->decrypt(kschedule, block, block);
- crypto_copyback(crp, crp->crp_payload_start + i,
- len, block);
- }
- } else
- error = EBADMSG;
- explicit_bzero(tag2, sizeof(tag2));
+ new->crp_opaque = crp;
+ new->crp_callback = ccr_soft_done;
+ error = crypto_dispatch(new);
+ if (error != 0) {
+ crp->crp_etype = error;
+ crypto_done(crp);
}
-
-out:
- zfree(kschedule, M_CCR);
- zfree(auth_ctx, M_CCR);
- explicit_bzero(block, sizeof(block));
- explicit_bzero(tag, sizeof(tag));
- crp->crp_etype = error;
- crypto_done(crp);
}
static void
@@ -2605,6 +2300,7 @@ ccr_choose_port(struct ccr_softc *sc)
static void
ccr_delete_session(struct ccr_session *s)
{
+ crypto_freesession(s->sw_session);
sglist_free(s->sg_input);
sglist_free(s->sg_output);
sglist_free(s->sg_ulptx);
@@ -2621,6 +2317,7 @@ ccr_newsession(device_t dev, crypto_session_t cses,
struct auth_hash *auth_hash;
unsigned int auth_mode, cipher_mode, mk_size;
unsigned int partial_digest_len;
+ int error;
switch (csp->csp_auth_alg) {
case CRYPTO_SHA1:
@@ -2712,6 +2409,15 @@ ccr_newsession(device_t dev, crypto_session_t cses,
return (ENOMEM);
}
+ if (csp->csp_mode == CSP_MODE_AEAD) {
+ error = crypto_newsession(&s->sw_session, csp,
+ CRYPTOCAP_F_SOFTWARE);
+ if (error) {
+ ccr_delete_session(s);
+ return (error);
+ }
+ }
+
sc = device_get_softc(dev);
mtx_lock(&sc->lock);
@@ -2881,16 +2587,11 @@ ccr_process(device_t dev, struct cryptop *crp, int hint)
ccr_aes_setkey(s, crp->crp_cipher_key,
csp->csp_cipher_klen);
}
- if (crp->crp_payload_length == 0) {
- mtx_unlock(&s->lock);
- ccr_gcm_soft(s, crp);
- return (0);
- }
error = ccr_gcm(sc, s, crp);
if (error == EMSGSIZE || error == EFBIG) {
counter_u64_add(sc->stats_sw_fallback, 1);
mtx_unlock(&s->lock);
- ccr_gcm_soft(s, crp);
+ ccr_soft(s, crp);
return (0);
}
if (error == 0) {
@@ -2909,7 +2610,7 @@ ccr_process(device_t dev, struct cryptop *crp, int hint)
if (error == EMSGSIZE || error == EFBIG) {
counter_u64_add(sc->stats_sw_fallback, 1);
mtx_unlock(&s->lock);
- ccr_ccm_soft(s, crp);
+ ccr_soft(s, crp);
return (0);
}
if (error == 0) {