git: 197ff4c35bf0 - main - ossl: Add support for AES-CBC cipher
- Go to: [ bottom of page ] [ top of archives ] [ this month ]
Date: Mon, 08 Nov 2021 09:53:47 UTC
The branch main has been updated by wma:
URL: https://cgit.FreeBSD.org/src/commit/?id=197ff4c35bf00fb61603592476d5213a16027987
commit 197ff4c35bf00fb61603592476d5213a16027987
Author: Kornel Duleba <mindal@semihalf.com>
AuthorDate: 2021-11-02 11:53:22 +0000
Commit: Wojciech Macek <wma@FreeBSD.org>
CommitDate: 2021-11-08 09:53:31 +0000
ossl: Add support for AES-CBC cipher
AES-CBC OpenSSL assembly is used underneath.
The glue layer(ossl_aes.c) is based on CHACHA20 implementation.
Contrary to the SHA and CHACHA20, AES OpenSSL assembly logic
does not have a fallback implementation in case CPU doesn't
support required instructions.
Because of that CPU caps are checked during initialization and AES
support is advertised only if available.
The feature is available on all architectures that ossl supports:
i386, amd64, arm64.
The biggest advantage of this patch over existing solutions
(aesni(4) and armv8crypto(4)) is that it supports SHA,
allowing for ETA operations.
Sponsored by: Stormshield
Obtained from: Semihalf
Reviewed by: jhb (previous version)
Differential revision: https://reviews.freebsd.org/D32099
---
share/man/man4/ossl.4 | 4 +-
sys/conf/files | 1 +
sys/conf/files.amd64 | 1 +
sys/conf/files.arm64 | 6 +-
sys/conf/files.i386 | 1 +
sys/crypto/openssl/ossl.c | 94 +++++++++++++++++------
sys/crypto/openssl/ossl.h | 36 ++++++++-
sys/crypto/openssl/ossl_aarch64.c | 23 +++++-
sys/crypto/openssl/ossl_aarch64.h | 31 ++++++++
sys/crypto/openssl/ossl_aes.c | 153 +++++++++++++++++++++++++++++++++++++
sys/crypto/openssl/ossl_chacha20.c | 18 ++++-
sys/crypto/openssl/ossl_cipher.h | 53 +++++++++++++
sys/crypto/openssl/ossl_x86.c | 15 +++-
sys/crypto/openssl/ossl_x86.h | 20 +++++
sys/modules/ossl/Makefile | 14 ++++
tests/sys/opencrypto/cryptotest.py | 2 +-
16 files changed, 436 insertions(+), 36 deletions(-)
diff --git a/share/man/man4/ossl.4 b/share/man/man4/ossl.4
index 5929e46e9fe3..039ce301ac29 100644
--- a/share/man/man4/ossl.4
+++ b/share/man/man4/ossl.4
@@ -26,7 +26,7 @@
.\"
.\" $FreeBSD$
.\"
-.Dd March 3, 2021
+.Dd September 24, 2021
.Dt OSSL 4
.Os
.Sh NAME
@@ -74,6 +74,8 @@ driver includes support for the following algorithms:
.Pp
.Bl -bullet -compact
.It
+AES-CBC
+.It
ChaCha20
.It
ChaCha20-Poly1305 (RFC 8439)
diff --git a/sys/conf/files b/sys/conf/files
index 5fa918b59f62..e1ee0e9fc9ef 100644
--- a/sys/conf/files
+++ b/sys/conf/files
@@ -716,6 +716,7 @@ crypto/chacha20/chacha-sw.c optional crypto | ipsec | ipsec_support
crypto/des/des_ecb.c optional netsmb
crypto/des/des_setkey.c optional netsmb
crypto/openssl/ossl.c optional ossl
+crypto/openssl/ossl_aes.c optional ossl
crypto/openssl/ossl_chacha20.c optional ossl
crypto/openssl/ossl_poly1305.c optional ossl
crypto/openssl/ossl_sha1.c optional ossl
diff --git a/sys/conf/files.amd64 b/sys/conf/files.amd64
index 37ff6404cdba..6b51c1a5a55d 100644
--- a/sys/conf/files.amd64
+++ b/sys/conf/files.amd64
@@ -88,6 +88,7 @@ cddl/dev/dtrace/amd64/dtrace_asm.S optional dtrace compile-with "${DTRACE_S}"
cddl/dev/dtrace/amd64/dtrace_subr.c optional dtrace compile-with "${DTRACE_C}"
crypto/aesni/aeskeys_amd64.S optional aesni
crypto/des/des_enc.c optional netsmb
+crypto/openssl/amd64/aesni-x86_64.S optional ossl
crypto/openssl/amd64/chacha-x86_64.S optional ossl
crypto/openssl/amd64/poly1305-x86_64.S optional ossl
crypto/openssl/amd64/sha1-x86_64.S optional ossl
diff --git a/sys/conf/files.arm64 b/sys/conf/files.arm64
index bb356c1babc1..a2cf15de1ac0 100644
--- a/sys/conf/files.arm64
+++ b/sys/conf/files.arm64
@@ -115,8 +115,8 @@ armv8_crypto_wrap.o optional armv8crypto \
compile-with "${CC} -c ${CFLAGS:C/^-O2$/-O3/:N-nostdinc:N-mgeneral-regs-only} -I$S/crypto/armv8/ ${WERROR} ${NO_WCAST_QUAL} -march=armv8-a+crypto ${.IMPSRC}" \
no-implicit-rule \
clean "armv8_crypto_wrap.o"
-aesv8-armx.o optional armv8crypto \
- dependency "$S/crypto/openssl/aarch64/aesv8-armx.S" \
+aesv8-armx.o optional armv8crypto | ossl \
+ dependency "$S/crypto/openssl/aarch64/aesv8-armx.S" \
compile-with "${CC} -c ${CFLAGS:C/^-O2$/-O3/:N-nostdinc:N-mgeneral-regs-only} -I$S/crypto/armv8/ ${WERROR} ${NO_WCAST_QUAL} -march=armv8-a+crypto ${.IMPSRC}" \
no-implicit-rule \
clean "aesv8-armx.o"
@@ -138,6 +138,8 @@ crypto/openssl/aarch64/sha256-armv8.S optional ossl \
compile-with "${CC} -c ${CFLAGS:N-mgeneral-regs-only} ${WERROR} ${.IMPSRC}"
crypto/openssl/aarch64/sha512-armv8.S optional ossl \
compile-with "${CC} -c ${CFLAGS:N-mgeneral-regs-only} ${WERROR} ${.IMPSRC}"
+crypto/openssl/aarch64/vpaes-armv8.S optional ossl \
+ compile-with "${CC} -c ${CFLAGS:N-mgeneral-regs-only} ${WERROR} ${.IMPSRC}"
dev/acpica/acpi_bus_if.m optional acpi
dev/acpica/acpi_if.m optional acpi
diff --git a/sys/conf/files.i386 b/sys/conf/files.i386
index e83bcd5a3492..0c681d6a84a0 100644
--- a/sys/conf/files.i386
+++ b/sys/conf/files.i386
@@ -15,6 +15,7 @@ cddl/dev/dtrace/i386/dtrace_asm.S optional dtrace compile-with "${DTRACE_S}"
cddl/dev/dtrace/i386/dtrace_subr.c optional dtrace compile-with "${DTRACE_C}"
crypto/aesni/aeskeys_i386.S optional aesni
crypto/des/arch/i386/des_enc.S optional netsmb
+crypto/openssl/i386/aesni-x86.S optional ossl
crypto/openssl/i386/chacha-x86.S optional ossl
crypto/openssl/i386/poly1305-x86.S optional ossl
crypto/openssl/i386/sha1-586.S optional ossl
diff --git a/sys/crypto/openssl/ossl.c b/sys/crypto/openssl/ossl.c
index ad9b93dd960d..f46b5a966bb1 100644
--- a/sys/crypto/openssl/ossl.c
+++ b/sys/crypto/openssl/ossl.c
@@ -49,24 +49,10 @@ __FBSDID("$FreeBSD$");
#include <crypto/openssl/ossl.h>
#include <crypto/openssl/ossl_chacha.h>
+#include <crypto/openssl/ossl_cipher.h>
#include "cryptodev_if.h"
-struct ossl_softc {
- int32_t sc_cid;
-};
-
-struct ossl_session_hash {
- struct ossl_hash_context ictx;
- struct ossl_hash_context octx;
- struct auth_hash *axf;
- u_int mlen;
-};
-
-struct ossl_session {
- struct ossl_session_hash hash;
-};
-
static MALLOC_DEFINE(M_OSSL, "ossl", "OpenSSL crypto");
static void
@@ -92,7 +78,7 @@ ossl_attach(device_t dev)
sc = device_get_softc(dev);
- ossl_cpuid();
+ ossl_cpuid(sc);
sc->sc_cid = crypto_get_driverid(dev, sizeof(struct ossl_session),
CRYPTOCAP_F_SOFTWARE | CRYPTOCAP_F_SYNC |
CRYPTOCAP_F_ACCEL_SOFTWARE);
@@ -143,9 +129,34 @@ ossl_lookup_hash(const struct crypto_session_params *csp)
}
}
+static struct ossl_cipher*
+ossl_lookup_cipher(const struct crypto_session_params *csp)
+{
+
+ switch (csp->csp_cipher_alg) {
+ case CRYPTO_AES_CBC:
+ switch (csp->csp_cipher_klen * 8) {
+ case 128:
+ case 192:
+ case 256:
+ break;
+ default:
+ return (NULL);
+ }
+ return (&ossl_cipher_aes_cbc);
+ case CRYPTO_CHACHA20:
+ if (csp->csp_cipher_klen != CHACHA_KEY_SIZE)
+ return (NULL);
+ return (&ossl_cipher_chacha20);
+ default:
+ return (NULL);
+ }
+}
+
static int
ossl_probesession(device_t dev, const struct crypto_session_params *csp)
{
+ struct ossl_softc *sc = device_get_softc(dev);
if ((csp->csp_flags & ~(CSP_F_SEPARATE_OUTPUT | CSP_F_SEPARATE_AAD)) !=
0)
@@ -156,14 +167,10 @@ ossl_probesession(device_t dev, const struct crypto_session_params *csp)
return (EINVAL);
break;
case CSP_MODE_CIPHER:
- switch (csp->csp_cipher_alg) {
- case CRYPTO_CHACHA20:
- if (csp->csp_cipher_klen != CHACHA_KEY_SIZE)
- return (EINVAL);
- break;
- default:
+ if (csp->csp_cipher_alg != CRYPTO_CHACHA20 && !sc->has_aes)
+ return (EINVAL);
+ if (ossl_lookup_cipher(csp) == NULL)
return (EINVAL);
- }
break;
case CSP_MODE_AEAD:
switch (csp->csp_cipher_alg) {
@@ -213,20 +220,57 @@ ossl_newsession_hash(struct ossl_session *s,
}
}
+static int
+ossl_newsession_cipher(struct ossl_session *s,
+ const struct crypto_session_params *csp)
+{
+ struct ossl_cipher *cipher;
+ int error = 0;
+
+ cipher = ossl_lookup_cipher(csp);
+ if (cipher == NULL)
+ return (EINVAL);
+
+ s->cipher.cipher = cipher;
+
+ if (csp->csp_cipher_key == NULL)
+ return (0);
+
+ fpu_kern_enter(curthread, NULL, FPU_KERN_NOCTX);
+ if (cipher->set_encrypt_key != NULL) {
+ error = cipher->set_encrypt_key(csp->csp_cipher_key,
+ 8 * csp->csp_cipher_klen, &s->cipher.enc_ctx);
+ if (error != 0) {
+ fpu_kern_leave(curthread, NULL);
+ return (error);
+ }
+ }
+ if (cipher->set_decrypt_key != NULL)
+ error = cipher->set_decrypt_key(csp->csp_cipher_key,
+ 8 * csp->csp_cipher_klen, &s->cipher.dec_ctx);
+ fpu_kern_leave(curthread, NULL);
+
+ return (error);
+}
+
static int
ossl_newsession(device_t dev, crypto_session_t cses,
const struct crypto_session_params *csp)
{
struct ossl_session *s;
+ int error = 0;
s = crypto_get_driver_session(cses);
switch (csp->csp_mode) {
case CSP_MODE_DIGEST:
ossl_newsession_hash(s, csp);
break;
+ case CSP_MODE_CIPHER:
+ error = ossl_newsession_cipher(s, csp);
+ break;
}
- return (0);
+ return (error);
}
static int
@@ -320,7 +364,7 @@ ossl_process(device_t dev, struct cryptop *crp, int hint)
error = ossl_process_hash(s, crp, csp);
break;
case CSP_MODE_CIPHER:
- error = ossl_chacha20(crp, csp);
+ error = s->cipher.cipher->process(&s->cipher, crp, csp);
break;
case CSP_MODE_AEAD:
if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op))
diff --git a/sys/crypto/openssl/ossl.h b/sys/crypto/openssl/ossl.h
index 11793dca037a..4f5353818add 100644
--- a/sys/crypto/openssl/ossl.h
+++ b/sys/crypto/openssl/ossl.h
@@ -36,20 +36,47 @@
struct cryptop;
struct crypto_session_params;
+struct ossl_softc;
+struct ossl_session;
-int ossl_chacha20(struct cryptop *crp,
- const struct crypto_session_params *csp);
int ossl_chacha20_poly1305_decrypt(struct cryptop *crp,
const struct crypto_session_params *csp);
int ossl_chacha20_poly1305_encrypt(struct cryptop *crp,
const struct crypto_session_params *csp);
-void ossl_cpuid(void);
+void ossl_cpuid(struct ossl_softc *sc);
+
+struct ossl_softc {
+ int32_t sc_cid;
+ bool has_aes;
+};
/* Needs to be big enough to hold any hash context. */
struct ossl_hash_context {
uint32_t dummy[61];
} __aligned(32);
+struct ossl_cipher_context {
+ uint32_t dummy[61];
+} __aligned(32);
+
+struct ossl_session_hash {
+ struct ossl_hash_context ictx;
+ struct ossl_hash_context octx;
+ struct auth_hash *axf;
+ u_int mlen;
+};
+
+struct ossl_session_cipher {
+ struct ossl_cipher_context dec_ctx;
+ struct ossl_cipher_context enc_ctx;
+ struct ossl_cipher *cipher;
+};
+
+struct ossl_session {
+ struct ossl_session_cipher cipher;
+ struct ossl_session_hash hash;
+};
+
extern struct auth_hash ossl_hash_poly1305;
extern struct auth_hash ossl_hash_sha1;
extern struct auth_hash ossl_hash_sha224;
@@ -57,4 +84,7 @@ extern struct auth_hash ossl_hash_sha256;
extern struct auth_hash ossl_hash_sha384;
extern struct auth_hash ossl_hash_sha512;
+extern struct ossl_cipher ossl_cipher_aes_cbc;
+extern struct ossl_cipher ossl_cipher_chacha20;
+
#endif /* !__OSSL_H__ */
diff --git a/sys/crypto/openssl/ossl_aarch64.c b/sys/crypto/openssl/ossl_aarch64.c
index 2a45a848808a..e4b87a75a403 100644
--- a/sys/crypto/openssl/ossl_aarch64.c
+++ b/sys/crypto/openssl/ossl_aarch64.c
@@ -36,6 +36,7 @@
#include <machine/md_var.h>
#include <crypto/openssl/ossl.h>
+#include <crypto/openssl/ossl_cipher.h>
#include <crypto/openssl/aarch64/arm_arch.h>
/*
@@ -43,8 +44,14 @@
*/
unsigned int OPENSSL_armcap_P;
+ossl_cipher_setkey_t aes_v8_set_encrypt_key;
+ossl_cipher_setkey_t aes_v8_set_decrypt_key;
+
+ossl_cipher_setkey_t vpaes_set_encrypt_key;
+ossl_cipher_setkey_t vpaes_set_decrypt_key;
+
void
-ossl_cpuid(void)
+ossl_cpuid(struct ossl_softc *sc)
{
/* SHA features */
if ((elf_hwcap & HWCAP_SHA1) != 0)
@@ -59,4 +66,18 @@ ossl_cpuid(void)
OPENSSL_armcap_P |= ARMV8_AES;
if ((elf_hwcap & HWCAP_PMULL) != 0)
OPENSSL_armcap_P |= ARMV8_PMULL;
+
+ if ((OPENSSL_armcap_P & ARMV8_AES) == 0 &&
+ (OPENSSL_armcap_P & ARMV7_NEON) == 0) {
+ sc->has_aes = false;
+ return;
+ }
+ sc->has_aes = true;
+ if (OPENSSL_armcap_P & ARMV8_AES) {
+ ossl_cipher_aes_cbc.set_encrypt_key = aes_v8_set_encrypt_key;
+ ossl_cipher_aes_cbc.set_decrypt_key = aes_v8_set_decrypt_key;
+ } else {
+ ossl_cipher_aes_cbc.set_encrypt_key = vpaes_set_encrypt_key;
+ ossl_cipher_aes_cbc.set_decrypt_key = vpaes_set_decrypt_key;
+ }
}
diff --git a/sys/crypto/openssl/ossl_aarch64.h b/sys/crypto/openssl/ossl_aarch64.h
new file mode 100644
index 000000000000..f933f862d009
--- /dev/null
+++ b/sys/crypto/openssl/ossl_aarch64.h
@@ -0,0 +1,31 @@
+/*
+ * Copyright 1995-2016 The OpenSSL Project Authors. All Rights Reserved.
+ *
+ * Licensed under the OpenSSL license (the "License"). You may not use
+ * this file except in compliance with the License. You can obtain a copy
+ * in the file LICENSE in the source distribution or at
+ * https://www.openssl.org/source/license.html
+ */
+
+#ifndef __OSSL_AARCH64__
+#define __OSSL_AARCH64__
+
+#include <crypto/openssl/ossl.h>
+#include <crypto/openssl/ossl_cipher.h>
+#include <crypto/openssl/aarch64/arm_arch.h>
+
+/* aesv8-armx.S */
+ossl_cipher_encrypt_t aes_v8_cbc_encrypt;
+/* vpaes-armv8.S */
+ossl_cipher_encrypt_t vpaes_cbc_encrypt;
+
+static void
+AES_CBC_ENCRYPT(const unsigned char *in, unsigned char *out,
+ size_t length, const void *key, unsigned char *iv, int encrypt)
+{
+ if (OPENSSL_armcap_P & ARMV8_AES)
+ aes_v8_cbc_encrypt(in, out, length, key, iv, encrypt);
+ else
+ vpaes_cbc_encrypt(in, out, length, key, iv, encrypt);
+}
+#endif
diff --git a/sys/crypto/openssl/ossl_aes.c b/sys/crypto/openssl/ossl_aes.c
new file mode 100644
index 000000000000..93f16e7dce55
--- /dev/null
+++ b/sys/crypto/openssl/ossl_aes.c
@@ -0,0 +1,153 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
+ *
+ * Copyright (c) 2021 Stormshield.
+ * Copyright (c) 2021 Semihalf.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/malloc.h>
+
+#include <opencrypto/cryptodev.h>
+
+#include <crypto/openssl/ossl.h>
+#include <crypto/openssl/ossl_cipher.h>
+
+#if defined(__amd64__) || defined(__i386__)
+#include <crypto/openssl/ossl_x86.h>
+#elif defined (__aarch64__)
+#include <crypto/openssl/ossl_aarch64.h>
+#endif
+
+static ossl_cipher_process_t ossl_aes_cbc;
+
+struct ossl_cipher ossl_cipher_aes_cbc = {
+ .type = CRYPTO_AES_CBC,
+ .blocksize = AES_BLOCK_LEN,
+ .ivsize = AES_BLOCK_LEN,
+
+ /* Filled during initialization based on CPU caps. */
+ .set_encrypt_key = NULL,
+ .set_decrypt_key = NULL,
+ .process = ossl_aes_cbc
+};
+
+static int
+ossl_aes_cbc(struct ossl_session_cipher *s, struct cryptop *crp,
+ const struct crypto_session_params *csp)
+{
+ struct crypto_buffer_cursor cc_in, cc_out;
+ unsigned char block[EALG_MAX_BLOCK_LEN];
+ unsigned char iv[EALG_MAX_BLOCK_LEN];
+ const unsigned char *in, *inseg;
+ unsigned char *out, *outseg;
+ size_t plen, seglen, inlen, outlen;
+ struct ossl_cipher_context key;
+ struct ossl_cipher *cipher;
+ int blocklen, error;
+ bool encrypt;
+
+ cipher = s->cipher;
+ encrypt = CRYPTO_OP_IS_ENCRYPT(crp->crp_op);
+ plen = crp->crp_payload_length;
+ blocklen = cipher->blocksize;
+
+ if (plen % blocklen)
+ return (EINVAL);
+
+ if (crp->crp_cipher_key != NULL) {
+ if (encrypt)
+ error = cipher->set_encrypt_key(crp->crp_cipher_key,
+ 8 * csp->csp_cipher_klen, &key);
+ else
+ error = cipher->set_decrypt_key(crp->crp_cipher_key,
+ 8 * csp->csp_cipher_klen, &key);
+ if (error)
+ return (error);
+ } else {
+ if (encrypt)
+ key = s->enc_ctx;
+ else
+ key = s->dec_ctx;
+ }
+
+ crypto_read_iv(crp, iv);
+
+ /* Derived from ossl_chacha20.c */
+ crypto_cursor_init(&cc_in, &crp->crp_buf);
+ crypto_cursor_advance(&cc_in, crp->crp_payload_start);
+ inseg = crypto_cursor_segment(&cc_in, &inlen);
+ if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) {
+ crypto_cursor_init(&cc_out, &crp->crp_obuf);
+ crypto_cursor_advance(&cc_out, crp->crp_payload_output_start);
+ } else {
+ cc_out = cc_in;
+ }
+ outseg = crypto_cursor_segment(&cc_out, &outlen);
+
+ while (plen >= blocklen) {
+ if (inlen < blocklen) {
+ crypto_cursor_copydata(&cc_in, blocklen, block);
+ in = block;
+ inlen = blocklen;
+ } else {
+ in = inseg;
+ }
+ if (outlen < blocklen) {
+ out = block;
+ outlen = blocklen;
+ } else {
+ out = outseg;
+ }
+
+ /* Figure out how many blocks we can encrypt/decrypt at once. */
+ seglen = rounddown(MIN(plen, MIN(inlen, outlen)), blocklen);
+
+ AES_CBC_ENCRYPT(in, out, seglen, &key, iv, encrypt);
+
+ if (out == block) {
+ crypto_cursor_copyback(&cc_out, blocklen, block);
+ outseg = crypto_cursor_segment(&cc_out, &outlen);
+ } else {
+ crypto_cursor_advance(&cc_out, seglen);
+ outseg += seglen;
+ outlen -= seglen;
+ }
+ if (in == block) {
+ inseg = crypto_cursor_segment(&cc_in, &inlen);
+ } else {
+ crypto_cursor_advance(&cc_in, seglen);
+ inseg += seglen;
+ inlen -= seglen;
+ }
+ plen -= seglen;
+ }
+
+ explicit_bzero(block, sizeof(block));
+ explicit_bzero(iv, sizeof(iv));
+ explicit_bzero(&key, sizeof(key));
+ return (0);
+}
diff --git a/sys/crypto/openssl/ossl_chacha20.c b/sys/crypto/openssl/ossl_chacha20.c
index aa125121e8b4..c21a28470a26 100644
--- a/sys/crypto/openssl/ossl_chacha20.c
+++ b/sys/crypto/openssl/ossl_chacha20.c
@@ -37,10 +37,24 @@
#include <crypto/openssl/ossl.h>
#include <crypto/openssl/ossl_chacha.h>
+#include <crypto/openssl/ossl_cipher.h>
#include <crypto/openssl/ossl_poly1305.h>
-int
-ossl_chacha20(struct cryptop *crp, const struct crypto_session_params *csp)
+static ossl_cipher_process_t ossl_chacha20;
+
+struct ossl_cipher ossl_cipher_chacha20 = {
+ .type = CRYPTO_CHACHA20,
+ .blocksize = CHACHA_BLK_SIZE,
+ .ivsize = CHACHA_CTR_SIZE,
+
+ .set_encrypt_key = NULL,
+ .set_decrypt_key = NULL,
+ .process = ossl_chacha20
+};
+
+static int
+ossl_chacha20(struct ossl_session_cipher *s, struct cryptop *crp,
+ const struct crypto_session_params *csp)
{
_Alignas(8) unsigned int key[CHACHA_KEY_SIZE / 4];
unsigned int counter[CHACHA_CTR_SIZE / 4];
diff --git a/sys/crypto/openssl/ossl_cipher.h b/sys/crypto/openssl/ossl_cipher.h
new file mode 100644
index 000000000000..d9e6ec29aafc
--- /dev/null
+++ b/sys/crypto/openssl/ossl_cipher.h
@@ -0,0 +1,53 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
+ *
+ * Copyright (c) 2021 Stormshield.
+ * Copyright (c) 2021 Semihalf.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef __OSSL_CIPHER_H__
+#define __OSSL_CIPHER_H__
+
+struct ossl_session_cipher;
+struct cryptop;
+struct crypto_session_params;
+
+typedef int (ossl_cipher_setkey_t)(const unsigned char*, int, void*);
+typedef int (ossl_cipher_process_t)(struct ossl_session_cipher*, struct cryptop*,
+ const struct crypto_session_params*);
+typedef void (ossl_cipher_encrypt_t)(const unsigned char*, unsigned char*, size_t,
+ const void*, unsigned char*, int);
+
+ossl_cipher_encrypt_t ossl_aes_cbc_encrypt;
+
+struct ossl_cipher {
+ int type;
+ uint16_t blocksize;
+ uint16_t ivsize;
+
+ ossl_cipher_setkey_t *set_encrypt_key;
+ ossl_cipher_setkey_t *set_decrypt_key;
+ ossl_cipher_process_t *process;
+};
+
+#endif
diff --git a/sys/crypto/openssl/ossl_x86.c b/sys/crypto/openssl/ossl_x86.c
index 60ff6fa0c759..a1e9a995948b 100644
--- a/sys/crypto/openssl/ossl_x86.c
+++ b/sys/crypto/openssl/ossl_x86.c
@@ -39,6 +39,7 @@
#include <x86/specialreg.h>
#include <crypto/openssl/ossl.h>
+#include <crypto/openssl/ossl_cipher.h>
/*
* See OPENSSL_ia32cap(3).
@@ -49,9 +50,13 @@
* [3] = 0
*/
unsigned int OPENSSL_ia32cap_P[4];
+#define AESNI_CAPABLE (OPENSSL_ia32cap_P[1]&(1<<(57-32)))
+
+ossl_cipher_setkey_t aesni_set_encrypt_key;
+ossl_cipher_setkey_t aesni_set_decrypt_key;
void
-ossl_cpuid(void)
+ossl_cpuid(struct ossl_softc *sc)
{
uint64_t xcr0;
u_int regs[4];
@@ -112,4 +117,12 @@ ossl_cpuid(void)
OPENSSL_ia32cap_P[1] &= ~(CPUID2_AVX | AMDID2_XOP | CPUID2_FMA);
OPENSSL_ia32cap_P[2] &= ~CPUID_STDEXT_AVX2;
}
+
+ if (!AESNI_CAPABLE) {
+ sc->has_aes = false;
+ return;
+ }
+ sc->has_aes = true;
+ ossl_cipher_aes_cbc.set_encrypt_key = aesni_set_encrypt_key;
+ ossl_cipher_aes_cbc.set_decrypt_key = aesni_set_decrypt_key;
}
diff --git a/sys/crypto/openssl/ossl_x86.h b/sys/crypto/openssl/ossl_x86.h
new file mode 100644
index 000000000000..12bd5a4eaddb
--- /dev/null
+++ b/sys/crypto/openssl/ossl_x86.h
@@ -0,0 +1,20 @@
+/*
+ * Copyright 1995-2016 The OpenSSL Project Authors. All Rights Reserved.
+ *
+ * Licensed under the OpenSSL license (the "License"). You may not use
+ * this file except in compliance with the License. You can obtain a copy
+ * in the file LICENSE in the source distribution or at
+ * https://www.openssl.org/source/license.html
+ */
+
+#ifndef __OSSL_X86__
+#define __OSSL_X86__
+
+#include <crypto/openssl/ossl.h>
+#include <crypto/openssl/ossl_cipher.h>
+
+/* aesni-x86_64.S, aesni-x86.S */
+ossl_cipher_encrypt_t aesni_cbc_encrypt;
+
+#define AES_CBC_ENCRYPT aesni_cbc_encrypt
+#endif
diff --git a/sys/modules/ossl/Makefile b/sys/modules/ossl/Makefile
index dfd82dcf6e1f..765e70a03edd 100644
--- a/sys/modules/ossl/Makefile
+++ b/sys/modules/ossl/Makefile
@@ -4,10 +4,12 @@
.PATH: ${SRCTOP}/sys/crypto/openssl/${MACHINE_CPUARCH}
KMOD= ossl
+OBJS+= ${OBJS.${MACHINE_CPUARCH}}
SRCS= bus_if.h \
cryptodev_if.h \
device_if.h \
ossl.c \
+ ossl_aes.c \
ossl_chacha20.c \
ossl_poly1305.c \
ossl_sha1.c \
@@ -21,9 +23,11 @@ SRCS.aarch64= \
sha1-armv8.S \
sha256-armv8.S \
sha512-armv8.S \
+ vpaes-armv8.S \
ossl_aarch64.c
SRCS.amd64= \
+ aesni-x86_64.S \
chacha-x86_64.S \
poly1305-x86_64.S \
sha1-x86_64.S \
@@ -32,6 +36,7 @@ SRCS.amd64= \
ossl_x86.c
SRCS.i386= \
+ aesni-x86.S \
chacha-x86.S \
poly1305-x86.S \
sha1-586.S \
@@ -45,4 +50,13 @@ ${SRCS.aarch64:M*.S:S/S/o/}: ${.TARGET:R}.S
${CC} -c ${CFLAGS:N-mgeneral-regs-only} ${WERROR} ${PROF} ${.IMPSRC}
${CTFCONVERT_CMD}
+# Based on modules/armv8crypto/Makefile.
+# Clang doesn't recognize "aes*" instructions without -march set.
+aesv8-armx.o: aesv8-armx.S
+ ${CC} -c ${CFLAGS:N-mgeneral-regs-only} ${WERROR} ${PROF} \
+ -march=armv8-a+crypto ${.IMPSRC}
+ ${CTFCONVERT_CMD}
+
+OBJS.aarch64= aesv8-armx.o
+
.include <bsd.kmod.mk>
diff --git a/tests/sys/opencrypto/cryptotest.py b/tests/sys/opencrypto/cryptotest.py
index 74ce62cee33d..447a7854b246 100644
--- a/tests/sys/opencrypto/cryptotest.py
+++ b/tests/sys/opencrypto/cryptotest.py
@@ -50,7 +50,7 @@ def katg(base, glob):
raise unittest.SkipTest("Missing %s test vectors" % (base))
return iglob(os.path.join(katdir, base, glob))
-aesmodules = [ 'cryptosoft0', 'aesni0', 'armv8crypto0', 'ccr0', 'ccp0', 'safexcel0', 'qat0' ]
+aesmodules = [ 'cryptosoft0', 'aesni0', 'armv8crypto0', 'ccr0', 'ccp0', 'ossl0', 'safexcel0', 'qat0' ]
shamodules = [ 'cryptosoft0', 'aesni0', 'armv8crypto0', 'ccr0', 'ccp0', 'ossl0', 'safexcel0', 'qat0' ]
def GenTestCase(cname):