git: ceb575553b22 - main - arm64: enable ossl accelerated AES-GCM on arm64
- Go to: [ bottom of page ] [ top of archives ] [ this month ]
Date: Tue, 20 Oct 2026 19:07:43 UTC
The branch main has been updated by gallatin:
URL: https://cgit.FreeBSD.org/src/commit/?id=ceb575553b222a310346f6a98b3c39bff49cb069
commit ceb575553b222a310346f6a98b3c39bff49cb069
Author: Andrew Gallatin <gallatin@FreeBSD.org>
AuthorDate: 2026-10-20 18:58:39 +0000
Commit: Andrew Gallatin <gallatin@FreeBSD.org>
CommitDate: 2026-10-20 19:07:39 +0000
arm64: enable ossl accelerated AES-GCM on arm64
Connect the baseline OpenSSL ARMv8 fused AES-GCM kernels to the OCF
ossl driver. Advertise the algorithm only when the system-wide capability set
includes both AES and PMULL.
Note that get_kernel_reg() is used to detect features such as PMULL
because the ossl device attaches before arm64 populates elf_hwcap when
ossl is built into the kernel or pre-loaded at boot.
Prefer OSSL over armv8crypto for AES-GCM sessions while retaining
armv8crypto as a fallback.
This provides about a 50% speedup for a Netflix ktls workload on a small
neoverse N1 board.
Reviewed by: markj
Constructive review feedback from: andrew, ngie
Differential Revision: https://reviews.freebsd.org/D60097
Sponsored by: Netflix
---
sys/conf/files.arm64 | 9 +++-
sys/crypto/armv8/armv8_crypto.c | 6 ++-
sys/crypto/openssl/ossl.c | 2 +-
sys/crypto/openssl/ossl_aarch64.c | 31 ++++++++++----
sys/crypto/openssl/ossl_aes_gcm.c | 88 +++++++++++++++++++++++++++++++++++++--
sys/modules/ossl/Makefile | 5 ++-
6 files changed, 126 insertions(+), 15 deletions(-)
diff --git a/sys/conf/files.arm64 b/sys/conf/files.arm64
index 7e604693eba8..105bc679acb8 100644
--- a/sys/conf/files.arm64
+++ b/sys/conf/files.arm64
@@ -197,14 +197,21 @@ aesv8-armx.o optional armv8crypto | ossl \
compile-with "${CC} -c ${CFLAGS:C/^-O2$/-O3/:N-nostdinc:N-mgeneral-regs-only} -I$S/crypto/armv8 -I$S/crypto/openssl ${WERROR} ${NO_WCAST_QUAL} ${CFLAGS:M-march=*:S/^$/-march=armv8-a/}+crypto ${.IMPSRC}" \
no-implicit-rule \
clean "aesv8-armx.o"
-ghashv8-armx.o optional armv8crypto \
+ghashv8-armx.o optional armv8crypto | ossl \
dependency "$S/crypto/openssl/aarch64/ghashv8-armx.S" \
compile-with "${CC} -c ${CFLAGS:C/^-O2$/-O3/:N-nostdinc:N-mgeneral-regs-only} -I$S/crypto/armv8 -I$S/crypto/openssl ${WERROR} ${NO_WCAST_QUAL} ${CFLAGS:M-march=*:S/^$/-march=armv8-a/}+crypto ${.IMPSRC}" \
no-implicit-rule \
clean "ghashv8-armx.o"
+aes-gcm-armv8_64.o optional ossl \
+ dependency "$S/crypto/openssl/aarch64/aes-gcm-armv8_64.S" \
+ compile-with "${CC} -c ${CFLAGS:N-nostdinc:N-mgeneral-regs-only} -I$S/crypto/openssl ${WERROR} ${NO_WCAST_QUAL} ${CFLAGS:M-march=*:S/^$/-march=armv8-a/}+crypto ${.IMPSRC}" \
+ no-implicit-rule \
+ clean "aes-gcm-armv8_64.o"
+
crypto/des/des_enc.c optional netsmb
crypto/openssl/ossl_aarch64.c optional ossl
+crypto/openssl/ossl_aes_gcm.c optional ossl
crypto/openssl/aarch64/chacha-armv8.S optional ossl \
compile-with "${CC} -c ${CFLAGS:N-mgeneral-regs-only} -I$S/crypto/openssl ${WERROR} ${.IMPSRC}"
crypto/openssl/aarch64/chacha-armv8-sve.S optional ossl \
diff --git a/sys/crypto/armv8/armv8_crypto.c b/sys/crypto/armv8/armv8_crypto.c
index 5b7ec4fbe125..624fa4e979e0 100644
--- a/sys/crypto/armv8/armv8_crypto.c
+++ b/sys/crypto/armv8/armv8_crypto.c
@@ -209,7 +209,11 @@ armv8_crypto_probesession(device_t dev,
default:
return (EINVAL);
}
- return (CRYPTODEV_PROBE_ACCEL_SOFTWARE);
+ /*
+ * Prefer other crypto implementations (eg, ossl) to armv8_crypto
+ * by reducing the probe priority.
+ */
+ return (CRYPTODEV_PROBE_ACCEL_SOFTWARE - 50);
}
static int
diff --git a/sys/crypto/openssl/ossl.c b/sys/crypto/openssl/ossl.c
index 203091c1e50b..04d8507cad4e 100644
--- a/sys/crypto/openssl/ossl.c
+++ b/sys/crypto/openssl/ossl.c
@@ -1,7 +1,7 @@
/*-
* SPDX-License-Identifier: BSD-2-Clause
*
- * Copyright (c) 2020 Netflix, Inc
+ * Copyright (c) 2020-2026 Netflix, Inc
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
diff --git a/sys/crypto/openssl/ossl_aarch64.c b/sys/crypto/openssl/ossl_aarch64.c
index a9d0b0bbe211..bf94d2bf2360 100644
--- a/sys/crypto/openssl/ossl_aarch64.c
+++ b/sys/crypto/openssl/ossl_aarch64.c
@@ -1,6 +1,7 @@
/*-
* SPDX-License-Identifier: BSD-2-Clause
*
+ * Copyright (c) 2026 Netflix, Inc
* Copyright (c) 2020 The FreeBSD Foundation
*
* This software was developed by Mitchell Horne <mhorne@FreeBSD.org>
@@ -29,9 +30,10 @@
*/
#include <sys/types.h>
+#include <sys/stdint.h>
-#include <machine/elf.h>
-#include <machine/md_var.h>
+#include <machine/armreg.h>
+#include <machine/cpu.h>
#include <crypto/openssl/ossl.h>
#include <crypto/openssl/ossl_cipher.h>
@@ -41,9 +43,13 @@
* Feature bits defined in arm_arch.h
*/
unsigned int OPENSSL_armcap_P;
+#define AES_PMULL_CAPABLE \
+ ((OPENSSL_armcap_P & (ARMV8_AES | ARMV8_PMULL)) == \
+ (ARMV8_AES | ARMV8_PMULL))
ossl_cipher_setkey_t aes_v8_set_encrypt_key;
ossl_cipher_setkey_t aes_v8_set_decrypt_key;
+ossl_cipher_setkey_t ossl_aes_gcm_setkey;
ossl_cipher_setkey_t vpaes_set_encrypt_key;
ossl_cipher_setkey_t vpaes_set_decrypt_key;
@@ -51,18 +57,23 @@ ossl_cipher_setkey_t vpaes_set_decrypt_key;
void
ossl_cpuid(struct ossl_softc *sc)
{
+ uint64_t isar0;
+
+ /* Use sanitized kernel features, available before elf_hwcap is set. */
+ get_kernel_reg(ID_AA64ISAR0_EL1, &isar0);
+
/* SHA features */
- if ((elf_hwcap & HWCAP_SHA1) != 0)
+ if (ID_AA64ISAR0_SHA1_VAL(isar0) >= ID_AA64ISAR0_SHA1_BASE)
OPENSSL_armcap_P |= ARMV8_SHA1;
- if ((elf_hwcap & HWCAP_SHA2) != 0)
+ if (ID_AA64ISAR0_SHA2_VAL(isar0) >= ID_AA64ISAR0_SHA2_BASE)
OPENSSL_armcap_P |= ARMV8_SHA256;
- if ((elf_hwcap & HWCAP_SHA512) != 0)
+ if (ID_AA64ISAR0_SHA2_VAL(isar0) >= ID_AA64ISAR0_SHA2_512)
OPENSSL_armcap_P |= ARMV8_SHA512;
/* AES features */
- if ((elf_hwcap & HWCAP_AES) != 0)
+ if (ID_AA64ISAR0_AES_VAL(isar0) >= ID_AA64ISAR0_AES_BASE)
OPENSSL_armcap_P |= ARMV8_AES;
- if ((elf_hwcap & HWCAP_PMULL) != 0)
+ if (ID_AA64ISAR0_AES_VAL(isar0) >= ID_AA64ISAR0_AES_PMULL)
OPENSSL_armcap_P |= ARMV8_PMULL;
if ((OPENSSL_armcap_P & ARMV8_AES) == 0 &&
@@ -78,4 +89,10 @@ ossl_cpuid(struct ossl_softc *sc)
ossl_cipher_aes_cbc.set_encrypt_key = vpaes_set_encrypt_key;
ossl_cipher_aes_cbc.set_decrypt_key = vpaes_set_decrypt_key;
}
+
+ if (AES_PMULL_CAPABLE) {
+ ossl_cipher_aes_gcm.set_encrypt_key = ossl_aes_gcm_setkey;
+ ossl_cipher_aes_gcm.set_decrypt_key = ossl_aes_gcm_setkey;
+ sc->has_aes_gcm = true;
+ }
}
diff --git a/sys/crypto/openssl/ossl_aes_gcm.c b/sys/crypto/openssl/ossl_aes_gcm.c
index e1cdc710c3aa..cceb9720dabe 100644
--- a/sys/crypto/openssl/ossl_aes_gcm.c
+++ b/sys/crypto/openssl/ossl_aes_gcm.c
@@ -10,10 +10,10 @@
*/
/*
- * This file contains an AES-GCM wrapper implementation from OpenSSL, using
- * AES-NI (x86) or POWER8 Crypto Extensions (ppc). It was ported from
- * cipher_aes_gcm_hw_aesni.inc and it makes use of a generic C implementation
- * for partial blocks, ported from gcm128.c with OPENSSL_SMALL_FOOTPRINT defined.
+ * This file contains AES-GCM wrapper implementations from OpenSSL, using
+ * AES-NI (x86), ARMv8 Crypto Extensions, or POWER8 Crypto Extensions. It
+ * makes use of a generic C implementation for partial blocks, ported from
+ * gcm128.c with OPENSSL_SMALL_FOOTPRINT defined.
*/
#include <sys/endian.h>
@@ -53,6 +53,86 @@ void GCM_gmult(uint64_t Xi[2], const __uint128_t Htable[16]);
void GCM_ghash(uint64_t Xi[2], const __uint128_t Htable[16], const void *in,
size_t len);
+#elif defined(__aarch64__)
+#define AES_set_encrypt_key aes_v8_set_encrypt_key
+#define AES_gcm_encrypt armv8_aes_gcm_encrypt
+#define AES_gcm_decrypt armv8_aes_gcm_decrypt
+#define AES_encrypt aes_v8_encrypt
+#define AES_ctr32_encrypt_blocks aes_v8_ctr32_encrypt_blocks
+#define GCM_init gcm_init_v8
+#define GCM_gmult gcm_gmult_v8
+#define GCM_ghash gcm_ghash_v8
+
+int AES_set_encrypt_key(const unsigned char *key, int bits, void *ctx);
+void AES_encrypt(const unsigned char *in, unsigned char *out, const void *ks);
+void AES_ctr32_encrypt_blocks(const unsigned char *in, unsigned char *out,
+ size_t blocks, const void *ks, const unsigned char *iv);
+
+void GCM_init(__uint128_t Htable[16], uint64_t Xi[2]);
+void GCM_gmult(uint64_t Xi[2], const __uint128_t Htable[16]);
+void GCM_ghash(uint64_t Xi[2], const __uint128_t Htable[16], const void *in,
+ size_t len);
+
+typedef size_t armv8_gcm_kernel_t(const unsigned char *, uint64_t,
+ unsigned char *, uint64_t *, unsigned char *, const void *);
+
+armv8_gcm_kernel_t aes_gcm_enc_128_kernel;
+armv8_gcm_kernel_t aes_gcm_enc_192_kernel;
+armv8_gcm_kernel_t aes_gcm_enc_256_kernel;
+armv8_gcm_kernel_t aes_gcm_dec_128_kernel;
+armv8_gcm_kernel_t aes_gcm_dec_192_kernel;
+armv8_gcm_kernel_t aes_gcm_dec_256_kernel;
+
+static size_t
+armv8_aes_gcm_encrypt(const unsigned char *in, unsigned char *out, size_t len,
+ const void *key, unsigned char ivec[16], uint64_t Xi[2])
+{
+ const struct ossl_aes_keysched *aes_key;
+ size_t bulk;
+
+ aes_key = key;
+ bulk = len - len % AES_BLOCK_LEN;
+ switch (aes_key->rounds) {
+ case 10:
+ (void)aes_gcm_enc_128_kernel(in, bulk * 8, out, Xi, ivec, key);
+ break;
+ case 12:
+ (void)aes_gcm_enc_192_kernel(in, bulk * 8, out, Xi, ivec, key);
+ break;
+ case 14:
+ (void)aes_gcm_enc_256_kernel(in, bulk * 8, out, Xi, ivec, key);
+ break;
+ default:
+ return (0);
+ }
+ return (bulk);
+}
+
+static size_t
+armv8_aes_gcm_decrypt(const unsigned char *in, unsigned char *out, size_t len,
+ const void *key, unsigned char ivec[16], uint64_t Xi[2])
+{
+ const struct ossl_aes_keysched *aes_key;
+ size_t bulk;
+
+ aes_key = key;
+ bulk = len - len % AES_BLOCK_LEN;
+ switch (aes_key->rounds) {
+ case 10:
+ (void)aes_gcm_dec_128_kernel(in, bulk * 8, out, Xi, ivec, key);
+ break;
+ case 12:
+ (void)aes_gcm_dec_192_kernel(in, bulk * 8, out, Xi, ivec, key);
+ break;
+ case 14:
+ (void)aes_gcm_dec_256_kernel(in, bulk * 8, out, Xi, ivec, key);
+ break;
+ default:
+ return (0);
+ }
+ return (bulk);
+}
+
#elif defined(__powerpc64__)
#define AES_set_encrypt_key aes_p8_set_encrypt_key
#define AES_gcm_encrypt(i,o,l,k,v,x) ppc_aes_gcm_crypt(i,o,l,k,v,x,1)
diff --git a/sys/modules/ossl/Makefile b/sys/modules/ossl/Makefile
index c516fe0c158d..35e0f30e07bb 100644
--- a/sys/modules/ossl/Makefile
+++ b/sys/modules/ossl/Makefile
@@ -28,14 +28,17 @@ SRCS.arm= \
ossl_aes_gcm_neon.c
SRCS.aarch64= \
+ aes-gcm-armv8_64.S \
chacha-armv8.S \
chacha-armv8-sve.S \
+ ghashv8-armx.S \
poly1305-armv8.S \
sha1-armv8.S \
sha256-armv8.S \
sha512-armv8.S \
vpaes-armv8.S \
- ossl_aarch64.c
+ ossl_aarch64.c \
+ ossl_aes_gcm.c
SRCS.amd64= \
aes-gcm-avx512.S \