svn commit: r308921 - in head: share/man/man4 sys/conf sys/crypto/armv8 sys/modules sys/modules/armv8crypto
Andrew Turner
andrew at FreeBSD.org
Mon Nov 21 11:18:02 UTC 2016
Author: andrew
Date: Mon Nov 21 11:18:00 2016
New Revision: 308921
URL: https://svnweb.freebsd.org/changeset/base/308921
Log:
Add accelerated AES with using the ARMv8 crypto instructions. This is based
on the AES-NI code, and modified as needed for use on ARMv8. When loaded
the driver will check the appropriate field in the id_aa64isar0_el1
register to see if AES is supported, and if so the probe function will
signal the driver should attach.
With this I have seen up to 2000Mb/s from the cryptotest test with a single
thread on a ThunderX Pass 2.0.
Reviewed by: imp
Obtained from: ABT Systems Ltd
MFC after: 1 week
Sponsored by: The FreeBSD Foundation
Differential Revision: https://reviews.freebsd.org/D8297
Added:
head/share/man/man4/armv8crypto.4 (contents, props changed)
head/sys/crypto/armv8/
head/sys/crypto/armv8/armv8_crypto.c (contents, props changed)
head/sys/crypto/armv8/armv8_crypto.h (contents, props changed)
head/sys/crypto/armv8/armv8_crypto_wrap.c (contents, props changed)
head/sys/modules/armv8crypto/
head/sys/modules/armv8crypto/Makefile (contents, props changed)
Modified:
head/share/man/man4/Makefile
head/sys/conf/files.arm64
head/sys/modules/Makefile
Modified: head/share/man/man4/Makefile
==============================================================================
--- head/share/man/man4/Makefile Mon Nov 21 10:49:37 2016 (r308920)
+++ head/share/man/man4/Makefile Mon Nov 21 11:18:00 2016 (r308921)
@@ -53,6 +53,7 @@ MAN= aac.4 \
${_aout.4} \
${_apic.4} \
arcmsr.4 \
+ ${_armv8crypto.4} \
${_asmc.4} \
ata.4 \
ath.4 \
@@ -746,6 +747,10 @@ MLINKS+=xe.4 if_xe.4
MLINKS+=xl.4 if_xl.4
MLINKS+=zyd.4 if_zyd.4
+.if ${MACHINE_CPUARCH} == "aarch64"
+_armv8crypto.4= armv8crypto.4
+.endif
+
.if ${MACHINE_CPUARCH} == "amd64" || ${MACHINE_CPUARCH} == "i386"
_acpi_asus.4= acpi_asus.4
_acpi_asus_wmi.4= acpi_asus_wmi.4
Added: head/share/man/man4/armv8crypto.4
==============================================================================
--- /dev/null 00:00:00 1970 (empty, because file is newly added)
+++ head/share/man/man4/armv8crypto.4 Mon Nov 21 11:18:00 2016 (r308921)
@@ -0,0 +1,83 @@
+.\" Copyright (c) 2016 The FreeBSD Foundation
+.\" All rights reserved.
+.\"
+.\" This software was developed by Andrew Turner under
+.\" sponsorship from the FreeBSD Foundation.
+.\"
+.\" Redistribution and use in source and binary forms, with or without
+.\" modification, are permitted provided that the following conditions
+.\" are met:
+.\" 1. Redistributions of source code must retain the above copyright
+.\" notice, this list of conditions and the following disclaimer.
+.\" 2. Redistributions in binary form must reproduce the above copyright
+.\" notice, this list of conditions and the following disclaimer in the
+.\" documentation and/or other materials provided with the distribution.
+.\"
+.\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+.\" ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+.\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+.\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+.\" SUCH DAMAGE.
+.\"
+.\" $FreeBSD$
+.\"
+.Dd October 20, 2016
+.Dt ARMV8CRYPTO 4
+.Os
+.Sh NAME
+.Nm armv8crypto
+.Nd "driver for the AES accelerator on ARM CPUs"
+.Sh SYNOPSIS
+To compile this driver into the kernel,
+place the following lines in your
+kernel configuration file:
+.Bd -ragged -offset indent
+.Cd "device crypto"
+.Cd "device armv8crypto"
+.Ed
+.Pp
+Alternatively, to load the driver as a
+module at boot time, place the following line in
+.Xr loader.conf 5 :
+.Bd -literal -offset indent
+armv8crypto_load="YES"
+.Ed
+.Sh DESCRIPTION
+Starting with the ARMv8 architecture ARM Limited has added optional
+cryptography instructions to accelerate AES, SHA-1, SHA-2, and
+finite field arithmetic.
+.Pp
+The processor capability is reported as AES in the Instruction Set
+Attributes 0 line at boot.
+The
+.Nm
+driver does not attach on systems that lack the required CPU capability.
+.Pp
+The
+.Nm
+driver registers itself to accelerate AES operations for
+.Xr crypto 4 .
+.Sh SEE ALSO
+.Xr crypt 3 ,
+.Xr crypto 4 ,
+.Xr intro 4 ,
+.Xr ipsec 4 ,
+.Xr random 4 ,
+.Xr crypto 9
+.Sh HISTORY
+The
+.Nm
+driver first appeared in
+.Fx 11.0 .
+.Sh AUTHORS
+.An -nosplit
+The
+.Nm
+driver was written by
+.An Andrew Turner Aq Mt andrew at FreeBSD.org .
Modified: head/sys/conf/files.arm64
==============================================================================
--- head/sys/conf/files.arm64 Mon Nov 21 10:49:37 2016 (r308920)
+++ head/sys/conf/files.arm64 Mon Nov 21 11:18:00 2016 (r308921)
@@ -136,6 +136,12 @@ contrib/vchiq/interface/vchiq_arm/vchiq_
compile-with "${NORMAL_C} -DUSE_VCHIQ_ARM -D__VCCOREVER__=0x04000000 -I$S/contrib/vchiq"
contrib/vchiq/interface/vchiq_arm/vchiq_util.c optional vchiq soc_brcm_bcm2837 \
compile-with "${NORMAL_C} -DUSE_VCHIQ_ARM -D__VCCOREVER__=0x04000000 -I$S/contrib/vchiq"
+crypto/armv8/armv8_crypto.c optional armv8crypto
+armv8_crypto_wrap.o optional armv8crypto \
+ dependency "$S/crypto/armv8/armv8_crypto_wrap.c" \
+ compile-with "${CC} -c ${CFLAGS:C/^-O2$/-O3/:N-nostdinc:N-mgeneral-regs-only} ${WERROR} ${NO_WCAST_QUAL} ${PROF} -march=armv8a+crypto ${.IMPSRC}" \
+ no-implicit-rule \
+ clean "armv8_crypto_wrap.o"
crypto/blowfish/bf_enc.c optional crypto | ipsec
crypto/des/des_enc.c optional crypto | ipsec | netsmb
dev/acpica/acpi_if.m optional acpi
Added: head/sys/crypto/armv8/armv8_crypto.c
==============================================================================
--- /dev/null 00:00:00 1970 (empty, because file is newly added)
+++ head/sys/crypto/armv8/armv8_crypto.c Mon Nov 21 11:18:00 2016 (r308921)
@@ -0,0 +1,565 @@
+/*-
+ * Copyright (c) 2005-2008 Pawel Jakub Dawidek <pjd at FreeBSD.org>
+ * Copyright (c) 2010 Konstantin Belousov <kib at FreeBSD.org>
+ * Copyright (c) 2014,2016 The FreeBSD Foundation
+ * All rights reserved.
+ *
+ * Portions of this software were developed by John-Mark Gurney
+ * under sponsorship of the FreeBSD Foundation and
+ * Rubicon Communications, LLC (Netgate).
+ *
+ * This software was developed by Andrew Turner under
+ * sponsorship from the FreeBSD Foundation.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+/*
+ * This is based on the aesni code.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/kernel.h>
+#include <sys/bus.h>
+#include <sys/endian.h>
+#include <sys/malloc.h>
+#include <sys/mbuf.h>
+#include <sys/module.h>
+#include <sys/queue.h>
+#include <sys/rwlock.h>
+#include <sys/smp.h>
+#include <sys/uio.h>
+
+#include <machine/vfp.h>
+
+#include <opencrypto/cryptodev.h>
+#include <cryptodev_if.h>
+#include <crypto/armv8/armv8_crypto.h>
+#include <crypto/rijndael/rijndael.h>
+
+struct armv8_crypto_softc {
+ int dieing;
+ int32_t cid;
+ uint32_t sid;
+ TAILQ_HEAD(armv8_crypto_sessions_head, armv8_crypto_session) sessions;
+ struct rwlock lock;
+};
+
+static struct mtx *ctx_mtx;
+static struct fpu_kern_ctx **ctx_vfp;
+
+#define AQUIRE_CTX(i, ctx) \
+ do { \
+ (i) = PCPU_GET(cpuid); \
+ mtx_lock(&ctx_mtx[(i)]); \
+ (ctx) = ctx_vfp[(i)]; \
+ } while (0)
+#define RELEASE_CTX(i, ctx) \
+ do { \
+ mtx_unlock(&ctx_mtx[(i)]); \
+ (i) = -1; \
+ (ctx) = NULL; \
+ } while (0)
+
+static void armv8_crypto_freesession_locked(struct armv8_crypto_softc *,
+ struct armv8_crypto_session *);
+static int armv8_crypto_cipher_process(struct armv8_crypto_session *,
+ struct cryptodesc *, struct cryptop *);
+
+MALLOC_DEFINE(M_ARMV8_CRYPTO, "armv8_crypto", "ARMv8 Crypto Data");
+
+static void
+armv8_crypto_identify(driver_t *drv, device_t parent)
+{
+
+ /* NB: order 10 is so we get attached after h/w devices */
+ if (device_find_child(parent, "armv8crypto", -1) == NULL &&
+ BUS_ADD_CHILD(parent, 10, "armv8crypto", -1) == 0)
+ panic("ARMv8 crypto: could not attach");
+}
+
+static int
+armv8_crypto_probe(device_t dev)
+{
+ uint64_t reg;
+ int ret = ENXIO;
+
+ reg = READ_SPECIALREG(id_aa64isar0_el1);
+
+ switch (ID_AA64ISAR0_AES(reg)) {
+ case ID_AA64ISAR0_AES_BASE:
+ case ID_AA64ISAR0_AES_PMULL:
+ ret = 0;
+ break;
+ }
+
+ device_set_desc_copy(dev, "AES-CBC");
+
+ /* TODO: Check more fields as we support more features */
+
+ return (ret);
+}
+
+static int
+armv8_crypto_attach(device_t dev)
+{
+ struct armv8_crypto_softc *sc;
+ int i;
+
+ sc = device_get_softc(dev);
+ TAILQ_INIT(&sc->sessions);
+ sc->dieing = 0;
+ sc->sid = 1;
+
+ sc->cid = crypto_get_driverid(dev, CRYPTOCAP_F_HARDWARE |
+ CRYPTOCAP_F_SYNC);
+ if (sc->cid < 0) {
+ device_printf(dev, "Could not get crypto driver id.\n");
+ return (ENOMEM);
+ }
+
+ rw_init(&sc->lock, "armv8crypto");
+
+ ctx_mtx = malloc(sizeof(*ctx_mtx) * (mp_maxid + 1), M_ARMV8_CRYPTO,
+ M_WAITOK|M_ZERO);
+ ctx_vfp = malloc(sizeof(*ctx_vfp) * (mp_maxid + 1), M_ARMV8_CRYPTO,
+ M_WAITOK|M_ZERO);
+
+ CPU_FOREACH(i) {
+ ctx_vfp[i] = fpu_kern_alloc_ctx(0);
+ mtx_init(&ctx_mtx[i], "armv8cryptoctx", NULL, MTX_DEF|MTX_NEW);
+ }
+
+ crypto_register(sc->cid, CRYPTO_AES_CBC, 0, 0);
+
+ return (0);
+}
+
+static int
+armv8_crypto_detach(device_t dev)
+{
+ struct armv8_crypto_softc *sc;
+ struct armv8_crypto_session *ses;
+ int i;
+
+ sc = device_get_softc(dev);
+
+ rw_wlock(&sc->lock);
+ TAILQ_FOREACH(ses, &sc->sessions, next) {
+ if (ses->used) {
+ rw_wunlock(&sc->lock);
+ device_printf(dev,
+ "Cannot detach, sessions still active.\n");
+ return (EBUSY);
+ }
+ }
+ sc->dieing = 1;
+ while ((ses = TAILQ_FIRST(&sc->sessions)) != NULL) {
+ TAILQ_REMOVE(&sc->sessions, ses, next);
+ free(ses, M_ARMV8_CRYPTO);
+ }
+ rw_wunlock(&sc->lock);
+ crypto_unregister_all(sc->cid);
+
+ rw_destroy(&sc->lock);
+
+ CPU_FOREACH(i) {
+ if (ctx_vfp[i] != NULL) {
+ mtx_destroy(&ctx_mtx[i]);
+ fpu_kern_free_ctx(ctx_vfp[i]);
+ }
+ ctx_vfp[i] = NULL;
+ }
+ free(ctx_mtx, M_ARMV8_CRYPTO);
+ ctx_mtx = NULL;
+ free(ctx_vfp, M_ARMV8_CRYPTO);
+ ctx_vfp = NULL;
+
+ return (0);
+}
+
+static int
+armv8_crypto_cipher_setup(struct armv8_crypto_session *ses,
+ struct cryptoini *encini)
+{
+ int i;
+
+ switch (ses->algo) {
+ case CRYPTO_AES_CBC:
+ switch (encini->cri_klen) {
+ case 128:
+ ses->rounds = AES128_ROUNDS;
+ break;
+ case 192:
+ ses->rounds = AES192_ROUNDS;
+ break;
+ case 256:
+ ses->rounds = AES256_ROUNDS;
+ break;
+ default:
+ CRYPTDEB("invalid CBC/ICM/GCM key length");
+ return (EINVAL);
+ }
+ break;
+ default:
+ return (EINVAL);
+ }
+
+ rijndaelKeySetupEnc(ses->enc_schedule, encini->cri_key,
+ encini->cri_klen);
+ rijndaelKeySetupDec(ses->dec_schedule, encini->cri_key,
+ encini->cri_klen);
+ for (i = 0; i < nitems(ses->enc_schedule); i++) {
+ ses->enc_schedule[i] = bswap32(ses->enc_schedule[i]);
+ ses->dec_schedule[i] = bswap32(ses->dec_schedule[i]);
+ }
+
+ return (0);
+}
+
+static int
+armv8_crypto_newsession(device_t dev, uint32_t *sidp, struct cryptoini *cri)
+{
+ struct armv8_crypto_softc *sc;
+ struct armv8_crypto_session *ses;
+ struct cryptoini *encini;
+ int error;
+
+ if (sidp == NULL || cri == NULL) {
+ CRYPTDEB("no sidp or cri");
+ return (EINVAL);
+ }
+
+ sc = device_get_softc(dev);
+ if (sc->dieing)
+ return (EINVAL);
+
+ ses = NULL;
+ encini = NULL;
+ for (; cri != NULL; cri = cri->cri_next) {
+ switch (cri->cri_alg) {
+ case CRYPTO_AES_CBC:
+ if (encini != NULL) {
+ CRYPTDEB("encini already set");
+ return (EINVAL);
+ }
+ encini = cri;
+ break;
+ default:
+ CRYPTDEB("unhandled algorithm");
+ return (EINVAL);
+ }
+ }
+ if (encini == NULL) {
+ CRYPTDEB("no cipher");
+ return (EINVAL);
+ }
+
+ rw_wlock(&sc->lock);
+ if (sc->dieing) {
+ rw_wunlock(&sc->lock);
+ return (EINVAL);
+ }
+
+ /*
+ * Free sessions goes first, so if first session is used, we need to
+ * allocate one.
+ */
+ ses = TAILQ_FIRST(&sc->sessions);
+ if (ses == NULL || ses->used) {
+ ses = malloc(sizeof(*ses), M_ARMV8_CRYPTO, M_NOWAIT | M_ZERO);
+ if (ses == NULL) {
+ rw_wunlock(&sc->lock);
+ return (ENOMEM);
+ }
+ ses->id = sc->sid++;
+ } else {
+ TAILQ_REMOVE(&sc->sessions, ses, next);
+ }
+ ses->used = 1;
+ TAILQ_INSERT_TAIL(&sc->sessions, ses, next);
+ rw_wunlock(&sc->lock);
+ ses->algo = encini->cri_alg;
+
+ error = armv8_crypto_cipher_setup(ses, encini);
+ if (error != 0) {
+ CRYPTDEB("setup failed");
+ rw_wlock(&sc->lock);
+ armv8_crypto_freesession_locked(sc, ses);
+ rw_wunlock(&sc->lock);
+ return (error);
+ }
+
+ *sidp = ses->id;
+ return (0);
+}
+
+static void
+armv8_crypto_freesession_locked(struct armv8_crypto_softc *sc,
+ struct armv8_crypto_session *ses)
+{
+ uint32_t sid;
+
+ rw_assert(&sc->lock, RA_WLOCKED);
+
+ sid = ses->id;
+ TAILQ_REMOVE(&sc->sessions, ses, next);
+ *ses = (struct armv8_crypto_session){};
+ ses->id = sid;
+ TAILQ_INSERT_HEAD(&sc->sessions, ses, next);
+}
+
+static int
+armv8_crypto_freesession(device_t dev, uint64_t tid)
+{
+ struct armv8_crypto_softc *sc;
+ struct armv8_crypto_session *ses;
+ uint32_t sid;
+
+ sc = device_get_softc(dev);
+ sid = ((uint32_t)tid) & 0xffffffff;
+ rw_wlock(&sc->lock);
+ TAILQ_FOREACH_REVERSE(ses, &sc->sessions, armv8_crypto_sessions_head,
+ next) {
+ if (ses->id == sid)
+ break;
+ }
+ if (ses == NULL) {
+ rw_wunlock(&sc->lock);
+ return (EINVAL);
+ }
+ armv8_crypto_freesession_locked(sc, ses);
+ rw_wunlock(&sc->lock);
+
+ return (0);
+}
+
+static int
+armv8_crypto_process(device_t dev, struct cryptop *crp, int hint __unused)
+{
+ struct armv8_crypto_softc *sc = device_get_softc(dev);
+ struct cryptodesc *crd, *enccrd;
+ struct armv8_crypto_session *ses;
+ int error;
+
+ error = 0;
+ enccrd = NULL;
+
+ /* Sanity check. */
+ if (crp == NULL)
+ return (EINVAL);
+
+ if (crp->crp_callback == NULL || crp->crp_desc == NULL) {
+ error = EINVAL;
+ goto out;
+ }
+
+ for (crd = crp->crp_desc; crd != NULL; crd = crd->crd_next) {
+ switch (crd->crd_alg) {
+ case CRYPTO_AES_CBC:
+ if (enccrd != NULL) {
+ error = EINVAL;
+ goto out;
+ }
+ enccrd = crd;
+ break;
+ default:
+ error = EINVAL;
+ goto out;
+ }
+ }
+
+ if (enccrd == NULL) {
+ error = EINVAL;
+ goto out;
+ }
+
+ /* We can only handle full blocks for now */
+ if ((enccrd->crd_len % AES_BLOCK_LEN) != 0) {
+ error = EINVAL;
+ goto out;
+ }
+
+ rw_rlock(&sc->lock);
+ TAILQ_FOREACH_REVERSE(ses, &sc->sessions, armv8_crypto_sessions_head,
+ next) {
+ if (ses->id == (crp->crp_sid & 0xffffffff))
+ break;
+ }
+ rw_runlock(&sc->lock);
+ if (ses == NULL) {
+ error = EINVAL;
+ goto out;
+ }
+
+ error = armv8_crypto_cipher_process(ses, enccrd, crp);
+
+out:
+ crp->crp_etype = error;
+ crypto_done(crp);
+ return (error);
+}
+
+static uint8_t *
+armv8_crypto_cipher_alloc(struct cryptodesc *enccrd, struct cryptop *crp,
+ int *allocated)
+{
+ struct mbuf *m;
+ struct uio *uio;
+ struct iovec *iov;
+ uint8_t *addr;
+
+ if (crp->crp_flags & CRYPTO_F_IMBUF) {
+ m = (struct mbuf *)crp->crp_buf;
+ if (m->m_next != NULL)
+ goto alloc;
+ addr = mtod(m, uint8_t *);
+ } else if (crp->crp_flags & CRYPTO_F_IOV) {
+ uio = (struct uio *)crp->crp_buf;
+ if (uio->uio_iovcnt != 1)
+ goto alloc;
+ iov = uio->uio_iov;
+ addr = (uint8_t *)iov->iov_base;
+ } else
+ addr = (uint8_t *)crp->crp_buf;
+ *allocated = 0;
+ addr += enccrd->crd_skip;
+ return (addr);
+
+alloc:
+ addr = malloc(enccrd->crd_len, M_ARMV8_CRYPTO, M_NOWAIT);
+ if (addr != NULL) {
+ *allocated = 1;
+ crypto_copydata(crp->crp_flags, crp->crp_buf, enccrd->crd_skip,
+ enccrd->crd_len, addr);
+ } else
+ *allocated = 0;
+ return (addr);
+}
+
+static int
+armv8_crypto_cipher_process(struct armv8_crypto_session *ses,
+ struct cryptodesc *enccrd, struct cryptop *crp)
+{
+ struct fpu_kern_ctx *ctx;
+ uint8_t *buf;
+ uint8_t iv[AES_BLOCK_LEN];
+ int allocated, error, i;
+ int encflag, ivlen;
+ int kt;
+
+ encflag = (enccrd->crd_flags & CRD_F_ENCRYPT) == CRD_F_ENCRYPT;
+
+ buf = armv8_crypto_cipher_alloc(enccrd, crp, &allocated);
+ if (buf == NULL)
+ return (ENOMEM);
+
+ error = 0;
+
+ kt = is_fpu_kern_thread(0);
+ if (!kt) {
+ AQUIRE_CTX(i, ctx);
+ error = fpu_kern_enter(curthread, ctx,
+ FPU_KERN_NORMAL | FPU_KERN_KTHR);
+ if (error != 0)
+ goto out;
+ }
+
+ if ((enccrd->crd_flags & CRD_F_KEY_EXPLICIT) != 0) {
+ panic("CRD_F_KEY_EXPLICIT");
+ }
+
+ switch (enccrd->crd_alg) {
+ case CRYPTO_AES_CBC:
+ ivlen = AES_BLOCK_LEN;
+ break;
+ }
+
+ /* Setup iv */
+ if (encflag) {
+ if ((enccrd->crd_flags & CRD_F_IV_EXPLICIT) != 0)
+ bcopy(enccrd->crd_iv, iv, ivlen);
+ else
+ arc4rand(iv, ivlen, 0);
+
+ if ((enccrd->crd_flags & CRD_F_IV_PRESENT) == 0)
+ crypto_copyback(crp->crp_flags, crp->crp_buf,
+ enccrd->crd_inject, ivlen, iv);
+ } else {
+ if ((enccrd->crd_flags & CRD_F_IV_EXPLICIT) != 0)
+ bcopy(enccrd->crd_iv, iv, ivlen);
+ else
+ crypto_copydata(crp->crp_flags, crp->crp_buf,
+ enccrd->crd_inject, ivlen, iv);
+ }
+
+ /* Do work */
+ switch (ses->algo) {
+ case CRYPTO_AES_CBC:
+ if (encflag)
+ armv8_aes_encrypt_cbc(ses->rounds, ses->enc_schedule,
+ enccrd->crd_len, buf, buf, iv);
+ else
+ armv8_aes_decrypt_cbc(ses->rounds, ses->dec_schedule,
+ enccrd->crd_len, buf, iv);
+ break;
+ }
+
+ if (allocated)
+ crypto_copyback(crp->crp_flags, crp->crp_buf, enccrd->crd_skip,
+ enccrd->crd_len, buf);
+
+ if (!kt) {
+ fpu_kern_leave(curthread, ctx);
+out:
+ RELEASE_CTX(i, ctx);
+ }
+ if (allocated) {
+ bzero(buf, enccrd->crd_len);
+ free(buf, M_ARMV8_CRYPTO);
+ }
+ return (error);
+}
+
+static device_method_t armv8_crypto_methods[] = {
+ DEVMETHOD(device_identify, armv8_crypto_identify),
+ DEVMETHOD(device_probe, armv8_crypto_probe),
+ DEVMETHOD(device_attach, armv8_crypto_attach),
+ DEVMETHOD(device_detach, armv8_crypto_detach),
+
+ DEVMETHOD(cryptodev_newsession, armv8_crypto_newsession),
+ DEVMETHOD(cryptodev_freesession, armv8_crypto_freesession),
+ DEVMETHOD(cryptodev_process, armv8_crypto_process),
+
+ DEVMETHOD_END,
+};
+
+static DEFINE_CLASS_0(armv8crypto, armv8_crypto_driver, armv8_crypto_methods,
+ sizeof(struct armv8_crypto_softc));
+static devclass_t armv8_crypto_devclass;
+
+DRIVER_MODULE(armv8crypto, nexus, armv8_crypto_driver, armv8_crypto_devclass,
+ 0, 0);
Added: head/sys/crypto/armv8/armv8_crypto.h
==============================================================================
--- /dev/null 00:00:00 1970 (empty, because file is newly added)
+++ head/sys/crypto/armv8/armv8_crypto.h Mon Nov 21 11:18:00 2016 (r308921)
@@ -0,0 +1,55 @@
+/*-
+ * Copyright (c) 2016 The FreeBSD Foundation
+ * All rights reserved.
+ *
+ * This software was developed by Andrew Turner under
+ * sponsorship from the FreeBSD Foundation.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _ARMV8_CRYPTO_H_
+#define _ARMV8_CRYPTO_H_
+
+#define AES128_ROUNDS 10
+#define AES192_ROUNDS 12
+#define AES256_ROUNDS 14
+#define AES_SCHED_LEN ((AES256_ROUNDS + 1) * AES_BLOCK_LEN)
+
+struct armv8_crypto_session {
+ uint32_t enc_schedule[AES_SCHED_LEN/4];
+ uint32_t dec_schedule[AES_SCHED_LEN/4];
+ int algo;
+ int rounds;
+ int used;
+ uint32_t id;
+ TAILQ_ENTRY(armv8_crypto_session) next;
+};
+
+void armv8_aes_encrypt_cbc(int, const void *, size_t, const uint8_t *,
+ uint8_t *, const uint8_t[static AES_BLOCK_LEN]);
+void armv8_aes_decrypt_cbc(int, const void *, size_t, uint8_t *,
+ const uint8_t[static AES_BLOCK_LEN]);
+
+#endif /* _ARMV8_CRYPTO_H_ */
Added: head/sys/crypto/armv8/armv8_crypto_wrap.c
==============================================================================
--- /dev/null 00:00:00 1970 (empty, because file is newly added)
+++ head/sys/crypto/armv8/armv8_crypto_wrap.c Mon Nov 21 11:18:00 2016 (r308921)
@@ -0,0 +1,128 @@
+/*-
+ * Copyright (c) 2016 The FreeBSD Foundation
+ * All rights reserved.
+ *
+ * This software was developed by Andrew Turner under
+ * sponsorship from the FreeBSD Foundation.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+/*
+ * This code is built with floating-point enabled. Make sure to have entered
+ * into floating-point context before calling any of these functions.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/malloc.h>
+#include <sys/queue.h>
+
+#include <opencrypto/cryptodev.h>
+#include <crypto/armv8/armv8_crypto.h>
+
+#include <arm_neon.h>
+
+static uint8x16_t
+armv8_aes_enc(int rounds, const uint8x16_t *keysched, const uint8x16_t from)
+{
+ uint8x16_t tmp;
+ int i;
+
+ tmp = from;
+ for (i = 0; i < rounds - 1; i += 2) {
+ tmp = vaeseq_u8(tmp, keysched[i]);
+ tmp = vaesmcq_u8(tmp);
+ tmp = vaeseq_u8(tmp, keysched[i + 1]);
+ tmp = vaesmcq_u8(tmp);
+ }
+
+ tmp = vaeseq_u8(tmp, keysched[rounds - 1]);
+ tmp = vaesmcq_u8(tmp);
+ tmp = vaeseq_u8(tmp, keysched[rounds]);
+ tmp = veorq_u8(tmp, keysched[rounds + 1]);
+
+ return (tmp);
+}
+
+static uint8x16_t
+armv8_aes_dec(int rounds, const uint8x16_t *keysched, const uint8x16_t from)
+{
+ uint8x16_t tmp;
+ int i;
+
+ tmp = from;
+ for (i = 0; i < rounds - 1; i += 2) {
+ tmp = vaesdq_u8(tmp, keysched[i]);
+ tmp = vaesimcq_u8(tmp);
+ tmp = vaesdq_u8(tmp, keysched[i+1]);
+ tmp = vaesimcq_u8(tmp);
+ }
+
+ tmp = vaesdq_u8(tmp, keysched[rounds - 1]);
+ tmp = vaesimcq_u8(tmp);
+ tmp = vaesdq_u8(tmp, keysched[rounds]);
+ tmp = veorq_u8(tmp, keysched[rounds + 1]);
+
+ return (tmp);
+}
+
+void
+armv8_aes_encrypt_cbc(int rounds, const void *key_schedule, size_t len,
+ const uint8_t *from, uint8_t *to, const uint8_t iv[static AES_BLOCK_LEN])
+{
+ uint8x16_t tot, ivreg, tmp;
+ size_t i;
+
+ len /= AES_BLOCK_LEN;
+ ivreg = vld1q_u8(iv);
+ for (i = 0; i < len; i++) {
+ tmp = vld1q_u8(from);
+ tot = armv8_aes_enc(rounds - 1, key_schedule,
+ veorq_u8(tmp, ivreg));
+ ivreg = tot;
+ vst1q_u8(to, tot);
+ from += AES_BLOCK_LEN;
+ to += AES_BLOCK_LEN;
+ }
+}
+
+void
+armv8_aes_decrypt_cbc(int rounds, const void *key_schedule, size_t len,
+ uint8_t *buf, const uint8_t iv[static AES_BLOCK_LEN])
+{
+ uint8x16_t ivreg, nextiv, tmp;
+ size_t i;
+
+ len /= AES_BLOCK_LEN;
+ ivreg = vld1q_u8(iv);
+ for (i = 0; i < len; i++) {
+ nextiv = vld1q_u8(buf);
+ tmp = armv8_aes_dec(rounds - 1, key_schedule, nextiv);
+ vst1q_u8(buf, veorq_u8(tmp, ivreg));
+ ivreg = nextiv;
+ buf += AES_BLOCK_LEN;
+ }
+}
Modified: head/sys/modules/Makefile
==============================================================================
--- head/sys/modules/Makefile Mon Nov 21 10:49:37 2016 (r308920)
+++ head/sys/modules/Makefile Mon Nov 21 11:18:00 2016 (r308921)
@@ -42,6 +42,7 @@ SUBDIR= \
${_apm} \
${_arcmsr} \
${_arcnet} \
+ ${_armv8crypto} \
${_asmc} \
ata \
ath \
@@ -539,6 +540,7 @@ _cxgb= cxgb
.endif
.if ${MACHINE_CPUARCH} == "aarch64"
+_armv8crypto= armv8crypto
_em= em
_igb= igb
.endif
Added: head/sys/modules/armv8crypto/Makefile
==============================================================================
--- /dev/null 00:00:00 1970 (empty, because file is newly added)
+++ head/sys/modules/armv8crypto/Makefile Mon Nov 21 11:18:00 2016 (r308921)
@@ -0,0 +1,20 @@
+# $FreeBSD$
+
+.PATH: ${.CURDIR}/../../crypto/armv8
+
+KMOD= armv8crypto
+SRCS= armv8_crypto.c
+SRCS+= device_if.h bus_if.h opt_bus.h cryptodev_if.h
+
+OBJS+= armv8_crypto_wrap.o
+
+# Remove -nostdinc so we can get the intrinsics.
+armv8_crypto_wrap.o: armv8_crypto_wrap.c
+ ${CC} -c ${CFLAGS:C/^-O2$/-O3/:N-nostdinc:N-mgeneral-regs-only} \
+ ${WERROR} ${PROF} \
+ -march=armv8a+crypto ${.IMPSRC}
+ ${CTFCONVERT_CMD}
+
+armv8_crypto_wrap.o: armv8_crypto.h
+
+.include <bsd.kmod.mk>
More information about the svn-src-all
mailing list