PERFORCE change 1198404 for review
John-Mark Gurney
jmg at FreeBSD.ORG
Tue Aug 5 21:56:29 UTC 2014
http://p4web.freebsd.org/@@1198404?ac=10
Change 1198404 by jmg at jmg_carbon2 on 2014/08/05 21:55:40
bring in the first rev of AES-GCM support...
This uses the NIST interface instead of the OpenBSD interface that
is only really useful for IPSEC, not for AEAD in general... This
currently uses the OpenBSD software GHASH algorithm, but I have a
faster version that will be integrated shortly...
This includes an AES-NI implementation of GHASH/GCM. This uses
code from the Intel white paper, and the license for usage of this
code is still pending.
Sponsored by: The FreeBSD Foundation
Affected files ...
... //depot/projects/opencrypto/sys/conf/files#2 edit
... //depot/projects/opencrypto/sys/conf/files.amd64#2 edit
... //depot/projects/opencrypto/sys/conf/files.i386#2 edit
... //depot/projects/opencrypto/sys/crypto/aesni/aesni.c#2 edit
... //depot/projects/opencrypto/sys/crypto/aesni/aesni.h#2 edit
... //depot/projects/opencrypto/sys/crypto/aesni/aesni_ghash.c#1 add
... //depot/projects/opencrypto/sys/crypto/aesni/aesni_wrap.c#2 edit
... //depot/projects/opencrypto/sys/modules/aesni/Makefile#2 edit
... //depot/projects/opencrypto/sys/modules/crypto/Makefile#2 edit
... //depot/projects/opencrypto/sys/opencrypto/crypto.c#2 edit
... //depot/projects/opencrypto/sys/opencrypto/cryptodev.c#2 edit
... //depot/projects/opencrypto/sys/opencrypto/cryptodev.h#2 edit
... //depot/projects/opencrypto/sys/opencrypto/cryptosoft.c#2 edit
... //depot/projects/opencrypto/sys/opencrypto/gmac.c#1 add
... //depot/projects/opencrypto/sys/opencrypto/gmac.h#1 add
... //depot/projects/opencrypto/sys/opencrypto/xform.c#2 edit
... //depot/projects/opencrypto/sys/opencrypto/xform.h#2 edit
Differences ...
==== //depot/projects/opencrypto/sys/conf/files#2 (text+ko) ====
@@ -3797,6 +3797,7 @@
opencrypto/cryptodev_if.m optional crypto
opencrypto/cryptosoft.c optional crypto
opencrypto/cryptodeflate.c optional crypto
+opencrypto/gmac.c optional crypto
opencrypto/rmd160.c optional crypto | ipsec
opencrypto/skipjack.c optional crypto
opencrypto/xform.c optional crypto
==== //depot/projects/opencrypto/sys/conf/files.amd64#2 (text+ko) ====
@@ -130,6 +130,11 @@
cddl/contrib/opensolaris/common/atomic/amd64/opensolaris_atomic.S optional zfs compile-with "${ZFS_S}"
crypto/aesni/aeskeys_amd64.S optional aesni
crypto/aesni/aesni.c optional aesni
+aesni_ghash.o optional aesni \
+ dependency "$S/crypto/aesni/aesni_ghash.c" \
+ compile-with "${CC} -c ${CFLAGS:C/^-O2$/-O3/:N-nostdinc} ${WERROR} ${PROF} -mmmx -msse -maes ${.IMPSRC}" \
+ no-implicit-rule \
+ clean "aesni_ghash.o"
aesni_wrap.o optional aesni \
dependency "$S/crypto/aesni/aesni_wrap.c" \
compile-with "${CC} -c ${CFLAGS:C/^-O2$/-O3/:N-nostdinc} ${WERROR} ${PROF} -mmmx -msse -maes ${.IMPSRC}" \
==== //depot/projects/opencrypto/sys/conf/files.i386#2 (text+ko) ====
@@ -115,6 +115,11 @@
no-implicit-rule
crypto/aesni/aeskeys_i386.S optional aesni
crypto/aesni/aesni.c optional aesni
+aesni_ghash.o optional aesni \
+ dependency "$S/crypto/aesni/aesni_ghash.c" \
+ compile-with "${CC} -c ${CFLAGS:C/^-O2$/-O3/:N-nostdinc} ${WERROR} ${PROF} -mmmx -msse -maes ${.IMPSRC}" \
+ no-implicit-rule \
+ clean "aesni_ghash.o"
aesni_wrap.o optional aesni \
dependency "$S/crypto/aesni/aesni_wrap.c" \
compile-with "${CC} -c ${CFLAGS:C/^-O2$/-O3/:N-nostdinc} ${WERROR} ${PROF} -mmmx -msse -maes ${.IMPSRC}" \
==== //depot/projects/opencrypto/sys/crypto/aesni/aesni.c#2 (text+ko) ====
@@ -1,8 +1,12 @@
/*-
* Copyright (c) 2005-2008 Pawel Jakub Dawidek <pjd at FreeBSD.org>
* Copyright (c) 2010 Konstantin Belousov <kib at FreeBSD.org>
+ * Copyright (c) 2014 The FreeBSD Foundation
* All rights reserved.
*
+ * Portions of this software were developed by John-Mark Gurney
+ * under sponsorship from the FreeBSD Foundation.
+ *
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
@@ -41,6 +45,7 @@
#include <sys/uio.h>
#include <crypto/aesni/aesni.h>
#include <cryptodev_if.h>
+#include <opencrypto/gmac.h>
struct aesni_softc {
int32_t cid;
@@ -56,7 +61,7 @@
static int aesni_cipher_setup(struct aesni_session *ses,
struct cryptoini *encini);
static int aesni_cipher_process(struct aesni_session *ses,
- struct cryptodesc *enccrd, struct cryptop *crp);
+ struct cryptodesc *enccrd, struct cryptodesc *authcrd, struct cryptop *crp);
MALLOC_DEFINE(M_AESNI, "aesni_data", "AESNI Data");
@@ -79,12 +84,12 @@
return (EINVAL);
}
- if ((cpu_feature & CPUID_SSE2) == 0) {
- device_printf(dev, "No SSE2 support but AESNI!?!\n");
+ if ((cpu_feature & CPUID2_SSE41) == 0) {
+ device_printf(dev, "No SSE4.1 support.\n");
return (EINVAL);
}
- device_set_desc_copy(dev, "AES-CBC,AES-XTS");
+ device_set_desc_copy(dev, "AES-CBC,AES-XTS,AES-GCM");
return (0);
}
@@ -106,6 +111,10 @@
rw_init(&sc->lock, "aesni_lock");
crypto_register(sc->cid, CRYPTO_AES_CBC, 0, 0);
crypto_register(sc->cid, CRYPTO_AES_XTS, 0, 0);
+ crypto_register(sc->cid, CRYPTO_AES_NIST_GCM_16, 0, 0);
+ crypto_register(sc->cid, CRYPTO_AES_128_NIST_GMAC, 0, 0);
+ crypto_register(sc->cid, CRYPTO_AES_192_NIST_GMAC, 0, 0);
+ crypto_register(sc->cid, CRYPTO_AES_256_NIST_GMAC, 0, 0);
return (0);
}
@@ -144,8 +153,10 @@
struct cryptoini *encini;
int error;
- if (sidp == NULL || cri == NULL)
+ if (sidp == NULL || cri == NULL) {
+ CRYPTDEB("no sidp or cri");
return (EINVAL);
+ }
sc = device_get_softc(dev);
ses = NULL;
@@ -154,16 +165,30 @@
switch (cri->cri_alg) {
case CRYPTO_AES_CBC:
case CRYPTO_AES_XTS:
- if (encini != NULL)
+ case CRYPTO_AES_NIST_GCM_16:
+ if (encini != NULL) {
+ CRYPTDEB("encini already set");
return (EINVAL);
+ }
encini = cri;
break;
+ case CRYPTO_AES_128_NIST_GMAC:
+ case CRYPTO_AES_192_NIST_GMAC:
+ case CRYPTO_AES_256_NIST_GMAC:
+ /*
+ * nothing to do here, maybe in the future cache some
+ * values for GHASH
+ */
+ break;
default:
+ CRYPTDEB("unhandled algorithm");
return (EINVAL);
}
}
- if (encini == NULL)
+ if (encini == NULL) {
+ CRYPTDEB("no cipher");
return (EINVAL);
+ }
rw_wlock(&sc->lock);
/*
@@ -195,6 +220,7 @@
error = aesni_cipher_setup(ses, encini);
if (error != 0) {
+ CRYPTDEB("setup failed");
rw_wlock(&sc->lock);
aesni_freesession_locked(sc, ses);
rw_wunlock(&sc->lock);
@@ -248,11 +274,13 @@
{
struct aesni_softc *sc = device_get_softc(dev);
struct aesni_session *ses = NULL;
- struct cryptodesc *crd, *enccrd;
- int error;
+ struct cryptodesc *crd, *enccrd, *authcrd;
+ int error, needauth;
error = 0;
enccrd = NULL;
+ authcrd = NULL;
+ needauth = 0;
/* Sanity check. */
if (crp == NULL)
@@ -273,11 +301,40 @@
}
enccrd = crd;
break;
+
+ case CRYPTO_AES_NIST_GCM_16:
+ if (enccrd != NULL) {
+ error = EINVAL;
+ goto out;
+ }
+ enccrd = crd;
+ needauth = 1;
+ break;
+
+ case CRYPTO_AES_128_NIST_GMAC:
+ case CRYPTO_AES_192_NIST_GMAC:
+ case CRYPTO_AES_256_NIST_GMAC:
+ if (authcrd != NULL) {
+ error = EINVAL;
+ goto out;
+ }
+ authcrd = crd;
+ needauth = 1;
+ break;
+
default:
return (EINVAL);
}
}
- if (enccrd == NULL || (enccrd->crd_len % AES_BLOCK_LEN) != 0) {
+
+ /* CBC & XTS can only handle full blocks for now */
+ if ((enccrd->crd_len == CRYPTO_AES_CBC || enccrd->crd_len ==
+ CRYPTO_AES_XTS) && (enccrd->crd_len % AES_BLOCK_LEN) != 0) {
+ error = EINVAL;
+ goto out;
+ }
+
+ if (enccrd == NULL || (needauth && authcrd == NULL)) {
error = EINVAL;
goto out;
}
@@ -293,7 +350,7 @@
goto out;
}
- error = aesni_cipher_process(ses, enccrd, crp);
+ error = aesni_cipher_process(ses, enccrd, authcrd, crp);
if (error != 0)
goto out;
@@ -376,18 +433,42 @@
return (error);
}
+static void
+printhexstr(uint8_t *ptr, int len)
+{
+ int i;
+
+ for (i = 0; i < len; i++)
+ printf("%02hhx", ptr[i]);
+}
+
static int
aesni_cipher_process(struct aesni_session *ses, struct cryptodesc *enccrd,
- struct cryptop *crp)
+ struct cryptodesc *authcrd, struct cryptop *crp)
{
+ uint8_t tag[GMAC_DIGEST_LEN];
struct thread *td;
- uint8_t *buf;
- int error, allocated;
+ uint8_t *buf, *authbuf;
+ int error, allocated, authallocated;
+ int ivlen, encflag;
+ int r;
+
+ encflag = (enccrd->crd_flags & CRD_F_ENCRYPT) == CRD_F_ENCRYPT;
buf = aesni_cipher_alloc(enccrd, crp, &allocated);
if (buf == NULL)
return (ENOMEM);
+ authbuf = NULL;
+ authallocated = 0;
+ if (authcrd != NULL) {
+ authbuf = aesni_cipher_alloc(authcrd, crp, &authallocated);
+ if (authbuf == NULL) {
+ free(buf, M_AESNI);
+ return (ENOMEM);
+ }
+ }
+
td = curthread;
error = fpu_kern_enter(td, ses->fpu_ctx, FPU_KERN_NORMAL |
FPU_KERN_KTHR);
@@ -401,42 +482,109 @@
goto out;
}
+ /* XXX - validate that enccrd and authcrd have/use same key? */
+ switch (enccrd->crd_alg) {
+ case CRYPTO_AES_CBC:
+ ivlen = 16;
+ break;
+ case CRYPTO_AES_XTS:
+ ivlen = 8;
+ break;
+ case CRYPTO_AES_NIST_GCM_16:
+ ivlen = 12; /* should support arbitarily larger */
+ break;
+ }
+
+ /* Setup ses->iv */
+ bzero(ses->iv, sizeof ses->iv);
+ /*printf("crd_flags: %#x, ivlen: %d, iv: ", enccrd->crd_flags, ivlen);*/
if ((enccrd->crd_flags & CRD_F_ENCRYPT) != 0) {
if ((enccrd->crd_flags & CRD_F_IV_EXPLICIT) != 0)
- bcopy(enccrd->crd_iv, ses->iv, AES_BLOCK_LEN);
+ bcopy(enccrd->crd_iv, ses->iv, ivlen);
if ((enccrd->crd_flags & CRD_F_IV_PRESENT) == 0)
crypto_copyback(crp->crp_flags, crp->crp_buf,
- enccrd->crd_inject, AES_BLOCK_LEN, ses->iv);
- if (ses->algo == CRYPTO_AES_CBC) {
+ enccrd->crd_inject, ivlen, ses->iv);
+ } else {
+ if ((enccrd->crd_flags & CRD_F_IV_EXPLICIT) != 0)
+ bcopy(enccrd->crd_iv, ses->iv, ivlen);
+ else
+ crypto_copydata(crp->crp_flags, crp->crp_buf,
+ enccrd->crd_inject, ivlen, ses->iv);
+ }
+ /*printhexstr(ses->iv, ivlen);
+ printf("\n");*/
+
+ if (authcrd != NULL && !encflag) {
+ crypto_copydata(crp->crp_flags, crp->crp_buf,
+ authcrd->crd_inject, GMAC_DIGEST_LEN, tag);
+ } else {
+ /*printf("ptag: ");
+ printhexstr(tag, sizeof tag);
+ printf("\n");*/
+ bzero(tag, sizeof tag);
+ }
+
+ /* Do work */
+ switch (ses->algo) {
+ case CRYPTO_AES_CBC:
+ if (encflag)
aesni_encrypt_cbc(ses->rounds, ses->enc_schedule,
enccrd->crd_len, buf, buf, ses->iv);
- } else /* if (ses->algo == CRYPTO_AES_XTS) */ {
+ else
+ aesni_decrypt_cbc(ses->rounds, ses->dec_schedule,
+ enccrd->crd_len, buf, ses->iv);
+ break;
+ case CRYPTO_AES_XTS:
+ if (encflag)
aesni_encrypt_xts(ses->rounds, ses->enc_schedule,
ses->xts_schedule, enccrd->crd_len, buf, buf,
ses->iv);
- }
- } else {
- if ((enccrd->crd_flags & CRD_F_IV_EXPLICIT) != 0)
- bcopy(enccrd->crd_iv, ses->iv, AES_BLOCK_LEN);
else
- crypto_copydata(crp->crp_flags, crp->crp_buf,
- enccrd->crd_inject, AES_BLOCK_LEN, ses->iv);
- if (ses->algo == CRYPTO_AES_CBC) {
- aesni_decrypt_cbc(ses->rounds, ses->dec_schedule,
- enccrd->crd_len, buf, ses->iv);
- } else /* if (ses->algo == CRYPTO_AES_XTS) */ {
aesni_decrypt_xts(ses->rounds, ses->dec_schedule,
ses->xts_schedule, enccrd->crd_len, buf, buf,
ses->iv);
+ break;
+ case CRYPTO_AES_NIST_GCM_16:
+ /*printf("GCM: %d\n", encflag);
+ printf("buf(%d): ", enccrd->crd_len);
+ printhexstr(buf, enccrd->crd_len);
+ printf("\nauthbuf(%d): ", authcrd->crd_len);
+ printhexstr(authbuf, authcrd->crd_len);
+ printf("\niv: ");
+ printhexstr(ses->iv, ivlen);
+ printf("\ntag: ");
+ printhexstr(tag, 16);
+ printf("\nsched: ");
+ printhexstr(ses->enc_schedule, 16 * (ses->rounds + 1));
+ printf("\n");*/
+ if (encflag)
+ AES_GCM_encrypt(buf, buf, authbuf, ses->iv, tag,
+ enccrd->crd_len, authcrd->crd_len, ivlen,
+ ses->enc_schedule, ses->rounds);
+ else {
+ r = AES_GCM_decrypt(buf, buf, authbuf, ses->iv, tag,
+ enccrd->crd_len, authcrd->crd_len, ivlen,
+ ses->enc_schedule, ses->rounds);
+ /*printf("dec r: %d\n", r);*/
}
+ break;
}
+
if (allocated)
crypto_copyback(crp->crp_flags, crp->crp_buf, enccrd->crd_skip,
enccrd->crd_len, buf);
+
+ /* OpenBSD doesn't copy this back. */
if ((enccrd->crd_flags & CRD_F_ENCRYPT) != 0)
crypto_copydata(crp->crp_flags, crp->crp_buf,
enccrd->crd_skip + enccrd->crd_len - AES_BLOCK_LEN,
AES_BLOCK_LEN, ses->iv);
+
+ if (authcrd != NULL) {
+ crypto_copyback(crp->crp_flags, crp->crp_buf,
+ authcrd->crd_inject, GMAC_DIGEST_LEN, tag);
+ }
+
out:
fpu_kern_leave(td, ses->fpu_ctx);
out1:
@@ -444,5 +592,9 @@
bzero(buf, enccrd->crd_len);
free(buf, M_AESNI);
}
+ if (authallocated) {
+ bzero(authbuf, authcrd->crd_len);
+ free(authbuf, M_AESNI);
+ }
return (error);
}
==== //depot/projects/opencrypto/sys/crypto/aesni/aesni.h#2 (text+ko) ====
@@ -96,6 +96,16 @@
const void *tweak_schedule /*__aligned(16)*/, size_t len,
const uint8_t *from, uint8_t *to, const uint8_t iv[AES_BLOCK_LEN]);
+/* GCM & GHASH functions */
+void AES_GCM_encrypt(const unsigned char *in, unsigned char *out,
+ const unsigned char *addt, const unsigned char *ivec,
+ unsigned char *tag, uint32_t nbytes, uint32_t abytes, int ibytes,
+ const unsigned char *key, int nr);
+int AES_GCM_decrypt(const unsigned char *in, unsigned char *out,
+ const unsigned char *addt, const unsigned char *ivec,
+ unsigned char *tag, uint32_t nbytes, uint32_t abytes, int ibytes,
+ const unsigned char *key, int nr);
+
int aesni_cipher_setup_common(struct aesni_session *ses, const uint8_t *key,
int keylen);
uint8_t *aesni_cipher_alloc(struct cryptodesc *enccrd, struct cryptop *crp,
==== //depot/projects/opencrypto/sys/crypto/aesni/aesni_wrap.c#2 (text+ko) ====
@@ -3,8 +3,12 @@
* Copyright (c) 2010 Konstantin Belousov <kib at FreeBSD.org>
* Copyright (c) 2010-2011 Pawel Jakub Dawidek <pawel at dawidek.net>
* Copyright 2012-2013 John-Mark Gurney <jmg at FreeBSD.org>
+ * Copyright (c) 2014 The FreeBSD Foundation
* All rights reserved.
*
+ * Portions of this software were developed by John-Mark Gurney
+ * under sponsorship from the FreeBSD Foundation.
+ *
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
@@ -29,14 +33,16 @@
#include <sys/cdefs.h>
__FBSDID("$FreeBSD: head/sys/crypto/aesni/aesni_wrap.c 267815 2014-06-24 06:55:49Z kib $");
-
+
#include <sys/param.h>
#include <sys/libkern.h>
#include <sys/malloc.h>
#include <sys/proc.h>
#include <sys/systm.h>
#include <crypto/aesni/aesni.h>
-
+
+#include <opencrypto/gmac.h>
+
#include "aesencdec.h"
MALLOC_DECLARE(M_AESNI);
@@ -336,6 +342,7 @@
switch (ses->algo) {
case CRYPTO_AES_CBC:
+ case CRYPTO_AES_NIST_GCM_16:
switch (keylen) {
case 128:
ses->rounds = AES128_ROUNDS;
@@ -347,6 +354,7 @@
ses->rounds = AES256_ROUNDS;
break;
default:
+ CRYPTDEB("invalid CBC/GCM key length");
return (EINVAL);
}
break;
@@ -359,6 +367,7 @@
ses->rounds = AES256_ROUNDS;
break;
default:
+ CRYPTDEB("invalid XTS key length");
return (EINVAL);
}
break;
@@ -368,7 +377,9 @@
aesni_set_enckey(key, ses->enc_schedule, ses->rounds);
aesni_set_deckey(ses->enc_schedule, ses->dec_schedule, ses->rounds);
- if (ses->algo == CRYPTO_AES_CBC)
+
+ /* setup IV */
+ if (ses->algo == CRYPTO_AES_CBC || ses->algo == CRYPTO_AES_NIST_GCM_16)
arc4rand(ses->iv, sizeof(ses->iv), 0);
else /* if (ses->algo == CRYPTO_AES_XTS) */ {
aesni_set_enckey(key + keylen / 16, ses->xts_schedule,
==== //depot/projects/opencrypto/sys/modules/aesni/Makefile#2 (text+ko) ====
@@ -7,13 +7,18 @@
SRCS+= aeskeys_${MACHINE_CPUARCH}.S
SRCS+= device_if.h bus_if.h opt_bus.h cryptodev_if.h
-OBJS+= aesni_wrap.o
+OBJS+= aesni_ghash.o aesni_wrap.o
# Remove -nostdinc so we can get the intrinsics.
+aesni_ghash.o: aesni_ghash.c
+ # XXX - gcc won't understand -mpclmul
+ ${CC} -c ${CFLAGS:C/^-O2$/-O3/:N-nostdinc} ${WERROR} ${PROF} \
+ -mmmx -msse -msse4 -maes -mpclmul ${.IMPSRC}
+ ${CTFCONVERT_CMD}
+
aesni_wrap.o: aesni_wrap.c
${CC} -c ${CFLAGS:C/^-O2$/-O3/:N-nostdinc} ${WERROR} ${PROF} \
-mmmx -msse -maes ${.IMPSRC}
${CTFCONVERT_CMD}
.include <bsd.kmod.mk>
-
==== //depot/projects/opencrypto/sys/modules/crypto/Makefile#2 (text+ko) ====
@@ -18,6 +18,7 @@
SRCS += des_ecb.c des_enc.c des_setkey.c
SRCS += sha1.c sha2.c sha256c.c
SRCS += siphash.c
+SRCS += gmac.c
SRCS += opt_param.h cryptodev_if.h bus_if.h device_if.h
SRCS += opt_ddb.h
==== //depot/projects/opencrypto/sys/opencrypto/crypto.c#2 (text+ko) ====
@@ -316,8 +316,12 @@
/* See if all the algorithms are supported. */
for (cr = cri; cr; cr = cr->cri_next)
- if (cap->cc_alg[cr->cri_alg] == 0)
+ if (cap->cc_alg[cr->cri_alg] == 0) {
+#ifdef DEBUG
+ printf("cr->cri_alg: %d\n", cr->cri_alg);
+#endif
return 0;
+ }
return 1;
}
@@ -421,9 +425,12 @@
(*sid) <<= 32;
(*sid) |= (lid & 0xffffffff);
cap->cc_sessions++;
- }
- } else
+ } else
+ CRYPTDEB("dev newsession failed");
+ } else {
+ CRYPTDEB("no driver");
err = EINVAL;
+ }
CRYPTO_DRIVER_UNLOCK();
return err;
}
==== //depot/projects/opencrypto/sys/opencrypto/cryptodev.c#2 (text+ko) ====
@@ -3,7 +3,12 @@
/*-
* Copyright (c) 2001 Theo de Raadt
* Copyright (c) 2002-2006 Sam Leffler, Errno Consulting
+ * Copyright (c) 2014 The FreeBSD Foundation
+ * All rights reserved.
*
+ * Portions of this software were developed by John-Mark Gurney
+ * under sponsorship from the FreeBSD Foundation.
+ *
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
@@ -317,6 +322,8 @@
static int cryptodev_op(struct csession *, struct crypt_op *,
struct ucred *, struct thread *td);
+static int cryptodev_aead(struct csession *, struct crypt_aead *,
+ struct ucred *, struct thread *);
static int cryptodev_key(struct crypt_kop *);
static int cryptodev_find(struct crypt_find_op *);
@@ -377,6 +384,7 @@
struct csession *cse;
struct session_op *sop;
struct crypt_op *cop;
+ struct crypt_aead *caead;
struct enc_xform *txform = NULL;
struct auth_hash *thash = NULL;
struct crypt_kop *kop;
@@ -437,7 +445,15 @@
case CRYPTO_CAMELLIA_CBC:
txform = &enc_xform_camellia;
break;
+ case CRYPTO_AES_NIST_CTR:
+ txform = &enc_xform_aes_nist_ctr;
+ break;
+ case CRYPTO_AES_NIST_GCM_16:
+ txform = &enc_xform_aes_nist_gcm;
+ break;
+
default:
+ CRYPTDEB("invalid cipher");
return (EINVAL);
}
@@ -462,6 +478,16 @@
case CRYPTO_RIPEMD160_HMAC:
thash = &auth_hash_hmac_ripemd_160;
break;
+ case CRYPTO_AES_128_NIST_GMAC:
+ thash = &auth_hash_nist_gmac_aes_128;
+ break;
+ case CRYPTO_AES_192_NIST_GMAC:
+ thash = &auth_hash_nist_gmac_aes_192;
+ break;
+ case CRYPTO_AES_256_NIST_GMAC:
+ thash = &auth_hash_nist_gmac_aes_256;
+ break;
+
#ifdef notdef
case CRYPTO_MD5:
thash = &auth_hash_md5;
@@ -474,6 +500,7 @@
thash = &auth_hash_null;
break;
default:
+ CRYPTDEB("invalid mac");
return (EINVAL);
}
@@ -485,6 +512,7 @@
crie.cri_klen = sop->keylen * 8;
if (sop->keylen > txform->maxkey ||
sop->keylen < txform->minkey) {
+ CRYPTDEB("invalid cipher parameters");
error = EINVAL;
goto bail;
}
@@ -492,8 +520,10 @@
crie.cri_key = malloc(crie.cri_klen / 8,
M_XDATA, M_WAITOK);
if ((error = copyin(sop->key, crie.cri_key,
- crie.cri_klen / 8)))
+ crie.cri_klen / 8))) {
+ CRYPTDEB("invalid key");
goto bail;
+ }
if (thash)
crie.cri_next = &cria;
}
@@ -502,6 +532,7 @@
cria.cri_alg = thash->type;
cria.cri_klen = sop->mackeylen * 8;
if (sop->mackeylen != thash->keysize) {
+ CRYPTDEB("invalid mac key length");
error = EINVAL;
goto bail;
}
@@ -510,8 +541,10 @@
cria.cri_key = malloc(cria.cri_klen / 8,
M_XDATA, M_WAITOK);
if ((error = copyin(sop->mackey, cria.cri_key,
- cria.cri_klen / 8)))
+ cria.cri_klen / 8))) {
+ CRYPTDEB("invalid mac key");
goto bail;
+ }
}
}
@@ -523,13 +556,17 @@
) {
crid = SES2(sop)->crid;
error = checkforsoftware(crid);
- if (error)
+ if (error) {
+ CRYPTDEB("checkforsoftware");
goto bail;
+ }
} else
crid = CRYPTOCAP_F_HARDWARE;
error = crypto_newsession(&sid, (txform ? &crie : &cria), crid);
- if (error)
+ if (error) {
+ CRYPTDEB("crypto_newsession");
goto bail;
+ }
cse = csecreate(fcr, sid, crie.cri_key, crie.cri_klen,
cria.cri_key, cria.cri_klen, sop->cipher, sop->mac, txform,
@@ -538,6 +575,7 @@
if (cse == NULL) {
crypto_freesession(sid);
error = EINVAL;
+ CRYPTDEB("csecreate");
goto bail;
}
sop->ses = cse->ses;
@@ -639,6 +677,13 @@
case CIOCFINDDEV:
error = cryptodev_find((struct crypt_find_op *)data);
break;
+ case CIOCCRYPTAEAD:
+ caead = (struct crypt_aead *)data;
+ cse = csefind(fcr, caead->ses);
+ if (cse == NULL)
+ return (EINVAL);
+ error = cryptodev_aead(cse, caead, active_cred, td);
+ break;
default:
error = EINVAL;
break;
@@ -816,6 +861,152 @@
}
static int
+cryptodev_aead(
+ struct csession *cse,
+ struct crypt_aead *caead,
+ struct ucred *active_cred,
+ struct thread *td)
+{
+ struct uio *uio;
+ struct cryptop *crp = NULL;
+ struct cryptodesc *crde = NULL, *crda = NULL;
+ int error;
+
+ if (caead->len > 256*1024-4)
+ return (E2BIG);
+
+ if (cse->txform == NULL || cse->thash == NULL || caead->tag == NULL ||
+ (caead->len % cse->txform->blocksize) != 0)
+ return (EINVAL);
+
+ uio = &cse->uio;
+ uio->uio_iov = &cse->iovec;
+ uio->uio_iovcnt = 1;
+ uio->uio_offset = 0;
+ uio->uio_resid = caead->len + caead->aadlen + cse->thash->hashsize;
+ uio->uio_segflg = UIO_SYSSPACE;
+ uio->uio_rw = UIO_WRITE;
+ uio->uio_td = td;
+ uio->uio_iov[0].iov_len = uio->uio_resid;
+
+ uio->uio_iov[0].iov_base = malloc(uio->uio_iov[0].iov_len,
+ M_XDATA, M_WAITOK);
+
+ crp = crypto_getreq(2);
+ if (crp == NULL) {
+ error = ENOMEM;
+ goto bail;
+ }
+
+ crda = crp->crp_desc;
+ crde = crda->crd_next;
+
+ if ((error = copyin(caead->src, cse->uio.uio_iov[0].iov_base,
+ caead->len)))
+ goto bail;
+
+ if ((error = copyin(caead->aad, (char *)cse->uio.uio_iov[0].iov_base +
+ caead->len, caead->aadlen)))
+ goto bail;
+
+ crda->crd_skip = caead->len;
+ crda->crd_len = caead->aadlen;
+ crda->crd_inject = caead->len + caead->aadlen;
+
+ crda->crd_alg = cse->mac;
+ crda->crd_key = cse->mackey;
+ crda->crd_klen = cse->mackeylen * 8;
+
+ if (caead->op == COP_ENCRYPT)
+ crde->crd_flags |= CRD_F_ENCRYPT;
+ else
+ crde->crd_flags &= ~CRD_F_ENCRYPT;
+ /* crde->crd_skip set below */
+ crde->crd_len = caead->len;
+ crde->crd_inject = 0;
+
+ crde->crd_alg = cse->cipher;
+ crde->crd_key = cse->key;
+ crde->crd_klen = cse->keylen * 8;
+
+ crp->crp_ilen = caead->len + caead->aadlen;
+ crp->crp_flags = CRYPTO_F_IOV | CRYPTO_F_CBIMM
+ | (caead->flags & COP_F_BATCH);
+ crp->crp_buf = (caddr_t)&cse->uio.uio_iov;
+ crp->crp_callback = (int (*) (struct cryptop *)) cryptodev_cb;
+ crp->crp_sid = cse->sid;
+ crp->crp_opaque = (void *)cse;
+
+ if (caead->iv) {
+ if (caead->ivlen > sizeof cse->tmp_iv) {
+ error = EINVAL;
+ printf("a: %d, b: %lu\n", caead->ivlen, sizeof cse->tmp_iv);
+ goto bail;
+ }
+
+ if ((error = copyin(caead->iv, cse->tmp_iv, caead->ivlen)))
+ goto bail;
+ bcopy(cse->tmp_iv, crde->crd_iv, caead->ivlen);
+ crde->crd_flags |= CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT;
+ crde->crd_skip = 0;
+ } else {
+ crde->crd_flags |= CRD_F_IV_PRESENT;
+ crde->crd_skip = cse->txform->blocksize;
+ crde->crd_len -= cse->txform->blocksize;
+ }
+
+ if ((error = copyin(caead->tag, (caddr_t)cse->uio.uio_iov[0].iov_base +
+ caead->len + caead->aadlen, cse->thash->hashsize)))
+ goto bail;
+again:
+ /*
+ * Let the dispatch run unlocked, then, interlock against the
+ * callback before checking if the operation completed and going
+ * to sleep. This insures drivers don't inherit our lock which
+ * results in a lock order reversal between crypto_dispatch forced
+ * entry and the crypto_done callback into us.
+ */
+ error = crypto_dispatch(crp);
+ mtx_lock(&cse->lock);
+ if (error == 0 && (crp->crp_flags & CRYPTO_F_DONE) == 0)
+ error = msleep(crp, &cse->lock, PWAIT, "crydev", 0);
+ mtx_unlock(&cse->lock);
+
+ if (error != 0)
+ goto bail;
+
+ if (crp->crp_etype == EAGAIN) {
+ crp->crp_etype = 0;
+ crp->crp_flags &= ~CRYPTO_F_DONE;
+ goto again;
+ }
+
+ if (crp->crp_etype != 0) {
+ error = crp->crp_etype;
+ goto bail;
+ }
+
+ if (cse->error) {
+ error = cse->error;
+ goto bail;
+ }
+
+ if (caead->dst && (error = copyout(cse->uio.uio_iov[0].iov_base,
+ caead->dst, caead->len)))
+ goto bail;
+
+ if ((error = copyout((caddr_t)cse->uio.uio_iov[0].iov_base +
+ caead->len + caead->aadlen, caead->tag, cse->thash->hashsize)))
+ goto bail;
+
+bail:
+ crypto_freereq(crp);
+ free(cse->uio.uio_iov[0].iov_base, M_XDATA);
+
+ return (error);
+}
+
+static int
cryptodev_cb(void *op)
{
struct cryptop *crp = (struct cryptop *) op;
==== //depot/projects/opencrypto/sys/opencrypto/cryptodev.h#2 (text+ko) ====
@@ -23,6 +23,11 @@
* PURPOSE.
*
* Copyright (c) 2001 Theo de Raadt
+ * Copyright (c) 2014 The FreeBSD Foundation
+ * All rights reserved.
+ *
+ * Portions of this software were developed by John-Mark Gurney
+ * under sponsorship from the FreeBSD Foundation.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
@@ -100,6 +105,9 @@
#define CAMELLIA_BLOCK_LEN 16
#define EALG_MAX_BLOCK_LEN AES_BLOCK_LEN /* Keep this updated */
+/* Maximum hash algorithm result length */
+#define AALG_MAX_RESULT_LEN 64 /* Keep this updated */
+
#define CRYPTO_ALGORITHM_MIN 1
#define CRYPTO_DES_CBC 1
#define CRYPTO_3DES_CBC 2
@@ -122,9 +130,15 @@
#define CRYPTO_SHA2_256_HMAC 18
#define CRYPTO_SHA2_384_HMAC 19
#define CRYPTO_SHA2_512_HMAC 20
-#define CRYPTO_CAMELLIA_CBC 21
+#define CRYPTO_CAMELLIA_CBC 21
#define CRYPTO_AES_XTS 22
-#define CRYPTO_ALGORITHM_MAX 22 /* Keep updated - see below */
+#define CRYPTO_AES_NIST_CTR 23
+#define CRYPTO_AES_NIST_GMAC 24 /* cipher side */
+#define CRYPTO_AES_NIST_GCM_16 25 /* 16 byte ICV */
+#define CRYPTO_AES_128_NIST_GMAC 26 /* auth side */
+#define CRYPTO_AES_192_NIST_GMAC 27 /* auth side */
+#define CRYPTO_AES_256_NIST_GMAC 28 /* auth side */
+#define CRYPTO_ALGORITHM_MAX 28 /* Keep updated - see below */
/* Algorithm flags */
#define CRYPTO_ALG_FLAG_SUPPORTED 0x01 /* Algorithm is supported */
@@ -182,6 +196,20 @@
caddr_t iv;
};
+/* op and flags the same as crypto_op */
+struct crypt_aead {
+ u_int32_t ses;
+ u_int16_t op; /* i.e. COP_ENCRYPT */
+ u_int16_t flags;
+ u_int len;
+ u_int aadlen;
+ u_int ivlen;
+ caddr_t src, dst; /* become iov[] inside kernel */
+ caddr_t aad; /* additional authenticated data */
+ caddr_t tag; /* must fit for chosen TAG length */
+ caddr_t iv;
+};
+
/*
* Parameters for looking up a crypto driver/device by
* device name or by id. The latter are returned for
@@ -239,6 +267,7 @@
#define CIOCGSESSION2 _IOWR('c', 106, struct session2_op)
#define CIOCKEY2 _IOWR('c', 107, struct crypt_kop)
#define CIOCFINDDEV _IOWR('c', 108, struct crypt_find_op)
+#define CIOCCRYPTAEAD _IOWR('c', 109, struct crypt_aead)
struct cryptotstat {
struct timespec acc; /* total accumulated time */
@@ -269,6 +298,14 @@
};
#ifdef _KERNEL
+
+#if 1
+#define CRYPTDEB(s) do { printf("%s:%d: %s\n", __FILE__, __LINE__, s); \
+ } while (0)
+#else
+#define CRYPTDEB(s) do { } while (0)
+#endif
+
/* Standard initialization structure beginning */
struct cryptoini {
int cri_alg; /* Algorithm to use */
@@ -292,10 +329,11 @@
place, so don't copy. */
#define CRD_F_IV_EXPLICIT 0x04 /* IV explicitly provided */
#define CRD_F_DSA_SHA_NEEDED 0x08 /* Compute SHA-1 of buffer for DSA */
+#define CRD_F_COMP 0x0f /* Set when doing compression */
#define CRD_F_KEY_EXPLICIT 0x10 /* Key explicitly provided */
-#define CRD_F_COMP 0x0f /* Set when doing compression */
struct cryptoini CRD_INI; /* Initialization/context data */
+#define crd_esn CRD_INI.cri_esn
>>> TRUNCATED FOR MAIL (1000 lines) <<<
More information about the p4-projects
mailing list