PERFORCE change 1199105 for review
John-Mark Gurney
jmg at FreeBSD.org
Mon Sep 8 04:51:05 UTC 2014
http://p4web.freebsd.org/@@1199105?ac=10
Change 1199105 by jmg at jmg_carbon2 on 2014/08/23 00:03:21
add support for ICM to aesni driver...
properly initalize error when authbuf fails to allocate
#if 0 out debug function...
Sponsored by: FreeBSD Foundation
Affected files ...
.. //depot/projects/opencrypto/sys/crypto/aesni/aesni.c#4 edit
.. //depot/projects/opencrypto/sys/crypto/aesni/aesni.h#3 edit
.. //depot/projects/opencrypto/sys/crypto/aesni/aesni_wrap.c#3 edit
Differences ...
==== //depot/projects/opencrypto/sys/crypto/aesni/aesni.c#4 (text+ko) ====
@@ -110,11 +110,12 @@
rw_init(&sc->lock, "aesni_lock");
crypto_register(sc->cid, CRYPTO_AES_CBC, 0, 0);
- crypto_register(sc->cid, CRYPTO_AES_XTS, 0, 0);
+ crypto_register(sc->cid, CRYPTO_AES_ICM, 0, 0);
crypto_register(sc->cid, CRYPTO_AES_NIST_GCM_16, 0, 0);
crypto_register(sc->cid, CRYPTO_AES_128_NIST_GMAC, 0, 0);
crypto_register(sc->cid, CRYPTO_AES_192_NIST_GMAC, 0, 0);
crypto_register(sc->cid, CRYPTO_AES_256_NIST_GMAC, 0, 0);
+ crypto_register(sc->cid, CRYPTO_AES_XTS, 0, 0);
return (0);
}
@@ -164,6 +165,7 @@
for (; cri != NULL; cri = cri->cri_next) {
switch (cri->cri_alg) {
case CRYPTO_AES_CBC:
+ case CRYPTO_AES_ICM:
case CRYPTO_AES_XTS:
case CRYPTO_AES_NIST_GCM_16:
if (encini != NULL) {
@@ -294,6 +296,7 @@
for (crd = crp->crp_desc; crd != NULL; crd = crd->crd_next) {
switch (crd->crd_alg) {
case CRYPTO_AES_CBC:
+ case CRYPTO_AES_ICM:
case CRYPTO_AES_XTS:
if (enccrd != NULL) {
error = EINVAL;
@@ -433,6 +436,7 @@
return (error);
}
+#if 0
static void
printhexstr(uint8_t *ptr, int len)
{
@@ -441,6 +445,7 @@
for (i = 0; i < len; i++)
printf("%02hhx", ptr[i]);
}
+#endif
/*
* authcrd contains the associated date.
@@ -458,6 +463,10 @@
encflag = (enccrd->crd_flags & CRD_F_ENCRYPT) == CRD_F_ENCRYPT;
+ if (enccrd->crd_alg == CRYPTO_AES_ICM &&
+ (enccrd->crd_flags & CRD_F_IV_EXPLICIT) == 0)
+ return (EINVAL);
+
buf = aesni_cipher_alloc(enccrd, crp, &allocated);
if (buf == NULL)
return (ENOMEM);
@@ -466,8 +475,10 @@
authallocated = 0;
if (authcrd != NULL) {
authbuf = aesni_cipher_alloc(authcrd, crp, &authallocated);
- if (authbuf == NULL)
+ if (authbuf == NULL) {
+ error = ENOMEM;
goto out1;
+ }
}
td = curthread;
@@ -486,6 +497,7 @@
/* XXX - validate that enccrd and authcrd have/use same key? */
switch (enccrd->crd_alg) {
case CRYPTO_AES_CBC:
+ case CRYPTO_AES_ICM:
ivlen = 16;
break;
case CRYPTO_AES_XTS:
@@ -535,6 +547,10 @@
aesni_decrypt_cbc(ses->rounds, ses->dec_schedule,
enccrd->crd_len, buf, ses->iv);
break;
+ case CRYPTO_AES_ICM:
+ aesni_encrypt_icm(ses->rounds, ses->enc_schedule,
+ enccrd->crd_len, buf, buf, ses->iv);
+ break;
case CRYPTO_AES_XTS:
if (encflag)
aesni_encrypt_xts(ses->rounds, ses->enc_schedule,
==== //depot/projects/opencrypto/sys/crypto/aesni/aesni.h#3 (text+ko) ====
@@ -88,6 +88,9 @@
size_t len, const uint8_t *from, uint8_t *to);
void aesni_decrypt_ecb(int rounds, const void *key_schedule /*__aligned(16)*/,
size_t len, const uint8_t *from, uint8_t *to);
+void aesni_encrypt_icm(int rounds, const void *key_schedule /*__aligned(16)*/,
+ size_t len, const uint8_t *from, uint8_t *to,
+ const uint8_t iv[AES_BLOCK_LEN]);
void aesni_encrypt_xts(int rounds, const void *data_schedule /*__aligned(16)*/,
const void *tweak_schedule /*__aligned(16)*/, size_t len,
==== //depot/projects/opencrypto/sys/crypto/aesni/aesni_wrap.c#3 (text+ko) ====
@@ -44,6 +44,7 @@
#include <opencrypto/gmac.h>
#include "aesencdec.h"
+#include <smmintrin.h>
MALLOC_DECLARE(M_AESNI);
@@ -182,6 +183,104 @@
}
}
+/*
+ * mixed endian increment, low 64bits stored in hi word to be compatible
+ * with _icm's BSWAP.
+ */
+static inline __m128i
+nextc(__m128i x)
+{
+ const __m128i ONE = _mm_setr_epi32(0, 0, 1, 0);
+ const __m128i ZERO = _mm_setzero_si128();
+
+ x = _mm_add_epi64(x, ONE);
+ __m128i t = _mm_cmpeq_epi64(x, ZERO);
+ t = _mm_unpackhi_epi64(t, ZERO);
+ x = _mm_sub_epi64(x, t);
+
+ return x;
+}
+
+void
+aesni_encrypt_icm(int rounds, const void *key_schedule, size_t len,
+ const uint8_t *from, uint8_t *to, const uint8_t iv[AES_BLOCK_LEN])
+{
+ __m128i tot;
+ __m128i tmp1, tmp2, tmp3, tmp4;
+ __m128i tmp5, tmp6, tmp7, tmp8;
+ __m128i ctr1, ctr2, ctr3, ctr4;
+ __m128i ctr5, ctr6, ctr7, ctr8;
+ __m128i BSWAP_EPI64;
+ __m128i tout[8];
+ struct blocks8 *top;
+ const struct blocks8 *blks;
+ size_t i, cnt;
+
+ BSWAP_EPI64 = _mm_set_epi8(8,9,10,11,12,13,14,15,0,1,2,3,4,5,6,7);
+
+ ctr1 = _mm_loadu_si128((__m128i*)iv);
+ ctr1 = _mm_shuffle_epi8(ctr1, BSWAP_EPI64);
+
+ cnt = len / AES_BLOCK_LEN / 8;
+ for (i = 0; i < cnt; i++) {
+ tmp1 = _mm_shuffle_epi8(ctr1, BSWAP_EPI64);
+ ctr2 = nextc(ctr1);
+ tmp2 = _mm_shuffle_epi8(ctr2, BSWAP_EPI64);
+ ctr3 = nextc(ctr2);
+ tmp3 = _mm_shuffle_epi8(ctr3, BSWAP_EPI64);
+ ctr4 = nextc(ctr3);
+ tmp4 = _mm_shuffle_epi8(ctr4, BSWAP_EPI64);
+ ctr5 = nextc(ctr4);
+ tmp5 = _mm_shuffle_epi8(ctr5, BSWAP_EPI64);
+ ctr6 = nextc(ctr5);
+ tmp6 = _mm_shuffle_epi8(ctr6, BSWAP_EPI64);
+ ctr7 = nextc(ctr6);
+ tmp7 = _mm_shuffle_epi8(ctr7, BSWAP_EPI64);
+ ctr8 = nextc(ctr7);
+ tmp8 = _mm_shuffle_epi8(ctr8, BSWAP_EPI64);
+ ctr1 = nextc(ctr8);
+
+ blks = (const struct blocks8 *)from;
+ top = (struct blocks8 *)to;
+ aesni_enc8(rounds - 1, key_schedule, tmp1, tmp2, tmp3, tmp4,
+ tmp5, tmp6, tmp7, tmp8, tout);
+
+ top->blk[0] = blks->blk[0] ^ tout[0];
+ top->blk[1] = blks->blk[1] ^ tout[1];
+ top->blk[2] = blks->blk[2] ^ tout[2];
+ top->blk[3] = blks->blk[3] ^ tout[3];
+ top->blk[4] = blks->blk[4] ^ tout[4];
+ top->blk[5] = blks->blk[5] ^ tout[5];
+ top->blk[6] = blks->blk[6] ^ tout[6];
+ top->blk[7] = blks->blk[7] ^ tout[7];
+
+ from += AES_BLOCK_LEN * 8;
+ to += AES_BLOCK_LEN * 8;
+ }
+ i *= 8;
+ cnt = len / AES_BLOCK_LEN;
+ for (; i < cnt; i++) {
+ tmp1 = _mm_shuffle_epi8(ctr1, BSWAP_EPI64);
+ ctr1 = nextc(ctr1);
+
+ tot = aesni_enc(rounds - 1, key_schedule, tmp1);
+
+ tot = tot ^ _mm_loadu_si128((const __m128i *)from);
+ _mm_storeu_si128((__m128i *)to, tot);
+
+ from += AES_BLOCK_LEN;
+ to += AES_BLOCK_LEN;
+ }
+
+ /* handle remaining partial round */
+ if (len % AES_BLOCK_LEN != 0) {
+ tmp1 = _mm_shuffle_epi8(ctr1, BSWAP_EPI64);
+ tot = aesni_enc(rounds - 1, key_schedule, tmp1);
+ tot = tot ^ _mm_loadu_si128((const __m128i *)from);
+ memcpy(to, &tot, len % AES_BLOCK_LEN);
+ }
+}
+
#define AES_XTS_BLOCKSIZE 16
#define AES_XTS_IVSIZE 8
#define AES_XTS_ALPHA 0x87 /* GF(2^128) generator polynomial */
@@ -342,6 +441,7 @@
switch (ses->algo) {
case CRYPTO_AES_CBC:
+ case CRYPTO_AES_ICM:
case CRYPTO_AES_NIST_GCM_16:
switch (keylen) {
case 128:
@@ -354,7 +454,7 @@
ses->rounds = AES256_ROUNDS;
break;
default:
- CRYPTDEB("invalid CBC/GCM key length");
+ CRYPTDEB("invalid CBC/ICM/GCM key length");
return (EINVAL);
}
break;
@@ -381,10 +481,9 @@
/* setup IV */
if (ses->algo == CRYPTO_AES_CBC || ses->algo == CRYPTO_AES_NIST_GCM_16)
arc4rand(ses->iv, sizeof(ses->iv), 0);
- else /* if (ses->algo == CRYPTO_AES_XTS) */ {
+ else if (ses->algo == CRYPTO_AES_XTS)
aesni_set_enckey(key + keylen / 16, ses->xts_schedule,
ses->rounds);
- }
return (0);
}
More information about the p4-projects
mailing list