aesni_wrap.c revision 230426
174429Sorion/*- 274429Sorion * Copyright (c) 2010 Konstantin Belousov <kib@FreeBSD.org> 374429Sorion * Copyright (c) 2010-2011 Pawel Jakub Dawidek <pawel@dawidek.net> 474429Sorion * All rights reserved. 574429Sorion * 674429Sorion * Redistribution and use in source and binary forms, with or without 774429Sorion * modification, are permitted provided that the following conditions 874429Sorion * are met: 974429Sorion * 1. Redistributions of source code must retain the above copyright 1074429Sorion * notice, this list of conditions and the following disclaimer. 1174429Sorion * 2. Redistributions in binary form must reproduce the above copyright 1274429Sorion * notice, this list of conditions and the following disclaimer in the 1374429Sorion * documentation and/or other materials provided with the distribution. 1474429Sorion * 1574429Sorion * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND 1674429Sorion * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 1774429Sorion * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 1874429Sorion * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE 1974429Sorion * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 2074429Sorion * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 2174429Sorion * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 2274429Sorion * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 2374429Sorion * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 2474429Sorion * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 2574429Sorion * SUCH DAMAGE. 2674429Sorion */ 2774429Sorion 2874429Sorion#include <sys/cdefs.h> 2974429Sorion__FBSDID("$FreeBSD: head/sys/crypto/aesni/aesni_wrap.c 230426 2012-01-21 17:45:27Z kib $"); 3074429Sorion 3174429Sorion#include <sys/param.h> 3274429Sorion#include <sys/libkern.h> 3374429Sorion#include <sys/malloc.h> 3474429Sorion#include <sys/proc.h> 3574429Sorion#include <sys/systm.h> 3674429Sorion#include <crypto/aesni/aesni.h> 3774429Sorion 3874429SorionMALLOC_DECLARE(M_AESNI); 3974429Sorion 4074429Sorionvoid 4174429Sorionaesni_encrypt_cbc(int rounds, const void *key_schedule, size_t len, 4274429Sorion const uint8_t *from, uint8_t *to, const uint8_t iv[AES_BLOCK_LEN]) 4374429Sorion{ 4474429Sorion const uint8_t *ivp; 4574429Sorion size_t i; 4674429Sorion 4774429Sorion len /= AES_BLOCK_LEN; 4874429Sorion ivp = iv; 4974429Sorion for (i = 0; i < len; i++) { 5074429Sorion aesni_enc(rounds - 1, key_schedule, from, to, ivp); 5174429Sorion ivp = to; 5274429Sorion from += AES_BLOCK_LEN; 5374429Sorion to += AES_BLOCK_LEN; 5474429Sorion } 5574429Sorion} 5674429Sorion 5774429Sorionvoid 5874429Sorionaesni_encrypt_ecb(int rounds, const void *key_schedule, size_t len, 5974429Sorion const uint8_t from[AES_BLOCK_LEN], uint8_t to[AES_BLOCK_LEN]) 6074429Sorion{ 6174429Sorion size_t i; 6274429Sorion 6374429Sorion len /= AES_BLOCK_LEN; 6474429Sorion for (i = 0; i < len; i++) { 6574429Sorion aesni_enc(rounds - 1, key_schedule, from, to, NULL); 6674429Sorion from += AES_BLOCK_LEN; 6774429Sorion to += AES_BLOCK_LEN; 6874429Sorion } 6974429Sorion} 7074429Sorion 7174429Sorionvoid 7274429Sorionaesni_decrypt_ecb(int rounds, const void *key_schedule, size_t len, 7374429Sorion const uint8_t from[AES_BLOCK_LEN], uint8_t to[AES_BLOCK_LEN]) 7474429Sorion{ 7574429Sorion size_t i; 7674429Sorion 7774429Sorion len /= AES_BLOCK_LEN; 7874429Sorion for (i = 0; i < len; i++) { 7974429Sorion aesni_dec(rounds - 1, key_schedule, from, to, NULL); 8074429Sorion from += AES_BLOCK_LEN; 8174429Sorion to += AES_BLOCK_LEN; 8274429Sorion } 8374429Sorion} 8474429Sorion 8574429Sorion#define AES_XTS_BLOCKSIZE 16 8674429Sorion#define AES_XTS_IVSIZE 8 8774429Sorion#define AES_XTS_ALPHA 0x87 /* GF(2^128) generator polynomial */ 8874429Sorion 8974429Sorionstatic void 9074429Sorionaesni_crypt_xts_block(int rounds, const void *key_schedule, uint64_t *tweak, 9174429Sorion const uint64_t *from, uint64_t *to, uint64_t *block, int do_encrypt) 9274429Sorion{ 9374429Sorion int carry; 9474429Sorion 9574429Sorion block[0] = from[0] ^ tweak[0]; 9674429Sorion block[1] = from[1] ^ tweak[1]; 9774429Sorion 9874429Sorion if (do_encrypt) 9974429Sorion aesni_enc(rounds - 1, key_schedule, (uint8_t *)block, (uint8_t *)to, NULL); 10074429Sorion else 10174429Sorion aesni_dec(rounds - 1, key_schedule, (uint8_t *)block, (uint8_t *)to, NULL); 10274429Sorion 10374429Sorion to[0] ^= tweak[0]; 10474429Sorion to[1] ^= tweak[1]; 10574429Sorion 10674429Sorion /* Exponentiate tweak. */ 10774429Sorion carry = ((tweak[0] & 0x8000000000000000ULL) > 0); 10874429Sorion tweak[0] <<= 1; 10974429Sorion if (tweak[1] & 0x8000000000000000ULL) { 11074429Sorion uint8_t *twk = (uint8_t *)tweak; 11174429Sorion 11274429Sorion twk[0] ^= AES_XTS_ALPHA; 11374429Sorion } 11474429Sorion tweak[1] <<= 1; 11574429Sorion if (carry) 11674429Sorion tweak[1] |= 1; 11774429Sorion} 11874429Sorion 11974429Sorionstatic void 12074429Sorionaesni_crypt_xts(int rounds, const void *data_schedule, 12174429Sorion const void *tweak_schedule, size_t len, const uint8_t *from, uint8_t *to, 12274429Sorion const uint8_t iv[AES_BLOCK_LEN], int do_encrypt) 12374429Sorion{ 12474429Sorion uint64_t block[AES_XTS_BLOCKSIZE / 8]; 12574429Sorion uint8_t tweak[AES_XTS_BLOCKSIZE]; 12674429Sorion size_t i; 12774429Sorion 12874429Sorion /* 12974429Sorion * Prepare tweak as E_k2(IV). IV is specified as LE representation 13074429Sorion * of a 64-bit block number which we allow to be passed in directly. 13174429Sorion */ 13274429Sorion#if BYTE_ORDER == LITTLE_ENDIAN 13374429Sorion bcopy(iv, tweak, AES_XTS_IVSIZE); 13474429Sorion /* Last 64 bits of IV are always zero. */ 13574429Sorion bzero(tweak + AES_XTS_IVSIZE, AES_XTS_IVSIZE); 13674429Sorion#else 13774429Sorion#error Only LITTLE_ENDIAN architectures are supported. 13874429Sorion#endif 13974429Sorion aesni_enc(rounds - 1, tweak_schedule, tweak, tweak, NULL); 14074429Sorion 14174429Sorion len /= AES_XTS_BLOCKSIZE; 14274429Sorion for (i = 0; i < len; i++) { 14374429Sorion aesni_crypt_xts_block(rounds, data_schedule, (uint64_t *)tweak, 14474429Sorion (const uint64_t *)from, (uint64_t *)to, block, do_encrypt); 14574429Sorion from += AES_XTS_BLOCKSIZE; 14674429Sorion to += AES_XTS_BLOCKSIZE; 14774429Sorion } 14874429Sorion 14974429Sorion bzero(tweak, sizeof(tweak)); 15074429Sorion bzero(block, sizeof(block)); 15174429Sorion} 15274429Sorion 15374429Sorionstatic void 15474429Sorionaesni_encrypt_xts(int rounds, const void *data_schedule, 15574429Sorion const void *tweak_schedule, size_t len, const uint8_t *from, uint8_t *to, 15674429Sorion const uint8_t iv[AES_BLOCK_LEN]) 15774429Sorion{ 15874429Sorion 15974429Sorion aesni_crypt_xts(rounds, data_schedule, tweak_schedule, len, from, to, 16074429Sorion iv, 1); 16174429Sorion} 16274429Sorion 16374429Sorionstatic void 16474429Sorionaesni_decrypt_xts(int rounds, const void *data_schedule, 16574429Sorion const void *tweak_schedule, size_t len, const uint8_t *from, uint8_t *to, 16674429Sorion const uint8_t iv[AES_BLOCK_LEN]) 16774429Sorion{ 16874429Sorion 16974429Sorion aesni_crypt_xts(rounds, data_schedule, tweak_schedule, len, from, to, 17074429Sorion iv, 0); 17174429Sorion} 17274429Sorion 17374429Sorionstatic int 17474429Sorionaesni_cipher_setup_common(struct aesni_session *ses, const uint8_t *key, 17574429Sorion int keylen) 17674429Sorion{ 17774429Sorion 17874429Sorion switch (ses->algo) { 17974429Sorion case CRYPTO_AES_CBC: 18074429Sorion switch (keylen) { 18174429Sorion case 128: 18274429Sorion ses->rounds = AES128_ROUNDS; 18374429Sorion break; 18474429Sorion case 192: 18574429Sorion ses->rounds = AES192_ROUNDS; 18674429Sorion break; 18774429Sorion case 256: 18874429Sorion ses->rounds = AES256_ROUNDS; 18974429Sorion break; 19074429Sorion default: 19174429Sorion return (EINVAL); 19274429Sorion } 19374429Sorion break; 19474429Sorion case CRYPTO_AES_XTS: 19574429Sorion switch (keylen) { 19674429Sorion case 256: 19774429Sorion ses->rounds = AES128_ROUNDS; 19874429Sorion break; 19974429Sorion case 512: 20074429Sorion ses->rounds = AES256_ROUNDS; 20174429Sorion break; 20274429Sorion default: 20374429Sorion return (EINVAL); 20474429Sorion } 20574429Sorion break; 20674429Sorion default: 20774429Sorion return (EINVAL); 20874429Sorion } 20974429Sorion 21074429Sorion aesni_set_enckey(key, ses->enc_schedule, ses->rounds); 21174429Sorion aesni_set_deckey(ses->enc_schedule, ses->dec_schedule, ses->rounds); 21274429Sorion if (ses->algo == CRYPTO_AES_CBC) 21374429Sorion arc4rand(ses->iv, sizeof(ses->iv), 0); 21474429Sorion else /* if (ses->algo == CRYPTO_AES_XTS) */ { 21574429Sorion aesni_set_enckey(key + keylen / 16, ses->xts_schedule, 21674429Sorion ses->rounds); 21774429Sorion } 21874429Sorion 21974429Sorion return (0); 22074429Sorion} 22174429Sorion 22274429Sorionint 22374429Sorionaesni_cipher_setup(struct aesni_session *ses, struct cryptoini *encini) 22474429Sorion{ 22574429Sorion struct thread *td; 22674429Sorion int error, saved_ctx; 22774429Sorion 22874429Sorion td = curthread; 22974429Sorion if (!is_fpu_kern_thread(0)) { 23074429Sorion error = fpu_kern_enter(td, ses->fpu_ctx, FPU_KERN_NORMAL); 23174429Sorion saved_ctx = 1; 23274429Sorion } else { 23374429Sorion error = 0; 23474429Sorion saved_ctx = 0; 23574429Sorion } 23674429Sorion if (error == 0) { 23774429Sorion error = aesni_cipher_setup_common(ses, encini->cri_key, 23874429Sorion encini->cri_klen); 23974429Sorion if (saved_ctx) 24074429Sorion fpu_kern_leave(td, ses->fpu_ctx); 24174429Sorion } 24274429Sorion return (error); 24374429Sorion} 24474429Sorion 24574429Sorionint 24674429Sorionaesni_cipher_process(struct aesni_session *ses, struct cryptodesc *enccrd, 24774429Sorion struct cryptop *crp) 24874429Sorion{ 24974429Sorion struct thread *td; 25074429Sorion uint8_t *buf; 25174429Sorion int error, allocated, saved_ctx; 25274429Sorion 25374429Sorion buf = aesni_cipher_alloc(enccrd, crp, &allocated); 25474429Sorion if (buf == NULL) 25574429Sorion return (ENOMEM); 25674429Sorion 25774429Sorion td = curthread; 25874429Sorion if (!is_fpu_kern_thread(0)) { 25974429Sorion error = fpu_kern_enter(td, ses->fpu_ctx, FPU_KERN_NORMAL); 26074429Sorion if (error != 0) 26174429Sorion goto out; 26274429Sorion saved_ctx = 1; 26374429Sorion } else { 26474429Sorion saved_ctx = 0; 26574429Sorion error = 0; 26674429Sorion } 26774429Sorion 26874429Sorion if ((enccrd->crd_flags & CRD_F_KEY_EXPLICIT) != 0) { 26974429Sorion error = aesni_cipher_setup_common(ses, enccrd->crd_key, 27074429Sorion enccrd->crd_klen); 27174429Sorion if (error != 0) 27274429Sorion goto out; 27374429Sorion } 27474429Sorion 27574429Sorion if ((enccrd->crd_flags & CRD_F_ENCRYPT) != 0) { 27674429Sorion if ((enccrd->crd_flags & CRD_F_IV_EXPLICIT) != 0) 27774429Sorion bcopy(enccrd->crd_iv, ses->iv, AES_BLOCK_LEN); 27874429Sorion if ((enccrd->crd_flags & CRD_F_IV_PRESENT) == 0) 27974429Sorion crypto_copyback(crp->crp_flags, crp->crp_buf, 28074429Sorion enccrd->crd_inject, AES_BLOCK_LEN, ses->iv); 28174429Sorion if (ses->algo == CRYPTO_AES_CBC) { 28274429Sorion aesni_encrypt_cbc(ses->rounds, ses->enc_schedule, 28374429Sorion enccrd->crd_len, buf, buf, ses->iv); 28474429Sorion } else /* if (ses->algo == CRYPTO_AES_XTS) */ { 28574429Sorion aesni_encrypt_xts(ses->rounds, ses->enc_schedule, 28674429Sorion ses->xts_schedule, enccrd->crd_len, buf, buf, 28774429Sorion ses->iv); 28874429Sorion } 28974429Sorion } else { 29074429Sorion if ((enccrd->crd_flags & CRD_F_IV_EXPLICIT) != 0) 29174429Sorion bcopy(enccrd->crd_iv, ses->iv, AES_BLOCK_LEN); 29274429Sorion else 29374429Sorion crypto_copydata(crp->crp_flags, crp->crp_buf, 29474429Sorion enccrd->crd_inject, AES_BLOCK_LEN, ses->iv); 29574429Sorion if (ses->algo == CRYPTO_AES_CBC) { 29674429Sorion aesni_decrypt_cbc(ses->rounds, ses->dec_schedule, 29774429Sorion enccrd->crd_len, buf, ses->iv); 29874429Sorion } else /* if (ses->algo == CRYPTO_AES_XTS) */ { 29974429Sorion aesni_decrypt_xts(ses->rounds, ses->dec_schedule, 30074429Sorion ses->xts_schedule, enccrd->crd_len, buf, buf, 30174429Sorion ses->iv); 30274429Sorion } 30374429Sorion } 30474429Sorion if (saved_ctx) 30574429Sorion fpu_kern_leave(td, ses->fpu_ctx); 30674429Sorion if (allocated) 30774429Sorion crypto_copyback(crp->crp_flags, crp->crp_buf, enccrd->crd_skip, 30874429Sorion enccrd->crd_len, buf); 30974429Sorion if ((enccrd->crd_flags & CRD_F_ENCRYPT) != 0) 31074429Sorion crypto_copydata(crp->crp_flags, crp->crp_buf, 31174429Sorion enccrd->crd_skip + enccrd->crd_len - AES_BLOCK_LEN, 31274429Sorion AES_BLOCK_LEN, ses->iv); 31374429Sorion out: 31474429Sorion if (allocated) { 31574429Sorion bzero(buf, enccrd->crd_len); 31674429Sorion free(buf, M_AESNI); 31774429Sorion } 31874429Sorion return (error); 31974429Sorion} 32074429Sorion