aesni_wrap.c (255187) | aesni_wrap.c (257757) |
---|---|
1/*- 2 * Copyright (C) 2008 Damien Miller <djm@mindrot.org> 3 * Copyright (c) 2010 Konstantin Belousov <kib@FreeBSD.org> 4 * Copyright (c) 2010-2011 Pawel Jakub Dawidek <pawel@dawidek.net> 5 * Copyright 2012-2013 John-Mark Gurney <jmg@FreeBSD.org> 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without --- 14 unchanged lines hidden (view full) --- 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 */ 29 30#include <sys/cdefs.h> | 1/*- 2 * Copyright (C) 2008 Damien Miller <djm@mindrot.org> 3 * Copyright (c) 2010 Konstantin Belousov <kib@FreeBSD.org> 4 * Copyright (c) 2010-2011 Pawel Jakub Dawidek <pawel@dawidek.net> 5 * Copyright 2012-2013 John-Mark Gurney <jmg@FreeBSD.org> 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without --- 14 unchanged lines hidden (view full) --- 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 */ 29 30#include <sys/cdefs.h> |
31__FBSDID("$FreeBSD: head/sys/crypto/aesni/aesni_wrap.c 255187 2013-09-03 18:31:23Z jmg $"); | 31__FBSDID("$FreeBSD: head/sys/crypto/aesni/aesni_wrap.c 257757 2013-11-06 19:14:49Z jmg $"); |
32 33#include <sys/param.h> 34#include <sys/libkern.h> 35#include <sys/malloc.h> 36#include <sys/proc.h> 37#include <sys/systm.h> 38#include <crypto/aesni/aesni.h> 39 40#include "aesencdec.h" 41 42MALLOC_DECLARE(M_AESNI); 43 | 32 33#include <sys/param.h> 34#include <sys/libkern.h> 35#include <sys/malloc.h> 36#include <sys/proc.h> 37#include <sys/systm.h> 38#include <crypto/aesni/aesni.h> 39 40#include "aesencdec.h" 41 42MALLOC_DECLARE(M_AESNI); 43 |
44struct blocks8 { 45 __m128i blk[8]; 46} __packed; 47 |
|
44void 45aesni_encrypt_cbc(int rounds, const void *key_schedule, size_t len, 46 const uint8_t *from, uint8_t *to, const uint8_t iv[AES_BLOCK_LEN]) 47{ 48 __m128i tot, ivreg; 49 size_t i; 50 51 len /= AES_BLOCK_LEN; --- 8 unchanged lines hidden (view full) --- 60 } 61} 62 63void 64aesni_decrypt_cbc(int rounds, const void *key_schedule, size_t len, 65 uint8_t *buf, const uint8_t iv[AES_BLOCK_LEN]) 66{ 67 __m128i blocks[8]; | 48void 49aesni_encrypt_cbc(int rounds, const void *key_schedule, size_t len, 50 const uint8_t *from, uint8_t *to, const uint8_t iv[AES_BLOCK_LEN]) 51{ 52 __m128i tot, ivreg; 53 size_t i; 54 55 len /= AES_BLOCK_LEN; --- 8 unchanged lines hidden (view full) --- 64 } 65} 66 67void 68aesni_decrypt_cbc(int rounds, const void *key_schedule, size_t len, 69 uint8_t *buf, const uint8_t iv[AES_BLOCK_LEN]) 70{ 71 __m128i blocks[8]; |
68 __m128i *bufs; | 72 struct blocks8 *blks; |
69 __m128i ivreg, nextiv; 70 size_t i, j, cnt; 71 72 ivreg = _mm_loadu_si128((const __m128i *)iv); 73 cnt = len / AES_BLOCK_LEN / 8; 74 for (i = 0; i < cnt; i++) { | 73 __m128i ivreg, nextiv; 74 size_t i, j, cnt; 75 76 ivreg = _mm_loadu_si128((const __m128i *)iv); 77 cnt = len / AES_BLOCK_LEN / 8; 78 for (i = 0; i < cnt; i++) { |
75 bufs = (__m128i *)buf; 76 aesni_dec8(rounds - 1, key_schedule, bufs[0], bufs[1], 77 bufs[2], bufs[3], bufs[4], bufs[5], bufs[6], 78 bufs[7], &blocks[0]); | 79 blks = (struct blocks8 *)buf; 80 aesni_dec8(rounds - 1, key_schedule, blks->blk[0], blks->blk[1], 81 blks->blk[2], blks->blk[3], blks->blk[4], blks->blk[5], 82 blks->blk[6], blks->blk[7], &blocks[0]); |
79 for (j = 0; j < 8; j++) { | 83 for (j = 0; j < 8; j++) { |
80 nextiv = bufs[j]; 81 bufs[j] = blocks[j] ^ ivreg; | 84 nextiv = blks->blk[j]; 85 blks->blk[j] = blocks[j] ^ ivreg; |
82 ivreg = nextiv; 83 } 84 buf += AES_BLOCK_LEN * 8; 85 } 86 i *= 8; 87 cnt = len / AES_BLOCK_LEN; 88 for (; i < cnt; i++) { | 86 ivreg = nextiv; 87 } 88 buf += AES_BLOCK_LEN * 8; 89 } 90 i *= 8; 91 cnt = len / AES_BLOCK_LEN; 92 for (; i < cnt; i++) { |
89 bufs = (__m128i *)buf; 90 nextiv = bufs[0]; 91 bufs[0] = aesni_dec(rounds - 1, key_schedule, bufs[0]) ^ ivreg; | 93 nextiv = _mm_loadu_si128((void *)buf); 94 _mm_storeu_si128((void *)buf, 95 aesni_dec(rounds - 1, key_schedule, nextiv) ^ ivreg); |
92 ivreg = nextiv; 93 buf += AES_BLOCK_LEN; 94 } 95} 96 97void 98aesni_encrypt_ecb(int rounds, const void *key_schedule, size_t len, 99 const uint8_t *from, uint8_t *to) 100{ 101 __m128i tot; | 96 ivreg = nextiv; 97 buf += AES_BLOCK_LEN; 98 } 99} 100 101void 102aesni_encrypt_ecb(int rounds, const void *key_schedule, size_t len, 103 const uint8_t *from, uint8_t *to) 104{ 105 __m128i tot; |
102 const __m128i *blocks; | 106 __m128i tout[8]; 107 struct blocks8 *top; 108 const struct blocks8 *blks; |
103 size_t i, cnt; 104 105 cnt = len / AES_BLOCK_LEN / 8; 106 for (i = 0; i < cnt; i++) { | 109 size_t i, cnt; 110 111 cnt = len / AES_BLOCK_LEN / 8; 112 for (i = 0; i < cnt; i++) { |
107 blocks = (const __m128i *)from; 108 aesni_enc8(rounds - 1, key_schedule, blocks[0], blocks[1], 109 blocks[2], blocks[3], blocks[4], blocks[5], blocks[6], 110 blocks[7], (__m128i *)to); | 113 blks = (const struct blocks8 *)from; 114 top = (struct blocks8 *)to; 115 aesni_enc8(rounds - 1, key_schedule, blks->blk[0], blks->blk[1], 116 blks->blk[2], blks->blk[3], blks->blk[4], blks->blk[5], 117 blks->blk[6], blks->blk[7], tout); 118 top->blk[0] = tout[0]; 119 top->blk[1] = tout[1]; 120 top->blk[2] = tout[2]; 121 top->blk[3] = tout[3]; 122 top->blk[4] = tout[4]; 123 top->blk[5] = tout[5]; 124 top->blk[6] = tout[6]; 125 top->blk[7] = tout[7]; |
111 from += AES_BLOCK_LEN * 8; 112 to += AES_BLOCK_LEN * 8; 113 } 114 i *= 8; 115 cnt = len / AES_BLOCK_LEN; 116 for (; i < cnt; i++) { 117 tot = aesni_enc(rounds - 1, key_schedule, 118 _mm_loadu_si128((const __m128i *)from)); 119 _mm_storeu_si128((__m128i *)to, tot); 120 from += AES_BLOCK_LEN; 121 to += AES_BLOCK_LEN; 122 } 123} 124 125void 126aesni_decrypt_ecb(int rounds, const void *key_schedule, size_t len, 127 const uint8_t from[AES_BLOCK_LEN], uint8_t to[AES_BLOCK_LEN]) 128{ 129 __m128i tot; | 126 from += AES_BLOCK_LEN * 8; 127 to += AES_BLOCK_LEN * 8; 128 } 129 i *= 8; 130 cnt = len / AES_BLOCK_LEN; 131 for (; i < cnt; i++) { 132 tot = aesni_enc(rounds - 1, key_schedule, 133 _mm_loadu_si128((const __m128i *)from)); 134 _mm_storeu_si128((__m128i *)to, tot); 135 from += AES_BLOCK_LEN; 136 to += AES_BLOCK_LEN; 137 } 138} 139 140void 141aesni_decrypt_ecb(int rounds, const void *key_schedule, size_t len, 142 const uint8_t from[AES_BLOCK_LEN], uint8_t to[AES_BLOCK_LEN]) 143{ 144 __m128i tot; |
130 const __m128i *blocks; | 145 __m128i tout[8]; 146 const struct blocks8 *blks; 147 struct blocks8 *top; |
131 size_t i, cnt; 132 133 cnt = len / AES_BLOCK_LEN / 8; 134 for (i = 0; i < cnt; i++) { | 148 size_t i, cnt; 149 150 cnt = len / AES_BLOCK_LEN / 8; 151 for (i = 0; i < cnt; i++) { |
135 blocks = (const __m128i *)from; 136 aesni_dec8(rounds - 1, key_schedule, blocks[0], blocks[1], 137 blocks[2], blocks[3], blocks[4], blocks[5], blocks[6], 138 blocks[7], (__m128i *)to); | 152 blks = (const struct blocks8 *)from; 153 top = (struct blocks8 *)to; 154 aesni_dec8(rounds - 1, key_schedule, blks->blk[0], blks->blk[1], 155 blks->blk[2], blks->blk[3], blks->blk[4], blks->blk[5], 156 blks->blk[6], blks->blk[7], tout); 157 top->blk[0] = tout[0]; 158 top->blk[1] = tout[1]; 159 top->blk[2] = tout[2]; 160 top->blk[3] = tout[3]; 161 top->blk[4] = tout[4]; 162 top->blk[5] = tout[5]; 163 top->blk[6] = tout[6]; 164 top->blk[7] = tout[7]; |
139 from += AES_BLOCK_LEN * 8; 140 to += AES_BLOCK_LEN * 8; 141 } 142 i *= 8; 143 cnt = len / AES_BLOCK_LEN; 144 for (; i < cnt; i++) { 145 tot = aesni_dec(rounds - 1, key_schedule, 146 _mm_loadu_si128((const __m128i *)from)); --- 21 unchanged lines hidden (view full) --- 168 /* next term */ 169 ret = _mm_slli_epi32(inp, 1); 170 ret ^= xtweak; 171 172 return ret; 173} 174 175static void | 165 from += AES_BLOCK_LEN * 8; 166 to += AES_BLOCK_LEN * 8; 167 } 168 i *= 8; 169 cnt = len / AES_BLOCK_LEN; 170 for (; i < cnt; i++) { 171 tot = aesni_dec(rounds - 1, key_schedule, 172 _mm_loadu_si128((const __m128i *)from)); --- 21 unchanged lines hidden (view full) --- 194 /* next term */ 195 ret = _mm_slli_epi32(inp, 1); 196 ret ^= xtweak; 197 198 return ret; 199} 200 201static void |
176aesni_crypt_xts_block(int rounds, const void *key_schedule, __m128i *tweak, 177 const __m128i *from, __m128i *to, int do_encrypt) | 202aesni_crypt_xts_block(int rounds, const __m128i *key_schedule, __m128i *tweak, 203 const uint8_t *from, uint8_t *to, int do_encrypt) |
178{ 179 __m128i block; 180 | 204{ 205 __m128i block; 206 |
181 block = *from ^ *tweak; | 207 block = _mm_loadu_si128((const __m128i *)from) ^ *tweak; |
182 183 if (do_encrypt) 184 block = aesni_enc(rounds - 1, key_schedule, block); 185 else 186 block = aesni_dec(rounds - 1, key_schedule, block); 187 | 208 209 if (do_encrypt) 210 block = aesni_enc(rounds - 1, key_schedule, block); 211 else 212 block = aesni_dec(rounds - 1, key_schedule, block); 213 |
188 *to = block ^ *tweak; | 214 _mm_storeu_si128((__m128i *)to, block ^ *tweak); |
189 190 *tweak = xts_crank_lfsr(*tweak); 191} 192 193static void | 215 216 *tweak = xts_crank_lfsr(*tweak); 217} 218 219static void |
194aesni_crypt_xts_block8(int rounds, const void *key_schedule, __m128i *tweak, 195 const __m128i *from, __m128i *to, int do_encrypt) | 220aesni_crypt_xts_block8(int rounds, const __m128i *key_schedule, __m128i *tweak, 221 const uint8_t *from, uint8_t *to, int do_encrypt) |
196{ 197 __m128i tmptweak; 198 __m128i a, b, c, d, e, f, g, h; 199 __m128i tweaks[8]; 200 __m128i tmp[8]; | 222{ 223 __m128i tmptweak; 224 __m128i a, b, c, d, e, f, g, h; 225 __m128i tweaks[8]; 226 __m128i tmp[8]; |
227 __m128i *top; 228 const __m128i *fromp; |
|
201 202 tmptweak = *tweak; 203 204 /* 205 * unroll the loop. This lets gcc put values directly in the 206 * register and saves memory accesses. 207 */ | 229 230 tmptweak = *tweak; 231 232 /* 233 * unroll the loop. This lets gcc put values directly in the 234 * register and saves memory accesses. 235 */ |
236 fromp = (const __m128i *)from; |
|
208#define PREPINP(v, pos) \ 209 do { \ 210 tweaks[(pos)] = tmptweak; \ | 237#define PREPINP(v, pos) \ 238 do { \ 239 tweaks[(pos)] = tmptweak; \ |
211 (v) = from[(pos)] ^ tmptweak; \ | 240 (v) = _mm_loadu_si128(&fromp[pos]) ^ \ 241 tmptweak; \ |
212 tmptweak = xts_crank_lfsr(tmptweak); \ 213 } while (0) 214 PREPINP(a, 0); 215 PREPINP(b, 1); 216 PREPINP(c, 2); 217 PREPINP(d, 3); 218 PREPINP(e, 4); 219 PREPINP(f, 5); 220 PREPINP(g, 6); 221 PREPINP(h, 7); 222 *tweak = tmptweak; 223 224 if (do_encrypt) 225 aesni_enc8(rounds - 1, key_schedule, a, b, c, d, e, f, g, h, 226 tmp); 227 else 228 aesni_dec8(rounds - 1, key_schedule, a, b, c, d, e, f, g, h, 229 tmp); 230 | 242 tmptweak = xts_crank_lfsr(tmptweak); \ 243 } while (0) 244 PREPINP(a, 0); 245 PREPINP(b, 1); 246 PREPINP(c, 2); 247 PREPINP(d, 3); 248 PREPINP(e, 4); 249 PREPINP(f, 5); 250 PREPINP(g, 6); 251 PREPINP(h, 7); 252 *tweak = tmptweak; 253 254 if (do_encrypt) 255 aesni_enc8(rounds - 1, key_schedule, a, b, c, d, e, f, g, h, 256 tmp); 257 else 258 aesni_dec8(rounds - 1, key_schedule, a, b, c, d, e, f, g, h, 259 tmp); 260 |
231 to[0] = tmp[0] ^ tweaks[0]; 232 to[1] = tmp[1] ^ tweaks[1]; 233 to[2] = tmp[2] ^ tweaks[2]; 234 to[3] = tmp[3] ^ tweaks[3]; 235 to[4] = tmp[4] ^ tweaks[4]; 236 to[5] = tmp[5] ^ tweaks[5]; 237 to[6] = tmp[6] ^ tweaks[6]; 238 to[7] = tmp[7] ^ tweaks[7]; | 261 top = (__m128i *)to; 262 _mm_storeu_si128(&top[0], tmp[0] ^ tweaks[0]); 263 _mm_storeu_si128(&top[1], tmp[1] ^ tweaks[1]); 264 _mm_storeu_si128(&top[2], tmp[2] ^ tweaks[2]); 265 _mm_storeu_si128(&top[3], tmp[3] ^ tweaks[3]); 266 _mm_storeu_si128(&top[4], tmp[4] ^ tweaks[4]); 267 _mm_storeu_si128(&top[5], tmp[5] ^ tweaks[5]); 268 _mm_storeu_si128(&top[6], tmp[6] ^ tweaks[6]); 269 _mm_storeu_si128(&top[7], tmp[7] ^ tweaks[7]); |
239} 240 241static void | 270} 271 272static void |
242aesni_crypt_xts(int rounds, const void *data_schedule, 243 const void *tweak_schedule, size_t len, const uint8_t *from, uint8_t *to, 244 const uint8_t iv[AES_BLOCK_LEN], int do_encrypt) | 273aesni_crypt_xts(int rounds, const __m128i *data_schedule, 274 const __m128i *tweak_schedule, size_t len, const uint8_t *from, 275 uint8_t *to, const uint8_t iv[AES_BLOCK_LEN], int do_encrypt) |
245{ 246 __m128i tweakreg; 247 uint8_t tweak[AES_XTS_BLOCKSIZE] __aligned(16); 248 size_t i, cnt; 249 250 /* 251 * Prepare tweak as E_k2(IV). IV is specified as LE representation 252 * of a 64-bit block number which we allow to be passed in directly. --- 6 unchanged lines hidden (view full) --- 259#error Only LITTLE_ENDIAN architectures are supported. 260#endif 261 tweakreg = _mm_loadu_si128((__m128i *)&tweak[0]); 262 tweakreg = aesni_enc(rounds - 1, tweak_schedule, tweakreg); 263 264 cnt = len / AES_XTS_BLOCKSIZE / 8; 265 for (i = 0; i < cnt; i++) { 266 aesni_crypt_xts_block8(rounds, data_schedule, &tweakreg, | 276{ 277 __m128i tweakreg; 278 uint8_t tweak[AES_XTS_BLOCKSIZE] __aligned(16); 279 size_t i, cnt; 280 281 /* 282 * Prepare tweak as E_k2(IV). IV is specified as LE representation 283 * of a 64-bit block number which we allow to be passed in directly. --- 6 unchanged lines hidden (view full) --- 290#error Only LITTLE_ENDIAN architectures are supported. 291#endif 292 tweakreg = _mm_loadu_si128((__m128i *)&tweak[0]); 293 tweakreg = aesni_enc(rounds - 1, tweak_schedule, tweakreg); 294 295 cnt = len / AES_XTS_BLOCKSIZE / 8; 296 for (i = 0; i < cnt; i++) { 297 aesni_crypt_xts_block8(rounds, data_schedule, &tweakreg, |
267 (const __m128i *)from, (__m128i *)to, do_encrypt); | 298 from, to, do_encrypt); |
268 from += AES_XTS_BLOCKSIZE * 8; 269 to += AES_XTS_BLOCKSIZE * 8; 270 } 271 i *= 8; 272 cnt = len / AES_XTS_BLOCKSIZE; 273 for (; i < cnt; i++) { 274 aesni_crypt_xts_block(rounds, data_schedule, &tweakreg, | 299 from += AES_XTS_BLOCKSIZE * 8; 300 to += AES_XTS_BLOCKSIZE * 8; 301 } 302 i *= 8; 303 cnt = len / AES_XTS_BLOCKSIZE; 304 for (; i < cnt; i++) { 305 aesni_crypt_xts_block(rounds, data_schedule, &tweakreg, |
275 (const __m128i *)from, (__m128i *)to, do_encrypt); | 306 from, to, do_encrypt); |
276 from += AES_XTS_BLOCKSIZE; 277 to += AES_XTS_BLOCKSIZE; 278 } 279} 280 281void 282aesni_encrypt_xts(int rounds, const void *data_schedule, 283 const void *tweak_schedule, size_t len, const uint8_t *from, uint8_t *to, --- 164 unchanged lines hidden --- | 307 from += AES_XTS_BLOCKSIZE; 308 to += AES_XTS_BLOCKSIZE; 309 } 310} 311 312void 313aesni_encrypt_xts(int rounds, const void *data_schedule, 314 const void *tweak_schedule, size_t len, const uint8_t *from, uint8_t *to, --- 164 unchanged lines hidden --- |