1/* 2 * arch/ubicom32/crypto/aes_ubicom32.c 3 * Ubicom32 implementation of the AES Cipher Algorithm. 4 * 5 * (C) Copyright 2009, Ubicom, Inc. 6 * 7 * This file is part of the Ubicom32 Linux Kernel Port. 8 * 9 * The Ubicom32 Linux Kernel Port is free software: you can redistribute 10 * it and/or modify it under the terms of the GNU General Public License 11 * as published by the Free Software Foundation, either version 2 of the 12 * License, or (at your option) any later version. 13 * 14 * The Ubicom32 Linux Kernel Port is distributed in the hope that it 15 * will be useful, but WITHOUT ANY WARRANTY; without even the implied 16 * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See 17 * the GNU General Public License for more details. 18 * 19 * You should have received a copy of the GNU General Public License 20 * along with the Ubicom32 Linux Kernel Port. If not, 21 * see <http://www.gnu.org/licenses/>. 22 * 23 * Ubicom32 implementation derived from (with many thanks): 24 * arch/m68knommu 25 * arch/blackfin 26 * arch/parisc 27 */ 28#include <crypto/aes.h> 29#include <crypto/algapi.h> 30#include <linux/err.h> 31#include <linux/module.h> 32#include <linux/init.h> 33#include <linux/spinlock.h> 34#include "crypto_ubicom32.h" 35#include <asm/linkage.h> 36 37struct ubicom32_aes_ctx { 38 u8 key[AES_MAX_KEY_SIZE]; 39 u32 ctrl; 40 int key_len; 41}; 42 43static inline void aes_hw_set_key(const u8 *key, u8 key_len) 44{ 45 /* 46 * switch case has more overhead than 4 move.4 instructions, so just copy 256 bits 47 */ 48 SEC_SET_KEY_256(key); 49} 50 51static inline void aes_hw_set_iv(const u8 *iv) 52{ 53 SEC_SET_IV_4W(iv); 54} 55 56static inline void aes_hw_cipher(u8 *out, const u8 *in) 57{ 58 SEC_SET_INPUT_4W(in); 59 60 asm volatile ( 61 " ; start AES by writing 0x40(SECURITY_BASE) \n\t" 62 " move.4 0x40(%0), #0x01 \n\t" 63 " pipe_flush 0 \n\t" 64 " \n\t" 65 " ; wait for the module to calculate the output \n\t" 66 " btst 0x04(%0), #0 \n\t" 67 " jmpne.f .-4 \n\t" 68 : 69 : "a" (SEC_BASE) 70 : "cc" 71 ); 72 73 SEC_GET_OUTPUT_4W(out); 74} 75 76static int __ocm_text aes_set_key(struct crypto_tfm *tfm, const u8 *in_key, 77 unsigned int key_len) 78{ 79 struct ubicom32_aes_ctx *uctx = crypto_tfm_ctx(tfm); 80 81 uctx->key_len = key_len; 82 memcpy(uctx->key, in_key, key_len); 83 84 /* 85 * leave out HASH_ALG (none = 0), CBC (no = 0), DIR (unknown) yet 86 */ 87 switch (uctx->key_len) { 88 case 16: 89 uctx->ctrl = SEC_KEY_128_BITS | SEC_ALG_AES; 90 break; 91 case 24: 92 uctx->ctrl = SEC_KEY_192_BITS | SEC_ALG_AES; 93 break; 94 case 32: 95 uctx->ctrl = SEC_KEY_256_BITS | SEC_ALG_AES; 96 break; 97 } 98 99 return 0; 100} 101 102static inline void aes_cipher(struct crypto_tfm *tfm, u8 *out, const u8 *in, u32 extra_flags) 103{ 104 const struct ubicom32_aes_ctx *uctx = crypto_tfm_ctx(tfm); 105 106 hw_crypto_lock(); 107 hw_crypto_check(); 108 hw_crypto_set_ctrl(uctx->ctrl | extra_flags); 109 110 aes_hw_set_key(uctx->key, uctx->key_len); 111 aes_hw_cipher(out, in); 112 113 hw_crypto_unlock(); 114} 115 116static void aes_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in) 117{ 118 aes_cipher(tfm, out, in, SEC_DIR_ENCRYPT); 119} 120 121static void aes_decrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in) 122{ 123 aes_cipher(tfm, out, in, SEC_DIR_DECRYPT); 124} 125 126static struct crypto_alg aes_alg = { 127 .cra_name = "aes", 128 .cra_driver_name = "aes-ubicom32", 129 .cra_priority = CRYPTO_UBICOM32_PRIORITY, 130 .cra_flags = CRYPTO_ALG_TYPE_CIPHER, 131 .cra_blocksize = AES_BLOCK_SIZE, 132 .cra_ctxsize = sizeof(struct ubicom32_aes_ctx), 133 .cra_alignmask = CRYPTO_UBICOM32_ALIGNMENT - 1, 134 .cra_module = THIS_MODULE, 135 .cra_list = LIST_HEAD_INIT(aes_alg.cra_list), 136 .cra_u = { 137 .cipher = { 138 .cia_min_keysize = AES_MIN_KEY_SIZE, 139 .cia_max_keysize = AES_MAX_KEY_SIZE, 140 .cia_setkey = aes_set_key, 141 .cia_encrypt = aes_encrypt, 142 .cia_decrypt = aes_decrypt, 143 } 144 } 145}; 146 147static void __ocm_text ecb_aes_crypt_loop(u8 *out, u8 *in, unsigned int n) 148{ 149 while (likely(n)) { 150 aes_hw_cipher(out, in); 151 out += AES_BLOCK_SIZE; 152 in += AES_BLOCK_SIZE; 153 n -= AES_BLOCK_SIZE; 154 } 155} 156 157static int __ocm_text ecb_aes_crypt(struct blkcipher_desc *desc, struct scatterlist *dst, 158 struct scatterlist *src, unsigned int nbytes, u32 extra_flags) 159{ 160 const struct ubicom32_aes_ctx *uctx = crypto_blkcipher_ctx(desc->tfm); 161 int ret; 162 163 struct blkcipher_walk walk; 164 blkcipher_walk_init(&walk, dst, src, nbytes); 165 ret = blkcipher_walk_virt(desc, &walk); 166 if (ret) { 167 return ret; 168 } 169 170 hw_crypto_lock(); 171 hw_crypto_check(); 172 173 hw_crypto_set_ctrl(uctx->ctrl | extra_flags); 174 aes_hw_set_key(uctx->key, uctx->key_len); 175 176 while (likely((nbytes = walk.nbytes))) { 177 /* only use complete blocks */ 178 unsigned int n = nbytes & ~(AES_BLOCK_SIZE - 1); 179 u8 *out = walk.dst.virt.addr; 180 u8 *in = walk.src.virt.addr; 181 182 /* finish n/16 blocks */ 183 ecb_aes_crypt_loop(out, in, n); 184 185 nbytes &= AES_BLOCK_SIZE - 1; 186 ret = blkcipher_walk_done(desc, &walk, nbytes); 187 } 188 189 hw_crypto_unlock(); 190 return ret; 191} 192 193static int ecb_aes_encrypt(struct blkcipher_desc *desc, 194 struct scatterlist *dst, struct scatterlist *src, 195 unsigned int nbytes) 196{ 197 return ecb_aes_crypt(desc, dst, src, nbytes, SEC_DIR_ENCRYPT); 198} 199 200static int ecb_aes_decrypt(struct blkcipher_desc *desc, 201 struct scatterlist *dst, struct scatterlist *src, 202 unsigned int nbytes) 203{ 204 return ecb_aes_crypt(desc, dst, src, nbytes, SEC_DIR_DECRYPT); 205} 206 207static struct crypto_alg ecb_aes_alg = { 208 .cra_name = "ecb(aes)", 209 .cra_driver_name = "ecb-aes-ubicom32", 210 .cra_priority = CRYPTO_UBICOM32_COMPOSITE_PRIORITY, 211 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 212 .cra_blocksize = AES_BLOCK_SIZE, 213 .cra_ctxsize = sizeof(struct ubicom32_aes_ctx), 214 .cra_alignmask = CRYPTO_UBICOM32_ALIGNMENT - 1, 215 .cra_type = &crypto_blkcipher_type, 216 .cra_module = THIS_MODULE, 217 .cra_list = LIST_HEAD_INIT(ecb_aes_alg.cra_list), 218 .cra_u = { 219 .blkcipher = { 220 .min_keysize = AES_MIN_KEY_SIZE, 221 .max_keysize = AES_MAX_KEY_SIZE, 222 .setkey = aes_set_key, 223 .encrypt = ecb_aes_encrypt, 224 .decrypt = ecb_aes_decrypt, 225 } 226 } 227}; 228 229#if CRYPTO_UBICOM32_LOOP_ASM 230void __ocm_text cbc_aes_encrypt_loop(u8 *out, u8 *in, u8 *iv, unsigned int n) 231{ 232 asm volatile ( 233 "; set init. iv 4w \n\t" 234 " move.4 0x50(%0), 0x0(%3) \n\t" 235 " move.4 0x54(%0), 0x4(%3) \n\t" 236 " move.4 0x58(%0), 0x8(%3) \n\t" 237 " move.4 0x5c(%0), 0xc(%3) \n\t" 238 " \n\t" 239 "; we know n > 0, so we can always \n\t" 240 "; load the first block \n\t" 241 "; set input 4w \n\t" 242 " move.4 0x30(%0), 0x0(%2) \n\t" 243 " move.4 0x34(%0), 0x4(%2) \n\t" 244 " move.4 0x38(%0), 0x8(%2) \n\t" 245 " move.4 0x3c(%0), 0xc(%2) \n\t" 246 " \n\t" 247 "; kickoff hw \n\t" 248 " move.4 0x40(%0), %2 \n\t" 249 " \n\t" 250 "; update n & flush \n\t" 251 " add.4 %4, #-16, %4 \n\t" 252 " pipe_flush 0 \n\t" 253 " \n\t" 254 "; while (n): work on 2nd block \n\t" 255 " 1: lsl.4 d15, %4, #0x0 \n\t" 256 " jmpeq.f 5f \n\t" 257 " \n\t" 258 "; set input 4w (2nd) \n\t" 259 " move.4 0x30(%0), 0x10(%2) \n\t" 260 " move.4 0x34(%0), 0x14(%2) \n\t" 261 " move.4 0x38(%0), 0x18(%2) \n\t" 262 " move.4 0x3c(%0), 0x1c(%2) \n\t" 263 " \n\t" 264 "; update n/in asap while waiting \n\t" 265 " add.4 %4, #-16, %4 \n\t" 266 " move.4 d15, 16(%2)++ \n\t" 267 " \n\t" 268 "; wait for the previous output \n\t" 269 " btst 0x04(%0), #0 \n\t" 270 " jmpne.f -4 \n\t" 271 " \n\t" 272 "; read previous output \n\t" 273 " move.4 0x0(%1), 0x50(%0) \n\t" 274 " move.4 0x4(%1), 0x54(%0) \n\t" 275 " move.4 0x8(%1), 0x58(%0) \n\t" 276 " move.4 0xc(%1), 0x5c(%0) \n\t" 277 " \n\t" 278 "; kick off hw for 2nd input \n\t" 279 " move.4 0x40(%0), %2 \n\t" 280 " \n\t" 281 "; update out asap \n\t" 282 " move.4 d15, 16(%1)++ \n\t" 283 " \n\t" 284 "; go back to loop \n\t" 285 " jmpt 1b \n\t" 286 " \n\t" 287 "; wait for last output \n\t" 288 " 5: btst 0x04(%0), #0 \n\t" 289 " jmpne.f -4 \n\t" 290 " \n\t" 291 "; read last output \n\t" 292 " move.4 0x0(%1), 0x50(%0) \n\t" 293 " move.4 0x4(%1), 0x54(%0) \n\t" 294 " move.4 0x8(%1), 0x58(%0) \n\t" 295 " move.4 0xc(%1), 0x5c(%0) \n\t" 296 " \n\t" 297 "; copy out iv \n\t" 298 " move.4 0x0(%3), 0x50(%0) \n\t" 299 " move.4 0x4(%3), 0x54(%0) \n\t" 300 " move.4 0x8(%3), 0x58(%0) \n\t" 301 " move.4 0xc(%3), 0x5c(%0) \n\t" 302 " \n\t" 303 : 304 : "a" (SEC_BASE), "a" (out), "a" (in), "a" (iv), "d" (n) 305 : "d15", "cc" 306 ); 307} 308 309#else 310 311static void __ocm_text cbc_aes_encrypt_loop(u8 *out, u8 *in, u8 *iv, unsigned int n) 312{ 313 aes_hw_set_iv(iv); 314 while (likely(n)) { 315 aes_hw_cipher(out, in); 316 out += AES_BLOCK_SIZE; 317 in += AES_BLOCK_SIZE; 318 n -= AES_BLOCK_SIZE; 319 } 320 SEC_COPY_4W(iv, out - AES_BLOCK_SIZE); 321} 322 323#endif 324 325static void __ocm_text cbc_aes_decrypt_loop(u8 *out, u8 *in, u8 *iv, unsigned int n) 326{ 327 while (likely(n)) { 328 aes_hw_set_iv(iv); 329 SEC_COPY_4W(iv, in); 330 aes_hw_cipher(out, in); 331 out += AES_BLOCK_SIZE; 332 in += AES_BLOCK_SIZE; 333 n -= AES_BLOCK_SIZE; 334 } 335} 336 337static int __ocm_text cbc_aes_crypt(struct blkcipher_desc *desc, 338 struct scatterlist *dst, struct scatterlist *src, 339 unsigned int nbytes, u32 extra_flags) 340{ 341 struct ubicom32_aes_ctx *uctx = crypto_blkcipher_ctx(desc->tfm); 342 int ret; 343 344 struct blkcipher_walk walk; 345 blkcipher_walk_init(&walk, dst, src, nbytes); 346 ret = blkcipher_walk_virt(desc, &walk); 347 if (unlikely(ret)) { 348 return ret; 349 } 350 351 hw_crypto_lock(); 352 hw_crypto_check(); 353 354 hw_crypto_set_ctrl(uctx->ctrl | extra_flags); 355 aes_hw_set_key(uctx->key, uctx->key_len); 356 357 while (likely((nbytes = walk.nbytes))) { 358 /* only use complete blocks */ 359 unsigned int n = nbytes & ~(AES_BLOCK_SIZE - 1); 360 if (likely(n)) { 361 u8 *out = walk.dst.virt.addr; 362 u8 *in = walk.src.virt.addr; 363 364 if (extra_flags & SEC_DIR_ENCRYPT) { 365 cbc_aes_encrypt_loop(out, in, walk.iv, n); 366 } else { 367 cbc_aes_decrypt_loop(out, in, walk.iv, n); 368 } 369 } 370 371 nbytes &= AES_BLOCK_SIZE - 1; 372 ret = blkcipher_walk_done(desc, &walk, nbytes); 373 } 374 hw_crypto_unlock(); 375 376 return ret; 377} 378 379static int __ocm_text cbc_aes_encrypt(struct blkcipher_desc *desc, 380 struct scatterlist *dst, struct scatterlist *src, 381 unsigned int nbytes) 382{ 383 return cbc_aes_crypt(desc, dst, src, nbytes, SEC_DIR_ENCRYPT | SEC_CBC_SET); 384} 385 386static int __ocm_text cbc_aes_decrypt(struct blkcipher_desc *desc, 387 struct scatterlist *dst, struct scatterlist *src, 388 unsigned int nbytes) 389{ 390 return cbc_aes_crypt(desc, dst, src, nbytes, SEC_DIR_DECRYPT | SEC_CBC_SET); 391} 392 393static struct crypto_alg cbc_aes_alg = { 394 .cra_name = "cbc(aes)", 395 .cra_driver_name = "cbc-aes-ubicom32", 396 .cra_priority = CRYPTO_UBICOM32_COMPOSITE_PRIORITY, 397 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 398 .cra_blocksize = AES_BLOCK_SIZE, 399 .cra_ctxsize = sizeof(struct ubicom32_aes_ctx), 400 .cra_alignmask = CRYPTO_UBICOM32_ALIGNMENT - 1, 401 .cra_type = &crypto_blkcipher_type, 402 .cra_module = THIS_MODULE, 403 .cra_list = LIST_HEAD_INIT(cbc_aes_alg.cra_list), 404 .cra_u = { 405 .blkcipher = { 406 .min_keysize = AES_MIN_KEY_SIZE, 407 .max_keysize = AES_MAX_KEY_SIZE, 408 .ivsize = AES_BLOCK_SIZE, 409 .setkey = aes_set_key, 410 .encrypt = cbc_aes_encrypt, 411 .decrypt = cbc_aes_decrypt, 412 } 413 } 414}; 415 416static int __init aes_init(void) 417{ 418 int ret; 419 420 hw_crypto_init(); 421 422 ret = crypto_register_alg(&aes_alg); 423 if (ret) 424 goto aes_err; 425 426 ret = crypto_register_alg(&ecb_aes_alg); 427 if (ret) 428 goto ecb_aes_err; 429 430 ret = crypto_register_alg(&cbc_aes_alg); 431 if (ret) 432 goto cbc_aes_err; 433 434out: 435 return ret; 436 437cbc_aes_err: 438 crypto_unregister_alg(&ecb_aes_alg); 439ecb_aes_err: 440 crypto_unregister_alg(&aes_alg); 441aes_err: 442 goto out; 443} 444 445static void __exit aes_fini(void) 446{ 447 crypto_unregister_alg(&cbc_aes_alg); 448 crypto_unregister_alg(&ecb_aes_alg); 449 crypto_unregister_alg(&aes_alg); 450} 451 452module_init(aes_init); 453module_exit(aes_fini); 454 455MODULE_ALIAS("aes"); 456 457MODULE_DESCRIPTION("Rijndael (AES) Cipher Algorithm"); 458MODULE_LICENSE("GPL"); 459