1/*
2 * arch/ubicom32/crypto/aes_ubicom32.c
3 *   Ubicom32 implementation of the AES Cipher Algorithm.
4 *
5 * (C) Copyright 2009, Ubicom, Inc.
6 *
7 * This file is part of the Ubicom32 Linux Kernel Port.
8 *
9 * The Ubicom32 Linux Kernel Port is free software: you can redistribute
10 * it and/or modify it under the terms of the GNU General Public License
11 * as published by the Free Software Foundation, either version 2 of the
12 * License, or (at your option) any later version.
13 *
14 * The Ubicom32 Linux Kernel Port is distributed in the hope that it
15 * will be useful, but WITHOUT ANY WARRANTY; without even the implied
16 * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See
17 * the GNU General Public License for more details.
18 *
19 * You should have received a copy of the GNU General Public License
20 * along with the Ubicom32 Linux Kernel Port.  If not,
21 * see <http://www.gnu.org/licenses/>.
22 *
23 * Ubicom32 implementation derived from (with many thanks):
24 *   arch/m68knommu
25 *   arch/blackfin
26 *   arch/parisc
27 */
28#include <crypto/aes.h>
29#include <crypto/algapi.h>
30#include <linux/err.h>
31#include <linux/module.h>
32#include <linux/init.h>
33#include <linux/spinlock.h>
34#include "crypto_ubicom32.h"
35#include <asm/linkage.h>
36
37struct ubicom32_aes_ctx {
38	u8 key[AES_MAX_KEY_SIZE];
39	u32 ctrl;
40	int key_len;
41};
42
43static inline void aes_hw_set_key(const u8 *key, u8 key_len)
44{
45	/*
46	 * switch case has more overhead than 4 move.4 instructions, so just copy 256 bits
47	 */
48	SEC_SET_KEY_256(key);
49}
50
51static inline void aes_hw_set_iv(const u8 *iv)
52{
53	SEC_SET_IV_4W(iv);
54}
55
56static inline void aes_hw_cipher(u8 *out, const u8 *in)
57{
58	SEC_SET_INPUT_4W(in);
59
60	asm volatile (
61	"	; start AES by writing 0x40(SECURITY_BASE)	\n\t"
62	"	move.4 0x40(%0), #0x01				\n\t"
63	"	pipe_flush 0                                    \n\t"
64	"							\n\t"
65	"	; wait for the module to calculate the output	\n\t"
66	"	btst 0x04(%0), #0				\n\t"
67	"	jmpne.f .-4					\n\t"
68		:
69		: "a" (SEC_BASE)
70		: "cc"
71	);
72
73	SEC_GET_OUTPUT_4W(out);
74}
75
76static int __ocm_text aes_set_key(struct crypto_tfm *tfm, const u8 *in_key,
77		       unsigned int key_len)
78{
79	struct ubicom32_aes_ctx *uctx = crypto_tfm_ctx(tfm);
80
81	uctx->key_len = key_len;
82	memcpy(uctx->key, in_key, key_len);
83
84	/*
85	 * leave out HASH_ALG (none = 0), CBC (no = 0), DIR (unknown) yet
86	 */
87	switch (uctx->key_len) {
88	case 16:
89		uctx->ctrl = SEC_KEY_128_BITS | SEC_ALG_AES;
90		break;
91	case 24:
92		uctx->ctrl = SEC_KEY_192_BITS | SEC_ALG_AES;
93		break;
94	case 32:
95		uctx->ctrl = SEC_KEY_256_BITS | SEC_ALG_AES;
96		break;
97	}
98
99	return 0;
100}
101
102static inline void aes_cipher(struct crypto_tfm *tfm, u8 *out, const u8 *in, u32 extra_flags)
103{
104	const struct ubicom32_aes_ctx *uctx = crypto_tfm_ctx(tfm);
105
106	hw_crypto_lock();
107	hw_crypto_check();
108	hw_crypto_set_ctrl(uctx->ctrl | extra_flags);
109
110	aes_hw_set_key(uctx->key, uctx->key_len);
111	aes_hw_cipher(out, in);
112
113	hw_crypto_unlock();
114}
115
116static void aes_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
117{
118	aes_cipher(tfm, out, in, SEC_DIR_ENCRYPT);
119}
120
121static void aes_decrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
122{
123	aes_cipher(tfm, out, in, SEC_DIR_DECRYPT);
124}
125
126static struct crypto_alg aes_alg = {
127	.cra_name		=	"aes",
128	.cra_driver_name	=	"aes-ubicom32",
129	.cra_priority		=	CRYPTO_UBICOM32_PRIORITY,
130	.cra_flags		=	CRYPTO_ALG_TYPE_CIPHER,
131	.cra_blocksize		=	AES_BLOCK_SIZE,
132	.cra_ctxsize		=	sizeof(struct ubicom32_aes_ctx),
133	.cra_alignmask		=	CRYPTO_UBICOM32_ALIGNMENT - 1,
134	.cra_module		=	THIS_MODULE,
135	.cra_list		=	LIST_HEAD_INIT(aes_alg.cra_list),
136	.cra_u			=	{
137		.cipher = {
138			.cia_min_keysize	=	AES_MIN_KEY_SIZE,
139			.cia_max_keysize	=	AES_MAX_KEY_SIZE,
140			.cia_setkey		=	aes_set_key,
141			.cia_encrypt		=	aes_encrypt,
142			.cia_decrypt		=	aes_decrypt,
143		}
144	}
145};
146
147static void __ocm_text ecb_aes_crypt_loop(u8 *out, u8 *in, unsigned int n)
148{
149	while (likely(n)) {
150		aes_hw_cipher(out, in);
151		out += AES_BLOCK_SIZE;
152		in += AES_BLOCK_SIZE;
153		n -= AES_BLOCK_SIZE;
154	}
155}
156
157static int __ocm_text ecb_aes_crypt(struct blkcipher_desc *desc, struct scatterlist *dst,
158			 struct scatterlist *src, unsigned int nbytes, u32 extra_flags)
159{
160	const struct ubicom32_aes_ctx *uctx = crypto_blkcipher_ctx(desc->tfm);
161	int ret;
162
163	struct blkcipher_walk walk;
164	blkcipher_walk_init(&walk, dst, src, nbytes);
165	ret = blkcipher_walk_virt(desc, &walk);
166        if (ret) {
167                return ret;
168        }
169
170	hw_crypto_lock();
171	hw_crypto_check();
172
173        hw_crypto_set_ctrl(uctx->ctrl | extra_flags);
174        aes_hw_set_key(uctx->key, uctx->key_len);
175
176	while (likely((nbytes = walk.nbytes))) {
177		/* only use complete blocks */
178		unsigned int n = nbytes & ~(AES_BLOCK_SIZE - 1);
179		u8 *out = walk.dst.virt.addr;
180		u8 *in = walk.src.virt.addr;
181
182		/* finish n/16 blocks */
183		ecb_aes_crypt_loop(out, in, n);
184
185		nbytes &= AES_BLOCK_SIZE - 1;
186		ret = blkcipher_walk_done(desc, &walk, nbytes);
187	}
188
189	hw_crypto_unlock();
190	return ret;
191}
192
193static int ecb_aes_encrypt(struct blkcipher_desc *desc,
194			   struct scatterlist *dst, struct scatterlist *src,
195			   unsigned int nbytes)
196{
197	return ecb_aes_crypt(desc, dst, src, nbytes, SEC_DIR_ENCRYPT);
198}
199
200static int ecb_aes_decrypt(struct blkcipher_desc *desc,
201			   struct scatterlist *dst, struct scatterlist *src,
202			   unsigned int nbytes)
203{
204	return ecb_aes_crypt(desc, dst, src, nbytes, SEC_DIR_DECRYPT);
205}
206
207static struct crypto_alg ecb_aes_alg = {
208	.cra_name		=	"ecb(aes)",
209	.cra_driver_name	=	"ecb-aes-ubicom32",
210	.cra_priority		=	CRYPTO_UBICOM32_COMPOSITE_PRIORITY,
211	.cra_flags		=	CRYPTO_ALG_TYPE_BLKCIPHER,
212	.cra_blocksize		=	AES_BLOCK_SIZE,
213	.cra_ctxsize		=	sizeof(struct ubicom32_aes_ctx),
214	.cra_alignmask          =       CRYPTO_UBICOM32_ALIGNMENT - 1,
215	.cra_type		=	&crypto_blkcipher_type,
216	.cra_module		=	THIS_MODULE,
217	.cra_list		=	LIST_HEAD_INIT(ecb_aes_alg.cra_list),
218	.cra_u			=	{
219		.blkcipher = {
220			.min_keysize		=	AES_MIN_KEY_SIZE,
221			.max_keysize		=	AES_MAX_KEY_SIZE,
222			.setkey			=	aes_set_key,
223			.encrypt		=	ecb_aes_encrypt,
224			.decrypt		=	ecb_aes_decrypt,
225		}
226	}
227};
228
229#if CRYPTO_UBICOM32_LOOP_ASM
230void __ocm_text cbc_aes_encrypt_loop(u8 *out, u8 *in, u8 *iv, unsigned int n)
231{
232	asm volatile (
233	"; set init. iv 4w			\n\t"
234	"	move.4 0x50(%0), 0x0(%3)	\n\t"
235	"	move.4 0x54(%0), 0x4(%3)	\n\t"
236	"	move.4 0x58(%0), 0x8(%3)	\n\t"
237	"	move.4 0x5c(%0), 0xc(%3)	\n\t"
238	"					\n\t"
239	"; we know n > 0, so we can always	\n\t"
240	"; load the first block			\n\t"
241	"; set input 4w				\n\t"
242	"	move.4 0x30(%0), 0x0(%2)	\n\t"
243	"	move.4 0x34(%0), 0x4(%2)	\n\t"
244	"	move.4 0x38(%0), 0x8(%2)	\n\t"
245	"	move.4 0x3c(%0), 0xc(%2)	\n\t"
246	"					\n\t"
247	"; kickoff hw				\n\t"
248	"	move.4 0x40(%0), %2		\n\t"
249	"					\n\t"
250	"; update n & flush			\n\t"
251	"	add.4 %4, #-16, %4		\n\t"
252	"	pipe_flush 0			\n\t"
253	"					\n\t"
254	"; while (n):  work on 2nd block	\n\t"
255	" 1:	lsl.4 d15, %4, #0x0		\n\t"
256	"	jmpeq.f 5f			\n\t"
257	"					\n\t"
258	"; set input 4w	(2nd)			\n\t"
259	"	move.4 0x30(%0), 0x10(%2)	\n\t"
260	"	move.4 0x34(%0), 0x14(%2)	\n\t"
261	"	move.4 0x38(%0), 0x18(%2)	\n\t"
262	"	move.4 0x3c(%0), 0x1c(%2)	\n\t"
263	"					\n\t"
264	"; update n/in asap while waiting	\n\t"
265	"	add.4 %4, #-16, %4		\n\t"
266	"	move.4 d15, 16(%2)++		\n\t"
267	"					\n\t"
268	"; wait for the previous output		\n\t"
269	"	btst 0x04(%0), #0		\n\t"
270	"	jmpne.f -4			\n\t"
271	"					\n\t"
272	"; read previous output			\n\t"
273	"	move.4 0x0(%1), 0x50(%0)	\n\t"
274	"	move.4 0x4(%1), 0x54(%0)	\n\t"
275	"	move.4 0x8(%1), 0x58(%0)	\n\t"
276	"	move.4 0xc(%1), 0x5c(%0)	\n\t"
277	"					\n\t"
278	"; kick off hw for 2nd input		\n\t"
279	"	move.4 0x40(%0), %2		\n\t"
280	"					\n\t"
281	"; update out asap			\n\t"
282	"	move.4 d15, 16(%1)++		\n\t"
283	"					\n\t"
284	"; go back to loop			\n\t"
285	"	jmpt 1b				\n\t"
286	"					\n\t"
287	"; wait for last output			\n\t"
288	" 5:	btst 0x04(%0), #0               \n\t"
289        "       jmpne.f -4                      \n\t"
290        "                                       \n\t"
291	"; read last output			\n\t"
292	"	move.4 0x0(%1), 0x50(%0)	\n\t"
293	"	move.4 0x4(%1), 0x54(%0)	\n\t"
294	"	move.4 0x8(%1), 0x58(%0)	\n\t"
295	"	move.4 0xc(%1), 0x5c(%0)	\n\t"
296        "                                       \n\t"
297	"; copy out iv				\n\t"
298	"	move.4 0x0(%3), 0x50(%0)	\n\t"
299	"	move.4 0x4(%3), 0x54(%0)	\n\t"
300	"	move.4 0x8(%3), 0x58(%0)	\n\t"
301	"	move.4 0xc(%3), 0x5c(%0)	\n\t"
302        "                                       \n\t"
303		:
304		: "a" (SEC_BASE), "a" (out), "a" (in), "a" (iv), "d" (n)
305		: "d15", "cc"
306	);
307}
308
309#else
310
311static void __ocm_text cbc_aes_encrypt_loop(u8 *out, u8 *in, u8 *iv, unsigned int n)
312{
313	aes_hw_set_iv(iv);
314	while (likely(n)) {
315		aes_hw_cipher(out, in);
316		out += AES_BLOCK_SIZE;
317		in += AES_BLOCK_SIZE;
318		n -= AES_BLOCK_SIZE;
319	}
320	SEC_COPY_4W(iv, out - AES_BLOCK_SIZE);
321}
322
323#endif
324
325static void __ocm_text cbc_aes_decrypt_loop(u8 *out, u8 *in, u8 *iv, unsigned int n)
326{
327        while (likely(n)) {
328                aes_hw_set_iv(iv);
329		SEC_COPY_4W(iv, in);
330                aes_hw_cipher(out, in);
331                out += AES_BLOCK_SIZE;
332                in += AES_BLOCK_SIZE;
333                n -= AES_BLOCK_SIZE;
334        }
335}
336
337static int __ocm_text cbc_aes_crypt(struct blkcipher_desc *desc,
338                           struct scatterlist *dst, struct scatterlist *src,
339                           unsigned int nbytes, u32 extra_flags)
340{
341	struct ubicom32_aes_ctx *uctx = crypto_blkcipher_ctx(desc->tfm);
342	int ret;
343
344        struct blkcipher_walk walk;
345        blkcipher_walk_init(&walk, dst, src, nbytes);
346	ret = blkcipher_walk_virt(desc, &walk);
347	if (unlikely(ret)) {
348		return ret;
349	}
350
351        hw_crypto_lock();
352	hw_crypto_check();
353
354        hw_crypto_set_ctrl(uctx->ctrl | extra_flags);
355        aes_hw_set_key(uctx->key, uctx->key_len);
356
357	while (likely((nbytes = walk.nbytes))) {
358                /* only use complete blocks */
359                unsigned int n = nbytes & ~(AES_BLOCK_SIZE - 1);
360		if (likely(n)) {
361	                u8 *out = walk.dst.virt.addr;
362		        u8 *in = walk.src.virt.addr;
363
364			if (extra_flags & SEC_DIR_ENCRYPT) {
365				cbc_aes_encrypt_loop(out, in, walk.iv, n);
366			} else {
367				cbc_aes_decrypt_loop(out, in, walk.iv, n);
368			}
369		}
370
371		nbytes &= AES_BLOCK_SIZE - 1;
372                ret = blkcipher_walk_done(desc, &walk, nbytes);
373	}
374	hw_crypto_unlock();
375
376	return ret;
377}
378
379static int __ocm_text cbc_aes_encrypt(struct blkcipher_desc *desc,
380			   struct scatterlist *dst, struct scatterlist *src,
381			   unsigned int nbytes)
382{
383	return cbc_aes_crypt(desc, dst, src, nbytes, SEC_DIR_ENCRYPT | SEC_CBC_SET);
384}
385
386static int __ocm_text cbc_aes_decrypt(struct blkcipher_desc *desc,
387			   struct scatterlist *dst, struct scatterlist *src,
388			   unsigned int nbytes)
389{
390	return cbc_aes_crypt(desc, dst, src, nbytes, SEC_DIR_DECRYPT | SEC_CBC_SET);
391}
392
393static struct crypto_alg cbc_aes_alg = {
394	.cra_name		=	"cbc(aes)",
395	.cra_driver_name	=	"cbc-aes-ubicom32",
396	.cra_priority		=	CRYPTO_UBICOM32_COMPOSITE_PRIORITY,
397	.cra_flags		=	CRYPTO_ALG_TYPE_BLKCIPHER,
398	.cra_blocksize		=	AES_BLOCK_SIZE,
399	.cra_ctxsize		=	sizeof(struct ubicom32_aes_ctx),
400	.cra_alignmask          =       CRYPTO_UBICOM32_ALIGNMENT - 1,
401	.cra_type		=	&crypto_blkcipher_type,
402	.cra_module		=	THIS_MODULE,
403	.cra_list		=	LIST_HEAD_INIT(cbc_aes_alg.cra_list),
404	.cra_u			=	{
405		.blkcipher = {
406			.min_keysize		=	AES_MIN_KEY_SIZE,
407			.max_keysize		=	AES_MAX_KEY_SIZE,
408			.ivsize			=	AES_BLOCK_SIZE,
409			.setkey			=	aes_set_key,
410			.encrypt		=	cbc_aes_encrypt,
411			.decrypt		=	cbc_aes_decrypt,
412		}
413	}
414};
415
416static int __init aes_init(void)
417{
418	int ret;
419
420	hw_crypto_init();
421
422	ret = crypto_register_alg(&aes_alg);
423	if (ret)
424		goto aes_err;
425
426	ret = crypto_register_alg(&ecb_aes_alg);
427	if (ret)
428		goto ecb_aes_err;
429
430	ret = crypto_register_alg(&cbc_aes_alg);
431	if (ret)
432		goto cbc_aes_err;
433
434out:
435	return ret;
436
437cbc_aes_err:
438	crypto_unregister_alg(&ecb_aes_alg);
439ecb_aes_err:
440	crypto_unregister_alg(&aes_alg);
441aes_err:
442	goto out;
443}
444
445static void __exit aes_fini(void)
446{
447	crypto_unregister_alg(&cbc_aes_alg);
448	crypto_unregister_alg(&ecb_aes_alg);
449	crypto_unregister_alg(&aes_alg);
450}
451
452module_init(aes_init);
453module_exit(aes_fini);
454
455MODULE_ALIAS("aes");
456
457MODULE_DESCRIPTION("Rijndael (AES) Cipher Algorithm");
458MODULE_LICENSE("GPL");
459