1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * aes-ce-ccm-glue.c - AES-CCM transform for ARMv8 with Crypto Extensions
4 *
5 * Copyright (C) 2013 - 2017 Linaro Ltd.
6 * Copyright (C) 2024 Google LLC
7 *
8 * Author: Ard Biesheuvel <ardb@kernel.org>
9 */
10
11#include <asm/neon.h>
12#include <asm/unaligned.h>
13#include <crypto/aes.h>
14#include <crypto/scatterwalk.h>
15#include <crypto/internal/aead.h>
16#include <crypto/internal/skcipher.h>
17#include <linux/module.h>
18
19#include "aes-ce-setkey.h"
20
21MODULE_IMPORT_NS(CRYPTO_INTERNAL);
22
23static int num_rounds(struct crypto_aes_ctx *ctx)
24{
25	/*
26	 * # of rounds specified by AES:
27	 * 128 bit key		10 rounds
28	 * 192 bit key		12 rounds
29	 * 256 bit key		14 rounds
30	 * => n byte key	=> 6 + (n/4) rounds
31	 */
32	return 6 + ctx->key_length / 4;
33}
34
35asmlinkage u32 ce_aes_mac_update(u8 const in[], u32 const rk[], int rounds,
36				 int blocks, u8 dg[], int enc_before,
37				 int enc_after);
38
39asmlinkage void ce_aes_ccm_encrypt(u8 out[], u8 const in[], u32 cbytes,
40				   u32 const rk[], u32 rounds, u8 mac[],
41				   u8 ctr[], u8 const final_iv[]);
42
43asmlinkage void ce_aes_ccm_decrypt(u8 out[], u8 const in[], u32 cbytes,
44				   u32 const rk[], u32 rounds, u8 mac[],
45				   u8 ctr[], u8 const final_iv[]);
46
47static int ccm_setkey(struct crypto_aead *tfm, const u8 *in_key,
48		      unsigned int key_len)
49{
50	struct crypto_aes_ctx *ctx = crypto_aead_ctx(tfm);
51
52	return ce_aes_expandkey(ctx, in_key, key_len);
53}
54
55static int ccm_setauthsize(struct crypto_aead *tfm, unsigned int authsize)
56{
57	if ((authsize & 1) || authsize < 4)
58		return -EINVAL;
59	return 0;
60}
61
62static int ccm_init_mac(struct aead_request *req, u8 maciv[], u32 msglen)
63{
64	struct crypto_aead *aead = crypto_aead_reqtfm(req);
65	__be32 *n = (__be32 *)&maciv[AES_BLOCK_SIZE - 8];
66	u32 l = req->iv[0] + 1;
67
68	/* verify that CCM dimension 'L' is set correctly in the IV */
69	if (l < 2 || l > 8)
70		return -EINVAL;
71
72	/* verify that msglen can in fact be represented in L bytes */
73	if (l < 4 && msglen >> (8 * l))
74		return -EOVERFLOW;
75
76	/*
77	 * Even if the CCM spec allows L values of up to 8, the Linux cryptoapi
78	 * uses a u32 type to represent msglen so the top 4 bytes are always 0.
79	 */
80	n[0] = 0;
81	n[1] = cpu_to_be32(msglen);
82
83	memcpy(maciv, req->iv, AES_BLOCK_SIZE - l);
84
85	/*
86	 * Meaning of byte 0 according to CCM spec (RFC 3610/NIST 800-38C)
87	 * - bits 0..2	: max # of bytes required to represent msglen, minus 1
88	 *                (already set by caller)
89	 * - bits 3..5	: size of auth tag (1 => 4 bytes, 2 => 6 bytes, etc)
90	 * - bit 6	: indicates presence of authenticate-only data
91	 */
92	maciv[0] |= (crypto_aead_authsize(aead) - 2) << 2;
93	if (req->assoclen)
94		maciv[0] |= 0x40;
95
96	memset(&req->iv[AES_BLOCK_SIZE - l], 0, l);
97	return 0;
98}
99
100static u32 ce_aes_ccm_auth_data(u8 mac[], u8 const in[], u32 abytes,
101				u32 macp, u32 const rk[], u32 rounds)
102{
103	int enc_after = (macp + abytes) % AES_BLOCK_SIZE;
104
105	do {
106		u32 blocks = abytes / AES_BLOCK_SIZE;
107
108		if (macp == AES_BLOCK_SIZE || (!macp && blocks > 0)) {
109			u32 rem = ce_aes_mac_update(in, rk, rounds, blocks, mac,
110						    macp, enc_after);
111			u32 adv = (blocks - rem) * AES_BLOCK_SIZE;
112
113			macp = enc_after ? 0 : AES_BLOCK_SIZE;
114			in += adv;
115			abytes -= adv;
116
117			if (unlikely(rem)) {
118				kernel_neon_end();
119				kernel_neon_begin();
120				macp = 0;
121			}
122		} else {
123			u32 l = min(AES_BLOCK_SIZE - macp, abytes);
124
125			crypto_xor(&mac[macp], in, l);
126			in += l;
127			macp += l;
128			abytes -= l;
129		}
130	} while (abytes > 0);
131
132	return macp;
133}
134
135static void ccm_calculate_auth_mac(struct aead_request *req, u8 mac[])
136{
137	struct crypto_aead *aead = crypto_aead_reqtfm(req);
138	struct crypto_aes_ctx *ctx = crypto_aead_ctx(aead);
139	struct __packed { __be16 l; __be32 h; u16 len; } ltag;
140	struct scatter_walk walk;
141	u32 len = req->assoclen;
142	u32 macp = AES_BLOCK_SIZE;
143
144	/* prepend the AAD with a length tag */
145	if (len < 0xff00) {
146		ltag.l = cpu_to_be16(len);
147		ltag.len = 2;
148	} else  {
149		ltag.l = cpu_to_be16(0xfffe);
150		put_unaligned_be32(len, &ltag.h);
151		ltag.len = 6;
152	}
153
154	macp = ce_aes_ccm_auth_data(mac, (u8 *)&ltag, ltag.len, macp,
155				    ctx->key_enc, num_rounds(ctx));
156	scatterwalk_start(&walk, req->src);
157
158	do {
159		u32 n = scatterwalk_clamp(&walk, len);
160		u8 *p;
161
162		if (!n) {
163			scatterwalk_start(&walk, sg_next(walk.sg));
164			n = scatterwalk_clamp(&walk, len);
165		}
166		p = scatterwalk_map(&walk);
167
168		macp = ce_aes_ccm_auth_data(mac, p, n, macp, ctx->key_enc,
169					    num_rounds(ctx));
170
171		len -= n;
172
173		scatterwalk_unmap(p);
174		scatterwalk_advance(&walk, n);
175		scatterwalk_done(&walk, 0, len);
176	} while (len);
177}
178
179static int ccm_encrypt(struct aead_request *req)
180{
181	struct crypto_aead *aead = crypto_aead_reqtfm(req);
182	struct crypto_aes_ctx *ctx = crypto_aead_ctx(aead);
183	struct skcipher_walk walk;
184	u8 __aligned(8) mac[AES_BLOCK_SIZE];
185	u8 orig_iv[AES_BLOCK_SIZE];
186	u32 len = req->cryptlen;
187	int err;
188
189	err = ccm_init_mac(req, mac, len);
190	if (err)
191		return err;
192
193	/* preserve the original iv for the final round */
194	memcpy(orig_iv, req->iv, AES_BLOCK_SIZE);
195
196	err = skcipher_walk_aead_encrypt(&walk, req, false);
197	if (unlikely(err))
198		return err;
199
200	kernel_neon_begin();
201
202	if (req->assoclen)
203		ccm_calculate_auth_mac(req, mac);
204
205	do {
206		u32 tail = walk.nbytes % AES_BLOCK_SIZE;
207		const u8 *src = walk.src.virt.addr;
208		u8 *dst = walk.dst.virt.addr;
209		u8 buf[AES_BLOCK_SIZE];
210		u8 *final_iv = NULL;
211
212		if (walk.nbytes == walk.total) {
213			tail = 0;
214			final_iv = orig_iv;
215		}
216
217		if (unlikely(walk.nbytes < AES_BLOCK_SIZE))
218			src = dst = memcpy(&buf[sizeof(buf) - walk.nbytes],
219					   src, walk.nbytes);
220
221		ce_aes_ccm_encrypt(dst, src, walk.nbytes - tail,
222				   ctx->key_enc, num_rounds(ctx),
223				   mac, walk.iv, final_iv);
224
225		if (unlikely(walk.nbytes < AES_BLOCK_SIZE))
226			memcpy(walk.dst.virt.addr, dst, walk.nbytes);
227
228		if (walk.nbytes) {
229			err = skcipher_walk_done(&walk, tail);
230		}
231	} while (walk.nbytes);
232
233	kernel_neon_end();
234
235	if (unlikely(err))
236		return err;
237
238	/* copy authtag to end of dst */
239	scatterwalk_map_and_copy(mac, req->dst, req->assoclen + req->cryptlen,
240				 crypto_aead_authsize(aead), 1);
241
242	return 0;
243}
244
245static int ccm_decrypt(struct aead_request *req)
246{
247	struct crypto_aead *aead = crypto_aead_reqtfm(req);
248	struct crypto_aes_ctx *ctx = crypto_aead_ctx(aead);
249	unsigned int authsize = crypto_aead_authsize(aead);
250	struct skcipher_walk walk;
251	u8 __aligned(8) mac[AES_BLOCK_SIZE];
252	u8 orig_iv[AES_BLOCK_SIZE];
253	u32 len = req->cryptlen - authsize;
254	int err;
255
256	err = ccm_init_mac(req, mac, len);
257	if (err)
258		return err;
259
260	/* preserve the original iv for the final round */
261	memcpy(orig_iv, req->iv, AES_BLOCK_SIZE);
262
263	err = skcipher_walk_aead_decrypt(&walk, req, false);
264	if (unlikely(err))
265		return err;
266
267	kernel_neon_begin();
268
269	if (req->assoclen)
270		ccm_calculate_auth_mac(req, mac);
271
272	do {
273		u32 tail = walk.nbytes % AES_BLOCK_SIZE;
274		const u8 *src = walk.src.virt.addr;
275		u8 *dst = walk.dst.virt.addr;
276		u8 buf[AES_BLOCK_SIZE];
277		u8 *final_iv = NULL;
278
279		if (walk.nbytes == walk.total) {
280			tail = 0;
281			final_iv = orig_iv;
282		}
283
284		if (unlikely(walk.nbytes < AES_BLOCK_SIZE))
285			src = dst = memcpy(&buf[sizeof(buf) - walk.nbytes],
286					   src, walk.nbytes);
287
288		ce_aes_ccm_decrypt(dst, src, walk.nbytes - tail,
289				   ctx->key_enc, num_rounds(ctx),
290				   mac, walk.iv, final_iv);
291
292		if (unlikely(walk.nbytes < AES_BLOCK_SIZE))
293			memcpy(walk.dst.virt.addr, dst, walk.nbytes);
294
295		if (walk.nbytes) {
296			err = skcipher_walk_done(&walk, tail);
297		}
298	} while (walk.nbytes);
299
300	kernel_neon_end();
301
302	if (unlikely(err))
303		return err;
304
305	/* compare calculated auth tag with the stored one */
306	scatterwalk_map_and_copy(orig_iv, req->src,
307				 req->assoclen + req->cryptlen - authsize,
308				 authsize, 0);
309
310	if (crypto_memneq(mac, orig_iv, authsize))
311		return -EBADMSG;
312	return 0;
313}
314
315static struct aead_alg ccm_aes_alg = {
316	.base = {
317		.cra_name		= "ccm(aes)",
318		.cra_driver_name	= "ccm-aes-ce",
319		.cra_priority		= 300,
320		.cra_blocksize		= 1,
321		.cra_ctxsize		= sizeof(struct crypto_aes_ctx),
322		.cra_module		= THIS_MODULE,
323	},
324	.ivsize		= AES_BLOCK_SIZE,
325	.chunksize	= AES_BLOCK_SIZE,
326	.maxauthsize	= AES_BLOCK_SIZE,
327	.setkey		= ccm_setkey,
328	.setauthsize	= ccm_setauthsize,
329	.encrypt	= ccm_encrypt,
330	.decrypt	= ccm_decrypt,
331};
332
333static int __init aes_mod_init(void)
334{
335	if (!cpu_have_named_feature(AES))
336		return -ENODEV;
337	return crypto_register_aead(&ccm_aes_alg);
338}
339
340static void __exit aes_mod_exit(void)
341{
342	crypto_unregister_aead(&ccm_aes_alg);
343}
344
345module_init(aes_mod_init);
346module_exit(aes_mod_exit);
347
348MODULE_DESCRIPTION("Synchronous AES in CCM mode using ARMv8 Crypto Extensions");
349MODULE_AUTHOR("Ard Biesheuvel <ardb@kernel.org>");
350MODULE_LICENSE("GPL v2");
351MODULE_ALIAS_CRYPTO("ccm(aes)");
352