1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Cryptographic API.
4 *
5 * Support for OMAP AES GCM HW acceleration.
6 *
7 * Copyright (c) 2016 Texas Instruments Incorporated
8 */
9
10#include <crypto/aes.h>
11#include <crypto/engine.h>
12#include <crypto/gcm.h>
13#include <crypto/internal/aead.h>
14#include <crypto/scatterwalk.h>
15#include <crypto/skcipher.h>
16#include <linux/errno.h>
17#include <linux/dma-mapping.h>
18#include <linux/dmaengine.h>
19#include <linux/interrupt.h>
20#include <linux/kernel.h>
21#include <linux/omap-dma.h>
22#include <linux/pm_runtime.h>
23#include <linux/scatterlist.h>
24#include <linux/string.h>
25
26#include "omap-crypto.h"
27#include "omap-aes.h"
28
29static int omap_aes_gcm_handle_queue(struct omap_aes_dev *dd,
30				     struct aead_request *req);
31
32static void omap_aes_gcm_finish_req(struct omap_aes_dev *dd, int ret)
33{
34	struct aead_request *req = dd->aead_req;
35
36	dd->in_sg = NULL;
37	dd->out_sg = NULL;
38
39	crypto_finalize_aead_request(dd->engine, req, ret);
40
41	pm_runtime_mark_last_busy(dd->dev);
42	pm_runtime_put_autosuspend(dd->dev);
43}
44
45static void omap_aes_gcm_done_task(struct omap_aes_dev *dd)
46{
47	u8 *tag;
48	int alen, clen, i, ret = 0, nsg;
49	struct omap_aes_reqctx *rctx;
50
51	alen = ALIGN(dd->assoc_len, AES_BLOCK_SIZE);
52	clen = ALIGN(dd->total, AES_BLOCK_SIZE);
53	rctx = aead_request_ctx(dd->aead_req);
54
55	nsg = !!(dd->assoc_len && dd->total);
56
57	dma_sync_sg_for_device(dd->dev, dd->out_sg, dd->out_sg_len,
58			       DMA_FROM_DEVICE);
59	dma_unmap_sg(dd->dev, dd->in_sg, dd->in_sg_len, DMA_TO_DEVICE);
60	dma_unmap_sg(dd->dev, dd->out_sg, dd->out_sg_len, DMA_FROM_DEVICE);
61	omap_aes_crypt_dma_stop(dd);
62
63	omap_crypto_cleanup(dd->out_sg, dd->orig_out,
64			    dd->aead_req->assoclen, dd->total,
65			    FLAGS_OUT_DATA_ST_SHIFT, dd->flags);
66
67	if (dd->flags & FLAGS_ENCRYPT)
68		scatterwalk_map_and_copy(rctx->auth_tag,
69					 dd->aead_req->dst,
70					 dd->total + dd->aead_req->assoclen,
71					 dd->authsize, 1);
72
73	omap_crypto_cleanup(&dd->in_sgl[0], NULL, 0, alen,
74			    FLAGS_ASSOC_DATA_ST_SHIFT, dd->flags);
75
76	omap_crypto_cleanup(&dd->in_sgl[nsg], NULL, 0, clen,
77			    FLAGS_IN_DATA_ST_SHIFT, dd->flags);
78
79	if (!(dd->flags & FLAGS_ENCRYPT)) {
80		tag = (u8 *)rctx->auth_tag;
81		for (i = 0; i < dd->authsize; i++) {
82			if (tag[i]) {
83				ret = -EBADMSG;
84			}
85		}
86	}
87
88	omap_aes_gcm_finish_req(dd, ret);
89}
90
91static int omap_aes_gcm_copy_buffers(struct omap_aes_dev *dd,
92				     struct aead_request *req)
93{
94	int alen, clen, cryptlen, assoclen, ret;
95	struct crypto_aead *aead = crypto_aead_reqtfm(req);
96	unsigned int authlen = crypto_aead_authsize(aead);
97	struct scatterlist *tmp, sg_arr[2];
98	int nsg;
99	u16 flags;
100
101	assoclen = req->assoclen;
102	cryptlen = req->cryptlen;
103
104	if (dd->flags & FLAGS_RFC4106_GCM)
105		assoclen -= 8;
106
107	if (!(dd->flags & FLAGS_ENCRYPT))
108		cryptlen -= authlen;
109
110	alen = ALIGN(assoclen, AES_BLOCK_SIZE);
111	clen = ALIGN(cryptlen, AES_BLOCK_SIZE);
112
113	nsg = !!(assoclen && cryptlen);
114
115	omap_aes_clear_copy_flags(dd);
116
117	sg_init_table(dd->in_sgl, nsg + 1);
118	if (assoclen) {
119		tmp = req->src;
120		ret = omap_crypto_align_sg(&tmp, assoclen,
121					   AES_BLOCK_SIZE, dd->in_sgl,
122					   OMAP_CRYPTO_COPY_DATA |
123					   OMAP_CRYPTO_ZERO_BUF |
124					   OMAP_CRYPTO_FORCE_SINGLE_ENTRY,
125					   FLAGS_ASSOC_DATA_ST_SHIFT,
126					   &dd->flags);
127		if (ret)
128			return ret;
129	}
130
131	if (cryptlen) {
132		tmp = scatterwalk_ffwd(sg_arr, req->src, req->assoclen);
133
134		if (nsg)
135			sg_unmark_end(dd->in_sgl);
136
137		ret = omap_crypto_align_sg(&tmp, cryptlen,
138					   AES_BLOCK_SIZE, &dd->in_sgl[nsg],
139					   OMAP_CRYPTO_COPY_DATA |
140					   OMAP_CRYPTO_ZERO_BUF |
141					   OMAP_CRYPTO_FORCE_SINGLE_ENTRY,
142					   FLAGS_IN_DATA_ST_SHIFT,
143					   &dd->flags);
144		if (ret)
145			return ret;
146	}
147
148	dd->in_sg = dd->in_sgl;
149	dd->total = cryptlen;
150	dd->assoc_len = assoclen;
151	dd->authsize = authlen;
152
153	dd->out_sg = req->dst;
154	dd->orig_out = req->dst;
155
156	dd->out_sg = scatterwalk_ffwd(sg_arr, req->dst, req->assoclen);
157
158	flags = 0;
159	if (req->src == req->dst || dd->out_sg == sg_arr)
160		flags |= OMAP_CRYPTO_FORCE_COPY;
161
162	if (cryptlen) {
163		ret = omap_crypto_align_sg(&dd->out_sg, cryptlen,
164					   AES_BLOCK_SIZE, &dd->out_sgl,
165					   flags,
166					   FLAGS_OUT_DATA_ST_SHIFT, &dd->flags);
167		if (ret)
168			return ret;
169	}
170
171	dd->in_sg_len = sg_nents_for_len(dd->in_sg, alen + clen);
172	dd->out_sg_len = sg_nents_for_len(dd->out_sg, clen);
173
174	return 0;
175}
176
177static int do_encrypt_iv(struct aead_request *req, u32 *tag, u32 *iv)
178{
179	struct omap_aes_gcm_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
180
181	aes_encrypt(&ctx->actx, (u8 *)tag, (u8 *)iv);
182	return 0;
183}
184
185void omap_aes_gcm_dma_out_callback(void *data)
186{
187	struct omap_aes_dev *dd = data;
188	struct omap_aes_reqctx *rctx;
189	int i, val;
190	u32 *auth_tag, tag[4];
191
192	if (!(dd->flags & FLAGS_ENCRYPT))
193		scatterwalk_map_and_copy(tag, dd->aead_req->src,
194					 dd->total + dd->aead_req->assoclen,
195					 dd->authsize, 0);
196
197	rctx = aead_request_ctx(dd->aead_req);
198	auth_tag = (u32 *)rctx->auth_tag;
199	for (i = 0; i < 4; i++) {
200		val = omap_aes_read(dd, AES_REG_TAG_N(dd, i));
201		auth_tag[i] = val ^ auth_tag[i];
202		if (!(dd->flags & FLAGS_ENCRYPT))
203			auth_tag[i] = auth_tag[i] ^ tag[i];
204	}
205
206	omap_aes_gcm_done_task(dd);
207}
208
209static int omap_aes_gcm_handle_queue(struct omap_aes_dev *dd,
210				     struct aead_request *req)
211{
212	if (req)
213		return crypto_transfer_aead_request_to_engine(dd->engine, req);
214
215	return 0;
216}
217
218static int omap_aes_gcm_prepare_req(struct aead_request *req,
219				    struct omap_aes_dev *dd)
220{
221	struct omap_aes_reqctx *rctx = aead_request_ctx(req);
222	struct omap_aes_gcm_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
223	int err;
224
225	dd->aead_req = req;
226
227	rctx->mode &= FLAGS_MODE_MASK;
228	dd->flags = (dd->flags & ~FLAGS_MODE_MASK) | rctx->mode;
229
230	err = omap_aes_gcm_copy_buffers(dd, req);
231	if (err)
232		return err;
233
234	dd->ctx = &ctx->octx;
235
236	return omap_aes_write_ctrl(dd);
237}
238
239static int omap_aes_gcm_crypt(struct aead_request *req, unsigned long mode)
240{
241	struct omap_aes_reqctx *rctx = aead_request_ctx(req);
242	struct crypto_aead *aead = crypto_aead_reqtfm(req);
243	unsigned int authlen = crypto_aead_authsize(aead);
244	struct omap_aes_dev *dd;
245	__be32 counter = cpu_to_be32(1);
246	int err, assoclen;
247
248	memset(rctx->auth_tag, 0, sizeof(rctx->auth_tag));
249	memcpy(rctx->iv + GCM_AES_IV_SIZE, &counter, 4);
250
251	err = do_encrypt_iv(req, (u32 *)rctx->auth_tag, (u32 *)rctx->iv);
252	if (err)
253		return err;
254
255	if (mode & FLAGS_RFC4106_GCM)
256		assoclen = req->assoclen - 8;
257	else
258		assoclen = req->assoclen;
259	if (assoclen + req->cryptlen == 0) {
260		scatterwalk_map_and_copy(rctx->auth_tag, req->dst, 0, authlen,
261					 1);
262		return 0;
263	}
264
265	dd = omap_aes_find_dev(rctx);
266	if (!dd)
267		return -ENODEV;
268	rctx->mode = mode;
269
270	return omap_aes_gcm_handle_queue(dd, req);
271}
272
273int omap_aes_gcm_encrypt(struct aead_request *req)
274{
275	struct omap_aes_reqctx *rctx = aead_request_ctx(req);
276
277	memcpy(rctx->iv, req->iv, GCM_AES_IV_SIZE);
278	return omap_aes_gcm_crypt(req, FLAGS_ENCRYPT | FLAGS_GCM);
279}
280
281int omap_aes_gcm_decrypt(struct aead_request *req)
282{
283	struct omap_aes_reqctx *rctx = aead_request_ctx(req);
284
285	memcpy(rctx->iv, req->iv, GCM_AES_IV_SIZE);
286	return omap_aes_gcm_crypt(req, FLAGS_GCM);
287}
288
289int omap_aes_4106gcm_encrypt(struct aead_request *req)
290{
291	struct omap_aes_gcm_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
292	struct omap_aes_reqctx *rctx = aead_request_ctx(req);
293
294	memcpy(rctx->iv, ctx->octx.nonce, 4);
295	memcpy(rctx->iv + 4, req->iv, 8);
296	return crypto_ipsec_check_assoclen(req->assoclen) ?:
297	       omap_aes_gcm_crypt(req, FLAGS_ENCRYPT | FLAGS_GCM |
298				  FLAGS_RFC4106_GCM);
299}
300
301int omap_aes_4106gcm_decrypt(struct aead_request *req)
302{
303	struct omap_aes_gcm_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
304	struct omap_aes_reqctx *rctx = aead_request_ctx(req);
305
306	memcpy(rctx->iv, ctx->octx.nonce, 4);
307	memcpy(rctx->iv + 4, req->iv, 8);
308	return crypto_ipsec_check_assoclen(req->assoclen) ?:
309	       omap_aes_gcm_crypt(req, FLAGS_GCM | FLAGS_RFC4106_GCM);
310}
311
312int omap_aes_gcm_setkey(struct crypto_aead *tfm, const u8 *key,
313			unsigned int keylen)
314{
315	struct omap_aes_gcm_ctx *ctx = crypto_aead_ctx(tfm);
316	int ret;
317
318	ret = aes_expandkey(&ctx->actx, key, keylen);
319	if (ret)
320		return ret;
321
322	memcpy(ctx->octx.key, key, keylen);
323	ctx->octx.keylen = keylen;
324
325	return 0;
326}
327
328int omap_aes_4106gcm_setkey(struct crypto_aead *tfm, const u8 *key,
329			    unsigned int keylen)
330{
331	struct omap_aes_gcm_ctx *ctx = crypto_aead_ctx(tfm);
332	int ret;
333
334	if (keylen < 4)
335		return -EINVAL;
336	keylen -= 4;
337
338	ret = aes_expandkey(&ctx->actx, key, keylen);
339	if (ret)
340		return ret;
341
342	memcpy(ctx->octx.key, key, keylen);
343	memcpy(ctx->octx.nonce, key + keylen, 4);
344	ctx->octx.keylen = keylen;
345
346	return 0;
347}
348
349int omap_aes_gcm_setauthsize(struct crypto_aead *tfm, unsigned int authsize)
350{
351	return crypto_gcm_check_authsize(authsize);
352}
353
354int omap_aes_4106gcm_setauthsize(struct crypto_aead *parent,
355				 unsigned int authsize)
356{
357	return crypto_rfc4106_check_authsize(authsize);
358}
359
360int omap_aes_gcm_crypt_req(struct crypto_engine *engine, void *areq)
361{
362	struct aead_request *req = container_of(areq, struct aead_request,
363						base);
364	struct omap_aes_reqctx *rctx = aead_request_ctx(req);
365	struct omap_aes_dev *dd = rctx->dd;
366	int ret;
367
368	if (!dd)
369		return -ENODEV;
370
371	ret = omap_aes_gcm_prepare_req(req, dd);
372	if (ret)
373		return ret;
374
375	if (dd->in_sg_len)
376		ret = omap_aes_crypt_dma_start(dd);
377	else
378		omap_aes_gcm_dma_out_callback(dd);
379
380	return ret;
381}
382
383int omap_aes_gcm_cra_init(struct crypto_aead *tfm)
384{
385	crypto_aead_set_reqsize(tfm, sizeof(struct omap_aes_reqctx));
386
387	return 0;
388}
389