1// SPDX-License-Identifier: GPL-2.0
2/*
3 * StarFive Public Key Algo acceleration driver
4 *
5 * Copyright (c) 2022 StarFive Technology
6 */
7
8#include <linux/crypto.h>
9#include <linux/iopoll.h>
10#include <crypto/akcipher.h>
11#include <crypto/algapi.h>
12#include <crypto/internal/akcipher.h>
13#include <crypto/internal/rsa.h>
14#include <crypto/scatterwalk.h>
15
16#include "jh7110-cryp.h"
17
18#define STARFIVE_PKA_REGS_OFFSET	0x400
19#define STARFIVE_PKA_CACR_OFFSET	(STARFIVE_PKA_REGS_OFFSET + 0x0)
20#define STARFIVE_PKA_CASR_OFFSET	(STARFIVE_PKA_REGS_OFFSET + 0x4)
21#define STARFIVE_PKA_CAAR_OFFSET	(STARFIVE_PKA_REGS_OFFSET + 0x8)
22#define STARFIVE_PKA_CAER_OFFSET	(STARFIVE_PKA_REGS_OFFSET + 0x108)
23#define STARFIVE_PKA_CANR_OFFSET	(STARFIVE_PKA_REGS_OFFSET + 0x208)
24
25/* R ^ 2 mod N and N0' */
26#define CRYPTO_CMD_PRE			0x0
27/* A * R mod N   ==> A */
28#define CRYPTO_CMD_ARN			0x5
29/* A * E * R mod N ==> A */
30#define CRYPTO_CMD_AERN			0x6
31/* A * A * R mod N ==> A */
32#define CRYPTO_CMD_AARN			0x7
33
34#define STARFIVE_RSA_MAX_KEYSZ		256
35#define STARFIVE_RSA_RESET		0x2
36
37static inline int starfive_pka_wait_done(struct starfive_cryp_ctx *ctx)
38{
39	struct starfive_cryp_dev *cryp = ctx->cryp;
40	u32 status;
41
42	return readl_relaxed_poll_timeout(cryp->base + STARFIVE_PKA_CASR_OFFSET, status,
43					  status & STARFIVE_PKA_DONE, 10, 100000);
44}
45
46static void starfive_rsa_free_key(struct starfive_rsa_key *key)
47{
48	kfree_sensitive(key->d);
49	kfree_sensitive(key->e);
50	kfree_sensitive(key->n);
51	memset(key, 0, sizeof(*key));
52}
53
54static unsigned int starfive_rsa_get_nbit(u8 *pa, u32 snum, int key_sz)
55{
56	u32 i;
57	u8 value;
58
59	i = snum >> 3;
60
61	value = pa[key_sz - i - 1];
62	value >>= snum & 0x7;
63	value &= 0x1;
64
65	return value;
66}
67
68static int starfive_rsa_montgomery_form(struct starfive_cryp_ctx *ctx,
69					u32 *out, u32 *in, u8 mont,
70					u32 *mod, int bit_len)
71{
72	struct starfive_cryp_dev *cryp = ctx->cryp;
73	struct starfive_cryp_request_ctx *rctx = ctx->rctx;
74	int count = rctx->total / sizeof(u32) - 1;
75	int loop;
76	u32 temp;
77	u8 opsize;
78
79	opsize = (bit_len - 1) >> 5;
80	rctx->csr.pka.v = 0;
81
82	writel(rctx->csr.pka.v, cryp->base + STARFIVE_PKA_CACR_OFFSET);
83
84	for (loop = 0; loop <= opsize; loop++)
85		writel(mod[opsize - loop], cryp->base + STARFIVE_PKA_CANR_OFFSET + loop * 4);
86
87	if (mont) {
88		rctx->csr.pka.v = 0;
89		rctx->csr.pka.cln_done = 1;
90		rctx->csr.pka.opsize = opsize;
91		rctx->csr.pka.exposize = opsize;
92		rctx->csr.pka.cmd = CRYPTO_CMD_PRE;
93		rctx->csr.pka.start = 1;
94		rctx->csr.pka.not_r2 = 1;
95		rctx->csr.pka.ie = 1;
96
97		writel(rctx->csr.pka.v, cryp->base + STARFIVE_PKA_CACR_OFFSET);
98
99		if (starfive_pka_wait_done(ctx))
100			return -ETIMEDOUT;
101
102		for (loop = 0; loop <= opsize; loop++)
103			writel(in[opsize - loop], cryp->base + STARFIVE_PKA_CAAR_OFFSET + loop * 4);
104
105		writel(0x1000000, cryp->base + STARFIVE_PKA_CAER_OFFSET);
106
107		for (loop = 1; loop <= opsize; loop++)
108			writel(0, cryp->base + STARFIVE_PKA_CAER_OFFSET + loop * 4);
109
110		rctx->csr.pka.v = 0;
111		rctx->csr.pka.cln_done = 1;
112		rctx->csr.pka.opsize = opsize;
113		rctx->csr.pka.exposize = opsize;
114		rctx->csr.pka.cmd = CRYPTO_CMD_AERN;
115		rctx->csr.pka.start = 1;
116		rctx->csr.pka.ie = 1;
117
118		writel(rctx->csr.pka.v, cryp->base + STARFIVE_PKA_CACR_OFFSET);
119
120		if (starfive_pka_wait_done(ctx))
121			return -ETIMEDOUT;
122	} else {
123		rctx->csr.pka.v = 0;
124		rctx->csr.pka.cln_done = 1;
125		rctx->csr.pka.opsize = opsize;
126		rctx->csr.pka.exposize = opsize;
127		rctx->csr.pka.cmd = CRYPTO_CMD_PRE;
128		rctx->csr.pka.start = 1;
129		rctx->csr.pka.pre_expf = 1;
130		rctx->csr.pka.ie = 1;
131
132		writel(rctx->csr.pka.v, cryp->base + STARFIVE_PKA_CACR_OFFSET);
133
134		if (starfive_pka_wait_done(ctx))
135			return -ETIMEDOUT;
136
137		for (loop = 0; loop <= count; loop++)
138			writel(in[count - loop], cryp->base + STARFIVE_PKA_CAER_OFFSET + loop * 4);
139
140		/*pad with 0 up to opsize*/
141		for (loop = count + 1; loop <= opsize; loop++)
142			writel(0, cryp->base + STARFIVE_PKA_CAER_OFFSET + loop * 4);
143
144		rctx->csr.pka.v = 0;
145		rctx->csr.pka.cln_done = 1;
146		rctx->csr.pka.opsize = opsize;
147		rctx->csr.pka.exposize = opsize;
148		rctx->csr.pka.cmd = CRYPTO_CMD_ARN;
149		rctx->csr.pka.start = 1;
150		rctx->csr.pka.ie = 1;
151
152		writel(rctx->csr.pka.v, cryp->base + STARFIVE_PKA_CACR_OFFSET);
153
154		if (starfive_pka_wait_done(ctx))
155			return -ETIMEDOUT;
156	}
157
158	for (loop = 0; loop <= opsize; loop++) {
159		temp = readl(cryp->base + STARFIVE_PKA_CAAR_OFFSET + 0x4 * loop);
160		out[opsize - loop] = temp;
161	}
162
163	return 0;
164}
165
166static int starfive_rsa_cpu_start(struct starfive_cryp_ctx *ctx, u32 *result,
167				  u8 *de, u32 *n, int key_sz)
168{
169	struct starfive_cryp_dev *cryp = ctx->cryp;
170	struct starfive_cryp_request_ctx *rctx = ctx->rctx;
171	struct starfive_rsa_key *key = &ctx->rsa_key;
172	u32 temp;
173	int ret = 0;
174	int opsize, mlen, loop;
175	unsigned int *mta;
176
177	opsize = (key_sz - 1) >> 2;
178
179	mta = kmalloc(key_sz, GFP_KERNEL);
180	if (!mta)
181		return -ENOMEM;
182
183	ret = starfive_rsa_montgomery_form(ctx, mta, (u32 *)rctx->rsa_data,
184					   0, n, key_sz << 3);
185	if (ret) {
186		dev_err_probe(cryp->dev, ret, "Conversion to Montgomery failed");
187		goto rsa_err;
188	}
189
190	for (loop = 0; loop <= opsize; loop++)
191		writel(mta[opsize - loop],
192		       cryp->base + STARFIVE_PKA_CAER_OFFSET + loop * 4);
193
194	for (loop = key->bitlen - 1; loop > 0; loop--) {
195		mlen = starfive_rsa_get_nbit(de, loop - 1, key_sz);
196
197		rctx->csr.pka.v = 0;
198		rctx->csr.pka.cln_done = 1;
199		rctx->csr.pka.opsize = opsize;
200		rctx->csr.pka.exposize = opsize;
201		rctx->csr.pka.cmd = CRYPTO_CMD_AARN;
202		rctx->csr.pka.start = 1;
203		rctx->csr.pka.ie = 1;
204
205		writel(rctx->csr.pka.v, cryp->base + STARFIVE_PKA_CACR_OFFSET);
206
207		ret = -ETIMEDOUT;
208		if (starfive_pka_wait_done(ctx))
209			goto rsa_err;
210
211		if (mlen) {
212			rctx->csr.pka.v = 0;
213			rctx->csr.pka.cln_done = 1;
214			rctx->csr.pka.opsize = opsize;
215			rctx->csr.pka.exposize = opsize;
216			rctx->csr.pka.cmd = CRYPTO_CMD_AERN;
217			rctx->csr.pka.start = 1;
218			rctx->csr.pka.ie = 1;
219
220			writel(rctx->csr.pka.v, cryp->base + STARFIVE_PKA_CACR_OFFSET);
221
222			if (starfive_pka_wait_done(ctx))
223				goto rsa_err;
224		}
225	}
226
227	for (loop = 0; loop <= opsize; loop++) {
228		temp = readl(cryp->base + STARFIVE_PKA_CAAR_OFFSET + 0x4 * loop);
229		result[opsize - loop] = temp;
230	}
231
232	ret = starfive_rsa_montgomery_form(ctx, result, result, 1, n, key_sz << 3);
233	if (ret)
234		dev_err_probe(cryp->dev, ret, "Conversion from Montgomery failed");
235rsa_err:
236	kfree(mta);
237	return ret;
238}
239
240static int starfive_rsa_start(struct starfive_cryp_ctx *ctx, u8 *result,
241			      u8 *de, u8 *n, int key_sz)
242{
243	return starfive_rsa_cpu_start(ctx, (u32 *)result, de, (u32 *)n, key_sz);
244}
245
246static int starfive_rsa_enc_core(struct starfive_cryp_ctx *ctx, int enc)
247{
248	struct starfive_cryp_dev *cryp = ctx->cryp;
249	struct starfive_cryp_request_ctx *rctx = ctx->rctx;
250	struct starfive_rsa_key *key = &ctx->rsa_key;
251	int ret = 0;
252
253	writel(STARFIVE_RSA_RESET, cryp->base + STARFIVE_PKA_CACR_OFFSET);
254
255	rctx->total = sg_copy_to_buffer(rctx->in_sg, rctx->nents,
256					rctx->rsa_data, rctx->total);
257
258	if (enc) {
259		key->bitlen = key->e_bitlen;
260		ret = starfive_rsa_start(ctx, rctx->rsa_data, key->e,
261					 key->n, key->key_sz);
262	} else {
263		key->bitlen = key->d_bitlen;
264		ret = starfive_rsa_start(ctx, rctx->rsa_data, key->d,
265					 key->n, key->key_sz);
266	}
267
268	if (ret)
269		goto err_rsa_crypt;
270
271	sg_copy_buffer(rctx->out_sg, sg_nents(rctx->out_sg),
272		       rctx->rsa_data, key->key_sz, 0, 0);
273
274err_rsa_crypt:
275	writel(STARFIVE_RSA_RESET, cryp->base + STARFIVE_PKA_CACR_OFFSET);
276	kfree(rctx->rsa_data);
277	return ret;
278}
279
280static int starfive_rsa_enc(struct akcipher_request *req)
281{
282	struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req);
283	struct starfive_cryp_ctx *ctx = akcipher_tfm_ctx(tfm);
284	struct starfive_cryp_dev *cryp = ctx->cryp;
285	struct starfive_rsa_key *key = &ctx->rsa_key;
286	struct starfive_cryp_request_ctx *rctx = akcipher_request_ctx(req);
287	int ret;
288
289	if (!key->key_sz) {
290		akcipher_request_set_tfm(req, ctx->akcipher_fbk);
291		ret = crypto_akcipher_encrypt(req);
292		akcipher_request_set_tfm(req, tfm);
293		return ret;
294	}
295
296	if (unlikely(!key->n || !key->e))
297		return -EINVAL;
298
299	if (req->dst_len < key->key_sz)
300		return dev_err_probe(cryp->dev, -EOVERFLOW,
301				     "Output buffer length less than parameter n\n");
302
303	rctx->in_sg = req->src;
304	rctx->out_sg = req->dst;
305	rctx->total = req->src_len;
306	rctx->nents = sg_nents(rctx->in_sg);
307	ctx->rctx = rctx;
308
309	return starfive_rsa_enc_core(ctx, 1);
310}
311
312static int starfive_rsa_dec(struct akcipher_request *req)
313{
314	struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req);
315	struct starfive_cryp_ctx *ctx = akcipher_tfm_ctx(tfm);
316	struct starfive_cryp_dev *cryp = ctx->cryp;
317	struct starfive_rsa_key *key = &ctx->rsa_key;
318	struct starfive_cryp_request_ctx *rctx = akcipher_request_ctx(req);
319	int ret;
320
321	if (!key->key_sz) {
322		akcipher_request_set_tfm(req, ctx->akcipher_fbk);
323		ret = crypto_akcipher_decrypt(req);
324		akcipher_request_set_tfm(req, tfm);
325		return ret;
326	}
327
328	if (unlikely(!key->n || !key->d))
329		return -EINVAL;
330
331	if (req->dst_len < key->key_sz)
332		return dev_err_probe(cryp->dev, -EOVERFLOW,
333				     "Output buffer length less than parameter n\n");
334
335	rctx->in_sg = req->src;
336	rctx->out_sg = req->dst;
337	ctx->rctx = rctx;
338	rctx->total = req->src_len;
339
340	return starfive_rsa_enc_core(ctx, 0);
341}
342
343static int starfive_rsa_set_n(struct starfive_rsa_key *rsa_key,
344			      const char *value, size_t vlen)
345{
346	const char *ptr = value;
347	unsigned int bitslen;
348	int ret;
349
350	while (!*ptr && vlen) {
351		ptr++;
352		vlen--;
353	}
354	rsa_key->key_sz = vlen;
355	bitslen = rsa_key->key_sz << 3;
356
357	/* check valid key size */
358	if (bitslen & 0x1f)
359		return -EINVAL;
360
361	ret = -ENOMEM;
362	rsa_key->n = kmemdup(ptr, rsa_key->key_sz, GFP_KERNEL);
363	if (!rsa_key->n)
364		goto err;
365
366	return 0;
367 err:
368	rsa_key->key_sz = 0;
369	rsa_key->n = NULL;
370	starfive_rsa_free_key(rsa_key);
371	return ret;
372}
373
374static int starfive_rsa_set_e(struct starfive_rsa_key *rsa_key,
375			      const char *value, size_t vlen)
376{
377	const char *ptr = value;
378	unsigned char pt;
379	int loop;
380
381	while (!*ptr && vlen) {
382		ptr++;
383		vlen--;
384	}
385	pt = *ptr;
386
387	if (!rsa_key->key_sz || !vlen || vlen > rsa_key->key_sz) {
388		rsa_key->e = NULL;
389		return -EINVAL;
390	}
391
392	rsa_key->e = kzalloc(rsa_key->key_sz, GFP_KERNEL);
393	if (!rsa_key->e)
394		return -ENOMEM;
395
396	for (loop = 8; loop > 0; loop--) {
397		if (pt >> (loop - 1))
398			break;
399	}
400
401	rsa_key->e_bitlen = (vlen - 1) * 8 + loop;
402
403	memcpy(rsa_key->e + (rsa_key->key_sz - vlen), ptr, vlen);
404
405	return 0;
406}
407
408static int starfive_rsa_set_d(struct starfive_rsa_key *rsa_key,
409			      const char *value, size_t vlen)
410{
411	const char *ptr = value;
412	unsigned char pt;
413	int loop;
414	int ret;
415
416	while (!*ptr && vlen) {
417		ptr++;
418		vlen--;
419	}
420	pt = *ptr;
421
422	ret = -EINVAL;
423	if (!rsa_key->key_sz || !vlen || vlen > rsa_key->key_sz)
424		goto err;
425
426	ret = -ENOMEM;
427	rsa_key->d = kzalloc(rsa_key->key_sz, GFP_KERNEL);
428	if (!rsa_key->d)
429		goto err;
430
431	for (loop = 8; loop > 0; loop--) {
432		if (pt >> (loop - 1))
433			break;
434	}
435
436	rsa_key->d_bitlen = (vlen - 1) * 8 + loop;
437
438	memcpy(rsa_key->d + (rsa_key->key_sz - vlen), ptr, vlen);
439
440	return 0;
441 err:
442	rsa_key->d = NULL;
443	return ret;
444}
445
446static int starfive_rsa_setkey(struct crypto_akcipher *tfm, const void *key,
447			       unsigned int keylen, bool private)
448{
449	struct starfive_cryp_ctx *ctx = akcipher_tfm_ctx(tfm);
450	struct rsa_key raw_key = {NULL};
451	struct starfive_rsa_key *rsa_key = &ctx->rsa_key;
452	int ret;
453
454	if (private)
455		ret = rsa_parse_priv_key(&raw_key, key, keylen);
456	else
457		ret = rsa_parse_pub_key(&raw_key, key, keylen);
458	if (ret < 0)
459		goto err;
460
461	starfive_rsa_free_key(rsa_key);
462
463	/* Use fallback for mod > 256 + 1 byte prefix */
464	if (raw_key.n_sz > STARFIVE_RSA_MAX_KEYSZ + 1)
465		return 0;
466
467	ret = starfive_rsa_set_n(rsa_key, raw_key.n, raw_key.n_sz);
468	if (ret)
469		return ret;
470
471	ret = starfive_rsa_set_e(rsa_key, raw_key.e, raw_key.e_sz);
472	if (ret)
473		goto err;
474
475	if (private) {
476		ret = starfive_rsa_set_d(rsa_key, raw_key.d, raw_key.d_sz);
477		if (ret)
478			goto err;
479	}
480
481	if (!rsa_key->n || !rsa_key->e) {
482		ret = -EINVAL;
483		goto err;
484	}
485
486	if (private && !rsa_key->d) {
487		ret = -EINVAL;
488		goto err;
489	}
490
491	return 0;
492 err:
493	starfive_rsa_free_key(rsa_key);
494	return ret;
495}
496
497static int starfive_rsa_set_pub_key(struct crypto_akcipher *tfm, const void *key,
498				    unsigned int keylen)
499{
500	struct starfive_cryp_ctx *ctx = akcipher_tfm_ctx(tfm);
501	int ret;
502
503	ret = crypto_akcipher_set_pub_key(ctx->akcipher_fbk, key, keylen);
504	if (ret)
505		return ret;
506
507	return starfive_rsa_setkey(tfm, key, keylen, false);
508}
509
510static int starfive_rsa_set_priv_key(struct crypto_akcipher *tfm, const void *key,
511				     unsigned int keylen)
512{
513	struct starfive_cryp_ctx *ctx = akcipher_tfm_ctx(tfm);
514	int ret;
515
516	ret = crypto_akcipher_set_priv_key(ctx->akcipher_fbk, key, keylen);
517	if (ret)
518		return ret;
519
520	return starfive_rsa_setkey(tfm, key, keylen, true);
521}
522
523static unsigned int starfive_rsa_max_size(struct crypto_akcipher *tfm)
524{
525	struct starfive_cryp_ctx *ctx = akcipher_tfm_ctx(tfm);
526
527	if (ctx->rsa_key.key_sz)
528		return ctx->rsa_key.key_sz;
529
530	return crypto_akcipher_maxsize(ctx->akcipher_fbk);
531}
532
533static int starfive_rsa_init_tfm(struct crypto_akcipher *tfm)
534{
535	struct starfive_cryp_ctx *ctx = akcipher_tfm_ctx(tfm);
536
537	ctx->akcipher_fbk = crypto_alloc_akcipher("rsa-generic", 0, 0);
538	if (IS_ERR(ctx->akcipher_fbk))
539		return PTR_ERR(ctx->akcipher_fbk);
540
541	ctx->cryp = starfive_cryp_find_dev(ctx);
542	if (!ctx->cryp) {
543		crypto_free_akcipher(ctx->akcipher_fbk);
544		return -ENODEV;
545	}
546
547	akcipher_set_reqsize(tfm, sizeof(struct starfive_cryp_request_ctx) +
548			     sizeof(struct crypto_akcipher) + 32);
549
550	return 0;
551}
552
553static void starfive_rsa_exit_tfm(struct crypto_akcipher *tfm)
554{
555	struct starfive_cryp_ctx *ctx = akcipher_tfm_ctx(tfm);
556	struct starfive_rsa_key *key = (struct starfive_rsa_key *)&ctx->rsa_key;
557
558	crypto_free_akcipher(ctx->akcipher_fbk);
559	starfive_rsa_free_key(key);
560}
561
562static struct akcipher_alg starfive_rsa = {
563	.encrypt = starfive_rsa_enc,
564	.decrypt = starfive_rsa_dec,
565	.sign = starfive_rsa_dec,
566	.verify = starfive_rsa_enc,
567	.set_pub_key = starfive_rsa_set_pub_key,
568	.set_priv_key = starfive_rsa_set_priv_key,
569	.max_size = starfive_rsa_max_size,
570	.init = starfive_rsa_init_tfm,
571	.exit = starfive_rsa_exit_tfm,
572	.base = {
573		.cra_name = "rsa",
574		.cra_driver_name = "starfive-rsa",
575		.cra_flags = CRYPTO_ALG_TYPE_AKCIPHER |
576			     CRYPTO_ALG_NEED_FALLBACK,
577		.cra_priority = 3000,
578		.cra_module = THIS_MODULE,
579		.cra_ctxsize = sizeof(struct starfive_cryp_ctx),
580	},
581};
582
583int starfive_rsa_register_algs(void)
584{
585	return crypto_register_akcipher(&starfive_rsa);
586}
587
588void starfive_rsa_unregister_algs(void)
589{
590	crypto_unregister_akcipher(&starfive_rsa);
591}
592