crypto/starfive/jh7110-aes.c

13 #include "jh7110-cryp.h"
52 static inline int starfive_aes_wait_busy(struct starfive_cryp_dev *cryp)
56 	return readl_relaxed_poll_timeout(cryp->base + STARFIVE_AES_CSR, status,
60 static inline int starfive_aes_wait_keydone(struct starfive_cryp_dev *cryp)
64 	return readl_relaxed_poll_timeout(cryp->base + STARFIVE_AES_CSR, status,
68 static inline int starfive_aes_wait_gcmdone(struct starfive_cryp_dev *cryp)
72 	return readl_relaxed_poll_timeout(cryp->base + STARFIVE_AES_CSR, status,
76 static inline int is_gcm(struct starfive_cryp_dev *cryp)
78 	return (cryp->flags & FLG_MODE_MASK) == STARFIVE_AES_MODE_GCM;
81 static inline bool is_encrypt(struct starfive_cryp_dev *cryp)
83 	return cryp->flags & FLG_ENCRYPT;
88 	struct starfive_cryp_dev *cryp = ctx->cryp;
93 		value = readl(ctx->cryp->base + STARFIVE_AES_CSR);
95 		writel(value, cryp->base + STARFIVE_AES_CSR);
96 		starfive_aes_wait_gcmdone(cryp);
99 		value = readl(ctx->cryp->base + STARFIVE_AES_CSR);
101 		writel(value, cryp->base + STARFIVE_AES_CSR);
108 	struct starfive_cryp_dev *cryp = ctx->cryp;
110 	writel(upper_32_bits(cryp->assoclen), cryp->base + STARFIVE_AES_ALEN0);
111 	writel(lower_32_bits(cryp->assoclen), cryp->base + STARFIVE_AES_ALEN1);
116 	struct starfive_cryp_dev *cryp = ctx->cryp;
118 	writel(upper_32_bits(cryp->total_in), cryp->base + STARFIVE_AES_MLEN0);
119 	writel(lower_32_bits(cryp->total_in), cryp->base + STARFIVE_AES_MLEN1);
133 	struct starfive_cryp_dev *cryp = ctx->cryp;
135 	writel(iv[0], cryp->base + STARFIVE_AES_IV0);
136 	writel(iv[1], cryp->base + STARFIVE_AES_IV1);
137 	writel(iv[2], cryp->base + STARFIVE_AES_IV2);
139 	if (is_gcm(cryp)) {
140 		if (starfive_aes_wait_gcmdone(cryp))
146 	writel(iv[3], cryp->base + STARFIVE_AES_IV3);
151 static inline void starfive_aes_get_iv(struct starfive_cryp_dev *cryp, u32 *iv)
153 	iv[0] = readl(cryp->base + STARFIVE_AES_IV0);
154 	iv[1] = readl(cryp->base + STARFIVE_AES_IV1);
155 	iv[2] = readl(cryp->base + STARFIVE_AES_IV2);
156 	iv[3] = readl(cryp->base + STARFIVE_AES_IV3);
161 	struct starfive_cryp_dev *cryp = ctx->cryp;
163 	writel(nonce[0], cryp->base + STARFIVE_AES_NONCE0);
164 	writel(nonce[1], cryp->base + STARFIVE_AES_NONCE1);
165 	writel(nonce[2], cryp->base + STARFIVE_AES_NONCE2);
166 	writel(nonce[3], cryp->base + STARFIVE_AES_NONCE3);
171 	struct starfive_cryp_dev *cryp = ctx->cryp;
175 		writel(key[0], cryp->base + STARFIVE_AES_KEY0);
176 		writel(key[1], cryp->base + STARFIVE_AES_KEY1);
177 		writel(key[2], cryp->base + STARFIVE_AES_KEY2);
178 		writel(key[3], cryp->base + STARFIVE_AES_KEY3);
182 		writel(key[4], cryp->base + STARFIVE_AES_KEY4);
183 		writel(key[5], cryp->base + STARFIVE_AES_KEY5);
187 		writel(key[6], cryp->base + STARFIVE_AES_KEY6);
188 		writel(key[7], cryp->base + STARFIVE_AES_KEY7);
191 	if (starfive_aes_wait_keydone(cryp))
199 	struct starfive_cryp_dev *cryp = ctx->cryp;
203 	memcpy(iv, cryp->req.areq->iv, AES_BLOCK_SIZE);
209 	b0[0] |= (8 * ((cryp->authsize - 2) / 2));
211 	if (cryp->assoclen)
214 	textlen = cryp->total_in;
227 	struct starfive_cryp_dev *cryp = ctx->cryp;
233 	writel(rctx->csr.aes.v, cryp->base + STARFIVE_AES_CSR);
236 	hw_mode = cryp->flags & FLG_MODE_MASK;
253 	rctx->csr.aes.cmode = !is_encrypt(cryp);
256 	if (cryp->side_chan) {
261 	writel(rctx->csr.aes.v, cryp->base + STARFIVE_AES_CSR);
263 	cryp->err = starfive_aes_write_key(ctx);
264 	if (cryp->err)
265 		return cryp->err;
271 		writel(GCM_AES_IV_SIZE, cryp->base + STARFIVE_AES_IVLEN);
273 		starfive_aes_write_iv(ctx, (void *)cryp->req.areq->iv);
283 		starfive_aes_write_iv(ctx, (void *)cryp->req.sreq->iv);
289 	return cryp->err;
294 	struct starfive_cryp_dev *cryp = ctx->cryp;
298 	if (starfive_aes_wait_busy(cryp))
299 		return dev_err_probe(cryp->dev, -ETIMEDOUT,
302 	if ((cryp->flags & FLG_MODE_MASK) == STARFIVE_AES_MODE_GCM) {
303 		cryp->tag_out[0] = readl(cryp->base + STARFIVE_AES_NONCE0);
304 		cryp->tag_out[1] = readl(cryp->base + STARFIVE_AES_NONCE1);
305 		cryp->tag_out[2] = readl(cryp->base + STARFIVE_AES_NONCE2);
306 		cryp->tag_out[3] = readl(cryp->base + STARFIVE_AES_NONCE3);
309 			cryp->tag_out[i] = readl(cryp->base + STARFIVE_AES_AESDIO0R);
312 	if (is_encrypt(cryp)) {
313 		scatterwalk_map_and_copy(cryp->tag_out, rctx->out_sg,
314 					 cryp->total_in, cryp->authsize, 1);
316 		if (crypto_memneq(cryp->tag_in, cryp->tag_out, cryp->authsize))
325 	struct starfive_cryp_dev *cryp = ctx->cryp;
326 	int err = cryp->err;
328 	if (!err && cryp->authsize)
331 	if (!err && ((cryp->flags & FLG_MODE_MASK) == STARFIVE_AES_MODE_CBC ||
332 		     (cryp->flags & FLG_MODE_MASK) == STARFIVE_AES_MODE_CTR))
333 		starfive_aes_get_iv(cryp, (void *)cryp->req.sreq->iv);
335 	if (cryp->authsize)
336 		crypto_finalize_aead_request(cryp->engine, cryp->req.areq, err);
338 		crypto_finalize_skcipher_request(cryp->engine, cryp->req.sreq,
344 	struct starfive_cryp_dev *cryp = ctx->cryp;
349 	total_len = ALIGN(cryp->assoclen, AES_BLOCK_SIZE) / sizeof(unsigned int);
353 		writel(*buffer, cryp->base + STARFIVE_AES_NONCE0);
355 		writel(*buffer, cryp->base + STARFIVE_AES_NONCE1);
357 		writel(*buffer, cryp->base + STARFIVE_AES_NONCE2);
359 		writel(*buffer, cryp->base + STARFIVE_AES_NONCE3);
363 	if (starfive_aes_wait_gcmdone(cryp))
364 		return dev_err_probe(cryp->dev, -ETIMEDOUT,
372 	struct starfive_cryp_dev *cryp = ctx->cryp;
378 	total_len = cryp->assoclen;
381 	writeb(*ci, cryp->base + STARFIVE_AES_AESDIO0R);
383 	writeb(*ci, cryp->base + STARFIVE_AES_AESDIO0R);
389 		writel(*buffer, cryp->base + STARFIVE_AES_AESDIO0R);
395 			writel(*buffer, cryp->base + STARFIVE_AES_AESDIO0R);
400 	if (starfive_aes_wait_busy(cryp))
401 		return dev_err_probe(cryp->dev, -ETIMEDOUT,
409 	struct starfive_cryp_dev *cryp = param;
411 	complete(&cryp->dma_done);
414 static void starfive_aes_dma_init(struct starfive_cryp_dev *cryp)
416 	cryp->cfg_in.direction = DMA_MEM_TO_DEV;
417 	cryp->cfg_in.src_addr_width = DMA_SLAVE_BUSWIDTH_16_BYTES;
418 	cryp->cfg_in.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
419 	cryp->cfg_in.src_maxburst = cryp->dma_maxburst;
420 	cryp->cfg_in.dst_maxburst = cryp->dma_maxburst;
421 	cryp->cfg_in.dst_addr = cryp->phys_base + STARFIVE_ALG_FIFO_OFFSET;
423 	dmaengine_slave_config(cryp->tx, &cryp->cfg_in);
425 	cryp->cfg_out.direction = DMA_DEV_TO_MEM;
426 	cryp->cfg_out.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
427 	cryp->cfg_out.dst_addr_width = DMA_SLAVE_BUSWIDTH_16_BYTES;
428 	cryp->cfg_out.src_maxburst = 4;
429 	cryp->cfg_out.dst_maxburst = 4;
430 	cryp->cfg_out.src_addr = cryp->phys_base + STARFIVE_ALG_FIFO_OFFSET;
432 	dmaengine_slave_config(cryp->rx, &cryp->cfg_out);
434 	init_completion(&cryp->dma_done);
437 static int starfive_aes_dma_xfer(struct starfive_cryp_dev *cryp,
449 	writel(alg_cr.v, cryp->base + STARFIVE_ALG_CR_OFFSET);
454 	writel(ALIGN(len, AES_BLOCK_SIZE), cryp->base + STARFIVE_DMA_IN_LEN_OFFSET);
455 	writel(ALIGN(len, AES_BLOCK_SIZE), cryp->base + STARFIVE_DMA_OUT_LEN_OFFSET);
460 	out_desc = dmaengine_prep_slave_sg(cryp->rx, dst, 1, DMA_DEV_TO_MEM,
468 	out_desc->callback_param = cryp;
470 	reinit_completion(&cryp->dma_done);
472 	dma_async_issue_pending(cryp->rx);
474 	in_desc = dmaengine_prep_slave_sg(cryp->tx, src, 1, DMA_MEM_TO_DEV,
482 	dma_async_issue_pending(cryp->tx);
484 	if (!wait_for_completion_timeout(&cryp->dma_done,
494 	writel(alg_cr.v, cryp->base + STARFIVE_ALG_CR_OFFSET);
499 static int starfive_aes_map_sg(struct starfive_cryp_dev *cryp,
505 	unsigned int remain = cryp->total_in;
512 			src_nents = dma_map_sg(cryp->dev, stsg, 1, DMA_BIDIRECTIONAL);
514 				return dev_err_probe(cryp->dev, -ENOMEM,
520 			ret = starfive_aes_dma_xfer(cryp, stsg, dtsg, len);
521 			dma_unmap_sg(cryp->dev, stsg, 1, DMA_BIDIRECTIONAL);
529 			src_nents = dma_map_sg(cryp->dev, stsg, 1, DMA_TO_DEVICE);
531 				return dev_err_probe(cryp->dev, -ENOMEM,
534 			dst_nents = dma_map_sg(cryp->dev, dtsg, 1, DMA_FROM_DEVICE);
536 				return dev_err_probe(cryp->dev, -ENOMEM,
542 			ret = starfive_aes_dma_xfer(cryp, stsg, dtsg, len);
543 			dma_unmap_sg(cryp->dev, stsg, 1, DMA_TO_DEVICE);
544 			dma_unmap_sg(cryp->dev, dtsg, 1, DMA_FROM_DEVICE);
575 	struct starfive_cryp_dev *cryp = ctx->cryp;
578 	cryp->req.sreq = req;
579 	cryp->total_in = req->cryptlen;
580 	cryp->total_out = req->cryptlen;
581 	cryp->assoclen = 0;
582 	cryp->authsize = 0;
593 	if (!cryp->total_in)
596 	starfive_aes_dma_init(cryp);
598 	ret = starfive_aes_map_sg(cryp, rctx->in_sg, rctx->out_sg);
613 	ctx->cryp = starfive_cryp_find_dev(ctx);
614 	if (!ctx->cryp)
620 		return dev_err_probe(ctx->cryp->dev, PTR_ERR(ctx->skcipher_fbk),
643 	struct starfive_cryp_dev *cryp = ctx->cryp;
648 	cryp->req.areq = req;
649 	cryp->assoclen = req->assoclen;
650 	cryp->authsize = crypto_aead_authsize(crypto_aead_reqtfm(req));
652 	rctx->in_sg = scatterwalk_ffwd(_src, req->src, cryp->assoclen);
656 		rctx->out_sg = scatterwalk_ffwd(_dst, req->dst, cryp->assoclen);
658 	if (is_encrypt(cryp)) {
659 		cryp->total_in = req->cryptlen;
660 		cryp->total_out = req->cryptlen;
662 		cryp->total_in = req->cryptlen - cryp->authsize;
663 		cryp->total_out = cryp->total_in;
664 		scatterwalk_map_and_copy(cryp->tag_in, req->src,
665 					 cryp->total_in + cryp->assoclen,
666 					 cryp->authsize, 0);
669 	if (cryp->assoclen) {
670 		rctx->adata = kzalloc(cryp->assoclen + AES_BLOCK_SIZE, GFP_KERNEL);
672 			return dev_err_probe(cryp->dev, -ENOMEM,
675 		if (sg_copy_to_buffer(req->src, sg_nents_for_len(req->src, cryp->assoclen),
676 				      rctx->adata, cryp->assoclen) != cryp->assoclen)
680 	if (cryp->total_in)
682 			       sg_dma_len(rctx->in_sg) - cryp->total_in,
683 			       cryp->total_in);
691 	if (!cryp->assoclen)
694 	if ((cryp->flags & FLG_MODE_MASK) == STARFIVE_AES_MODE_CCM)
705 	if (!cryp->total_in)
708 	starfive_aes_dma_init(cryp);
710 	ret = starfive_aes_map_sg(cryp, rctx->in_sg, rctx->out_sg);
724 	ctx->cryp = starfive_cryp_find_dev(ctx);
725 	if (!ctx->cryp)
731 		return dev_err_probe(ctx->cryp->dev, PTR_ERR(ctx->aead_fbk),
748 static bool starfive_aes_check_unaligned(struct starfive_cryp_dev *cryp,
792 	struct starfive_cryp_dev *cryp = ctx->cryp;
795 	cryp->flags = flags;
797 	if ((cryp->flags & FLG_MODE_MASK) == STARFIVE_AES_MODE_ECB ||
798 	    (cryp->flags & FLG_MODE_MASK) == STARFIVE_AES_MODE_CBC)
802 	if (starfive_aes_check_unaligned(cryp, req->src, req->dst))
803 		return starfive_aes_do_fallback(req, is_encrypt(cryp));
805 	return crypto_transfer_skcipher_request_to_engine(cryp->engine, req);
829 	struct starfive_cryp_dev *cryp = ctx->cryp;
832 	cryp->flags = flags;
837 	if (((cryp->flags & FLG_MODE_MASK) == STARFIVE_AES_MODE_CCM) &&
838 	    !is_encrypt(cryp))
848 	if (starfive_aes_check_unaligned(cryp, src, dst))
849 		return starfive_aes_aead_do_fallback(req, is_encrypt(cryp));
851 	return crypto_transfer_aead_request_to_engine(cryp->engine, req);