1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * PowerPC P10 (ppc64le) accelerated ChaCha and XChaCha stream ciphers,
4 * including ChaCha20 (RFC7539)
5 *
6 * Copyright 2023- IBM Corp. All rights reserved.
7 */
8
9#include <crypto/algapi.h>
10#include <crypto/internal/chacha.h>
11#include <crypto/internal/simd.h>
12#include <crypto/internal/skcipher.h>
13#include <linux/kernel.h>
14#include <linux/module.h>
15#include <linux/cpufeature.h>
16#include <linux/sizes.h>
17#include <asm/simd.h>
18#include <asm/switch_to.h>
19
20asmlinkage void chacha_p10le_8x(u32 *state, u8 *dst, const u8 *src,
21				unsigned int len, int nrounds);
22
23static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_p10);
24
25static void vsx_begin(void)
26{
27	preempt_disable();
28	enable_kernel_vsx();
29}
30
31static void vsx_end(void)
32{
33	disable_kernel_vsx();
34	preempt_enable();
35}
36
37static void chacha_p10_do_8x(u32 *state, u8 *dst, const u8 *src,
38			     unsigned int bytes, int nrounds)
39{
40	unsigned int l = bytes & ~0x0FF;
41
42	if (l > 0) {
43		chacha_p10le_8x(state, dst, src, l, nrounds);
44		bytes -= l;
45		src += l;
46		dst += l;
47		state[12] += l / CHACHA_BLOCK_SIZE;
48	}
49
50	if (bytes > 0)
51		chacha_crypt_generic(state, dst, src, bytes, nrounds);
52}
53
54void hchacha_block_arch(const u32 *state, u32 *stream, int nrounds)
55{
56	hchacha_block_generic(state, stream, nrounds);
57}
58EXPORT_SYMBOL(hchacha_block_arch);
59
60void chacha_init_arch(u32 *state, const u32 *key, const u8 *iv)
61{
62	chacha_init_generic(state, key, iv);
63}
64EXPORT_SYMBOL(chacha_init_arch);
65
66void chacha_crypt_arch(u32 *state, u8 *dst, const u8 *src, unsigned int bytes,
67		       int nrounds)
68{
69	if (!static_branch_likely(&have_p10) || bytes <= CHACHA_BLOCK_SIZE ||
70	    !crypto_simd_usable())
71		return chacha_crypt_generic(state, dst, src, bytes, nrounds);
72
73	do {
74		unsigned int todo = min_t(unsigned int, bytes, SZ_4K);
75
76		vsx_begin();
77		chacha_p10_do_8x(state, dst, src, todo, nrounds);
78		vsx_end();
79
80		bytes -= todo;
81		src += todo;
82		dst += todo;
83	} while (bytes);
84}
85EXPORT_SYMBOL(chacha_crypt_arch);
86
87static int chacha_p10_stream_xor(struct skcipher_request *req,
88				 const struct chacha_ctx *ctx, const u8 *iv)
89{
90	struct skcipher_walk walk;
91	u32 state[16];
92	int err;
93
94	err = skcipher_walk_virt(&walk, req, false);
95	if (err)
96		return err;
97
98	chacha_init_generic(state, ctx->key, iv);
99
100	while (walk.nbytes > 0) {
101		unsigned int nbytes = walk.nbytes;
102
103		if (nbytes < walk.total)
104			nbytes = rounddown(nbytes, walk.stride);
105
106		if (!crypto_simd_usable()) {
107			chacha_crypt_generic(state, walk.dst.virt.addr,
108					     walk.src.virt.addr, nbytes,
109					     ctx->nrounds);
110		} else {
111			vsx_begin();
112			chacha_p10_do_8x(state, walk.dst.virt.addr,
113				      walk.src.virt.addr, nbytes, ctx->nrounds);
114			vsx_end();
115		}
116		err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
117		if (err)
118			break;
119	}
120
121	return err;
122}
123
124static int chacha_p10(struct skcipher_request *req)
125{
126	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
127	struct chacha_ctx *ctx = crypto_skcipher_ctx(tfm);
128
129	return chacha_p10_stream_xor(req, ctx, req->iv);
130}
131
132static int xchacha_p10(struct skcipher_request *req)
133{
134	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
135	struct chacha_ctx *ctx = crypto_skcipher_ctx(tfm);
136	struct chacha_ctx subctx;
137	u32 state[16];
138	u8 real_iv[16];
139
140	chacha_init_generic(state, ctx->key, req->iv);
141	hchacha_block_arch(state, subctx.key, ctx->nrounds);
142	subctx.nrounds = ctx->nrounds;
143
144	memcpy(&real_iv[0], req->iv + 24, 8);
145	memcpy(&real_iv[8], req->iv + 16, 8);
146	return chacha_p10_stream_xor(req, &subctx, real_iv);
147}
148
149static struct skcipher_alg algs[] = {
150	{
151		.base.cra_name		= "chacha20",
152		.base.cra_driver_name	= "chacha20-p10",
153		.base.cra_priority	= 300,
154		.base.cra_blocksize	= 1,
155		.base.cra_ctxsize	= sizeof(struct chacha_ctx),
156		.base.cra_module	= THIS_MODULE,
157
158		.min_keysize		= CHACHA_KEY_SIZE,
159		.max_keysize		= CHACHA_KEY_SIZE,
160		.ivsize			= CHACHA_IV_SIZE,
161		.chunksize		= CHACHA_BLOCK_SIZE,
162		.setkey			= chacha20_setkey,
163		.encrypt		= chacha_p10,
164		.decrypt		= chacha_p10,
165	}, {
166		.base.cra_name		= "xchacha20",
167		.base.cra_driver_name	= "xchacha20-p10",
168		.base.cra_priority	= 300,
169		.base.cra_blocksize	= 1,
170		.base.cra_ctxsize	= sizeof(struct chacha_ctx),
171		.base.cra_module	= THIS_MODULE,
172
173		.min_keysize		= CHACHA_KEY_SIZE,
174		.max_keysize		= CHACHA_KEY_SIZE,
175		.ivsize			= XCHACHA_IV_SIZE,
176		.chunksize		= CHACHA_BLOCK_SIZE,
177		.setkey			= chacha20_setkey,
178		.encrypt		= xchacha_p10,
179		.decrypt		= xchacha_p10,
180	}, {
181		.base.cra_name		= "xchacha12",
182		.base.cra_driver_name	= "xchacha12-p10",
183		.base.cra_priority	= 300,
184		.base.cra_blocksize	= 1,
185		.base.cra_ctxsize	= sizeof(struct chacha_ctx),
186		.base.cra_module	= THIS_MODULE,
187
188		.min_keysize		= CHACHA_KEY_SIZE,
189		.max_keysize		= CHACHA_KEY_SIZE,
190		.ivsize			= XCHACHA_IV_SIZE,
191		.chunksize		= CHACHA_BLOCK_SIZE,
192		.setkey			= chacha12_setkey,
193		.encrypt		= xchacha_p10,
194		.decrypt		= xchacha_p10,
195	}
196};
197
198static int __init chacha_p10_init(void)
199{
200	if (!cpu_has_feature(CPU_FTR_ARCH_31))
201		return 0;
202
203	static_branch_enable(&have_p10);
204
205	return crypto_register_skciphers(algs, ARRAY_SIZE(algs));
206}
207
208static void __exit chacha_p10_exit(void)
209{
210	if (!static_branch_likely(&have_p10))
211		return;
212
213	crypto_unregister_skciphers(algs, ARRAY_SIZE(algs));
214}
215
216module_init(chacha_p10_init);
217module_exit(chacha_p10_exit);
218
219MODULE_DESCRIPTION("ChaCha and XChaCha stream ciphers (P10 accelerated)");
220MODULE_AUTHOR("Danny Tsen <dtsen@linux.ibm.com>");
221MODULE_LICENSE("GPL v2");
222MODULE_ALIAS_CRYPTO("chacha20");
223MODULE_ALIAS_CRYPTO("chacha20-p10");
224MODULE_ALIAS_CRYPTO("xchacha20");
225MODULE_ALIAS_CRYPTO("xchacha20-p10");
226MODULE_ALIAS_CRYPTO("xchacha12");
227MODULE_ALIAS_CRYPTO("xchacha12-p10");
228