1/*
2 * Cryptographic API.
3 *
4 * Glue code for the SHA256 Secure Hash Algorithm assembler implementations
5 * using SSSE3, AVX, AVX2, and SHA-NI instructions.
6 *
7 * This file is based on sha256_generic.c
8 *
9 * Copyright (C) 2013 Intel Corporation.
10 *
11 * Author:
12 *     Tim Chen <tim.c.chen@linux.intel.com>
13 *
14 * This program is free software; you can redistribute it and/or modify it
15 * under the terms of the GNU General Public License as published by the Free
16 * Software Foundation; either version 2 of the License, or (at your option)
17 * any later version.
18 *
19 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
20 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
21 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
22 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
23 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
24 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
25 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
26 * SOFTWARE.
27 */
28
29
30#define pr_fmt(fmt)	KBUILD_MODNAME ": " fmt
31
32#include <crypto/internal/hash.h>
33#include <crypto/internal/simd.h>
34#include <linux/init.h>
35#include <linux/module.h>
36#include <linux/mm.h>
37#include <linux/types.h>
38#include <crypto/sha2.h>
39#include <crypto/sha256_base.h>
40#include <linux/string.h>
41#include <asm/cpu_device_id.h>
42#include <asm/simd.h>
43
44asmlinkage void sha256_transform_ssse3(struct sha256_state *state,
45				       const u8 *data, int blocks);
46
47static const struct x86_cpu_id module_cpu_ids[] = {
48#ifdef CONFIG_AS_SHA256_NI
49	X86_MATCH_FEATURE(X86_FEATURE_SHA_NI, NULL),
50#endif
51	X86_MATCH_FEATURE(X86_FEATURE_AVX2, NULL),
52	X86_MATCH_FEATURE(X86_FEATURE_AVX, NULL),
53	X86_MATCH_FEATURE(X86_FEATURE_SSSE3, NULL),
54	{}
55};
56MODULE_DEVICE_TABLE(x86cpu, module_cpu_ids);
57
58static int _sha256_update(struct shash_desc *desc, const u8 *data,
59			  unsigned int len, sha256_block_fn *sha256_xform)
60{
61	struct sha256_state *sctx = shash_desc_ctx(desc);
62
63	if (!crypto_simd_usable() ||
64	    (sctx->count % SHA256_BLOCK_SIZE) + len < SHA256_BLOCK_SIZE)
65		return crypto_sha256_update(desc, data, len);
66
67	/*
68	 * Make sure struct sha256_state begins directly with the SHA256
69	 * 256-bit internal state, as this is what the asm functions expect.
70	 */
71	BUILD_BUG_ON(offsetof(struct sha256_state, state) != 0);
72
73	kernel_fpu_begin();
74	sha256_base_do_update(desc, data, len, sha256_xform);
75	kernel_fpu_end();
76
77	return 0;
78}
79
80static int sha256_finup(struct shash_desc *desc, const u8 *data,
81	      unsigned int len, u8 *out, sha256_block_fn *sha256_xform)
82{
83	if (!crypto_simd_usable())
84		return crypto_sha256_finup(desc, data, len, out);
85
86	kernel_fpu_begin();
87	if (len)
88		sha256_base_do_update(desc, data, len, sha256_xform);
89	sha256_base_do_finalize(desc, sha256_xform);
90	kernel_fpu_end();
91
92	return sha256_base_finish(desc, out);
93}
94
95static int sha256_ssse3_update(struct shash_desc *desc, const u8 *data,
96			 unsigned int len)
97{
98	return _sha256_update(desc, data, len, sha256_transform_ssse3);
99}
100
101static int sha256_ssse3_finup(struct shash_desc *desc, const u8 *data,
102	      unsigned int len, u8 *out)
103{
104	return sha256_finup(desc, data, len, out, sha256_transform_ssse3);
105}
106
107/* Add padding and return the message digest. */
108static int sha256_ssse3_final(struct shash_desc *desc, u8 *out)
109{
110	return sha256_ssse3_finup(desc, NULL, 0, out);
111}
112
113static int sha256_ssse3_digest(struct shash_desc *desc, const u8 *data,
114	      unsigned int len, u8 *out)
115{
116	return sha256_base_init(desc) ?:
117	       sha256_ssse3_finup(desc, data, len, out);
118}
119
120static struct shash_alg sha256_ssse3_algs[] = { {
121	.digestsize	=	SHA256_DIGEST_SIZE,
122	.init		=	sha256_base_init,
123	.update		=	sha256_ssse3_update,
124	.final		=	sha256_ssse3_final,
125	.finup		=	sha256_ssse3_finup,
126	.digest		=	sha256_ssse3_digest,
127	.descsize	=	sizeof(struct sha256_state),
128	.base		=	{
129		.cra_name	=	"sha256",
130		.cra_driver_name =	"sha256-ssse3",
131		.cra_priority	=	150,
132		.cra_blocksize	=	SHA256_BLOCK_SIZE,
133		.cra_module	=	THIS_MODULE,
134	}
135}, {
136	.digestsize	=	SHA224_DIGEST_SIZE,
137	.init		=	sha224_base_init,
138	.update		=	sha256_ssse3_update,
139	.final		=	sha256_ssse3_final,
140	.finup		=	sha256_ssse3_finup,
141	.descsize	=	sizeof(struct sha256_state),
142	.base		=	{
143		.cra_name	=	"sha224",
144		.cra_driver_name =	"sha224-ssse3",
145		.cra_priority	=	150,
146		.cra_blocksize	=	SHA224_BLOCK_SIZE,
147		.cra_module	=	THIS_MODULE,
148	}
149} };
150
151static int register_sha256_ssse3(void)
152{
153	if (boot_cpu_has(X86_FEATURE_SSSE3))
154		return crypto_register_shashes(sha256_ssse3_algs,
155				ARRAY_SIZE(sha256_ssse3_algs));
156	return 0;
157}
158
159static void unregister_sha256_ssse3(void)
160{
161	if (boot_cpu_has(X86_FEATURE_SSSE3))
162		crypto_unregister_shashes(sha256_ssse3_algs,
163				ARRAY_SIZE(sha256_ssse3_algs));
164}
165
166asmlinkage void sha256_transform_avx(struct sha256_state *state,
167				     const u8 *data, int blocks);
168
169static int sha256_avx_update(struct shash_desc *desc, const u8 *data,
170			 unsigned int len)
171{
172	return _sha256_update(desc, data, len, sha256_transform_avx);
173}
174
175static int sha256_avx_finup(struct shash_desc *desc, const u8 *data,
176		      unsigned int len, u8 *out)
177{
178	return sha256_finup(desc, data, len, out, sha256_transform_avx);
179}
180
181static int sha256_avx_final(struct shash_desc *desc, u8 *out)
182{
183	return sha256_avx_finup(desc, NULL, 0, out);
184}
185
186static int sha256_avx_digest(struct shash_desc *desc, const u8 *data,
187		      unsigned int len, u8 *out)
188{
189	return sha256_base_init(desc) ?:
190	       sha256_avx_finup(desc, data, len, out);
191}
192
193static struct shash_alg sha256_avx_algs[] = { {
194	.digestsize	=	SHA256_DIGEST_SIZE,
195	.init		=	sha256_base_init,
196	.update		=	sha256_avx_update,
197	.final		=	sha256_avx_final,
198	.finup		=	sha256_avx_finup,
199	.digest		=	sha256_avx_digest,
200	.descsize	=	sizeof(struct sha256_state),
201	.base		=	{
202		.cra_name	=	"sha256",
203		.cra_driver_name =	"sha256-avx",
204		.cra_priority	=	160,
205		.cra_blocksize	=	SHA256_BLOCK_SIZE,
206		.cra_module	=	THIS_MODULE,
207	}
208}, {
209	.digestsize	=	SHA224_DIGEST_SIZE,
210	.init		=	sha224_base_init,
211	.update		=	sha256_avx_update,
212	.final		=	sha256_avx_final,
213	.finup		=	sha256_avx_finup,
214	.descsize	=	sizeof(struct sha256_state),
215	.base		=	{
216		.cra_name	=	"sha224",
217		.cra_driver_name =	"sha224-avx",
218		.cra_priority	=	160,
219		.cra_blocksize	=	SHA224_BLOCK_SIZE,
220		.cra_module	=	THIS_MODULE,
221	}
222} };
223
224static bool avx_usable(void)
225{
226	if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL)) {
227		if (boot_cpu_has(X86_FEATURE_AVX))
228			pr_info("AVX detected but unusable.\n");
229		return false;
230	}
231
232	return true;
233}
234
235static int register_sha256_avx(void)
236{
237	if (avx_usable())
238		return crypto_register_shashes(sha256_avx_algs,
239				ARRAY_SIZE(sha256_avx_algs));
240	return 0;
241}
242
243static void unregister_sha256_avx(void)
244{
245	if (avx_usable())
246		crypto_unregister_shashes(sha256_avx_algs,
247				ARRAY_SIZE(sha256_avx_algs));
248}
249
250asmlinkage void sha256_transform_rorx(struct sha256_state *state,
251				      const u8 *data, int blocks);
252
253static int sha256_avx2_update(struct shash_desc *desc, const u8 *data,
254			 unsigned int len)
255{
256	return _sha256_update(desc, data, len, sha256_transform_rorx);
257}
258
259static int sha256_avx2_finup(struct shash_desc *desc, const u8 *data,
260		      unsigned int len, u8 *out)
261{
262	return sha256_finup(desc, data, len, out, sha256_transform_rorx);
263}
264
265static int sha256_avx2_final(struct shash_desc *desc, u8 *out)
266{
267	return sha256_avx2_finup(desc, NULL, 0, out);
268}
269
270static int sha256_avx2_digest(struct shash_desc *desc, const u8 *data,
271		      unsigned int len, u8 *out)
272{
273	return sha256_base_init(desc) ?:
274	       sha256_avx2_finup(desc, data, len, out);
275}
276
277static struct shash_alg sha256_avx2_algs[] = { {
278	.digestsize	=	SHA256_DIGEST_SIZE,
279	.init		=	sha256_base_init,
280	.update		=	sha256_avx2_update,
281	.final		=	sha256_avx2_final,
282	.finup		=	sha256_avx2_finup,
283	.digest		=	sha256_avx2_digest,
284	.descsize	=	sizeof(struct sha256_state),
285	.base		=	{
286		.cra_name	=	"sha256",
287		.cra_driver_name =	"sha256-avx2",
288		.cra_priority	=	170,
289		.cra_blocksize	=	SHA256_BLOCK_SIZE,
290		.cra_module	=	THIS_MODULE,
291	}
292}, {
293	.digestsize	=	SHA224_DIGEST_SIZE,
294	.init		=	sha224_base_init,
295	.update		=	sha256_avx2_update,
296	.final		=	sha256_avx2_final,
297	.finup		=	sha256_avx2_finup,
298	.descsize	=	sizeof(struct sha256_state),
299	.base		=	{
300		.cra_name	=	"sha224",
301		.cra_driver_name =	"sha224-avx2",
302		.cra_priority	=	170,
303		.cra_blocksize	=	SHA224_BLOCK_SIZE,
304		.cra_module	=	THIS_MODULE,
305	}
306} };
307
308static bool avx2_usable(void)
309{
310	if (avx_usable() && boot_cpu_has(X86_FEATURE_AVX2) &&
311		    boot_cpu_has(X86_FEATURE_BMI2))
312		return true;
313
314	return false;
315}
316
317static int register_sha256_avx2(void)
318{
319	if (avx2_usable())
320		return crypto_register_shashes(sha256_avx2_algs,
321				ARRAY_SIZE(sha256_avx2_algs));
322	return 0;
323}
324
325static void unregister_sha256_avx2(void)
326{
327	if (avx2_usable())
328		crypto_unregister_shashes(sha256_avx2_algs,
329				ARRAY_SIZE(sha256_avx2_algs));
330}
331
332#ifdef CONFIG_AS_SHA256_NI
333asmlinkage void sha256_ni_transform(struct sha256_state *digest,
334				    const u8 *data, int rounds);
335
336static int sha256_ni_update(struct shash_desc *desc, const u8 *data,
337			 unsigned int len)
338{
339	return _sha256_update(desc, data, len, sha256_ni_transform);
340}
341
342static int sha256_ni_finup(struct shash_desc *desc, const u8 *data,
343		      unsigned int len, u8 *out)
344{
345	return sha256_finup(desc, data, len, out, sha256_ni_transform);
346}
347
348static int sha256_ni_final(struct shash_desc *desc, u8 *out)
349{
350	return sha256_ni_finup(desc, NULL, 0, out);
351}
352
353static int sha256_ni_digest(struct shash_desc *desc, const u8 *data,
354		      unsigned int len, u8 *out)
355{
356	return sha256_base_init(desc) ?:
357	       sha256_ni_finup(desc, data, len, out);
358}
359
360static struct shash_alg sha256_ni_algs[] = { {
361	.digestsize	=	SHA256_DIGEST_SIZE,
362	.init		=	sha256_base_init,
363	.update		=	sha256_ni_update,
364	.final		=	sha256_ni_final,
365	.finup		=	sha256_ni_finup,
366	.digest		=	sha256_ni_digest,
367	.descsize	=	sizeof(struct sha256_state),
368	.base		=	{
369		.cra_name	=	"sha256",
370		.cra_driver_name =	"sha256-ni",
371		.cra_priority	=	250,
372		.cra_blocksize	=	SHA256_BLOCK_SIZE,
373		.cra_module	=	THIS_MODULE,
374	}
375}, {
376	.digestsize	=	SHA224_DIGEST_SIZE,
377	.init		=	sha224_base_init,
378	.update		=	sha256_ni_update,
379	.final		=	sha256_ni_final,
380	.finup		=	sha256_ni_finup,
381	.descsize	=	sizeof(struct sha256_state),
382	.base		=	{
383		.cra_name	=	"sha224",
384		.cra_driver_name =	"sha224-ni",
385		.cra_priority	=	250,
386		.cra_blocksize	=	SHA224_BLOCK_SIZE,
387		.cra_module	=	THIS_MODULE,
388	}
389} };
390
391static int register_sha256_ni(void)
392{
393	if (boot_cpu_has(X86_FEATURE_SHA_NI))
394		return crypto_register_shashes(sha256_ni_algs,
395				ARRAY_SIZE(sha256_ni_algs));
396	return 0;
397}
398
399static void unregister_sha256_ni(void)
400{
401	if (boot_cpu_has(X86_FEATURE_SHA_NI))
402		crypto_unregister_shashes(sha256_ni_algs,
403				ARRAY_SIZE(sha256_ni_algs));
404}
405
406#else
407static inline int register_sha256_ni(void) { return 0; }
408static inline void unregister_sha256_ni(void) { }
409#endif
410
411static int __init sha256_ssse3_mod_init(void)
412{
413	if (!x86_match_cpu(module_cpu_ids))
414		return -ENODEV;
415
416	if (register_sha256_ssse3())
417		goto fail;
418
419	if (register_sha256_avx()) {
420		unregister_sha256_ssse3();
421		goto fail;
422	}
423
424	if (register_sha256_avx2()) {
425		unregister_sha256_avx();
426		unregister_sha256_ssse3();
427		goto fail;
428	}
429
430	if (register_sha256_ni()) {
431		unregister_sha256_avx2();
432		unregister_sha256_avx();
433		unregister_sha256_ssse3();
434		goto fail;
435	}
436
437	return 0;
438fail:
439	return -ENODEV;
440}
441
442static void __exit sha256_ssse3_mod_fini(void)
443{
444	unregister_sha256_ni();
445	unregister_sha256_avx2();
446	unregister_sha256_avx();
447	unregister_sha256_ssse3();
448}
449
450module_init(sha256_ssse3_mod_init);
451module_exit(sha256_ssse3_mod_fini);
452
453MODULE_LICENSE("GPL");
454MODULE_DESCRIPTION("SHA256 Secure Hash Algorithm, Supplemental SSE3 accelerated");
455
456MODULE_ALIAS_CRYPTO("sha256");
457MODULE_ALIAS_CRYPTO("sha256-ssse3");
458MODULE_ALIAS_CRYPTO("sha256-avx");
459MODULE_ALIAS_CRYPTO("sha256-avx2");
460MODULE_ALIAS_CRYPTO("sha224");
461MODULE_ALIAS_CRYPTO("sha224-ssse3");
462MODULE_ALIAS_CRYPTO("sha224-avx");
463MODULE_ALIAS_CRYPTO("sha224-avx2");
464#ifdef CONFIG_AS_SHA256_NI
465MODULE_ALIAS_CRYPTO("sha256-ni");
466MODULE_ALIAS_CRYPTO("sha224-ni");
467#endif
468