1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * sha2-ce-glue.c - SHA-224/SHA-256 using ARMv8 Crypto Extensions
4 *
5 * Copyright (C) 2015 Linaro Ltd <ard.biesheuvel@linaro.org>
6 */
7
8#include <crypto/internal/hash.h>
9#include <crypto/internal/simd.h>
10#include <crypto/sha2.h>
11#include <crypto/sha256_base.h>
12#include <linux/cpufeature.h>
13#include <linux/crypto.h>
14#include <linux/module.h>
15
16#include <asm/hwcap.h>
17#include <asm/simd.h>
18#include <asm/neon.h>
19#include <asm/unaligned.h>
20
21#include "sha256_glue.h"
22
23MODULE_DESCRIPTION("SHA-224/SHA-256 secure hash using ARMv8 Crypto Extensions");
24MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
25MODULE_LICENSE("GPL v2");
26
27asmlinkage void sha2_ce_transform(struct sha256_state *sst, u8 const *src,
28				  int blocks);
29
30static int sha2_ce_update(struct shash_desc *desc, const u8 *data,
31			  unsigned int len)
32{
33	struct sha256_state *sctx = shash_desc_ctx(desc);
34
35	if (!crypto_simd_usable() ||
36	    (sctx->count % SHA256_BLOCK_SIZE) + len < SHA256_BLOCK_SIZE)
37		return crypto_sha256_arm_update(desc, data, len);
38
39	kernel_neon_begin();
40	sha256_base_do_update(desc, data, len,
41			      (sha256_block_fn *)sha2_ce_transform);
42	kernel_neon_end();
43
44	return 0;
45}
46
47static int sha2_ce_finup(struct shash_desc *desc, const u8 *data,
48			 unsigned int len, u8 *out)
49{
50	if (!crypto_simd_usable())
51		return crypto_sha256_arm_finup(desc, data, len, out);
52
53	kernel_neon_begin();
54	if (len)
55		sha256_base_do_update(desc, data, len,
56				      (sha256_block_fn *)sha2_ce_transform);
57	sha256_base_do_finalize(desc, (sha256_block_fn *)sha2_ce_transform);
58	kernel_neon_end();
59
60	return sha256_base_finish(desc, out);
61}
62
63static int sha2_ce_final(struct shash_desc *desc, u8 *out)
64{
65	return sha2_ce_finup(desc, NULL, 0, out);
66}
67
68static struct shash_alg algs[] = { {
69	.init			= sha224_base_init,
70	.update			= sha2_ce_update,
71	.final			= sha2_ce_final,
72	.finup			= sha2_ce_finup,
73	.descsize		= sizeof(struct sha256_state),
74	.digestsize		= SHA224_DIGEST_SIZE,
75	.base			= {
76		.cra_name		= "sha224",
77		.cra_driver_name	= "sha224-ce",
78		.cra_priority		= 300,
79		.cra_blocksize		= SHA256_BLOCK_SIZE,
80		.cra_module		= THIS_MODULE,
81	}
82}, {
83	.init			= sha256_base_init,
84	.update			= sha2_ce_update,
85	.final			= sha2_ce_final,
86	.finup			= sha2_ce_finup,
87	.descsize		= sizeof(struct sha256_state),
88	.digestsize		= SHA256_DIGEST_SIZE,
89	.base			= {
90		.cra_name		= "sha256",
91		.cra_driver_name	= "sha256-ce",
92		.cra_priority		= 300,
93		.cra_blocksize		= SHA256_BLOCK_SIZE,
94		.cra_module		= THIS_MODULE,
95	}
96} };
97
98static int __init sha2_ce_mod_init(void)
99{
100	return crypto_register_shashes(algs, ARRAY_SIZE(algs));
101}
102
103static void __exit sha2_ce_mod_fini(void)
104{
105	crypto_unregister_shashes(algs, ARRAY_SIZE(algs));
106}
107
108module_cpu_feature_match(SHA2, sha2_ce_mod_init);
109module_exit(sha2_ce_mod_fini);
110