1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * GHASH using the RISC-V vector crypto extensions
4 *
5 * Copyright (C) 2023 VRULL GmbH
6 * Author: Heiko Stuebner <heiko.stuebner@vrull.eu>
7 *
8 * Copyright (C) 2023 SiFive, Inc.
9 * Author: Jerry Shih <jerry.shih@sifive.com>
10 */
11
12#include <asm/simd.h>
13#include <asm/vector.h>
14#include <crypto/ghash.h>
15#include <crypto/internal/hash.h>
16#include <crypto/internal/simd.h>
17#include <linux/linkage.h>
18#include <linux/module.h>
19
20asmlinkage void ghash_zvkg(be128 *accumulator, const be128 *key, const u8 *data,
21			   size_t len);
22
23struct riscv64_ghash_tfm_ctx {
24	be128 key;
25};
26
27struct riscv64_ghash_desc_ctx {
28	be128 accumulator;
29	u8 buffer[GHASH_BLOCK_SIZE];
30	u32 bytes;
31};
32
33static int riscv64_ghash_setkey(struct crypto_shash *tfm, const u8 *key,
34				unsigned int keylen)
35{
36	struct riscv64_ghash_tfm_ctx *tctx = crypto_shash_ctx(tfm);
37
38	if (keylen != GHASH_BLOCK_SIZE)
39		return -EINVAL;
40
41	memcpy(&tctx->key, key, GHASH_BLOCK_SIZE);
42
43	return 0;
44}
45
46static int riscv64_ghash_init(struct shash_desc *desc)
47{
48	struct riscv64_ghash_desc_ctx *dctx = shash_desc_ctx(desc);
49
50	*dctx = (struct riscv64_ghash_desc_ctx){};
51
52	return 0;
53}
54
55static inline void
56riscv64_ghash_blocks(const struct riscv64_ghash_tfm_ctx *tctx,
57		     struct riscv64_ghash_desc_ctx *dctx,
58		     const u8 *src, size_t srclen)
59{
60	/* The srclen is nonzero and a multiple of 16. */
61	if (crypto_simd_usable()) {
62		kernel_vector_begin();
63		ghash_zvkg(&dctx->accumulator, &tctx->key, src, srclen);
64		kernel_vector_end();
65	} else {
66		do {
67			crypto_xor((u8 *)&dctx->accumulator, src,
68				   GHASH_BLOCK_SIZE);
69			gf128mul_lle(&dctx->accumulator, &tctx->key);
70			src += GHASH_BLOCK_SIZE;
71			srclen -= GHASH_BLOCK_SIZE;
72		} while (srclen);
73	}
74}
75
76static int riscv64_ghash_update(struct shash_desc *desc, const u8 *src,
77				unsigned int srclen)
78{
79	const struct riscv64_ghash_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm);
80	struct riscv64_ghash_desc_ctx *dctx = shash_desc_ctx(desc);
81	unsigned int len;
82
83	if (dctx->bytes) {
84		if (dctx->bytes + srclen < GHASH_BLOCK_SIZE) {
85			memcpy(dctx->buffer + dctx->bytes, src, srclen);
86			dctx->bytes += srclen;
87			return 0;
88		}
89		memcpy(dctx->buffer + dctx->bytes, src,
90		       GHASH_BLOCK_SIZE - dctx->bytes);
91		riscv64_ghash_blocks(tctx, dctx, dctx->buffer,
92				     GHASH_BLOCK_SIZE);
93		src += GHASH_BLOCK_SIZE - dctx->bytes;
94		srclen -= GHASH_BLOCK_SIZE - dctx->bytes;
95		dctx->bytes = 0;
96	}
97
98	len = round_down(srclen, GHASH_BLOCK_SIZE);
99	if (len) {
100		riscv64_ghash_blocks(tctx, dctx, src, len);
101		src += len;
102		srclen -= len;
103	}
104
105	if (srclen) {
106		memcpy(dctx->buffer, src, srclen);
107		dctx->bytes = srclen;
108	}
109
110	return 0;
111}
112
113static int riscv64_ghash_final(struct shash_desc *desc, u8 *out)
114{
115	const struct riscv64_ghash_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm);
116	struct riscv64_ghash_desc_ctx *dctx = shash_desc_ctx(desc);
117	int i;
118
119	if (dctx->bytes) {
120		for (i = dctx->bytes; i < GHASH_BLOCK_SIZE; i++)
121			dctx->buffer[i] = 0;
122
123		riscv64_ghash_blocks(tctx, dctx, dctx->buffer,
124				     GHASH_BLOCK_SIZE);
125	}
126
127	memcpy(out, &dctx->accumulator, GHASH_DIGEST_SIZE);
128	return 0;
129}
130
131static struct shash_alg riscv64_ghash_alg = {
132	.init = riscv64_ghash_init,
133	.update = riscv64_ghash_update,
134	.final = riscv64_ghash_final,
135	.setkey = riscv64_ghash_setkey,
136	.descsize = sizeof(struct riscv64_ghash_desc_ctx),
137	.digestsize = GHASH_DIGEST_SIZE,
138	.base = {
139		.cra_blocksize = GHASH_BLOCK_SIZE,
140		.cra_ctxsize = sizeof(struct riscv64_ghash_tfm_ctx),
141		.cra_priority = 300,
142		.cra_name = "ghash",
143		.cra_driver_name = "ghash-riscv64-zvkg",
144		.cra_module = THIS_MODULE,
145	},
146};
147
148static int __init riscv64_ghash_mod_init(void)
149{
150	if (riscv_isa_extension_available(NULL, ZVKG) &&
151	    riscv_vector_vlen() >= 128)
152		return crypto_register_shash(&riscv64_ghash_alg);
153
154	return -ENODEV;
155}
156
157static void __exit riscv64_ghash_mod_exit(void)
158{
159	crypto_unregister_shash(&riscv64_ghash_alg);
160}
161
162module_init(riscv64_ghash_mod_init);
163module_exit(riscv64_ghash_mod_exit);
164
165MODULE_DESCRIPTION("GHASH (RISC-V accelerated)");
166MODULE_AUTHOR("Heiko Stuebner <heiko.stuebner@vrull.eu>");
167MODULE_LICENSE("GPL");
168MODULE_ALIAS_CRYPTO("ghash");
169