1/*-
2 * Copyright (c) 2018 Conrad Meyer <cem@FreeBSD.org>
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 *    notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 *    notice, this list of conditions and the following disclaimer in the
12 *    documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 */
26
27#include <sys/cdefs.h>
28__FBSDID("$FreeBSD$");
29
30#include <sys/param.h>
31#include <sys/bus.h>
32#include <sys/kernel.h>
33#include <sys/kobj.h>
34#include <sys/lock.h>
35#include <sys/malloc.h>
36#include <sys/module.h>
37#include <sys/mutex.h>
38#include <sys/rwlock.h>
39#include <sys/smp.h>
40
41#include <blake2.h>
42
43#include <opencrypto/cryptodev.h>
44#include <cryptodev_if.h>
45
46#include <machine/fpu.h>
47
48struct blake2_session {
49	size_t mlen;
50};
51CTASSERT((size_t)BLAKE2B_KEYBYTES > (size_t)BLAKE2S_KEYBYTES);
52
53struct blake2_softc {
54	bool	dying;
55	int32_t cid;
56	struct rwlock lock;
57};
58
59static struct mtx_padalign *ctx_mtx;
60static struct fpu_kern_ctx **ctx_fpu;
61
62#define ACQUIRE_CTX(i, ctx)					\
63	do {							\
64		(i) = PCPU_GET(cpuid);				\
65		mtx_lock(&ctx_mtx[(i)]);			\
66		(ctx) = ctx_fpu[(i)];				\
67	} while (0)
68#define RELEASE_CTX(i, ctx)					\
69	do {							\
70		mtx_unlock(&ctx_mtx[(i)]);			\
71		(i) = -1;					\
72		(ctx) = NULL;					\
73	} while (0)
74
75static int blake2_cipher_setup(struct blake2_session *ses,
76    const struct crypto_session_params *csp);
77static int blake2_cipher_process(struct blake2_session *ses,
78    struct cryptop *crp);
79
80MALLOC_DEFINE(M_BLAKE2, "blake2_data", "Blake2 Data");
81
82static void
83blake2_identify(driver_t *drv, device_t parent)
84{
85
86	/* NB: order 10 is so we get attached after h/w devices */
87	if (device_find_child(parent, "blaketwo", -1) == NULL &&
88	    BUS_ADD_CHILD(parent, 10, "blaketwo", -1) == 0)
89		panic("blaketwo: could not attach");
90}
91
92static int
93blake2_probe(device_t dev)
94{
95	device_set_desc(dev, "Blake2");
96	return (0);
97}
98
99static void
100blake2_cleanctx(void)
101{
102	int i;
103
104	/* XXX - no way to return driverid */
105	CPU_FOREACH(i) {
106		if (ctx_fpu[i] != NULL) {
107			mtx_destroy(&ctx_mtx[i]);
108			fpu_kern_free_ctx(ctx_fpu[i]);
109		}
110		ctx_fpu[i] = NULL;
111	}
112	free(ctx_mtx, M_BLAKE2);
113	ctx_mtx = NULL;
114	free(ctx_fpu, M_BLAKE2);
115	ctx_fpu = NULL;
116}
117
118static int
119blake2_attach(device_t dev)
120{
121	struct blake2_softc *sc;
122	int i;
123
124	sc = device_get_softc(dev);
125	sc->dying = false;
126
127	sc->cid = crypto_get_driverid(dev, sizeof(struct blake2_session),
128	    CRYPTOCAP_F_SOFTWARE | CRYPTOCAP_F_SYNC |
129	    CRYPTOCAP_F_ACCEL_SOFTWARE);
130	if (sc->cid < 0) {
131		device_printf(dev, "Could not get crypto driver id.\n");
132		return (ENOMEM);
133	}
134
135	ctx_mtx = malloc(sizeof(*ctx_mtx) * (mp_maxid + 1), M_BLAKE2,
136	    M_WAITOK | M_ZERO);
137	ctx_fpu = malloc(sizeof(*ctx_fpu) * (mp_maxid + 1), M_BLAKE2,
138	    M_WAITOK | M_ZERO);
139
140	CPU_FOREACH(i) {
141#ifdef __amd64__
142		ctx_fpu[i] = fpu_kern_alloc_ctx_domain(
143		    pcpu_find(i)->pc_domain, FPU_KERN_NORMAL);
144#else
145		ctx_fpu[i] = fpu_kern_alloc_ctx(FPU_KERN_NORMAL);
146#endif
147		mtx_init(&ctx_mtx[i], "bl2fpumtx", NULL, MTX_DEF | MTX_NEW);
148	}
149
150	rw_init(&sc->lock, "blake2_lock");
151
152	return (0);
153}
154
155static int
156blake2_detach(device_t dev)
157{
158	struct blake2_softc *sc;
159
160	sc = device_get_softc(dev);
161
162	rw_wlock(&sc->lock);
163	sc->dying = true;
164	rw_wunlock(&sc->lock);
165	crypto_unregister_all(sc->cid);
166
167	rw_destroy(&sc->lock);
168
169	blake2_cleanctx();
170
171	return (0);
172}
173
174static int
175blake2_probesession(device_t dev, const struct crypto_session_params *csp)
176{
177
178	if (csp->csp_flags != 0)
179		return (EINVAL);
180	switch (csp->csp_mode) {
181	case CSP_MODE_DIGEST:
182		switch (csp->csp_auth_alg) {
183		case CRYPTO_BLAKE2B:
184		case CRYPTO_BLAKE2S:
185			break;
186		default:
187			return (EINVAL);
188		}
189		break;
190	default:
191		return (EINVAL);
192	}
193	return (CRYPTODEV_PROBE_ACCEL_SOFTWARE);
194}
195
196static int
197blake2_newsession(device_t dev, crypto_session_t cses,
198    const struct crypto_session_params *csp)
199{
200	struct blake2_softc *sc;
201	struct blake2_session *ses;
202	int error;
203
204	sc = device_get_softc(dev);
205
206	ses = crypto_get_driver_session(cses);
207
208	rw_rlock(&sc->lock);
209	if (sc->dying) {
210		rw_runlock(&sc->lock);
211		return (EINVAL);
212	}
213	rw_runlock(&sc->lock);
214
215	error = blake2_cipher_setup(ses, csp);
216	if (error != 0) {
217		CRYPTDEB("setup failed");
218		return (error);
219	}
220
221	return (0);
222}
223
224static int
225blake2_process(device_t dev, struct cryptop *crp, int hint __unused)
226{
227	struct blake2_session *ses;
228	int error;
229
230	ses = crypto_get_driver_session(crp->crp_session);
231	error = blake2_cipher_process(ses, crp);
232
233	crp->crp_etype = error;
234	crypto_done(crp);
235	return (0);
236}
237
238static device_method_t blake2_methods[] = {
239	DEVMETHOD(device_identify, blake2_identify),
240	DEVMETHOD(device_probe, blake2_probe),
241	DEVMETHOD(device_attach, blake2_attach),
242	DEVMETHOD(device_detach, blake2_detach),
243
244	DEVMETHOD(cryptodev_probesession, blake2_probesession),
245	DEVMETHOD(cryptodev_newsession, blake2_newsession),
246	DEVMETHOD(cryptodev_process, blake2_process),
247
248	DEVMETHOD_END
249};
250
251static driver_t blake2_driver = {
252	"blaketwo",
253	blake2_methods,
254	sizeof(struct blake2_softc),
255};
256static devclass_t blake2_devclass;
257
258DRIVER_MODULE(blake2, nexus, blake2_driver, blake2_devclass, 0, 0);
259MODULE_VERSION(blake2, 1);
260MODULE_DEPEND(blake2, crypto, 1, 1, 1);
261
262static bool
263blake2_check_klen(const struct crypto_session_params *csp, unsigned klen)
264{
265
266	if (csp->csp_auth_alg == CRYPTO_BLAKE2S)
267		return (klen <= BLAKE2S_KEYBYTES);
268	else
269		return (klen <= BLAKE2B_KEYBYTES);
270}
271
272static int
273blake2_cipher_setup(struct blake2_session *ses,
274    const struct crypto_session_params *csp)
275{
276	int hashlen;
277
278	CTASSERT((size_t)BLAKE2S_OUTBYTES <= (size_t)BLAKE2B_OUTBYTES);
279
280	if (!blake2_check_klen(csp, csp->csp_auth_klen))
281		return (EINVAL);
282
283	if (csp->csp_auth_mlen < 0)
284		return (EINVAL);
285
286	switch (csp->csp_auth_alg) {
287	case CRYPTO_BLAKE2S:
288		hashlen = BLAKE2S_OUTBYTES;
289		break;
290	case CRYPTO_BLAKE2B:
291		hashlen = BLAKE2B_OUTBYTES;
292		break;
293	default:
294		return (EINVAL);
295	}
296
297	if (csp->csp_auth_mlen > hashlen)
298		return (EINVAL);
299
300	if (csp->csp_auth_mlen == 0)
301		ses->mlen = hashlen;
302	else
303		ses->mlen = csp->csp_auth_mlen;
304	return (0);
305}
306
307static int
308blake2b_applicator(void *state, const void *buf, u_int len)
309{
310	int rc;
311
312	rc = blake2b_update(state, buf, len);
313	if (rc != 0)
314		return (EINVAL);
315	return (0);
316}
317
318static int
319blake2s_applicator(void *state, const void *buf, u_int len)
320{
321	int rc;
322
323	rc = blake2s_update(state, buf, len);
324	if (rc != 0)
325		return (EINVAL);
326	return (0);
327}
328
329static int
330blake2_cipher_process(struct blake2_session *ses, struct cryptop *crp)
331{
332	union {
333		blake2b_state sb;
334		blake2s_state ss;
335	} bctx;
336	char res[BLAKE2B_OUTBYTES], res2[BLAKE2B_OUTBYTES];
337	const struct crypto_session_params *csp;
338	struct fpu_kern_ctx *ctx;
339	const void *key;
340	int ctxidx;
341	bool kt;
342	int error, rc;
343	unsigned klen;
344
345	ctx = NULL;
346	ctxidx = 0;
347	error = EINVAL;
348
349	kt = is_fpu_kern_thread(0);
350	if (!kt) {
351		ACQUIRE_CTX(ctxidx, ctx);
352		fpu_kern_enter(curthread, ctx,
353		    FPU_KERN_NORMAL | FPU_KERN_KTHR);
354	}
355
356	csp = crypto_get_params(crp->crp_session);
357	if (crp->crp_auth_key != NULL)
358		key = crp->crp_auth_key;
359	else
360		key = csp->csp_auth_key;
361	klen = csp->csp_auth_klen;
362	switch (csp->csp_auth_alg) {
363	case CRYPTO_BLAKE2B:
364		if (klen > 0)
365			rc = blake2b_init_key(&bctx.sb, ses->mlen, key, klen);
366		else
367			rc = blake2b_init(&bctx.sb, ses->mlen);
368		if (rc != 0)
369			goto out;
370		error = crypto_apply(crp, crp->crp_payload_start,
371		    crp->crp_payload_length, blake2b_applicator, &bctx.sb);
372		if (error != 0)
373			goto out;
374		rc = blake2b_final(&bctx.sb, res, ses->mlen);
375		if (rc != 0) {
376			error = EINVAL;
377			goto out;
378		}
379		break;
380	case CRYPTO_BLAKE2S:
381		if (klen > 0)
382			rc = blake2s_init_key(&bctx.ss, ses->mlen, key, klen);
383		else
384			rc = blake2s_init(&bctx.ss, ses->mlen);
385		if (rc != 0)
386			goto out;
387		error = crypto_apply(crp, crp->crp_payload_start,
388		    crp->crp_payload_length, blake2s_applicator, &bctx.ss);
389		if (error != 0)
390			goto out;
391		rc = blake2s_final(&bctx.ss, res, ses->mlen);
392		if (rc != 0) {
393			error = EINVAL;
394			goto out;
395		}
396		break;
397	default:
398		panic("unreachable");
399	}
400
401	if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) {
402		crypto_copydata(crp, crp->crp_digest_start, ses->mlen, res2);
403		if (timingsafe_bcmp(res, res2, ses->mlen) != 0)
404			return (EBADMSG);
405	} else
406		crypto_copyback(crp, crp->crp_digest_start, ses->mlen, res);
407
408out:
409	if (!kt) {
410		fpu_kern_leave(curthread, ctx);
411		RELEASE_CTX(ctxidx, ctx);
412	}
413	return (error);
414}
415