1/*-
2 * Copyright (c) 2005-2008 Pawel Jakub Dawidek <pjd@FreeBSD.org>
3 * Copyright (c) 2010 Konstantin Belousov <kib@FreeBSD.org>
4 * Copyright (c) 2014,2016 The FreeBSD Foundation
5 * Copyright (c) 2020 Ampere Computing
6 * All rights reserved.
7 *
8 * Portions of this software were developed by John-Mark Gurney
9 * under sponsorship of the FreeBSD Foundation and
10 * Rubicon Communications, LLC (Netgate).
11 *
12 * This software was developed by Andrew Turner under
13 * sponsorship from the FreeBSD Foundation.
14 *
15 * Redistribution and use in source and binary forms, with or without
16 * modification, are permitted provided that the following conditions
17 * are met:
18 * 1. Redistributions of source code must retain the above copyright
19 *    notice, this list of conditions and the following disclaimer.
20 * 2. Redistributions in binary form must reproduce the above copyright
21 *    notice, this list of conditions and the following disclaimer in the
22 *    documentation and/or other materials provided with the distribution.
23 *
24 * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
25 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
26 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
27 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
28 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
29 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
30 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
31 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
32 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
33 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
34 * SUCH DAMAGE.
35 */
36
37/*
38 * This is based on the aesni code.
39 */
40
41#include <sys/cdefs.h>
42__FBSDID("$FreeBSD$");
43
44#include <sys/param.h>
45#include <sys/systm.h>
46#include <sys/kernel.h>
47#include <sys/bus.h>
48#include <sys/endian.h>
49#include <sys/lock.h>
50#include <sys/malloc.h>
51#include <sys/mbuf.h>
52#include <sys/module.h>
53#include <sys/mutex.h>
54#include <sys/queue.h>
55#include <sys/rwlock.h>
56#include <sys/smp.h>
57#include <sys/uio.h>
58
59#include <machine/vfp.h>
60
61#include <opencrypto/cryptodev.h>
62#include <opencrypto/gmac.h>
63#include <cryptodev_if.h>
64#include <crypto/armv8/armv8_crypto.h>
65#include <crypto/rijndael/rijndael.h>
66
67struct armv8_crypto_softc {
68	int		dieing;
69	int32_t		cid;
70	struct rwlock	lock;
71	bool		has_pmul;
72};
73
74static struct mtx *ctx_mtx;
75static struct fpu_kern_ctx **ctx_vfp;
76
77#define AQUIRE_CTX(i, ctx)					\
78	do {							\
79		(i) = PCPU_GET(cpuid);				\
80		mtx_lock(&ctx_mtx[(i)]);			\
81		(ctx) = ctx_vfp[(i)];				\
82	} while (0)
83#define RELEASE_CTX(i, ctx)					\
84	do {							\
85		mtx_unlock(&ctx_mtx[(i)]);			\
86		(i) = -1;					\
87		(ctx) = NULL;					\
88	} while (0)
89
90static int armv8_crypto_cipher_process(struct armv8_crypto_session *,
91    struct cryptop *);
92
93MALLOC_DEFINE(M_ARMV8_CRYPTO, "armv8_crypto", "ARMv8 Crypto Data");
94
95static void
96armv8_crypto_identify(driver_t *drv, device_t parent)
97{
98
99	/* NB: order 10 is so we get attached after h/w devices */
100	if (device_find_child(parent, "armv8crypto", -1) == NULL &&
101	    BUS_ADD_CHILD(parent, 10, "armv8crypto", -1) == 0)
102		panic("ARMv8 crypto: could not attach");
103}
104
105static int
106armv8_crypto_probe(device_t dev)
107{
108	uint64_t reg;
109	int ret = ENXIO;
110
111	reg = READ_SPECIALREG(id_aa64isar0_el1);
112
113	switch (ID_AA64ISAR0_AES_VAL(reg)) {
114	case ID_AA64ISAR0_AES_BASE:
115		ret = 0;
116		device_set_desc(dev, "AES-CBC,AES-XTS");
117		break;
118	case ID_AA64ISAR0_AES_PMULL:
119		ret = 0;
120		device_set_desc(dev, "AES-CBC,AES-XTS,AES-GCM");
121		break;
122	default:
123		break;
124	case ID_AA64ISAR0_AES_NONE:
125		device_printf(dev, "CPU lacks AES instructions\n");
126		break;
127	}
128
129	/* TODO: Check more fields as we support more features */
130
131	return (ret);
132}
133
134static int
135armv8_crypto_attach(device_t dev)
136{
137	struct armv8_crypto_softc *sc;
138	uint64_t reg;
139	int i;
140
141	sc = device_get_softc(dev);
142	sc->dieing = 0;
143
144	reg = READ_SPECIALREG(id_aa64isar0_el1);
145
146	if (ID_AA64ISAR0_AES_VAL(reg) == ID_AA64ISAR0_AES_PMULL)
147		sc->has_pmul = true;
148
149	sc->cid = crypto_get_driverid(dev, sizeof(struct armv8_crypto_session),
150	    CRYPTOCAP_F_SOFTWARE | CRYPTOCAP_F_SYNC | CRYPTOCAP_F_ACCEL_SOFTWARE);
151	if (sc->cid < 0) {
152		device_printf(dev, "Could not get crypto driver id.\n");
153		return (ENOMEM);
154	}
155
156	rw_init(&sc->lock, "armv8crypto");
157
158	ctx_mtx = malloc(sizeof(*ctx_mtx) * (mp_maxid + 1), M_ARMV8_CRYPTO,
159	    M_WAITOK|M_ZERO);
160	ctx_vfp = malloc(sizeof(*ctx_vfp) * (mp_maxid + 1), M_ARMV8_CRYPTO,
161	    M_WAITOK|M_ZERO);
162
163	CPU_FOREACH(i) {
164		ctx_vfp[i] = fpu_kern_alloc_ctx(0);
165		mtx_init(&ctx_mtx[i], "armv8cryptoctx", NULL, MTX_DEF|MTX_NEW);
166	}
167
168	return (0);
169}
170
171static int
172armv8_crypto_detach(device_t dev)
173{
174	struct armv8_crypto_softc *sc;
175	int i;
176
177	sc = device_get_softc(dev);
178
179	rw_wlock(&sc->lock);
180	sc->dieing = 1;
181	rw_wunlock(&sc->lock);
182	crypto_unregister_all(sc->cid);
183
184	rw_destroy(&sc->lock);
185
186	CPU_FOREACH(i) {
187		if (ctx_vfp[i] != NULL) {
188			mtx_destroy(&ctx_mtx[i]);
189			fpu_kern_free_ctx(ctx_vfp[i]);
190		}
191		ctx_vfp[i] = NULL;
192	}
193	free(ctx_mtx, M_ARMV8_CRYPTO);
194	ctx_mtx = NULL;
195	free(ctx_vfp, M_ARMV8_CRYPTO);
196	ctx_vfp = NULL;
197
198	return (0);
199}
200
201#define SUPPORTED_SES (CSP_F_SEPARATE_OUTPUT | CSP_F_SEPARATE_AAD)
202
203static int
204armv8_crypto_probesession(device_t dev,
205    const struct crypto_session_params *csp)
206{
207	struct armv8_crypto_softc *sc;
208
209	sc = device_get_softc(dev);
210
211	if ((csp->csp_flags & ~(SUPPORTED_SES)) != 0)
212		return (EINVAL);
213
214	switch (csp->csp_mode) {
215	case CSP_MODE_AEAD:
216		switch (csp->csp_cipher_alg) {
217		case CRYPTO_AES_NIST_GCM_16:
218			if (!sc->has_pmul)
219				return (EINVAL);
220			if (csp->csp_ivlen != AES_GCM_IV_LEN)
221				return (EINVAL);
222			if (csp->csp_auth_mlen != 0 &&
223			    csp->csp_auth_mlen != GMAC_DIGEST_LEN)
224				return (EINVAL);
225			switch (csp->csp_cipher_klen * 8) {
226			case 128:
227			case 192:
228			case 256:
229				break;
230			default:
231				return (EINVAL);
232			}
233			break;
234		default:
235			return (EINVAL);
236		}
237		break;
238	case CSP_MODE_CIPHER:
239		switch (csp->csp_cipher_alg) {
240		case CRYPTO_AES_CBC:
241			if (csp->csp_ivlen != AES_BLOCK_LEN)
242				return (EINVAL);
243			switch (csp->csp_cipher_klen * 8) {
244			case 128:
245			case 192:
246			case 256:
247				break;
248			default:
249				return (EINVAL);
250			}
251			break;
252		case CRYPTO_AES_XTS:
253			if (csp->csp_ivlen != AES_XTS_IV_LEN)
254				return (EINVAL);
255			switch (csp->csp_cipher_klen * 8) {
256			case 256:
257			case 512:
258				break;
259			default:
260				return (EINVAL);
261			}
262			break;
263		default:
264			return (EINVAL);
265		}
266		break;
267	default:
268		return (EINVAL);
269	}
270	return (CRYPTODEV_PROBE_ACCEL_SOFTWARE);
271}
272
273static int
274armv8_crypto_cipher_setup(struct armv8_crypto_session *ses,
275    const struct crypto_session_params *csp, const uint8_t *key, int keylen)
276{
277	__uint128_val_t H;
278	struct fpu_kern_ctx *ctx;
279	int kt, i;
280
281	if (csp->csp_cipher_alg == CRYPTO_AES_XTS)
282		keylen /= 2;
283
284	switch (keylen * 8) {
285	case 128:
286	case 192:
287	case 256:
288		break;
289	default:
290		return (EINVAL);
291	}
292
293	kt = is_fpu_kern_thread(0);
294	if (!kt) {
295		AQUIRE_CTX(i, ctx);
296		fpu_kern_enter(curthread, ctx,
297		    FPU_KERN_NORMAL | FPU_KERN_KTHR);
298	}
299
300	aes_v8_set_encrypt_key(key,
301	    keylen * 8, &ses->enc_schedule);
302
303	if ((csp->csp_cipher_alg == CRYPTO_AES_XTS) ||
304	    (csp->csp_cipher_alg == CRYPTO_AES_CBC))
305		aes_v8_set_decrypt_key(key,
306		    keylen * 8, &ses->dec_schedule);
307
308	if (csp->csp_cipher_alg == CRYPTO_AES_XTS)
309		aes_v8_set_encrypt_key(key + keylen, keylen * 8, &ses->xts_schedule);
310
311	if (csp->csp_cipher_alg == CRYPTO_AES_NIST_GCM_16) {
312		memset(H.c, 0, sizeof(H.c));
313		aes_v8_encrypt(H.c, H.c, &ses->enc_schedule);
314		H.u[0] = bswap64(H.u[0]);
315		H.u[1] = bswap64(H.u[1]);
316		gcm_init_v8(ses->Htable, H.u);
317	}
318
319	if (!kt) {
320		fpu_kern_leave(curthread, ctx);
321		RELEASE_CTX(i, ctx);
322	}
323
324	return (0);
325}
326
327static int
328armv8_crypto_newsession(device_t dev, crypto_session_t cses,
329    const struct crypto_session_params *csp)
330{
331	struct armv8_crypto_softc *sc;
332	struct armv8_crypto_session *ses;
333	int error;
334
335	sc = device_get_softc(dev);
336	rw_wlock(&sc->lock);
337	if (sc->dieing) {
338		rw_wunlock(&sc->lock);
339		return (EINVAL);
340	}
341
342	ses = crypto_get_driver_session(cses);
343	error = armv8_crypto_cipher_setup(ses, csp, csp->csp_cipher_key,
344	    csp->csp_cipher_klen);
345	rw_wunlock(&sc->lock);
346	return (error);
347}
348
349static int
350armv8_crypto_process(device_t dev, struct cryptop *crp, int hint __unused)
351{
352	struct armv8_crypto_session *ses;
353
354	ses = crypto_get_driver_session(crp->crp_session);
355	crp->crp_etype = armv8_crypto_cipher_process(ses, crp);
356	crypto_done(crp);
357	return (0);
358}
359
360static uint8_t *
361armv8_crypto_cipher_alloc(struct cryptop *crp, int start, int length, int *allocated)
362{
363	uint8_t *addr;
364
365	addr = crypto_contiguous_subsegment(crp, start, length);
366	if (addr != NULL) {
367		*allocated = 0;
368		return (addr);
369	}
370	addr = malloc(crp->crp_payload_length, M_ARMV8_CRYPTO, M_NOWAIT);
371	if (addr != NULL) {
372		*allocated = 1;
373		crypto_copydata(crp, start, length, addr);
374	} else
375		*allocated = 0;
376	return (addr);
377}
378
379static int
380armv8_crypto_cipher_process(struct armv8_crypto_session *ses,
381    struct cryptop *crp)
382{
383	const struct crypto_session_params *csp;
384	struct fpu_kern_ctx *ctx;
385	uint8_t *buf, *authbuf, *outbuf;
386	uint8_t iv[AES_BLOCK_LEN], tag[GMAC_DIGEST_LEN];
387	int allocated, authallocated, outallocated, i;
388	int encflag;
389	int kt;
390	int error;
391	bool outcopy;
392
393	csp = crypto_get_params(crp->crp_session);
394	encflag = CRYPTO_OP_IS_ENCRYPT(crp->crp_op);
395
396	allocated = 0;
397	outallocated = 0;
398	authallocated = 0;
399	authbuf = NULL;
400	kt = 1;
401
402	buf = armv8_crypto_cipher_alloc(crp, crp->crp_payload_start,
403	    crp->crp_payload_length, &allocated);
404	if (buf == NULL)
405		return (ENOMEM);
406
407	if (csp->csp_cipher_alg == CRYPTO_AES_NIST_GCM_16) {
408		if (crp->crp_aad != NULL)
409			authbuf = crp->crp_aad;
410		else
411			authbuf = armv8_crypto_cipher_alloc(crp, crp->crp_aad_start,
412			    crp->crp_aad_length, &authallocated);
413		if (authbuf == NULL) {
414			error = ENOMEM;
415			goto out;
416		}
417	}
418
419	if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) {
420		outbuf = crypto_buffer_contiguous_subsegment(&crp->crp_obuf,
421		    crp->crp_payload_output_start, crp->crp_payload_length);
422		if (outbuf == NULL) {
423			outcopy = true;
424			if (allocated)
425				outbuf = buf;
426			else {
427				outbuf = malloc(crp->crp_payload_length,
428				    M_ARMV8_CRYPTO, M_NOWAIT);
429				if (outbuf == NULL) {
430					error = ENOMEM;
431					goto out;
432				}
433				outallocated = true;
434			}
435		} else
436			outcopy = false;
437	} else {
438		outbuf = buf;
439		outcopy = allocated;
440	}
441
442	kt = is_fpu_kern_thread(0);
443	if (!kt) {
444		AQUIRE_CTX(i, ctx);
445		fpu_kern_enter(curthread, ctx,
446		    FPU_KERN_NORMAL | FPU_KERN_KTHR);
447	}
448
449	if (crp->crp_cipher_key != NULL) {
450		armv8_crypto_cipher_setup(ses, csp, crp->crp_cipher_key,
451		    csp->csp_cipher_klen);
452	}
453
454	crypto_read_iv(crp, iv);
455
456	/* Do work */
457	switch (csp->csp_cipher_alg) {
458	case CRYPTO_AES_CBC:
459		if ((crp->crp_payload_length % AES_BLOCK_LEN) != 0) {
460			error = EINVAL;
461			goto out;
462		}
463		if (encflag)
464			armv8_aes_encrypt_cbc(&ses->enc_schedule,
465			    crp->crp_payload_length, buf, buf, iv);
466		else
467			armv8_aes_decrypt_cbc(&ses->dec_schedule,
468			    crp->crp_payload_length, buf, iv);
469		break;
470	case CRYPTO_AES_XTS:
471		if (encflag)
472			armv8_aes_encrypt_xts(&ses->enc_schedule,
473			    &ses->xts_schedule.aes_key, crp->crp_payload_length, buf,
474			    buf, iv);
475		else
476			armv8_aes_decrypt_xts(&ses->dec_schedule,
477			    &ses->xts_schedule.aes_key, crp->crp_payload_length, buf,
478			    buf, iv);
479		break;
480	case CRYPTO_AES_NIST_GCM_16:
481		if (encflag) {
482			memset(tag, 0, sizeof(tag));
483			armv8_aes_encrypt_gcm(&ses->enc_schedule,
484			    crp->crp_payload_length,
485			    buf, outbuf,
486			    crp->crp_aad_length, authbuf,
487			    tag, iv, ses->Htable);
488			crypto_copyback(crp, crp->crp_digest_start, sizeof(tag),
489			    tag);
490		} else {
491			crypto_copydata(crp, crp->crp_digest_start, sizeof(tag),
492			    tag);
493			if (armv8_aes_decrypt_gcm(&ses->enc_schedule,
494			    crp->crp_payload_length,
495			    buf, outbuf,
496			    crp->crp_aad_length, authbuf,
497			    tag, iv, ses->Htable) != 0) {
498				error = EBADMSG;
499				goto out;
500			}
501		}
502		break;
503	}
504
505	if (outcopy)
506		crypto_copyback(crp, CRYPTO_HAS_OUTPUT_BUFFER(crp) ?
507		    crp->crp_payload_output_start : crp->crp_payload_start,
508		    crp->crp_payload_length, outbuf);
509
510	error = 0;
511out:
512	if (!kt) {
513		fpu_kern_leave(curthread, ctx);
514		RELEASE_CTX(i, ctx);
515	}
516
517	if (allocated)
518		zfree(buf, M_ARMV8_CRYPTO);
519	if (authallocated)
520		zfree(authbuf, M_ARMV8_CRYPTO);
521	if (outallocated)
522		zfree(outbuf, M_ARMV8_CRYPTO);
523	explicit_bzero(iv, sizeof(iv));
524	explicit_bzero(tag, sizeof(tag));
525
526	return (error);
527}
528
529static device_method_t armv8_crypto_methods[] = {
530	DEVMETHOD(device_identify,	armv8_crypto_identify),
531	DEVMETHOD(device_probe,		armv8_crypto_probe),
532	DEVMETHOD(device_attach,	armv8_crypto_attach),
533	DEVMETHOD(device_detach,	armv8_crypto_detach),
534
535	DEVMETHOD(cryptodev_probesession, armv8_crypto_probesession),
536	DEVMETHOD(cryptodev_newsession,	armv8_crypto_newsession),
537	DEVMETHOD(cryptodev_process,	armv8_crypto_process),
538
539	DEVMETHOD_END,
540};
541
542static DEFINE_CLASS_0(armv8crypto, armv8_crypto_driver, armv8_crypto_methods,
543    sizeof(struct armv8_crypto_softc));
544static devclass_t armv8_crypto_devclass;
545
546DRIVER_MODULE(armv8crypto, nexus, armv8_crypto_driver, armv8_crypto_devclass,
547    0, 0);
548