1/*-
2 * Copyright (c) 2005-2008 Pawel Jakub Dawidek <pjd@FreeBSD.org>
3 * Copyright (c) 2010 Konstantin Belousov <kib@FreeBSD.org>
4 * Copyright (c) 2014,2016 The FreeBSD Foundation
5 * Copyright (c) 2020 Ampere Computing
6 * All rights reserved.
7 *
8 * Portions of this software were developed by John-Mark Gurney
9 * under sponsorship of the FreeBSD Foundation and
10 * Rubicon Communications, LLC (Netgate).
11 *
12 * This software was developed by Andrew Turner under
13 * sponsorship from the FreeBSD Foundation.
14 *
15 * Redistribution and use in source and binary forms, with or without
16 * modification, are permitted provided that the following conditions
17 * are met:
18 * 1. Redistributions of source code must retain the above copyright
19 *    notice, this list of conditions and the following disclaimer.
20 * 2. Redistributions in binary form must reproduce the above copyright
21 *    notice, this list of conditions and the following disclaimer in the
22 *    documentation and/or other materials provided with the distribution.
23 *
24 * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
25 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
26 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
27 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
28 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
29 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
30 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
31 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
32 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
33 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
34 * SUCH DAMAGE.
35 */
36
37/*
38 * This is based on the aesni code.
39 */
40
41#include <sys/param.h>
42#include <sys/systm.h>
43#include <sys/kernel.h>
44#include <sys/bus.h>
45#include <sys/endian.h>
46#include <sys/malloc.h>
47#include <sys/mbuf.h>
48#include <sys/module.h>
49#include <sys/queue.h>
50#include <sys/smp.h>
51#include <sys/uio.h>
52
53#include <machine/vfp.h>
54
55#include <opencrypto/cryptodev.h>
56#include <opencrypto/gmac.h>
57#include <cryptodev_if.h>
58#include <crypto/armv8/armv8_crypto.h>
59#include <crypto/rijndael/rijndael.h>
60
61struct armv8_crypto_softc {
62	int32_t		cid;
63	bool		has_pmul;
64};
65
66static int armv8_crypto_cipher_process(struct armv8_crypto_session *,
67    struct cryptop *);
68
69MALLOC_DEFINE(M_ARMV8_CRYPTO, "armv8_crypto", "ARMv8 Crypto Data");
70
71static void
72armv8_crypto_identify(driver_t *drv, device_t parent)
73{
74
75	/* NB: order 10 is so we get attached after h/w devices */
76	if (device_find_child(parent, "armv8crypto", -1) == NULL &&
77	    BUS_ADD_CHILD(parent, 10, "armv8crypto", -1) == 0)
78		panic("ARMv8 crypto: could not attach");
79}
80
81static int
82armv8_crypto_probe(device_t dev)
83{
84	uint64_t reg;
85	int ret = ENXIO;
86
87	reg = READ_SPECIALREG(id_aa64isar0_el1);
88
89	switch (ID_AA64ISAR0_AES_VAL(reg)) {
90	case ID_AA64ISAR0_AES_BASE:
91		ret = 0;
92		device_set_desc(dev, "AES-CBC,AES-XTS");
93		break;
94	case ID_AA64ISAR0_AES_PMULL:
95		ret = 0;
96		device_set_desc(dev, "AES-CBC,AES-XTS,AES-GCM");
97		break;
98	default:
99		break;
100	case ID_AA64ISAR0_AES_NONE:
101		device_printf(dev, "CPU lacks AES instructions\n");
102		break;
103	}
104
105	/* TODO: Check more fields as we support more features */
106
107	return (ret);
108}
109
110static int
111armv8_crypto_attach(device_t dev)
112{
113	struct armv8_crypto_softc *sc;
114	uint64_t reg;
115
116	sc = device_get_softc(dev);
117
118	reg = READ_SPECIALREG(id_aa64isar0_el1);
119
120	if (ID_AA64ISAR0_AES_VAL(reg) == ID_AA64ISAR0_AES_PMULL)
121		sc->has_pmul = true;
122
123	sc->cid = crypto_get_driverid(dev, sizeof(struct armv8_crypto_session),
124	    CRYPTOCAP_F_SOFTWARE | CRYPTOCAP_F_SYNC | CRYPTOCAP_F_ACCEL_SOFTWARE);
125	if (sc->cid < 0) {
126		device_printf(dev, "Could not get crypto driver id.\n");
127		return (ENOMEM);
128	}
129
130	return (0);
131}
132
133static int
134armv8_crypto_detach(device_t dev)
135{
136	struct armv8_crypto_softc *sc;
137
138	sc = device_get_softc(dev);
139
140	crypto_unregister_all(sc->cid);
141
142	return (0);
143}
144
145#define SUPPORTED_SES (CSP_F_SEPARATE_OUTPUT | CSP_F_SEPARATE_AAD)
146
147static int
148armv8_crypto_probesession(device_t dev,
149    const struct crypto_session_params *csp)
150{
151	struct armv8_crypto_softc *sc;
152
153	sc = device_get_softc(dev);
154
155	if ((csp->csp_flags & ~(SUPPORTED_SES)) != 0)
156		return (EINVAL);
157
158	switch (csp->csp_mode) {
159	case CSP_MODE_AEAD:
160		switch (csp->csp_cipher_alg) {
161		case CRYPTO_AES_NIST_GCM_16:
162			if (!sc->has_pmul)
163				return (EINVAL);
164			if (csp->csp_auth_mlen != 0 &&
165			    csp->csp_auth_mlen != GMAC_DIGEST_LEN)
166				return (EINVAL);
167			switch (csp->csp_cipher_klen * 8) {
168			case 128:
169			case 192:
170			case 256:
171				break;
172			default:
173				return (EINVAL);
174			}
175			break;
176		default:
177			return (EINVAL);
178		}
179		break;
180	case CSP_MODE_CIPHER:
181		switch (csp->csp_cipher_alg) {
182		case CRYPTO_AES_CBC:
183			if (csp->csp_ivlen != AES_BLOCK_LEN)
184				return (EINVAL);
185			switch (csp->csp_cipher_klen * 8) {
186			case 128:
187			case 192:
188			case 256:
189				break;
190			default:
191				return (EINVAL);
192			}
193			break;
194		case CRYPTO_AES_XTS:
195			if (csp->csp_ivlen != AES_XTS_IV_LEN)
196				return (EINVAL);
197			switch (csp->csp_cipher_klen * 8) {
198			case 256:
199			case 512:
200				break;
201			default:
202				return (EINVAL);
203			}
204			break;
205		default:
206			return (EINVAL);
207		}
208		break;
209	default:
210		return (EINVAL);
211	}
212	return (CRYPTODEV_PROBE_ACCEL_SOFTWARE);
213}
214
215static int
216armv8_crypto_cipher_setup(struct armv8_crypto_session *ses,
217    const struct crypto_session_params *csp, const uint8_t *key, int keylen)
218{
219	__uint128_val_t H;
220
221	if (csp->csp_cipher_alg == CRYPTO_AES_XTS)
222		keylen /= 2;
223
224	switch (keylen * 8) {
225	case 128:
226	case 192:
227	case 256:
228		break;
229	default:
230		return (EINVAL);
231	}
232
233	fpu_kern_enter(curthread, NULL, FPU_KERN_NORMAL | FPU_KERN_NOCTX);
234
235	aes_v8_set_encrypt_key(key,
236	    keylen * 8, &ses->enc_schedule);
237
238	if ((csp->csp_cipher_alg == CRYPTO_AES_XTS) ||
239	    (csp->csp_cipher_alg == CRYPTO_AES_CBC))
240		aes_v8_set_decrypt_key(key,
241		    keylen * 8, &ses->dec_schedule);
242
243	if (csp->csp_cipher_alg == CRYPTO_AES_XTS)
244		aes_v8_set_encrypt_key(key + keylen, keylen * 8, &ses->xts_schedule);
245
246	if (csp->csp_cipher_alg == CRYPTO_AES_NIST_GCM_16) {
247		memset(H.c, 0, sizeof(H.c));
248		aes_v8_encrypt(H.c, H.c, &ses->enc_schedule);
249		H.u[0] = bswap64(H.u[0]);
250		H.u[1] = bswap64(H.u[1]);
251		gcm_init_v8(ses->Htable, H.u);
252	}
253
254	fpu_kern_leave(curthread, NULL);
255
256	return (0);
257}
258
259static int
260armv8_crypto_newsession(device_t dev, crypto_session_t cses,
261    const struct crypto_session_params *csp)
262{
263	struct armv8_crypto_session *ses;
264	int error;
265
266	ses = crypto_get_driver_session(cses);
267	error = armv8_crypto_cipher_setup(ses, csp, csp->csp_cipher_key,
268	    csp->csp_cipher_klen);
269	return (error);
270}
271
272static int
273armv8_crypto_process(device_t dev, struct cryptop *crp, int hint __unused)
274{
275	struct armv8_crypto_session *ses;
276
277	ses = crypto_get_driver_session(crp->crp_session);
278	crp->crp_etype = armv8_crypto_cipher_process(ses, crp);
279	crypto_done(crp);
280	return (0);
281}
282
283static uint8_t *
284armv8_crypto_cipher_alloc(struct cryptop *crp, int start, int length, int *allocated)
285{
286	uint8_t *addr;
287
288	addr = crypto_contiguous_subsegment(crp, start, length);
289	if (addr != NULL) {
290		*allocated = 0;
291		return (addr);
292	}
293	addr = malloc(crp->crp_payload_length, M_ARMV8_CRYPTO, M_NOWAIT);
294	if (addr != NULL) {
295		*allocated = 1;
296		crypto_copydata(crp, start, length, addr);
297	} else
298		*allocated = 0;
299	return (addr);
300}
301
302static int
303armv8_crypto_cipher_process(struct armv8_crypto_session *ses,
304    struct cryptop *crp)
305{
306	struct crypto_buffer_cursor fromc, toc;
307	const struct crypto_session_params *csp;
308	uint8_t *authbuf;
309	uint8_t iv[AES_BLOCK_LEN], tag[GMAC_DIGEST_LEN];
310	int authallocated;
311	int encflag;
312	int error;
313
314	csp = crypto_get_params(crp->crp_session);
315	encflag = CRYPTO_OP_IS_ENCRYPT(crp->crp_op);
316
317	authallocated = 0;
318	authbuf = NULL;
319
320	if (csp->csp_cipher_alg == CRYPTO_AES_NIST_GCM_16) {
321		if (crp->crp_aad != NULL)
322			authbuf = crp->crp_aad;
323		else
324			authbuf = armv8_crypto_cipher_alloc(crp, crp->crp_aad_start,
325			    crp->crp_aad_length, &authallocated);
326		if (authbuf == NULL)
327			return (ENOMEM);
328	}
329	crypto_cursor_init(&fromc, &crp->crp_buf);
330	crypto_cursor_advance(&fromc, crp->crp_payload_start);
331	if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) {
332		crypto_cursor_init(&toc, &crp->crp_obuf);
333		crypto_cursor_advance(&toc, crp->crp_payload_output_start);
334	} else {
335		crypto_cursor_copy(&fromc, &toc);
336	}
337
338	if (crp->crp_cipher_key != NULL) {
339		armv8_crypto_cipher_setup(ses, csp, crp->crp_cipher_key,
340		    csp->csp_cipher_klen);
341	}
342
343	crypto_read_iv(crp, iv);
344
345	fpu_kern_enter(curthread, NULL, FPU_KERN_NORMAL | FPU_KERN_NOCTX);
346
347	error = 0;
348	switch (csp->csp_cipher_alg) {
349	case CRYPTO_AES_CBC:
350		if ((crp->crp_payload_length % AES_BLOCK_LEN) != 0) {
351			error = EINVAL;
352			break;
353		}
354		if (encflag)
355			armv8_aes_encrypt_cbc(&ses->enc_schedule,
356			    crp->crp_payload_length, &fromc, &toc, iv);
357		else
358			armv8_aes_decrypt_cbc(&ses->dec_schedule,
359			    crp->crp_payload_length, &fromc, &toc, iv);
360		break;
361	case CRYPTO_AES_XTS:
362		if (encflag)
363			armv8_aes_encrypt_xts(&ses->enc_schedule,
364			    &ses->xts_schedule.aes_key, crp->crp_payload_length,
365			    &fromc, &toc, iv);
366		else
367			armv8_aes_decrypt_xts(&ses->dec_schedule,
368			    &ses->xts_schedule.aes_key, crp->crp_payload_length,
369			    &fromc, &toc, iv);
370		break;
371	case CRYPTO_AES_NIST_GCM_16:
372		if (encflag) {
373			memset(tag, 0, sizeof(tag));
374			armv8_aes_encrypt_gcm(&ses->enc_schedule,
375			    crp->crp_payload_length, &fromc, &toc,
376			    crp->crp_aad_length, authbuf, tag, iv, ses->Htable);
377			crypto_copyback(crp, crp->crp_digest_start, sizeof(tag),
378			    tag);
379		} else {
380			crypto_copydata(crp, crp->crp_digest_start, sizeof(tag),
381			    tag);
382			error = armv8_aes_decrypt_gcm(&ses->enc_schedule,
383			    crp->crp_payload_length, &fromc, &toc,
384			    crp->crp_aad_length, authbuf, tag, iv, ses->Htable);
385		}
386		break;
387	}
388
389	fpu_kern_leave(curthread, NULL);
390
391	if (authallocated)
392		zfree(authbuf, M_ARMV8_CRYPTO);
393	explicit_bzero(iv, sizeof(iv));
394	explicit_bzero(tag, sizeof(tag));
395
396	return (error);
397}
398
399static device_method_t armv8_crypto_methods[] = {
400	DEVMETHOD(device_identify,	armv8_crypto_identify),
401	DEVMETHOD(device_probe,		armv8_crypto_probe),
402	DEVMETHOD(device_attach,	armv8_crypto_attach),
403	DEVMETHOD(device_detach,	armv8_crypto_detach),
404
405	DEVMETHOD(cryptodev_probesession, armv8_crypto_probesession),
406	DEVMETHOD(cryptodev_newsession,	armv8_crypto_newsession),
407	DEVMETHOD(cryptodev_process,	armv8_crypto_process),
408
409	DEVMETHOD_END,
410};
411
412static DEFINE_CLASS_0(armv8crypto, armv8_crypto_driver, armv8_crypto_methods,
413    sizeof(struct armv8_crypto_softc));
414
415DRIVER_MODULE(armv8crypto, nexus, armv8_crypto_driver, 0, 0);
416