aesni.c revision 267815 
1/*-
2 * Copyright (c) 2005-2008 Pawel Jakub Dawidek <pjd@FreeBSD.org>
3 * Copyright (c) 2010 Konstantin Belousov <kib@FreeBSD.org>
4 * All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 *    notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 *    notice, this list of conditions and the following disclaimer in the
13 *    documentation and/or other materials provided with the distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25 * SUCH DAMAGE.
26 */
27
28#include <sys/cdefs.h>
29__FBSDID("$FreeBSD: head/sys/crypto/aesni/aesni.c 267815 2014-06-24 06:55:49Z kib $");
30
31#include <sys/param.h>
32#include <sys/systm.h>
33#include <sys/kernel.h>
34#include <sys/kobj.h>
35#include <sys/libkern.h>
36#include <sys/lock.h>
37#include <sys/module.h>
38#include <sys/malloc.h>
39#include <sys/rwlock.h>
40#include <sys/bus.h>
41#include <sys/uio.h>
42#include <crypto/aesni/aesni.h>
43#include <cryptodev_if.h>
44
45struct aesni_softc {
46	int32_t cid;
47	uint32_t sid;
48	TAILQ_HEAD(aesni_sessions_head, aesni_session) sessions;
49	struct rwlock lock;
50};
51
52static int aesni_newsession(device_t, uint32_t *sidp, struct cryptoini *cri);
53static int aesni_freesession(device_t, uint64_t tid);
54static void aesni_freesession_locked(struct aesni_softc *sc,
55    struct aesni_session *ses);
56static int aesni_cipher_setup(struct aesni_session *ses,
57    struct cryptoini *encini);
58static int aesni_cipher_process(struct aesni_session *ses,
59    struct cryptodesc *enccrd, struct cryptop *crp);
60
61MALLOC_DEFINE(M_AESNI, "aesni_data", "AESNI Data");
62
63static void
64aesni_identify(driver_t *drv, device_t parent)
65{
66
67	/* NB: order 10 is so we get attached after h/w devices */
68	if (device_find_child(parent, "aesni", -1) == NULL &&
69	    BUS_ADD_CHILD(parent, 10, "aesni", -1) == 0)
70		panic("aesni: could not attach");
71}
72
73static int
74aesni_probe(device_t dev)
75{
76
77	if ((cpu_feature2 & CPUID2_AESNI) == 0) {
78		device_printf(dev, "No AESNI support.\n");
79		return (EINVAL);
80	}
81
82	if ((cpu_feature & CPUID_SSE2) == 0) {
83		device_printf(dev, "No SSE2 support but AESNI!?!\n");
84		return (EINVAL);
85	}
86
87	device_set_desc_copy(dev, "AES-CBC,AES-XTS");
88	return (0);
89}
90
91static int
92aesni_attach(device_t dev)
93{
94	struct aesni_softc *sc;
95
96	sc = device_get_softc(dev);
97	TAILQ_INIT(&sc->sessions);
98	sc->sid = 1;
99	sc->cid = crypto_get_driverid(dev, CRYPTOCAP_F_HARDWARE |
100	    CRYPTOCAP_F_SYNC);
101	if (sc->cid < 0) {
102		device_printf(dev, "Could not get crypto driver id.\n");
103		return (ENOMEM);
104	}
105
106	rw_init(&sc->lock, "aesni_lock");
107	crypto_register(sc->cid, CRYPTO_AES_CBC, 0, 0);
108	crypto_register(sc->cid, CRYPTO_AES_XTS, 0, 0);
109	return (0);
110}
111
112static int
113aesni_detach(device_t dev)
114{
115	struct aesni_softc *sc;
116	struct aesni_session *ses;
117
118	sc = device_get_softc(dev);
119	rw_wlock(&sc->lock);
120	TAILQ_FOREACH(ses, &sc->sessions, next) {
121		if (ses->used) {
122			rw_wunlock(&sc->lock);
123			device_printf(dev,
124			    "Cannot detach, sessions still active.\n");
125			return (EBUSY);
126		}
127	}
128	while ((ses = TAILQ_FIRST(&sc->sessions)) != NULL) {
129		TAILQ_REMOVE(&sc->sessions, ses, next);
130		fpu_kern_free_ctx(ses->fpu_ctx);
131		free(ses, M_AESNI);
132	}
133	rw_wunlock(&sc->lock);
134	rw_destroy(&sc->lock);
135	crypto_unregister_all(sc->cid);
136	return (0);
137}
138
139static int
140aesni_newsession(device_t dev, uint32_t *sidp, struct cryptoini *cri)
141{
142	struct aesni_softc *sc;
143	struct aesni_session *ses;
144	struct cryptoini *encini;
145	int error;
146
147	if (sidp == NULL || cri == NULL)
148		return (EINVAL);
149
150	sc = device_get_softc(dev);
151	ses = NULL;
152	encini = NULL;
153	for (; cri != NULL; cri = cri->cri_next) {
154		switch (cri->cri_alg) {
155		case CRYPTO_AES_CBC:
156		case CRYPTO_AES_XTS:
157			if (encini != NULL)
158				return (EINVAL);
159			encini = cri;
160			break;
161		default:
162			return (EINVAL);
163		}
164	}
165	if (encini == NULL)
166		return (EINVAL);
167
168	rw_wlock(&sc->lock);
169	/*
170	 * Free sessions goes first, so if first session is used, we need to
171	 * allocate one.
172	 */
173	ses = TAILQ_FIRST(&sc->sessions);
174	if (ses == NULL || ses->used) {
175		ses = malloc(sizeof(*ses), M_AESNI, M_NOWAIT | M_ZERO);
176		if (ses == NULL) {
177			rw_wunlock(&sc->lock);
178			return (ENOMEM);
179		}
180		ses->fpu_ctx = fpu_kern_alloc_ctx(FPU_KERN_NORMAL |
181		    FPU_KERN_NOWAIT);
182		if (ses->fpu_ctx == NULL) {
183			free(ses, M_AESNI);
184			rw_wunlock(&sc->lock);
185			return (ENOMEM);
186		}
187		ses->id = sc->sid++;
188	} else {
189		TAILQ_REMOVE(&sc->sessions, ses, next);
190	}
191	ses->used = 1;
192	TAILQ_INSERT_TAIL(&sc->sessions, ses, next);
193	rw_wunlock(&sc->lock);
194	ses->algo = encini->cri_alg;
195
196	error = aesni_cipher_setup(ses, encini);
197	if (error != 0) {
198		rw_wlock(&sc->lock);
199		aesni_freesession_locked(sc, ses);
200		rw_wunlock(&sc->lock);
201		return (error);
202	}
203
204	*sidp = ses->id;
205	return (0);
206}
207
208static void
209aesni_freesession_locked(struct aesni_softc *sc, struct aesni_session *ses)
210{
211	struct fpu_kern_ctx *ctx;
212	uint32_t sid;
213
214	sid = ses->id;
215	TAILQ_REMOVE(&sc->sessions, ses, next);
216	ctx = ses->fpu_ctx;
217	bzero(ses, sizeof(*ses));
218	ses->id = sid;
219	ses->fpu_ctx = ctx;
220	TAILQ_INSERT_HEAD(&sc->sessions, ses, next);
221}
222
223static int
224aesni_freesession(device_t dev, uint64_t tid)
225{
226	struct aesni_softc *sc;
227	struct aesni_session *ses;
228	uint32_t sid;
229
230	sc = device_get_softc(dev);
231	sid = ((uint32_t)tid) & 0xffffffff;
232	rw_wlock(&sc->lock);
233	TAILQ_FOREACH_REVERSE(ses, &sc->sessions, aesni_sessions_head, next) {
234		if (ses->id == sid)
235			break;
236	}
237	if (ses == NULL) {
238		rw_wunlock(&sc->lock);
239		return (EINVAL);
240	}
241	aesni_freesession_locked(sc, ses);
242	rw_wunlock(&sc->lock);
243	return (0);
244}
245
246static int
247aesni_process(device_t dev, struct cryptop *crp, int hint __unused)
248{
249	struct aesni_softc *sc = device_get_softc(dev);
250	struct aesni_session *ses = NULL;
251	struct cryptodesc *crd, *enccrd;
252	int error;
253
254	error = 0;
255	enccrd = NULL;
256
257	/* Sanity check. */
258	if (crp == NULL)
259		return (EINVAL);
260
261	if (crp->crp_callback == NULL || crp->crp_desc == NULL) {
262		error = EINVAL;
263		goto out;
264	}
265
266	for (crd = crp->crp_desc; crd != NULL; crd = crd->crd_next) {
267		switch (crd->crd_alg) {
268		case CRYPTO_AES_CBC:
269		case CRYPTO_AES_XTS:
270			if (enccrd != NULL) {
271				error = EINVAL;
272				goto out;
273			}
274			enccrd = crd;
275			break;
276		default:
277			return (EINVAL);
278		}
279	}
280	if (enccrd == NULL || (enccrd->crd_len % AES_BLOCK_LEN) != 0) {
281		error = EINVAL;
282		goto out;
283	}
284
285	rw_rlock(&sc->lock);
286	TAILQ_FOREACH_REVERSE(ses, &sc->sessions, aesni_sessions_head, next) {
287		if (ses->id == (crp->crp_sid & 0xffffffff))
288			break;
289	}
290	rw_runlock(&sc->lock);
291	if (ses == NULL) {
292		error = EINVAL;
293		goto out;
294	}
295
296	error = aesni_cipher_process(ses, enccrd, crp);
297	if (error != 0)
298		goto out;
299
300out:
301	crp->crp_etype = error;
302	crypto_done(crp);
303	return (error);
304}
305
306uint8_t *
307aesni_cipher_alloc(struct cryptodesc *enccrd, struct cryptop *crp,
308    int *allocated)
309{
310	struct uio *uio;
311	struct iovec *iov;
312	uint8_t *addr;
313
314	if (crp->crp_flags & CRYPTO_F_IMBUF)
315		goto alloc;
316	else if (crp->crp_flags & CRYPTO_F_IOV) {
317		uio = (struct uio *)crp->crp_buf;
318		if (uio->uio_iovcnt != 1)
319			goto alloc;
320		iov = uio->uio_iov;
321		addr = (u_char *)iov->iov_base + enccrd->crd_skip;
322	} else
323		addr = (u_char *)crp->crp_buf;
324	*allocated = 0;
325	return (addr);
326
327alloc:
328	addr = malloc(enccrd->crd_len, M_AESNI, M_NOWAIT);
329	if (addr != NULL) {
330		*allocated = 1;
331		crypto_copydata(crp->crp_flags, crp->crp_buf, enccrd->crd_skip,
332		    enccrd->crd_len, addr);
333	} else
334		*allocated = 0;
335	return (addr);
336}
337
338static device_method_t aesni_methods[] = {
339	DEVMETHOD(device_identify, aesni_identify),
340	DEVMETHOD(device_probe, aesni_probe),
341	DEVMETHOD(device_attach, aesni_attach),
342	DEVMETHOD(device_detach, aesni_detach),
343
344	DEVMETHOD(cryptodev_newsession, aesni_newsession),
345	DEVMETHOD(cryptodev_freesession, aesni_freesession),
346	DEVMETHOD(cryptodev_process, aesni_process),
347
348	{0, 0},
349};
350
351static driver_t aesni_driver = {
352	"aesni",
353	aesni_methods,
354	sizeof(struct aesni_softc),
355};
356static devclass_t aesni_devclass;
357
358DRIVER_MODULE(aesni, nexus, aesni_driver, aesni_devclass, 0, 0);
359MODULE_VERSION(aesni, 1);
360MODULE_DEPEND(aesni, crypto, 1, 1, 1);
361
362static int
363aesni_cipher_setup(struct aesni_session *ses, struct cryptoini *encini)
364{
365	struct thread *td;
366	int error;
367
368	td = curthread;
369	error = fpu_kern_enter(td, ses->fpu_ctx, FPU_KERN_NORMAL |
370	    FPU_KERN_KTHR);
371	if (error != 0)
372		return (error);
373	error = aesni_cipher_setup_common(ses, encini->cri_key,
374	    encini->cri_klen);
375	fpu_kern_leave(td, ses->fpu_ctx);
376	return (error);
377}
378
379static int
380aesni_cipher_process(struct aesni_session *ses, struct cryptodesc *enccrd,
381    struct cryptop *crp)
382{
383	struct thread *td;
384	uint8_t *buf;
385	int error, allocated;
386
387	buf = aesni_cipher_alloc(enccrd, crp, &allocated);
388	if (buf == NULL)
389		return (ENOMEM);
390
391	td = curthread;
392	error = fpu_kern_enter(td, ses->fpu_ctx, FPU_KERN_NORMAL |
393	    FPU_KERN_KTHR);
394	if (error != 0)
395		goto out1;
396
397	if ((enccrd->crd_flags & CRD_F_KEY_EXPLICIT) != 0) {
398		error = aesni_cipher_setup_common(ses, enccrd->crd_key,
399		    enccrd->crd_klen);
400		if (error != 0)
401			goto out;
402	}
403
404	if ((enccrd->crd_flags & CRD_F_ENCRYPT) != 0) {
405		if ((enccrd->crd_flags & CRD_F_IV_EXPLICIT) != 0)
406			bcopy(enccrd->crd_iv, ses->iv, AES_BLOCK_LEN);
407		if ((enccrd->crd_flags & CRD_F_IV_PRESENT) == 0)
408			crypto_copyback(crp->crp_flags, crp->crp_buf,
409			    enccrd->crd_inject, AES_BLOCK_LEN, ses->iv);
410		if (ses->algo == CRYPTO_AES_CBC) {
411			aesni_encrypt_cbc(ses->rounds, ses->enc_schedule,
412			    enccrd->crd_len, buf, buf, ses->iv);
413		} else /* if (ses->algo == CRYPTO_AES_XTS) */ {
414			aesni_encrypt_xts(ses->rounds, ses->enc_schedule,
415			    ses->xts_schedule, enccrd->crd_len, buf, buf,
416			    ses->iv);
417		}
418	} else {
419		if ((enccrd->crd_flags & CRD_F_IV_EXPLICIT) != 0)
420			bcopy(enccrd->crd_iv, ses->iv, AES_BLOCK_LEN);
421		else
422			crypto_copydata(crp->crp_flags, crp->crp_buf,
423			    enccrd->crd_inject, AES_BLOCK_LEN, ses->iv);
424		if (ses->algo == CRYPTO_AES_CBC) {
425			aesni_decrypt_cbc(ses->rounds, ses->dec_schedule,
426			    enccrd->crd_len, buf, ses->iv);
427		} else /* if (ses->algo == CRYPTO_AES_XTS) */ {
428			aesni_decrypt_xts(ses->rounds, ses->dec_schedule,
429			    ses->xts_schedule, enccrd->crd_len, buf, buf,
430			    ses->iv);
431		}
432	}
433	if (allocated)
434		crypto_copyback(crp->crp_flags, crp->crp_buf, enccrd->crd_skip,
435		    enccrd->crd_len, buf);
436	if ((enccrd->crd_flags & CRD_F_ENCRYPT) != 0)
437		crypto_copydata(crp->crp_flags, crp->crp_buf,
438		    enccrd->crd_skip + enccrd->crd_len - AES_BLOCK_LEN,
439		    AES_BLOCK_LEN, ses->iv);
440out:
441	fpu_kern_leave(td, ses->fpu_ctx);
442out1:
443	if (allocated) {
444		bzero(buf, enccrd->crd_len);
445		free(buf, M_AESNI);
446	}
447	return (error);
448}
449