aesni.c revision 258399 
1/*-
2 * Copyright (c) 2005-2008 Pawel Jakub Dawidek <pjd@FreeBSD.org>
3 * Copyright (c) 2010 Konstantin Belousov <kib@FreeBSD.org>
4 * All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 *    notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 *    notice, this list of conditions and the following disclaimer in the
13 *    documentation and/or other materials provided with the distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25 * SUCH DAMAGE.
26 */
27
28#include <sys/cdefs.h>
29__FBSDID("$FreeBSD: head/sys/crypto/aesni/aesni.c 258399 2013-11-20 20:25:27Z jmg $");
30
31#include <sys/param.h>
32#include <sys/systm.h>
33#include <sys/kernel.h>
34#include <sys/kobj.h>
35#include <sys/libkern.h>
36#include <sys/lock.h>
37#include <sys/module.h>
38#include <sys/malloc.h>
39#include <sys/rwlock.h>
40#include <sys/bus.h>
41#include <sys/uio.h>
42#include <crypto/aesni/aesni.h>
43#include <cryptodev_if.h>
44
45struct aesni_softc {
46	int32_t cid;
47	uint32_t sid;
48	TAILQ_HEAD(aesni_sessions_head, aesni_session) sessions;
49	struct rwlock lock;
50};
51
52static int aesni_newsession(device_t, uint32_t *sidp, struct cryptoini *cri);
53static int aesni_freesession(device_t, uint64_t tid);
54static void aesni_freesession_locked(struct aesni_softc *sc,
55    struct aesni_session *ses);
56
57MALLOC_DEFINE(M_AESNI, "aesni_data", "AESNI Data");
58
59static void
60aesni_identify(driver_t *drv, device_t parent)
61{
62
63	/* NB: order 10 is so we get attached after h/w devices */
64	if (device_find_child(parent, "aesni", -1) == NULL &&
65	    BUS_ADD_CHILD(parent, 10, "aesni", -1) == 0)
66		panic("aesni: could not attach");
67}
68
69static int
70aesni_probe(device_t dev)
71{
72
73	if ((cpu_feature2 & CPUID2_AESNI) == 0) {
74		device_printf(dev, "No AESNI support.\n");
75		return (EINVAL);
76	}
77
78	if ((cpu_feature & CPUID_SSE2) == 0) {
79		device_printf(dev, "No SSE2 support but AESNI!?!\n");
80		return (EINVAL);
81	}
82
83	device_set_desc_copy(dev, "AES-CBC,AES-XTS");
84	return (0);
85}
86
87static int
88aesni_attach(device_t dev)
89{
90	struct aesni_softc *sc;
91
92	sc = device_get_softc(dev);
93	TAILQ_INIT(&sc->sessions);
94	sc->sid = 1;
95	sc->cid = crypto_get_driverid(dev,
96	    CRYPTOCAP_F_HARDWARE|CRYPTOCAP_F_SYNC);
97	if (sc->cid < 0) {
98		device_printf(dev, "Could not get crypto driver id.\n");
99		return (ENOMEM);
100	}
101
102	rw_init(&sc->lock, "aesni_lock");
103	crypto_register(sc->cid, CRYPTO_AES_CBC, 0, 0);
104	crypto_register(sc->cid, CRYPTO_AES_XTS, 0, 0);
105	return (0);
106}
107
108static int
109aesni_detach(device_t dev)
110{
111	struct aesni_softc *sc;
112	struct aesni_session *ses;
113
114	sc = device_get_softc(dev);
115	rw_wlock(&sc->lock);
116	TAILQ_FOREACH(ses, &sc->sessions, next) {
117		if (ses->used) {
118			rw_wunlock(&sc->lock);
119			device_printf(dev,
120			    "Cannot detach, sessions still active.\n");
121			return (EBUSY);
122		}
123	}
124	while ((ses = TAILQ_FIRST(&sc->sessions)) != NULL) {
125		TAILQ_REMOVE(&sc->sessions, ses, next);
126		fpu_kern_free_ctx(ses->fpu_ctx);
127		free(ses, M_AESNI);
128	}
129	rw_wunlock(&sc->lock);
130	rw_destroy(&sc->lock);
131	crypto_unregister_all(sc->cid);
132	return (0);
133}
134
135static int
136aesni_newsession(device_t dev, uint32_t *sidp, struct cryptoini *cri)
137{
138	struct aesni_softc *sc;
139	struct aesni_session *ses;
140	struct cryptoini *encini;
141	int error;
142
143	if (sidp == NULL || cri == NULL)
144		return (EINVAL);
145
146	sc = device_get_softc(dev);
147	ses = NULL;
148	encini = NULL;
149	for (; cri != NULL; cri = cri->cri_next) {
150		switch (cri->cri_alg) {
151		case CRYPTO_AES_CBC:
152		case CRYPTO_AES_XTS:
153			if (encini != NULL)
154				return (EINVAL);
155			encini = cri;
156			break;
157		default:
158			return (EINVAL);
159		}
160	}
161	if (encini == NULL)
162		return (EINVAL);
163
164	rw_wlock(&sc->lock);
165	/*
166	 * Free sessions goes first, so if first session is used, we need to
167	 * allocate one.
168	 */
169	ses = TAILQ_FIRST(&sc->sessions);
170	if (ses == NULL || ses->used) {
171		ses = malloc(sizeof(*ses), M_AESNI, M_NOWAIT | M_ZERO);
172		if (ses == NULL) {
173			rw_wunlock(&sc->lock);
174			return (ENOMEM);
175		}
176		ses->fpu_ctx = fpu_kern_alloc_ctx(FPU_KERN_NORMAL |
177		    FPU_KERN_NOWAIT);
178		if (ses->fpu_ctx == NULL) {
179			free(ses, M_AESNI);
180			rw_wunlock(&sc->lock);
181			return (ENOMEM);
182		}
183		ses->id = sc->sid++;
184	} else {
185		TAILQ_REMOVE(&sc->sessions, ses, next);
186	}
187	ses->used = 1;
188	TAILQ_INSERT_TAIL(&sc->sessions, ses, next);
189	rw_wunlock(&sc->lock);
190	ses->algo = encini->cri_alg;
191
192	error = aesni_cipher_setup(ses, encini);
193	if (error != 0) {
194		rw_wlock(&sc->lock);
195		aesni_freesession_locked(sc, ses);
196		rw_wunlock(&sc->lock);
197		return (error);
198	}
199
200	*sidp = ses->id;
201	return (0);
202}
203
204static void
205aesni_freesession_locked(struct aesni_softc *sc, struct aesni_session *ses)
206{
207	struct fpu_kern_ctx *ctx;
208	uint32_t sid;
209
210	sid = ses->id;
211	TAILQ_REMOVE(&sc->sessions, ses, next);
212	ctx = ses->fpu_ctx;
213	bzero(ses, sizeof(*ses));
214	ses->id = sid;
215	ses->fpu_ctx = ctx;
216	TAILQ_INSERT_HEAD(&sc->sessions, ses, next);
217}
218
219static int
220aesni_freesession(device_t dev, uint64_t tid)
221{
222	struct aesni_softc *sc;
223	struct aesni_session *ses;
224	uint32_t sid;
225
226	sc = device_get_softc(dev);
227	sid = ((uint32_t)tid) & 0xffffffff;
228	rw_wlock(&sc->lock);
229	TAILQ_FOREACH_REVERSE(ses, &sc->sessions, aesni_sessions_head, next) {
230		if (ses->id == sid)
231			break;
232	}
233	if (ses == NULL) {
234		rw_wunlock(&sc->lock);
235		return (EINVAL);
236	}
237	aesni_freesession_locked(sc, ses);
238	rw_wunlock(&sc->lock);
239	return (0);
240}
241
242static int
243aesni_process(device_t dev, struct cryptop *crp, int hint __unused)
244{
245	struct aesni_softc *sc = device_get_softc(dev);
246	struct aesni_session *ses = NULL;
247	struct cryptodesc *crd, *enccrd;
248	int error;
249
250	error = 0;
251	enccrd = NULL;
252
253	/* Sanity check. */
254	if (crp == NULL)
255		return (EINVAL);
256
257	if (crp->crp_callback == NULL || crp->crp_desc == NULL) {
258		error = EINVAL;
259		goto out;
260	}
261
262	for (crd = crp->crp_desc; crd != NULL; crd = crd->crd_next) {
263		switch (crd->crd_alg) {
264		case CRYPTO_AES_CBC:
265		case CRYPTO_AES_XTS:
266			if (enccrd != NULL) {
267				error = EINVAL;
268				goto out;
269			}
270			enccrd = crd;
271			break;
272		default:
273			return (EINVAL);
274		}
275	}
276	if (enccrd == NULL || (enccrd->crd_len % AES_BLOCK_LEN) != 0) {
277		error = EINVAL;
278		goto out;
279	}
280
281	rw_rlock(&sc->lock);
282	TAILQ_FOREACH_REVERSE(ses, &sc->sessions, aesni_sessions_head, next) {
283		if (ses->id == (crp->crp_sid & 0xffffffff))
284			break;
285	}
286	rw_runlock(&sc->lock);
287	if (ses == NULL) {
288		error = EINVAL;
289		goto out;
290	}
291
292	error = aesni_cipher_process(ses, enccrd, crp);
293	if (error != 0)
294		goto out;
295
296out:
297	crp->crp_etype = error;
298	crypto_done(crp);
299	return (error);
300}
301
302uint8_t *
303aesni_cipher_alloc(struct cryptodesc *enccrd, struct cryptop *crp,
304    int *allocated)
305{
306	struct uio *uio;
307	struct iovec *iov;
308	uint8_t *addr;
309
310	if (crp->crp_flags & CRYPTO_F_IMBUF)
311		goto alloc;
312	else if (crp->crp_flags & CRYPTO_F_IOV) {
313		uio = (struct uio *)crp->crp_buf;
314		if (uio->uio_iovcnt != 1)
315			goto alloc;
316		iov = uio->uio_iov;
317		addr = (u_char *)iov->iov_base + enccrd->crd_skip;
318	} else
319		addr = (u_char *)crp->crp_buf;
320	*allocated = 0;
321	return (addr);
322
323alloc:
324	addr = malloc(enccrd->crd_len, M_AESNI, M_NOWAIT);
325	if (addr != NULL) {
326		*allocated = 1;
327		crypto_copydata(crp->crp_flags, crp->crp_buf, enccrd->crd_skip,
328		    enccrd->crd_len, addr);
329	} else
330		*allocated = 0;
331	return (addr);
332}
333
334static device_method_t aesni_methods[] = {
335	DEVMETHOD(device_identify, aesni_identify),
336	DEVMETHOD(device_probe, aesni_probe),
337	DEVMETHOD(device_attach, aesni_attach),
338	DEVMETHOD(device_detach, aesni_detach),
339
340	DEVMETHOD(cryptodev_newsession, aesni_newsession),
341	DEVMETHOD(cryptodev_freesession, aesni_freesession),
342	DEVMETHOD(cryptodev_process, aesni_process),
343
344	{0, 0},
345};
346
347static driver_t aesni_driver = {
348	"aesni",
349	aesni_methods,
350	sizeof(struct aesni_softc),
351};
352static devclass_t aesni_devclass;
353
354DRIVER_MODULE(aesni, nexus, aesni_driver, aesni_devclass, 0, 0);
355MODULE_VERSION(aesni, 1);
356MODULE_DEPEND(aesni, crypto, 1, 1, 1);
357