aesni.c revision 230426 
1/*-
2 * Copyright (c) 2005-2008 Pawel Jakub Dawidek <pjd@FreeBSD.org>
3 * Copyright (c) 2010 Konstantin Belousov <kib@FreeBSD.org>
4 * All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 *    notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 *    notice, this list of conditions and the following disclaimer in the
13 *    documentation and/or other materials provided with the distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25 * SUCH DAMAGE.
26 */
27
28#include <sys/cdefs.h>
29__FBSDID("$FreeBSD: head/sys/crypto/aesni/aesni.c 230426 2012-01-21 17:45:27Z kib $");
30
31#include <sys/param.h>
32#include <sys/systm.h>
33#include <sys/kernel.h>
34#include <sys/kobj.h>
35#include <sys/libkern.h>
36#include <sys/lock.h>
37#include <sys/module.h>
38#include <sys/malloc.h>
39#include <sys/rwlock.h>
40#include <sys/bus.h>
41#include <sys/uio.h>
42#include <crypto/aesni/aesni.h>
43#include "cryptodev_if.h"
44
45struct aesni_softc {
46	int32_t cid;
47	uint32_t sid;
48	TAILQ_HEAD(aesni_sessions_head, aesni_session) sessions;
49	struct rwlock lock;
50};
51
52static int aesni_newsession(device_t, uint32_t *sidp, struct cryptoini *cri);
53static int aesni_freesession(device_t, uint64_t tid);
54static void aesni_freesession_locked(struct aesni_softc *sc,
55    struct aesni_session *ses);
56
57MALLOC_DEFINE(M_AESNI, "aesni_data", "AESNI Data");
58
59static void
60aesni_identify(driver_t *drv, device_t parent)
61{
62
63	/* NB: order 10 is so we get attached after h/w devices */
64	if (device_find_child(parent, "aesni", -1) == NULL &&
65	    BUS_ADD_CHILD(parent, 10, "aesni", -1) == 0)
66		panic("aesni: could not attach");
67}
68
69static int
70aesni_probe(device_t dev)
71{
72
73	if ((cpu_feature2 & CPUID2_AESNI) == 0) {
74		device_printf(dev, "No AESNI support.\n");
75		return (EINVAL);
76	}
77	device_set_desc_copy(dev, "AES-CBC,AES-XTS");
78	return (0);
79}
80
81static int
82aesni_attach(device_t dev)
83{
84	struct aesni_softc *sc;
85
86	sc = device_get_softc(dev);
87	TAILQ_INIT(&sc->sessions);
88	sc->sid = 1;
89	sc->cid = crypto_get_driverid(dev, CRYPTOCAP_F_HARDWARE);
90	if (sc->cid < 0) {
91		device_printf(dev, "Could not get crypto driver id.\n");
92		return (ENOMEM);
93	}
94
95	rw_init(&sc->lock, "aesni_lock");
96	crypto_register(sc->cid, CRYPTO_AES_CBC, 0, 0);
97	crypto_register(sc->cid, CRYPTO_AES_XTS, 0, 0);
98	return (0);
99}
100
101static int
102aesni_detach(device_t dev)
103{
104	struct aesni_softc *sc;
105	struct aesni_session *ses;
106
107	sc = device_get_softc(dev);
108	rw_wlock(&sc->lock);
109	TAILQ_FOREACH(ses, &sc->sessions, next) {
110		if (ses->used) {
111			rw_wunlock(&sc->lock);
112			device_printf(dev,
113			    "Cannot detach, sessions still active.\n");
114			return (EBUSY);
115		}
116	}
117	while ((ses = TAILQ_FIRST(&sc->sessions)) != NULL) {
118		TAILQ_REMOVE(&sc->sessions, ses, next);
119		fpu_kern_free_ctx(ses->fpu_ctx);
120		free(ses, M_AESNI);
121	}
122	rw_wunlock(&sc->lock);
123	rw_destroy(&sc->lock);
124	crypto_unregister_all(sc->cid);
125	return (0);
126}
127
128static int
129aesni_newsession(device_t dev, uint32_t *sidp, struct cryptoini *cri)
130{
131	struct aesni_softc *sc;
132	struct aesni_session *ses;
133	struct cryptoini *encini;
134	int error;
135
136	if (sidp == NULL || cri == NULL)
137		return (EINVAL);
138
139	sc = device_get_softc(dev);
140	ses = NULL;
141	encini = NULL;
142	for (; cri != NULL; cri = cri->cri_next) {
143		switch (cri->cri_alg) {
144		case CRYPTO_AES_CBC:
145		case CRYPTO_AES_XTS:
146			if (encini != NULL)
147				return (EINVAL);
148			encini = cri;
149			break;
150		default:
151			return (EINVAL);
152		}
153	}
154	if (encini == NULL)
155		return (EINVAL);
156
157	rw_wlock(&sc->lock);
158	/*
159	 * Free sessions goes first, so if first session is used, we need to
160	 * allocate one.
161	 */
162	ses = TAILQ_FIRST(&sc->sessions);
163	if (ses == NULL || ses->used) {
164		ses = malloc(sizeof(*ses), M_AESNI, M_NOWAIT | M_ZERO);
165		if (ses == NULL) {
166			rw_wunlock(&sc->lock);
167			return (ENOMEM);
168		}
169		ses->fpu_ctx = fpu_kern_alloc_ctx(FPU_KERN_NORMAL |
170		    FPU_KERN_NOWAIT);
171		if (ses->fpu_ctx == NULL) {
172			free(ses, M_AESNI);
173			rw_wunlock(&sc->lock);
174			return (ENOMEM);
175		}
176		ses->id = sc->sid++;
177	} else {
178		TAILQ_REMOVE(&sc->sessions, ses, next);
179	}
180	ses->used = 1;
181	TAILQ_INSERT_TAIL(&sc->sessions, ses, next);
182	rw_wunlock(&sc->lock);
183	ses->algo = encini->cri_alg;
184
185	error = aesni_cipher_setup(ses, encini);
186	if (error != 0) {
187		rw_wlock(&sc->lock);
188		aesni_freesession_locked(sc, ses);
189		rw_wunlock(&sc->lock);
190		return (error);
191	}
192
193	*sidp = ses->id;
194	return (0);
195}
196
197static void
198aesni_freesession_locked(struct aesni_softc *sc, struct aesni_session *ses)
199{
200	struct fpu_kern_ctx *ctx;
201	uint32_t sid;
202
203	sid = ses->id;
204	TAILQ_REMOVE(&sc->sessions, ses, next);
205	ctx = ses->fpu_ctx;
206	bzero(ses, sizeof(*ses));
207	ses->id = sid;
208	ses->fpu_ctx = ctx;
209	TAILQ_INSERT_HEAD(&sc->sessions, ses, next);
210}
211
212static int
213aesni_freesession(device_t dev, uint64_t tid)
214{
215	struct aesni_softc *sc;
216	struct aesni_session *ses;
217	uint32_t sid;
218
219	sc = device_get_softc(dev);
220	sid = ((uint32_t)tid) & 0xffffffff;
221	rw_wlock(&sc->lock);
222	TAILQ_FOREACH_REVERSE(ses, &sc->sessions, aesni_sessions_head, next) {
223		if (ses->id == sid)
224			break;
225	}
226	if (ses == NULL) {
227		rw_wunlock(&sc->lock);
228		return (EINVAL);
229	}
230	aesni_freesession_locked(sc, ses);
231	rw_wunlock(&sc->lock);
232	return (0);
233}
234
235static int
236aesni_process(device_t dev, struct cryptop *crp, int hint __unused)
237{
238	struct aesni_softc *sc = device_get_softc(dev);
239	struct aesni_session *ses = NULL;
240	struct cryptodesc *crd, *enccrd;
241	int error;
242
243	error = 0;
244	enccrd = NULL;
245
246	/* Sanity check. */
247	if (crp == NULL)
248		return (EINVAL);
249
250	if (crp->crp_callback == NULL || crp->crp_desc == NULL) {
251		error = EINVAL;
252		goto out;
253	}
254
255	for (crd = crp->crp_desc; crd != NULL; crd = crd->crd_next) {
256		switch (crd->crd_alg) {
257		case CRYPTO_AES_CBC:
258		case CRYPTO_AES_XTS:
259			if (enccrd != NULL) {
260				error = EINVAL;
261				goto out;
262			}
263			enccrd = crd;
264			break;
265		default:
266			return (EINVAL);
267		}
268	}
269	if (enccrd == NULL || (enccrd->crd_len % AES_BLOCK_LEN) != 0) {
270		error = EINVAL;
271		goto out;
272	}
273
274	rw_rlock(&sc->lock);
275	TAILQ_FOREACH_REVERSE(ses, &sc->sessions, aesni_sessions_head, next) {
276		if (ses->id == (crp->crp_sid & 0xffffffff))
277			break;
278	}
279	rw_runlock(&sc->lock);
280	if (ses == NULL) {
281		error = EINVAL;
282		goto out;
283	}
284
285	error = aesni_cipher_process(ses, enccrd, crp);
286	if (error != 0)
287		goto out;
288
289out:
290	crp->crp_etype = error;
291	crypto_done(crp);
292	return (error);
293}
294
295uint8_t *
296aesni_cipher_alloc(struct cryptodesc *enccrd, struct cryptop *crp,
297    int *allocated)
298{
299	struct uio *uio;
300	struct iovec *iov;
301	uint8_t *addr;
302
303	if (crp->crp_flags & CRYPTO_F_IMBUF)
304		goto alloc;
305	else if (crp->crp_flags & CRYPTO_F_IOV) {
306		uio = (struct uio *)crp->crp_buf;
307		if (uio->uio_iovcnt != 1)
308			goto alloc;
309		iov = uio->uio_iov;
310		addr = (u_char *)iov->iov_base + enccrd->crd_skip;
311	} else
312		addr = (u_char *)crp->crp_buf;
313	*allocated = 0;
314	return (addr);
315
316alloc:
317	addr = malloc(enccrd->crd_len, M_AESNI, M_NOWAIT);
318	if (addr != NULL) {
319		*allocated = 1;
320		crypto_copydata(crp->crp_flags, crp->crp_buf, enccrd->crd_skip,
321		    enccrd->crd_len, addr);
322	} else
323		*allocated = 0;
324	return (addr);
325}
326
327static device_method_t aesni_methods[] = {
328	DEVMETHOD(device_identify, aesni_identify),
329	DEVMETHOD(device_probe, aesni_probe),
330	DEVMETHOD(device_attach, aesni_attach),
331	DEVMETHOD(device_detach, aesni_detach),
332
333	DEVMETHOD(cryptodev_newsession, aesni_newsession),
334	DEVMETHOD(cryptodev_freesession, aesni_freesession),
335	DEVMETHOD(cryptodev_process, aesni_process),
336
337	{0, 0},
338};
339
340static driver_t aesni_driver = {
341	"aesni",
342	aesni_methods,
343	sizeof(struct aesni_softc),
344};
345static devclass_t aesni_devclass;
346
347DRIVER_MODULE(aesni, nexus, aesni_driver, aesni_devclass, 0, 0);
348MODULE_VERSION(aesni, 1);
349MODULE_DEPEND(aesni, crypto, 1, 1, 1);
350