aesni.c revision 213064 
1/*-
2 * Copyright (c) 2005-2008 Pawel Jakub Dawidek <pjd@FreeBSD.org>
3 * Copyright (c) 2010 Konstantin Belousov <kib@FreeBSD.org>
4 * All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 *    notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 *    notice, this list of conditions and the following disclaimer in the
13 *    documentation and/or other materials provided with the distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25 * SUCH DAMAGE.
26 */
27
28#include <sys/cdefs.h>
29__FBSDID("$FreeBSD: head/sys/crypto/aesni/aesni.c 213064 2010-09-23 11:26:38Z pjd $");
30
31#include <sys/param.h>
32#include <sys/systm.h>
33#include <sys/kernel.h>
34#include <sys/kobj.h>
35#include <sys/libkern.h>
36#include <sys/lock.h>
37#include <sys/module.h>
38#include <sys/malloc.h>
39#include <sys/rwlock.h>
40#include <sys/bus.h>
41#include <sys/uio.h>
42#include <crypto/aesni/aesni.h>
43#include "cryptodev_if.h"
44
45struct aesni_softc {
46	int32_t cid;
47	uint32_t sid;
48	TAILQ_HEAD(aesni_sessions_head, aesni_session) sessions;
49	struct rwlock lock;
50};
51
52static int aesni_newsession(device_t, uint32_t *sidp, struct cryptoini *cri);
53static int aesni_freesession(device_t, uint64_t tid);
54static void aesni_freesession_locked(struct aesni_softc *sc,
55    struct aesni_session *ses);
56
57MALLOC_DEFINE(M_AESNI, "aesni_data", "AESNI Data");
58
59static void
60aesni_identify(driver_t *drv, device_t parent)
61{
62
63	/* NB: order 10 is so we get attached after h/w devices */
64	if (device_find_child(parent, "aesni", -1) == NULL &&
65	    BUS_ADD_CHILD(parent, 10, "aesni", -1) == 0)
66		panic("aesni: could not attach");
67}
68
69static int
70aesni_probe(device_t dev)
71{
72
73	if ((cpu_feature2 & CPUID2_AESNI) == 0) {
74		device_printf(dev, "No AESNI support.\n");
75		return (EINVAL);
76	}
77	device_set_desc_copy(dev, "AES-CBC");
78	return (0);
79}
80
81static int
82aesni_attach(device_t dev)
83{
84	struct aesni_softc *sc;
85
86	sc = device_get_softc(dev);
87	TAILQ_INIT(&sc->sessions);
88	sc->sid = 1;
89	sc->cid = crypto_get_driverid(dev, CRYPTOCAP_F_HARDWARE);
90	if (sc->cid < 0) {
91		device_printf(dev, "Could not get crypto driver id.\n");
92		return (ENOMEM);
93	}
94
95	rw_init(&sc->lock, "aesni_lock");
96	crypto_register(sc->cid, CRYPTO_AES_CBC, 0, 0);
97	return (0);
98}
99
100static int
101aesni_detach(device_t dev)
102{
103	struct aesni_softc *sc;
104	struct aesni_session *ses;
105
106	sc = device_get_softc(dev);
107	rw_wlock(&sc->lock);
108	TAILQ_FOREACH(ses, &sc->sessions, next) {
109		if (ses->used) {
110			rw_wunlock(&sc->lock);
111			device_printf(dev,
112			    "Cannot detach, sessions still active.\n");
113			return (EBUSY);
114		}
115	}
116	while ((ses = TAILQ_FIRST(&sc->sessions)) != NULL) {
117		TAILQ_REMOVE(&sc->sessions, ses, next);
118		free(ses, M_AESNI);
119	}
120	rw_wunlock(&sc->lock);
121	rw_destroy(&sc->lock);
122	crypto_unregister_all(sc->cid);
123	return (0);
124}
125
126static int
127aesni_newsession(device_t dev, uint32_t *sidp, struct cryptoini *cri)
128{
129	struct aesni_softc *sc;
130	struct aesni_session *ses;
131	struct cryptoini *encini;
132	int error;
133
134	if (sidp == NULL || cri == NULL)
135		return (EINVAL);
136
137	sc = device_get_softc(dev);
138	ses = NULL;
139	encini = NULL;
140	for (; cri != NULL; cri = cri->cri_next) {
141		switch (cri->cri_alg) {
142		case CRYPTO_AES_CBC:
143			if (encini != NULL)
144				return (EINVAL);
145			encini = cri;
146			break;
147		default:
148			return (EINVAL);
149		}
150	}
151	if (encini == NULL)
152		return (EINVAL);
153
154	rw_wlock(&sc->lock);
155	/*
156	 * Free sessions goes first, so if first session is used, we need to
157	 * allocate one.
158	 */
159	ses = TAILQ_FIRST(&sc->sessions);
160	if (ses == NULL || ses->used) {
161		ses = malloc(sizeof(*ses), M_AESNI, M_NOWAIT | M_ZERO);
162		if (ses == NULL) {
163			rw_wunlock(&sc->lock);
164			return (ENOMEM);
165		}
166		KASSERT(((uintptr_t)ses) % 0x10 == 0,
167		    ("malloc returned unaligned pointer"));
168		ses->id = sc->sid++;
169	} else {
170		TAILQ_REMOVE(&sc->sessions, ses, next);
171	}
172	ses->used = 1;
173	TAILQ_INSERT_TAIL(&sc->sessions, ses, next);
174	rw_wunlock(&sc->lock);
175
176	error = aesni_cipher_setup(ses, encini);
177	if (error != 0) {
178		rw_wlock(&sc->lock);
179		aesni_freesession_locked(sc, ses);
180		rw_wunlock(&sc->lock);
181		return (error);
182	}
183
184	*sidp = ses->id;
185	return (0);
186}
187
188static void
189aesni_freesession_locked(struct aesni_softc *sc, struct aesni_session *ses)
190{
191	uint32_t sid;
192
193	sid = ses->id;
194	TAILQ_REMOVE(&sc->sessions, ses, next);
195	bzero(ses, sizeof(*ses));
196	ses->id = sid;
197	TAILQ_INSERT_HEAD(&sc->sessions, ses, next);
198}
199
200static int
201aesni_freesession(device_t dev, uint64_t tid)
202{
203	struct aesni_softc *sc;
204	struct aesni_session *ses;
205	uint32_t sid;
206
207	sc = device_get_softc(dev);
208	sid = ((uint32_t)tid) & 0xffffffff;
209	rw_wlock(&sc->lock);
210	TAILQ_FOREACH_REVERSE(ses, &sc->sessions, aesni_sessions_head, next) {
211		if (ses->id == sid)
212			break;
213	}
214	if (ses == NULL) {
215		rw_wunlock(&sc->lock);
216		return (EINVAL);
217	}
218	aesni_freesession_locked(sc, ses);
219	rw_wunlock(&sc->lock);
220	return (0);
221}
222
223static int
224aesni_process(device_t dev, struct cryptop *crp, int hint __unused)
225{
226	struct aesni_softc *sc = device_get_softc(dev);
227	struct aesni_session *ses = NULL;
228	struct cryptodesc *crd, *enccrd;
229	int error;
230
231	error = 0;
232	enccrd = NULL;
233
234	/* Sanity check. */
235	if (crp == NULL)
236		return (EINVAL);
237
238	if (crp->crp_callback == NULL || crp->crp_desc == NULL) {
239		error = EINVAL;
240		goto out;
241	}
242
243	for (crd = crp->crp_desc; crd != NULL; crd = crd->crd_next) {
244		switch (crd->crd_alg) {
245		case CRYPTO_AES_CBC:
246			if (enccrd != NULL) {
247				error = EINVAL;
248				goto out;
249			}
250			enccrd = crd;
251			break;
252		default:
253			return (EINVAL);
254		}
255	}
256	if (enccrd == NULL || (enccrd->crd_len % AES_BLOCK_LEN) != 0) {
257		error = EINVAL;
258		goto out;
259	}
260
261	rw_rlock(&sc->lock);
262	TAILQ_FOREACH_REVERSE(ses, &sc->sessions, aesni_sessions_head, next) {
263		if (ses->id == (crp->crp_sid & 0xffffffff))
264			break;
265	}
266	rw_runlock(&sc->lock);
267	if (ses == NULL) {
268		error = EINVAL;
269		goto out;
270	}
271
272	error = aesni_cipher_process(ses, enccrd, crp);
273	if (error != 0)
274		goto out;
275
276out:
277	crp->crp_etype = error;
278	crypto_done(crp);
279	return (error);
280}
281
282uint8_t *
283aesni_cipher_alloc(struct cryptodesc *enccrd, struct cryptop *crp,
284    int *allocated)
285{
286	struct uio *uio;
287	struct iovec *iov;
288	uint8_t *addr;
289
290	if (crp->crp_flags & CRYPTO_F_IMBUF)
291		goto alloc;
292	else if (crp->crp_flags & CRYPTO_F_IOV) {
293		uio = (struct uio *)crp->crp_buf;
294		if (uio->uio_iovcnt != 1)
295			goto alloc;
296		iov = uio->uio_iov;
297		addr = (u_char *)iov->iov_base + enccrd->crd_skip;
298	} else
299		addr = (u_char *)crp->crp_buf;
300	*allocated = 0;
301	return (addr);
302
303alloc:
304	addr = malloc(enccrd->crd_len, M_AESNI, M_NOWAIT);
305	if (addr != NULL) {
306		*allocated = 1;
307		crypto_copydata(crp->crp_flags, crp->crp_buf, enccrd->crd_skip,
308		    enccrd->crd_len, addr);
309	} else
310		*allocated = 0;
311	return (addr);
312}
313
314static device_method_t aesni_methods[] = {
315	DEVMETHOD(device_identify, aesni_identify),
316	DEVMETHOD(device_probe, aesni_probe),
317	DEVMETHOD(device_attach, aesni_attach),
318	DEVMETHOD(device_detach, aesni_detach),
319
320	DEVMETHOD(cryptodev_newsession, aesni_newsession),
321	DEVMETHOD(cryptodev_freesession, aesni_freesession),
322	DEVMETHOD(cryptodev_process, aesni_process),
323
324	{0, 0},
325};
326
327static driver_t aesni_driver = {
328	"aesni",
329	aesni_methods,
330	sizeof(struct aesni_softc),
331};
332static devclass_t aesni_devclass;
333
334DRIVER_MODULE(aesni, nexus, aesni_driver, aesni_devclass, 0, 0);
335MODULE_VERSION(aesni, 1);
336MODULE_DEPEND(aesni, crypto, 1, 1, 1);
337