1/*-
2 * Copyright (c) 2005-2008 Pawel Jakub Dawidek <pjd@FreeBSD.org>
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 *    notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 *    notice, this list of conditions and the following disclaimer in the
12 *    documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 */
26
27#include <sys/cdefs.h>
28__FBSDID("$FreeBSD: releng/10.3/sys/crypto/via/padlock.c 268033 2014-06-30 09:48:44Z kib $");
29
30#include <sys/param.h>
31#include <sys/systm.h>
32#include <sys/kernel.h>
33#include <sys/module.h>
34#include <sys/lock.h>
35#include <sys/rwlock.h>
36#include <sys/malloc.h>
37#include <sys/libkern.h>
38#if defined(__amd64__) || (defined(__i386__) && !defined(PC98))
39#include <machine/cpufunc.h>
40#include <machine/cputypes.h>
41#include <machine/md_var.h>
42#include <machine/specialreg.h>
43#endif
44
45#include <opencrypto/cryptodev.h>
46
47#include <crypto/via/padlock.h>
48
49#include <sys/kobj.h>
50#include <sys/bus.h>
51#include "cryptodev_if.h"
52
53/*
54 * Technical documentation about the PadLock engine can be found here:
55 *
56 * http://www.via.com.tw/en/downloads/whitepapers/initiatives/padlock/programming_guide.pdf
57 */
58
59struct padlock_softc {
60	int32_t		sc_cid;
61	uint32_t	sc_sid;
62	TAILQ_HEAD(padlock_sessions_head, padlock_session) sc_sessions;
63	struct rwlock	sc_sessions_lock;
64};
65
66static int padlock_newsession(device_t, uint32_t *sidp, struct cryptoini *cri);
67static int padlock_freesession(device_t, uint64_t tid);
68static void padlock_freesession_one(struct padlock_softc *sc,
69    struct padlock_session *ses, int locked);
70static int padlock_process(device_t, struct cryptop *crp, int hint __unused);
71
72MALLOC_DEFINE(M_PADLOCK, "padlock_data", "PadLock Data");
73
74static void
75padlock_identify(driver_t *drv, device_t parent)
76{
77	/* NB: order 10 is so we get attached after h/w devices */
78	if (device_find_child(parent, "padlock", -1) == NULL &&
79	    BUS_ADD_CHILD(parent, 10, "padlock", -1) == 0)
80		panic("padlock: could not attach");
81}
82
83static int
84padlock_probe(device_t dev)
85{
86	char capp[256];
87
88#if defined(__amd64__) || (defined(__i386__) && !defined(PC98))
89	/* If there is no AES support, we has nothing to do here. */
90	if (!(via_feature_xcrypt & VIA_HAS_AES)) {
91		device_printf(dev, "No ACE support.\n");
92		return (EINVAL);
93	}
94	strlcpy(capp, "AES-CBC", sizeof(capp));
95#if 0
96	strlcat(capp, ",AES-EBC", sizeof(capp));
97	strlcat(capp, ",AES-CFB", sizeof(capp));
98	strlcat(capp, ",AES-OFB", sizeof(capp));
99#endif
100	if (via_feature_xcrypt & VIA_HAS_SHA) {
101		strlcat(capp, ",SHA1", sizeof(capp));
102		strlcat(capp, ",SHA256", sizeof(capp));
103	}
104#if 0
105	if (via_feature_xcrypt & VIA_HAS_AESCTR)
106		strlcat(capp, ",AES-CTR", sizeof(capp));
107	if (via_feature_xcrypt & VIA_HAS_MM)
108		strlcat(capp, ",RSA", sizeof(capp));
109#endif
110	device_set_desc_copy(dev, capp);
111	return (0);
112#else
113	return (EINVAL);
114#endif
115}
116
117static int
118padlock_attach(device_t dev)
119{
120	struct padlock_softc *sc = device_get_softc(dev);
121
122	TAILQ_INIT(&sc->sc_sessions);
123	sc->sc_sid = 1;
124
125	sc->sc_cid = crypto_get_driverid(dev, CRYPTOCAP_F_HARDWARE);
126	if (sc->sc_cid < 0) {
127		device_printf(dev, "Could not get crypto driver id.\n");
128		return (ENOMEM);
129	}
130
131	rw_init(&sc->sc_sessions_lock, "padlock_lock");
132	crypto_register(sc->sc_cid, CRYPTO_AES_CBC, 0, 0);
133	crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC, 0, 0);
134	crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC, 0, 0);
135	crypto_register(sc->sc_cid, CRYPTO_RIPEMD160_HMAC, 0, 0);
136	crypto_register(sc->sc_cid, CRYPTO_SHA2_256_HMAC, 0, 0);
137	crypto_register(sc->sc_cid, CRYPTO_SHA2_384_HMAC, 0, 0);
138	crypto_register(sc->sc_cid, CRYPTO_SHA2_512_HMAC, 0, 0);
139	return (0);
140}
141
142static int
143padlock_detach(device_t dev)
144{
145	struct padlock_softc *sc = device_get_softc(dev);
146	struct padlock_session *ses;
147
148	rw_wlock(&sc->sc_sessions_lock);
149	TAILQ_FOREACH(ses, &sc->sc_sessions, ses_next) {
150		if (ses->ses_used) {
151			rw_wunlock(&sc->sc_sessions_lock);
152			device_printf(dev,
153			    "Cannot detach, sessions still active.\n");
154			return (EBUSY);
155		}
156	}
157	while ((ses = TAILQ_FIRST(&sc->sc_sessions)) != NULL) {
158		TAILQ_REMOVE(&sc->sc_sessions, ses, ses_next);
159		fpu_kern_free_ctx(ses->ses_fpu_ctx);
160		free(ses, M_PADLOCK);
161	}
162	rw_destroy(&sc->sc_sessions_lock);
163	crypto_unregister_all(sc->sc_cid);
164	return (0);
165}
166
167static int
168padlock_newsession(device_t dev, uint32_t *sidp, struct cryptoini *cri)
169{
170	struct padlock_softc *sc = device_get_softc(dev);
171	struct padlock_session *ses = NULL;
172	struct cryptoini *encini, *macini;
173	struct thread *td;
174	int error;
175
176	if (sidp == NULL || cri == NULL)
177		return (EINVAL);
178
179	encini = macini = NULL;
180	for (; cri != NULL; cri = cri->cri_next) {
181		switch (cri->cri_alg) {
182		case CRYPTO_NULL_HMAC:
183		case CRYPTO_MD5_HMAC:
184		case CRYPTO_SHA1_HMAC:
185		case CRYPTO_RIPEMD160_HMAC:
186		case CRYPTO_SHA2_256_HMAC:
187		case CRYPTO_SHA2_384_HMAC:
188		case CRYPTO_SHA2_512_HMAC:
189			if (macini != NULL)
190				return (EINVAL);
191			macini = cri;
192			break;
193		case CRYPTO_AES_CBC:
194			if (encini != NULL)
195				return (EINVAL);
196			encini = cri;
197			break;
198		default:
199			return (EINVAL);
200		}
201	}
202
203	/*
204	 * We only support HMAC algorithms to be able to work with
205	 * ipsec(4), so if we are asked only for authentication without
206	 * encryption, don't pretend we can accellerate it.
207	 */
208	if (encini == NULL)
209		return (EINVAL);
210
211	/*
212	 * Let's look for a free session structure.
213	 */
214	rw_wlock(&sc->sc_sessions_lock);
215	/*
216	 * Free sessions goes first, so if first session is used, we need to
217	 * allocate one.
218	 */
219	ses = TAILQ_FIRST(&sc->sc_sessions);
220	if (ses == NULL || ses->ses_used) {
221		ses = malloc(sizeof(*ses), M_PADLOCK, M_NOWAIT | M_ZERO);
222		if (ses == NULL) {
223			rw_wunlock(&sc->sc_sessions_lock);
224			return (ENOMEM);
225		}
226		ses->ses_fpu_ctx = fpu_kern_alloc_ctx(FPU_KERN_NORMAL |
227		    FPU_KERN_NOWAIT);
228		if (ses->ses_fpu_ctx == NULL) {
229			free(ses, M_PADLOCK);
230			rw_wunlock(&sc->sc_sessions_lock);
231			return (ENOMEM);
232		}
233		ses->ses_id = sc->sc_sid++;
234	} else {
235		TAILQ_REMOVE(&sc->sc_sessions, ses, ses_next);
236	}
237	ses->ses_used = 1;
238	TAILQ_INSERT_TAIL(&sc->sc_sessions, ses, ses_next);
239	rw_wunlock(&sc->sc_sessions_lock);
240
241	error = padlock_cipher_setup(ses, encini);
242	if (error != 0) {
243		padlock_freesession_one(sc, ses, 0);
244		return (error);
245	}
246
247	if (macini != NULL) {
248		td = curthread;
249		error = fpu_kern_enter(td, ses->ses_fpu_ctx, FPU_KERN_NORMAL |
250		    FPU_KERN_KTHR);
251		if (error == 0) {
252			error = padlock_hash_setup(ses, macini);
253			fpu_kern_leave(td, ses->ses_fpu_ctx);
254		}
255		if (error != 0) {
256			padlock_freesession_one(sc, ses, 0);
257			return (error);
258		}
259	}
260
261	*sidp = ses->ses_id;
262	return (0);
263}
264
265static void
266padlock_freesession_one(struct padlock_softc *sc, struct padlock_session *ses,
267    int locked)
268{
269	struct fpu_kern_ctx *ctx;
270	uint32_t sid = ses->ses_id;
271
272	if (!locked)
273		rw_wlock(&sc->sc_sessions_lock);
274	TAILQ_REMOVE(&sc->sc_sessions, ses, ses_next);
275	padlock_hash_free(ses);
276	ctx = ses->ses_fpu_ctx;
277	bzero(ses, sizeof(*ses));
278	ses->ses_used = 0;
279	ses->ses_id = sid;
280	ses->ses_fpu_ctx = ctx;
281	TAILQ_INSERT_HEAD(&sc->sc_sessions, ses, ses_next);
282	if (!locked)
283		rw_wunlock(&sc->sc_sessions_lock);
284}
285
286static int
287padlock_freesession(device_t dev, uint64_t tid)
288{
289	struct padlock_softc *sc = device_get_softc(dev);
290	struct padlock_session *ses;
291	uint32_t sid = ((uint32_t)tid) & 0xffffffff;
292
293	rw_wlock(&sc->sc_sessions_lock);
294	TAILQ_FOREACH_REVERSE(ses, &sc->sc_sessions, padlock_sessions_head,
295	    ses_next) {
296		if (ses->ses_id == sid)
297			break;
298	}
299	if (ses == NULL) {
300		rw_wunlock(&sc->sc_sessions_lock);
301		return (EINVAL);
302	}
303	padlock_freesession_one(sc, ses, 1);
304	rw_wunlock(&sc->sc_sessions_lock);
305	return (0);
306}
307
308static int
309padlock_process(device_t dev, struct cryptop *crp, int hint __unused)
310{
311	struct padlock_softc *sc = device_get_softc(dev);
312	struct padlock_session *ses = NULL;
313	struct cryptodesc *crd, *enccrd, *maccrd;
314	int error = 0;
315
316	enccrd = maccrd = NULL;
317
318	/* Sanity check. */
319	if (crp == NULL)
320		return (EINVAL);
321
322	if (crp->crp_callback == NULL || crp->crp_desc == NULL) {
323		error = EINVAL;
324		goto out;
325	}
326
327	for (crd = crp->crp_desc; crd != NULL; crd = crd->crd_next) {
328		switch (crd->crd_alg) {
329		case CRYPTO_NULL_HMAC:
330		case CRYPTO_MD5_HMAC:
331		case CRYPTO_SHA1_HMAC:
332		case CRYPTO_RIPEMD160_HMAC:
333		case CRYPTO_SHA2_256_HMAC:
334		case CRYPTO_SHA2_384_HMAC:
335		case CRYPTO_SHA2_512_HMAC:
336			if (maccrd != NULL) {
337				error = EINVAL;
338				goto out;
339			}
340			maccrd = crd;
341			break;
342		case CRYPTO_AES_CBC:
343			if (enccrd != NULL) {
344				error = EINVAL;
345				goto out;
346			}
347			enccrd = crd;
348			break;
349		default:
350			return (EINVAL);
351		}
352	}
353	if (enccrd == NULL || (enccrd->crd_len % AES_BLOCK_LEN) != 0) {
354		error = EINVAL;
355		goto out;
356	}
357
358	rw_rlock(&sc->sc_sessions_lock);
359	TAILQ_FOREACH_REVERSE(ses, &sc->sc_sessions, padlock_sessions_head,
360	    ses_next) {
361		if (ses->ses_id == (crp->crp_sid & 0xffffffff))
362			break;
363	}
364	rw_runlock(&sc->sc_sessions_lock);
365	if (ses == NULL) {
366		error = EINVAL;
367		goto out;
368	}
369
370	/* Perform data authentication if requested before encryption. */
371	if (maccrd != NULL && maccrd->crd_next == enccrd) {
372		error = padlock_hash_process(ses, maccrd, crp);
373		if (error != 0)
374			goto out;
375	}
376
377	error = padlock_cipher_process(ses, enccrd, crp);
378	if (error != 0)
379		goto out;
380
381	/* Perform data authentication if requested after encryption. */
382	if (maccrd != NULL && enccrd->crd_next == maccrd) {
383		error = padlock_hash_process(ses, maccrd, crp);
384		if (error != 0)
385			goto out;
386	}
387
388out:
389#if 0
390	/*
391	 * This code is not necessary, because contexts will be freed on next
392	 * padlock_setup_mackey() call or at padlock_freesession() call.
393	 */
394	if (ses != NULL && maccrd != NULL &&
395	    (maccrd->crd_flags & CRD_F_KEY_EXPLICIT) != 0) {
396		padlock_free_ctx(ses->ses_axf, ses->ses_ictx);
397		padlock_free_ctx(ses->ses_axf, ses->ses_octx);
398	}
399#endif
400	crp->crp_etype = error;
401	crypto_done(crp);
402	return (error);
403}
404
405static device_method_t padlock_methods[] = {
406	DEVMETHOD(device_identify,	padlock_identify),
407	DEVMETHOD(device_probe,		padlock_probe),
408	DEVMETHOD(device_attach,	padlock_attach),
409	DEVMETHOD(device_detach,	padlock_detach),
410
411	DEVMETHOD(cryptodev_newsession,	padlock_newsession),
412	DEVMETHOD(cryptodev_freesession,padlock_freesession),
413	DEVMETHOD(cryptodev_process,	padlock_process),
414
415	{0, 0},
416};
417
418static driver_t padlock_driver = {
419	"padlock",
420	padlock_methods,
421	sizeof(struct padlock_softc),
422};
423static devclass_t padlock_devclass;
424
425/* XXX where to attach */
426DRIVER_MODULE(padlock, nexus, padlock_driver, padlock_devclass, 0, 0);
427MODULE_VERSION(padlock, 1);
428MODULE_DEPEND(padlock, crypto, 1, 1, 1);
429